From c09cd0c805dd39254b64162f5f97f17fc7539975 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:35:59 +0000 Subject: [PATCH 01/32] fix(build): count Python checks in the build's completion line A build whose Python check failed exited non-zero but printed "Completed successfully", because the completion footer counted SQL results and Python nodes but not Python check results. Failed checks now count as FAIL and make the line "Completed with errors", warned checks count as WARN, passing checks as PASS, and failed checks are listed under the footer's failures. The check docs also called ctx.fail, ctx.warn and ctx.pass_ with a positional message, which raises TypeError because the message is keyword-only; the examples now pass message=. --- Makefile | 1 + .../_helpers/build_command/completion.rs | 48 +++++++++-- .../_helpers/build_progress/footer.rs | 16 ++++ .../docs/concepts/python-nodes/checks.md | 8 +- .../docs/concepts/python-nodes/factories.md | 2 +- .../cli/commands/main/build/_test_types.py | 11 +++ .../main/build/test_python_check_outcomes.py | 79 +++++++++++++++++++ .../docs/concepts/python-nodes/checks.mdx | 8 +- .../docs/concepts/python-nodes/factories.mdx | 2 +- 9 files changed, 158 insertions(+), 17 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_check_outcomes.py diff --git a/Makefile b/Makefile index b9bf0cd60c..1e216addbd 100644 --- a/Makefile +++ b/Makefile @@ -113,6 +113,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_load_sources.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_storage_key.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_source_end_audits.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_check_outcomes.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_unbuilt_dependencies.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index 4e931cd9b7..dc10a996b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -19,7 +19,8 @@ use crate::bindings::_helpers::build_command::invocation::BuildContext; use crate::bindings::_helpers::build_command::support::{ boolean, emit, joined, none, text, utc_now, }; -use crate::bindings::_helpers::build_progress::footer::format_build_footer; +use crate::bindings::_helpers::build_progress::footer::format_build_footer_with; +use crate::bindings::_helpers::check::check_results::check_result_text; use crate::bindings::_helpers::compile_driver::pipeline_entries::plan_has_executable_work; use crate::bindings::_helpers::compile_outputs::runtime_target::{ write_python_check_runtime_target, write_runtime_target, @@ -35,8 +36,12 @@ use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::result_output::documents::format_build_execution_json; use crate::bindings::_helpers::result_output::documents::format_degradable_build_execution_json; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; +use sqlbuild_core::build_progress::models::PythonNodeText; +use sqlbuild_core::check_command::models::{CheckResultText, CheckStatus}; const SUCCESS_STATUS: &str = "success"; +const CHECK_KIND: &str = "check"; +const CHECK_WARNED: &str = "check warned"; const PENDING_MESSAGE: &str = "Build cost collection has not completed."; const INTERRUPTED_MESSAGE: &str = "Build was interrupted before cost collection completed."; @@ -313,6 +318,30 @@ fn any_failed( Ok(false) } +/// A Python check result as the footer counts it: a failure fails the build, a warning warns. +fn check_node_text(check: &CheckResultText) -> PythonNodeText { + let message: Option = check + .error_message + .clone() + .or_else(|| check.message.clone()); + PythonNodeText { + name: check.name.clone(), + kind: CHECK_KIND.to_owned(), + status: if check.status == CheckStatus::Fail { + "failed" + } else { + SUCCESS_STATUS + } + .to_owned(), + warnings: if check.status == CheckStatus::Warn { + vec![message.clone().unwrap_or_else(|| CHECK_WARNED.to_owned())] + } else { + Vec::new() + }, + error_message: message, + } +} + /// Persist and render a completed direct build and return its exit code. pub(crate) fn complete_direct_build<'py>( context: &BuildContext<'py>, @@ -356,14 +385,19 @@ pub(crate) fn complete_direct_build<'py>( ); let cost_seconds: f64 = finish_cost_timing(context, cost_started); let record: Bound<'py, PyAny> = record?; - let footer: String = format_build_footer( + let checks: Vec = check_results + .try_iter()? + .map(|check| Ok(check_node_text(&check_result_text(&check?)?))) + .collect::>()?; + let footer: String = format_build_footer_with( &result.clone(), - native_field(&preparation.callbacks, "elapsed")?.extract::()?, - boolean(py, context.use_color).extract::()?, + ( + native_field(&preparation.callbacks, "elapsed")?.extract::()?, + context.use_color, + ), Some(&outcome.python_results.clone()).filter(|value| !value.is_none()), - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? - .extract()?; + checks, + )?; emit(&context.stream, &format!("\n{footer}\n"))?; let payload: Bound<'py, PyAny> = format_degradable_build_execution_json( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs index 72ba57bd60..894c07caaf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs @@ -16,6 +16,21 @@ pub(crate) fn format_build_footer( elapsed: f64, use_color: bool, python_node_results: Option<&Bound<'_, PyAny>>, +) -> PyResult { + format_build_footer_with( + result, + (elapsed, use_color), + python_node_results, + Vec::new(), + ) +} + +/// The build footer, counting `checks` (Python check results) with the Python node results. +pub(crate) fn format_build_footer_with( + result: &Bound<'_, PyAny>, + (elapsed, use_color): (f64, bool), + python_node_results: Option<&Bound<'_, PyAny>>, + checks: Vec, ) -> PyResult { let mut python: Vec = Vec::new(); if let Some(results) = python_node_results { @@ -23,6 +38,7 @@ pub(crate) fn format_build_footer( python.push(python_node_text(&node?)?); } } + python.extend(checks); Ok(footer( &build_text(result)?, &python, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md index eca32a9470..4dc3e69973 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md @@ -34,8 +34,8 @@ def check_orders_exported(ctx: CheckContext): """Check orders exported.""" result = ctx.result_of(export_orders) if result.metadata.get("rows", 0) == 0: - return ctx.fail("no orders exported") - return ctx.pass_("orders exported") + return ctx.fail(message="no orders exported") + return ctx.pass_(message="orders exported") ``` The check receives a `CheckContext` and reads its dependencies' persisted results with `ctx.result_of(...)`. @@ -51,7 +51,7 @@ def orders_present(ctx): """Orders present.""" orders = ctx.relation(model("fact_orders")) count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0] - return ctx.pass_() if count else ctx.fail("no orders") + return ctx.pass_() if count else ctx.fail(message="no orders") ``` Do not hard-code relation names in check SQL; see [Hard-coded relation names](sql-references.md#hard-coded-relation-names). @@ -86,7 +86,7 @@ Returning `None` is not allowed - checks must be explicit. def orders_freshness(ctx): """Orders freshness.""" if stale(): - return ctx.fail("export is stale") # recorded as a warning, does not fail the build + return ctx.fail(message="export is stale") # recorded as a warning, does not fail the build return ctx.pass_() ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md index e0cdfed182..afec5249a9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md @@ -69,7 +69,7 @@ def orders_pipeline(): @check(depends_on=export) def orders_export_check(ctx): """Check that the orders export is ready.""" - return ctx.pass_("export ready") + return ctx.pass_(message="export ready") return [load, export, orders_export_check] ``` diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index d065846d30..19ea6c1b35 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1461,3 +1461,14 @@ class SourceEndAuditBuildE2ETestCase: build_arguments: tuple[str, ...] expected_exit_code: int expected_audit_reported: bool + + +@dataclass(frozen=True) +class PythonCheckBuildE2ETestCase: + """A build whose Python check returns `check_body`, and the footer and exit it must end with.""" + + description: str + check_decorator: str + check_body: str + expected_exit_code: int + expected_footer_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_check_outcomes.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_check_outcomes.py new file mode 100644 index 0000000000..ebe2b25d60 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_check_outcomes.py @@ -0,0 +1,79 @@ +"""E2E coverage that a build's completion line and exit code account for its Python checks.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + PythonCheckBuildE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + PythonCheckBuildE2ETestCase( + description="a failing check fails the build", + check_decorator='@check(depends_on=model("fact_orders"))', + check_body='return ctx.fail(message="orders present")', + expected_exit_code=1, + expected_footer_fragment="✗ Completed with errors PASS=1 WARN=0 FAIL=1", + ), + PythonCheckBuildE2ETestCase( + description="a check that raises fails the build", + check_decorator='@check(depends_on=model("fact_orders"))', + check_body='raise RuntimeError("boom")', + expected_exit_code=1, + expected_footer_fragment="✗ Completed with errors PASS=1 WARN=0 FAIL=1", + ), + PythonCheckBuildE2ETestCase( + description="a failing warn-severity check warns", + check_decorator='@check(depends_on=model("fact_orders"), severity="warn")', + check_body='return ctx.fail(message="orders present")', + expected_exit_code=0, + expected_footer_fragment="Completed with warnings PASS=1 WARN=1 FAIL=0", + ), + PythonCheckBuildE2ETestCase( + description="a passing check passes", + check_decorator='@check(depends_on=model("fact_orders"))', + check_body='return ctx.pass_(message="orders checked")', + expected_exit_code=0, + expected_footer_fragment="✓ Completed successfully PASS=2 WARN=0 FAIL=0", + ), + ], + ids=lambda case: case.description, +) +def test_given_python_check_when_building_then_completion_line_reports_its_outcome( + test_case: PythonCheckBuildE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="python_check_outcomes", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "python_check_outcomes"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "warehouse.duckdb"\n\n[defaults]\nmaterialized = "table"\n' + ), + "models/fact_orders.sql": ( + "MODEL (description 'Test model fact_orders.');\nSELECT 1 AS order_id\n" + ), + "python/checks/orders.py": ( + "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n\n\n" + f"{test_case.check_decorator}\n" + "def orders_checked(ctx):\n" + " '''Test check orders_checked.'''\n" + f" {test_case.check_body}\n" + ), + }, + ) + + built: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert built.returncode == test_case.expected_exit_code, built.stdout + built.stderr + assert test_case.expected_footer_fragment in built.stdout, built.stdout diff --git a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx index c4d5355e9e..dd93a67b34 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx @@ -22,8 +22,8 @@ def check_orders_exported(ctx: CheckContext): """Check orders exported.""" result = ctx.result_of(export_orders) if result.metadata.get("rows", 0) == 0: - return ctx.fail("no orders exported") - return ctx.pass_("orders exported") + return ctx.fail(message="no orders exported") + return ctx.pass_(message="orders exported") ``` The check receives a `CheckContext` and reads its dependencies' persisted results with `ctx.result_of(...)`. @@ -40,7 +40,7 @@ def orders_present(ctx): """Orders present.""" orders = ctx.relation(model("fact_orders")) count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0] - return ctx.pass_() if count else ctx.fail("no orders") + return ctx.pass_() if count else ctx.fail(message="no orders") ``` Do not hard-code relation names in check SQL; see [Hard-coded relation names](/docs/concepts/python-nodes/sql-references#hard-coded-relation-names). @@ -75,7 +75,7 @@ Returning `None` is not allowed - checks must be explicit. def orders_freshness(ctx): """Orders freshness.""" if stale(): - return ctx.fail("export is stale") # recorded as a warning, does not fail the build + return ctx.fail(message="export is stale") # recorded as a warning, does not fail the build return ctx.pass_() ``` diff --git a/website/src/content/docs/docs/concepts/python-nodes/factories.mdx b/website/src/content/docs/docs/concepts/python-nodes/factories.mdx index 6994e8fbee..0af21a86e8 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/factories.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/factories.mdx @@ -68,7 +68,7 @@ def orders_pipeline(): @check(depends_on=export) def orders_export_check(ctx): """Check that the orders export is ready.""" - return ctx.pass_("export ready") + return ctx.pass_(message="export ready") return [load, export, orders_export_check] ``` From d88e7a8a5a57f391503d687cbb9ad2394d8172ae Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:51:02 +0000 Subject: [PATCH 02/32] fix(test): find mocked references on both sides of a parenthesised join SQL-test planning collects the relation markers a query reads to swap in their mocks. The walk skipped the left side of a parenthesised join, so a model reading FROM (__ref("orders") AS o JOIN __ref("customers") AS c ON ...) failed with "references __ref(\"orders\") which has no mock" although the test mocked it. It also skipped the relations of UPDATE ... FROM. Both are collected now, and the JSON oracle the walk is tested against covers them. --- .../_helpers/sql_tests/relation_markers.rs | 15 +++- .../src/compiler/tests/helpers.rs | 15 ++-- .../tests/test_sql_test_relation_markers.rs | 12 ++++ .../cli/commands/main/test/_test_types.py | 10 +++ .../test/test_parenthesised_join_mocks.py | 71 +++++++++++++++++++ 5 files changed, 115 insertions(+), 8 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/test/test_parenthesised_join_mocks.py diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs index 959d41a9fc..8f8e991ae1 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/relation_markers.rs @@ -24,7 +24,20 @@ fn node_marker_calls(mut calls: Vec, node: &Expression) -> Vec calls.extend(from_markers(from)), Expression::Delete(delete) => calls.extend(join_markers(&delete.joins)), - Expression::JoinedTable(table) => calls.extend(join_markers(&table.joins)), + Expression::JoinedTable(table) => { + calls.extend( + marker(&table.left) + .into_iter() + .chain(join_markers(&table.joins)), + ); + } + Expression::Update(update) => calls.extend( + update + .from_clause + .iter() + .flat_map(from_markers) + .chain(join_markers(&update.from_joins)), + ), Expression::DataDeletionProperty(property) => { return property.on.dfs().fold(calls, node_marker_calls); } diff --git a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs index ef30aaadb8..dea0e94b09 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/helpers.rs @@ -1574,17 +1574,18 @@ type MarkerCalls = Option>; /// The original JSON-tree walk, kept as the oracle the direct relation walk must match. fn json_relation_marker_calls(value: &Value) -> Vec<(String, String)> { - let from_markers = value - .get("from") - .and_then(|from| from.get("expressions")) - .and_then(Value::as_array) + let from_markers = ["from", "from_clause"] .into_iter() + .filter_map(|key| value.get(key)) + .filter_map(|from| from.get("expressions")) + .filter_map(Value::as_array) .flatten() + .chain(value.get("left").filter(|_| value.get("joins").is_some())) .filter_map(json_marker); - let join_markers = value - .get("joins") - .and_then(Value::as_array) + let join_markers = ["joins", "from_joins"] .into_iter() + .filter_map(|key| value.get(key)) + .filter_map(Value::as_array) .flatten() .filter_map(|join| join.get("this")) .filter_map(json_marker); diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs index f054efe809..f22adc6a39 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_relation_markers.rs @@ -63,6 +63,18 @@ fn given_parsed_statements_when_collecting_relation_markers_then_json_walk_order sql: "SELECT __ref(\"orders\") AS value FROM __table_fn(\"orders\", 2) WHERE id IN (SELECT id FROM t)", expected_calls: &[], }, + RelationMarkersTestCase { + description: "a parenthesised join keeps its left side", + dialect: "duckdb", + sql: "SELECT * FROM (__ref(\"orders\") JOIN __ref(\"customers\") ON TRUE)", + expected_calls: &[("__ref", "orders"), ("__ref", "customers")], + }, + RelationMarkersTestCase { + description: "update from relations are collected", + dialect: "postgres", + sql: "UPDATE t SET x = 1 FROM __ref(\"orders\") JOIN __ref(\"customers\") ON TRUE", + expected_calls: &[("__ref", "orders"), ("__ref", "customers")], + }, RelationMarkersTestCase { description: "set operations walk each branch", dialect: "postgres", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index e48ffb8404..9de88c687c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -593,3 +593,13 @@ class MacroMockAliasTestCase: command: tuple[str, ...] expected_exit_code: int expected_fragment: str + + +@dataclass(frozen=True) +class ParenthesisedJoinMockTestCase: + """A model reading mocked references through a parenthesised join, and the test outcome.""" + + description: str + model_from: str + expected_exit_code: int + expected_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_parenthesised_join_mocks.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_parenthesised_join_mocks.py new file mode 100644 index 0000000000..2eb13148af --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_parenthesised_join_mocks.py @@ -0,0 +1,71 @@ +"""E2E coverage of SQL-test mocks for references read through a parenthesised join.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + ParenthesisedJoinMockTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_PROJECT_TOML: str = ( + 'name = "shop"\nadapter = "duckdb"\n\n[connection]\ndatabase = "shop.duckdb"\n\n' + "[scopes]\nenforce_placement = false\n" +) +_TEST: str = ( + "TEST ();\n\n" + "WITH\n" + "__ref__orders AS (SELECT 7 AS order_id),\n" + "__ref__customers AS (SELECT 7 AS order_id),\n" + "__expected__order_customers AS (SELECT 7 AS order_id)\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + ParenthesisedJoinMockTestCase( + description="both sides of a parenthesised join use their mocks", + model_from=( + '(__ref("orders") AS o JOIN __ref("customers") AS c ON o.order_id = c.order_id)' + ), + expected_exit_code=0, + expected_fragment="PASS=1 FAIL=0", + ), + ParenthesisedJoinMockTestCase( + description="an unparenthesised join uses its mocks", + model_from='__ref("orders") AS o JOIN __ref("customers") AS c ON o.order_id = c.order_id', + expected_exit_code=0, + expected_fragment="PASS=1 FAIL=0", + ), + ], + ids=lambda case: case.description, +) +def test_given_parenthesised_join_when_testing_then_both_references_use_their_mocks( + test_case: ParenthesisedJoinMockTestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/orders.sql": 'MODEL (description "Orders.");\n\nSELECT 1 AS order_id\n', + "models/customers.sql": 'MODEL (description "Customers.");\n\nSELECT 1 AS order_id\n', + "models/order_customers.sql": ( + 'MODEL (description "Orders with customers.");\n\n' + f"SELECT o.order_id FROM {test_case.model_from}\n" + ), + "tests/unit/test_order_customers.sql": _TEST, + }, + ) + + tested: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "test"), project_dir=project_dir + ) + + assert tested.returncode == test_case.expected_exit_code, tested.stdout + tested.stderr + assert test_case.expected_fragment in tested.stdout From 449b96282786ce6e6354fe2ab590250601e1515c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:09:20 +0000 Subject: [PATCH 03/32] fix(macros): locate a nested macro call that makes a dict key unhashable A dict key built from a nested macro call, such as @py.show({@py.items(): 2}) where items() returns a list, crashed compilation with a raw TypeError traceback. It now fails with the located P001 macro argument error, pointing at the nested call whose value cannot be hashed and naming its type. --- .../_helpers/render/macro_evaluation.rs | 41 ++++++++- .../cli/commands/main/compile/_test_types.py | 10 +++ .../compile/test_macro_argument_dict_keys.py | 87 +++++++++++++++++++ 3 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_macro_argument_dict_keys.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs index b4da44c688..cdae298365 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs @@ -26,6 +26,7 @@ const SURROGATE_HELP: &str = "Return valid Unicode text from the macro; lone sur come from bytes decoded with errors='surrogateescape'"; const UNARY_HELP: &str = "Return a number from the nested macro, or apply the sign inside the macro"; +const NESTED_KEY_HELP: &str = "Return a string, number, True, False, None, or a tuple of them from a macro used as a dict key"; /// Text addressed by code point offsets, as the native scan reports them. pub(crate) struct CallText<'a> { @@ -344,8 +345,11 @@ impl<'py> ArgumentSource<'_, 'py> { ArgumentValue::Dict(pairs) => { let mapping: Bound<'py, PyDict> = PyDict::new(py); for (key, item) in pairs { - let key: Bound<'py, PyAny> = self.built(key)?; - mapping.set_item(key, self.built(item)?)?; + let built_key: Bound<'py, PyAny> = self.built(key)?; + if let Err(error) = built_key.hash() { + return Err(self.unhashable_key(key, error)); + } + mapping.set_item(built_key, self.built(item)?)?; } mapping.into_any() } @@ -384,6 +388,23 @@ impl<'py> ArgumentSource<'_, 'py> { Err(self.argument_error(self.nested[call].0, &detail, UNARY_HELP)) } + /// The error for a dict key a nested macro call made unhashable, at that call. + fn unhashable_key(&self, key: &ArgumentValue, error: PyErr) -> PyErr { + let Some(call) = nested_calls(key) + .into_iter() + .find(|call| self.values[*call].hash().is_err()) + else { + return error; + }; + let type_name: String = match self.values[call].get_type().name() { + Ok(name) => name.to_string(), + Err(failure) => return failure, + }; + let detail: String = + format!("use the value of a nested macro call, which is a {type_name}, as a dict key"); + self.argument_error(self.nested[call].0, &detail, NESTED_KEY_HELP) + } + /// The `MacroArgumentError` for a native parse error at its line and column. fn parse_error(&self, error: &ArgumentError) -> PyErr { let mut offset: usize = 0; @@ -410,6 +431,22 @@ impl<'py> ArgumentSource<'_, 'py> { } } +/// The nested macro calls inside `value`, in authored order. +fn nested_calls(value: &ArgumentValue) -> Vec { + match value { + ArgumentValue::NestedCall(index) => vec![*index], + ArgumentValue::List(items) | ArgumentValue::Tuple(items) => { + items.iter().flat_map(nested_calls).collect() + } + ArgumentValue::Dict(pairs) => pairs + .iter() + .flat_map(|(key, item)| nested_calls(key).into_iter().chain(nested_calls(item))) + .collect(), + ArgumentValue::Negative(inner) | ArgumentValue::Positive(inner) => nested_calls(inner), + _ => Vec::new(), + } +} + /// `MacroArgumentError` of the call `name` at code point `offset` of `text`. pub(crate) fn macro_argument_error( (expansion, classes): (&NativeMacroExpansion, &ExpansionClasses<'_>), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index f9f228811d..691861929f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1312,3 +1312,13 @@ class TypedCallContractTestCase: projection: str expected_exit_code: int expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class MacroDictKeyCompileE2ETestCase: + """A macro call whose dict argument has a key, and the compile outcome it must give.""" + + description: str + argument: str + expected_exit_code: int + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_macro_argument_dict_keys.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_macro_argument_dict_keys.py new file mode 100644 index 0000000000..9a8b3717d0 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_macro_argument_dict_keys.py @@ -0,0 +1,87 @@ +"""E2E coverage of macro argument dict keys that Python cannot hash.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + MacroDictKeyCompileE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_MACROS: str = ( + "def show(value):\n" + ' """Show a value."""\n' + " return repr(str(value))\n\n\n" + "def items():\n" + ' """A list of items."""\n' + " return [1]\n\n\n" + "def label():\n" + ' """A label."""\n' + ' return "orders"\n' +) +_HELP: str = ( + "help: Return a string, number, True, False, None, or a tuple of them from a macro used as a " + "dict key" +) + + +@pytest.mark.parametrize( + "test_case", + [ + MacroDictKeyCompileE2ETestCase( + description="a nested call returning a list as a dict key is a located P001", + argument="{@py.items(): 2}", + expected_exit_code=1, + expected_fragments=( + "error[P001]: Macro arguments of '@py.show'", + "use the value of a nested macro call, which is a list, as a dict key at line 2, " + "column 18", + _HELP, + ), + ), + MacroDictKeyCompileE2ETestCase( + description="the unhashable call inside a tuple key is the one reported", + argument="{(@py.label(), @py.items()): 2}", + expected_exit_code=1, + expected_fragments=("which is a list, as a dict key at line 2, column 32", _HELP), + ), + MacroDictKeyCompileE2ETestCase( + description="a nested call returning a string is a valid dict key", + argument="{@py.label(): 2}", + expected_exit_code=0, + expected_fragments=("0 errors",), + ), + ], + ids=lambda case: case.description, +) +def test_given_macro_dict_key_from_nested_call_when_compiling_then_unhashable_keys_are_located( + test_case: MacroDictKeyCompileE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "shop"\nadapter = "duckdb"\n\n[connection]\ndatabase = "shop.duckdb"\n\n' + "[scopes]\nenforce_placement = false\n" + ), + "macros/python/values.py": _MACROS, + "models/orders.sql": ( + "MODEL (description 'Test model orders.');\n" + f"SELECT @py.show({test_case.argument}) AS shown\n" + ), + }, + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + + output: str = compiled.stdout + compiled.stderr + assert compiled.returncode == test_case.expected_exit_code, output + assert "Traceback" not in output + assert all(fragment in output for fragment in test_case.expected_fragments), output From cd389399db0ff5821ee417c696f6705159ca24e7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:21:40 +0000 Subject: [PATCH 04/32] fix(refactor): let sqb mv move a model that Python code names sqb mv refused any move of a model whose name appeared in project Python, such as @check(depends_on=model("orders")), although a move to a folder or to a file of the same name keeps the model's name, so those references stay valid. The Python name search now runs only for renames: sqb rename, and sqb mv to a file with a new name. --- .../_helpers/refactor/refactor_facts.rs | 17 ++++- .../sqlbuild/references/docs/cli/rename.md | 2 +- .../cli/commands/main/refactor/_test_types.py | 10 +++ .../main/refactor/test_python_named_moves.py | 70 +++++++++++++++++++ website/src/content/docs/docs/cli/rename.mdx | 2 +- 5 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_python_named_moves.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs index e976064a6f..4dea3dd407 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs @@ -19,6 +19,7 @@ use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; use crate::bindings::_helpers::scopes::scope_reports::{scope_move_preview, scope_rebound_usages}; use crate::bindings::_helpers::scopes::scope_validation::relocated_declarations; +const SQL_SUFFIX: &str = ".sql"; const GENERIC_DIALECT: &str = "generic"; const MATERIALIZED_KEY: &str = "materialized"; const MIGRATE_FROM_KEY: &str = "migrate_from"; @@ -305,6 +306,18 @@ fn python_locations( Ok(locations) } +/// Whether `request` moves a model without renaming it: a folder, or a file of the same name. +fn move_keeps_name(request: &RefactorRequest) -> bool { + if request.operation != RefactorOperation::MoveModel { + return false; + } + let destination: &str = request.destination.as_deref().unwrap_or_default(); + match destination.strip_suffix(SQL_SUFFIX) { + Some(path) => path.rsplit('/').next() == Some(request.model_name.as_str()), + None => true, + } +} + /// The facts one refactoring plan of `requests` reads. pub(crate) fn refactor_facts( project: &Bound<'_, PyAny>, @@ -312,7 +325,9 @@ pub(crate) fn refactor_facts( ) -> PyResult { let mut locations: Vec = Vec::new(); for request in requests { - locations.extend(python_locations(project, request)?); + if !move_keeps_name(request) { + locations.extend(python_locations(project, request)?); + } } let compiled: Bound<'_, PyAny> = native_field(&native_field(project, "graph")?, "project")?; let discovered: Bound<'_, PyAny> = native_field(project, "discovered")?; diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md index 5680d39da6..882cf0a160 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md @@ -167,7 +167,7 @@ The command refuses, and writes nothing, when: - a reference is produced by a macro, or a column is read inside SQL that a macro generates; - a column reaches a model through `SELECT *` in a subquery, a `USING` or `NATURAL` join, or an unqualified name that could belong to several tables; - a downstream model's `SELECT *` would silently rename its output, without `--cascade`; -- Python code or Python node SQL strings name the model or column; +- Python code or Python node SQL strings name a model being renamed, or a column being renamed. A move that keeps the model's name leaves Python references to it as they are; - a Python import of a moved Python file cannot be rewritten safely. A rename whose new name belongs to another layer's folder, under the layer-folder rules `SQBRPROJECT102` and `SQBRPROJECT105`, fails verification because `sqb rename` keeps the file where it is. When those rules are the only failures, the output says where the model belongs and prints the one-step [`sqb mv`](#rename-or-move) command, with that layer's folder and the current sub-folders filled in: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py index d017b36f43..ab99b5aa0e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/_test_types.py @@ -289,3 +289,13 @@ class LayoutRepairDryRunE2ETestCase: files: dict[str, str] arguments: tuple[str, ...] expected_lines: tuple[str, ...] + + +@dataclass(frozen=True) +class PythonNamedMoveE2ETestCase: + """A move of a model a Python check names, and whether `sqb mv` carries it out.""" + + description: str + destination: str + expected_exit_code: int + expected_model_path: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_python_named_moves.py b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_python_named_moves.py new file mode 100644 index 0000000000..e0bfe88366 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/refactor/test_python_named_moves.py @@ -0,0 +1,70 @@ +"""CLI e2e coverage for `sqb mv` on a model that Python code names.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.refactor._test_types import ( + PythonNamedMoveE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_CHECK: str = ( + "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n\n\n" + '@check(depends_on=model("orders"))\n' + "def orders_present(ctx):\n" + " '''Test check orders_present.'''\n" + " return ctx.pass_()\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonNamedMoveE2ETestCase( + description="a move to a folder keeps the name and moves", + destination="models/marts/", + expected_exit_code=0, + expected_model_path="models/marts/orders.sql", + ), + PythonNamedMoveE2ETestCase( + description="a move to a file of the same name moves", + destination="models/marts/orders.sql", + expected_exit_code=0, + expected_model_path="models/marts/orders.sql", + ), + PythonNamedMoveE2ETestCase( + description="a move that renames the model is refused", + destination="models/marts/fct_orders.sql", + expected_exit_code=1, + expected_model_path="models/orders.sql", + ), + ], + ids=lambda case: case.description, +) +def test_given_model_named_by_python_when_moving_then_only_renames_are_refused( + test_case: PythonNamedMoveE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "shop"\nadapter = "duckdb"\n\n[connection]\ndatabase = "shop.duckdb"\n\n' + "[scopes]\nenforce_placement = false\n" + ), + "models/orders.sql": "MODEL (description 'Test model orders.');\nSELECT 1 AS order_id\n", + "python/checks/orders.py": _CHECK, + }, + ) + + moved: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "mv", "orders", test_case.destination), project_dir=project_dir + ) + + assert moved.returncode == test_case.expected_exit_code, moved.stdout + moved.stderr + assert (project_dir / test_case.expected_model_path).is_file() + assert (project_dir / "python/checks/orders.py").read_text(encoding="utf-8") == _CHECK diff --git a/website/src/content/docs/docs/cli/rename.mdx b/website/src/content/docs/docs/cli/rename.mdx index 84f956f65d..f0d4911c2a 100644 --- a/website/src/content/docs/docs/cli/rename.mdx +++ b/website/src/content/docs/docs/cli/rename.mdx @@ -149,7 +149,7 @@ The command refuses, and writes nothing, when: - a reference is produced by a macro, or a column is read inside SQL that a macro generates; - a column reaches a model through `SELECT *` in a subquery, a `USING` or `NATURAL` join, or an unqualified name that could belong to several tables; - a downstream model's `SELECT *` would silently rename its output, without `--cascade`; -- Python code or Python node SQL strings name the model or column; +- Python code or Python node SQL strings name a model being renamed, or a column being renamed. A move that keeps the model's name leaves Python references to it as they are; - a Python import of a moved Python file cannot be rewritten safely. A rename whose new name belongs to another layer's folder, under the layer-folder rules `SQBRPROJECT102` and `SQBRPROJECT105`, fails verification because `sqb rename` keeps the file where it is. When those rules are the only failures, the output says where the model belongs and prints the one-step [`sqb mv`](#rename-or-move) command, with that layer's folder and the current sub-folders filled in: From c7ef6bfb0056fca0cc204904928a23cdfed43eeb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:32:11 +0000 Subject: [PATCH 05/32] fix(sqlite): accept text columns as timestamp cursors SQLite stores timestamps as ISO-8601 text, and its cursor windows already compare cursors as normalised text, but compile rejected a cursor_type timestamp column of type TEXT, such as strftime('%Y-%m-%d 00:00:00', ordered_at), with B301. On SQLite a text column is now a valid timestamp cursor; other adapters still reject it. --- .../_helpers/metadata_checks/checks.rs | 12 ++- .../docs/concepts/adapters/sqlite.md | 3 + .../cli/commands/main/sqlite/_test_types.py | 12 +++ .../commands/main/sqlite/test_text_cursors.py | 83 +++++++++++++++++++ .../docs/docs/concepts/adapters/sqlite.mdx | 3 + 5 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_text_cursors.py diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs index 5e96bc534d..5a272e337d 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/checks.rs @@ -30,6 +30,8 @@ static SQL_TEST_CTE: LazyLock> = LazyLock::new(|| compiled(SQL_TEST_CTE_PATTERN)); const INTEGER_CURSOR_FAMILIES: &[TypeFamily] = &[TypeFamily::Integer, TypeFamily::Decimal]; +/// The dialect that stores timestamps as ISO-8601 text, so a text column is a timestamp cursor. +const TEXT_TIMESTAMP_DIALECT: &str = "sqlite"; const TEMPORAL_CURSOR_FAMILIES: &[TypeFamily] = &[ TypeFamily::Timestamp, TypeFamily::Datetime, @@ -170,7 +172,7 @@ fn reference_errors( )?); } } - if let Some(error) = cursor_error(model, shape, context.families)? { + if let Some(error) = cursor_error(model, shape, (context.families, context.dialect))? { errors.push(error); } Ok(errors) @@ -180,7 +182,7 @@ fn reference_errors( fn cursor_error( model: &MetadataModel, shape: &[(String, String)], - families: &Families, + (families, dialect): (&Families, Option<&str>), ) -> Result, SemanticFailure> { let (Some(cursor), Some(cursor_type)) = (model.cursor.as_deref(), model.cursor_type.as_deref()) else { @@ -203,6 +205,12 @@ fn cursor_error( if family == TypeFamily::Other || allowed.contains(&family) { return Ok(None); } + if family == TypeFamily::String + && allowed == TEMPORAL_CURSOR_FAMILIES + && dialect == Some(TEXT_TIMESTAMP_DIALECT) + { + return Ok(None); + } let message = format!("cursor_type {cursor_type} does not match column '{cursor}' type {actual}"); Ok(Some(model_error( diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md index 1d0dcc04fa..cf41d899c8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md @@ -84,6 +84,9 @@ SQLite stores values by type affinity, so SQLBuild keeps column types explicitly or a UTC offset fall in the right window. A value with an offset belongs to the window of its UTC time; DuckDB's `TIMESTAMP` instead drops the offset. Comparisons you write yourself against `__cursor_start()` and `__cursor_end()` compare the stored text as is. +- A `cursor_type timestamp` column may be text, such as + `strftime('%Y-%m-%d 00:00:00', ordered_at)`, because SQLite stores timestamps as text. Other + adapters reject a text timestamp cursor with `B301`. - Booleans are stored as `0` and `1`. - JSON, arrays and objects are stored as JSON text. - `DECIMAL(p, s)` means exact canonical text. A column whose `DECIMAL(p, s)` or `NUMERIC(p, s)` diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py index 6020e57fa0..14f7e92203 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py @@ -96,3 +96,15 @@ class SqliteLoaderE2ETestCase: relations: tuple[str, ...] expected_rows: tuple[tuple[object, ...], ...] expected_declared_types: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class TextTimestampCursorE2ETestCase: + """A daily model whose timestamp cursor is a text expression, on one adapter.""" + + description: str + adapter: str + connection: str + day_expression: str + expected_exit_code: int + expected_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_text_cursors.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_text_cursors.py new file mode 100644 index 0000000000..e97a7139db --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_text_cursors.py @@ -0,0 +1,83 @@ +"""E2E coverage of text columns as timestamp cursors, which SQLite stores timestamps as.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.sqlite._test_types import ( + TextTimestampCursorE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_TWO_ORDERS: str = ( + "MODEL (description 'Test model orders.');\n" + "SELECT 1 AS order_id, '2026-01-01 09:30:00' AS ordered_at\n" + "UNION ALL SELECT 2 AS order_id, '2026-01-02 09:30:00' AS ordered_at\n" +) +_THREE_ORDERS: str = _TWO_ORDERS + ( + "UNION ALL SELECT 3 AS order_id, '2026-01-03 09:30:00' AS ordered_at\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + TextTimestampCursorE2ETestCase( + description="sqlite accepts a text timestamp cursor and builds incrementally", + adapter="sqlite", + connection='path = "shop.sqlite"', + day_expression="strftime('%Y-%m-%d 00:00:00', o.ordered_at)", + expected_exit_code=0, + expected_fragment="✓ Completed successfully PASS=2", + ), + TextTimestampCursorE2ETestCase( + description="duckdb still rejects a text timestamp cursor", + adapter="duckdb", + connection='database = "shop.duckdb"', + day_expression="CAST(o.ordered_at AS VARCHAR)", + expected_exit_code=1, + expected_fragment="cursor_type TIMESTAMP does not match column 'order_day' type", + ), + ], + ids=lambda case: case.description, +) +def test_given_text_timestamp_cursor_when_building_twice_then_only_sqlite_accepts_it( + test_case: TextTimestampCursorE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": ( + f'name = "shop"\nadapter = "{test_case.adapter}"\n\n' + f"[connection]\n{test_case.connection}\n\n" + '[defaults]\nmaterialized = "table"\n\n[scopes]\nenforce_placement = false\n' + ), + "models/orders.sql": _TWO_ORDERS, + "models/daily_orders.sql": ( + "MODEL (description 'Test model daily_orders.', materialized incremental,\n" + " incremental_strategy delete_insert, cursor order_day, cursor_type timestamp,\n" + " cursor_grain day, cursor_inputs (orders ordered_at));\n" + f"SELECT o.order_id, {test_case.day_expression} AS order_day\n" + 'FROM __ref("orders") AS o\n' + ), + }, + ) + first: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + _ = (project_dir / "models/orders.sql").write_text(_THREE_ORDERS, encoding="utf-8") + + second: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + output: str = first.stdout + first.stderr + second.stdout + second.stderr + assert (first.returncode, second.returncode) == ( + test_case.expected_exit_code, + test_case.expected_exit_code, + ), output + assert test_case.expected_fragment in output diff --git a/website/src/content/docs/docs/concepts/adapters/sqlite.mdx b/website/src/content/docs/docs/concepts/adapters/sqlite.mdx index 48bdd38330..3fceb698e1 100644 --- a/website/src/content/docs/docs/concepts/adapters/sqlite.mdx +++ b/website/src/content/docs/docs/concepts/adapters/sqlite.mdx @@ -70,6 +70,9 @@ SQLite stores values by type affinity, so SQLBuild keeps column types explicitly or a UTC offset fall in the right window. A value with an offset belongs to the window of its UTC time; DuckDB's `TIMESTAMP` instead drops the offset. Comparisons you write yourself against `__cursor_start()` and `__cursor_end()` compare the stored text as is. +- A `cursor_type timestamp` column may be text, such as + `strftime('%Y-%m-%d 00:00:00', ordered_at)`, because SQLite stores timestamps as text. Other + adapters reject a text timestamp cursor with `B301`. - Booleans are stored as `0` and `1`. - JSON, arrays and objects are stored as JSON text. - `DECIMAL(p, s)` means exact canonical text. A column whose `DECIMAL(p, s)` or `NUMERIC(p, s)` From d4a52e635167128143e6528743e07367b8f13e45 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:02:06 +0000 Subject: [PATCH 06/32] feat(sqlite): warn at compile about casts to date and time types On SQLite, CAST(x AS TIMESTAMP), DATE, DATETIME or TIME takes NUMERIC affinity and keeps only the leading number, so ISO-8601 text such as '2026-01-01 10:00:00' becomes 2026. Compile now reports a located B104 warning at each such cast in model SQL, with help to keep the text or use datetime(), date() or strftime(). Other adapters are unaffected. --- .../src/binding_locations/constants.rs | 2 + .../main/locate_binding_diagnostics.rs | 9 +- .../src/semantic_checks/constants.rs | 5 + .../_helpers/diagnostics.rs | 4 + .../src/semantic_validation/_helpers/mod.rs | 1 + .../_helpers/sqlite_temporal_casts.rs | 120 +++++++ .../src/semantic_validation/tests.rs | 2 + .../tests/test_sqlite_temporal_casts.rs | 61 ++++ .../semantic_validation/tests/test_types.rs | 12 + .../_helpers/analysis_corpus/failure_cases.py | 16 + .../docs/concepts/adapters/sqlite.md | 2 +- .../docs/concepts/semantic-compilation.md | 1 + .../cli/commands/main/sqlite/_test_types.py | 9 + .../main/sqlite/test_temporal_casts.py | 73 +++++ .../analysis-sqlite-temporal-cast.json | 294 ++++++++++++++++++ .../docs/docs/concepts/adapters/sqlite.mdx | 2 +- .../docs/concepts/semantic-compilation.mdx | 1 + 17 files changed, 610 insertions(+), 4 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/semantic_validation/_helpers/sqlite_temporal_casts.rs create mode 100644 crates/sqlbuild-analysis/src/semantic_validation/tests/test_sqlite_temporal_casts.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_temporal_casts.py create mode 100644 tests/goldens/compiler/failure/analysis-sqlite-temporal-cast.json diff --git a/crates/sqlbuild-analysis/src/binding_locations/constants.rs b/crates/sqlbuild-analysis/src/binding_locations/constants.rs index 9bc5d6c902..93fce95212 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/constants.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/constants.rs @@ -4,3 +4,5 @@ pub(crate) const UNKNOWN_TABLE_INTERNAL_CODE: &str = "B000"; /// The severity whose repeats at a known span compile drops. pub(crate) const ERROR_SEVERITY: &str = "error"; +/// SQLite temporal-cast warnings, which every binding pass reports, so they are kept once. +pub(crate) const SQLITE_TEMPORAL_CAST_CODE: &str = "B104"; diff --git a/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs b/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs index 7c717f791d..c305fbf6c4 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs @@ -2,7 +2,9 @@ use std::collections::HashSet; -use crate::binding_locations::constants::{ERROR_SEVERITY, UNKNOWN_TABLE_INTERNAL_CODE}; +use crate::binding_locations::constants::{ + ERROR_SEVERITY, SQLITE_TEMPORAL_CAST_CODE, UNKNOWN_TABLE_INTERNAL_CODE, +}; use crate::binding_locations::models::{ AuthoredSpan, BindingSource, LocatedDiagnostic, SeenDiagnostic, }; @@ -56,7 +58,10 @@ pub fn locate_binding_diagnostics( line, column, }; - if seen.contains(&key) && start.is_some() && severity == ERROR_SEVERITY { + if seen.contains(&key) + && start.is_some() + && (severity == ERROR_SEVERITY || code == SQLITE_TEMPORAL_CAST_CODE) + { continue; } seen.insert(key); diff --git a/crates/sqlbuild-analysis/src/semantic_checks/constants.rs b/crates/sqlbuild-analysis/src/semantic_checks/constants.rs index 0f5f5702c9..98800e3e4d 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/constants.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/constants.rs @@ -132,6 +132,11 @@ pub(crate) const SEMANTIC_HELP: &[(&str, &str)] = &[ "rewrite the truth-value test null-safely, for example COALESCE(x, FALSE) for x IS TRUE, \ and NOT COALESCE(x, FALSE) or x IS DISTINCT FROM TRUE for x IS NOT TRUE", ), + ( + "B104", + "keep the ISO-8601 text without a cast, or convert it with datetime(), date() or \ + strftime()", + ), ( "B210", "use an expression whose inferred type satisfies this SQL construct", diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs index 17175e9868..b198ae8a54 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs @@ -2,6 +2,7 @@ use crate::semantic_validation::_helpers::catalog::diagnostic_row; use crate::semantic_validation::_helpers::function_names; +use crate::semantic_validation::_helpers::sqlite_temporal_casts; use crate::semantic_validation::_helpers::truth_tests; use crate::semantic_validation::_helpers::type_locations::locate_type_findings; use crate::semantic_validation::models::DiagnosticRow; @@ -167,6 +168,9 @@ pub(crate) fn map_diagnostics( sql, dialect, probes, )); errors.extend(truth_tests::unsupported_truth_test_errors(sql, dialect)); + errors.extend(sqlite_temporal_casts::sqlite_temporal_cast_warnings( + sql, dialect, + )); let valid = !errors .iter() .any(|error| error.severity == ValidationSeverity::Error); diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/mod.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/mod.rs index b75be9835b..664231fa4b 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/mod.rs @@ -4,5 +4,6 @@ pub mod function_names; pub mod identifiers; pub mod normalization; pub mod positions; +pub mod sqlite_temporal_casts; pub mod truth_tests; pub mod type_locations; diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/sqlite_temporal_casts.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/sqlite_temporal_casts.rs new file mode 100644 index 0000000000..c3a37312e8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/sqlite_temporal_casts.rs @@ -0,0 +1,120 @@ +//! `CAST(... AS TIMESTAMP)` and its date and time siblings on SQLite, which keep only a number. + +use crate::semantic_validation::_helpers::function_names::{Cursor, is_word_byte}; +use polyglot_sql::{DialectType, ValidationError}; +use sqlbuild_sqltext::sql_scan::main::non_code_end::non_code_end; +use sqlbuild_sqltext::sql_scan::models::{QuotePolicy, Unclosed}; + +pub(crate) const SQLITE_TEMPORAL_CAST_CODE: &str = "B104"; +const CAST_KEYWORD: &str = "CAST"; +const AS_KEYWORD: &str = "AS"; +const TEMPORAL_TYPES: [&str; 5] = ["TIMESTAMP", "TIMESTAMPTZ", "DATETIME", "DATE", "TIME"]; + +/// One authored `CAST(... AS )`, spanning `CAST` through the type name. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct TemporalCast { + pub(crate) start: usize, + pub(crate) end: usize, + pub(crate) type_name: String, +} + +/// Located B104 warnings for temporal casts, which SQLite applies with NUMERIC affinity. +pub(crate) fn sqlite_temporal_cast_warnings( + sql: &str, + dialect: DialectType, +) -> Vec { + if dialect != DialectType::SQLite { + return Vec::new(); + } + let mut cursor: Cursor = Cursor::new(); + temporal_casts(sql) + .into_iter() + .map(|cast| { + let message: String = format!( + "SQLite casts to {} with NUMERIC affinity and keeps only the leading number, so \ + '2026-01-01 10:00:00' becomes 2026", + cast.type_name + ); + cursor.advance(sql, cast.start); + let start: usize = cursor.characters; + let end: usize = start + sql[cast.start..cast.end].chars().count(); + ValidationError::warning(message, SQLITE_TEMPORAL_CAST_CODE) + .with_location(cursor.line, cursor.column) + .with_span(Some(start), Some(end)) + }) + .collect() +} + +/// Every top-level `AS ` inside a `CAST(...)`, outside comments and quoted text. +pub(crate) fn temporal_casts(sql: &str) -> Vec { + let bytes: &[u8] = sql.as_bytes(); + let mut casts: Vec = Vec::new(); + let mut open_casts: Vec<(usize, usize)> = Vec::new(); + let mut depth: usize = 0; + let mut pending_cast: Option = None; + let mut after_as: Option = None; + let mut index: usize = 0; + while index < bytes.len() { + let byte: u8 = bytes[index]; + if byte.is_ascii_whitespace() { + index += 1; + continue; + } + match non_code_end(bytes, index, QuotePolicy::COMPILER) { + Ok(Some(end)) => { + index = end; + continue; + } + Ok(None) => {} + Err(Unclosed::BlockComment | Unclosed::Quote | Unclosed::Parenthesis) => break, + } + if byte == b'(' { + depth += 1; + if let Some(start) = pending_cast.take() { + open_casts.push((start, depth)); + } + after_as = None; + index += 1; + continue; + } + pending_cast = None; + if byte == b')' { + if open_casts.last().is_some_and(|(_, level)| *level == depth) { + open_casts.pop(); + } + depth = depth.saturating_sub(1); + after_as = None; + index += 1; + continue; + } + if !is_word_byte(byte) { + after_as = None; + index += 1; + continue; + } + let word_start: usize = index; + while index < bytes.len() && is_word_byte(bytes[index]) { + index += 1; + } + let word: &str = &sql[word_start..index]; + if let Some(cast_start) = after_as.take() { + let upper: String = word.to_ascii_uppercase(); + if TEMPORAL_TYPES.contains(&upper.as_str()) { + casts.push(TemporalCast { + start: cast_start, + end: index, + type_name: upper, + }); + } + continue; + } + if word.eq_ignore_ascii_case(CAST_KEYWORD) { + pending_cast = Some(word_start); + } else if word.eq_ignore_ascii_case(AS_KEYWORD) + && let Some((cast_start, _)) = open_casts.last().filter(|(_, level)| *level == depth) + { + after_as = Some(*cast_start); + } + } + casts +} diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests.rs index 21e41d1335..e5b49dd034 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests.rs @@ -6,6 +6,8 @@ mod normalize_batch; mod positions; #[path = "tests/test_semantic_validation.rs"] mod semantic_validation; +#[path = "tests/test_sqlite_temporal_casts.rs"] +mod sqlite_temporal_casts; #[path = "tests/test_types.rs"] mod test_types; #[path = "tests/test_truth_tests.rs"] diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_sqlite_temporal_casts.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_sqlite_temporal_casts.rs new file mode 100644 index 0000000000..b0d4f5d2e5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_sqlite_temporal_casts.rs @@ -0,0 +1,61 @@ +use crate::semantic_validation::_helpers::sqlite_temporal_casts::{ + TemporalCast, sqlite_temporal_cast_warnings, temporal_casts, +}; +use crate::semantic_validation::tests::test_types::{ + TemporalCastDialectTestCase, TemporalCastScanTestCase, +}; +use polyglot_sql::DialectType; + +#[test] +fn given_authored_sql_when_scanning_temporal_casts_then_finds_only_cast_types() { + let test_cases = [ + TemporalCastScanTestCase { + description: "timestamp and date casts are found in any case", + sql: "SELECT CAST(a AS TIMESTAMP), cast(b as date), CAST(c AS TEXT) FROM t", + expected_casts: &[(7, 26, "TIMESTAMP"), (29, 43, "DATE")], + }, + TemporalCastScanTestCase { + description: "a nested cast's type is not the outer cast's", + sql: "SELECT CAST(COALESCE(a, CAST(1 AS TEXT)) AS DATETIME) FROM t", + expected_casts: &[(7, 52, "DATETIME")], + }, + TemporalCastScanTestCase { + description: "aliases, quoted text and comments are ignored", + sql: "SELECT (SELECT 1 AS date) AS d, 'CAST(x AS DATE)' -- CAST(y AS DATE)\nFROM t", + expected_casts: &[], + }, + ]; + for test_case in test_cases { + let found: Vec = temporal_casts(test_case.sql); + let spans: Vec<(usize, usize, &str)> = found + .iter() + .map(|cast| (cast.start, cast.end, cast.type_name.as_str())) + .collect(); + assert_eq!(spans, test_case.expected_casts, "{}", test_case.description); + } +} + +#[test] +fn given_dialect_when_checking_temporal_casts_then_only_sqlite_warns() { + let test_cases = [ + TemporalCastDialectTestCase { + description: "SQLite warns about a timestamp cast", + dialect: DialectType::SQLite, + expected_warnings: 1, + }, + TemporalCastDialectTestCase { + description: "DuckDB casts timestamps natively", + dialect: DialectType::DuckDB, + expected_warnings: 0, + }, + ]; + for test_case in test_cases { + assert_eq!( + sqlite_temporal_cast_warnings("SELECT CAST(a AS TIMESTAMP) FROM t", test_case.dialect) + .len(), + test_case.expected_warnings, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs index 9f58f8b22e..201a5dc6fc 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_types.rs @@ -73,3 +73,15 @@ pub(crate) struct LocatedTruthTestDiagnosticTestCase { pub(crate) expected_location: (Option, Option), pub(crate) expected_span: (Option, Option), } + +pub(crate) struct TemporalCastScanTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_casts: &'static [(usize, usize, &'static str)], +} + +pub(crate) struct TemporalCastDialectTestCase { + pub(crate) description: &'static str, + pub(crate) dialect: polyglot_sql::DialectType, + pub(crate) expected_warnings: usize, +} diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py index 8f95cc214a..f0b3c21434 100644 --- a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py @@ -38,6 +38,9 @@ 'name = "failure_corpus"\nadapter = "snowflake"\n\n' '[connection]\naccount = "example"\nuser = "builder"\n' ) +_SQLITE_CONFIG: str = ( + 'name = "failure_corpus"\nadapter = "sqlite"\n\n[connection]\npath = "failure.sqlite"\n' +) _POSTGRES_CONFIG: str = ( 'name = "failure_corpus"\nadapter = "postgres"\n\n' '[connection]\nhost = "localhost"\ndatabase = "failure"\nschema = "public"\n' @@ -154,6 +157,19 @@ def _binding_cases() -> tuple[FailureCase, ...]: ), }, ), + failure_case( + name="analysis-sqlite-temporal-cast", + expected_code="B002", + expected_warning_code="B104", + expected_codes=("B002", "B104"), + files={ + FAILURE_CONFIG_PATH: _SQLITE_CONFIG, + **_mart( + "SELECT customer_id, CAST(amount AS TIMESTAMP) AS total_amount, missing\n" + f"FROM {_ORDERS}\n" + ), + }, + ), ) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md index cf41d899c8..2a329bd9a4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md @@ -76,7 +76,7 @@ SQLite stores values by type affinity, so SQLBuild keeps column types explicitly its declared type; a computed column gets the type SQLBuild infers at compile time. - Timestamps and dates are ISO-8601 text. `CAST(x AS TIMESTAMP)` and `CAST(x AS DATE)` do not produce timestamps in SQLite (they keep only the leading number), so write timestamps as text - literals or use SQLite date functions. SQLBuild never applies such a cast itself: source, + literals or use SQLite date functions. Compile warns with `B104` at each such cast in model SQL. SQLBuild never applies such a cast itself: source, contract and type-change casts to a date or time type keep the stored text, and source loaders write Python `datetime` values as UTC text in the form `YYYY-MM-DD HH:MM:SS[.ffffff]`. - Cursor windows, watermarks and window deletes compare timestamp cursors as UTC text in the diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md index 9da9c2427e..6dc5b1d9ee 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md @@ -207,6 +207,7 @@ the notice never reduces them to a count alone. | B101 | E202 or SQLBuild | Unknown function, or a built-in spelling the dialect does not accept | | B102 | E203 or project declaration | Invalid function arity | | B103 | SQLBuild | Truth-value test the dialect does not accept | +| B104 | SQLBuild | Warning: a SQLite `CAST` to a date or time type, which keeps only the leading number | | B210–B219 | E210–E219 | Proven native type/shape errors on enabled dialects | | B211–B218 | W210–W216 | Implicit conversions; blocking, fixed by an explicit `CAST` or comparison | | B216 | E216 | Set-operation, subquery, or row arity | diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py index 14f7e92203..3173523ee5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py @@ -108,3 +108,12 @@ class TextTimestampCursorE2ETestCase: day_expression: str expected_exit_code: int expected_fragment: str + + +@dataclass(frozen=True) +class SqliteTemporalCastE2ETestCase: + """A SQLite model selecting one expression, and the B104 warnings compile reports for it.""" + + description: str + expression: str + expected_warning_count: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_temporal_casts.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_temporal_casts.py new file mode 100644 index 0000000000..b8dc12dc73 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_temporal_casts.py @@ -0,0 +1,73 @@ +"""E2E coverage of the compile warning for date and time casts, which SQLite turns into numbers.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.sqlite._test_types import ( + SqliteTemporalCastE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteTemporalCastE2ETestCase( + description="a timestamp cast warns once at the cast", + expression="CAST(o.ordered_at AS TIMESTAMP)", + expected_warning_count=1, + ), + SqliteTemporalCastE2ETestCase( + description="a date cast around a text cast warns once", + expression="CAST(CAST(o.ordered_at AS TEXT) AS DATE)", + expected_warning_count=1, + ), + SqliteTemporalCastE2ETestCase( + description="a SQLite date function does not warn", + expression="datetime(o.ordered_at)", + expected_warning_count=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_sqlite_temporal_cast_when_compiling_then_b104_warns( + test_case: SqliteTemporalCastE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "shop"\nadapter = "sqlite"\n\n[connection]\npath = "shop.sqlite"\n\n' + "[scopes]\nenforce_placement = false\n" + ), + "models/orders.sql": ( + "MODEL (description 'Test model orders.');\n" + "SELECT 1 AS order_id, '2026-01-01 10:00:00' AS ordered_at\n" + ), + "models/order_times.sql": ( + "MODEL (description 'Test model order_times.');\n" + f'SELECT o.order_id, {test_case.expression} AS ordered_at FROM __ref("orders") AS o\n' + ), + }, + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", "--json"), project_dir=project_dir + ) + + assert compiled.returncode == 0, compiled.stdout + compiled.stderr + report: dict[str, Any] = json.loads(compiled.stdout) + located: list[tuple[str, str, int, int]] = [ + (item["code"], item["path"], item["line"], item["column"]) for item in report["diagnostics"] + ] + assert located.count(("B104", "models/order_times.sql", 2, 20)) == ( + test_case.expected_warning_count + ), test_case.description + assert [item[0] for item in located].count("B104") == test_case.expected_warning_count diff --git a/tests/goldens/compiler/failure/analysis-sqlite-temporal-cast.json b/tests/goldens/compiler/failure/analysis-sqlite-temporal-cast.json new file mode 100644 index 0000000000..3de33d2fe5 --- /dev/null +++ b/tests/goldens/compiler/failure/analysis-sqlite-temporal-cast.json @@ -0,0 +1,294 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "B002", + "message": "Unknown column 'missing' in stg_orders", + "resource_type": "model", + "resource_name": "customer_totals", + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 64, + "location": { + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 64, + "end_line": 5, + "end_column": 71 + }, + "notes": [ + "stg_orders has: amount, customer_id, order_id, status" + ] + }, + { + "phase": "compile", + "severity": "warning", + "code": "B104", + "message": "SQLite casts to TIMESTAMP with NUMERIC affinity and keeps only the leading number, so '2026-01-01 10:00:00' becomes 2026", + "resource_type": "model", + "resource_name": "customer_totals", + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 21, + "location": { + "path": "models/marts/customer_totals.sql", + "line": 5, + "column": 21, + "end_line": 5, + "end_column": 45 + }, + "help": "keep the ISO-8601 text without a cast, or convert it with datetime(), date() or strftime()" + } + ] + } + ], + "compiled": { + "models/marts/customer_totals.sql": [ + "SELECT customer_id, CAST(amount AS TIMESTAMP) AS total_amount, missing", + "FROM __ref(\"stg_orders\")", + "" + ], + "models/staging/stg_orders.sql": [ + "SELECT order_id, customer_id, amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, + "manifest": { + "nodes": { + "model.failure_corpus.customer_totals": { + "database": null, + "schema": "main", + "name": "customer_totals", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/marts/customer_totals.sql", + "original_file_path": "models/marts/customer_totals.sql", + "unique_id": "model.failure_corpus.customer_totals", + "fqn": [ + "failure_corpus", + "models", + "marts", + "customer_totals" + ], + "alias": "customer_totals", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Order totals per customer", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "customer_totals", + "depends_on": { + "macros": [], + "nodes": [ + "model.failure_corpus.stg_orders" + ] + } + }, + "model.failure_corpus.stg_orders": { + "database": null, + "schema": "main", + "name": "stg_orders", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/staging/stg_orders.sql", + "original_file_path": "models/staging/stg_orders.sql", + "unique_id": "model.failure_corpus.stg_orders", + "fqn": [ + "failure_corpus", + "models", + "staging", + "stg_orders" + ], + "alias": "stg_orders", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Staged orders", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "stg_orders", + "depends_on": { + "macros": [], + "nodes": [ + "source.failure_corpus.raw_orders" + ] + } + } + }, + "sources": { + "source.failure_corpus.raw_orders": { + "database": null, + "schema": "", + "name": "raw_orders", + "resource_type": "source", + "package_name": "failure_corpus", + "path": "sources/raw.yml", + "original_file_path": "sources/raw.yml", + "unique_id": "source.failure_corpus.raw_orders", + "fqn": [ + "failure_corpus", + "raw_orders" + ], + "source_name": "raw_orders", + "source_description": "", + "loader": "", + "identifier": "raw_orders", + "quoting": {}, + "loaded_at_field": null, + "freshness": null, + "external": null, + "description": "Orders feed.", + "columns": { + "order_id": { + "name": "order_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "customer_id": { + "name": "customer_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "amount": { + "name": "amount", + "description": "", + "meta": {}, + "data_type": "DOUBLE", + "constraints": [], + "quote": null, + "tags": [] + }, + "status": { + "name": "status", + "description": "", + "meta": {}, + "data_type": "VARCHAR", + "constraints": [], + "quote": null, + "tags": [] + } + }, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": {}, + "patch_path": null, + "unrendered_config": {}, + "relation_name": null + } + }, + "macros": {}, + "exposures": {}, + "metrics": {}, + "groups": {}, + "selectors": {}, + "disabled": {}, + "parent_map": { + "model.failure_corpus.customer_totals": [ + "model.failure_corpus.stg_orders" + ], + "model.failure_corpus.stg_orders": [ + "source.failure_corpus.raw_orders" + ], + "source.failure_corpus.raw_orders": [] + }, + "group_map": {}, + "docs": {}, + "saved_queries": {}, + "semantic_models": {}, + "unit_tests": {} + } +} diff --git a/website/src/content/docs/docs/concepts/adapters/sqlite.mdx b/website/src/content/docs/docs/concepts/adapters/sqlite.mdx index 3fceb698e1..e458073b1d 100644 --- a/website/src/content/docs/docs/concepts/adapters/sqlite.mdx +++ b/website/src/content/docs/docs/concepts/adapters/sqlite.mdx @@ -62,7 +62,7 @@ SQLite stores values by type affinity, so SQLBuild keeps column types explicitly its declared type; a computed column gets the type SQLBuild infers at compile time. - Timestamps and dates are ISO-8601 text. `CAST(x AS TIMESTAMP)` and `CAST(x AS DATE)` do not produce timestamps in SQLite (they keep only the leading number), so write timestamps as text - literals or use SQLite date functions. SQLBuild never applies such a cast itself: source, + literals or use SQLite date functions. Compile warns with `B104` at each such cast in model SQL. SQLBuild never applies such a cast itself: source, contract and type-change casts to a date or time type keep the stored text, and source loaders write Python `datetime` values as UTC text in the form `YYYY-MM-DD HH:MM:SS[.ffffff]`. - Cursor windows, watermarks and window deletes compare timestamp cursors as UTC text in the diff --git a/website/src/content/docs/docs/concepts/semantic-compilation.mdx b/website/src/content/docs/docs/concepts/semantic-compilation.mdx index 0b6eefbe1b..1387a963ec 100644 --- a/website/src/content/docs/docs/concepts/semantic-compilation.mdx +++ b/website/src/content/docs/docs/concepts/semantic-compilation.mdx @@ -197,6 +197,7 @@ the notice never reduces them to a count alone. | B101 | E202 or SQLBuild | Unknown function, or a built-in spelling the dialect does not accept | | B102 | E203 or project declaration | Invalid function arity | | B103 | SQLBuild | Truth-value test the dialect does not accept | +| B104 | SQLBuild | Warning: a SQLite `CAST` to a date or time type, which keeps only the leading number | | B210–B219 | E210–E219 | Proven native type/shape errors on enabled dialects | | B211–B218 | W210–W216 | Implicit conversions; blocking, fixed by an explicit `CAST` or comparison | | B216 | E216 | Set-operation, subquery, or row arity | From a507c96fb33b63b1665f919b6c1201e456e5a83e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:13:48 +0000 Subject: [PATCH 07/32] fix(dbt): reject --json on sqb dbt run and sqb dbt build sqb dbt run --json and sqb dbt build --json accepted the flag but wrote no JSON document: stdout held SQLBuild result rows and dbt output, so a caller parsing it failed. Both commands now reject --json with C238 before dbt starts and point at sqb dbt plan --json, the command that has a JSON document. BREAKING CHANGE: sqb dbt run and sqb dbt build fail with C238 when given --json, instead of ignoring it. --- .../_helpers/dbt_interop/runtime_config.rs | 14 +++++- .../sqlbuild/references/docs/cli/dbt.md | 3 ++ .../cli/commands/main/dbt/_test_types.py | 10 +++++ .../cli/commands/main/dbt/test_json_option.py | 45 +++++++++++++++++++ website/src/content/docs/docs/cli/dbt.mdx | 3 ++ 5 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_json_option.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs index 09300d8282..54133326f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs @@ -28,6 +28,10 @@ const UNRECOGNIZED_HELP: &str = "SQLBuild flags go before `--`; pass raw dbt fla const PARSER_HELP: &str = "Run `-h` to list supported flags; pass raw dbt flags after `--`."; const EXECUTION_COMMANDS: [&str; 3] = ["plan", "run", "build"]; const DISPLAY_FLAGS: [&str; 3] = ["--json", "--verbose", "-v"]; +const PLAN_COMMAND: &str = "plan"; +const JSON_FLAG: &str = "--json"; +const JSON_HELP: &str = + "Use sqb dbt plan --json for a machine-readable preview of the combined work."; const DEFER_FLAG: &str = "--defer"; const VARS_FLAG: &str = "--vars"; const STATE_FLAG: &str = "--state"; @@ -173,7 +177,15 @@ pub(crate) fn validate_dbt_execution_args( if !EXECUTION_COMMANDS.contains(&name.as_str()) { return Ok(()); } - let display_free: Vec = texts(args)? + let given: Vec = texts(args)?; + if name != PLAN_COMMAND && given.iter().any(|arg| arg == JSON_FLAG) { + return Err(argument_error( + command.py(), + format!("sqb dbt {name} does not support --json"), + JSON_HELP, + )); + } + let display_free: Vec = given .into_iter() .filter(|arg| !DISPLAY_FLAGS.contains(&arg.as_str())) .collect(); diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dbt.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dbt.md index fa539c7253..0dfdb25eaa 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dbt.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/dbt.md @@ -34,6 +34,9 @@ Build dbt models first (including dbt tests), then SQLBuild models with audits. sqb dbt build [--select ...] [--exclude ...] ``` +`sqb dbt run` and `sqb dbt build` have no JSON output: they reject `--json` with `C238` before +dbt starts. Use `sqb dbt plan --json` for a machine-readable preview of the combined work. + ### Plan safety policies After dbt finishes, `sqb dbt run` and `sqb dbt build` apply the same plan safety policies as diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py index 2ff3651c63..ac342829ef 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py @@ -130,3 +130,13 @@ class DbtTargetConcurrencyE2ETestCase: target_settings: str command: tuple[str, ...] expected_fragment: str + + +@dataclass(frozen=True) +class DbtJsonOptionE2ETestCase: + """A `sqb dbt` command given `--json`, and whether it is rejected before dbt runs.""" + + description: str + command: tuple[str, ...] + expected_returncode: int + expected_stderr_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_json_option.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_json_option.py new file mode 100644 index 0000000000..7ed4894e24 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_json_option.py @@ -0,0 +1,45 @@ +"""E2E coverage that `sqb dbt run` and `sqb dbt build` reject `--json`, which they cannot honour.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.dbt._test_types import DbtJsonOptionE2ETestCase +from tests.e2e.src.sqlbuild.cli.commands.main.dbt.helpers import prepare_dbt_interop_project +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + DbtJsonOptionE2ETestCase( + description="dbt run rejects --json", + command=("dbt", "run", "--json", "--select", "local_only"), + expected_returncode=1, + expected_stderr_fragment="error[C238]: sqb dbt run does not support --json", + ), + DbtJsonOptionE2ETestCase( + description="dbt build rejects a leading global --json", + command=("--json", "dbt", "build"), + expected_returncode=1, + expected_stderr_fragment="error[C238]: sqb dbt build does not support --json", + ), + ], + ids=lambda case: case.description, +) +def test_given_json_option_when_running_dbt_interop_then_it_is_rejected_before_dbt_runs( + test_case: DbtJsonOptionE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_dbt_interop_project(tmp_path=tmp_path) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + assert result.returncode == test_case.expected_returncode, result.stdout + result.stderr + assert test_case.expected_stderr_fragment in result.stderr, test_case.description + assert "sqb dbt plan --json" in result.stderr + assert not (project_dir / "dbt_interop.duckdb").exists() diff --git a/website/src/content/docs/docs/cli/dbt.mdx b/website/src/content/docs/docs/cli/dbt.mdx index 49e6e0a693..1238052e98 100644 --- a/website/src/content/docs/docs/cli/dbt.mdx +++ b/website/src/content/docs/docs/cli/dbt.mdx @@ -33,6 +33,9 @@ Build dbt models first (including dbt tests), then SQLBuild models with audits. sqb dbt build [--select ...] [--exclude ...] ``` +`sqb dbt run` and `sqb dbt build` have no JSON output: they reject `--json` with `C238` before +dbt starts. Use `sqb dbt plan --json` for a machine-readable preview of the combined work. + ### Plan safety policies After dbt finishes, `sqb dbt run` and `sqb dbt build` apply the same plan safety policies as From 4600aa4ac81a2f77c936155076cdc22cb0de2efa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:26:30 +0000 Subject: [PATCH 08/32] fix(scenario): load replayed VARIANT values from their JSON text Local scenario replay inserted VARIANT fixture values as Python objects, which DuckDB stores as variants of different member types than a JSON parse gives, so an expected '{"qty": 2}'::JSON::VARIANT never matched the replayed row and strict replay failed with X506. The loader now loads each VARIANT value through CAST(CAST(? AS JSON) AS VARIANT): DuckDB captures are serialised to JSON, and a string captured from another warehouse, whose driver returns semi-structured values as JSON text, is used as that text when it parses. --- .../scenario/_helpers/local/snapshots.py | 44 +++++++++++++-- src/sqlbuild/executor/scenario/constants.py | 4 ++ .../cli/commands/main/scenario/helpers.py | 13 +++-- .../main/scenario/test_local_variant.py | 9 +++ .../executor/scenario/_helpers/_test_types.py | 10 ++++ .../scenario/_helpers/test_local_snapshots.py | 56 +++++++++++++++++++ 6 files changed, 127 insertions(+), 9 deletions(-) diff --git a/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py b/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py index 313c83e736..0cedb40aa3 100644 --- a/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py +++ b/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from pathlib import Path from typing import Any @@ -15,12 +16,15 @@ scenario_snapshot_root, ) from sqlbuild.executor.scenario.constants import ( + LOCAL_CAPTURE_ADAPTER, + LOCAL_TYPE_VARIANT, SCENARIO_LOCAL_JSONL_INVALID, SCENARIO_LOCAL_LOAD_FAILED, SCENARIO_LOCAL_MANIFEST_INVALID, SCENARIO_LOCAL_SNAPSHOT_MISSING, SCENARIO_LOCAL_SNAPSHOT_STALE, SCENARIO_LOCAL_TYPE_INVALID, + VARIANT_FROM_JSON_PLACEHOLDER, ) from sqlbuild.executor.scenario.models import ( ScenarioLocalSnapshotLoadedRelation, @@ -111,6 +115,7 @@ def load_scenario_snapshot_into_duckdb( snapshot_root=snapshot_root, manifest_path=manifest_path, relation=relation, + capture_adapter=manifest.capture_adapter, ) ) @@ -214,6 +219,7 @@ def _load_snapshot_relation( snapshot_root: Path, manifest_path: Path, relation: ScenarioSnapshotRelation, + capture_adapter: str, ) -> ScenarioLocalSnapshotLoadedRelation: del manifest_path file_path: Path = snapshot_root / relation.file_path @@ -242,6 +248,7 @@ def _load_snapshot_relation( relation=relation, table_name=table_name, rows=rows, + native_values=capture_adapter == LOCAL_CAPTURE_ADAPTER, ) return ScenarioLocalSnapshotLoadedRelation( kind=relation.kind, @@ -276,16 +283,25 @@ def _insert_rows( relation: ScenarioSnapshotRelation, table_name: str, rows: tuple[dict[str, object], ...], + native_values: bool, ) -> None: if not rows: return column_names: tuple[str, ...] = tuple(column.name for column in relation.columns) - insert_sql: str = _build_insert_sql(table_name=table_name, column_names=column_names) + variant_columns: tuple[bool, ...] = tuple( + column.local_type.strip().upper() == LOCAL_TYPE_VARIANT for column in relation.columns + ) + insert_sql: str = _build_insert_sql( + table_name=table_name, column_names=column_names, variant_columns=variant_columns + ) row_values: list[tuple[object, ...]] = [] for row in rows: values: list[object] = [] - for column_name in column_names: - values.append(_row_value(row=row, column_name=column_name)) + for column_name, variant in zip(column_names, variant_columns, strict=True): + value: object = _row_value(row=row, column_name=column_name) + values.append( + _variant_json_text(value=value, native_values=native_values) if variant else value + ) row_values.append(tuple(values)) try: connection.executemany(insert_sql, row_values) @@ -340,9 +356,27 @@ def _build_create_table_sql(*, table_name: str, columns: tuple[ScenarioSnapshotC return f"CREATE TABLE {_quote_identifier(table_name)} ({column_defs})" -def _build_insert_sql(*, table_name: str, column_names: tuple[str, ...]) -> str: +def _variant_json_text(*, value: object, native_values: bool) -> str | None: + """A VARIANT value as JSON text; another adapter's string that parses is already JSON text.""" + + if value is None: + return None + if isinstance(value, str) and not native_values: + try: + json.loads(value) + except ValueError: + return json.dumps(value) + return value + return json.dumps(value) + + +def _build_insert_sql( + *, table_name: str, column_names: tuple[str, ...], variant_columns: tuple[bool, ...] +) -> str: columns: str = ", ".join(_quote_identifier(column_name) for column_name in column_names) - placeholders: str = ", ".join("?" for _ in column_names) + placeholders: str = ", ".join( + VARIANT_FROM_JSON_PLACEHOLDER if variant else "?" for variant in variant_columns + ) return f"INSERT INTO {_quote_identifier(table_name)} ({columns}) VALUES ({placeholders})" diff --git a/src/sqlbuild/executor/scenario/constants.py b/src/sqlbuild/executor/scenario/constants.py index 44ca0876c3..e24205edb9 100644 --- a/src/sqlbuild/executor/scenario/constants.py +++ b/src/sqlbuild/executor/scenario/constants.py @@ -38,6 +38,7 @@ LOCAL_TYPE_TIME: str = "TIME" LOCAL_TYPE_TIMESTAMP: str = "TIMESTAMP" LOCAL_TYPE_TIMESTAMPTZ: str = "TIMESTAMPTZ" +LOCAL_TYPE_VARIANT: str = "VARIANT" LOCAL_TYPE_UINT128: str = "UINT128" LOCAL_TYPE_VARCHAR: str = "VARCHAR" @@ -162,3 +163,6 @@ UNTYPED_ARRAY_TYPE: str = "[]" JSONL_BLANK_LINE: str = "" LOCAL_SCENARIO_DIALECT: str = "duckdb" + +LOCAL_CAPTURE_ADAPTER: str = "duckdb" +VARIANT_FROM_JSON_PLACEHOLDER: str = "CAST(CAST(? AS JSON) AS VARIANT)" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py index a0f92daa25..8497adcbbd 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py @@ -1137,7 +1137,11 @@ def build_connect_by_scenario_project_files() -> dict[str, str]: } -def build_local_variant_scenario_project_files() -> dict[str, str]: +def build_local_variant_scenario_project_files( + *, + output: str = "o.payload IS NOT NULL AS has_payload", + expected: str = "TRUE AS has_payload", +) -> dict[str, str]: """Build an in-memory DuckDB scenario whose source fixture holds a VARIANT column.""" return { @@ -1152,15 +1156,16 @@ def build_local_variant_scenario_project_files() -> dict[str, str]: ), "models/stg_orders.sql": ( "MODEL (description 'Test model stg_orders.');\n" - "SELECT o.order_id, o.payload IS NOT NULL AS has_payload\n" + f"SELECT o.order_id, {output}\n" 'FROM __source("raw_orders") AS o\n' ), "tests/scenarios/orders__keep_payload.sql": ( "SCENARIO (description 'Test scenario orders__keep_payload.');\n" "WITH\n" "__source__raw_orders AS (\n" - " SELECT 1 AS order_id, '{\"qty\": 2}'::JSON::VARIANT AS payload\n),\n" - "__expected__stg_orders AS (SELECT 1 AS order_id, TRUE AS has_payload)\n" + " SELECT 1 AS order_id,\n" + ' \'{"qty": 2, "tags": ["a"]}\'::JSON::VARIANT AS payload\n),\n' + f"__expected__stg_orders AS (SELECT 1 AS order_id, {expected})\n" "SELECT 1\n" ), } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py index 82ab7d6f4a..802a8cb242 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py @@ -25,6 +25,15 @@ scenario_name="orders__keep_payload", expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", ), + LocalVariantScenarioTestCase( + description="an expected VARIANT object compares equal to the replayed fixture", + repo_files=build_local_variant_scenario_project_files( + output="o.payload", + expected='\'{"qty": 2, "tags": ["a"]}\'::JSON::VARIANT AS payload', + ), + scenario_name="orders__keep_payload", + expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py index 63e0cc8ae0..3f0e8bfdc8 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py @@ -192,3 +192,13 @@ class LocalScenarioSqlTestCase: source_dialect: str | None sql: str expected_sql: str + + +@dataclass(frozen=True) +class ScenarioLocalVariantLoadTestCase: + """A Snowflake-captured VARIANT JSONL value and the variant type it must load as.""" + + description: str + value: object + expected_variant_type: str + expected_matches_parsed_json: bool diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_local_snapshots.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_local_snapshots.py index 891bf8bb83..5f9f20be74 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_local_snapshots.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_local_snapshots.py @@ -21,6 +21,7 @@ from tests.unit.src.sqlbuild.executor.scenario._helpers._test_types import ( ScenarioLocalSnapshotLoadErrorTestCase, ScenarioLocalSnapshotLoadTestCase, + ScenarioLocalVariantLoadTestCase, ) from tests.unit.src.sqlbuild.executor.scenario._helpers.helpers import ( build_local_snapshot_load_manifest, @@ -178,3 +179,58 @@ def test_given_invalid_local_snapshot_when_loading_then_raises_coded_error( expected_fragment: str for expected_fragment in test_case.expected_error_fragments: assert expected_fragment in error_text + + +@pytest.mark.parametrize( + "test_case", + [ + ScenarioLocalVariantLoadTestCase( + description="driver JSON text of an object loads as that object", + value='{\n "qty": 2\n}', + expected_variant_type="OBJECT(qty)", + expected_matches_parsed_json=True, + ), + ScenarioLocalVariantLoadTestCase( + description="driver JSON text of a string loads as the string", + value='"placed"', + expected_variant_type="VARCHAR", + expected_matches_parsed_json=False, + ), + ScenarioLocalVariantLoadTestCase( + description="text that is not JSON loads as a string", + value="placed", + expected_variant_type="VARCHAR", + expected_matches_parsed_json=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_variant_text_when_loading_into_duckdb_then_it_loads_as_parsed_json( + tmp_path: Path, + test_case: ScenarioLocalVariantLoadTestCase, +) -> None: + manifest: ScenarioSnapshotManifest = build_local_snapshot_load_manifest( + input_fingerprint=INPUT_FINGERPRINT, + columns=(("payload", "VARIANT", "VARIANT"),), + ) + write_local_snapshot_load_test_files( + project_dir=tmp_path, manifest=manifest, rows=({"PAYLOAD": test_case.value},) + ) + connection: duckdb.DuckDBPyConnection = duckdb.connect(":memory:") + + result: ScenarioLocalSnapshotLoadResult = load_scenario_snapshot_into_duckdb( + project_dir=tmp_path, + scenario_name=SCENARIO_NAME, + current_input_fingerprint=INPUT_FINGERPRINT, + connection=connection, + ) + loaded: tuple[object, ...] | None = connection.execute( + "SELECT variant_typeof(payload), " + "payload IN (SELECT CAST(CAST('{\"qty\": 2}' AS JSON) AS VARIANT)) " + f'FROM "{result.relations[0].table_name}"' + ).fetchone() + + assert loaded == ( + test_case.expected_variant_type, + test_case.expected_matches_parsed_json, + ), test_case.description From e47d174beab91760b25959b60a697189554b42c6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:35:38 +0000 Subject: [PATCH 09/32] fix(contracts): match Snowflake star outputs to lowercase contract columns On Snowflake a star expands to the upstream's columns as Snowflake names them, upper case, so SELECT o.* under a contract declaring order_id and status reported K005 for ORDER_ID and STATUS although Snowflake treats the unquoted names as the same columns. On Snowflake a declared column now also names the upper-case form of its name; an undeclared column a star adds is still reported. --- .../src/contracts/_helpers/columns.rs | 23 +++++- .../src/contracts/_helpers/type_comparison.rs | 6 ++ .../src/contracts/constants.rs | 2 + .../cli/commands/main/compile/_test_types.py | 10 +++ .../compile/test_snowflake_star_contracts.py | 73 +++++++++++++++++++ 5 files changed, 110 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_snowflake_star_contracts.py diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs index 8603bf2d2d..64a04fda35 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/columns.rs @@ -72,7 +72,12 @@ pub(crate) fn model_contract_diagnostics( .map(|column| (column.name.as_str(), column)) .collect(); if contract_enforced { - diagnostics.extend(extra_columns(model, schema, inferred_columns)); + diagnostics.extend(extra_columns( + model, + schema, + inferred_columns, + types.folds_upper(), + )); } let unchecked: HashSet<&str> = model .unchecked_output_columns @@ -80,7 +85,12 @@ pub(crate) fn model_contract_diagnostics( .map(String::as_str) .collect(); for (index, declared) in schema.columns.iter().enumerate() { - let Some(inferred) = inferred_by_name.get(declared.name.as_str()) else { + let Some(inferred) = inferred_by_name.get(declared.name.as_str()).or_else(|| { + types + .folds_upper() + .then(|| inferred_by_name.get(declared.name.to_uppercase().as_str())) + .flatten() + }) else { if validate_declared_shape && !model.fast_lineage_has_star { diagnostics.push(missing_column(model, index, declared, contract_enforced)); } @@ -147,15 +157,20 @@ fn missing_declarations(model: &ContractModel) -> ContractDiagnostic { } } +/// Inferred columns no declared column names; a folded `order_id` also names a star's `ORDER_ID`. fn extra_columns( model: &ContractModel, schema: &ContractSchema, inferred_columns: &[InferredOutputColumn], + folds_upper: bool, ) -> Vec { - let declared_names: HashSet<&str> = schema + let declared_names: HashSet = schema .columns .iter() - .map(|column| column.name.as_str()) + .flat_map(|column| { + let folded: Option = folds_upper.then(|| column.name.to_uppercase()); + std::iter::once(column.name.clone()).chain(folded) + }) .collect(); let help: &str = if schema.named_schema { EXTRA_COLUMN_NAMED_SCHEMA_HELP diff --git a/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs b/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs index a922a1dc2a..b72f041818 100644 --- a/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs +++ b/crates/sqlbuild-analysis/src/contracts/_helpers/type_comparison.rs @@ -3,6 +3,7 @@ use std::cell::RefCell; use std::collections::HashMap; +use crate::contracts::constants::UPPER_FOLDING_DIALECT; use crate::type_system::errors::TypeNormalizationError; use crate::type_system::main::normalize_type::normalize_type; use crate::type_system::models::{NormalizedType, TypeNormalization}; @@ -21,6 +22,11 @@ pub(crate) struct TypeComparer<'a> { } impl<'a> TypeComparer<'a> { + /// Whether the dialect folds unquoted identifiers to upper case, as star expansion names them. + pub(crate) fn folds_upper(&self) -> bool { + self.dialect == UPPER_FOLDING_DIALECT + } + pub(crate) fn new(dialect: &'a str) -> Self { Self { dialect, diff --git a/crates/sqlbuild-analysis/src/contracts/constants.rs b/crates/sqlbuild-analysis/src/contracts/constants.rs index 26e6e274b7..35d840309d 100644 --- a/crates/sqlbuild-analysis/src/contracts/constants.rs +++ b/crates/sqlbuild-analysis/src/contracts/constants.rs @@ -39,3 +39,5 @@ pub(crate) const PROMOTION_REMOVE_CONTRACT_HELP: &str = "or remove the model's e pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; pub(crate) const ADDITIONAL_HELP_SEPARATOR: &str = "\n = help: "; +/// The dialect that folds unquoted identifiers to upper case. +pub(crate) const UPPER_FOLDING_DIALECT: &str = "snowflake"; diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 691861929f..a31c7012e5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1322,3 +1322,13 @@ class MacroDictKeyCompileE2ETestCase: argument: str expected_exit_code: int expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class SnowflakeStarContractE2ETestCase: + """A Snowflake model reading a star under a lowercase contract, and the K005 it reports.""" + + description: str + upstream_columns: str + expected_exit_code: int + expected_k005_columns: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_snowflake_star_contracts.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_snowflake_star_contracts.py new file mode 100644 index 0000000000..160caf3bdd --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_snowflake_star_contracts.py @@ -0,0 +1,73 @@ +"""E2E coverage of Snowflake star outputs, which fold to upper case, against lowercase contracts.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + SnowflakeStarContractE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_PROJECT_TOML: str = ( + 'name = "shop"\nadapter = "snowflake"\n\n[connection]\naccount = "example"\n' + 'user = "builder"\n\n[defaults]\ndatabase = "ANALYTICS"\nschema = "MARTS"\n\n' + "[scopes]\nenforce_placement = false\n" +) +_CONTRACT: str = "contract enforced, columns (order_id (type INT), status (type VARCHAR))" + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeStarContractE2ETestCase( + description="a star over the declared columns satisfies the lowercase contract", + upstream_columns="1 AS order_id, CAST('a' AS VARCHAR) AS status", + expected_exit_code=0, + expected_k005_columns=(), + ), + SnowflakeStarContractE2ETestCase( + description="an undeclared column a star adds is still reported", + upstream_columns="1 AS order_id, CAST('a' AS VARCHAR) AS status, 2 AS extra", + expected_exit_code=1, + expected_k005_columns=("EXTRA",), + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_star_when_compiling_lowercase_contract_then_folded_names_match( + test_case: SnowflakeStarContractE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/orders.sql": ( + f"MODEL (description 'Test model orders.');\nSELECT {test_case.upstream_columns}\n" + ), + "models/order_rows.sql": ( + f"MODEL (description 'Test model order_rows.', {_CONTRACT});\n" + 'SELECT o.* FROM __ref("orders") AS o\n' + ), + }, + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", "--json"), project_dir=project_dir + ) + + report: dict[str, Any] = json.loads(compiled.stdout) + codes: list[str] = [item["code"] for item in report["diagnostics"]] + messages: list[str] = [item["message"] for item in report["diagnostics"]] + assert compiled.returncode == test_case.expected_exit_code, compiled.stdout + compiled.stderr + assert codes.count("K005") == len(test_case.expected_k005_columns), messages + assert set(messages) >= { + f"column '{name}' is not declared in enforced contract for model 'order_rows'" + for name in test_case.expected_k005_columns + }, test_case.description From d144678adbf1913c7f2ebe2dc305644851b62aa9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:46:00 +0000 Subject: [PATCH 10/32] feat(sqlite): load dlt and ingestr sources into SQLite projects sqb load ran Python loaders on SQLite, but dlt and ingestr integration loaders failed with "Adapter 'sqlite' does not support ... loaders". dlt now writes through its sqlalchemy destination into the project's database file, dataset main unless the source names one, and ingestr gets a sqlite:/// destination URI for the same file. --- .../src/dlt_destination/constants.rs | 5 ++ .../main/dlt_destination_plan.rs | 16 ++++- .../tests/test_dlt_destination_plan.rs | 16 +++++ .../ingestr_command/_helpers/destinations.rs | 8 +++ .../main/ingestr_destination_uri.rs | 3 +- .../tests/test_ingestr_command.rs | 7 ++ .../docs/concepts/adapters/sqlite.md | 1 + .../references/docs/integrations/dlt.md | 2 +- .../references/docs/integrations/ingestr.md | 1 + .../cli/commands/main/load/test_dlt.py | 66 +++++++++++++++++++ .../integrations/ingestr/test_ingestr.py | 53 +++++++++++++++ .../docs/docs/concepts/adapters/sqlite.mdx | 1 + .../content/docs/docs/integrations/dlt.mdx | 2 +- .../docs/docs/integrations/ingestr.mdx | 1 + 14 files changed, 178 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-core/src/dlt_destination/constants.rs b/crates/sqlbuild-core/src/dlt_destination/constants.rs index 920e01ac55..a924286ff8 100644 --- a/crates/sqlbuild-core/src/dlt_destination/constants.rs +++ b/crates/sqlbuild-core/src/dlt_destination/constants.rs @@ -13,6 +13,8 @@ pub(crate) const SQLBUILD_OWNED_DESTINATION_KEYS: [&str; 4] = [ ]; /// The dataset DuckDB destinations land in without an explicit one. pub(crate) const DUCKDB_DEFAULT_DATASET: &str = "main"; +/// The dataset of SQLite's own database file; dlt attaches a file per other dataset. +pub(crate) const SQLITE_MAIN_DATASET: &str = "main"; /// The suffix Snowflake account and host settings drop. pub(crate) const SNOWFLAKE_ACCOUNT_SUFFIX: &str = ".snowflakecomputing.com"; /// SQL Server's default port number. @@ -24,6 +26,9 @@ pub(crate) const SNOWFLAKE_OPTIONAL_SETTINGS: [&str; 4] = pub(crate) const BIGQUERY_ARGUMENTS: [(&str, &str); 2] = [("project_id", "project"), ("location", "location")]; pub(crate) const DUCKDB_ADAPTER: &str = "duckdb"; +pub(crate) const SQLITE_ADAPTER: &str = "sqlite"; +/// The dlt destination factory that writes SQLite through SQLAlchemy. +pub(crate) const SQLALCHEMY_FACTORY: &str = "sqlalchemy"; pub(crate) const MOTHERDUCK_ADAPTER: &str = "motherduck"; pub(crate) const POSTGRES_ADAPTER: &str = "postgres"; pub(crate) const SNOWFLAKE_ADAPTER: &str = "snowflake"; diff --git a/crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs b/crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs index 3e06044eb3..07f5ea30cf 100644 --- a/crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs +++ b/crates/sqlbuild-core/src/dlt_destination/main/dlt_destination_plan.rs @@ -2,7 +2,9 @@ use crate::dlt_destination::_helpers::credentials::{adapter_call, credentials, required}; use crate::dlt_destination::_helpers::request::{check_dataset_name, check_destination_keys}; -use crate::dlt_destination::constants::{DUCKDB_ADAPTER, DUCKDB_DEFAULT_DATASET}; +use crate::dlt_destination::constants::{ + DUCKDB_ADAPTER, DUCKDB_DEFAULT_DATASET, SQLALCHEMY_FACTORY, SQLITE_ADAPTER, SQLITE_MAIN_DATASET, +}; use crate::dlt_destination::errors::DltDestinationError; use crate::dlt_destination::models::{ DltDataset, DltDestinationPlan, DltDestinationRequest, DltValue, @@ -26,6 +28,18 @@ pub fn dlt_destination_plan( }, }); } + if adapter == SQLITE_ADAPTER { + let path: &str = required(request.connection, "path", SQLITE_ADAPTER)?; + return Ok(DltDestinationPlan { + factory: SQLALCHEMY_FACTORY, + arguments: credentials(DltValue::Text(format!("sqlite:///{path}"))), + dataset: if request.dataset_name.truthy { + DltDataset::Given + } else { + DltDataset::Default(SQLITE_MAIN_DATASET) + }, + }); + } check_dataset_name(adapter, request.dataset_name)?; let Some((factory, arguments)) = adapter_call(adapter, request.connection)? else { return Err(DltDestinationError { diff --git a/crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs b/crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs index 418fd98879..5174d1a6d8 100644 --- a/crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs +++ b/crates/sqlbuild-core/src/dlt_destination/tests/test_dlt_destination_plan.rs @@ -23,6 +23,22 @@ fn given_adapter_connection_when_planning_dlt_destination_then_matches_python_ma dataset: DltDataset::Default("main"), }), }, + DestinationPlanTestCase { + description: "sqlite writes its database file through sqlalchemy into main", + adapter: "sqlite", + strings: &[("path", "/data/warehouse.sqlite")], + port: None, + destination_keys: &[], + dataset_name: None, + expected_plan: Ok(DltDestinationPlan { + factory: "sqlalchemy", + arguments: vec![( + "credentials", + DltValue::Text("sqlite:////data/warehouse.sqlite".to_owned()), + )], + dataset: DltDataset::Default("main"), + }), + }, DestinationPlanTestCase { description: "postgres connection string uses host default and port", adapter: "postgres", diff --git a/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs b/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs index 86b6532aa8..f485354ef2 100644 --- a/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs +++ b/crates/sqlbuild-core/src/ingestr_command/_helpers/destinations.rs @@ -19,6 +19,14 @@ pub(crate) fn duckdb(connection: &LoaderConnection) -> Result Result { + Ok(format!( + "sqlite:///{}", + required(connection, "path", "sqlite")? + )) +} + /// MotherDuck's database URI with its optional token. pub(crate) fn motherduck(connection: &LoaderConnection) -> String { let database: &str = optional(connection, "database").unwrap_or_default(); diff --git a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs index 198bf35e36..3a472007b0 100644 --- a/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs +++ b/crates/sqlbuild-core/src/ingestr_command/main/ingestr_destination_uri.rs @@ -1,7 +1,7 @@ //! The ingestr destination URI for a SQLBuild warehouse connection. use crate::ingestr_command::_helpers::destinations::{ - bigquery, databricks, duckdb, motherduck, postgres, snowflake, sqlserver, + bigquery, databricks, duckdb, motherduck, postgres, snowflake, sqlite, sqlserver, }; use crate::ingestr_command::errors::IngestrCommandError; use crate::loader_connection::models::LoaderConnection; @@ -13,6 +13,7 @@ pub fn ingestr_destination_uri( ) -> Result { match adapter_name { "duckdb" => duckdb(connection), + "sqlite" => sqlite(connection), "motherduck" => Ok(motherduck(connection)), "postgres" => postgres(connection), "bigquery" => bigquery(connection), diff --git a/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs b/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs index e35dd77651..16bf812ffe 100644 --- a/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs +++ b/crates/sqlbuild-core/src/ingestr_command/tests/test_ingestr_command.rs @@ -18,6 +18,13 @@ fn given_connection_when_building_destination_uri_then_matches_python_quoting() port: None, expected_uri: Ok("duckdb:///warehouse.duckdb"), }, + DestinationUriTestCase { + description: "sqlite path", + adapter_name: "sqlite", + strings: &[("path", "/data/warehouse.sqlite")], + port: None, + expected_uri: Ok("sqlite:////data/warehouse.sqlite"), + }, DestinationUriTestCase { description: "motherduck prefix and token", adapter_name: "motherduck", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md index 2a329bd9a4..4a2a535e3a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md @@ -110,6 +110,7 @@ SQLite stores values by type affinity, so SQLBuild keeps column types explicitly | `sqb diff`, `sqb clone` | Supported; clones are physical copies | | Scenarios, `sqb scenario capture`, `--local` replay | Supported; see below | | Statement time limits | Connection interrupt | +| `sqb load`: Python, dlt and ingestr loaders | Supported; dlt writes through its `sqlalchemy` destination | ## Scenarios diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md index 851ae4f9cc..180d5af6a7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md @@ -115,7 +115,7 @@ Use SQLBuild's `write_disposition` (dlt's term), not `write_strategy`. `delete_i ### Destination -By default each resource loads into the database SQLBuild's adapter manages, with the dataset/schema derived from your target. An optional `destination` mapping passes extra settings to the dlt destination, but it cannot set `credentials`, `dataset_name`, or `default_schema_name` (SQLBuild manages those): +By default each resource loads into the database SQLBuild's adapter manages, with the dataset/schema derived from your target. On SQLite, dlt writes through its `sqlalchemy` destination into the project's database file (dataset `main`); the `sqlbuild[dlt]` extra installs the SQLAlchemy it needs. An optional `destination` mapping passes extra settings to the dlt destination, but it cannot set `credentials`, `dataset_name`, or `default_schema_name` (SQLBuild manages those): ```yaml dlt_sources: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md index 58cd89e2b4..60544e7351 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md @@ -138,6 +138,7 @@ SQLBuild automatically builds the ingestr destination URI from your adapter conn | Adapter | Destination URI format | |---------|----------------------| | DuckDB | `duckdb:///path/to/db.duckdb` | +| SQLite | `sqlite:///path/to/db.sqlite` | | MotherDuck | `motherduck://database` | | PostgreSQL | `postgresql://user:pass@host:port/db` | | Snowflake | `snowflake://user:pass@account/db/schema` | diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py index 21adbee791..17b823732c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_dlt.py @@ -2,7 +2,9 @@ from __future__ import annotations +import sqlite3 import subprocess +from contextlib import closing from pathlib import Path import pytest @@ -248,3 +250,67 @@ def test_given_dlt_rest_api_source_when_loading_then_table_is_materialized_and_q sql="SELECT order_id, amount FROM order_totals ORDER BY order_id", ) assert tuple(model_rows) == test_case.expected_model_rows + + +@pytest.mark.parametrize( + "test_case", + [ + DltLocalSourceE2ETestCase( + description="loads local jsonl filesystem source into sqlite", + expected_loaded_rows=((1, 10), (2, 20), (3, 30)), + expected_model_rows=((2, 20), (3, 30)), + ) + ], + ids=lambda case: case.description, +) +def test_given_sqlite_project_when_loading_dlt_filesystem_source_then_models_read_it( + tmp_path: Path, test_case: DltLocalSourceE2ETestCase +) -> None: + project_root: Path = tmp_path / "dlt_sqlite_project" + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="dlt_sqlite_project", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "dlt_sqlite_project"\nadapter = "sqlite"\n\n' + '[connection]\npath = "warehouse.sqlite"\n\n' + '[defaults]\nmaterialized = "table"\n' + ), + "sources/raw.yml": ( + "dlt_sources:\n" + " - type: filesystem\n" + " config:\n" + f' bucket_url: "file://{project_root / "data"}"\n' + ' file_glob: "orders.jsonl"\n' + " resources:\n" + " - name: raw_orders\n" + " description: Test source raw_orders.\n" + " reader: jsonl\n" + " write_disposition: replace\n" + ), + "data/orders.jsonl": ( + '{"order_id": 1, "amount": 10}\n' + '{"order_id": 2, "amount": 20}\n' + '{"order_id": 3, "amount": 30}\n' + ), + "models/order_totals.sql": ( + "MODEL (description 'Test model order_totals.', materialized table);\n\n" + 'SELECT order_id, amount FROM __source("raw_orders") WHERE amount >= 20\n' + ), + }, + ) + + built: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert built.returncode == 0, built.stdout + built.stderr + with closing(sqlite3.connect(project_dir / "warehouse.sqlite")) as warehouse: + loaded_rows: list[tuple[object, ...]] = warehouse.execute( + "SELECT order_id, amount FROM raw_orders ORDER BY order_id" + ).fetchall() + model_rows: list[tuple[object, ...]] = warehouse.execute( + "SELECT order_id, amount FROM order_totals ORDER BY order_id" + ).fetchall() + assert tuple(loaded_rows) == test_case.expected_loaded_rows + assert tuple(model_rows) == test_case.expected_model_rows diff --git a/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py b/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py index cbb700083b..4db0cacd83 100644 --- a/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py +++ b/tests/e2e/src/sqlbuild/integrations/ingestr/test_ingestr.py @@ -1,6 +1,8 @@ from __future__ import annotations +import sqlite3 import subprocess +from contextlib import closing from pathlib import Path import pytest @@ -79,6 +81,57 @@ def test_given_postgres_source_when_loading_with_ingestr_then_duckdb_target_has_ ) +@pytest.mark.parametrize( + "test_case", + [ + IngestrE2ETestCase( + description="loads a sqlite source into a sqlite destination", + expected_rows=((1, "new"), (2, "paid")), + expected_stdout_fragments=("ingestr execution", "raw_sqlite_orders", "OK"), + ) + ], + ids=lambda case: case.description, +) +def test_given_sqlite_project_when_loading_with_ingestr_then_target_has_rows( + tmp_path: Path, + test_case: IngestrE2ETestCase, +) -> None: + source_path: Path = tmp_path / "source.db" + with closing(sqlite3.connect(source_path)) as source, source: + source.execute("CREATE TABLE orders (order_id INTEGER, status TEXT)") + source.execute("INSERT INTO orders VALUES (1, 'new'), (2, 'paid')") + project_dir: Path = tmp_path / "sqlite_to_sqlite" + write_project_files( + project_dir=project_dir, + files={ + "sqlbuild_project.toml": ( + 'name = "sqlite_to_sqlite"\nadapter = "sqlite"\n\n' + '[connection]\npath = "target.sqlite"\n' + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_sqlite_orders\n description: Test source raw_sqlite_orders.\n" + " ingestr:\n" + f' source_uri: "sqlite:///{source_path}"\n' + " source_table: orders\n" + ), + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb_with_ingestr( + command=("--no-color", "load"), project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + for fragment in test_case.expected_stdout_fragments: + assert fragment in result.stdout + with closing(sqlite3.connect(project_dir / "target.sqlite")) as target: + rows: list[tuple[object, ...]] = target.execute( + "SELECT order_id, status FROM raw_sqlite_orders ORDER BY order_id" + ).fetchall() + assert tuple(rows) == test_case.expected_rows + + @pytest.mark.parametrize( "test_case", [ diff --git a/website/src/content/docs/docs/concepts/adapters/sqlite.mdx b/website/src/content/docs/docs/concepts/adapters/sqlite.mdx index e458073b1d..594b94f6fe 100644 --- a/website/src/content/docs/docs/concepts/adapters/sqlite.mdx +++ b/website/src/content/docs/docs/concepts/adapters/sqlite.mdx @@ -96,6 +96,7 @@ SQLite stores values by type affinity, so SQLBuild keeps column types explicitly | `sqb diff`, `sqb clone` | Supported; clones are physical copies | | Scenarios, `sqb scenario capture`, `--local` replay | Supported; see below | | Statement time limits | Connection interrupt | +| `sqb load`: Python, dlt and ingestr loaders | Supported; dlt writes through its `sqlalchemy` destination | ## Scenarios diff --git a/website/src/content/docs/docs/integrations/dlt.mdx b/website/src/content/docs/docs/integrations/dlt.mdx index c1e4b8cc4e..851049c889 100644 --- a/website/src/content/docs/docs/integrations/dlt.mdx +++ b/website/src/content/docs/docs/integrations/dlt.mdx @@ -111,7 +111,7 @@ Use SQLBuild's `write_disposition` (dlt's term), not `write_strategy`. `delete_i ### Destination -By default each resource loads into the database SQLBuild's adapter manages, with the dataset/schema derived from your target. An optional `destination` mapping passes extra settings to the dlt destination, but it cannot set `credentials`, `dataset_name`, or `default_schema_name` (SQLBuild manages those): +By default each resource loads into the database SQLBuild's adapter manages, with the dataset/schema derived from your target. On SQLite, dlt writes through its `sqlalchemy` destination into the project's database file (dataset `main`); the `sqlbuild[dlt]` extra installs the SQLAlchemy it needs. An optional `destination` mapping passes extra settings to the dlt destination, but it cannot set `credentials`, `dataset_name`, or `default_schema_name` (SQLBuild manages those): ```yaml dlt_sources: diff --git a/website/src/content/docs/docs/integrations/ingestr.mdx b/website/src/content/docs/docs/integrations/ingestr.mdx index d57f6ecb41..7704837775 100644 --- a/website/src/content/docs/docs/integrations/ingestr.mdx +++ b/website/src/content/docs/docs/integrations/ingestr.mdx @@ -128,6 +128,7 @@ SQLBuild automatically builds the ingestr destination URI from your adapter conn | Adapter | Destination URI format | |---------|----------------------| | DuckDB | `duckdb:///path/to/db.duckdb` | +| SQLite | `sqlite:///path/to/db.sqlite` | | MotherDuck | `motherduck://database` | | PostgreSQL | `postgresql://user:pass@host:port/db` | | Snowflake | `snowflake://user:pass@account/db/schema` | From 6372efcb5b6bc95b3609f414292eee62d40f05d0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:52:15 +0000 Subject: [PATCH 11/32] feat(postgres): add a connection option to turn off temporary test functions sqb test copies each SQL function a test calls into the session's pg_temp schema. Behind a transaction-mode pooler a later statement can run in a session without the copy and fail. The PostgreSQL connection option temporary_test_functions = false stops the copies, so tests call the built functions as before. --- .../docs/concepts/adapters/postgres.md | 4 +- .../postgres/classes/postgres_adapter.py | 10 +- src/sqlbuild/adapters/postgres/constants.py | 1 + .../cli/commands/main/postgres/_test_types.py | 10 ++ .../postgres/test_temporary_test_functions.py | 95 +++++++++++++++++++ .../docs/docs/concepts/adapters/postgres.mdx | 4 +- 6 files changed, 120 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_temporary_test_functions.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/postgres.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/postgres.md index 97337b9aa1..3517b77ccf 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/postgres.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/postgres.md @@ -40,6 +40,7 @@ schema = "analytics_dev" | `user` | Database user | | `password` | Database password | | `dbname` | Database name | +| `temporary_test_functions` | `false` stops `sqb test` from creating session-temporary (`pg_temp`) copies of project SQL functions, so tests call the built functions. Use it behind a transaction-mode pooler such as PgBouncer, where later statements can run in a session without the copies (default: `true`) | Connection fields are passed to `psycopg.connect()`. See the [psycopg documentation](https://www.psycopg.org/psycopg3/docs/api/connections.html) for all available options. The target remains authoritative for database and schema qualification. @@ -70,4 +71,5 @@ schema = "dev" `sqb test` creates each SQL function the selected tests call in the session's `pg_temp` schema and calls it as `pg_temp.`, so tests use the current project definition without a build. Temporary functions belong to one server session: behind a transaction-mode pooler, run `sqb test` through a -direct connection or a session-mode pool. See [Functions in `sqb test`](../testing.md#functions-in-sqb-test). +direct connection or a session-mode pool, or set `temporary_test_functions = false` so tests call the +built functions. See [Functions in `sqb test`](../testing.md#functions-in-sqb-test). diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index a0b5c6ac3e..5dd568d615 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -111,6 +111,7 @@ from sqlbuild.adapters.postgres.constants import ( ROW_COUNT_RELATION_KINDS, TABLE_FUNCTION_RETURN_TYPE, + TEMPORARY_TEST_FUNCTIONS_KEY, ) from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.diagnostics.main.log_sql import log_sql @@ -140,6 +141,7 @@ def get_columns_for_relations( escape_string_prefix=True, nested_block_comments=True ) max_identifier_length: ClassVar[int] = 63 + temporary_test_functions: bool = True _snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect( timestamp_type="TIMESTAMP", distinct_condition=render_is_distinct_from, @@ -564,7 +566,7 @@ def render_create_temporary_function( ) -> TemporaryFunction | None: """Copy a SQL function into the session's `pg_temp` schema, called schema-qualified.""" - if definition.language != FunctionLanguage.SQL: + if definition.language != FunctionLanguage.SQL or not self.temporary_test_functions: return None target: str = f"pg_temp.{self.render_identifier(name)}" return TemporaryFunction( @@ -2088,7 +2090,11 @@ def connect(self, config: dict[str, Any]) -> _PostgresConnection: code="A401", ) from error - raw_connection: Any = psycopg.connect(**config, autocommit=True) + connect_config: dict[str, Any] = dict(config) + self.temporary_test_functions = bool( + connect_config.pop(TEMPORARY_TEST_FUNCTIONS_KEY, self.temporary_test_functions) + ) + raw_connection: Any = psycopg.connect(**connect_config, autocommit=True) return _PostgresConnection(raw_connection) def _execute(self, *, connection: _PostgresConnection, sql: str) -> Any: diff --git a/src/sqlbuild/adapters/postgres/constants.py b/src/sqlbuild/adapters/postgres/constants.py index 8910dbb44a..ef548a4e26 100644 --- a/src/sqlbuild/adapters/postgres/constants.py +++ b/src/sqlbuild/adapters/postgres/constants.py @@ -3,3 +3,4 @@ TABLE_FUNCTION_RETURN_TYPE: str = "TABLE" ROW_COUNT_RELATION_KINDS: frozenset[str] = frozenset({"r", "m"}) TRANSACTION_OPEN_STATUS: str = "INTRANS" +TEMPORARY_TEST_FUNCTIONS_KEY: str = "temporary_test_functions" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py index 4388eba3f8..354f07f82e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py @@ -243,3 +243,13 @@ class PostgresLongModelNameE2ETestCase: model_columns: str expected_exit_code: int expected_scenario_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class PostgresTemporaryTestFunctionsE2ETestCase: + """A `sqb test` of an edited function, with or without temporary function copies.""" + + description: str + connection_option: str + expected_exit_code: int + expected_stdout_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_temporary_test_functions.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_temporary_test_functions.py new file mode 100644 index 0000000000..d974db8467 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_temporary_test_functions.py @@ -0,0 +1,95 @@ +"""E2E coverage of the PostgreSQL option that turns off temporary function copies in `sqb test`.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.postgres._test_types import ( + PostgresTemporaryTestFunctionsE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.postgres.helpers import ( + build_postgres_project_toml, + build_unique_schema_name, + cleanup_postgres_schema, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_FUNCTION_PATH: str = "functions/sql/is_completed_order.sql" +_FUNCTION_HEADER: str = ( + "FUNCTION (description 'Whether an order status counts as completed.', " + "arguments (order_status VARCHAR), returns BOOLEAN);\n" +) +_TEST: str = ( + 'TEST (name "orders_case");\nWITH\n' + "__ref__order_statuses AS (SELECT 'completed' AS order_status),\n" + "__expected__orders AS (SELECT 'completed' AS order_status, TRUE AS is_completed)\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + PostgresTemporaryTestFunctionsE2ETestCase( + description="by default the test runs the edited definition", + connection_option="", + expected_exit_code=1, + expected_stdout_fragment="FAIL=1", + ), + PostgresTemporaryTestFunctionsE2ETestCase( + description="with copies off the test runs the built function", + connection_option="temporary_test_functions = false\n", + expected_exit_code=0, + expected_stdout_fragment="PASS=1", + ), + ], + ids=lambda case: case.description, +) +def test_given_edited_function_when_testing_then_option_chooses_the_definition( + test_case: PostgresTemporaryTestFunctionsE2ETestCase, + tmp_path: Path, + postgres_e2e_config: dict[str, object], +) -> None: + schema_name: str = build_unique_schema_name(prefix="sqb_temp_functions") + project_toml: str = build_postgres_project_toml( + project_name="orders_temporary_functions", + schema_name=schema_name, + config=postgres_e2e_config, + ).replace("[connection]\n", f"[connection]\n{test_case.connection_option}", 1) + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_temporary_functions", + repo_files={ + "sqlbuild_project.toml": project_toml, + "models/order_statuses.sql": ( + "MODEL (description 'Test model order_statuses.', materialized table);\n" + "SELECT 'completed' AS order_status\n" + ), + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized table);\n" + 'SELECT order_status, __udf("is_completed_order")(order_status) AS is_completed ' + 'FROM __ref("order_statuses")\n' + ), + "tests/unit/orders_case.sql": _TEST, + _FUNCTION_PATH: _FUNCTION_HEADER + "SELECT order_status = 'completed'", + }, + ) + try: + built: subprocess.CompletedProcess[str] = run_sqb( + command=("build", "--no-tests", "--no-audits"), project_dir=project_dir + ) + _ = (project_dir / _FUNCTION_PATH).write_text( + _FUNCTION_HEADER + "SELECT order_status = 'shipped'", encoding="utf-8" + ) + + tested: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "test"), project_dir=project_dir + ) + finally: + cleanup_postgres_schema(schema_name=schema_name, config=postgres_e2e_config) + + assert built.returncode == 0, built.stdout + built.stderr + assert tested.returncode == test_case.expected_exit_code, tested.stdout + tested.stderr + assert test_case.expected_stdout_fragment in tested.stdout, test_case.description diff --git a/website/src/content/docs/docs/concepts/adapters/postgres.mdx b/website/src/content/docs/docs/concepts/adapters/postgres.mdx index e0bb8c2153..c38c0a2714 100644 --- a/website/src/content/docs/docs/concepts/adapters/postgres.mdx +++ b/website/src/content/docs/docs/concepts/adapters/postgres.mdx @@ -37,6 +37,7 @@ schema = "analytics_dev" | `user` | Database user | | `password` | Database password | | `dbname` | Database name | +| `temporary_test_functions` | `false` stops `sqb test` from creating session-temporary (`pg_temp`) copies of project SQL functions, so tests call the built functions. Use it behind a transaction-mode pooler such as PgBouncer, where later statements can run in a session without the copies (default: `true`) | Connection fields are passed to `psycopg.connect()`. See the [psycopg documentation](https://www.psycopg.org/psycopg3/docs/api/connections.html) for all available options. The target remains authoritative for database and schema qualification. @@ -67,4 +68,5 @@ schema = "dev" `sqb test` creates each SQL function the selected tests call in the session's `pg_temp` schema and calls it as `pg_temp.`, so tests use the current project definition without a build. Temporary functions belong to one server session: behind a transaction-mode pooler, run `sqb test` through a -direct connection or a session-mode pool. See [Functions in `sqb test`](/docs/concepts/testing#functions-in-sqb-test). +direct connection or a session-mode pool, or set `temporary_test_functions = false` so tests call the +built functions. See [Functions in `sqb test`](/docs/concepts/testing#functions-in-sqb-test). From 6c4ff0fab350f5358b4b5a67468301715c74c229 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:55:41 +0000 Subject: [PATCH 12/32] fix(test): register DuckDB Python UDFs on every sqb test connection sqb test registered a project's DuckDB Python UDFs on its first test connection only. Connections to one database file share them, but an in-memory database is separate per connection, so tests on the other connections failed calling the UDF. The UDFs are now registered on every test connection, including replacement connections; a connection that already sees the function keeps it. --- .../executor/pipeline/_helpers/testing.py | 91 ++++++++++++++++--- src/sqlbuild/executor/pipeline/models.py | 9 +- .../cli/commands/main/test/_test_types.py | 10 ++ .../main/test/test_python_udf_connections.py | 80 ++++++++++++++++ 4 files changed, 177 insertions(+), 13 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_udf_connections.py diff --git a/src/sqlbuild/executor/pipeline/_helpers/testing.py b/src/sqlbuild/executor/pipeline/_helpers/testing.py index 7ff175af98..267c97b175 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/testing.py +++ b/src/sqlbuild/executor/pipeline/_helpers/testing.py @@ -8,7 +8,7 @@ from collections.abc import Callable, Iterable from concurrent.futures import Future, ThreadPoolExecutor from contextvars import copy_context -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any, cast from uuid import uuid4 @@ -36,6 +36,7 @@ install_temporary_test_functions, rewrite_test_entry, ) +from sqlbuild.executor.pipeline.exceptions import TemporaryTestFunctionError from sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure import ( raise_warehouse_connection_failure, ) @@ -132,11 +133,18 @@ def run_test_pipeline( connection=connections[0], ) for ordinal, connection in enumerate(connections[1:], start=2): + label: str = f"test connection {ordinal} of {len(connections)}" install_temporary_test_functions( adapter=adapter, connection=connection, functions=temporary_functions, - connection_label=f"test connection {ordinal} of {len(connections)}", + connection_label=label, + ) + _install_python_test_functions( + adapter=adapter, + connection=connection, + functions=temporary_functions, + connection_label=label, ) installed_functions.append(temporary_functions) entries = tuple( @@ -330,6 +338,12 @@ def _connect_test_connection( functions=functions, connection_label="a replacement test connection", ) + _install_python_test_functions( + adapter=adapter, + connection=connection, + functions=functions, + connection_label="a replacement test connection", + ) except BaseException: adapter.close(connection) raise @@ -407,12 +421,13 @@ def _prepare_test_functions( name=key.name, reason=temporary_functions.failures.get(key) ) - _register_connection_scoped_python_functions( + registered: tuple[FunctionPlanEntry, ...] = _register_connection_scoped_python_functions( missing_by_key=missing_by_key, function_entries=function_entries, adapter=adapter, connection=connection, ) + temporary_functions = replace(temporary_functions, python_functions=registered) missing_by_test: dict[CompiledObjectKey, tuple[str, ...]] = {} for test_entry in plan.test_entries: @@ -445,24 +460,76 @@ def _register_connection_scoped_python_functions( function_entries: dict[CompiledObjectKey, FunctionPlanEntry], adapter: BaseAdapter, connection: Any, -) -> None: +) -> tuple[FunctionPlanEntry, ...]: if adapter.persists_python_functions(): - return + return () + registered: list[FunctionPlanEntry] = [] key: CompiledObjectKey for key in tuple(missing_by_key): function_entry: FunctionPlanEntry | None = function_entries.get(key) if function_entry is None or function_entry.language != FunctionLanguage.PYTHON: continue - result: FunctionExecutionResult = execute_function( - function_entry=function_entry, - adapter=adapter, - connection=connection, - statement_recorder=StatementRecorder(), - run_id="test", - query_change_tracking=False, + result: FunctionExecutionResult = _register_python_function( + adapter=adapter, connection=connection, function_entry=function_entry ) if result.status == ExecutionStatus.SUCCESS: del missing_by_key[key] + registered.append(function_entry) + return tuple(registered) + + +def _register_python_function( + *, adapter: BaseAdapter, connection: Any, function_entry: FunctionPlanEntry +) -> FunctionExecutionResult: + return execute_function( + function_entry=function_entry, + adapter=adapter, + connection=connection, + statement_recorder=StatementRecorder(), + run_id="test", + query_change_tracking=False, + ) + + +def _install_python_test_functions( + *, + adapter: BaseAdapter, + connection: Any, + functions: TemporaryTestFunctions, + connection_label: str, +) -> None: + """Register the first connection's Python functions on another, or fail with T006.""" + + function_entry: FunctionPlanEntry + for function_entry in functions.python_functions: + result: FunctionExecutionResult = _register_python_function( + adapter=adapter, connection=connection, function_entry=function_entry + ) + if result.status == ExecutionStatus.SUCCESS or _function_exists( + adapter=adapter, connection=connection, function_entry=function_entry + ): + continue + raise TemporaryTestFunctionError( + function_name=function_entry.name, + connection=connection_label, + reason=result.error_message or result.status.value, + ) + + +def _function_exists( + *, adapter: BaseAdapter, connection: Any, function_entry: FunctionPlanEntry +) -> bool: + """Whether the connection already sees the function, as connections to one database file do.""" + + destination: CompiledRelationLocation = function_entry.destination + return bool( + adapter.list_functions( + connection=connection, + database=destination.database, + schemas=(destination.schema,) if destination.schema is not None else None, + names=(destination.name,), + ) + ) def _group_function_names( diff --git a/src/sqlbuild/executor/pipeline/models.py b/src/sqlbuild/executor/pipeline/models.py index c821813fe3..f8b73cae46 100644 --- a/src/sqlbuild/executor/pipeline/models.py +++ b/src/sqlbuild/executor/pipeline/models.py @@ -6,7 +6,12 @@ from dataclasses import dataclass, field from typing import Any -from sqlbuild._native import AuditPlanEntry, CompiledObjectKey, SqlTestPlanEntry +from sqlbuild._native import ( + AuditPlanEntry, + CompiledObjectKey, + FunctionPlanEntry, + SqlTestPlanEntry, +) from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import ( BuildCallbacks, @@ -52,6 +57,8 @@ class TemporaryTestFunctions: created_keys: frozenset[CompiledObjectKey] = frozenset() replacements: dict[str, str] = field(default_factory=dict) failures: dict[CompiledObjectKey, str] = field(default_factory=dict) + python_functions: tuple[FunctionPlanEntry, ...] = () + """Connection-scoped Python functions the first connection registered, in order.""" @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index 9de88c687c..f047989255 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -603,3 +603,13 @@ class ParenthesisedJoinMockTestCase: model_from: str expected_exit_code: int expected_fragment: str + + +@dataclass(frozen=True) +class PythonUdfConnectionsE2ETestCase: + """Unit tests calling a DuckDB Python UDF over several test connections to one database.""" + + description: str + database: str + concurrency: int + expected_stdout_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_udf_connections.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_udf_connections.py new file mode 100644 index 0000000000..a22a256835 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_python_udf_connections.py @@ -0,0 +1,80 @@ +"""E2E coverage that every `sqb test` connection can call the project's DuckDB Python UDFs.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ( + PythonUdfConnectionsE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_UDF: str = ( + "from sqlbuild.functions import udf\n\n\n" + '@udf(arguments={"order_status": "STRING"}, returns="BOOLEAN", runtime_version="3.11")\n' + "def main(order_status: str | None) -> bool:\n" + ' """Test function main."""\n' + ' return order_status == "completed"\n' +) +_TEST: str = ( + "TEST ();\nWITH\n" + "__ref__order_statuses AS (SELECT 'completed' AS order_status),\n" + "__expected__orders AS (SELECT 'completed' AS order_status, TRUE AS is_completed)\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonUdfConnectionsE2ETestCase( + description="in-memory databases get the UDF on every connection", + database=":memory:", + concurrency=4, + expected_stdout_fragment="PASS=8 FAIL=0 TOTAL=8", + ), + PythonUdfConnectionsE2ETestCase( + description="connections to one database file share the UDF", + database="orders.duckdb", + concurrency=4, + expected_stdout_fragment="PASS=8 FAIL=0 TOTAL=8", + ), + ], + ids=lambda case: case.description, +) +def test_given_python_udf_when_testing_on_several_connections_then_every_test_can_call_it( + test_case: PythonUdfConnectionsE2ETestCase, tmp_path: Path +) -> None: + tests: dict[str, str] = {f"tests/unit/test_orders_{index}.sql": _TEST for index in range(1, 9)} + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_python_udf", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "orders_python_udf"\nadapter = "duckdb"\n\n' + f'[connection]\ndatabase = "{test_case.database}"\n\n' + "[scopes]\nenforce_placement = false\n" + ), + "functions/python/is_completed_order.py": _UDF, + "models/order_statuses.sql": ( + "MODEL (description 'Test model order_statuses.');\n" + "SELECT 'completed' AS order_status\n" + ), + "models/orders.sql": ( + "MODEL (description 'Test model orders.');\n" + 'SELECT s.order_status, __udf("is_completed_order")(s.order_status) AS is_completed\n' + 'FROM __ref("order_statuses") AS s\n' + ), + **tests, + }, + ) + + tested: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "test", "--concurrency", str(test_case.concurrency)), + project_dir=project_dir, + ) + + assert tested.returncode == 0, tested.stdout + tested.stderr + assert test_case.expected_stdout_fragment in tested.stdout, test_case.description From af906c9f81d483904a26056900cf4eebf8050af2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:04:14 +0000 Subject: [PATCH 13/32] fix(duckdb): attach a database once per project file and from the project directory A DuckDB connection's attach entries failed on a command's second connection with "Unique file handle conflict ... already attached", because connections to one project database share the attachment, and a relative attach path resolved from the working directory, so running sqb from outside the project created or read another file. Attach now uses ATTACH IF NOT EXISTS, and a relative DuckDB or SQLite attach path resolves from the project directory, like database. BREAKING CHANGE: a relative DuckDB attach path now resolves from the project directory instead of the current working directory. --- Makefile | 1 + .../_helpers/command_runtime/connection.rs | 76 ++++++++++++++++--- .../docs/concepts/adapters/duckdb.md | 4 + .../contract/classes/duckdb_backed_adapter.py | 2 +- .../cli/commands/main/build/_test_types.py | 9 +++ .../commands/main/build/test_duckdb_attach.py | 75 ++++++++++++++++++ .../adapters/duckdb/_helpers/test_sql.py | 10 +-- .../docs/docs/concepts/adapters/duckdb.mdx | 4 + 8 files changed, 165 insertions(+), 16 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_attach.py diff --git a/Makefile b/Makefile index 1e216addbd..4bb3692469 100644 --- a/Makefile +++ b/Makefile @@ -114,6 +114,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_storage_key.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_source_end_audits.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_check_outcomes.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_attach.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_unbuilt_dependencies.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 1e502bab25..9eeaceadfc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -6,7 +6,7 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_config::command_connection::constants::{ - CONNECTION_SOURCE_KEY, DBT_PROFILE_CONNECTION_SOURCE, DBT_PROFILE_ROUTING_KEYS, + CONNECTION_SOURCE_KEY, DBT_PROFILE_CONNECTION_SOURCE, DBT_PROFILE_ROUTING_KEYS, DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, LOCAL_DATABASE_PATH_KEYS, }; use sqlbuild_config::command_connection::main::missing_connection_error::missing_connection_error; @@ -25,6 +25,14 @@ use crate::bindings::_helpers::dbt_interop::profile_connection::resolve_raw_dbt_ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; +const ATTACH_KEY: &str = "attach"; +const ATTACH_PATH_KEY: &str = "path"; +const ATTACH_TYPE_KEY: &str = "type"; +/// `ATTACH ... (TYPE ...)` values whose path is a local database file. +const LOCAL_ATTACH_TYPES: [&str; 2] = ["duckdb", "sqlite"]; +const URL_SCHEME_SEPARATOR: &str = "://"; +const MOTHERDUCK_PATH_PREFIX: &str = "md:"; + const CONNECTION_OPTIONS: (&[&str], &[&str]) = ( &["raw_config", "project_dir", "adapter_name"], &["discovered_inputs", "cli_vars", "selected_target"], @@ -108,28 +116,76 @@ pub(crate) fn anchor_local_database_path( config: &Bound<'_, PyDict>, (project_dir, adapter_name): (&Bound<'_, PyAny>, &str), ) -> PyResult<()> { - let py: Python<'_> = config.py(); let Some((_, path_key)) = LOCAL_DATABASE_PATH_KEYS .iter() .find(|(adapter, _)| *adapter == adapter_name) else { return Ok(()); }; + if adapter_name == DUCKDB_ADAPTER { + anchor_attach_paths(config, project_dir)?; + } let Some(database) = config.get_item(*path_key)? else { return Ok(()); }; - if !database.is_instance_of::() { - return Ok(()); + if let Some(anchored) = anchored_file_path(&database, project_dir)? { + config.set_item(*path_key, anchored)?; + } + Ok(()) +} + +/// `project_dir / value` for a relative local file path, else None. +fn anchored_file_path<'py>( + value: &Bound<'py, PyAny>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = value.py(); + if !value.is_instance_of::() { + return Ok(None); } - if path(py, &database)? - .call_method0("is_absolute")? - .is_truthy()? - || database.extract::()? == DUCKDB_MEMORY_DATABASE + if path(py, value)?.call_method0("is_absolute")?.is_truthy()? + || value.extract::()? == DUCKDB_MEMORY_DATABASE { + return Ok(None); + } + Ok(Some(path(py, project_dir)?.div(value)?.str()?.into_any())) +} + +/// Anchor each relative DuckDB or SQLite file an `attach` entry names; other paths are DSNs or URLs. +fn anchor_attach_paths(config: &Bound<'_, PyDict>, project_dir: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = config.py(); + let Some(entries) = config.get_item(ATTACH_KEY)? else { return Ok(()); + }; + let Ok(entries) = entries.try_iter() else { + return Ok(()); + }; + let mut anchored: Vec> = Vec::new(); + for entry in entries { + let entry: Bound<'_, PyAny> = entry?; + let Ok(mapping) = entry.cast::() else { + anchored.push(entry); + continue; + }; + let local_file: bool = match mapping.get_item(ATTACH_TYPE_KEY)? { + None => true, + Some(kind) => kind + .extract::() + .is_ok_and(|kind| LOCAL_ATTACH_TYPES.contains(&kind.to_ascii_lowercase().as_str())), + }; + let copy: Bound<'_, PyDict> = mapping.copy()?; + if let Some(path_value) = mapping.get_item(ATTACH_PATH_KEY)? + && local_file + && !path_value.extract::().is_ok_and(|text| { + text.contains(URL_SCHEME_SEPARATOR) || text.starts_with(MOTHERDUCK_PATH_PREFIX) + }) + && let Some(file) = anchored_file_path(&path_value, project_dir)? + { + copy.set_item(ATTACH_PATH_KEY, file)?; + } + anchored.push(copy.into_any()); } - let anchored: Bound<'_, PyAny> = path(py, project_dir)?.div(&database)?.str()?.into_any(); - config.set_item(*path_key, anchored) + config.set_item(ATTACH_KEY, pyo3::types::PyList::new(py, anchored)?) } /// Resolve dbt profiles, relative DuckDB paths, and the command group's warehouse. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md index 35506f687c..e88fe61a3a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md @@ -52,6 +52,10 @@ alias = "external" read_only = true ``` +A relative `path` of a DuckDB or SQLite file is relative to the project directory, like `database`. +Every connection of a command attaches it once: connections to the same project database share the +attachment. + ## Table promotion mode DuckDB defaults to `staged` promotion: tables are materialized into a staging table, audited, then swapped into the target. This is configurable in `settings`: diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 32829b302b..b4f0982e88 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1190,7 +1190,7 @@ def duckdb_build_attach_sql(self, attach_entry: dict[str, object]) -> str: """Build a DuckDB ATTACH statement from one attach config entry.""" path: str = str(attach_entry["path"]) - sql: str = f"ATTACH '{path}'" + sql: str = f"ATTACH IF NOT EXISTS '{path}'" alias: object | None = attach_entry.get("alias") if alias is not None: sql += f" AS {alias}" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 19ea6c1b35..33a2d69cf0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1472,3 +1472,12 @@ class PythonCheckBuildE2ETestCase: check_body: str expected_exit_code: int expected_footer_fragment: str + + +@dataclass(frozen=True) +class DuckDbAttachE2ETestCase: + """A DuckDB project attaching a second database file, built with some connections.""" + + description: str + concurrency: int + expected_archived_rows: list[tuple[object, ...]] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_attach.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_attach.py new file mode 100644 index 0000000000..061ba86be8 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_attach.py @@ -0,0 +1,75 @@ +"""E2E coverage of DuckDB `attach` entries across connections, run from the repository root.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import DuckDbAttachE2ETestCase +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + REPO_ROOT, + prepare_inline_project, + query_duckdb, + run_sqb, +) + +_PROJECT_TOML: str = ( + 'name = "shop"\nadapter = "duckdb"\n\n[connection]\ndatabase = "shop.duckdb"\n\n' + '[[connection.attach]]\npath = "archive.duckdb"\nalias = "archive"\n\n' + "[scopes]\nenforce_placement = false\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + DuckDbAttachE2ETestCase( + description="one connection attaches the file from the project directory", + concurrency=1, + expected_archived_rows=[(1,)], + ), + DuckDbAttachE2ETestCase( + description="several connections attach the same file once", + concurrency=2, + expected_archived_rows=[(1,)], + ), + ], + ids=lambda case: case.description, +) +def test_given_attached_duckdb_file_when_building_then_models_land_in_it( + test_case: DuckDbAttachE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/archived_orders.sql": ( + "MODEL (description 'Test model archived_orders.', materialized table, " + "database archive, schema main);\nSELECT 1 AS order_id\n" + ), + "models/customers.sql": ( + "MODEL (description 'Test model customers.', materialized table);\n" + "SELECT 1 AS customer_id\n" + ), + "models/order_ids.sql": ( + "MODEL (description 'Test model order_ids.', materialized table);\n" + 'SELECT o.order_id FROM __ref("archived_orders") AS o\n' + ), + }, + ) + built: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--concurrency", str(test_case.concurrency)), + project_dir=project_dir, + ) + + assert built.returncode == 0, built.stdout + built.stderr + assert ( + query_duckdb( + db_path=project_dir / "archive.duckdb", sql="SELECT order_id FROM archived_orders" + ) + == test_case.expected_archived_rows + ), test_case.description + assert not (REPO_ROOT / "archive.duckdb").exists() diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/_helpers/test_sql.py b/tests/integration/src/sqlbuild/adapters/duckdb/_helpers/test_sql.py index 80e42963d6..ca01171fc0 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/_helpers/test_sql.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/_helpers/test_sql.py @@ -17,12 +17,12 @@ BuildAttachSqlTestCase( description="attaches database with path only", attach_entry={"path": "/tmp/other.duckdb"}, - expected_sql="ATTACH '/tmp/other.duckdb'", + expected_sql="ATTACH IF NOT EXISTS '/tmp/other.duckdb'", ), BuildAttachSqlTestCase( description="attaches database with alias", attach_entry={"path": "/tmp/other.duckdb", "alias": "other"}, - expected_sql="ATTACH '/tmp/other.duckdb' AS other", + expected_sql="ATTACH IF NOT EXISTS '/tmp/other.duckdb' AS other", ), BuildAttachSqlTestCase( description="attaches database with type and read_only", @@ -32,17 +32,17 @@ "type": "sqlite", "read_only": True, }, - expected_sql="ATTACH 'sqlite.db' AS sqlite_db (TYPE sqlite, READ_ONLY)", + expected_sql="ATTACH IF NOT EXISTS 'sqlite.db' AS sqlite_db (TYPE sqlite, READ_ONLY)", ), BuildAttachSqlTestCase( description="attaches database with read_only false omits option", attach_entry={"path": "/tmp/data.duckdb", "read_only": False}, - expected_sql="ATTACH '/tmp/data.duckdb'", + expected_sql="ATTACH IF NOT EXISTS '/tmp/data.duckdb'", ), BuildAttachSqlTestCase( description="attaches database with type only", attach_entry={"path": "pg.db", "type": "postgres"}, - expected_sql="ATTACH 'pg.db' (TYPE postgres)", + expected_sql="ATTACH IF NOT EXISTS 'pg.db' (TYPE postgres)", ), ], ids=lambda case: case.description, diff --git a/website/src/content/docs/docs/concepts/adapters/duckdb.mdx b/website/src/content/docs/docs/concepts/adapters/duckdb.mdx index 90b0e70a7d..3b3d4d499b 100644 --- a/website/src/content/docs/docs/concepts/adapters/duckdb.mdx +++ b/website/src/content/docs/docs/concepts/adapters/duckdb.mdx @@ -49,6 +49,10 @@ alias = "external" read_only = true ``` +A relative `path` of a DuckDB or SQLite file is relative to the project directory, like `database`. +Every connection of a command attaches it once: connections to the same project database share the +attachment. + ## Table promotion mode DuckDB defaults to `staged` promotion: tables are materialized into a staging table, audited, then swapped into the target. This is configurable in `settings`: From fc9c777d601e10a38d509e944149d3314a899ebb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:14:27 +0000 Subject: [PATCH 14/32] fix(sqlserver): read timestamp cursor bounds as ISO 8601 text The planner read cursor bounds with CAST(MAX(cursor) AS VARCHAR). On SQL Server that cast renders a datetime as 'Apr 4 2026 2:30PM', so the second, incremental build of a timestamp-cursor model failed with C001 "invalid timestamp cursor value". Adapters now render the bound as text through render_cursor_text, and SQL Server converts timestamp bounds with CONVERT(VARCHAR(33), ..., 126). The SQL Server waffle-shop E2E, which builds twice, covers the incremental path. --- .../main/cursor_query_sql.rs | 12 +++--- .../tests/test_cursor_queries.rs | 5 ++- .../_helpers/planner_warehouse/cursors.rs | 40 ++++++++++++++++++- .../contract/classes/strict_adapter.py | 6 +++ .../sqlserver/classes/sqlserver_adapter.py | 8 ++++ src/sqlbuild/adapters/sqlserver/constants.py | 1 + 6 files changed, 63 insertions(+), 9 deletions(-) diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_sql.rs b/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_sql.rs index b86da84c9f..949b41cb4e 100644 --- a/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_sql.rs +++ b/crates/sqlbuild-analysis/src/planner_warehouse/main/cursor_query_sql.rs @@ -2,19 +2,19 @@ use crate::planner_warehouse::models::PhysicalCursorQuery; -/// Select the requested bounds of the cursor column as text. -pub fn cursor_query_sql(query: &PhysicalCursorQuery) -> String { +/// Select the requested bounds of the cursor column as text, each through `as_text`. +pub fn cursor_query_sql(query: &PhysicalCursorQuery, as_text: &dyn Fn(&str) -> String) -> String { let mut parts: Vec = Vec::new(); if !query.min_tags.is_empty() { parts.push(format!( - "CAST(MIN({}) AS VARCHAR) AS _min", - query.cursor_column + "{} AS _min", + as_text(&format!("MIN({})", query.cursor_column)) )); } if !query.max_tags.is_empty() { parts.push(format!( - "CAST(MAX({}) AS VARCHAR) AS _max", - query.cursor_column + "{} AS _max", + as_text(&format!("MAX({})", query.cursor_column)) )); } format!("SELECT {} FROM {}", parts.join(", "), query.relation) diff --git a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs index 10b026f12e..df78d914d7 100644 --- a/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs +++ b/crates/sqlbuild-analysis/src/planner_warehouse/tests/test_cursor_queries.rs @@ -47,7 +47,10 @@ fn given_logical_bounds_when_grouping_then_reads_each_relation_once_in_first_ord ]; for test_case in test_cases { let groups = cursor_query_groups(&test_case.requests); - let sql: Vec = groups.iter().map(cursor_query_sql).collect(); + let sql: Vec = groups + .iter() + .map(|group| cursor_query_sql(group, &|bound| format!("CAST({bound} AS VARCHAR)"))) + .collect(); let identities: Vec = groups .iter() .enumerate() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs index 0a82820a15..b2c7779c8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/cursors.rs @@ -45,11 +45,15 @@ use sqlbuild_analysis::planner_warehouse::main::cursor_query_groups::cursor_quer use sqlbuild_analysis::planner_warehouse::main::cursor_query_identity::cursor_query_identity; use sqlbuild_analysis::planner_warehouse::main::cursor_query_sql::cursor_query_sql; use sqlbuild_analysis::planner_warehouse::models::{CursorTagRequest, PhysicalCursorQuery}; +use std::cell::RefCell; use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Instant; +/// The adapter method that renders one cursor bound expression as text. +const CURSOR_TEXT_METHOD: &str = "render_cursor_text"; + /// Selection and policy inputs for cursor warehouse inspection. pub(crate) struct CursorInputs<'py> { pub(crate) selected_keys: Bound<'py, PyAny>, @@ -692,6 +696,34 @@ fn fan_out<'py>(query: &PhysicalCursorQuery, row: &Bound<'py, PyAny>) -> PyResul Ok(values) } +/// The query's MIN/MAX statement, each bound read as text the way the adapter renders it. +fn adapter_cursor_sql( + request: &SnapshotRequest<'_>, + query: &PhysicalCursorQuery, +) -> PyResult { + let py = request.adapter.py(); + let render = native_field(&request.adapter, CURSOR_TEXT_METHOD)?; + let failure: RefCell> = RefCell::new(None); + let sql: String = cursor_query_sql(query, &|expression: &str| { + call_with( + &render, + &[ + ("expression", text(py, expression)), + ("cursor_type", text(py, &query.cursor_type)), + ], + ) + .and_then(|rendered| rendered.extract::()) + .unwrap_or_else(|error| { + failure.replace(Some(error)); + String::new() + }) + }); + match failure.into_inner() { + Some(error) => Err(error), + None => Ok(sql), + } +} + fn execute_queries<'py>( request: &SnapshotRequest<'py>, queries: &[PhysicalCursorQuery], @@ -708,7 +740,7 @@ fn execute_queries<'py>( if !on_progress.is_none() { on_progress.call1((format!("Inspecting cursor bounds {identity}..."),))?; } - let outcome = read_cursor_query(py, &cursor_query_sql(query), handles)?; + let outcome = read_cursor_query(py, &adapter_cursor_sql(request, query)?, handles)?; report_cursor_query(py, query, (&outcome, &identity), on_progress)?; outcomes.push(outcome); } @@ -722,7 +754,11 @@ fn execute_queries<'py>( } let mut tasks: Vec<_> = Vec::new(); for query in queries { - tasks.push(bound_task(py, cursor_query_sql(query), handles)?); + tasks.push(bound_task( + py, + adapter_cursor_sql(request, query)?, + handles, + )?); } call_with( &host::entry(py, RUN_BOUNDED)?, diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index a9a37916aa..ae44b4191c 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -140,6 +140,12 @@ def render_cursor_operand(self, *, expression: str, cursor_type: str | None) -> del cursor_type return expression + def render_cursor_text(self, *, expression: str, cursor_type: str) -> str: + """Optional: a cursor bound expression as the text the planner parses as a cursor value.""" + + del cursor_type + return f"CAST({expression} AS VARCHAR)" + def render_create_temporary_function( self, *, name: str, definition: FunctionDefinition ) -> TemporaryFunction | None: diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index afab51a217..2f200b531d 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -114,6 +114,7 @@ DIFF_UNSUPPORTED_COMPARISON_TYPES, INFORMATION_SCHEMA_NULLABLE_VALUE, INTEGER_TYPE_TOKEN, + ISO8601_CONVERT_STYLE, USER_TABLE_OBJECT_TYPE, ) from sqlbuild.compiler.compile.types import FunctionLanguage @@ -990,6 +991,13 @@ def render_loader_value_literal( def _quote_sql_string(self, value: str) -> str: return "'" + value.replace("'", "''") + "'" + def render_cursor_text(self, *, expression: str, cursor_type: str) -> str: + """Render a timestamp bound in ISO 8601 (style 126), not CAST's 'Apr 4 2026 2:30PM'.""" + + if cursor_type == CursorKind.TIMESTAMP: + return f"CONVERT(VARCHAR(33), {expression}, {ISO8601_CONVERT_STYLE})" + return super().render_cursor_text(expression=expression, cursor_type=cursor_type) + def render_cursor_bound_literal(self, *, value: str, cursor_type: str | None) -> str: if cursor_type == CursorKind.INTEGER: return value diff --git a/src/sqlbuild/adapters/sqlserver/constants.py b/src/sqlbuild/adapters/sqlserver/constants.py index 8c8b78377b..611c496afc 100644 --- a/src/sqlbuild/adapters/sqlserver/constants.py +++ b/src/sqlbuild/adapters/sqlserver/constants.py @@ -12,3 +12,4 @@ INFORMATION_SCHEMA_NULLABLE_VALUE: str = "YES" INTEGER_TYPE_TOKEN: str = "INT" USER_TABLE_OBJECT_TYPE: str = "U" +ISO8601_CONVERT_STYLE: int = 126 From bb0df8cfa1662a6a51c83292520a0bbb0508ae6c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:16:00 +0000 Subject: [PATCH 15/32] fix(databricks): check schema existence in the current catalog when none is set DatabricksAdapter.schema_exists returned False whenever the target had no catalog, so sqb review's post-drop check and the janitor's leftover-schema filter always saw such schemas as missing. Without a catalog it now queries the session's current catalog through the unqualified information_schema.schemata. --- .../databricks/classes/databricks_adapter.py | 11 +++-- src/sqlbuild/adapters/databricks/constants.py | 1 + .../adapters/databricks/_test_types.py | 11 +++++ .../adapters/databricks/test_client.py | 44 +++++++++++++++++++ 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 2165dabf5d..edf2f88a2f 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -121,6 +121,7 @@ from sqlbuild.adapters.databricks._helpers.relation_usage import read_relation_usage from sqlbuild.adapters.databricks.classes.databricks_connection import _DatabricksConnection from sqlbuild.adapters.databricks.constants import ( + CURRENT_CATALOG_INFORMATION_SCHEMA, DELTA_COLUMN_MAPPING_NAME_MODE, DELTA_COLUMN_MAPPING_PROPERTY, DELTA_DEFAULT_DELETED_FILE_RETENTION_DAYS, @@ -3126,11 +3127,13 @@ def schema_exists( database: str | None, schema: str, ) -> bool: - if database is None: - return False + schemata: str = ( + CURRENT_CATALOG_INFORMATION_SCHEMA + if database is None + else self._information_schema(database) + ) query: str = ( - f"SELECT 1 FROM {self._information_schema(database)}.schemata " - f"WHERE schema_name = {self._string_literal(schema)}" + f"SELECT 1 FROM {schemata}.schemata WHERE schema_name = {self._string_literal(schema)}" ) cursor: Any = connection.cursor() try: diff --git a/src/sqlbuild/adapters/databricks/constants.py b/src/sqlbuild/adapters/databricks/constants.py index 1557862edf..bc09f33568 100644 --- a/src/sqlbuild/adapters/databricks/constants.py +++ b/src/sqlbuild/adapters/databricks/constants.py @@ -34,3 +34,4 @@ ) USAGE_UNAVAILABLE_SQLSTATES: frozenset[str] = frozenset({"42501", "42P01", "42704"}) LINEAGE_VIEW_SOURCE_TYPES: tuple[str, ...] = ("VIEW", "MATERIALIZED_VIEW", "METRIC_VIEW") +CURRENT_CATALOG_INFORMATION_SCHEMA: str = "information_schema" diff --git a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py index 4f82e53f10..3f61d0ccf0 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/_test_types.py @@ -189,3 +189,14 @@ class DatabricksRelationUsageFailureTestCase: message: str sqlstate: str | None expected_error_match: str + + +@dataclass(frozen=True) +class DatabricksSchemaExistsTestCase: + """A schema existence check with or without a catalog, and what it must query and answer.""" + + description: str + database: str | None + rows: list[tuple[object, ...]] + expected_exists: bool + expected_query_fragment: str diff --git a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py index 046e6ad499..122d6bce96 100644 --- a/tests/unit/src/sqlbuild/adapters/databricks/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/databricks/test_client.py @@ -27,6 +27,7 @@ DatabricksRenderDurableCloneTestCase, DatabricksRenderPythonFunctionTestCase, DatabricksRenderTableFunctionTestCase, + DatabricksSchemaExistsTestCase, DatabricksStringTypeCastRenderingTestCase, DatabricksTableFreshnessBatchTestCase, DatabricksTableFreshnessFallbackTestCase, @@ -752,3 +753,46 @@ def test_given_candidate_relations_when_adding_age_metadata_then_databricks_read str(enriched[0].created_at), str(enriched[0].last_altered_at), ) == test_case.expected_timestamps + + +@pytest.mark.parametrize( + "test_case", + [ + DatabricksSchemaExistsTestCase( + description="a schema in the session's current catalog is found without a catalog", + database=None, + rows=[(1,)], + expected_exists=True, + expected_query_fragment="SELECT 1 FROM information_schema.schemata", + ), + DatabricksSchemaExistsTestCase( + description="a schema missing from the current catalog is reported missing", + database=None, + rows=[], + expected_exists=False, + expected_query_fragment="SELECT 1 FROM information_schema.schemata", + ), + DatabricksSchemaExistsTestCase( + description="an explicit catalog is queried through its information schema", + database="main", + rows=[(1,)], + expected_exists=True, + expected_query_fragment="SELECT 1 FROM `main`.information_schema.schemata", + ), + ], + ids=lambda case: case.description, +) +def test_given_databricks_schema_when_checking_existence_then_current_catalog_is_used_without_one( + test_case: DatabricksSchemaExistsTestCase, +) -> None: + cursor: FakeDatabricksMetadataCursor = FakeDatabricksMetadataCursor(rows=test_case.rows) + + exists: bool = DatabricksAdapter().schema_exists( + connection=cast(Any, FakeDatabricksMetadataConnection((cursor,))), + database=test_case.database, + schema="analytics", + ) + + assert exists is test_case.expected_exists, test_case.description + assert cursor.executed_sql is not None + assert test_case.expected_query_fragment in cursor.executed_sql, test_case.description From 2d5fd90713928d515173cffc55f5219b7b3343dd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:32:02 +0000 Subject: [PATCH 16/32] fix(rules): report model rule columns from the file when the query shares a line Model rule findings counted columns from the query's own text. When the query began on the MODEL header's line, as in MODEL (...); WITH ... SELECT * FROM o, SQBRMODEL102 pointed at column 35 instead of the star's file column. A finding on the query's first content line is now shifted by the characters before the query on that file line; findings on later lines and on queries below the header are unchanged. --- .../src/engine/_helpers/project_rows.rs | 26 +++++++++ .../src/engine/_helpers/retained.rs | 2 + .../src/engine/_helpers/rows.rs | 1 + .../src/engine/tests/helpers.rs | 1 + .../src/engine/tests/test_evaluation.rs | 2 +- crates/sqlbuild-rules/src/models.rs | 5 ++ .../sqlbuild-rules/src/query_facts/models.rs | 22 +++++++- .../src/rules/_helpers/evaluation.rs | 6 +- .../cli/commands/main/rules/_test_types.py | 9 +++ .../main/rules/test_rule_positions.py | 56 +++++++++++++++++++ 10 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_positions.py diff --git a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs index 2370e9df16..77074eb3a6 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs @@ -29,6 +29,11 @@ pub(crate) fn model_row(facts: &CompiledModelFacts) -> Result &facts.authored_query_sql, &facts.query_sql, ), + query_column_offset: query_column_offset( + &facts.authored_sql, + &facts.authored_query_sql, + &facts.query_sql, + ), config: facts.config.values.clone(), authored_config_keys: facts.config.header_keys.clone(), logical_schema: facts.config.layer_schema.clone(), @@ -107,6 +112,27 @@ pub(crate) fn declaration(fact: &DeclarationFact) -> Declaration { } } +/// The file-line characters before the query's first content that the query itself lacks. +fn query_column_offset(file: &str, authored_query: &str, query: &str) -> u64 { + let line_prefix = |text: &str, at: usize| { + text[..at] + .rsplit('\n') + .next() + .unwrap_or_default() + .chars() + .count() + }; + let content_start = |text: &str| text.len() - text.trim_start().len(); + if authored_query.is_empty() { + return 0; + } + file.rfind(authored_query).map_or(0, |start| { + let in_file: usize = line_prefix(file, start + content_start(authored_query)); + let in_query: usize = line_prefix(query, content_start(query)); + in_file.saturating_sub(in_query) as u64 + }) +} + /// The lines of `file` above `query`, the rendered form of its `authored_query`; 0 if unknown. fn query_line_offset(file: &str, authored_query: &str, query: &str) -> u64 { let leading_lines = |text: &str| { diff --git a/crates/sqlbuild-rules/src/engine/_helpers/retained.rs b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs index bb56b7b76c..8d9c843ad8 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/retained.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs @@ -133,6 +133,7 @@ impl RowWriter { query_sql: _, authored_sql: _, query_line_offset: _, + query_column_offset: _, typed_calls: _, config, authored_config_keys, @@ -351,6 +352,7 @@ impl RowReader<'_> { query_sql: String::new(), authored_sql: String::new(), query_line_offset: 0, + query_column_offset: 0, typed_calls: Vec::new(), config: { let count: usize = self.count()?; diff --git a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs index 8cf1544beb..5cd8ac5cfb 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs @@ -84,6 +84,7 @@ fn build_model(row: ModelRow, inputs: &ModelInputs<'_, '_>) -> Result Model { constant_declarations: vec![declaration], typed_calls: Vec::new(), query_line_offset: 0, + query_column_offset: 0, declared_audit_count: 3, targeting_test_count: 4, empty_input_only_test_count: 0, diff --git a/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs b/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs index cebbf4fb43..819e600020 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_evaluation.rs @@ -1227,7 +1227,7 @@ fn given_model_sql_when_stubbing_shape_then_offsets_map_back_to_source_positions .sql .find(test_case.expected_fragment) .ok_or("fragment missing from the model SQL")?; - let source = ModelSource::new(test_case.sql, &shape.offsets, 0); + let source = ModelSource::new(test_case.sql, &shape.offsets, (0, 0)); let end = normalized_start + test_case.expected_fragment.chars().count(); assert_eq!( source.text(normalized_start, end), diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index 8f3a910df5..06e2ed730d 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -562,6 +562,9 @@ pub struct Model { /// The lines of the model file above `query_sql`, which locations inside it skip. #[serde(default)] pub query_line_offset: u64, + /// The characters before `query_sql`'s first line on its file line, when it shares one. + #[serde(default)] + pub query_column_offset: u64, pub config: BTreeMap, pub authored_config_keys: Vec, pub logical_schema: Option, @@ -875,6 +878,8 @@ pub struct ModelRow { pub authored_sql: String, /// The lines of the model file above `query_sql`. pub query_line_offset: u64, + /// The characters before `query_sql`'s first line on its file line. + pub query_column_offset: u64, pub config: BTreeMap, pub authored_config_keys: Vec, pub logical_schema: Option, diff --git a/crates/sqlbuild-rules/src/query_facts/models.rs b/crates/sqlbuild-rules/src/query_facts/models.rs index ace5ffb435..78617ee31d 100644 --- a/crates/sqlbuild-rules/src/query_facts/models.rs +++ b/crates/sqlbuild-rules/src/query_facts/models.rs @@ -227,10 +227,18 @@ pub(crate) struct ModelSource<'a> { offsets: &'a [usize], /// The lines of the model file above this SQL. line_offset: u64, + /// The characters before this SQL's first content line on its file line. + column_offset: u64, + /// The 1-based line of this SQL's first content. + first_content_line: usize, } impl<'a> ModelSource<'a> { - pub(crate) fn new(sql: &'a str, offsets: &'a [usize], line_offset: u64) -> Self { + pub(crate) fn new( + sql: &'a str, + offsets: &'a [usize], + (line_offset, column_offset): (u64, u64), + ) -> Self { let mut bytes: Vec = Vec::with_capacity(sql.len() + 1); let mut lines: Vec = vec![0]; for (character, (byte, value)) in sql.char_indices().enumerate() { @@ -246,6 +254,11 @@ impl<'a> ModelSource<'a> { lines, offsets, line_offset, + column_offset, + first_content_line: sql[..sql.len() - sql.trim_start().len()] + .matches('\n') + .count() + + 1, } } @@ -269,7 +282,12 @@ impl<'a> ModelSource<'a> { let character = self.original(offset); let line = self.lines.partition_point(|start| *start <= character); let column = character - self.lines[line - 1] + 1; - (line as u64 + self.line_offset, column as u64) + let shifted: u64 = if line == self.first_content_line { + self.column_offset + } else { + 0 + }; + (line as u64 + self.line_offset, column as u64 + shifted) } /// The model's own text between normalized characters `start` and `end`. diff --git a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs index 373f72499a..2ce4d6291d 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs @@ -144,7 +144,11 @@ fn evaluate_model_inner(request: ModelEvaluationRequest<'_>) -> Result None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "shop"\nadapter = "duckdb"\n\n[connection]\ndatabase = "shop.duckdb"\n\n' + "[scopes]\nenforce_placement = false\n" + ), + "models/orders.sql": test_case.model_sql, + }, + ) + + ran: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "rules", "run", "SQBRMODEL102"), project_dir=project_dir + ) + + assert test_case.expected_location in ran.stdout + ran.stderr, ran.stdout + ran.stderr From ed0c4db4c1bd22df510dd2a13e34ced783d9caef Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:41:25 +0000 Subject: [PATCH 17/32] fix(clickhouse): treat # and // line comments as comments ClickHouse and Polyglot's ClickHouse tokenizer read # and // as line comments, but SQLBuild's ClickHouse lexical syntax and the rules tokenizer listed only --. An apostrophe in such a comment then opened a string to SQLBuild's scanners, so compile and sqb rules failed with P001 "SQL reference contains an unclosed quoted string". Both tables now list --, # and //. --- .../src/query_facts/_helpers/shape.rs | 2 +- .../clickhouse/classes/clickhouse_adapter.py | 2 +- .../cli/commands/main/rules/_test_types.py | 9 +++ .../main/rules/test_clickhouse_comments.py | 59 +++++++++++++++++++ 4 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/rules/test_clickhouse_comments.py diff --git a/crates/sqlbuild-rules/src/query_facts/_helpers/shape.rs b/crates/sqlbuild-rules/src/query_facts/_helpers/shape.rs index d000202686..0f2b3ea81d 100644 --- a/crates/sqlbuild-rules/src/query_facts/_helpers/shape.rs +++ b/crates/sqlbuild-rules/src/query_facts/_helpers/shape.rs @@ -96,7 +96,7 @@ fn lexical_syntax(dialect: &str) -> LexicalSyntax { }, "clickhouse" => LexicalSyntax { backslash_escape_quotes: strings(&["'"]), - line_comment_prefixes: lines(&["--"]), + line_comment_prefixes: lines(&["--", "#", "//"]), ..LexicalSyntax::default() }, "databricks" | "duckdb" => LexicalSyntax { diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 745f207132..a92876ca3d 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -160,7 +160,7 @@ class ClickHouseAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): sql_analysis_dialect_name: ClassVar[str | None] = "clickhouse" sql_lexical_syntax: ClassVar[SqlLexicalSyntax] = SqlLexicalSyntax( backslash_escape_quotes=frozenset({"'", '"', "`"}), - line_comment_prefixes=frozenset({"--"}), + line_comment_prefixes=frozenset({"--", "#", "//"}), ) max_identifier_length: ClassVar[int] = MAX_IDENTIFIER_LENGTH relation_grants_supported: ClassVar[bool] = False diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 0a630c77d8..d1c861433d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -345,3 +345,12 @@ class RulePositionE2ETestCase: description: str model_sql: str expected_location: str + + +@dataclass(frozen=True) +class ClickHouseCommentE2ETestCase: + """A ClickHouse model with a line comment holding a quote, and the finding rules report.""" + + description: str + comment: str + expected_location: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_clickhouse_comments.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_clickhouse_comments.py new file mode 100644 index 0000000000..0cc24a9f8b --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_clickhouse_comments.py @@ -0,0 +1,59 @@ +"""E2E coverage that ClickHouse `#` and `//` line comments are comments to SQLBuild's scanners.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import ( + ClickHouseCommentE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_PROJECT_TOML: str = ( + 'name = "shop"\nadapter = "clickhouse"\n\n[connection]\nhost = "localhost"\n' + 'database = "default"\n\n[defaults]\nschema = "analytics"\n\n' + "[scopes]\nenforce_placement = false\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + ClickHouseCommentE2ETestCase( + description="a hash comment with an apostrophe is skipped", + comment="# don't keep the star", + expected_location="models/orders.sql:4:8 [SQBRMODEL102]", + ), + ClickHouseCommentE2ETestCase( + description="a double-slash comment with an apostrophe is skipped", + comment="// don't keep the star", + expected_location="models/orders.sql:4:8 [SQBRMODEL102]", + ), + ], + ids=lambda case: case.description, +) +def test_given_clickhouse_line_comment_when_running_rules_then_it_is_not_scanned_as_code( + test_case: ClickHouseCommentE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/orders.sql": ( + "MODEL (description 'Test model orders.', materialized table);\n" + f"WITH o AS (SELECT 1 AS id)\n{test_case.comment}\nSELECT * FROM o\n" + ), + }, + ) + + ran: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "rules", "run", "SQBRMODEL102"), project_dir=project_dir + ) + + output: str = ran.stdout + ran.stderr + assert "P001" not in output, output + assert test_case.expected_location in output, test_case.description From 91f5d6b6c31b262dba27f8fd6622ad632f17b9d3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:53:06 +0000 Subject: [PATCH 18/32] docs(contributing): explain why maturin develop needs --skip-install A plain maturin develop installs the pinned sqlbuild-cli from the package index instead of the workspace crate, so the sqb executable is a released build, and the install fails between a release commit and its publication. The contributor guide now says to always pass --skip-install or use make dev-native or make fast-native. --- CONTRIBUTING.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f51ee10759..c75806fac3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -62,6 +62,13 @@ Do not compare development-profile timings with release benchmarks. Dependency o changes still require `uv sync --all-extras`; `--skip-install` does not refresh installed metadata, dependencies or console scripts. +Always pass `--skip-install` to `maturin develop`, or use `make dev-native` or `make fast-native`. +Without it, maturin installs the `sqlbuild-cli` version that `pyproject.toml` pins from the package +index, not this checkout's `crates/sqlbuild-cli`: the `sqb` executable is then a released build, +and between a release commit and its publication the install fails because that version does not +exist yet. `uv sync` and `uv run` install the workspace `sqlbuild-cli` and rebuild the extension +when Rust sources change. + Before opening a pull request, run: ```bash From 9e9e976ec225545b3443a0612825fa027720b030 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 05:53:42 +0000 Subject: [PATCH 19/32] fix(make): stop make skills installing the development skill globally make skills, which regenerates the repository's bundled skill files, also ran sqb skills --global --target opencode and so wrote this checkout's development skill into the user's real OpenCode skills directory. It now regenerates only the repository files; the global install is the explicit make install-global-skill target. --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4bb3692469..e1ffa034ee 100644 --- a/Makefile +++ b/Makefile @@ -372,9 +372,13 @@ planner-differential: skills: uv run python -m scripts.generate_docs_skill - uv run sqb skills --global --target opencode uv run fensu skills +# Installs this checkout's development skill into your user-level OpenCode skills; never run by CI. +.PHONY: install-global-skill +install-global-skill: + uv run sqb skills --global --target opencode + DBT_EXECUTABLE ?= dbt export DBT_EXECUTABLE From 0c4224608e871a7c76413ee494ca4f6aa81ebbd4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 09:25:07 +0000 Subject: [PATCH 20/32] fix(analysis): take Polyglot's parser, typing and star-expansion fixes Update Polyglot to a revision that fixes several analysis gaps: - a star over a derived table expands to its columns, so SELECT * FROM (SELECT ...) [AS s] no longer reports column_count 0; - star EXCLUDE, REPLACE and RENAME apply when stars expand, so an EXCLUDE no longer causes a false B216 between set operation arms; - column-alias lists on table functions and tables name their columns; - Snowflake typed lambda parameters, (total DOUBLE, item VARIANT) ->, and a subscript before a colon path, f[0]:name, parse; - Snowflake reports CONNECT_BY_ISLEAF and CONNECT_BY_ISCYCLE, which it does not support, as unknown columns; - ClickHouse DATE_TRUNC keeps its date or timestamp type instead of text, so a DATE_TRUNC timestamp cursor no longer fails B301; - BigQuery RANGE, RANGE and RANGE parse; - truncated T-SQL such as SELECT IF(region > 1, re is a parse error instead of a parser panic, and nested set operations keep their grouping when transpiled to SQLite. A new compile E2E covers the dialect cases, and the tests that used the T-SQL panic now assert its parse error. --- Cargo.lock | 6 +- Cargo.toml | 4 +- .../lineage/tests/test_build_fast_lineage.rs | 7 +- .../cli/commands/main/compile/_test_types.py | 12 ++ .../cli/commands/main/compile/helpers.py | 6 + .../main/compile/test_dialect_syntax.py | 180 ++++++++++++++++++ .../sqlbuild/compiler/lineage/_test_types.py | 6 +- .../lineage/test_native_fast_lineage.py | 24 ++- .../compiler/project_assembly/_test_types.py | 6 +- .../test_native_project_assembly.py | 20 +- 10 files changed, 232 insertions(+), 39 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_syntax.py diff --git a/Cargo.lock b/Cargo.lock index 60356bd71e..a03b6b41b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -711,7 +711,7 @@ checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "polyglot-sql" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=1ef5056c2b315e69bd13023bd90ffe60d94880a3#1ef5056c2b315e69bd13023bd90ffe60d94880a3" +source = "git+https://github.com/chio-labs/polyglot.git?rev=f76a2e36abb352f40dd43c829b6c336867ef7585#f76a2e36abb352f40dd43c829b6c336867ef7585" dependencies = [ "polyglot-sql-ast-derive", "polyglot-sql-function-catalogs", @@ -726,7 +726,7 @@ dependencies = [ [[package]] name = "polyglot-sql-ast-derive" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=1ef5056c2b315e69bd13023bd90ffe60d94880a3#1ef5056c2b315e69bd13023bd90ffe60d94880a3" +source = "git+https://github.com/chio-labs/polyglot.git?rev=f76a2e36abb352f40dd43c829b6c336867ef7585#f76a2e36abb352f40dd43c829b6c336867ef7585" dependencies = [ "proc-macro2", "quote", @@ -736,7 +736,7 @@ dependencies = [ [[package]] name = "polyglot-sql-function-catalogs" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=1ef5056c2b315e69bd13023bd90ffe60d94880a3#1ef5056c2b315e69bd13023bd90ffe60d94880a3" +source = "git+https://github.com/chio-labs/polyglot.git?rev=f76a2e36abb352f40dd43c829b6c336867ef7585#f76a2e36abb352f40dd43c829b6c336867ef7585" [[package]] name = "portable-atomic" diff --git a/Cargo.toml b/Cargo.toml index 5f2e8c9c6a..093d94e748 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,14 +72,14 @@ junction = { version = "2.1.0", default-features = false } mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] } proc-macro2 = { version = "1", features = ["span-locations"] } pyo3 = { version = "0.26", features = ["abi3-py312"] } -polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "1ef5056c2b315e69bd13023bd90ffe60d94880a3", default-features = false, features = [ +polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "f76a2e36abb352f40dd43c829b6c336867ef7585", default-features = false, features = [ "ast-tools", "builder", "all-dialects", "semantic", "stacker", ] } -polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "1ef5056c2b315e69bd13023bd90ffe60d94880a3", default-features = false, features = [ +polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "f76a2e36abb352f40dd43c829b6c336867ef7585", default-features = false, features = [ "dialect-duckdb", ] } blake3 = { version = "1", features = ["rayon"] } diff --git a/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs b/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs index a384f09acf..9998ac560d 100644 --- a/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs +++ b/crates/sqlbuild-analysis/src/lineage/tests/test_build_fast_lineage.rs @@ -196,16 +196,15 @@ fn given_models_without_compact_facts_when_building_then_matches_python_lineage( expected_detail: Some("Expected 1 statement, found 2"), }, ParsedLineageTestCase { - description: "a parser panic on truncated T-SQL fails as an internal error", + description: "truncated T-SQL reports its parse error", sql: "SELECT IF(region > 1, re", dialect: Some("tsql"), inferred_columns: &[], - expected_status: "error", + expected_status: "unparsed", expected_lines: &[], expected_has_star: false, expected_detail: Some( - "NativeCompilerError: native SQL compilation panicked \ - (native fast lineage of request model 0)", + "Parse error at line 1, column 25: Expected ')' to close the IF condition", ), }, ParsedLineageTestCase { diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index a31c7012e5..44fad988b9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1332,3 +1332,15 @@ class SnowflakeStarContractE2ETestCase: upstream_columns: str expected_exit_code: int expected_k005_columns: tuple[str, ...] + + +@dataclass(frozen=True) +class DialectSyntaxCompileE2ETestCase: + """A project using dialect syntax the SQL parser must analyse, and the compile outcome.""" + + description: str + adapter: str + models: dict[str, str] + expected_exit_code: int + expected_codes: tuple[str, ...] + expected_column_counts: dict[str, int] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 291e7ac39e..c11c87a534 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -5338,3 +5338,9 @@ def run_benchmark_generator( check=False, timeout=300, ) + + +def dialect_syntax_model(description: str, sql: str, config: str = "") -> str: + """Build a model file with a description, optional extra header config and its query.""" + + return f"MODEL (description '{description}'{config});\n{sql}\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_syntax.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_syntax.py new file mode 100644 index 0000000000..3a72dece78 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_syntax.py @@ -0,0 +1,180 @@ +"""E2E coverage of dialect syntax that compile analysis parses, types and expands.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + DialectSyntaxCompileE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import dialect_syntax_model +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_CONNECTIONS: dict[str, str] = { + "duckdb": '[connection]\ndatabase = "shop.duckdb"\n', + "snowflake": ( + '[connection]\naccount = "example"\nuser = "builder"\n\n' + '[defaults]\ndatabase = "ANALYTICS"\nschema = "MARTS"\n' + ), + "clickhouse": ( + '[connection]\nhost = "localhost"\ndatabase = "default"\n\n[defaults]\nschema = "default"\n' + ), + "bigquery": ( + '[connection]\nproject = "example"\ndataset = "marts"\n\n' + '[defaults]\ndatabase = "example"\nschema = "marts"\n' + ), +} + + +@pytest.mark.parametrize( + "test_case", + [ + DialectSyntaxCompileE2ETestCase( + description="stars over derived tables expand to their columns", + adapter="duckdb", + models={ + "unaliased": dialect_syntax_model( + "Unaliased.", "SELECT * FROM (SELECT 1 AS order_id, 'a' AS status)" + ), + "set_operation": dialect_syntax_model( + "Set operation.", + "SELECT * FROM (SELECT 1 AS order_id UNION ALL SELECT 2) AS s", + ), + "renamed": dialect_syntax_model( + "Renamed.", "SELECT * FROM (SELECT 1 AS order_id, 'a' AS status) AS s(a, b)" + ), + }, + expected_exit_code=0, + expected_codes=(), + expected_column_counts={"unaliased": 2, "set_operation": 1, "renamed": 2}, + ), + DialectSyntaxCompileE2ETestCase( + description="star EXCLUDE drops a column before set operation arms are compared", + adapter="duckdb", + models={ + "orders": dialect_syntax_model( + "Orders.", + "SELECT * EXCLUDE (status) FROM (SELECT 1 AS order_id, 'a' AS status) AS t " + "UNION ALL SELECT 2 AS order_id", + ), + }, + expected_exit_code=0, + expected_codes=(), + expected_column_counts={"orders": 1}, + ), + DialectSyntaxCompileE2ETestCase( + description="column alias lists name table function and table columns", + adapter="duckdb", + models={ + "series": dialect_syntax_model( + "Series.", "SELECT g.n FROM generate_series(1, 3) AS g(n)" + ), + "orders": dialect_syntax_model("Orders.", "SELECT 1 AS id, 'a' AS items"), + "renamed_orders": dialect_syntax_model( + "Renamed orders.", 'SELECT o.a, o.b FROM __ref("orders") AS o(a, b)' + ), + }, + expected_exit_code=0, + expected_codes=(), + expected_column_counts={"series": 1, "renamed_orders": 2}, + ), + DialectSyntaxCompileE2ETestCase( + description="Snowflake typed lambdas and subscripts before a colon path parse", + adapter="snowflake", + models={ + "totals": dialect_syntax_model( + "Totals.", + "SELECT REDUCE(ARRAY_CONSTRUCT(1, 2), 0::DOUBLE, " + "(total DOUBLE, item VARIANT) -> total + item::DOUBLE) AS total", + ), + "names": dialect_syntax_model( + "Names.", + "SELECT o.f[0]:name::VARCHAR AS name " + "FROM (SELECT PARSE_JSON('[{\"name\": 1}]') AS f) AS o", + ), + }, + expected_exit_code=0, + expected_codes=(), + expected_column_counts={"totals": 1, "names": 1}, + ), + DialectSyntaxCompileE2ETestCase( + description="Snowflake rejects Oracle's CONNECT_BY_ISLEAF pseudo-column", + adapter="snowflake", + models={ + "tree": dialect_syntax_model( + "Tree.", + "SELECT e.id, CONNECT_BY_ISLEAF AS leaf, LEVEL AS depth " + "FROM (SELECT 1 AS id, NULL AS parent_id) AS e " + "START WITH e.parent_id IS NULL CONNECT BY PRIOR e.id = e.parent_id", + ), + }, + expected_exit_code=1, + expected_codes=("B002",), + expected_column_counts={}, + ), + DialectSyntaxCompileE2ETestCase( + description="a ClickHouse DATE_TRUNC timestamp cursor keeps its timestamp type", + adapter="clickhouse", + models={ + "events": dialect_syntax_model("Events.", "SELECT CAST(now() AS DateTime) AS ts"), + "hourly_events": dialect_syntax_model( + "Hourly events.", + "SELECT DATE_TRUNC('hour', e.ts) AS hour_ts FROM __ref(\"events\") AS e", + ", materialized incremental, incremental_strategy append, cursor hour_ts, " + "cursor_type timestamp, cursor_grain hour", + ), + }, + expected_exit_code=0, + expected_codes=(), + expected_column_counts={"hourly_events": 1}, + ), + DialectSyntaxCompileE2ETestCase( + description="BigQuery RANGE types parse in casts", + adapter="bigquery", + models={ + "periods": dialect_syntax_model( + "Periods.", + "SELECT CAST(NULL AS RANGE) AS period, " + "CAST(NULL AS RANGE) AS window", + ), + }, + expected_exit_code=0, + expected_codes=(), + expected_column_counts={"periods": 2}, + ), + ], + ids=lambda case: case.description, +) +def test_given_dialect_syntax_when_compiling_then_analysis_parses_and_types_it( + test_case: DialectSyntaxCompileE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="shop", + repo_files={ + "sqlbuild_project.toml": ( + f'name = "shop"\nadapter = "{test_case.adapter}"\n\n' + f"{_CONNECTIONS[test_case.adapter]}\n[scopes]\nenforce_placement = false\n" + ), + **{f"models/{name}.sql": sql for name, sql in test_case.models.items()}, + }, + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile", "--json"), project_dir=project_dir + ) + + report: dict[str, Any] = json.loads(compiled.stdout) + column_counts: dict[str, int] = { + model["name"]: model["column_count"] for model in report["resources"]["models"] + } + assert compiled.returncode == test_case.expected_exit_code, compiled.stdout + compiled.stderr + assert tuple(item["code"] for item in report["diagnostics"]) == test_case.expected_codes + assert { + name: column_counts[name] for name in test_case.expected_column_counts + } == test_case.expected_column_counts, test_case.description diff --git a/tests/integration/src/sqlbuild/compiler/lineage/_test_types.py b/tests/integration/src/sqlbuild/compiler/lineage/_test_types.py index b8ae9adfd3..398b3d8e02 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/_test_types.py @@ -37,10 +37,10 @@ class UnparsedLineageTestCase: @dataclass(frozen=True) -class ParserPanicLineageTestCase: - """A model whose SQL panics the native parser, failing lineage as an internal error.""" +class TruncatedSqlLineageTestCase: + """A model whose SQL ends mid-expression, which fast lineage leaves out.""" description: str dialect: str query_sql: str - expected_message: str + expected_has_model: bool diff --git a/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py b/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py index e9c5cec76c..3dcaebd13c 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py @@ -11,7 +11,6 @@ import pytest -import sqlbuild._native as native_module from sqlbuild._native import ( ProjectColumnLineage, compiled_model_record, @@ -29,7 +28,7 @@ from tests.integration.src.sqlbuild.compiler.lineage._test_types import ( FormerlyDeferredLineageTestCase, GeneratedLineageParityTestCase, - ParserPanicLineageTestCase, + TruncatedSqlLineageTestCase, UnparsedLineageTestCase, ) from tests.integration.src.sqlbuild.compiler.lineage.helpers import ( @@ -241,20 +240,17 @@ def test_given_unknown_dialect_when_building_fast_lineage_then_python_error_rais @pytest.mark.parametrize( "test_case", [ - ParserPanicLineageTestCase( - description="truncated T-SQL the parser panics on, which crashes the wheel's process", + TruncatedSqlLineageTestCase( + description="truncated T-SQL the parser once panicked on is left out", dialect="tsql", query_sql="SELECT IF(region > 1, re", - expected_message=( - "NativeCompilerError: native SQL compilation panicked " - "(native fast lineage of request model 0)" - ), + expected_has_model=False, ), ], ids=lambda case: case.description, ) -def test_given_parser_panic_when_building_fast_lineage_then_raises_native_compiler_error( - test_case: ParserPanicLineageTestCase, tmp_path: Path +def test_given_truncated_sql_when_building_fast_lineage_then_omits_the_model( + test_case: TruncatedSqlLineageTestCase, tmp_path: Path ) -> None: project: CompiledProjectView = compiled_project( project_dir=tmp_path / "project", @@ -272,10 +268,12 @@ def test_given_parser_panic_when_building_fast_lineage_then_raises_native_compil ), ) - with pytest.raises(native_module.NativeCompilerError) as raised: - _ = fast_column_lineage(project=broken, dialect=test_case.dialect, model_names=None) + native: ProjectColumnLineage | None = fast_column_lineage( + project=broken, dialect=test_case.dialect, model_names=None + ) - assert str(raised.value) == test_case.expected_message + assert native is not None + assert native.has_model(project.models[0].name) is test_case.expected_has_model if __name__ == "__main__": diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py index 17fd7216d5..c7047fc3ae 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/_test_types.py @@ -53,10 +53,10 @@ class OptOutTestCase: @dataclass(frozen=True) -class NativePanicTestCase: - """A project whose native assembly panics in the parser.""" +class TruncatedHookTestCase: + """A project whose hook SQL ends mid-expression, which assembly reports as invalid input.""" description: str files: dict[str, str] dialect: str - expected_message: str + expected_message_fragment: str diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py index e3bc480ef9..e8f15d0d8e 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/test_native_project_assembly.py @@ -10,7 +10,6 @@ import pytest -import sqlbuild._native as native_module from sqlbuild._native import assemble_project from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -26,8 +25,8 @@ AssemblyDeferralTestCase, EdgeAssemblyTestCase, GeneratedAssemblyParityTestCase, - NativePanicTestCase, OptOutTestCase, + TruncatedHookTestCase, WindowsEnvironmentTestCase, ) from tests.integration.src.sqlbuild.compiler.project_assembly.helpers import ( @@ -255,8 +254,8 @@ def test_given_sql_analysis_opt_outs_when_attaching_natively_then_python_rejecti @pytest.mark.parametrize( "test_case", [ - NativePanicTestCase( - description="a T-SQL hook the parser panics on", + TruncatedHookTestCase( + description="a T-SQL hook the parser once panicked on", files={ "sqlbuild_project.toml": _PROJECT_TOML, "models/orders.sql": ( @@ -265,25 +264,24 @@ def test_given_sql_analysis_opt_outs_when_attaching_natively_then_python_rejecti ), }, dialect="tsql", - expected_message=( - "NativeCompilerError: native SQL compilation panicked (native project assembly)" - ), + expected_message_fragment="Expected ')' to close the IF condition", ), ], ids=lambda case: case.description, ) -def test_given_native_assembly_panics_when_assembling_then_raises_without_python_retry( - test_case: NativePanicTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +def test_given_truncated_hook_sql_when_assembling_then_reports_it_without_python_retry( + test_case: TruncatedHookTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: inputs: CompileInputsView = project_inputs( project_dir=tmp_path / "project", files=test_case.files ) calls: Counter[str] = python_resource_calls(monkeypatch=monkeypatch) - with pytest.raises(native_module.NativeCompilerError) as raised: + with pytest.raises(CompileInputError) as raised: _ = assemble_project( inputs=inputs, inference_profile=ExpressionInferenceProfile(sql_analysis_dialect=test_case.dialect), ) - assert (str(raised.value), calls) == (test_case.expected_message, Counter()) + assert test_case.expected_message_fragment in str(raised.value) + assert calls == Counter() From 829a701bc13200511b343719c2108a78003d7646 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 09:28:32 +0000 Subject: [PATCH 21/32] test(scenario): describe leaked lifecycle events when the pipeline order check fails The scenario pipeline orchestration test has failed on CI with two unexpected statement events between the connection operation and the scenario attempts, and it does not reproduce locally. Its event-order assertion now reports each event's invocation, run, adapter, purpose and statement kind, so the next failure shows whether the statements came from this test's pipeline or leaked from another test. --- .../sqlbuild/executor/pipeline/_helpers/helpers.py | 11 +++++++++++ .../executor/pipeline/_helpers/test_scenario.py | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py index 9c59b2342d..d67ac5307a 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/helpers.py @@ -409,3 +409,14 @@ def order_status_callee_test_plan(*, callee_body_sql: str, test_count: int) -> P ), function_locations={**plan.function_locations, "status_matches": callee_destination}, ) + + +def describe_lifecycle_events(events: list[LifecycleEvent]) -> list[str]: + """Describe events for an assertion message, with the invocation and run that produced them.""" + + return [ + f"{event.event_type} invocation={event.invocation_id} run={event.run_id} " + f"adapter={event.payload.get('adapter')} purpose={event.payload.get('purpose')} " + f"kind={event.payload.get('statement_kind')} resource={event.resource_id}" + for event in events + ] diff --git a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py index 2a276b5d77..d6fdebf610 100644 --- a/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py +++ b/tests/unit/src/sqlbuild/executor/pipeline/_helpers/test_scenario.py @@ -43,6 +43,7 @@ ScenarioPipelinePlanBuilder, ScenarioPipelineTestAdapter, build_scenario_pipeline_result, + describe_lifecycle_events, local_snapshot_loader_for_test_case, ) @@ -139,7 +140,7 @@ def complete_scenario( "resource_attempt_completed", "resource_attempt_started", "resource_attempt_failed", - ) + ), describe_lifecycle_events(lifecycle_events) assert tuple(event.resource_id for event in resource_events) == ( "sql_scenario:passing_scenario", "sql_scenario:passing_scenario", From f9b23f8f677b356cf1a13718ca090c1eeb5f0502 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 09:33:45 +0000 Subject: [PATCH 22/32] test(recall): dump every thread's stack when a Recall test's CLI run hangs The four-concurrent-builds Recall test once timed out after 600 s inside sqb recall query under heavy load, and the hang left no evidence. The Recall test helpers now run the CLI with PYTHONFAULTHANDLER, and a CLI run, or a concurrent build, still running after 600 s is aborted with SIGABRT so the failure carries every thread's stack. --- .../src/sqlbuild/cli/recall/helpers.py | 47 ++++++++++++++----- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index 64cc99889a..c88a32de38 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -24,6 +24,8 @@ EMITTED_STDERR: bytes = "warning \u2713 from the task\n".encode() EMITTED_TAIL: bytes = b"after the warning\n" MEBIBYTE: int = 1024 * 1024 +HANG_SECONDS: int = 600 +HANG_DUMP_GRACE_SECONDS: int = 30 EMPTY_RESULT: bytes = b'{"rows": []}' NUMBERED_LINE: re.Pattern[bytes] = re.compile(rb"^(\d+) std(?:out|err)$", re.MULTILINE) PROJECT_TOML: str = ( @@ -108,22 +110,36 @@ def write_project(project_dir: Path, *, extra_toml: str = "", targets: int = 1) def environment(**overrides: str) -> dict[str, str]: - """The process environment for one CLI run.""" + """The process environment for one CLI run; a hang dump prints every thread's stack.""" - return {**os.environ, "NO_COLOR": "1", **overrides} + return {**os.environ, "NO_COLOR": "1", "PYTHONFAULTHANDLER": "1", **overrides} def run(project_dir: Path, *args: str, **env: str) -> subprocess.CompletedProcess[bytes]: - """Run the Python CLI once in the project and capture exact streams.""" + """Run the Python CLI once in the project and capture exact streams, or fail with its stacks.""" + + process: subprocess.Popen[bytes] = start(project_dir, *args, **env) + try: + stdout, stderr = process.communicate(timeout=HANG_SECONDS) + except subprocess.TimeoutExpired: + raise AssertionError( + f"sqlbuild {shlex.join(args)} hung for {HANG_SECONDS}s:\n{hang_dump(process)}" + ) from None + return subprocess.CompletedProcess(process.args, process.returncode, stdout, stderr) - return subprocess.run( - [str(PYTHON_CLI), *args], - cwd=project_dir, - capture_output=True, - check=False, - env=environment(**env), - timeout=600, - ) + +def hang_dump(process: subprocess.Popen[bytes]) -> str: + """Abort a hung CLI's process group so it prints every thread's stack, and return its stderr.""" + + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGABRT) + try: + _, stderr = process.communicate(timeout=HANG_DUMP_GRACE_SECONDS) + except subprocess.TimeoutExpired: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + _, stderr = process.communicate() + return stderr.decode(errors="replace") def start(project_dir: Path, *args: str, **env: str) -> subprocess.Popen[bytes]: @@ -267,15 +283,22 @@ def concurrent_builds(project_dir: Path, targets: tuple[str, ...]) -> tuple[list ] reads: int = 0 unexpected_failures: int = 0 - while any(build.poll() is None for build in builds): + deadline: float = time.monotonic() + HANG_SECONDS + while any(build.poll() is None for build in builds) and time.monotonic() < deadline: reader: subprocess.CompletedProcess[bytes] = run( project_dir, "recall", "query", "--json", "SELECT COUNT(*) FROM invocations" ) reads += 1 unexpected_failures += int(reader.returncode != 0 and b"[C971]" not in reader.stderr) + hung: list[str] = list(map(hang_dump, filter(_still_running, builds))) + assert not hung, f"{len(hung)} builds still ran after {HANG_SECONDS}s:\n" + "\n".join(hung) return [build.wait(timeout=600) for build in builds], reads, unexpected_failures +def _still_running(process: subprocess.Popen[bytes]) -> bool: + return process.poll() is None + + def timed_build(project_dir: Path) -> tuple[subprocess.CompletedProcess[bytes], float, float]: """A build with the seconds until its first stderr byte and until it exited.""" From 6f90c22d11c8a0908447a21e992ecf3a3a6151d7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 09:51:09 +0000 Subject: [PATCH 23/32] fix(compile): log why a warm compile could not keep cached SQL test artifacts A warm compile has intermittently re-planned SQL tests on CI, printing an extra SQL test planning line, and the miss left no evidence of its cause. When the SQL test artifact cache holds records but some tests cannot reuse theirs, compile now logs one debug event giving the count per reason: not recorded, identity changed, unsafe path, artifact missing, size changed, unreadable mtime or content changed. The event goes to the invocation's diagnostic log and to --debug output. --- .../main/reusable_sql_test_record.rs | 32 +++++---- crates/sqlbuild-cache/src/artifacts/models.rs | 26 +++++++ crates/sqlbuild-cache/src/artifacts/tests.rs | 2 + .../src/artifacts/tests/helpers.rs | 10 +++ .../artifacts/tests/test_reusable_record.rs | 68 +++++++++++++++++++ .../src/artifacts/tests/test_types.rs | 9 +++ .../_helpers/compile_command/target_writer.rs | 38 +++++++++-- .../src/bindings/_helpers/native_host/host.rs | 18 +++++ .../cli/commands/main/compile/_test_types.py | 9 +++ .../cli/commands/main/compile/helpers.py | 14 ++++ .../compile/test_sql_test_cache_misses.py | 59 ++++++++++++++++ 11 files changed, 266 insertions(+), 19 deletions(-) create mode 100644 crates/sqlbuild-cache/src/artifacts/tests/test_reusable_record.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_sql_test_cache_misses.py diff --git a/crates/sqlbuild-cache/src/artifacts/main/reusable_sql_test_record.rs b/crates/sqlbuild-cache/src/artifacts/main/reusable_sql_test_record.rs index 73d58f8154..87bb3c3c47 100644 --- a/crates/sqlbuild-cache/src/artifacts/main/reusable_sql_test_record.rs +++ b/crates/sqlbuild-cache/src/artifacts/main/reusable_sql_test_record.rs @@ -6,31 +6,35 @@ use std::path::{Path, PathBuf}; use crate::artifacts::_helpers::sql_test_records::{ file_sha256, is_safe_relative_sql_path, metadata, mtime_ns, }; -use crate::artifacts::models::SqlTestArtifactRecord; +use crate::artifacts::models::{SqlTestArtifactRecord, SqlTestRecordMiss}; -/// The record with its artifact's current mtime, when the artifact is unchanged. -#[must_use] +/// The record with its artifact's current mtime when the artifact is unchanged, else why not. pub fn reusable_sql_test_record( tests_root: &Path, record: &SqlTestArtifactRecord, identity: &str, -) -> Option { - if record.identity != identity || !is_safe_relative_sql_path(&record.relative_path) { - return None; +) -> Result { + if record.identity != identity { + return Err(SqlTestRecordMiss::IdentityChanged); + } + if !is_safe_relative_sql_path(&record.relative_path) { + return Err(SqlTestRecordMiss::UnsafePath); } let artifact_path: PathBuf = tests_root.join(&record.relative_path); - let metadata: Metadata = metadata(&artifact_path)?; - if !metadata.is_file() || metadata.len() != record.size { - return None; + let metadata: Metadata = metadata(&artifact_path) + .filter(Metadata::is_file) + .ok_or(SqlTestRecordMiss::ArtifactMissing)?; + if metadata.len() != record.size { + return Err(SqlTestRecordMiss::SizeChanged); } - let current_mtime_ns: i64 = mtime_ns(&metadata)?; + let current_mtime_ns: i64 = mtime_ns(&metadata).ok_or(SqlTestRecordMiss::UnreadableMtime)?; if current_mtime_ns == record.mtime_ns { - return Some(record.clone()); + return Ok(record.clone()); } - if file_sha256(&artifact_path)? != record.content_sha256 { - return None; + if file_sha256(&artifact_path).as_deref() != Some(record.content_sha256.as_str()) { + return Err(SqlTestRecordMiss::ContentChanged); } - Some(SqlTestArtifactRecord { + Ok(SqlTestArtifactRecord { mtime_ns: current_mtime_ns, ..record.clone() }) diff --git a/crates/sqlbuild-cache/src/artifacts/models.rs b/crates/sqlbuild-cache/src/artifacts/models.rs index d3eb9963e5..fb8d1c2a05 100644 --- a/crates/sqlbuild-cache/src/artifacts/models.rs +++ b/crates/sqlbuild-cache/src/artifacts/models.rs @@ -33,6 +33,32 @@ pub struct SqlTestArtifactRecord { pub content_sha256: String, } +/// Why a cached SQL test artifact could not be kept. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SqlTestRecordMiss { + IdentityChanged, + UnsafePath, + ArtifactMissing, + SizeChanged, + UnreadableMtime, + ContentChanged, +} + +impl SqlTestRecordMiss { + /// The reason as a short snake-case label for debug logs. + #[must_use] + pub const fn label(self) -> &'static str { + match self { + Self::IdentityChanged => "identity_changed", + Self::UnsafePath => "unsafe_path", + Self::ArtifactMissing => "artifact_missing", + Self::SizeChanged => "size_changed", + Self::UnreadableMtime => "unreadable_mtime", + Self::ContentChanged => "content_changed", + } + } +} + /// The fields of one cache file record, already checked to be strings and non-negative integers. #[derive(Clone, Debug, PartialEq, Eq)] pub struct StoredSqlTestRecord { diff --git a/crates/sqlbuild-cache/src/artifacts/tests.rs b/crates/sqlbuild-cache/src/artifacts/tests.rs index 6d280821bd..7a54e11b92 100644 --- a/crates/sqlbuild-cache/src/artifacts/tests.rs +++ b/crates/sqlbuild-cache/src/artifacts/tests.rs @@ -3,6 +3,8 @@ mod helpers; #[cfg(unix)] #[path = "tests/test_remove_stale.rs"] mod remove_stale; +#[path = "tests/test_reusable_record.rs"] +mod reusable_record; #[path = "tests/test_types.rs"] mod test_types; #[path = "tests/test_write_artifacts.rs"] diff --git a/crates/sqlbuild-cache/src/artifacts/tests/helpers.rs b/crates/sqlbuild-cache/src/artifacts/tests/helpers.rs index 114464f151..9262e9b2e9 100644 --- a/crates/sqlbuild-cache/src/artifacts/tests/helpers.rs +++ b/crates/sqlbuild-cache/src/artifacts/tests/helpers.rs @@ -47,3 +47,13 @@ pub(crate) fn linked_compiled_root(target: &Path) { std::fs::create_dir_all(target.join("elsewhere")).expect("linked folder"); std::os::unix::fs::symlink(target.join("elsewhere"), target.join("compiled")).expect("link"); } + +/// Move `root/t.sql`'s mtime five seconds ahead without changing its bytes. +pub(crate) fn bump_mtime(root: &Path) { + let file = std::fs::File::options() + .write(true) + .open(root.join("t.sql")) + .expect("artifact"); + let later = std::time::SystemTime::now() + std::time::Duration::from_secs(5); + file.set_modified(later).expect("mtime"); +} diff --git a/crates/sqlbuild-cache/src/artifacts/tests/test_reusable_record.rs b/crates/sqlbuild-cache/src/artifacts/tests/test_reusable_record.rs new file mode 100644 index 0000000000..e7037a63f9 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/tests/test_reusable_record.rs @@ -0,0 +1,68 @@ +use crate::artifacts::main::new_sql_test_record::new_sql_test_record; +use crate::artifacts::main::reusable_sql_test_record::reusable_sql_test_record; +use crate::artifacts::models::SqlTestRecordMiss; +use crate::artifacts::tests::helpers::{bump_mtime, place}; +use crate::artifacts::tests::test_types::ReusableRecordTestCase; + +const IDENTITY: &str = "identity-1"; + +#[test] +fn given_recorded_artifact_when_checking_reuse_then_reports_why_it_missed() { + let test_cases = [ + ReusableRecordTestCase { + description: "an unchanged artifact is kept", + change: |_| {}, + identity: IDENTITY, + expected_outcome: Ok(()), + }, + ReusableRecordTestCase { + description: "a touched artifact with the same bytes is kept", + change: bump_mtime, + identity: IDENTITY, + expected_outcome: Ok(()), + }, + ReusableRecordTestCase { + description: "a new identity misses", + change: |_| {}, + identity: "identity-2", + expected_outcome: Err(SqlTestRecordMiss::IdentityChanged), + }, + ReusableRecordTestCase { + description: "a deleted artifact misses", + change: |root| std::fs::remove_file(root.join("t.sql")).expect("removed"), + identity: IDENTITY, + expected_outcome: Err(SqlTestRecordMiss::ArtifactMissing), + }, + ReusableRecordTestCase { + description: "a resized artifact misses", + change: |root| place(&root.join("t.sql"), b"select 10\n"), + identity: IDENTITY, + expected_outcome: Err(SqlTestRecordMiss::SizeChanged), + }, + ReusableRecordTestCase { + description: "same-size new bytes miss", + change: |root| { + place(&root.join("t.sql"), b"select 2\n"); + bump_mtime(root); + }, + identity: IDENTITY, + expected_outcome: Err(SqlTestRecordMiss::ContentChanged), + }, + ]; + for test_case in test_cases { + let root = tempfile::tempdir().expect("tests folder"); + place(&root.path().join("t.sql"), b"select 1\n"); + let record = new_sql_test_record(root.path(), &root.path().join("t.sql"), IDENTITY) + .expect("recorded artifact"); + (test_case.change)(root.path()); + + let outcome = reusable_sql_test_record(root.path(), &record, test_case.identity); + + assert_eq!( + outcome.map(|_| ()), + test_case.expected_outcome, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cache/src/artifacts/tests/test_types.rs b/crates/sqlbuild-cache/src/artifacts/tests/test_types.rs index a475d71bf5..12583ab0a3 100644 --- a/crates/sqlbuild-cache/src/artifacts/tests/test_types.rs +++ b/crates/sqlbuild-cache/src/artifacts/tests/test_types.rs @@ -48,3 +48,12 @@ pub(crate) struct WalkRaceTestCase { pub(crate) change_during_walk: fn(&std::path::Path), pub(crate) expected_stamped: bool, } + +/// A change to a recorded SQL test artifact and whether, or why not, the record is kept. +pub(crate) struct ReusableRecordTestCase { + pub(crate) description: &'static str, + /// Changes the tests folder after the artifact `t.sql` was recorded. + pub(crate) change: fn(&std::path::Path), + pub(crate) identity: &'static str, + pub(crate) expected_outcome: Result<(), crate::artifacts::models::SqlTestRecordMiss>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index ffeb324443..1a10104441 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -1,6 +1,6 @@ //! Write compiled project output to `target/`: models, functions, audits, SQL tests, manifest. -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::path::{Path, PathBuf}; use pyo3::exceptions::PyValueError; @@ -11,7 +11,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; use pyo3::{IntoPyObjectExt, pyclass, pyfunction, wrap_pyfunction}; use sqlbuild_cache::artifacts::main::new_sql_test_record::new_sql_test_record; use sqlbuild_cache::artifacts::main::reusable_sql_test_record::reusable_sql_test_record; -use sqlbuild_cache::artifacts::models::SqlTestArtifactRecord; +use sqlbuild_cache::artifacts::models::{SqlTestArtifactRecord, SqlTestRecordMiss}; use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_host::entries::models::ExecutorHost; @@ -58,6 +58,9 @@ const MANIFEST_FILE: &str = "manifest.json"; /// The compile cache file stamping the compiled tree the last stale-file walk left. const COMPILED_TREE_STAMP_FILE: &str = "compiled-tree.bin"; const SQL_FILE_SUFFIX: &str = ".sql"; +const SQL_TEST_CACHE_LOGGER: &str = "sqlbuild.compile.reuse"; +/// The miss reason of a SQL test the cache file holds no record for. +const NOT_RECORDED: &str = "not_recorded"; /// `functions/` path components that already lead a function's path. const LANGUAGE_DIR_DEPTH: usize = 2; const PUBLISHED_STAGING_CHANGED: &str = "changed"; @@ -448,6 +451,20 @@ fn test_identities<'py>( Ok(identities) } +/// Log why a warm SQL test artifact cache could not keep some artifacts, by reason. +fn log_cache_misses(py: Python<'_>, misses: &BTreeMap<&'static str, usize>) -> PyResult<()> { + let reasons: String = misses + .iter() + .map(|(reason, count)| format!("{reason}={count}")) + .collect::>() + .join(","); + host::log_debug_message( + py, + SQL_TEST_CACHE_LOGGER, + &format!("SQL test artifact cache missed: {reasons}"), + ) +} + /// Plan every SQL test whose cached artifact under `target_dir` cannot be reused. fn plan_tests<'py>( target_dir: &Bound<'py, PyAny>, @@ -483,15 +500,23 @@ fn plan_tests<'py>( let mut pending: Vec = Vec::new(); let mut cached_paths: Vec> = Vec::new(); let mut cached_records: Vec<(String, SqlTestArtifactRecord)> = Vec::new(); + let mut misses: BTreeMap<&'static str, usize> = BTreeMap::new(); for (index, (test, identity)) in tests.into_iter().zip(identities).enumerate() { let Some((record_key, identity)) = identity else { pending.push((test.unbind(), None, None)); continue; }; - let reusable: Option = cached + let reusable: Result = cached .get(&record_key) - .and_then(|record| reusable_sql_test_record(&tests_root_path, record, &identity)); - if let Some(record) = reusable { + .ok_or(NOT_RECORDED) + .and_then(|record| { + reusable_sql_test_record(&tests_root_path, record, &identity) + .map_err(SqlTestRecordMiss::label) + }); + if let Err(reason) = reusable { + *misses.entry(reason).or_default() += 1; + } + if let Ok(record) = reusable { let path: Bound<'py, PyAny> = tests_root.div(&record.relative_path)?; artifact_kept(recorder, &path, (record.size, record.mtime_ns))?; cached_paths.push(path.unbind()); @@ -501,6 +526,9 @@ fn plan_tests<'py>( } yield_interpreter(py, index); } + if !cached.is_empty() && !misses.is_empty() { + log_cache_misses(py, &misses)?; + } let pending_tests: Vec> = pending .iter() .map(|(test, _, _)| test.bind(py).clone()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs index 5ed20ed5e2..c2904c48dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/host.rs @@ -64,3 +64,21 @@ impl<'py> HostEntries for PythonHost<'py> { pub(crate) fn entry<'py>(py: Python<'py>, id: impl Into) -> PyResult> { PythonHost(py).entry(id.into()) } + +/// `log_debug_event(logger=logging.getLogger(logger), message=message)`. +pub(crate) fn log_debug_message(py: Python<'_>, logger: &str, message: &str) -> PyResult<()> { + let kwargs: Bound<'_, PyDict> = PyDict::new(py); + kwargs.set_item( + "logger", + py.import("logging")? + .getattr("getLogger")? + .call1((logger,))?, + )?; + kwargs.set_item("message", message)?; + entry( + py, + sqlbuild_host::entries::models::ExecutorHost::LogDebugEvent, + )? + .call((), Some(&kwargs)) + .map(|_| ()) +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 44fad988b9..b668045bc4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1344,3 +1344,12 @@ class DialectSyntaxCompileE2ETestCase: expected_exit_code: int expected_codes: tuple[str, ...] expected_column_counts: dict[str, int] + + +@dataclass(frozen=True) +class SqlTestCacheMissE2ETestCase: + """A change to compiled SQL test artifacts and the miss reasons a warm compile logs.""" + + description: str + change: Callable[[Path], None] + expected_debug_line: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index c11c87a534..a3838018fe 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -5344,3 +5344,17 @@ def dialect_syntax_model(description: str, sql: str, config: str = "") -> str: """Build a model file with a description, optional extra header config and its query.""" return f"MODEL (description '{description}'{config});\n{sql}\n" + + +def append_to_compiled_sql_tests(project_dir: Path) -> None: + """Append a comment to every compiled SQL test artifact, changing its size.""" + + for artifact in sorted((project_dir / "target/compiled/tests").rglob("*.sql")): + _ = artifact.write_text(artifact.read_text() + "-- changed\n") + + +def flip_compiled_sql_test_bytes(project_dir: Path) -> None: + """Swap one character of every compiled SQL test artifact, keeping its size.""" + + for artifact in sorted((project_dir / "target/compiled/tests").rglob("*.sql")): + _ = artifact.write_text(artifact.read_text().replace("SELECT", "select", 1)) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_sql_test_cache_misses.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_sql_test_cache_misses.py new file mode 100644 index 0000000000..2f9750e3d5 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_sql_test_cache_misses.py @@ -0,0 +1,59 @@ +"""E2E coverage that a warm compile logs why it could not keep cached SQL test artifacts.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + SqlTestCacheMissE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + COMPILE_CACHE_REGION_ENV_VAR, + append_to_compiled_sql_tests, + flip_compiled_sql_test_bytes, + run_installed_sqb, +) + +_MISS_LINE: str = "SQL test artifact cache missed" +_ENV: dict[str, str] = {COMPILE_CACHE_REGION_ENV_VAR: "east"} + + +@pytest.mark.parametrize( + "test_case", + [ + SqlTestCacheMissE2ETestCase( + description="resized artifacts log size_changed", + change=append_to_compiled_sql_tests, + expected_debug_line=f"{_MISS_LINE}: size_changed=", + ), + SqlTestCacheMissE2ETestCase( + description="same-size rewritten artifacts log content_changed", + change=flip_compiled_sql_test_bytes, + expected_debug_line=f"{_MISS_LINE}: content_changed=", + ), + ], + ids=lambda case: case.description, +) +def test_given_changed_sql_test_artifacts_when_compiling_warm_then_debug_log_names_the_miss( + cache_invalidation_project: Path, test_case: SqlTestCacheMissE2ETestCase +) -> None: + project_dir: Path = cache_invalidation_project + debug_compile: tuple[str, ...] = ("--debug", "compile") + cold: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, args=debug_compile, env=_ENV + ) + warm: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, args=debug_compile, env=_ENV + ) + + test_case.change(project_dir) + changed: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, args=debug_compile, env=_ENV + ) + + assert (cold.returncode, warm.returncode, changed.returncode) == (0, 0, 0), changed.stderr + assert _MISS_LINE not in warm.stderr + assert test_case.expected_debug_line in changed.stderr, changed.stderr[-2000:] From 7e5bb13d267f77aa1783cccbddf4f7bf24d19a7d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 10:16:57 +0000 Subject: [PATCH 24/32] test(macros): expect a macro call in a star REPLACE to take its returns type With Polyglot applying star REPLACE during expansion, a SQL macro call\nin SELECT * REPLACE (...) now takes the macro's returns type, so the\nstrict xfail that recorded the gap passed and failed the suite. --- .../main/compile/sql_macros/test_sql_macro_typing_matrix.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py index 32bd9cb50d..52b91805db 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/sql_macros/test_sql_macro_typing_matrix.py @@ -480,9 +480,6 @@ def test_given_a_typed_call_case_when_compiling_then_it_reports_as_expected( assert fragment not in printed, printed -@pytest.mark.xfail( - reason="a * REPLACE item takes its source column's type; a known polyglot gap", strict=True -) @pytest.mark.parametrize( "test_case", [ From 3236459d8eb6305002f730094846cb07f6941f9d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 10:16:57 +0000 Subject: [PATCH 25/32] fix(rules): map model rule positions back through macro and declaration expansion Model rules located findings in the expanded query, so every macro\ncall that expanded into more lines than it was written in moved the\nfindings below it down; SQBRCONTRACT105 for a model with several\nmulti-line macro calls pointed many lines past the flagged output.\nCompiled model facts now keep the interpolation, declaration and macro\nexpansion passes, and rule positions map through them to the authored\nfile. A position inside an expansion points at the call that produced\nit. When the passes do not fit the query, positions fall back to the\nprevious query-relative mapping. --- .../src/compiled_project/_helpers/plain.rs | 1 + .../src/compiled_project/models.rs | 5 ++ .../_helpers/compiled_project/project.rs | 1 + .../_helpers/input_loops/model_loop_host.rs | 11 +++ .../src/engine/_helpers/project_rows.rs | 40 ++++++++++- .../src/engine/_helpers/retained.rs | 2 + .../src/engine/_helpers/rows.rs | 1 + .../src/engine/tests/helpers.rs | 1 + .../src/lint_text/main/map_expanded_offset.rs | 10 +++ .../sqlbuild-rules/src/lint_text/main/mod.rs | 1 + crates/sqlbuild-rules/src/lint_text/models.rs | 2 +- crates/sqlbuild-rules/src/models.rs | 16 +++++ .../sqlbuild-rules/src/query_facts/models.rs | 68 +++++++++++++++++++ .../src/rules/_helpers/evaluation.rs | 3 +- .../cli/commands/main/rules/_test_types.py | 6 +- .../main/rules/test_rule_positions.py | 43 +++++++++++- 16 files changed, 203 insertions(+), 8 deletions(-) create mode 100644 crates/sqlbuild-rules/src/lint_text/main/map_expanded_offset.rs diff --git a/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs b/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs index 3f6e1724f9..da86454d6c 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/_helpers/plain.rs @@ -40,5 +40,6 @@ pub(crate) fn plain_model_facts(model: PlainModel) -> CompiledModelFacts { constant_declarations: model.constant_declarations, analysis: None, typed_calls: model.typed_calls, + expansion_passes: model.expansion_passes, } } diff --git a/crates/sqlbuild-analysis/src/compiled_project/models.rs b/crates/sqlbuild-analysis/src/compiled_project/models.rs index 309221dca8..455cceb3f5 100644 --- a/crates/sqlbuild-analysis/src/compiled_project/models.rs +++ b/crates/sqlbuild-analysis/src/compiled_project/models.rs @@ -5,6 +5,7 @@ use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use serde_json::Value; +use sqlbuild_sqltext::compiler::models::CharSpan; /// One compile's project facts; models keep the order the compile-input stage produced them. #[derive(Clone, Debug, Default, PartialEq)] @@ -34,6 +35,8 @@ pub struct CompiledModelFacts { pub analysis: Option, /// Each expression SQL macro call in `query_sql`, by byte range. pub typed_calls: Vec, + /// The expansion passes from `authored_query_sql` to `query_sql`; empty when unknown. + pub expansion_passes: Vec>, } /// A model's effective config values with sorted keys, as the rules request has always read them. @@ -217,6 +220,8 @@ pub struct PlainModel { pub constant_declarations: Vec, /// Each expression SQL macro call in `query_sql`, by byte range. pub typed_calls: Vec, + /// The expansion passes from `authored_query_sql` to `query_sql`; empty when unknown. + pub expansion_passes: Vec>, } /// A one-based line and column, counted in characters. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index 2a7f6664d5..8f3f6575fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -587,6 +587,7 @@ pub(crate) fn model_facts(model: ModelInputPy<'_>) -> PyResult( }, }; let (values, encoding_error) = config_values(&stripped); + let expansion_passes: Vec> = if *record.query_sql == *record.expansion.expanded_sql + { + vec![ + record.expansion.interpolation_spans.clone(), + record.expansion.declaration_spans.clone(), + record.expansion.macro_spans.clone(), + ] + } else { + Vec::new() + }; let facts: CompiledModelFacts = plain_model_facts(PlainModel { name, relative_path, @@ -291,6 +301,7 @@ pub(crate) fn finished_record<'py>( enum_declarations: declarations(enums)?, constant_declarations: declarations(constants)?, typed_calls: finished.typed_calls, + expansion_passes, }); Ok((record, facts)) } diff --git a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs index 77074eb3a6..0c6f68f4a4 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/project_rows.rs @@ -3,10 +3,12 @@ use sqlbuild_analysis::compiled_project::models::{ CompiledModelFacts, DeclarationFact, ModelSchemaFact, }; +use sqlbuild_sqltext::compiler::models::CharSpan; +use crate::lint_text::models::ExpansionSpan; use crate::models::{ - Declaration, DeclarationMember, DynamicFamilyRow, DynamicProofRow, ModelRow, Reference, - SchemaColumnRow, SchemaRow, + AuthoredQueryMap, Declaration, DeclarationMember, DynamicFamilyRow, DynamicProofRow, ModelRow, + Reference, SchemaColumnRow, SchemaRow, }; /// The row the rules builder reads for one model; the authored query wins over the whole file. @@ -34,6 +36,7 @@ pub(crate) fn model_row(facts: &CompiledModelFacts) -> Result &facts.authored_query_sql, &facts.query_sql, ), + authored_query: authored_query_map(facts), config: facts.config.values.clone(), authored_config_keys: facts.config.header_keys.clone(), logical_schema: facts.config.layer_schema.clone(), @@ -112,6 +115,39 @@ pub(crate) fn declaration(fact: &DeclarationFact) -> Declaration { } } +/// The expansion passes back to the authored query and where that query starts in the file. +fn authored_query_map(facts: &CompiledModelFacts) -> Option { + let span = |span: &CharSpan| ExpansionSpan { + source_start: character_offset(span.source_start), + source_end: character_offset(span.source_end), + output_start: character_offset(span.output_start), + output_end: character_offset(span.output_end), + }; + if facts.authored_query_sql.is_empty() || facts.expansion_passes.iter().all(Vec::is_empty) { + return None; + } + let start: usize = facts.authored_sql.rfind(&*facts.authored_query_sql)?; + let above: &str = &facts.authored_sql[..start]; + Some(AuthoredQueryMap { + passes: facts + .expansion_passes + .iter() + .map(|pass| pass.iter().map(span).collect()) + .collect(), + line_offset: above.matches('\n').count() as u64, + column_offset: above + .rsplit('\n') + .next() + .unwrap_or_default() + .chars() + .count() as u64, + }) +} + +fn character_offset(offset: usize) -> i64 { + i64::try_from(offset).unwrap_or(i64::MAX) +} + /// The file-line characters before the query's first content that the query itself lacks. fn query_column_offset(file: &str, authored_query: &str, query: &str) -> u64 { let line_prefix = |text: &str, at: usize| { diff --git a/crates/sqlbuild-rules/src/engine/_helpers/retained.rs b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs index 8d9c843ad8..006c1474f4 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/retained.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/retained.rs @@ -134,6 +134,7 @@ impl RowWriter { authored_sql: _, query_line_offset: _, query_column_offset: _, + authored_query: _, typed_calls: _, config, authored_config_keys, @@ -353,6 +354,7 @@ impl RowReader<'_> { authored_sql: String::new(), query_line_offset: 0, query_column_offset: 0, + authored_query: None, typed_calls: Vec::new(), config: { let count: usize = self.count()?; diff --git a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs index 5cd8ac5cfb..3e47c5c188 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs @@ -85,6 +85,7 @@ fn build_model(row: ModelRow, inputs: &ModelInputs<'_, '_>) -> Result Model { typed_calls: Vec::new(), query_line_offset: 0, query_column_offset: 0, + authored_query: None, declared_audit_count: 3, targeting_test_count: 4, empty_input_only_test_count: 0, diff --git a/crates/sqlbuild-rules/src/lint_text/main/map_expanded_offset.rs b/crates/sqlbuild-rules/src/lint_text/main/map_expanded_offset.rs new file mode 100644 index 0000000000..be4a4d6ba2 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/map_expanded_offset.rs @@ -0,0 +1,10 @@ +//! Map an offset of expanded SQL back to its authored text. + +use crate::lint_text::_helpers::sql_diagnostics::map_offset; +use crate::lint_text::models::ExpansionSpan; + +/// The authored offset of expanded character `offset`, and whether expansion generated it. +#[must_use] +pub fn map_expanded_offset(offset: i64, passes: &[Vec]) -> (i64, bool) { + map_offset(offset, passes) +} diff --git a/crates/sqlbuild-rules/src/lint_text/main/mod.rs b/crates/sqlbuild-rules/src/lint_text/main/mod.rs index 61981d45cf..73885850a0 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/mod.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/mod.rs @@ -4,6 +4,7 @@ pub mod format_fixture_blocks; pub mod format_sql_bodies; pub mod frame_sql_macro_expression; pub mod lint_sql_bodies; +pub mod map_expanded_offset; pub mod named_declaration_role_kind; pub mod neutralize_audit_parameters; pub mod neutralize_context_interpolation; diff --git a/crates/sqlbuild-rules/src/lint_text/models.rs b/crates/sqlbuild-rules/src/lint_text/models.rs index 6d4ddb5c81..3634cbf666 100644 --- a/crates/sqlbuild-rules/src/lint_text/models.rs +++ b/crates/sqlbuild-rules/src/lint_text/models.rs @@ -95,7 +95,7 @@ pub(crate) struct PreparedBody { } /// One substitution of an expansion pass: authored source range and generated output range. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)] pub struct ExpansionSpan { pub source_start: i64, pub source_end: i64, diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index 06e2ed730d..c78de2bcf8 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -565,6 +565,9 @@ pub struct Model { /// The characters before `query_sql`'s first line on its file line, when it shares one. #[serde(default)] pub query_column_offset: u64, + /// How `query_sql` maps back to the authored query, when its expansion is known. + #[serde(default)] + pub authored_query: Option, pub config: BTreeMap, pub authored_config_keys: Vec, pub logical_schema: Option, @@ -869,6 +872,17 @@ impl fensu_policy::policy::types::PolicyRule for RuleMetadata { } } +/// The expansion passes from a model's authored query to `query_sql`, and the query's file place. +#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)] +pub struct AuthoredQueryMap { + /// Each pass's substitutions in characters, the first pass applied to the authored query. + pub passes: Vec>, + /// The lines of the model file above the authored query. + pub line_offset: u64, + /// The characters before the authored query on its first file line. + pub column_offset: u64, +} + /// One compiled model's facts as the rules request builder receives them. #[derive(Clone, Debug, Default)] pub struct ModelRow { @@ -880,6 +894,8 @@ pub struct ModelRow { pub query_line_offset: u64, /// The characters before `query_sql`'s first line on its file line. pub query_column_offset: u64, + /// How `query_sql` maps back to the authored query, when its expansion is known. + pub authored_query: Option, pub config: BTreeMap, pub authored_config_keys: Vec, pub logical_schema: Option, diff --git a/crates/sqlbuild-rules/src/query_facts/models.rs b/crates/sqlbuild-rules/src/query_facts/models.rs index 78617ee31d..35dd961830 100644 --- a/crates/sqlbuild-rules/src/query_facts/models.rs +++ b/crates/sqlbuild-rules/src/query_facts/models.rs @@ -1,6 +1,9 @@ use polyglot_sql::Expression; use std::collections::BTreeSet; +use crate::lint_text::main::map_expanded_offset::map_expanded_offset; +use crate::models::AuthoredQueryMap; + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum CallKind { Ref, @@ -231,6 +234,15 @@ pub(crate) struct ModelSource<'a> { column_offset: u64, /// The 1-based line of this SQL's first content. first_content_line: usize, + authored: Option>, +} + +/// The authored query that expansion turned into a model's SQL, as positions are read from it. +#[derive(Debug)] +struct AuthoredText<'a> { + characters: Vec, + lines: Vec, + map: &'a AuthoredQueryMap, } impl<'a> ModelSource<'a> { @@ -259,7 +271,60 @@ impl<'a> ModelSource<'a> { .matches('\n') .count() + 1, + authored: None, + } + } + + /// Report positions in `authored_sql`, the query `map`'s passes expanded into this SQL. + pub(crate) fn with_authored( + mut self, + authored_sql: &str, + map: Option<&'a AuthoredQueryMap>, + ) -> Self { + self.authored = map.map(|map| { + let characters: Vec = authored_sql.chars().collect(); + let mut lines: Vec = vec![0]; + for (index, character) in characters.iter().enumerate() { + if *character == '\n' { + lines.push(index + 1); + } + } + AuthoredText { + characters, + lines, + map, + } + }); + self + } + + /// The file line and column of SQL character `character` in the authored query, if it maps. + fn authored_line_column(&self, character: usize) -> Option<(u64, u64)> { + let authored: &AuthoredText<'_> = self.authored.as_ref()?; + let offset: i64 = i64::try_from(character).unwrap_or(i64::MAX); + let (mapped, generated) = map_expanded_offset(offset, &authored.map.passes); + let mapped: usize = usize::try_from(mapped).unwrap_or(usize::MAX); + let byte: usize = self.bytes.get(character).copied().unwrap_or(self.sql.len()); + let expected: Option = self.sql.get(byte..).and_then(|rest| rest.chars().next()); + let fits: bool = if generated { + mapped < authored.characters.len() + } else { + authored.characters.get(mapped).copied() == expected + }; + if !fits { + return None; } + let line: usize = authored.lines.partition_point(|start| *start <= mapped); + let column: usize = mapped - authored.lines[line - 1] + 1; + let shifted: u64 = if line == 1 { + authored.map.column_offset + } else { + 0 + }; + Some(( + line as u64 + authored.map.line_offset, + column as u64 + shifted, + )) } fn original(&self, offset: usize) -> usize { @@ -280,6 +345,9 @@ impl<'a> ModelSource<'a> { /// The 1-based line and column of normalized character `offset`. pub(crate) fn line_column(&self, offset: usize) -> (u64, u64) { let character = self.original(offset); + if let Some(position) = self.authored_line_column(character) { + return position; + } let line = self.lines.partition_point(|start| *start <= character); let column = character - self.lines[line - 1] + 1; let shifted: u64 = if line == self.first_content_line { diff --git a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs index 2ce4d6291d..733cabe15d 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs @@ -148,7 +148,8 @@ fn evaluate_model_inner(request: ModelEvaluationRequest<'_>) -> Result Date: Sun, 25 Oct 2026 10:17:56 +0000 Subject: [PATCH 26/32] test(microbatch): report state rows and build output when a failure-window check fails The completion-write failure-window test has failed intermittently and\ndoes not reproduce locally (36 of 36 runs pass at 12-way concurrency).\nIts checks after the failed build now report the microbatch state rows\nand that build's output, so the next failure shows which window\ndiverged. --- .../main/build/test_microbatch_failure_windows.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py index 757f96b935..46aa758a33 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py @@ -114,17 +114,25 @@ def test_given_completion_write_failure_during_rebuild_when_retried_then_live_ta "AND partition_end = '2026-01-01T02:00:00'" ), )[0][0] + state_after_failure: list[tuple[object, ...]] = query_duckdb( + db_path=db_path, + sql=( + "SELECT record_type, partition_start, partition_end FROM main._sqlbuild_microbatches " + "ORDER BY partition_start, record_type" + ), + ) recovered: subprocess.CompletedProcess[str] = run_sqb( command=("--no-color", "build"), project_dir=project_dir, working_dir=project_dir, ) + failure_evidence: str = f"state={state_after_failure}\n{failed.stdout}{failed.stderr}" assert initial.returncode == test_case.expected_exit_code, initial.stdout + initial.stderr - assert failed.returncode != test_case.expected_exit_code + assert failed.returncode != test_case.expected_exit_code, failure_evidence assert "simulated completion write failure" in failed.stdout + failed.stderr - assert target_after_failed_state_write == [(1, "a")] - assert completion_count_after_failure == 0 + assert target_after_failed_state_write == [(1, "a")], failure_evidence + assert completion_count_after_failure == 0, failure_evidence assert recovered.returncode == test_case.expected_exit_code, recovered.stdout + recovered.stderr assert ( query_duckdb( From 6d6ee1917f69287aaa7dc4105a6e44b1d31b7aa5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 10:22:32 +0000 Subject: [PATCH 27/32] test: align unit guards with dbt --json rejection and Databricks schema lookup The dbt wrapper case passed --json to sqb dbt run, which now rejects it, so it covers only the stripped --verbose flag and stdout progress. The metadata-SQL ownership guard now allows the Databricks constants module, which holds the information_schema name the no-catalog schema lookup uses. --- .../adapter/relations/test_metadata_sql_ownership.py | 1 + tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py b/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py index 97d0d5d5d4..e00348448c 100644 --- a/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py +++ b/tests/unit/src/sqlbuild/adapter/relations/test_metadata_sql_ownership.py @@ -18,6 +18,7 @@ "adapters/bigquery/_helpers/relation_usage.py", "adapters/bigquery/classes/bigquery_adapter.py", "adapters/databricks/classes/databricks_adapter.py", + "adapters/databricks/constants.py", "adapters/postgres/classes/postgres_adapter.py", "adapters/snowflake/_helpers/metadata_types.py", "adapters/snowflake/_helpers/show_metadata.py", diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py index f3acc3ef7a..caaa9c3b86 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dbt/test_dbt.py @@ -100,11 +100,11 @@ def plan_dbt_interop_from_project( "test_case", [ DbtExecutionWrapperTestCase( - description="dbt run strips local json and verbose flags before execution", + description="dbt run strips the local verbose flag before execution", command_name="run", - args=("--json", "--verbose", "--select", "tag:nightly"), + args=("--verbose", "--select", "tag:nightly"), expected_forwarded_args=("--select", "tag:nightly"), - expected_progress_stream=sys.stderr, + expected_progress_stream=sys.stdout, ), DbtExecutionWrapperTestCase( description="dbt build keeps human output on stdout", From d6906ce44053cada46df86899f27c07a9243b3d6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 11:18:55 +0000 Subject: [PATCH 28/32] fix(scenario): compare replayed VARIANT columns as VARIANT values A Snowflake capture stores a VARIANT as Snowflake's pretty-printed JSON text in its own key order, and local replay transpiles an expected PARSE_JSON to a DuckDB JSON value. DuckDB resolves a VARIANT column set against a JSON one to JSON and compares the text, so equal objects with other key order or whitespace failed strict replay with X506. The expected comparison now casts each column the actual DuckDB relation stores as VARIANT to VARIANT on both sides, where objects compare by key and value; arrays, value types and missing keys still differ. --- .../_helpers/lifecycle/variant_comparison.py | 39 +++++++++ src/sqlbuild/executor/scenario/constants.py | 1 + .../scenario/main/_expected_expectations.py | 20 +++-- .../cli/commands/main/scenario/_test_types.py | 11 +++ .../cli/commands/main/scenario/helpers.py | 87 +++++++++++++++++++ .../main/scenario/test_local_variant.py | 62 +++++++++++++ 6 files changed, 214 insertions(+), 6 deletions(-) create mode 100644 src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py diff --git a/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py b/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py new file mode 100644 index 0000000000..b7b0675b83 --- /dev/null +++ b/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py @@ -0,0 +1,39 @@ +"""Compare DuckDB VARIANT outputs as VARIANT values, not as JSON text.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapter.relations.types import RelationLocation +from sqlbuild.executor.scenario.constants import LOCAL_TYPE_VARIANT, VARIANT_COMPARISON_TEMPLATE + + +def variant_compared_columns( + *, + adapter: BaseAdapter, + connection: Any, + location: RelationLocation, + compared_columns: tuple[str, ...] | None, +) -> tuple[str, ...] | None: + """The compared columns, a DuckDB actual relation's VARIANT columns compared as VARIANT.""" + + if compared_columns is None or not isinstance(adapter, DuckDbBackedAdapter): + return compared_columns + actual: tuple[ColumnInfo, ...] = adapter.get_columns( + connection=connection, + database=location.database, + schema=location.schema, + name=location.name, + ) + variants: frozenset[str] = frozenset( + column.name.lower() + for column in actual + if column.type.strip().upper() == LOCAL_TYPE_VARIANT + ) + return tuple( + VARIANT_COMPARISON_TEMPLATE.format(column=column) if column.lower() in variants else column + for column in compared_columns + ) diff --git a/src/sqlbuild/executor/scenario/constants.py b/src/sqlbuild/executor/scenario/constants.py index e24205edb9..7ac93b4e65 100644 --- a/src/sqlbuild/executor/scenario/constants.py +++ b/src/sqlbuild/executor/scenario/constants.py @@ -166,3 +166,4 @@ LOCAL_CAPTURE_ADAPTER: str = "duckdb" VARIANT_FROM_JSON_PLACEHOLDER: str = "CAST(CAST(? AS JSON) AS VARIANT)" +VARIANT_COMPARISON_TEMPLATE: str = "CAST({column} AS VARIANT) AS {column}" diff --git a/src/sqlbuild/executor/scenario/main/_expected_expectations.py b/src/sqlbuild/executor/scenario/main/_expected_expectations.py index 873065cd4c..9206a76ad0 100644 --- a/src/sqlbuild/executor/scenario/main/_expected_expectations.py +++ b/src/sqlbuild/executor/scenario/main/_expected_expectations.py @@ -9,6 +9,9 @@ from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) +from sqlbuild.executor.scenario._helpers.lifecycle.variant_comparison import ( + variant_compared_columns, +) from sqlbuild.executor.scenario.constants import ( SCENARIO_EXEC_EXPECTED_ERRORED, SCENARIO_EXEC_EXPECTED_FAILED, @@ -30,13 +33,18 @@ def execute_scenario_expected_expectation( actual_relation: str = resolve_relation_location_qualified_name( adapter=adapter, location=expectation.actual_destination ) - comparison_sql: str = build_scenario_expected_comparison_sql( - actual_sql=f"SELECT * FROM {actual_relation}", - expected_sql=expectation.expected_sql, - set_difference_operator=adapter.render_set_difference_operator(), - compared_columns=expectation.compared_columns, - ) try: + comparison_sql: str = build_scenario_expected_comparison_sql( + actual_sql=f"SELECT * FROM {actual_relation}", + expected_sql=expectation.expected_sql, + set_difference_operator=adapter.render_set_difference_operator(), + compared_columns=variant_compared_columns( + adapter=adapter, + connection=connection, + location=expectation.actual_destination, + compared_columns=expectation.compared_columns, + ), + ) cursor: Any = adapter.execute(connection=connection, sql=comparison_sql) row: Any | None = cursor.fetchone() except Exception as exc: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py index 067698c193..5d6b26e112 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py @@ -340,3 +340,14 @@ class SnapshotHelperEditTestCase: edited_to: str expected_exit_code: int expected_stdout_fragment: str + + +@dataclass(frozen=True) +class CapturedVariantTextScenarioTestCase: + """A Snowflake-captured VARIANT stored as JSON text, compared with a PARSE_JSON expected.""" + + description: str + captured_payload_text: str + expected_payload_json: str + expected_returncode: int + expected_stdout_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py index 8497adcbbd..62ebbb4c90 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py @@ -1197,3 +1197,90 @@ def build_helper_fixture_scenario_project_files() -> dict[str, str]: "SELECT 1\n" ), } + + +SNOWFLAKE_VARIANT_FIXTURE_SQL: str = ( + 'SELECT 1 AS order_id, \'a\' AS status, PARSE_JSON(\'{"qty": 2, "note": "x"}\') AS payload' +) + + +def build_snowflake_variant_scenario_project_files(*, expected_payload_json: str) -> dict[str, str]: + """Build an offline Snowflake project whose scenario compares a VARIANT with PARSE_JSON.""" + + return { + "sqlbuild_project.toml": build_offline_snowflake_project_toml(), + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + " schema: main\n table: raw_orders\n columns:\n" + ' - {name: order_id, type: "NUMBER(38,0)"}\n' + " - {name: status, type: VARCHAR}\n" + " - {name: payload, type: VARIANT}\n" + ), + "models/stg_payloads.sql": ( + 'MODEL (description "Order payloads.");\n' + 'SELECT order_id, payload FROM __source("raw_orders")\n' + ), + "tests/scenarios/payloads__variant_compare.sql": ( + "SCENARIO (description 'Payload VARIANT values compare equal locally.');\n" + "WITH\n" + f"__source__raw_orders AS ({SNOWFLAKE_VARIANT_FIXTURE_SQL}),\n" + "__expected__stg_payloads AS (SELECT 1 AS order_id, " + f"PARSE_JSON('{expected_payload_json}') AS payload)\n" + "SELECT 1\n" + ), + } + + +def write_snowflake_variant_snapshot( + *, project_dir: Path, scenario_name: str, payload_text: str +) -> None: + """Write the snapshot a Snowflake capture stores: the VARIANT as Snowflake's JSON text.""" + + snapshot_root: Path = project_dir / "tests" / "_scenario_snapshots" / scenario_name + (snapshot_root / "sources").mkdir(parents=True) + row: str = json.dumps({"ORDER_ID": 1, "PAYLOAD": payload_text, "STATUS": "a"}) + "\n" + (snapshot_root / "sources" / "raw_orders.jsonl").write_text(row, encoding="utf-8") + manifest: dict[str, object] = { + "capture_adapter": "snowflake", + "capture_dialect": "snowflake", + "captured_at": "2026-10-25T11:10:54Z", + "format": "jsonl", + "input_fingerprint": build_scenario_snapshot_input_fingerprint( + scenario_name=scenario_name, + input_specs=( + ScenarioSnapshotInputSpec( + kind=ScenarioArtifactKind.SOURCE, + logical_name="raw_orders", + file_path=Path("sources") / "raw_orders.jsonl", + capture_sql=SNOWFLAKE_VARIANT_FIXTURE_SQL, + ), + ), + capture_adapter="snowflake", + capture_dialect="snowflake", + ), + "relations": [ + { + "bytes": len(row.encode("utf-8")), + "columns": [ + {"local_type": local, "name": name, "warehouse_type": warehouse} + for name, local, warehouse in ( + ("order_id", "DECIMAL(1, 0)", "NUMBER(1,0)"), + ("status", "TEXT(1)", "VARCHAR(1)"), + ("payload", "VARIANT", "VARIANT"), + ) + ], + "file": "sources/raw_orders.jsonl", + "kind": "source", + "logical_name": "raw_orders", + "row_count": 1, + } + ], + "scenario_name": scenario_name, + "sqlbuild_version": "0.177.0", + "total_bytes": len(row.encode("utf-8")), + "total_rows": 1, + "version": 1, + } + (snapshot_root / "scenario.json").write_text( + json.dumps(manifest, indent=2) + "\n", encoding="utf-8" + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py index 802a8cb242..b4b2cb2226 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py @@ -8,10 +8,13 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.scenario._test_types import ( + CapturedVariantTextScenarioTestCase, LocalVariantScenarioTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.scenario.helpers import ( build_local_variant_scenario_project_files, + build_snowflake_variant_scenario_project_files, + write_snowflake_variant_snapshot, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb @@ -64,3 +67,62 @@ def test_given_captured_variant_fixture_when_replaying_locally_then_the_scenario assert replayed.returncode == 0, replayed.stdout + replayed.stderr assert "X699" not in replayed.stdout assert test_case.expected_stdout_fragment in replayed.stdout + + +@pytest.mark.parametrize( + "test_case", + [ + CapturedVariantTextScenarioTestCase( + description="pretty-printed JSON text in another key order equals the expected value", + captured_payload_text='{\n "note": "x",\n "qty": 2\n}', + expected_payload_json='{"qty": 2, "note": "x"}', + expected_returncode=0, + expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", + ), + CapturedVariantTextScenarioTestCase( + description="nested objects in another key order equal the expected value", + captured_payload_text='{\n "a": [\n 1,\n {\n "d": 2,\n "c": 3\n }\n ],\n "b": null\n}', + expected_payload_json='{"b": null, "a": [1, {"c": 3, "d": 2}]}', + expected_returncode=0, + expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", + ), + CapturedVariantTextScenarioTestCase( + description="a different value still fails the comparison", + captured_payload_text='{\n "note": "x",\n "qty": 3\n}', + expected_payload_json='{"qty": 2, "note": "x"}', + expected_returncode=1, + expected_stdout_fragment="mismatched=1 missing=1", + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_variant_text_snapshot_when_replaying_locally_then_json_values_compare_semantically( + test_case: CapturedVariantTextScenarioTestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="snowflake_variant", + repo_files=build_snowflake_variant_scenario_project_files( + expected_payload_json=test_case.expected_payload_json + ), + ) + write_snowflake_variant_snapshot( + project_dir=project_dir, + scenario_name="payloads__variant_compare", + payload_text=test_case.captured_payload_text, + ) + + replayed: subprocess.CompletedProcess[str] = run_sqb( + command=( + "--no-color", + "scenario", + "test", + "--local", + "--strict", + "payloads__variant_compare", + ), + project_dir=project_dir, + ) + + assert replayed.returncode == test_case.expected_returncode, replayed.stdout + replayed.stderr + assert test_case.expected_stdout_fragment in replayed.stdout, replayed.stdout From e1c222837673298958d1201752407824cb4794c8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 11:27:20 +0000 Subject: [PATCH 29/32] fix(scenario): compare VARIANT and JSON outputs by canonical JSON text Comparing replayed VARIANT columns as VARIANT values still depended on how each side's VARIANT was built. Re-encoding DuckDB captures as JSON changed the member types of cast-built values such as {'qty': 2}::VARIANT or 2::VARIANT, so their scenarios failed X506 though the values were equal. The expected comparison now compares each column the actual DuckDB relation stores as VARIANT or JSON through one canonical JSON text on both sides: sorted keys, no whitespace and numbers in their shortest exact form. A Python function on the replay connection computes it. Equal values match however they were built, whether cast-built, JSON-built or captured as Snowflake's pretty-printed text in its own key order; arrays, value types and missing keys still differ. DuckDB captures load their values natively again; another warehouse's JSON text still loads through JSON. --- .../_helpers/lifecycle/canonical_json.py | 34 ++++++++++++++ .../_helpers/lifecycle/variant_comparison.py | 36 ++++++++++++--- .../scenario/_helpers/local/snapshots.py | 13 +++--- src/sqlbuild/executor/scenario/constants.py | 6 ++- .../cli/commands/main/scenario/_test_types.py | 1 + .../cli/commands/main/scenario/helpers.py | 4 +- .../main/scenario/test_local_variant.py | 31 ++++++++++++- .../executor/scenario/_helpers/_test_types.py | 9 ++++ .../scenario/_helpers/test_canonical_json.py | 46 +++++++++++++++++++ 9 files changed, 162 insertions(+), 18 deletions(-) create mode 100644 src/sqlbuild/executor/scenario/_helpers/lifecycle/canonical_json.py create mode 100644 tests/unit/src/sqlbuild/executor/scenario/_helpers/test_canonical_json.py diff --git a/src/sqlbuild/executor/scenario/_helpers/lifecycle/canonical_json.py b/src/sqlbuild/executor/scenario/_helpers/lifecycle/canonical_json.py new file mode 100644 index 0000000000..8ae5db92c7 --- /dev/null +++ b/src/sqlbuild/executor/scenario/_helpers/lifecycle/canonical_json.py @@ -0,0 +1,34 @@ +"""One canonical JSON text per JSON value, so equal values compare equal as text.""" + +from __future__ import annotations + +import json +from decimal import Decimal + + +def canonical_json(text: str | None) -> str | None: + """`text` as JSON with sorted keys, no whitespace and numbers in their shortest exact form.""" + + if text is None: + return None + return _canonical(json.loads(text, parse_float=Decimal, parse_int=Decimal)) + + +def _canonical(value: object) -> str: + if isinstance(value, dict): + members: list[str] = [ + f"{json.dumps(key, ensure_ascii=False)}:{_canonical(value[key])}" + for key in sorted(value) + ] + return "{" + ",".join(members) + "}" + if isinstance(value, list): + return "[" + ",".join(_canonical(item) for item in value) + "]" + if isinstance(value, Decimal): + return _canonical_number(value) + return json.dumps(value, ensure_ascii=False) + + +def _canonical_number(number: Decimal) -> str: + if number.is_finite() and number == number.to_integral_value(): + return str(int(number)) + return format(number.normalize(), "f") diff --git a/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py b/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py index b7b0675b83..992d364d9c 100644 --- a/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py +++ b/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py @@ -1,4 +1,4 @@ -"""Compare DuckDB VARIANT outputs as VARIANT values, not as JSON text.""" +"""Compare DuckDB VARIANT and JSON outputs by their canonical JSON text.""" from __future__ import annotations @@ -8,7 +8,12 @@ from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.relations.types import RelationLocation -from sqlbuild.executor.scenario.constants import LOCAL_TYPE_VARIANT, VARIANT_COMPARISON_TEMPLATE +from sqlbuild.executor.scenario._helpers.lifecycle.canonical_json import canonical_json +from sqlbuild.executor.scenario.constants import ( + CANONICAL_JSON_FUNCTION, + SEMI_STRUCTURED_COMPARISON_TEMPLATE, + SEMI_STRUCTURED_LOCAL_TYPES, +) def variant_compared_columns( @@ -18,7 +23,7 @@ def variant_compared_columns( location: RelationLocation, compared_columns: tuple[str, ...] | None, ) -> tuple[str, ...] | None: - """The compared columns, a DuckDB actual relation's VARIANT columns compared as VARIANT.""" + """The compared columns, a DuckDB actual relation's VARIANT and JSON ones canonicalised.""" if compared_columns is None or not isinstance(adapter, DuckDbBackedAdapter): return compared_columns @@ -28,12 +33,29 @@ def variant_compared_columns( schema=location.schema, name=location.name, ) - variants: frozenset[str] = frozenset( + semi_structured: frozenset[str] = frozenset( column.name.lower() for column in actual - if column.type.strip().upper() == LOCAL_TYPE_VARIANT + if column.type.strip().upper() in SEMI_STRUCTURED_LOCAL_TYPES ) - return tuple( - VARIANT_COMPARISON_TEMPLATE.format(column=column) if column.lower() in variants else column + canonicalised: tuple[str, ...] = tuple( + SEMI_STRUCTURED_COMPARISON_TEMPLATE.format(column=column) + if column.lower() in semi_structured + else column for column in compared_columns ) + if canonicalised != compared_columns: + _register_canonical_json(adapter=adapter, connection=connection) + return canonicalised + + +def _register_canonical_json(*, adapter: DuckDbBackedAdapter, connection: Any) -> None: + registered: object = adapter.execute( + connection=connection, + sql=( + "SELECT COUNT(*) FROM duckdb_functions() " + f"WHERE function_name = '{CANONICAL_JSON_FUNCTION}'" + ), + ).fetchone() + if registered is not None and registered[0] == 0: + connection.create_function(CANONICAL_JSON_FUNCTION, canonical_json, ["VARCHAR"], "VARCHAR") diff --git a/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py b/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py index 0cedb40aa3..ff719a1ee7 100644 --- a/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py +++ b/src/sqlbuild/executor/scenario/_helpers/local/snapshots.py @@ -289,7 +289,8 @@ def _insert_rows( return column_names: tuple[str, ...] = tuple(column.name for column in relation.columns) variant_columns: tuple[bool, ...] = tuple( - column.local_type.strip().upper() == LOCAL_TYPE_VARIANT for column in relation.columns + not native_values and column.local_type.strip().upper() == LOCAL_TYPE_VARIANT + for column in relation.columns ) insert_sql: str = _build_insert_sql( table_name=table_name, column_names=column_names, variant_columns=variant_columns @@ -299,9 +300,7 @@ def _insert_rows( values: list[object] = [] for column_name, variant in zip(column_names, variant_columns, strict=True): value: object = _row_value(row=row, column_name=column_name) - values.append( - _variant_json_text(value=value, native_values=native_values) if variant else value - ) + values.append(_variant_json_text(value=value) if variant else value) row_values.append(tuple(values)) try: connection.executemany(insert_sql, row_values) @@ -356,12 +355,12 @@ def _build_create_table_sql(*, table_name: str, columns: tuple[ScenarioSnapshotC return f"CREATE TABLE {_quote_identifier(table_name)} ({column_defs})" -def _variant_json_text(*, value: object, native_values: bool) -> str | None: - """A VARIANT value as JSON text; another adapter's string that parses is already JSON text.""" +def _variant_json_text(*, value: object) -> str | None: + """Another adapter's VARIANT value as JSON text; a string that parses is already JSON text.""" if value is None: return None - if isinstance(value, str) and not native_values: + if isinstance(value, str): try: json.loads(value) except ValueError: diff --git a/src/sqlbuild/executor/scenario/constants.py b/src/sqlbuild/executor/scenario/constants.py index 7ac93b4e65..91cff7d22d 100644 --- a/src/sqlbuild/executor/scenario/constants.py +++ b/src/sqlbuild/executor/scenario/constants.py @@ -166,4 +166,8 @@ LOCAL_CAPTURE_ADAPTER: str = "duckdb" VARIANT_FROM_JSON_PLACEHOLDER: str = "CAST(CAST(? AS JSON) AS VARIANT)" -VARIANT_COMPARISON_TEMPLATE: str = "CAST({column} AS VARIANT) AS {column}" +CANONICAL_JSON_FUNCTION: str = "sqb_canonical_json" +SEMI_STRUCTURED_COMPARISON_TEMPLATE: str = ( + CANONICAL_JSON_FUNCTION + "(CAST({column} AS JSON)) AS {column}" +) +SEMI_STRUCTURED_LOCAL_TYPES: frozenset[str] = frozenset({"VARIANT", "JSON"}) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py index 5d6b26e112..060d9a9f14 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py @@ -326,6 +326,7 @@ class LocalVariantScenarioTestCase: repo_files: dict[str, str] scenario_name: str expected_stdout_fragment: str + expected_returncode: int = 0 @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py index 62ebbb4c90..c028a31c81 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py @@ -1141,6 +1141,7 @@ def build_local_variant_scenario_project_files( *, output: str = "o.payload IS NOT NULL AS has_payload", expected: str = "TRUE AS has_payload", + fixture: str = '\'{"qty": 2, "tags": ["a"]}\'::JSON::VARIANT', ) -> dict[str, str]: """Build an in-memory DuckDB scenario whose source fixture holds a VARIANT column.""" @@ -1163,8 +1164,7 @@ def build_local_variant_scenario_project_files( "SCENARIO (description 'Test scenario orders__keep_payload.');\n" "WITH\n" "__source__raw_orders AS (\n" - " SELECT 1 AS order_id,\n" - ' \'{"qty": 2, "tags": ["a"]}\'::JSON::VARIANT AS payload\n),\n' + f" SELECT 1 AS order_id,\n {fixture} AS payload\n),\n" f"__expected__stg_orders AS (SELECT 1 AS order_id, {expected})\n" "SELECT 1\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py index b4b2cb2226..e16615e5cc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py @@ -37,6 +37,35 @@ scenario_name="orders__keep_payload", expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", ), + LocalVariantScenarioTestCase( + description="a cast-built VARIANT object compares equal to the replayed fixture", + repo_files=build_local_variant_scenario_project_files( + fixture="{'qty': 2, 'note': 'x'}::VARIANT", + output="o.payload", + expected="{'note': 'x', 'qty': 2}::VARIANT AS payload", + ), + scenario_name="orders__keep_payload", + expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", + ), + LocalVariantScenarioTestCase( + description="a cast-built VARIANT scalar compares equal to the replayed fixture", + repo_files=build_local_variant_scenario_project_files( + fixture="2::VARIANT", output="o.payload", expected="2::VARIANT AS payload" + ), + scenario_name="orders__keep_payload", + expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", + ), + LocalVariantScenarioTestCase( + description="a different cast-built VARIANT value still fails the comparison", + repo_files=build_local_variant_scenario_project_files( + fixture="{'qty': 2}::VARIANT", + output="o.payload", + expected="{'qty': 3}::VARIANT AS payload", + ), + scenario_name="orders__keep_payload", + expected_stdout_fragment="mismatched=1 missing=1", + expected_returncode=1, + ), ], ids=lambda case: case.description, ) @@ -64,7 +93,7 @@ def test_given_captured_variant_fixture_when_replaying_locally_then_the_scenario project_dir=project_dir, ) - assert replayed.returncode == 0, replayed.stdout + replayed.stderr + assert replayed.returncode == test_case.expected_returncode, replayed.stdout + replayed.stderr assert "X699" not in replayed.stdout assert test_case.expected_stdout_fragment in replayed.stdout diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py index 3f0e8bfdc8..bd4f84044e 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py @@ -202,3 +202,12 @@ class ScenarioLocalVariantLoadTestCase: value: object expected_variant_type: str expected_matches_parsed_json: bool + + +@dataclass(frozen=True) +class CanonicalJsonTestCase: + """JSON text and the canonical text equal values of it share.""" + + description: str + text: str | None + expected_canonical: str | None diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_canonical_json.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_canonical_json.py new file mode 100644 index 0000000000..73dae70228 --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_canonical_json.py @@ -0,0 +1,46 @@ +"""Tests for the canonical JSON text scenario comparisons use for VARIANT and JSON values.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.executor.scenario._helpers.lifecycle.canonical_json import canonical_json +from tests.unit.src.sqlbuild.executor.scenario._helpers._test_types import CanonicalJsonTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + CanonicalJsonTestCase( + description="pretty-printed keys in another order", + text='{\n "qty": 2,\n "note": "x"\n}', + expected_canonical='{"note":"x","qty":2}', + ), + CanonicalJsonTestCase( + description="nested objects are sorted and arrays keep their order", + text='{"b": [3, {"z": 1, "a": null}], "a": true}', + expected_canonical='{"a":true,"b":[3,{"a":null,"z":1}]}', + ), + CanonicalJsonTestCase( + description="integral numbers lose trailing zeros", + text='{"a": 2.0, "b": 1E1}', + expected_canonical='{"a":2,"b":10}', + ), + CanonicalJsonTestCase( + description="fractional numbers keep exact digits", + text="[2.50, 0.0000001]", + expected_canonical="[2.5,0.0000001]", + ), + CanonicalJsonTestCase( + description="strings stay strings and keep non-ASCII text", + text='["2", "caf\\u00e9"]', + expected_canonical='["2","café"]', + ), + CanonicalJsonTestCase(description="NULL stays NULL", text=None, expected_canonical=None), + ], + ids=lambda case: case.description, +) +def test_given_json_text_when_canonicalising_then_equal_values_share_one_text( + test_case: CanonicalJsonTestCase, +) -> None: + assert canonical_json(test_case.text) == test_case.expected_canonical From 368742eb0ee718b4697c7bd15c7952f5dea3f142 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 11:39:32 +0000 Subject: [PATCH 30/32] fix(duckdb): resolve ~ attach and database paths from the home directory Anchoring relative DuckDB file paths to the project directory also anchored ~/archive.duckdb, which DuckDB expands from HOME, so it opened /~/archive.duckdb and failed. A path that is ~ or starts with ~/ now resolves from HOME before anchoring, and only other relative paths are joined to the project directory. The DuckDB docs also say that attach entries from a dbt profile resolve from the SQLBuild project directory. --- .../_helpers/command_runtime/connection.rs | 30 ++++++++++++++++--- .../docs/concepts/adapters/duckdb.md | 5 +++- .../cli/commands/main/build/_test_types.py | 2 ++ .../commands/main/build/test_duckdb_attach.py | 18 +++++++++-- .../docs/docs/concepts/adapters/duckdb.mdx | 5 +++- 5 files changed, 51 insertions(+), 9 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 9eeaceadfc..8a824ae6e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -1,5 +1,7 @@ //! Connection config resolution for CLI commands. +use std::path::PathBuf; + use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -32,6 +34,10 @@ const ATTACH_TYPE_KEY: &str = "type"; const LOCAL_ATTACH_TYPES: [&str; 2] = ["duckdb", "sqlite"]; const URL_SCHEME_SEPARATOR: &str = "://"; const MOTHERDUCK_PATH_PREFIX: &str = "md:"; +/// A path naming the home directory, which DuckDB expands from `HOME`. +const HOME_PREFIX: &str = "~"; +const HOME_DIRECTORY_PREFIX: &str = "~/"; +const HOME_VARIABLE: &str = "HOME"; const CONNECTION_OPTIONS: (&[&str], &[&str]) = ( &["raw_config", "project_dir", "adapter_name"], @@ -134,7 +140,7 @@ pub(crate) fn anchor_local_database_path( Ok(()) } -/// `project_dir / value` for a relative local file path, else None. +/// The file a local path names: `~` expanded, then `project_dir / path` when still relative. fn anchored_file_path<'py>( value: &Bound<'py, PyAny>, project_dir: &Bound<'py, PyAny>, @@ -143,14 +149,30 @@ fn anchored_file_path<'py>( if !value.is_instance_of::() { return Ok(None); } - if path(py, value)?.call_method0("is_absolute")?.is_truthy()? - || value.extract::()? == DUCKDB_MEMORY_DATABASE - { + let text: String = value.extract()?; + if text == DUCKDB_MEMORY_DATABASE { + return Ok(None); + } + if let Some(rest) = home_relative_rest(&text) { + return Ok(std::env::var_os(HOME_VARIABLE).map(|home| { + let file: PathBuf = PathBuf::from(home).join(rest); + PyString::new(py, &file.to_string_lossy()).into_any() + })); + } + if path(py, value)?.call_method0("is_absolute")?.is_truthy()? { return Ok(None); } Ok(Some(path(py, project_dir)?.div(value)?.str()?.into_any())) } +/// What follows `~` or `~/`, the home directory DuckDB expands a path from. +fn home_relative_rest(text: &str) -> Option<&str> { + if text == HOME_PREFIX { + return Some(""); + } + text.strip_prefix(HOME_DIRECTORY_PREFIX) +} + /// Anchor each relative DuckDB or SQLite file an `attach` entry names; other paths are DSNs or URLs. fn anchor_attach_paths(config: &Bound<'_, PyDict>, project_dir: &Bound<'_, PyAny>) -> PyResult<()> { let py: Python<'_> = config.py(); diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md index e88fe61a3a..dd8164d1d1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md @@ -52,7 +52,10 @@ alias = "external" read_only = true ``` -A relative `path` of a DuckDB or SQLite file is relative to the project directory, like `database`. +A relative `path` of a DuckDB or SQLite file is relative to the project directory, like `database`, +and a path starting with `~/` is relative to your home directory. This also applies to `attach` +entries that come from a dbt profile: they resolve from the SQLBuild project directory, not from +the dbt project or the working directory. Every connection of a command attaches it once: connections to the same project database share the attachment. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 33a2d69cf0..75cf231937 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1481,3 +1481,5 @@ class DuckDbAttachE2ETestCase: description: str concurrency: int expected_archived_rows: list[tuple[object, ...]] + attach_path: str = "archive.duckdb" + expected_archive_root: str = "project" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_attach.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_attach.py index 061ba86be8..0798368e7e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_attach.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_attach.py @@ -17,7 +17,7 @@ _PROJECT_TOML: str = ( 'name = "shop"\nadapter = "duckdb"\n\n[connection]\ndatabase = "shop.duckdb"\n\n' - '[[connection.attach]]\npath = "archive.duckdb"\nalias = "archive"\n\n' + '[[connection.attach]]\npath = "{attach_path}"\nalias = "archive"\n\n' "[scopes]\nenforce_placement = false\n" ) @@ -35,6 +35,13 @@ concurrency=2, expected_archived_rows=[(1,)], ), + DuckDbAttachE2ETestCase( + description="a home-relative path attaches the file in the home directory", + concurrency=1, + expected_archived_rows=[(1,)], + attach_path="~/archive.duckdb", + expected_archive_root="home", + ), ], ids=lambda case: case.description, ) @@ -45,7 +52,7 @@ def test_given_attached_duckdb_file_when_building_then_models_land_in_it( tmp_path=tmp_path, project_name="shop", repo_files={ - "sqlbuild_project.toml": _PROJECT_TOML, + "sqlbuild_project.toml": _PROJECT_TOML.format(attach_path=test_case.attach_path), "models/archived_orders.sql": ( "MODEL (description 'Test model archived_orders.', materialized table, " "database archive, schema main);\nSELECT 1 AS order_id\n" @@ -60,16 +67,21 @@ def test_given_attached_duckdb_file_when_building_then_models_land_in_it( ), }, ) + home: Path = tmp_path / "home" + home.mkdir() built: subprocess.CompletedProcess[str] = run_sqb( command=("--no-color", "build", "--concurrency", str(test_case.concurrency)), project_dir=project_dir, + env={"HOME": str(home)}, ) + archive_root: Path = {"project": project_dir, "home": home}[test_case.expected_archive_root] assert built.returncode == 0, built.stdout + built.stderr assert ( query_duckdb( - db_path=project_dir / "archive.duckdb", sql="SELECT order_id FROM archived_orders" + db_path=archive_root / "archive.duckdb", sql="SELECT order_id FROM archived_orders" ) == test_case.expected_archived_rows ), test_case.description assert not (REPO_ROOT / "archive.duckdb").exists() + assert not (project_dir / "~").exists() diff --git a/website/src/content/docs/docs/concepts/adapters/duckdb.mdx b/website/src/content/docs/docs/concepts/adapters/duckdb.mdx index 3b3d4d499b..f09a229587 100644 --- a/website/src/content/docs/docs/concepts/adapters/duckdb.mdx +++ b/website/src/content/docs/docs/concepts/adapters/duckdb.mdx @@ -49,7 +49,10 @@ alias = "external" read_only = true ``` -A relative `path` of a DuckDB or SQLite file is relative to the project directory, like `database`. +A relative `path` of a DuckDB or SQLite file is relative to the project directory, like `database`, +and a path starting with `~/` is relative to your home directory. This also applies to `attach` +entries that come from a dbt profile: they resolve from the SQLBuild project directory, not from +the dbt project or the working directory. Every connection of a command attaches it once: connections to the same project database share the attachment. From 52c5cc586cb55b35587e483d9bc49879bf4eda65 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 12:04:14 +0000 Subject: [PATCH 31/32] fix(analysis): reject a bare VALUES CTE body on Snowflake Update Polyglot to a revision that reports WITH v AS (VALUES ...) as a syntax error on Snowflake, which rejects it, instead of accepting the model until it ran. Compile now fails such a model with P001 and a message that says to select from the list; VALUES in FROM stays valid. --- Cargo.lock | 6 ++-- Cargo.toml | 4 +-- .../main/compile/test_dialect_syntax.py | 29 ++++++++++++++++++- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a03b6b41b1..b59a21d102 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -711,7 +711,7 @@ checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "polyglot-sql" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=f76a2e36abb352f40dd43c829b6c336867ef7585#f76a2e36abb352f40dd43c829b6c336867ef7585" +source = "git+https://github.com/chio-labs/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" dependencies = [ "polyglot-sql-ast-derive", "polyglot-sql-function-catalogs", @@ -726,7 +726,7 @@ dependencies = [ [[package]] name = "polyglot-sql-ast-derive" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=f76a2e36abb352f40dd43c829b6c336867ef7585#f76a2e36abb352f40dd43c829b6c336867ef7585" +source = "git+https://github.com/chio-labs/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" dependencies = [ "proc-macro2", "quote", @@ -736,7 +736,7 @@ dependencies = [ [[package]] name = "polyglot-sql-function-catalogs" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=f76a2e36abb352f40dd43c829b6c336867ef7585#f76a2e36abb352f40dd43c829b6c336867ef7585" +source = "git+https://github.com/chio-labs/polyglot.git?rev=8767d688c5b504ad609645e27ab77a260d91c5a1#8767d688c5b504ad609645e27ab77a260d91c5a1" [[package]] name = "portable-atomic" diff --git a/Cargo.toml b/Cargo.toml index 093d94e748..a630a3e6c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,14 +72,14 @@ junction = { version = "2.1.0", default-features = false } mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] } proc-macro2 = { version = "1", features = ["span-locations"] } pyo3 = { version = "0.26", features = ["abi3-py312"] } -polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "f76a2e36abb352f40dd43c829b6c336867ef7585", default-features = false, features = [ +polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [ "ast-tools", "builder", "all-dialects", "semantic", "stacker", ] } -polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "f76a2e36abb352f40dd43c829b6c336867ef7585", default-features = false, features = [ +polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "8767d688c5b504ad609645e27ab77a260d91c5a1", default-features = false, features = [ "dialect-duckdb", ] } blake3 = { version = "1", features = ["rayon"] } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_syntax.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_syntax.py index 3a72dece78..7c55560bcd 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_syntax.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_syntax.py @@ -117,6 +117,32 @@ expected_codes=("B002",), expected_column_counts={}, ), + DialectSyntaxCompileE2ETestCase( + description="Snowflake rejects a CTE whose body is a bare VALUES list", + adapter="snowflake", + models={ + "codes": dialect_syntax_model( + "Codes.", "WITH v AS (VALUES (1), (2)) SELECT v.column1 AS code FROM v" + ), + }, + expected_exit_code=1, + expected_codes=("P001",), + expected_column_counts={}, + ), + DialectSyntaxCompileE2ETestCase( + description="Snowflake accepts VALUES in FROM inside a CTE", + adapter="snowflake", + models={ + "codes": dialect_syntax_model( + "Codes.", + "WITH v AS (SELECT t.code FROM (VALUES (1), (2)) AS t(code)) " + "SELECT v.code FROM v", + ), + }, + expected_exit_code=0, + expected_codes=(), + expected_column_counts={"codes": 1}, + ), DialectSyntaxCompileE2ETestCase( description="a ClickHouse DATE_TRUNC timestamp cursor keeps its timestamp type", adapter="clickhouse", @@ -171,7 +197,8 @@ def test_given_dialect_syntax_when_compiling_then_analysis_parses_and_types_it( report: dict[str, Any] = json.loads(compiled.stdout) column_counts: dict[str, int] = { - model["name"]: model["column_count"] for model in report["resources"]["models"] + model["name"]: model["column_count"] + for model in report.get("resources", {}).get("models", []) } assert compiled.returncode == test_case.expected_exit_code, compiled.stdout + compiled.stderr assert tuple(item["code"] for item in report["diagnostics"]) == test_case.expected_codes From 2544c0ada462391b595ae5f803c7313cc8952343 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 25 Oct 2026 13:53:36 +0000 Subject: [PATCH 32/32] fix(scenario): compare quoted VARIANT output columns by JSON value A quoted output column such as o.payload AS "Payload" reached the comparison with its quotes, so it never matched the actual relation's VARIANT column, skipped the JSON-value comparison and failed X506. The column's unquoted name is now matched. The scenario docs now say that local replay compares VARIANT and JSON columns by JSON value, so values of non-JSON types with the same JSON text compare equal. --- .../sqlbuild/references/docs/concepts/scenarios.md | 4 ++++ .../_helpers/lifecycle/variant_comparison.py | 14 +++++++++++--- src/sqlbuild/executor/scenario/constants.py | 1 + .../commands/main/scenario/test_local_variant.py | 10 ++++++++++ .../src/content/docs/docs/concepts/scenarios.mdx | 4 ++++ 5 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md index 6896a8c95e..091f62fa3c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md @@ -393,5 +393,9 @@ See the [CLI reference](../cli/scenario.md) for full command documentation. ## Limitations - Custom materializations are not supported in scenarios yet. Scenario models using custom materializations will fail with a clear error. +- Local replay compares VARIANT and JSON output columns by their JSON value, not their type: key + order, whitespace and number spelling do not matter, so a Snowflake capture's pretty-printed + JSON equals an expected `PARSE_JSON(...)`. Values of non-JSON types with the same JSON text also + compare equal, for example `DATE '2024-01-01'::VARIANT` and `'2024-01-01'::VARIANT`. - Local replay transpiles SQL from the project adapter dialect to DuckDB. Adapter-specific SQL that cannot be translated will produce a clear error with the failing resource name and reason. - Outside local replay, scenario fixture, expected and assertion SQL runs as authored: SQLBuild replaces `__ref`, `__source`, `__seed` and `__dbt_ref` calls with relation names and may wrap a partial fixture to add its missing columns, but never regenerates the query. Calls inside comments and quoted text are left unchanged. Quoted text and comments follow the project adapter's rules, including with `--local`: backslash escapes on Snowflake, BigQuery and Databricks, `E'...'` strings on DuckDB and Postgres, nested block comments on DuckDB, Postgres, Databricks and SQL Server, `#` line comments on BigQuery, and `//` line comments on Snowflake. A call naming a relation the scenario does not provide, or SQL with an unclosed string or comment, fails planning with `S511` naming the scenario file. diff --git a/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py b/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py index 992d364d9c..4a5172ba28 100644 --- a/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py +++ b/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py @@ -1,4 +1,4 @@ -"""Compare DuckDB VARIANT and JSON outputs by their canonical JSON text.""" +"""Compare DuckDB VARIANT and JSON outputs by their JSON value, so equal JSON text matches.""" from __future__ import annotations @@ -11,6 +11,7 @@ from sqlbuild.executor.scenario._helpers.lifecycle.canonical_json import canonical_json from sqlbuild.executor.scenario.constants import ( CANONICAL_JSON_FUNCTION, + IDENTIFIER_QUOTE, SEMI_STRUCTURED_COMPARISON_TEMPLATE, SEMI_STRUCTURED_LOCAL_TYPES, ) @@ -23,7 +24,7 @@ def variant_compared_columns( location: RelationLocation, compared_columns: tuple[str, ...] | None, ) -> tuple[str, ...] | None: - """The compared columns, a DuckDB actual relation's VARIANT and JSON ones canonicalised.""" + """The compared columns, with VARIANT and JSON ones compared by JSON value, not by type.""" if compared_columns is None or not isinstance(adapter, DuckDbBackedAdapter): return compared_columns @@ -40,7 +41,7 @@ def variant_compared_columns( ) canonicalised: tuple[str, ...] = tuple( SEMI_STRUCTURED_COMPARISON_TEMPLATE.format(column=column) - if column.lower() in semi_structured + if _identifier_name(column).lower() in semi_structured else column for column in compared_columns ) @@ -49,6 +50,13 @@ def variant_compared_columns( return canonicalised +def _identifier_name(column: str) -> str: + inner: str = column.removeprefix(IDENTIFIER_QUOTE).removesuffix(IDENTIFIER_QUOTE) + if len(inner) + len(IDENTIFIER_QUOTE) * 2 != len(column): + return column + return inner.replace(IDENTIFIER_QUOTE * 2, IDENTIFIER_QUOTE) + + def _register_canonical_json(*, adapter: DuckDbBackedAdapter, connection: Any) -> None: registered: object = adapter.execute( connection=connection, diff --git a/src/sqlbuild/executor/scenario/constants.py b/src/sqlbuild/executor/scenario/constants.py index 91cff7d22d..1f87454379 100644 --- a/src/sqlbuild/executor/scenario/constants.py +++ b/src/sqlbuild/executor/scenario/constants.py @@ -171,3 +171,4 @@ CANONICAL_JSON_FUNCTION + "(CAST({column} AS JSON)) AS {column}" ) SEMI_STRUCTURED_LOCAL_TYPES: frozenset[str] = frozenset({"VARIANT", "JSON"}) +IDENTIFIER_QUOTE: str = '"' diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py index e16615e5cc..6f8f81b177 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py @@ -55,6 +55,16 @@ scenario_name="orders__keep_payload", expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", ), + LocalVariantScenarioTestCase( + description="a quoted VARIANT output column compares by its JSON value", + repo_files=build_local_variant_scenario_project_files( + fixture="{'qty': 2, 'note': 'x'}::VARIANT", + output='o.payload AS "Payload"', + expected='\'{"note": "x", "qty": 2}\'::JSON::VARIANT AS "Payload"', + ), + scenario_name="orders__keep_payload", + expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", + ), LocalVariantScenarioTestCase( description="a different cast-built VARIANT value still fails the comparison", repo_files=build_local_variant_scenario_project_files( diff --git a/website/src/content/docs/docs/concepts/scenarios.mdx b/website/src/content/docs/docs/concepts/scenarios.mdx index 0a70f20061..828b5835d8 100644 --- a/website/src/content/docs/docs/concepts/scenarios.mdx +++ b/website/src/content/docs/docs/concepts/scenarios.mdx @@ -381,5 +381,9 @@ See the [CLI reference](/docs/cli/scenario) for full command documentation. ## Limitations - Custom materializations are not supported in scenarios yet. Scenario models using custom materializations will fail with a clear error. +- Local replay compares VARIANT and JSON output columns by their JSON value, not their type: key + order, whitespace and number spelling do not matter, so a Snowflake capture's pretty-printed + JSON equals an expected `PARSE_JSON(...)`. Values of non-JSON types with the same JSON text also + compare equal, for example `DATE '2024-01-01'::VARIANT` and `'2024-01-01'::VARIANT`. - Local replay transpiles SQL from the project adapter dialect to DuckDB. Adapter-specific SQL that cannot be translated will produce a clear error with the failing resource name and reason. - Outside local replay, scenario fixture, expected and assertion SQL runs as authored: SQLBuild replaces `__ref`, `__source`, `__seed` and `__dbt_ref` calls with relation names and may wrap a partial fixture to add its missing columns, but never regenerates the query. Calls inside comments and quoted text are left unchanged. Quoted text and comments follow the project adapter's rules, including with `--local`: backslash escapes on Snowflake, BigQuery and Databricks, `E'...'` strings on DuckDB and Postgres, nested block comments on DuckDB, Postgres, Databricks and SQL Server, `#` line comments on BigQuery, and `//` line comments on Snowflake. A call naming a relation the scenario does not provide, or SQL with an unclosed string or comment, fails planning with `S511` naming the scenario file.