From b45ea2057b04b505b4595936fda9f7d2219fe949 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 26 Oct 2026 06:43:30 +0000 Subject: [PATCH 01/16] build(deps)!: require DuckDB 2.0 Raise the duckdb requirement from 1.3.2 to 2.0.0 and lock 2.0.0. Fresh installs already resolved 2.0.0 because the requirement had no upper bound; this makes 2.0 the tested and supported version. DuckDB 2.0 writes new database files in storage version v2.0.0+, which DuckDB 1.x cannot open. SQLBuild still leaves the warehouse file at DuckDB's default, so the storage tests now expect v2.0.0+, and the DuckDB adapter page explains how to keep a new file readable by 1.x. BREAKING CHANGE: SQLBuild now requires duckdb>=2.0.0. Database files SQLBuild creates use DuckDB 2.0 storage and cannot be opened by DuckDB 1.x; existing files keep their storage version. --- pyproject.toml | 2 +- .../main/build/test_duckdb_storage_key.py | 4 +- .../sqlbuild/adapters/duckdb/test_client.py | 4 +- uv.lock | 38 ++++++------------- .../docs/docs/concepts/adapters/duckdb.mdx | 7 ++++ 5 files changed, 24 insertions(+), 31 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index acc6a4447f..c1857e1c7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ description = "Typed, test-first SQL pipelines with local E2E testing" readme = "README.md" requires-python = ">=3.12,<3.15" dependencies = [ - "duckdb>=1.3.2", + "duckdb>=2.0.0", "filelock>=3.20.3", "orjson>=3.11.7", "polyglot-sql-chio==0.13.9", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_storage_key.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_storage_key.py index 44ff171421..7af693c641 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_storage_key.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_storage_key.py @@ -23,12 +23,12 @@ ConnectionStorageKeyBuildE2ETestCase( description="a storage version key is ignored like any unknown connection key", connection_key="storage_compatibility_version", - expected_storage_version="v1.0.0+", + expected_storage_version="v2.0.0+", ), ConnectionStorageKeyBuildE2ETestCase( description="an unknown connection key is ignored", connection_key="unknown_option", - expected_storage_version="v1.0.0+", + expected_storage_version="v2.0.0+", ), ], ids=lambda case: case.description, diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py index 4d7a50849e..d4fe064842 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py @@ -196,12 +196,12 @@ def test_given_existing_database_when_opening_for_local_replay_then_its_tables_r UserStorageVersionKeyTestCase( description="a storage version in user config is ignored like any unknown key", config_key="storage_compatibility_version", - expected_storage_version="v1.0.0+", + expected_storage_version="v2.0.0+", ), UserStorageVersionKeyTestCase( description="an unknown connection key is ignored", config_key="unknown_option", - expected_storage_version="v1.0.0+", + expected_storage_version="v2.0.0+", ), ], ids=lambda case: case.description, diff --git a/uv.lock b/uv.lock index 9ec9bf35c6..3ae0630e48 100644 --- a/uv.lock +++ b/uv.lock @@ -1301,31 +1301,17 @@ wheels = [ [[package]] name = "duckdb" -version = "1.5.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7d/19/e57151753576373c6696a12022648546cca6038e8833fda2908ee2342d9b/duckdb-1.5.5.tar.gz", hash = "sha256:72f33ee57ca7595b23957671a2cc7f7fe2be0ecc2d68f63abedcfcaa3a5c1238", size = 18066741, upload-time = "2026-07-22T10:55:17.819Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d6/40/2e05d324400fdaa5656c9f48d6298da421cb034d85e509fa0e6e325cf04b/duckdb-1.5.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d4dd65f8941a604b947e0b9b4b4f7165988e29a23ec0b69b4038520956d9933e", size = 32753858, upload-time = "2026-07-22T10:54:05.514Z" }, - { url = "https://files.pythonhosted.org/packages/79/15/5ceb58ffb5bb8a62b3fd7abb39c41467cdf94850ece02e6d88664dfc75ce/duckdb-1.5.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:33db46679b071f108d57139493dee2d37e1f5efcf5c5c039c2969eed11a6c8a7", size = 17368293, upload-time = "2026-07-22T10:54:09.139Z" }, - { url = "https://files.pythonhosted.org/packages/bf/5c/bf02da0b354fe83cca4f95a4fbf762181af466f7d551ab2a093f7698882a/duckdb-1.5.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f0b88535a5d86fdd63dba6ea02ab68c003dfb9e4892b11256ef24c4da208baae", size = 15509131, upload-time = "2026-07-22T10:54:12.228Z" }, - { url = "https://files.pythonhosted.org/packages/ea/a9/5f1f09da421d8e930e0b063d11c1b3f90363f40ede74438cd188afdd13a2/duckdb-1.5.5-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f316eae2323d9a851883fdf2dee91c1f9efe251ab33e14a2272f82a913422ed6", size = 19391959, upload-time = "2026-07-22T10:54:15.551Z" }, - { url = "https://files.pythonhosted.org/packages/4f/98/6549769f158126fa64fd6c1ac2eb59a18282146c939867a3eb31b7c1db07/duckdb-1.5.5-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7a6d2d11859d82a936ebdcb30ce3d8a1cbb3e990bff05c12abb9b54c44fa7bd1", size = 21510909, upload-time = "2026-07-22T10:54:19.681Z" }, - { url = "https://files.pythonhosted.org/packages/af/b7/5753b41d3124838f868f9f523362812d9fc45409e9e4dd70dcbb0a25826e/duckdb-1.5.5-cp312-cp312-win_amd64.whl", hash = "sha256:ddfbdb096c11d51ee22492397d342c90a82e62c5d09961477895934d0a25372f", size = 13168544, upload-time = "2026-07-22T10:54:22.789Z" }, - { url = "https://files.pythonhosted.org/packages/5c/28/44b679c7d46245f8398feae7edac959d1b83d4eb143e25b3fce0630b78bd/duckdb-1.5.5-cp312-cp312-win_arm64.whl", hash = "sha256:2725d2b9ace3a4e75d72fc5a239f6a44b502c580edadb8fb2676db772c5f9282", size = 13988684, upload-time = "2026-07-22T10:54:26.003Z" }, - { url = "https://files.pythonhosted.org/packages/47/37/4a38116e7700720fd152c666292214fd3abdf916496991296d8d1f66efbf/duckdb-1.5.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:cd98829b67788609017e65c761bd42a5dd0f9129441bed8bda4d6881ccf819f0", size = 32754294, upload-time = "2026-07-22T10:54:29.822Z" }, - { url = "https://files.pythonhosted.org/packages/66/42/7d392f1ba1eee0eaf4ab4c8c7a604bfe3536cd63f979cf5c98798664f807/duckdb-1.5.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:feead93c56679b79592d437c62975d39cb67adedffa7592c763baf8160ac7366", size = 17368211, upload-time = "2026-07-22T10:54:33.359Z" }, - { url = "https://files.pythonhosted.org/packages/9f/a5/0a6f4fa60562faa615e55e15bd1953a2f2b17a8edd8105e5cda215e43457/duckdb-1.5.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:49c963d9469373d7aba8d750d9ea565ab823e94166efed953f184dd9b169b98c", size = 15509136, upload-time = "2026-07-22T10:54:36.369Z" }, - { url = "https://files.pythonhosted.org/packages/e4/cb/023c89f51978545b9fab318581bba0c457a58e7530d2d933e54ae7d8647c/duckdb-1.5.5-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a736217825461732b5442d05a220f3da2e23a0dae114efbf08c9bf171b53098a", size = 19392147, upload-time = "2026-07-22T10:54:39.551Z" }, - { url = "https://files.pythonhosted.org/packages/3e/c5/41bef391fb8b23dbc133c9f2ba016e7a7a8124513d2cc1b430f1897d87e4/duckdb-1.5.5-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:078e6a60dd8eedde5832f45422ca5c4a6b8c837aeabd8a56ca0b7d933f588053", size = 21511060, upload-time = "2026-07-22T10:54:42.788Z" }, - { url = "https://files.pythonhosted.org/packages/07/9f/c44dfc1f924ac29b3252dc1b91393c01d009dbfe9f8ed33f10b986151bd1/duckdb-1.5.5-cp313-cp313-win_amd64.whl", hash = "sha256:6826504277dba513c0c5d71d828456c94d729c9d2482f94b2e289f90a9167e28", size = 13168028, upload-time = "2026-07-22T10:54:46.127Z" }, - { url = "https://files.pythonhosted.org/packages/ca/88/591384b2cd59abddd6f5dc175e60374f9abae6064429f0c4402854c10f44/duckdb-1.5.5-cp313-cp313-win_arm64.whl", hash = "sha256:baa9c5702002fabb559ded2a39008f9f421fcbc7237d388b8213eff1e08858de", size = 13989955, upload-time = "2026-07-22T10:54:49.262Z" }, - { url = "https://files.pythonhosted.org/packages/3e/56/12c65bfa2d2605b81981b264788891bcf11ec72227889554cead5d8d13b9/duckdb-1.5.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8e6413dd40facb7b8ab21bd844450cd8f549b29e138635be9cf090ef4d2049e2", size = 32761946, upload-time = "2026-07-22T10:54:53.412Z" }, - { url = "https://files.pythonhosted.org/packages/b9/46/682ce155f17e0d2822d4f13ee3db9ca4b5b7c2da61b841b2629035e1f4bc/duckdb-1.5.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:64078acfd16541132ac6e191eb81b2845554444a0305cc1aa581ba107e514aa8", size = 17375069, upload-time = "2026-07-22T10:54:57.269Z" }, - { url = "https://files.pythonhosted.org/packages/39/ce/a24bcbd3289c8f305a430759c5fc12242740b4af3e17f7593f3a34e333d2/duckdb-1.5.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8c11775cc99a447618d5f1840126db17f2652f3eae05529df4f81f40e2df7151", size = 15519791, upload-time = "2026-07-22T10:55:00.681Z" }, - { url = "https://files.pythonhosted.org/packages/d9/76/3a01afbc615c1d418c0de58a6b68ac5ce2a8563232c0464bfbc2ce552398/duckdb-1.5.5-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77bbc1e6ba12e1e06f9020117bdf848627ecfdf36f907550e62e008e6109dece", size = 19398251, upload-time = "2026-07-22T10:55:04.168Z" }, - { url = "https://files.pythonhosted.org/packages/a1/43/3a5e81d1728f4d234c79bfe385808ee7c04834f7c37a4b5c257459c25614/duckdb-1.5.5-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fbf0f2d48b43c6c304d00463b463c27ead6c4b01c3c1816b750f728decf71afe", size = 21513851, upload-time = "2026-07-22T10:55:07.864Z" }, - { url = "https://files.pythonhosted.org/packages/91/41/fc7c829172c60ca22485251eab285f4f1a0d87b486a024c726f21471d86e/duckdb-1.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:9dc826c4b50e64f6c4e4d07a3a9cb075ef70ba3899dc43ec5493dc3d7b04b353", size = 13691858, upload-time = "2026-07-22T10:55:11.181Z" }, - { url = "https://files.pythonhosted.org/packages/e1/2c/95d9216b79e9273689d7ebce125a54503ed0c9bd7da931f0265888e99779/duckdb-1.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:63e48d4b74b15aeacd688976432a7225163df8c226eddeb8536bba2d4d4ff433", size = 14470180, upload-time = "2026-07-22T10:55:14.445Z" }, +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c2/7c/dfdf5e9cebefd032f57f09b224993e861ecb55f61e36c709544b04b44f70/duckdb-2.0.0.tar.gz", hash = "sha256:09de511ef0dc76a5213e8d3d87a8b89bb4d3e8a6fe73a9f60dc001269946b44f", size = 16771077, upload-time = "2026-10-21T12:51:13.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/28/1fb52f9ea10205a55a07c32e37cd4fd68fa9d1dc2520a12405ec9bb59cac/duckdb-2.0.0-cp312-abi3-macosx_10_15_universal2.whl", hash = "sha256:cb1cc30318397c3bcffa6b683196293698052588de3c1ba1e144ba0c72560ce9", size = 37003294, upload-time = "2026-10-21T12:50:56.489Z" }, + { url = "https://files.pythonhosted.org/packages/07/5b/fbe92644709fb9162cc51753f92f892b1f333658079d1f11efaed43341d2/duckdb-2.0.0-cp312-abi3-macosx_10_15_x86_64.whl", hash = "sha256:46c7a299ff5f45be9068cfc0ceecb3fd574af9535de8cb6908ae8b52d2846cbd", size = 19689207, upload-time = "2026-10-21T12:50:59.331Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a3/4d23938223626980cdcf18fd6b0dbf040823b5cf15beabbf4ab7ee276631/duckdb-2.0.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:b5d758b53751c930e09d371230241005ea5203c8edc65a1e1fafca83fcc906ef", size = 17597612, upload-time = "2026-10-21T12:51:01.818Z" }, + { url = "https://files.pythonhosted.org/packages/92/4a/985fb9d93d7f9f8e82092ddfe05ec8c9a1d0df7bdb0f562a10ced30e3e8f/duckdb-2.0.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e38daf0acbad9de7ce892d402b68409519291ee16af3c2cba85ace3b085c7eaa", size = 21655920, upload-time = "2026-10-21T12:51:04.36Z" }, + { url = "https://files.pythonhosted.org/packages/cc/33/01ad2535ee897db2bad70cbc800655167c53afd0b955eed84f0759c2f417/duckdb-2.0.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ab51ec57d329d6f3df37c0287b55d71c3fc07bb54b61d1afed09104f953b1431", size = 24371005, upload-time = "2026-10-21T12:51:06.86Z" }, + { url = "https://files.pythonhosted.org/packages/09/d3/9e8ca1242b98f04041bef09686ecfe098b41f820aa23240f63f360b64eaf/duckdb-2.0.0-cp312-abi3-win_amd64.whl", hash = "sha256:f61c96547ed432f54dae42a717fd88563eda55cb47549715fac9f61f770c205a", size = 15368860, upload-time = "2026-10-21T12:51:09.395Z" }, + { url = "https://files.pythonhosted.org/packages/37/81/6ea5dc1050f86ec4dd038a2774766b42247ba6104e7785f13a0c600e2b33/duckdb-2.0.0-cp312-abi3-win_arm64.whl", hash = "sha256:c5e6708efaa8647e0914a16f1914354419e529a1925929950e7d610418e45873", size = 15726489, upload-time = "2026-10-21T12:51:11.643Z" }, ] [[package]] @@ -4331,7 +4317,7 @@ requires-dist = [ { name = "databricks-sql-connector", extras = ["pyarrow"], marker = "extra == 'databricks'", specifier = ">=4.1.2" }, { name = "dbc", marker = "extra == 'ingestr'", specifier = ">=0.3.1" }, { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], marker = "extra == 'dlt'", specifier = ">=1.28.0,<2" }, - { name = "duckdb", specifier = ">=1.3.2" }, + { name = "duckdb", specifier = ">=2.0.0" }, { name = "filelock", specifier = ">=3.20.3" }, { name = "google-cloud-bigquery", marker = "extra == 'bigquery'", specifier = ">=3.37.0" }, { name = "ingestr", marker = "extra == 'ingestr'", specifier = ">=1.1.73" }, diff --git a/website/src/content/docs/docs/concepts/adapters/duckdb.mdx b/website/src/content/docs/docs/concepts/adapters/duckdb.mdx index f09a229587..e525647c89 100644 --- a/website/src/content/docs/docs/concepts/adapters/duckdb.mdx +++ b/website/src/content/docs/docs/concepts/adapters/duckdb.mdx @@ -5,6 +5,13 @@ description: "DuckDB adapter configuration for SQLBuild." DuckDB is included as a core dependency. No extra installation needed. +SQLBuild requires DuckDB 2.0 or later. DuckDB 2.0 writes new database files in a storage format +that DuckDB 1.x cannot open, so a database file SQLBuild creates can only be read by DuckDB 2.0 +clients. A file that already exists keeps its storage version, so a file created by DuckDB 1.x +stays readable by 1.x. To create a new file that 1.x can read, create it before the first build +with an explicit storage version, for example +`duckdb my_project.duckdb -storage-version v1.5.0 -c "SELECT 1"`. + ## Connection config ```toml From 0cf6bc33b75c3e00e4bffbeaf131da4de2fa7dcd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 26 Oct 2026 06:43:43 +0000 Subject: [PATCH 02/16] fix(duckdb): keep table and view switches working on DuckDB 2.0 DuckDB 2.0 lets ALTER TABLE ... RENAME rename a view, where 1.x refused. Promoting a table over a model that was a view swapped the view into the staging name, then failed to drop it as a table, so a model changed from view to table failed in promotion. The DuckDB swap now refuses a view on its left before renaming anything, so promotion takes its existing path for a view destination: drop the view and rename the staged table into place. --- .../contract/classes/duckdb_backed_adapter.py | 19 +++++++ .../sqlbuild/adapters/duckdb/_test_types.py | 10 ++++ .../sqlbuild/adapters/duckdb/test_client.py | 49 +++++++++++++++++++ 3 files changed, 78 insertions(+) diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index b4f0982e88..9d5cbd8919 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1645,6 +1645,9 @@ def swap( right: str, statement_recorder: StatementRecorder, ) -> None: + # DuckDB 2 lets ALTER TABLE rename a view, which would swap a view into the staging name. + if self._is_view(connection=connection, relation=left): + raise AdapterUserError(message=f"DuckDB cannot swap the view {left} with a table") statements: tuple[str, ...] = self.render_swap(left=left, right=right) statement_recorder.record_many(statements) with self.transaction(connection): @@ -1652,6 +1655,22 @@ def swap( for stmt in statements: self.execute(connection=connection, sql=stmt) + def _is_view(self, *, connection: Any, relation: str) -> bool: + parts: list[str] = [_unquote_duckdb_identifier(part) for part in relation.split(".")] + name: str = parts[-1] + schema_sql: str = ( + _duckdb_string_literal(parts[-2]) if len(parts) >= 2 else "current_schema()" + ) + database_sql: str = ( + _duckdb_string_literal(parts[-3]) if len(parts) >= 3 else "current_database()" + ) + query: str = ( + "SELECT 1 FROM duckdb_views() WHERE NOT internal " + f"AND view_name = {_duckdb_string_literal(name)} " + f"AND schema_name = {schema_sql} AND database_name = {database_sql}" + ) + return self.execute(connection=connection, sql=query).fetchone() is not None + def render_clone( self, *, diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py index c9a9638a43..97cbb8fa36 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py @@ -129,6 +129,16 @@ class SwapTestCase: expected_right_value: str +@dataclass(frozen=True) +class ViewSwapTestCase: + """A swap whose left relation is a view, which DuckDB must not rename as a table.""" + + description: str + setup_sql: tuple[str, ...] + left: str + expected_relations: list[tuple[str, str]] + + @dataclass(frozen=True) class DeleteInsertTestCase: description: str diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py index d4fe064842..9cd4168929 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py @@ -9,6 +9,7 @@ from sqlbuild._native import InferredNullability, SeedCsvSettings from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import ( ColumnInfo, ExpressionInferenceProfile, @@ -48,6 +49,7 @@ SwapTestCase, TransactionalAtomicityTestCase, UserStorageVersionKeyTestCase, + ViewSwapTestCase, ) from tests.integration.src.sqlbuild.adapters.duckdb.helpers import count_relation_rows @@ -781,6 +783,53 @@ def test_given_two_tables_when_swapping_then_contents_are_exchanged( assert right_val[0] == test_case.expected_right_value +@pytest.mark.parametrize( + "test_case", + [ + ViewSwapTestCase( + description="an unqualified view is refused and left in place", + setup_sql=( + "CREATE VIEW orders AS SELECT 1 AS order_id", + "CREATE TABLE orders__staging AS SELECT 2 AS order_id", + ), + left="orders", + expected_relations=[("orders", "VIEW"), ("orders__staging", "BASE TABLE")], + ), + ViewSwapTestCase( + description="a schema-qualified view is refused and left in place", + setup_sql=( + "CREATE VIEW main.orders AS SELECT 1 AS order_id", + "CREATE TABLE main.orders__staging AS SELECT 2 AS order_id", + ), + left="main.orders", + expected_relations=[("orders", "VIEW"), ("orders__staging", "BASE TABLE")], + ), + ], + ids=lambda case: case.description, +) +def test_given_view_on_left_when_swapping_then_swap_is_refused_without_renaming( + test_case: ViewSwapTestCase, + adapter: DuckDbAdapter, + connection: Any, +) -> None: + statement: str + for statement in test_case.setup_sql: + connection.execute(statement) + + with pytest.raises(AdapterUserError, match="cannot swap the view"): + adapter.swap( + connection=connection, + left=test_case.left, + right="main.orders__staging", + statement_recorder=StatementRecorder(), + ) + + relations: list[tuple[str, str]] = connection.execute( + "SELECT table_name, table_type FROM information_schema.tables ORDER BY table_name" + ).fetchall() + assert relations == test_case.expected_relations + + @pytest.mark.parametrize( "test_case", [ From 7cb8a5f53fe648d5a3d8b295bc73d4f012b321b5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 26 Oct 2026 06:59:10 +0000 Subject: [PATCH 03/16] fix(duckdb): register Python functions after a partly read result DuckDB 2.0 refuses connection calls other than execute() while a result from execute() is not read to its end, and a fetchone() never reads to the end. Registering the VARIANT comparison function after a fetchone() therefore failed every local scenario that compares VARIANT or JSON output, and a Python UDF registered after such a read would fail the same way. Both registrations now go through one DuckDB adapter helper that finishes the open result before calling create_function. --- .../contract/classes/duckdb_backed_adapter.py | 23 ++++++++++- .../_helpers/lifecycle/variant_comparison.py | 8 +++- .../sqlbuild/adapters/duckdb/_test_types.py | 10 +++++ .../sqlbuild/adapters/duckdb/test_client.py | 38 +++++++++++++++++++ 4 files changed, 77 insertions(+), 2 deletions(-) diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 9d5cbd8919..69f5cc89b6 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1478,7 +1478,28 @@ def create_function( f"RETURNS {returns}" ) statement_recorder.record(registration_sql) - connection.create_function(function_name, callable_function, parameter_types, returns) + self.register_python_function( + connection=connection, + name=function_name, + function=callable_function, + parameter_types=parameter_types, + returns=returns, + ) + + def register_python_function( + self, + *, + connection: Any, + name: str, + function: Any, + parameter_types: list[str], + returns: str, + ) -> None: + """Register a Python function on the connection as a connection-scoped DuckDB function.""" + + # DuckDB 2 refuses connection calls other than execute() while a result is partly read. + connection.complete() + connection.create_function(name, function, parameter_types, returns) def _load_python_udf_callable(self, *, source_file_path: Path, entry_point: str) -> Any: module_name: str = "sqlbuild_python_udf_" + "_".join( diff --git a/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py b/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py index 4a5172ba28..5e4dab04a3 100644 --- a/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py +++ b/src/sqlbuild/executor/scenario/_helpers/lifecycle/variant_comparison.py @@ -66,4 +66,10 @@ def _register_canonical_json(*, adapter: DuckDbBackedAdapter, connection: Any) - ), ).fetchone() if registered is not None and registered[0] == 0: - connection.create_function(CANONICAL_JSON_FUNCTION, canonical_json, ["VARCHAR"], "VARCHAR") + adapter.register_python_function( + connection=connection, + name=CANONICAL_JSON_FUNCTION, + function=canonical_json, + parameter_types=["VARCHAR"], + returns="VARCHAR", + ) diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py index 97cbb8fa36..d6f9449719 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py @@ -269,3 +269,13 @@ class UserStorageVersionKeyTestCase: description: str config_key: str expected_storage_version: str + + +@dataclass(frozen=True) +class PythonFunctionRegistrationTestCase: + """A Python function registered while an earlier result is only partly read.""" + + description: str + open_result_sql: str + call_sql: str + expected_rows: list[tuple[object, ...]] diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py index 9cd4168929..588a72fb1e 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py @@ -41,6 +41,7 @@ MaterializeTestCase, MergeTestCase, PhysicalRelationGenerationTestCase, + PythonFunctionRegistrationTestCase, QueryTestCase, RecorderWriteTestCase, RelationExistsTestCase, @@ -1654,3 +1655,40 @@ def test_given_missing_right_table_when_swapping_then_left_table_preserved( connection.execute(test_case.verify_sql).fetchall() ) assert actual == test_case.expected_rows_after_failure + + +@pytest.mark.parametrize( + "test_case", + [ + PythonFunctionRegistrationTestCase( + description="a one-row aggregate result with its row read", + open_result_sql="SELECT COUNT(*) FROM range(3)", + call_sql="SELECT double_amount(21)", + expected_rows=[(42,)], + ), + PythonFunctionRegistrationTestCase( + description="a many-row result with one row read", + open_result_sql="SELECT * FROM range(100)", + call_sql="SELECT double_amount(5)", + expected_rows=[(10,)], + ), + ], + ids=lambda case: case.description, +) +def test_given_partly_read_result_when_registering_python_function_then_function_is_callable( + test_case: PythonFunctionRegistrationTestCase, + adapter: DuckDbAdapter, + connection: Any, +) -> None: + cursor: Any = connection.execute(test_case.open_result_sql) + _ = cursor.fetchmany(1) + + adapter.register_python_function( + connection=connection, + name="double_amount", + function=lambda amount: amount * 2, + parameter_types=["BIGINT"], + returns="BIGINT", + ) + + assert connection.execute(test_case.call_sql).fetchall() == test_case.expected_rows From cb4074f361673561f95ef39dc072aa48fa194735 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 26 Oct 2026 06:59:11 +0000 Subject: [PATCH 04/16] fix(scenario): cancel the running statement when local replay is interrupted Local replay relied on DuckDB stopping a statement when Ctrl-C raised KeyboardInterrupt in the main thread. DuckDB 2.0 no longer does, so an interrupted local replay kept running its statement until it finished. Local replay now registers its run-scoped connection with the running scenarios, like target runs, so Ctrl-C interrupts the statement. --- src/sqlbuild/executor/pipeline/_helpers/scenario.py | 7 +++++++ src/sqlbuild/executor/scenario/_helpers/local/execution.py | 6 ++++++ src/sqlbuild/executor/scenario/main/_local.py | 3 +++ 3 files changed, 16 insertions(+) diff --git a/src/sqlbuild/executor/pipeline/_helpers/scenario.py b/src/sqlbuild/executor/pipeline/_helpers/scenario.py index decc58ee66..fdc81cfee4 100644 --- a/src/sqlbuild/executor/pipeline/_helpers/scenario.py +++ b/src/sqlbuild/executor/pipeline/_helpers/scenario.py @@ -293,6 +293,7 @@ def run_scenario_local_test_pipeline( promotion_mode: TablePromotionMode = resolve_promotion_mode( settings=pipeline_result.project.settings, adapter=adapter ) + running: RunningScenarios = RunningScenarios(adapter=adapter) def execute(scenario_plan: ScenarioExecutionPlan) -> ScenarioRunResult: return execute_local_scenario_load_only_run( @@ -303,6 +304,7 @@ def execute(scenario_plan: ScenarioExecutionPlan) -> ScenarioRunResult: promotion_mode=promotion_mode, capture_adapter=replay_source.capture_adapter, capture_dialect=replay_source.capture_dialect, + running=running, ) def failed(*, scenario_name: str, exc: Exception) -> ScenarioRunResult: @@ -326,6 +328,11 @@ def failed(*, scenario_name: str, exc: Exception) -> ScenarioRunResult: failed=failed, on_scenario_start=on_scenario_start, on_scenario_complete=on_scenario_complete, + scheduling=_ScenarioScheduling( + interrupts=ScenarioInterrupts( + cancel_statements=partial(running.interrupt, close_uncancellable=False) + ) + ), ) diff --git a/src/sqlbuild/executor/scenario/_helpers/local/execution.py b/src/sqlbuild/executor/scenario/_helpers/local/execution.py index a29675dd13..fc112e07ba 100644 --- a/src/sqlbuild/executor/scenario/_helpers/local/execution.py +++ b/src/sqlbuild/executor/scenario/_helpers/local/execution.py @@ -47,6 +47,7 @@ build_scenario_snapshot_input_specs, classify_scenario_snapshot_state, ) +from sqlbuild.executor.scenario.classes.running_scenarios import RunningScenarios from sqlbuild.executor.scenario.constants import ( SCENARIO_EXEC_ASSERTION_ERRORED, SCENARIO_EXEC_ASSERTION_FAILED, @@ -86,6 +87,7 @@ def execute_local_scenario_load_only_run( promotion_mode: TablePromotionMode, capture_adapter: str | None = None, capture_dialect: str | None = None, + running: RunningScenarios | None = None, ) -> ScenarioRunResult: """Run one scenario locally against a run-scoped DuckDB database.""" @@ -139,6 +141,8 @@ def execute_local_scenario_load_only_run( if isinstance(adapter, DuckDbBackedAdapter) else adapter.connect({"database": str(duckdb_path)}) ) + if running is not None: + running.start(scenario_plan=scenario_plan, connection=connection) try: input_fingerprint: str = build_scenario_snapshot_input_fingerprint( scenario_name=scenario_plan.name, @@ -178,6 +182,8 @@ def execute_local_scenario_load_only_run( error_message=error_message(exc), ) finally: + if running is not None: + running.finish(connection=connection) adapter.close(connection) return result diff --git a/src/sqlbuild/executor/scenario/main/_local.py b/src/sqlbuild/executor/scenario/main/_local.py index 388ce36cbf..6a90e738e9 100644 --- a/src/sqlbuild/executor/scenario/main/_local.py +++ b/src/sqlbuild/executor/scenario/main/_local.py @@ -10,6 +10,7 @@ from sqlbuild.executor.scenario._helpers.local.execution import ( execute_local_scenario_load_only_run as _execute_local_scenario_load_only_run, ) +from sqlbuild.executor.scenario.classes.running_scenarios import RunningScenarios from sqlbuild.executor.scenario.models import ScenarioRunResult @@ -22,6 +23,7 @@ def execute_local_scenario_load_only_run( promotion_mode: TablePromotionMode, capture_adapter: str | None = None, capture_dialect: str | None = None, + running: RunningScenarios | None = None, ) -> ScenarioRunResult: """Run one scenario locally against a run-scoped DuckDB database.""" @@ -33,4 +35,5 @@ def execute_local_scenario_load_only_run( promotion_mode=promotion_mode, capture_adapter=capture_adapter, capture_dialect=capture_dialect, + running=running, ) From ee40b8dcfd52d3093350c147e78b481bbd8a6174 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 26 Oct 2026 06:59:11 +0000 Subject: [PATCH 05/16] test(scenario): write fixture lambdas in the syntax DuckDB 2.0 accepts DuckDB 2.0 disables the single-arrow lambda syntax by default, so the scenario fixture that checks lambda parameters are not mock columns now uses lambda x: syntax. --- .../e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py index c028a31c81..89ecb8afe3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py @@ -1017,11 +1017,11 @@ def build_lambda_scenario_project_files() -> dict[str, str]: 'WITH sess AS (SELECT * FROM __ref("sessions")),\n' 'crt AS (SELECT c.session_id, c.quantities FROM __ref("carts") AS c)\n' "SELECT\n s.session_id,\n" - " list_reduce(list_transform(c.quantities, q -> q * 2), (acc, x) -> acc + x)" + " list_reduce(list_transform(c.quantities, lambda q: q * 2), lambda acc, x: acc + x)" " AS total,\n" - " len(list_filter(c.quantities, i -> i > 1)) AS big\n" + " len(list_filter(c.quantities, lambda i: i > 1)) AS big\n" "FROM sess AS s\nINNER JOIN crt AS c ON s.session_id = c.session_id\n" - "WHERE len(list_filter(c.quantities, w -> w < 0)) = 0\n" + "WHERE len(list_filter(c.quantities, lambda w: w < 0)) = 0\n" ), "tests/scenarios/cart_totals_doubled.sql": ( "SCENARIO (description 'Test scenario cart_totals_doubled.');\n" From 4912676d7e33e83b34dae8943d7793f930e1e73d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 26 Oct 2026 07:10:05 +0000 Subject: [PATCH 06/16] test: match DuckDB 2.0 view text and relation API in E2E helpers DuckDB 2.0 stores a view's join in parentheses, so the clone test expects that text. Relations no longer have fetchone(), so the CTE-name helper reads json_serialize_sql through execute(). --- .../cli/commands/main/clone/test_clone_preparation.py | 4 ++-- tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone_preparation.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone_preparation.py index a64f7f7583..2c2906edf4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone_preparation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone_preparation.py @@ -33,8 +33,8 @@ "dev", "orders_summary", "CREATE VIEW dev.orders_summary AS SELECT f.*, p.amount_cents AS paid_cents " - "FROM dev.fct_orders AS f LEFT JOIN dev.stg_payments AS p " - "ON ((p.order_id = f.order_id));", + "FROM (dev.fct_orders AS f LEFT JOIN dev.stg_payments AS p " + "ON ((p.order_id = f.order_id)));", ), ( "dev", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index 5199d8dbec..7cd9ded090 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -1711,8 +1711,8 @@ def build_shared_cte_name_chain_project_files( def rendered_cte_names(sql: str) -> tuple[set[str], set[str]]: """Return a rendered test query's top-level CTE names and every CTE name nested below them.""" - row: tuple[Any, ...] | None = duckdb.sql( - "SELECT json_serialize_sql($sql)", params={"sql": sql} + row: tuple[Any, ...] | None = duckdb.execute( + "SELECT json_serialize_sql($sql)", {"sql": sql} ).fetchone() assert row is not None entries: list[dict[str, Any]] = json.loads(row[0])["statements"][0]["node"]["cte_map"]["map"] From 2684b3d2f5ac4f730c85c0f31389e4e8d3757db6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 26 Oct 2026 07:18:59 +0000 Subject: [PATCH 07/16] docs: regenerate the DuckDB skill reference page --- .../sqlbuild/references/docs/concepts/adapters/duckdb.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md index dd8164d1d1..e302516d10 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md @@ -8,6 +8,13 @@ Online: https://sqlbuild.com/docs/concepts/adapters/duckdb/ DuckDB is included as a core dependency. No extra installation needed. +SQLBuild requires DuckDB 2.0 or later. DuckDB 2.0 writes new database files in a storage format +that DuckDB 1.x cannot open, so a database file SQLBuild creates can only be read by DuckDB 2.0 +clients. A file that already exists keeps its storage version, so a file created by DuckDB 1.x +stays readable by 1.x. To create a new file that 1.x can read, create it before the first build +with an explicit storage version, for example +`duckdb my_project.duckdb -storage-version v1.5.0 -c "SELECT 1"`. + ## Connection config ```toml From ecb4954e481b14a30ef165cb850d5fa92ac7fc50 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 26 Oct 2026 07:25:28 +0000 Subject: [PATCH 08/16] refactor(duckdb): state the DuckDB 2 reasons in docstrings --- .../contract/classes/duckdb_backed_adapter.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 69f5cc89b6..a04c8367c9 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1495,9 +1495,8 @@ def register_python_function( parameter_types: list[str], returns: str, ) -> None: - """Register a Python function on the connection as a connection-scoped DuckDB function.""" + """Register a Python function after finishing a partly read result, which DuckDB 2 needs.""" - # DuckDB 2 refuses connection calls other than execute() while a result is partly read. connection.complete() connection.create_function(name, function, parameter_types, returns) @@ -1666,7 +1665,6 @@ def swap( right: str, statement_recorder: StatementRecorder, ) -> None: - # DuckDB 2 lets ALTER TABLE rename a view, which would swap a view into the staging name. if self._is_view(connection=connection, relation=left): raise AdapterUserError(message=f"DuckDB cannot swap the view {left} with a table") statements: tuple[str, ...] = self.render_swap(left=left, right=right) @@ -1677,13 +1675,14 @@ def swap( self.execute(connection=connection, sql=stmt) def _is_view(self, *, connection: Any, relation: str) -> bool: - parts: list[str] = [_unquote_duckdb_identifier(part) for part in relation.split(".")] - name: str = parts[-1] + """Whether a relation is a view; DuckDB 2 lets ALTER TABLE rename a view, so swap checks.""" + + *qualifiers, name = [_unquote_duckdb_identifier(part) for part in relation.split(".")] schema_sql: str = ( - _duckdb_string_literal(parts[-2]) if len(parts) >= 2 else "current_schema()" + _duckdb_string_literal(qualifiers[-1]) if qualifiers else "current_schema()" ) database_sql: str = ( - _duckdb_string_literal(parts[-3]) if len(parts) >= 3 else "current_database()" + _duckdb_string_literal(qualifiers[-2]) if len(qualifiers) > 1 else "current_database()" ) query: str = ( "SELECT 1 FROM duckdb_views() WHERE NOT internal " From cd16f43a4a65a243a31945877a346f588dacc2fe Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 26 Oct 2026 08:10:09 +0000 Subject: [PATCH 09/16] fix(duckdb): replace a live view when a full refresh promotes its rebuild A build-aside full refresh moves the live relation to __sqb_prev__ with ALTER TABLE ... RENAME. On DuckDB 2.0 that also renames a view, so a model changed from a view to a snapshot or microbatch model left a __sqb_prev__ view behind, and the next full refresh failed with R008 when it dropped that name as a table. DuckDB rename now refuses a view, as swap already does, with a dedicated error. The full-refresh promotion catches it, drops the live view and renames the rebuild into place; a view has no data to keep as the previous generation. --- .../contract/classes/duckdb_backed_adapter.py | 8 +- src/sqlbuild/adapter/contract/exceptions.py | 4 + .../_helpers/materializations/full_refresh.py | 50 +++++++++-- .../cli/commands/main/build/_test_types.py | 11 +++ .../build/test_materialization_type_change.py | 88 +++++++++++++++++++ 5 files changed, 150 insertions(+), 11 deletions(-) diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index a04c8367c9..acb96c15a1 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -47,7 +47,7 @@ LATEST_DUCKDB_STORAGE_VERSION, QUALIFIED_NAME_SEPARATOR, ) -from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.exceptions import AdapterUserError, ViewRenameRefusedError from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage @@ -1651,6 +1651,10 @@ def rename( destination: str, statement_recorder: StatementRecorder, ) -> None: + if self._is_view(connection=connection, relation=origin): + raise ViewRenameRefusedError( + message=f"DuckDB cannot rename the view {origin} as a table" + ) statements: tuple[str, ...] = self.render_rename(origin=origin, destination=destination) statement_recorder.record_many(statements) stmt: str @@ -1666,7 +1670,7 @@ def swap( statement_recorder: StatementRecorder, ) -> None: if self._is_view(connection=connection, relation=left): - raise AdapterUserError(message=f"DuckDB cannot swap the view {left} with a table") + raise ViewRenameRefusedError(message=f"DuckDB cannot swap the view {left} with a table") statements: tuple[str, ...] = self.render_swap(left=left, right=right) statement_recorder.record_many(statements) with self.transaction(connection): diff --git a/src/sqlbuild/adapter/contract/exceptions.py b/src/sqlbuild/adapter/contract/exceptions.py index df357405cd..c997c740da 100644 --- a/src/sqlbuild/adapter/contract/exceptions.py +++ b/src/sqlbuild/adapter/contract/exceptions.py @@ -17,6 +17,10 @@ def __init__(self, message: str, *, code: str | None = None, help: str | None = self.help = help +class ViewRenameRefusedError(AdapterUserError): + """Raised before renaming a view with table DDL, which DuckDB 2 would otherwise accept.""" + + class SqlStatementExecutionError(RuntimeError): """Warehouse error attributed to the exact SQL statement that raised it.""" diff --git a/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py b/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py index f5d8c71f22..53cd231d8d 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py @@ -7,6 +7,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.exceptions import ViewRenameRefusedError from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( @@ -107,19 +108,50 @@ def promote_full_refresh_rebuild( if_exists=True, statement_recorder=statement_recorder, ) - promote_staged_relation( - adapter=adapter, - connection=connection, - target_qualified=relations.target_qualified, - staged_qualified=relations.rebuild_qualified, - displaced_qualified=relations.previous_qualified, - target_exists=target_exists, - statement_recorder=statement_recorder, - ) + try: + promote_staged_relation( + adapter=adapter, + connection=connection, + target_qualified=relations.target_qualified, + staged_qualified=relations.rebuild_qualified, + displaced_qualified=relations.previous_qualified, + target_exists=target_exists, + statement_recorder=statement_recorder, + ) + except ViewRenameRefusedError: + _replace_live_view( + adapter=adapter, + connection=connection, + relations=relations, + statement_recorder=statement_recorder, + ) _ = guard.rebind() lifecycle.completed(metadata={"changed_count": 1}) +def _replace_live_view( + *, + adapter: BaseAdapter, + connection: Any, + relations: FullRefreshRelations, + statement_recorder: StatementRecorder, +) -> None: + """Drop a live view the rebuild replaces; a view keeps no data to retain as prev.""" + + adapter.drop_view( + connection=connection, + destination=relations.target_qualified, + if_exists=True, + statement_recorder=statement_recorder, + ) + adapter.rename( + connection=connection, + origin=relations.rebuild_qualified, + destination=relations.target_qualified, + statement_recorder=statement_recorder, + ) + + def promote_staged_relation( *, adapter: BaseAdapter, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 75cf231937..9b1c1f78a4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1376,6 +1376,17 @@ class MaterializationTypeChangeTestCase: expected_table_types: tuple[str, ...] +@dataclass(frozen=True) +class ViewToFullRefreshTestCase: + """A view model switched to a build-aside materialization, then fully refreshed repeatedly.""" + + description: str + model_header: str + full_refresh_count: int + expected_relations: list[tuple[str, str]] + expected_rows: list[tuple[object, ...]] + + @dataclass(frozen=True) class TargetConcurrencyE2ETestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_materialization_type_change.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_materialization_type_change.py index 8c30847451..96d74c565e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_materialization_type_change.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_materialization_type_change.py @@ -10,6 +10,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( MaterializationTypeChangeTestCase, + ViewToFullRefreshTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import order_totals_model_sql from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( @@ -77,5 +78,92 @@ def test_given_built_model_when_materialization_changes_then_build_replaces_rela assert rows == [(1, 250)], test_case.description +_SNAPSHOT_PROJECT_TOML: str = ( + 'name = "customers"\nadapter = "duckdb"\n[connection]\ndatabase = "customers.duckdb"\n' + '[snapshots]\ncurrent_state_full_refresh = "allow"\n' +) +_RAW_CUSTOMERS_YML: str = ( + "sources:\n - name: raw_customers\n description: Raw customers.\n" + " schema: main\n table: raw_customers\n" +) +_CUSTOMER_SNAPSHOT_SQL: str = ( + 'SELECT customer_id, plan, updated_at FROM __source("raw_customers")\n' +) + + +@pytest.mark.parametrize( + "test_case", + [ + ViewToFullRefreshTestCase( + description="a view switched to a snapshot fully refreshes three times", + model_header=( + "MODEL (description 'Customer snapshot.', materialized snapshot, " + "unique_key [customer_id], snapshot_strategy timestamp, updated_at updated_at);" + ), + full_refresh_count=3, + expected_relations=[ + ("__sqb_prev__customer_snapshot", "BASE TABLE"), + ("customer_snapshot", "BASE TABLE"), + ], + expected_rows=[(1, "basic")], + ), + ], + ids=lambda case: case.description, +) +def test_given_view_model_when_switched_and_fully_refreshed_then_each_refresh_replaces_it( + test_case: ViewToFullRefreshTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="customers_project", + repo_files={ + "sqlbuild_project.toml": _SNAPSHOT_PROJECT_TOML, + "sources/raw.yml": _RAW_CUSTOMERS_YML, + }, + ) + import duckdb + + connection: duckdb.DuckDBPyConnection = duckdb.connect(str(project_dir / "customers.duckdb")) + connection.execute( + "CREATE TABLE main.raw_customers AS " + "SELECT 1 AS customer_id, 'basic' AS plan, TIMESTAMP '2024-01-01' AS updated_at" + ) + connection.close() + model_path: Path = project_dir / "models/customer_snapshot.sql" + model_path.parent.mkdir(parents=True, exist_ok=True) + _ = model_path.write_text( + "MODEL (description 'Customer snapshot.', materialized view);\n\n" + _CUSTOMER_SNAPSHOT_SQL, + encoding="utf-8", + ) + builds: list[subprocess.CompletedProcess[str]] = [ + run_sqb(project_dir=project_dir, command=("--no-color", "build")) + ] + _ = model_path.write_text( + test_case.model_header + "\n\n" + _CUSTOMER_SNAPSHOT_SQL, encoding="utf-8" + ) + for _refresh in range(test_case.full_refresh_count): + builds.append( + run_sqb(project_dir=project_dir, command=("--no-color", "build", "--full-refresh")) + ) + relations: list[tuple[Any, ...]] = query_duckdb( + db_path=project_dir / "customers.duckdb", + sql=( + "SELECT table_name, table_type FROM information_schema.tables " + "WHERE table_name LIKE '%customer_snapshot%' ORDER BY table_name" + ), + ) + rows: list[tuple[Any, ...]] = query_duckdb( + db_path=project_dir / "customers.duckdb", + sql="SELECT customer_id, plan FROM main.customer_snapshot", + ) + + assert [build.returncode for build in builds] == [0] * len(builds), [ + build.stdout + build.stderr for build in builds + ] + assert relations == test_case.expected_relations + assert rows == test_case.expected_rows + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) From 0b202e8f5926caf1bb6eab734368a1fb3f09b3c4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 26 Oct 2026 08:10:19 +0000 Subject: [PATCH 10/16] fix(duckdb): match database-qualified compatibility views on DuckDB 2.0 DuckDB 2.0 stores a view's SQL without its own catalog prefix, so a view at a database-qualified location never matched the three-part SELECT SQLBuild renders for it. Old-name compatibility views were then recreated on every refresh, and the janitor stopped recognising them. When the plain comparison fails, the DuckDB adapter drops the view's own catalog qualifier from both texts and compares again. --- .../contract/classes/duckdb_backed_adapter.py | 26 +++++- .../sqlbuild/adapters/duckdb/_test_types.py | 13 +++ .../sqlbuild/adapters/duckdb/test_client.py | 81 +++++++++++++++++++ 3 files changed, 119 insertions(+), 1 deletion(-) diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index acb96c15a1..5768865086 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -5,6 +5,7 @@ import importlib.util import json import logging +import re from datetime import date, datetime from decimal import Decimal from importlib.machinery import ModuleSpec @@ -1817,7 +1818,19 @@ def view_definition_matches( definition: str | None = self.read_view_definition( connection=connection, database=database, schema=schema, name=name ) - return definition is not None and same_view_definition(definition=definition, sql=sql) + if definition is None: + return False + if same_view_definition(definition=definition, sql=sql): + return True + catalog: str = database if database is not None else self._current_database(connection) + return same_view_definition( + definition=_without_catalog_qualifier(text=definition, catalog=catalog), + sql=_without_catalog_qualifier(text=sql, catalog=catalog), + ) + + def _current_database(self, connection: Any) -> str: + row: Any = self.execute(connection=connection, sql="SELECT current_database()").fetchone() + return str(row[0]) def render_relation_revokes( self, *, grants: tuple[RelationGrant, ...], destination: str @@ -2757,6 +2770,17 @@ def _duckdb_string_literal(self, value: str) -> str: return value.replace("'", "''") +def _without_catalog_qualifier(*, text: str, catalog: str) -> str: + """Drop a view's own catalog prefix, which DuckDB 2 leaves out of stored view SQL.""" + + quoted_catalog: str = re.escape('"' + catalog.replace('"', '""') + '"') + pattern: re.Pattern[str] = re.compile( + rf"""(? str: return "'" + value.replace("'", "''") + "'" diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py index d6f9449719..0e9e638a17 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/_test_types.py @@ -279,3 +279,16 @@ class PythonFunctionRegistrationTestCase: open_result_sql: str call_sql: str expected_rows: list[tuple[object, ...]] + + +@dataclass(frozen=True) +class QualifiedViewDefinitionTestCase: + """A view at a database-qualified location compared with the SELECT SQLBuild renders.""" + + description: str + setup_sql: tuple[str, ...] + database: str | None + schema: str + name: str + sql: str + expected_match: bool diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py index 588a72fb1e..d79ab7d5b3 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py @@ -42,6 +42,7 @@ MergeTestCase, PhysicalRelationGenerationTestCase, PythonFunctionRegistrationTestCase, + QualifiedViewDefinitionTestCase, QueryTestCase, RecorderWriteTestCase, RelationExistsTestCase, @@ -1692,3 +1693,83 @@ def test_given_partly_read_result_when_registering_python_function_then_function ) assert connection.execute(test_case.call_sql).fetchall() == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + QualifiedViewDefinitionTestCase( + description="a view in an attached database matches its three-part SELECT", + setup_sql=( + "CREATE SCHEMA shop.sales", + "CREATE TABLE shop.sales.orders AS SELECT 1 AS order_id", + "CREATE VIEW shop.sales.old_orders AS SELECT * FROM shop.sales.orders", + ), + database="shop", + schema="sales", + name="old_orders", + sql="SELECT * FROM shop.sales.orders", + expected_match=True, + ), + QualifiedViewDefinitionTestCase( + description="a quoted three-part SELECT matches too", + setup_sql=( + "CREATE SCHEMA shop.sales", + "CREATE TABLE shop.sales.orders AS SELECT 1 AS order_id", + "CREATE VIEW shop.sales.old_orders AS SELECT * FROM shop.sales.orders", + ), + database="shop", + schema="sales", + name="old_orders", + sql='SELECT * FROM "shop"."sales"."orders"', + expected_match=True, + ), + QualifiedViewDefinitionTestCase( + description="a view in the default database matches with no database given", + setup_sql=( + "CREATE TABLE memory.main.orders AS SELECT 1 AS order_id", + "CREATE VIEW memory.main.old_orders AS SELECT * FROM memory.main.orders", + ), + database=None, + schema="main", + name="old_orders", + sql="SELECT * FROM memory.main.orders", + expected_match=True, + ), + QualifiedViewDefinitionTestCase( + description="a three-part SELECT of another relation still differs", + setup_sql=( + "CREATE SCHEMA shop.sales", + "CREATE TABLE shop.sales.orders AS SELECT 1 AS order_id", + "CREATE TABLE shop.sales.customers AS SELECT 1 AS customer_id", + "CREATE VIEW shop.sales.old_orders AS SELECT * FROM shop.sales.orders", + ), + database="shop", + schema="sales", + name="old_orders", + sql="SELECT * FROM shop.sales.customers", + expected_match=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_database_qualified_view_when_comparing_definition_then_catalog_prefix_is_ignored( + test_case: QualifiedViewDefinitionTestCase, + adapter: DuckDbAdapter, + connection: Any, + tmp_path: Path, +) -> None: + connection.execute(f"ATTACH '{tmp_path / 'shop.duckdb'}' AS shop") + statement: str + for statement in test_case.setup_sql: + connection.execute(statement) + + matches: bool = adapter.view_definition_matches( + connection=connection, + database=test_case.database, + schema=test_case.schema, + name=test_case.name, + sql=test_case.sql, + ) + + assert matches is test_case.expected_match From 46810d3d7bdebb3820c738e686a9295842fc58cd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 26 Oct 2026 08:13:01 +0000 Subject: [PATCH 11/16] fix(motherduck): explain that MotherDuck does not support DuckDB 2.0 yet MotherDuck's extension refuses to load on DuckDB 2.0 and supports DuckDB up to 1.5.6. Connecting now reports that MotherDuck does not support the installed DuckDB yet and that MotherDuck users should stay on the last SQLBuild release that supports DuckDB 1.5, instead of only the extension's initialisation error. Other connection errors are raised unchanged. The MotherDuck adapter page says the same. --- .../docs/concepts/adapters/motherduck.md | 8 ++++ .../motherduck/classes/motherduck_adapter.py | 16 +++++++- src/sqlbuild/adapters/motherduck/constants.py | 5 +++ .../adapters/motherduck/_test_types.py | 8 ++++ .../sqlbuild/adapters/motherduck/helpers.py | 18 +++++++-- .../adapters/motherduck/test_client.py | 40 +++++++++++++++++++ .../docs/concepts/adapters/motherduck.mdx | 8 ++++ 7 files changed, 99 insertions(+), 4 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/motherduck.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/motherduck.md index cf92a5859c..4c4d3e4f15 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/motherduck.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/motherduck.md @@ -8,6 +8,14 @@ Online: https://sqlbuild.com/docs/concepts/adapters/motherduck/ MotherDuck uses DuckDB's built-in `md:` connection support. No extra installation needed beyond the core DuckDB dependency. +## DuckDB 2.0 support + +SQLBuild requires DuckDB 2.0, and MotherDuck does not support DuckDB 2.0 yet: its extension +supports DuckDB up to 1.5.6 and refuses to load on 2.0. Connecting reports +`MotherDuck does not support DuckDB 2.0.0 yet`. If you use MotherDuck, stay on the last SQLBuild +release that supports DuckDB 1.5, as listed in the changelog, until MotherDuck supports +DuckDB 2.0. + ## Connection config ```toml diff --git a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py index e12099b630..775f4906f4 100644 --- a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py +++ b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py @@ -10,9 +10,12 @@ from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.types import BuiltinAdapter, LoaderLogicalType from sqlbuild.adapters.motherduck.constants import ( EMPTY_CONNECTION_VALUE, + MOTHERDUCK_UNSUPPORTED_DUCKDB_HELP, + MOTHERDUCK_UNSUPPORTED_DUCKDB_MARKER, URI_ADDITIONAL_PARAMETER_SEPARATOR, URI_QUERY_SEPARATOR, ) @@ -154,7 +157,18 @@ def connect(self, config: dict[str, Any]) -> Any: import duckdb database: str = str(duckdb_config.get("database", ":memory:")) - connection: duckdb.DuckDBPyConnection = duckdb.connect(database=database) + try: + connection: duckdb.DuckDBPyConnection = duckdb.connect(database=database) + except duckdb.Error as error: + if MOTHERDUCK_UNSUPPORTED_DUCKDB_MARKER not in str(error): + raise + raise AdapterUserError( + message=( + f"MotherDuck does not support DuckDB {duckdb.__version__} yet. " + f"{MOTHERDUCK_UNSUPPORTED_DUCKDB_HELP}" + ), + help=MOTHERDUCK_UNSUPPORTED_DUCKDB_HELP, + ) from error extensions: list[str] | tuple[str, ...] = duckdb_config.get("extensions", ()) extension_name: str diff --git a/src/sqlbuild/adapters/motherduck/constants.py b/src/sqlbuild/adapters/motherduck/constants.py index afdf13abdc..8d790d42ee 100644 --- a/src/sqlbuild/adapters/motherduck/constants.py +++ b/src/sqlbuild/adapters/motherduck/constants.py @@ -3,3 +3,8 @@ EMPTY_CONNECTION_VALUE: str = "" URI_QUERY_SEPARATOR: str = "?" URI_ADDITIONAL_PARAMETER_SEPARATOR: str = "&" +MOTHERDUCK_UNSUPPORTED_DUCKDB_MARKER: str = "is not yet supported by MotherDuck" +MOTHERDUCK_UNSUPPORTED_DUCKDB_HELP: str = ( + "This SQLBuild release requires DuckDB 2.0. To use MotherDuck, stay on the last SQLBuild " + "release that supports DuckDB 1.5 until MotherDuck supports DuckDB 2.0." +) diff --git a/tests/unit/src/sqlbuild/adapters/motherduck/_test_types.py b/tests/unit/src/sqlbuild/adapters/motherduck/_test_types.py index e92c6ac103..954b081726 100644 --- a/tests/unit/src/sqlbuild/adapters/motherduck/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/motherduck/_test_types.py @@ -24,3 +24,11 @@ class MotherDuckPruneSqlTestCase: schema: str retain_versions: int expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class MotherDuckConnectErrorTestCase: + description: str + connect_error: str + expected_error_type: str + expected_help_fragment: str | None diff --git a/tests/unit/src/sqlbuild/adapters/motherduck/helpers.py b/tests/unit/src/sqlbuild/adapters/motherduck/helpers.py index 2d4fa4fc43..b8bf1fe7b9 100644 --- a/tests/unit/src/sqlbuild/adapters/motherduck/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/motherduck/helpers.py @@ -16,23 +16,35 @@ def execute(self, sql: str) -> FakeDuckDbConnection: return self +class FakeDuckDbError(Exception): + """Stand-in for duckdb.Error.""" + + class FakeDuckDbModule: """Module double for duckdb.connect.""" - def __init__(self) -> None: + Error: type[Exception] = FakeDuckDbError + __version__: str = "2.0.0" + + def __init__(self, *, connect_error: str | None = None) -> None: self.connected_databases: list[str] = [] self.connection: FakeDuckDbConnection = FakeDuckDbConnection() + self.connect_error: str | None = connect_error def connect(self, *, database: str) -> FakeDuckDbConnection: self.connected_databases.append(database) + if self.connect_error is not None: + raise FakeDuckDbError(self.connect_error) return self.connection -def install_fake_duckdb_module(monkeypatch: Any) -> FakeDuckDbModule: +def install_fake_duckdb_module( + monkeypatch: Any, *, connect_error: str | None = None +) -> FakeDuckDbModule: """Install a fake duckdb module for adapter connection tests.""" import sys - fake_module: FakeDuckDbModule = FakeDuckDbModule() + fake_module: FakeDuckDbModule = FakeDuckDbModule(connect_error=connect_error) monkeypatch.setitem(sys.modules, "duckdb", fake_module) return fake_module diff --git a/tests/unit/src/sqlbuild/adapters/motherduck/test_client.py b/tests/unit/src/sqlbuild/adapters/motherduck/test_client.py index 95770e4c32..6d68782017 100644 --- a/tests/unit/src/sqlbuild/adapters/motherduck/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/motherduck/test_client.py @@ -5,6 +5,7 @@ from sqlbuild.adapters.motherduck.classes.motherduck_adapter import MotherDuckAdapter from tests.unit.src.sqlbuild.adapters.motherduck._test_types import ( MotherDuckAdapterDefaultsTestCase, + MotherDuckConnectErrorTestCase, MotherDuckConnectionDatabaseTestCase, MotherDuckPruneSqlTestCase, ) @@ -57,6 +58,45 @@ def test_given_connection_config_when_connecting_then_uses_motherduck_database( assert fake_duckdb.connected_databases == [test_case.expected_database] +@pytest.mark.parametrize( + "test_case", + [ + MotherDuckConnectErrorTestCase( + description="an unsupported DuckDB version is reported with what to do", + connect_error=( + 'Initialization function "motherduck_duckdb_cpp_init" threw an exception: ' + '"Your DuckDB version (v2.0.0) is not yet supported by MotherDuck."' + ), + expected_error_type="AdapterUserError", + expected_help_fragment="stay on the last SQLBuild release that supports DuckDB 1.5", + ), + MotherDuckConnectErrorTestCase( + description="any other connection error is raised unchanged", + connect_error="IO Error: unauthorized", + expected_error_type="FakeDuckDbError", + expected_help_fragment=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_motherduck_connect_when_connecting_then_unsupported_version_has_help( + test_case: MotherDuckConnectErrorTestCase, + monkeypatch: pytest.MonkeyPatch, +) -> None: + adapter: MotherDuckAdapter = MotherDuckAdapter() + _ = install_fake_duckdb_module(monkeypatch, connect_error=test_case.connect_error) + + with pytest.raises(Exception) as raised: + adapter.connect({"database": "md:analytics"}) + + assert type(raised.value).__name__ == test_case.expected_error_type + help_text: object = getattr(raised.value, "help", None) + if test_case.expected_help_fragment is None: + assert help_text is None + else: + assert isinstance(help_text, str) and test_case.expected_help_fragment in help_text + + @pytest.mark.parametrize( "test_case", [ diff --git a/website/src/content/docs/docs/concepts/adapters/motherduck.mdx b/website/src/content/docs/docs/concepts/adapters/motherduck.mdx index 40963c1323..e74d9b00d1 100644 --- a/website/src/content/docs/docs/concepts/adapters/motherduck.mdx +++ b/website/src/content/docs/docs/concepts/adapters/motherduck.mdx @@ -5,6 +5,14 @@ description: "MotherDuck adapter configuration for SQLBuild." MotherDuck uses DuckDB's built-in `md:` connection support. No extra installation needed beyond the core DuckDB dependency. +## DuckDB 2.0 support + +SQLBuild requires DuckDB 2.0, and MotherDuck does not support DuckDB 2.0 yet: its extension +supports DuckDB up to 1.5.6 and refuses to load on 2.0. Connecting reports +`MotherDuck does not support DuckDB 2.0.0 yet`. If you use MotherDuck, stay on the last SQLBuild +release that supports DuckDB 1.5, as listed in the changelog, until MotherDuck supports +DuckDB 2.0. + ## Connection config ```toml From 5008074d9f617740d5c2cb7e063a23f0bcdd83d5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 26 Oct 2026 08:13:01 +0000 Subject: [PATCH 12/16] docs(duckdb): list the DuckDB 2.0 changes that affect model SQL The DuckDB adapter page now covers arrow lambdas, partial ~ matching, MERGE with duplicate source rows, untyped NULL columns, integer division and modulo by zero, and dlt's dataset reads. The rename page's lambda example uses the lambda x: syntax DuckDB 2.0 accepts. --- .../sqlbuild/references/docs/cli/rename.md | 2 +- .../docs/concepts/adapters/duckdb.md | 19 +++++++++++++++++++ website/src/content/docs/docs/cli/rename.mdx | 2 +- .../docs/docs/concepts/adapters/duckdb.mdx | 19 +++++++++++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md index 882cf0a160..e578057321 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/rename.md @@ -132,7 +132,7 @@ Renaming a column rewrites the model that produces it and every query that reads - the producing projection, as `amount AS revenue`, or the existing alias; - every downstream reference, including filters, joins, grouping, and ordering; - column declarations, including the model's own `MODEL (columns (...))` entries with or without commas between them, and column-valued config such as `unique_key`, `cursor`, and `partition_by`, and `cursor_inputs` and `relationships (field ...)` entries that point at the column; -- string arguments of audits attached in `MODEL` headers, such as `where_condition "is_flagged = TRUE"` or `expression "amount > 0"`, when the generic audit's SQL definition splices the argument in as SQL with `@where_condition`. These arguments are parsed as SQL expressions, so only references to the audited model's column change: string literals stay as written, and a subquery reading another model, such as `customer_id IN (SELECT customer_id FROM __ref("vip_customers"))`, keeps that model's column. You are asked to edit an argument by hand when it cannot be parsed, when it reads a relation whose columns SQLBuild does not know, when the audit quotes it as a string with `@'prefix'`, when it joins on the column with `USING` or `NATURAL`, or when no SQL definition of the audit is found. A lambda parameter named like the column, as in `list_filter(xs, customer_id -> customer_id > 0)`, keeps its name; +- string arguments of audits attached in `MODEL` headers, such as `where_condition "is_flagged = TRUE"` or `expression "amount > 0"`, when the generic audit's SQL definition splices the argument in as SQL with `@where_condition`. These arguments are parsed as SQL expressions, so only references to the audited model's column change: string literals stay as written, and a subquery reading another model, such as `customer_id IN (SELECT customer_id FROM __ref("vip_customers"))`, keeps that model's column. You are asked to edit an argument by hand when it cannot be parsed, when it reads a relation whose columns SQLBuild does not know, when the audit quotes it as a string with `@'prefix'`, when it joins on the column with `USING` or `NATURAL`, or when no SQL definition of the audit is found. A lambda parameter named like the column, as in `list_filter(xs, lambda customer_id: customer_id > 0)`, keeps its name; - fixture columns of `__ref__` and `__expected__` CTEs. By default a column rename is one step: downstream models keep their output names, so `SELECT o.amount` becomes `SELECT o.revenue AS amount`. With `--cascade`, a downstream model that passes the column through unchanged, as a bare column, a same-named alias, or `SELECT *`, renames its output too, and the rename continues through its own consumers. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md index e302516d10..47be44e3bb 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md @@ -15,6 +15,25 @@ stays readable by 1.x. To create a new file that 1.x can read, create it before with an explicit storage version, for example `duckdb my_project.duckdb -storage-version v1.5.0 -c "SELECT 1"`. +## SQL that behaves differently on DuckDB 2.0 + +DuckDB 2.0 changes how some SQL runs. SQLBuild runs model SQL as written, so check models for +these before upgrading: + +- Arrow lambdas such as `list_filter(xs, x -> x > 0)` are rejected by default. Write + `list_filter(xs, lambda x: x > 0)` instead. DuckDB 2.1 removes the arrow syntax. +- `~` and `!~` match a regular expression anywhere in the string, as in PostgreSQL, so + `'abc' ~ 'b'` is now true. Use `regexp_full_match` for a match of the whole string. +- `MERGE` fails when one target row matches several source rows, instead of picking one. Keep + an incremental model's new rows unique on its `unique_key`. +- An untyped `NULL`, as in `SELECT NULL AS discount`, creates a column of type `"NULL"` that + rejects later non-null values. Cast it, for example `CAST(NULL AS DECIMAL(18, 2)) AS discount`. +- Integer division `//` and modulo `%` by zero raise an error instead of returning `NULL`. Wrap + the expression in `TRY(...)` to get `NULL`. + +dlt's dataset read API (reading a loaded table back as a dataframe or Arrow table) calls DuckDB +methods that DuckDB 2.0 removed, so those reads fail. Loading data with dlt is not affected. + ## Connection config ```toml diff --git a/website/src/content/docs/docs/cli/rename.mdx b/website/src/content/docs/docs/cli/rename.mdx index f0d4911c2a..165673bb4e 100644 --- a/website/src/content/docs/docs/cli/rename.mdx +++ b/website/src/content/docs/docs/cli/rename.mdx @@ -114,7 +114,7 @@ Renaming a column rewrites the model that produces it and every query that reads - the producing projection, as `amount AS revenue`, or the existing alias; - every downstream reference, including filters, joins, grouping, and ordering; - column declarations, including the model's own `MODEL (columns (...))` entries with or without commas between them, and column-valued config such as `unique_key`, `cursor`, and `partition_by`, and `cursor_inputs` and `relationships (field ...)` entries that point at the column; -- string arguments of audits attached in `MODEL` headers, such as `where_condition "is_flagged = TRUE"` or `expression "amount > 0"`, when the generic audit's SQL definition splices the argument in as SQL with `@where_condition`. These arguments are parsed as SQL expressions, so only references to the audited model's column change: string literals stay as written, and a subquery reading another model, such as `customer_id IN (SELECT customer_id FROM __ref("vip_customers"))`, keeps that model's column. You are asked to edit an argument by hand when it cannot be parsed, when it reads a relation whose columns SQLBuild does not know, when the audit quotes it as a string with `@'prefix'`, when it joins on the column with `USING` or `NATURAL`, or when no SQL definition of the audit is found. A lambda parameter named like the column, as in `list_filter(xs, customer_id -> customer_id > 0)`, keeps its name; +- string arguments of audits attached in `MODEL` headers, such as `where_condition "is_flagged = TRUE"` or `expression "amount > 0"`, when the generic audit's SQL definition splices the argument in as SQL with `@where_condition`. These arguments are parsed as SQL expressions, so only references to the audited model's column change: string literals stay as written, and a subquery reading another model, such as `customer_id IN (SELECT customer_id FROM __ref("vip_customers"))`, keeps that model's column. You are asked to edit an argument by hand when it cannot be parsed, when it reads a relation whose columns SQLBuild does not know, when the audit quotes it as a string with `@'prefix'`, when it joins on the column with `USING` or `NATURAL`, or when no SQL definition of the audit is found. A lambda parameter named like the column, as in `list_filter(xs, lambda customer_id: customer_id > 0)`, keeps its name; - fixture columns of `__ref__` and `__expected__` CTEs. By default a column rename is one step: downstream models keep their output names, so `SELECT o.amount` becomes `SELECT o.revenue AS amount`. With `--cascade`, a downstream model that passes the column through unchanged, as a bare column, a same-named alias, or `SELECT *`, renames its output too, and the rename continues through its own consumers. diff --git a/website/src/content/docs/docs/concepts/adapters/duckdb.mdx b/website/src/content/docs/docs/concepts/adapters/duckdb.mdx index e525647c89..0f787c1427 100644 --- a/website/src/content/docs/docs/concepts/adapters/duckdb.mdx +++ b/website/src/content/docs/docs/concepts/adapters/duckdb.mdx @@ -12,6 +12,25 @@ stays readable by 1.x. To create a new file that 1.x can read, create it before with an explicit storage version, for example `duckdb my_project.duckdb -storage-version v1.5.0 -c "SELECT 1"`. +## SQL that behaves differently on DuckDB 2.0 + +DuckDB 2.0 changes how some SQL runs. SQLBuild runs model SQL as written, so check models for +these before upgrading: + +- Arrow lambdas such as `list_filter(xs, x -> x > 0)` are rejected by default. Write + `list_filter(xs, lambda x: x > 0)` instead. DuckDB 2.1 removes the arrow syntax. +- `~` and `!~` match a regular expression anywhere in the string, as in PostgreSQL, so + `'abc' ~ 'b'` is now true. Use `regexp_full_match` for a match of the whole string. +- `MERGE` fails when one target row matches several source rows, instead of picking one. Keep + an incremental model's new rows unique on its `unique_key`. +- An untyped `NULL`, as in `SELECT NULL AS discount`, creates a column of type `"NULL"` that + rejects later non-null values. Cast it, for example `CAST(NULL AS DECIMAL(18, 2)) AS discount`. +- Integer division `//` and modulo `%` by zero raise an error instead of returning `NULL`. Wrap + the expression in `TRY(...)` to get `NULL`. + +dlt's dataset read API (reading a loaded table back as a dataframe or Arrow table) calls DuckDB +methods that DuckDB 2.0 removed, so those reads fail. Loading data with dlt is not affected. + ## Connection config ```toml From a45876e919dc53ae1f3f003f48f9650af73429bd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:12:19 +0000 Subject: [PATCH 13/16] test(scenario): replay a Snowflake arrow lambda locally Local replay transpiles captured Snowflake SQL to DuckDB, and DuckDB 2.0 rejects the arrow lambda the transpiler wrote. This E2E test replays a Snowflake FILTER(..., x -> ...) model locally. It passes once the pinned polyglot-sql-chio release writes DuckDB lambdas as LAMBDA x : body. --- Makefile | 1 + .../cli/commands/main/scenario/_test_types.py | 9 ++ .../cli/commands/main/scenario/helpers.py | 83 +++++++++++++++++++ .../main/scenario/test_local_lambda.py | 53 ++++++++++++ 4 files changed, 146 insertions(+) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_lambda.py diff --git a/Makefile b/Makefile index e1ffa034ee..aff8652a6a 100644 --- a/Makefile +++ b/Makefile @@ -155,6 +155,7 @@ E2E_DUCKDB_CLI_DATA_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_helper_ctes.py \ tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_fixture_columns.py \ tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_variant.py \ + tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_lambda.py \ tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_snapshot_staleness.py \ tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_unmocked_inputs.py diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py index 060d9a9f14..fb52762696 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py @@ -352,3 +352,12 @@ class CapturedVariantTextScenarioTestCase: expected_payload_json: str expected_returncode: int expected_stdout_fragment: str + + +@dataclass(frozen=True) +class CapturedLambdaScenarioTestCase: + """A Snowflake-captured scenario whose model uses an arrow lambda, replayed in DuckDB.""" + + description: str + scenario_name: str + expected_stdout_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py index 89ecb8afe3..0cc7b5aa84 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py @@ -1284,3 +1284,86 @@ def write_snowflake_variant_snapshot( (snapshot_root / "scenario.json").write_text( json.dumps(manifest, indent=2) + "\n", encoding="utf-8" ) + + +SNOWFLAKE_ORDER_IDS_FIXTURE_SQL: str = "SELECT 1 AS order_id" + + +def build_snowflake_lambda_scenario_project_files() -> dict[str, str]: + """Build an offline Snowflake project whose model filters an array with an arrow lambda.""" + + return { + "sqlbuild_project.toml": build_offline_snowflake_project_toml(), + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n description: Raw orders.\n" + " schema: main\n table: raw_orders\n columns:\n" + ' - {name: order_id, type: "NUMBER(38,0)"}\n' + ), + "models/order_quantities.sql": ( + 'MODEL (description "Quantities above each order id.");\n' + "SELECT order_id,\n" + " FILTER(ARRAY_CONSTRUCT(1, 2, 3), x -> x > order_id) AS big_quantities\n" + 'FROM __source("raw_orders")\n' + ), + "tests/scenarios/order_quantities__lambda.sql": ( + "SCENARIO (description 'An arrow lambda replays locally.');\n" + "WITH\n" + f"__source__raw_orders AS ({SNOWFLAKE_ORDER_IDS_FIXTURE_SQL}),\n" + "__expected__order_quantities AS (\n" + " SELECT 1 AS order_id, ARRAY_CONSTRUCT(2, 3) AS big_quantities\n" + ")\n" + "SELECT 1\n" + ), + } + + +def write_snowflake_order_ids_snapshot(*, project_dir: Path, scenario_name: str) -> None: + """Write the snapshot a Snowflake capture stores for a one-column order source.""" + + snapshot_root: Path = project_dir / "tests" / "_scenario_snapshots" / scenario_name + (snapshot_root / "sources").mkdir(parents=True) + row: str = json.dumps({"ORDER_ID": 1}) + "\n" + (snapshot_root / "sources" / "raw_orders.jsonl").write_text(row, encoding="utf-8") + manifest: dict[str, object] = { + "capture_adapter": "snowflake", + "capture_dialect": "snowflake", + "captured_at": "2026-10-25T11:10:54Z", + "format": "jsonl", + "input_fingerprint": build_scenario_snapshot_input_fingerprint( + scenario_name=scenario_name, + input_specs=( + ScenarioSnapshotInputSpec( + kind=ScenarioArtifactKind.SOURCE, + logical_name="raw_orders", + file_path=Path("sources") / "raw_orders.jsonl", + capture_sql=SNOWFLAKE_ORDER_IDS_FIXTURE_SQL, + ), + ), + capture_adapter="snowflake", + capture_dialect="snowflake", + ), + "relations": [ + { + "bytes": len(row.encode("utf-8")), + "columns": [ + { + "local_type": "DECIMAL(1, 0)", + "name": "order_id", + "warehouse_type": "NUMBER(1,0)", + } + ], + "file": "sources/raw_orders.jsonl", + "kind": "source", + "logical_name": "raw_orders", + "row_count": 1, + } + ], + "scenario_name": scenario_name, + "sqlbuild_version": "0.177.0", + "total_bytes": len(row.encode("utf-8")), + "total_rows": 1, + "version": 1, + } + (snapshot_root / "scenario.json").write_text( + json.dumps(manifest, indent=2) + "\n", encoding="utf-8" + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_lambda.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_lambda.py new file mode 100644 index 0000000000..707c1e5e6f --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_lambda.py @@ -0,0 +1,53 @@ +"""E2E coverage of local replay of captured-dialect lambdas, which DuckDB 2.0 needs as lambda x:.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.scenario._test_types import ( + CapturedLambdaScenarioTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.scenario.helpers import ( + build_snowflake_lambda_scenario_project_files, + write_snowflake_order_ids_snapshot, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + CapturedLambdaScenarioTestCase( + description="a Snowflake FILTER with an arrow lambda replays locally", + scenario_name="order_quantities__lambda", + expected_stdout_fragment="PASS=1 FAIL=0 ERROR=0 SKIP=0 TOTAL=1", + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_arrow_lambda_when_replaying_locally_then_the_scenario_passes( + test_case: CapturedLambdaScenarioTestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="snowflake_lambda", + repo_files=build_snowflake_lambda_scenario_project_files(), + ) + write_snowflake_order_ids_snapshot( + project_dir=project_dir, scenario_name=test_case.scenario_name + ) + + replayed: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "scenario", "test", "--local", "--strict", test_case.scenario_name), + project_dir=project_dir, + ) + + assert replayed.returncode == 0, replayed.stdout + replayed.stderr + assert test_case.expected_stdout_fragment in replayed.stdout, replayed.stdout + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From b46eb7f73ba888c6d06984d9066de018488a0fc9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:32:59 +0000 Subject: [PATCH 14/16] fix(duckdb): check the live type in full refresh instead of in every rename Guarding DuckDB rename added a catalog read to every rename, so a janitor run archiving many tables issued one extra query per table. Only the build-aside full refresh renames a target that may be a view, so it now reads the live relation type once, through the relation metadata cache, on DuckDB and MotherDuck, and replaces a live view instead of moving it aside. DuckDB rename no longer reads the catalog. --- .../contract/classes/duckdb_backed_adapter.py | 4 -- .../_helpers/materializations/full_refresh.py | 44 ++++++++++++++----- src/sqlbuild/executor/run/models.py | 2 + .../sqlbuild/executor/run/_helpers/helpers.py | 6 ++- .../run/_helpers/test_staged_promotion.py | 4 +- 5 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 5768865086..2a0c9d48a7 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1652,10 +1652,6 @@ def rename( destination: str, statement_recorder: StatementRecorder, ) -> None: - if self._is_view(connection=connection, relation=origin): - raise ViewRenameRefusedError( - message=f"DuckDB cannot rename the view {origin} as a table" - ) statements: tuple[str, ...] = self.render_rename(origin=origin, destination=destination) statement_recorder.record_many(statements) stmt: str diff --git a/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py b/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py index 53cd231d8d..86322ba5ec 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/full_refresh.py @@ -7,9 +7,9 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.exceptions import ViewRenameRefusedError -from sqlbuild.adapter.contract.types import BuiltinAdapter +from sqlbuild.adapter.contract.types import BuiltinAdapter, RelationType from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists +from sqlbuild.adapter.relations.main.cached_relation_type import cached_relation_type from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) @@ -23,6 +23,9 @@ ) _PREVIOUS_PREFIX: str = "__sqb_prev__" +_TABLE_RENAME_MOVES_VIEWS: frozenset[str] = frozenset( + {BuiltinAdapter.DUCKDB, BuiltinAdapter.MOTHERDUCK} +) _REBUILD_PREFIX: str = "__sqb_rebuild__" @@ -51,6 +54,8 @@ def resolve_full_refresh_relations( target_qualified=_qualified( adapter=adapter, database=database, schema=schema, name=target_name ), + database=database, + schema=schema, rebuild_name=rebuild_name, rebuild_qualified=_qualified( adapter=adapter, database=database, schema=schema, name=rebuild_name @@ -108,7 +113,18 @@ def promote_full_refresh_rebuild( if_exists=True, statement_recorder=statement_recorder, ) - try: + if ( + target_exists + and adapter.adapter_name in _TABLE_RENAME_MOVES_VIEWS + and _live_view(adapter=adapter, connection=connection, relations=relations) + ): + _replace_live_view( + adapter=adapter, + connection=connection, + relations=relations, + statement_recorder=statement_recorder, + ) + else: promote_staged_relation( adapter=adapter, connection=connection, @@ -118,17 +134,25 @@ def promote_full_refresh_rebuild( target_exists=target_exists, statement_recorder=statement_recorder, ) - except ViewRenameRefusedError: - _replace_live_view( - adapter=adapter, - connection=connection, - relations=relations, - statement_recorder=statement_recorder, - ) _ = guard.rebind() lifecycle.completed(metadata={"changed_count": 1}) +def _live_view(*, adapter: BaseAdapter, connection: Any, relations: FullRefreshRelations) -> bool: + """Whether the live relation is a view, which table renames must not move aside.""" + + return ( + cached_relation_type( + adapter=adapter, + connection=connection, + database=relations.database, + schema=relations.schema, + name=relations.target_name, + ) + == RelationType.VIEW + ) + + def _replace_live_view( *, adapter: BaseAdapter, diff --git a/src/sqlbuild/executor/run/models.py b/src/sqlbuild/executor/run/models.py index 9b82f68de4..d324f736fb 100644 --- a/src/sqlbuild/executor/run/models.py +++ b/src/sqlbuild/executor/run/models.py @@ -600,6 +600,8 @@ class FullRefreshRelations: target_name: str target_qualified: str + database: str | None + schema: str | None rebuild_name: str rebuild_qualified: str previous_name: str diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py index 9d0d0d1ae9..0f5adee649 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py @@ -285,8 +285,12 @@ class RecordingConnection: def __init__(self) -> None: self.executed: list[str] = [] - def execute(self, sql: str) -> None: + def execute(self, sql: str) -> "RecordingConnection": self.executed.append(sql) + return self + + def fetchall(self) -> list[tuple[object, ...]]: + return [] class RecordingSnowflakeAdapter(SnowflakeAdapter): diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_staged_promotion.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_staged_promotion.py index bd65b3d594..6a5cd3f48d 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_staged_promotion.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_staged_promotion.py @@ -83,11 +83,13 @@ def test_given_staged_relation_when_promoting_then_nothing_is_dropped( expected_statements=("ALTER TABLE main.__sqb_rebuild__orders RENAME TO orders",), ), FullRefreshPromotionTestCase( - description="rename adapters drop the previous copy then rename twice", + description="duckdb drops the previous copy, checks the live type, then renames twice", adapter_name="duckdb", target_exists=True, expected_statements=( "DROP TABLE IF EXISTS main.__sqb_prev__orders", + "SELECT table_name, table_schema, table_type FROM information_schema.tables " + "WHERE 1=1 AND table_schema IN ('main') AND table_name IN ('orders')", "ALTER TABLE main.orders RENAME TO __sqb_prev__orders", "ALTER TABLE main.__sqb_rebuild__orders RENAME TO orders", ), From 4f33b2dc2ee0c014c874fb8f816ef46349f95a8e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:41:57 +0000 Subject: [PATCH 15/16] test: keep new test setup branch-free and constants first --- .../build/test_materialization_type_change.py | 24 ++++++------- .../adapters/motherduck/_test_types.py | 2 +- .../sqlbuild/adapters/motherduck/helpers.py | 35 ++++++++++++++----- .../adapters/motherduck/test_client.py | 14 ++++---- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_materialization_type_change.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_materialization_type_change.py index 96d74c565e..d1bb294f8f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_materialization_type_change.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_materialization_type_change.py @@ -23,6 +23,17 @@ 'name = "orders"\nadapter = "duckdb"\n[connection]\ndatabase = "orders.duckdb"\n' ) _MODEL_PATH: str = "models/order_totals.sql" +_SNAPSHOT_PROJECT_TOML: str = ( + 'name = "customers"\nadapter = "duckdb"\n[connection]\ndatabase = "customers.duckdb"\n' + '[snapshots]\ncurrent_state_full_refresh = "allow"\n' +) +_RAW_CUSTOMERS_YML: str = ( + "sources:\n - name: raw_customers\n description: Raw customers.\n" + " schema: main\n table: raw_customers\n" +) +_CUSTOMER_SNAPSHOT_SQL: str = ( + 'SELECT customer_id, plan, updated_at FROM __source("raw_customers")\n' +) @pytest.mark.parametrize( @@ -78,19 +89,6 @@ def test_given_built_model_when_materialization_changes_then_build_replaces_rela assert rows == [(1, 250)], test_case.description -_SNAPSHOT_PROJECT_TOML: str = ( - 'name = "customers"\nadapter = "duckdb"\n[connection]\ndatabase = "customers.duckdb"\n' - '[snapshots]\ncurrent_state_full_refresh = "allow"\n' -) -_RAW_CUSTOMERS_YML: str = ( - "sources:\n - name: raw_customers\n description: Raw customers.\n" - " schema: main\n table: raw_customers\n" -) -_CUSTOMER_SNAPSHOT_SQL: str = ( - 'SELECT customer_id, plan, updated_at FROM __source("raw_customers")\n' -) - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/adapters/motherduck/_test_types.py b/tests/unit/src/sqlbuild/adapters/motherduck/_test_types.py index 954b081726..8ae02e160d 100644 --- a/tests/unit/src/sqlbuild/adapters/motherduck/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/motherduck/_test_types.py @@ -31,4 +31,4 @@ class MotherDuckConnectErrorTestCase: description: str connect_error: str expected_error_type: str - expected_help_fragment: str | None + expected_help: str | None diff --git a/tests/unit/src/sqlbuild/adapters/motherduck/helpers.py b/tests/unit/src/sqlbuild/adapters/motherduck/helpers.py index b8bf1fe7b9..282733baca 100644 --- a/tests/unit/src/sqlbuild/adapters/motherduck/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/motherduck/helpers.py @@ -26,25 +26,44 @@ class FakeDuckDbModule: Error: type[Exception] = FakeDuckDbError __version__: str = "2.0.0" - def __init__(self, *, connect_error: str | None = None) -> None: + def __init__(self) -> None: self.connected_databases: list[str] = [] self.connection: FakeDuckDbConnection = FakeDuckDbConnection() - self.connect_error: str | None = connect_error def connect(self, *, database: str) -> FakeDuckDbConnection: self.connected_databases.append(database) - if self.connect_error is not None: - raise FakeDuckDbError(self.connect_error) return self.connection -def install_fake_duckdb_module( - monkeypatch: Any, *, connect_error: str | None = None -) -> FakeDuckDbModule: +class FailingFakeDuckDbModule(FakeDuckDbModule): + """Module double whose duckdb.connect raises a DuckDB error.""" + + def __init__(self, *, connect_error: str) -> None: + super().__init__() + self.connect_error: str = connect_error + + def connect(self, *, database: str) -> FakeDuckDbConnection: + self.connected_databases.append(database) + raise FakeDuckDbError(self.connect_error) + + +def install_failing_fake_duckdb_module( + monkeypatch: Any, *, connect_error: str +) -> FailingFakeDuckDbModule: + """Install a fake duckdb module whose connect fails with the given message.""" + + import sys + + fake_module: FailingFakeDuckDbModule = FailingFakeDuckDbModule(connect_error=connect_error) + monkeypatch.setitem(sys.modules, "duckdb", fake_module) + return fake_module + + +def install_fake_duckdb_module(monkeypatch: Any) -> FakeDuckDbModule: """Install a fake duckdb module for adapter connection tests.""" import sys - fake_module: FakeDuckDbModule = FakeDuckDbModule(connect_error=connect_error) + fake_module: FakeDuckDbModule = FakeDuckDbModule() monkeypatch.setitem(sys.modules, "duckdb", fake_module) return fake_module diff --git a/tests/unit/src/sqlbuild/adapters/motherduck/test_client.py b/tests/unit/src/sqlbuild/adapters/motherduck/test_client.py index 6d68782017..c952115efe 100644 --- a/tests/unit/src/sqlbuild/adapters/motherduck/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/motherduck/test_client.py @@ -3,6 +3,7 @@ import pytest from sqlbuild.adapters.motherduck.classes.motherduck_adapter import MotherDuckAdapter +from sqlbuild.adapters.motherduck.constants import MOTHERDUCK_UNSUPPORTED_DUCKDB_HELP from tests.unit.src.sqlbuild.adapters.motherduck._test_types import ( MotherDuckAdapterDefaultsTestCase, MotherDuckConnectErrorTestCase, @@ -11,6 +12,7 @@ ) from tests.unit.src.sqlbuild.adapters.motherduck.helpers import ( FakeDuckDbModule, + install_failing_fake_duckdb_module, install_fake_duckdb_module, ) @@ -68,13 +70,13 @@ def test_given_connection_config_when_connecting_then_uses_motherduck_database( '"Your DuckDB version (v2.0.0) is not yet supported by MotherDuck."' ), expected_error_type="AdapterUserError", - expected_help_fragment="stay on the last SQLBuild release that supports DuckDB 1.5", + expected_help=MOTHERDUCK_UNSUPPORTED_DUCKDB_HELP, ), MotherDuckConnectErrorTestCase( description="any other connection error is raised unchanged", connect_error="IO Error: unauthorized", expected_error_type="FakeDuckDbError", - expected_help_fragment=None, + expected_help=None, ), ], ids=lambda case: case.description, @@ -84,17 +86,13 @@ def test_given_failing_motherduck_connect_when_connecting_then_unsupported_versi monkeypatch: pytest.MonkeyPatch, ) -> None: adapter: MotherDuckAdapter = MotherDuckAdapter() - _ = install_fake_duckdb_module(monkeypatch, connect_error=test_case.connect_error) + _ = install_failing_fake_duckdb_module(monkeypatch, connect_error=test_case.connect_error) with pytest.raises(Exception) as raised: adapter.connect({"database": "md:analytics"}) assert type(raised.value).__name__ == test_case.expected_error_type - help_text: object = getattr(raised.value, "help", None) - if test_case.expected_help_fragment is None: - assert help_text is None - else: - assert isinstance(help_text, str) and test_case.expected_help_fragment in help_text + assert getattr(raised.value, "help", None) == test_case.expected_help @pytest.mark.parametrize( From 33dc4875cc9690db5aedfa6a7e44b921aff10d3a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:54:41 +0000 Subject: [PATCH 16/16] docs(motherduck): name 0.182.0 as the last DuckDB 1.5 release; state dlt's whole-table Arrow reads The MotherDuck connection error no longer repeats its help text in the message, the dlt note says only whole-table Arrow reads fail, and the local lambda replay test is an expected failure until the polyglot wheel that writes DuckDB lambdas as LAMBDA x : body is published. --- .../sqlbuild/references/docs/concepts/adapters/duckdb.md | 4 ++-- .../sqlbuild/references/docs/concepts/adapters/motherduck.md | 5 ++--- .../adapters/motherduck/classes/motherduck_adapter.py | 5 +---- src/sqlbuild/adapters/motherduck/constants.py | 4 ++-- .../sqlbuild/cli/commands/main/scenario/test_local_lambda.py | 4 ++++ website/src/content/docs/docs/concepts/adapters/duckdb.mdx | 4 ++-- .../src/content/docs/docs/concepts/adapters/motherduck.mdx | 5 ++--- 7 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md index 47be44e3bb..d8d1dc708f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/duckdb.md @@ -31,8 +31,8 @@ these before upgrading: - Integer division `//` and modulo `%` by zero raise an error instead of returning `NULL`. Wrap the expression in `TRY(...)` to get `NULL`. -dlt's dataset read API (reading a loaded table back as a dataframe or Arrow table) calls DuckDB -methods that DuckDB 2.0 removed, so those reads fail. Loading data with dlt is not affected. +dlt's dataset read API calls a DuckDB method that DuckDB 2.0 removed when it reads a loaded table +back as one whole Arrow table, so that read fails. Dataframe and chunked Arrow reads still work. Loading data with dlt is not affected. ## Connection config diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/motherduck.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/motherduck.md index 4c4d3e4f15..0d7d82f76a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/motherduck.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/motherduck.md @@ -12,9 +12,8 @@ MotherDuck uses DuckDB's built-in `md:` connection support. No extra installatio SQLBuild requires DuckDB 2.0, and MotherDuck does not support DuckDB 2.0 yet: its extension supports DuckDB up to 1.5.6 and refuses to load on 2.0. Connecting reports -`MotherDuck does not support DuckDB 2.0.0 yet`. If you use MotherDuck, stay on the last SQLBuild -release that supports DuckDB 1.5, as listed in the changelog, until MotherDuck supports -DuckDB 2.0. +`MotherDuck does not support DuckDB 2.0.0 yet`. If you use MotherDuck, stay on SQLBuild 0.182.0, +the last release that supports DuckDB 1.5, until MotherDuck supports DuckDB 2.0. ## Connection config diff --git a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py index 775f4906f4..434c33a209 100644 --- a/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py +++ b/src/sqlbuild/adapters/motherduck/classes/motherduck_adapter.py @@ -163,10 +163,7 @@ def connect(self, config: dict[str, Any]) -> Any: if MOTHERDUCK_UNSUPPORTED_DUCKDB_MARKER not in str(error): raise raise AdapterUserError( - message=( - f"MotherDuck does not support DuckDB {duckdb.__version__} yet. " - f"{MOTHERDUCK_UNSUPPORTED_DUCKDB_HELP}" - ), + message=f"MotherDuck does not support DuckDB {duckdb.__version__} yet.", help=MOTHERDUCK_UNSUPPORTED_DUCKDB_HELP, ) from error diff --git a/src/sqlbuild/adapters/motherduck/constants.py b/src/sqlbuild/adapters/motherduck/constants.py index 8d790d42ee..7e2626ec6a 100644 --- a/src/sqlbuild/adapters/motherduck/constants.py +++ b/src/sqlbuild/adapters/motherduck/constants.py @@ -5,6 +5,6 @@ URI_ADDITIONAL_PARAMETER_SEPARATOR: str = "&" MOTHERDUCK_UNSUPPORTED_DUCKDB_MARKER: str = "is not yet supported by MotherDuck" MOTHERDUCK_UNSUPPORTED_DUCKDB_HELP: str = ( - "This SQLBuild release requires DuckDB 2.0. To use MotherDuck, stay on the last SQLBuild " - "release that supports DuckDB 1.5 until MotherDuck supports DuckDB 2.0." + "This SQLBuild release requires DuckDB 2.0. To use MotherDuck, stay on SQLBuild 0.182.0, " + "the last release that supports DuckDB 1.5, until MotherDuck supports DuckDB 2.0." ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_lambda.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_lambda.py index 707c1e5e6f..642a7575f9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_lambda.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_local_lambda.py @@ -17,6 +17,10 @@ from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb +@pytest.mark.xfail( + strict=True, + reason="needs the polyglot-sql-chio wheel that writes DuckDB lambdas as LAMBDA x : body", +) @pytest.mark.parametrize( "test_case", [ diff --git a/website/src/content/docs/docs/concepts/adapters/duckdb.mdx b/website/src/content/docs/docs/concepts/adapters/duckdb.mdx index 0f787c1427..e526cf06d3 100644 --- a/website/src/content/docs/docs/concepts/adapters/duckdb.mdx +++ b/website/src/content/docs/docs/concepts/adapters/duckdb.mdx @@ -28,8 +28,8 @@ these before upgrading: - Integer division `//` and modulo `%` by zero raise an error instead of returning `NULL`. Wrap the expression in `TRY(...)` to get `NULL`. -dlt's dataset read API (reading a loaded table back as a dataframe or Arrow table) calls DuckDB -methods that DuckDB 2.0 removed, so those reads fail. Loading data with dlt is not affected. +dlt's dataset read API calls a DuckDB method that DuckDB 2.0 removed when it reads a loaded table +back as one whole Arrow table, so that read fails. Dataframe and chunked Arrow reads still work. Loading data with dlt is not affected. ## Connection config diff --git a/website/src/content/docs/docs/concepts/adapters/motherduck.mdx b/website/src/content/docs/docs/concepts/adapters/motherduck.mdx index e74d9b00d1..31a55a561b 100644 --- a/website/src/content/docs/docs/concepts/adapters/motherduck.mdx +++ b/website/src/content/docs/docs/concepts/adapters/motherduck.mdx @@ -9,9 +9,8 @@ MotherDuck uses DuckDB's built-in `md:` connection support. No extra installatio SQLBuild requires DuckDB 2.0, and MotherDuck does not support DuckDB 2.0 yet: its extension supports DuckDB up to 1.5.6 and refuses to load on 2.0. Connecting reports -`MotherDuck does not support DuckDB 2.0.0 yet`. If you use MotherDuck, stay on the last SQLBuild -release that supports DuckDB 1.5, as listed in the changelog, until MotherDuck supports -DuckDB 2.0. +`MotherDuck does not support DuckDB 2.0.0 yet`. If you use MotherDuck, stay on SQLBuild 0.182.0, +the last release that supports DuckDB 1.5, until MotherDuck supports DuckDB 2.0. ## Connection config