From b9f54810807629886a8b0bef085087fe15f18a70 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:52:23 +0000 Subject: [PATCH] fix: match configured snapshot keys and cursors to quoted columns Snapshot required and generated columns, snapshot key exclusion, the microbatch cursor check and merge immutable columns now compare names with the shared column-name key, so a quoted mixed-case name such as '"listingId"' matches its warehouse column on every adapter. A snapshot or microbatch build now stops with a clear error when an unquoted setting names a column the warehouse lists only as a quoted, case-sensitive identifier, instead of failing with a misleading missing-column error or generating SQL for the wrong column. --- benchmarks/duckshop/README.md | 4 +- .../snap_edge_partner_listing_export.sql | 31 ++++++ .../test_snap_edge_partner_listing_export.sql | 30 ++++++ .../docs/concepts/adapters/snowflake.md | 24 +++++ .../adapter/contract/classes/base_adapter.py | 5 +- .../contract/classes/duckdb_backed_adapter.py | 5 +- .../snowflake/classes/snowflake_adapter.py | 5 +- .../_helpers/materializations/microbatch.py | 14 ++- .../run/_helpers/materializations/snapshot.py | 42 ++++++-- .../_helpers/validation/configured_columns.py | 37 ++++++++ .../cli/commands/main/build/_test_types.py | 11 +++ .../build/test_quoted_mixed_case_columns.py | 95 +++++++++++++++++++ .../build/snowflake_metadata/_test_types.py | 13 +++ .../build/snowflake_metadata/helpers.py | 39 ++++++++ .../test_quoted_mixed_case_columns.py | 81 ++++++++++++++++ .../sqlbuild/adapters/duckdb/_test_types.py | 2 + .../sqlbuild/adapters/duckdb/test_client.py | 17 +++- .../executor/run/_helpers/_test_types.py | 10 ++ .../sqlbuild/executor/run/_helpers/helpers.py | 17 ++++ .../run/_helpers/test_configured_columns.py | 66 +++++++++++++ .../docs/docs/concepts/adapters/snowflake.mdx | 23 +++++ 21 files changed, 549 insertions(+), 22 deletions(-) create mode 100644 benchmarks/duckshop/project/models/marts/edge_cases/snap_edge_partner_listing_export.sql create mode 100644 benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_snap_edge_partner_listing_export.sql create mode 100644 src/sqlbuild/executor/run/_helpers/validation/configured_columns.py create mode 100644 tests/unit/src/sqlbuild/executor/run/_helpers/test_configured_columns.py diff --git a/benchmarks/duckshop/README.md b/benchmarks/duckshop/README.md index 505f262150..00a284348c 100644 --- a/benchmarks/duckshop/README.md +++ b/benchmarks/duckshop/README.md @@ -119,7 +119,7 @@ project and does not change it: - raw `edge__*` sources in `sources/edge_cases.yml`, loaded by `python/loaders/edge_cases.py` from fixed rows that `data_scale` does not change, so core data and core expectations stay the same. -It has 38 models (7 staging, 22 intermediate, 9 marts), 62 unit tests and 4 scenarios, in four +It has 39 models (7 staging, 22 intermediate, 10 marts), 63 unit tests and 4 scenarios, in four groups: | Group | Covers | @@ -127,7 +127,7 @@ groups: | Semi-structured | JSON null against SQL NULL and missing keys; document text with sorted, case-differing and escaped keys; JSON number fidelity and casts out of `VARIANT`; path-like keys (`''`, `*`, `$`-prefixed, dotted) with `GET` and `GET_IGNORE_CASE`; `OBJECT_CONSTRUCT_KEEP_NULL`, `OBJECT_INSERT`, `OBJECT_DELETE`, `OBJECT_PICK`; array builders, `ARRAY_EXCEPT` and number spellings in `ARRAY_POSITION`; `FLATTEN` outer; lenient and malformed `TRY_PARSE_JSON`; typed lambdas | | Dates and times | `TRY_TO_TIMESTAMP_NTZ` and its siblings over free text; epoch milliseconds as numbers and text; daylight-saving gaps and overlaps; fractional offsets; nanosecond truncation and epoch parts after 2242; pre-1970 text with sub-microsecond digits; date-part normalisation; month ends and leap days; week numbering across a year; sentinel timestamps | | Ordering, numbers and text | NULL ordering under `DESC`; ties in ordered aggregates, `MAX_BY`, `MODE` and `ANY_VALUE`; default window frames; `UNION ALL BY NAME`; `ROUND` half-way values, decimal division, `MOD` and 38-digit amounts; integers past 2^53 against `FLOAT`; regular expressions and backslash escapes; `LIKE ... ESCAPE` and `ILIKE ANY`; `SPLIT_PART` and `SUBSTR` before the start; `NOT IN` with a NULL; names in many scripts | -| Scenarios and lifecycle | typed values across model boundaries; `VARIANT` expected output; a star under a lower-case contract and a quoted camelCase export; a cursor-window incremental model over far-apart fixture loads; a change log merged across two loads, checked against a full rebuild | +| Scenarios and lifecycle | typed values across model boundaries; `VARIANT` expected output; a star under a lower-case contract and a quoted camelCase export; a snapshot keyed and compared on quoted camelCase columns; a cursor-window incremental model over far-apart fixture loads; a change log merged across two loads, checked against a full rebuild | Unit-test expectations in the edge-case area were captured on Snowflake under the `edge` target's pinned session, rather than derived by hand; scenario expectations, and the assertions in tests diff --git a/benchmarks/duckshop/project/models/marts/edge_cases/snap_edge_partner_listing_export.sql b/benchmarks/duckshop/project/models/marts/edge_cases/snap_edge_partner_listing_export.sql new file mode 100644 index 0000000000..46e3d2e762 --- /dev/null +++ b/benchmarks/duckshop/project/models/marts/edge_cases/snap_edge_partner_listing_export.sql @@ -0,0 +1,31 @@ +MODEL ( + description """ + Version history of the partner feed export for listings with a known status, keyed and compared + on its quoted camelCase columns + """, + tags [edge_case], + materialized snapshot, + unique_key ['"listingId"'], + snapshot_strategy check, + check_columns ['"unitPrice"', '"isLive"'], + columns ( + "listingId" (nullable false, audits [not_null]), + ), +); + +WITH export AS ( + SELECT * + FROM __ref("rpt_edge_partner_listing_export") +), + +final AS ( + SELECT + e."listingId", + e."unitPrice", + e."isLive" + FROM export AS e + WHERE e."isLive" IS NOT NULL +) + +SELECT * +FROM final diff --git a/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_snap_edge_partner_listing_export.sql b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_snap_edge_partner_listing_export.sql new file mode 100644 index 0000000000..7ed41cbd81 --- /dev/null +++ b/benchmarks/duckshop/project/tests/unit/marts/edge_cases/test_snap_edge_partner_listing_export.sql @@ -0,0 +1,30 @@ +TEST (name "snap_edge_partner_listing_export__tracks_listings_with_a_known_status"); + +WITH __ref__rpt_edge_partner_listing_export AS ( + SELECT + 'LS-EIDER-03' AS "listingId", + 0.05::NUMBER(18, 2) AS "unitPrice", + CAST(NULL AS BOOLEAN) AS "isLive" + UNION ALL + SELECT + 'LS-MALLARD-01' AS "listingId", + 12.99::NUMBER(18, 2) AS "unitPrice", + TRUE AS "isLive" + UNION ALL + SELECT + 'LS-TEAL-02' AS "listingId", + 8.99::NUMBER(18, 2) AS "unitPrice", + FALSE AS "isLive" +), + +__expected__snap_edge_partner_listing_export AS ( + SELECT + 'LS-MALLARD-01' AS "listingId", + 12.99::NUMBER(18, 2) AS "unitPrice", + TRUE AS "isLive" + UNION ALL + SELECT + 'LS-TEAL-02' AS "listingId", + 8.99::NUMBER(18, 2) AS "unitPrice", + FALSE AS "isLive" +) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index ca81724ba2..fbd6184f4b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -10,6 +10,7 @@ Online: https://sqlbuild.com/docs/concepts/adapters/snowflake/ - Connection config - Session initialization +- Case-sensitive column names - Planning metadata - Build metadata - State tables @@ -63,6 +64,29 @@ one warehouse for build-style commands and another for ad-hoc commands such as ` `sqb diff`, and `--warehouse` overrides both for one invocation. See [command-group warehouses](../project-configuration.md#command-group-warehouses). +## Case-sensitive column names + +Snowflake stores an unquoted column name in upper case, so `order_id` and `ORDER_ID` name the +same column. A column created with a quoted name in mixed or lower case, such as `"orderId"`, +keeps its exact case, and only the quoted name reads it. + +Model settings that name such a column (`unique_key`, `updated_at`, `observed_at`, +`check_columns`, `cursor` and `merge_exclude_columns`) take the quoted name inside a string: + +```sql +MODEL ( + description 'Order history.', + materialized snapshot, + unique_key ['"orderId"'], + snapshot_strategy timestamp, + updated_at '"updatedAt"', +); +``` + +A snapshot or microbatch build stops when an unquoted setting names a case-sensitive column, with +an error such as +`unique_key 'orderId' names the case-sensitive column "orderId"; write it with its quotes, as '"orderId"'`. + ## Planning metadata Planning reads relation and column metadata with `SHOW TABLES`, `SHOW VIEWS`, and `SHOW COLUMNS` diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 97d8f9b5b4..6acc8921ad 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -38,6 +38,7 @@ AdapterUserError, UnsupportedTypedSqlRenderingError, ) +from sqlbuild.adapter.contract.main.column_name_key import column_name_key from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition from sqlbuild.adapter.contract.main.unsupported_relation_usage import unsupported_relation_usage @@ -969,7 +970,7 @@ def render_merge( exclude_columns: tuple[str, ...] = (), ) -> tuple[str, ...]: immutable_columns: frozenset[str] = frozenset( - column.lower() for column in (*unique_key, *exclude_columns) + column_name_key(column) for column in (*unique_key, *exclude_columns) ) join_condition: str = " AND ".join( f"__target.{self.render_identifier(k)} = __source.{self.render_identifier(k)}" @@ -978,7 +979,7 @@ def render_merge( update_assignments: str = ", ".join( f"{self.render_identifier(col)} = __source.{self.render_identifier(col)}" for col in source_columns - if col.lower() not in immutable_columns + if column_name_key(col) not in immutable_columns ) insert_columns: str = ", ".join(self.render_identifier(col) for col in source_columns) insert_values: str = ", ".join( diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index a7b499bea5..d376a6a228 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -49,6 +49,7 @@ QUALIFIED_NAME_SEPARATOR, ) from sqlbuild.adapter.contract.exceptions import AdapterUserError, ViewRenameRefusedError +from sqlbuild.adapter.contract.main.column_name_key import column_name_key from sqlbuild.adapter.contract.main.is_quoted_identifier import is_quoted_identifier from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition @@ -2079,7 +2080,7 @@ def render_merge( exclude_columns: tuple[str, ...] = (), ) -> tuple[str, ...]: immutable_columns: frozenset[str] = frozenset( - column.lower() for column in (*unique_key, *exclude_columns) + column_name_key(column) for column in (*unique_key, *exclude_columns) ) join_condition: str = " AND ".join( f"__target.{self.render_identifier(k)} = __source.{self.render_identifier(k)}" @@ -2088,7 +2089,7 @@ def render_merge( update_assignments: str = ", ".join( f"{self.render_identifier(col)} = __source.{self.render_identifier(col)}" for col in source_columns - if col.lower() not in immutable_columns + if column_name_key(col) not in immutable_columns ) insert_columns: str = ", ".join(self.render_identifier(col) for col in source_columns) insert_values: str = ", ".join( diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 9dbd19841c..49736eb0af 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -47,6 +47,7 @@ DIFF_RIGHT_SIDE, ) from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.main.column_name_key import column_name_key from sqlbuild.adapter.contract.main.complete_table_freshness_results import ( complete_table_freshness_results, ) @@ -1220,7 +1221,7 @@ def render_merge( exclude_columns: tuple[str, ...] = (), ) -> tuple[str, ...]: immutable_columns: frozenset[str] = frozenset( - column.lower() for column in (*unique_key, *exclude_columns) + column_name_key(column) for column in (*unique_key, *exclude_columns) ) join_condition: str = " AND ".join( f"__target.{self.render_identifier(k)} = __source.{self.render_identifier(k)}" @@ -1229,7 +1230,7 @@ def render_merge( update_assignments: str = ", ".join( f"{self.render_identifier(col)} = __source.{self.render_identifier(col)}" for col in source_columns - if col.lower() not in immutable_columns + if column_name_key(col) not in immutable_columns ) insert_columns: str = ", ".join(self.render_identifier(col) for col in source_columns) insert_values: str = ", ".join( diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index 850714cf33..ffd067fd81 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -58,6 +58,7 @@ from sqlbuild._native import cursor_split_into_batches as split_into_batches from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.main.column_name_key import column_name_key from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( @@ -97,6 +98,7 @@ _widenable_columns, ) from sqlbuild.executor.run._helpers.reuse.fingerprinting import try_write_fingerprint +from sqlbuild.executor.run._helpers.validation.configured_columns import require_quoted_spelling from sqlbuild.executor.run._helpers.validation.cursor_bounds import ( build_runtime_cursor_spec, empty_input_rebuild_error, @@ -3531,8 +3533,16 @@ def _validate_cursor_output_columns( cursor_column: str | None = entry.cursor_column if cursor_column is None: return - delta_names: frozenset[str] = frozenset(column.name.lower() for column in delta_columns) - if cursor_column.lower() in delta_names: + delta_names: frozenset[str] = frozenset( + column_name_key(column.name) for column in delta_columns + ) + if column_name_key(cursor_column) in delta_names: + require_quoted_spelling( + model_name=entry.name, + setting="cursor", + configured=(cursor_column,), + listed_columns=delta_columns, + ) return raise ExecutorInputError( f"microbatch cursor column '{cursor_column}' is not produced by model output; " diff --git a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py index abefe46ae5..08144f58d1 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py @@ -18,6 +18,7 @@ ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.main.column_name_key import column_name_key from sqlbuild.adapter.contract.models import ( ColumnInfo, NormalizedType, @@ -56,6 +57,7 @@ resolve_full_refresh_relations, ) from sqlbuild.executor.run._helpers.reuse.fingerprinting import try_write_fingerprint +from sqlbuild.executor.run._helpers.validation.configured_columns import require_quoted_spelling from sqlbuild.executor.run._helpers.validation.contracts import validate_runtime_contract from sqlbuild.executor.run.constants import ( SNAPSHOT_ALL_CHECK_COLUMNS, @@ -634,7 +636,9 @@ def _validate_delta_columns( delta_columns: tuple[ColumnInfo, ...], check_columns: tuple[str, ...], ) -> None: - column_names: frozenset[str] = frozenset(column.name.lower() for column in delta_columns) + column_names: frozenset[str] = frozenset( + column_name_key(column.name) for column in delta_columns + ) required_columns: tuple[str, ...] if entry.snapshot_strategy == SnapshotStrategy.TIMESTAMP: required_columns = (*entry.unique_key, _require_updated_at(entry)) @@ -654,17 +658,35 @@ def _validate_delta_columns( ): required_columns = (*required_columns, entry.observed_at_column) missing_columns: list[str] = [ - column for column in required_columns if column.lower() not in column_names + column for column in required_columns if column_name_key(column) not in column_names ] if missing_columns: raise ExecutorInputError( f"snapshot model '{entry.name}' query output is missing required columns: " f"{', '.join(missing_columns)}" ) + setting: str + configured: tuple[str | None, ...] + for setting, configured in ( + ("unique_key", entry.unique_key), + ("updated_at", (entry.updated_at_column,)), + ("observed_at", (entry.observed_at_column,)), + ("check_columns", check_columns), + ): + require_quoted_spelling( + model_name=entry.name, + setting=setting, + configured=tuple( + name for name in configured if name is not None and name in required_columns + ), + listed_columns=delta_columns, + ) valid_from_column: str = _valid_from_column(entry) valid_to_column: str = _valid_to_column(entry) generated_collisions: list[str] = [ - column for column in (valid_from_column, valid_to_column) if column.lower() in column_names + column + for column in (valid_from_column, valid_to_column) + if column_name_key(column) in column_names ] if generated_collisions: raise ExecutorInputError( @@ -941,16 +963,18 @@ def _expanded_check_columns( ): return entry.check_columns - excluded_names: set[str] = {column.lower() for column in entry.unique_key} - excluded_names.add(_valid_from_column(entry).lower()) - excluded_names.add(_valid_to_column(entry).lower()) + excluded_names: set[str] = {column_name_key(column) for column in entry.unique_key} + excluded_names.add(column_name_key(_valid_from_column(entry))) + excluded_names.add(column_name_key(_valid_to_column(entry))) if entry.observed_at_column is not None: - excluded_names.add(entry.observed_at_column.lower()) + excluded_names.add(column_name_key(entry.observed_at_column)) if entry.updated_at_column is not None: - excluded_names.add(entry.updated_at_column.lower()) + excluded_names.add(column_name_key(entry.updated_at_column)) expanded_columns: tuple[str, ...] = tuple( - column.name for column in delta_columns if column.name.lower() not in excluded_names + column.name + for column in delta_columns + if column_name_key(column.name) not in excluded_names ) if not expanded_columns: raise ExecutorInputError( diff --git a/src/sqlbuild/executor/run/_helpers/validation/configured_columns.py b/src/sqlbuild/executor/run/_helpers/validation/configured_columns.py new file mode 100644 index 0000000000..2a8c32d805 --- /dev/null +++ b/src/sqlbuild/executor/run/_helpers/validation/configured_columns.py @@ -0,0 +1,37 @@ +"""Runtime validation of the column names a model's configuration names.""" + +from __future__ import annotations + +from collections.abc import Iterable + +from sqlbuild.adapter.contract.main.column_name_key import column_name_key +from sqlbuild.adapter.contract.main.is_quoted_identifier import is_quoted_identifier +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.errors.contracts.exceptions import ExecutorInputError + + +def require_quoted_spelling( + *, + model_name: str, + setting: str, + configured: Iterable[str], + listed_columns: tuple[ColumnInfo, ...], +) -> None: + """Reject an unquoted configured name that only a quoted, case-sensitive column matches.""" + + spellings: dict[str, list[str]] = {} + column: ColumnInfo + for column in listed_columns: + spellings.setdefault(column_name_key(column.name), []).append(column.name) + name: str + for name in configured: + if is_quoted_identifier(name): + continue + matches: list[str] = spellings.get(column_name_key(name), []) + if not matches or not all(is_quoted_identifier(match) for match in matches): + continue + raise ExecutorInputError( + f"model '{model_name}' {setting} '{name}' names the case-sensitive column " + f"{matches[0]}; write it with its quotes, as '{matches[0]}', so the generated SQL " + "keeps its case" + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index c114f1cb3b..aef671c86b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1513,3 +1513,14 @@ class QuotedContractE2ETestCase: description: str declared_name: str expected_rows: list[tuple[object, ...]] + + +@dataclass(frozen=True) +class QuotedConfiguredColumnsE2ETestCase: + """A snapshot key and a microbatch cursor configured with quoted camelCase names.""" + + description: str + unique_key: str + cursor: str + expected_history_rows: list[tuple[object, ...]] + expected_activity_rows: list[tuple[object, ...]] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_quoted_mixed_case_columns.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_quoted_mixed_case_columns.py index bfa49dd09c..d74ce1b65b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_quoted_mixed_case_columns.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_quoted_mixed_case_columns.py @@ -3,12 +3,14 @@ from __future__ import annotations import subprocess +from datetime import datetime from decimal import Decimal from pathlib import Path import pytest from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + QuotedConfiguredColumnsE2ETestCase, QuotedContractE2ETestCase, QuotedMixedCaseColumnsBuildE2ETestCase, ) @@ -161,5 +163,98 @@ def test_given_enforced_contract_on_quoted_alias_when_building_then_contract_and assert rows == test_case.expected_rows +@pytest.mark.parametrize( + "test_case", + [ + QuotedConfiguredColumnsE2ETestCase( + description="a quoted snapshot key and microbatch cursor build twice", + unique_key="'\"listingId\"'", + cursor="'\"eventAt\"'", + expected_history_rows=[("L1", 1299, True), ("L1", 1499, False)], + expected_activity_rows=[ + ("L1", datetime(2026, 1, 1, 0, 30)), + ("L2", datetime(2026, 1, 1, 1, 30)), + ("L3", datetime(2026, 1, 1, 2, 30)), + ], + ), + ], + ids=lambda case: case.description, +) +def test_given_quoted_configured_columns_when_building_twice_then_snapshot_and_batches_apply( + test_case: QuotedConfiguredColumnsE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="partner_feed", + repo_files={ + "sqlbuild_project.toml": _PROJECT_FILES["sqlbuild_project.toml"], + "sources/raw.yml": ( + "sources:\n" + " - name: raw_listings\n description: Partner listings.\n" + " schema: main\n table: raw_listings\n" + " - name: raw_listing_events\n description: Partner listing events.\n" + " schema: main\n table: raw_listing_events\n" + ), + "models/listing_history.sql": ( + "MODEL (\n description 'Listing price history.',\n materialized snapshot,\n" + f" unique_key [{test_case.unique_key}],\n snapshot_strategy timestamp,\n" + " updated_at '\"updatedAt\"',\n);\n\n" + 'SELECT "listingId", "unitPriceMinor", "updatedAt" FROM __source("raw_listings")\n' + ), + "models/listing_activity.sql": ( + "MODEL (\n description 'Listing events by hour.',\n materialized incremental,\n" + " incremental_strategy delete_insert,\n incremental_mode microbatch,\n" + " microbatch_strategy watermark,\n cursor_watermark_mode all,\n" + f" cursor {test_case.cursor},\n cursor_type timestamp,\n cursor_grain hour,\n" + " cursor_inputs (\n" + ' raw_listing_events (column "eventAt", roles [filter, watermark]),\n ),\n' + " batch_size 1h,\n);\n\n" + 'SELECT "listingId", "eventAt" FROM __source("raw_listing_events")\n' + ), + }, + ) + db_path: Path = project_dir / "partner_feed.duckdb" + execute_duckdb( + db_path=db_path, + sql=( + "CREATE TABLE main.raw_listings AS SELECT 'L1' AS \"listingId\", " + '1299::BIGINT AS "unitPriceMinor", TIMESTAMP \'2026-01-01 00:00:00\' AS "updatedAt";\n' + "CREATE TABLE main.raw_listing_events AS SELECT * FROM (VALUES " + "('L1', TIMESTAMP '2026-01-01 00:30:00'), ('L2', TIMESTAMP '2026-01-01 01:30:00')) " + 'AS v("listingId", "eventAt")' + ), + ) + first: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + execute_duckdb( + db_path=db_path, + sql=( + 'UPDATE main.raw_listings SET "unitPriceMinor" = 1499, ' + "\"updatedAt\" = TIMESTAMP '2026-01-02 00:00:00';\n" + "INSERT INTO main.raw_listing_events VALUES ('L3', TIMESTAMP '2026-01-01 02:30:00')" + ), + ) + second: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + for build in (first, second): + assert build.returncode == 0, build.stdout + build.stderr + history: list[tuple[object, ...]] = query_duckdb( + db_path=db_path, + sql=( + 'SELECT "listingId", "unitPriceMinor", valid_to IS NOT NULL ' + "FROM main.listing_history ORDER BY valid_from" + ), + ) + activity: list[tuple[object, ...]] = query_duckdb( + db_path=db_path, + sql='SELECT "listingId", "eventAt" FROM main.listing_activity ORDER BY "eventAt"', + ) + assert history == test_case.expected_history_rows + assert activity == test_case.expected_activity_rows + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py index e787f39fb2..136eec98e4 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/_test_types.py @@ -119,3 +119,16 @@ class QuotedContractBuildTestCase: declared_name: str produced_columns: tuple[str, ...] expected_exit_code: int = 0 + + +@dataclass(frozen=True) +class QuotedConfiguredColumnTestCase: + """A snapshot key and a microbatch cursor configured for quoted mixed-case columns.""" + + description: str + unique_key: str + updated_at: str + cursor: str + expected_exit_code: int + expected_fragments: tuple[str, ...] = () + expected_errors: tuple[str, ...] = () diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py index f65ad62662..cd508ad9ed 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/helpers.py @@ -801,3 +801,42 @@ def schema_listing_counts(statements: tuple[str, ...]) -> dict[str, int]: ] matches: list[re.Match[str]] = list(filter(None, searches)) return dict(Counter(f"{match['kind']} {match['scope']}" for match in matches)) + + +def quoted_configured_column_files( + *, unique_key: str, updated_at: str, cursor: str +) -> dict[str, str]: + """A snapshot and a microbatch model over listings stored with quoted camelCase columns.""" + + return { + "sqlbuild_project.toml": ( + 'name = "partner_feed"\nadapter = "snowflake"\n\n' + '[connection]\naccount = "example"\nuser = "example"\n' + 'database = "analytics"\nschema = "marts"\n\n' + '[defaults]\ndatabase = "analytics"\nschema = "marts"\nmaterialized = "table"\n' + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_listings\n description: Partner listings in quoted camelCase.\n" + " database: analytics\n schema: raw\n table: listings\n" + " - name: raw_listing_events\n description: Partner listing events.\n" + " database: analytics\n schema: raw\n table: listing_events\n" + ), + "models/listing_history.sql": ( + "MODEL (\n description 'Listing price history.',\n materialized snapshot,\n" + f" unique_key [{unique_key}],\n snapshot_strategy timestamp,\n" + f" updated_at {updated_at},\n);\n\n" + 'SELECT "listingId", "unitPrice", "updatedAt" FROM __source("raw_listings")\n' + ), + "models/listing_activity.sql": ( + "MODEL (\n description 'Listing events by hour.',\n materialized incremental,\n" + " incremental_strategy delete_insert,\n incremental_mode microbatch,\n" + " microbatch_strategy watermark,\n cursor_watermark_mode all,\n" + f" cursor {cursor},\n cursor_type timestamp,\n cursor_grain day,\n" + " cursor_start '2026-01-01',\n cursor_end '2026-01-03',\n" + " cursor_inputs (\n" + " raw_listing_events (column '\"eventAt\"', roles [filter, watermark]),\n ),\n" + " batch_size 1d,\n);\n\n" + 'SELECT "listingId", "eventAt" FROM __source("raw_listing_events")\n' + ), + } diff --git a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_quoted_mixed_case_columns.py b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_quoted_mixed_case_columns.py index 1037660e61..961f7c6652 100644 --- a/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_quoted_mixed_case_columns.py +++ b/tests/integration/src/sqlbuild/executor/build/snowflake_metadata/test_quoted_mixed_case_columns.py @@ -8,11 +8,13 @@ from tests.integration.src.sqlbuild.executor.build.snowflake_metadata._test_types import ( QuotedColumnBuildTestCase, + QuotedConfiguredColumnTestCase, QuotedContractBuildTestCase, ) from tests.integration.src.sqlbuild.executor.build.snowflake_metadata.helpers import ( SimulatedSnowflakeWarehouse, build_with_cli, + quoted_configured_column_files, ) from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import ( FakeColumn, @@ -153,5 +155,84 @@ def test_given_enforced_contract_on_quoted_alias_when_building_then_runtime_cont assert "runtime contract" not in output, output +@pytest.mark.parametrize( + "test_case", + [ + QuotedConfiguredColumnTestCase( + description="quoted names keep their quotes in snapshot and microbatch SQL", + unique_key="'\"listingId\"'", + updated_at="'\"updatedAt\"'", + cursor="'\"eventAt\"'", + expected_exit_code=0, + expected_fragments=( + 'GROUP BY "listingId"', + '__sqb_rebuild__listing_activity WHERE "eventAt" >=', + ), + ), + QuotedConfiguredColumnTestCase( + description="unquoted names for case-sensitive columns are rejected clearly", + unique_key="listingId", + updated_at="updatedAt", + cursor="eventAt", + expected_exit_code=1, + expected_errors=( + "unique_key 'listingId' names the case-sensitive column \"listingId\"; " + "write it with its quotes, as '\"listingId\"'", + "cursor 'eventAt' names the case-sensitive column \"eventAt\"", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_configured_quoted_columns_when_building_then_sql_keeps_quotes_or_rejects( + test_case: QuotedConfiguredColumnTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + for relative, content in quoted_configured_column_files( + unique_key=test_case.unique_key, updated_at=test_case.updated_at, cursor=test_case.cursor + ).items(): + path: Path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(content, encoding="utf-8") + listing_columns: tuple[FakeColumn, ...] = ( + FakeColumn(name="listingId"), + FakeColumn(name="unitPrice", data_type="NUMBER"), + FakeColumn(name="updatedAt", data_type="TIMESTAMP_NTZ"), + ) + event_columns: tuple[FakeColumn, ...] = ( + FakeColumn(name="listingId"), + FakeColumn(name="eventAt", data_type="TIMESTAMP_NTZ"), + ) + warehouse: SimulatedSnowflakeWarehouse = SimulatedSnowflakeWarehouse( + relations=( + FakeRelation( + database="ANALYTICS", schema="RAW", name="LISTINGS", columns=listing_columns + ), + FakeRelation( + database="ANALYTICS", schema="RAW", name="LISTING_EVENTS", columns=event_columns + ), + ), + ) + warehouse.set_model_columns(model="listing_history", columns=listing_columns) + warehouse.set_model_columns(model="listing_activity", columns=event_columns) + + builds: tuple[tuple[int, tuple[str, ...]], ...] = tuple( + build_with_cli(project_dir=tmp_path, warehouse=warehouse, monkeypatch=monkeypatch) + for _ in range(2) + ) + + output: str = capsys.readouterr().out + assert tuple(build[0] for build in builds) == (test_case.expected_exit_code,) * 2, output + statements: str = "\n".join(("\n".join(builds[0][1]), "\n".join(builds[1][1]))) + fragment: str + for fragment in test_case.expected_fragments: + assert fragment in statements, statements + error: str + for error in test_case.expected_errors: + assert error in output, output + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/duckdb/_test_types.py b/tests/unit/src/sqlbuild/adapters/duckdb/_test_types.py index 8500d1ab73..4440fb80c6 100644 --- a/tests/unit/src/sqlbuild/adapters/duckdb/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/duckdb/_test_types.py @@ -8,6 +8,8 @@ class DuckDbMergeExclusionTestCase: description: str expected_update_clause: str expected_insert_clause: str + unique_key: tuple[str, ...] = ("order_id", "line_id") + source_columns: tuple[str, ...] = ("order_id", "line_id", "status", "updated_at") @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/adapters/duckdb/test_client.py b/tests/unit/src/sqlbuild/adapters/duckdb/test_client.py index bfa13727c1..1d34e3a5c6 100644 --- a/tests/unit/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/duckdb/test_client.py @@ -359,7 +359,18 @@ def test_given_subquery_source_when_rendering_freshness_query_then_duckdb_omits_ 'VALUES (__source."order_id", __source."line_id", __source."status", ' '__source."updated_at")' ), - ) + ), + DuckDbMergeExclusionTestCase( + description="a quoted camelCase key matches its listed column", + expected_update_clause='"status" = __source."status"', + expected_insert_clause=( + 'INSERT ("orderId", "line_id", "status", "updated_at") ' + 'VALUES (__source."orderId", __source."line_id", __source."status", ' + '__source."updated_at")' + ), + unique_key=('"orderId"', "line_id"), + source_columns=("orderId", "line_id", "status", "updated_at"), + ), ], ids=lambda case: case.description, ) @@ -371,8 +382,8 @@ def test_given_composite_keys_when_rendering_merge_then_uses_expected_columns( (merge_sql,) = adapter.render_merge( destination="analytics.orders", sql="SELECT order_id, line_id, status, updated_at FROM staging_orders", - unique_key=("order_id", "line_id"), - source_columns=("order_id", "line_id", "status", "updated_at"), + unique_key=test_case.unique_key, + source_columns=test_case.source_columns, exclude_columns=("UPDATED_AT",), ) diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py index 481ec6c633..5409d0c525 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/_test_types.py @@ -484,3 +484,13 @@ class CustomQueryChangeTestCase: description: str reason: PlanReason expected_query_changed: bool + + +@dataclass(frozen=True) +class QuotedSpellingTestCase: + """A configured column name, the warehouse-listed names, and whether it is rejected.""" + + description: str + configured: str + listed_names: tuple[str, ...] + expected_error: str | None = None diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py index 0f5adee649..8473afc61e 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py @@ -29,8 +29,10 @@ AuditRunScope, AuditSeverity, ) +from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.run._helpers.execution.hooks import execute_hooks +from sqlbuild.executor.run._helpers.validation.configured_columns import require_quoted_spelling from sqlbuild.executor.run.models import HookContext, HookRelationLookup, HookRunContext from sqlbuild.executor.run.types import HookPhase from sqlbuild.python_nodes.models import SqlResourceRef @@ -369,3 +371,18 @@ def run_python_hook_with_relations( enforce_explicit_references=enforce_explicit_references, ), ) + + +def quoted_spelling_rejection(*, configured: str, listed_names: tuple[str, ...]) -> str | None: + """The error a unique_key ``configured`` gets against ``listed_names``, or None.""" + + try: + require_quoted_spelling( + model_name="orders", + setting="unique_key", + configured=(configured,), + listed_columns=tuple(ColumnInfo(name=name, type="VARCHAR") for name in listed_names), + ) + except ExecutorInputError as error: + return str(error) + return None diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_configured_columns.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_configured_columns.py new file mode 100644 index 0000000000..3c58d4ed1a --- /dev/null +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_configured_columns.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import pytest + +from tests.unit.src.sqlbuild.executor.run._helpers._test_types import QuotedSpellingTestCase +from tests.unit.src.sqlbuild.executor.run._helpers.helpers import quoted_spelling_rejection + + +@pytest.mark.parametrize( + "test_case", + [ + QuotedSpellingTestCase( + description="a quoted name matches its quoted column", + configured='"orderId"', + listed_names=('"orderId"', "amount"), + ), + QuotedSpellingTestCase( + description="an unquoted name matches a case-folded column", + configured="ORDER_ID", + listed_names=("order_id",), + ), + QuotedSpellingTestCase( + description="an unquoted name matches a case-folded column beside a quoted one", + configured="orderId", + listed_names=("orderid", '"orderId"'), + ), + QuotedSpellingTestCase( + description="a name no column matches is left to the other checks", + configured="missing", + listed_names=('"orderId"',), + ), + QuotedSpellingTestCase( + description="an unquoted name for a case-sensitive column is rejected", + configured="orderId", + listed_names=('"orderId"',), + expected_error=( + "model 'orders' unique_key 'orderId' names the case-sensitive column " + '"orderId"; write it with its quotes, as \'"orderId"\', so the generated SQL ' + "keeps its case" + ), + ), + QuotedSpellingTestCase( + description="an unquoted lower-case name for a quoted lower-case column is rejected", + configured="orderid", + listed_names=('"orderid"',), + expected_error=( + "model 'orders' unique_key 'orderid' names the case-sensitive column " + '"orderid"; write it with its quotes, as \'"orderid"\', so the generated SQL ' + "keeps its case" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_configured_column_when_checking_quoted_spelling_then_rejects_only_case_sensitive( + test_case: QuotedSpellingTestCase, +) -> None: + rejection: str | None = quoted_spelling_rejection( + configured=test_case.configured, listed_names=test_case.listed_names + ) + + assert rejection == test_case.expected_error + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index 477f4ea2bc..e04ec2e193 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -49,6 +49,29 @@ one warehouse for build-style commands and another for ad-hoc commands such as ` `sqb diff`, and `--warehouse` overrides both for one invocation. See [command-group warehouses](/docs/concepts/project-configuration#command-group-warehouses). +## Case-sensitive column names + +Snowflake stores an unquoted column name in upper case, so `order_id` and `ORDER_ID` name the +same column. A column created with a quoted name in mixed or lower case, such as `"orderId"`, +keeps its exact case, and only the quoted name reads it. + +Model settings that name such a column (`unique_key`, `updated_at`, `observed_at`, +`check_columns`, `cursor` and `merge_exclude_columns`) take the quoted name inside a string: + +```sql +MODEL ( + description 'Order history.', + materialized snapshot, + unique_key ['"orderId"'], + snapshot_strategy timestamp, + updated_at '"updatedAt"', +); +``` + +A snapshot or microbatch build stops when an unquoted setting names a case-sensitive column, with +an error such as +`unique_key 'orderId' names the case-sensitive column "orderId"; write it with its quotes, as '"orderId"'`. + ## Planning metadata Planning reads relation and column metadata with `SHOW TABLES`, `SHOW VIEWS`, and `SHOW COLUMNS`