From 3327d5ede49c6834b213beaeed0ad6472e46d943 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 20 Sep 2026 13:16:03 +0100 Subject: [PATCH] fix: type null-only SQL test fixtures --- .../adapter/contract/classes/base_adapter.py | 5 + .../contract/classes/duckdb_backed_adapter.py | 5 + .../contract/classes/strict_adapter.py | 5 + .../bigquery/classes/bigquery_adapter.py | 3 + .../databricks/classes/databricks_adapter.py | 3 + .../postgres/classes/postgres_adapter.py | 5 + .../snowflake/classes/snowflake_adapter.py | 5 + .../sqlserver/classes/sqlserver_adapter.py | 3 + .../_helpers/analysis/fixture_columns.py | 151 ++++++++++++++++++ .../compile/main/_infer_fixture_columns.py | 106 +----------- src/sqlbuild/compiler/compile/models.py | 9 ++ .../planner/_helpers/fixtures/completion.py | 78 ++++++++- .../_helpers/sql_tests/fixture_validation.py | 6 +- .../cli/commands/main/test/helpers.py | 36 +++++ .../cli/commands/main/test/test_test.py | 10 ++ .../planner/_helpers/test_fixture_planning.py | 68 ++++++++ .../adapters/snowflake/test_client.py | 21 +++ .../compiler/planner/_helpers/_test_types.py | 2 + .../_helpers/test_fixture_planning_work.py | 69 +++++++- 19 files changed, 482 insertions(+), 108 deletions(-) create mode 100644 src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index 2d98e9ae44..174f4b76a0 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -1954,6 +1954,11 @@ def render_identifier(self, name: str) -> str: return '"' + name.replace('"', '""') + '"' + def render_exact_identifier(self, name: str) -> str: + """Render one explicitly quoted identifier without changing its logical spelling.""" + + return self.render_identifier(name) + def render_framework_type(self, type_name: FrameworkType) -> str: """Render one framework-internal logical type using generic SQL defaults.""" diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 0d9fc65fb9..31f77c7009 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -291,6 +291,11 @@ def render_identifier(self, name: str) -> str: return '"' + name.replace('"', '""') + '"' + def render_exact_identifier(self, name: str) -> str: + """Render one explicitly quoted DuckDB identifier with its exact case.""" + + return self.render_identifier(name) + def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: match type_name: case LoaderLogicalType.BOOLEAN: diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index b349568fd8..e76502d06d 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -680,6 +680,11 @@ def render_identifier(self, name: str) -> str: """Render one SQL identifier for this adapter.""" ... + @abstractmethod + def render_exact_identifier(self, name: str) -> str: + """Render one explicitly quoted identifier without changing its logical spelling.""" + ... + @abstractmethod def render_framework_type(self, type_name: FrameworkType) -> str: """Render one framework-internal logical type for this adapter.""" diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 28b1f00a0c..d37debc4bd 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -972,6 +972,9 @@ def _bigquery_loader_rows_projection_sql( def render_identifier(self, name: str) -> str: return "`" + name.replace("`", "``") + "`" + def render_exact_identifier(self, name: str) -> str: + return self.render_identifier(name) + def _quote_sql_string(self, value: str) -> str: return "'" + value.replace("'", "''") + "'" diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 66ba6f9379..743883bbc2 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -728,6 +728,9 @@ def sql_analysis_dialect(self) -> str | None: def render_identifier(self, name: str) -> str: return "`" + name.replace("`", "``") + "`" + def render_exact_identifier(self, name: str) -> str: + return self.render_identifier(name) + def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: match type_name: case LoaderLogicalType.BOOLEAN: diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index f77aad4ff0..6b123118f8 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -1660,6 +1660,11 @@ def render_identifier(self, name: str) -> str: return '"' + name.replace('"', '""') + '"' + def render_exact_identifier(self, name: str) -> str: + """Render one explicitly quoted PostgreSQL identifier with its exact case.""" + + return self.render_identifier(name) + def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: match type_name: case LoaderLogicalType.BOOLEAN: diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 5f4aa7702f..7bd0f37264 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -1064,6 +1064,11 @@ def render_identifier(self, name: str) -> str: return '"' + name.upper().replace('"', '""') + '"' + def render_exact_identifier(self, name: str) -> str: + """Render one explicitly quoted Snowflake identifier with its exact case.""" + + return '"' + name.replace('"', '""') + '"' + def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: match type_name: case LoaderLogicalType.BOOLEAN: diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index 09755ece86..4e60c013e1 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -954,6 +954,9 @@ def merge( def render_identifier(self, name: str) -> str: return "[" + name.replace("]", "]]") + "]" + def render_exact_identifier(self, name: str) -> str: + return self.render_identifier(name) + def render_framework_type(self, type_name: FrameworkType) -> str: match type_name: case FrameworkType.STRING: diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py b/src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py new file mode 100644 index 0000000000..78f0be56cd --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py @@ -0,0 +1,151 @@ +"""Fixture-column inference implementation.""" + +from __future__ import annotations + +from typing import Any, cast + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile._helpers.analysis.columns import ( + analyze_columns_and_lineage_with_polyglot, +) +from sqlbuild.compiler.compile.constants import ( + POLYGLOT_ARRAY_KIND, + POLYGLOT_FUNCTION_KIND, + POLYGLOT_LITERAL_KIND, + POLYGLOT_STRUCTURED_KINDS, +) +from sqlbuild.compiler.compile.models import ( + FixtureColumnInference, + InferredColumn, + PolyglotAnalysisResult, +) +from sqlbuild.compiler.sql_analysis.constants import POLYGLOT_KIND_NULL +from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql + + +def infer_fixture_column_facts( + *, query_sql: str, inference_profile: ExpressionInferenceProfile +) -> FixtureColumnInference | None: + """Infer fixture columns and identify explicitly untyped NULL projections.""" + + analysis: PolyglotAnalysisResult = analyze_columns_and_lineage_with_polyglot( + query_sql=query_sql, + inference_profile=inference_profile, + ) + if not analysis.analysis_succeeded or analysis.has_star or analysis.columns is None: + return None + simple_types, null_literal_names, quoted_names = _simple_projection_facts( + query_sql=query_sql, + dialect=inference_profile.sql_analysis_dialect, + ) + return FixtureColumnInference( + columns=tuple( + InferredColumn( + name=column.name, + type=column.type or simple_types.get(column.name.casefold()), + nullability=column.nullability, + ) + for column in analysis.columns + ), + null_literal_names=null_literal_names, + quoted_names=quoted_names, + ) + + +def _simple_projection_facts( + *, query_sql: str, dialect: str | None +) -> tuple[dict[str, str], frozenset[str], frozenset[str]]: + polyglot_sql: Any = import_polyglot_sql() + try: + payload: dict[str, Any] = polyglot_sql.parse_one( + query_sql, dialect=dialect or "generic" + ).to_dict() + except polyglot_sql.PolyglotError: + return {}, frozenset(), frozenset() + direct_select_payload: object = payload.get("select") + select_payload: dict[str, Any] | None = _first_select(payload) + if select_payload is None: + return {}, frozenset(), frozenset() + types, null_literal_names, quoted_names = _collect_projection_facts( + select_payload=select_payload + ) + effective_null_literal_names: frozenset[str] = ( + null_literal_names if isinstance(direct_select_payload, dict) else frozenset() + ) + return types, effective_null_literal_names, quoted_names + + +def _collect_projection_facts( + *, select_payload: dict[str, Any] +) -> tuple[dict[str, str], frozenset[str], frozenset[str]]: + types: dict[str, str] = {} + null_literal_names: set[str] = set() + quoted_names: set[str] = set() + for expression in select_payload.get("expressions", []): + alias_payload: object = expression.get("alias") if isinstance(expression, dict) else None + if not isinstance(alias_payload, dict): + continue + alias: object = alias_payload.get("alias") + value: object = alias_payload.get("this") + if not isinstance(alias, dict) or not isinstance(value, dict): + continue + name: object = alias.get("name") + if not isinstance(name, str): + continue + if alias.get("quoted") is True: + quoted_names.add(name.casefold()) + raw_value_kind: object = next(iter(value), "") + if not isinstance(raw_value_kind, str): + continue + value_payload: object = value.get(raw_value_kind) + if raw_value_kind == POLYGLOT_KIND_NULL: + null_literal_names.add(name.casefold()) + continue + inferred_type: str | None = _simple_expression_type( + value_kind=raw_value_kind, + value_payload=value_payload, + ) + if inferred_type is not None: + types[name.casefold()] = inferred_type + return types, frozenset(null_literal_names), frozenset(quoted_names) + + +def _simple_expression_type(*, value_kind: str, value_payload: object) -> str | None: + if value_kind == POLYGLOT_LITERAL_KIND and isinstance(value_payload, dict): + value_dict: dict[object, object] = cast(dict[object, object], value_payload) + return { + "string": "VARCHAR", + "number": "NUMBER", + "boolean": "BOOLEAN", + }.get(str(value_dict.get("literal_type"))) + if value_kind == POLYGLOT_ARRAY_KIND: + return "ARRAY" + if value_kind in POLYGLOT_STRUCTURED_KINDS: + return "VARIANT" + if value_kind == POLYGLOT_FUNCTION_KIND and isinstance(value_payload, dict): + value_dict = cast(dict[object, object], value_payload) + function_name: str = str(value_dict.get("name") or "").upper() + return { + "ARRAY_CONSTRUCT": "ARRAY", + "OBJECT_CONSTRUCT": "OBJECT", + "PARSE_JSON": "VARIANT", + }.get(function_name) + return None + + +def _first_select(payload: object) -> dict[str, Any] | None: + if isinstance(payload, dict): + payload_dict: dict[object, object] = cast(dict[object, object], payload) + select_payload: object = payload_dict.get("select") + if isinstance(select_payload, dict): + return cast(dict[str, Any], select_payload) + for value in payload_dict.values(): + found: dict[str, Any] | None = _first_select(value) + if found is not None: + return found + if isinstance(payload, list): + for value in payload: + found = _first_select(value) + if found is not None: + return found + return None diff --git a/src/sqlbuild/compiler/compile/main/_infer_fixture_columns.py b/src/sqlbuild/compiler/compile/main/_infer_fixture_columns.py index 0417802d17..b07f9e25de 100644 --- a/src/sqlbuild/compiler/compile/main/_infer_fixture_columns.py +++ b/src/sqlbuild/compiler/compile/main/_infer_fixture_columns.py @@ -2,111 +2,19 @@ from __future__ import annotations -from typing import Any, cast - from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile._helpers.analysis.columns import ( - analyze_columns_and_lineage_with_polyglot, -) -from sqlbuild.compiler.compile.constants import ( - POLYGLOT_ARRAY_KIND, - POLYGLOT_FUNCTION_KIND, - POLYGLOT_LITERAL_KIND, - POLYGLOT_STRUCTURED_KINDS, +from sqlbuild.compiler.compile._helpers.analysis.fixture_columns import ( + infer_fixture_column_facts as _infer_fixture_column_facts, ) -from sqlbuild.compiler.compile.models import InferredColumn, PolyglotAnalysisResult -from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql +from sqlbuild.compiler.compile.models import FixtureColumnInference -def infer_fixture_columns( +def infer_fixture_column_facts( *, query_sql: str, inference_profile: ExpressionInferenceProfile -) -> tuple[InferredColumn, ...] | None: - """Infer fixture columns and conservative simple-literal types without warehouse access.""" +) -> FixtureColumnInference | None: + """Infer fixture columns and identify explicitly untyped NULL projections.""" - analysis: PolyglotAnalysisResult = analyze_columns_and_lineage_with_polyglot( + return _infer_fixture_column_facts( query_sql=query_sql, inference_profile=inference_profile, ) - if not analysis.analysis_succeeded or analysis.has_star or analysis.columns is None: - return None - simple_types: dict[str, str] = _simple_projection_types( - query_sql=query_sql, - dialect=inference_profile.sql_analysis_dialect, - ) - return tuple( - InferredColumn( - name=column.name, - type=column.type or simple_types.get(column.name.casefold()), - nullability=column.nullability, - ) - for column in analysis.columns - ) - - -def _simple_projection_types(*, query_sql: str, dialect: str | None) -> dict[str, str]: - polyglot_sql: Any = import_polyglot_sql() - try: - payload: dict[str, Any] = polyglot_sql.parse_one( - query_sql, dialect=dialect or "generic" - ).to_dict() - except polyglot_sql.PolyglotError: - return {} - select_payload: dict[str, Any] | None = _first_select(payload) - if select_payload is None: - return {} - result: dict[str, str] = {} - for expression in select_payload.get("expressions", []): - alias_payload: object = expression.get("alias") if isinstance(expression, dict) else None - if not isinstance(alias_payload, dict): - continue - alias: object = alias_payload.get("alias") - value: object = alias_payload.get("this") - if not isinstance(alias, dict) or not isinstance(value, dict): - continue - name: object = alias.get("name") - if not isinstance(name, str): - continue - raw_value_kind: object = next(iter(value), "") - if not isinstance(raw_value_kind, str): - continue - value_kind: str = raw_value_kind - value_payload: object = value.get(value_kind) - inferred_type: str | None = None - if value_kind == POLYGLOT_LITERAL_KIND and isinstance(value_payload, dict): - inferred_type = { - "string": "VARCHAR", - "number": "NUMBER", - "boolean": "BOOLEAN", - }.get(str(value_payload.get("literal_type"))) - elif value_kind == POLYGLOT_ARRAY_KIND: - inferred_type = "ARRAY" - elif value_kind in POLYGLOT_STRUCTURED_KINDS: - inferred_type = "VARIANT" - elif value_kind == POLYGLOT_FUNCTION_KIND and isinstance(value_payload, dict): - function_name: str = str(value_payload.get("name") or "").upper() - inferred_type = { - "ARRAY_CONSTRUCT": "ARRAY", - "OBJECT_CONSTRUCT": "OBJECT", - "PARSE_JSON": "VARIANT", - }.get(function_name) - if inferred_type is not None: - result[name.casefold()] = inferred_type - return result - - -def _first_select(payload: object) -> dict[str, Any] | None: - if isinstance(payload, dict): - payload_dict: dict[object, object] = cast(dict[object, object], payload) - select_payload: object = payload_dict.get("select") - if isinstance(select_payload, dict): - return cast(dict[str, Any], select_payload) - for value in payload_dict.values(): - found: dict[str, Any] | None = _first_select(value) - if found is not None: - return found - if isinstance(payload, list): - for value in payload: - found = _first_select(value) - if found is not None: - return found - return None diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 999eabe754..752d8de490 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -144,6 +144,15 @@ class InferredColumn: nullability: InferredNullability = InferredNullability.UNKNOWN +@dataclass(frozen=True) +class FixtureColumnInference: + """Inferred fixture columns plus outputs that are explicitly bare NULL literals.""" + + columns: tuple[InferredColumn, ...] + null_literal_names: frozenset[str] + quoted_names: frozenset[str] + + @dataclass(frozen=True) class DynamicColumnFamilyProof: """Compiler evidence for one declared runtime-generated column family.""" diff --git a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py b/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py index bb398968df..fd7b9d405a 100644 --- a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py +++ b/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py @@ -10,7 +10,7 @@ from sqlbuild.compiler.compile.main._analyze_columns_and_lineage import ( analyze_resolved_column_reads, ) -from sqlbuild.compiler.compile.main._infer_fixture_columns import infer_fixture_columns +from sqlbuild.compiler.compile.main._infer_fixture_columns import infer_fixture_column_facts from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, CompiledLineageSourceFact, @@ -18,6 +18,7 @@ CompiledProject, CompileSqlReference, DynamicColumnContractProof, + FixtureColumnInference, InferredColumn, ) from sqlbuild.compiler.compile.types import CompiledResourceType @@ -58,16 +59,20 @@ def build_relation_fixture_completion( model_map: dict[str, CompiledModel] = context.models_by_name fixture_sql_by_key: dict[FixtureKey, str] = {} inferred_by_fixture: dict[FixtureKey, tuple[InferredColumn, ...]] = {} + null_literal_names_by_fixture: dict[FixtureKey, frozenset[str]] = {} + quoted_names_by_fixture: dict[FixtureKey, frozenset[str]] = {} for resource_type, fixtures in fixture_groups: for name, sql in fixtures.items(): key: FixtureKey = (resource_type, name) fixture_sql_by_key[key] = sql - inferred: tuple[InferredColumn, ...] | None = infer_fixture_columns( + inference: FixtureColumnInference | None = infer_fixture_column_facts( query_sql=sql, inference_profile=adapter.expression_inference_profile(), ) - if inferred is not None: - inferred_by_fixture[key] = inferred + if inference is not None: + inferred_by_fixture[key] = inference.columns + null_literal_names_by_fixture[key] = inference.null_literal_names + quoted_names_by_fixture[key] = inference.quoted_names relations: dict[FixtureKey, FixtureRelationMetadata] = context.relations fixture_keys_requiring_analysis: frozenset[FixtureKey] = _fixture_keys_requiring_analysis( @@ -89,6 +94,10 @@ def build_relation_fixture_completion( ordered_model_names=ordered_model_names, model_map=model_map, ) + typed_nulls_by_fixture: dict[FixtureKey, dict[str, str]] = _typed_null_fixture_types( + null_literal_names_by_fixture=null_literal_names_by_fixture, + expected_types=expected_types, + ) diagnostics: list[RelationFixtureDiagnostic] = list( _unknown_fixture_column_diagnostics( inferred_by_fixture=inferred_by_fixture, @@ -96,6 +105,7 @@ def build_relation_fixture_completion( ) ) + completed_keys: set[FixtureKey] = set() for key, required_columns in required.items(): inferred: tuple[InferredColumn, ...] | None = inferred_by_fixture.get(key) if inferred is None: @@ -104,7 +114,8 @@ def build_relation_fixture_completion( missing: tuple[str, ...] = tuple( sorted(name for name in required_columns if name.casefold() not in available_names) ) - if not missing: + typed_nulls: dict[str, str] = typed_nulls_by_fixture.get(key, {}) + if not missing and not typed_nulls: continue relation: FixtureRelationMetadata | None = relations.get(key) metadata: tuple[FixtureColumnMetadata, ...] = ( @@ -172,9 +183,26 @@ def build_relation_fixture_completion( nullable_columns.sort(key=lambda column: metadata_order[column.name.casefold()]) fixture_sql_by_key[key] = _completed_fixture_sql( sql=fixture_sql_by_key[key], + inferred=inferred, + typed_nulls=typed_nulls, + quoted_names=quoted_names_by_fixture.get(key, frozenset()), completed=tuple(nullable_columns), adapter=adapter, ) + completed_keys.add(key) + + for key, typed_nulls in typed_nulls_by_fixture.items(): + if key in completed_keys: + continue + inferred = inferred_by_fixture[key] + fixture_sql_by_key[key] = _completed_fixture_sql( + sql=fixture_sql_by_key[key], + inferred=inferred, + typed_nulls=typed_nulls, + quoted_names=quoted_names_by_fixture.get(key, frozenset()), + completed=(), + adapter=adapter, + ) return RelationFixtureCompletion( fixture_sql_by_key=fixture_sql_by_key, @@ -184,6 +212,24 @@ def build_relation_fixture_completion( ) +def _typed_null_fixture_types( + *, + null_literal_names_by_fixture: dict[FixtureKey, frozenset[str]], + expected_types: dict[FixtureKey, dict[str, str]], +) -> dict[FixtureKey, dict[str, str]]: + result: dict[FixtureKey, dict[str, str]] = {} + for key, null_literal_names in null_literal_names_by_fixture.items(): + fixture_types: dict[str, str] = {} + expected_fixture_types: dict[str, str] = expected_types.get(key, {}) + for name in null_literal_names: + expected_type: str | None = expected_fixture_types.get(name) + if expected_type is not None: + fixture_types[name] = expected_type + if fixture_types: + result[key] = fixture_types + return result + + def _fixture_keys_requiring_analysis( *, inferred_by_fixture: dict[FixtureKey, tuple[InferredColumn, ...]], @@ -607,11 +653,31 @@ def _supplied_columns_form_schema_prefix( def _completed_fixture_sql( *, sql: str, + inferred: tuple[InferredColumn, ...], + typed_nulls: dict[str, str], + quoted_names: frozenset[str], completed: tuple[FixtureColumnMetadata, ...], adapter: BaseAdapter, ) -> str: alias: str = adapter.render_identifier(_PARTIAL_FIXTURE_ALIAS) - projections: list[str] = [f"{alias}.*"] + projections: list[str] + if typed_nulls: + projections = [] + for supplied_column in inferred: + rendered_name: str = ( + adapter.render_exact_identifier(supplied_column.name) + if supplied_column.name.casefold() in quoted_names + else supplied_column.name + ) + source_expression: str = f"{alias}.{rendered_name}" + expected_type: str | None = typed_nulls.get(supplied_column.name.casefold()) + projections.append( + source_expression + if expected_type is None + else f"CAST({source_expression} AS {expected_type}) AS {rendered_name}" + ) + else: + projections = [f"{alias}.*"] for completed_column in completed: projections.append( f"CAST(NULL AS {completed_column.type}) AS " diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py index 4561c6da77..a3876e350a 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py @@ -5,10 +5,11 @@ import re from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.main._infer_fixture_columns import infer_fixture_columns +from sqlbuild.compiler.compile.main._infer_fixture_columns import infer_fixture_column_facts from sqlbuild.compiler.compile.models import ( CompiledProject, CompiledSqlTest, + FixtureColumnInference, InferredColumn, ) from sqlbuild.compiler.compile.types import CompiledResourceType @@ -65,10 +66,11 @@ def build_validated_test_fixtures( for name, sql in fixtures.items(): inferred_columns: tuple[InferredColumn, ...] | None if resource_type == CompiledResourceType.SQL_TEST: - inferred_columns = infer_fixture_columns( + inference: FixtureColumnInference | None = infer_fixture_column_facts( query_sql=sql, inference_profile=adapter.expression_inference_profile(), ) + inferred_columns = inference.columns if inference is not None else None else: inferred_columns = completion.inferred_by_fixture.get((resource_type, name)) if inferred_columns is None: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index 378d464141..cac701b55d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -570,6 +570,42 @@ def build_explicit_typed_null_fixture_project_files() -> dict[str, str]: return files +def build_untyped_null_fixture_project_files() -> dict[str, str]: + """Build a complete empty fixture whose bare null needs its contract type.""" + + return { + "sqlbuild_project.toml": ( + 'name = "untyped_null_fixture"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + 'database = "untyped_null_fixture.duckdb"\n' + ), + "models/orders.sql": ( + "MODEL ();\n\n" + "SELECT EXTRACT(YEAR FROM ordered_at) AS order_year\n" + 'FROM __source("raw_orders")\n' + ), + "sources/raw_orders.yml": ( + "sources:\n" + " - name: raw_orders\n" + " schema: main\n" + " table: raw_orders\n" + " contract: enforced\n" + " columns:\n" + " - name: ordered_at\n type: DATE\n nullable: true\n" + ), + "tests/unit/test_orders.sql": ( + "TEST();\n\n" + "WITH\n" + "__source__raw_orders AS (SELECT NULL AS ordered_at WHERE FALSE),\n" + "__expected__orders AS (\n" + " SELECT CAST(NULL AS BIGINT) AS order_year WHERE FALSE\n" + ")\n" + "SELECT 1\n" + ), + } + + def build_qualified_star_other_relation_project_files() -> dict[str, str]: """Build a qualified star that does not target the only mocked relation.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py index c305282c66..f9c908ff7b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py @@ -51,6 +51,7 @@ build_transformed_collection_project_files, build_unsatisfied_leaf_test_project_files, build_unspecified_nullability_fixture_project_files, + build_untyped_null_fixture_project_files, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( assert_fragments_in_order, @@ -151,6 +152,15 @@ expected_artifact_fragments=("CAST(NULL AS TEXT) AS status",), unexpected_artifact_fragments=("__sqlbuild_partial_fixture",), ), + PartialFixtureE2ETestCase( + description="supplied bare null receives authoritative fixture type", + repo_files=build_untyped_null_fixture_project_files(), + expected_stdout_fragment="PASS=1", + expected_artifact_fragments=( + 'CAST("__sqlbuild_partial_fixture".ordered_at AS DATE) AS ordered_at', + "__sqlbuild_partial_fixture", + ), + ), PartialFixtureE2ETestCase( description="qualified star on real relation does not expand mocked relation", repo_files=build_qualified_star_other_relation_project_files(), diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py index fcd38e6410..2542323465 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning.py @@ -82,5 +82,73 @@ def test_given_partial_fixture_when_compiling_and_planning_then_completes_requir assert test_case.expected_sql_fragment in entry.chain[0].resolved_sql +@pytest.mark.parametrize( + "test_case", + ( + FixturePlanningIntegrationTestCase( + description="supplied null-only column receives its authoritative type", + expected_sql_fragment=( + 'CAST("__sqlbuild_partial_fixture".status AS VARCHAR) AS status' + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_untyped_null_fixture_when_planning_then_applies_authoritative_type( + test_case: FixturePlanningIntegrationTestCase, + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], +) -> None: + write_repo_files( + tmp_path, + { + "sqlbuild_project.toml": ( + 'name = "fixture_type_demo"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + 'database = ":memory:"\n' + ), + "models/orders.sql": ( + "MODEL ();\n\nSELECT TRY_CAST(status AS DOUBLE) AS parsed_status " + 'FROM __source("raw_orders")\n' + ), + "sources/raw_orders.yml": ( + "sources:\n" + " - name: raw_orders\n" + " expression: SELECT '1.5' AS status\n" + " contract: enforced\n" + " columns:\n" + " - name: status\n type: VARCHAR\n nullable: true\n" + ), + "tests/unit/test_orders.sql": ( + "TEST();\n\n" + "WITH\n" + "__source__raw_orders AS (SELECT NULL AS status WHERE FALSE),\n" + "__expected__orders AS (\n" + " SELECT CAST(NULL AS DOUBLE) AS parsed_status WHERE FALSE\n" + ")\n" + "SELECT 1\n" + ), + }, + ) + discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + adapter: DuckDbAdapter = DuckDbAdapter() + + project: CompiledProject = compile_project( + discovered_inputs=discovered_inputs, + adapter=adapter, + ) + entries, warnings = build_selected_test_entries( + project=project, + adapter=adapter, + selected_keys=frozenset((project.models[0].key,)), + ) + + assert warnings == [] + assert len(entries) == 1 + entry: SqlTestPlanEntry = entries[0] + assert test_case.expected_sql_fragment in entry.chain[0].resolved_sql + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py index facd416002..4014ff176d 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py @@ -839,6 +839,27 @@ def test_given_identifier_when_rendering_then_snowflake_quotes_uppercase_identif assert identifier == test_case.expected_identifier +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeRenderIdentifierTestCase( + description="preserves explicitly quoted mixed-case identifier", + name="Order_ID", + expected_identifier='"Order_ID"', + ), + ], + ids=lambda case: case.description, +) +def test_given_exact_identifier_when_rendering_then_snowflake_preserves_case( + test_case: SnowflakeRenderIdentifierTestCase, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + + identifier: str = adapter.render_exact_identifier(test_case.name) + + assert identifier == test_case.expected_identifier + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 505aa80a30..2882afb428 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -58,7 +58,9 @@ class FixturePlanningWorkTestCase: @dataclass(frozen=True) class CompleteFixturePlanningTestCase: description: str + fixture_sql: str expected_analysis_calls: int + expected_inference_calls: int expected_fixture_sql: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index 374d6684ef..ec8d96b5bb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -123,9 +123,69 @@ def test_given_many_tests_when_building_entries_then_project_fixture_metadata_is ( CompleteFixturePlanningTestCase( description="complete authoritative fixture", + fixture_sql="SELECT 1 AS order_id", expected_analysis_calls=0, + expected_inference_calls=1, expected_fixture_sql="SELECT 1 AS order_id", ), + CompleteFixturePlanningTestCase( + description="explicit typed null fixture", + fixture_sql="SELECT CAST(NULL AS INTEGER) AS order_id WHERE FALSE", + expected_analysis_calls=0, + expected_inference_calls=1, + expected_fixture_sql="SELECT CAST(NULL AS INTEGER) AS order_id WHERE FALSE", + ), + CompleteFixturePlanningTestCase( + description="untyped null fixture", + fixture_sql="SELECT NULL AS order_id WHERE FALSE", + expected_analysis_calls=0, + expected_inference_calls=1, + expected_fixture_sql=( + 'SELECT\n CAST("__sqlbuild_partial_fixture".order_id AS INTEGER) ' + "AS order_id\nFROM (\nSELECT NULL AS order_id WHERE FALSE\n) " + 'AS "__sqlbuild_partial_fixture"' + ), + ), + CompleteFixturePlanningTestCase( + description="untyped null before typed union value", + fixture_sql=( + "SELECT NULL AS order_id UNION ALL SELECT CAST(1.5 AS DOUBLE) AS order_id" + ), + expected_analysis_calls=0, + expected_inference_calls=1, + expected_fixture_sql=( + "SELECT NULL AS order_id UNION ALL SELECT CAST(1.5 AS DOUBLE) AS order_id" + ), + ), + CompleteFixturePlanningTestCase( + description="unquoted mixed-case null fixture", + fixture_sql="SELECT NULL AS Order_ID WHERE FALSE", + expected_analysis_calls=0, + expected_inference_calls=1, + expected_fixture_sql=( + 'SELECT\n CAST("__sqlbuild_partial_fixture".Order_ID AS INTEGER) ' + "AS Order_ID\nFROM (\nSELECT NULL AS Order_ID WHERE FALSE\n) " + 'AS "__sqlbuild_partial_fixture"' + ), + ), + CompleteFixturePlanningTestCase( + description="quoted mixed-case null fixture", + fixture_sql='SELECT NULL AS "Order_ID" WHERE FALSE', + expected_analysis_calls=0, + expected_inference_calls=1, + expected_fixture_sql=( + 'SELECT\n CAST("__sqlbuild_partial_fixture"."Order_ID" AS INTEGER) ' + 'AS "Order_ID"\nFROM (\nSELECT NULL AS "Order_ID" WHERE FALSE\n) ' + 'AS "__sqlbuild_partial_fixture"' + ), + ), + CompleteFixturePlanningTestCase( + description="unknown non-null expression fixture", + fixture_sql="SELECT custom_value() AS order_id", + expected_analysis_calls=0, + expected_inference_calls=1, + expected_fixture_sql="SELECT custom_value() AS order_id", + ), ), ids=lambda case: case.description, ) @@ -175,18 +235,25 @@ def test_given_complete_authoritative_fixture_when_completing_then_skips_column_ "analyze_resolved_column_reads", fallback_analyzer, ) + fixture_inference: Mock = Mock(wraps=fixture_completion.infer_fixture_column_facts) + monkeypatch.setattr( + fixture_completion, + "infer_fixture_column_facts", + fixture_inference, + ) completed: RelationFixtureCompletion = fixture_completion.build_relation_fixture_completion( project=replace(project, models=(model,)), adapter=PlannerTestAdapter(), ordered_model_names=(model.name,), - fixture_groups=((CompiledResourceType.SOURCE, {"raw_orders": "SELECT 1 AS order_id"}),), + fixture_groups=((CompiledResourceType.SOURCE, {"raw_orders": test_case.fixture_sql}),), planning_context=context, ) assert completed.diagnostics == () assert completed.fixture_sql_by_key[source_key] == test_case.expected_fixture_sql assert fallback_analyzer.call_count == test_case.expected_analysis_calls + assert fixture_inference.call_count == test_case.expected_inference_calls if __name__ == "__main__":