From 26627c2a7805c3149ac5aa638f5593fc127c3b7d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 21 Sep 2026 04:52:52 +0100 Subject: [PATCH] feat: add contract-driven test fixtures --- README.md | 6 + .../cli/commands/main/project/_format.py | 50 ++- .../compile/_helpers/sql_tests/core.py | 15 +- .../_helpers/sql_tests/empty_fixture.py | 16 + .../planner/_helpers/fixtures/completion.py | 161 ++++++++-- .../_helpers/fixtures/empty_fixture.py | 14 + .../planner/_helpers/sql_tests/assembly.py | 2 +- .../_helpers/sql_tests/fixture_formatting.py | 272 +++++++++++++++++ .../_helpers/sql_tests/fixture_validation.py | 34 ++- .../planner/classes/fixture_null_autofix.py | 32 ++ .../lint/_helpers/fixture_null_candidates.py | 30 ++ .../main/has_fixture_typed_null_candidates.py | 13 + src/sqlbuild/lint/main/run_format.py | 36 ++- .../cli/commands/main/test/helpers.py | 68 ++++- .../cli/commands/main/test/test_test.py | 31 ++ .../compiler/compile/_helpers/test_tests.py | 23 ++ tests/unit/src/sqlbuild/lint/helpers.py | 42 +++ .../unit/src/sqlbuild/lint/test_cli_format.py | 289 ++++++++++++++++++ 18 files changed, 1078 insertions(+), 56 deletions(-) create mode 100644 src/sqlbuild/compiler/compile/_helpers/sql_tests/empty_fixture.py create mode 100644 src/sqlbuild/compiler/planner/_helpers/fixtures/empty_fixture.py create mode 100644 src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py create mode 100644 src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py create mode 100644 src/sqlbuild/lint/_helpers/fixture_null_candidates.py create mode 100644 src/sqlbuild/lint/main/has_fixture_typed_null_candidates.py create mode 100644 tests/unit/src/sqlbuild/lint/helpers.py diff --git a/README.md b/README.md index 2e3f9c71a6..35bff33ae7 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,12 @@ model SQL or warehouse defaults. Required columns declared with `nullable false` unknown types must be supplied explicitly. When the relation's complete column set is authoritative, misspelled or unknown supplied fixture columns are rejected. +A direct `NULL AS column_name` projection receives the authoritative relation type in compiled test +SQL, including expected-output CTEs for contracted models. To represent a contracted upstream with +no rows, use `SELECT * FROM __empty_fixture()` as the complete body of a `__ref__`, `__source__`, or +`__seed__` fixture CTE, or a contracted `__expected__` CTE. SQLBuild expands it to the relation's +full typed schema with a false filter. + See the [documentation](https://docs.sqlbuild.com) for incremental models, scenarios, loaders, and more. ### Python project layout diff --git a/src/sqlbuild/cli/commands/main/project/_format.py b/src/sqlbuild/cli/commands/main/project/_format.py index e1afee35a7..15df1bb9a8 100644 --- a/src/sqlbuild/cli/commands/main/project/_format.py +++ b/src/sqlbuild/cli/commands/main/project/_format.py @@ -6,13 +6,18 @@ from dataclasses import replace from pathlib import Path +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.commands._helpers.lint.runs import ( prepare_lint_run, render_lint_result, render_lint_result_json, ) from sqlbuild.cli.commands._helpers.lint.selection import resolve_lint_inputs -from sqlbuild.compiler.compile.types import TypedSqlValueRenderer +from sqlbuild.compiler.pipeline.main.graph import build_project_graph +from sqlbuild.compiler.pipeline.models import ProjectGraph +from sqlbuild.lint.main.has_fixture_typed_null_candidates import ( + has_fixture_typed_null_candidates, +) from sqlbuild.lint.main.run_format import run_format from sqlbuild.lint.models import LintConfig, LintRunResult from sqlbuild.presentation.main.supports_color import supports_color @@ -34,24 +39,43 @@ def run_format_command( prepared: tuple[LintConfig, str | None] = prepare_lint_run(project_dir=base_dir) if prepared[1] is not None: print(f"WARN {prepared[1]}") - value_renderer: TypedSqlValueRenderer | None = None - selected_paths: frozenset[Path] | None = None config: LintConfig = prepared[0] - if select or exclude: - value_renderer, selected_paths, _discovered_inputs = resolve_lint_inputs( - project_dir=base_dir, - select=select, - exclude=exclude, - ) - config = replace( - config, - dialect=value_renderer.sql_analysis_dialect_name or "generic", - ) + value_renderer: BaseAdapter | None = None + selected_paths: frozenset[Path] | None = None + graph: ProjectGraph | None = None + has_fixture_candidates: bool = has_fixture_typed_null_candidates(project_dir=base_dir) + if select or exclude or has_fixture_candidates: + try: + value_renderer, selected_paths, discovered_inputs = resolve_lint_inputs( + project_dir=base_dir, + select=select, + exclude=exclude, + ) + except Exception: + if select or exclude: + raise + else: + config = replace( + config, + dialect=value_renderer.sql_analysis_dialect_name or "generic", + ) + if has_fixture_candidates and selected_paths is None: + try: + graph = build_project_graph( + discovered_inputs=discovered_inputs, + adapter=value_renderer, + ) + except Exception: + graph = None result: LintRunResult = run_format( project_dir=base_dir, config=config, value_renderer=value_renderer, selected_paths=selected_paths, + compiled_project=( + graph.project if graph is not None and graph.project.settings.sql_analysis else None + ), + adapter=value_renderer, write=not (check or diff), ) if diff: diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py index 838063da54..ebd5de1363 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py @@ -13,6 +13,7 @@ ) from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references from sqlbuild.compiler.compile._helpers.render.macros import find_macro_call_names +from sqlbuild.compiler.compile._helpers.sql_tests.empty_fixture import is_empty_fixture_query from sqlbuild.compiler.compile.constants import ( ASSERT_TEST_CTE_PREFIX, DBT_REF_TEST_CTE_PREFIX, @@ -489,7 +490,11 @@ def _classify_model_sql_test_ctes( file_label=file_label, ) ) - _validate_expected_cte_query(cte=cte, file_label=file_label) + _validate_expected_cte_query( + cte=cte, + file_label=file_label, + allow_empty_fixture=True, + ) expected_ctes.append(cte) continue if cte.name.startswith(ASSERT_TEST_CTE_PREFIX): @@ -736,8 +741,14 @@ def _read_sql_string_literal(*, sql: str, start: int) -> tuple[str, int]: def _validate_expected_cte_query( - *, cte: CompileSqlTestCte, file_label: str, label: str = "__expected__" + *, + cte: CompileSqlTestCte, + file_label: str, + label: str = "__expected__", + allow_empty_fixture: bool = False, ) -> None: + if allow_empty_fixture and is_empty_fixture_query(cte.sql_body): + return if _contains_select_star(cte.sql_body): raise CompileInputError(f"SQL test '{file_label}' must not use SELECT * in {label} CTEs") branch_column_names: tuple[tuple[str, ...], ...] = _extract_expected_branch_column_names( diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/empty_fixture.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/empty_fixture.py new file mode 100644 index 0000000000..2b287d3dbd --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/empty_fixture.py @@ -0,0 +1,16 @@ +"""Recognition for the first-class empty relation fixture marker.""" + +from __future__ import annotations + +import re + +_EMPTY_FIXTURE_PATTERN: re.Pattern[str] = re.compile( + r"^\s*SELECT\s+\*\s+FROM\s+__empty_fixture\s*\(\s*\)\s*$", + re.IGNORECASE, +) + + +def is_empty_fixture_query(sql: str) -> bool: + """Return whether SQL is exactly the empty relation fixture marker.""" + + return _EMPTY_FIXTURE_PATTERN.fullmatch(sql) is not None diff --git a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py b/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py index fd7b9d405a..6281ecab86 100644 --- a/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py +++ b/src/sqlbuild/compiler/planner/_helpers/fixtures/completion.py @@ -25,8 +25,8 @@ from sqlbuild.compiler.lineage.types import ( ColumnLineageConfidence, ColumnTransformKind, - InferredNullability, ) +from sqlbuild.compiler.planner._helpers.fixtures.empty_fixture import is_empty_fixture_query from sqlbuild.compiler.planner.models import ( FixtureColumnMetadata, FixtureRelationMetadata, @@ -57,24 +57,24 @@ def build_relation_fixture_completion( project=project ) model_map: dict[str, CompiledModel] = context.models_by_name - fixture_sql_by_key: dict[FixtureKey, str] = {} - inferred_by_fixture: dict[FixtureKey, tuple[InferredColumn, ...]] = {} - null_literal_names_by_fixture: dict[FixtureKey, frozenset[str]] = {} - quoted_names_by_fixture: dict[FixtureKey, frozenset[str]] = {} - for resource_type, fixtures in fixture_groups: - for name, sql in fixtures.items(): - key: FixtureKey = (resource_type, name) - fixture_sql_by_key[key] = sql - inference: FixtureColumnInference | None = infer_fixture_column_facts( - query_sql=sql, - inference_profile=adapter.expression_inference_profile(), - ) - if inference is not None: - inferred_by_fixture[key] = inference.columns - null_literal_names_by_fixture[key] = inference.null_literal_names - quoted_names_by_fixture[key] = inference.quoted_names - relations: dict[FixtureKey, FixtureRelationMetadata] = context.relations + fixture_sql_by_key: dict[FixtureKey, str] + inferred_by_fixture: dict[FixtureKey, tuple[InferredColumn, ...]] + null_literal_names_by_fixture: dict[FixtureKey, frozenset[str]] + quoted_names_by_fixture: dict[FixtureKey, frozenset[str]] + diagnostics: list[RelationFixtureDiagnostic] + ( + fixture_sql_by_key, + inferred_by_fixture, + null_literal_names_by_fixture, + quoted_names_by_fixture, + diagnostics, + ) = _prepare_fixture_inputs( + fixture_groups=fixture_groups, + relations=relations, + adapter=adapter, + ) + fixture_keys_requiring_analysis: frozenset[FixtureKey] = _fixture_keys_requiring_analysis( inferred_by_fixture=inferred_by_fixture, authoritative_columns=context.authoritative_columns, @@ -98,10 +98,9 @@ def build_relation_fixture_completion( null_literal_names_by_fixture=null_literal_names_by_fixture, expected_types=expected_types, ) - diagnostics: list[RelationFixtureDiagnostic] = list( + diagnostics.extend( _unknown_fixture_column_diagnostics( - inferred_by_fixture=inferred_by_fixture, - relations=relations, + inferred_by_fixture=inferred_by_fixture, relations=relations ) ) @@ -212,6 +211,116 @@ def build_relation_fixture_completion( ) +def _prepare_fixture_inputs( + *, + fixture_groups: FixtureGroups, + relations: dict[FixtureKey, FixtureRelationMetadata], + adapter: BaseAdapter, +) -> tuple[ + dict[FixtureKey, str], + dict[FixtureKey, tuple[InferredColumn, ...]], + dict[FixtureKey, frozenset[str]], + dict[FixtureKey, frozenset[str]], + list[RelationFixtureDiagnostic], +]: + fixture_sql_by_key: dict[FixtureKey, str] = {} + inferred_by_fixture: dict[FixtureKey, tuple[InferredColumn, ...]] = {} + null_literal_names_by_fixture: dict[FixtureKey, frozenset[str]] = {} + quoted_names_by_fixture: dict[FixtureKey, frozenset[str]] = {} + diagnostics: list[RelationFixtureDiagnostic] = [] + for resource_type, fixtures in fixture_groups: + for name, sql in fixtures.items(): + key: FixtureKey = (resource_type, name) + fixture_sql: str = sql + if is_empty_fixture_query(fixture_sql): + empty_sql, empty_error = complete_empty_fixture_sql( + sql=fixture_sql, + relation=relations.get(key), + adapter=adapter, + ) + if empty_error is not None: + diagnostics.append( + RelationFixtureDiagnostic( + key=key, + message=f"mock {key[0].value} '{key[1]}' {empty_error}", + ) + ) + fixture_sql_by_key[key] = fixture_sql + continue + fixture_sql = empty_sql + fixture_sql_by_key[key] = fixture_sql + inference: FixtureColumnInference | None = infer_fixture_column_facts( + query_sql=fixture_sql, + inference_profile=adapter.expression_inference_profile(), + ) + if inference is not None: + inferred_by_fixture[key] = inference.columns + null_literal_names_by_fixture[key] = inference.null_literal_names + quoted_names_by_fixture[key] = inference.quoted_names + return ( + fixture_sql_by_key, + inferred_by_fixture, + null_literal_names_by_fixture, + quoted_names_by_fixture, + diagnostics, + ) + + +def complete_typed_null_columns( + *, + sql: str, + inference: FixtureColumnInference, + expected_types: dict[str, str], + adapter: BaseAdapter, +) -> str: + """Apply authoritative types to direct bare-NULL projections.""" + + typed_nulls: dict[str, str] = { + name: expected_types[name] + for name in inference.null_literal_names + if name in expected_types + } + if not typed_nulls: + return sql + return _completed_fixture_sql( + sql=sql, + inferred=inference.columns, + typed_nulls=typed_nulls, + quoted_names=inference.quoted_names, + completed=(), + adapter=adapter, + ) + + +def complete_empty_fixture_sql( + *, + sql: str, + relation: FixtureRelationMetadata | None, + adapter: BaseAdapter, +) -> tuple[str, str | None]: + """Expand an empty-fixture marker from authoritative relation metadata.""" + + if not is_empty_fixture_query(sql): + return sql, None + if relation is None or not relation.authoritative_names: + return sql, "uses __empty_fixture() but its column schema is not authoritative" + unknown_types: tuple[str, ...] = tuple( + column.name for column in relation.columns if column.type is None + ) + if unknown_types: + return ( + sql, + f"uses __empty_fixture() but columns have unknown types: {', '.join(unknown_types)}", + ) + if not relation.columns: + return sql, "uses __empty_fixture() but its schema has no columns" + projections: str = ",\n ".join( + f"CAST(NULL AS {column.type}) AS {adapter.render_identifier(column.name)}" + for column in relation.columns + ) + return f"SELECT\n {projections}\nWHERE FALSE", None + + def _typed_null_fixture_types( *, null_literal_names_by_fixture: dict[FixtureKey, frozenset[str]], @@ -520,7 +629,7 @@ def _model_fixture_metadata(*, model: CompiledModel) -> FixtureRelationMetadata: columns_by_name[key] = FixtureColumnMetadata( name=column.name, type=column.type, - nullable=_inferred_nullable(column.nullability), + nullable=None, ) for column in model.schema_entry.columns if model.schema_entry is not None else (): key = column.name.casefold() @@ -621,14 +730,6 @@ def _unknown_fixture_column_diagnostics( return tuple(diagnostics) -def _inferred_nullable(value: InferredNullability) -> bool | None: - if value == InferredNullability.NULLABLE: - return True - if value == InferredNullability.NON_NULL: - return False - return None - - def _models_reading_columns( *, column_names: tuple[str, ...], required_columns: dict[str, set[str]] ) -> tuple[str, ...]: diff --git a/src/sqlbuild/compiler/planner/_helpers/fixtures/empty_fixture.py b/src/sqlbuild/compiler/planner/_helpers/fixtures/empty_fixture.py new file mode 100644 index 0000000000..868fdbf5ad --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/fixtures/empty_fixture.py @@ -0,0 +1,14 @@ +"""Recognition for the planner's empty relation fixture marker.""" + +import re + +_EMPTY_FIXTURE_PATTERN: re.Pattern[str] = re.compile( + r"^\s*SELECT\s+\*\s+FROM\s+__empty_fixture\s*\(\s*\)\s*$", + re.IGNORECASE, +) + + +def is_empty_fixture_query(sql: str) -> bool: + """Return whether SQL is exactly the empty relation fixture marker.""" + + return _EMPTY_FIXTURE_PATTERN.fullmatch(sql) is not None diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py index 540038ec0d..36dda5c77f 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/assembly.py @@ -240,7 +240,7 @@ def plan_test( mock_ref_names=frozenset(mock_refs), ) if sql_analysis_enabled and validate_fixtures: - mock_refs, mock_sources, mock_seeds = build_validated_test_fixtures( + mock_refs, mock_sources, mock_seeds, expected_map = build_validated_test_fixtures( test=test, project=project, adapter=adapter, diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py new file mode 100644 index 0000000000..f9a9cbccd1 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py @@ -0,0 +1,272 @@ +"""Contract-aware formatting of SQL test fixture projections.""" + +from __future__ import annotations + +import re +from pathlib import Path + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.compiler.compile.constants import ( + REF_TEST_CTE_PREFIX, + SEED_TEST_CTE_PREFIX, + SOURCE_TEST_CTE_PREFIX, +) +from sqlbuild.compiler.compile.main._infer_fixture_columns import infer_fixture_column_facts +from sqlbuild.compiler.compile.models import ( + CompiledModelSqlTestPayload, + CompiledProject, + CompiledSqlTest, + CompileSqlTestCte, + FixtureColumnInference, +) +from sqlbuild.compiler.compile.types import CompiledResourceType, SqlTestMode +from sqlbuild.compiler.planner._helpers.fixtures.completion import ( + build_relation_fixture_completion, + build_relation_fixture_context, +) +from sqlbuild.compiler.planner._helpers.sql_tests.assembly import ( + build_sql_test_planning_context, +) +from sqlbuild.compiler.planner.models import ( + FixtureColumnMetadata, + FixtureRelationMetadata, + RelationFixtureCompletion, + RelationFixturePlanningContext, + SqlTestPlanningContext, +) +from sqlbuild.compiler.planner.types import FixtureGroups, FixtureKey + +_TYPED_NULL_LINE_PATTERN: re.Pattern[str] = re.compile( + r"^(?P[ \t]*)(?:" + r"CAST\s*\(\s*NULL\s+AS\s+(?P" + r"[A-Za-z_][A-Za-z0-9_]*(?:\s*\([^()]*\))?" + r"(?:\s+[A-Za-z_][A-Za-z0-9_]*(?:\s*\([^()]*\))?)*" + r")\)" + r"|NULL\s*::\s*(?P[A-Za-z_][A-Za-z0-9_]*(?:\([^)]*\))?)" + r")\s+AS\s+(?P\"[^\"]+\"|`[^`]+`|[A-Za-z_][A-Za-z0-9_]*)" + r"(?P\s*,)?[ \t]*$", + re.IGNORECASE, +) +_CLAUSE_LINE_PATTERN: re.Pattern[str] = re.compile( + r"^\s*(?:FROM|WHERE|GROUP|HAVING|QUALIFY|ORDER|LIMIT|UNION|INTERSECT|EXCEPT)\b", + re.IGNORECASE, +) +_SET_OPERATION_PATTERN: re.Pattern[str] = re.compile( + r"\b(?:UNION|INTERSECT|EXCEPT)\b", re.IGNORECASE +) + + +def format_redundant_fixture_nulls( + *, + files: dict[Path, str], + project_dir: Path, + project: CompiledProject, + adapter: BaseAdapter, +) -> dict[Path, str]: + """Remove typed-null fixture projections that planner completion can reproduce.""" + + planning_context: SqlTestPlanningContext = build_sql_test_planning_context( + project=project, + tests=project.sql_tests, + ) + fixture_context: RelationFixturePlanningContext = build_relation_fixture_context( + project=project + ) + updated: dict[Path, str] = {} + processed_blocks: set[tuple[Path, int]] = set() + for test in project.sql_tests: + source_path: Path | None = test.source_path + block_index: int | None = test.block_index + if source_path is None or block_index is None: + continue + block_key: tuple[Path, int] = (source_path, block_index) + if block_key in processed_blocks or test.mode is not SqlTestMode.MODEL: + continue + processed_blocks.add(block_key) + file_path: Path = (project_dir / source_path).resolve() + contents: str | None = updated.get(file_path, files.get(file_path)) + if contents is None: + continue + fixed_block: str = _format_test_block( + test=test, + project=project, + adapter=adapter, + planning_context=planning_context, + fixture_context=fixture_context, + ) + if fixed_block == test.test_block.sql_body: + continue + rewritten: str = _replace_test_block( + contents=contents, + original=test.test_block.sql_body, + replacement=fixed_block, + ) + if rewritten != contents: + updated[file_path] = rewritten + return updated + + +def _format_test_block( + *, + test: CompiledSqlTest, + project: CompiledProject, + adapter: BaseAdapter, + planning_context: SqlTestPlanningContext, + fixture_context: RelationFixturePlanningContext, +) -> str: + payload: object = test.payload + if not isinstance(payload, CompiledModelSqlTestPayload): + return test.test_block.sql_body + if test.sql_body != test.test_block.sql_body: + return test.test_block.sql_body + ordered_model_names: tuple[str, ...] | None = planning_context.chain_names_by_test_key.get( + test.key + ) + if ordered_model_names is None: + return test.test_block.sql_body + fixture_ctes: tuple[CompileSqlTestCte, ...] = tuple( + cte for cte in payload.authored_ctes if _fixture_key(cte.name) is not None + ) + fixture_sql: dict[tuple[CompiledResourceType, str], str] = { + key: cte.sql_body for cte in fixture_ctes if (key := _fixture_key(cte.name)) is not None + } + changed_bodies: list[tuple[str, str]] = [] + for cte in fixture_ctes: + key: FixtureKey | None = _fixture_key(cte.name) + if key is None: + continue + relation: FixtureRelationMetadata | None = fixture_context.relations.get(key) + if relation is None or not relation.authoritative_names: + continue + candidate: str = _remove_redundant_typed_null_lines( + sql=fixture_sql[key], + relation=relation, + adapter=adapter, + ) + if candidate == fixture_sql[key]: + continue + candidate_fixture_sql: dict[tuple[CompiledResourceType, str], str] = { + **fixture_sql, + key: candidate, + } + completion: RelationFixtureCompletion = build_relation_fixture_completion( + project=project, + adapter=adapter, + ordered_model_names=ordered_model_names, + fixture_groups=_fixture_groups(candidate_fixture_sql), + planning_context=fixture_context, + ) + if completion.diagnostics: + continue + fixture_sql[key] = candidate + changed_bodies.append((cte.sql_body, candidate)) + if not changed_bodies: + return test.test_block.sql_body + return _replace_cte_bodies( + sql=test.test_block.sql_body, + replacements=tuple(changed_bodies), + ) + + +def _remove_redundant_typed_null_lines( + *, sql: str, relation: FixtureRelationMetadata, adapter: BaseAdapter +) -> str: + if _SET_OPERATION_PATTERN.search(sql) is not None: + return sql + metadata_by_name: dict[str, FixtureColumnMetadata] = { + column.name.casefold(): column for column in relation.columns + } + lines: list[str] = sql.splitlines() + typed_null_indexes: set[int] = set() + remove_indexes: set[int] = set() + for index, line in enumerate(lines): + match: re.Match[str] | None = _TYPED_NULL_LINE_PATTERN.fullmatch(line) + if match is None: + continue + alias: str = match.group("alias").strip('"`') + column: FixtureColumnMetadata | None = metadata_by_name.get(alias.casefold()) + if column is None or column.type is None: + continue + typed_null_indexes.add(index) + if column.nullable is False: + continue + remove_indexes.add(index) + empty_retained: list[str] = [ + line for index, line in enumerate(lines) if index not in typed_null_indexes + ] + empty_retained = _without_trailing_projection_comma(lines=empty_retained) + empty_candidate: str = "\n".join(empty_retained) + if re.fullmatch(r"\s*SELECT\s+WHERE\s+FALSE\s*", empty_candidate, re.IGNORECASE): + return "SELECT * FROM __empty_fixture()" + if not remove_indexes: + return sql + retained: list[str] = [line for index, line in enumerate(lines) if index not in remove_indexes] + retained = _without_trailing_projection_comma(lines=retained) + candidate: str = "\n".join(retained) + inference: FixtureColumnInference | None = infer_fixture_column_facts( + query_sql=candidate, + inference_profile=adapter.expression_inference_profile(), + ) + if inference is None or not inference.columns: + return sql + return candidate + + +def _without_trailing_projection_comma(*, lines: list[str]) -> list[str]: + updated: list[str] = list(lines) + clause_index: int | None = next( + (index for index, line in enumerate(updated) if _CLAUSE_LINE_PATTERN.match(line)), + None, + ) + search_end: int = len(updated) if clause_index is None else clause_index + for index in range(search_end - 1, -1, -1): + if not updated[index].strip(): + continue + updated[index] = re.sub(r",\s*$", "", updated[index]) + break + return updated + + +def _fixture_key(name: str) -> tuple[CompiledResourceType, str] | None: + for prefix, resource_type in ( + (REF_TEST_CTE_PREFIX, CompiledResourceType.MODEL), + (SOURCE_TEST_CTE_PREFIX, CompiledResourceType.SOURCE), + (SEED_TEST_CTE_PREFIX, CompiledResourceType.SEED), + ): + if name.startswith(prefix): + return resource_type, name.removeprefix(prefix) + return None + + +def _fixture_groups( + fixture_sql: dict[tuple[CompiledResourceType, str], str], +) -> FixtureGroups: + groups: list[tuple[CompiledResourceType, dict[str, str]]] = [] + resource_types: tuple[CompiledResourceType, ...] = ( + CompiledResourceType.MODEL, + CompiledResourceType.SOURCE, + CompiledResourceType.SEED, + ) + for resource_type in resource_types: + fixtures: dict[str, str] = {} + for (candidate_type, name), sql in fixture_sql.items(): + if candidate_type == resource_type: + fixtures[name] = sql + groups.append((resource_type, fixtures)) + return tuple(groups) + + +def _replace_cte_bodies(*, sql: str, replacements: tuple[tuple[str, str], ...]) -> str: + result: str = sql + for original, replacement in replacements: + if result.count(original) != 1: + continue + result = result.replace(original, replacement, 1) + return result + + +def _replace_test_block(*, contents: str, original: str, replacement: str) -> str: + start: int = contents.find(original) + if start < 0: + return contents + return f"{contents[:start]}{replacement}{contents[start + len(original) :]}" diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py index a3876e350a..f7592c32b8 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_validation.py @@ -15,6 +15,9 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner._helpers.fixtures.completion import ( build_relation_fixture_completion, + build_relation_fixture_context, + complete_empty_fixture_sql, + complete_typed_null_columns, ) from sqlbuild.compiler.planner.exceptions import SqlTestFixtureValidationError from sqlbuild.compiler.planner.models import ( @@ -39,7 +42,7 @@ def build_validated_test_fixtures( mock_seeds: dict[str, str], expected_outputs: dict[str, str], planning_context: RelationFixturePlanningContext | None = None, -) -> tuple[dict[str, str], dict[str, str], dict[str, str]]: +) -> tuple[dict[str, str], dict[str, str], dict[str, str], dict[str, str]]: """Report all statically provable missing columns and collection-type mismatches.""" fixture_groups: tuple[tuple[CompiledResourceType, dict[str, str]], ...] = ( @@ -47,17 +50,21 @@ def build_validated_test_fixtures( (CompiledResourceType.SOURCE, mock_sources), (CompiledResourceType.SEED, mock_seeds), ) + fixture_context: RelationFixturePlanningContext = ( + planning_context or build_relation_fixture_context(project=project) + ) completion: RelationFixtureCompletion = build_relation_fixture_completion( project=project, adapter=adapter, ordered_model_names=ordered_model_names, fixture_groups=fixture_groups, - planning_context=planning_context, + planning_context=fixture_context, ) errors: list[str] = [ _located_diagnostic(test=test, diagnostic=diagnostic) for diagnostic in completion.diagnostics ] + completed_expected_outputs: dict[str, str] = dict(expected_outputs) type_fixture_groups: tuple[tuple[CompiledResourceType, dict[str, str]], ...] = ( *fixture_groups, (CompiledResourceType.SQL_TEST, expected_outputs), @@ -66,11 +73,31 @@ def build_validated_test_fixtures( for name, sql in fixtures.items(): inferred_columns: tuple[InferredColumn, ...] | None if resource_type == CompiledResourceType.SQL_TEST: + completed_sql, empty_error = complete_empty_fixture_sql( + sql=sql, + relation=fixture_context.relations.get((CompiledResourceType.MODEL, name)), + adapter=adapter, + ) + if empty_error is not None: + errors.append( + f"{_fixture_location(test=test, resource_type=resource_type, name=name)}: " + f"expected output '{name}' {empty_error}" + ) + completed_expected_outputs[name] = completed_sql inference: FixtureColumnInference | None = infer_fixture_column_facts( - query_sql=sql, + query_sql=completed_sql, inference_profile=adapter.expression_inference_profile(), ) inferred_columns = inference.columns if inference is not None else None + if inference is not None: + completed_expected_outputs[name] = complete_typed_null_columns( + sql=completed_sql, + inference=inference, + expected_types=completion.expected_types.get( + (CompiledResourceType.MODEL, name), {} + ), + adapter=adapter, + ) else: inferred_columns = completion.inferred_by_fixture.get((resource_type, name)) if inferred_columns is None: @@ -118,6 +145,7 @@ def build_validated_test_fixtures( fixtures=mock_seeds, completion=completion, ), + completed_expected_outputs, ) diff --git a/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py b/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py new file mode 100644 index 0000000000..380124d2a8 --- /dev/null +++ b/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py @@ -0,0 +1,32 @@ +"""Compiler-backed safe fixture-null source rewrites.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.planner._helpers.sql_tests.fixture_formatting import ( + format_redundant_fixture_nulls, +) + + +class FixtureNullAutofix: + """Apply fixture-null rewrites proven safe by planner completion.""" + + @staticmethod + def apply( + *, + files: dict[Path, str], + project_dir: Path, + project: CompiledProject, + adapter: BaseAdapter, + ) -> dict[Path, str]: + """Return contract-safe fixture source rewrites.""" + + return format_redundant_fixture_nulls( + files=files, + project_dir=project_dir, + project=project, + adapter=adapter, + ) diff --git a/src/sqlbuild/lint/_helpers/fixture_null_candidates.py b/src/sqlbuild/lint/_helpers/fixture_null_candidates.py new file mode 100644 index 0000000000..c1e4266c26 --- /dev/null +++ b/src/sqlbuild/lint/_helpers/fixture_null_candidates.py @@ -0,0 +1,30 @@ +"""Cheap source scan for contract-aware fixture formatting candidates.""" + +from __future__ import annotations + +import re +from pathlib import Path + +_FIXTURE_TYPED_NULL_PATTERN: re.Pattern[str] = re.compile( + r"(?:CAST\s*\(\s*NULL\s+AS\b|NULL\s*::)", re.IGNORECASE +) +_FIXTURE_CTE_MARKERS: tuple[str, ...] = ("__ref__", "__source__", "__seed__") +_IGNORED_SEARCH_DIRECTORIES: frozenset[str] = frozenset({".git", ".venv", "target", "node_modules"}) + + +def has_fixture_typed_null_candidates(*, project_dir: Path) -> bool: + """Return whether source files may contain safe fixture-null autofixes.""" + + for file_path in project_dir.rglob("*.sql"): + if any(part in _IGNORED_SEARCH_DIRECTORIES for part in file_path.parts): + continue + try: + contents: str = file_path.read_text(encoding="utf-8") + except OSError: + continue + folded: str = contents.casefold() + if not any(marker in folded for marker in _FIXTURE_CTE_MARKERS): + continue + if _FIXTURE_TYPED_NULL_PATTERN.search(contents) is not None: + return True + return False diff --git a/src/sqlbuild/lint/main/has_fixture_typed_null_candidates.py b/src/sqlbuild/lint/main/has_fixture_typed_null_candidates.py new file mode 100644 index 0000000000..e65cbbd865 --- /dev/null +++ b/src/sqlbuild/lint/main/has_fixture_typed_null_candidates.py @@ -0,0 +1,13 @@ +"""Public lint entry for fixture-null candidate discovery.""" + +from pathlib import Path + +from sqlbuild.lint._helpers.fixture_null_candidates import ( + has_fixture_typed_null_candidates as _has_fixture_typed_null_candidates, +) + + +def has_fixture_typed_null_candidates(*, project_dir: Path) -> bool: + """Return whether source files may contain safe fixture-null autofixes.""" + + return _has_fixture_typed_null_candidates(project_dir=project_dir) diff --git a/src/sqlbuild/lint/main/run_format.py b/src/sqlbuild/lint/main/run_format.py index 7cb10342ee..5688d9941c 100644 --- a/src/sqlbuild/lint/main/run_format.py +++ b/src/sqlbuild/lint/main/run_format.py @@ -4,7 +4,10 @@ from pathlib import Path +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import TypedSqlValueRenderer +from sqlbuild.compiler.planner.classes.fixture_null_autofix import FixtureNullAutofix from sqlbuild.lint._helpers.headers import scan_headers from sqlbuild.lint._helpers.native import format_native_headers, lint_native_headers from sqlbuild.lint._helpers.native_format import ( @@ -29,6 +32,8 @@ def run_format( config: LintConfig, value_renderer: TypedSqlValueRenderer | None = None, selected_paths: frozenset[Path] | None = None, + compiled_project: CompiledProject | None = None, + adapter: BaseAdapter | None = None, write: bool = True, ) -> LintRunResult: """Format all DSL files in place and report the violations that remain.""" @@ -40,7 +45,11 @@ def run_format( file_path: newline_style(contents=contents) for file_path, contents in files.items() } updated_contents: dict[Path, str] = _apply_fixes( - files=files, config=config, project_dir=project_dir + files=files, + config=config, + project_dir=project_dir, + compiled_project=compiled_project, + adapter=adapter, ) formatted: list[Path] = [] changes: list[FormatChange] = [] @@ -91,14 +100,31 @@ def run_format( def _apply_fixes( - *, files: dict[Path, str], config: LintConfig, project_dir: Path + *, + files: dict[Path, str], + config: LintConfig, + project_dir: Path, + compiled_project: CompiledProject | None, + adapter: BaseAdapter | None, ) -> dict[Path, str]: """Return contents after native header and supported SQL body fixes.""" - updated: dict[Path, str] = {} + updated: dict[Path, str] = ( + FixtureNullAutofix.apply( + files=files, + project_dir=project_dir, + project=compiled_project, + adapter=adapter, + ) + if compiled_project is not None and adapter is not None + else {} + ) file_path: Path contents: str - for file_path, contents in sorted(files.items()): + current_files: dict[Path, str] = { + file_path: updated.get(file_path, contents) for file_path, contents in files.items() + } + for file_path, contents in sorted(current_files.items()): native_result: tuple[str, tuple[LintViolation, ...]] = format_native_headers( contents=contents, file_path=file_path, @@ -108,7 +134,7 @@ def _apply_fixes( updated[file_path] = native_result[0] if not config.native_enabled: return updated - current_files: dict[Path, str] = { + current_files = { file_path: updated.get(file_path, contents) for file_path, contents in files.items() } updated.update( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index cac701b55d..ddb554cb7a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -581,7 +581,10 @@ def build_untyped_null_fixture_project_files() -> dict[str, str]: 'database = "untyped_null_fixture.duckdb"\n' ), "models/orders.sql": ( - "MODEL ();\n\n" + "MODEL (\n" + " contract enforced,\n" + " columns (order_year (type BIGINT)),\n" + ");\n\n" "SELECT EXTRACT(YEAR FROM ordered_at) AS order_year\n" 'FROM __source("raw_orders")\n' ), @@ -599,13 +602,74 @@ def build_untyped_null_fixture_project_files() -> dict[str, str]: "WITH\n" "__source__raw_orders AS (SELECT NULL AS ordered_at WHERE FALSE),\n" "__expected__orders AS (\n" - " SELECT CAST(NULL AS BIGINT) AS order_year WHERE FALSE\n" + " SELECT NULL AS order_year WHERE FALSE\n" ")\n" "SELECT 1\n" ), } +def build_contract_empty_fixture_project_files() -> dict[str, str]: + """Build empty input and expected fixtures whose shapes come from contracts.""" + + files: dict[str, str] = build_partial_source_fixture_project_files() + files["models/orders.sql"] = ( + "MODEL (\n" + " contract enforced,\n" + " columns (\n" + " order_id (type INTEGER),\n" + " status (type VARCHAR),\n" + " ),\n" + ");\n\n" + 'SELECT order_id, status FROM __source("raw_orders")\n' + ) + files["sources/raw_orders.yml"] = files["sources/raw_orders.yml"].replace( + " table: raw_orders\n", + " table: raw_orders\n contract: enforced\n", + ) + files["tests/unit/test_orders.sql"] = ( + "TEST();\n\n" + "WITH\n" + "__source__raw_orders AS (SELECT * FROM __empty_fixture()),\n" + "__expected__orders AS (SELECT * FROM __empty_fixture())\n" + "SELECT 1\n" + ) + return files + + +def build_open_source_empty_fixture_project_files() -> dict[str, str]: + """Build an empty source fixture without an authoritative source shape.""" + + files: dict[str, str] = build_partial_source_fixture_project_files() + files["tests/unit/test_orders.sql"] = ( + "TEST();\n\n" + "WITH\n" + "__source__raw_orders AS (SELECT * FROM __empty_fixture()),\n" + "__expected__orders AS (\n" + " SELECT 1 AS order_id, CAST(NULL AS VARCHAR) AS status\n" + ")\n" + "SELECT 1\n" + ) + return files + + +def build_open_expected_empty_fixture_project_files() -> dict[str, str]: + """Build an empty expected fixture without an authoritative model shape.""" + + files: dict[str, str] = build_partial_source_fixture_project_files() + files["models/orders.sql"] = 'MODEL ();\n\nSELECT * FROM __source("raw_orders")\n' + files["tests/unit/test_orders.sql"] = ( + "TEST();\n\n" + "WITH\n" + "__source__raw_orders AS (\n" + " SELECT 1 AS order_id, CAST(NULL AS VARCHAR) AS status\n" + "),\n" + "__expected__orders AS (SELECT * FROM __empty_fixture())\n" + "SELECT 1\n" + ) + return files + + def build_qualified_star_other_relation_project_files() -> dict[str, str]: """Build a qualified star that does not target the only mocked relation.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py index f9c908ff7b..d3094845cf 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py @@ -25,6 +25,7 @@ build_chain_test_project_files, build_clause_partial_source_fixture_project_files, build_complex_values_fixture_project_files, + build_contract_empty_fixture_project_files, build_cte_derived_output_fixture_project_files, build_cte_partial_source_fixture_project_files, build_dynamic_pivot_test_project_files, @@ -38,7 +39,9 @@ build_mixed_case_partial_fixture_project_files, build_mock_boundary_test_project_files, build_multiple_invalid_fixtures_project_files, + build_open_expected_empty_fixture_project_files, build_open_schema_ref_fixture_project_files, + build_open_source_empty_fixture_project_files, build_parameterized_test_project_files, build_partial_ref_fixture_project_files, build_partial_seed_fixture_project_files, @@ -158,9 +161,21 @@ expected_stdout_fragment="PASS=1", expected_artifact_fragments=( 'CAST("__sqlbuild_partial_fixture".ordered_at AS DATE) AS ordered_at', + 'CAST("__sqlbuild_partial_fixture".order_year AS BIGINT) AS order_year', "__sqlbuild_partial_fixture", ), ), + PartialFixtureE2ETestCase( + description="empty fixture receives complete authoritative relation shape", + repo_files=build_contract_empty_fixture_project_files(), + expected_stdout_fragment="PASS=1", + expected_artifact_fragments=( + 'CAST(NULL AS INT) AS "order_id"', + 'CAST(NULL AS TEXT) AS "status"', + "FALSE", + ), + unexpected_artifact_fragments=("__empty_fixture",), + ), PartialFixtureE2ETestCase( description="qualified star on real relation does not expand mocked relation", repo_files=build_qualified_star_other_relation_project_files(), @@ -400,6 +415,22 @@ def test_given_complex_strings_in_values_when_processing_then_literals_remain_in "missing required columns: status", ), ), + SqlTestFixtureValidationE2ETestCase( + description="empty source fixture requires an authoritative shape", + repo_files=build_open_source_empty_fixture_project_files(), + expected_stderr_fragments=( + "mock source 'raw_orders' uses __empty_fixture()", + "column schema is not authoritative", + ), + ), + SqlTestFixtureValidationE2ETestCase( + description="empty expected fixture requires an authoritative shape", + repo_files=build_open_expected_empty_fixture_project_files(), + expected_stderr_fragments=( + "expected output 'orders' uses __empty_fixture()", + "column schema is not authoritative", + ), + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py index 339344ac15..6d9b6a2586 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py @@ -49,6 +49,19 @@ expected_expected_model_names=("orders",), expected_mock_table_function_names=("customer_orders",), ), + ExtractSqlTestCtesTestCase( + description="allows empty fixture marker for model expected output", + sql=""" + WITH + __source__raw_orders AS (SELECT 1 AS order_id), + __expected__orders AS (SELECT * FROM __empty_fixture()) + SELECT 1 + """.strip(), + expected_authored_cte_names=("__source__raw_orders",), + expected_mock_model_names=(), + expected_mock_source_names=("raw_orders",), + expected_expected_model_names=("orders",), + ), ExtractSqlTestCtesTestCase( description="extracts zero-row assertion ctes", sql=""" @@ -634,6 +647,16 @@ def test_given_direct_logic_sql_test_cte_variants_when_extracting_then_it_return """.strip(), expected_error_fragment=r"must not use SELECT \* in __expected__ CTEs", ), + ExtractSqlTestCtesErrorTestCase( + description="rejects empty fixture marker for direct logic expected output", + sql=""" + WITH __macro_actual__ AS (SELECT 'paid' AS status), + __macro_expected__ AS (SELECT * FROM __empty_fixture()) + SELECT 1 + """.strip(), + mode=SqlTestMode.MACRO, + expected_error_fragment=r"must not use SELECT \* in __macro_expected__ CTEs", + ), ExtractSqlTestCtesErrorTestCase( description="raises when union branch projection order differs", sql=""" diff --git a/tests/unit/src/sqlbuild/lint/helpers.py b/tests/unit/src/sqlbuild/lint/helpers.py new file mode 100644 index 0000000000..a47ffe97a6 --- /dev/null +++ b/tests/unit/src/sqlbuild/lint/helpers.py @@ -0,0 +1,42 @@ +"""Project writers shared by lint behavior tests.""" + +from pathlib import Path + +_PROJECT_TOML: str = 'name = "demo"\nadapter = "duckdb"\n' + + +def write_fixture_format_project( + *, tmp_path: Path, fixture_sql: str, project_toml: str = _PROJECT_TOML +) -> Path: + """Write a contracted two-model project with one SQL fixture.""" + + (tmp_path / "sqlbuild_project.toml").write_text(project_toml, encoding="utf-8") + staging_model: Path = tmp_path / "models" / "stg_rows.sql" + staging_model.parent.mkdir() + staging_model.write_text( + "MODEL (\n" + ' description "Staged rows",\n' + " contract enforced,\n" + " columns (\n" + " id (type INTEGER, nullable false),\n" + " flag (type BOOLEAN, nullable true),\n" + " ),\n" + ");\n\n" + "SELECT 1 AS id, TRUE AS flag\n", + encoding="utf-8", + ) + (tmp_path / "models" / "rows.sql").write_text( + 'MODEL (description "Rows");\n\nSELECT id, flag FROM __ref("stg_rows")\n', + encoding="utf-8", + ) + test_file: Path = tmp_path / "tests" / "unit" / "test_rows.sql" + test_file.parent.mkdir(parents=True) + test_file.write_text( + "TEST();\n\n" + "WITH\n" + f"__ref__stg_rows AS ({fixture_sql}),\n" + "__expected__rows AS (SELECT 1 AS id, TRUE AS flag)\n" + "SELECT 1\n", + encoding="utf-8", + ) + return test_file diff --git a/tests/unit/src/sqlbuild/lint/test_cli_format.py b/tests/unit/src/sqlbuild/lint/test_cli_format.py index 07842f1bfa..06941f517c 100644 --- a/tests/unit/src/sqlbuild/lint/test_cli_format.py +++ b/tests/unit/src/sqlbuild/lint/test_cli_format.py @@ -8,6 +8,7 @@ from sqlbuild.cli.commands.main.entrypoint.entry import main from tests.unit.src.sqlbuild.lint._test_types import FormatCliTestCase +from tests.unit.src.sqlbuild.lint.helpers import write_fixture_format_project PROJECT_TOML: str = 'name = "demo"\nadapter = "duckdb"\n' @@ -85,3 +86,291 @@ def test_given_unselected_sql_rule_finding_when_formatting_then_rule_is_not_enfo assert exit_code == test_case.expected_exit_code assert test_case.expected_fragment not in capsys.readouterr().out + + +@pytest.mark.parametrize( + "test_case", + [ + FormatCliTestCase( + "typed fixture null diff", + ("--diff",), + 0, + "test_orders.sql", + False, + ), + FormatCliTestCase( + "typed fixture null apply", + (), + 0, + "Formatted files:", + True, + ), + ], + ids=lambda case: case.description, +) +def test_given_redundant_typed_fixture_null_when_formatting_then_safe_fix_uses_contract( + test_case: FormatCliTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") + staging_model: Path = tmp_path / "models" / "stg_orders.sql" + staging_model.parent.mkdir() + staging_model.write_text( + "MODEL (\n" + ' description "Staged orders",\n' + " contract enforced,\n" + " columns (\n" + " order_id (type INTEGER, nullable false),\n" + " status (type VARCHAR, nullable true),\n" + " ),\n" + ");\n\n" + "SELECT 1 AS order_id, CAST(NULL AS VARCHAR) AS status\n", + encoding="utf-8", + ) + (tmp_path / "models" / "orders.sql").write_text( + 'MODEL (description "Orders");\n\nSELECT order_id, status FROM __ref("stg_orders")\n', + encoding="utf-8", + ) + test_file: Path = tmp_path / "tests" / "unit" / "test_orders.sql" + test_file.parent.mkdir(parents=True) + original: str = ( + "TEST();\n\n" + "WITH\n" + "helper AS (SELECT '__ref__stg_orders AS (keep)' AS label),\n" + "__ref__stg_orders AS (\n" + " SELECT\n" + " 1 AS order_id,\n" + " CAST(NULL AS VARCHAR(100)) AS status\n" + "),\n" + "__expected__orders AS (\n" + " SELECT 1 AS order_id, CAST(NULL AS VARCHAR) AS status\n" + ")\n" + "SELECT 1\n" + ) + test_file.write_text(original, encoding="utf-8") + + exit_code: int = main(["--project-dir", str(tmp_path), "format", *test_case.arguments]) + + assert exit_code == test_case.expected_exit_code + assert test_case.expected_fragment in capsys.readouterr().out + formatted: str = test_file.read_text(encoding="utf-8") + fixture_prefix, expected_suffix = formatted.split("__expected__orders", maxsplit=1) + assert ( + "CAST(NULL AS VARCHAR(100)) AS status" not in fixture_prefix + ) is test_case.expected_writes + assert "CAST(NULL AS VARCHAR) AS status" in expected_suffix + assert "'__ref__stg_orders AS (keep)'" in formatted + + +@pytest.mark.parametrize( + "test_case", + [FormatCliTestCase("all-null empty fixture", (), 0, "__EMPTY_FIXTURE()")], + ids=lambda case: case.description, +) +def test_given_all_null_empty_fixture_when_formatting_then_uses_empty_fixture_marker( + test_case: FormatCliTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + model.write_text( + "MODEL (\n" + ' description "Orders",\n' + " contract enforced,\n" + " columns (\n" + " order_id (type INTEGER, nullable false),\n" + " status (type VARCHAR),\n" + " ),\n" + ");\n\n" + "SELECT 1 AS order_id, 'open' AS status\n", + encoding="utf-8", + ) + test_file: Path = tmp_path / "tests" / "unit" / "test_orders.sql" + test_file.parent.mkdir(parents=True) + test_file.write_text( + "TEST();\n\n" + "WITH\n" + "__ref__orders AS (\n" + " SELECT\n" + " CAST(NULL AS INTEGER) AS order_id,\n" + " CAST(NULL AS VARCHAR) AS status\n" + " WHERE FALSE\n" + "),\n" + "__expected__orders AS (SELECT 1 AS order_id, 'open' AS status)\n" + "SELECT 1\n", + encoding="utf-8", + ) + + exit_code: int = main(["--project-dir", str(tmp_path), "format"]) + + assert exit_code == test_case.expected_exit_code + assert "Formatted files:" in capsys.readouterr().out + formatted: str = test_file.read_text(encoding="utf-8") + assert test_case.expected_fragment in formatted + assert "__ref__orders AS (\n SELECT\n *\n FROM __EMPTY_FIXTURE()\n)" in formatted + assert "CAST(NULL" not in formatted.split("__expected__orders", maxsplit=1)[0] + + +@pytest.mark.parametrize( + "test_case", + [FormatCliTestCase("uncompilable project fallback", (), 0, "CAST(NULL AS INTEGER)")], + ids=lambda case: case.description, +) +def test_given_uncompilable_project_with_fixture_null_when_formatting_then_formats_without_autofix( + test_case: FormatCliTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + model.write_text( + 'MODEL (description "Orders");\n\nselect * from __ref("missing_orders")\n', + encoding="utf-8", + ) + test_file: Path = tmp_path / "tests" / "unit" / "test_orders.sql" + test_file.parent.mkdir(parents=True) + test_file.write_text( + "TEST();\n\n" + "WITH\n" + "__ref__missing_orders AS (\n" + " SELECT CAST(NULL AS INTEGER) AS order_id WHERE FALSE\n" + "),\n" + "__expected__orders AS (SELECT 1 AS order_id)\n" + "SELECT 1\n", + encoding="utf-8", + ) + + exit_code: int = main(["--project-dir", str(tmp_path), "format"]) + + assert exit_code == test_case.expected_exit_code + assert "Formatted files:" in capsys.readouterr().out + formatted: str = test_file.read_text(encoding="utf-8") + assert test_case.expected_fragment in formatted + assert "CAST(NULL AS INTEGER) AS order_id" in formatted + assert "WITH __ref__missing_orders AS" in formatted + + +@pytest.mark.parametrize( + "test_case", + [FormatCliTestCase("non-null expression", (), 0, "IS NULL OR (")], + ids=lambda case: case.description, +) +def test_given_typed_null_inside_boolean_expression_when_formatting_then_expression_is_preserved( + test_case: FormatCliTestCase, + tmp_path: Path, +) -> None: + test_file: Path = write_fixture_format_project( + tmp_path=tmp_path, + fixture_sql=( + "\n SELECT\n 1 AS id,\n CAST(NULL AS BOOLEAN) IS NULL OR (TRUE) AS flag\n" + ), + ) + + exit_code: int = main(["--project-dir", str(tmp_path), "format"]) + + assert exit_code == test_case.expected_exit_code + formatted: str = test_file.read_text(encoding="utf-8") + assert test_case.expected_fragment in formatted + assert "CAST(NULL AS BOOLEAN) IS NULL OR (" in formatted + assert ") AS flag" in formatted + + +@pytest.mark.parametrize( + "test_case", + [FormatCliTestCase("union branches", (), 0, "UNION ALL")], + ids=lambda case: case.description, +) +def test_given_union_fixture_when_formatting_then_branch_shapes_are_preserved( + test_case: FormatCliTestCase, + tmp_path: Path, +) -> None: + test_file: Path = write_fixture_format_project( + tmp_path=tmp_path, + fixture_sql=( + "\n SELECT\n" + " 1 AS id,\n" + " CAST(NULL AS BOOLEAN) AS flag\n" + " UNION ALL\n" + " SELECT 2 AS id, TRUE AS flag\n" + ), + ) + + exit_code: int = main(["--project-dir", str(tmp_path), "format"]) + + assert exit_code == test_case.expected_exit_code + formatted: str = test_file.read_text(encoding="utf-8") + assert test_case.expected_fragment in formatted + assert "CAST(NULL AS BOOLEAN) AS flag" in formatted + assert "UNION ALL" in formatted + + +@pytest.mark.parametrize( + "test_case", + [FormatCliTestCase("SQL analysis disabled", (), 0, "CAST(NULL AS BOOLEAN)")], + ids=lambda case: case.description, +) +def test_given_sql_analysis_disabled_when_formatting_then_fixture_null_is_preserved( + test_case: FormatCliTestCase, + tmp_path: Path, +) -> None: + test_file: Path = write_fixture_format_project( + tmp_path=tmp_path, + fixture_sql=("\n SELECT\n 1 AS id,\n CAST(NULL AS BOOLEAN) AS flag\n"), + project_toml=PROJECT_TOML + "\n[settings]\nsql_analysis = false\n", + ) + + exit_code: int = main(["--project-dir", str(tmp_path), "format"]) + + assert exit_code == test_case.expected_exit_code + assert test_case.expected_fragment in test_file.read_text(encoding="utf-8") + + +@pytest.mark.parametrize( + "test_case", + [FormatCliTestCase("macro-expanded fixture", (), 0, "@mock_rows()")], + ids=lambda case: case.description, +) +def test_given_macro_expanded_fixture_when_formatting_then_authored_source_is_not_rewritten( + test_case: FormatCliTestCase, + tmp_path: Path, +) -> None: + test_file: Path = write_fixture_format_project( + tmp_path=tmp_path, + fixture_sql="@mock_rows()", + ) + macro_file: Path = tmp_path / "macros" / "rows.py" + macro_file.parent.mkdir() + macro_file.write_text( + "def mock_rows() -> str:\n" + " return (\n" + ' "SELECT\\n"\n' + ' " 1 AS id,\\n"\n' + ' " CAST(NULL AS BOOLEAN) AS flag"\n' + " )\n", + encoding="utf-8", + ) + authored: str = test_file.read_text(encoding="utf-8") + test_file.write_text( + authored.replace( + "WITH\n", + "WITH\n" + "helper AS (\n" + " SELECT 'SELECT\n" + " 1 AS id,\n" + " CAST(NULL AS BOOLEAN) AS flag' AS fixture_text\n" + "),\n", + 1, + ), + encoding="utf-8", + ) + + exit_code: int = main(["--project-dir", str(tmp_path), "format"]) + + assert exit_code == test_case.expected_exit_code + formatted: str = test_file.read_text(encoding="utf-8") + assert test_case.expected_fragment in formatted + assert "CAST(NULL AS BOOLEAN) AS flag' AS fixture_text" in formatted