From faf96749b2163aa0dc330bf150c11f07582b8c85 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 02:51:01 +0100 Subject: [PATCH 1/4] test(compiler): record Polyglot wheel calls by calling site for the differential When SQLBUILD_ANALYSIS_RECORD_DIR is set, the guarded Polyglot proxy wraps every wheel callable it hands out and counts calls by the calling file:function and API, writing one JSON file per process at exit. With the variable unset the proxy returns exactly what it did before. The compiler differential reads these records to show which compile-path wheel sites each corpus reaches. --- .../sql_analysis/_helpers/polyglot.py | 74 ++++++++++++++++++- .../compiler/sql_analysis/constants.py | 2 + .../compiler/sql_analysis/__init__.py | 1 + .../compiler/sql_analysis/_test_types.py | 15 ++++ .../sqlbuild/compiler/sql_analysis/helpers.py | 18 +++++ .../test_polyglot_site_recorder.py | 65 ++++++++++++++++ 6 files changed, 173 insertions(+), 2 deletions(-) create mode 100644 tests/integration/src/sqlbuild/compiler/sql_analysis/__init__.py create mode 100644 tests/integration/src/sqlbuild/compiler/sql_analysis/_test_types.py create mode 100644 tests/integration/src/sqlbuild/compiler/sql_analysis/helpers.py create mode 100644 tests/integration/src/sqlbuild/compiler/sql_analysis/test_polyglot_site_recorder.py diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot.py b/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot.py index 2252e43013..e85d3d7642 100644 --- a/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot.py +++ b/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot.py @@ -2,11 +2,20 @@ from __future__ import annotations +import atexit +import inspect +import json +import os +import sys +import threading +from collections import Counter +from pathlib import Path from typing import Any import polyglot_sql from sqlbuild.compiler.sql_analysis.constants import ( + ANALYSIS_RECORD_DIR_ENV_VAR, POLYGLOT_ANALYZE_QUERY_ENTRY_POINT, POLYGLOT_ANALYZE_QUERY_OPTIONS_ARGUMENT_COUNT, POLYGLOT_ANALYZE_QUERY_OPTIONS_ARGUMENT_INDEX, @@ -14,6 +23,7 @@ POLYGLOT_COMPLEXITY_GUARD_PARAMETER, POLYGLOT_MAX_FUNCTION_CALL_DEPTH, POLYGLOT_MAX_FUNCTION_CALL_DEPTH_OPTION, + POLYGLOT_SITE_RECORD_PREFIX, ) _COMPLEXITY_GUARD: dict[str, int] = { @@ -57,19 +67,79 @@ def _has_explicit_options_guard(self, *, args: tuple[Any, ...], kwargs: dict[str return isinstance(options, dict) and POLYGLOT_COMPLEXITY_GUARD_OPTION in options +class _SiteRecorder: + """Debug-only count of Polyglot wheel calls by calling `file:function`, written at exit.""" + + def __init__(self, *, directory: Path) -> None: + self._directory: Path = directory + self._package_root: Path = Path(__file__).resolve().parents[3] + self._calls: Counter[tuple[str, str]] = Counter() + self._lock: threading.Lock = threading.Lock() + _ = atexit.register(self._write) + + @classmethod + def from_environment(cls) -> _SiteRecorder | None: + """Return a recorder when the debug record directory is set, otherwise None.""" + + directory: str | None = os.environ.get(ANALYSIS_RECORD_DIR_ENV_VAR) + return cls(directory=Path(directory)) if directory else None + + def record(self, *, api: str, caller: Any) -> None: + code: Any = caller.f_code + path: Path = Path(code.co_filename).resolve() + site: str = ( + path.relative_to(self._package_root).as_posix() + if path.is_relative_to(self._package_root) + else path.name + ) + with self._lock: + self._calls[(f"{site}:{code.co_qualname}", api)] += 1 + + def _write(self) -> None: + if not self._calls: + return + self._directory.mkdir(parents=True, exist_ok=True) + rows: list[list[object]] = [ + [site, api, count] for (site, api), count in sorted(self._calls.items()) + ] + target: Path = self._directory / f"{POLYGLOT_SITE_RECORD_PREFIX}{os.getpid()}.json" + _ = target.write_text(json.dumps({"calls": rows}) + "\n", encoding="utf-8") + + +class _RecordedEntryPoint: + """One Polyglot callable that records its calling site before running.""" + + def __init__(self, *, name: str, function: Any, recorder: _SiteRecorder) -> None: + self._name: str = name + self._function: Any = function + self._recorder: _SiteRecorder = recorder + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + self._recorder.record(api=self._name, caller=sys._getframe(1)) + return self._function(*args, **kwargs) + + class _PolyglotSqlModule: """Proxy Polyglot parsing through SQLBuild's trusted-SQL policy.""" + def __init__(self, *, recorder: _SiteRecorder | None) -> None: + self._recorder: _SiteRecorder | None = recorder + def __getattr__(self, name: str) -> Any: attribute: Any = getattr(polyglot_sql, name) - return ( + entry_point: Any = ( _GuardedEntryPoint(name=name, function=attribute) if name in _GUARDED_ENTRY_POINTS else attribute ) + if self._recorder is None or not callable(attribute) or inspect.isclass(attribute): + return entry_point + return _RecordedEntryPoint(name=name, function=entry_point, recorder=self._recorder) -_POLYGLOT_SQL_MODULE: _PolyglotSqlModule = _PolyglotSqlModule() +_POLYGLOT_SQL_MODULE: _PolyglotSqlModule = _PolyglotSqlModule( + recorder=_SiteRecorder.from_environment() +) def import_polyglot() -> Any: diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index 87feca62c3..d40fe049bf 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -31,6 +31,8 @@ POLYGLOT_COMPLEXITY_GUARD_OPTION: str = "complexityGuard" POLYGLOT_MAX_FUNCTION_CALL_DEPTH_OPTION: str = "maxFunctionCallDepth" POLYGLOT_MAX_FUNCTION_CALL_DEPTH: int = 512 +ANALYSIS_RECORD_DIR_ENV_VAR: str = "SQLBUILD_ANALYSIS_RECORD_DIR" +POLYGLOT_SITE_RECORD_PREFIX: str = "polyglot-sites-" BINDING_UNKNOWN_TABLE_INTERNAL_CODE: str = "B000" POLYGLOT_KIND_ALIAS: str = "alias" diff --git a/tests/integration/src/sqlbuild/compiler/sql_analysis/__init__.py b/tests/integration/src/sqlbuild/compiler/sql_analysis/__init__.py new file mode 100644 index 0000000000..5618f65ea9 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/sql_analysis/__init__.py @@ -0,0 +1 @@ +"""SQL analysis integration tests.""" diff --git a/tests/integration/src/sqlbuild/compiler/sql_analysis/_test_types.py b/tests/integration/src/sqlbuild/compiler/sql_analysis/_test_types.py new file mode 100644 index 0000000000..5af0584a8f --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/sql_analysis/_test_types.py @@ -0,0 +1,15 @@ +"""Test case types for SQL analysis integration tests.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class SiteRecorderTestCase: + """One process calling the wheel through a product site, with or without the record dir.""" + + description: str + record_dir: str + expected_rows: tuple[tuple[str, str, int], ...] + expected_entry_point: str diff --git a/tests/integration/src/sqlbuild/compiler/sql_analysis/helpers.py b/tests/integration/src/sqlbuild/compiler/sql_analysis/helpers.py new file mode 100644 index 0000000000..6f91e00bbe --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/sql_analysis/helpers.py @@ -0,0 +1,18 @@ +"""Read the wheel-site records a test process wrote.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from sqlbuild.compiler.sql_analysis.constants import POLYGLOT_SITE_RECORD_PREFIX + + +def recorded_rows(root: Path) -> tuple[tuple[str, str, int], ...]: + """Return every `(site, api, calls)` row the site recorder wrote below `root`.""" + + rows: list[tuple[str, str, int]] = [] + for path in sorted(root.rglob(f"{POLYGLOT_SITE_RECORD_PREFIX}*.json")): + for site, api, count in json.loads(path.read_text(encoding="utf-8"))["calls"]: + rows.append((str(site), str(api), int(count))) + return tuple(rows) diff --git a/tests/integration/src/sqlbuild/compiler/sql_analysis/test_polyglot_site_recorder.py b/tests/integration/src/sqlbuild/compiler/sql_analysis/test_polyglot_site_recorder.py new file mode 100644 index 0000000000..de009bcc7f --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/sql_analysis/test_polyglot_site_recorder.py @@ -0,0 +1,65 @@ +"""The debug site recorder counts wheel calls by product site only when its directory is set.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR +from tests.integration.src.sqlbuild.compiler.sql_analysis._test_types import SiteRecorderTestCase +from tests.integration.src.sqlbuild.compiler.sql_analysis.helpers import recorded_rows + +_PROCESS: str = ( + "from sqlbuild.compiler.references.main.extract_relation_names import " + "extract_relation_names\n" + "from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql\n" + "for _ in range(2):\n" + " assert extract_relation_names(sql='SELECT 1 FROM main.orders', dialect='duckdb')\n" + "module = import_polyglot_sql()\n" + "print(type(module.parse_one).__name__, isinstance(module.PolyglotError, type))\n" +) +_SITE: str = "compiler/references/_helpers/relation_names.py:extract_relation_names_impl" + + +@pytest.mark.parametrize( + "test_case", + [ + SiteRecorderTestCase( + description="recording", + record_dir="records", + expected_rows=((_SITE, "parse", 2),), + expected_entry_point="_RecordedEntryPoint", + ), + SiteRecorderTestCase( + description="disabled", + record_dir="", + expected_rows=(), + expected_entry_point="_GuardedEntryPoint", + ), + ], + ids=lambda case: case.description, +) +def test_given_record_dir_when_process_calls_the_wheel_then_sites_are_counted_at_exit( + test_case: SiteRecorderTestCase, tmp_path: Path +) -> None: + completed: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, "-c", _PROCESS], + cwd=tmp_path, + env={**os.environ, ANALYSIS_RECORD_DIR_ENV_VAR: test_case.record_dir}, + capture_output=True, + text=True, + check=False, + timeout=120, + ) + + assert completed.returncode == 0, completed.stderr + assert recorded_rows(tmp_path) == test_case.expected_rows + assert completed.stdout.split() == [test_case.expected_entry_point, "True"] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From fa310939b7ff928d8196918fa079b875196cf6f2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 02:51:11 +0100 Subject: [PATCH 2/4] test(compiler): compare analysis input edits with uncached compiles Add an incremental edit sequence that changes the inputs SQL analysis publishes shapes from: an upstream column type under a star chain, a contract type, a source expression, a UDF return type and a source column's declared nullability. Every step must match a --no-cache compile of the same project. --- .../cli/commands/main/compile/helpers.py | 33 +++++++++++++++++++ .../compile/test_compile_incremental_edits.py | 16 +++++++++ 2 files changed, 49 insertions(+) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 59ba8d67f5..7efeec91b1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -3965,6 +3965,39 @@ def fact_comment(root: Path) -> None: replace_project_text(root, FACT_ORDERS_MODEL, "FROM __ref", "-- leaf\nFROM __ref") +def payment_expression_type_change(root: Path) -> None: + """Widen one column of the payments source expression, which changes its inferred shape.""" + + replace_project_text( + root, + "sources/raw.yml", + "SELECT 1 AS id, 1 AS order_id,", + "SELECT 1 AS id, CAST(1 AS BIGINT) AS order_id,", + ) + + +def completed_order_udf_signature_change(root: Path) -> None: + """Change the completed-order UDF's declared return type, which its callers' shapes read.""" + + replace_project_text( + root, + "functions/sql/udf__is_completed_order.sql", + "returns BOOLEAN,\n);\n\norder_status = 'completed'", + "returns INTEGER,\n);\n\nCAST(order_status = 'completed' AS INTEGER)", + ) + + +def order_status_nullability_change(root: Path) -> None: + """Declare one raw orders source column non-null in its schema entry.""" + + replace_project_text( + root, + "sources/raw.yml", + " - name: status\n type: VARCHAR\n", + " - name: status\n type: VARCHAR\n nullable: false\n", + ) + + def staging_column_removed(root: Path) -> None: """Drop the quantity column from staging orders, which downstream models and tests read.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py index b4100aaac1..90bd684ca7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py @@ -24,6 +24,7 @@ compile_in_process, compile_in_process_output, compiled_text, + completed_order_udf_signature_change, disable_change_detection, edit_and_compile, enable_compile_reuse, @@ -37,6 +38,8 @@ json_report_keys, model_edit_step, move_project_file, + order_status_nullability_change, + payment_expression_type_change, prime_render_store, prime_render_store_in_process, random_edit_plan, @@ -50,6 +53,7 @@ staging_header_change, staging_new_column, staging_type_change, + star_chain_added, write_external_flavor, write_generated_edit_models, write_project_file, @@ -208,6 +212,18 @@ model_edit_step("edit_after_resources", fact_comment), ), ), + IncrementalEditSequenceTestCase( + description="analysis_input_edits", + steps=( + full_edit_step("star_chain_added", star_chain_added), + model_edit_step("upstream_type_flips_downstream_shapes", staging_type_change), + model_edit_step("contract_type_edit", staging_contract_change), + full_edit_step("source_expression_edit", payment_expression_type_change), + full_edit_step("udf_signature_edit", completed_order_udf_signature_change), + full_edit_step("schema_entry_nullability_edit", order_status_nullability_change), + model_edit_step("edit_after_analysis_inputs", fact_comment), + ), + ), ], ids=lambda case: case.description, ) From 799137309323769a2d51ad592b27db650b2f3c66 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 02:51:32 +0100 Subject: [PATCH 3/4] test(compiler): cover every analysis input in the compiler differential Generate SQL analysis blocks in the seeded projects: typed, untyped and star outputs over complete, published and open inputs; set operations and CTEs; a contracted model under a three-level chain; scalar and table UDFs with typed arguments; a proven dynamic pivot; quoted identifiers; metadata audits; a SQL test with a macro mock; a needed sql_analysis opt-out under require_sql_analysis; literal SQL in a Python task; and select-limited, analysis-off and rich-lineage compiles. The first seed of a run also compiles its analysis blocks under the postgres and snowflake dialects. Classify CompiledProject captures into 56 analysis kinds, crediting a kind only from analysed evidence, and add --require-analysis-coverage to the per-PR job. Add an analysis failure corpus with one case per reachable B, K, S and syntax code plus ordering cases, checking the first error's code, message, help, notes and location and every case's code order. The code set comes from the help catalogue and the analysis-stage modules; unreachable codes are listed with reasons. Each native analysis lane gets its own empty corpus module already wired into the shared list. --wheel-site-report prints and writes Polyglot wheel calls per engine, site and corpus, plus analysis deferral counts read from the same record directory; CI uploads it as an artifact. Nothing in it gates. --- .github/workflows/verify.yml | 11 +- .../_helpers/analysis_corpus/__init__.py | 1 + .../analysis_corpus/analysis_session.py | 9 + .../_helpers/analysis_corpus/assembly.py | 9 + .../_helpers/analysis_corpus/contracts.py | 9 + .../_helpers/analysis_corpus/failure_cases.py | 561 ++++++++++++++++++ .../_helpers/analysis_corpus/lineage.py | 9 + .../analysis_corpus/semantic_checks.py | 9 + .../_helpers/analysis_corpus/sql_test_glue.py | 9 + .../_helpers/comparing/comparison.py | 7 + .../_helpers/comparing/project_comparison.py | 3 + .../_helpers/corpus/case_builder.py | 10 +- .../_helpers/corpus/corpus.py | 24 +- .../_helpers/corpus/emitted_codes.py | 48 ++ .../_helpers/corpus/failure_cases.py | 4 + .../_helpers/coverage/analysis.py | 539 +++++++++++++++++ .../_helpers/running/execution.py | 24 +- .../_helpers/running/records.py | 84 +++ .../_helpers/running/report.py | 12 + .../_helpers/running/run.py | 2 + .../classes/analysis_features.py | 348 +++++++++++ .../classes/project_builder.py | 31 +- scripts/compiler_differential/constants.py | 144 ++++- .../main/differential.py | 41 +- .../main/generate_project.py | 8 +- scripts/compiler_differential/models.py | 22 + src/sqlbuild/compiler/frontier/constants.py | 1 + .../compiler_differential/_test_types.py | 31 + .../scripts/compiler_differential/helpers.py | 12 + .../test_analysis_capture.py | 114 ++++ .../test_compiler_differential.py | 93 ++- .../test_failure_corpus_codes.py | 48 +- .../_helpers/corpus/_test_types.py | 21 +- .../_helpers/corpus/test_analysis_codes.py | 71 +++ .../_helpers/corpus/test_corpus.py | 38 ++ .../_helpers/coverage/_test_types.py | 19 + .../_helpers/coverage/helpers.py | 66 ++- .../_helpers/coverage/test_analysis.py | 288 +++++++++ .../_helpers/running/_test_types.py | 11 + .../_helpers/running/test_records.py | 82 +++ 40 files changed, 2852 insertions(+), 21 deletions(-) create mode 100644 scripts/compiler_differential/_helpers/analysis_corpus/__init__.py create mode 100644 scripts/compiler_differential/_helpers/analysis_corpus/analysis_session.py create mode 100644 scripts/compiler_differential/_helpers/analysis_corpus/assembly.py create mode 100644 scripts/compiler_differential/_helpers/analysis_corpus/contracts.py create mode 100644 scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py create mode 100644 scripts/compiler_differential/_helpers/analysis_corpus/lineage.py create mode 100644 scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py create mode 100644 scripts/compiler_differential/_helpers/analysis_corpus/sql_test_glue.py create mode 100644 scripts/compiler_differential/_helpers/coverage/analysis.py create mode 100644 scripts/compiler_differential/_helpers/running/records.py create mode 100644 scripts/compiler_differential/classes/analysis_features.py create mode 100644 tests/e2e/scripts/compiler_differential/test_analysis_capture.py create mode 100644 tests/unit/scripts/compiler_differential/_helpers/corpus/test_analysis_codes.py create mode 100644 tests/unit/scripts/compiler_differential/_helpers/coverage/test_analysis.py create mode 100644 tests/unit/scripts/compiler_differential/_helpers/running/test_records.py diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 86e9f59e05..27e8cb57cc 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -294,7 +294,16 @@ jobs: run: >- make compiler-differential COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage - --require-render-coverage" + --require-render-coverage --require-analysis-coverage + --wheel-site-report ${{ runner.temp }}/compiler-differential-wheel-sites.json" + - name: Upload the Polyglot wheel-site and deferral report + if: always() + uses: actions/upload-artifact@v4 + with: + name: compiler-differential-wheel-sites-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/compiler-differential-wheel-sites.json + if-no-files-found: warn + retention-days: 30 - name: Compare generated seeds and the failure corpus (python vs shipped native) run: make compiler-differential-shipped diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/__init__.py b/scripts/compiler_differential/_helpers/analysis_corpus/__init__.py new file mode 100644 index 0000000000..934b344cf3 --- /dev/null +++ b/scripts/compiler_differential/_helpers/analysis_corpus/__init__.py @@ -0,0 +1 @@ +"""Analysis-stage failure cases, one module per native analysis lane.""" diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/analysis_session.py b/scripts/compiler_differential/_helpers/analysis_corpus/analysis_session.py new file mode 100644 index 0000000000..9faaa527da --- /dev/null +++ b/scripts/compiler_differential/_helpers/analysis_corpus/analysis_session.py @@ -0,0 +1,9 @@ +"""Failure cases owned by the native model analysis session lane.""" + +from scripts.compiler_differential.models import FailureCase + + +def analysis_session_failure_cases() -> tuple[FailureCase, ...]: + """Return this lane's failure cases; the lane appends here without editing shared lists.""" + + return () diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/assembly.py b/scripts/compiler_differential/_helpers/analysis_corpus/assembly.py new file mode 100644 index 0000000000..7cb668e3bb --- /dev/null +++ b/scripts/compiler_differential/_helpers/analysis_corpus/assembly.py @@ -0,0 +1,9 @@ +"""Failure cases owned by the native project assembly and target validation lane.""" + +from scripts.compiler_differential.models import FailureCase + + +def assembly_failure_cases() -> tuple[FailureCase, ...]: + """Return this lane's failure cases; the lane appends here without editing shared lists.""" + + return () diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/contracts.py b/scripts/compiler_differential/_helpers/analysis_corpus/contracts.py new file mode 100644 index 0000000000..db6cbc7038 --- /dev/null +++ b/scripts/compiler_differential/_helpers/analysis_corpus/contracts.py @@ -0,0 +1,9 @@ +"""Failure cases owned by the native contracts and promotion conflicts lane.""" + +from scripts.compiler_differential.models import FailureCase + + +def contracts_failure_cases() -> tuple[FailureCase, ...]: + """Return this lane's failure cases; the lane appends here without editing shared lists.""" + + return () diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py new file mode 100644 index 0000000000..00956f4030 --- /dev/null +++ b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py @@ -0,0 +1,561 @@ +"""Minimal projects that fail SQL analysis, semantic, contract, or target validation.""" + +from scripts.compiler_differential._helpers.analysis_corpus.analysis_session import ( + analysis_session_failure_cases, +) +from scripts.compiler_differential._helpers.analysis_corpus.assembly import ( + assembly_failure_cases, +) +from scripts.compiler_differential._helpers.analysis_corpus.contracts import ( + contracts_failure_cases, +) +from scripts.compiler_differential._helpers.analysis_corpus.lineage import ( + lineage_failure_cases, +) +from scripts.compiler_differential._helpers.analysis_corpus.semantic_checks import ( + semantic_checks_failure_cases, +) +from scripts.compiler_differential._helpers.analysis_corpus.sql_test_glue import ( + sql_test_glue_failure_cases, +) +from scripts.compiler_differential._helpers.corpus.case_builder import ( + failure_case, + mart_body_files, + staging_files, +) +from scripts.compiler_differential.constants import ( + FAILURE_BASE_CONFIG, + FAILURE_BASE_STAGING, + FAILURE_CONFIG_PATH, + FAILURE_MART_PATH, + FAILURE_SOURCES_PATH, +) +from scripts.compiler_differential.models import FailureCase + +_MART_HEADER: str = '"Order totals per customer",' +_ORDERS: str = '__ref("stg_orders")' +_SNOWFLAKE_CONFIG: str = ( + 'name = "failure_corpus"\nadapter = "snowflake"\n\n' + '[connection]\naccount = "example"\nuser = "builder"\n' +) +_POSTGRES_CONFIG: str = ( + 'name = "failure_corpus"\nadapter = "postgres"\n\n' + '[connection]\nhost = "localhost"\ndatabase = "failure"\nschema = "public"\n' +) +_FUNCTION_PATH: str = "functions/sql/scaled_amount.sql" +_SCALED_AMOUNT: str = ( + "FUNCTION (\n" + ' description "Scale an amount by an integer factor",\n' + " arguments (p_amount DOUBLE, p_factor INTEGER),\n" + " returns DOUBLE,\n);\n\n" + "p_amount * p_factor\n" +) +_HOOK_PATH: str = "hooks/sql/record_refresh.sql" +_EXPRESSION_AUDIT: str = 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE NOT (@expression)\n' +_EXPRESSION_AUDIT_PATH: str = "models/marts/_sqlbuild/_audits/generic/expression_is_true.sql" +_TEST_PATH: str = "tests/unit/test_stg_orders.sql" +_ORDER_ROW: str = ( + " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount, 'placed' AS status\n" +) + + +def _mart(body: str) -> dict[str, str]: + return mart_body_files(body) + + +def _mart_header(*, extra: str, body: str | None = None) -> dict[str, str]: + files: dict[str, str] = mart_body_files( + body + or 'SELECT customer_id, SUM(amount) AS total_amount\nFROM __ref("stg_orders")\n' + "GROUP BY customer_id\n" + ) + return { + FAILURE_MART_PATH: files[FAILURE_MART_PATH].replace( + _MART_HEADER, f"{_MART_HEADER}\n{extra}" + ) + } + + +def analysis_failure_cases() -> tuple[FailureCase, ...]: + """Return the analysis-stage cases, then every analysis lane's own cases.""" + + return ( + *_binding_cases(), + *_type_cases(), + *_grouping_cases(), + *_metadata_cases(), + *_contract_cases(), + *_target_cases(), + *_syntax_cases(), + *_ordering_cases(), + *analysis_session_failure_cases(), + *semantic_checks_failure_cases(), + *contracts_failure_cases(), + *lineage_failure_cases(), + *sql_test_glue_failure_cases(), + *assembly_failure_cases(), + ) + + +def _binding_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="analysis-ambiguous-column", + expected_code="B003", + files=_mart( + "SELECT order_id\n" + f"FROM {_ORDERS} a\nJOIN {_ORDERS} b ON a.customer_id = b.customer_id\n" + ), + ), + failure_case( + name="analysis-unknown-qualifier", + expected_code="B004", + expected_message="Unknown table or alias 'x' referenced by column 'order_id'", + files=_mart(f"SELECT x.order_id\nFROM {_ORDERS} o\n"), + ), + failure_case( + name="analysis-alias-column-list-width", + expected_code="B005", + files=_mart( + "SELECT o.a AS first_column\n" + f"FROM (SELECT order_id, customer_id FROM {_ORDERS}) AS o(a, b, c)\n" + ), + ), + failure_case( + name="analysis-unknown-function", + expected_code="B101", + files=_mart( + f"SELECT customer_id, not_a_function(amount) AS total_amount\nFROM {_ORDERS}\n" + ), + ), + failure_case( + name="analysis-function-argument-count", + expected_code="B102", + files=_mart(f"SELECT customer_id, SUBSTRING(status) AS total_amount\nFROM {_ORDERS}\n"), + ), + failure_case( + name="analysis-udf-argument-count", + expected_code="B102", + files={ + _FUNCTION_PATH: _SCALED_AMOUNT, + **_mart( + 'SELECT customer_id, __udf("scaled_amount")(amount) AS total_amount\n' + f"FROM {_ORDERS}\n" + ), + }, + ), + ) + + +def _type_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="analysis-non-boolean-predicate", + expected_code="B211", + files=_mart( + f"SELECT customer_id, amount AS total_amount\nFROM {_ORDERS}\nWHERE status\n" + ), + ), + failure_case( + name="analysis-arithmetic-operands", + expected_code="B212", + files=_mart(f"SELECT customer_id, status + 1 AS total_amount\nFROM {_ORDERS}\n"), + ), + failure_case( + name="analysis-function-argument-type", + expected_code="B213", + files=_mart(f"SELECT customer_id, ABS(status) AS total_amount\nFROM {_ORDERS}\n"), + ), + failure_case( + name="analysis-set-operation-branch-types", + expected_code="B215", + files=_mart( + f"SELECT customer_id, amount FROM {_ORDERS}\n" + f"UNION ALL\nSELECT customer_id, DATE '2026-01-01' FROM {_ORDERS}\n" + ), + ), + failure_case( + name="analysis-unsupported-cast", + expected_code="B218", + files=_mart(f"SELECT customer_id, DATE '2026-13-45' AS placed_on\nFROM {_ORDERS}\n"), + ), + ) + + +def _grouping_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="analysis-aggregate-in-where", + expected_code="B231", + files=_mart( + f"SELECT customer_id\nFROM {_ORDERS}\nWHERE SUM(amount) > 1\nGROUP BY customer_id\n" + ), + ), + failure_case( + name="analysis-window-without-over", + expected_code="B232", + files=_mart(f"SELECT customer_id, ROW_NUMBER() AS position\nFROM {_ORDERS}\n"), + ), + failure_case( + name="analysis-duplicate-cte-name", + expected_code="B233", + files=_mart( + f"WITH totals AS (SELECT customer_id FROM {_ORDERS}),\n" + f"totals AS (SELECT customer_id FROM {_ORDERS})\n" + "SELECT customer_id FROM totals\n" + ), + ), + failure_case( + name="analysis-non-constant-limit", + expected_code="B234", + files=_mart(f"SELECT customer_id\nFROM {_ORDERS}\nLIMIT customer_id\n"), + ), + ) + + +def _metadata_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="analysis-unique-key-unknown-column", + expected_code="B300", + expected_message="unique_key references unknown column 'order_key'", + files=_mart_header(extra=" materialized table,\n unique_key [order_key],"), + ), + failure_case( + name="analysis-relationships-unknown-field", + expected_code="B300", + expected_message="relationships target has no column 'customer_key'", + files=_mart_header( + extra=" columns (\n" + ' customer_id (audits [relationships (to __ref("stg_orders"), ' + "field customer_key)]),\n )," + ), + ), + failure_case( + name="analysis-accepted-values-type", + expected_code="B301", + expected_message="accepted_values", + files=_mart_header( + extra=" columns (\n" + " customer_id (audits [accepted_values (values [DATE '2026-01-01'])]),\n )," + ), + ), + failure_case( + name="analysis-cursor-type-family", + expected_code="B301", + expected_message="does not match column 'customer_id' type INT", + files={ + FAILURE_MART_PATH: ( + 'MODEL (\n description "Order totals per customer",\n' + " materialized incremental,\n incremental_strategy delete_insert,\n" + " unique_key [customer_id],\n" + " cursor customer_id,\n cursor_type timestamp,\n cursor_grain day,\n" + " cursor_inputs (\n stg_orders customer_id,\n ),\n);\n\n" + "SELECT customer_id, SUM(amount) AS total_amount\n" + f"FROM {_ORDERS}\nGROUP BY customer_id\n" + ) + }, + ), + failure_case( + name="analysis-udf-argument-family", + expected_code="B301", + expected_message="Function 'scaled_amount' argument 'p_amount' expects DOUBLE", + files={ + _FUNCTION_PATH: _SCALED_AMOUNT, + **_mart( + 'SELECT customer_id, __udf("scaled_amount")(status, 2) AS total_amount\n' + f"FROM {_ORDERS}\n" + ), + }, + ), + failure_case( + name="analysis-expression-audit-unknown-column", + expected_code="B002", + expected_message="Unknown column 'discount' in output", + files={ + _EXPRESSION_AUDIT_PATH: _EXPRESSION_AUDIT, + **_mart_header(extra=' audits [expression_is_true (expression "discount >= 0")],'), + }, + ), + failure_case( + name="analysis-test-fixture-unknown-column", + expected_code="B302", + expected_message="SQL test '__source__raw_orders' names unknown column 'discount'", + files={ + _TEST_PATH: ( + "TEST();\n\nWITH\n__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount," + " 'placed' AS status, 2 AS discount\n),\n" + "__expected__stg_orders AS (\n SELECT 1 AS order_id\n)\nSELECT 1\n" + ) + }, + ), + ) + + +def _contract_cases() -> tuple[FailureCase, ...]: + contract_header: str = ' description "Staged orders",\n contract enforced,\n' + query: str = 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' + return ( + failure_case( + name="contract-nullability", + expected_code="K004", + expected_message="column 'customer_id' is declared non-null but may be nullable", + files=staging_files( + f"MODEL (\n{contract_header}" + " columns (\n order_id (type INTEGER),\n" + " customer_id (type INTEGER, nullable false),\n" + " amount (type DOUBLE),\n status (type VARCHAR),\n ),\n);\n\n" + "SELECT order_id, CAST(NULL AS INTEGER) AS customer_id, amount, status\n" + 'FROM __source("raw_orders")\n' + ), + ), + failure_case( + name="contract-without-columns", + expected_code="K006", + expected_message="model 'stg_orders' has contract enforced but declares no columns", + files=staging_files(f"MODEL (\n{contract_header});\n\n{query}"), + ), + failure_case( + name="contract-immediate-promotion", + expected_code="K011", + expected_message="contract enforced requires staged table promotion", + files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + + '\n[settings]\ntable_promotion_mode = "immediate"\n', + **staging_files( + f"MODEL (\n{contract_header} materialized table,\n" + " columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n" + " amount (type DOUBLE),\n status (type VARCHAR),\n ),\n);\n\n" + f"{query}" + ), + }, + ), + failure_case( + name="contract-dynamic-columns-unproven", + expected_code="K011", + expected_message="dynamic column contract is not proven", + files=_mart_header( + extra=" materialized table,\n columns (customer_id (type INTEGER)),\n" + " dynamic_columns (\n status_amounts (\n pivot_column status,\n" + " value_column amount,\n aggregate MAX,\n type DOUBLE\n )\n ),", + body=f"PIVOT {_ORDERS}\nON status\nUSING MAX(amount)\nGROUP BY customer_id\n", + ), + ), + failure_case( + name="contract-dynamic-columns-unsupported-dialect", + expected_code="K011", + expected_message="does not support compiler-proven", + files={ + FAILURE_CONFIG_PATH: _POSTGRES_CONFIG, + **_mart_header( + extra=" materialized table,\n columns (customer_id (type INTEGER)),\n" + " dynamic_columns (\n status_amounts (\n pivot_column status,\n" + " value_column amount,\n aggregate MAX,\n type DOUBLE\n" + " )\n ),", + body=( + "SELECT customer_id, MAX(amount) AS placed\n" + f"FROM {_ORDERS}\nGROUP BY customer_id\n" + ), + ), + }, + ), + ) + + +def _target_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="target-missing-database", + expected_code="S101", + expected_message="snowflake execution requires explicit target database", + expected_help="my_database", + files={FAILURE_CONFIG_PATH: _SNOWFLAKE_CONFIG + '\n[defaults]\nschema = "ANALYTICS"\n'}, + ), + failure_case( + name="target-managed-loader-collision", + expected_code="S102", + expected_message="Managed loader write collision", + files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + + '\n[targets.dev]\nschema = "dev"\nloader_schema = "raw"\n' + + '\n[targets.prod]\nschema = "prod"\nloader_schema = "raw"\n', + "python/loaders/orders.py": ( + "from sqlbuild.loaders import loader\n\n\n" + "@loader\n" + "def loaded_orders(ctx):\n" + ' """Load raw orders."""\n' + ' return [{"order_id": 1}]\n' + ), + "sources/loaded.yml": ( + "sources:\n - name: loaded_orders\n description: Loaded orders.\n" + " managed: true\n write_strategy: table\n" + " columns:\n - name: order_id\n type: INTEGER\n" + ), + }, + ), + failure_case( + name="target-missing-write-schema", + expected_code="S103", + expected_message="has no explicitly resolved physical write schema", + files={FAILURE_CONFIG_PATH: _SNOWFLAKE_CONFIG + 'database = "ANALYTICS"\n'}, + ), + failure_case( + name="target-named-without-schema", + expected_code="S104", + expected_message="Named target 'dev' must explicitly set schema", + files={ + FAILURE_CONFIG_PATH: _SNOWFLAKE_CONFIG.replace( + 'adapter = "snowflake"\n', 'adapter = "snowflake"\ndefault_target = "dev"\n' + ) + + 'database = "ANALYTICS"\n\n[targets.dev]\ndatabase = "ANALYTICS"\n' + }, + ), + ) + + +def _syntax_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="syntax-hook", + expected_code="P001", + expected_message="record_refresh", + files={ + _HOOK_PATH: ('HOOK (\n description "Record a refresh"\n);\n\nSELECT FROM WHERE\n'), + **_mart_header(extra=' post_hooks [sql("record_refresh")],'), + }, + ), + failure_case( + name="syntax-inline-hook", + expected_code="P001", + expected_message="hook", + files=_mart_header(extra=' post_hooks [inline_sql("SELECT FROM WHERE")],'), + ), + failure_case( + name="syntax-function", + expected_code="P001", + expected_message="scaled_amount", + files={ + _FUNCTION_PATH: _SCALED_AMOUNT.replace( + "p_amount * p_factor", "p_amount * (p_factor" + ), + **_mart( + 'SELECT customer_id, __udf("scaled_amount")(amount, 2) AS total_amount\n' + f"FROM {_ORDERS}\n" + ), + }, + ), + failure_case( + name="syntax-source-expression", + expected_code="P001", + expected_message="raw_orders", + files={ + FAILURE_SOURCES_PATH: ( + "sources:\n - name: raw_orders\n description: Orders feed.\n" + " expression: >-\n" + " (SELECT 1 AS order_id,, 10 AS customer_id,\n" + " CAST(5 AS DOUBLE) AS amount, 'placed' AS status)\n" + ) + }, + ), + failure_case( + name="analysis-rejected-opt-out-hides-findings", + expected_code="P009", + expected_message="`sql_analysis false` is not needed for model 'customer_totals'", + files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + + "\n[settings]\nrequire_sql_analysis = true\n", + **_mart_header( + extra=" sql_analysis false,", + body=( + "SELECT customer_id, SUM(refund) AS total_amount\n" + f"FROM {_ORDERS}\nGROUP BY customer_id\n" + ), + ), + }, + ), + ) + + +def _ordering_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="order-two-failing-models", + expected_code="B002", + expected_message="Unknown column 'missing_column' in stg_orders", + expected_codes=("B002", "B002"), + files={ + **staging_files( + FAILURE_BASE_STAGING.replace("amount, status", "amount, discount, status") + ), + "models/marts/order_counts.sql": ( + 'MODEL (\n description "Order counts per customer",\n);\n\n' + "SELECT customer_id, COUNT(missing_column) AS order_count\n" + f"FROM {_ORDERS}\nGROUP BY customer_id\n" + ), + }, + ), + failure_case( + name="order-binding-metadata-contract", + expected_code="B002", + expected_codes=("B002", "B300", "K001"), + files={ + **staging_files( + 'MODEL (\n description "Staged orders",\n contract enforced,\n' + " columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n" + " amount (type DOUBLE),\n status (type VARCHAR),\n" + " discount (type DOUBLE),\n ),\n);\n\n" + 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' + ), + **_mart_header( + extra=" materialized table,\n unique_key [order_key],", + body=( + "SELECT customer_id, SUM(refund) AS total_amount\n" + f"FROM {_ORDERS}\nGROUP BY customer_id\n" + ), + ), + }, + ), + failure_case( + name="order-poisoned-downstream-uses", + expected_code="B002", + expected_message="Unknown column 'refund' in raw_orders", + expected_notes=( + "1 downstream output uses were not type-checked because of this error", + ), + files={ + **staging_files( + FAILURE_BASE_STAGING.replace("amount, status", "amount, refund, status") + ), + FAILURE_MART_PATH: ( + 'MODEL (\n description "Order totals per customer",\n);\n\n' + "SELECT customer_id, SUM(refund) AS total_refund\n" + f"FROM {_ORDERS}\nGROUP BY customer_id\n" + ), + }, + ), + failure_case( + name="order-contract-error-then-warning", + expected_code="K005", + expected_warning_code="K003", + expected_codes=("K005", "K003"), + files=staging_files( + 'MODEL (\n description "Staged orders",\n contract enforced,\n' + " columns (\n order_id (type INTEGER),\n note (type VARCHAR),\n" + " customer_id (type INTEGER),\n amount (type DOUBLE),\n ),\n);\n\n" + "SELECT order_id, NULL AS note, customer_id, amount, status\n" + 'FROM __source("raw_orders")\n' + ), + ), + failure_case( + name="order-closest-column-help", + expected_code="B002", + expected_help="did you mean 'amount'?", + expected_notes=("stg_orders has: amount, status, customer_id, order_id",), + expected_location=(5, 25), + files=_mart( + "SELECT customer_id, SUM(amonut) AS total_amount\n" + f"FROM {_ORDERS}\nGROUP BY customer_id\n" + ), + ), + ) diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/lineage.py b/scripts/compiler_differential/_helpers/analysis_corpus/lineage.py new file mode 100644 index 0000000000..fe6729e247 --- /dev/null +++ b/scripts/compiler_differential/_helpers/analysis_corpus/lineage.py @@ -0,0 +1,9 @@ +"""Failure cases owned by the native FAST lineage facts lane.""" + +from scripts.compiler_differential.models import FailureCase + + +def lineage_failure_cases() -> tuple[FailureCase, ...]: + """Return this lane's failure cases; the lane appends here without editing shared lists.""" + + return () diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py b/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py new file mode 100644 index 0000000000..069b7d2986 --- /dev/null +++ b/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py @@ -0,0 +1,9 @@ +"""Failure cases owned by the native semantic completion lane.""" + +from scripts.compiler_differential.models import FailureCase + + +def semantic_checks_failure_cases() -> tuple[FailureCase, ...]: + """Return this lane's failure cases; the lane appends here without editing shared lists.""" + + return () diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/sql_test_glue.py b/scripts/compiler_differential/_helpers/analysis_corpus/sql_test_glue.py new file mode 100644 index 0000000000..456d975490 --- /dev/null +++ b/scripts/compiler_differential/_helpers/analysis_corpus/sql_test_glue.py @@ -0,0 +1,9 @@ +"""Failure cases owned by the native SQL-test planning glue lane.""" + +from scripts.compiler_differential.models import FailureCase + + +def sql_test_glue_failure_cases() -> tuple[FailureCase, ...]: + """Return this lane's failure cases; the lane appends here without editing shared lists.""" + + return () diff --git a/scripts/compiler_differential/_helpers/comparing/comparison.py b/scripts/compiler_differential/_helpers/comparing/comparison.py index f10225a405..53c5976c0b 100644 --- a/scripts/compiler_differential/_helpers/comparing/comparison.py +++ b/scripts/compiler_differential/_helpers/comparing/comparison.py @@ -86,6 +86,13 @@ def first_diagnostic_message(*, outcome: CommandOutcome, severity: str) -> str | return str(diagnostics[0].get("message")) if diagnostics else None +def first_diagnostic(*, outcome: CommandOutcome, severity: str) -> dict[str, object]: + """Return a JSON compile report's first diagnostic of one severity, or an empty mapping.""" + + diagnostics: list[dict[str, object]] = _diagnostics(outcome=outcome, severity=severity) + return diagnostics[0] if diagnostics else {} + + def _diagnostics(*, outcome: CommandOutcome, severity: str | None) -> list[dict[str, object]]: payload: dict[str, object] = as_json_object(_loads(outcome.stdout)) or {} diagnostics: object = payload.get("diagnostics") diff --git a/scripts/compiler_differential/_helpers/comparing/project_comparison.py b/scripts/compiler_differential/_helpers/comparing/project_comparison.py index 7e5c1397c4..ac14b3d1fd 100644 --- a/scripts/compiler_differential/_helpers/comparing/project_comparison.py +++ b/scripts/compiler_differential/_helpers/comparing/project_comparison.py @@ -10,6 +10,7 @@ compare_engine_runs, diagnostic_codes, ) +from scripts.compiler_differential._helpers.coverage.analysis import project_analysis_kinds from scripts.compiler_differential._helpers.coverage.discovery import project_discovery_kinds from scripts.compiler_differential._helpers.coverage.render import project_render_kinds from scripts.compiler_differential._helpers.running.execution import run_engine @@ -74,6 +75,8 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) -> else frozenset() ), rendered_kinds=project_render_kinds(left.captures) if counted else frozenset(), + analysed_kinds=project_analysis_kinds(left.captures) if counted else frozenset(), + records=(left.records, right.records) if options.analysis_records else None, ) diff --git a/scripts/compiler_differential/_helpers/corpus/case_builder.py b/scripts/compiler_differential/_helpers/corpus/case_builder.py index 7e3bc2886e..bd6f445942 100644 --- a/scripts/compiler_differential/_helpers/corpus/case_builder.py +++ b/scripts/compiler_differential/_helpers/corpus/case_builder.py @@ -19,8 +19,12 @@ def failure_case( files: dict[str, str], expected_warning_code: str | None = None, expected_message: str | None = None, + expected_help: str | None = None, + expected_notes: tuple[str, ...] = (), + expected_location: tuple[int, int] | None = None, + expected_codes: tuple[str, ...] | None = None, ) -> FailureCase: - """Return the base project with `files` added or replaced.""" + """Return the base project with `files` added or replaced; notes are whole note lines.""" return FailureCase( files={**FAILURE_BASE_FILES, **files}, @@ -28,6 +32,10 @@ def failure_case( expected_code=expected_code, expected_warning_code=expected_warning_code, expected_message=expected_message, + expected_help=expected_help, + expected_notes=expected_notes, + expected_location=expected_location, + expected_codes=expected_codes, ) diff --git a/scripts/compiler_differential/_helpers/corpus/corpus.py b/scripts/compiler_differential/_helpers/corpus/corpus.py index b217c7f0c0..9fecf60b33 100644 --- a/scripts/compiler_differential/_helpers/corpus/corpus.py +++ b/scripts/compiler_differential/_helpers/corpus/corpus.py @@ -2,6 +2,7 @@ from __future__ import annotations +import itertools import tomllib from pathlib import Path @@ -9,6 +10,7 @@ from scripts.compiler_differential.classes.dense_project import DenseProject from scripts.compiler_differential.classes.project_builder import ProjectBuilder from scripts.compiler_differential.constants import ( + ANALYSIS_DIALECT_VARIANTS, COLD_COMPILE, CORPUS_DENSE, CORPUS_EXAMPLES, @@ -21,6 +23,7 @@ FIXTURE_EXPECTED_OUTCOMES, FIXTURE_PROJECT_SUBDIRECTORIES, FIXTURE_ROOT, + GENERATOR_DIALECT_BLOCKS, PLAN, PROJECT_CONFIG_FILE, STORE_WARM_COMPILE, @@ -76,6 +79,10 @@ def build_corpus( ) if CORPUS_SEEDS in corpora: entries.extend(_seed_project(seed) for seed in seeds) + entries.extend( + _dialect_seed_project(seed=seed, dialect=dialect) + for seed, dialect in itertools.product(seeds[:1], ANALYSIS_DIALECT_VARIANTS) + ) if CORPUS_FAILURES in corpora: entries.extend( CorpusProject( @@ -138,7 +145,7 @@ def _seed_project(seed: int) -> CorpusProject: generated: GeneratedProject = ProjectBuilder(seed).build() return CorpusProject( name=f"seed/{seed}", - commands=(COLD_COMPILE, STORE_WARM_COMPILE, PLAN), + commands=(COLD_COMPILE, STORE_WARM_COMPILE, PLAN, *generated.extra_commands), expected=ExpectedOutcome( error_code=generated.expected_error_code, succeeding_commands=generated.succeeding_commands, @@ -146,3 +153,18 @@ def _seed_project(seed: int) -> CorpusProject: writer=generated.write, seed_coverage=True, ) + + +def _dialect_seed_project(*, seed: int, dialect: str) -> CorpusProject: + """Compile a seed's analysis blocks under another adapter's dialect; plan needs a warehouse.""" + + generated: GeneratedProject = ProjectBuilder( + seed, blocks=GENERATOR_DIALECT_BLOCKS, dialect=dialect + ).build() + return CorpusProject( + name=f"seed/{seed}-{dialect}", + commands=(COLD_COMPILE,), + expected=EXPECT_SUCCESS, + writer=generated.write, + seed_coverage=True, + ) diff --git a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py index 90b28c7d88..c7c2af8612 100644 --- a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py +++ b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py @@ -8,11 +8,13 @@ from scripts.compiler_differential._helpers.comparing.comparison import ( diagnostic_codes, + first_diagnostic, first_diagnostic_message, ) from scripts.compiler_differential._helpers.corpus.corpus import build_corpus from scripts.compiler_differential._helpers.running.execution import run_engine from scripts.compiler_differential.constants import ( + ANALYSIS_CODE_SCAN_ROOTS, CORPUS_FAILURES, DIAGNOSTIC_CODE_PATTERN, ERROR_SEVERITY, @@ -30,6 +32,7 @@ ) from sqlbuild.compiler.auditing.constants import BUILT_IN_AUDIT_SHADOW_CODE from sqlbuild.compiler.compile import constants as compile_constants +from sqlbuild.compiler.compile._helpers.diagnostics.help import semantic_help_catalogue from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.exceptions import DiscoveryError from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError @@ -91,6 +94,17 @@ def render_error_codes() -> frozenset[str]: ) +def analysis_error_codes(compiler_root: Path) -> frozenset[str]: + """Return the help-catalogue codes plus every code literal in the analysis-stage modules.""" + + found: set[str] = set(semantic_help_catalogue()) + for root in ANALYSIS_CODE_SCAN_ROOTS: + path: Path = compiler_root / root + for module in sorted(path.rglob("*.py")) if path.is_dir() else (path,): + found.update(_module_codes(module)) + return frozenset(found) + + def inline_compile_codes(compile_root: Path) -> dict[str, tuple[str, ...]]: """Map each `code="X###"` literal in render-stage compile modules to where it is written.""" @@ -118,6 +132,30 @@ def unaccounted_inline_codes( } +def _module_codes(path: Path) -> set[str]: + codes: set[str] = set() + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + literal: str | None = _code_literal(node) or _code_constant(node) + if literal is not None: + codes.add(literal) + return codes + + +def _code_constant(node: ast.AST) -> str | None: + if not isinstance(node, (ast.Assign, ast.AnnAssign)): + return None + targets: list[ast.expr] = node.targets if isinstance(node, ast.Assign) else [node.target] + value: ast.expr | None = node.value + if not ( + any(isinstance(target, ast.Name) and target.id.endswith("_CODE") for target in targets) + and isinstance(value, ast.Constant) + and isinstance(value.value, str) + and DIAGNOSTIC_CODE_PATTERN.fullmatch(value.value) + ): + return None + return value.value + + def _code_literal(node: ast.AST) -> str | None: if not ( isinstance(node, ast.keyword) @@ -143,10 +181,20 @@ def _emitted_codes(*, project: CorpusProject, options: DifferentialOptions) -> E side=LEFT_SIDE, options=options, ) + first: dict[str, object] = first_diagnostic(outcome=run.outcomes[0], severity=ERROR_SEVERITY) + notes: object = first.get("notes") + line: object = first.get("line") + column: object = first.get("column") return EmittedCodes( errors=diagnostic_codes(outcome=run.outcomes[0], severity=ERROR_SEVERITY), warnings=diagnostic_codes(outcome=run.outcomes[0], severity=WARNING_SEVERITY), first_error_message=first_diagnostic_message( outcome=run.outcomes[0], severity=ERROR_SEVERITY ), + first_error_help=None if first.get("help") is None else str(first.get("help")), + first_error_notes=tuple(str(note) for note in notes) if isinstance(notes, list) else (), + first_error_location=( + (line, column) if isinstance(line, int) and isinstance(column, int) else None + ), + codes=diagnostic_codes(outcome=run.outcomes[0], severity=None), ) diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index 4ff02ce71e..f041e96d90 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -1,5 +1,8 @@ """Minimal failing projects, each derived from the shared base project.""" +from scripts.compiler_differential._helpers.analysis_corpus.failure_cases import ( + analysis_failure_cases, +) from scripts.compiler_differential._helpers.corpus.case_builder import ( failure_case, mart_body_files, @@ -45,6 +48,7 @@ def all_failure_cases() -> tuple[FailureCase, ...]: *render_failure_cases(), *scope_failure_cases(), *reference_failure_cases(), + *analysis_failure_cases(), ) diff --git a/scripts/compiler_differential/_helpers/coverage/analysis.py b/scripts/compiler_differential/_helpers/coverage/analysis.py new file mode 100644 index 0000000000..e4e6c4e7d8 --- /dev/null +++ b/scripts/compiler_differential/_helpers/coverage/analysis.py @@ -0,0 +1,539 @@ +"""Classify `CompiledProject` captures into analysis kinds; only analysed evidence counts.""" + +from __future__ import annotations + +import dataclasses +import re +from collections.abc import Iterator +from pathlib import Path + +from scripts.compiler_differential._helpers.comparing.compare import as_json_object +from scripts.compiler_differential._helpers.coverage.capture_values import ( + capture_problems, + enum_name, + has_content, + records, +) +from scripts.compiler_differential.classes.capture_file import expand_capture_text +from scripts.compiler_differential.constants import ( + ANALYSIS_CALLABLE_CAPTURE_FIELDS, + ANALYSIS_DETAIL_KINDS, + ANALYSIS_EXPLICIT_WRITE_SCHEMA_DIALECTS, + ANALYSIS_NON_COLLECTION_FIELDS, + ANALYSIS_OPAQUE_CAPTURE_VALUES, + ANALYSIS_STAGE_CAPTURE_SUFFIX, + ANALYSIS_UPSTREAM_CHAIN_HOPS, + COMPILED_PROJECT_CAPTURE_TYPE, +) +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_OMITTED_ATTRIBUTES + +_SET_OPERATION: re.Pattern[str] = re.compile(r"\b(?:UNION|INTERSECT|EXCEPT)\b", re.IGNORECASE) +_LEADING_CTE: re.Pattern[str] = re.compile(r"^\s*WITH\s+\w+\s+AS\s*\(", re.IGNORECASE) +_QUOTED_IDENTIFIER: re.Pattern[str] = re.compile(r'"([^"]+)"') +_SIZED_TYPE: re.Pattern[str] = re.compile(r"\((\d+)\)\s*$") +_RUNTIME_PLACEHOLDER: re.Pattern[str] = re.compile(r"@@@([A-Za-z_]\w*)") +_CURSOR_INTRINSIC: str = "__cursor_start()" +_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") +_EXPECTED_CTE_PREFIX: str = "__expected__" +_SQL_HOOK_TYPE: str = "SqlHookEntry" +_NON_NULL: str = "non_null" +_ENFORCED: str = "enforced" +_TABLE: str = "table" +_INCREMENTAL: str = "incremental" +_STAGED: str = "staged" +_MODEL_REFERENCE: str = "ref" +_SOURCE_REFERENCE: str = "source" +_SEED_REFERENCE: str = "seed" +_UDF_REFERENCE: str = "udf" +_TABLE_FUNCTION_REFERENCE: str = "table_function" +_AUDIT_KINDS: dict[str, str] = { + "accepted_values": "accepted_values_audit", + "relationships": "relationships_audit", + "expression_is_true": "expression_audit", +} +_METADATA_COLUMN_KEYS: tuple[str, ...] = ("unique_key", "cursor", "partition_column") + + +@dataclasses.dataclass(frozen=True) +class _Project: + """The parts of one `CompiledProject` capture the analysis kinds read.""" + + capture: dict[str, object] + models: dict[str, dict[str, object]] + sources: dict[str, dict[str, object]] + seeds: dict[str, dict[str, object]] + functions: dict[str, dict[str, object]] + + +def required_analysis_kinds() -> tuple[str, ...]: + """Return every `CompiledProject` collection field plus the finer required kinds.""" + + return (*analysis_collection_kinds(), *ANALYSIS_DETAIL_KINDS) + + +def analysis_collection_kinds() -> tuple[str, ...]: + """Return the `CompiledProject` fields that hold compiled resources or facts.""" + + return tuple( + field.name + for field in dataclasses.fields(CompiledProject) + if field.name not in ANALYSIS_NON_COLLECTION_FIELDS + ) + + +def project_analysis_kinds(captures: dict[str, dict[str, Path]]) -> frozenset[str]: + """Return the analysis kinds every `CompiledProject` capture of one project proves.""" + + kinds: set[str] = set() + for files in captures.values(): + for name, path in files.items(): + if name.endswith(ANALYSIS_STAGE_CAPTURE_SUFFIX): + kinds.update(analysed_input_kinds(path.read_text(encoding="utf-8"))) + return frozenset(kinds) + + +def analysis_capture_problems(capture_text: str) -> tuple[str, ...]: + """Return why a compiled-project capture is unsound; the catalog's native handle is allowed.""" + + return capture_problems( + capture_text=ANALYSIS_OPAQUE_CAPTURE_VALUES.sub("null", capture_text), + fields=tuple( + field.name + for field in dataclasses.fields(CompiledProject) + if field.name not in _omitted_fields() + ), + callable_fields=ANALYSIS_CALLABLE_CAPTURE_FIELDS, + ) + + +def analysed_input_kinds(capture_text: str) -> frozenset[str]: + """Return the analysis input kinds one canonical `CompiledProject` capture proves.""" + + capture: dict[str, object] = as_json_object(expand_capture_text(capture_text)) or {} + project: _Project = _Project( + capture=capture, + models=_by_name(capture.get("models")), + sources={ + str((as_json_object(source.get("source_entry")) or {}).get("name")): source + for source in records(capture.get("sources")) + }, + seeds=_by_name(capture.get("seeds")), + functions=_by_name(capture.get("functions")), + ) + kinds: set[str] = { + kind for kind in analysis_collection_kinds() if has_content(capture.get(kind)) + } + kinds.update(_inference_kinds(project)) + kinds.update(_binding_path_kinds(project)) + kinds.update(_shape_publication_kinds(project)) + kinds.update(_input_resource_kinds(project)) + kinds.update(_configuration_kinds(project)) + kinds.update(_metadata_kinds(project)) + kinds.update(_attachment_kinds(project)) + dialect: object = capture.get("sql_analysis_dialect") + if isinstance(dialect, str) and any(_analysed(model) for model in project.models.values()): + kinds.add(f"dialect_{dialect}") + return frozenset(kinds) + + +def _omitted_fields() -> frozenset[str]: + return STAGE_CAPTURE_OMITTED_ATTRIBUTES.get(COMPILED_PROJECT_CAPTURE_TYPE, frozenset()) + + +def _by_name(value: object) -> dict[str, dict[str, object]]: + return {str(record.get("name")): record for record in records(value)} + + +def _analysed(model: dict[str, object]) -> bool: + """Whether binding validation ran on the model's SQL.""" + + return model.get("binding_validated") is True + + +def _columns(model: dict[str, object]) -> list[dict[str, object]]: + return records(model.get("inferred_columns")) + + +def _values(resource: dict[str, object]) -> dict[str, object]: + return as_json_object((as_json_object(resource.get("config")) or {}).get("values")) or {} + + +def _references(model: dict[str, object]) -> Iterator[tuple[str, str]]: + for reference in records(model.get("references")): + yield str(enum_name(reference.get("ref_kind"))), str(reference.get("ref_name")) + + +def _referenced(*, model: dict[str, object], kind: str) -> list[str]: + return [name for ref_kind, name in _references(model) if ref_kind == kind] + + +def _contract_enforced(model: dict[str, object]) -> bool: + return _values(model).get("contract") == _ENFORCED + + +def _schema_columns(resource: dict[str, object]) -> list[dict[str, object]]: + return records((as_json_object(resource.get("schema_entry")) or {}).get("columns")) + + +def _nullability(column: dict[str, object]) -> str | None: + return enum_name(column.get("nullability")) + + +def _inference_kinds(project: _Project) -> set[str]: + kinds: set[str] = set() + for model in project.models.values(): + if not _analysed(model): + continue + columns: list[dict[str, object]] = _columns(model) + if any(column.get("type") is not None for column in columns): + kinds.add("typed_column") + if any(column.get("type") is None for column in columns): + kinds.add("untyped_column") + upstream_models: list[str] = [ + name + for name in _referenced(model=model, kind=_MODEL_REFERENCE) + if name in project.models + ] + if model.get("fast_lineage_has_star") is True: + if model.get("fast_lineage_star_resolved") is not True: + if model.get("dynamic_column_contract") is None: + kinds.add("star_unresolved") + elif upstream_models: + kinds.add("star_over_published_shape") + else: + kinds.add("star_over_complete_input") + query: str = str(model.get("query_sql", "")) + if _SET_OPERATION.search(query) and upstream_models and columns: + kinds.add("set_operation") + if _LEADING_CTE.search(query) and columns: + kinds.add("cte") + if _passes_cte_type_through(project=project, model=model, upstream=upstream_models): + kinds.add("cte_passthrough_type") + if _QUOTED_IDENTIFIER.search(query) and _quoted_case_folded( + project=project, model=model, upstream=upstream_models + ): + kinds.add("quoted_identifier_case") + proof: dict[str, object] = as_json_object(model.get("dynamic_column_contract")) or {} + if proof.get("output_proven") is True and records(proof.get("families")): + kinds.add("dynamic_column_proof") + values: dict[str, object] = _values(model) + if ( + _CURSOR_INTRINSIC in query + and values.get("materialized") == _INCREMENTAL + and values.get("cursor") is not None + ): + kinds.add("cursor_intrinsic") + placeholders: set[str] = set(_RUNTIME_PLACEHOLDER.findall(query)) + defaults: dict[str, object] = as_json_object(values.get("placeholders")) or {} + if placeholders and placeholders <= set(defaults): + kinds.add("runtime_placeholder") + return kinds + + +def _passes_cte_type_through( + *, project: _Project, model: dict[str, object], upstream: list[str] +) -> bool: + """Whether a CTE model keeps an upstream column's exact inferred type.""" + + upstream_types: set[tuple[str, str]] = set().union( + *(_typed_columns(project.models[name]) for name in upstream) + ) + return bool(_typed_columns(model) & upstream_types) + + +def _typed_columns(model: dict[str, object]) -> set[tuple[str, str]]: + """Return the model's `(name, type)` pairs for every inferred column with a type.""" + + return { + (str(column.get("name")), str(column.get("type"))) + for column in _columns(model) + if column.get("type") is not None + } + + +def _quoted_case_folded( + *, project: _Project, model: dict[str, object], upstream: list[str] +) -> bool: + """Whether a quoted identifier bound to an upstream column only by ignoring its case.""" + + names: set[str] = set().union(*(_column_names_of(project.models[name]) for name in upstream)) + folded: set[str] = {name.casefold() for name in names} + return not records(model.get("binding_diagnostics")) and any( + quoted not in names and quoted.casefold() in folded + for quoted in _QUOTED_IDENTIFIER.findall(str(model.get("query_sql", ""))) + ) + + +def _column_names_of(model: dict[str, object]) -> set[str]: + return {str(column.get("name")) for column in _columns(model)} + + +def _binding_path_kinds(project: _Project) -> set[str]: + """Distinguish the dependency-ordered binding dataflow from the one-batch path.""" + + kinds: set[str] = set() + analysed: list[dict[str, object]] = [ + model for model in project.models.values() if _analysed(model) + ] + complete: list[bool] = [ + _reads_complete_inputs(project=project, model=model) for model in analysed + ] + if analysed and all(complete): + kinds.add("batch_binding") + if not all(complete): + kinds.add("dataflow_binding") + analysable: list[dict[str, object]] = [ + model + for model in project.models.values() + if _values(model).get("sql_analysis") is not False and records(model.get("references")) + ] + if analysed and any( + model.get("inferred_columns") is None and not _analysed(model) for model in analysable + ): + kinds.add("select_limited_analysis") + if ( + project.models + and not analysed + and all( + model.get("inferred_columns") is None + for model in analysable + if model.get("dynamic_column_contract") is None + ) + ): + kinds.add("sql_analysis_disabled") + if _longest_analysed_chain(project) >= ANALYSIS_UPSTREAM_CHAIN_HOPS: + kinds.add("upstream_chain") + return kinds + + +def _reads_complete_inputs(*, project: _Project, model: dict[str, object]) -> bool: + """Whether every relation the model reads has a schema known before any model is analysed.""" + + for kind, name in _references(model): + if kind == _MODEL_REFERENCE: + upstream: dict[str, object] | None = project.models.get(name) + if upstream is None or not (_contract_enforced(upstream) and _schema_columns(upstream)): + return False + elif kind == _SOURCE_REFERENCE: + entry: dict[str, object] = ( + as_json_object((project.sources.get(name) or {}).get("source_entry")) or {} + ) + if not entry.get("expression") and not ( + entry.get("contract") == _ENFORCED and records(entry.get("columns")) + ): + return False + elif kind == _SEED_REFERENCE: + if not _schema_columns(project.seeds.get(name) or {}): + return False + elif kind == _TABLE_FUNCTION_REFERENCE: + if not records((project.functions.get(name) or {}).get("return_columns")): + return False + return True + + +def _longest_analysed_chain(project: _Project) -> int: + """Return the most model-to-model hops along analysed models, ignoring reference cycles.""" + + hops: dict[str, int] = dict.fromkeys( + (name for name, model in project.models.items() if _analysed(model)), 0 + ) + for _ in range(len(hops)): + hops = {name: _upstream_hops(project=project, name=name, hops=hops) for name in hops} + return max(hops.values(), default=0) + + +def _upstream_hops(*, project: _Project, name: str, hops: dict[str, int]) -> int: + """Return one more hop than the deepest analysed upstream model, capped by the model count.""" + + upstream: list[int] = [ + hops[upstream_name] + for upstream_name in _referenced(model=project.models[name], kind=_MODEL_REFERENCE) + if upstream_name in hops + ] + return min(max(upstream, default=-1) + 1, len(hops)) + + +def _shape_publication_kinds(project: _Project) -> set[str]: + """Prove a contract's declared types and nullability reached a downstream model's shape.""" + + kinds: set[str] = set() + for model in project.models.values(): + if not _analysed(model): + continue + downstream: dict[str, dict[str, object]] = { + str(column.get("name")): column for column in _columns(model) + } + for name in _referenced(model=model, kind=_MODEL_REFERENCE): + upstream: dict[str, object] | None = project.models.get(name) + if upstream is None or not _contract_enforced(upstream): + continue + inferred: dict[str, dict[str, object]] = { + str(column.get("name")): column for column in _columns(upstream) + } + for declared in _schema_columns(upstream): + column_name: str = str(declared.get("name")) + published: dict[str, object] | None = downstream.get(column_name) + if published is None: + continue + size: re.Match[str] | None = _SIZED_TYPE.search(str(declared.get("type") or "")) + published_size: re.Match[str] | None = _SIZED_TYPE.search( + str(published.get("type") or "") + ) + if size and published_size and size.group(1) == published_size.group(1): + kinds.add("sized_contract_shape") + if ( + declared.get("nullable") is False + and _nullability(published) == _NON_NULL + and _nullability(inferred.get(column_name) or {}) != _NON_NULL + ): + kinds.add("contract_nullability_shape") + return kinds + + +def _input_resource_kinds(project: _Project) -> set[str]: + kinds: set[str] = set() + expression_shapes: object = (as_json_object(project.capture.get("binding_catalog")) or {}).get( + "expression_shapes" + ) + for model in project.models.values(): + if not _analysed(model): + continue + for kind, name in _references(model): + if kind == _SOURCE_REFERENCE and has_content(expression_shapes): + entry: dict[str, object] = ( + as_json_object((project.sources.get(name) or {}).get("source_entry")) or {} + ) + if entry.get("expression"): + kinds.add("source_expression_shape") + elif kind == _SEED_REFERENCE and any( + column.get("type") for column in _schema_columns(project.seeds.get(name) or {}) + ): + kinds.add("seed_column_types") + elif kind in (_UDF_REFERENCE, _TABLE_FUNCTION_REFERENCE): + kinds.update(_function_kinds(function=project.functions.get(name), kind=kind)) + return kinds + + +def _function_kinds(*, function: dict[str, object] | None, kind: str) -> set[str]: + if function is None: + return set() + arguments: list[dict[str, object]] = records(function.get("arguments")) + kinds: set[str] = set() + if arguments and all(argument.get("type") for argument in arguments): + kinds.add("udf_argument_types") + if kind == _TABLE_FUNCTION_REFERENCE and records(function.get("return_columns")): + kinds.add("table_function") + elif kind == _UDF_REFERENCE and function.get("returns"): + kinds.add("scalar_udf") + return kinds + + +def _configuration_kinds(project: _Project) -> set[str]: + kinds: set[str] = set() + settings: dict[str, object] = as_json_object(project.capture.get("settings")) or {} + for model in project.models.values(): + values: dict[str, object] = _values(model) + if ( + settings.get("require_sql_analysis") is True + and values.get("sql_analysis") is False + and model.get("inferred_columns") is None + and model.get("rejected_sql_analysis_opt_out") is None + ): + kinds.add("sql_analysis_opt_out") + for key in _HOOK_KEYS: + hooks: list[dict[str, object]] = [ + hook + for hook in records(values.get(key)) + if str(hook.get("__type__", "")).endswith(_SQL_HOOK_TYPE) + ] + kinds.update( + "named_sql_hook" if hook.get("definition_sql") else "inline_sql_hook" + for hook in hooks + ) + if len(hooks) > 1: + kinds.add("hook_list") + if ( + settings.get("table_promotion_mode") == _STAGED + and _contract_enforced(model) + and values.get("materialized", _TABLE) == _TABLE + ): + kinds.add("explicit_promotion_mode") + if project.capture.get("enforce_explicit_references") is True and has_content( + project.capture.get("unmatched_literal_sql_relations") + ): + kinds.add("python_sql_literal_relation") + destinations: list[dict[str, object]] = [ + as_json_object(resource.get("destination")) or {} + for resource in (*project.models.values(), *project.seeds.values()) + ] + if ( + str(project.capture.get("sql_analysis_dialect")) in ANALYSIS_EXPLICIT_WRITE_SCHEMA_DIALECTS + and destinations + and all(destination.get("schema") for destination in destinations) + ): + kinds.add("managed_write_schema") + return kinds + + +def _metadata_kinds(project: _Project) -> set[str]: + kinds: set[str] = set() + for model in project.models.values(): + if not _analysed(model): + continue + entry: dict[str, object] = as_json_object(model.get("schema_entry")) or {} + audits: list[dict[str, object]] = records(entry.get("audits")) + for column in records(entry.get("columns")): + audits.extend(records(column.get("audits"))) + kinds.update( + _AUDIT_KINDS[name] + for audit in audits + if (name := str(audit.get("definition_name"))) in _AUDIT_KINDS + ) + values: dict[str, object] = _values(model) + output: set[str] = {str(column.get("name")).casefold() for column in _columns(model)} + if any( + (names := _column_names(values.get(key))) and names <= output + for key in _METADATA_COLUMN_KEYS + ): + kinds.add("metadata_column_reference") + return kinds + + +def _column_names(value: object) -> set[str]: + """Return the case-folded column names a metadata key such as `unique_key` names.""" + + if isinstance(value, str): + return {value.casefold()} + if isinstance(value, list): + return {str(name).casefold() for name in value} + return set() + + +def _attachment_kinds(project: _Project) -> set[str]: + kinds: set[str] = set() + for test in records(project.capture.get("sql_tests")): + payload: dict[str, object] = as_json_object(test.get("payload")) or {} + expected: list[str] = [ + str(cte.get("name", "")).removeprefix(_EXPECTED_CTE_PREFIX) + for cte in records(payload.get("expected_ctes")) + ] + if any(_analysed(project.models.get(name) or {}) for name in expected): + kinds.add("sql_test_expected_columns") + mocks: list[str] = [ + str(mock) for mock in (as_json_object(payload.get("macro_mocks")) or {}).values() + ] + overrides: dict[str, object] = as_json_object(payload.get("model_query_overrides")) or {} + if any( + _mock_rendered(project=project, mocks=mocks, name=name, override=str(override)) + for name, override in overrides.items() + ): + kinds.add("sql_test_macro_mock") + return kinds + + +def _mock_rendered(*, project: _Project, mocks: list[str], name: str, override: str) -> bool: + """Whether a test's re-expanded query holds a mocked macro value the model's own SQL lacks.""" + + compiled: str = str((project.models.get(name) or {}).get("query_sql", "")) + return any(mock in override and mock not in compiled for mock in mocks) diff --git a/scripts/compiler_differential/_helpers/running/execution.py b/scripts/compiler_differential/_helpers/running/execution.py index 3c4b283cb0..c58f5d32a8 100644 --- a/scripts/compiler_differential/_helpers/running/execution.py +++ b/scripts/compiler_differential/_helpers/running/execution.py @@ -7,7 +7,9 @@ import subprocess from pathlib import Path +from scripts.compiler_differential._helpers.running.records import read_analysis_records from scripts.compiler_differential.constants import ( + ANALYSIS_RECORD_ENV_VAR, CAPTURES_DIRECTORY, COMMAND_TIMEOUT_SECONDS, COMPILED_DIRECTORY, @@ -18,6 +20,7 @@ GENERATOR_ENVIRONMENT, MANIFEST_FILE, PROJECT_DIRECTORY, + RECORDS_DIRECTORY, SQB_ENTRY, STAGE_CAPTURE_ENV_VAR, ) @@ -60,17 +63,27 @@ def run_engine( ) capture_root: Path = case_dir / f"{side}-{engine}-{CAPTURES_DIRECTORY}" shutil.rmtree(capture_root, ignore_errors=True) + record_root: Path = case_dir / f"{side}-{engine}-{RECORDS_DIRECTORY}" + shutil.rmtree(record_root, ignore_errors=True) outcomes: list[CommandOutcome] = [] for index, command in enumerate(project.commands): capture_dir: Path | None = ( capture_root / f"{index}-{command.label}" if options.stage_captures else None ) + record_dir: Path | None = ( + record_root / f"{index}-{command.label}" if options.analysis_records else None + ) outcomes.append( _run_command( command=command, project_dir=project_dir, environment={ - **_command_environment(engine=engine, options=options, capture_dir=capture_dir), + **_command_environment( + engine=engine, + options=options, + capture_dir=capture_dir, + record_dir=record_dir, + ), **dict(command.environment), }, python=options.python, @@ -83,6 +96,7 @@ def run_engine( manifest=_optional_text(project_dir / MANIFEST_FILE), dag=_optional_text(project_dir / DAG_FILE), captures=_captures(capture_root), + records=read_analysis_records(record_root) if options.analysis_records else None, ) retained: Path = case_dir / engine shutil.rmtree(retained, ignore_errors=True) @@ -105,7 +119,11 @@ def harness_environment() -> dict[str, str]: def _command_environment( - *, engine: str, options: DifferentialOptions, capture_dir: Path | None + *, + engine: str, + options: DifferentialOptions, + capture_dir: Path | None, + record_dir: Path | None, ) -> dict[str, str]: environment: dict[str, str] = { **harness_environment(), @@ -114,6 +132,8 @@ def _command_environment( } if capture_dir is not None: environment[STAGE_CAPTURE_ENV_VAR] = str(capture_dir) + if record_dir is not None: + environment[ANALYSIS_RECORD_ENV_VAR] = str(record_dir) return environment diff --git a/scripts/compiler_differential/_helpers/running/records.py b/scripts/compiler_differential/_helpers/running/records.py new file mode 100644 index 0000000000..b1ab9e0145 --- /dev/null +++ b/scripts/compiler_differential/_helpers/running/records.py @@ -0,0 +1,84 @@ +"""Read the debug analysis records one engine's processes wrote: wheel call sites and deferrals.""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path +from typing import cast + +from scripts.compiler_differential._helpers.comparing.compare import as_json_object +from scripts.compiler_differential.constants import ( + ANALYSIS_DEFERRAL_RECORD_PREFIX, + WHEEL_SITE_RECORD_PREFIX, +) +from scripts.compiler_differential.models import AnalysisRecords, ProjectComparison + + +def read_analysis_records(root: Path) -> AnalysisRecords: + """Sum every process's wheel-site counts and `{"kind", "site"}` deferral lines below `root`.""" + + sites: Counter[tuple[str, str]] = Counter() + deferrals: Counter[tuple[str, str]] = Counter() + for path in sorted(root.rglob(f"{WHEEL_SITE_RECORD_PREFIX}*.json")) if root.is_dir() else (): + payload: dict[str, object] = as_json_object(json.loads(path.read_text("utf-8"))) or {} + rows: object = payload.get("calls") + for row in rows if isinstance(rows, list) else (): + site, api, count = cast(list[object], row) + sites[(str(site), str(api))] += int(str(count)) + for path in ( + sorted(root.rglob(f"{ANALYSIS_DEFERRAL_RECORD_PREFIX}*.jsonl")) if root.is_dir() else () + ): + for line in path.read_text("utf-8").splitlines(): + record: dict[str, object] = as_json_object(json.loads(line)) or {} + deferrals[(str(record.get("kind")), str(record.get("site")))] += 1 + return AnalysisRecords(wheel_sites=dict(sites), deferrals=dict(deferrals)) + + +def wheel_site_report( + *, comparisons: list[ProjectComparison], engines: tuple[str, str] +) -> dict[str, object]: + """Return wheel calls per engine, site and API, split by corpus, plus deferral counts.""" + + report: dict[str, object] = {} + for side, engine in enumerate(engines): + sites: dict[str, dict[str, int]] = {} + deferrals: dict[str, dict[str, int]] = {} + for comparison in comparisons: + records: AnalysisRecords | None = ( + comparison.records[side] if comparison.records else None + ) + if records is None: + continue + corpus: str = comparison.project.split("/", 1)[0] + for (site, api), count in records.wheel_sites.items(): + by_corpus: dict[str, int] = sites.setdefault(f"{site} {api}", {}) + by_corpus[corpus] = by_corpus.get(corpus, 0) + count + for (kind, site), count in records.deferrals.items(): + by_corpus = deferrals.setdefault(f"{kind} {site}", {}) + by_corpus[corpus] = by_corpus.get(corpus, 0) + count + report[engine] = { + "wheel_sites": dict(sorted(sites.items())), + "deferrals": dict(sorted(deferrals.items())), + } + return report + + +def format_wheel_site_report(report: dict[str, object]) -> str: + """Render the wheel-site and deferral counts for the run log; nothing here gates.""" + + lines: list[str] = [] + for engine, raw in report.items(): + entry: dict[str, object] = as_json_object(raw) or {} + for label, key in ( + ("Polyglot wheel calls", "wheel_sites"), + ("Analysis deferrals", "deferrals"), + ): + counts: dict[str, object] = as_json_object(entry.get(key)) or {} + lines.append(f"{label} ({engine}): {'none recorded' if not counts else ''}".rstrip()) + for name, by_corpus in counts.items(): + corpora: dict[str, object] = as_json_object(by_corpus) or {} + split: str = ", ".join(f"{corpus} {count}" for corpus, count in corpora.items()) + total: int = sum(int(str(count)) for count in corpora.values()) + lines.append(f" {total:>7} {name} ({split})") + return "\n".join(lines) diff --git a/scripts/compiler_differential/_helpers/running/report.py b/scripts/compiler_differential/_helpers/running/report.py index d8e835850d..ba7d523fbb 100644 --- a/scripts/compiler_differential/_helpers/running/report.py +++ b/scripts/compiler_differential/_helpers/running/report.py @@ -3,6 +3,7 @@ from __future__ import annotations from scripts.compiler_differential.constants import ( + ANALYSIS_INDIRECT_KINDS, CONFIG_ONLY_KINDS, RENDER_CONFIG_ONLY_KINDS, RENDER_INDIRECT_KINDS, @@ -81,6 +82,17 @@ def format_render_coverage(*, covered: frozenset[str], required: tuple[str, ...] ) +def format_analysis_coverage(*, covered: frozenset[str], required: tuple[str, ...]) -> str: + """Report how many required analysis input kinds the seed corpus exercised.""" + + return _format_coverage( + stage="Analysis", + covered=covered, + required=required, + notes=(("indirect kinds", "credited because", ANALYSIS_INDIRECT_KINDS),), + ) + + def _format_coverage( *, stage: str, diff --git a/scripts/compiler_differential/_helpers/running/run.py b/scripts/compiler_differential/_helpers/running/run.py index 44ce33fec8..d6dbbb668f 100644 --- a/scripts/compiler_differential/_helpers/running/run.py +++ b/scripts/compiler_differential/_helpers/running/run.py @@ -30,6 +30,8 @@ def differential_options(*, args: argparse.Namespace, work_dir: Path) -> Differe engine_environment=parse_engine_environment(args.engine_env), require_discovery_coverage=args.require_discovery_coverage, require_render_coverage=args.require_render_coverage, + require_analysis_coverage=args.require_analysis_coverage, + analysis_records=args.wheel_site_report is not None, ) diff --git a/scripts/compiler_differential/classes/analysis_features.py b/scripts/compiler_differential/classes/analysis_features.py new file mode 100644 index 0000000000..e45a65edaa --- /dev/null +++ b/scripts/compiler_differential/classes/analysis_features.py @@ -0,0 +1,348 @@ +"""Write the optional project inputs that make generated seeds cover every analysis input kind.""" + +from __future__ import annotations + +import random +from collections.abc import Callable + +from scripts.compiler_differential.constants import ( + GENERATOR_ANALYSIS_FOLDER, + GENERATOR_ANALYSIS_MODE_COMMANDS, + GENERATOR_ANALYSIS_TEST_FOLDER, + GENERATOR_OPEN_SOURCE, + GENERATOR_SELECTED_MODEL, +) +from scripts.compiler_differential.models import DifferentialCommand, ModelPlan + + +class AnalysisFeatureWriter: + """Add self-contained SQL analysis blocks to one generated project, chosen per seed.""" + + def __init__( + self, + *, + blocks: tuple[str, ...], + rng: random.Random, + files: dict[str, str], + features: set[str], + staging: list[ModelPlan], + sources: list[str], + ) -> None: + self._blocks: tuple[str, ...] = blocks + self._random: random.Random = rng + self._files: dict[str, str] = files + self._features: set[str] = features + self._staging: list[ModelPlan] = staging + self._sources: list[str] = sources + self.config_lines: list[str] = [] + self.settings_lines: list[str] = [] + self.commands: tuple[DifferentialCommand, ...] = () + + def write(self) -> None: + """Write the selected analysis blocks in their canonical order.""" + + writers: dict[str, Callable[[], None]] = { + "column_shapes": self._column_shapes, + "contract_chain": self._contract_chain, + "udf_signatures": self._udf_signatures, + "dynamic_columns": self._dynamic_columns, + "quoted_identifiers": self._quoted_identifiers, + "metadata_checks": self._metadata_checks, + "sql_test_mocks": self._sql_test_mocks, + "analysis_opt_out": self._analysis_opt_out, + "python_sql": self._python_sql, + "analysis_modes": self._analysis_modes, + } + for block in self._blocks: + writer: Callable[[], None] | None = writers.get(block) + if writer is not None: + writer() + + def _base(self) -> ModelPlan: + return self._random.choice(self._staging) + + def _model(self, *, name: str, header: str, body: str, description: str) -> None: + self._files[f"{GENERATOR_ANALYSIS_FOLDER}/{name}.sql"] = ( + f'MODEL (\n description "{description}",\n{header});\n\n{body}' + ) + + def _column_shapes(self) -> None: + self._features.add("column_shapes") + base: ModelPlan = self._base() + other: ModelPlan = self._base() + source: str = self._random.choice(self._sources) + self._model( + name=f"typed_{base.name}", + description=f"Typed and untyped outputs of {base.name}", + header="", + body=( + "SELECT id, amount * 2 AS doubled_amount, CAST(status AS VARCHAR) AS status_text,\n" + " created_at, NULL AS missing_note\n" + f'FROM __ref("{base.name}")\n' + ), + ) + self._model( + name=f"every_{source}", + description=f"Every column of {source}", + header="", + body=f'SELECT *\nFROM __source("{source}")\n', + ) + self._model( + name=f"every_{base.name}", + description=f"Every column of {base.name}", + header="", + body=f'SELECT *\nFROM __ref("{base.name}")\n', + ) + self._files["sources/open.yml"] = ( + "sources:\n" + f" - name: {GENERATOR_OPEN_SOURCE}\n" + " description: Events whose columns are only known to the warehouse.\n" + ) + self._model( + name="every_open_event", + description="Every column of an open source", + header="", + body=f'SELECT *\nFROM __source("{GENERATOR_OPEN_SOURCE}")\n', + ) + self._model( + name=f"combined_{base.name}", + description=f"Recent and older rows of {base.name} and {other.name}", + header="", + body=( + "WITH recent AS (\n" + f' SELECT id, created_at FROM __ref("{base.name}") WHERE amount > 0\n' + "),\nolder AS (\n" + f' SELECT id, created_at FROM __ref("{other.name}")\n' + ")\n" + "SELECT id, created_at FROM recent\nUNION ALL\nSELECT id, created_at FROM older\n" + ), + ) + self._model( + name=f"passed_{base.name}", + description=f"Columns of {base.name} passed through a CTE", + header="", + body=( + f'WITH passthrough AS (\n SELECT id, created_at FROM __ref("{base.name}")\n)\n' + "SELECT id, created_at\nFROM passthrough\n" + ), + ) + + def _contract_chain(self) -> None: + self._features.add("contract_chain") + base: ModelPlan = self._base() + width: int = self._random.randint(16, 40) + source: str = base.inputs[0].split(":", 1)[1] + contracted: str = f"contracted_{base.name}" + self._model( + name=contracted, + description=f"Contracted copy of {source}", + header=( + " materialized table,\n" + " contract enforced,\n" + " columns (\n" + " id (type INTEGER, nullable false),\n" + f" label (type VARCHAR({width})),\n" + " amount (type DOUBLE),\n" + " ),\n" + ), + body=( + f"SELECT CAST(id AS INTEGER) AS id, CAST(status AS VARCHAR({width})) AS label,\n" + " amount\n" + f'FROM __source("{source}")\n' + ), + ) + upstream: str = contracted + for level, expression in enumerate(("label", "UPPER(label) AS label", "label"), start=1): + name: str = f"chain_{level}_{base.name}" + self._model( + name=name, + description=f"Level {level} below the contracted copy of {base.name}", + header="", + body=f'SELECT id, {expression}, amount\nFROM __ref("{upstream}")\n', + ) + upstream = name + self.settings_lines.append('table_promotion_mode = "staged"') + + def _udf_signatures(self) -> None: + self._features.add("udf_signatures") + base: ModelPlan = self._base() + factor: int = self._random.randint(2, 9) + self._files["functions/sql/scaled_amount.sql"] = ( + "FUNCTION (\n" + ' description "Scale an amount by an integer factor",\n' + " arguments (p_amount DOUBLE, p_factor INTEGER),\n" + " returns DOUBLE,\n);\n\n" + "p_amount * p_factor\n" + ) + self._files["functions/sql/table_fn__amount_bands.sql"] = ( + "FUNCTION (\n" + ' description "Band staged amounts above a floor",\n' + " arguments (p_floor DOUBLE),\n" + " returns table (\n id INTEGER,\n band VARCHAR\n ),\n);\n\n" + "SELECT CAST(id AS INTEGER) AS id,\n" + " CASE WHEN amount > p_floor THEN 'high' ELSE 'low' END AS band\n" + f'FROM __ref("{base.name}")\n' + ) + self._model( + name=f"scaled_{base.name}", + description=f"Scaled amounts of {base.name}", + header="", + body=( + f'SELECT id, __udf("scaled_amount")(amount, {factor}) AS scaled_amount\n' + f'FROM __ref("{base.name}")\n' + ), + ) + self._model( + name="banded_amounts", + description="Amount bands from a table function", + header="", + body=( + "SELECT bands.id, bands.band\n" + f'FROM __table_fn("table_fn__amount_bands")({factor}.5) AS bands\n' + ), + ) + + def _dynamic_columns(self) -> None: + self._features.add("dynamic_columns") + base: ModelPlan = self._base() + pivot_input: str = f"pivot_input_{base.name}" + self._model( + name=pivot_input, + description=f"Contracted pivot input from {base.name}", + header=( + " materialized table,\n" + " contract enforced,\n" + " columns (\n" + " id (type INTEGER),\n" + " status (type VARCHAR),\n" + " amount (type DOUBLE),\n" + " ),\n" + ), + body=( + "SELECT CAST(id AS INTEGER) AS id, CAST(status AS VARCHAR) AS status, amount\n" + f'FROM __ref("{base.name}")\n' + ), + ) + self._model( + name=f"pivoted_{base.name}", + description=f"Amounts of {base.name} pivoted by status", + header=( + " materialized table,\n" + " columns (id (type INTEGER)),\n" + " dynamic_columns (\n" + " status_amounts (\n" + " pivot_column status,\n" + " value_column amount,\n" + " aggregate MAX,\n" + " type DOUBLE\n" + " )\n" + " ),\n" + ), + body=f'PIVOT __ref("{pivot_input}")\nON status\nUSING MAX(amount)\nGROUP BY id\n', + ) + + def _quoted_identifiers(self) -> None: + self._features.add("quoted_identifiers") + base: ModelPlan = self._base() + self._model( + name=f"quoted_{base.name}", + description=f"Quoted identifiers over {base.name}", + header="", + body=( + 'SELECT "ID" AS "OrderKey", "Amount", "Status" AS "Status"\n' + f'FROM __ref("{base.name}")\n' + ), + ) + + def _metadata_checks(self) -> None: + self._features.add("metadata_checks") + base: ModelPlan = self._base() + self._files[ + f"{GENERATOR_ANALYSIS_FOLDER}/_sqlbuild/_audits/generic/expression_is_true.sql" + ] = 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE NOT (@expression)\n' + self._model( + name=f"checked_{base.name}", + description=f"Metadata-checked copy of {base.name}", + header=( + " materialized table,\n" + " unique_key [id],\n" + " audits [\n" + ' expression_is_true (expression "amount >= -1000", severity warn),\n' + " ],\n" + " columns (\n" + f' id (audits [relationships (to __ref("{base.name}"), field id)]),\n' + " status (audits [accepted_values (values " + '["placed", "expédié", "取消", "returned ✓"], severity warn)]),\n' + " ),\n" + ), + body=f'SELECT id, amount, status\nFROM __ref("{base.name}")\n', + ) + + def _sql_test_mocks(self) -> None: + self._features.add("sql_test_mocks") + base: ModelPlan = self._base() + region: str = self._random.choice(("north", "south", "east")) + self._files[f"{GENERATOR_ANALYSIS_FOLDER}/_sqlbuild/_macros/regions.py"] = ( + "def analysis_region() -> str:\n" + ' """Return the region rows are assigned to."""\n' + f" return \"'{region}'\"\n" + ) + model: str = f"regional_{base.name}" + self._model( + name=model, + description=f"Regional copy of {base.name}", + header="", + body=f'SELECT id, amount, @analysis_region() AS region\nFROM __ref("{base.name}")\n', + ) + self._files[f"{GENERATOR_ANALYSIS_TEST_FOLDER}/test_{model}.sql"] = ( + "TEST();\n\nWITH\n" + "__macro__analysis_region AS (\n SELECT '''west'''\n),\n" + f"__ref__{base.name} AS (\n" + " SELECT 1 AS id, CAST(4 AS DOUBLE) AS amount, 'placed' AS status,\n" + " TIMESTAMP '2026-01-01 00:00:00' AS created_at\n" + "),\n" + f"__expected__{model} AS (\n" + " SELECT 1 AS id, CAST(4 AS DOUBLE) AS amount, 'west' AS region\n" + ")\nSELECT 1\n" + ) + + def _analysis_opt_out(self) -> None: + self._features.add("analysis_opt_out") + base: ModelPlan = self._base() + self._model( + name=f"unanalysed_{base.name}", + description=f"Copy of {base.name} that SQL analysis cannot parse", + header=" sql_analysis false,\n", + body=f'SELECT id, amount\nFROM __ref("{base.name}")\nWHERE id = ANY [1, 2]\n', + ) + self.settings_lines.append("require_sql_analysis = true") + + def _python_sql(self) -> None: + self._features.add("python_sql") + self._files["python/tasks/flag_exports.py"] = ( + "from sqlbuild.tasks import TaskContext, task\n\n\n" + "@task\n" + "def count_flagged_events(ctx: TaskContext):\n" + ' """Count externally flagged events."""\n' + ' ctx.query("SELECT COUNT(*) FROM main.external_event_flags")\n' + " return ctx.result()\n" + ) + + def _analysis_modes(self) -> None: + self._features.add("analysis_modes") + base: ModelPlan = self._base() + self.commands = tuple( + _selected_command(label=label, arguments=arguments, model=base.name) + for label, arguments in GENERATOR_ANALYSIS_MODE_COMMANDS + ) + + +def _selected_command(*, label: str, arguments: tuple[str, ...], model: str) -> DifferentialCommand: + """Return one analysis-mode command with the selected model substituted.""" + + return DifferentialCommand( + label=label, + arguments=tuple( + model if argument == GENERATOR_SELECTED_MODEL else argument for argument in arguments + ), + ) diff --git a/scripts/compiler_differential/classes/project_builder.py b/scripts/compiler_differential/classes/project_builder.py index 03b776ddbf..6827d59e50 100644 --- a/scripts/compiler_differential/classes/project_builder.py +++ b/scripts/compiler_differential/classes/project_builder.py @@ -6,6 +6,7 @@ from collections.abc import Callable from dataclasses import replace +from scripts.compiler_differential.classes.analysis_features import AnalysisFeatureWriter from scripts.compiler_differential.classes.discovery_features import ( DiscoveryFeatureWriter, feature_blocks_for_seed, @@ -15,6 +16,8 @@ DUCKDB_ADAPTER, GENERATOR_CONSTANT_KIND, GENERATOR_CROSS_DOMAIN_SHARE, + GENERATOR_DIALECT_CONNECTIONS, + GENERATOR_DIALECT_TARGETS, GENERATOR_DOMAINS, GENERATOR_ENUM_KIND, GENERATOR_FLOAT_VALUES, @@ -44,9 +47,12 @@ class ProjectBuilder: """Accumulate the files of one generated project from a seeded random source.""" - def __init__(self, seed: int, *, blocks: tuple[str, ...] | None = None) -> None: + def __init__( + self, seed: int, *, blocks: tuple[str, ...] | None = None, dialect: str | None = None + ) -> None: self._seed: int = seed self._forced_blocks: tuple[str, ...] | None = blocks + self._dialect: str | None = dialect self._random: random.Random = random.Random(seed) self._files: dict[str, str] = {} self._features: set[str] = set() @@ -80,6 +86,7 @@ def build(self) -> GeneratedProject: self._random.choice(sorted(GENERATOR_INVALID_CODES)) if self._random.random() < GENERATOR_INVALID_SHARE and not set(blocks) & set(GENERATOR_RARE_FEATURE_BLOCKS) + and self._dialect is None else None ) self._write_declarations() @@ -107,10 +114,20 @@ def build(self) -> GeneratedProject: label=self._label, ) rendering.write() + analysis: AnalysisFeatureWriter = AnalysisFeatureWriter( + blocks=blocks, + rng=self._random, + files=self._files, + features=self._features, + staging=[model for model in self._models if model.layer == _STAGING], + sources=self._sources, + ) + analysis.write() self._write_config( domains=domains, adapter=extras.adapter, - extra_lines=[*extras.config_lines, *rendering.config_lines], + extra_lines=[*extras.config_lines, *rendering.config_lines, *analysis.config_lines], + settings_lines=analysis.settings_lines, templated_target=rendering.templated_target, ) if invalid_kind is not None: @@ -125,6 +142,7 @@ def build(self) -> GeneratedProject: ), features=tuple(sorted(self._features)), succeeding_commands=extras.succeeding_commands, + extra_commands=analysis.commands, ) def _target_schema(self, *, schema: str, templated: bool) -> str: @@ -144,15 +162,18 @@ def _write_config( domains: list[str], adapter: str | None, extra_lines: list[str], + settings_lines: list[str], templated_target: bool, ) -> None: lines: list[str] = [ f'name = "generated_{self._seed}"', - f'adapter = "{adapter or DUCKDB_ADAPTER}"', + f'adapter = "{self._dialect or adapter or DUCKDB_ADAPTER}"', 'default_target = "dev"', "", "[connection]", - 'database = "generated.duckdb"', + *GENERATOR_DIALECT_CONNECTIONS.get( + self._dialect or "", ('database = "generated.duckdb"',) + ), "", "[vars]", f'region = "{self._random.choice(("north", "south", "east"))}"', @@ -165,9 +186,11 @@ def _write_config( schema=self._random.choice(("analytics", "main")), templated=templated_target ) ), + *GENERATOR_DIALECT_TARGETS.get(self._dialect or "", ()), "", "[settings]", f'default_audit_severity = "{self._random.choice(("warn", "error"))}"', + *settings_lines, "", "[defaults]", f'materialized = "{self._random.choice(("table", "view"))}"', diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index de07f6da70..cb6c6a469c 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -7,11 +7,15 @@ ENGINE_ENV_VAR: str = "SQLBUILD_COMPILER_ENGINE" STAGE_CAPTURE_ENV_VAR: str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR" +ANALYSIS_RECORD_ENV_VAR: str = "SQLBUILD_ANALYSIS_RECORD_DIR" +WHEEL_SITE_RECORD_PREFIX: str = "polyglot-sites-" +ANALYSIS_DEFERRAL_RECORD_PREFIX: str = "analysis-deferrals-" +RECORDS_DIRECTORY: str = "records" ENGINE_NAMES: tuple[str, ...] = ("python", "native", "native-preview") DEFAULT_ENGINES: tuple[str, str] = ("python", "native-preview") SQB_ENTRY: str = "import sys; from sqlbuild.cli.entry.main.entry import main; sys.exit(main())" EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset( - {"VIRTUAL_ENV", ENGINE_ENV_VAR, STAGE_CAPTURE_ENV_VAR} + {"VIRTUAL_ENV", ENGINE_ENV_VAR, STAGE_CAPTURE_ENV_VAR, ANALYSIS_RECORD_ENV_VAR} ) EXCLUDED_ENVIRONMENT_PREFIX: str = "DBT_" PROJECT_CONFIG_FILE: str = "sqlbuild_project.toml" @@ -198,6 +202,18 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "resource_audits", "model_config", ) +GENERATOR_ANALYSIS_BLOCKS: tuple[str, ...] = ( + "column_shapes", + "contract_chain", + "udf_signatures", + "dynamic_columns", + "quoted_identifiers", + "metadata_checks", + "sql_test_mocks", + "analysis_opt_out", + "python_sql", + "analysis_modes", +) GENERATOR_FEATURE_BLOCKS: tuple[str, ...] = ( "incremental_append", "incremental_delete_insert", @@ -227,6 +243,7 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "dbt_ref", "line_endings", *GENERATOR_RENDER_BLOCKS, + *GENERATOR_ANALYSIS_BLOCKS, ) GENERATOR_FEATURE_STRIDE: int = 4 GENERATOR_OPTIONAL_FEATURE_SHARE: float = 0.2 @@ -236,6 +253,26 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: GENERATOR_FEATURE_FOLDER: str = "models/features" GENERATOR_RENDER_FOLDER: str = "models/rendering" GENERATOR_RENDER_TEST_FOLDER: str = "tests/unit/rendering" +GENERATOR_ANALYSIS_FOLDER: str = "models/analysis" +GENERATOR_ANALYSIS_TEST_FOLDER: str = "tests/unit/analysis" +GENERATOR_OPEN_SOURCE: str = "open_events" +GENERATOR_DIALECT_CONNECTIONS: dict[str, tuple[str, ...]] = { + "postgres": ('host = "localhost"', 'database = "generated"'), + "snowflake": ('account = "example"', 'user = "builder"', 'database = "GENERATED"'), +} +GENERATOR_DIALECT_TARGETS: dict[str, tuple[str, ...]] = {"snowflake": ('database = "GENERATED"',)} +GENERATOR_DIALECT_EXCLUDED_BLOCKS: frozenset[str] = frozenset( + {"dynamic_columns", "quoted_identifiers", "analysis_modes"} +) +GENERATOR_DIALECT_BLOCKS: tuple[str, ...] = tuple( + block for block in GENERATOR_ANALYSIS_BLOCKS if block not in GENERATOR_DIALECT_EXCLUDED_BLOCKS +) +GENERATOR_SELECTED_MODEL: str = "{model}" +GENERATOR_ANALYSIS_MODE_COMMANDS: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("compile-select", ("compile", "--json", "--select", GENERATOR_SELECTED_MODEL)), + ("compile-no-analysis", ("compile", "--json", "--no-sql-analysis")), + ("compile-rich-lineage", ("compile", "--json", "--lineage-mode", "rich")), +) GENERATOR_CHANNEL_ENV_VAR: str = "SQB_DIFFERENTIAL_CHANNEL" GENERATOR_MISSING_ENV_VAR: str = "SQB_DIFFERENTIAL_UNSET" GENERATOR_ENVIRONMENT: dict[str, str] = {GENERATOR_CHANNEL_ENV_VAR: "web-orders"} @@ -506,3 +543,108 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "S022": _TOLERANT_SCOPE_ONLY, "S023": _TOLERANT_SCOPE_ONLY, } +ANALYSIS_STAGE_CAPTURE_SUFFIX: str = "-compiled_project.json" +COMPILED_PROJECT_CAPTURE_TYPE: str = "sqlbuild.compiler.compile.models:CompiledProject" +ANALYSIS_NON_COLLECTION_FIELDS: frozenset[str] = frozenset( + { + "run_id", + "effective_target_name", + "effective_connection", + "effective_vars", + "binding_catalog", + "effective_target_database", + "effective_target_schema", + "sql_analysis_dialect", + "sql_lexical_syntax", + "compile_cache_dir", + "settings", + "scenario", + "enforce_explicit_references", + "diagnostics", + "external_sql_reference_resolver", + "scope_index", + "native_session", + } +) +ANALYSIS_CALLABLE_CAPTURE_FIELDS: frozenset[str] = CALLABLE_CAPTURE_FIELDS +ANALYSIS_OPAQUE_CAPTURE_VALUES: re.Pattern[str] = re.compile( + r'\{"__opaque__":\s*"sqlbuild\._native:ProjectCatalog"\}' +) +ANALYSIS_EXPLICIT_WRITE_SCHEMA_DIALECTS: frozenset[str] = frozenset({"snowflake"}) +ANALYSIS_DIALECT_VARIANTS: tuple[str, ...] = ("postgres", "snowflake") +ANALYSIS_UPSTREAM_CHAIN_HOPS: int = 3 +ANALYSIS_DETAIL_KINDS: tuple[str, ...] = ( + "typed_column", + "untyped_column", + "star_over_complete_input", + "star_over_published_shape", + "star_unresolved", + "set_operation", + "cte", + "cte_passthrough_type", + "quoted_identifier_case", + "dynamic_column_proof", + "cursor_intrinsic", + "runtime_placeholder", + "batch_binding", + "dataflow_binding", + "select_limited_analysis", + "sql_analysis_disabled", + "upstream_chain", + "sized_contract_shape", + "contract_nullability_shape", + "source_expression_shape", + "seed_column_types", + "scalar_udf", + "table_function", + "udf_argument_types", + "sql_analysis_opt_out", + "inline_sql_hook", + "named_sql_hook", + "hook_list", + "explicit_promotion_mode", + "python_sql_literal_relation", + "managed_write_schema", + "accepted_values_audit", + "relationships_audit", + "expression_audit", + "metadata_column_reference", + "sql_test_expected_columns", + "sql_test_macro_mock", + "dialect_duckdb", + "dialect_postgresql", + "dialect_snowflake", +) +ANALYSIS_INDIRECT_KINDS: dict[str, str] = { + "managed_write_schema": ( + "every destination resolved an explicit schema under an adapter that requires one; " + "the capture does not record the write-schema check itself" + ), +} +ANALYSIS_CODE_SCAN_ROOTS: tuple[str, ...] = ( + "compile/_helpers/assembly", + "compile/_helpers/analysis", + "compile/_helpers/diagnostics", + "contracts", + "pipeline/_helpers/target_validation.py", + "sql_analysis", +) +_NEVER_REPORTED_BY_POLYGLOT: str = ( + "polyglot defines the finding but never reports it, and no SQLBuild check maps to it" +) +ANALYSIS_UNREACHABLE_CODES: dict[str, str] = { + "B000": ( + "the binding layer's internal unknown-relation code; compile drops it because an " + "unknown relation is an open input" + ), + "B210": _NEVER_REPORTED_BY_POLYGLOT, + "B214": ( + "polyglot reports assignment types only for INSERT statements; compiled models, tests " + "and audits are queries" + ), + "B219": _NEVER_REPORTED_BY_POLYGLOT, + "P003": ( + "the contract check raises it for a declared column without a type, which it skips " + "first; compile reports P003 only as the built-in audit shadow warning" + ), +} diff --git a/scripts/compiler_differential/main/differential.py b/scripts/compiler_differential/main/differential.py index 314fc3ceab..446b25b09e 100644 --- a/scripts/compiler_differential/main/differential.py +++ b/scripts/compiler_differential/main/differential.py @@ -3,16 +3,23 @@ from __future__ import annotations import argparse +import json import os import sys import tempfile import time from pathlib import Path +from scripts.compiler_differential._helpers.coverage.analysis import required_analysis_kinds from scripts.compiler_differential._helpers.coverage.discovery import required_discovery_kinds from scripts.compiler_differential._helpers.coverage.render import required_render_kinds from scripts.compiler_differential._helpers.running.options import parse_expected_outcome +from scripts.compiler_differential._helpers.running.records import ( + format_wheel_site_report, + wheel_site_report, +) from scripts.compiler_differential._helpers.running.report import ( + format_analysis_coverage, format_discovery_coverage, format_render_coverage, format_summary, @@ -52,6 +59,15 @@ def run_compiler_differential(argv: list[str] | None = None) -> int: if options.stage_captures and CORPUS_SEEDS in args.corpus else {} ) + if args.wheel_site_report is not None: + report: dict[str, object] = wheel_site_report( + comparisons=comparisons, engines=options.engines + ) + args.wheel_site_report.parent.mkdir(parents=True, exist_ok=True) + _ = args.wheel_site_report.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(format_wheel_site_report(report)) print( format_summary( comparisons=comparisons, @@ -71,7 +87,7 @@ def run_compiler_differential(argv: list[str] | None = None) -> int: def _seed_coverage( *, comparisons: list[ProjectComparison], options: DifferentialOptions ) -> dict[str, tuple[str, ...]]: - """Print discovery and render coverage; return the missing kinds each requirement enforces.""" + """Print each stage's coverage; return the missing kinds each requirement enforces.""" discovered: frozenset[str] = frozenset().union( *(comparison.discovered_kinds for comparison in comparisons) @@ -80,14 +96,21 @@ def _seed_coverage( *(comparison.rendered_kinds for comparison in comparisons) ) required_discovery: tuple[str, ...] = required_discovery_kinds() + analysed: frozenset[str] = frozenset().union( + *(comparison.analysed_kinds for comparison in comparisons) + ) required_render: tuple[str, ...] = required_render_kinds() + required_analysis: tuple[str, ...] = required_analysis_kinds() print(format_discovery_coverage(covered=discovered, required=required_discovery)) print(format_render_coverage(covered=rendered, required=required_render)) + print(format_analysis_coverage(covered=analysed, required=required_analysis)) missing: dict[str, tuple[str, ...]] = {} if options.require_discovery_coverage: missing["discovery"] = tuple(kind for kind in required_discovery if kind not in discovered) if options.require_render_coverage: missing["render"] = tuple(kind for kind in required_render if kind not in rendered) + if options.require_analysis_coverage: + missing["analysis"] = tuple(kind for kind in required_analysis if kind not in analysed) return missing @@ -141,6 +164,21 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: action="store_true", help="fail unless the seed corpus exercises every render input kind (needs captures)", ) + parser.add_argument( + "--require-analysis-coverage", + action="store_true", + help="fail unless the seed corpus exercises every analysis input kind (needs captures)", + ) + parser.add_argument( + "--wheel-site-report", + type=Path, + default=None, + metavar="PATH", + help=( + "record Polyglot wheel calls by calling site and analysis deferrals in every engine " + "process, print them and write them to PATH as JSON (reported, never gated)" + ), + ) parser.add_argument( "--engine-env", action="append", @@ -154,6 +192,7 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: for flag, required in ( ("--require-discovery-coverage", args.require_discovery_coverage), ("--require-render-coverage", args.require_render_coverage), + ("--require-analysis-coverage", args.require_analysis_coverage), ): if required and not (args.stage_captures and CORPUS_SEEDS in args.corpus): parser.error(f"{flag} needs --stage-captures and the seeds corpus") diff --git a/scripts/compiler_differential/main/generate_project.py b/scripts/compiler_differential/main/generate_project.py index 76f581b12a..15587490e3 100644 --- a/scripts/compiler_differential/main/generate_project.py +++ b/scripts/compiler_differential/main/generate_project.py @@ -4,7 +4,9 @@ from scripts.compiler_differential.models import GeneratedProject -def generate_project(*, seed: int, blocks: tuple[str, ...] | None = None) -> GeneratedProject: - """Return the seed's deterministic project; `blocks` overrides its optional feature blocks.""" +def generate_project( + *, seed: int, blocks: tuple[str, ...] | None = None, dialect: str | None = None +) -> GeneratedProject: + """Return the seed's project; `blocks` overrides its blocks, `dialect` its adapter.""" - return ProjectBuilder(seed, blocks=blocks).build() + return ProjectBuilder(seed, blocks=blocks, dialect=dialect).build() diff --git a/scripts/compiler_differential/models.py b/scripts/compiler_differential/models.py index 3af73b087b..0a9cd20d14 100644 --- a/scripts/compiler_differential/models.py +++ b/scripts/compiler_differential/models.py @@ -49,6 +49,14 @@ class CommandOutcome: timed_out: bool = False +@dataclass(frozen=True) +class AnalysisRecords: + """Wheel calls by `(site, api)` and analysis deferrals by `(kind, site)` for one run.""" + + wheel_sites: dict[tuple[str, str], int] = field(default_factory=dict) + deferrals: dict[tuple[str, str], int] = field(default_factory=dict) + + @dataclass(frozen=True) class EngineRun: """Everything one engine produced for one corpus project.""" @@ -59,6 +67,7 @@ class EngineRun: manifest: str | None dag: str | None captures: dict[str, dict[str, Path]] = field(default_factory=dict) + records: AnalysisRecords | None = None @dataclass(frozen=True) @@ -91,6 +100,8 @@ class ProjectComparison: seconds: float discovered_kinds: frozenset[str] = frozenset() rendered_kinds: frozenset[str] = frozenset() + analysed_kinds: frozenset[str] = frozenset() + records: tuple[AnalysisRecords | None, AnalysisRecords | None] | None = None @dataclass(frozen=True) @@ -105,6 +116,8 @@ class DifferentialOptions: engine_environment: dict[str, dict[str, str]] require_discovery_coverage: bool = False require_render_coverage: bool = False + require_analysis_coverage: bool = False + analysis_records: bool = False @dataclass(frozen=True) @@ -130,6 +143,7 @@ class GeneratedProject(WritableProject): expected_error_code: str | None features: tuple[str, ...] succeeding_commands: tuple[str, ...] = () + extra_commands: tuple[DifferentialCommand, ...] = () @dataclass(frozen=True) @@ -140,6 +154,10 @@ class FailureCase(WritableProject): expected_code: str expected_warning_code: str | None = None expected_message: str | None = None + expected_help: str | None = None + expected_notes: tuple[str, ...] = () + expected_location: tuple[int, int] | None = None + expected_codes: tuple[str, ...] | None = None @dataclass(frozen=True) @@ -177,6 +195,10 @@ class EmittedCodes: errors: tuple[str, ...] warnings: tuple[str, ...] first_error_message: str | None = None + first_error_help: str | None = None + first_error_notes: tuple[str, ...] = () + first_error_location: tuple[int, int] | None = None + codes: tuple[str, ...] = () @property def first_error(self) -> str | None: diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index b10c7aab0c..949292fc67 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -57,6 +57,7 @@ ), "sqlbuild.compiler.discovery.models:DiscoveredProjectInputs": frozenset({"native_session"}), "sqlbuild.compiler.compile.models:DeclarationScopeResolver": frozenset({"native_contexts"}), + "sqlbuild.compiler.compile.models:CompiledProject": frozenset({"native_session"}), } STAGE_CAPTURE_DECODED_SEQUENCES: frozenset[str] = frozenset( {"sqlbuild.compiler.compile.models:CompactLineageFacts"} diff --git a/tests/e2e/scripts/compiler_differential/_test_types.py b/tests/e2e/scripts/compiler_differential/_test_types.py index 9c434d9283..e488fe198a 100644 --- a/tests/e2e/scripts/compiler_differential/_test_types.py +++ b/tests/e2e/scripts/compiler_differential/_test_types.py @@ -61,6 +61,12 @@ class FailureCorpusCodesTestCase: expected_discovery_codes: frozenset[str] expected_render_codes: frozenset[str] unreachable_render_codes: frozenset[str] + expected_first_helps: dict[str, str] + expected_first_notes: dict[str, frozenset[str]] + expected_first_locations: dict[str, tuple[int, int]] + expected_code_orders: dict[str, tuple[str, ...]] + expected_analysis_codes: frozenset[str] + unreachable_analysis_codes: frozenset[str] @dataclass(frozen=True) @@ -83,3 +89,28 @@ class RenderCaptureTestCase: blocks: tuple[str, ...] command: tuple[str, ...] expected_kinds: frozenset[str] + + +@dataclass(frozen=True) +class AnalysisCaptureTestCase: + """A generated project, the commands it runs, and the analysis kinds their captures prove.""" + + description: str + seed: int + blocks: tuple[str, ...] + dialect: str | None + commands: tuple[tuple[str, ...], ...] + generated_command_count: int + expected_kinds: frozenset[str] + expected_absent: frozenset[str] + + +@dataclass(frozen=True) +class WheelSiteReportTestCase: + """A harness run that records wheel sites, optionally with an injected native deferral.""" + + description: str + perturbation: str + expected_lines: tuple[str, ...] + expected_sites: frozenset[str] + expected_deferrals: frozenset[str] diff --git a/tests/e2e/scripts/compiler_differential/helpers.py b/tests/e2e/scripts/compiler_differential/helpers.py index 61cb287cc4..dff1c5eb7f 100644 --- a/tests/e2e/scripts/compiler_differential/helpers.py +++ b/tests/e2e/scripts/compiler_differential/helpers.py @@ -73,6 +73,18 @@ def _perturbed(*, until, **stages): frontier.native_frontier = _perturbed """ +DEFERRAL_PERTURBATION: str = """ +import json +import os +from pathlib import Path + +_directory = os.environ.get("SQLBUILD_ANALYSIS_RECORD_DIR") +if _directory: + Path(_directory).mkdir(parents=True, exist_ok=True) + with open(Path(_directory) / f"analysis-deferrals-{os.getpid()}.jsonl", "a") as _record: + _record.write(json.dumps({"kind": "legacy_fallback", "site": "orders.sql"}) + "\\n") +""" + _BROKEN_REF_FILES: dict[str, str] = { "sqlbuild_project.toml": ( 'name = "broken_orders"\nadapter = "duckdb"\n\n' diff --git a/tests/e2e/scripts/compiler_differential/test_analysis_capture.py b/tests/e2e/scripts/compiler_differential/test_analysis_capture.py new file mode 100644 index 0000000000..0cd6b1209a --- /dev/null +++ b/tests/e2e/scripts/compiler_differential/test_analysis_capture.py @@ -0,0 +1,114 @@ +"""Real compiles capture every compiled-project field canonically and prove every analysis kind.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +from scripts.compiler_differential._helpers.coverage.analysis import ( + analysed_input_kinds, + analysis_capture_problems, + required_analysis_kinds, +) +from scripts.compiler_differential._helpers.running.execution import harness_environment +from scripts.compiler_differential.constants import ( + ANALYSIS_STAGE_CAPTURE_SUFFIX, + GENERATOR_DIALECT_BLOCKS, + GENERATOR_FEATURE_BLOCKS, + SQB_ENTRY, + STAGE_CAPTURE_ENV_VAR, +) +from scripts.compiler_differential.main.generate_project import generate_project +from scripts.compiler_differential.models import GeneratedProject +from tests.e2e.scripts.compiler_differential._test_types import AnalysisCaptureTestCase + +# dbt_ref fails compile without its manifest; macro_generated_reference turns off the explicit +# references that Python SQL relation checks need. +_CONFLICTING_BLOCKS: frozenset[str] = frozenset({"dbt_ref", "macro_generated_reference"}) +_OTHER_DIALECT_KINDS: frozenset[str] = frozenset( + {"dialect_postgresql", "dialect_snowflake", "managed_write_schema"} +) + + +@pytest.mark.parametrize( + "test_case", + [ + AnalysisCaptureTestCase( + description="every_compiling_feature_block_in_every_analysis_mode", + seed=11, + blocks=tuple( + block for block in GENERATOR_FEATURE_BLOCKS if block not in _CONFLICTING_BLOCKS + ), + dialect=None, + commands=(("compile", "--json"), ("plan", "--json")), + generated_command_count=3, + expected_kinds=frozenset(required_analysis_kinds()) - _OTHER_DIALECT_KINDS, + expected_absent=_OTHER_DIALECT_KINDS, + ), + AnalysisCaptureTestCase( + description="analysis_blocks_compiled_under_snowflake", + seed=11, + blocks=GENERATOR_DIALECT_BLOCKS, + dialect="snowflake", + commands=(("compile", "--json"),), + generated_command_count=0, + expected_kinds=frozenset( + {"dialect_snowflake", "managed_write_schema", "upstream_chain", "typed_column"} + ), + expected_absent=frozenset({"dialect_duckdb", "dialect_postgresql"}), + ), + AnalysisCaptureTestCase( + description="analysis_blocks_compiled_under_postgres", + seed=11, + blocks=GENERATOR_DIALECT_BLOCKS, + dialect="postgres", + commands=(("compile", "--json"),), + generated_command_count=0, + expected_kinds=frozenset({"dialect_postgresql", "sized_contract_shape", "scalar_udf"}), + expected_absent=frozenset({"dialect_duckdb", "managed_write_schema"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_project_with_every_analysis_input_when_capturing_then_capture_is_complete( + test_case: AnalysisCaptureTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "project" + generated: GeneratedProject = generate_project( + seed=test_case.seed, blocks=test_case.blocks, dialect=test_case.dialect + ) + generated.write(project_dir) + generated_commands: tuple[tuple[str, ...], ...] = tuple( + command.arguments for command in generated.extra_commands + )[: test_case.generated_command_count] + commands: tuple[tuple[str, ...], ...] = (*test_case.commands, *generated_commands) + + kinds: set[str] = set() + for index, command in enumerate(commands): + capture_dir: Path = tmp_path / f"captures-{index}" + completed: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, "-c", SQB_ENTRY, "--no-color", *command], + cwd=project_dir, + env={**harness_environment(), STAGE_CAPTURE_ENV_VAR: str(capture_dir)}, + capture_output=True, + text=True, + check=False, + timeout=600, + ) + assert completed.returncode == 0, (command, completed.stdout + completed.stderr) + text: str = next(capture_dir.glob(f"*{ANALYSIS_STAGE_CAPTURE_SUFFIX}")).read_text( + encoding="utf-8" + ) + assert analysis_capture_problems(text) == (), command + kinds.update(analysed_input_kinds(text)) + + assert len(generated_commands) == test_case.generated_command_count + assert test_case.expected_kinds <= kinds, test_case.expected_kinds - kinds + assert not test_case.expected_absent & kinds, test_case.expected_absent & kinds + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py index 42665b68ca..9ab1471e8e 100644 --- a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py +++ b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import re import shutil from pathlib import Path @@ -14,9 +15,11 @@ HarnessRunTestCase, ProjectExpectationTestCase, SharedAnalysisSeedTestCase, + WheelSiteReportTestCase, ) from tests.e2e.scripts.compiler_differential.helpers import ( COMPILED_PROJECT_CAPTURE, + DEFERRAL_PERTURBATION, DISCOVERY_PERTURBATION, NATIVE_ONLY_STDERR_LINE, RENDER_PERTURBATION, @@ -240,7 +243,7 @@ def test_given_both_engines_fail_alike_when_outcome_is_unexpected_then_harness_f expected_exit_code=1, expected_lines=( "OK seed/0", - "Compiler differential FAILED: 0 of 1 projects differ", + "Compiler differential FAILED: 0 of 3 projects differ", "Required discovery coverage missing: ", ), expected_absent=("passed",), @@ -259,11 +262,32 @@ def test_given_both_engines_fail_alike_when_outcome_is_unexpected_then_harness_f expected_lines=( "OK seed/0", "Render coverage: ", - "Compiler differential FAILED: 0 of 1 projects differ", + "Compiler differential FAILED: 0 of 3 projects differ", "Required render coverage missing: ", ), expected_absent=("passed", "Required discovery coverage missing"), ), + CoverageFailureTestCase( + description="one_seed_cannot_cover_every_analysis_kind", + extra_arguments=( + "--corpus", + "seeds", + "--seeds", + "1", + "--stage-captures", + "--require-analysis-coverage", + ), + expected_exit_code=1, + expected_lines=( + "OK seed/0", + "OK seed/0-postgres", + "OK seed/0-snowflake", + "Analysis coverage: ", + "Compiler differential FAILED: 0 of 3 projects differ", + "Required analysis coverage missing: ", + ), + expected_absent=("passed", "Required render coverage missing"), + ), ], ids=lambda case: case.description, ) @@ -288,7 +312,7 @@ def test_given_required_coverage_missing_when_comparing_then_harness_fails_and_s seed=SHARED_ANALYSIS_SEED, expected_lines=( f"OK seed/{SHARED_ANALYSIS_SEED}", - "Compiler differential passed: 1 projects identical (python vs python)", + "Compiler differential passed: 3 projects identical (python vs python)", ), expected_capture_sides=("left-python-captures", "right-python-captures"), expected_compile_exit_code=0, @@ -374,5 +398,68 @@ def test_given_failure_expectation_when_code_is_not_the_first_error_then_harness assert all(line in output for line in test_case.expected_lines), output +@pytest.mark.parametrize( + "test_case", + [ + WheelSiteReportTestCase( + description="python_and_preview_wheel_calls_without_deferrals", + perturbation="", + expected_lines=( + "Polyglot wheel calls (python):", + "Polyglot wheel calls (native-preview):", + "Analysis deferrals (python): none recorded", + "Analysis deferrals (native-preview): none recorded", + "Compiler differential passed: 1 projects identical", + ), + expected_sites=frozenset( + { + "compiler/compile/_helpers/analysis/validation.py:" + "_validate_sql_syntax_with_message parse_one", + } + ), + expected_deferrals=frozenset(), + ), + WheelSiteReportTestCase( + description="preview_deferrals_are_counted_per_kind_and_site", + perturbation=DEFERRAL_PERTURBATION, + expected_lines=( + "Analysis deferrals (python): none recorded", + "Analysis deferrals (native-preview):", + " legacy_fallback orders.sql (project ", + ), + expected_sites=frozenset(), + expected_deferrals=frozenset({"legacy_fallback orders.sql"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_wheel_site_report_when_comparing_then_sites_and_deferrals_are_reported( + test_case: WheelSiteReportTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + report_path: Path = tmp_path / "report" / "wheel-sites.json" + exit_code: int = run_compiler_differential( + harness_arguments( + work_dir=tmp_path / "work", + extra=( + *perturbation_arguments(tmp_path / "perturbation", source=test_case.perturbation), + "--wheel-site-report", + str(report_path), + ), + ) + ) + + output: str = capsys.readouterr().out + report: dict[str, dict[str, dict[str, dict[str, int]]]] = json.loads( + report_path.read_text(encoding="utf-8") + ) + preview: dict[str, dict[str, dict[str, int]]] = report["native-preview"] + assert exit_code == 0, output + assert all(line in output for line in test_case.expected_lines), output + assert test_case.expected_sites <= set(report["python"]["wheel_sites"]), report + assert set(preview["deferrals"]) == test_case.expected_deferrals + assert all(sum(by_corpus.values()) > 0 for by_corpus in preview["deferrals"].values()) + assert report["python"]["deferrals"] == {} + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py b/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py index 8f07f996bd..172a53588d 100644 --- a/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py +++ b/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py @@ -1,4 +1,5 @@ -"""Every failure case emits its declared code first, reaching every discovery and render code.""" +"""Every failure case emits its declared first error, reaching every discovery, render and +analysis code.""" from __future__ import annotations @@ -8,12 +9,15 @@ import pytest +import sqlbuild.compiler as compiler_package from scripts.compiler_differential._helpers.corpus.emitted_codes import ( + analysis_error_codes, discovery_error_codes, emitted_failure_codes, render_error_codes, ) from scripts.compiler_differential.constants import ( + ANALYSIS_UNREACHABLE_CODES, DISCOVERY_UNREACHABLE_CODES, RENDER_UNREACHABLE_CODES, ) @@ -21,6 +25,8 @@ from scripts.compiler_differential.models import DifferentialOptions, EmittedCodes from tests.e2e.scripts.compiler_differential._test_types import FailureCorpusCodesTestCase +_COMPILER_ROOT: Path = Path(compiler_package.__file__).parent + @pytest.mark.parametrize( "test_case", @@ -44,6 +50,29 @@ - frozenset(DISCOVERY_UNREACHABLE_CODES), expected_render_codes=render_error_codes() - frozenset(RENDER_UNREACHABLE_CODES), unreachable_render_codes=frozenset(RENDER_UNREACHABLE_CODES), + expected_first_helps={ + f"failure/{case.name}": case.expected_help + for case in failure_cases() + if case.expected_help is not None + }, + expected_first_notes={ + f"failure/{case.name}": frozenset(case.expected_notes) + for case in failure_cases() + if case.expected_notes + }, + expected_first_locations={ + f"failure/{case.name}": case.expected_location + for case in failure_cases() + if case.expected_location is not None + }, + expected_code_orders={ + f"failure/{case.name}": case.expected_codes + for case in failure_cases() + if case.expected_codes is not None + }, + expected_analysis_codes=analysis_error_codes(_COMPILER_ROOT) + - frozenset(ANALYSIS_UNREACHABLE_CODES), + unreachable_analysis_codes=frozenset(ANALYSIS_UNREACHABLE_CODES), ) ], ids=lambda case: case.description, @@ -84,6 +113,23 @@ def test_given_failure_corpus_when_compiling_then_first_errors_match_and_cover_e assert test_case.expected_render_codes <= reached, test_case.expected_render_codes - reached assert not test_case.unreachable_render_codes & set(first_errors.values()) assert test_case.unreachable_render_codes <= render_error_codes() + assert all( + fragment in (emitted[name].first_error_help or "") + for name, fragment in test_case.expected_first_helps.items() + ), {name: emitted[name].first_error_help for name in test_case.expected_first_helps} + assert all( + notes <= set(emitted[name].first_error_notes) + for name, notes in test_case.expected_first_notes.items() + ), {name: emitted[name].first_error_notes for name in test_case.expected_first_notes} + assert { + name: emitted[name].first_error_location for name in test_case.expected_first_locations + } == test_case.expected_first_locations + assert { + name: emitted[name].codes for name in test_case.expected_code_orders + } == test_case.expected_code_orders + assert test_case.expected_analysis_codes <= reached, test_case.expected_analysis_codes - reached + assert not test_case.unreachable_analysis_codes & set(first_errors.values()) + assert test_case.unreachable_analysis_codes <= analysis_error_codes(_COMPILER_ROOT) if __name__ == "__main__": diff --git a/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py index 48f6053e2b..5f250d89bd 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py +++ b/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from pathlib import Path -from scripts.compiler_differential.models import ExpectedOutcome +from scripts.compiler_differential.models import DifferentialCommand, ExpectedOutcome @dataclass(frozen=True) @@ -25,3 +25,22 @@ class InlineCompileCodesTestCase: compile_root: Path accounted_codes: frozenset[str] expected_unaccounted: dict[str, tuple[str, ...]] + + +@dataclass(frozen=True) +class DialectVariantTestCase: + """A seed range and the dialect variants its first seed must add to the corpus.""" + + description: str + seeds: range + expected_names: tuple[str, ...] + expected_variant_commands: tuple[DifferentialCommand, ...] + + +@dataclass(frozen=True) +class AnalysisCodesTestCase: + """The compiler package and the analysis codes its scan must find and document.""" + + description: str + compiler_root: Path + expected_codes: frozenset[str] diff --git a/tests/unit/scripts/compiler_differential/_helpers/corpus/test_analysis_codes.py b/tests/unit/scripts/compiler_differential/_helpers/corpus/test_analysis_codes.py new file mode 100644 index 0000000000..f774280585 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/corpus/test_analysis_codes.py @@ -0,0 +1,71 @@ +"""The analysis code scan finds every B, K, S and syntax code, and documents each unreachable one.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import sqlbuild.compiler as compiler_package +from scripts.compiler_differential._helpers.analysis_corpus.failure_cases import ( + analysis_failure_cases, +) +from scripts.compiler_differential._helpers.corpus.emitted_codes import analysis_error_codes +from scripts.compiler_differential.constants import ANALYSIS_UNREACHABLE_CODES +from scripts.compiler_differential.main.failure_cases import failure_cases +from scripts.compiler_differential.models import FailureCase +from tests.unit.scripts.compiler_differential._helpers.corpus._test_types import ( + AnalysisCodesTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + AnalysisCodesTestCase( + description="compiler_package", + compiler_root=Path(compiler_package.__file__).parent, + expected_codes=frozenset( + { + *(f"B00{number}" for number in (0, 2, 3, 4, 5)), + "B101", + "B102", + *(f"B2{number}" for number in range(10, 20)), + *(f"B23{number}" for number in range(5)), + "B300", + "B301", + "B302", + *(f"K00{number}" for number in range(1, 7)), + "K011", + "P001", + "P003", + "S101", + "S102", + "S103", + "S104", + } + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_compiler_when_scanning_analysis_codes_then_each_is_cased_or_unreachable( + test_case: AnalysisCodesTestCase, +) -> None: + codes: frozenset[str] = analysis_error_codes(test_case.compiler_root) + cases: tuple[FailureCase, ...] = (*failure_cases(), *analysis_failure_cases()) + cased: set[str | None] = {case.expected_code for case in cases} | { + case.expected_warning_code for case in cases + } + + assert test_case.expected_codes <= codes, test_case.expected_codes - codes + assert set(ANALYSIS_UNREACHABLE_CODES) <= codes + assert all(reason.strip() for reason in ANALYSIS_UNREACHABLE_CODES.values()) + assert codes - set(ANALYSIS_UNREACHABLE_CODES) <= cased, ( + codes - set(ANALYSIS_UNREACHABLE_CODES) - cased + ) + assert not set(ANALYSIS_UNREACHABLE_CODES) & {case.expected_code for case in cases} + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/corpus/test_corpus.py b/tests/unit/scripts/compiler_differential/_helpers/corpus/test_corpus.py index e1f4c0299b..3bba3bdc8a 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/corpus/test_corpus.py +++ b/tests/unit/scripts/compiler_differential/_helpers/corpus/test_corpus.py @@ -7,9 +7,11 @@ import pytest from scripts.compiler_differential._helpers.corpus.corpus import build_corpus +from scripts.compiler_differential.constants import COLD_COMPILE from scripts.compiler_differential.models import CorpusProject, ExpectedOutcome from tests.unit.scripts.compiler_differential._helpers.corpus._test_types import ( CorpusExpectationTestCase, + DialectVariantTestCase, ) _REPO_ROOT: Path = Path(__file__).resolve().parents[6] @@ -65,5 +67,41 @@ def test_given_corpus_selection_when_building_then_entries_declare_expected_outc } == test_case.expected_outcomes +@pytest.mark.parametrize( + "test_case", + [ + DialectVariantTestCase( + description="first_seed_compiles_under_each_dialect", + seeds=range(5, 7), + expected_names=( + "seed/5", + "seed/6", + "seed/5-postgres", + "seed/5-snowflake", + ), + expected_variant_commands=(COLD_COMPILE,), + ) + ], + ids=lambda case: case.description, +) +def test_given_seed_range_when_building_then_first_seed_adds_compile_only_dialect_variants( + test_case: DialectVariantTestCase, +) -> None: + corpus: list[CorpusProject] = build_corpus( + repo_root=_REPO_ROOT, + corpora=("seeds",), + seeds=test_case.seeds, + dense_models=20, + projects=(), + project_expectation=ExpectedOutcome(), + ) + variants: list[CorpusProject] = corpus[len(test_case.seeds) :] + + assert tuple(entry.name for entry in corpus) == test_case.expected_names + assert all(entry.seed_coverage for entry in corpus) + assert {entry.commands for entry in variants} == {test_case.expected_variant_commands} + assert all(entry.expected == ExpectedOutcome() for entry in variants) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py index a9e3ca07af..d873fc67d9 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py @@ -71,3 +71,22 @@ class RenderEncodedInputsTestCase: description: str inputs: CompileProjectInputs expected_kinds: frozenset[str] + + +@dataclass(frozen=True) +class AnalysisCaptureKindsTestCase: + """One compiled-project capture fragment and the analysis kinds it must and must not prove.""" + + description: str + capture: dict[str, object] + expected_present: frozenset[str] + expected_absent: frozenset[str] + + +@dataclass(frozen=True) +class AnalysisCaptureProblemsTestCase: + """A compiled-project capture text and how many problems make it unsound.""" + + description: str + capture_text: str + expected_problem_count: int diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py index 0c0d785c4d..6ca8ca73bc 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py @@ -1,4 +1,4 @@ -"""Capture fragments for discovery coverage tests.""" +"""Capture fragments for discovery, render and analysis coverage tests.""" from __future__ import annotations @@ -6,7 +6,7 @@ from itertools import filterfalse from typing import cast -from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild.compiler.compile.models import CompiledProject, CompileProjectInputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_OMITTED_ATTRIBUTES @@ -106,3 +106,65 @@ def audit_input(*, definition: str, template: str, rendered: str) -> dict[str, o "attached_target_name": "orders", "attached_column_name": "amount", } + + +_NULLABILITY: str = "sqlbuild.compiler.lineage.types:InferredNullability" + + +def empty_compiled_capture(**overrides: object) -> dict[str, object]: + """Return a capture with every captured `CompiledProject` field, in order, then overrides.""" + + type_name: str = "sqlbuild.compiler.compile.models:CompiledProject" + omitted: frozenset[str] = STAGE_CAPTURE_OMITTED_ATTRIBUTES.get(type_name, frozenset()) + names: list[str] = [field.name for field in dataclasses.fields(CompiledProject)] + return { + "__type__": type_name, + **dict.fromkeys(filterfalse(omitted.__contains__, names), []), + **overrides, + } + + +def inferred_column(*, name: str, type_sql: str | None, nullability: str = "UNKNOWN") -> object: + """Return one captured inferred output column.""" + + return { + "name": name, + "type": type_sql, + "nullability": {"__enum__": f"{_NULLABILITY}.{nullability}"}, + } + + +def compiled_model( + *, + name: str, + query_sql: str = "SELECT 1 AS id", + columns: list[object] | None = None, + references: list[dict[str, object]] | None = None, + validated: bool = True, + values: dict[str, object] | None = None, + schema_columns: list[dict[str, object]] | None = None, + star: tuple[bool, bool] = (False, False), +) -> dict[str, object]: + """Return one captured compiled model; `columns=None` means it was not analysed.""" + + return { + "name": name, + "query_sql": query_sql, + "config": {"values": values or {}}, + "references": references or [], + "schema_entry": {"columns": schema_columns or [], "audits": []}, + "inferred_columns": columns, + "binding_validated": validated, + "binding_diagnostics": [], + "fast_lineage_has_star": star[0], + "fast_lineage_star_resolved": star[1], + "dynamic_column_contract": None, + "rejected_sql_analysis_opt_out": None, + "destination": {"schema": "analytics"}, + } + + +def compiled_source(*, name: str, expression: str | None) -> dict[str, object]: + """Return one captured compiled source with an optional SQL expression.""" + + return {"source_entry": {"name": name, "expression": expression, "columns": []}} diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_analysis.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_analysis.py new file mode 100644 index 0000000000..92da94d8a3 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_analysis.py @@ -0,0 +1,288 @@ +"""Compiled-project captures are classified into the analysis kinds the seed corpus must exercise.""" + +from __future__ import annotations + +import json + +import pytest + +from scripts.compiler_differential._helpers.coverage.analysis import ( + analysed_input_kinds, + analysis_capture_problems, + required_analysis_kinds, +) +from scripts.compiler_differential.constants import ANALYSIS_NON_COLLECTION_FIELDS +from tests.unit.scripts.compiler_differential._helpers.coverage._test_types import ( + AnalysisCaptureKindsTestCase, + AnalysisCaptureProblemsTestCase, + RequiredKindsTestCase, +) +from tests.unit.scripts.compiler_differential._helpers.coverage.helpers import ( + compiled_model, + compiled_source, + empty_compiled_capture, + inferred_column, + reference, +) + +_TYPED_ORDERS: list[object] = [ + inferred_column(name="id", type_sql="INT"), + inferred_column(name="placed_at", type_sql="TIMESTAMP"), +] +_CONTRACT_COLUMNS: list[dict[str, object]] = [ + {"name": "id", "type": "INTEGER", "nullable": False}, + {"name": "label", "type": "VARCHAR(24)", "nullable": None}, +] +_CONTRACT: dict[str, object] = {"contract": "enforced", "materialized": "table"} + + +@pytest.mark.parametrize( + "test_case", + [ + AnalysisCaptureKindsTestCase( + description="analysed_star_over_a_source_expression_and_a_model", + capture=empty_compiled_capture( + sources=[compiled_source(name="raw_orders", expression="(SELECT 1 AS id)")], + binding_catalog={"expression_shapes": {"(SELECT 1 AS id)": {"id": "INT"}}}, + sql_analysis_dialect="duckdb", + models=[ + compiled_model( + name="stg_orders", + columns=[*_TYPED_ORDERS, inferred_column(name="note", type_sql=None)], + references=[reference(kind="SOURCE", name="raw_orders")], + star=(True, True), + ), + compiled_model( + name="every_order", + columns=_TYPED_ORDERS, + references=[reference(kind="REF", name="stg_orders")], + star=(True, True), + ), + ], + ), + expected_present=frozenset( + { + "models", + "sources", + "typed_column", + "untyped_column", + "star_over_complete_input", + "star_over_published_shape", + "source_expression_shape", + "dataflow_binding", + "dialect_duckdb", + } + ), + expected_absent=frozenset({"star_unresolved", "batch_binding", "upstream_chain"}), + ), + AnalysisCaptureKindsTestCase( + description="columns_without_binding_validation_prove_nothing", + capture=empty_compiled_capture( + sql_analysis_dialect="duckdb", + unchecked_output_columns={"__set__": []}, + models=[ + compiled_model( + name="stg_orders", + query_sql="WITH a AS (SELECT 1 AS id) SELECT id FROM a UNION SELECT 2", + columns=_TYPED_ORDERS, + references=[reference(kind="REF", name="upstream")], + validated=False, + star=(True, False), + ), + compiled_model(name="upstream", columns=None, validated=False), + ], + ), + expected_present=frozenset({"models"}), + expected_absent=frozenset( + { + "sql_analysis_disabled", + "typed_column", + "star_unresolved", + "set_operation", + "cte", + "dialect_duckdb", + "unchecked_output_columns", + } + ), + ), + AnalysisCaptureKindsTestCase( + description="contract_types_and_nullability_published_downstream", + capture=empty_compiled_capture( + models=[ + compiled_model( + name="contracted", + columns=[ + inferred_column(name="id", type_sql="INT"), + inferred_column(name="label", type_sql="VARCHAR(24)"), + ], + values=_CONTRACT, + schema_columns=_CONTRACT_COLUMNS, + ), + compiled_model( + name="below", + columns=[ + inferred_column(name="id", type_sql="INT", nullability="NON_NULL"), + inferred_column(name="label", type_sql="TEXT(24)"), + ], + references=[reference(kind="REF", name="contracted")], + ), + ], + settings={"table_promotion_mode": "staged"}, + ), + expected_present=frozenset( + { + "sized_contract_shape", + "contract_nullability_shape", + "batch_binding", + "explicit_promotion_mode", + } + ), + expected_absent=frozenset({"dataflow_binding"}), + ), + AnalysisCaptureKindsTestCase( + description="nullability_already_inferred_upstream_is_not_contract_evidence", + capture=empty_compiled_capture( + models=[ + compiled_model( + name="contracted", + columns=[ + inferred_column(name="id", type_sql="INT", nullability="NON_NULL") + ], + values=_CONTRACT, + schema_columns=_CONTRACT_COLUMNS, + ), + compiled_model( + name="below", + columns=[ + inferred_column(name="id", type_sql="INT", nullability="NON_NULL") + ], + references=[reference(kind="REF", name="contracted")], + ), + ], + ), + expected_present=frozenset({"typed_column"}), + expected_absent=frozenset({"contract_nullability_shape", "sized_contract_shape"}), + ), + AnalysisCaptureKindsTestCase( + description="selection_leaves_unselected_models_unanalysed", + capture=empty_compiled_capture( + sources=[compiled_source(name="raw_orders", expression="(SELECT 1 AS id)")], + models=[ + compiled_model( + name="stg_orders", + columns=_TYPED_ORDERS, + references=[reference(kind="SOURCE", name="raw_orders")], + ), + compiled_model( + name="totals", + columns=None, + references=[reference(kind="REF", name="stg_orders")], + validated=False, + ), + ], + ), + expected_present=frozenset({"select_limited_analysis", "batch_binding"}), + expected_absent=frozenset({"sql_analysis_disabled", "dataflow_binding"}), + ), + AnalysisCaptureKindsTestCase( + description="literal_sql_relations_need_explicit_references", + capture=empty_compiled_capture( + enforce_explicit_references=False, + unmatched_literal_sql_relations=[{"owner_label": "task:export"}], + ), + expected_present=frozenset({"unmatched_literal_sql_relations"}), + expected_absent=frozenset({"python_sql_literal_relation"}), + ), + AnalysisCaptureKindsTestCase( + description="quoted_identifier_bound_only_by_ignoring_case", + capture=empty_compiled_capture( + models=[ + compiled_model(name="stg_orders", columns=_TYPED_ORDERS), + compiled_model( + name="quoted", + query_sql='SELECT "ID" AS "OrderKey" FROM __ref("stg_orders")', + columns=[inferred_column(name="OrderKey", type_sql="INT")], + references=[reference(kind="REF", name="stg_orders")], + ), + ], + ), + expected_present=frozenset({"quoted_identifier_case"}), + expected_absent=frozenset({"cte", "set_operation"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_compiled_capture_when_classifying_then_only_analysed_evidence_counts( + test_case: AnalysisCaptureKindsTestCase, +) -> None: + kinds: frozenset[str] = analysed_input_kinds(json.dumps(test_case.capture)) + + assert test_case.expected_present <= kinds, test_case.expected_present - kinds + assert not test_case.expected_absent & kinds, test_case.expected_absent & kinds + + +@pytest.mark.parametrize( + "test_case", + [ + RequiredKindsTestCase( + description="compiled_collections_and_detail_kinds", + expected_required=frozenset( + {"models", "sql_tests", "sql_expansions", "typed_column", "dialect_snowflake"} + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_compiled_project_fields_when_listing_required_kinds_then_settings_are_excluded( + test_case: RequiredKindsTestCase, +) -> None: + required: tuple[str, ...] = required_analysis_kinds() + + assert test_case.expected_required <= set(required) + assert not ANALYSIS_NON_COLLECTION_FIELDS & set(required) + assert len(required) == len(set(required)) + + +@pytest.mark.parametrize( + "test_case", + [ + AnalysisCaptureProblemsTestCase( + description="binding_catalog_native_handle_is_expected", + capture_text=json.dumps( + empty_compiled_capture( + binding_catalog={"native": {"__opaque__": "sqlbuild._native:ProjectCatalog"}} + ) + ), + expected_problem_count=0, + ), + AnalysisCaptureProblemsTestCase( + description="any_other_opaque_value_is_a_problem", + capture_text=json.dumps( + empty_compiled_capture(models=[{"__opaque__": "sqlbuild._native:Unexpected"}]) + ), + expected_problem_count=1, + ), + AnalysisCaptureProblemsTestCase( + description="missing_field_is_a_problem", + capture_text=json.dumps( + { + key: value + for key, value in empty_compiled_capture().items() + if key != "sql_scenarios" + } + ), + expected_problem_count=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_compiled_capture_when_checking_soundness_then_problems_are_counted( + test_case: AnalysisCaptureProblemsTestCase, +) -> None: + assert len(analysis_capture_problems(test_case.capture_text)) == ( + test_case.expected_problem_count + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py index 87744bb7f6..6233c40c85 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py +++ b/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py @@ -64,3 +64,14 @@ class SummaryTestCase: missing_coverage: dict[str, tuple[str, ...]] expected_lines: tuple[str, ...] expected_absent: tuple[str, ...] + + +@dataclass(frozen=True) +class AnalysisRecordsTestCase: + """Record files two engine processes wrote and the report lines they must produce.""" + + description: str + files: dict[str, str] + expected_wheel_sites: dict[tuple[str, str], int] + expected_deferrals: dict[tuple[str, str], int] + expected_lines: tuple[str, ...] diff --git a/tests/unit/scripts/compiler_differential/_helpers/running/test_records.py b/tests/unit/scripts/compiler_differential/_helpers/running/test_records.py new file mode 100644 index 0000000000..2f3a00bccd --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/running/test_records.py @@ -0,0 +1,82 @@ +"""Wheel-site and deferral records are summed per engine, site and corpus, never gated.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from scripts.compiler_differential._helpers.running.records import ( + format_wheel_site_report, + read_analysis_records, + wheel_site_report, +) +from scripts.compiler_differential.models import AnalysisRecords, ProjectComparison +from tests.unit.scripts.compiler_differential._helpers.running._test_types import ( + AnalysisRecordsTestCase, +) + +_SITE: str = "compiler/compile/_helpers/analysis/columns.py:_infer_columns_with_polyglot" + + +@pytest.mark.parametrize( + "test_case", + [ + AnalysisRecordsTestCase( + description="two_processes_and_one_deferral", + files={ + "0-compile/polyglot-sites-11.json": json.dumps( + {"calls": [[_SITE, "parse_one", 2]]} + ), + "2-plan/polyglot-sites-12.json": json.dumps({"calls": [[_SITE, "parse_one", 3]]}), + "2-plan/analysis-deferrals-12.jsonl": ( + json.dumps({"kind": "legacy_fallback", "site": "compact.py"}) + "\n" + ), + }, + expected_wheel_sites={(_SITE, "parse_one"): 5}, + expected_deferrals={("legacy_fallback", "compact.py"): 1}, + expected_lines=( + "Polyglot wheel calls (python):", + f" 5 {_SITE} parse_one (seed 5)", + "Analysis deferrals (python):", + " 1 legacy_fallback compact.py (seed 1)", + "Polyglot wheel calls (native-preview): none recorded", + "Analysis deferrals (native-preview): none recorded", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_record_files_when_reporting_then_counts_are_summed_per_engine_and_corpus( + test_case: AnalysisRecordsTestCase, tmp_path: Path +) -> None: + for relative_path, contents in test_case.files.items(): + path: Path = tmp_path / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + + records: AnalysisRecords = read_analysis_records(tmp_path) + report: str = format_wheel_site_report( + wheel_site_report( + comparisons=[ + ProjectComparison( + project="seed/3", + differences=(), + seconds=1.0, + records=(records, read_analysis_records(tmp_path / "missing")), + ) + ], + engines=("python", "native-preview"), + ) + ) + + assert (records.wheel_sites, records.deferrals) == ( + test_case.expected_wheel_sites, + test_case.expected_deferrals, + ) + assert report.splitlines() == list(test_case.expected_lines) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 14e286043e557a04f6f0a3327e6adce24351c28d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 03:48:48 +0100 Subject: [PATCH 4/4] test(compiler): address differential coverage review findings Reach P003 through the sql_analysis/sql_validation conflict, keep the analysis record directory and the compile-reuse switch out of compile reuse and macro-call store identity, credit hook and promotion kinds only on analysed models, mirror the product's complete binding schemas, and copy the site counter under its lock. --- .../_helpers/analysis_corpus/failure_cases.py | 6 ++ .../_helpers/coverage/analysis.py | 44 +++++++++++-- scripts/compiler_differential/constants.py | 9 +-- src/sqlbuild/cli/compile_reuse/constants.py | 11 +++- src/sqlbuild/compiler/frontier/constants.py | 1 + .../compiler/macro_bridge/constants.py | 14 ++++- .../sql_analysis/_helpers/polyglot.py | 6 +- .../main/compile/test_compile_reuse.py | 6 ++ .../_helpers/coverage/helpers.py | 28 ++++++++- .../_helpers/coverage/test_analysis.py | 63 +++++++++++++++++++ 10 files changed, 171 insertions(+), 17 deletions(-) diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py index 00956f4030..e95a90f5fd 100644 --- a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py @@ -474,6 +474,12 @@ def _syntax_cases() -> tuple[FailureCase, ...]: ), }, ), + failure_case( + name="analysis-opt-out-conflicts-with-legacy-key", + expected_code="P003", + expected_message="MODEL sql_analysis conflicts with legacy sql_validation", + files=_mart_header(extra=" sql_analysis false,\n sql_validation true,"), + ), ) diff --git a/scripts/compiler_differential/_helpers/coverage/analysis.py b/scripts/compiler_differential/_helpers/coverage/analysis.py index e4e6c4e7d8..c7e4576ccd 100644 --- a/scripts/compiler_differential/_helpers/coverage/analysis.py +++ b/scripts/compiler_differential/_helpers/coverage/analysis.py @@ -4,12 +4,13 @@ import dataclasses import re -from collections.abc import Iterator +from collections.abc import Iterator, Sequence from pathlib import Path from scripts.compiler_differential._helpers.comparing.compare import as_json_object from scripts.compiler_differential._helpers.coverage.capture_values import ( capture_problems, + decoded, enum_name, has_content, records, @@ -308,18 +309,19 @@ def _binding_path_kinds(project: _Project) -> set[str]: def _reads_complete_inputs(*, project: _Project, model: dict[str, object]) -> bool: - """Whether every relation the model reads has a schema known before any model is analysed.""" + """Mirror `build_complete_binding_schemas` plus resolved expression-source shapes.""" + resolved_expressions: frozenset[str] = _resolved_expression_sources(project) for kind, name in _references(model): if kind == _MODEL_REFERENCE: upstream: dict[str, object] | None = project.models.get(name) - if upstream is None or not (_contract_enforced(upstream) and _schema_columns(upstream)): + if upstream is None or not _declares_complete_schema(upstream): return False elif kind == _SOURCE_REFERENCE: entry: dict[str, object] = ( as_json_object((project.sources.get(name) or {}).get("source_entry")) or {} ) - if not entry.get("expression") and not ( + if str(entry.get("expression")) not in resolved_expressions and not ( entry.get("contract") == _ENFORCED and records(entry.get("columns")) ): return False @@ -332,6 +334,38 @@ def _reads_complete_inputs(*, project: _Project, model: dict[str, object]) -> bo return True +def _declares_complete_schema(model: dict[str, object]) -> bool: + schema_entry: dict[str, object] = as_json_object(model.get("schema_entry")) or {} + return ( + _contract_enforced(model) + and bool(_schema_columns(model)) + and not has_content(schema_entry.get("dynamic_columns")) + ) + + +def _resolved_expression_sources(project: _Project) -> frozenset[str]: + """Return the source expressions whose shape compile resolved before analysis.""" + + shapes: object = decoded( + (as_json_object(project.capture.get("binding_catalog")) or {}).get("expression_shapes") + ) + pairs: Sequence[object] = ( + list(shapes.items()) + if isinstance(shapes, dict) + else shapes + if isinstance(shapes, list) + else [] + ) + resolved: set[str] = set() + for pair in pairs: + match pair: + case [expression, shape] if shape is not None: + resolved.add(str(expression)) + case _: + pass + return frozenset(resolved) + + def _longest_analysed_chain(project: _Project) -> int: """Return the most model-to-model hops along analysed models, ignoring reference cycles.""" @@ -441,6 +475,8 @@ def _configuration_kinds(project: _Project) -> set[str]: and model.get("rejected_sql_analysis_opt_out") is None ): kinds.add("sql_analysis_opt_out") + if model.get("inferred_columns") is None: + continue for key in _HOOK_KEYS: hooks: list[dict[str, object]] = [ hook diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index cb6c6a469c..d67572e0b5 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -624,6 +624,7 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: ANALYSIS_CODE_SCAN_ROOTS: tuple[str, ...] = ( "compile/_helpers/assembly", "compile/_helpers/analysis", + "compile/_helpers/attachment/model_config.py", "compile/_helpers/diagnostics", "contracts", "pipeline/_helpers/target_validation.py", @@ -639,12 +640,8 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: ), "B210": _NEVER_REPORTED_BY_POLYGLOT, "B214": ( - "polyglot reports assignment types only for INSERT statements; compiled models, tests " - "and audits are queries" + "polyglot reports assignment types only for INSERT and UPDATE statements; compiled " + "models, tests and audits are queries" ), "B219": _NEVER_REPORTED_BY_POLYGLOT, - "P003": ( - "the contract check raises it for a declared column without a type, which it skips " - "first; compile reports P003 only as the built-in audit shadow warning" - ), } diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index 0f9e81a3b0..2bd4a76bac 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -8,9 +8,11 @@ CompileArtifactWriteRegistry, ) from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR, COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR, ) +from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR REUSE_FORMAT_VERSION: int = 1 REUSE_ENTRY_DIRECTORY_NAME: str = "project-reuse-v1" @@ -25,12 +27,17 @@ REUSE_ENTRY_BYTE_ORDER: Literal["big"] = "big" REUSE_MAX_ENTRY_BYTES: int = 512 * 1024 * 1024 REUSE_MAX_STORED_ENTRIES: int = 8 -REUSE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_REUSE" +REUSE_DISABLE_ENV_VAR: str = COMPILE_REUSE_DISABLE_ENV_VAR REUSE_DISABLE_VALUE: str = "1" REUSE_LOGGER_NAME: str = "sqlbuild.compile.reuse" TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] = ("SQLBUILD_", "SQB_") UNTRACKED_ENVIRONMENT_NAMES: frozenset[str] = frozenset( - {COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR} + { + ANALYSIS_RECORD_DIR_ENV_VAR, + COMPILE_REUSE_DISABLE_ENV_VAR, + COMPILER_ENGINE_ENV_VAR, + STAGE_CAPTURE_DIR_ENV_VAR, + } ) MISSING_ENVIRONMENT_VALUE: str = "\0missing" MISSING_FILE_DIGEST: str = "\0missing" diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index 949292fc67..4e0ddf0b0e 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -34,6 +34,7 @@ CACHE_DIRECTORY_NAME: str = "cache" COMPILER_CACHE_DIRECTORY_NAME: str = "compiler" STAGE_CAPTURE_DIR_ENV_VAR: str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR" +COMPILE_REUSE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_REUSE" STAGE_CAPTURE_SUFFIX: str = ".json" STAGE_CAPTURE_SHARED_MARKER: str = "__shared__" STAGE_CAPTURE_ROOT_KEY: str = "__capture__" diff --git a/src/sqlbuild/compiler/macro_bridge/constants.py b/src/sqlbuild/compiler/macro_bridge/constants.py index 50b84b4929..af15940fe6 100644 --- a/src/sqlbuild/compiler/macro_bridge/constants.py +++ b/src/sqlbuild/compiler/macro_bridge/constants.py @@ -5,8 +5,13 @@ import itertools from contextvars import ContextVar -from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR, + COMPILER_ENGINE_ENV_VAR, + STAGE_CAPTURE_DIR_ENV_VAR, +) from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR MACRO_USE_EVENT: int = 0 DECLARATION_READ_EVENT: int = 1 @@ -31,5 +36,10 @@ INTERPRETER_MODULE_ORIGINS: frozenset[str] = frozenset({"built-in", "frozen"}) STORE_TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] = ("SQLBUILD_", "SQB_") STORE_UNTRACKED_ENVIRONMENT_NAMES: frozenset[str] = frozenset( - {COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR} + { + ANALYSIS_RECORD_DIR_ENV_VAR, + COMPILE_REUSE_DISABLE_ENV_VAR, + COMPILER_ENGINE_ENV_VAR, + STAGE_CAPTURE_DIR_ENV_VAR, + } ) diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot.py b/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot.py index e85d3d7642..d6afc038ed 100644 --- a/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot.py +++ b/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot.py @@ -96,11 +96,13 @@ def record(self, *, api: str, caller: Any) -> None: self._calls[(f"{site}:{code.co_qualname}", api)] += 1 def _write(self) -> None: - if not self._calls: + with self._lock: + calls: Counter[tuple[str, str]] = self._calls.copy() + if not calls: return self._directory.mkdir(parents=True, exist_ok=True) rows: list[list[object]] = [ - [site, api, count] for (site, api), count in sorted(self._calls.items()) + [site, api, count] for (site, api), count in sorted(calls.items()) ] target: Path = self._directory / f"{POLYGLOT_SITE_RECORD_PREFIX}{os.getpid()}.json" _ = target.write_text(json.dumps({"calls": rows}) + "\n", encoding="utf-8") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py index 01bd824022..daf0d72598 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py @@ -9,6 +9,7 @@ import pytest +from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileReuseBypassTestCase, CompileReuseConcurrentWriterTestCase, @@ -279,6 +280,11 @@ def test_given_reused_compile_when_input_changes_then_it_misses_and_matches_unca edit=lambda _root: None, edited_env={"ORDERS_UNRELATED_SETTING": "1"}, ), + CompileReuseHitTestCase( + description="analysis_record_directory", + edit=lambda _root: None, + edited_env={ANALYSIS_RECORD_DIR_ENV_VAR: "target/analysis-records"}, + ), CompileReuseHitTestCase( description="editor_settings_added", edit=lambda root: write_project_file(root, ".vscode/settings.json", "{}\n"), diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py index 6ca8ca73bc..9a71c56c7a 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py @@ -143,6 +143,7 @@ def compiled_model( validated: bool = True, values: dict[str, object] | None = None, schema_columns: list[dict[str, object]] | None = None, + dynamic_columns: list[dict[str, object]] | None = None, star: tuple[bool, bool] = (False, False), ) -> dict[str, object]: """Return one captured compiled model; `columns=None` means it was not analysed.""" @@ -152,7 +153,11 @@ def compiled_model( "query_sql": query_sql, "config": {"values": values or {}}, "references": references or [], - "schema_entry": {"columns": schema_columns or [], "audits": []}, + "schema_entry": { + "columns": schema_columns or [], + "dynamic_columns": dynamic_columns or [], + "audits": [], + }, "inferred_columns": columns, "binding_validated": validated, "binding_diagnostics": [], @@ -168,3 +173,24 @@ def compiled_source(*, name: str, expression: str | None) -> dict[str, object]: """Return one captured compiled source with an optional SQL expression.""" return {"source_entry": {"name": name, "expression": expression, "columns": []}} + + +def hooked_compiled_capture(*, columns: list[object] | None) -> dict[str, object]: + """Return a staged-promotion capture whose contracted model has inline and named SQL hooks.""" + + hooks: list[dict[str, object]] = [ + {"__type__": "sqlbuild:SqlHookEntry", "definition_sql": None}, + {"__type__": "sqlbuild:SqlHookEntry", "definition_sql": "SELECT 1"}, + ] + return empty_compiled_capture( + settings={"table_promotion_mode": "staged"}, + models=[ + compiled_model( + name="contracted", + columns=columns, + validated=columns is not None, + values={"contract": "enforced", "materialized": "table", "post_hooks": hooks}, + schema_columns=[{"name": "id", "type": "INTEGER", "nullable": False}], + ) + ], + ) diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_analysis.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_analysis.py index 92da94d8a3..b118c6a866 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_analysis.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_analysis.py @@ -21,6 +21,7 @@ compiled_model, compiled_source, empty_compiled_capture, + hooked_compiled_capture, inferred_column, reference, ) @@ -34,6 +35,12 @@ {"name": "label", "type": "VARCHAR(24)", "nullable": None}, ] _CONTRACT: dict[str, object] = {"contract": "enforced", "materialized": "table"} +_CONFIG_KINDS: tuple[str, ...] = ( + "explicit_promotion_mode", + "inline_sql_hook", + "named_sql_hook", + "hook_list", +) @pytest.mark.parametrize( @@ -167,6 +174,11 @@ description="selection_leaves_unselected_models_unanalysed", capture=empty_compiled_capture( sources=[compiled_source(name="raw_orders", expression="(SELECT 1 AS id)")], + binding_catalog={ + "expression_shapes": { + "__unordered_mapping__": [["(SELECT 1 AS id)", {"id": "INT"}]] + } + }, models=[ compiled_model( name="stg_orders", @@ -184,6 +196,57 @@ expected_present=frozenset({"select_limited_analysis", "batch_binding"}), expected_absent=frozenset({"sql_analysis_disabled", "dataflow_binding"}), ), + AnalysisCaptureKindsTestCase( + description="unresolved_expression_source_is_not_a_complete_input", + capture=empty_compiled_capture( + sources=[compiled_source(name="raw_orders", expression="(SELECT 1 AS id)")], + binding_catalog={ + "expression_shapes": {"__unordered_mapping__": [["(SELECT 1 AS id)", None]]} + }, + models=[ + compiled_model( + name="stg_orders", + columns=_TYPED_ORDERS, + references=[reference(kind="SOURCE", name="raw_orders")], + ) + ], + ), + expected_present=frozenset({"dataflow_binding"}), + expected_absent=frozenset({"batch_binding"}), + ), + AnalysisCaptureKindsTestCase( + description="dynamic_column_contract_is_not_a_complete_input", + capture=empty_compiled_capture( + models=[ + compiled_model( + name="contracted", + columns=[inferred_column(name="id", type_sql="INT")], + values=_CONTRACT, + schema_columns=_CONTRACT_COLUMNS, + dynamic_columns=[{"name": "status_amounts"}], + ), + compiled_model( + name="below", + columns=[inferred_column(name="id", type_sql="INT")], + references=[reference(kind="REF", name="contracted")], + ), + ], + ), + expected_present=frozenset({"dataflow_binding"}), + expected_absent=frozenset({"batch_binding"}), + ), + AnalysisCaptureKindsTestCase( + description="hooks_and_promotion_on_an_analysed_model", + capture=hooked_compiled_capture(columns=_TYPED_ORDERS), + expected_present=frozenset(_CONFIG_KINDS), + expected_absent=frozenset(), + ), + AnalysisCaptureKindsTestCase( + description="hooks_and_promotion_on_an_unanalysed_model_are_not_credited", + capture=hooked_compiled_capture(columns=None), + expected_present=frozenset({"models"}), + expected_absent=frozenset(_CONFIG_KINDS), + ), AnalysisCaptureKindsTestCase( description="literal_sql_relations_need_explicit_references", capture=empty_compiled_capture(