diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index d007476831..1645e78582 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -293,7 +293,8 @@ jobs: - name: Compare fixtures, examples, generated seeds and the failure corpus run: >- make compiler-differential - COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage" + COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage + --require-render-coverage" format_performance: name: Formatter performance guards diff --git a/scripts/compiler_differential/_helpers/comparing/comparison.py b/scripts/compiler_differential/_helpers/comparing/comparison.py index 71c3ae0851..97dc2c2eea 100644 --- a/scripts/compiler_differential/_helpers/comparing/comparison.py +++ b/scripts/compiler_differential/_helpers/comparing/comparison.py @@ -72,14 +72,28 @@ def compare_engine_runs(*, project: str, left: EngineRun, right: EngineRun) -> l def diagnostic_codes(*, outcome: CommandOutcome, severity: str | None) -> tuple[str, ...]: """Return a JSON compile report's diagnostic codes in order, optionally of one severity.""" + return tuple( + str(diagnostic.get("code")) + for diagnostic in _diagnostics(outcome=outcome, severity=severity) + ) + + +def first_diagnostic_message(*, outcome: CommandOutcome, severity: str) -> str | None: + """Return the message of a JSON compile report's first diagnostic of one severity.""" + + diagnostics: list[dict[str, object]] = _diagnostics(outcome=outcome, severity=severity) + return str(diagnostics[0].get("message")) if diagnostics else None + + +def _diagnostics(*, outcome: CommandOutcome, severity: str | None) -> list[dict[str, object]]: payload: dict[str, object] = as_json_object(_loads(outcome.stdout)) or {} diagnostics: object = payload.get("diagnostics") - found: list[str] = [] + found: list[dict[str, object]] = [] for raw_diagnostic in diagnostics if isinstance(diagnostics, list) else (): diagnostic: dict[str, object] | None = as_json_object(raw_diagnostic) if diagnostic is not None and (severity is None or diagnostic.get("severity") == severity): - found.append(str(diagnostic.get("code"))) - return tuple(found) + found.append(diagnostic) + return found def _command_differences( diff --git a/scripts/compiler_differential/_helpers/comparing/project_comparison.py b/scripts/compiler_differential/_helpers/comparing/project_comparison.py index 308d7b7791..c1e2f3af67 100644 --- a/scripts/compiler_differential/_helpers/comparing/project_comparison.py +++ b/scripts/compiler_differential/_helpers/comparing/project_comparison.py @@ -11,6 +11,7 @@ diagnostic_codes, ) from scripts.compiler_differential._helpers.coverage.discovery import project_discovery_kinds +from scripts.compiler_differential._helpers.coverage.render import project_render_kinds from scripts.compiler_differential._helpers.running.execution import run_engine from scripts.compiler_differential.constants import ERROR_SEVERITY, WARNING_SEVERITY from scripts.compiler_differential.models import ( @@ -50,6 +51,7 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) -> engine=right_engine, options=options, ) + counted: bool = project.seed_coverage and options.stage_captures differences: list[Difference] = [ *_timeout_differences(project=project, runs=(left, right)), *_corpus_differences(project=project, run=left), @@ -61,9 +63,10 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) -> seconds=time.monotonic() - started, discovered_kinds=( project_discovery_kinds(captures=left.captures, source_dir=source_dir) - if project.discovery_coverage and options.stage_captures + if counted else frozenset() ), + rendered_kinds=project_render_kinds(left.captures) if counted else frozenset(), ) diff --git a/scripts/compiler_differential/_helpers/corpus/case_builder.py b/scripts/compiler_differential/_helpers/corpus/case_builder.py new file mode 100644 index 0000000000..7e3bc2886e --- /dev/null +++ b/scripts/compiler_differential/_helpers/corpus/case_builder.py @@ -0,0 +1,49 @@ +"""Build failure cases as small edits of the shared base project.""" + +from scripts.compiler_differential.constants import ( + FAILURE_BASE_CONFIG, + FAILURE_BASE_FILES, + FAILURE_CONFIG_PATH, + FAILURE_MART_PATH, + FAILURE_STAGING_PATH, +) +from scripts.compiler_differential.models import FailureCase + +_MART_HEADER: str = 'MODEL (\n description "Order totals per customer",\n);\n\n' + + +def failure_case( + *, + name: str, + expected_code: str, + files: dict[str, str], + expected_warning_code: str | None = None, + expected_message: str | None = None, +) -> FailureCase: + """Return the base project with `files` added or replaced.""" + + return FailureCase( + files={**FAILURE_BASE_FILES, **files}, + name=name, + expected_code=expected_code, + expected_warning_code=expected_warning_code, + expected_message=expected_message, + ) + + +def staging_files(sql: str) -> dict[str, str]: + """Replace the staging model.""" + + return {FAILURE_STAGING_PATH: sql} + + +def mart_body_files(body: str) -> dict[str, str]: + """Replace the mart model's query, keeping its header.""" + + return {FAILURE_MART_PATH: _MART_HEADER + body} + + +def config_files(extra: str) -> dict[str, str]: + """Append `extra` to the base project configuration.""" + + return {FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + extra} diff --git a/scripts/compiler_differential/_helpers/corpus/corpus.py b/scripts/compiler_differential/_helpers/corpus/corpus.py index d1d51c4127..805f8aa782 100644 --- a/scripts/compiler_differential/_helpers/corpus/corpus.py +++ b/scripts/compiler_differential/_helpers/corpus/corpus.py @@ -143,5 +143,5 @@ def _seed_project(seed: int) -> CorpusProject: succeeding_commands=generated.succeeding_commands, ), writer=generated.write, - discovery_coverage=True, + seed_coverage=True, ) diff --git a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py index 858e2864ee..09701b1305 100644 --- a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py @@ -1,5 +1,9 @@ """Minimal projects that fail inside project discovery, one per diagnostic family.""" +from scripts.compiler_differential._helpers.corpus.case_builder import ( + config_files, + failure_case, +) from scripts.compiler_differential.constants import ( FAILURE_BASE_CONFIG, FAILURE_BASE_FILES, @@ -33,16 +37,6 @@ ) -def _case(*, name: str, expected_code: str, files: dict[str, str]) -> FailureCase: - return FailureCase( - files={**FAILURE_BASE_FILES, **files}, name=name, expected_code=expected_code - ) - - -def _config(extra: str) -> dict[str, str]: - return {FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + extra} - - def discovery_failure_cases() -> tuple[FailureCase, ...]: """Return the discovery-time failure cases in a stable order.""" @@ -58,47 +52,47 @@ def discovery_failure_cases() -> tuple[FailureCase, ...]: def _config_cases() -> tuple[FailureCase, ...]: return ( - _case( + failure_case( name="config-bom", expected_code="D001", files={FAILURE_CONFIG_PATH: GENERATOR_BOM + FAILURE_BASE_CONFIG}, ), - _case( + failure_case( name="local-config-unknown-key", expected_code="D001", files={"sqlbuild_local.toml": "[settings]\nsql_analysiss = false\n"}, ), - _case( + failure_case( name="target-unknown-key", expected_code="D001", - files=_config('\n[targets.ci]\nschema = "ci"\nwarehouse_size = "large"\n'), + files=config_files('\n[targets.ci]\nschema = "ci"\nwarehouse_size = "large"\n'), ), - _case( + failure_case( name="dbt-unknown-key", expected_code="D001", - files=_config('\n[dbt]\nproject_dir = "dbt"\nmanifest = "dbt/manifest.json"\n'), + files=config_files('\n[dbt]\nproject_dir = "dbt"\nmanifest = "dbt/manifest.json"\n'), ), - _case( + failure_case( name="path-defaults-unmatched", expected_code="D007", - files=_config('\n[path_defaults."reporting/finance"]\nmaterialized = "table"\n'), + files=config_files('\n[path_defaults."reporting/finance"]\nmaterialized = "table"\n'), ), ) def _sql_file_cases() -> tuple[FailureCase, ...]: return ( - _case( + failure_case( name="model-bom", expected_code="D002", files={FAILURE_STAGING_PATH: GENERATOR_BOM + FAILURE_BASE_STAGING}, ), - _case( + failure_case( name="model-missing-header", expected_code="D002", files={FAILURE_STAGING_PATH: 'SELECT order_id\nFROM __source("raw_orders")\n'}, ), - _case( + failure_case( name="model-unknown-header-key", expected_code="D002", files={ @@ -107,12 +101,12 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="model-reserved-name", expected_code="D016", files={"models/marts/_chain_.sql": FAILURE_BASE_MART}, ), - _case( + failure_case( name="test-case-missing-parameter", expected_code="D003", files={ @@ -127,7 +121,7 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="generic-audit-header", expected_code="D004", files={ @@ -136,7 +130,7 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="list-constant-mixed-types", expected_code="D013", files={ @@ -145,7 +139,7 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="enum-duplicate-member", expected_code="D013", files={ @@ -154,7 +148,7 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="sql-hook-unknown-key", expected_code="D014", files={ @@ -163,7 +157,7 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="sql-function-header", expected_code="D002", files={ @@ -173,7 +167,7 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="column-nullable-not-null", expected_code="P002", files={ @@ -189,7 +183,7 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: def _layout_cases() -> tuple[FailureCase, ...]: return tuple( - _case(name=name, expected_code="D013", files={path: contents}) + failure_case(name=name, expected_code="D013", files={path: contents}) for name, path, contents in ( ("scoped-root-at-project-root", "_enums/order_status.sql", ""), ("declaration-group-below-root", "models/_sqlbuild/macros/money.py", _MACRO_MODULE), @@ -205,7 +199,7 @@ def _layout_cases() -> tuple[FailureCase, ...]: def _yaml_file_cases() -> tuple[FailureCase, ...]: return ( - _case( + failure_case( name="schema-yml-model-metadata", expected_code="D005", files={ @@ -214,7 +208,7 @@ def _yaml_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="seed-yaml-extension", expected_code="D005", files={ @@ -222,7 +216,7 @@ def _yaml_file_cases() -> tuple[FailureCase, ...]: "seeds/order_channels.csv": "id,label\n1,web\n", }, ), - _case( + failure_case( name="source-unknown-key", expected_code="D006", files={ @@ -232,12 +226,12 @@ def _yaml_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="seed-declaration-without-csv", expected_code="D008", files={"seeds/order_channels.yml": _SEED_DECLARATION}, ), - _case( + failure_case( name="seed-header-mismatch", expected_code="D008", files={ @@ -250,12 +244,12 @@ def _yaml_file_cases() -> tuple[FailureCase, ...]: def _conflict_cases() -> tuple[FailureCase, ...]: return ( - _case( + failure_case( name="duplicate-source", expected_code="D007", files={"sources/more.yml": FAILURE_BASE_FILES[FAILURE_SOURCES_PATH]}, ), - _case( + failure_case( name="duplicate-scenario", expected_code="D007", files={ @@ -269,7 +263,7 @@ def _conflict_cases() -> tuple[FailureCase, ...]: for folder in ("north", "south") }, ), - _case( + failure_case( name="resource-name-collision", expected_code="D007", files={ @@ -277,7 +271,7 @@ def _conflict_cases() -> tuple[FailureCase, ...]: "seeds/stg_orders.csv": "id,label\n1,web\n", }, ), - _case( + failure_case( name="managed-source-without-loader", expected_code="D007", files={ @@ -287,7 +281,7 @@ def _conflict_cases() -> tuple[FailureCase, ...]: " columns:\n - name: id\n type: INTEGER\n" }, ), - _case( + failure_case( name="duplicate-python-node", expected_code="D007", files={ @@ -295,7 +289,7 @@ def _conflict_cases() -> tuple[FailureCase, ...]: "python/tasks/second.py": _TASK_MODULE, }, ), - _case( + failure_case( name="check-depends-on-check", expected_code="D007", files={ @@ -313,12 +307,12 @@ def _conflict_cases() -> tuple[FailureCase, ...]: def _python_cases() -> tuple[FailureCase, ...]: return ( - _case( + failure_case( name="python-node-import-error", expected_code="D011", files={"python/tasks/exports.py": "import missing_orders_client\n" + _TASK_MODULE}, ), - _case( + failure_case( name="duplicate-hook", expected_code="D011", files={ @@ -326,7 +320,7 @@ def _python_cases() -> tuple[FailureCase, ...]: "hooks/python/second.py": _HOOK_MODULE, }, ), - _case( + failure_case( name="provider-invalid-settings", expected_code="D012", files={ @@ -338,7 +332,7 @@ def _python_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="duplicate-provider", expected_code="D012", files={ @@ -351,12 +345,12 @@ def _python_cases() -> tuple[FailureCase, ...]: for module in ("first", "second") }, ), - _case( + failure_case( name="legacy-event-exporters-directory", expected_code="D015", files={"event_exporters/publish.py": "VALUE = 1\n"}, ), - _case( + failure_case( name="lifecycle-sink-default-parameter", expected_code="D015", files={ @@ -368,7 +362,7 @@ def _python_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="lifecycle-sink-unknown-provider", expected_code="D015", files={ @@ -380,7 +374,7 @@ def _python_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="command-output-sink-unknown-provider", expected_code="D015", files={ @@ -392,7 +386,7 @@ def _python_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="materialization-import-error", expected_code="D011", files={ @@ -403,7 +397,7 @@ def _python_cases() -> tuple[FailureCase, ...]: ), }, ), - _case( + failure_case( name="python-outside-extension-root", expected_code="D016", files={"helpers/formatting.py": "VALUE = 1\n"}, diff --git a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py index fdfb77c540..810e3f1480 100644 --- a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py +++ b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py @@ -2,16 +2,23 @@ from __future__ import annotations +import ast from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from scripts.compiler_differential._helpers.comparing.comparison import diagnostic_codes +from scripts.compiler_differential._helpers.comparing.comparison import ( + diagnostic_codes, + first_diagnostic_message, +) from scripts.compiler_differential._helpers.corpus.corpus import build_corpus from scripts.compiler_differential._helpers.running.execution import run_engine from scripts.compiler_differential.constants import ( CORPUS_FAILURES, + DIAGNOSTIC_CODE_PATTERN, ERROR_SEVERITY, EXPECT_SUCCESS, + RENDER_CODE_SCAN_EXCLUDED, + RENDER_RAISED_CODES, WARNING_SEVERITY, ) from scripts.compiler_differential.models import ( @@ -20,10 +27,15 @@ EmittedCodes, EngineRun, ) +from sqlbuild.compiler.auditing.constants import BUILT_IN_AUDIT_SHADOW_CODE +from sqlbuild.compiler.compile import constants as compile_constants +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.exceptions import DiscoveryError from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError +from sqlbuild.compiler.scopes.types import ScopeDiagnosticCode _SOURCE_DIRECTORY: str = "source" +_CODE_KEYWORD: str = "code" def emitted_failure_codes(*, options: DifferentialOptions) -> dict[str, EmittedCodes]: @@ -57,6 +69,66 @@ def discovery_error_codes() -> frozenset[str]: return frozenset(codes) +def render_error_codes() -> frozenset[str]: + """Return every code render, scope, and attachment can report, from compiler constants.""" + + compile_codes: set[str] = { + value + for name, value in vars(compile_constants).items() + if name.endswith("_CODE") + and isinstance(value, str) + and DIAGNOSTIC_CODE_PATTERN.fullmatch(value) + } + return frozenset( + { + CompileInputError.code, + BUILT_IN_AUDIT_SHADOW_CODE, + *compile_codes, + *(code.value for code in ScopeDiagnosticCode), + *RENDER_RAISED_CODES, + } + ) + + +def inline_compile_codes(compile_root: Path) -> dict[str, tuple[str, ...]]: + """Map each `code="X###"` literal in render-stage compile modules to where it is written.""" + + found: dict[str, list[str]] = {} + for path in sorted(compile_root.rglob("*.py")): + relative: str = path.relative_to(compile_root).as_posix() + if any(relative.startswith(f"{excluded}/") for excluded in RENDER_CODE_SCAN_EXCLUDED): + continue + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + code: str | None = _code_literal(node) + if code is not None: + found.setdefault(code, []).append(f"{relative}:{getattr(node, 'lineno', 0)}") + return {code: tuple(locations) for code, locations in found.items()} + + +def unaccounted_inline_codes( + *, compile_root: Path, accounted: frozenset[str] +) -> dict[str, tuple[str, ...]]: + """Return the inline render-stage codes that are neither required nor documented.""" + + return { + code: locations + for code, locations in inline_compile_codes(compile_root).items() + if code not in accounted + } + + +def _code_literal(node: ast.AST) -> str | None: + if not ( + isinstance(node, ast.keyword) + and node.arg == _CODE_KEYWORD + and isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + and DIAGNOSTIC_CODE_PATTERN.fullmatch(node.value.value) + ): + return None + return node.value.value + + def _emitted_codes(*, project: CorpusProject, options: DifferentialOptions) -> EmittedCodes: case_dir: Path = options.work_dir / project.name.replace("/", "__") source_dir: Path = case_dir / _SOURCE_DIRECTORY @@ -72,4 +144,7 @@ def _emitted_codes(*, project: CorpusProject, options: DifferentialOptions) -> E return EmittedCodes( errors=diagnostic_codes(outcome=run.outcomes[0], severity=ERROR_SEVERITY), warnings=diagnostic_codes(outcome=run.outcomes[0], severity=WARNING_SEVERITY), + first_error_message=first_diagnostic_message( + outcome=run.outcomes[0], severity=ERROR_SEVERITY + ), ) diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index 2cf0947ae8..c15ed2a7fc 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -1,46 +1,29 @@ """Minimal failing projects, each derived from the shared base project.""" +from scripts.compiler_differential._helpers.corpus.case_builder import ( + failure_case, + mart_body_files, + staging_files, +) from scripts.compiler_differential._helpers.corpus.discovery_failure_cases import ( discovery_failure_cases, ) +from scripts.compiler_differential._helpers.corpus.render_failure_cases import ( + render_failure_cases, +) from scripts.compiler_differential.constants import ( FAILURE_BASE_CONFIG, - FAILURE_BASE_FILES, FAILURE_BASE_MART, FAILURE_BASE_STAGING, FAILURE_CONFIG_PATH, FAILURE_MART_PATH, FAILURE_SOURCES_PATH, - FAILURE_STAGING_PATH, ) from scripts.compiler_differential.models import FailureCase _RULES_CONFIG: str = '\n[rules]\nselect = ["{codes}"]\n' -def _case( - *, - name: str, - expected_code: str, - files: dict[str, str], - expected_warning_code: str | None = None, -) -> FailureCase: - return FailureCase( - files={**FAILURE_BASE_FILES, **files}, - name=name, - expected_code=expected_code, - expected_warning_code=expected_warning_code, - ) - - -def _staging(sql: str) -> dict[str, str]: - return {FAILURE_STAGING_PATH: sql} - - -def _mart_body(body: str) -> dict[str, str]: - return {FAILURE_MART_PATH: 'MODEL (\n description "Order totals per customer",\n);\n\n' + body} - - def _rules(*codes: str) -> dict[str, str]: return { FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + _RULES_CONFIG.format(codes='", "'.join(codes)) @@ -50,82 +33,84 @@ def _rules(*codes: str) -> dict[str, str]: def all_failure_cases() -> tuple[FailureCase, ...]: """Return every failure case in a stable order.""" - return (*_compile_failure_cases(), *discovery_failure_cases()) + return (*_compile_failure_cases(), *discovery_failure_cases(), *render_failure_cases()) def _compile_failure_cases() -> tuple[FailureCase, ...]: return ( - _case( + failure_case( name="config-toml-syntax", expected_code="D001", files={ FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + "[defaults\nmaterialized = 'table'\n" }, ), - _case( + failure_case( name="config-unknown-default-key", expected_code="D001", files={ FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + '\n[defaults]\ndescription = "Shared"\n' }, ), - _case( + failure_case( name="config-unknown-adapter", expected_code="C601", files={FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG.replace('"duckdb"', '"warehouse9000"')}, ), - _case( + failure_case( name="model-header-syntax", expected_code="D002", - files=_staging( + files=staging_files( FAILURE_BASE_STAGING.replace( '"Staged orders",', '"Staged orders",\n materialized (,' ) ), ), - _case( + failure_case( name="model-sql-syntax", expected_code="P001", - files=_staging(FAILURE_BASE_STAGING.replace("SELECT order_id,", "SELECT order_id,, ")), + files=staging_files( + FAILURE_BASE_STAGING.replace("SELECT order_id,", "SELECT order_id,, ") + ), ), - _case( + failure_case( name="test-parse", expected_code="D003", files={"tests/unit/test_stg_orders.sql": "TEST(\n\nSELECT 1\n"}, ), - _case( + failure_case( name="audit-parse", expected_code="D004", files={ "models/marts/_sqlbuild/audits/singular/broken.sql": "AUDIT (name);\n\nSELECT 1\n" }, ), - _case( + failure_case( name="schema-unterminated-declaration", expected_code="D013", files={"models/staging/_sqlbuild/_schemas/orders.sql": "SCHEMA (\n name orders,\n"}, ), - _case( + failure_case( name="source-parse", expected_code="D006", files={FAILURE_SOURCES_PATH: "sources:\n - name: raw_orders\n columns: [\n"}, ), - _case( + failure_case( name="duplicate-model", expected_code="D007", files={"models/marts/stg_orders.sql": FAILURE_BASE_STAGING}, ), - _case( + failure_case( name="seed-without-header", expected_code="D008", files={"seeds/lookups.csv": ""}, ), - _case( + failure_case( name="scenario-parse", expected_code="D009", files={"tests/scenarios/orders.sql": "SCENARIO (\n\nSELECT 1\n"}, ), - _case( + failure_case( name="python-node-missing-description", expected_code="P010", files={ @@ -135,22 +120,22 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="enum-parse", expected_code="D013", files={"enums/order_status.sql": "ENUM (\n name order_status,\n members [,\n);\n"}, ), - _case( + failure_case( name="hook-without-header", expected_code="D014", files={"hooks/sql/record_refresh.sql": "SELECT 1\n"}, ), - _case( + failure_case( name="invalid-resource-name", expected_code="D016", files={"models/marts/order-totals.sql": FAILURE_BASE_MART}, ), - _case( + failure_case( name="built-in-audit-shadow", expected_code="S010", expected_warning_code="P003", @@ -160,7 +145,7 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="singular-audit-one-resource", expected_code="P004", files={ @@ -170,7 +155,7 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="macro-generated-reference", expected_code="P006", files={ @@ -179,40 +164,42 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ' """Return the staged orders relation."""\n' " return '__ref(\"stg_orders\")'\n" ), - **_mart_body( + **mart_body_files( "SELECT customer_id, SUM(amount) AS total_amount\n" "FROM @staged_orders()\nGROUP BY customer_id\n" ), }, ), - _case( + failure_case( name="missing-description", expected_code="P010", - files=_staging(FAILURE_BASE_STAGING.replace(' description "Staged orders",\n', "")), + files=staging_files( + FAILURE_BASE_STAGING.replace(' description "Staged orders",\n', "") + ), ), - _case( + failure_case( name="unknown-ref", expected_code="P001", - files=_mart_body( + files=mart_body_files( 'SELECT customer_id, SUM(amount) AS total_amount\nFROM __ref("stg_payments")\n' "GROUP BY customer_id\n" ), ), - _case( + failure_case( name="unknown-source", expected_code="P001", - files=_staging( + files=staging_files( FAILURE_BASE_STAGING.replace('__source("raw_orders")', '__source("raw_payments")') ), ), - _case( + failure_case( name="unknown-macro", expected_code="P001", - files=_staging( + files=staging_files( FAILURE_BASE_STAGING.replace("amount, status", '@cents("amount") AS amount, status') ), ), - _case( + failure_case( name="failing-macro", expected_code="P001", files={ @@ -221,41 +208,41 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ' """Refuse every input."""\n' ' raise ValueError(f"cannot convert {expression}")\n' ), - **_staging( + **staging_files( FAILURE_BASE_STAGING.replace( "amount, status", '@cents("amount") AS amount, status' ) ), }, ), - _case( + failure_case( name="unknown-enum-member", expected_code="P001", files={ "models/staging/_sqlbuild/_enums/order_status.sql": ( "ENUM (\n name order_status,\n members [PLACED, SHIPPED],\n);\n" ), - **_staging( + **staging_files( FAILURE_BASE_STAGING + 'WHERE status = @enum("order_status").RETURNED\n' ), }, ), - _case( + failure_case( name="unknown-constant", expected_code="P001", - files=_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + files=staging_files(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), ), - _case( + failure_case( name="invalid-decimal-constant", expected_code="D013", files={ "models/staging/_sqlbuild/_constants/limits.sql": ( 'CONSTANT (name minimum_amount, type decimal, value "ten");\n' ), - **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + **staging_files(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), }, ), - _case( + failure_case( name="duplicate-declaration", expected_code="P001", files={ @@ -263,28 +250,28 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: "CONSTANT (name minimum_amount, value 1);\n" "CONSTANT (name minimum_amount, value 2);\n" ), - **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + **staging_files(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), }, ), - _case( + failure_case( name="inaccessible-declaration", expected_code="P001", files={ "models/marts/_sqlbuild/_constants/limits.sql": ( "CONSTANT (name minimum_amount, value 1);\n" ), - **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + **staging_files(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), }, ), - _case( + failure_case( name="over-broad-global", expected_code="S024", files={ "constants/limits.sql": "CONSTANT (name minimum_amount, value 1);\n", - **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + **staging_files(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), }, ), - _case( + failure_case( name="unused-declaration", expected_code="S010", files={ @@ -293,7 +280,7 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="macro-import-cycle", expected_code="P001", files={ @@ -311,10 +298,10 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ), }, ), - _case( + failure_case( name="contract-missing-column", expected_code="K001", - files=_staging( + files=staging_files( 'MODEL (\n description "Staged orders",\n contract enforced,\n' " columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n" " amount (type DOUBLE),\n status (type VARCHAR),\n" @@ -323,51 +310,51 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' ), ), - _case( + failure_case( name="contract-type-mismatch", expected_code="K002", - files=_staging( + files=staging_files( 'MODEL (\n description "Staged orders",\n contract enforced,\n' " columns (\n order_id (type VARCHAR),\n customer_id (type INTEGER),\n" " amount (type DOUBLE),\n status (type VARCHAR),\n ),\n);\n\n" 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' ), ), - _case( + failure_case( name="set-operation-width", expected_code="B216", - files=_mart_body( + files=mart_body_files( 'SELECT customer_id, amount FROM __ref("stg_orders")\n' 'UNION ALL\nSELECT customer_id FROM __ref("stg_orders")\n' ), ), - _case( + failure_case( name="comparison-type", expected_code="B217", - files=_mart_body( + files=mart_body_files( "SELECT customer_id, SUM(amount) AS total_amount\n" "FROM __ref(\"stg_orders\")\nWHERE amount = DATE '2026-01-01'\n" "GROUP BY customer_id\n" ), ), - _case( + failure_case( name="unknown-column", expected_code="B002", - files=_mart_body( + files=mart_body_files( 'SELECT customer_id, SUM(discount) AS total_amount\nFROM __ref("stg_orders")\n' "GROUP BY customer_id\n" ), ), - _case( + failure_case( name="ungrouped-column", expected_code="B230", - files=_mart_body( + files=mart_body_files( "SELECT customer_id, status, SUM(amount) AS total_amount\n" 'FROM __ref("stg_orders")\n' "GROUP BY customer_id\n" ), ), - _case( + failure_case( name="test-expected-unknown-column", expected_code="B302", files={ @@ -380,19 +367,19 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="rule-unused-cte", expected_code="SQBRSQL005", files={ **_rules("SQBRSQL005"), - **_mart_body( + **mart_body_files( "WITH spare AS (SELECT 1 AS one)\n" "SELECT customer_id, SUM(amount) AS total_amount\n" 'FROM __ref("stg_orders")\nGROUP BY customer_id\n' ), }, ), - _case( + failure_case( name="rule-custom-finding", expected_code="XSQBRDIFF001", files={ @@ -410,11 +397,13 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ), }, ), - _case( + failure_case( name="multiple-errors-order", expected_code="P010", files={ - **_staging(FAILURE_BASE_STAGING.replace(' description "Staged orders",\n', "")), + **staging_files( + FAILURE_BASE_STAGING.replace(' description "Staged orders",\n', "") + ), FAILURE_MART_PATH: FAILURE_BASE_MART.replace( ' description "Order totals per customer",\n', "" ), @@ -424,29 +413,29 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ), }, ), - _case( + failure_case( name="contract-extra-column", expected_code="K005", - files=_staging( + files=staging_files( 'MODEL (\n description "Staged orders",\n contract enforced,\n' " columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n" " amount (type DOUBLE),\n ),\n);\n\n" 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' ), ), - _case( + failure_case( name="contract-unknown-type", expected_code="K002", - files=_staging( + files=staging_files( 'MODEL (\n description "Staged orders",\n' " columns (\n order_id (type MYSTERY_TYPE),\n ),\n);\n\n" 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' ), ), - _case( + failure_case( name="cursor-unknown-column", expected_code="B300", - files=_mart_body( + files=mart_body_files( "SELECT customer_id, SUM(amount) AS total_amount\n" 'FROM __ref("stg_orders")\nGROUP BY customer_id\n' ) @@ -461,7 +450,7 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="cursor-model-without-inputs", expected_code="P011", files={ @@ -473,22 +462,22 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="invalid-declaration-name", expected_code="D016", files={ "models/staging/_sqlbuild/_constants/limits.sql": ( "CONSTANT (name MinimumAmount, value 1);\n" ), - **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("MinimumAmount")\n'), + **staging_files(FAILURE_BASE_STAGING + 'WHERE amount > @const("MinimumAmount")\n'), }, ), - _case( + failure_case( name="python-node-syntax", expected_code="D011", files={"python/tasks/exports.py": "def broken(:\n pass\n"}, ), - _case( + failure_case( name="hook-unknown-argument", expected_code="P001", files={ @@ -501,20 +490,20 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ), }, ), - _case( + failure_case( name="model-private-enum-misuse", expected_code="P001", - files=_staging( + files=staging_files( 'MODEL (\n description "Staged orders",\n' " enums (\n _state [OPEN, CLOSED],\n ),\n);\n\n" "SELECT order_id, customer_id, amount, status\n" 'FROM __source("raw_orders")\nWHERE status = @enum("_state").PENDING\n' ), ), - _case( + failure_case( name="variable-not-defined", expected_code="P001", - files=_staging( + files=staging_files( FAILURE_BASE_STAGING.replace("status\n", "status, '@@region' AS region\n", 1) ), ), diff --git a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py new file mode 100644 index 0000000000..55db1432e5 --- /dev/null +++ b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py @@ -0,0 +1,354 @@ +"""Minimal projects that fail while rendering, scoping, or attaching compile inputs.""" + +from scripts.compiler_differential._helpers.corpus.case_builder import ( + config_files, + failure_case, + mart_body_files, + staging_files, +) +from scripts.compiler_differential.constants import ( + FAILURE_BASE_MART, + FAILURE_BASE_STAGING, + FAILURE_MART_PATH, +) +from scripts.compiler_differential.models import FailureCase + +_STAGING_HEADER: str = '"Staged orders",' +_MART_HEADER: str = '"Order totals per customer",' +_STAGING_COLUMNS: str = "amount, status" +_STAGING_MACROS: str = "models/staging/_sqlbuild/_macros/orders.py" +_MART_MACROS: str = "models/marts/_sqlbuild/_macros/orders.py" +_CENTS_MACRO: str = ( + 'def cents(expression: str) -> str:\n """Convert to cents."""\n' + ' return f"({expression} * 100)"\n' +) +_ORDERS_FOR: str = ( + "FUNCTION (\n" + ' description "Orders for one customer",\n' + " arguments (p_customer INTEGER),\n" + " returns table (\n order_id INTEGER\n ),\n);\n\n" + 'SELECT order_id\nFROM __ref("stg_orders")\nWHERE customer_id = p_customer\n' +) +_FLOOR_AUDIT: str = 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE amount < @minimum\n' +_RECORD_HOOK: str = "HOOK (\n description \"Record a refresh\"\n);\n\nSELECT @'label' AS label\n" +_TOTALS_READER: str = ( + 'AUDIT ();\n\nSELECT *\nFROM __ref("@model") o\n' + 'WHERE o.customer_id NOT IN (SELECT customer_id FROM __ref("customer_totals"))\n' +) +_HOOK_READING_TOTALS: str = ( + "from sqlbuild.hooks import hook\nfrom sqlbuild.refs import model\n\n\n" + '@hook(reads=[model("customer_totals")])\n' + 'def note_refresh(ctx):\n """Note a refresh."""\n return None\n' +) +_UPSTREAM_DBT_PROJECT: str = "name: upstream\nversion: '1.0.0'\nprofile: upstream\n" + + +def _staging_header(extra: str) -> dict[str, str]: + return staging_files( + FAILURE_BASE_STAGING.replace(_STAGING_HEADER, f"{_STAGING_HEADER}\n{extra}") + ) + + +def _staging_columns(*, columns: str, files: dict[str, str] | None = None) -> dict[str, str]: + return { + **(files or {}), + **staging_files(FAILURE_BASE_STAGING.replace(_STAGING_COLUMNS, columns, 1)), + } + + +def render_failure_cases() -> tuple[FailureCase, ...]: + """Return the render-time failure cases in a stable order.""" + + return (*_macro_cases(), *_interpolation_cases(), *_reference_cases(), *_attachment_cases()) + + +def _macro_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="macro-returns-non-string", + expected_code="P001", + expected_message="must return a SQL string", + files=_staging_columns( + columns="amount, status, @answer() AS answer", + files={ + _STAGING_MACROS: ( + 'def answer() -> int:\n """Return a number."""\n return 42\n' + ) + }, + ), + ), + failure_case( + name="macro-output-calls-macro", + expected_code="P001", + expected_message="returned SQLBuild call(s) @cents()", + files=_staging_columns( + columns='amount, status, @wrapped("amount") AS wrapped', + files={ + _STAGING_MACROS: _CENTS_MACRO + + '\n\ndef wrapped(expression: str) -> str:\n """Wrap a call."""\n' + + " return f'@cents(\"{expression}\")'\n" + }, + ), + ), + failure_case( + name="macro-non-literal-argument", + expected_code="P001", + expected_message="must use only Python literals", + files=_staging_columns( + columns="amount, status, @cents(amount) AS cents", + files={_STAGING_MACROS: _CENTS_MACRO}, + ), + ), + failure_case( + name="macro-in-sibling-scope", + expected_code="P001", + expected_message="is inaccessible", + files=_staging_columns( + columns='amount, status, @cents("amount") AS cents', + files={_MART_MACROS: _CENTS_MACRO}, + ), + ), + failure_case( + name="local-macro-needed-below-owner", + expected_code="S007", + files={ + _STAGING_MACROS: _CENTS_MACRO, + "models/staging/_sqlbuild/macros/wrapping.py": ( + "from models.staging._sqlbuild._macros.orders import cents\n\n\n" + 'def wrapped(expression: str) -> str:\n """Wrap cents."""\n' + " return cents(expression)\n" + ), + "models/staging/regional/stg_regional_orders.sql": ( + 'MODEL (\n description "Regional orders",\n);\n\n' + 'SELECT order_id, @wrapped("amount") AS amount_cents\n' + 'FROM __ref("stg_orders")\n' + ), + }, + ), + ) + + +def _interpolation_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="unset-environment-variable", + expected_code="P001", + expected_message="unknown environment variable '@@ENV:SQB_DIFFERENTIAL_UNSET'", + files=_staging_columns( + columns="amount, status, '@@ENV:SQB_DIFFERENTIAL_UNSET' AS channel" + ), + ), + failure_case( + name="context-variable-in-model-sql", + expected_code="P001", + expected_message="does not allow @@CTX templates", + files=_staging_columns( + columns="amount, status, '@@CTX:destination.qualified' AS destination" + ), + ), + failure_case( + name="runtime-placeholder-without-default", + expected_code="P001", + expected_message="@@@placeholders without default values", + files={ + "materializations/copy_table.py": ( + "from sqlbuild.executor.custom.models import MaterializationContext, " + "MaterializationResult\n\n\n" + "def materialize(ctx: MaterializationContext) -> MaterializationResult:\n" + " return MaterializationResult(relation=ctx.destination, audit_results=())\n" + ), + **staging_files( + FAILURE_BASE_STAGING.replace( + _STAGING_HEADER, f"{_STAGING_HEADER}\n materialized copy_table," + ).replace( + _STAGING_COLUMNS, + "amount, status, CAST(@@@window_start AS DATE) AS window_start", + ) + ), + }, + ), + failure_case( + name="cursor-intrinsic-outside-incremental", + expected_code="P001", + expected_message="uses cursor intrinsics but is not a built-in incremental model", + files=_staging_columns(columns="amount, status, __cursor_start() AS window_start"), + ), + failure_case( + name="named-hook-missing-argument", + expected_code="P001", + expected_message="is missing argument 'label'", + files={ + "hooks/sql/record_refresh.sql": _RECORD_HOOK, + FAILURE_MART_PATH: FAILURE_BASE_MART.replace( + _MART_HEADER, f'{_MART_HEADER}\n post_hooks [sql("record_refresh")],' + ), + }, + ), + failure_case( + name="generic-audit-missing-argument", + expected_code="P001", + expected_message="is missing argument 'minimum' for generic audit 'floor'", + files={ + "models/staging/_sqlbuild/_audits/generic/floor.sql": _FLOOR_AUDIT, + **_staging_header(" audits [floor],"), + }, + ), + ) + + +def _reference_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="table-function-arity", + expected_code="P001", + expected_message="expects 1 argument but received 2", + files={ + "functions/sql/table_fn__orders_for.sql": _ORDERS_FOR, + **mart_body_files( + "SELECT order_id AS customer_id, 1 AS total_amount\n" + 'FROM __table_fn("table_fn__orders_for")(1, 2)\n' + ), + }, + ), + failure_case( + name="unknown-table-function", + expected_code="P001", + expected_message="references unknown table function 'table_fn__missing'", + files=mart_body_files( + "SELECT order_id AS customer_id, 1 AS total_amount\n" + 'FROM __table_fn("table_fn__missing")(1)\n' + ), + ), + failure_case( + name="unknown-scalar-function", + expected_code="P001", + expected_message="references unknown SQL function 'is_large'", + files=_staging_columns(columns='amount, status, __udf("is_large")(amount) AS is_large'), + ), + failure_case( + name="test-expects-unknown-model", + expected_code="P001", + expected_message="expects unknown model 'stg_payments'", + files={ + "tests/unit/test_stg_payments.sql": ( + "TEST();\n\nWITH\n__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount," + " 'placed' AS status\n),\n" + "__expected__stg_payments AS (\n SELECT 1 AS order_id\n)\nSELECT 1\n" + ) + }, + ), + failure_case( + name="dbt-reference-without-manifest", + expected_code="C214", + files={ + **config_files('\n[dbt]\nproject_dir = "dbt"\n'), + "dbt/dbt_project.yml": _UPSTREAM_DBT_PROJECT, + **mart_body_files( + "SELECT customer_id, COUNT(*) AS order_count\n" + 'FROM __dbt_ref("upstream", "upstream_orders")\nGROUP BY customer_id\n' + ), + }, + ), + failure_case( + name="hard-coded-relation-in-python", + expected_code="P008", + files={ + "python/tasks/exports.py": ( + "from sqlbuild.tasks import task\n\n\n" + '@task\ndef export_orders(ctx):\n """Export orders."""\n' + ' return ctx.query("SELECT * FROM stg_orders")\n' + ) + }, + ), + ) + + +def _attachment_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="attached-audit-reads-dependant", + expected_code="P005", + files={ + "models/staging/_sqlbuild/_audits/generic/matches_totals.sql": _TOTALS_READER, + **_staging_header(" audits [matches_totals],"), + }, + ), + failure_case( + name="hook-reads-dependant", + expected_code="P007", + files={ + "models/staging/_sqlbuild/_hooks/python/notes.py": _HOOK_READING_TOTALS, + **_staging_header(' post_hooks [python("note_refresh")],'), + }, + ), + failure_case( + name="unneeded-sql-analysis-opt-out", + expected_code="P009", + files={ + **config_files("\n[settings]\nrequire_sql_analysis = true\n"), + **_staging_header(" sql_analysis false,"), + }, + ), + failure_case( + name="snapshot-schema-change-with-contract", + expected_code="K012", + files=staging_files( + 'MODEL (\n description "Staged orders",\n materialized snapshot,\n' + " unique_key [order_id],\n snapshot_strategy check,\n" + " check_columns [status],\n snapshot_schema_change append_new_columns,\n" + " contract enforced,\n columns (\n order_id (type INTEGER),\n" + " customer_id (type INTEGER),\n amount (type DOUBLE),\n" + " status (type VARCHAR),\n ),\n);\n\n" + 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' + ), + ), + failure_case( + name="schema-from-sibling-folder", + expected_code="S006", + files={ + "models/marts/_sqlbuild/_schemas/order_shape.sql": ( + 'SCHEMA (\n name order_shape,\n description "Order shape",\n' + " columns (\n order_id (type INTEGER),\n ),\n);\n" + ), + **_staging_header(" model_schema order_shape,"), + }, + ), + failure_case( + name="inherited-declaration-used-in-one-folder", + expected_code="S008", + files={ + "models/staging/_sqlbuild/constants/limits.sql": ( + "CONSTANT (name minimum_amount, value 1);\n" + ), + **staging_files(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + }, + ), + failure_case( + name="singular-audit-across-trees", + expected_code="S009", + files={ + "models/marts/_sqlbuild/audits/singular/totals_have_orders.sql": ( + 'AUDIT (name "totals_have_orders");\n\n' + 'SELECT t.customer_id\nFROM __ref("customer_totals") t\n' + 'LEFT JOIN __source("raw_orders") o ON t.customer_id = o.customer_id\n' + "WHERE o.customer_id IS NULL\n" + ) + }, + ), + failure_case( + name="orphan-audit-factory", + expected_code="S024", + expected_warning_code="C216", + files={ + "python/audits/quality.py": ( + "from sqlbuild.audits import AuditCase, audit_factory\n\n\n" + "@audit_factory\ndef order_quality():\n" + ' return [AuditCase(name="id_present", definition="not_null", ' + 'arguments={"column": "order_id"})]\n' + ), + "constants/limits.sql": "CONSTANT (name minimum_amount, value 1);\n", + **staging_files(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + }, + ), + ) diff --git a/scripts/compiler_differential/_helpers/coverage/capture_values.py b/scripts/compiler_differential/_helpers/coverage/capture_values.py new file mode 100644 index 0000000000..40e2d90287 --- /dev/null +++ b/scripts/compiler_differential/_helpers/coverage/capture_values.py @@ -0,0 +1,105 @@ +"""Read values out of canonical stage captures without crediting encoded empty collections.""" + +from __future__ import annotations + +import json +import re +from collections.abc import Iterator + +from scripts.compiler_differential._helpers.comparing.compare import as_json_object +from scripts.compiler_differential.constants import ( + CALLABLE_MARKER, + COLLECTION_CAPTURE_MARKERS, + TYPE_MARKER, + UNCANONICAL_CAPTURE_MARKERS, +) + +_ENUM_VALUE: re.Pattern[str] = re.compile(r"\.([A-Z_]+)$") + + +def decoded(value: object) -> object: + """Unwrap the capture's set and non-string-key mapping encodings into plain lists.""" + + encoded: dict[str, object] | None = as_json_object(value) + if encoded is None or len(encoded) != 1: + return value + marker: str = next(iter(encoded)) + return encoded[marker] if marker in COLLECTION_CAPTURE_MARKERS else value + + +def has_content(value: object) -> bool: + """Whether a captured value holds anything; encoded empty sets and mappings do not.""" + + return bool(decoded(value)) + + +def records(value: object) -> list[dict[str, object]]: + """Return the objects of a captured list or set.""" + + items: object = decoded(value) + if not isinstance(items, list): + return [] + return [record for item in items if (record := as_json_object(item)) is not None] + + +def mapping_records(value: object) -> list[dict[str, object]]: + """Return the object values of a captured string-keyed mapping, such as macros by name.""" + + mapping: dict[str, object] = as_json_object(value) or {} + return [ + record + for key, item in mapping.items() + if key != TYPE_MARKER and (record := as_json_object(item)) is not None + ] + + +def mapping_size(value: object) -> int: + """Return how many entries a captured mapping or list holds.""" + + items: object = decoded(value) + return len(items) if isinstance(items, (dict, list)) else 0 + + +def enum_name(value: object) -> str | None: + """Return the lower-case member name of a captured enum value.""" + + encoded: dict[str, object] | None = as_json_object(value) + match: re.Match[str] | None = ( + None if encoded is None else _ENUM_VALUE.search(str(encoded.get("__enum__", ""))) + ) + return None if match is None else match.group(1).lower() + + +def named_values(*, value: object, names: frozenset[str]) -> Iterator[tuple[str, object]]: + """Yield every `(key, value)` pair below `value` whose key is one of `names`.""" + + record: dict[str, object] | None = as_json_object(value) + children: list[object] = list(value) if isinstance(value, list) else [] + if record is not None: + yield from ((key, item) for key, item in record.items() if key in names) + children.extend(record.values()) + for child in children: + yield from named_values(value=child, names=names) + + +def capture_problems( + *, capture_text: str, fields: tuple[str, ...], callable_fields: frozenset[str] +) -> tuple[str, ...]: + """Return why a capture is incomplete or not canonical; empty when it is sound.""" + + capture: dict[str, object] = as_json_object(json.loads(capture_text)) or {} + problems: list[str] = [] + captured: tuple[str, ...] = tuple(key for key in capture if key != TYPE_MARKER) + if captured != fields: + problems.append(f"captured fields {captured} differ from {fields}") + problems.extend( + f"capture contains {marker}" + for marker in UNCANONICAL_CAPTURE_MARKERS + if f'"{marker}"' in capture_text + ) + problems.extend( + f"{field} is not a qualified callable name: {value}" + for field, value in named_values(value=capture, names=callable_fields) + if set(as_json_object(value) or {}) != {CALLABLE_MARKER} + ) + return tuple(problems) diff --git a/scripts/compiler_differential/_helpers/coverage/discovery.py b/scripts/compiler_differential/_helpers/coverage/discovery.py index 7492e86d42..37e800e26f 100644 --- a/scripts/compiler_differential/_helpers/coverage/discovery.py +++ b/scripts/compiler_differential/_helpers/coverage/discovery.py @@ -9,10 +9,16 @@ from pathlib import Path from scripts.compiler_differential._helpers.comparing.compare import as_json_object +from scripts.compiler_differential._helpers.coverage.capture_values import ( + capture_problems, + enum_name, + has_content, + mapping_size, + named_values, + records, +) from scripts.compiler_differential.constants import ( CALLABLE_CAPTURE_FIELDS, - CALLABLE_MARKER, - COLLECTION_CAPTURE_MARKERS, CRLF_BYTES, DBT_REF_CALL, DISCOVERY_DECLARATION_FILE_COLLECTIONS, @@ -27,13 +33,10 @@ SINGULAR_AUDIT_KIND, STRATEGY_HEADER_KEYS, TAB, - TYPE_MARKER, - UNCANONICAL_CAPTURE_MARKERS, ) from sqlbuild.adapter.discovery.constants import BUILTIN_ADAPTER_IMPORTS from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -_ENUM_VALUE: re.Pattern[str] = re.compile(r"\.([A-Z_]+)$") _CROSS_FILE_MACRO_IMPORT: re.Pattern[str] = re.compile(r"^(?:from|import) macros\.", re.MULTILINE) _NON_ASCII_COMMENT: re.Pattern[str] = re.compile(r"--[^\n]*[^\x00-\x7f]") _NON_ASCII_STRING: re.Pattern[str] = re.compile(r"'[^'\n]*[^\x00-\x7f][^'\n]*'") @@ -75,7 +78,7 @@ def discovered_relative_paths(capture_text: str) -> frozenset[str]: capture: object = json.loads(capture_text) return frozenset( str(path) - for _, value in _named_values(value=capture, names=RELATIVE_PATH_FIELDS) + for _, value in named_values(value=capture, names=RELATIVE_PATH_FIELDS) if (path := (as_json_object(value) or {}).get(PATH_MARKER)) is not None ) @@ -103,7 +106,7 @@ def discovered_input_kinds(capture_text: str) -> frozenset[str]: capture: dict[str, object] = as_json_object(json.loads(capture_text)) or {} kinds: set[str] = { - kind for kind in discovery_collection_kinds() if _has_content(capture.get(kind)) + kind for kind in discovery_collection_kinds() if has_content(capture.get(kind)) } kinds.update(_model_kinds(capture)) kinds.update(_source_kinds(capture)) @@ -117,79 +120,19 @@ def discovered_input_kinds(capture_text: str) -> frozenset[str]: def discovery_capture_problems(capture_text: str) -> tuple[str, ...]: """Return why a discovery capture is incomplete or not canonical; empty when it is sound.""" - capture: dict[str, object] = as_json_object(json.loads(capture_text)) or {} - fields: tuple[str, ...] = tuple( - field.name for field in dataclasses.fields(DiscoveredProjectInputs) - ) - problems: list[str] = [] - captured: tuple[str, ...] = tuple(key for key in capture if key != TYPE_MARKER) - if captured != fields: - problems.append(f"captured fields {captured} differ from {fields}") - problems.extend( - f"capture contains {marker}" - for marker in UNCANONICAL_CAPTURE_MARKERS - if f'"{marker}"' in capture_text - ) - problems.extend( - f"{field} is not a qualified callable name: {value}" - for field, value in _named_values(value=capture, names=CALLABLE_CAPTURE_FIELDS) - if set(as_json_object(value) or {}) != {CALLABLE_MARKER} - ) - return tuple(problems) - - -def _named_values(*, value: object, names: frozenset[str]) -> Iterator[tuple[str, object]]: - record: dict[str, object] | None = as_json_object(value) - children: list[object] = list(value) if isinstance(value, list) else [] - if record is not None: - yield from ((key, item) for key, item in record.items() if key in names) - children.extend(record.values()) - for child in children: - yield from _named_values(value=child, names=names) - - -def _decoded(value: object) -> object: - """Unwrap the capture's set and non-string-key mapping encodings into plain lists.""" - - encoded: dict[str, object] | None = as_json_object(value) - if encoded is None or len(encoded) != 1: - return value - marker: str = next(iter(encoded)) - return encoded[marker] if marker in COLLECTION_CAPTURE_MARKERS else value - - -def _has_content(value: object) -> bool: - """Whether a captured value holds anything; encoded empty sets and mappings do not.""" - - return bool(_decoded(value)) - - -def _records(value: object) -> list[dict[str, object]]: - items: object = _decoded(value) - if not isinstance(items, list): - return [] - return [record for item in items if (record := as_json_object(item)) is not None] - - -def _mapping_size(value: object) -> int: - items: object = _decoded(value) - return len(items) if isinstance(items, (dict, list)) else 0 - - -def _enum_name(value: object) -> str | None: - encoded: dict[str, object] | None = as_json_object(value) - match: re.Match[str] | None = ( - None if encoded is None else _ENUM_VALUE.search(str(encoded.get("__enum__", ""))) + return capture_problems( + capture_text=capture_text, + fields=tuple(field.name for field in dataclasses.fields(DiscoveredProjectInputs)), + callable_fields=CALLABLE_CAPTURE_FIELDS, ) - return None if match is None else match.group(1).lower() def _model_kinds(capture: dict[str, object]) -> set[str]: kinds: set[str] = set() materializations: set[str] = { - str(record.get("name")) for record in _records(capture.get("materialization_files")) + str(record.get("name")) for record in records(capture.get("materialization_files")) } - for model in _records(capture.get("model_files")): + for model in records(capture.get("model_files")): header: dict[str, object] = as_json_object(model.get("header_values")) or {} materialized: object = header.get("materialized") if header.get("incremental_mode") == MICROBATCH_MODE: @@ -198,7 +141,7 @@ def _model_kinds(capture: dict[str, object]) -> set[str]: kinds.add(f"{materialized}_{header.get(STRATEGY_HEADER_KEYS[str(materialized)])}") elif materialized in materializations: kinds.add("custom_materialized_model") - if _records(model.get("constant_declarations")) or _records(model.get("enum_declarations")): + if records(model.get("constant_declarations")) or records(model.get("enum_declarations")): kinds.add("model_local_declaration") if DBT_REF_CALL in str(model.get("contents", "")): kinds.add("dbt_ref_model") @@ -207,8 +150,8 @@ def _model_kinds(capture: dict[str, object]) -> set[str]: def _source_kinds(capture: dict[str, object]) -> set[str]: kinds: set[str] = set() - for source_file in _records(capture.get("source_files")): - for entry in _records(source_file.get("source_entries")): + for source_file in records(capture.get("source_files")): + for entry in records(source_file.get("source_entries")): kinds.add("managed_source" if entry.get("managed") is True else "unmanaged_source") return kinds @@ -216,29 +159,29 @@ def _source_kinds(capture: dict[str, object]) -> set[str]: def _declaration_kinds(capture: dict[str, object]) -> set[str]: kinds: set[str] = set() for collection in DISCOVERY_DECLARATION_FILE_COLLECTIONS: - for record in _records(capture.get(collection)): - scope: str | None = _enum_name(record.get("scope_kind")) + for record in records(capture.get(collection)): + scope: str | None = enum_name(record.get("scope_kind")) if scope is not None: kinds.add(f"scope_{scope}") - for constant_file in _records(capture.get("constant_files")): - for declaration in _records(constant_file.get("declarations")): + for constant_file in records(capture.get("constant_files")): + for declaration in records(constant_file.get("declarations")): value: dict[str, object] = as_json_object(declaration.get("value")) or {} logical: dict[str, object] = as_json_object(value.get("logical_type")) or {} - kind: str | None = _enum_name(logical.get("kind")) + kind: str | None = enum_name(logical.get("kind")) if kind is not None: kinds.add(f"constant_{kind}") if not json.dumps(value.get("value"), ensure_ascii=False).isascii(): kinds.add("non_ascii_constant") - for enum_file in _records(capture.get("enum_files")): - for declaration in _records(enum_file.get("declarations")): - members: list[dict[str, object]] = _records(declaration.get("members")) + for enum_file in records(capture.get("enum_files")): + for declaration in records(enum_file.get("declarations")): + members: list[dict[str, object]] = records(declaration.get("members")) kinds.update( "enum_integer" if isinstance(member.get("value"), int) else "enum_string" for member in members ) if any( _CROSS_FILE_MACRO_IMPORT.search(str(record.get("contents", ""))) - for record in _records(capture.get("macro_files")) + for record in records(capture.get("macro_files")) ): kinds.add("cross_file_macro_import") return kinds @@ -246,15 +189,15 @@ def _declaration_kinds(capture: dict[str, object]) -> set[str]: def _test_and_audit_kinds(capture: dict[str, object]) -> set[str]: kinds: set[str] = set() - for test_file in _records(capture.get("test_files")): - for block in _records(test_file.get("blocks")): - mode: str | None = _enum_name(block.get("mode")) + for test_file in records(capture.get("test_files")): + for block in records(test_file.get("blocks")): + mode: str | None = enum_name(block.get("mode")) if mode is not None: kinds.add(f"test_mode_{mode}") - if _records(block.get("cases")): + if records(block.get("cases")): kinds.add("parameterized_test") - for audit_file in _records(capture.get("audit_files")): - declaration_kind: str | None = _enum_name(audit_file.get("declaration_kind")) + for audit_file in records(capture.get("audit_files")): + declaration_kind: str | None = enum_name(audit_file.get("declaration_kind")) kinds.add( SINGULAR_AUDIT_KIND if declaration_kind == SINGULAR_AUDIT_KIND else "generic_audit" ) @@ -265,15 +208,15 @@ def _config_kinds(capture: dict[str, object]) -> set[str]: kinds: set[str] = set() project: dict[str, object] = as_json_object(capture.get("project_config")) or {} local: dict[str, object] = as_json_object(capture.get("local_config")) or {} - if _has_content(project.get("path_defaults")): + if has_content(project.get("path_defaults")): kinds.add("path_defaults") if ( - _has_content(local.get("vars")) - or _has_content(local.get("setting_overrides")) + has_content(local.get("vars")) + or has_content(local.get("setting_overrides")) or local.get("target") is not None ): kinds.add("local_config") - if local.get("target") is not None and _mapping_size(project.get("targets")) > 1: + if local.get("target") is not None and mapping_size(project.get("targets")) > 1: kinds.add("target_override") dbt: dict[str, object] = as_json_object(project.get("dbt")) or {} if dbt.get("target_path") is not None: @@ -286,7 +229,7 @@ def _config_kinds(capture: dict[str, object]) -> set[str]: def _text_kinds(capture: dict[str, object]) -> set[str]: kinds: set[str] = set() for collection in DISCOVERY_TEXT_FILE_COLLECTIONS: - for record in _records(capture.get(collection)): + for record in records(capture.get(collection)): contents: str = str(record.get("contents", "")) if TAB in contents: kinds.add("tab") diff --git a/scripts/compiler_differential/_helpers/coverage/macro_sources.py b/scripts/compiler_differential/_helpers/coverage/macro_sources.py new file mode 100644 index 0000000000..0798dadb07 --- /dev/null +++ b/scripts/compiler_differential/_helpers/coverage/macro_sources.py @@ -0,0 +1,41 @@ +"""Read one macro function out of the captured source of the file that defines it.""" + +from __future__ import annotations + +import ast + +from scripts.compiler_differential.constants import TYPED_REFERENCE_ANNOTATION + + +def macro_function_source(*, file_source: str, name: str) -> str: + """Return the source of the top-level function `name`, or "" when the file has none.""" + + function: ast.FunctionDef | None = _function(file_source=file_source, name=name) + if function is None: + return "" + return ast.get_source_segment(file_source, function) or "" + + +def takes_typed_reference(*, file_source: str, name: str) -> bool: + """Whether the function `name` annotates a parameter with the typed reference type.""" + + function: ast.FunctionDef | None = _function(file_source=file_source, name=name) + if function is None: + return False + arguments: ast.arguments = function.args + return any( + argument.annotation is not None + and TYPED_REFERENCE_ANNOTATION in ast.unparse(argument.annotation) + for argument in (*arguments.posonlyargs, *arguments.args, *arguments.kwonlyargs) + ) + + +def _function(*, file_source: str, name: str) -> ast.FunctionDef | None: + try: + module: ast.Module = ast.parse(file_source) + except SyntaxError: + return None + return next( + (node for node in module.body if isinstance(node, ast.FunctionDef) and node.name == name), + None, + ) diff --git a/scripts/compiler_differential/_helpers/coverage/render.py b/scripts/compiler_differential/_helpers/coverage/render.py new file mode 100644 index 0000000000..ffd99a14da --- /dev/null +++ b/scripts/compiler_differential/_helpers/coverage/render.py @@ -0,0 +1,420 @@ +"""Classify `CompileProjectInputs` captures into render kinds; only rendered evidence counts.""" + +from __future__ import annotations + +import dataclasses +import json +import re +from collections.abc import Iterator +from pathlib import PurePosixPath + +from scripts.compiler_differential._helpers.comparing.compare import as_json_object +from scripts.compiler_differential._helpers.coverage.capture_values import ( + capture_problems, + enum_name, + has_content, + mapping_records, + records, +) +from scripts.compiler_differential._helpers.coverage.macro_sources import ( + macro_function_source, + takes_typed_reference, +) +from scripts.compiler_differential.constants import ( + RENDER_CALLABLE_CAPTURE_FIELDS, + RENDER_DETAIL_KINDS, + RENDER_HOOK_KEYS, + RENDER_MACRO_SOURCE_READS, + RENDER_NON_COLLECTION_FIELDS, + RENDER_STAGE_CAPTURE_SUFFIX, +) +from sqlbuild.compiler.auditing.constants import BUILT_IN_AUDIT_NAMES +from sqlbuild.compiler.compile.models import CompileProjectInputs + +_MACRO_CALL: re.Pattern[str] = re.compile(r"@([A-Za-z_]\w*)\(") +_REFERENCE_CALL: re.Pattern[str] = re.compile(r"__(ref|source|seed)\(\s*\"([^\"]+)\"\s*\)") +_PROJECT_VARIABLE: re.Pattern[str] = re.compile(r"(? tuple[str, ...]: + """Return every `CompileProjectInputs` collection field plus the finer required kinds.""" + + return (*render_collection_kinds(), *RENDER_DETAIL_KINDS) + + +def render_collection_kinds() -> tuple[str, ...]: + """Return the `CompileProjectInputs` fields that hold rendered inputs.""" + + return tuple( + field.name + for field in dataclasses.fields(CompileProjectInputs) + if field.name not in RENDER_NON_COLLECTION_FIELDS + ) + + +def project_render_kinds(captures: dict[str, dict[str, str]]) -> frozenset[str]: + """Return the render kinds every `CompileProjectInputs` capture of one project proves.""" + + kinds: set[str] = set() + for files in captures.values(): + for name, text in files.items(): + if name.endswith(RENDER_STAGE_CAPTURE_SUFFIX): + kinds.update(rendered_input_kinds(text)) + return frozenset(kinds) + + +def render_capture_problems(capture_text: str) -> tuple[str, ...]: + """Return why a render capture is incomplete or not canonical; empty when it is sound.""" + + return capture_problems( + capture_text=capture_text, + fields=tuple(field.name for field in dataclasses.fields(CompileProjectInputs)), + callable_fields=RENDER_CALLABLE_CAPTURE_FIELDS, + ) + + +def rendered_input_kinds(capture_text: str) -> frozenset[str]: + """Return the render input kinds one canonical `CompileProjectInputs` capture proves.""" + + capture: dict[str, object] = as_json_object(json.loads(capture_text)) or {} + kinds: set[str] = {kind for kind in render_collection_kinds() if has_content(capture.get(kind))} + usages: list[tuple[str, dict[str, object]]] = list(_usages(capture)) + kinds.update(_macro_kinds(capture=capture, usages=usages)) + kinds.update(_declaration_kinds(capture=capture, usages=usages)) + kinds.update(_interpolation_kinds(capture)) + kinds.update(_hook_kinds(capture)) + kinds.update(_reference_kinds(capture)) + kinds.update(_attachment_kinds(capture)) + return frozenset(kinds) + + +def _usages(capture: dict[str, object]) -> Iterator[tuple[str, dict[str, object]]]: + """Yield `(consumer collection label, usage)` for every recorded declaration usage.""" + + for model in records(capture.get("model_inputs")): + for field in ("macro_usages", "declaration_usages"): + yield from ((_MODEL_RESOURCE, usage) for usage in records(model.get(field))) + for collection, label in _CONSUMER_COLLECTIONS.items(): + for item in records(capture.get(collection)): + yield from ((label, usage) for usage in records(item.get("declaration_usages"))) + + +def _declaration_kind(usage: dict[str, object]) -> str | None: + return enum_name((as_json_object(usage.get("declaration")) or {}).get("kind")) + + +def _macro_kinds( + *, capture: dict[str, object], usages: list[tuple[str, dict[str, object]]] +) -> set[str]: + kinds: set[str] = set() + used_macros: set[str] = set() + for consumer, usage in usages: + if _declaration_kind(usage) != _MACRO_DECLARATION: + continue + used_macros.add(str((as_json_object(usage.get("declaration")) or {}).get("name"))) + kinds.add(f"macro_in_{consumer}") + typed_macros: set[str] = set() + for macro in mapping_records(capture.get("loaded_macros")): + name: str = str(macro.get("name")) + file_source: str = str(macro.get("raw_source", "")) + if takes_typed_reference(file_source=file_source, name=name): + typed_macros.add(name) + if name not in used_macros: + continue + if records(macro.get("dependencies")): + kinds.add("cross_file_macro_import") + kinds.update(_macro_source_reads(macro_function_source(file_source=file_source, name=name))) + for model in records(capture.get("model_inputs")): + kinds.update(_model_macro_kinds(model=model, typed_macros=typed_macros)) + for test in records(capture.get("test_inputs")): + if enum_name(test.get("mode")) == _MACRO_DECLARATION and any( + _declaration_kind(usage) == _MACRO_DECLARATION + for usage in records(test.get("declaration_usages")) + ): + kinds.add("tested_macro") + return kinds + + +def _macro_source_reads(source: str) -> set[str]: + kinds: set[str] = set() + for kind, markers in RENDER_MACRO_SOURCE_READS.items(): + if any(marker in source for marker in markers): + kinds.add(kind) + return kinds + + +def _model_macro_kinds(*, model: dict[str, object], typed_macros: set[str]) -> set[str]: + kinds: set[str] = set() + authored: str = str(model.get("macro_source_sql", "")) + rendered: str = str(model.get("query_sql", "")) + calls: list[tuple[str, str]] = list(_macro_calls(authored)) + if not calls or _MACRO_CALL.search(rendered): + return kinds + if any(_MACRO_CALL.search(arguments) for _, arguments in calls): + kinds.add("nested_macro_call") + referenced: set[tuple[str, str]] = { + (str(enum_name(reference.get("ref_kind"))), str(reference.get("ref_name"))) + for reference in records(model.get("references")) + } + passed: set[tuple[str, str]] = set() + for name, arguments in calls: + if name in typed_macros: + passed.update(_REFERENCE_CALL.findall(arguments)) + if passed and passed <= referenced: + kinds.add("typed_reference_argument") + written: set[tuple[str, str]] = set(_REFERENCE_CALL.findall(authored)) + if {item for item in referenced if item[0] in _RELATION_REFERENCE_KINDS} - written: + kinds.add("macro_generated_reference") + return kinds + + +def _macro_calls(sql: str) -> Iterator[tuple[str, str]]: + """Yield the name and argument text of every macro call, matching nested parentheses.""" + + for match in _MACRO_CALL.finditer(sql): + depth: int = 1 + index: int = match.end() + while index < len(sql) and depth: + depth += {"(": 1, ")": -1}.get(sql[index], 0) + index += 1 + yield match.group(1), sql[match.end() : index - 1] + + +def _declaration_kinds( + *, capture: dict[str, object], usages: list[tuple[str, dict[str, object]]] +) -> set[str]: + kinds: set[str] = set() + scopes: dict[str, str] = { + json.dumps(record.get("identity"), sort_keys=True): str(enum_name(record.get("scope"))) + for record in records( + (as_json_object(capture.get("scope_index")) or {}).get("declarations") + ) + } + list_constants: set[str] = { + str(declaration.get("name")) + for declaration in mapping_records(capture.get("public_constants")) + if enum_name( + ( + as_json_object((as_json_object(declaration.get("value")) or {}).get("logical_type")) + or {} + ).get("kind") + ) + == _LIST_VALUE + } + for consumer, usage in usages: + declaration: dict[str, object] = as_json_object(usage.get("declaration")) or {} + kind: str | None = _declaration_kind(usage) + if declaration.get("owner") is not None: + kinds.add("scope_private_use") + scope: str | None = scopes.get(json.dumps(declaration, sort_keys=True)) + if scope is not None: + kinds.add(f"scope_{scope}_use") + if kind == _ENUM_DECLARATION and usage.get("enum_member") is not None: + kinds.add("enum_member") + if kind == _CONSTANT_DECLARATION: + kinds.add( + "list_constant" if declaration.get("name") in list_constants else "scalar_constant" + ) + if consumer != _MODEL_RESOURCE and usage.get("through") is not None: + kinds.add("expected_model_grant") + for model in records(capture.get("model_inputs")): + if has_content(model.get("enum_columns")): + kinds.add("enum_column_contract") + return kinds + + +def _interpolation_kinds(capture: dict[str, object]) -> set[str]: + kinds: set[str] = set() + for model in records(capture.get("model_inputs")): + model_file: dict[str, object] = as_json_object(model.get("model_file")) or {} + authored: str = str(model_file.get("query_sql", "")) + rendered: str = str(model.get("query_sql", "")) + if _PROJECT_VARIABLE.search(authored) and not _PROJECT_VARIABLE.search(rendered): + kinds.add("project_variable") + if _ENVIRONMENT_VARIABLE in authored and _ENVIRONMENT_VARIABLE not in rendered: + kinds.add("environment_variable") + values: dict[str, object] = ( + as_json_object((as_json_object(model.get("config")) or {}).get("values")) or {} + ) + placeholders: set[str] = set(_RUNTIME_PLACEHOLDER.findall(rendered)) + defaults: dict[str, object] = as_json_object(values.get(_PLACEHOLDERS_KEY)) or {} + if placeholders and placeholders <= set(defaults): + kinds.add("runtime_placeholder") + if ( + _CURSOR_INTRINSIC in rendered + and values.get(_MATERIALIZED_KEY) == _INCREMENTAL + and values.get(_CURSOR_KEY) is not None + ): + kinds.add("cursor_intrinsic") + discovered: dict[str, object] = as_json_object(capture.get("discovered_inputs")) or {} + authored_expressions: dict[str, str] = {} + for source_file in records(discovered.get("source_files")): + for entry in records(source_file.get("source_entries")): + authored_expressions[str(entry.get("name"))] = str(entry.get("expression")) + for source in records(capture.get("source_inputs")): + entry: dict[str, object] = as_json_object(source.get("source_entry")) or {} + if authored_expressions.get(str(entry.get("name"))) not in (None, entry.get("expression")): + kinds.add("source_expression_rendered") + if _audit_arguments_rendered(capture): + kinds.add("audit_arguments") + return kinds + + +def _audit_arguments_rendered(capture: dict[str, object]) -> bool: + """Whether an authored generic-audit argument's value is bound into its rendered SQL.""" + + authored: dict[tuple[str, str | None, str], dict[str, object]] = _authored_audit_arguments( + capture + ) + for audit in records(capture.get("audit_inputs")): + relative_path: dict[str, object] = ( + as_json_object((as_json_object(audit.get("audit_file")) or {}).get("relative_path")) + or {} + ) + definition: str = PurePosixPath(str(relative_path.get("__path__", ""))).stem + key: tuple[str, str | None, str] = ( + str(audit.get("attached_target_name")), + _optional_text(audit.get("attached_column_name")), + definition, + ) + template: str = str((as_json_object(audit.get("audit_block")) or {}).get("sql_body", "")) + rendered: str = str(audit.get("sql_body", "")) + for name, value in authored.get(key, {}).items(): + literal: re.Pattern[str] = re.compile(rf"(? dict[tuple[str, str | None, str], dict[str, object]]: + """Map `(target, column, definition)` to authored arguments, except built-ins and `column`.""" + + found: dict[tuple[str, str | None, str], dict[str, object]] = {} + for model in records(capture.get("model_inputs")): + entry: dict[str, object] = as_json_object(model.get("schema_entry")) or {} + target: str = str(entry.get("name")) + attachments: list[tuple[str | None, dict[str, object]]] = [ + (None, instance) for instance in records(entry.get("audits")) + ] + for column in records(entry.get("columns")): + attachments.extend( + (str(column.get("name")), instance) for instance in records(column.get("audits")) + ) + for column_name, instance in attachments: + definition: str = str(instance.get("definition_name")) + if definition not in BUILT_IN_AUDIT_NAMES: + found[(target, column_name, definition)] = _authored_arguments(instance) + return found + + +def _authored_arguments(instance: dict[str, object]) -> dict[str, object]: + arguments: dict[str, object] = as_json_object(instance.get("arguments")) or {} + return { + name: value + for name, value in arguments.items() + if name != _IMPLICIT_AUDIT_ARGUMENT and isinstance(value, (int, float, str)) + } + + +def _optional_text(value: object) -> str | None: + return None if value is None else str(value) + + +def _hook_kinds(capture: dict[str, object]) -> set[str]: + kinds: set[str] = set() + for model in records(capture.get("model_inputs")): + header: dict[str, object] = ( + as_json_object((as_json_object(model.get("model_file")) or {}).get("header_values")) + or {} + ) + values: dict[str, object] = ( + as_json_object((as_json_object(model.get("config")) or {}).get("values")) or {} + ) + for key in RENDER_HOOK_KEYS: + authored_hooks: list[dict[str, object]] = records(header.get(key)) + rendered_hooks: list[dict[str, object]] = records(values.get(key)) + for authored, rendered in zip(authored_hooks, rendered_hooks, strict=False): + kinds.update(_hook_entry_kinds(authored=authored, rendered=rendered)) + return kinds + + +def _hook_entry_kinds(*, authored: dict[str, object], rendered: dict[str, object]) -> set[str]: + if str(rendered.get("__type__", "")).endswith(_PYTHON_HOOK_TYPE): + return {"python_hook"} + statement: str = str(rendered.get("statement", "")) + authored_text: str = json.dumps(authored, ensure_ascii=False) + kinds: set[str] = {"named_sql_hook" if rendered.get("definition_sql") else "inline_sql_hook"} + if _CONTEXT_VARIABLE in authored_text and _CONTEXT_VARIABLE not in statement: + kinds.add("hook_context_variable") + if _MACRO_CALL.search(authored_text) and not _MACRO_CALL.search(statement): + kinds.add("macro_in_hook") + if ( + has_content(rendered.get("kwargs")) + and _TEMPLATE_ARGUMENT.search(str(rendered.get("definition_sql", ""))) + and not _TEMPLATE_ARGUMENT.search(statement) + ): + kinds.add("named_hook_arguments") + return kinds + + +def _reference_kinds(capture: dict[str, object]) -> set[str]: + kinds: set[str] = set() + for collection in _REFERENCE_COLLECTIONS: + for item in records(capture.get(collection)): + kinds.update( + f"reference_{enum_name(reference.get('ref_kind'))}" + for reference in records(item.get("references")) + ) + return kinds + + +def _attachment_kinds(capture: dict[str, object]) -> set[str]: + kinds: set[str] = set() + for test in records(capture.get("test_inputs")): + kinds.add(f"{enum_name(test.get('mode'))}_test") + if test.get("case_name") is not None: + kinds.add("parameterized_test_case") + for audit in records(capture.get("audit_inputs")): + target: str | None = enum_name(audit.get("attached_target_kind")) + if target is None: + kinds.add("singular_audit") + elif audit.get("attached_column_name") is not None: + kinds.add(f"{target}_column_audit") + else: + kinds.add(f"{target}_audit") + for model in records(capture.get("model_inputs")): + config: dict[str, object] = as_json_object(model.get("config")) or {} + if config.get("matched_path_default") is not None: + kinds.add("path_default") + return kinds diff --git a/scripts/compiler_differential/_helpers/running/execution.py b/scripts/compiler_differential/_helpers/running/execution.py index 06bd80c6c2..4835622891 100644 --- a/scripts/compiler_differential/_helpers/running/execution.py +++ b/scripts/compiler_differential/_helpers/running/execution.py @@ -15,6 +15,7 @@ ENGINE_ENV_VAR, EXCLUDED_ENVIRONMENT_KEYS, EXCLUDED_ENVIRONMENT_PREFIX, + GENERATOR_ENVIRONMENT, MANIFEST_FILE, PROJECT_DIRECTORY, SQB_ENTRY, @@ -88,12 +89,16 @@ def run_engine( def harness_environment() -> dict[str, str]: - """Return this process's environment without engine, capture, and dbt settings.""" + """Return this environment without engine, capture or dbt settings, plus generator vars.""" return { - key: value - for key, value in os.environ.items() - if key not in EXCLUDED_ENVIRONMENT_KEYS and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX) + **{ + key: value + for key, value in os.environ.items() + if key not in EXCLUDED_ENVIRONMENT_KEYS + and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX) + }, + **GENERATOR_ENVIRONMENT, } diff --git a/scripts/compiler_differential/_helpers/running/report.py b/scripts/compiler_differential/_helpers/running/report.py index 79a4e76dab..d8e835850d 100644 --- a/scripts/compiler_differential/_helpers/running/report.py +++ b/scripts/compiler_differential/_helpers/running/report.py @@ -2,7 +2,11 @@ from __future__ import annotations -from scripts.compiler_differential.constants import CONFIG_ONLY_KINDS +from scripts.compiler_differential.constants import ( + CONFIG_ONLY_KINDS, + RENDER_CONFIG_ONLY_KINDS, + RENDER_INDIRECT_KINDS, +) from scripts.compiler_differential.models import Difference, ProjectComparison @@ -25,13 +29,16 @@ def format_summary( comparisons: list[ProjectComparison], engines: tuple[str, str], seconds: float, - missing_coverage: tuple[str, ...] = (), + missing_coverage: dict[str, tuple[str, ...]] | None = None, ) -> str: - """Summarize the run; any difference or required-but-missing coverage reports FAILED.""" + """Summarize the run; any difference or missing coverage, keyed by stage, reports FAILED.""" differing: list[ProjectComparison] = [item for item in comparisons if item.differences] + gaps: dict[str, tuple[str, ...]] = { + stage: kinds for stage, kinds in (missing_coverage or {}).items() if kinds + } left_engine, right_engine = engines - if not differing and not missing_coverage: + if not differing and not gaps: return ( f"Compiler differential passed: {len(comparisons)} projects identical " f"({left_engine} vs {right_engine}) in {seconds:.1f}s" @@ -43,21 +50,57 @@ def format_summary( if differing: first: Difference = differing[0].differences[0] lines.append(f"First difference: {first.project}: {first.artifact} at {first.location}") - if missing_coverage: - lines.append(f"Required discovery coverage missing: {', '.join(missing_coverage)}") + lines.extend( + f"Required {stage} coverage missing: {', '.join(kinds)}" for stage, kinds in gaps.items() + ) return "\n".join(lines) def format_discovery_coverage(*, covered: frozenset[str], required: tuple[str, ...]) -> str: """Report how many required discovery input kinds the seed corpus exercised.""" - missing: list[str] = [kind for kind in required if kind not in covered] - line: str = ( - f"Discovery coverage: {len(required) - len(missing)} of {len(required)} input kinds " - "exercised by the seed corpus" + return _format_coverage( + stage="Discovery", + covered=covered, + required=required, + notes=(("config-only kinds", "only proves", CONFIG_ONLY_KINDS),), ) - config_only: str = "; ".join( - f"{kind} (only proves {meaning})" for kind, meaning in CONFIG_ONLY_KINDS.items() + + +def format_render_coverage(*, covered: frozenset[str], required: tuple[str, ...]) -> str: + """Report how many required render input kinds the seed corpus exercised.""" + + return _format_coverage( + stage="Render", + covered=covered, + required=required, + notes=( + ("config-only kinds", "only proves", RENDER_CONFIG_ONLY_KINDS), + ("indirect kinds", "credited because", RENDER_INDIRECT_KINDS), + ), ) - line = f"{line}\n config-only kinds: {config_only}" - return line if not missing else f"{line}\n missing: {', '.join(missing)}" + + +def _format_coverage( + *, + stage: str, + covered: frozenset[str], + required: tuple[str, ...], + notes: tuple[tuple[str, str, dict[str, str]], ...], +) -> str: + missing: list[str] = [kind for kind in required if kind not in covered] + lines: list[str] = [ + f"{stage} coverage: {len(required) - len(missing)} of {len(required)} input kinds " + "exercised by the seed corpus" + ] + for label, verb, kinds in notes: + by_meaning: dict[str, list[str]] = {} + for kind, meaning in kinds.items(): + by_meaning.setdefault(meaning, []).append(kind) + described: str = "; ".join( + f"{', '.join(names)} ({verb} {meaning})" for meaning, names in by_meaning.items() + ) + lines.append(f" {label}: {described or 'none'}") + if missing: + lines.append(f" missing: {', '.join(missing)}") + return "\n".join(lines) diff --git a/scripts/compiler_differential/_helpers/running/run.py b/scripts/compiler_differential/_helpers/running/run.py index 49ed382374..44ce33fec8 100644 --- a/scripts/compiler_differential/_helpers/running/run.py +++ b/scripts/compiler_differential/_helpers/running/run.py @@ -29,6 +29,7 @@ def differential_options(*, args: argparse.Namespace, work_dir: Path) -> Differe python=args.python, engine_environment=parse_engine_environment(args.engine_env), require_discovery_coverage=args.require_discovery_coverage, + require_render_coverage=args.require_render_coverage, ) diff --git a/scripts/compiler_differential/classes/discovery_features.py b/scripts/compiler_differential/classes/discovery_features.py index 63625986e6..d0a30ddd73 100644 --- a/scripts/compiler_differential/classes/discovery_features.py +++ b/scripts/compiler_differential/classes/discovery_features.py @@ -108,7 +108,9 @@ def write(self) -> None: "line_endings": self._line_endings, } for block in self._blocks: - writers[block]() + writer: Callable[[], None] | None = writers.get(block) + if writer is not None: + writer() def _base(self) -> ModelPlan: return self._random.choice(self._staging) diff --git a/scripts/compiler_differential/classes/project_builder.py b/scripts/compiler_differential/classes/project_builder.py index bdffa11e73..43138485c0 100644 --- a/scripts/compiler_differential/classes/project_builder.py +++ b/scripts/compiler_differential/classes/project_builder.py @@ -10,6 +10,7 @@ DiscoveryFeatureWriter, feature_blocks_for_seed, ) +from scripts.compiler_differential.classes.render_features import RenderFeatureWriter from scripts.compiler_differential.constants import ( DUCKDB_ADAPTER, GENERATOR_CONSTANT_KIND, @@ -95,7 +96,21 @@ def build(self) -> GeneratedProject: label=self._label, ) extras.write() - self._write_config(domains=domains, adapter=extras.adapter, extra_lines=extras.config_lines) + rendering: RenderFeatureWriter = RenderFeatureWriter( + blocks=blocks, + rng=self._random, + files=self._files, + features=self._features, + staging=[model for model in self._models if model.layer == _STAGING], + sources=self._sources, + label=self._label, + ) + rendering.write() + self._write_config( + domains=domains, + adapter=extras.adapter, + extra_lines=[*extras.config_lines, *rendering.config_lines], + ) if invalid_kind is not None: self._inject_invalid(kind=invalid_kind) return GeneratedProject( diff --git a/scripts/compiler_differential/classes/render_features.py b/scripts/compiler_differential/classes/render_features.py new file mode 100644 index 0000000000..fa791e7ed0 --- /dev/null +++ b/scripts/compiler_differential/classes/render_features.py @@ -0,0 +1,417 @@ +"""Write the optional project inputs that make generated seeds cover every render input kind.""" + +from __future__ import annotations + +import random +from collections.abc import Callable + +from scripts.compiler_differential.constants import ( + GENERATOR_CHANNEL_ENV_VAR, + GENERATOR_RENDER_FOLDER, + GENERATOR_RENDER_TEST_FOLDER, +) +from scripts.compiler_differential.models import ModelPlan + + +class RenderFeatureWriter: + """Add self-contained rendering blocks to one generated project, chosen per seed.""" + + def __init__( + self, + *, + blocks: tuple[str, ...], + rng: random.Random, + files: dict[str, str], + features: set[str], + staging: list[ModelPlan], + sources: list[str], + label: Callable[[], str], + ) -> None: + self._blocks: tuple[str, ...] = blocks + self._random: random.Random = rng + self._files: dict[str, str] = files + self._features: set[str] = features + self._staging: list[ModelPlan] = staging + self._sources: list[str] = sources + self._label: Callable[[], str] = label + self.config_lines: list[str] = [] + + def write(self) -> None: + """Write the selected rendering blocks in their canonical order.""" + + writers: dict[str, Callable[[], None]] = { + "macro_context_reads": self._macro_context_reads, + "typed_reference_macro": self._typed_reference_macro, + "macro_generated_reference": self._macro_generated_reference, + "table_function": self._table_function, + "interpolation": self._interpolation, + "cursor_bounds": self._cursor_bounds, + "macros_across_resources": self._macros_across_resources, + "enum_column_contract": self._enum_column_contract, + "resource_audits": self._resource_audits, + } + for block in self._blocks: + writer: Callable[[], None] | None = writers.get(block) + if writer is not None: + writer() + + def _base(self) -> ModelPlan: + return self._random.choice(self._staging) + + def _model(self, *, name: str, header: str, body: str, description: str) -> None: + self._files[f"{GENERATOR_RENDER_FOLDER}/{name}.sql"] = ( + f'MODEL (\n description "{description}",\n{header});\n\n{body}' + ) + + def _owned(self, *, role: str, name: str) -> str: + return f"{GENERATOR_RENDER_FOLDER}/_sqlbuild/_{role}/{name}" + + def _macro_context_reads(self) -> None: + self._features.add("macro_context_reads") + base: ModelPlan = self._base() + minimum: int = self._random.randint(1, 4) + self._files[self._owned(role="constants", name="render_limits.sql")] = ( + f"CONSTANT (name minimum_render_amount, value {minimum});\n" + ) + self._files[self._owned(role="enums", name="render_channel.sql")] = ( + 'ENUM (\n name render_channel,\n members (WEB "web", STORE "store"),\n);\n' + ) + self._files[self._owned(role="macros", name="policy.py")] = ( + "import os\n\n\n" + "def minimum_amount_filter(ctx, column: str) -> str:\n" + ' """Keep rows at or above the configured minimum amount."""\n' + ' if ctx.constants["minimum_render_amount"] is None:\n' + ' return "TRUE"\n' + " return f\"{column} >= {ctx.render_constant('minimum_render_amount')}\"\n\n\n" + "def channel_label(ctx) -> str:\n" + ' """Label rows with the web channel, target and adapter."""\n' + ' channel = ctx.render_enum_member(enum_name="render_channel", member_name="WEB")\n' + ' region = ctx.vars["region"]\n' + " return f\"{channel} || '/{ctx.target_name}/{ctx.adapter_name}/{region}'\"\n\n\n" + "def with_deployment(expression: str) -> str:\n" + ' """Suffix an expression with the deployment channel from the environment."""\n' + f' channel = os.environ.get("{GENERATOR_CHANNEL_ENV_VAR}", "none")\n' + " return f\"({expression} || '@{channel}')\"\n" + ) + self._model( + name=f"policy_{base.name}", + description=f"Policy-filtered copy of {base.name}", + header="", + body=( + "SELECT id, amount, @with_deployment(@channel_label()) AS channel,\n" + ' @const("minimum_render_amount") AS minimum_amount,\n' + ' @enum("render_channel").STORE AS fallback_channel\n' + f'FROM __ref("{base.name}")\n' + 'WHERE @minimum_amount_filter("amount")\n' + ), + ) + + def _typed_reference_macro(self) -> None: + self._features.add("typed_reference_macro") + first, second = ( + self._random.sample(self._staging, k=2) + if len(self._staging) > 1 + else (self._staging * 2) + ) + source: str = self._random.choice(self._sources) + self._files[self._owned(role="macros", name="relations.py")] = ( + "from sqlbuild.refs import SqlResourceRef\n\n\n" + "def union_ids(relations: list[SqlResourceRef]) -> str:\n" + ' """Union the ids of several relations."""\n' + ' selects = [f"SELECT id FROM {relation}" for relation in relations]\n' + ' return " UNION ALL ".join(selects)\n' + "\n\n" + "def latest_rows(relation: SqlResourceRef, limit: int = 2) -> str:\n" + ' """Return the newest rows of one relation."""\n' + ' return f"(SELECT id, amount FROM {relation} ORDER BY id DESC LIMIT {limit})"\n' + ) + self._model( + name="unioned_event_ids", + description="Ids from two staged relations", + header="", + body=( + "SELECT id\n" + f'FROM (@union_ids([__ref("{first.name}"), __ref("{second.name}")])) AS unioned\n' + ), + ) + self._model( + name=f"latest_{source}", + description=f"Newest rows of {source}", + header="", + body=f'SELECT id, amount\nFROM @latest_rows(__source("{source}"), limit=2) AS latest\n', + ) + + def _macro_generated_reference(self) -> None: + self._features.add("macro_generated_reference") + base: ModelPlan = self._base() + self.config_lines.extend(["", "[references]", "enforce_explicit = false"]) + self._files[self._owned(role="macros", name="legacy.py")] = ( + "def legacy_events() -> str:\n" + ' """Return a staged relation the way a migrated project wrote it."""\n' + f" return '__ref(\"{base.name}\")'\n" + ) + self._model( + name="legacy_event_ids", + description="Ids read through a macro-generated reference", + header="", + body="SELECT id\nFROM @legacy_events() AS legacy\n", + ) + + def _table_function(self) -> None: + self._features.add("table_function") + base: ModelPlan = self._base() + self._files["functions/sql/table_fn__events_above.sql"] = ( + "FUNCTION (\n" + ' description "Events above an amount",\n' + " arguments (p_minimum DOUBLE),\n" + " returns table (\n id INTEGER,\n amount DOUBLE\n ),\n);\n\n" + f'SELECT id, amount\nFROM __ref("{base.name}")\nWHERE amount > p_minimum\n' + ) + self._model( + name="large_events", + description="Events above a threshold from a table function", + header="", + body=( + "SELECT id, amount\n" + f'FROM __table_fn("table_fn__events_above")({self._random.choice(("1.5", "4"))})\n' + ), + ) + + def _interpolation(self) -> None: + self._features.add("interpolation") + base: ModelPlan = self._base() + self._files[self._owned(role="hooks/sql", name="announce_refresh.sql")] = ( + 'HOOK (\n description "Announce one refreshed relation"\n);\n\n' + "SELECT @'relation' AS refreshed_relation, @'label' AS refresh_label\n" + ) + self._files[self._owned(role="macros", name="notes.py")] = ( + "def refresh_note(relation: str) -> str:\n" + ' """Describe one refreshed relation."""\n' + " return f\"'refreshed ' || '{relation}'\"\n" + ) + self._model( + name=f"interpolated_{base.name}", + description=f"Interpolated copy of {base.name}", + header=( + " materialized table,\n" + " post_hooks [\n" + ' sql("announce_refresh", relation: "@@CTX:destination.qualified", ' + f'label: "{self._label()}"),\n' + " inline_sql(\"SELECT @refresh_note('@@CTX:destination.qualified') AS note\"),\n" + " inline_sql(\"SELECT '@@CTX:run.target' AS target, " + "'@@CTX:model.name' AS model\"),\n" + " ],\n" + ), + body=( + "SELECT id,\n" + " '@@site_label' AS site,\n" + f" '@@ENV:{GENERATOR_CHANNEL_ENV_VAR}' AS channel,\n" + " amount * @@discount_rate AS discounted\n" + f'FROM __ref("{base.name}")\n' + ), + ) + self._deferred_placeholder(base) + + def _deferred_placeholder(self, base: ModelPlan) -> None: + self._files["materializations/windowed_copy.py"] = ( + "from sqlbuild.executor.custom.models import MaterializationContext, " + "MaterializationResult\n\n\n" + "def materialize(ctx: MaterializationContext) -> MaterializationResult:\n" + ' sql = ctx.sql.replace("@@@window_start", "DATE \'2026-01-01\'")\n' + ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS {sql}")\n' + " return MaterializationResult(relation=ctx.destination, audit_results=())\n" + ) + self._model( + name=f"windowed_{base.name}", + description=f"Runtime-windowed copy of {base.name}", + header=( + " materialized windowed_copy,\n" + " placeholders (\n window_start \"'2026-01-01'\",\n ),\n" + " config (\n window_column created_at,\n ),\n" + ), + body=( + f'SELECT id, created_at\nFROM __ref("{base.name}")\n' + "WHERE CAST(created_at AS DATE) >= CAST(@@@window_start AS DATE)\n" + ), + ) + + def _cursor_bounds(self) -> None: + self._features.add("cursor_bounds") + base: ModelPlan = self._base() + self._model( + name=f"bounded_{base.name}", + description=f"Cursor-bounded copy of {base.name}", + header=( + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " unique_key [id],\n" + " cursor created_at,\n cursor_type timestamp,\n cursor_grain day,\n" + " incremental_mode microbatch,\n" + " microbatch_strategy rolling_window,\n" + " lookback 1d,\n" + " batch_size 1d,\n" + f" cursor_inputs (\n {base.name} created_at,\n ),\n" + ), + body=( + "SELECT id, amount, created_at\n" + f'FROM __ref("{base.name}")\n' + "WHERE created_at >= __cursor_start()\n AND created_at < __cursor_end()\n" + ), + ) + + def _macros_across_resources(self) -> None: + self._features.add("macros_across_resources") + base: ModelPlan = self._base() + source: str = base.inputs[0].split(":", 1)[1] + self._files["macros/labels.py"] = ( + "def tidy_label(expression: str) -> str:\n" + ' """Trim and lower-case a label."""\n' + ' return f"LOWER(TRIM({expression}))"\n' + ) + self._files[self._owned(role="audits/generic", name="tidy_status.sql")] = ( + 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE @column <> @tidy_label("@column")\n' + ) + self._files["functions/sql/tidy_status.sql"] = ( + "FUNCTION (\n" + ' description "Normalise a status label.",\n' + " arguments (raw_status STRING),\n" + " returns STRING,\n);\n\n" + '@tidy_label("raw_status")\n' + ) + self._model( + name=f"labelled_{base.name}", + description=f"Labelled copy of {base.name}", + header=" columns (\n status (audits [tidy_status]),\n ),\n", + body=( + 'SELECT id, @tidy_label("status") AS status,\n' + ' __udf("tidy_status")(status) AS tidied\n' + f'FROM __ref("{base.name}")\n' + ), + ) + self._files["sources/labelled.yml"] = ( + "sources:\n" + " - name: labelled_channels\n" + " description: Channel labels normalised in the source expression.\n" + ' expression: "(SELECT 1 AS id, @tidy_label(\\"\' Web \'\\") AS channel)"\n' + " columns:\n" + " - name: id\n type: INTEGER\n" + " - name: channel\n type: VARCHAR\n" + ) + self._model( + name="channel_labels", + description="Normalised channel labels", + header="", + body='SELECT id, channel\nFROM __source("labelled_channels")\n', + ) + self._files[ + f"{GENERATOR_RENDER_FOLDER}/_sqlbuild/audits/singular/labelled_channels_match.sql" + ] = ( + 'AUDIT (name "labelled_channels_match");\n\n' + f'SELECT l.id\nFROM __ref("labelled_{base.name}") l\n' + 'JOIN __ref("channel_labels") c ON l.id = c.id\nWHERE l.status = c.channel\n' + ) + self._files[f"tests/scenarios/labelled_{base.name}_scenario.sql"] = ( + 'SCENARIO (\n description "Labels are normalised end to end"\n);\n\n' + "WITH\n" + f"__source__{source} AS (\n" + " SELECT 3 AS id, CAST(1 AS DOUBLE) AS amount, ' Placed ' AS status,\n" + " TIMESTAMP '2026-01-03 00:00:00' AS created_at\n" + "),\n" + f"__expected__labelled_{base.name} AS (\n" + " SELECT 3 AS id, @tidy_label(\"' Placed '\") AS status\n" + ")\nSELECT 1\n" + ) + self._files[f"{GENERATOR_RENDER_TEST_FOLDER}/_sqlbuild/_macros/fixtures.py"] = ( + "def event_rows(count: int = 2) -> str:\n" + ' """Return fixture event rows."""\n' + ' return " UNION ALL ".join(\n' + ' f"SELECT {index} AS id, CAST({index} AS DOUBLE) AS amount, "\n' + " f\"'placed' AS status, \"\n" + " f\"TIMESTAMP '2026-01-0{index} 00:00:00' AS created_at\"\n" + " for index in range(1, count + 1)\n" + " )\n" + ) + self._files[f"{GENERATOR_RENDER_TEST_FOLDER}/test_labelled_{base.name}.sql"] = ( + "TEST();\n\nWITH\n" + f"__source__{source} AS (\n @event_rows(2)\n),\n" + f"__expected__labelled_{base.name} AS (\n" + " SELECT 1 AS id, @tidy_label(\"'placed'\") AS status\n" + " UNION ALL\n" + " SELECT 2 AS id, 'placed' AS status\n" + ")\nSELECT 1\n" + ) + + def _enum_column_contract(self) -> None: + self._features.add("enum_column_contract") + base: ModelPlan = self._base() + self._files[self._owned(role="enums", name="render_state.sql")] = ( + "ENUM (\n name render_state,\n members (\n" + ' PLACED "placed",\n SHIPPED "expédié",\n CANCELLED "取消",\n' + ' RETURNED "returned ✓",\n ),\n);\n' + ) + self._model( + name=f"stated_{base.name}", + description=f"Enum-typed copy of {base.name}", + header=( + " contract enforced,\n" + " columns (\n" + " id (type INTEGER),\n" + " status (type render_state),\n" + " tier (type VARCHAR),\n" + " ),\n" + f" constants (\n _tier_floor {self._random.randint(1, 9)},\n ),\n" + " enums (\n _tier [LOW, HIGH],\n ),\n" + ), + body=( + "SELECT CAST(id AS INTEGER) AS id, status,\n" + ' CASE WHEN amount >= @const("_tier_floor") THEN @enum("_tier").HIGH\n' + ' ELSE @enum("_tier").LOW END AS tier\n' + f'FROM __ref("{base.name}")\n' + ), + ) + + def _resource_audits(self) -> None: + self._features.add("resource_audits") + self._files["sources/audited.yml"] = ( + "sources:\n" + " - name: audited_events\n" + " description: Events with source audits.\n" + ' expression: "(SELECT 1 AS id, CAST(@@discount_rate AS DOUBLE) AS amount)"\n' + " columns:\n" + " - name: id\n" + " type: INTEGER\n" + " audits:\n" + " - not_null\n" + " - unique\n" + " - accepted_values:\n" + " values: [1, 2]\n" + " - name: amount\n" + " type: DOUBLE\n" + ) + self._files["seeds/channel_codes.csv"] = f'id,label\n1,"{self._label()}"\n2,web\n' + self._files["seeds/channel_codes.yml"] = ( + "seeds:\n" + " - name: channel_codes\n" + " description: Channel codes with seed audits.\n" + " columns:\n" + " - name: id\n" + " type: INTEGER\n" + " audits:\n" + " - not_null\n" + " - name: label\n" + " type: VARCHAR\n" + ) + self._files[self._owned(role="audits/generic", name="amount_floor.sql")] = ( + 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE amount < @minimum\n' + ) + self._model( + name="audited_event_totals", + description="Totals of the audited events per channel", + header=f" audits [amount_floor (minimum {self._random.randint(-40, -2)})],\n", + body=( + "SELECT c.label, SUM(e.amount) AS amount\n" + 'FROM __source("audited_events") e\n' + 'JOIN __seed("channel_codes") c ON e.id = c.id\n' + "GROUP BY c.label\n" + ), + ) diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 7d64fbea60..e9429d3521 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -181,6 +181,17 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: GENERATOR_MACRO_KIND: "macros", GENERATOR_HOOK_KIND: "hooks", } +GENERATOR_RENDER_BLOCKS: tuple[str, ...] = ( + "macro_context_reads", + "typed_reference_macro", + "macro_generated_reference", + "table_function", + "interpolation", + "cursor_bounds", + "macros_across_resources", + "enum_column_contract", + "resource_audits", +) GENERATOR_FEATURE_BLOCKS: tuple[str, ...] = ( "incremental_append", "incremental_delete_insert", @@ -209,6 +220,7 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "target_override", "dbt_ref", "line_endings", + *GENERATOR_RENDER_BLOCKS, ) GENERATOR_FEATURE_STRIDE: int = 4 GENERATOR_OPTIONAL_FEATURE_SHARE: float = 0.2 @@ -216,6 +228,10 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: GENERATOR_RARE_FEATURE_BLOCKS: dict[str, int] = {"dbt_ref": 1} GENERATOR_DBT_REF_ERROR_CODE: str = "C214" GENERATOR_FEATURE_FOLDER: str = "models/features" +GENERATOR_RENDER_FOLDER: str = "models/rendering" +GENERATOR_RENDER_TEST_FOLDER: str = "tests/unit/rendering" +GENERATOR_CHANNEL_ENV_VAR: str = "SQB_DIFFERENTIAL_CHANNEL" +GENERATOR_ENVIRONMENT: dict[str, str] = {GENERATOR_CHANNEL_ENV_VAR: "web-orders"} GENERATOR_PROJECT_ADAPTER: str = "generated_duckdb" GENERATOR_CRLF: str = "\r\n" GENERATOR_BOM: str = "\ufeff" @@ -352,3 +368,128 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "the config sets [dbt] target_path; the manifest is read after discovery" ), } +RENDER_STAGE_CAPTURE_SUFFIX: str = "-compile_project_inputs.json" +RENDER_NON_COLLECTION_FIELDS: frozenset[str] = frozenset( + { + "project_config", + "local_config", + "discovered_inputs", + "sql_lexical_syntax", + "run_id", + "effective_target_name", + "effective_target", + "compile_cache_dir", + "effective_connection", + "effective_settings", + "no_sql_validation", + "effective_vars", + "macro_context", + "diagnostics", + "external_sql_reference_resolver", + "scope_index", + "declaration_scope", + "analysis_reuse", + } +) +RENDER_CALLABLE_CAPTURE_FIELDS: frozenset[str] = CALLABLE_CAPTURE_FIELDS +TYPED_REFERENCE_ANNOTATION: str = "SqlResourceRef" +RENDER_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") +RENDER_MACRO_SOURCE_READS: dict[str, tuple[str, ...]] = { + "macro_reads_vars": ("ctx.vars",), + "macro_reads_constants": ("ctx.constants", "ctx.render_constant"), + "macro_reads_enums": ("ctx.enums", "ctx.render_enum_member"), + "macro_reads_target": ("ctx.target_name", "ctx.adapter_name"), + "macro_reads_environment": ("os.environ",), +} +RENDER_DETAIL_KINDS: tuple[str, ...] = ( + "macro_in_model", + "macro_in_test", + "macro_in_scenario", + "macro_in_audit", + "macro_in_function", + "macro_in_source_expression", + "macro_in_hook", + "nested_macro_call", + "typed_reference_argument", + "macro_generated_reference", + "cross_file_macro_import", + "tested_macro", + *RENDER_MACRO_SOURCE_READS, + "scope_global_use", + "scope_inherited_use", + "scope_local_use", + "scope_private_use", + "enum_member", + "scalar_constant", + "list_constant", + "expected_model_grant", + "enum_column_contract", + "project_variable", + "environment_variable", + "runtime_placeholder", + "cursor_intrinsic", + "source_expression_rendered", + "audit_arguments", + "python_hook", + "named_sql_hook", + "inline_sql_hook", + "hook_context_variable", + "named_hook_arguments", + "reference_ref", + "reference_source", + "reference_seed", + "reference_udf", + "reference_table_function", + "reference_dbt_ref", + "model_test", + "macro_test", + "udf_test", + "parameterized_test_case", + "singular_audit", + "model_audit", + "model_column_audit", + "source_column_audit", + "seed_column_audit", + "path_default", +) +RENDER_INDIRECT_KINDS: dict[str, str] = { + kind: "the used macro's own function source reads it; the capture does not record context reads" + for kind in RENDER_MACRO_SOURCE_READS +} +RENDER_CONFIG_ONLY_KINDS: dict[str, str] = {} +DIAGNOSTIC_CODE_PATTERN: re.Pattern[str] = re.compile(r"[A-Z][0-9]{3}") +RENDER_RAISED_CODES: frozenset[str] = frozenset({"C214", "C216", "K012"}) +RENDER_CODE_SCAN_EXCLUDED: dict[str, str] = { + "_helpers/assembly": "semantic and metadata validation of the assembled CompiledProject (M4)", + "_helpers/analysis": "SQL analysis of rendered SQL, after the CompileProjectInputs frontier", +} +_SCOPE_QUERY_ONLY: str = "reported only by sqb scope queries, never by compile" +_TOLERANT_SCOPE_ONLY: str = ( + "reported only by the tolerant index sqb scope builds; compile discovery fails first " + "with a D-code" +) +_NEVER_EMITTED: str = "defined in ScopeDiagnosticCode but no compiler path reports it" +_FOLDED_INTO_P001: str = ( + "compile raises the strict scope index error, which reports it inside a P001 " + "'Invalid declaration scope index' message" +) +RENDER_UNREACHABLE_CODES: dict[str, str] = { + "S001": _SCOPE_QUERY_ONLY, + "S002": _SCOPE_QUERY_ONLY, + "S003": _FOLDED_INTO_P001, + "S004": "discovery rejects the identity first with D016", + "S005": "discovery rejects the identity first with D016", + "S011": _SCOPE_QUERY_ONLY, + "S012": _NEVER_EMITTED, + "S013": _SCOPE_QUERY_ONLY, + "S014": _SCOPE_QUERY_ONLY, + "S015": _NEVER_EMITTED, + "S016": _NEVER_EMITTED, + "S017": _NEVER_EMITTED, + "S018": "discovery rejects the duplicate resource first with D007", + "S019": _TOLERANT_SCOPE_ONLY, + "S020": _TOLERANT_SCOPE_ONLY, + "S021": _TOLERANT_SCOPE_ONLY, + "S022": _TOLERANT_SCOPE_ONLY, + "S023": _TOLERANT_SCOPE_ONLY, +} diff --git a/scripts/compiler_differential/main/differential.py b/scripts/compiler_differential/main/differential.py index a7d7974f1a..fdb49af5c1 100644 --- a/scripts/compiler_differential/main/differential.py +++ b/scripts/compiler_differential/main/differential.py @@ -10,9 +10,11 @@ from pathlib import Path from scripts.compiler_differential._helpers.coverage.discovery import required_discovery_kinds +from scripts.compiler_differential._helpers.coverage.render import required_render_kinds from scripts.compiler_differential._helpers.running.options import parse_expected_outcome from scripts.compiler_differential._helpers.running.report import ( format_discovery_coverage, + format_render_coverage, format_summary, ) from scripts.compiler_differential._helpers.running.run import ( @@ -44,15 +46,11 @@ def run_compiler_differential(argv: list[str] | None = None) -> int: comparisons: list[ProjectComparison] = compare_corpus( corpus=selected_corpus(args=args, repo_root=repo_root), options=options ) - missing_coverage: tuple[str, ...] = () - if options.stage_captures and CORPUS_SEEDS in args.corpus: - covered: frozenset[str] = frozenset().union( - *(comparison.discovered_kinds for comparison in comparisons) - ) - required: tuple[str, ...] = required_discovery_kinds() - print(format_discovery_coverage(covered=covered, required=required)) - if options.require_discovery_coverage: - missing_coverage = tuple(kind for kind in required if kind not in covered) + missing_coverage: dict[str, tuple[str, ...]] = ( + _seed_coverage(comparisons=comparisons, options=options) + if options.stage_captures and CORPUS_SEEDS in args.corpus + else {} + ) print( format_summary( comparisons=comparisons, @@ -61,7 +59,35 @@ def run_compiler_differential(argv: list[str] | None = None) -> int: missing_coverage=missing_coverage, ) ) - return 1 if missing_coverage or any(comparison.differences for comparison in comparisons) else 0 + return ( + 1 + if any(missing_coverage.values()) + or any(comparison.differences for comparison in comparisons) + else 0 + ) + + +def _seed_coverage( + *, comparisons: list[ProjectComparison], options: DifferentialOptions +) -> dict[str, tuple[str, ...]]: + """Print discovery and render coverage; return the missing kinds each requirement enforces.""" + + discovered: frozenset[str] = frozenset().union( + *(comparison.discovered_kinds for comparison in comparisons) + ) + rendered: frozenset[str] = frozenset().union( + *(comparison.rendered_kinds for comparison in comparisons) + ) + required_discovery: tuple[str, ...] = required_discovery_kinds() + required_render: tuple[str, ...] = required_render_kinds() + print(format_discovery_coverage(covered=discovered, required=required_discovery)) + print(format_render_coverage(covered=rendered, required=required_render)) + missing: dict[str, tuple[str, ...]] = {} + if options.require_discovery_coverage: + missing["discovery"] = tuple(kind for kind in required_discovery if kind not in discovered) + if options.require_render_coverage: + missing["render"] = tuple(kind for kind in required_render if kind not in rendered) + return missing def _parse_args(argv: list[str] | None) -> argparse.Namespace: @@ -99,6 +125,11 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: action="store_true", help="fail unless the seed corpus exercises every discovery input kind (needs captures)", ) + parser.add_argument( + "--require-render-coverage", + action="store_true", + help="fail unless the seed corpus exercises every render input kind (needs captures)", + ) parser.add_argument( "--engine-env", action="append", @@ -109,8 +140,10 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: parser.add_argument("--work-dir", type=Path, default=None, help="keep run directories here") parser.add_argument("--python", type=Path, default=Path(sys.executable)) args: argparse.Namespace = parser.parse_args(argv) - if args.require_discovery_coverage and not ( - args.stage_captures and CORPUS_SEEDS in args.corpus + for flag, required in ( + ("--require-discovery-coverage", args.require_discovery_coverage), + ("--require-render-coverage", args.require_render_coverage), ): - parser.error("--require-discovery-coverage needs --stage-captures and the seeds corpus") + if required and not (args.stage_captures and CORPUS_SEEDS in args.corpus): + parser.error(f"{flag} needs --stage-captures and the seeds corpus") return args diff --git a/scripts/compiler_differential/models.py b/scripts/compiler_differential/models.py index cb2bf0b149..9237e10793 100644 --- a/scripts/compiler_differential/models.py +++ b/scripts/compiler_differential/models.py @@ -34,7 +34,7 @@ class CorpusProject: source_dir: Path | None = None writer: Callable[[Path], None] | None = None project_subdirectory: str | None = None - discovery_coverage: bool = False + seed_coverage: bool = False @dataclass(frozen=True) @@ -89,6 +89,7 @@ class ProjectComparison: differences: tuple[Difference, ...] seconds: float discovered_kinds: frozenset[str] = frozenset() + rendered_kinds: frozenset[str] = frozenset() @dataclass(frozen=True) @@ -102,6 +103,7 @@ class DifferentialOptions: python: Path engine_environment: dict[str, dict[str, str]] require_discovery_coverage: bool = False + require_render_coverage: bool = False @dataclass(frozen=True) @@ -136,6 +138,7 @@ class FailureCase(WritableProject): name: str expected_code: str expected_warning_code: str | None = None + expected_message: str | None = None @dataclass(frozen=True) @@ -172,6 +175,7 @@ class EmittedCodes: errors: tuple[str, ...] warnings: tuple[str, ...] + first_error_message: str | None = None @property def first_error(self) -> str | None: diff --git a/tests/e2e/scripts/compiler_differential/_test_types.py b/tests/e2e/scripts/compiler_differential/_test_types.py index e26a034b4b..2056886b3d 100644 --- a/tests/e2e/scripts/compiler_differential/_test_types.py +++ b/tests/e2e/scripts/compiler_differential/_test_types.py @@ -43,8 +43,11 @@ class FailureCorpusCodesTestCase: description: str expected_first_errors: dict[str, str] + expected_first_messages: dict[str, str] expected_warnings: dict[str, str] expected_discovery_codes: frozenset[str] + expected_render_codes: frozenset[str] + unreachable_render_codes: frozenset[str] @dataclass(frozen=True) @@ -56,3 +59,14 @@ class CoverageFailureTestCase: expected_exit_code: int expected_lines: tuple[str, ...] expected_absent: tuple[str, ...] + + +@dataclass(frozen=True) +class RenderCaptureTestCase: + """A generated project with every feature block and the render kinds its capture proves.""" + + description: str + seed: int + blocks: tuple[str, ...] + command: tuple[str, ...] + expected_kinds: frozenset[str] diff --git a/tests/e2e/scripts/compiler_differential/helpers.py b/tests/e2e/scripts/compiler_differential/helpers.py index e28a9accab..ea3af574c8 100644 --- a/tests/e2e/scripts/compiler_differential/helpers.py +++ b/tests/e2e/scripts/compiler_differential/helpers.py @@ -49,6 +49,27 @@ def _perturbed(*, until, **stages): return dataclasses.replace(result, model_files=tuple(reversed(result.model_files))) +frontier.native_frontier = _perturbed +""" + +RENDER_PERTURBATION: str = """ +import dataclasses + +import sqlbuild.compiler.frontier.main._compile_frontier as frontier +from sqlbuild.compiler.frontier.types import CompilerStage + +_original = frontier.native_frontier + + +def _perturbed(*, until, **stages): + result = _original(until=until, **stages) + if until is not CompilerStage.COMPILE_PROJECT_INPUTS or not result.model_inputs: + return result + first = result.model_inputs[0] + changed = dataclasses.replace(first, macro_deps=(*first.macro_deps, "perturbed_by_test")) + return dataclasses.replace(result, model_inputs=(changed, *result.model_inputs[1:])) + + frontier.native_frontier = _perturbed """ diff --git a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py index 204b4a628c..ebf2ba845c 100644 --- a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py +++ b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py @@ -16,6 +16,7 @@ from tests.e2e.scripts.compiler_differential.helpers import ( DISCOVERY_PERTURBATION, NATIVE_ONLY_STDERR_LINE, + RENDER_PERTURBATION, harness_arguments, perturbation_arguments, write_broken_ref_project, @@ -129,6 +130,42 @@ def test_given_perturbed_native_discovery_when_comparing_then_discovery_capture_ assert all(re.search(pattern, output) for pattern in test_case.expected_patterns), output +@pytest.mark.parametrize( + "test_case", + [ + HarnessRunTestCase( + description="perturbed_native_compile_inputs", + extra_arguments=(), + expected_exit_code=1, + expected_lines=("DIFF project/waffle_shop",), + expected_patterns=( + r"- stage capture 0-compile/002-compile_project_inputs\.json at " + r"/model_inputs/0/macro_deps/", + r"- stage capture 2-plan/002-compile_project_inputs\.json at " + r"/model_inputs/0/macro_deps/", + ), + expected_absent=("001-discovered_project_inputs.json",), + ) + ], + ids=lambda case: case.description, +) +def test_given_perturbed_native_render_when_comparing_then_compile_inputs_capture_names_the_stage( + test_case: HarnessRunTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + exit_code: int = run_compiler_differential( + harness_arguments( + work_dir=tmp_path / "work", + extra=perturbation_arguments(tmp_path / "perturbation", source=RENDER_PERTURBATION), + ) + ) + + output: str = capsys.readouterr().out + assert exit_code == test_case.expected_exit_code, output + assert all(line in output for line in test_case.expected_lines), output + assert all(re.search(pattern, output) for pattern in test_case.expected_patterns), output + assert not any(text in output for text in test_case.expected_absent), output + + @pytest.mark.parametrize( "test_case", [ @@ -195,7 +232,26 @@ def test_given_both_engines_fail_alike_when_outcome_is_unexpected_then_harness_f "Required discovery coverage missing: ", ), expected_absent=("passed",), - ) + ), + CoverageFailureTestCase( + description="one_seed_cannot_cover_every_render_kind", + extra_arguments=( + "--corpus", + "seeds", + "--seeds", + "1", + "--stage-captures", + "--require-render-coverage", + ), + expected_exit_code=1, + expected_lines=( + "OK seed/0", + "Render coverage: ", + "Compiler differential FAILED: 0 of 1 projects differ", + "Required render coverage missing: ", + ), + expected_absent=("passed", "Required discovery coverage missing"), + ), ], ids=lambda case: case.description, ) diff --git a/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py b/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py index f4c96b1253..b0694a6d7b 100644 --- a/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py +++ b/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py @@ -1,8 +1,9 @@ -"""Every failure case emits its declared code as the first error, reaching every discovery code.""" +"""Every failure case emits its declared code first, reaching every discovery and render code.""" from __future__ import annotations import sys +from itertools import chain from pathlib import Path import pytest @@ -10,7 +11,9 @@ from scripts.compiler_differential._helpers.corpus.emitted_codes import ( discovery_error_codes, emitted_failure_codes, + render_error_codes, ) +from scripts.compiler_differential.constants import RENDER_UNREACHABLE_CODES from scripts.compiler_differential.main.failure_cases import failure_cases from scripts.compiler_differential.models import DifferentialOptions, EmittedCodes from tests.e2e.scripts.compiler_differential._test_types import FailureCorpusCodesTestCase @@ -24,13 +27,24 @@ expected_first_errors={ f"failure/{case.name}": case.expected_code for case in failure_cases() }, - expected_warnings={"failure/built-in-audit-shadow": "P003"}, + expected_first_messages={ + f"failure/{case.name}": case.expected_message + for case in failure_cases() + if case.expected_message is not None + }, + expected_warnings={ + f"failure/{case.name}": case.expected_warning_code + for case in failure_cases() + if case.expected_warning_code is not None + }, expected_discovery_codes=discovery_error_codes(), + expected_render_codes=render_error_codes() - frozenset(RENDER_UNREACHABLE_CODES), + unreachable_render_codes=frozenset(RENDER_UNREACHABLE_CODES), ) ], ids=lambda case: case.description, ) -def test_given_failure_corpus_when_compiling_then_emitted_first_errors_match_and_cover_discovery( +def test_given_failure_corpus_when_compiling_then_first_errors_match_and_cover_every_stage( test_case: FailureCorpusCodesTestCase, tmp_path: Path ) -> None: emitted: dict[str, EmittedCodes] = emitted_failure_codes( @@ -51,7 +65,21 @@ def test_given_failure_corpus_when_compiling_then_emitted_first_errors_match_and assert all( code in emitted[name].warnings for name, code in test_case.expected_warnings.items() ), emitted + first_messages: dict[str, str] = { + name: emitted[name].first_error_message or "" for name in test_case.expected_first_messages + } + assert all( + fragment in first_messages[name] + for name, fragment in test_case.expected_first_messages.items() + ), first_messages + reached: set[str | None] = { + *first_errors.values(), + *chain.from_iterable(codes.warnings for codes in emitted.values()), + } assert test_case.expected_discovery_codes <= set(first_errors.values()) + assert test_case.expected_render_codes <= reached, test_case.expected_render_codes - reached + assert not test_case.unreachable_render_codes & set(first_errors.values()) + assert test_case.unreachable_render_codes <= render_error_codes() if __name__ == "__main__": diff --git a/tests/e2e/scripts/compiler_differential/test_render_capture.py b/tests/e2e/scripts/compiler_differential/test_render_capture.py new file mode 100644 index 0000000000..ff23f217c2 --- /dev/null +++ b/tests/e2e/scripts/compiler_differential/test_render_capture.py @@ -0,0 +1,68 @@ +"""A real plan captures every compile-input field canonically and proves every render kind.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +from scripts.compiler_differential._helpers.coverage.render import ( + render_capture_problems, + rendered_input_kinds, + required_render_kinds, +) +from scripts.compiler_differential._helpers.running.execution import harness_environment +from scripts.compiler_differential.constants import ( + GENERATOR_FEATURE_BLOCKS, + RENDER_STAGE_CAPTURE_SUFFIX, + SQB_ENTRY, + STAGE_CAPTURE_ENV_VAR, +) +from scripts.compiler_differential.main.generate_project import generate_project +from tests.e2e.scripts.compiler_differential._test_types import RenderCaptureTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + RenderCaptureTestCase( + description="every_feature_block_planned", + seed=11, + blocks=GENERATOR_FEATURE_BLOCKS, + command=("plan", "--json"), + expected_kinds=frozenset(required_render_kinds()), + ) + ], + ids=lambda case: case.description, +) +def test_given_project_with_every_render_input_when_capturing_then_capture_is_complete( + test_case: RenderCaptureTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "project" + capture_dir: Path = tmp_path / "captures" + generate_project(seed=test_case.seed, blocks=test_case.blocks).write(project_dir) + + completed: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, "-c", SQB_ENTRY, "--no-color", *test_case.command], + cwd=project_dir, + env={**harness_environment(), STAGE_CAPTURE_ENV_VAR: str(capture_dir)}, + capture_output=True, + text=True, + check=False, + timeout=600, + ) + + assert completed.returncode == 0, completed.stdout + completed.stderr + text: str = next(capture_dir.glob(f"*{RENDER_STAGE_CAPTURE_SUFFIX}")).read_text( + encoding="utf-8" + ) + assert render_capture_problems(text) == () + assert test_case.expected_kinds <= rendered_input_kinds(text), ( + test_case.expected_kinds - rendered_input_kinds(text) + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py index eab6d1cbbc..48f6053e2b 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py +++ b/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass +from pathlib import Path from scripts.compiler_differential.models import ExpectedOutcome @@ -14,3 +15,13 @@ class CorpusExpectationTestCase: description: str corpora: tuple[str, ...] expected_outcomes: dict[str, ExpectedOutcome] + + +@dataclass(frozen=True) +class InlineCompileCodesTestCase: + """The compile package and the codes every inline render-stage literal must belong to.""" + + description: str + compile_root: Path + accounted_codes: frozenset[str] + expected_unaccounted: dict[str, tuple[str, ...]] diff --git a/tests/unit/scripts/compiler_differential/_helpers/corpus/test_render_codes.py b/tests/unit/scripts/compiler_differential/_helpers/corpus/test_render_codes.py new file mode 100644 index 0000000000..ba18491aff --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/corpus/test_render_codes.py @@ -0,0 +1,45 @@ +"""Every inline diagnostic code in render-stage compile modules is required or documented.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import sqlbuild.compiler.compile as compile_package +from scripts.compiler_differential._helpers.corpus.emitted_codes import ( + inline_compile_codes, + render_error_codes, + unaccounted_inline_codes, +) +from scripts.compiler_differential.constants import RENDER_UNREACHABLE_CODES +from tests.unit.scripts.compiler_differential._helpers.corpus._test_types import ( + InlineCompileCodesTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + InlineCompileCodesTestCase( + description="compile_package", + compile_root=Path(compile_package.__file__).parent, + accounted_codes=render_error_codes() | frozenset(RENDER_UNREACHABLE_CODES), + expected_unaccounted={}, + ) + ], + ids=lambda case: case.description, +) +def test_given_compile_modules_when_scanning_inline_codes_then_each_is_required_or_unreachable( + test_case: InlineCompileCodesTestCase, +) -> None: + unaccounted: dict[str, tuple[str, ...]] = unaccounted_inline_codes( + compile_root=test_case.compile_root, accounted=test_case.accounted_codes + ) + + assert inline_compile_codes(test_case.compile_root) + assert unaccounted == test_case.expected_unaccounted + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py index ed46332262..a9e3ca07af 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py @@ -4,6 +4,7 @@ from dataclasses import dataclass +from sqlbuild.compiler.compile.models import CompileProjectInputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs @@ -51,3 +52,22 @@ class EncodedInputsKindsTestCase: description: str inputs: DiscoveredProjectInputs expected_kinds: frozenset[str] + + +@dataclass(frozen=True) +class RenderCaptureKindsTestCase: + """One render capture fragment and the render kinds it must and must not prove.""" + + description: str + capture: dict[str, object] + expected_present: frozenset[str] + expected_absent: frozenset[str] + + +@dataclass(frozen=True) +class RenderEncodedInputsTestCase: + """Real compile inputs, encoded by the stage capture, and the render kinds they prove.""" + + description: str + inputs: CompileProjectInputs + expected_kinds: frozenset[str] diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py index 6774d6a053..7ba7b7441c 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py @@ -3,7 +3,9 @@ from __future__ import annotations import dataclasses +from typing import cast +from sqlbuild.compiler.compile.models import CompileProjectInputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs _VALUE_KIND: str = "sqlbuild.sql_values.types:SqlValueKind" @@ -25,3 +27,76 @@ def empty_inputs_capture(**overrides: object) -> dict[str, object]: **{field.name: [] for field in dataclasses.fields(DiscoveredProjectInputs)}, **overrides, } + + +_DECLARATION_KIND: str = "sqlbuild.compiler.scopes.types:DeclarationKind" +_RESOURCE_KIND: str = "sqlbuild.compiler.scopes.types:ResourceKind" +_AUDIT_TARGET: str = "sqlbuild.compiler.compile.types:AttachedAuditTargetKind" +_REFERENCE_KIND: str = "sqlbuild.compiler.references.types:SqlReferenceKind" + + +def empty_render_capture(**overrides: object) -> dict[str, object]: + """Return a capture with every `CompileProjectInputs` field, in order, then overrides.""" + + return { + "__type__": "sqlbuild.compiler.compile.models:CompileProjectInputs", + **{field.name: [] for field in dataclasses.fields(CompileProjectInputs)}, + **overrides, + } + + +def usage(*, kind: str, name: str, enum_member: str | None = None) -> dict[str, object]: + """Return one captured direct usage of a project declaration.""" + + return { + "declaration": { + "kind": {"__enum__": f"{_DECLARATION_KIND}.{kind}"}, + "name": name, + "owner": None, + }, + "through": None, + "enum_member": enum_member, + } + + +def private_usage(*, kind: str, name: str, owner: str) -> dict[str, object]: + """Return one captured usage of a declaration private to the model `owner`.""" + + direct: dict[str, object] = usage(kind=kind, name=name) + return { + **direct, + "declaration": {**_as_object(direct["declaration"]), "owner": _model(owner)}, + } + + +def granted_usage(*, kind: str, name: str, through: str) -> dict[str, object]: + """Return one captured usage granted through the expected model `through`.""" + + return {**usage(kind=kind, name=name), "through": _model(through)} + + +def _model(name: str) -> dict[str, object]: + return {"kind": {"__enum__": f"{_RESOURCE_KIND}.MODEL"}, "name": name} + + +def _as_object(value: object) -> dict[str, object]: + return cast(dict[str, object], value) + + +def reference(*, kind: str, name: str) -> dict[str, object]: + """Return one captured SQL reference.""" + + return {"ref_kind": {"__enum__": f"{_REFERENCE_KIND}.{kind}"}, "ref_name": name} + + +def audit_input(*, definition: str, template: str, rendered: str) -> dict[str, object]: + """Return one captured generic audit attached to column `amount` of model `orders`.""" + + return { + "audit_file": {"relative_path": {"__path__": f"audits/generic/{definition}.sql"}}, + "audit_block": {"sql_body": template}, + "sql_body": rendered, + "attached_target_kind": {"__enum__": f"{_AUDIT_TARGET}.MODEL"}, + "attached_target_name": "orders", + "attached_column_name": "amount", + } diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py new file mode 100644 index 0000000000..5f84d87d99 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py @@ -0,0 +1,484 @@ +"""Render captures are classified into the input kinds the seed corpus must exercise.""" + +from __future__ import annotations + +import dataclasses +import json + +import pytest + +from scripts.compiler_differential._helpers.coverage.render import ( + project_render_kinds, + render_capture_problems, + rendered_input_kinds, + required_render_kinds, +) +from scripts.compiler_differential.constants import RENDER_NON_COLLECTION_FIELDS +from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig +from tests.unit.scripts.compiler_differential._helpers.coverage._test_types import ( + CaptureProblemsTestCase, + RenderCaptureKindsTestCase, + RenderEncodedInputsTestCase, + RequiredKindsTestCase, +) +from tests.unit.scripts.compiler_differential._helpers.coverage.helpers import ( + audit_input, + empty_render_capture, + granted_usage, + private_usage, + reference, + usage, +) + +_HOOK: str = "sqlbuild.compiler.discovery.models" +_TARGET: str = "sqlbuild.compiler.compile.types:AttachedAuditTargetKind" +_TEST_MODE: str = "sqlbuild.compiler.compile.types:SqlTestMode" +_SCOPE: str = "sqlbuild.compiler.scopes.types:ScopeKind" +_VALUE_KIND: str = "sqlbuild.sql_values.types:SqlValueKind" +_SHARED_MACRO_FILE: str = ( + "import os\n\n\n" + "def round_it(ctx, value):\n return f\"ROUND({value}) + {ctx.vars['x']}\"\n\n\n" + "def region_tag():\n return os.environ.get('REGION', 'east')\n" +) +_TYPED_MACRO_FILE: str = ( + "from sqlbuild.refs import SqlResourceRef\n\n\n" + "def union_ids(relations: list[SqlResourceRef]) -> str:\n" + ' return " UNION ALL ".join(f"SELECT id FROM {item}" for item in relations)\n' +) + + +@pytest.mark.parametrize( + "test_case", + [ + RenderCaptureKindsTestCase( + description="macro_calls_rendered_in_a_model", + capture=empty_render_capture( + model_inputs=[ + { + "macro_source_sql": ( + 'SELECT @round_it(@double("amount")) AS amount\n' + 'FROM @union_ids([__ref("orders"), __ref("returns")])\n' + "JOIN @legacy_customers() c ON TRUE" + ), + "query_sql": ( + "SELECT ROUND(amount * 2) AS amount\n" + 'FROM (SELECT id FROM __ref("orders") UNION ALL ' + 'SELECT id FROM __ref("returns"))\n' + 'JOIN __ref("customers") c ON TRUE' + ), + "references": [ + reference(kind="REF", name="orders"), + reference(kind="REF", name="returns"), + reference(kind="REF", name="customers"), + ], + "macro_usages": [ + usage(kind="MACRO", name="round_it"), + usage(kind="MACRO", name="union_ids"), + ], + } + ], + loaded_macros={ + "round_it": { + "name": "round_it", + "raw_source": _SHARED_MACRO_FILE, + "dependencies": [{"kind": "macro", "name": "base"}], + }, + "region_tag": { + "name": "region_tag", + "raw_source": _SHARED_MACRO_FILE, + "dependencies": [{"kind": "macro", "name": "base"}], + }, + "union_ids": { + "name": "union_ids", + "raw_source": _TYPED_MACRO_FILE, + "dependencies": [], + }, + }, + ), + expected_present=frozenset( + { + "model_inputs", + "loaded_macros", + "macro_in_model", + "nested_macro_call", + "typed_reference_argument", + "macro_generated_reference", + "cross_file_macro_import", + "macro_reads_vars", + "reference_ref", + } + ), + expected_absent=frozenset({"macro_reads_environment", "macro_in_test"}), + ), + RenderCaptureKindsTestCase( + description="unused_or_untyped_macros_prove_nothing", + capture=empty_render_capture( + model_inputs=[ + { + "macro_source_sql": 'SELECT * FROM @round_it(__ref("orders"))', + "query_sql": 'SELECT * FROM ROUND(__ref("orders"))', + "references": [reference(kind="REF", name="orders")], + } + ], + loaded_macros={ + "round_it": { + "name": "round_it", + "raw_source": _SHARED_MACRO_FILE, + "dependencies": [{"kind": "macro", "name": "base"}], + }, + }, + ), + expected_present=frozenset({"model_inputs", "reference_ref"}), + expected_absent=frozenset( + { + "typed_reference_argument", + "cross_file_macro_import", + "macro_reads_vars", + "macro_reads_environment", + } + ), + ), + RenderCaptureKindsTestCase( + description="unexpanded_macro_text_proves_nothing", + capture=empty_render_capture( + model_inputs=[ + { + "macro_source_sql": 'SELECT @round_it(@double("amount")) FROM t', + "query_sql": 'SELECT @round_it(@double("amount")) FROM t', + } + ] + ), + expected_present=frozenset({"model_inputs"}), + expected_absent=frozenset( + {"nested_macro_call", "typed_reference_argument", "macro_generated_reference"} + ), + ), + RenderCaptureKindsTestCase( + description="interpolated_templates", + capture=empty_render_capture( + model_inputs=[ + { + "model_file": { + "query_sql": "SELECT '@@site' AS site, '@@ENV:CHANNEL' AS channel" + }, + "query_sql": ( + "SELECT 'north' AS site, 'web' AS channel\n" + "WHERE d >= __cursor_start() AND d >= @@@window_start" + ), + "config": { + "values": { + "materialized": "incremental", + "cursor": "d", + "placeholders": {"window_start": "'2026-01-01'"}, + } + }, + "schema_entry": { + "name": "orders", + "audits": [], + "columns": [ + { + "name": "amount", + "audits": [ + {"definition_name": "floor", "arguments": {"floor": -17}} + ], + } + ], + }, + } + ], + source_inputs=[ + {"source_entry": {"name": "orders", "expression": "(SELECT 0.5 AS rate)"}} + ], + audit_inputs=[ + audit_input( + definition="floor", + template="SELECT * FROM t WHERE @column < @floor", + rendered="SELECT * FROM t WHERE amount < -17", + ) + ], + discovered_inputs={ + "source_files": [ + { + "source_entries": [ + {"name": "orders", "expression": "(SELECT @@rate AS rate)"} + ] + } + ] + }, + ), + expected_present=frozenset( + { + "project_variable", + "environment_variable", + "cursor_intrinsic", + "runtime_placeholder", + "source_expression_rendered", + "audit_arguments", + "model_column_audit", + } + ), + expected_absent=frozenset({"model_audit", "singular_audit"}), + ), + RenderCaptureKindsTestCase( + description="surviving_tokens_without_config_and_built_in_audit_tokens_prove_nothing", + capture=empty_render_capture( + model_inputs=[ + { + "query_sql": "SELECT __cursor_start() AS s, @@@window_start AS w", + "config": {"values": {"materialized": "view"}}, + "schema_entry": { + "name": "orders", + "audits": [], + "columns": [ + { + "name": "amount", + "audits": [{"definition_name": "not_null", "arguments": {}}], + } + ], + }, + } + ], + audit_inputs=[ + audit_input( + definition="not_null", + template="SELECT @column FROM @relation WHERE @column IS NULL", + rendered='SELECT amount FROM __ref("orders") WHERE amount IS NULL', + ) + ], + ), + expected_present=frozenset({"model_column_audit"}), + expected_absent=frozenset( + {"cursor_intrinsic", "runtime_placeholder", "audit_arguments"} + ), + ), + RenderCaptureKindsTestCase( + description="unsubstituted_variables_prove_nothing", + capture=empty_render_capture( + model_inputs=[ + { + "model_file": {"query_sql": "SELECT '@@site', '@@ENV:CHANNEL'"}, + "query_sql": "SELECT '@@site', '@@ENV:CHANNEL'", + } + ] + ), + expected_present=frozenset({"model_inputs"}), + expected_absent=frozenset({"project_variable", "environment_variable"}), + ), + RenderCaptureKindsTestCase( + description="hooks_rendered_from_their_authored_entries", + capture=empty_render_capture( + model_inputs=[ + { + "model_file": { + "header_values": { + "post_hooks": [ + { + "__type__": f"{_HOOK}:SqlHookEntry", + "statement": "SELECT @note('@@CTX:destination.qualified')", + }, + { + "__type__": f"{_HOOK}:NamedSqlHookEntry", + "kwargs": {"label": "orders"}, + }, + {"__type__": f"{_HOOK}:PythonHookEntry", "name": "log"}, + ] + } + }, + "config": { + "values": { + "post_hooks": [ + { + "__type__": f"{_HOOK}:SqlHookEntry", + "statement": "SELECT 'refreshed main.orders'", + "definition_sql": None, + }, + { + "__type__": f"{_HOOK}:SqlHookEntry", + "statement": "SELECT 'orders' AS label", + "definition_sql": "SELECT @'label' AS label", + "kwargs": {"label": "orders"}, + }, + {"__type__": f"{_HOOK}:PythonHookEntry", "name": "log"}, + ] + }, + "matched_path_default": "marts", + }, + } + ] + ), + expected_present=frozenset( + { + "inline_sql_hook", + "named_sql_hook", + "python_hook", + "hook_context_variable", + "macro_in_hook", + "named_hook_arguments", + "path_default", + } + ), + expected_absent=frozenset(), + ), + RenderCaptureKindsTestCase( + description="declarations_used_by_resources", + capture=empty_render_capture( + model_inputs=[ + { + "declaration_usages": [ + usage(kind="ENUM", name="status", enum_member="OPEN"), + usage(kind="CONSTANT", name="weights"), + private_usage(kind="CONSTANT", name="_floor", owner="orders"), + ], + "enum_columns": {"status": {"name": "status"}}, + } + ], + test_inputs=[ + { + "mode": {"__enum__": f"{_TEST_MODE}.MACRO"}, + "declaration_usages": [ + granted_usage(kind="MACRO", name="double", through="orders") + ], + "case_name": "small", + } + ], + public_constants={ + "weights": { + "name": "weights", + "value": {"logical_type": {"kind": {"__enum__": f"{_VALUE_KIND}.LIST"}}}, + } + }, + scope_index={ + "declarations": [ + { + "identity": usage(kind="ENUM", name="status")["declaration"], + "scope": {"__enum__": f"{_SCOPE}.INHERITED"}, + } + ] + }, + ), + expected_present=frozenset( + { + "enum_member", + "list_constant", + "scope_private_use", + "scope_inherited_use", + "enum_column_contract", + "macro_in_test", + "tested_macro", + "macro_test", + "expected_model_grant", + "parameterized_test_case", + } + ), + expected_absent=frozenset({"scope_local_use", "scope_global_use"}), + ), + RenderCaptureKindsTestCase( + description="encoded_empty_collections", + capture=empty_render_capture( + loaded_macros={}, + public_enums={"__mapping__": []}, + model_inputs=[{"enum_columns": {"__mapping__": []}}], + ), + expected_present=frozenset({"model_inputs"}), + expected_absent=frozenset({"loaded_macros", "public_enums", "enum_column_contract"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_render_capture_when_classifying_then_only_rendered_inputs_are_credited( + test_case: RenderCaptureKindsTestCase, +) -> None: + kinds: frozenset[str] = rendered_input_kinds(json.dumps(test_case.capture)) + + assert test_case.expected_present <= kinds, test_case.expected_present - kinds + assert not test_case.expected_absent & kinds, test_case.expected_absent & kinds + + +@pytest.mark.parametrize( + "test_case", + [ + RenderEncodedInputsTestCase( + description="empty_project", + inputs=CompileProjectInputs( + project_config=ProjectConfig(name="orders", adapter="duckdb"), + local_config=LocalConfig(), + discovered_inputs=DiscoveredProjectInputs( + project_config=ProjectConfig(name="orders", adapter="duckdb"), + local_config=LocalConfig(), + ), + sql_lexical_syntax=SqlLexicalSyntax(), + ), + expected_kinds=frozenset(), + ) + ], + ids=lambda case: case.description, +) +def test_given_encoded_compile_inputs_when_classifying_then_encoded_empty_values_prove_nothing( + test_case: RenderEncodedInputsTestCase, +) -> None: + capture: str = render_stage_capture(test_case.inputs) + + assert rendered_input_kinds(capture) == test_case.expected_kinds + assert project_render_kinds({"0-compile": {"002-compile_project_inputs.json": capture}}) == ( + test_case.expected_kinds + ) + + +@pytest.mark.parametrize( + "test_case", + [ + RequiredKindsTestCase( + description="every_rendered_collection", + expected_required=frozenset( + field.name for field in dataclasses.fields(CompileProjectInputs) + ) + - RENDER_NON_COLLECTION_FIELDS, + ) + ], + ids=lambda case: case.description, +) +def test_given_compile_project_inputs_when_listing_required_kinds_then_collections_are_required( + test_case: RequiredKindsTestCase, +) -> None: + assert test_case.expected_required <= set(required_render_kinds()) + + +@pytest.mark.parametrize( + "test_case", + [ + CaptureProblemsTestCase( + description="sound_capture", + capture=empty_render_capture( + loaded_macros={"double": {"function": {"__callable__": "macros.math:double"}}} + ), + expected_problem_count=0, + ), + CaptureProblemsTestCase( + description="opaque_value_and_unnamed_callable", + capture=empty_render_capture( + loaded_macros={"double": {"function": {"__opaque__": "macros.math:Doubler"}}} + ), + expected_problem_count=2, + ), + CaptureProblemsTestCase( + description="missing_field", + capture={"__type__": "CompileProjectInputs", "model_inputs": []}, + expected_problem_count=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_render_capture_when_checking_then_incomplete_or_opaque_parts_are_reported( + test_case: CaptureProblemsTestCase, +) -> None: + problems: tuple[str, ...] = render_capture_problems(json.dumps(test_case.capture)) + + assert len(problems) == test_case.expected_problem_count, problems + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py index 0249e11836..87744bb7f6 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py +++ b/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass from scripts.compiler_differential.models import ExpectedOutcome, ProjectComparison @@ -48,6 +49,7 @@ class CoverageReportTestCase: """Covered kinds and the coverage report lines the harness must print.""" description: str + formatter: Callable[..., str] covered: frozenset[str] required: tuple[str, ...] expected_lines: tuple[str, ...] @@ -59,6 +61,6 @@ class SummaryTestCase: description: str comparisons: list[ProjectComparison] - missing_coverage: tuple[str, ...] + missing_coverage: dict[str, tuple[str, ...]] expected_lines: tuple[str, ...] expected_absent: tuple[str, ...] diff --git a/tests/unit/scripts/compiler_differential/_helpers/running/test_report.py b/tests/unit/scripts/compiler_differential/_helpers/running/test_report.py index 1662c3efb9..16dc259015 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/running/test_report.py +++ b/tests/unit/scripts/compiler_differential/_helpers/running/test_report.py @@ -6,6 +6,7 @@ from scripts.compiler_differential._helpers.running.report import ( format_discovery_coverage, + format_render_coverage, format_summary, ) from scripts.compiler_differential.models import Difference, ProjectComparison @@ -31,6 +32,7 @@ [ CoverageReportTestCase( description="complete", + formatter=format_discovery_coverage, covered=frozenset({"model_files", "crlf"}), required=("model_files", "crlf"), expected_lines=( @@ -40,6 +42,7 @@ ), CoverageReportTestCase( description="missing_kinds_are_named_in_required_order", + formatter=format_discovery_coverage, covered=frozenset({"model_files"}), required=("providers", "model_files", "crlf"), expected_lines=( @@ -47,13 +50,26 @@ " missing: providers, crlf", ), ), + CoverageReportTestCase( + description="render_names_indirect_and_missing_kinds", + formatter=format_render_coverage, + covered=frozenset({"model_inputs", "macro_reads_vars"}), + required=("model_inputs", "macro_reads_vars", "typed_reference_argument"), + expected_lines=( + "Render coverage: 2 of 3 input kinds exercised by the seed corpus", + " config-only kinds: none", + "indirect kinds: macro_reads_vars, macro_reads_constants, macro_reads_enums,", + "(credited because the used macro's own function source reads it", + " missing: typed_reference_argument", + ), + ), ], ids=lambda case: case.description, ) def test_given_covered_kinds_when_reporting_then_missing_kinds_are_named( test_case: CoverageReportTestCase, ) -> None: - report: str = format_discovery_coverage(covered=test_case.covered, required=test_case.required) + report: str = test_case.formatter(covered=test_case.covered, required=test_case.required) assert all(line in report for line in test_case.expected_lines), report @@ -64,24 +80,25 @@ def test_given_covered_kinds_when_reporting_then_missing_kinds_are_named( SummaryTestCase( description="identical_and_covered", comparisons=[_IDENTICAL], - missing_coverage=(), + missing_coverage={"discovery": (), "render": ()}, expected_lines=("Compiler differential passed: 1 projects identical",), expected_absent=("FAILED",), ), SummaryTestCase( description="identical_but_coverage_missing", comparisons=[_IDENTICAL], - missing_coverage=("providers", "crlf"), + missing_coverage={"discovery": ("providers", "crlf"), "render": ("path_default",)}, expected_lines=( "Compiler differential FAILED: 0 of 1 projects differ", "Required discovery coverage missing: providers, crlf", + "Required render coverage missing: path_default", ), expected_absent=("passed",), ), SummaryTestCase( description="differences_and_coverage_missing", comparisons=[_IDENTICAL, _DIFFERING], - missing_coverage=("providers",), + missing_coverage={"discovery": ("providers",)}, expected_lines=( "Compiler differential FAILED: 1 of 2 projects differ", "First difference: seed/1: `compile` stdout at /models/0", diff --git a/tests/unit/scripts/compiler_differential/main/_test_types.py b/tests/unit/scripts/compiler_differential/main/_test_types.py index 9dedbb9673..66189fc444 100644 --- a/tests/unit/scripts/compiler_differential/main/_test_types.py +++ b/tests/unit/scripts/compiler_differential/main/_test_types.py @@ -66,3 +66,12 @@ class RareFeatureSeedTestCase: expected_error_code: str expected_succeeding_commands: tuple[str, ...] other_seeds: tuple[int, ...] + + +@dataclass(frozen=True) +class CoverageFlagUsageTestCase: + """A coverage requirement given without what it needs, and the usage error it reports.""" + + description: str + arguments: tuple[str, ...] + expected_message: str diff --git a/tests/unit/scripts/compiler_differential/main/test_differential_usage.py b/tests/unit/scripts/compiler_differential/main/test_differential_usage.py new file mode 100644 index 0000000000..d88235215d --- /dev/null +++ b/tests/unit/scripts/compiler_differential/main/test_differential_usage.py @@ -0,0 +1,47 @@ +"""Coverage requirements are rejected unless the run captures stages for the seed corpus.""" + +from __future__ import annotations + +import pytest + +from scripts.compiler_differential.main.differential import run_compiler_differential +from tests.unit.scripts.compiler_differential.main._test_types import CoverageFlagUsageTestCase + +_USAGE_ERROR_EXIT_CODE: int = 2 + + +@pytest.mark.parametrize( + "test_case", + [ + CoverageFlagUsageTestCase( + description="render_coverage_without_captures", + arguments=("--corpus", "seeds", "--require-render-coverage"), + expected_message="--require-render-coverage needs --stage-captures and the seeds corpus", + ), + CoverageFlagUsageTestCase( + description="render_coverage_without_seeds", + arguments=("--corpus", "fixtures", "--stage-captures", "--require-render-coverage"), + expected_message="--require-render-coverage needs --stage-captures and the seeds corpus", + ), + CoverageFlagUsageTestCase( + description="discovery_coverage_without_captures", + arguments=("--corpus", "seeds", "--require-discovery-coverage"), + expected_message=( + "--require-discovery-coverage needs --stage-captures and the seeds corpus" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_coverage_requirement_without_captured_seeds_when_parsing_then_usage_error( + test_case: CoverageFlagUsageTestCase, capsys: pytest.CaptureFixture[str] +) -> None: + with pytest.raises(SystemExit) as raised: + _ = run_compiler_differential(list(test_case.arguments)) + + assert raised.value.code == _USAGE_ERROR_EXIT_CODE + assert test_case.expected_message in capsys.readouterr().err + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"])