diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index bad18bf43d..d007476831 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -291,7 +291,9 @@ jobs: - name: Install dependencies run: uv sync --locked --all-groups --all-extras - name: Compare fixtures, examples, generated seeds and the failure corpus - run: make compiler-differential COMPILER_DIFFERENTIAL_ARGS=--stage-captures + run: >- + make compiler-differential + COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage" format_performance: name: Formatter performance guards diff --git a/scripts/compiler_differential/_helpers/comparing/__init__.py b/scripts/compiler_differential/_helpers/comparing/__init__.py new file mode 100644 index 0000000000..2e317c6ec3 --- /dev/null +++ b/scripts/compiler_differential/_helpers/comparing/__init__.py @@ -0,0 +1 @@ +"""Compare what two engines produced and locate the first differences.""" diff --git a/scripts/compiler_differential/_helpers/compare.py b/scripts/compiler_differential/_helpers/comparing/compare.py similarity index 100% rename from scripts/compiler_differential/_helpers/compare.py rename to scripts/compiler_differential/_helpers/comparing/compare.py diff --git a/scripts/compiler_differential/_helpers/comparison.py b/scripts/compiler_differential/_helpers/comparing/comparison.py similarity index 95% rename from scripts/compiler_differential/_helpers/comparison.py rename to scripts/compiler_differential/_helpers/comparing/comparison.py index cfd54bfbae..71c3ae0851 100644 --- a/scripts/compiler_differential/_helpers/comparison.py +++ b/scripts/compiler_differential/_helpers/comparing/comparison.py @@ -5,20 +5,19 @@ import json from collections.abc import Callable -from scripts.compiler_differential._helpers.compare import ( +from scripts.compiler_differential._helpers.comparing.compare import ( as_json_object, first_document_difference, first_json_difference, first_text_difference, ) -from scripts.compiler_differential._helpers.normalize import ( +from scripts.compiler_differential._helpers.comparing.normalize import ( normalize_artifact_text, normalize_stderr, strip_manifest_metadata, strip_report_fields, ) from scripts.compiler_differential.constants import ( - ERROR_SEVERITY, MISSING_VALUE, MODEL_RESOURCE_TYPE, PLAN_LABEL, @@ -70,17 +69,15 @@ def compare_engine_runs(*, project: str, left: EngineRun, right: EngineRun) -> l ] -def diagnostic_codes(*, outcome: CommandOutcome, errors_only: bool) -> tuple[str, ...]: - """Return the diagnostic codes of a JSON compile report, in report order.""" +def diagnostic_codes(*, outcome: CommandOutcome, severity: str | None) -> tuple[str, ...]: + """Return a JSON compile report's diagnostic codes in order, optionally of one severity.""" payload: dict[str, object] = as_json_object(_loads(outcome.stdout)) or {} diagnostics: object = payload.get("diagnostics") found: list[str] = [] for raw_diagnostic in diagnostics if isinstance(diagnostics, list) else (): diagnostic: dict[str, object] | None = as_json_object(raw_diagnostic) - if diagnostic is not None and ( - not errors_only or diagnostic.get("severity") == ERROR_SEVERITY - ): + if diagnostic is not None and (severity is None or diagnostic.get("severity") == severity): found.append(str(diagnostic.get("code"))) return tuple(found) diff --git a/scripts/compiler_differential/_helpers/normalize.py b/scripts/compiler_differential/_helpers/comparing/normalize.py similarity index 95% rename from scripts/compiler_differential/_helpers/normalize.py rename to scripts/compiler_differential/_helpers/comparing/normalize.py index a81a9f8987..728ed1c84f 100644 --- a/scripts/compiler_differential/_helpers/normalize.py +++ b/scripts/compiler_differential/_helpers/comparing/normalize.py @@ -2,7 +2,7 @@ from __future__ import annotations -from scripts.compiler_differential._helpers.compare import as_json_object +from scripts.compiler_differential._helpers.comparing.compare import as_json_object from scripts.compiler_differential.constants import ( ELAPSED_TIME_MASK, ELAPSED_TIME_PATTERN, diff --git a/scripts/compiler_differential/_helpers/project_comparison.py b/scripts/compiler_differential/_helpers/comparing/project_comparison.py similarity index 61% rename from scripts/compiler_differential/_helpers/project_comparison.py rename to scripts/compiler_differential/_helpers/comparing/project_comparison.py index 3741cdb565..308d7b7791 100644 --- a/scripts/compiler_differential/_helpers/project_comparison.py +++ b/scripts/compiler_differential/_helpers/comparing/project_comparison.py @@ -6,8 +6,13 @@ import time from pathlib import Path -from scripts.compiler_differential._helpers.comparison import compare_engine_runs, diagnostic_codes -from scripts.compiler_differential._helpers.execution import run_engine +from scripts.compiler_differential._helpers.comparing.comparison import ( + compare_engine_runs, + diagnostic_codes, +) +from scripts.compiler_differential._helpers.coverage.discovery import project_discovery_kinds +from scripts.compiler_differential._helpers.running.execution import run_engine +from scripts.compiler_differential.constants import ERROR_SEVERITY, WARNING_SEVERITY from scripts.compiler_differential.models import ( CommandOutcome, CorpusProject, @@ -54,6 +59,11 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) -> project=project.name, differences=tuple(differences), seconds=time.monotonic() - started, + discovered_kinds=( + project_discovery_kinds(captures=left.captures, source_dir=source_dir) + if project.discovery_coverage and options.stage_captures + else frozenset() + ), ) @@ -68,23 +78,21 @@ def _materialized_source(*, project: CorpusProject, case_dir: Path) -> Path: def _corpus_differences(*, project: CorpusProject, run: EngineRun) -> list[Difference]: first: CommandOutcome = run.outcomes[0] - errors: tuple[str, ...] = diagnostic_codes(outcome=first, errors_only=True) - expected_code: str | None = project.expected.error_code - if expected_code is not None: - codes: tuple[str, ...] = diagnostic_codes(outcome=first, errors_only=False) - met: bool = first.exit_code != 0 and bool(errors) and expected_code in codes - return ( - [] - if met - else [ + errors: tuple[str, ...] = diagnostic_codes(outcome=first, severity=ERROR_SEVERITY) + if project.expected.error_code is not None: + return [ + *_failure_differences(project=project, outcome=first, errors=errors), + *( _expectation_difference( project=project, - location="expected failure", - expected=f"non-zero exit reporting {expected_code}", - actual=f"exit {first.exit_code}: {', '.join(codes) or ''}", + location=f"`{outcome.label}` expected success", + expected="exit 0", + actual=f"exit {outcome.exit_code}", ) - ] - ) + for outcome in run.outcomes + if outcome.label in project.expected.succeeding_commands and outcome.exit_code != 0 + ), + ] problems: list[str] = [ f"`{outcome.label}` exit {outcome.exit_code}" for outcome in run.outcomes @@ -104,6 +112,34 @@ def _corpus_differences(*, project: CorpusProject, run: EngineRun) -> list[Diffe ] +def _failure_differences( + *, project: CorpusProject, outcome: CommandOutcome, errors: tuple[str, ...] +) -> list[Difference]: + expected_code: str | None = project.expected.error_code + differences: list[Difference] = [] + if outcome.exit_code == 0 or not errors or errors[0] != expected_code: + differences.append( + _expectation_difference( + project=project, + location="expected failure", + expected=f"non-zero exit whose first error is {expected_code}", + actual=f"exit {outcome.exit_code}: errors {', '.join(errors) or ''}", + ) + ) + warning_code: str | None = project.expected.warning_code + warnings: tuple[str, ...] = diagnostic_codes(outcome=outcome, severity=WARNING_SEVERITY) + if warning_code is not None and warning_code not in warnings: + differences.append( + _expectation_difference( + project=project, + location="expected warning", + expected=f"warning {warning_code}", + actual=f"warnings {', '.join(warnings) or ''}", + ) + ) + return differences + + def _timeout_differences( *, project: CorpusProject, runs: tuple[EngineRun, ...] ) -> list[Difference]: diff --git a/scripts/compiler_differential/_helpers/corpus/__init__.py b/scripts/compiler_differential/_helpers/corpus/__init__.py new file mode 100644 index 0000000000..0ccdb4d17d --- /dev/null +++ b/scripts/compiler_differential/_helpers/corpus/__init__.py @@ -0,0 +1 @@ +"""Assemble the corpus projects, including the failure corpus.""" diff --git a/scripts/compiler_differential/_helpers/corpus.py b/scripts/compiler_differential/_helpers/corpus/corpus.py similarity index 90% rename from scripts/compiler_differential/_helpers/corpus.py rename to scripts/compiler_differential/_helpers/corpus/corpus.py index 531a0e554f..d1d51c4127 100644 --- a/scripts/compiler_differential/_helpers/corpus.py +++ b/scripts/compiler_differential/_helpers/corpus/corpus.py @@ -5,7 +5,7 @@ import tomllib from pathlib import Path -from scripts.compiler_differential._helpers.failure_cases import all_failure_cases +from scripts.compiler_differential._helpers.corpus.failure_cases import all_failure_cases from scripts.compiler_differential.classes.dense_project import DenseProject from scripts.compiler_differential.classes.project_builder import ProjectBuilder from scripts.compiler_differential.constants import ( @@ -80,7 +80,9 @@ def build_corpus( CorpusProject( name=f"failure/{case.name}", commands=(COLD_COMPILE,), - expected=ExpectedOutcome(error_code=case.expected_code), + expected=ExpectedOutcome( + error_code=case.expected_code, warning_code=case.expected_warning_code + ), writer=case.write, ) for case in all_failure_cases() @@ -136,6 +138,10 @@ def _seed_project(seed: int) -> CorpusProject: return CorpusProject( name=f"seed/{seed}", commands=(COLD_COMPILE, PLAN), - expected=ExpectedOutcome(error_code=generated.expected_error_code), + expected=ExpectedOutcome( + error_code=generated.expected_error_code, + succeeding_commands=generated.succeeding_commands, + ), writer=generated.write, + discovery_coverage=True, ) diff --git a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py new file mode 100644 index 0000000000..c2cf26daa0 --- /dev/null +++ b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py @@ -0,0 +1,393 @@ +"""Minimal projects that fail inside project discovery, one per diagnostic family.""" + +from scripts.compiler_differential.constants import ( + FAILURE_BASE_CONFIG, + FAILURE_BASE_FILES, + FAILURE_BASE_MART, + FAILURE_BASE_STAGING, + FAILURE_CONFIG_PATH, + FAILURE_MART_PATH, + FAILURE_SOURCES_PATH, + FAILURE_STAGING_PATH, + GENERATOR_BOM, +) +from scripts.compiler_differential.models import FailureCase + +_TASK_MODULE: str = ( + "from sqlbuild.tasks import task\n\n\n" + "@task\ndef export_orders(ctx):\n" + ' """Export orders."""\n' + " return None\n" +) +_HOOK_MODULE: str = ( + "from sqlbuild.hooks import hook\n\n\n" + "@hook\ndef note_refresh(ctx):\n" + ' """Log one refresh."""\n' + " return None\n" +) +_SEED_DECLARATION: str = ( + "seeds:\n - name: order_channels\n description: Order channels.\n" + " columns:\n - name: id\n type: INTEGER\n" + " - name: label\n type: VARCHAR\n" +) + + +def _case(*, name: str, expected_code: str, files: dict[str, str]) -> FailureCase: + return FailureCase( + files={**FAILURE_BASE_FILES, **files}, name=name, expected_code=expected_code + ) + + +def _config(extra: str) -> dict[str, str]: + return {FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + extra} + + +def discovery_failure_cases() -> tuple[FailureCase, ...]: + """Return the discovery-time failure cases in a stable order.""" + + return ( + *_config_cases(), + *_sql_file_cases(), + *_yaml_file_cases(), + *_conflict_cases(), + *_python_cases(), + ) + + +def _config_cases() -> tuple[FailureCase, ...]: + return ( + _case( + name="config-bom", + expected_code="D001", + files={FAILURE_CONFIG_PATH: GENERATOR_BOM + FAILURE_BASE_CONFIG}, + ), + _case( + name="local-config-unknown-key", + expected_code="D001", + files={"sqlbuild_local.toml": "[settings]\nsql_analysiss = false\n"}, + ), + _case( + name="target-unknown-key", + expected_code="D001", + files=_config('\n[targets.ci]\nschema = "ci"\nwarehouse_size = "large"\n'), + ), + _case( + name="dbt-unknown-key", + expected_code="D001", + files=_config('\n[dbt]\nproject_dir = "dbt"\nmanifest = "dbt/manifest.json"\n'), + ), + _case( + name="path-defaults-unmatched", + expected_code="D007", + files=_config('\n[path_defaults."reporting/finance"]\nmaterialized = "table"\n'), + ), + ) + + +def _sql_file_cases() -> tuple[FailureCase, ...]: + return ( + _case( + name="model-bom", + expected_code="D002", + files={FAILURE_STAGING_PATH: GENERATOR_BOM + FAILURE_BASE_STAGING}, + ), + _case( + name="model-missing-header", + expected_code="D002", + files={FAILURE_STAGING_PATH: 'SELECT order_id\nFROM __source("raw_orders")\n'}, + ), + _case( + name="model-unknown-header-key", + expected_code="D002", + files={ + FAILURE_STAGING_PATH: FAILURE_BASE_STAGING.replace( + '"Staged orders",', '"Staged orders",\n materialised table,' + ) + }, + ), + _case( + name="model-reserved-name", + expected_code="D016", + files={"models/marts/_chain_.sql": FAILURE_BASE_MART}, + ), + _case( + name="test-case-missing-parameter", + expected_code="D003", + files={ + "tests/unit/test_order_cases.sql": ( + 'TEST (\n name "keeps_status",\n' + " parameters (\n status_value string,\n amount_value float,\n ),\n" + ' cases (\n placed (status_value "placed"),\n ),\n);\n\n' + "WITH\n__source__raw_orders AS (\n" + ' SELECT 1 AS order_id, 10 AS customer_id, @param("amount_value") AS amount,' + ' @param("status_value") AS status\n),\n' + "__expected__stg_orders AS (\n SELECT 1 AS order_id\n)\nSELECT 1\n" + ) + }, + ), + _case( + name="generic-audit-header", + expected_code="D004", + files={ + "models/marts/_sqlbuild/_audits/generic/at_least.sql": ( + "AUDIT (unknown_option 1);\n\nSELECT * FROM @relation WHERE @column < 0\n" + ) + }, + ), + _case( + name="list-constant-mixed-types", + expected_code="D013", + files={ + "models/staging/_sqlbuild/_constants/statuses.sql": ( + 'CONSTANT (name kept_statuses, value ["placed", 3]);\n' + ) + }, + ), + _case( + name="enum-duplicate-member", + expected_code="D013", + files={ + "models/staging/_sqlbuild/_enums/order_status.sql": ( + "ENUM (\n name order_status,\n members [PLACED, PLACED],\n);\n" + ) + }, + ), + _case( + name="sql-hook-unknown-key", + expected_code="D014", + files={ + "hooks/sql/record_refresh.sql": ( + 'HOOK (\n description "Record a refresh",\n phase post,\n);\n\nSELECT 1\n' + ) + }, + ), + _case( + name="sql-function-header", + expected_code="D002", + files={ + "functions/sql/is_placed.sql": ( + 'FUNCTION (\n description "Whether placed",\n returns,\n);\n\n' + "order_status = 'placed'\n" + ) + }, + ), + _case( + name="column-nullable-not-null", + expected_code="P002", + files={ + FAILURE_STAGING_PATH: FAILURE_BASE_STAGING.replace( + '"Staged orders",', + '"Staged orders",\n columns (\n' + " order_id (nullable true, audits [not_null]),\n ),", + ) + }, + ), + ) + + +def _yaml_file_cases() -> tuple[FailureCase, ...]: + return ( + _case( + name="schema-yml-model-metadata", + expected_code="D005", + files={ + "models/marts/schema.yml": ( + "models:\n - name: customer_totals\n description: Totals.\n" + ) + }, + ), + _case( + name="seed-yaml-extension", + expected_code="D005", + files={ + "seeds/order_channels.yaml": _SEED_DECLARATION, + "seeds/order_channels.csv": "id,label\n1,web\n", + }, + ), + _case( + name="source-unknown-key", + expected_code="D006", + files={ + FAILURE_SOURCES_PATH: FAILURE_BASE_FILES[FAILURE_SOURCES_PATH].replace( + " description: Orders feed.\n", + " description: Orders feed.\n freshnes: 1d\n", + ) + }, + ), + _case( + name="seed-declaration-without-csv", + expected_code="D008", + files={"seeds/order_channels.yml": _SEED_DECLARATION}, + ), + _case( + name="seed-header-mismatch", + expected_code="D008", + files={ + "seeds/order_channels.yml": _SEED_DECLARATION, + "seeds/order_channels.csv": "id,name\n1,web\n", + }, + ), + ) + + +def _conflict_cases() -> tuple[FailureCase, ...]: + return ( + _case( + name="duplicate-source", + expected_code="D007", + files={"sources/more.yml": FAILURE_BASE_FILES[FAILURE_SOURCES_PATH]}, + ), + _case( + name="duplicate-scenario", + expected_code="D007", + files={ + f"tests/scenarios/{folder}/orders.sql": ( + 'SCENARIO (description "Orders world");\n\n' + "WITH\n__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount," + " 'placed' AS status\n),\n" + "__expected__stg_orders AS (\n SELECT 1 AS order_id\n)\nSELECT 1\n" + ) + for folder in ("north", "south") + }, + ), + _case( + name="resource-name-collision", + expected_code="D007", + files={ + "seeds/stg_orders.yml": _SEED_DECLARATION.replace("order_channels", "stg_orders"), + "seeds/stg_orders.csv": "id,label\n1,web\n", + }, + ), + _case( + name="managed-source-without-loader", + expected_code="D007", + files={ + FAILURE_SOURCES_PATH: FAILURE_BASE_FILES[FAILURE_SOURCES_PATH] + + " - name: loaded_products\n description: Loaded products.\n" + " managed: true\n write_strategy: table\n" + " columns:\n - name: id\n type: INTEGER\n" + }, + ), + _case( + name="duplicate-python-node", + expected_code="D007", + files={ + "python/tasks/first.py": _TASK_MODULE, + "python/tasks/second.py": _TASK_MODULE, + }, + ), + _case( + name="check-depends-on-check", + expected_code="D007", + files={ + "python/checks/orders.py": ( + "from sqlbuild.checks import check\nfrom sqlbuild.refs import model\n\n\n" + '@check(depends_on=model("stg_orders"))\ndef orders_present(ctx):\n' + ' """Orders present."""\n return ctx.pass_()\n\n\n' + "@check(depends_on=orders_present)\ndef orders_checked(ctx):\n" + ' """Orders checked."""\n return ctx.pass_()\n' + ) + }, + ), + ) + + +def _python_cases() -> tuple[FailureCase, ...]: + return ( + _case( + name="python-node-import-error", + expected_code="D011", + files={"python/tasks/exports.py": "import missing_orders_client\n" + _TASK_MODULE}, + ), + _case( + name="duplicate-hook", + expected_code="D011", + files={ + "hooks/python/first.py": _HOOK_MODULE, + "hooks/python/second.py": _HOOK_MODULE, + }, + ), + _case( + name="provider-invalid-settings", + expected_code="D012", + files={ + "providers/export_settings.py": ( + "from sqlbuild.providers import Provider\n\n\n" + "class ExportSettings(Provider):\n" + ' """Export settings."""\n\n' + " api_token: str\n" + ) + }, + ), + _case( + name="duplicate-provider", + expected_code="D012", + files={ + f"providers/{module}.py": ( + "from sqlbuild.providers import Provider\n\n\n" + "class ExportSettings(Provider):\n" + ' """Export settings."""\n\n' + ' region: str = "north"\n' + ) + for module in ("first", "second") + }, + ), + _case( + name="legacy-event-exporters-directory", + expected_code="D015", + files={"event_exporters/publish.py": "VALUE = 1\n"}, + ), + _case( + name="lifecycle-sink-default-parameter", + expected_code="D015", + files={ + "sinks/lifecycle.py": ( + "from sqlbuild.sinks import LifecycleEvent, lifecycle_event_sink\n\n\n" + "@lifecycle_event_sink()\n" + "def ignore_events(event: LifecycleEvent, limit=3) -> None:\n" + " return None\n" + ) + }, + ), + _case( + name="lifecycle-sink-unknown-provider", + expected_code="D015", + files={ + "sinks/lifecycle.py": ( + "from sqlbuild.sinks import LifecycleEvent, lifecycle_event_sink\n\n\n" + "@lifecycle_event_sink()\n" + "def publish_events(event: LifecycleEvent, destination_client) -> None:\n" + " return None\n" + ) + }, + ), + _case( + name="command-output-sink-unknown-provider", + expected_code="D015", + files={ + "sinks/output.py": ( + "from sqlbuild.sinks import CommandOutputRecord, command_output_sink\n\n\n" + '@command_output_sink(streams={"stdout"})\n' + "def publish_output(record: CommandOutputRecord, destination_client) -> None:\n" + " return None\n" + ) + }, + ), + _case( + name="materialization-import-error", + expected_code="D011", + files={ + "materializations/copy_table.py": "import missing_warehouse_client\n", + FAILURE_MART_PATH: FAILURE_BASE_MART.replace( + '"Order totals per customer",', + '"Order totals per customer",\n materialized copy_table,', + ), + }, + ), + _case( + name="python-outside-extension-root", + expected_code="D016", + files={"helpers/formatting.py": "VALUE = 1\n"}, + ), + ) diff --git a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py new file mode 100644 index 0000000000..fdfb77c540 --- /dev/null +++ b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py @@ -0,0 +1,75 @@ +"""Run the failure corpus under one engine and report the codes each case really emits.""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from scripts.compiler_differential._helpers.comparing.comparison import diagnostic_codes +from scripts.compiler_differential._helpers.corpus.corpus import build_corpus +from scripts.compiler_differential._helpers.running.execution import run_engine +from scripts.compiler_differential.constants import ( + CORPUS_FAILURES, + ERROR_SEVERITY, + EXPECT_SUCCESS, + WARNING_SEVERITY, +) +from scripts.compiler_differential.models import ( + CorpusProject, + DifferentialOptions, + EmittedCodes, + EngineRun, +) +from sqlbuild.compiler.discovery.exceptions import DiscoveryError +from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError + +_SOURCE_DIRECTORY: str = "source" + + +def emitted_failure_codes(*, options: DifferentialOptions) -> dict[str, EmittedCodes]: + """Compile every failure case under the first engine and return its emitted codes by name.""" + + corpus: list[CorpusProject] = build_corpus( + repo_root=options.work_dir, + corpora=(CORPUS_FAILURES,), + seeds=range(0), + dense_models=0, + projects=(), + project_expectation=EXPECT_SUCCESS, + ) + with ThreadPoolExecutor(max_workers=options.jobs) as pool: + emitted: list[EmittedCodes] = list( + pool.map(lambda project: _emitted_codes(project=project, options=options), corpus) + ) + return {project.name: codes for project, codes in zip(corpus, emitted, strict=True)} + + +def discovery_error_codes() -> frozenset[str]: + """Return the code of every `DiscoveryError` subclass, walking the whole hierarchy.""" + + _ = ResourceIdentityError + codes: set[str] = set() + pending: list[type[DiscoveryError]] = list(DiscoveryError.__subclasses__()) + while pending: + error_class: type[DiscoveryError] = pending.pop() + codes.add(str(error_class.code)) + pending.extend(error_class.__subclasses__()) + return frozenset(codes) + + +def _emitted_codes(*, project: CorpusProject, options: DifferentialOptions) -> EmittedCodes: + case_dir: Path = options.work_dir / project.name.replace("/", "__") + source_dir: Path = case_dir / _SOURCE_DIRECTORY + if project.writer is not None: + project.writer(source_dir) + run: EngineRun = run_engine( + project=project, + source_dir=source_dir, + case_dir=case_dir, + engine=options.engines[0], + options=options, + ) + return EmittedCodes( + errors=diagnostic_codes(outcome=run.outcomes[0], severity=ERROR_SEVERITY), + warnings=diagnostic_codes(outcome=run.outcomes[0], severity=WARNING_SEVERITY), + ) diff --git a/scripts/compiler_differential/_helpers/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py similarity index 96% rename from scripts/compiler_differential/_helpers/failure_cases.py rename to scripts/compiler_differential/_helpers/corpus/failure_cases.py index a49d3df4e5..2cf0947ae8 100644 --- a/scripts/compiler_differential/_helpers/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -1,5 +1,8 @@ """Minimal failing projects, each derived from the shared base project.""" +from scripts.compiler_differential._helpers.corpus.discovery_failure_cases import ( + discovery_failure_cases, +) from scripts.compiler_differential.constants import ( FAILURE_BASE_CONFIG, FAILURE_BASE_FILES, @@ -15,9 +18,18 @@ _RULES_CONFIG: str = '\n[rules]\nselect = ["{codes}"]\n' -def _case(*, name: str, expected_code: str, files: dict[str, str]) -> FailureCase: +def _case( + *, + name: str, + expected_code: str, + files: dict[str, str], + expected_warning_code: str | None = None, +) -> FailureCase: return FailureCase( - files={**FAILURE_BASE_FILES, **files}, name=name, expected_code=expected_code + files={**FAILURE_BASE_FILES, **files}, + name=name, + expected_code=expected_code, + expected_warning_code=expected_warning_code, ) @@ -38,6 +50,10 @@ def _rules(*codes: str) -> dict[str, str]: def all_failure_cases() -> tuple[FailureCase, ...]: """Return every failure case in a stable order.""" + return (*_compile_failure_cases(), *discovery_failure_cases()) + + +def _compile_failure_cases() -> tuple[FailureCase, ...]: return ( _case( name="config-toml-syntax", @@ -136,7 +152,8 @@ def all_failure_cases() -> tuple[FailureCase, ...]: ), _case( name="built-in-audit-shadow", - expected_code="P003", + expected_code="S010", + expected_warning_code="P003", files={ "models/staging/_sqlbuild/_audits/generic/not_null.sql": ( 'AUDIT (name "not_null");\n\nSELECT 1 WHERE FALSE\n' diff --git a/scripts/compiler_differential/_helpers/coverage/__init__.py b/scripts/compiler_differential/_helpers/coverage/__init__.py new file mode 100644 index 0000000000..90914209f0 --- /dev/null +++ b/scripts/compiler_differential/_helpers/coverage/__init__.py @@ -0,0 +1 @@ +"""Measure which discovery input kinds the corpus exercises.""" diff --git a/scripts/compiler_differential/_helpers/coverage/discovery.py b/scripts/compiler_differential/_helpers/coverage/discovery.py new file mode 100644 index 0000000000..7492e86d42 --- /dev/null +++ b/scripts/compiler_differential/_helpers/coverage/discovery.py @@ -0,0 +1,299 @@ +"""Classify discovery captures into input kinds; CONFIG_ONLY_KINDS lists the known gaps.""" + +from __future__ import annotations + +import dataclasses +import json +import re +from collections.abc import Iterator +from pathlib import Path + +from scripts.compiler_differential._helpers.comparing.compare import as_json_object +from scripts.compiler_differential.constants import ( + CALLABLE_CAPTURE_FIELDS, + CALLABLE_MARKER, + COLLECTION_CAPTURE_MARKERS, + CRLF_BYTES, + DBT_REF_CALL, + DISCOVERY_DECLARATION_FILE_COLLECTIONS, + DISCOVERY_DETAIL_KINDS, + DISCOVERY_NON_COLLECTION_FIELDS, + DISCOVERY_STAGE_CAPTURE_SUFFIX, + DISCOVERY_TEXT_FILE_COLLECTIONS, + GENERATOR_BOM, + MICROBATCH_MODE, + PATH_MARKER, + RELATIVE_PATH_FIELDS, + SINGULAR_AUDIT_KIND, + STRATEGY_HEADER_KEYS, + TAB, + TYPE_MARKER, + UNCANONICAL_CAPTURE_MARKERS, +) +from sqlbuild.adapter.discovery.constants import BUILTIN_ADAPTER_IMPORTS +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs + +_ENUM_VALUE: re.Pattern[str] = re.compile(r"\.([A-Z_]+)$") +_CROSS_FILE_MACRO_IMPORT: re.Pattern[str] = re.compile(r"^(?:from|import) macros\.", re.MULTILINE) +_NON_ASCII_COMMENT: re.Pattern[str] = re.compile(r"--[^\n]*[^\x00-\x7f]") +_NON_ASCII_STRING: re.Pattern[str] = re.compile(r"'[^'\n]*[^\x00-\x7f][^'\n]*'") + + +def required_discovery_kinds() -> tuple[str, ...]: + """Return every `DiscoveredProjectInputs` collection field plus the finer required kinds.""" + + return (*discovery_collection_kinds(), *DISCOVERY_DETAIL_KINDS) + + +def discovery_collection_kinds() -> tuple[str, ...]: + """Return the `DiscoveredProjectInputs` fields that hold discovered inputs.""" + + return tuple( + field.name + for field in dataclasses.fields(DiscoveredProjectInputs) + if field.name not in DISCOVERY_NON_COLLECTION_FIELDS + ) + + +def project_discovery_kinds( + *, captures: dict[str, dict[str, str]], source_dir: Path +) -> frozenset[str]: + """Return the kinds of every discovery capture plus CRLF, which only the bytes show.""" + + kinds: set[str] = set() + read_paths: set[str] = set() + for text in _discovery_capture_texts(captures): + kinds.update(discovered_input_kinds(text)) + read_paths.update(discovered_relative_paths(text)) + kinds.update(authored_byte_kinds(source_dir=source_dir, relative_paths=frozenset(read_paths))) + return frozenset(kinds) + + +def discovered_relative_paths(capture_text: str) -> frozenset[str]: + """Return the project-relative path of every file a discovery capture says it read.""" + + capture: object = json.loads(capture_text) + return frozenset( + str(path) + for _, value in _named_values(value=capture, names=RELATIVE_PATH_FIELDS) + if (path := (as_json_object(value) or {}).get(PATH_MARKER)) is not None + ) + + +def authored_byte_kinds(*, source_dir: Path, relative_paths: frozenset[str]) -> frozenset[str]: + """Return the line-ending kinds of the authored files discovery read.""" + + paths: list[Path] = [source_dir / relative_path for relative_path in sorted(relative_paths)] + return frozenset( + {"crlf"} + if any(path.is_file() and CRLF_BYTES in path.read_bytes() for path in paths) + else set() + ) + + +def _discovery_capture_texts(captures: dict[str, dict[str, str]]) -> Iterator[str]: + for files in captures.values(): + for name, text in files.items(): + if name.endswith(DISCOVERY_STAGE_CAPTURE_SUFFIX): + yield text + + +def discovered_input_kinds(capture_text: str) -> frozenset[str]: + """Return the input kinds one canonical discovery capture proves were discovered.""" + + capture: dict[str, object] = as_json_object(json.loads(capture_text)) or {} + kinds: set[str] = { + kind for kind in discovery_collection_kinds() if _has_content(capture.get(kind)) + } + kinds.update(_model_kinds(capture)) + kinds.update(_source_kinds(capture)) + kinds.update(_declaration_kinds(capture)) + kinds.update(_test_and_audit_kinds(capture)) + kinds.update(_config_kinds(capture)) + kinds.update(_text_kinds(capture)) + return frozenset(kinds) + + +def discovery_capture_problems(capture_text: str) -> tuple[str, ...]: + """Return why a discovery capture is incomplete or not canonical; empty when it is sound.""" + + capture: dict[str, object] = as_json_object(json.loads(capture_text)) or {} + fields: tuple[str, ...] = tuple( + field.name for field in dataclasses.fields(DiscoveredProjectInputs) + ) + problems: list[str] = [] + captured: tuple[str, ...] = tuple(key for key in capture if key != TYPE_MARKER) + if captured != fields: + problems.append(f"captured fields {captured} differ from {fields}") + problems.extend( + f"capture contains {marker}" + for marker in UNCANONICAL_CAPTURE_MARKERS + if f'"{marker}"' in capture_text + ) + problems.extend( + f"{field} is not a qualified callable name: {value}" + for field, value in _named_values(value=capture, names=CALLABLE_CAPTURE_FIELDS) + if set(as_json_object(value) or {}) != {CALLABLE_MARKER} + ) + return tuple(problems) + + +def _named_values(*, value: object, names: frozenset[str]) -> Iterator[tuple[str, object]]: + record: dict[str, object] | None = as_json_object(value) + children: list[object] = list(value) if isinstance(value, list) else [] + if record is not None: + yield from ((key, item) for key, item in record.items() if key in names) + children.extend(record.values()) + for child in children: + yield from _named_values(value=child, names=names) + + +def _decoded(value: object) -> object: + """Unwrap the capture's set and non-string-key mapping encodings into plain lists.""" + + encoded: dict[str, object] | None = as_json_object(value) + if encoded is None or len(encoded) != 1: + return value + marker: str = next(iter(encoded)) + return encoded[marker] if marker in COLLECTION_CAPTURE_MARKERS else value + + +def _has_content(value: object) -> bool: + """Whether a captured value holds anything; encoded empty sets and mappings do not.""" + + return bool(_decoded(value)) + + +def _records(value: object) -> list[dict[str, object]]: + items: object = _decoded(value) + if not isinstance(items, list): + return [] + return [record for item in items if (record := as_json_object(item)) is not None] + + +def _mapping_size(value: object) -> int: + items: object = _decoded(value) + return len(items) if isinstance(items, (dict, list)) else 0 + + +def _enum_name(value: object) -> str | None: + encoded: dict[str, object] | None = as_json_object(value) + match: re.Match[str] | None = ( + None if encoded is None else _ENUM_VALUE.search(str(encoded.get("__enum__", ""))) + ) + return None if match is None else match.group(1).lower() + + +def _model_kinds(capture: dict[str, object]) -> set[str]: + kinds: set[str] = set() + materializations: set[str] = { + str(record.get("name")) for record in _records(capture.get("materialization_files")) + } + for model in _records(capture.get("model_files")): + header: dict[str, object] = as_json_object(model.get("header_values")) or {} + materialized: object = header.get("materialized") + if header.get("incremental_mode") == MICROBATCH_MODE: + kinds.add(f"microbatch_{header.get('microbatch_strategy')}") + elif materialized in STRATEGY_HEADER_KEYS: + kinds.add(f"{materialized}_{header.get(STRATEGY_HEADER_KEYS[str(materialized)])}") + elif materialized in materializations: + kinds.add("custom_materialized_model") + if _records(model.get("constant_declarations")) or _records(model.get("enum_declarations")): + kinds.add("model_local_declaration") + if DBT_REF_CALL in str(model.get("contents", "")): + kinds.add("dbt_ref_model") + return kinds + + +def _source_kinds(capture: dict[str, object]) -> set[str]: + kinds: set[str] = set() + for source_file in _records(capture.get("source_files")): + for entry in _records(source_file.get("source_entries")): + kinds.add("managed_source" if entry.get("managed") is True else "unmanaged_source") + return kinds + + +def _declaration_kinds(capture: dict[str, object]) -> set[str]: + kinds: set[str] = set() + for collection in DISCOVERY_DECLARATION_FILE_COLLECTIONS: + for record in _records(capture.get(collection)): + scope: str | None = _enum_name(record.get("scope_kind")) + if scope is not None: + kinds.add(f"scope_{scope}") + for constant_file in _records(capture.get("constant_files")): + for declaration in _records(constant_file.get("declarations")): + value: dict[str, object] = as_json_object(declaration.get("value")) or {} + logical: dict[str, object] = as_json_object(value.get("logical_type")) or {} + kind: str | None = _enum_name(logical.get("kind")) + if kind is not None: + kinds.add(f"constant_{kind}") + if not json.dumps(value.get("value"), ensure_ascii=False).isascii(): + kinds.add("non_ascii_constant") + for enum_file in _records(capture.get("enum_files")): + for declaration in _records(enum_file.get("declarations")): + members: list[dict[str, object]] = _records(declaration.get("members")) + kinds.update( + "enum_integer" if isinstance(member.get("value"), int) else "enum_string" + for member in members + ) + if any( + _CROSS_FILE_MACRO_IMPORT.search(str(record.get("contents", ""))) + for record in _records(capture.get("macro_files")) + ): + kinds.add("cross_file_macro_import") + return kinds + + +def _test_and_audit_kinds(capture: dict[str, object]) -> set[str]: + kinds: set[str] = set() + for test_file in _records(capture.get("test_files")): + for block in _records(test_file.get("blocks")): + mode: str | None = _enum_name(block.get("mode")) + if mode is not None: + kinds.add(f"test_mode_{mode}") + if _records(block.get("cases")): + kinds.add("parameterized_test") + for audit_file in _records(capture.get("audit_files")): + declaration_kind: str | None = _enum_name(audit_file.get("declaration_kind")) + kinds.add( + SINGULAR_AUDIT_KIND if declaration_kind == SINGULAR_AUDIT_KIND else "generic_audit" + ) + return kinds + + +def _config_kinds(capture: dict[str, object]) -> set[str]: + kinds: set[str] = set() + project: dict[str, object] = as_json_object(capture.get("project_config")) or {} + local: dict[str, object] = as_json_object(capture.get("local_config")) or {} + if _has_content(project.get("path_defaults")): + kinds.add("path_defaults") + if ( + _has_content(local.get("vars")) + or _has_content(local.get("setting_overrides")) + or local.get("target") is not None + ): + kinds.add("local_config") + if local.get("target") is not None and _mapping_size(project.get("targets")) > 1: + kinds.add("target_override") + dbt: dict[str, object] = as_json_object(project.get("dbt")) or {} + if dbt.get("target_path") is not None: + kinds.add("dbt_target_path_config") + if project.get("adapter") not in BUILTIN_ADAPTER_IMPORTS: + kinds.add("project_adapter_config") + return kinds + + +def _text_kinds(capture: dict[str, object]) -> set[str]: + kinds: set[str] = set() + for collection in DISCOVERY_TEXT_FILE_COLLECTIONS: + for record in _records(capture.get(collection)): + contents: str = str(record.get("contents", "")) + if TAB in contents: + kinds.add("tab") + if contents.startswith(GENERATOR_BOM): + kinds.add("bom") + if _NON_ASCII_COMMENT.search(contents): + kinds.add("non_ascii_comment") + if _NON_ASCII_STRING.search(contents): + kinds.add("non_ascii_string") + return kinds diff --git a/scripts/compiler_differential/_helpers/report.py b/scripts/compiler_differential/_helpers/report.py deleted file mode 100644 index fccf230595..0000000000 --- a/scripts/compiler_differential/_helpers/report.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Render differential results for terminals and CI logs.""" - -from __future__ import annotations - -from scripts.compiler_differential.models import Difference, ProjectComparison - - -def format_comparison(*, comparison: ProjectComparison, engines: tuple[str, str]) -> str: - """Render one project's verdict and the first difference in every differing artifact.""" - - status: str = "OK " if not comparison.differences else "DIFF" - lines: list[str] = [f"{status} {comparison.project} ({comparison.seconds:.1f}s)"] - for difference in comparison.differences: - left_label, right_label = difference.labels or engines - width: int = max(len(left_label), len(right_label)) - lines.append(f" - {difference.artifact} at {difference.location}") - lines.append(f" {left_label:<{width}}: {difference.left}") - lines.append(f" {right_label:<{width}}: {difference.right}") - return "\n".join(lines) - - -def format_summary( - *, comparisons: list[ProjectComparison], engines: tuple[str, str], seconds: float -) -> str: - """Summarize the run, naming the first difference of the first differing project.""" - - differing: list[ProjectComparison] = [item for item in comparisons if item.differences] - left_engine, right_engine = engines - if not differing: - return ( - f"Compiler differential passed: {len(comparisons)} projects identical " - f"({left_engine} vs {right_engine}) in {seconds:.1f}s" - ) - first: Difference = differing[0].differences[0] - return ( - f"Compiler differential FAILED: {len(differing)} of {len(comparisons)} projects differ " - f"({left_engine} vs {right_engine}) in {seconds:.1f}s\n" - f"First difference: {first.project}: {first.artifact} at {first.location}" - ) diff --git a/scripts/compiler_differential/_helpers/running/__init__.py b/scripts/compiler_differential/_helpers/running/__init__.py new file mode 100644 index 0000000000..ae6b558bcb --- /dev/null +++ b/scripts/compiler_differential/_helpers/running/__init__.py @@ -0,0 +1 @@ +"""Run engines, parse options, and report results.""" diff --git a/scripts/compiler_differential/_helpers/execution.py b/scripts/compiler_differential/_helpers/running/execution.py similarity index 100% rename from scripts/compiler_differential/_helpers/execution.py rename to scripts/compiler_differential/_helpers/running/execution.py diff --git a/scripts/compiler_differential/_helpers/options.py b/scripts/compiler_differential/_helpers/running/options.py similarity index 100% rename from scripts/compiler_differential/_helpers/options.py rename to scripts/compiler_differential/_helpers/running/options.py diff --git a/scripts/compiler_differential/_helpers/running/report.py b/scripts/compiler_differential/_helpers/running/report.py new file mode 100644 index 0000000000..79a4e76dab --- /dev/null +++ b/scripts/compiler_differential/_helpers/running/report.py @@ -0,0 +1,63 @@ +"""Render differential results for terminals and CI logs.""" + +from __future__ import annotations + +from scripts.compiler_differential.constants import CONFIG_ONLY_KINDS +from scripts.compiler_differential.models import Difference, ProjectComparison + + +def format_comparison(*, comparison: ProjectComparison, engines: tuple[str, str]) -> str: + """Render one project's verdict and the first difference in every differing artifact.""" + + status: str = "OK " if not comparison.differences else "DIFF" + lines: list[str] = [f"{status} {comparison.project} ({comparison.seconds:.1f}s)"] + for difference in comparison.differences: + left_label, right_label = difference.labels or engines + width: int = max(len(left_label), len(right_label)) + lines.append(f" - {difference.artifact} at {difference.location}") + lines.append(f" {left_label:<{width}}: {difference.left}") + lines.append(f" {right_label:<{width}}: {difference.right}") + return "\n".join(lines) + + +def format_summary( + *, + comparisons: list[ProjectComparison], + engines: tuple[str, str], + seconds: float, + missing_coverage: tuple[str, ...] = (), +) -> str: + """Summarize the run; any difference or required-but-missing coverage reports FAILED.""" + + differing: list[ProjectComparison] = [item for item in comparisons if item.differences] + left_engine, right_engine = engines + if not differing and not missing_coverage: + return ( + f"Compiler differential passed: {len(comparisons)} projects identical " + f"({left_engine} vs {right_engine}) in {seconds:.1f}s" + ) + lines: list[str] = [ + f"Compiler differential FAILED: {len(differing)} of {len(comparisons)} projects differ " + f"({left_engine} vs {right_engine}) in {seconds:.1f}s" + ] + if differing: + first: Difference = differing[0].differences[0] + lines.append(f"First difference: {first.project}: {first.artifact} at {first.location}") + if missing_coverage: + lines.append(f"Required discovery coverage missing: {', '.join(missing_coverage)}") + return "\n".join(lines) + + +def format_discovery_coverage(*, covered: frozenset[str], required: tuple[str, ...]) -> str: + """Report how many required discovery input kinds the seed corpus exercised.""" + + missing: list[str] = [kind for kind in required if kind not in covered] + line: str = ( + f"Discovery coverage: {len(required) - len(missing)} of {len(required)} input kinds " + "exercised by the seed corpus" + ) + config_only: str = "; ".join( + f"{kind} (only proves {meaning})" for kind, meaning in CONFIG_ONLY_KINDS.items() + ) + line = f"{line}\n config-only kinds: {config_only}" + return line if not missing else f"{line}\n missing: {', '.join(missing)}" diff --git a/scripts/compiler_differential/_helpers/run.py b/scripts/compiler_differential/_helpers/running/run.py similarity index 83% rename from scripts/compiler_differential/_helpers/run.py rename to scripts/compiler_differential/_helpers/running/run.py index 0d200afe02..49ed382374 100644 --- a/scripts/compiler_differential/_helpers/run.py +++ b/scripts/compiler_differential/_helpers/running/run.py @@ -7,10 +7,10 @@ from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from scripts.compiler_differential._helpers.corpus import build_corpus -from scripts.compiler_differential._helpers.options import parse_engine_environment -from scripts.compiler_differential._helpers.project_comparison import compare_project -from scripts.compiler_differential._helpers.report import format_comparison +from scripts.compiler_differential._helpers.comparing.project_comparison import compare_project +from scripts.compiler_differential._helpers.corpus.corpus import build_corpus +from scripts.compiler_differential._helpers.running.options import parse_engine_environment +from scripts.compiler_differential._helpers.running.report import format_comparison from scripts.compiler_differential.models import ( CorpusProject, DifferentialOptions, @@ -28,6 +28,7 @@ def differential_options(*, args: argparse.Namespace, work_dir: Path) -> Differe stage_captures=args.stage_captures, python=args.python, engine_environment=parse_engine_environment(args.engine_env), + require_discovery_coverage=args.require_discovery_coverage, ) diff --git a/scripts/compiler_differential/classes/discovery_features.py b/scripts/compiler_differential/classes/discovery_features.py new file mode 100644 index 0000000000..63625986e6 --- /dev/null +++ b/scripts/compiler_differential/classes/discovery_features.py @@ -0,0 +1,555 @@ +"""Write the optional project inputs that make generated seeds cover every discovery input kind.""" + +from __future__ import annotations + +import random +from collections.abc import Callable +from functools import partial + +from scripts.compiler_differential.constants import ( + GENERATOR_BOM, + GENERATOR_CRLF, + GENERATOR_DBT_MANIFEST, + GENERATOR_DBT_MODEL, + GENERATOR_DBT_REF_ERROR_CODE, + GENERATOR_FEATURE_BLOCKS, + GENERATOR_FEATURE_FOLDER, + GENERATOR_FEATURE_STRIDE, + GENERATOR_OPTIONAL_FEATURE_SHARE, + GENERATOR_PROJECT_ADAPTER, + GENERATOR_RARE_FEATURE_BLOCKS, + GENERATOR_RARE_FEATURE_PERIOD, + PLAN_LABEL, +) +from scripts.compiler_differential.models import ModelPlan + +_UNIQUE_KEY: str = " unique_key [id],\n" +_CURSOR_HEADER: str = " cursor created_at,\n cursor_type timestamp,\n cursor_grain day,\n" + + +class DiscoveryFeatureWriter: + """Add self-contained feature blocks to one generated project, chosen per seed.""" + + def __init__( + self, + *, + blocks: tuple[str, ...], + rng: random.Random, + files: dict[str, str], + features: set[str], + staging: list[ModelPlan], + label: Callable[[], str], + ) -> None: + self._blocks: tuple[str, ...] = blocks + self._random: random.Random = rng + self._files: dict[str, str] = files + self._features: set[str] = features + self._staging: list[ModelPlan] = staging + self._label: Callable[[], str] = label + self.config_lines: list[str] = [] + self.adapter: str | None = None + self.expected_error_code: str | None = None + self.succeeding_commands: tuple[str, ...] = () + + def write(self) -> None: + """Write the selected blocks in their canonical order.""" + + writers: dict[str, Callable[[], None]] = { + "incremental_append": partial(self._incremental, strategy="append", unique_key=""), + "incremental_delete_insert": partial( + self._incremental, strategy="delete_insert", unique_key=_UNIQUE_KEY + ), + "incremental_merge": partial( + self._incremental, strategy="merge", unique_key=_UNIQUE_KEY + ), + "microbatch_watermark": partial( + self._microbatch, + strategy="watermark", + cursor_input="(column created_at, roles [filter, watermark])", + options=" cursor_watermark_mode all,\n", + ), + "microbatch_rolling_window": partial( + self._microbatch, + strategy="rolling_window", + cursor_input="created_at", + options=" lookback 2d,\n", + ), + "snapshot_timestamp": partial( + self._snapshot, + strategy="timestamp", + options=" updated_at created_at,\n", + columns="status, created_at", + ), + "snapshot_check": partial( + self._snapshot, + strategy="check", + options=" check_columns [status, amount],\n", + columns="status, amount", + ), + "python_hook": self._python_hook, + "managed_source_loader": self._managed_source_loader, + "asset_and_check": self._asset_and_check, + "audit_factory": self._audit_factory, + "provider": self._provider, + "lifecycle_sink": self._lifecycle_sink, + "command_output_sink": self._command_output_sink, + "custom_materialization": self._custom_materialization, + "project_adapter": self._project_adapter, + "model_schema": self._model_schema, + "list_constant": self._list_constant, + "cross_file_macro_import": self._cross_file_macro_import, + "macro_test_mode": self._macro_test_mode, + "parameterized_test": self._parameterized_test, + "generic_audit": self._generic_audit, + "functions": self._functions, + "local_config": self._local_config, + "target_override": self._target_override, + "dbt_ref": self._dbt_ref, + "line_endings": self._line_endings, + } + for block in self._blocks: + writers[block]() + + def _base(self) -> ModelPlan: + return self._random.choice(self._staging) + + def _model(self, *, name: str, header: str, body: str, description: str) -> str: + path: str = f"{GENERATOR_FEATURE_FOLDER}/{name}.sql" + self._files[path] = f'MODEL (\n description "{description}",\n{header});\n\n{body}' + return path + + def _incremental(self, *, strategy: str, unique_key: str) -> None: + self._features.add(f"incremental_{strategy}") + base: ModelPlan = self._base() + _ = self._model( + name=f"{strategy}_{base.name}", + description=f"Incremental {strategy} copy of {base.name}", + header=( + " materialized incremental,\n" + f" incremental_strategy {strategy},\n" + f"{unique_key}{_CURSOR_HEADER}" + f" cursor_inputs (\n {base.name} created_at,\n ),\n" + ), + body=f'SELECT id, amount, status, created_at\nFROM __ref("{base.name}")\n', + ) + + def _microbatch(self, *, strategy: str, cursor_input: str, options: str) -> None: + self._features.add(f"microbatch_{strategy}") + base: ModelPlan = self._base() + _ = self._model( + name=f"batched_{strategy}_{base.name}", + description=f"Microbatched {strategy} totals of {base.name}", + header=( + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + f"{_CURSOR_HEADER}" + " incremental_mode microbatch,\n" + f" microbatch_strategy {strategy},\n" + f"{options}" + " batch_size 1d,\n" + f" cursor_inputs (\n {base.name} {cursor_input},\n ),\n" + ), + body=( + "SELECT CAST(created_at AS DATE) AS created_at, SUM(amount) AS amount\n" + f'FROM __ref("{base.name}")\nGROUP BY CAST(created_at AS DATE)\n' + ), + ) + + def _snapshot(self, *, strategy: str, options: str, columns: str) -> None: + self._features.add(f"snapshot_{strategy}") + base: ModelPlan = self._base() + _ = self._model( + name=f"{strategy}_history_{base.name}", + description=f"{strategy.capitalize()} history of {base.name}", + header=( + " materialized snapshot,\n unique_key [id],\n" + f" snapshot_strategy {strategy},\n{options}" + ), + body=f'SELECT id, {columns}\nFROM __ref("{base.name}")\n', + ) + + def _python_hook(self) -> None: + self._features.add("python_hook") + base: ModelPlan = self._base() + self._files[f"{GENERATOR_FEATURE_FOLDER}/_sqlbuild/_hooks/python/notify.py"] = ( + "from sqlbuild.hooks import hook\n\n\n" + "@hook\n" + 'def note_refresh(ctx, channel="#builds"):\n' + ' """Log one refresh."""\n' + ' ctx.log(f"{channel}: {ctx.model_name} refreshed during {ctx.phase}")\n' + ) + _ = self._model( + name=f"hooked_{base.name}", + description=f"Hooked copy of {base.name}", + header=( + " materialized table,\n" + f' post_hooks [python("note_refresh", channel: "{self._label()}")],\n' + ), + body=f'SELECT id, amount\nFROM __ref("{base.name}")\n', + ) + + def _managed_source_loader(self) -> None: + self._features.add("managed_source_loader") + self._files["sources/managed.yml"] = ( + "sources:\n" + " - name: loaded_products\n" + " description: Products loaded by a project loader.\n" + " managed: true\n" + " write_strategy: table\n" + " columns:\n" + " - name: id\n type: INTEGER\n" + " - name: label\n type: VARCHAR\n" + ) + self._files["python/loaders/products.py"] = ( + "from sqlbuild.loaders import loader\n\n\n" + "@loader\n" + "def loaded_products(ctx):\n" + ' """Load the product list."""\n' + f' return [{{"id": 1, "label": "{self._label()}"}}, {{"id": 2, "label": "plain"}}]\n' + ) + _ = self._model( + name="product_labels", + description="Product labels from the loader", + header=" materialized view,\n", + body='SELECT id, label\nFROM __source("loaded_products")\n', + ) + + def _asset_and_check(self) -> None: + self._features.add("asset_and_check") + base: ModelPlan = self._base() + self._files["python/assets/exports.py"] = ( + "from sqlbuild.assets import asset\n" + "from sqlbuild.checks import check\n" + "from sqlbuild.refs import model\n\n\n" + f'@asset(depends_on=model("{base.name}"), tags=("export",))\n' + f"def export_{base.name}(ctx):\n" + ' """Export one staged model."""\n' + ' return ctx.result(metadata={"rows": 3}, materialized=True)\n\n\n' + f'@check(depends_on=export_{base.name}, severity="warn")\n' + f"def export_{base.name}_ready(ctx):\n" + ' """Check that the export ran."""\n' + " return ctx.pass_()\n" + ) + + def _audit_factory(self) -> None: + self._features.add("audit_factory") + base: ModelPlan = self._base() + self._files["python/audits/quality.py"] = ( + "from sqlbuild.audits import AuditCase, audit_factory\n\n\n" + "@audit_factory\n" + "def event_quality():\n" + " return [\n" + ' AuditCase(name="id_present", definition="not_null", ' + 'arguments={"column": "id"}),\n' + " ]\n" + ) + _ = self._model( + name=f"audited_{base.name}", + description=f"Factory-audited copy of {base.name}", + header=" audit_factories [event_quality],\n", + body=f'SELECT id, amount\nFROM __ref("{base.name}")\n', + ) + + def _provider(self) -> None: + self._features.add("provider") + self._files["providers/export_settings.py"] = ( + "from sqlbuild.providers import Provider\n\n\n" + "class ExportSettings(Provider):\n" + ' """Settings for generated exports."""\n\n' + f' region: str = "{self._random.choice(("north", "south"))}"\n' + " batch_size: int = 50\n" + ) + self._files["python/tasks/provided.py"] = ( + "from sqlbuild.tasks import task\n\n\n" + "@task\n" + "def provided_export(ctx, export_settings):\n" + ' """Export with provider settings."""\n' + ' return ctx.result(payload={"region": export_settings.region})\n' + ) + + def _lifecycle_sink(self) -> None: + self._features.add("lifecycle_sink") + self._files["sinks/lifecycle.py"] = ( + "from sqlbuild.sinks import LifecycleEvent, LifecycleEventKind, lifecycle_event_sink\n" + "\n\n" + "@lifecycle_event_sink(\n" + " event_kinds={LifecycleEventKind.RUN, LifecycleEventKind.RESOURCE}\n" + ")\n" + "def ignore_lifecycle(event: LifecycleEvent) -> None:\n" + " return None\n" + ) + + def _command_output_sink(self) -> None: + self._features.add("command_output_sink") + self._files["sinks/output.py"] = ( + "from sqlbuild.sinks import CommandOutputRecord, command_output_sink\n\n\n" + '@command_output_sink(streams={"stderr"})\n' + "def ignore_output(record: CommandOutputRecord) -> None:\n" + " return None\n" + ) + + def _custom_materialization(self) -> None: + self._features.add("custom_materialization") + base: ModelPlan = self._base() + self._files["materializations/copy_table.py"] = ( + "from sqlbuild.executor.custom.models import MaterializationContext, " + "MaterializationResult\n\n\n" + "def materialize(ctx: MaterializationContext) -> MaterializationResult:\n" + ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS {ctx.sql}")\n' + " return MaterializationResult(relation=ctx.destination, audit_results=())\n" + ) + _ = self._model( + name=f"copied_{base.name}", + description=f"Custom copy of {base.name}", + header=" materialized copy_table,\n", + body=f'SELECT id, status\nFROM __ref("{base.name}")\n', + ) + + def _project_adapter(self) -> None: + self._features.add("project_adapter") + self.adapter = GENERATOR_PROJECT_ADAPTER + self._files["adapters/generated_duckdb.py"] = ( + "from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter\n\n\n" + "class GeneratedDuckDbAdapter(DuckDbAdapter):\n" + f' adapter_name = "{GENERATOR_PROJECT_ADAPTER}"\n' + ) + self._files["adapter.py"] = ( + "from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter\n\n\n" + "class ProjectOverrideAdapter(DuckDbAdapter):\n" + ' """Project adapter override."""\n\n' + f' adapter_name = "{GENERATOR_PROJECT_ADAPTER}_override"\n' + ) + + def _model_schema(self) -> None: + self._features.add("model_schema") + base: ModelPlan = self._base() + self._files[f"{GENERATOR_FEATURE_FOLDER}/_sqlbuild/_schemas/event_shape.sql"] = ( + "SCHEMA (\n name event_shape,\n" + ' description "Canonical event shape",\n' + " columns (\n" + " id (type INTEGER, audits [not_null]),\n" + " status (type VARCHAR),\n" + " ),\n);\n" + ) + _ = self._model( + name=f"shaped_{base.name}", + description=f"Schema-shaped copy of {base.name}", + header=" model_schema event_shape,\n contract enforced,\n", + body=f'SELECT CAST(id AS INTEGER) AS id, status\nFROM __ref("{base.name}")\n', + ) + + def _list_constant(self) -> None: + self._features.add("list_constant") + base: ModelPlan = self._base() + values: list[str] = self._random.sample(["placed", "expédié", "取消", "returned ✓"], k=2) + self._files[f"{GENERATOR_FEATURE_FOLDER}/_sqlbuild/_constants/statuses.sql"] = ( + "CONSTANT (\n name kept_statuses,\n value [" + + ", ".join(f'"{value}"' for value in values) + + "],\n);\n" + "CONSTANT (\n name amount_weights,\n value [0.5, 1.25, 2500.75],\n);\n" + ) + _ = self._model( + name=f"kept_{base.name}", + description=f"Statuses kept from {base.name}", + header="", + body=( + f'SELECT id, status, amount * 0.5 AS weighted\nFROM __ref("{base.name}")\n' + 'WHERE status IN @const("kept_statuses")\n' + ' AND amount * 1.0 IN @const("amount_weights") OR id > 0\n' + ), + ) + + def _cross_file_macro_import(self) -> None: + self._features.add("cross_file_macro_import") + base: ModelPlan = self._base() + self._files["macros/rounding.py"] = ( + "def round_amount(expression: str, places: int = 2) -> str:\n" + ' """Round an amount."""\n' + ' return f"ROUND({expression}, {places})"\n' + ) + self._files["macros/tax.py"] = ( + "from macros.rounding import round_amount\n\n\n" + "def taxed(expression: str) -> str:\n" + ' """Add tax to an amount, rounded."""\n' + ' return round_amount(f"{expression} * 1.2")\n' + ) + _ = self._model( + name=f"taxed_{base.name}", + description=f"Taxed copy of {base.name}", + header="", + body=( + 'SELECT id, @taxed("amount") AS taxed, @round_amount("amount", 1) AS rounded\n' + f'FROM __ref("{base.name}")\n' + ), + ) + self._files["tests/unit/test_taxed_macro.sql"] = ( + 'TEST (mode macro, name "taxes_amounts");\n\n' + "WITH\ninput_values AS (\n SELECT 10 AS amount\n),\n" + '__macro_actual__ AS (\n SELECT @taxed("amount") AS taxed\n FROM input_values\n),\n' + "__macro_expected__ AS (\n SELECT 12.0 AS taxed\n)\nSELECT 1\n" + ) + + def _macro_test_mode(self) -> None: + self._features.add("macro_test_mode") + self._files["tests/unit/_macros/scaling.py"] = ( + "def doubled(expression: str) -> str:\n" + ' """Double an amount."""\n' + ' return f"({expression} * 2)"\n' + ) + self._files["tests/unit/test_doubled_macro.sql"] = ( + 'TEST (mode macro, name "doubles_amounts");\n\n' + "WITH\ninput_values AS (\n SELECT 4 AS amount\n),\n" + '__macro_actual__ AS (\n SELECT @doubled("amount") AS doubled\n' + " FROM input_values\n),\n" + "__macro_expected__ AS (\n SELECT 8 AS doubled\n)\nSELECT 1\n" + ) + + def _parameterized_test(self) -> None: + self._features.add("parameterized_test") + base: ModelPlan = self._base() + source: str = base.inputs[0].split(":", 1)[1] + self._files[f"tests/unit/test_{base.name}_cases.sql"] = ( + "TEST (\n" + f' name "{base.name}_keeps_status",\n' + " parameters (\n status_value string,\n amount_value float,\n ),\n" + " cases (\n" + f' placed (status_value "placed", amount_value 1.5),\n' + f' labelled (status_value "{self._label()}", amount_value 2.25),\n' + " ),\n);\n\n" + "WITH\n" + f"__source__{source} AS (\n" + ' SELECT 1 AS id, CAST(@param("amount_value") AS DOUBLE) AS amount,\n' + ' @param("status_value") AS status,\n' + " TIMESTAMP '2026-01-01 00:00:00' AS created_at\n" + "),\n" + f"__expected__{base.name} AS (\n" + " SELECT 1 AS id\n" + ")\nSELECT 1\n" + ) + + def _generic_audit(self) -> None: + self._features.add("generic_audit") + base: ModelPlan = self._base() + self._files[f"{GENERATOR_FEATURE_FOLDER}/_sqlbuild/_audits/generic/amount_at_least.sql"] = ( + 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE @column < @minimum\n' + ) + _ = self._model( + name=f"guarded_{base.name}", + description=f"Generic-audited copy of {base.name}", + header=" columns (\n amount (audits [amount_at_least (minimum 0)]),\n ),\n", + body=f'SELECT id, amount\nFROM __ref("{base.name}")\n', + ) + + def _functions(self) -> None: + self._features.add("functions") + self._files["functions/sql/is_placed.sql"] = ( + "FUNCTION (\n" + ' description "Whether a status is placed.",\n' + " arguments (event_status STRING),\n" + " returns BOOLEAN,\n);\n\n" + "event_status = 'placed'\n" + ) + self._files["functions/python/is_large.py"] = ( + "from sqlbuild.functions import udf\n\n\n" + '@udf(arguments={"amount": "DOUBLE"}, returns="BOOLEAN", runtime_version="3.11")\n' + "def main(amount: float | None) -> bool:\n" + ' """Whether an amount is large."""\n' + " return amount is not None and amount > 10\n" + ) + self._files["tests/unit/test_is_placed_udf.sql"] = ( + 'TEST (mode udf, name "detects_placed_events");\n\n' + "WITH\ninput_values AS (\n SELECT 'placed' AS event_status\n),\n" + "__udf_actual__ AS (\n" + ' SELECT event_status, __udf("is_placed")(event_status) AS is_placed\n' + " FROM input_values\n),\n" + "__udf_expected__ AS (\n SELECT 'placed' AS event_status, TRUE AS is_placed\n)\n" + "SELECT 1\n" + ) + base: ModelPlan = self._base() + _ = self._model( + name=f"flagged_{base.name}", + description=f"Function-flagged copy of {base.name}", + header="", + body=( + 'SELECT id, __udf("is_placed")(status) AS is_placed,\n' + ' __udf("is_large")(amount) AS is_large\n' + f'FROM __ref("{base.name}")\n' + ), + ) + + def _local_config(self) -> None: + self._features.add("local_config") + self._files["sqlbuild_local.toml"] = ( + "[vars]\n" + f'site_label = "{self._label()}"\n\n' + "[settings]\n" + f"query_change_tracking = {self._random.choice(('true', 'false'))}\n" + ) + + def _target_override(self) -> None: + self._features.add("target_override") + self.config_lines.extend( + ["", "[targets.ci]", 'schema = "ci_checks"', "", "[targets.ci.vars]", 'region = "west"'] + ) + self._files.setdefault("sqlbuild_local.toml", "") + self._files["sqlbuild_local.toml"] = ( + 'target = "ci"\n\n' + self._files["sqlbuild_local.toml"] + ) + + def _dbt_ref(self) -> None: + self._features.add("dbt_ref") + self.expected_error_code = GENERATOR_DBT_REF_ERROR_CODE + self.succeeding_commands = (PLAN_LABEL,) + self.config_lines.extend( + ["", "[dbt]", 'project_dir = "dbt"', 'target_path = "dbt/artifacts"'] + ) + self._files["dbt/dbt_project.yml"] = "name: upstream\nversion: '1.0.0'\nprofile: upstream\n" + self._files["dbt/artifacts/manifest.json"] = GENERATOR_DBT_MANIFEST + _ = self._model( + name="dbt_order_counts", + description="Counts from the upstream dbt model", + header="", + body=( + "SELECT customer_id, COUNT(*) AS order_count\n" + f'FROM __dbt_ref("upstream", "{GENERATOR_DBT_MODEL}")\nGROUP BY customer_id\n' + ), + ) + + def _line_endings(self) -> None: + self._features.add("line_endings") + base: ModelPlan = self._base() + path: str = self._model( + name=f"spaced_{base.name}", + description=f"Copy of {base.name} with {self._label()}", + header=' tags ["línea", "東京"],\n', + body=( + f"-- Généré pour {self._label()}\n" + "SELECT\n" + " id,\n" + f" '{self._label()}' AS label, -- étiquette ✓\n" + " amount\n" + f'FROM __ref("{base.name}")\n' + ), + ) + self._files[path] = self._files[path].replace("\n ", "\n\t").replace("\n", GENERATOR_CRLF) + self._files["sources/raw.yml"] = GENERATOR_BOM + self._files["sources/raw.yml"].replace( + "\n", GENERATOR_CRLF + ) + + +def feature_blocks_for_seed(*, seed: int, rng: random.Random) -> tuple[str, ...]: + """Return stride-forced blocks plus a random share of the rest; rare blocks are residue-only.""" + + blocks: list[str] = [] + for index, block in enumerate(GENERATOR_FEATURE_BLOCKS): + rare_residue: int | None = GENERATOR_RARE_FEATURE_BLOCKS.get(block) + if rare_residue is not None: + selected: bool = seed % GENERATOR_RARE_FEATURE_PERIOD == rare_residue + else: + selected = ( + index % GENERATOR_FEATURE_STRIDE == seed % GENERATOR_FEATURE_STRIDE + or rng.random() < GENERATOR_OPTIONAL_FEATURE_SHARE + ) + if selected: + blocks.append(block) + return tuple(blocks) diff --git a/scripts/compiler_differential/classes/project_builder.py b/scripts/compiler_differential/classes/project_builder.py index beb7b1d465..bdffa11e73 100644 --- a/scripts/compiler_differential/classes/project_builder.py +++ b/scripts/compiler_differential/classes/project_builder.py @@ -6,7 +6,12 @@ from collections.abc import Callable from dataclasses import replace +from scripts.compiler_differential.classes.discovery_features import ( + DiscoveryFeatureWriter, + feature_blocks_for_seed, +) from scripts.compiler_differential.constants import ( + DUCKDB_ADAPTER, GENERATOR_CONSTANT_KIND, GENERATOR_CROSS_DOMAIN_SHARE, GENERATOR_DOMAINS, @@ -20,6 +25,7 @@ GENERATOR_MACRO_KIND, GENERATOR_NON_ASCII_DESCRIPTION_SHARE, GENERATOR_NON_ASCII_LABELS, + GENERATOR_RARE_FEATURE_BLOCKS, GENERATOR_REGION_ENV_VAR, GENERATOR_ROLES, GENERATOR_SOURCE_COLUMNS, @@ -36,8 +42,9 @@ class ProjectBuilder: """Accumulate the files of one generated project from a seeded random source.""" - def __init__(self, seed: int) -> None: + def __init__(self, seed: int, *, blocks: tuple[str, ...] | None = None) -> None: self._seed: int = seed + self._forced_blocks: tuple[str, ...] | None = blocks self._random: random.Random = random.Random(seed) self._files: dict[str, str] = {} self._features: set[str] = set() @@ -62,9 +69,15 @@ def build(self) -> GeneratedProject: self._plan_constant(index=index) for index in range(self._random.randint(1, 3)): self._plan_macro(index=index) + blocks: tuple[str, ...] = ( + feature_blocks_for_seed(seed=self._seed, rng=self._random) + if self._forced_blocks is None + else self._forced_blocks + ) invalid_kind: str | None = ( self._random.choice(sorted(GENERATOR_INVALID_CODES)) if self._random.random() < GENERATOR_INVALID_SHARE + and not set(blocks) & set(GENERATOR_RARE_FEATURE_BLOCKS) else None ) self._write_declarations() @@ -73,16 +86,28 @@ def build(self) -> GeneratedProject: self._write_python_nodes() self._write_tests_and_scenarios() self._write_singular_audit() - self._write_config(domains=domains) + extras: DiscoveryFeatureWriter = DiscoveryFeatureWriter( + blocks=blocks, + rng=self._random, + files=self._files, + features=self._features, + staging=[model for model in self._models if model.layer == _STAGING], + label=self._label, + ) + extras.write() + self._write_config(domains=domains, adapter=extras.adapter, extra_lines=extras.config_lines) if invalid_kind is not None: self._inject_invalid(kind=invalid_kind) return GeneratedProject( files=dict(sorted(self._files.items())), seed=self._seed, expected_error_code=( - None if invalid_kind is None else GENERATOR_INVALID_CODES[invalid_kind] + GENERATOR_INVALID_CODES[invalid_kind] + if invalid_kind is not None + else extras.expected_error_code ), features=tuple(sorted(self._features)), + succeeding_commands=extras.succeeding_commands, ) def _chance(self, share: float) -> bool: @@ -91,10 +116,12 @@ def _chance(self, share: float) -> bool: def _label(self) -> str: return self._random.choice(GENERATOR_NON_ASCII_LABELS) - def _write_config(self, *, domains: list[str]) -> None: + def _write_config( + self, *, domains: list[str], adapter: str | None, extra_lines: list[str] + ) -> None: lines: list[str] = [ f'name = "generated_{self._seed}"', - 'adapter = "duckdb"', + f'adapter = "{adapter or DUCKDB_ADAPTER}"', 'default_target = "dev"', "", "[connection]", @@ -130,6 +157,7 @@ def _write_config(self, *, domains: list[str]) -> None: ] ) self._features.add("path_defaults_wildcard") + lines.extend(extra_lines) self._files["sqlbuild_project.toml"] = "\n".join(lines) + "\n" def _write_sources(self) -> None: diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 012ee68423..7d64fbea60 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -1,5 +1,6 @@ """Stable constants for the compiler engine differential harness.""" +import json import re from scripts.compiler_differential.models import DifferentialCommand, ExpectedOutcome @@ -33,6 +34,7 @@ DEFAULT_SEED_COUNT: int = 12 PLAN_LABEL: str = "plan" ERROR_SEVERITY: str = "error" +WARNING_SEVERITY: str = "warning" MODEL_RESOURCE_TYPE: str = "model" DEFAULT_DENSE_MODELS: int = 3000 FIXTURE_ROOT: str = "tests/e2e/fixtures" @@ -179,8 +181,174 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: GENERATOR_MACRO_KIND: "macros", GENERATOR_HOOK_KIND: "hooks", } +GENERATOR_FEATURE_BLOCKS: tuple[str, ...] = ( + "incremental_append", + "incremental_delete_insert", + "incremental_merge", + "microbatch_watermark", + "microbatch_rolling_window", + "snapshot_timestamp", + "snapshot_check", + "python_hook", + "managed_source_loader", + "asset_and_check", + "audit_factory", + "provider", + "lifecycle_sink", + "command_output_sink", + "custom_materialization", + "project_adapter", + "model_schema", + "list_constant", + "cross_file_macro_import", + "macro_test_mode", + "parameterized_test", + "generic_audit", + "functions", + "local_config", + "target_override", + "dbt_ref", + "line_endings", +) +GENERATOR_FEATURE_STRIDE: int = 4 +GENERATOR_OPTIONAL_FEATURE_SHARE: float = 0.2 +GENERATOR_RARE_FEATURE_PERIOD: int = 12 +GENERATOR_RARE_FEATURE_BLOCKS: dict[str, int] = {"dbt_ref": 1} +GENERATOR_DBT_REF_ERROR_CODE: str = "C214" +GENERATOR_FEATURE_FOLDER: str = "models/features" +GENERATOR_PROJECT_ADAPTER: str = "generated_duckdb" +GENERATOR_CRLF: str = "\r\n" +GENERATOR_BOM: str = "\ufeff" +GENERATOR_DBT_MODEL: str = "upstream_orders" +GENERATOR_DBT_MANIFEST: str = ( + json.dumps( + { + "metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12.json"}, + "nodes": { + f"model.upstream.{GENERATOR_DBT_MODEL}": { + "resource_type": "model", + "package_name": "upstream", + "name": GENERATOR_DBT_MODEL, + "database": "generated", + "schema": "upstream", + "alias": GENERATOR_DBT_MODEL, + "fqn": ["upstream", GENERATOR_DBT_MODEL], + "checksum": {"name": "sha256", "checksum": "0" * 64}, + "depends_on": {"nodes": []}, + "config": {"materialized": "table"}, + "columns": { + "order_id": {"name": "order_id", "data_type": "integer"}, + "customer_id": {"name": "customer_id", "data_type": "integer"}, + }, + "compiled_code": "SELECT 1 AS order_id, 10 AS customer_id", + } + }, + "sources": {}, + }, + indent=2, + ) + + "\n" +) GENERATOR_SUFFIXES: dict[str, str] = { GENERATOR_ENUM_KIND: ".sql", GENERATOR_CONSTANT_KIND: ".sql", GENERATOR_MACRO_KIND: ".py", } +DISCOVERY_STAGE_CAPTURE_SUFFIX: str = "-discovered_project_inputs.json" +DISCOVERY_NON_COLLECTION_FIELDS: frozenset[str] = frozenset( + {"project_config", "local_config", "project_dir"} +) +DISCOVERY_DECLARATION_FILE_COLLECTIONS: tuple[str, ...] = ( + "enum_files", + "constant_files", + "model_schema_files", + "sql_hook_files", + "audit_files", + "macro_files", + "hook_functions", +) +DISCOVERY_TEXT_FILE_COLLECTIONS: tuple[str, ...] = ( + "model_files", + "enum_files", + "constant_files", + "model_schema_files", + "sql_function_files", + "sql_hook_files", + "python_function_files", + "schema_files", + "source_files", + "test_files", + "scenario_files", + "audit_files", + "macro_files", +) +DISCOVERY_DETAIL_KINDS: tuple[str, ...] = ( + "incremental_append", + "incremental_delete_insert", + "incremental_merge", + "microbatch_watermark", + "microbatch_rolling_window", + "snapshot_timestamp", + "snapshot_check", + "custom_materialized_model", + "model_local_declaration", + "dbt_ref_model", + "dbt_target_path_config", + "managed_source", + "unmanaged_source", + "scope_global", + "scope_inherited", + "scope_local", + "constant_string", + "constant_integer", + "constant_boolean", + "constant_float", + "constant_decimal", + "constant_list", + "non_ascii_constant", + "enum_string", + "enum_integer", + "cross_file_macro_import", + "test_mode_model", + "test_mode_macro", + "test_mode_udf", + "parameterized_test", + "singular_audit", + "generic_audit", + "path_defaults", + "local_config", + "target_override", + "project_adapter_config", + "crlf", + "tab", + "bom", + "non_ascii_comment", + "non_ascii_string", +) +MICROBATCH_MODE: str = "microbatch" +STRATEGY_HEADER_KEYS: dict[str, str] = { + "incremental": "incremental_strategy", + "snapshot": "snapshot_strategy", +} +DBT_REF_CALL: str = "__dbt_ref(" +SINGULAR_AUDIT_KIND: str = "singular_audit" +TAB: str = "\t" +CRLF_BYTES: bytes = b"\r\n" +TYPE_MARKER: str = "__type__" +CALLABLE_MARKER: str = "__callable__" +UNCANONICAL_CAPTURE_MARKERS: tuple[str, ...] = ("__opaque__", "__cycle__") +CALLABLE_CAPTURE_FIELDS: frozenset[str] = frozenset({"function", "provider_class"}) +COLLECTION_CAPTURE_MARKERS: frozenset[str] = frozenset( + {"__set__", "__mapping__", "__unordered_mapping__"} +) +PATH_MARKER: str = "__path__" +RELATIVE_PATH_FIELDS: frozenset[str] = frozenset({"relative_path"}) +CONFIG_ONLY_KINDS: dict[str, str] = { + "project_adapter_config": ( + "the config names a non-built-in adapter; adapters/ modules are resolved after " + "discovery and are not in the capture" + ), + "dbt_target_path_config": ( + "the config sets [dbt] target_path; the manifest is read after discovery" + ), +} diff --git a/scripts/compiler_differential/main/differential.py b/scripts/compiler_differential/main/differential.py index ea25888d9d..a7d7974f1a 100644 --- a/scripts/compiler_differential/main/differential.py +++ b/scripts/compiler_differential/main/differential.py @@ -9,15 +9,20 @@ import time from pathlib import Path -from scripts.compiler_differential._helpers.options import parse_expected_outcome -from scripts.compiler_differential._helpers.report import format_summary -from scripts.compiler_differential._helpers.run import ( +from scripts.compiler_differential._helpers.coverage.discovery import required_discovery_kinds +from scripts.compiler_differential._helpers.running.options import parse_expected_outcome +from scripts.compiler_differential._helpers.running.report import ( + format_discovery_coverage, + format_summary, +) +from scripts.compiler_differential._helpers.running.run import ( compare_corpus, differential_options, selected_corpus, ) from scripts.compiler_differential.constants import ( CORPUS_NAMES, + CORPUS_SEEDS, DEFAULT_DENSE_MODELS, DEFAULT_ENGINES, DEFAULT_SEED_COUNT, @@ -39,14 +44,24 @@ def run_compiler_differential(argv: list[str] | None = None) -> int: comparisons: list[ProjectComparison] = compare_corpus( corpus=selected_corpus(args=args, repo_root=repo_root), options=options ) + missing_coverage: tuple[str, ...] = () + if options.stage_captures and CORPUS_SEEDS in args.corpus: + covered: frozenset[str] = frozenset().union( + *(comparison.discovered_kinds for comparison in comparisons) + ) + required: tuple[str, ...] = required_discovery_kinds() + print(format_discovery_coverage(covered=covered, required=required)) + if options.require_discovery_coverage: + missing_coverage = tuple(kind for kind in required if kind not in covered) print( format_summary( comparisons=comparisons, engines=options.engines, seconds=time.monotonic() - started, + missing_coverage=missing_coverage, ) ) - return 1 if any(comparison.differences for comparison in comparisons) else 0 + return 1 if missing_coverage or any(comparison.differences for comparison in comparisons) else 0 def _parse_args(argv: list[str] | None) -> argparse.Namespace: @@ -79,6 +94,11 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: action="store_true", help="also dump and diff every frontier object to localise a mismatch to a stage", ) + parser.add_argument( + "--require-discovery-coverage", + action="store_true", + help="fail unless the seed corpus exercises every discovery input kind (needs captures)", + ) parser.add_argument( "--engine-env", action="append", @@ -88,4 +108,9 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: ) parser.add_argument("--work-dir", type=Path, default=None, help="keep run directories here") parser.add_argument("--python", type=Path, default=Path(sys.executable)) - return parser.parse_args(argv) + args: argparse.Namespace = parser.parse_args(argv) + if args.require_discovery_coverage and not ( + args.stage_captures and CORPUS_SEEDS in args.corpus + ): + parser.error("--require-discovery-coverage needs --stage-captures and the seeds corpus") + return args diff --git a/scripts/compiler_differential/main/failure_cases.py b/scripts/compiler_differential/main/failure_cases.py index 5d6352bdf7..3d912a813f 100644 --- a/scripts/compiler_differential/main/failure_cases.py +++ b/scripts/compiler_differential/main/failure_cases.py @@ -1,6 +1,6 @@ """The failure corpus: one minimal failing project per diagnostic family member.""" -from scripts.compiler_differential._helpers.failure_cases import all_failure_cases +from scripts.compiler_differential._helpers.corpus.failure_cases import all_failure_cases from scripts.compiler_differential.models import FailureCase diff --git a/scripts/compiler_differential/main/generate_project.py b/scripts/compiler_differential/main/generate_project.py index 5dd0c654d9..76f581b12a 100644 --- a/scripts/compiler_differential/main/generate_project.py +++ b/scripts/compiler_differential/main/generate_project.py @@ -4,7 +4,7 @@ from scripts.compiler_differential.models import GeneratedProject -def generate_project(*, seed: int) -> GeneratedProject: - """Return the same project files and expectation for the same seed on every platform.""" +def generate_project(*, seed: int, blocks: tuple[str, ...] | None = None) -> GeneratedProject: + """Return the seed's deterministic project; `blocks` overrides its optional feature blocks.""" - return ProjectBuilder(seed).build() + return ProjectBuilder(seed, blocks=blocks).build() diff --git a/scripts/compiler_differential/models.py b/scripts/compiler_differential/models.py index 7da950905d..cb2bf0b149 100644 --- a/scripts/compiler_differential/models.py +++ b/scripts/compiler_differential/models.py @@ -17,9 +17,11 @@ class DifferentialCommand: @dataclass(frozen=True) class ExpectedOutcome: - """What a corpus project must produce: success, or failure with one diagnostic code.""" + """Success, or failure whose first error is `error_code`, plus an optional expected warning.""" error_code: str | None = None + succeeding_commands: tuple[str, ...] = () + warning_code: str | None = None @dataclass(frozen=True) @@ -32,6 +34,7 @@ class CorpusProject: source_dir: Path | None = None writer: Callable[[Path], None] | None = None project_subdirectory: str | None = None + discovery_coverage: bool = False @dataclass(frozen=True) @@ -85,6 +88,7 @@ class ProjectComparison: project: str differences: tuple[Difference, ...] seconds: float + discovered_kinds: frozenset[str] = frozenset() @dataclass(frozen=True) @@ -97,6 +101,7 @@ class DifferentialOptions: stage_captures: bool python: Path engine_environment: dict[str, dict[str, str]] + require_discovery_coverage: bool = False @dataclass(frozen=True) @@ -121,6 +126,7 @@ class GeneratedProject(WritableProject): seed: int expected_error_code: str | None features: tuple[str, ...] + succeeding_commands: tuple[str, ...] = () @dataclass(frozen=True) @@ -129,6 +135,7 @@ class FailureCase(WritableProject): name: str expected_code: str + expected_warning_code: str | None = None @dataclass(frozen=True) @@ -157,3 +164,17 @@ class DeclarationPlan: consumers: tuple[ModelPlan, ...] body: str = "" private: bool = False + + +@dataclass(frozen=True) +class EmittedCodes: + """The error and warning codes one compile reported, in report order.""" + + errors: tuple[str, ...] + warnings: tuple[str, ...] + + @property + def first_error(self) -> str | None: + """Return the first reported error code, if any.""" + + return self.errors[0] if self.errors else None diff --git a/tests/e2e/scripts/compiler_differential/_test_types.py b/tests/e2e/scripts/compiler_differential/_test_types.py index 9e0b02f814..e26a034b4b 100644 --- a/tests/e2e/scripts/compiler_differential/_test_types.py +++ b/tests/e2e/scripts/compiler_differential/_test_types.py @@ -25,3 +25,34 @@ class ProjectExpectationTestCase: extra_arguments: tuple[str, ...] expected_exit_code: int expected_lines: tuple[str, ...] + + +@dataclass(frozen=True) +class DiscoveryCaptureTestCase: + """A generated project with every feature block and what its discovery capture must hold.""" + + description: str + seed: int + blocks: tuple[str, ...] + expected_collections: frozenset[str] + + +@dataclass(frozen=True) +class FailureCorpusCodesTestCase: + """The failure corpus run for real and the codes its cases must emit.""" + + description: str + expected_first_errors: dict[str, str] + expected_warnings: dict[str, str] + expected_discovery_codes: frozenset[str] + + +@dataclass(frozen=True) +class CoverageFailureTestCase: + """A seed range too small for full coverage and how the harness must report it.""" + + description: str + extra_arguments: tuple[str, ...] + expected_exit_code: int + expected_lines: tuple[str, ...] + expected_absent: tuple[str, ...] diff --git a/tests/e2e/scripts/compiler_differential/helpers.py b/tests/e2e/scripts/compiler_differential/helpers.py index a5488ddd3c..efadb94b07 100644 --- a/tests/e2e/scripts/compiler_differential/helpers.py +++ b/tests/e2e/scripts/compiler_differential/helpers.py @@ -4,6 +4,8 @@ from pathlib import Path +from scripts.compiler_differential.main.failure_cases import failure_cases + NATIVE_ONLY_STDERR_LINE: str = "native engine perturbation active" WAFFLE_SHOP_FIXTURE: Path = Path(__file__).resolve().parents[2] / "fixtures" / "waffle_shop" _SITECUSTOMIZE: str = f''' @@ -31,6 +33,25 @@ def _perturbed(*, until, python_stage): print("{NATIVE_ONLY_STDERR_LINE}", file=sys.stderr) ''' +DISCOVERY_PERTURBATION: str = """ +import dataclasses + +import sqlbuild.compiler.frontier.main._compile_frontier as frontier +from sqlbuild.compiler.frontier.types import CompilerStage + +_original = frontier.native_frontier + + +def _perturbed(*, until, python_stage): + result = _original(until=until, python_stage=python_stage) + if until is not CompilerStage.DISCOVERED_PROJECT_INPUTS or len(result.model_files) < 2: + return result + return dataclasses.replace(result, model_files=tuple(reversed(result.model_files))) + + +frontier.native_frontier = _perturbed +""" + _BROKEN_REF_FILES: dict[str, str] = { "sqlbuild_project.toml": ( 'name = "broken_orders"\nadapter = "duckdb"\n\n' @@ -52,11 +73,21 @@ def write_broken_ref_project(directory: Path) -> Path: return directory -def write_native_perturbation(directory: Path) -> Path: - """Write a sitecustomize module that changes the first compiled model on the native path.""" +def write_failure_case_project(directory: Path, *, name: str) -> Path: + """Write one named failure-corpus project below a directory.""" + + {case.name: case for case in failure_cases()}[name].write(directory) + return directory + + +def write_native_perturbation(directory: Path, *, source: str = _SITECUSTOMIZE) -> Path: + """Write a sitecustomize module that perturbs one frontier object on the native path. + + By default it changes the first compiled model. + """ directory.mkdir(parents=True, exist_ok=True) - _ = (directory / "sitecustomize.py").write_text(_SITECUSTOMIZE, encoding="utf-8") + _ = (directory / "sitecustomize.py").write_text(source, encoding="utf-8") return directory @@ -77,11 +108,11 @@ def harness_arguments( ] -def perturbation_arguments(directory: Path) -> tuple[str, ...]: +def perturbation_arguments(directory: Path, *, source: str = _SITECUSTOMIZE) -> tuple[str, ...]: """Return the options that load the perturbation into native-engine processes only.""" return ( "--stage-captures", "--engine-env", - f"native:PYTHONPATH={write_native_perturbation(directory)}", + f"native:PYTHONPATH={write_native_perturbation(directory, source=source)}", ) diff --git a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py index 54e646dd33..204b4a628c 100644 --- a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py +++ b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py @@ -9,14 +9,17 @@ from scripts.compiler_differential.main.differential import run_compiler_differential from tests.e2e.scripts.compiler_differential._test_types import ( + CoverageFailureTestCase, HarnessRunTestCase, ProjectExpectationTestCase, ) from tests.e2e.scripts.compiler_differential.helpers import ( + DISCOVERY_PERTURBATION, NATIVE_ONLY_STDERR_LINE, harness_arguments, perturbation_arguments, write_broken_ref_project, + write_failure_case_project, ) @@ -93,6 +96,39 @@ def test_given_perturbed_native_engine_when_comparing_then_every_artifact_is_loc assert not any(text in output for text in test_case.expected_absent), output +@pytest.mark.parametrize( + "test_case", + [ + HarnessRunTestCase( + description="perturbed_native_discovery", + extra_arguments=(), + expected_exit_code=1, + expected_lines=("DIFF project/waffle_shop",), + expected_patterns=( + r"- stage capture 0-compile/001-discovered_project_inputs\.json at /model_files/0/", + r"- stage capture 2-plan/001-discovered_project_inputs\.json at /model_files/0/", + ), + expected_absent=(), + ) + ], + ids=lambda case: case.description, +) +def test_given_perturbed_native_discovery_when_comparing_then_discovery_capture_names_the_stage( + test_case: HarnessRunTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + exit_code: int = run_compiler_differential( + harness_arguments( + work_dir=tmp_path / "work", + extra=perturbation_arguments(tmp_path / "perturbation", source=DISCOVERY_PERTURBATION), + ) + ) + + output: str = capsys.readouterr().out + assert exit_code == test_case.expected_exit_code, output + assert all(line in output for line in test_case.expected_lines), output + assert all(re.search(pattern, output) for pattern in test_case.expected_patterns), output + + @pytest.mark.parametrize( "test_case", [ @@ -139,5 +175,81 @@ def test_given_both_engines_fail_alike_when_outcome_is_unexpected_then_harness_f assert all(line in output for line in test_case.expected_lines), output +@pytest.mark.parametrize( + "test_case", + [ + CoverageFailureTestCase( + description="one_seed_cannot_cover_every_kind", + extra_arguments=( + "--corpus", + "seeds", + "--seeds", + "1", + "--stage-captures", + "--require-discovery-coverage", + ), + expected_exit_code=1, + expected_lines=( + "OK seed/0", + "Compiler differential FAILED: 0 of 1 projects differ", + "Required discovery coverage missing: ", + ), + expected_absent=("passed",), + ) + ], + ids=lambda case: case.description, +) +def test_given_required_coverage_missing_when_comparing_then_harness_fails_and_says_so( + test_case: CoverageFailureTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + exit_code: int = run_compiler_differential( + ["--jobs", "1", "--work-dir", str(tmp_path / "work"), *test_case.extra_arguments] + ) + + output: str = capsys.readouterr().out + assert exit_code == test_case.expected_exit_code, output + assert all(line in output for line in test_case.expected_lines), output + assert not any(text in output for text in test_case.expected_absent), output + + +@pytest.mark.parametrize( + "test_case", + [ + ProjectExpectationTestCase( + description="expected_code_only_reported_as_warning", + extra_arguments=("--expect", "failure:P003"), + expected_exit_code=1, + expected_lines=( + "DIFF project/built-in-audit-shadow", + "- corpus expectation at expected failure", + ), + ), + ProjectExpectationTestCase( + description="expected_code_is_the_first_error", + extra_arguments=("--expect", "failure:S010"), + expected_exit_code=0, + expected_lines=("OK project/built-in-audit-shadow",), + ), + ], + ids=lambda case: case.description, +) +def test_given_failure_expectation_when_code_is_not_the_first_error_then_harness_fails( + test_case: ProjectExpectationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + exit_code: int = run_compiler_differential( + harness_arguments( + work_dir=tmp_path / "work", + extra=test_case.extra_arguments, + project=write_failure_case_project( + tmp_path / "built-in-audit-shadow", name="built-in-audit-shadow" + ), + ) + ) + + output: str = capsys.readouterr().out + assert exit_code == test_case.expected_exit_code, output + assert all(line in output for line in test_case.expected_lines), output + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/scripts/compiler_differential/test_discovery_capture.py b/tests/e2e/scripts/compiler_differential/test_discovery_capture.py new file mode 100644 index 0000000000..f455b3d8cb --- /dev/null +++ b/tests/e2e/scripts/compiler_differential/test_discovery_capture.py @@ -0,0 +1,64 @@ +"""A real compile captures every discovered collection canonically, with callables as names.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +from scripts.compiler_differential._helpers.coverage.discovery import ( + discovered_input_kinds, + discovery_capture_problems, + discovery_collection_kinds, +) +from scripts.compiler_differential._helpers.running.execution import harness_environment +from scripts.compiler_differential.constants import ( + DISCOVERY_STAGE_CAPTURE_SUFFIX, + GENERATOR_FEATURE_BLOCKS, + SQB_ENTRY, + STAGE_CAPTURE_ENV_VAR, +) +from scripts.compiler_differential.main.generate_project import generate_project +from tests.e2e.scripts.compiler_differential._test_types import DiscoveryCaptureTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + DiscoveryCaptureTestCase( + description="every_feature_block", + seed=11, + blocks=GENERATOR_FEATURE_BLOCKS, + expected_collections=frozenset(discovery_collection_kinds()), + ) + ], + ids=lambda case: case.description, +) +def test_given_project_with_every_input_kind_when_capturing_discovery_then_capture_is_complete( + test_case: DiscoveryCaptureTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "project" + capture_dir: Path = tmp_path / "captures" + generate_project(seed=test_case.seed, blocks=test_case.blocks).write(project_dir) + + _ = subprocess.run( + [sys.executable, "-c", SQB_ENTRY, "--no-color", "compile", "--json"], + cwd=project_dir, + env={**harness_environment(), STAGE_CAPTURE_ENV_VAR: str(capture_dir)}, + capture_output=True, + text=True, + check=False, + timeout=600, + ) + + text: str = next(capture_dir.glob(f"*{DISCOVERY_STAGE_CAPTURE_SUFFIX}")).read_text( + encoding="utf-8" + ) + assert discovery_capture_problems(text) == () + assert test_case.expected_collections <= discovered_input_kinds(text) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py b/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py new file mode 100644 index 0000000000..f4c96b1253 --- /dev/null +++ b/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py @@ -0,0 +1,58 @@ +"""Every failure case emits its declared code as the first error, reaching every discovery code.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +from scripts.compiler_differential._helpers.corpus.emitted_codes import ( + discovery_error_codes, + emitted_failure_codes, +) +from scripts.compiler_differential.main.failure_cases import failure_cases +from scripts.compiler_differential.models import DifferentialOptions, EmittedCodes +from tests.e2e.scripts.compiler_differential._test_types import FailureCorpusCodesTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + FailureCorpusCodesTestCase( + description="python_engine", + expected_first_errors={ + f"failure/{case.name}": case.expected_code for case in failure_cases() + }, + expected_warnings={"failure/built-in-audit-shadow": "P003"}, + expected_discovery_codes=discovery_error_codes(), + ) + ], + ids=lambda case: case.description, +) +def test_given_failure_corpus_when_compiling_then_emitted_first_errors_match_and_cover_discovery( + test_case: FailureCorpusCodesTestCase, tmp_path: Path +) -> None: + emitted: dict[str, EmittedCodes] = emitted_failure_codes( + options=DifferentialOptions( + engines=("python", "python"), + work_dir=tmp_path, + jobs=4, + stage_captures=False, + python=Path(sys.executable), + engine_environment={}, + ) + ) + + first_errors: dict[str, str | None] = { + name: codes.first_error for name, codes in emitted.items() + } + assert first_errors == test_case.expected_first_errors + assert all( + code in emitted[name].warnings for name, code in test_case.expected_warnings.items() + ), emitted + assert test_case.expected_discovery_codes <= set(first_errors.values()) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/_test_types.py deleted file mode 100644 index d605e21b48..0000000000 --- a/tests/unit/scripts/compiler_differential/_helpers/_test_types.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Test case types for the differential harness comparison helpers.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from scripts.compiler_differential.models import ExpectedOutcome - - -@dataclass(frozen=True) -class JsonDifferenceTestCase: - """Two JSON documents and where the harness must report their first difference.""" - - description: str - left: object - right: object - expected_location: str | None - - -@dataclass(frozen=True) -class TextDifferenceTestCase: - """Two texts and the first differing line the harness must report.""" - - description: str - left: str - right: str - expected_location: str | None - - -@dataclass(frozen=True) -class PreviewTestCase: - """A long differing value and the bounded preview the report shows.""" - - description: str - left: object - right: object - expected_max_length: int - expected_suffix: str - - -@dataclass(frozen=True) -class NormalizationTestCase: - """Raw output and the normalized text the harness compares.""" - - description: str - raw: str - expected_text: str - - -@dataclass(frozen=True) -class PayloadStripTestCase: - """A decoded report or manifest and what remains once volatile fields are removed.""" - - description: str - payload: object - expected_payload: object - - -@dataclass(frozen=True) -class EngineEnvironmentTestCase: - """Repeated --engine-env values and the per-engine overrides they produce.""" - - description: str - values: tuple[str, ...] - expected_environment: dict[str, dict[str, str]] - - -@dataclass(frozen=True) -class EngineEnvironmentErrorTestCase: - """A malformed --engine-env value and the usage error it raises.""" - - description: str - value: str - expected_message: str - - -@dataclass(frozen=True) -class ExpectedOutcomeTestCase: - """One --expect value and the outcome it parses to.""" - - description: str - value: str - expected_outcome: ExpectedOutcome - - -@dataclass(frozen=True) -class ExpectedOutcomeErrorTestCase: - """One malformed --expect value.""" - - description: str - value: str - expected_message: str - - -@dataclass(frozen=True) -class CorpusExpectationTestCase: - """A corpus selection and the expectation each named entry must declare.""" - - description: str - corpora: tuple[str, ...] - expected_outcomes: dict[str, ExpectedOutcome] diff --git a/tests/unit/scripts/compiler_differential/_helpers/comparing/__init__.py b/tests/unit/scripts/compiler_differential/_helpers/comparing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/compiler_differential/_helpers/comparing/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/comparing/_test_types.py new file mode 100644 index 0000000000..f799bdcdf4 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/comparing/_test_types.py @@ -0,0 +1,54 @@ +"""Test case types for the differential harness comparing helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class JsonDifferenceTestCase: + """Two JSON documents and where the harness must report their first difference.""" + + description: str + left: object + right: object + expected_location: str | None + + +@dataclass(frozen=True) +class TextDifferenceTestCase: + """Two texts and the first differing line the harness must report.""" + + description: str + left: str + right: str + expected_location: str | None + + +@dataclass(frozen=True) +class PreviewTestCase: + """A long differing value and the bounded preview the report shows.""" + + description: str + left: object + right: object + expected_max_length: int + expected_suffix: str + + +@dataclass(frozen=True) +class NormalizationTestCase: + """Raw output and the normalized text the harness compares.""" + + description: str + raw: str + expected_text: str + + +@dataclass(frozen=True) +class PayloadStripTestCase: + """A decoded report or manifest and what remains once volatile fields are removed.""" + + description: str + payload: object + expected_payload: object diff --git a/tests/unit/scripts/compiler_differential/_helpers/test_compare.py b/tests/unit/scripts/compiler_differential/_helpers/comparing/test_compare.py similarity index 96% rename from tests/unit/scripts/compiler_differential/_helpers/test_compare.py rename to tests/unit/scripts/compiler_differential/_helpers/comparing/test_compare.py index 44c711296c..06426329f1 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/test_compare.py +++ b/tests/unit/scripts/compiler_differential/_helpers/comparing/test_compare.py @@ -4,12 +4,12 @@ import pytest -from scripts.compiler_differential._helpers.compare import ( +from scripts.compiler_differential._helpers.comparing.compare import ( first_json_difference, first_text_difference, ) from scripts.compiler_differential.models import Divergence -from tests.unit.scripts.compiler_differential._helpers._test_types import ( +from tests.unit.scripts.compiler_differential._helpers.comparing._test_types import ( JsonDifferenceTestCase, PreviewTestCase, TextDifferenceTestCase, diff --git a/tests/unit/scripts/compiler_differential/_helpers/test_normalize.py b/tests/unit/scripts/compiler_differential/_helpers/comparing/test_normalize.py similarity index 95% rename from tests/unit/scripts/compiler_differential/_helpers/test_normalize.py rename to tests/unit/scripts/compiler_differential/_helpers/comparing/test_normalize.py index a89a893598..05f84df19b 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/test_normalize.py +++ b/tests/unit/scripts/compiler_differential/_helpers/comparing/test_normalize.py @@ -4,13 +4,13 @@ import pytest -from scripts.compiler_differential._helpers.normalize import ( +from scripts.compiler_differential._helpers.comparing.normalize import ( normalize_artifact_text, normalize_stderr, strip_manifest_metadata, strip_report_fields, ) -from tests.unit.scripts.compiler_differential._helpers._test_types import ( +from tests.unit.scripts.compiler_differential._helpers.comparing._test_types import ( NormalizationTestCase, PayloadStripTestCase, ) diff --git a/tests/unit/scripts/compiler_differential/_helpers/corpus/__init__.py b/tests/unit/scripts/compiler_differential/_helpers/corpus/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py new file mode 100644 index 0000000000..eab6d1cbbc --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py @@ -0,0 +1,16 @@ +"""Test case types for the differential harness corpus helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from scripts.compiler_differential.models import ExpectedOutcome + + +@dataclass(frozen=True) +class CorpusExpectationTestCase: + """A corpus selection and the expectation each named entry must declare.""" + + description: str + corpora: tuple[str, ...] + expected_outcomes: dict[str, ExpectedOutcome] diff --git a/tests/unit/scripts/compiler_differential/_helpers/test_corpus.py b/tests/unit/scripts/compiler_differential/_helpers/corpus/test_corpus.py similarity index 90% rename from tests/unit/scripts/compiler_differential/_helpers/test_corpus.py rename to tests/unit/scripts/compiler_differential/_helpers/corpus/test_corpus.py index f35dfd35e1..e1f4c0299b 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/test_corpus.py +++ b/tests/unit/scripts/compiler_differential/_helpers/corpus/test_corpus.py @@ -6,13 +6,13 @@ import pytest -from scripts.compiler_differential._helpers.corpus import build_corpus +from scripts.compiler_differential._helpers.corpus.corpus import build_corpus from scripts.compiler_differential.models import CorpusProject, ExpectedOutcome -from tests.unit.scripts.compiler_differential._helpers._test_types import ( +from tests.unit.scripts.compiler_differential._helpers.corpus._test_types import ( CorpusExpectationTestCase, ) -_REPO_ROOT: Path = Path(__file__).resolve().parents[5] +_REPO_ROOT: Path = Path(__file__).resolve().parents[6] @pytest.mark.parametrize( diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/__init__.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py new file mode 100644 index 0000000000..ed46332262 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py @@ -0,0 +1,53 @@ +"""Test case types for discovery coverage classification.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs + + +@dataclass(frozen=True) +class DiscoveryKindsTestCase: + """One discovery capture fragment and the input kinds it must prove.""" + + description: str + capture: dict[str, object] + expected_present: frozenset[str] + expected_absent: frozenset[str] + + +@dataclass(frozen=True) +class AuthoredByteKindsTestCase: + """Authored files, the paths discovery read, and whether CRLF must be credited.""" + + description: str + files: dict[str, bytes] + read_paths: tuple[str, ...] + expected_crlf: bool + + +@dataclass(frozen=True) +class RequiredKindsTestCase: + """Collections that must be required because discovery produces them.""" + + description: str + expected_required: frozenset[str] + + +@dataclass(frozen=True) +class CaptureProblemsTestCase: + """A discovery capture and the problems that make it incomplete or not canonical.""" + + description: str + capture: dict[str, object] + expected_problem_count: int + + +@dataclass(frozen=True) +class EncodedInputsKindsTestCase: + """Real discovered inputs, encoded by the stage capture, and the kinds they prove.""" + + description: str + inputs: DiscoveredProjectInputs + expected_kinds: frozenset[str] diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py new file mode 100644 index 0000000000..6774d6a053 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py @@ -0,0 +1,27 @@ +"""Capture fragments for discovery coverage tests.""" + +from __future__ import annotations + +import dataclasses + +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs + +_VALUE_KIND: str = "sqlbuild.sql_values.types:SqlValueKind" + + +def constant_declaration(*, kind: str, value: object) -> dict[str, object]: + """Return one captured constant declaration of a logical kind.""" + + return { + "value": {"logical_type": {"kind": {"__enum__": f"{_VALUE_KIND}.{kind}"}}, "value": value} + } + + +def empty_inputs_capture(**overrides: object) -> dict[str, object]: + """Return a capture with every `DiscoveredProjectInputs` field, in order, then overrides.""" + + return { + "__type__": "sqlbuild.compiler.discovery.models:DiscoveredProjectInputs", + **{field.name: [] for field in dataclasses.fields(DiscoveredProjectInputs)}, + **overrides, + } diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py new file mode 100644 index 0000000000..8acee8e92d --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_discovery.py @@ -0,0 +1,305 @@ +"""Discovery captures are classified into the input kinds the seed corpus must exercise.""" + +from __future__ import annotations + +import dataclasses +import json +from pathlib import Path + +import pytest + +from scripts.compiler_differential._helpers.coverage.discovery import ( + discovered_input_kinds, + discovery_capture_problems, + project_discovery_kinds, + required_discovery_kinds, +) +from scripts.compiler_differential.constants import DISCOVERY_NON_COLLECTION_FIELDS +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture +from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig +from tests.unit.scripts.compiler_differential._helpers.coverage._test_types import ( + AuthoredByteKindsTestCase, + CaptureProblemsTestCase, + DiscoveryKindsTestCase, + EncodedInputsKindsTestCase, + RequiredKindsTestCase, +) +from tests.unit.scripts.compiler_differential._helpers.coverage.helpers import ( + constant_declaration, + empty_inputs_capture, +) + +_SCOPE: str = "sqlbuild.compiler.scopes.types:ScopeKind" + + +@pytest.mark.parametrize( + "test_case", + [ + DiscoveryKindsTestCase( + description="collections_and_model_strategies", + capture={ + "project_config": {"adapter": "duckdb", "path_defaults": {"staging": {}}}, + "local_config": {"target": None, "vars": {}}, + "model_files": [ + { + "header_values": { + "materialized": "incremental", + "incremental_strategy": "merge", + } + }, + { + "header_values": { + "materialized": "incremental", + "incremental_mode": "microbatch", + "microbatch_strategy": "watermark", + } + }, + {"header_values": {"materialized": "snapshot", "snapshot_strategy": "check"}}, + {"header_values": {"materialized": "copy_table"}, "contents": "-- café\n"}, + ], + "materialization_files": [{"name": "copy_table"}], + "adapter_file": {"relative_path": {"__path__": "adapter.py"}}, + }, + expected_present=frozenset( + { + "model_files", + "materialization_files", + "adapter_file", + "incremental_merge", + "microbatch_watermark", + "snapshot_check", + "custom_materialized_model", + "path_defaults", + "non_ascii_comment", + } + ), + expected_absent=frozenset( + { + "incremental_delete_insert", + "local_config", + "project_adapter_config", + "enum_files", + } + ), + ), + DiscoveryKindsTestCase( + description="declarations_tests_and_config", + capture={ + "project_config": { + "adapter": "orders_duckdb", + "targets": {"dev": {}, "ci": {}}, + "dbt": {"target_path": "dbt/artifacts"}, + }, + "local_config": {"target": "ci", "vars": {"region": "north"}}, + "constant_files": [ + { + "scope_kind": {"__enum__": f"{_SCOPE}.INHERITED"}, + "contents": "\ufeffCONSTANT (name labels, value ['東京']);\n", + "declarations": [ + constant_declaration(kind="LIST", value=["東京"]), + constant_declaration(kind="FLOAT", value=0.5), + ], + } + ], + "enum_files": [{"declarations": [{"members": [{"name": "LOW", "value": 1}]}]}], + "macro_files": [ + {"contents": "from macros.rounding import round_amount\n"}, + ], + "test_files": [ + { + "contents": "TEST(\n\tmode macro);", + "blocks": [ + { + "mode": { + "__enum__": "sqlbuild.compiler.compile.types:SqlTestMode.MACRO" + }, + "cases": [{"name": "placed"}], + } + ], + } + ], + "audit_files": [ + { + "declaration_kind": { + "__enum__": "sqlbuild.compiler.scopes.types:DeclarationKind.SINGULAR_AUDIT" + } + } + ], + "source_files": [{"source_entries": [{"managed": True}, {"managed": False}]}], + }, + expected_present=frozenset( + { + "scope_inherited", + "constant_list", + "constant_float", + "non_ascii_constant", + "enum_integer", + "cross_file_macro_import", + "test_mode_macro", + "parameterized_test", + "singular_audit", + "managed_source", + "unmanaged_source", + "local_config", + "target_override", + "dbt_target_path_config", + "project_adapter_config", + "bom", + "tab", + "non_ascii_string", + } + ), + expected_absent=frozenset({"generic_audit", "enum_string", "scope_global"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_discovery_capture_when_classifying_then_kinds_follow_discovered_facts( + test_case: DiscoveryKindsTestCase, +) -> None: + kinds: frozenset[str] = discovered_input_kinds(json.dumps(test_case.capture)) + + assert test_case.expected_present <= kinds + assert not test_case.expected_absent & kinds + + +@pytest.mark.parametrize( + "test_case", + [ + EncodedInputsKindsTestCase( + description="project_without_any_optional_input", + inputs=DiscoveredProjectInputs( + project_config=ProjectConfig(name="orders", adapter="duckdb"), + local_config=LocalConfig(), + ), + expected_kinds=frozenset(), + ), + EncodedInputsKindsTestCase( + description="local_config_with_only_an_empty_override_set", + inputs=DiscoveredProjectInputs( + project_config=ProjectConfig(name="orders", adapter="duckdb"), + local_config=LocalConfig(setting_overrides=frozenset()), + ), + expected_kinds=frozenset(), + ), + EncodedInputsKindsTestCase( + description="local_config_with_a_setting_override", + inputs=DiscoveredProjectInputs( + project_config=ProjectConfig(name="orders", adapter="duckdb"), + local_config=LocalConfig(setting_overrides=frozenset({"sql_analysis"})), + ), + expected_kinds=frozenset({"local_config"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_encoded_inputs_when_classifying_then_encoded_empty_values_prove_nothing( + test_case: EncodedInputsKindsTestCase, +) -> None: + assert discovered_input_kinds(render_stage_capture(test_case.inputs)) == ( + test_case.expected_kinds + ) + + +@pytest.mark.parametrize( + "test_case", + [ + AuthoredByteKindsTestCase( + description="crlf_in_discovered_model", + files={"models/orders.sql": b"MODEL ();\r\nSELECT 1\r\n"}, + read_paths=("models/orders.sql",), + expected_crlf=True, + ), + AuthoredByteKindsTestCase( + description="lf_only", + files={"models/orders.sql": b"MODEL ();\nSELECT 1\n"}, + read_paths=("models/orders.sql",), + expected_crlf=False, + ), + AuthoredByteKindsTestCase( + description="crlf_only_in_a_file_discovery_did_not_read", + files={ + "models/orders.sql": b"MODEL ();\nSELECT 1\n", + "notes/readme.txt": b"orders\r\n", + }, + read_paths=("models/orders.sql",), + expected_crlf=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_authored_bytes_when_classifying_then_crlf_counts_only_for_discovered_files( + test_case: AuthoredByteKindsTestCase, tmp_path: Path +) -> None: + for relative_path, contents in test_case.files.items(): + path: Path = tmp_path / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_bytes(contents) + capture: str = json.dumps( + empty_inputs_capture( + model_files=[{"relative_path": {"__path__": path}} for path in test_case.read_paths] + ) + ) + + kinds: frozenset[str] = project_discovery_kinds( + captures={"0-compile": {"001-discovered_project_inputs.json": capture}}, source_dir=tmp_path + ) + + assert ("crlf" in kinds) is test_case.expected_crlf + + +@pytest.mark.parametrize( + "test_case", + [ + RequiredKindsTestCase( + description="every_discovered_collection", + expected_required=frozenset( + field.name for field in dataclasses.fields(DiscoveredProjectInputs) + ) + - DISCOVERY_NON_COLLECTION_FIELDS, + ) + ], + ids=lambda case: case.description, +) +def test_given_discovered_project_inputs_when_listing_required_kinds_then_collections_are_required( + test_case: RequiredKindsTestCase, +) -> None: + assert test_case.expected_required <= set(required_discovery_kinds()) + + +@pytest.mark.parametrize( + "test_case", + [ + CaptureProblemsTestCase( + description="sound_capture", + capture=empty_inputs_capture( + task_functions=[{"function": {"__callable__": "python.tasks.orders:export"}}] + ), + expected_problem_count=0, + ), + CaptureProblemsTestCase( + description="opaque_value_and_unnamed_callable", + capture=empty_inputs_capture( + task_functions=[{"function": {"__opaque__": "orders:Exporter"}}] + ), + expected_problem_count=2, + ), + CaptureProblemsTestCase( + description="missing_field", + capture={"__type__": "DiscoveredProjectInputs", "model_files": []}, + expected_problem_count=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_discovery_capture_when_checking_then_incomplete_or_opaque_parts_are_reported( + test_case: CaptureProblemsTestCase, +) -> None: + problems: tuple[str, ...] = discovery_capture_problems(json.dumps(test_case.capture)) + + assert len(problems) == test_case.expected_problem_count, problems + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/running/__init__.py b/tests/unit/scripts/compiler_differential/_helpers/running/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py new file mode 100644 index 0000000000..0249e11836 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py @@ -0,0 +1,64 @@ +"""Test case types for the differential harness running helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from scripts.compiler_differential.models import ExpectedOutcome, ProjectComparison + + +@dataclass(frozen=True) +class EngineEnvironmentTestCase: + """Repeated --engine-env values and the per-engine overrides they produce.""" + + description: str + values: tuple[str, ...] + expected_environment: dict[str, dict[str, str]] + + +@dataclass(frozen=True) +class EngineEnvironmentErrorTestCase: + """A malformed --engine-env value and the usage error it raises.""" + + description: str + value: str + expected_message: str + + +@dataclass(frozen=True) +class ExpectedOutcomeTestCase: + """One --expect value and the outcome it parses to.""" + + description: str + value: str + expected_outcome: ExpectedOutcome + + +@dataclass(frozen=True) +class ExpectedOutcomeErrorTestCase: + """One malformed --expect value.""" + + description: str + value: str + expected_message: str + + +@dataclass(frozen=True) +class CoverageReportTestCase: + """Covered kinds and the coverage report lines the harness must print.""" + + description: str + covered: frozenset[str] + required: tuple[str, ...] + expected_lines: tuple[str, ...] + + +@dataclass(frozen=True) +class SummaryTestCase: + """Project results, missing required coverage, and the summary the harness must print.""" + + description: str + comparisons: list[ProjectComparison] + missing_coverage: tuple[str, ...] + expected_lines: tuple[str, ...] + expected_absent: tuple[str, ...] diff --git a/tests/unit/scripts/compiler_differential/_helpers/test_options.py b/tests/unit/scripts/compiler_differential/_helpers/running/test_options.py similarity index 95% rename from tests/unit/scripts/compiler_differential/_helpers/test_options.py rename to tests/unit/scripts/compiler_differential/_helpers/running/test_options.py index 83a3bd61ac..8c7e4d3423 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/test_options.py +++ b/tests/unit/scripts/compiler_differential/_helpers/running/test_options.py @@ -6,13 +6,13 @@ import pytest -from scripts.compiler_differential._helpers.options import ( +from scripts.compiler_differential._helpers.running.options import ( parse_engine_environment, parse_expected_outcome, ) from scripts.compiler_differential.exceptions import DifferentialUsageError from scripts.compiler_differential.models import ExpectedOutcome -from tests.unit.scripts.compiler_differential._helpers._test_types import ( +from tests.unit.scripts.compiler_differential._helpers.running._test_types import ( EngineEnvironmentErrorTestCase, EngineEnvironmentTestCase, ExpectedOutcomeErrorTestCase, diff --git a/tests/unit/scripts/compiler_differential/_helpers/running/test_report.py b/tests/unit/scripts/compiler_differential/_helpers/running/test_report.py new file mode 100644 index 0000000000..1662c3efb9 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/running/test_report.py @@ -0,0 +1,110 @@ +"""The harness reports how much of the discovery input space the seed corpus exercised.""" + +from __future__ import annotations + +import pytest + +from scripts.compiler_differential._helpers.running.report import ( + format_discovery_coverage, + format_summary, +) +from scripts.compiler_differential.models import Difference, ProjectComparison +from tests.unit.scripts.compiler_differential._helpers.running._test_types import ( + CoverageReportTestCase, + SummaryTestCase, +) + +_IDENTICAL: ProjectComparison = ProjectComparison(project="seed/0", differences=(), seconds=1.0) +_DIFFERING: ProjectComparison = ProjectComparison( + project="seed/1", + differences=( + Difference( + project="seed/1", artifact="`compile` stdout", location="/models/0", left="a", right="b" + ), + ), + seconds=1.0, +) + + +@pytest.mark.parametrize( + "test_case", + [ + CoverageReportTestCase( + description="complete", + covered=frozenset({"model_files", "crlf"}), + required=("model_files", "crlf"), + expected_lines=( + "Discovery coverage: 2 of 2 input kinds exercised by the seed corpus", + "config-only kinds: project_adapter_config (only proves the config names", + ), + ), + CoverageReportTestCase( + description="missing_kinds_are_named_in_required_order", + covered=frozenset({"model_files"}), + required=("providers", "model_files", "crlf"), + expected_lines=( + "Discovery coverage: 1 of 3 input kinds exercised by the seed corpus", + " missing: providers, crlf", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_covered_kinds_when_reporting_then_missing_kinds_are_named( + test_case: CoverageReportTestCase, +) -> None: + report: str = format_discovery_coverage(covered=test_case.covered, required=test_case.required) + + assert all(line in report for line in test_case.expected_lines), report + + +@pytest.mark.parametrize( + "test_case", + [ + SummaryTestCase( + description="identical_and_covered", + comparisons=[_IDENTICAL], + missing_coverage=(), + expected_lines=("Compiler differential passed: 1 projects identical",), + expected_absent=("FAILED",), + ), + SummaryTestCase( + description="identical_but_coverage_missing", + comparisons=[_IDENTICAL], + missing_coverage=("providers", "crlf"), + expected_lines=( + "Compiler differential FAILED: 0 of 1 projects differ", + "Required discovery coverage missing: providers, crlf", + ), + expected_absent=("passed",), + ), + SummaryTestCase( + description="differences_and_coverage_missing", + comparisons=[_IDENTICAL, _DIFFERING], + missing_coverage=("providers",), + expected_lines=( + "Compiler differential FAILED: 1 of 2 projects differ", + "First difference: seed/1: `compile` stdout at /models/0", + "Required discovery coverage missing: providers", + ), + expected_absent=("passed",), + ), + ], + ids=lambda case: case.description, +) +def test_given_results_and_coverage_when_summarizing_then_failures_never_read_as_passed( + test_case: SummaryTestCase, +) -> None: + summary: str = format_summary( + comparisons=test_case.comparisons, + engines=("python", "native"), + seconds=2.0, + missing_coverage=test_case.missing_coverage, + ) + + assert all(line in summary for line in test_case.expected_lines), summary + assert not any(text in summary for text in test_case.expected_absent), summary + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/main/_test_types.py b/tests/unit/scripts/compiler_differential/main/_test_types.py index 78f387505f..9dedbb9673 100644 --- a/tests/unit/scripts/compiler_differential/main/_test_types.py +++ b/tests/unit/scripts/compiler_differential/main/_test_types.py @@ -45,3 +45,24 @@ class FailureCaseWriteTestCase: description: str case: FailureCase expected_files: tuple[str, ...] + + +@dataclass(frozen=True) +class FeatureBlockWindowTestCase: + """A window of consecutive seeds and the feature blocks it must force.""" + + description: str + seeds: range + expected_blocks: frozenset[str] + + +@dataclass(frozen=True) +class RareFeatureSeedTestCase: + """A seed carrying a rare block and the outcome it must declare.""" + + description: str + seed: int + expected_feature: str + expected_error_code: str + expected_succeeding_commands: tuple[str, ...] + other_seeds: tuple[int, ...] diff --git a/tests/unit/scripts/compiler_differential/main/test_generate_project.py b/tests/unit/scripts/compiler_differential/main/test_generate_project.py index bde93149c8..5ee8f700ec 100644 --- a/tests/unit/scripts/compiler_differential/main/test_generate_project.py +++ b/tests/unit/scripts/compiler_differential/main/test_generate_project.py @@ -2,11 +2,23 @@ from __future__ import annotations +import random + import pytest +from scripts.compiler_differential.classes.discovery_features import feature_blocks_for_seed +from scripts.compiler_differential.constants import ( + DEFAULT_SEED_COUNT, + GENERATOR_FEATURE_BLOCKS, + GENERATOR_FEATURE_STRIDE, + GENERATOR_RARE_FEATURE_BLOCKS, + PLAN_LABEL, +) from scripts.compiler_differential.main.generate_project import generate_project from scripts.compiler_differential.models import GeneratedProject from tests.unit.scripts.compiler_differential.main._test_types import ( + FeatureBlockWindowTestCase, + RareFeatureSeedTestCase, SeedDeterminismTestCase, SeedRangeTestCase, ) @@ -63,6 +75,7 @@ def test_given_same_seed_when_generating_twice_then_projects_are_identical( "scenario", "singular_audit", "column_audits", + *GENERATOR_FEATURE_BLOCKS, } ), expected_distinct_projects=60, @@ -86,5 +99,54 @@ def test_given_seed_range_when_generating_then_features_and_failures_are_covered ) +@pytest.mark.parametrize( + "test_case", + [ + FeatureBlockWindowTestCase( + description=f"seeds_{start}_to_{start + GENERATOR_FEATURE_STRIDE - 1}", + seeds=range(start, start + GENERATOR_FEATURE_STRIDE), + expected_blocks=frozenset(GENERATOR_FEATURE_BLOCKS) + - set(GENERATOR_RARE_FEATURE_BLOCKS), + ) + for start in (0, 5, 37) + ], + ids=lambda case: case.description, +) +def test_given_consecutive_seeds_when_choosing_blocks_then_every_common_block_is_forced( + test_case: FeatureBlockWindowTestCase, +) -> None: + chosen: set[str] = set() + for seed in test_case.seeds: + chosen.update(feature_blocks_for_seed(seed=seed, rng=random.Random(seed))) + + assert test_case.expected_blocks <= chosen + + +@pytest.mark.parametrize( + "test_case", + [ + RareFeatureSeedTestCase( + description="dbt_ref_seed_fails_compile_but_plans", + seed=GENERATOR_RARE_FEATURE_BLOCKS["dbt_ref"], + expected_feature="dbt_ref", + expected_error_code="C214", + expected_succeeding_commands=(PLAN_LABEL,), + other_seeds=(0, *range(2, DEFAULT_SEED_COUNT)), + ) + ], + ids=lambda case: case.description, +) +def test_given_rare_block_seed_when_generating_then_outcome_is_declared( + test_case: RareFeatureSeedTestCase, +) -> None: + project: GeneratedProject = generate_project(seed=test_case.seed) + others: list[GeneratedProject] = [generate_project(seed=seed) for seed in test_case.other_seeds] + + assert test_case.expected_feature in project.features + assert project.expected_error_code == test_case.expected_error_code + assert project.succeeding_commands == test_case.expected_succeeding_commands + assert not any(test_case.expected_feature in other.features for other in others) + + if __name__ == "__main__": pytest.main([__file__, "-vv"])