From f3bb20119f31c0a34f498522f2c2808d8627b90c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 08:50:56 +0100 Subject: [PATCH 1/4] test(compiler): generate every render input in the differential seeds Add rendering feature blocks to the seeded generator: macros reading context, constants, enums and the environment, nested and typed-reference macro calls, macro-generated references with relaxed explicit-reference rules, table functions, variable, environment and hook-context interpolation, runtime placeholders, cursor intrinsics, macros in tests, scenarios, audits, functions and source expressions, enum-typed columns, model-private declarations, named hook arguments, and model, source and seed audits. The harness provides the fixed environment variable the generated projects read. --- .../_helpers/running/execution.py | 13 +- .../classes/discovery_features.py | 4 +- .../classes/project_builder.py | 17 +- .../classes/render_features.py | 417 ++++++++++++++++++ scripts/compiler_differential/constants.py | 16 + 5 files changed, 461 insertions(+), 6 deletions(-) create mode 100644 scripts/compiler_differential/classes/render_features.py diff --git a/scripts/compiler_differential/_helpers/running/execution.py b/scripts/compiler_differential/_helpers/running/execution.py index 06bd80c6c2..4835622891 100644 --- a/scripts/compiler_differential/_helpers/running/execution.py +++ b/scripts/compiler_differential/_helpers/running/execution.py @@ -15,6 +15,7 @@ ENGINE_ENV_VAR, EXCLUDED_ENVIRONMENT_KEYS, EXCLUDED_ENVIRONMENT_PREFIX, + GENERATOR_ENVIRONMENT, MANIFEST_FILE, PROJECT_DIRECTORY, SQB_ENTRY, @@ -88,12 +89,16 @@ def run_engine( def harness_environment() -> dict[str, str]: - """Return this process's environment without engine, capture, and dbt settings.""" + """Return this environment without engine, capture or dbt settings, plus generator vars.""" return { - key: value - for key, value in os.environ.items() - if key not in EXCLUDED_ENVIRONMENT_KEYS and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX) + **{ + key: value + for key, value in os.environ.items() + if key not in EXCLUDED_ENVIRONMENT_KEYS + and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX) + }, + **GENERATOR_ENVIRONMENT, } diff --git a/scripts/compiler_differential/classes/discovery_features.py b/scripts/compiler_differential/classes/discovery_features.py index 63625986e6..d0a30ddd73 100644 --- a/scripts/compiler_differential/classes/discovery_features.py +++ b/scripts/compiler_differential/classes/discovery_features.py @@ -108,7 +108,9 @@ def write(self) -> None: "line_endings": self._line_endings, } for block in self._blocks: - writers[block]() + writer: Callable[[], None] | None = writers.get(block) + if writer is not None: + writer() def _base(self) -> ModelPlan: return self._random.choice(self._staging) diff --git a/scripts/compiler_differential/classes/project_builder.py b/scripts/compiler_differential/classes/project_builder.py index bdffa11e73..43138485c0 100644 --- a/scripts/compiler_differential/classes/project_builder.py +++ b/scripts/compiler_differential/classes/project_builder.py @@ -10,6 +10,7 @@ DiscoveryFeatureWriter, feature_blocks_for_seed, ) +from scripts.compiler_differential.classes.render_features import RenderFeatureWriter from scripts.compiler_differential.constants import ( DUCKDB_ADAPTER, GENERATOR_CONSTANT_KIND, @@ -95,7 +96,21 @@ def build(self) -> GeneratedProject: label=self._label, ) extras.write() - self._write_config(domains=domains, adapter=extras.adapter, extra_lines=extras.config_lines) + rendering: RenderFeatureWriter = RenderFeatureWriter( + blocks=blocks, + rng=self._random, + files=self._files, + features=self._features, + staging=[model for model in self._models if model.layer == _STAGING], + sources=self._sources, + label=self._label, + ) + rendering.write() + self._write_config( + domains=domains, + adapter=extras.adapter, + extra_lines=[*extras.config_lines, *rendering.config_lines], + ) if invalid_kind is not None: self._inject_invalid(kind=invalid_kind) return GeneratedProject( diff --git a/scripts/compiler_differential/classes/render_features.py b/scripts/compiler_differential/classes/render_features.py new file mode 100644 index 0000000000..fa791e7ed0 --- /dev/null +++ b/scripts/compiler_differential/classes/render_features.py @@ -0,0 +1,417 @@ +"""Write the optional project inputs that make generated seeds cover every render input kind.""" + +from __future__ import annotations + +import random +from collections.abc import Callable + +from scripts.compiler_differential.constants import ( + GENERATOR_CHANNEL_ENV_VAR, + GENERATOR_RENDER_FOLDER, + GENERATOR_RENDER_TEST_FOLDER, +) +from scripts.compiler_differential.models import ModelPlan + + +class RenderFeatureWriter: + """Add self-contained rendering blocks to one generated project, chosen per seed.""" + + def __init__( + self, + *, + blocks: tuple[str, ...], + rng: random.Random, + files: dict[str, str], + features: set[str], + staging: list[ModelPlan], + sources: list[str], + label: Callable[[], str], + ) -> None: + self._blocks: tuple[str, ...] = blocks + self._random: random.Random = rng + self._files: dict[str, str] = files + self._features: set[str] = features + self._staging: list[ModelPlan] = staging + self._sources: list[str] = sources + self._label: Callable[[], str] = label + self.config_lines: list[str] = [] + + def write(self) -> None: + """Write the selected rendering blocks in their canonical order.""" + + writers: dict[str, Callable[[], None]] = { + "macro_context_reads": self._macro_context_reads, + "typed_reference_macro": self._typed_reference_macro, + "macro_generated_reference": self._macro_generated_reference, + "table_function": self._table_function, + "interpolation": self._interpolation, + "cursor_bounds": self._cursor_bounds, + "macros_across_resources": self._macros_across_resources, + "enum_column_contract": self._enum_column_contract, + "resource_audits": self._resource_audits, + } + for block in self._blocks: + writer: Callable[[], None] | None = writers.get(block) + if writer is not None: + writer() + + def _base(self) -> ModelPlan: + return self._random.choice(self._staging) + + def _model(self, *, name: str, header: str, body: str, description: str) -> None: + self._files[f"{GENERATOR_RENDER_FOLDER}/{name}.sql"] = ( + f'MODEL (\n description "{description}",\n{header});\n\n{body}' + ) + + def _owned(self, *, role: str, name: str) -> str: + return f"{GENERATOR_RENDER_FOLDER}/_sqlbuild/_{role}/{name}" + + def _macro_context_reads(self) -> None: + self._features.add("macro_context_reads") + base: ModelPlan = self._base() + minimum: int = self._random.randint(1, 4) + self._files[self._owned(role="constants", name="render_limits.sql")] = ( + f"CONSTANT (name minimum_render_amount, value {minimum});\n" + ) + self._files[self._owned(role="enums", name="render_channel.sql")] = ( + 'ENUM (\n name render_channel,\n members (WEB "web", STORE "store"),\n);\n' + ) + self._files[self._owned(role="macros", name="policy.py")] = ( + "import os\n\n\n" + "def minimum_amount_filter(ctx, column: str) -> str:\n" + ' """Keep rows at or above the configured minimum amount."""\n' + ' if ctx.constants["minimum_render_amount"] is None:\n' + ' return "TRUE"\n' + " return f\"{column} >= {ctx.render_constant('minimum_render_amount')}\"\n\n\n" + "def channel_label(ctx) -> str:\n" + ' """Label rows with the web channel, target and adapter."""\n' + ' channel = ctx.render_enum_member(enum_name="render_channel", member_name="WEB")\n' + ' region = ctx.vars["region"]\n' + " return f\"{channel} || '/{ctx.target_name}/{ctx.adapter_name}/{region}'\"\n\n\n" + "def with_deployment(expression: str) -> str:\n" + ' """Suffix an expression with the deployment channel from the environment."""\n' + f' channel = os.environ.get("{GENERATOR_CHANNEL_ENV_VAR}", "none")\n' + " return f\"({expression} || '@{channel}')\"\n" + ) + self._model( + name=f"policy_{base.name}", + description=f"Policy-filtered copy of {base.name}", + header="", + body=( + "SELECT id, amount, @with_deployment(@channel_label()) AS channel,\n" + ' @const("minimum_render_amount") AS minimum_amount,\n' + ' @enum("render_channel").STORE AS fallback_channel\n' + f'FROM __ref("{base.name}")\n' + 'WHERE @minimum_amount_filter("amount")\n' + ), + ) + + def _typed_reference_macro(self) -> None: + self._features.add("typed_reference_macro") + first, second = ( + self._random.sample(self._staging, k=2) + if len(self._staging) > 1 + else (self._staging * 2) + ) + source: str = self._random.choice(self._sources) + self._files[self._owned(role="macros", name="relations.py")] = ( + "from sqlbuild.refs import SqlResourceRef\n\n\n" + "def union_ids(relations: list[SqlResourceRef]) -> str:\n" + ' """Union the ids of several relations."""\n' + ' selects = [f"SELECT id FROM {relation}" for relation in relations]\n' + ' return " UNION ALL ".join(selects)\n' + "\n\n" + "def latest_rows(relation: SqlResourceRef, limit: int = 2) -> str:\n" + ' """Return the newest rows of one relation."""\n' + ' return f"(SELECT id, amount FROM {relation} ORDER BY id DESC LIMIT {limit})"\n' + ) + self._model( + name="unioned_event_ids", + description="Ids from two staged relations", + header="", + body=( + "SELECT id\n" + f'FROM (@union_ids([__ref("{first.name}"), __ref("{second.name}")])) AS unioned\n' + ), + ) + self._model( + name=f"latest_{source}", + description=f"Newest rows of {source}", + header="", + body=f'SELECT id, amount\nFROM @latest_rows(__source("{source}"), limit=2) AS latest\n', + ) + + def _macro_generated_reference(self) -> None: + self._features.add("macro_generated_reference") + base: ModelPlan = self._base() + self.config_lines.extend(["", "[references]", "enforce_explicit = false"]) + self._files[self._owned(role="macros", name="legacy.py")] = ( + "def legacy_events() -> str:\n" + ' """Return a staged relation the way a migrated project wrote it."""\n' + f" return '__ref(\"{base.name}\")'\n" + ) + self._model( + name="legacy_event_ids", + description="Ids read through a macro-generated reference", + header="", + body="SELECT id\nFROM @legacy_events() AS legacy\n", + ) + + def _table_function(self) -> None: + self._features.add("table_function") + base: ModelPlan = self._base() + self._files["functions/sql/table_fn__events_above.sql"] = ( + "FUNCTION (\n" + ' description "Events above an amount",\n' + " arguments (p_minimum DOUBLE),\n" + " returns table (\n id INTEGER,\n amount DOUBLE\n ),\n);\n\n" + f'SELECT id, amount\nFROM __ref("{base.name}")\nWHERE amount > p_minimum\n' + ) + self._model( + name="large_events", + description="Events above a threshold from a table function", + header="", + body=( + "SELECT id, amount\n" + f'FROM __table_fn("table_fn__events_above")({self._random.choice(("1.5", "4"))})\n' + ), + ) + + def _interpolation(self) -> None: + self._features.add("interpolation") + base: ModelPlan = self._base() + self._files[self._owned(role="hooks/sql", name="announce_refresh.sql")] = ( + 'HOOK (\n description "Announce one refreshed relation"\n);\n\n' + "SELECT @'relation' AS refreshed_relation, @'label' AS refresh_label\n" + ) + self._files[self._owned(role="macros", name="notes.py")] = ( + "def refresh_note(relation: str) -> str:\n" + ' """Describe one refreshed relation."""\n' + " return f\"'refreshed ' || '{relation}'\"\n" + ) + self._model( + name=f"interpolated_{base.name}", + description=f"Interpolated copy of {base.name}", + header=( + " materialized table,\n" + " post_hooks [\n" + ' sql("announce_refresh", relation: "@@CTX:destination.qualified", ' + f'label: "{self._label()}"),\n' + " inline_sql(\"SELECT @refresh_note('@@CTX:destination.qualified') AS note\"),\n" + " inline_sql(\"SELECT '@@CTX:run.target' AS target, " + "'@@CTX:model.name' AS model\"),\n" + " ],\n" + ), + body=( + "SELECT id,\n" + " '@@site_label' AS site,\n" + f" '@@ENV:{GENERATOR_CHANNEL_ENV_VAR}' AS channel,\n" + " amount * @@discount_rate AS discounted\n" + f'FROM __ref("{base.name}")\n' + ), + ) + self._deferred_placeholder(base) + + def _deferred_placeholder(self, base: ModelPlan) -> None: + self._files["materializations/windowed_copy.py"] = ( + "from sqlbuild.executor.custom.models import MaterializationContext, " + "MaterializationResult\n\n\n" + "def materialize(ctx: MaterializationContext) -> MaterializationResult:\n" + ' sql = ctx.sql.replace("@@@window_start", "DATE \'2026-01-01\'")\n' + ' ctx.execute_sql(f"CREATE OR REPLACE TABLE {ctx.destination} AS {sql}")\n' + " return MaterializationResult(relation=ctx.destination, audit_results=())\n" + ) + self._model( + name=f"windowed_{base.name}", + description=f"Runtime-windowed copy of {base.name}", + header=( + " materialized windowed_copy,\n" + " placeholders (\n window_start \"'2026-01-01'\",\n ),\n" + " config (\n window_column created_at,\n ),\n" + ), + body=( + f'SELECT id, created_at\nFROM __ref("{base.name}")\n' + "WHERE CAST(created_at AS DATE) >= CAST(@@@window_start AS DATE)\n" + ), + ) + + def _cursor_bounds(self) -> None: + self._features.add("cursor_bounds") + base: ModelPlan = self._base() + self._model( + name=f"bounded_{base.name}", + description=f"Cursor-bounded copy of {base.name}", + header=( + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " unique_key [id],\n" + " cursor created_at,\n cursor_type timestamp,\n cursor_grain day,\n" + " incremental_mode microbatch,\n" + " microbatch_strategy rolling_window,\n" + " lookback 1d,\n" + " batch_size 1d,\n" + f" cursor_inputs (\n {base.name} created_at,\n ),\n" + ), + body=( + "SELECT id, amount, created_at\n" + f'FROM __ref("{base.name}")\n' + "WHERE created_at >= __cursor_start()\n AND created_at < __cursor_end()\n" + ), + ) + + def _macros_across_resources(self) -> None: + self._features.add("macros_across_resources") + base: ModelPlan = self._base() + source: str = base.inputs[0].split(":", 1)[1] + self._files["macros/labels.py"] = ( + "def tidy_label(expression: str) -> str:\n" + ' """Trim and lower-case a label."""\n' + ' return f"LOWER(TRIM({expression}))"\n' + ) + self._files[self._owned(role="audits/generic", name="tidy_status.sql")] = ( + 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE @column <> @tidy_label("@column")\n' + ) + self._files["functions/sql/tidy_status.sql"] = ( + "FUNCTION (\n" + ' description "Normalise a status label.",\n' + " arguments (raw_status STRING),\n" + " returns STRING,\n);\n\n" + '@tidy_label("raw_status")\n' + ) + self._model( + name=f"labelled_{base.name}", + description=f"Labelled copy of {base.name}", + header=" columns (\n status (audits [tidy_status]),\n ),\n", + body=( + 'SELECT id, @tidy_label("status") AS status,\n' + ' __udf("tidy_status")(status) AS tidied\n' + f'FROM __ref("{base.name}")\n' + ), + ) + self._files["sources/labelled.yml"] = ( + "sources:\n" + " - name: labelled_channels\n" + " description: Channel labels normalised in the source expression.\n" + ' expression: "(SELECT 1 AS id, @tidy_label(\\"\' Web \'\\") AS channel)"\n' + " columns:\n" + " - name: id\n type: INTEGER\n" + " - name: channel\n type: VARCHAR\n" + ) + self._model( + name="channel_labels", + description="Normalised channel labels", + header="", + body='SELECT id, channel\nFROM __source("labelled_channels")\n', + ) + self._files[ + f"{GENERATOR_RENDER_FOLDER}/_sqlbuild/audits/singular/labelled_channels_match.sql" + ] = ( + 'AUDIT (name "labelled_channels_match");\n\n' + f'SELECT l.id\nFROM __ref("labelled_{base.name}") l\n' + 'JOIN __ref("channel_labels") c ON l.id = c.id\nWHERE l.status = c.channel\n' + ) + self._files[f"tests/scenarios/labelled_{base.name}_scenario.sql"] = ( + 'SCENARIO (\n description "Labels are normalised end to end"\n);\n\n' + "WITH\n" + f"__source__{source} AS (\n" + " SELECT 3 AS id, CAST(1 AS DOUBLE) AS amount, ' Placed ' AS status,\n" + " TIMESTAMP '2026-01-03 00:00:00' AS created_at\n" + "),\n" + f"__expected__labelled_{base.name} AS (\n" + " SELECT 3 AS id, @tidy_label(\"' Placed '\") AS status\n" + ")\nSELECT 1\n" + ) + self._files[f"{GENERATOR_RENDER_TEST_FOLDER}/_sqlbuild/_macros/fixtures.py"] = ( + "def event_rows(count: int = 2) -> str:\n" + ' """Return fixture event rows."""\n' + ' return " UNION ALL ".join(\n' + ' f"SELECT {index} AS id, CAST({index} AS DOUBLE) AS amount, "\n' + " f\"'placed' AS status, \"\n" + " f\"TIMESTAMP '2026-01-0{index} 00:00:00' AS created_at\"\n" + " for index in range(1, count + 1)\n" + " )\n" + ) + self._files[f"{GENERATOR_RENDER_TEST_FOLDER}/test_labelled_{base.name}.sql"] = ( + "TEST();\n\nWITH\n" + f"__source__{source} AS (\n @event_rows(2)\n),\n" + f"__expected__labelled_{base.name} AS (\n" + " SELECT 1 AS id, @tidy_label(\"'placed'\") AS status\n" + " UNION ALL\n" + " SELECT 2 AS id, 'placed' AS status\n" + ")\nSELECT 1\n" + ) + + def _enum_column_contract(self) -> None: + self._features.add("enum_column_contract") + base: ModelPlan = self._base() + self._files[self._owned(role="enums", name="render_state.sql")] = ( + "ENUM (\n name render_state,\n members (\n" + ' PLACED "placed",\n SHIPPED "expédié",\n CANCELLED "取消",\n' + ' RETURNED "returned ✓",\n ),\n);\n' + ) + self._model( + name=f"stated_{base.name}", + description=f"Enum-typed copy of {base.name}", + header=( + " contract enforced,\n" + " columns (\n" + " id (type INTEGER),\n" + " status (type render_state),\n" + " tier (type VARCHAR),\n" + " ),\n" + f" constants (\n _tier_floor {self._random.randint(1, 9)},\n ),\n" + " enums (\n _tier [LOW, HIGH],\n ),\n" + ), + body=( + "SELECT CAST(id AS INTEGER) AS id, status,\n" + ' CASE WHEN amount >= @const("_tier_floor") THEN @enum("_tier").HIGH\n' + ' ELSE @enum("_tier").LOW END AS tier\n' + f'FROM __ref("{base.name}")\n' + ), + ) + + def _resource_audits(self) -> None: + self._features.add("resource_audits") + self._files["sources/audited.yml"] = ( + "sources:\n" + " - name: audited_events\n" + " description: Events with source audits.\n" + ' expression: "(SELECT 1 AS id, CAST(@@discount_rate AS DOUBLE) AS amount)"\n' + " columns:\n" + " - name: id\n" + " type: INTEGER\n" + " audits:\n" + " - not_null\n" + " - unique\n" + " - accepted_values:\n" + " values: [1, 2]\n" + " - name: amount\n" + " type: DOUBLE\n" + ) + self._files["seeds/channel_codes.csv"] = f'id,label\n1,"{self._label()}"\n2,web\n' + self._files["seeds/channel_codes.yml"] = ( + "seeds:\n" + " - name: channel_codes\n" + " description: Channel codes with seed audits.\n" + " columns:\n" + " - name: id\n" + " type: INTEGER\n" + " audits:\n" + " - not_null\n" + " - name: label\n" + " type: VARCHAR\n" + ) + self._files[self._owned(role="audits/generic", name="amount_floor.sql")] = ( + 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE amount < @minimum\n' + ) + self._model( + name="audited_event_totals", + description="Totals of the audited events per channel", + header=f" audits [amount_floor (minimum {self._random.randint(-40, -2)})],\n", + body=( + "SELECT c.label, SUM(e.amount) AS amount\n" + 'FROM __source("audited_events") e\n' + 'JOIN __seed("channel_codes") c ON e.id = c.id\n' + "GROUP BY c.label\n" + ), + ) diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 7d64fbea60..7f58c28ad4 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -181,6 +181,17 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: GENERATOR_MACRO_KIND: "macros", GENERATOR_HOOK_KIND: "hooks", } +GENERATOR_RENDER_BLOCKS: tuple[str, ...] = ( + "macro_context_reads", + "typed_reference_macro", + "macro_generated_reference", + "table_function", + "interpolation", + "cursor_bounds", + "macros_across_resources", + "enum_column_contract", + "resource_audits", +) GENERATOR_FEATURE_BLOCKS: tuple[str, ...] = ( "incremental_append", "incremental_delete_insert", @@ -209,6 +220,7 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "target_override", "dbt_ref", "line_endings", + *GENERATOR_RENDER_BLOCKS, ) GENERATOR_FEATURE_STRIDE: int = 4 GENERATOR_OPTIONAL_FEATURE_SHARE: float = 0.2 @@ -216,6 +228,10 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: GENERATOR_RARE_FEATURE_BLOCKS: dict[str, int] = {"dbt_ref": 1} GENERATOR_DBT_REF_ERROR_CODE: str = "C214" GENERATOR_FEATURE_FOLDER: str = "models/features" +GENERATOR_RENDER_FOLDER: str = "models/rendering" +GENERATOR_RENDER_TEST_FOLDER: str = "tests/unit/rendering" +GENERATOR_CHANNEL_ENV_VAR: str = "SQB_DIFFERENTIAL_CHANNEL" +GENERATOR_ENVIRONMENT: dict[str, str] = {GENERATOR_CHANNEL_ENV_VAR: "web-orders"} GENERATOR_PROJECT_ADAPTER: str = "generated_duckdb" GENERATOR_CRLF: str = "\r\n" GENERATOR_BOM: str = "\ufeff" From 7a4eee476cb602f44af29e9a3213ff551ee18e00 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 08:51:54 +0100 Subject: [PATCH 2/4] test(compiler): require render coverage in the compiler differential Classify CompileProjectInputs stage captures into render input kinds, crediting a kind only from rendered evidence: a template absent from the rendered SQL, a recorded usage, reference, grant, hook entry or attached audit. Encoded empty sets and mappings prove nothing. Kinds credited from a used macro's source text are reported as indirect. Add --require-render-coverage and require it in the per-PR compiler differential job. Share the capture readers with discovery coverage. --- .github/workflows/verify.yml | 3 +- .../_helpers/comparing/project_comparison.py | 5 +- .../_helpers/corpus/corpus.py | 2 +- .../_helpers/coverage/capture_values.py | 105 ++++ .../_helpers/coverage/discovery.py | 135 ++--- .../_helpers/coverage/macro_sources.py | 41 ++ .../_helpers/coverage/render.py | 420 +++++++++++++++ .../_helpers/running/report.py | 71 ++- .../_helpers/running/run.py | 1 + scripts/compiler_differential/constants.py | 89 ++++ .../main/differential.py | 59 ++- scripts/compiler_differential/models.py | 4 +- .../test_compiler_differential.py | 21 +- .../_helpers/coverage/_test_types.py | 20 + .../_helpers/coverage/helpers.py | 75 +++ .../_helpers/coverage/test_render.py | 484 ++++++++++++++++++ .../_helpers/running/_test_types.py | 4 +- .../_helpers/running/test_report.py | 25 +- .../compiler_differential/main/_test_types.py | 9 + .../main/test_differential_usage.py | 47 ++ 20 files changed, 1487 insertions(+), 133 deletions(-) create mode 100644 scripts/compiler_differential/_helpers/coverage/capture_values.py create mode 100644 scripts/compiler_differential/_helpers/coverage/macro_sources.py create mode 100644 scripts/compiler_differential/_helpers/coverage/render.py create mode 100644 tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py create mode 100644 tests/unit/scripts/compiler_differential/main/test_differential_usage.py diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index d007476831..1645e78582 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -293,7 +293,8 @@ jobs: - name: Compare fixtures, examples, generated seeds and the failure corpus run: >- make compiler-differential - COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage" + COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage + --require-render-coverage" format_performance: name: Formatter performance guards diff --git a/scripts/compiler_differential/_helpers/comparing/project_comparison.py b/scripts/compiler_differential/_helpers/comparing/project_comparison.py index 308d7b7791..c1e2f3af67 100644 --- a/scripts/compiler_differential/_helpers/comparing/project_comparison.py +++ b/scripts/compiler_differential/_helpers/comparing/project_comparison.py @@ -11,6 +11,7 @@ diagnostic_codes, ) from scripts.compiler_differential._helpers.coverage.discovery import project_discovery_kinds +from scripts.compiler_differential._helpers.coverage.render import project_render_kinds from scripts.compiler_differential._helpers.running.execution import run_engine from scripts.compiler_differential.constants import ERROR_SEVERITY, WARNING_SEVERITY from scripts.compiler_differential.models import ( @@ -50,6 +51,7 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) -> engine=right_engine, options=options, ) + counted: bool = project.seed_coverage and options.stage_captures differences: list[Difference] = [ *_timeout_differences(project=project, runs=(left, right)), *_corpus_differences(project=project, run=left), @@ -61,9 +63,10 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) -> seconds=time.monotonic() - started, discovered_kinds=( project_discovery_kinds(captures=left.captures, source_dir=source_dir) - if project.discovery_coverage and options.stage_captures + if counted else frozenset() ), + rendered_kinds=project_render_kinds(left.captures) if counted else frozenset(), ) diff --git a/scripts/compiler_differential/_helpers/corpus/corpus.py b/scripts/compiler_differential/_helpers/corpus/corpus.py index d1d51c4127..805f8aa782 100644 --- a/scripts/compiler_differential/_helpers/corpus/corpus.py +++ b/scripts/compiler_differential/_helpers/corpus/corpus.py @@ -143,5 +143,5 @@ def _seed_project(seed: int) -> CorpusProject: succeeding_commands=generated.succeeding_commands, ), writer=generated.write, - discovery_coverage=True, + seed_coverage=True, ) diff --git a/scripts/compiler_differential/_helpers/coverage/capture_values.py b/scripts/compiler_differential/_helpers/coverage/capture_values.py new file mode 100644 index 0000000000..40e2d90287 --- /dev/null +++ b/scripts/compiler_differential/_helpers/coverage/capture_values.py @@ -0,0 +1,105 @@ +"""Read values out of canonical stage captures without crediting encoded empty collections.""" + +from __future__ import annotations + +import json +import re +from collections.abc import Iterator + +from scripts.compiler_differential._helpers.comparing.compare import as_json_object +from scripts.compiler_differential.constants import ( + CALLABLE_MARKER, + COLLECTION_CAPTURE_MARKERS, + TYPE_MARKER, + UNCANONICAL_CAPTURE_MARKERS, +) + +_ENUM_VALUE: re.Pattern[str] = re.compile(r"\.([A-Z_]+)$") + + +def decoded(value: object) -> object: + """Unwrap the capture's set and non-string-key mapping encodings into plain lists.""" + + encoded: dict[str, object] | None = as_json_object(value) + if encoded is None or len(encoded) != 1: + return value + marker: str = next(iter(encoded)) + return encoded[marker] if marker in COLLECTION_CAPTURE_MARKERS else value + + +def has_content(value: object) -> bool: + """Whether a captured value holds anything; encoded empty sets and mappings do not.""" + + return bool(decoded(value)) + + +def records(value: object) -> list[dict[str, object]]: + """Return the objects of a captured list or set.""" + + items: object = decoded(value) + if not isinstance(items, list): + return [] + return [record for item in items if (record := as_json_object(item)) is not None] + + +def mapping_records(value: object) -> list[dict[str, object]]: + """Return the object values of a captured string-keyed mapping, such as macros by name.""" + + mapping: dict[str, object] = as_json_object(value) or {} + return [ + record + for key, item in mapping.items() + if key != TYPE_MARKER and (record := as_json_object(item)) is not None + ] + + +def mapping_size(value: object) -> int: + """Return how many entries a captured mapping or list holds.""" + + items: object = decoded(value) + return len(items) if isinstance(items, (dict, list)) else 0 + + +def enum_name(value: object) -> str | None: + """Return the lower-case member name of a captured enum value.""" + + encoded: dict[str, object] | None = as_json_object(value) + match: re.Match[str] | None = ( + None if encoded is None else _ENUM_VALUE.search(str(encoded.get("__enum__", ""))) + ) + return None if match is None else match.group(1).lower() + + +def named_values(*, value: object, names: frozenset[str]) -> Iterator[tuple[str, object]]: + """Yield every `(key, value)` pair below `value` whose key is one of `names`.""" + + record: dict[str, object] | None = as_json_object(value) + children: list[object] = list(value) if isinstance(value, list) else [] + if record is not None: + yield from ((key, item) for key, item in record.items() if key in names) + children.extend(record.values()) + for child in children: + yield from named_values(value=child, names=names) + + +def capture_problems( + *, capture_text: str, fields: tuple[str, ...], callable_fields: frozenset[str] +) -> tuple[str, ...]: + """Return why a capture is incomplete or not canonical; empty when it is sound.""" + + capture: dict[str, object] = as_json_object(json.loads(capture_text)) or {} + problems: list[str] = [] + captured: tuple[str, ...] = tuple(key for key in capture if key != TYPE_MARKER) + if captured != fields: + problems.append(f"captured fields {captured} differ from {fields}") + problems.extend( + f"capture contains {marker}" + for marker in UNCANONICAL_CAPTURE_MARKERS + if f'"{marker}"' in capture_text + ) + problems.extend( + f"{field} is not a qualified callable name: {value}" + for field, value in named_values(value=capture, names=callable_fields) + if set(as_json_object(value) or {}) != {CALLABLE_MARKER} + ) + return tuple(problems) diff --git a/scripts/compiler_differential/_helpers/coverage/discovery.py b/scripts/compiler_differential/_helpers/coverage/discovery.py index 7492e86d42..37e800e26f 100644 --- a/scripts/compiler_differential/_helpers/coverage/discovery.py +++ b/scripts/compiler_differential/_helpers/coverage/discovery.py @@ -9,10 +9,16 @@ from pathlib import Path from scripts.compiler_differential._helpers.comparing.compare import as_json_object +from scripts.compiler_differential._helpers.coverage.capture_values import ( + capture_problems, + enum_name, + has_content, + mapping_size, + named_values, + records, +) from scripts.compiler_differential.constants import ( CALLABLE_CAPTURE_FIELDS, - CALLABLE_MARKER, - COLLECTION_CAPTURE_MARKERS, CRLF_BYTES, DBT_REF_CALL, DISCOVERY_DECLARATION_FILE_COLLECTIONS, @@ -27,13 +33,10 @@ SINGULAR_AUDIT_KIND, STRATEGY_HEADER_KEYS, TAB, - TYPE_MARKER, - UNCANONICAL_CAPTURE_MARKERS, ) from sqlbuild.adapter.discovery.constants import BUILTIN_ADAPTER_IMPORTS from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -_ENUM_VALUE: re.Pattern[str] = re.compile(r"\.([A-Z_]+)$") _CROSS_FILE_MACRO_IMPORT: re.Pattern[str] = re.compile(r"^(?:from|import) macros\.", re.MULTILINE) _NON_ASCII_COMMENT: re.Pattern[str] = re.compile(r"--[^\n]*[^\x00-\x7f]") _NON_ASCII_STRING: re.Pattern[str] = re.compile(r"'[^'\n]*[^\x00-\x7f][^'\n]*'") @@ -75,7 +78,7 @@ def discovered_relative_paths(capture_text: str) -> frozenset[str]: capture: object = json.loads(capture_text) return frozenset( str(path) - for _, value in _named_values(value=capture, names=RELATIVE_PATH_FIELDS) + for _, value in named_values(value=capture, names=RELATIVE_PATH_FIELDS) if (path := (as_json_object(value) or {}).get(PATH_MARKER)) is not None ) @@ -103,7 +106,7 @@ def discovered_input_kinds(capture_text: str) -> frozenset[str]: capture: dict[str, object] = as_json_object(json.loads(capture_text)) or {} kinds: set[str] = { - kind for kind in discovery_collection_kinds() if _has_content(capture.get(kind)) + kind for kind in discovery_collection_kinds() if has_content(capture.get(kind)) } kinds.update(_model_kinds(capture)) kinds.update(_source_kinds(capture)) @@ -117,79 +120,19 @@ def discovered_input_kinds(capture_text: str) -> frozenset[str]: def discovery_capture_problems(capture_text: str) -> tuple[str, ...]: """Return why a discovery capture is incomplete or not canonical; empty when it is sound.""" - capture: dict[str, object] = as_json_object(json.loads(capture_text)) or {} - fields: tuple[str, ...] = tuple( - field.name for field in dataclasses.fields(DiscoveredProjectInputs) - ) - problems: list[str] = [] - captured: tuple[str, ...] = tuple(key for key in capture if key != TYPE_MARKER) - if captured != fields: - problems.append(f"captured fields {captured} differ from {fields}") - problems.extend( - f"capture contains {marker}" - for marker in UNCANONICAL_CAPTURE_MARKERS - if f'"{marker}"' in capture_text - ) - problems.extend( - f"{field} is not a qualified callable name: {value}" - for field, value in _named_values(value=capture, names=CALLABLE_CAPTURE_FIELDS) - if set(as_json_object(value) or {}) != {CALLABLE_MARKER} - ) - return tuple(problems) - - -def _named_values(*, value: object, names: frozenset[str]) -> Iterator[tuple[str, object]]: - record: dict[str, object] | None = as_json_object(value) - children: list[object] = list(value) if isinstance(value, list) else [] - if record is not None: - yield from ((key, item) for key, item in record.items() if key in names) - children.extend(record.values()) - for child in children: - yield from _named_values(value=child, names=names) - - -def _decoded(value: object) -> object: - """Unwrap the capture's set and non-string-key mapping encodings into plain lists.""" - - encoded: dict[str, object] | None = as_json_object(value) - if encoded is None or len(encoded) != 1: - return value - marker: str = next(iter(encoded)) - return encoded[marker] if marker in COLLECTION_CAPTURE_MARKERS else value - - -def _has_content(value: object) -> bool: - """Whether a captured value holds anything; encoded empty sets and mappings do not.""" - - return bool(_decoded(value)) - - -def _records(value: object) -> list[dict[str, object]]: - items: object = _decoded(value) - if not isinstance(items, list): - return [] - return [record for item in items if (record := as_json_object(item)) is not None] - - -def _mapping_size(value: object) -> int: - items: object = _decoded(value) - return len(items) if isinstance(items, (dict, list)) else 0 - - -def _enum_name(value: object) -> str | None: - encoded: dict[str, object] | None = as_json_object(value) - match: re.Match[str] | None = ( - None if encoded is None else _ENUM_VALUE.search(str(encoded.get("__enum__", ""))) + return capture_problems( + capture_text=capture_text, + fields=tuple(field.name for field in dataclasses.fields(DiscoveredProjectInputs)), + callable_fields=CALLABLE_CAPTURE_FIELDS, ) - return None if match is None else match.group(1).lower() def _model_kinds(capture: dict[str, object]) -> set[str]: kinds: set[str] = set() materializations: set[str] = { - str(record.get("name")) for record in _records(capture.get("materialization_files")) + str(record.get("name")) for record in records(capture.get("materialization_files")) } - for model in _records(capture.get("model_files")): + for model in records(capture.get("model_files")): header: dict[str, object] = as_json_object(model.get("header_values")) or {} materialized: object = header.get("materialized") if header.get("incremental_mode") == MICROBATCH_MODE: @@ -198,7 +141,7 @@ def _model_kinds(capture: dict[str, object]) -> set[str]: kinds.add(f"{materialized}_{header.get(STRATEGY_HEADER_KEYS[str(materialized)])}") elif materialized in materializations: kinds.add("custom_materialized_model") - if _records(model.get("constant_declarations")) or _records(model.get("enum_declarations")): + if records(model.get("constant_declarations")) or records(model.get("enum_declarations")): kinds.add("model_local_declaration") if DBT_REF_CALL in str(model.get("contents", "")): kinds.add("dbt_ref_model") @@ -207,8 +150,8 @@ def _model_kinds(capture: dict[str, object]) -> set[str]: def _source_kinds(capture: dict[str, object]) -> set[str]: kinds: set[str] = set() - for source_file in _records(capture.get("source_files")): - for entry in _records(source_file.get("source_entries")): + for source_file in records(capture.get("source_files")): + for entry in records(source_file.get("source_entries")): kinds.add("managed_source" if entry.get("managed") is True else "unmanaged_source") return kinds @@ -216,29 +159,29 @@ def _source_kinds(capture: dict[str, object]) -> set[str]: def _declaration_kinds(capture: dict[str, object]) -> set[str]: kinds: set[str] = set() for collection in DISCOVERY_DECLARATION_FILE_COLLECTIONS: - for record in _records(capture.get(collection)): - scope: str | None = _enum_name(record.get("scope_kind")) + for record in records(capture.get(collection)): + scope: str | None = enum_name(record.get("scope_kind")) if scope is not None: kinds.add(f"scope_{scope}") - for constant_file in _records(capture.get("constant_files")): - for declaration in _records(constant_file.get("declarations")): + for constant_file in records(capture.get("constant_files")): + for declaration in records(constant_file.get("declarations")): value: dict[str, object] = as_json_object(declaration.get("value")) or {} logical: dict[str, object] = as_json_object(value.get("logical_type")) or {} - kind: str | None = _enum_name(logical.get("kind")) + kind: str | None = enum_name(logical.get("kind")) if kind is not None: kinds.add(f"constant_{kind}") if not json.dumps(value.get("value"), ensure_ascii=False).isascii(): kinds.add("non_ascii_constant") - for enum_file in _records(capture.get("enum_files")): - for declaration in _records(enum_file.get("declarations")): - members: list[dict[str, object]] = _records(declaration.get("members")) + for enum_file in records(capture.get("enum_files")): + for declaration in records(enum_file.get("declarations")): + members: list[dict[str, object]] = records(declaration.get("members")) kinds.update( "enum_integer" if isinstance(member.get("value"), int) else "enum_string" for member in members ) if any( _CROSS_FILE_MACRO_IMPORT.search(str(record.get("contents", ""))) - for record in _records(capture.get("macro_files")) + for record in records(capture.get("macro_files")) ): kinds.add("cross_file_macro_import") return kinds @@ -246,15 +189,15 @@ def _declaration_kinds(capture: dict[str, object]) -> set[str]: def _test_and_audit_kinds(capture: dict[str, object]) -> set[str]: kinds: set[str] = set() - for test_file in _records(capture.get("test_files")): - for block in _records(test_file.get("blocks")): - mode: str | None = _enum_name(block.get("mode")) + for test_file in records(capture.get("test_files")): + for block in records(test_file.get("blocks")): + mode: str | None = enum_name(block.get("mode")) if mode is not None: kinds.add(f"test_mode_{mode}") - if _records(block.get("cases")): + if records(block.get("cases")): kinds.add("parameterized_test") - for audit_file in _records(capture.get("audit_files")): - declaration_kind: str | None = _enum_name(audit_file.get("declaration_kind")) + for audit_file in records(capture.get("audit_files")): + declaration_kind: str | None = enum_name(audit_file.get("declaration_kind")) kinds.add( SINGULAR_AUDIT_KIND if declaration_kind == SINGULAR_AUDIT_KIND else "generic_audit" ) @@ -265,15 +208,15 @@ def _config_kinds(capture: dict[str, object]) -> set[str]: kinds: set[str] = set() project: dict[str, object] = as_json_object(capture.get("project_config")) or {} local: dict[str, object] = as_json_object(capture.get("local_config")) or {} - if _has_content(project.get("path_defaults")): + if has_content(project.get("path_defaults")): kinds.add("path_defaults") if ( - _has_content(local.get("vars")) - or _has_content(local.get("setting_overrides")) + has_content(local.get("vars")) + or has_content(local.get("setting_overrides")) or local.get("target") is not None ): kinds.add("local_config") - if local.get("target") is not None and _mapping_size(project.get("targets")) > 1: + if local.get("target") is not None and mapping_size(project.get("targets")) > 1: kinds.add("target_override") dbt: dict[str, object] = as_json_object(project.get("dbt")) or {} if dbt.get("target_path") is not None: @@ -286,7 +229,7 @@ def _config_kinds(capture: dict[str, object]) -> set[str]: def _text_kinds(capture: dict[str, object]) -> set[str]: kinds: set[str] = set() for collection in DISCOVERY_TEXT_FILE_COLLECTIONS: - for record in _records(capture.get(collection)): + for record in records(capture.get(collection)): contents: str = str(record.get("contents", "")) if TAB in contents: kinds.add("tab") diff --git a/scripts/compiler_differential/_helpers/coverage/macro_sources.py b/scripts/compiler_differential/_helpers/coverage/macro_sources.py new file mode 100644 index 0000000000..0798dadb07 --- /dev/null +++ b/scripts/compiler_differential/_helpers/coverage/macro_sources.py @@ -0,0 +1,41 @@ +"""Read one macro function out of the captured source of the file that defines it.""" + +from __future__ import annotations + +import ast + +from scripts.compiler_differential.constants import TYPED_REFERENCE_ANNOTATION + + +def macro_function_source(*, file_source: str, name: str) -> str: + """Return the source of the top-level function `name`, or "" when the file has none.""" + + function: ast.FunctionDef | None = _function(file_source=file_source, name=name) + if function is None: + return "" + return ast.get_source_segment(file_source, function) or "" + + +def takes_typed_reference(*, file_source: str, name: str) -> bool: + """Whether the function `name` annotates a parameter with the typed reference type.""" + + function: ast.FunctionDef | None = _function(file_source=file_source, name=name) + if function is None: + return False + arguments: ast.arguments = function.args + return any( + argument.annotation is not None + and TYPED_REFERENCE_ANNOTATION in ast.unparse(argument.annotation) + for argument in (*arguments.posonlyargs, *arguments.args, *arguments.kwonlyargs) + ) + + +def _function(*, file_source: str, name: str) -> ast.FunctionDef | None: + try: + module: ast.Module = ast.parse(file_source) + except SyntaxError: + return None + return next( + (node for node in module.body if isinstance(node, ast.FunctionDef) and node.name == name), + None, + ) diff --git a/scripts/compiler_differential/_helpers/coverage/render.py b/scripts/compiler_differential/_helpers/coverage/render.py new file mode 100644 index 0000000000..ffd99a14da --- /dev/null +++ b/scripts/compiler_differential/_helpers/coverage/render.py @@ -0,0 +1,420 @@ +"""Classify `CompileProjectInputs` captures into render kinds; only rendered evidence counts.""" + +from __future__ import annotations + +import dataclasses +import json +import re +from collections.abc import Iterator +from pathlib import PurePosixPath + +from scripts.compiler_differential._helpers.comparing.compare import as_json_object +from scripts.compiler_differential._helpers.coverage.capture_values import ( + capture_problems, + enum_name, + has_content, + mapping_records, + records, +) +from scripts.compiler_differential._helpers.coverage.macro_sources import ( + macro_function_source, + takes_typed_reference, +) +from scripts.compiler_differential.constants import ( + RENDER_CALLABLE_CAPTURE_FIELDS, + RENDER_DETAIL_KINDS, + RENDER_HOOK_KEYS, + RENDER_MACRO_SOURCE_READS, + RENDER_NON_COLLECTION_FIELDS, + RENDER_STAGE_CAPTURE_SUFFIX, +) +from sqlbuild.compiler.auditing.constants import BUILT_IN_AUDIT_NAMES +from sqlbuild.compiler.compile.models import CompileProjectInputs + +_MACRO_CALL: re.Pattern[str] = re.compile(r"@([A-Za-z_]\w*)\(") +_REFERENCE_CALL: re.Pattern[str] = re.compile(r"__(ref|source|seed)\(\s*\"([^\"]+)\"\s*\)") +_PROJECT_VARIABLE: re.Pattern[str] = re.compile(r"(? tuple[str, ...]: + """Return every `CompileProjectInputs` collection field plus the finer required kinds.""" + + return (*render_collection_kinds(), *RENDER_DETAIL_KINDS) + + +def render_collection_kinds() -> tuple[str, ...]: + """Return the `CompileProjectInputs` fields that hold rendered inputs.""" + + return tuple( + field.name + for field in dataclasses.fields(CompileProjectInputs) + if field.name not in RENDER_NON_COLLECTION_FIELDS + ) + + +def project_render_kinds(captures: dict[str, dict[str, str]]) -> frozenset[str]: + """Return the render kinds every `CompileProjectInputs` capture of one project proves.""" + + kinds: set[str] = set() + for files in captures.values(): + for name, text in files.items(): + if name.endswith(RENDER_STAGE_CAPTURE_SUFFIX): + kinds.update(rendered_input_kinds(text)) + return frozenset(kinds) + + +def render_capture_problems(capture_text: str) -> tuple[str, ...]: + """Return why a render capture is incomplete or not canonical; empty when it is sound.""" + + return capture_problems( + capture_text=capture_text, + fields=tuple(field.name for field in dataclasses.fields(CompileProjectInputs)), + callable_fields=RENDER_CALLABLE_CAPTURE_FIELDS, + ) + + +def rendered_input_kinds(capture_text: str) -> frozenset[str]: + """Return the render input kinds one canonical `CompileProjectInputs` capture proves.""" + + capture: dict[str, object] = as_json_object(json.loads(capture_text)) or {} + kinds: set[str] = {kind for kind in render_collection_kinds() if has_content(capture.get(kind))} + usages: list[tuple[str, dict[str, object]]] = list(_usages(capture)) + kinds.update(_macro_kinds(capture=capture, usages=usages)) + kinds.update(_declaration_kinds(capture=capture, usages=usages)) + kinds.update(_interpolation_kinds(capture)) + kinds.update(_hook_kinds(capture)) + kinds.update(_reference_kinds(capture)) + kinds.update(_attachment_kinds(capture)) + return frozenset(kinds) + + +def _usages(capture: dict[str, object]) -> Iterator[tuple[str, dict[str, object]]]: + """Yield `(consumer collection label, usage)` for every recorded declaration usage.""" + + for model in records(capture.get("model_inputs")): + for field in ("macro_usages", "declaration_usages"): + yield from ((_MODEL_RESOURCE, usage) for usage in records(model.get(field))) + for collection, label in _CONSUMER_COLLECTIONS.items(): + for item in records(capture.get(collection)): + yield from ((label, usage) for usage in records(item.get("declaration_usages"))) + + +def _declaration_kind(usage: dict[str, object]) -> str | None: + return enum_name((as_json_object(usage.get("declaration")) or {}).get("kind")) + + +def _macro_kinds( + *, capture: dict[str, object], usages: list[tuple[str, dict[str, object]]] +) -> set[str]: + kinds: set[str] = set() + used_macros: set[str] = set() + for consumer, usage in usages: + if _declaration_kind(usage) != _MACRO_DECLARATION: + continue + used_macros.add(str((as_json_object(usage.get("declaration")) or {}).get("name"))) + kinds.add(f"macro_in_{consumer}") + typed_macros: set[str] = set() + for macro in mapping_records(capture.get("loaded_macros")): + name: str = str(macro.get("name")) + file_source: str = str(macro.get("raw_source", "")) + if takes_typed_reference(file_source=file_source, name=name): + typed_macros.add(name) + if name not in used_macros: + continue + if records(macro.get("dependencies")): + kinds.add("cross_file_macro_import") + kinds.update(_macro_source_reads(macro_function_source(file_source=file_source, name=name))) + for model in records(capture.get("model_inputs")): + kinds.update(_model_macro_kinds(model=model, typed_macros=typed_macros)) + for test in records(capture.get("test_inputs")): + if enum_name(test.get("mode")) == _MACRO_DECLARATION and any( + _declaration_kind(usage) == _MACRO_DECLARATION + for usage in records(test.get("declaration_usages")) + ): + kinds.add("tested_macro") + return kinds + + +def _macro_source_reads(source: str) -> set[str]: + kinds: set[str] = set() + for kind, markers in RENDER_MACRO_SOURCE_READS.items(): + if any(marker in source for marker in markers): + kinds.add(kind) + return kinds + + +def _model_macro_kinds(*, model: dict[str, object], typed_macros: set[str]) -> set[str]: + kinds: set[str] = set() + authored: str = str(model.get("macro_source_sql", "")) + rendered: str = str(model.get("query_sql", "")) + calls: list[tuple[str, str]] = list(_macro_calls(authored)) + if not calls or _MACRO_CALL.search(rendered): + return kinds + if any(_MACRO_CALL.search(arguments) for _, arguments in calls): + kinds.add("nested_macro_call") + referenced: set[tuple[str, str]] = { + (str(enum_name(reference.get("ref_kind"))), str(reference.get("ref_name"))) + for reference in records(model.get("references")) + } + passed: set[tuple[str, str]] = set() + for name, arguments in calls: + if name in typed_macros: + passed.update(_REFERENCE_CALL.findall(arguments)) + if passed and passed <= referenced: + kinds.add("typed_reference_argument") + written: set[tuple[str, str]] = set(_REFERENCE_CALL.findall(authored)) + if {item for item in referenced if item[0] in _RELATION_REFERENCE_KINDS} - written: + kinds.add("macro_generated_reference") + return kinds + + +def _macro_calls(sql: str) -> Iterator[tuple[str, str]]: + """Yield the name and argument text of every macro call, matching nested parentheses.""" + + for match in _MACRO_CALL.finditer(sql): + depth: int = 1 + index: int = match.end() + while index < len(sql) and depth: + depth += {"(": 1, ")": -1}.get(sql[index], 0) + index += 1 + yield match.group(1), sql[match.end() : index - 1] + + +def _declaration_kinds( + *, capture: dict[str, object], usages: list[tuple[str, dict[str, object]]] +) -> set[str]: + kinds: set[str] = set() + scopes: dict[str, str] = { + json.dumps(record.get("identity"), sort_keys=True): str(enum_name(record.get("scope"))) + for record in records( + (as_json_object(capture.get("scope_index")) or {}).get("declarations") + ) + } + list_constants: set[str] = { + str(declaration.get("name")) + for declaration in mapping_records(capture.get("public_constants")) + if enum_name( + ( + as_json_object((as_json_object(declaration.get("value")) or {}).get("logical_type")) + or {} + ).get("kind") + ) + == _LIST_VALUE + } + for consumer, usage in usages: + declaration: dict[str, object] = as_json_object(usage.get("declaration")) or {} + kind: str | None = _declaration_kind(usage) + if declaration.get("owner") is not None: + kinds.add("scope_private_use") + scope: str | None = scopes.get(json.dumps(declaration, sort_keys=True)) + if scope is not None: + kinds.add(f"scope_{scope}_use") + if kind == _ENUM_DECLARATION and usage.get("enum_member") is not None: + kinds.add("enum_member") + if kind == _CONSTANT_DECLARATION: + kinds.add( + "list_constant" if declaration.get("name") in list_constants else "scalar_constant" + ) + if consumer != _MODEL_RESOURCE and usage.get("through") is not None: + kinds.add("expected_model_grant") + for model in records(capture.get("model_inputs")): + if has_content(model.get("enum_columns")): + kinds.add("enum_column_contract") + return kinds + + +def _interpolation_kinds(capture: dict[str, object]) -> set[str]: + kinds: set[str] = set() + for model in records(capture.get("model_inputs")): + model_file: dict[str, object] = as_json_object(model.get("model_file")) or {} + authored: str = str(model_file.get("query_sql", "")) + rendered: str = str(model.get("query_sql", "")) + if _PROJECT_VARIABLE.search(authored) and not _PROJECT_VARIABLE.search(rendered): + kinds.add("project_variable") + if _ENVIRONMENT_VARIABLE in authored and _ENVIRONMENT_VARIABLE not in rendered: + kinds.add("environment_variable") + values: dict[str, object] = ( + as_json_object((as_json_object(model.get("config")) or {}).get("values")) or {} + ) + placeholders: set[str] = set(_RUNTIME_PLACEHOLDER.findall(rendered)) + defaults: dict[str, object] = as_json_object(values.get(_PLACEHOLDERS_KEY)) or {} + if placeholders and placeholders <= set(defaults): + kinds.add("runtime_placeholder") + if ( + _CURSOR_INTRINSIC in rendered + and values.get(_MATERIALIZED_KEY) == _INCREMENTAL + and values.get(_CURSOR_KEY) is not None + ): + kinds.add("cursor_intrinsic") + discovered: dict[str, object] = as_json_object(capture.get("discovered_inputs")) or {} + authored_expressions: dict[str, str] = {} + for source_file in records(discovered.get("source_files")): + for entry in records(source_file.get("source_entries")): + authored_expressions[str(entry.get("name"))] = str(entry.get("expression")) + for source in records(capture.get("source_inputs")): + entry: dict[str, object] = as_json_object(source.get("source_entry")) or {} + if authored_expressions.get(str(entry.get("name"))) not in (None, entry.get("expression")): + kinds.add("source_expression_rendered") + if _audit_arguments_rendered(capture): + kinds.add("audit_arguments") + return kinds + + +def _audit_arguments_rendered(capture: dict[str, object]) -> bool: + """Whether an authored generic-audit argument's value is bound into its rendered SQL.""" + + authored: dict[tuple[str, str | None, str], dict[str, object]] = _authored_audit_arguments( + capture + ) + for audit in records(capture.get("audit_inputs")): + relative_path: dict[str, object] = ( + as_json_object((as_json_object(audit.get("audit_file")) or {}).get("relative_path")) + or {} + ) + definition: str = PurePosixPath(str(relative_path.get("__path__", ""))).stem + key: tuple[str, str | None, str] = ( + str(audit.get("attached_target_name")), + _optional_text(audit.get("attached_column_name")), + definition, + ) + template: str = str((as_json_object(audit.get("audit_block")) or {}).get("sql_body", "")) + rendered: str = str(audit.get("sql_body", "")) + for name, value in authored.get(key, {}).items(): + literal: re.Pattern[str] = re.compile(rf"(? dict[tuple[str, str | None, str], dict[str, object]]: + """Map `(target, column, definition)` to authored arguments, except built-ins and `column`.""" + + found: dict[tuple[str, str | None, str], dict[str, object]] = {} + for model in records(capture.get("model_inputs")): + entry: dict[str, object] = as_json_object(model.get("schema_entry")) or {} + target: str = str(entry.get("name")) + attachments: list[tuple[str | None, dict[str, object]]] = [ + (None, instance) for instance in records(entry.get("audits")) + ] + for column in records(entry.get("columns")): + attachments.extend( + (str(column.get("name")), instance) for instance in records(column.get("audits")) + ) + for column_name, instance in attachments: + definition: str = str(instance.get("definition_name")) + if definition not in BUILT_IN_AUDIT_NAMES: + found[(target, column_name, definition)] = _authored_arguments(instance) + return found + + +def _authored_arguments(instance: dict[str, object]) -> dict[str, object]: + arguments: dict[str, object] = as_json_object(instance.get("arguments")) or {} + return { + name: value + for name, value in arguments.items() + if name != _IMPLICIT_AUDIT_ARGUMENT and isinstance(value, (int, float, str)) + } + + +def _optional_text(value: object) -> str | None: + return None if value is None else str(value) + + +def _hook_kinds(capture: dict[str, object]) -> set[str]: + kinds: set[str] = set() + for model in records(capture.get("model_inputs")): + header: dict[str, object] = ( + as_json_object((as_json_object(model.get("model_file")) or {}).get("header_values")) + or {} + ) + values: dict[str, object] = ( + as_json_object((as_json_object(model.get("config")) or {}).get("values")) or {} + ) + for key in RENDER_HOOK_KEYS: + authored_hooks: list[dict[str, object]] = records(header.get(key)) + rendered_hooks: list[dict[str, object]] = records(values.get(key)) + for authored, rendered in zip(authored_hooks, rendered_hooks, strict=False): + kinds.update(_hook_entry_kinds(authored=authored, rendered=rendered)) + return kinds + + +def _hook_entry_kinds(*, authored: dict[str, object], rendered: dict[str, object]) -> set[str]: + if str(rendered.get("__type__", "")).endswith(_PYTHON_HOOK_TYPE): + return {"python_hook"} + statement: str = str(rendered.get("statement", "")) + authored_text: str = json.dumps(authored, ensure_ascii=False) + kinds: set[str] = {"named_sql_hook" if rendered.get("definition_sql") else "inline_sql_hook"} + if _CONTEXT_VARIABLE in authored_text and _CONTEXT_VARIABLE not in statement: + kinds.add("hook_context_variable") + if _MACRO_CALL.search(authored_text) and not _MACRO_CALL.search(statement): + kinds.add("macro_in_hook") + if ( + has_content(rendered.get("kwargs")) + and _TEMPLATE_ARGUMENT.search(str(rendered.get("definition_sql", ""))) + and not _TEMPLATE_ARGUMENT.search(statement) + ): + kinds.add("named_hook_arguments") + return kinds + + +def _reference_kinds(capture: dict[str, object]) -> set[str]: + kinds: set[str] = set() + for collection in _REFERENCE_COLLECTIONS: + for item in records(capture.get(collection)): + kinds.update( + f"reference_{enum_name(reference.get('ref_kind'))}" + for reference in records(item.get("references")) + ) + return kinds + + +def _attachment_kinds(capture: dict[str, object]) -> set[str]: + kinds: set[str] = set() + for test in records(capture.get("test_inputs")): + kinds.add(f"{enum_name(test.get('mode'))}_test") + if test.get("case_name") is not None: + kinds.add("parameterized_test_case") + for audit in records(capture.get("audit_inputs")): + target: str | None = enum_name(audit.get("attached_target_kind")) + if target is None: + kinds.add("singular_audit") + elif audit.get("attached_column_name") is not None: + kinds.add(f"{target}_column_audit") + else: + kinds.add(f"{target}_audit") + for model in records(capture.get("model_inputs")): + config: dict[str, object] = as_json_object(model.get("config")) or {} + if config.get("matched_path_default") is not None: + kinds.add("path_default") + return kinds diff --git a/scripts/compiler_differential/_helpers/running/report.py b/scripts/compiler_differential/_helpers/running/report.py index 79a4e76dab..d8e835850d 100644 --- a/scripts/compiler_differential/_helpers/running/report.py +++ b/scripts/compiler_differential/_helpers/running/report.py @@ -2,7 +2,11 @@ from __future__ import annotations -from scripts.compiler_differential.constants import CONFIG_ONLY_KINDS +from scripts.compiler_differential.constants import ( + CONFIG_ONLY_KINDS, + RENDER_CONFIG_ONLY_KINDS, + RENDER_INDIRECT_KINDS, +) from scripts.compiler_differential.models import Difference, ProjectComparison @@ -25,13 +29,16 @@ def format_summary( comparisons: list[ProjectComparison], engines: tuple[str, str], seconds: float, - missing_coverage: tuple[str, ...] = (), + missing_coverage: dict[str, tuple[str, ...]] | None = None, ) -> str: - """Summarize the run; any difference or required-but-missing coverage reports FAILED.""" + """Summarize the run; any difference or missing coverage, keyed by stage, reports FAILED.""" differing: list[ProjectComparison] = [item for item in comparisons if item.differences] + gaps: dict[str, tuple[str, ...]] = { + stage: kinds for stage, kinds in (missing_coverage or {}).items() if kinds + } left_engine, right_engine = engines - if not differing and not missing_coverage: + if not differing and not gaps: return ( f"Compiler differential passed: {len(comparisons)} projects identical " f"({left_engine} vs {right_engine}) in {seconds:.1f}s" @@ -43,21 +50,57 @@ def format_summary( if differing: first: Difference = differing[0].differences[0] lines.append(f"First difference: {first.project}: {first.artifact} at {first.location}") - if missing_coverage: - lines.append(f"Required discovery coverage missing: {', '.join(missing_coverage)}") + lines.extend( + f"Required {stage} coverage missing: {', '.join(kinds)}" for stage, kinds in gaps.items() + ) return "\n".join(lines) def format_discovery_coverage(*, covered: frozenset[str], required: tuple[str, ...]) -> str: """Report how many required discovery input kinds the seed corpus exercised.""" - missing: list[str] = [kind for kind in required if kind not in covered] - line: str = ( - f"Discovery coverage: {len(required) - len(missing)} of {len(required)} input kinds " - "exercised by the seed corpus" + return _format_coverage( + stage="Discovery", + covered=covered, + required=required, + notes=(("config-only kinds", "only proves", CONFIG_ONLY_KINDS),), ) - config_only: str = "; ".join( - f"{kind} (only proves {meaning})" for kind, meaning in CONFIG_ONLY_KINDS.items() + + +def format_render_coverage(*, covered: frozenset[str], required: tuple[str, ...]) -> str: + """Report how many required render input kinds the seed corpus exercised.""" + + return _format_coverage( + stage="Render", + covered=covered, + required=required, + notes=( + ("config-only kinds", "only proves", RENDER_CONFIG_ONLY_KINDS), + ("indirect kinds", "credited because", RENDER_INDIRECT_KINDS), + ), ) - line = f"{line}\n config-only kinds: {config_only}" - return line if not missing else f"{line}\n missing: {', '.join(missing)}" + + +def _format_coverage( + *, + stage: str, + covered: frozenset[str], + required: tuple[str, ...], + notes: tuple[tuple[str, str, dict[str, str]], ...], +) -> str: + missing: list[str] = [kind for kind in required if kind not in covered] + lines: list[str] = [ + f"{stage} coverage: {len(required) - len(missing)} of {len(required)} input kinds " + "exercised by the seed corpus" + ] + for label, verb, kinds in notes: + by_meaning: dict[str, list[str]] = {} + for kind, meaning in kinds.items(): + by_meaning.setdefault(meaning, []).append(kind) + described: str = "; ".join( + f"{', '.join(names)} ({verb} {meaning})" for meaning, names in by_meaning.items() + ) + lines.append(f" {label}: {described or 'none'}") + if missing: + lines.append(f" missing: {', '.join(missing)}") + return "\n".join(lines) diff --git a/scripts/compiler_differential/_helpers/running/run.py b/scripts/compiler_differential/_helpers/running/run.py index 49ed382374..44ce33fec8 100644 --- a/scripts/compiler_differential/_helpers/running/run.py +++ b/scripts/compiler_differential/_helpers/running/run.py @@ -29,6 +29,7 @@ def differential_options(*, args: argparse.Namespace, work_dir: Path) -> Differe python=args.python, engine_environment=parse_engine_environment(args.engine_env), require_discovery_coverage=args.require_discovery_coverage, + require_render_coverage=args.require_render_coverage, ) diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 7f58c28ad4..43d558123c 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -368,3 +368,92 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "the config sets [dbt] target_path; the manifest is read after discovery" ), } +RENDER_STAGE_CAPTURE_SUFFIX: str = "-compile_project_inputs.json" +RENDER_NON_COLLECTION_FIELDS: frozenset[str] = frozenset( + { + "project_config", + "local_config", + "discovered_inputs", + "sql_lexical_syntax", + "run_id", + "effective_target_name", + "effective_target", + "compile_cache_dir", + "effective_connection", + "effective_settings", + "no_sql_validation", + "effective_vars", + "macro_context", + "diagnostics", + "external_sql_reference_resolver", + "scope_index", + "declaration_scope", + "analysis_reuse", + } +) +RENDER_CALLABLE_CAPTURE_FIELDS: frozenset[str] = CALLABLE_CAPTURE_FIELDS +TYPED_REFERENCE_ANNOTATION: str = "SqlResourceRef" +RENDER_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") +RENDER_MACRO_SOURCE_READS: dict[str, tuple[str, ...]] = { + "macro_reads_vars": ("ctx.vars",), + "macro_reads_constants": ("ctx.constants", "ctx.render_constant"), + "macro_reads_enums": ("ctx.enums", "ctx.render_enum_member"), + "macro_reads_target": ("ctx.target_name", "ctx.adapter_name"), + "macro_reads_environment": ("os.environ",), +} +RENDER_DETAIL_KINDS: tuple[str, ...] = ( + "macro_in_model", + "macro_in_test", + "macro_in_scenario", + "macro_in_audit", + "macro_in_function", + "macro_in_source_expression", + "macro_in_hook", + "nested_macro_call", + "typed_reference_argument", + "macro_generated_reference", + "cross_file_macro_import", + "tested_macro", + *RENDER_MACRO_SOURCE_READS, + "scope_global_use", + "scope_inherited_use", + "scope_local_use", + "scope_private_use", + "enum_member", + "scalar_constant", + "list_constant", + "expected_model_grant", + "enum_column_contract", + "project_variable", + "environment_variable", + "runtime_placeholder", + "cursor_intrinsic", + "source_expression_rendered", + "audit_arguments", + "python_hook", + "named_sql_hook", + "inline_sql_hook", + "hook_context_variable", + "named_hook_arguments", + "reference_ref", + "reference_source", + "reference_seed", + "reference_udf", + "reference_table_function", + "reference_dbt_ref", + "model_test", + "macro_test", + "udf_test", + "parameterized_test_case", + "singular_audit", + "model_audit", + "model_column_audit", + "source_column_audit", + "seed_column_audit", + "path_default", +) +RENDER_INDIRECT_KINDS: dict[str, str] = { + kind: "the used macro's own function source reads it; the capture does not record context reads" + for kind in RENDER_MACRO_SOURCE_READS +} +RENDER_CONFIG_ONLY_KINDS: dict[str, str] = {} diff --git a/scripts/compiler_differential/main/differential.py b/scripts/compiler_differential/main/differential.py index a7d7974f1a..fdb49af5c1 100644 --- a/scripts/compiler_differential/main/differential.py +++ b/scripts/compiler_differential/main/differential.py @@ -10,9 +10,11 @@ from pathlib import Path from scripts.compiler_differential._helpers.coverage.discovery import required_discovery_kinds +from scripts.compiler_differential._helpers.coverage.render import required_render_kinds from scripts.compiler_differential._helpers.running.options import parse_expected_outcome from scripts.compiler_differential._helpers.running.report import ( format_discovery_coverage, + format_render_coverage, format_summary, ) from scripts.compiler_differential._helpers.running.run import ( @@ -44,15 +46,11 @@ def run_compiler_differential(argv: list[str] | None = None) -> int: comparisons: list[ProjectComparison] = compare_corpus( corpus=selected_corpus(args=args, repo_root=repo_root), options=options ) - missing_coverage: tuple[str, ...] = () - if options.stage_captures and CORPUS_SEEDS in args.corpus: - covered: frozenset[str] = frozenset().union( - *(comparison.discovered_kinds for comparison in comparisons) - ) - required: tuple[str, ...] = required_discovery_kinds() - print(format_discovery_coverage(covered=covered, required=required)) - if options.require_discovery_coverage: - missing_coverage = tuple(kind for kind in required if kind not in covered) + missing_coverage: dict[str, tuple[str, ...]] = ( + _seed_coverage(comparisons=comparisons, options=options) + if options.stage_captures and CORPUS_SEEDS in args.corpus + else {} + ) print( format_summary( comparisons=comparisons, @@ -61,7 +59,35 @@ def run_compiler_differential(argv: list[str] | None = None) -> int: missing_coverage=missing_coverage, ) ) - return 1 if missing_coverage or any(comparison.differences for comparison in comparisons) else 0 + return ( + 1 + if any(missing_coverage.values()) + or any(comparison.differences for comparison in comparisons) + else 0 + ) + + +def _seed_coverage( + *, comparisons: list[ProjectComparison], options: DifferentialOptions +) -> dict[str, tuple[str, ...]]: + """Print discovery and render coverage; return the missing kinds each requirement enforces.""" + + discovered: frozenset[str] = frozenset().union( + *(comparison.discovered_kinds for comparison in comparisons) + ) + rendered: frozenset[str] = frozenset().union( + *(comparison.rendered_kinds for comparison in comparisons) + ) + required_discovery: tuple[str, ...] = required_discovery_kinds() + required_render: tuple[str, ...] = required_render_kinds() + print(format_discovery_coverage(covered=discovered, required=required_discovery)) + print(format_render_coverage(covered=rendered, required=required_render)) + missing: dict[str, tuple[str, ...]] = {} + if options.require_discovery_coverage: + missing["discovery"] = tuple(kind for kind in required_discovery if kind not in discovered) + if options.require_render_coverage: + missing["render"] = tuple(kind for kind in required_render if kind not in rendered) + return missing def _parse_args(argv: list[str] | None) -> argparse.Namespace: @@ -99,6 +125,11 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: action="store_true", help="fail unless the seed corpus exercises every discovery input kind (needs captures)", ) + parser.add_argument( + "--require-render-coverage", + action="store_true", + help="fail unless the seed corpus exercises every render input kind (needs captures)", + ) parser.add_argument( "--engine-env", action="append", @@ -109,8 +140,10 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: parser.add_argument("--work-dir", type=Path, default=None, help="keep run directories here") parser.add_argument("--python", type=Path, default=Path(sys.executable)) args: argparse.Namespace = parser.parse_args(argv) - if args.require_discovery_coverage and not ( - args.stage_captures and CORPUS_SEEDS in args.corpus + for flag, required in ( + ("--require-discovery-coverage", args.require_discovery_coverage), + ("--require-render-coverage", args.require_render_coverage), ): - parser.error("--require-discovery-coverage needs --stage-captures and the seeds corpus") + if required and not (args.stage_captures and CORPUS_SEEDS in args.corpus): + parser.error(f"{flag} needs --stage-captures and the seeds corpus") return args diff --git a/scripts/compiler_differential/models.py b/scripts/compiler_differential/models.py index cb2bf0b149..f9c89633c9 100644 --- a/scripts/compiler_differential/models.py +++ b/scripts/compiler_differential/models.py @@ -34,7 +34,7 @@ class CorpusProject: source_dir: Path | None = None writer: Callable[[Path], None] | None = None project_subdirectory: str | None = None - discovery_coverage: bool = False + seed_coverage: bool = False @dataclass(frozen=True) @@ -89,6 +89,7 @@ class ProjectComparison: differences: tuple[Difference, ...] seconds: float discovered_kinds: frozenset[str] = frozenset() + rendered_kinds: frozenset[str] = frozenset() @dataclass(frozen=True) @@ -102,6 +103,7 @@ class DifferentialOptions: python: Path engine_environment: dict[str, dict[str, str]] require_discovery_coverage: bool = False + require_render_coverage: bool = False @dataclass(frozen=True) diff --git a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py index 204b4a628c..0d80bd3002 100644 --- a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py +++ b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py @@ -195,7 +195,26 @@ def test_given_both_engines_fail_alike_when_outcome_is_unexpected_then_harness_f "Required discovery coverage missing: ", ), expected_absent=("passed",), - ) + ), + CoverageFailureTestCase( + description="one_seed_cannot_cover_every_render_kind", + extra_arguments=( + "--corpus", + "seeds", + "--seeds", + "1", + "--stage-captures", + "--require-render-coverage", + ), + expected_exit_code=1, + expected_lines=( + "OK seed/0", + "Render coverage: ", + "Compiler differential FAILED: 0 of 1 projects differ", + "Required render coverage missing: ", + ), + expected_absent=("passed", "Required discovery coverage missing"), + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py index ed46332262..a9e3ca07af 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py @@ -4,6 +4,7 @@ from dataclasses import dataclass +from sqlbuild.compiler.compile.models import CompileProjectInputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs @@ -51,3 +52,22 @@ class EncodedInputsKindsTestCase: description: str inputs: DiscoveredProjectInputs expected_kinds: frozenset[str] + + +@dataclass(frozen=True) +class RenderCaptureKindsTestCase: + """One render capture fragment and the render kinds it must and must not prove.""" + + description: str + capture: dict[str, object] + expected_present: frozenset[str] + expected_absent: frozenset[str] + + +@dataclass(frozen=True) +class RenderEncodedInputsTestCase: + """Real compile inputs, encoded by the stage capture, and the render kinds they prove.""" + + description: str + inputs: CompileProjectInputs + expected_kinds: frozenset[str] diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py index 6774d6a053..7ba7b7441c 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py @@ -3,7 +3,9 @@ from __future__ import annotations import dataclasses +from typing import cast +from sqlbuild.compiler.compile.models import CompileProjectInputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs _VALUE_KIND: str = "sqlbuild.sql_values.types:SqlValueKind" @@ -25,3 +27,76 @@ def empty_inputs_capture(**overrides: object) -> dict[str, object]: **{field.name: [] for field in dataclasses.fields(DiscoveredProjectInputs)}, **overrides, } + + +_DECLARATION_KIND: str = "sqlbuild.compiler.scopes.types:DeclarationKind" +_RESOURCE_KIND: str = "sqlbuild.compiler.scopes.types:ResourceKind" +_AUDIT_TARGET: str = "sqlbuild.compiler.compile.types:AttachedAuditTargetKind" +_REFERENCE_KIND: str = "sqlbuild.compiler.references.types:SqlReferenceKind" + + +def empty_render_capture(**overrides: object) -> dict[str, object]: + """Return a capture with every `CompileProjectInputs` field, in order, then overrides.""" + + return { + "__type__": "sqlbuild.compiler.compile.models:CompileProjectInputs", + **{field.name: [] for field in dataclasses.fields(CompileProjectInputs)}, + **overrides, + } + + +def usage(*, kind: str, name: str, enum_member: str | None = None) -> dict[str, object]: + """Return one captured direct usage of a project declaration.""" + + return { + "declaration": { + "kind": {"__enum__": f"{_DECLARATION_KIND}.{kind}"}, + "name": name, + "owner": None, + }, + "through": None, + "enum_member": enum_member, + } + + +def private_usage(*, kind: str, name: str, owner: str) -> dict[str, object]: + """Return one captured usage of a declaration private to the model `owner`.""" + + direct: dict[str, object] = usage(kind=kind, name=name) + return { + **direct, + "declaration": {**_as_object(direct["declaration"]), "owner": _model(owner)}, + } + + +def granted_usage(*, kind: str, name: str, through: str) -> dict[str, object]: + """Return one captured usage granted through the expected model `through`.""" + + return {**usage(kind=kind, name=name), "through": _model(through)} + + +def _model(name: str) -> dict[str, object]: + return {"kind": {"__enum__": f"{_RESOURCE_KIND}.MODEL"}, "name": name} + + +def _as_object(value: object) -> dict[str, object]: + return cast(dict[str, object], value) + + +def reference(*, kind: str, name: str) -> dict[str, object]: + """Return one captured SQL reference.""" + + return {"ref_kind": {"__enum__": f"{_REFERENCE_KIND}.{kind}"}, "ref_name": name} + + +def audit_input(*, definition: str, template: str, rendered: str) -> dict[str, object]: + """Return one captured generic audit attached to column `amount` of model `orders`.""" + + return { + "audit_file": {"relative_path": {"__path__": f"audits/generic/{definition}.sql"}}, + "audit_block": {"sql_body": template}, + "sql_body": rendered, + "attached_target_kind": {"__enum__": f"{_AUDIT_TARGET}.MODEL"}, + "attached_target_name": "orders", + "attached_column_name": "amount", + } diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py new file mode 100644 index 0000000000..5f84d87d99 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_render.py @@ -0,0 +1,484 @@ +"""Render captures are classified into the input kinds the seed corpus must exercise.""" + +from __future__ import annotations + +import dataclasses +import json + +import pytest + +from scripts.compiler_differential._helpers.coverage.render import ( + project_render_kinds, + render_capture_problems, + rendered_input_kinds, + required_render_kinds, +) +from scripts.compiler_differential.constants import RENDER_NON_COLLECTION_FIELDS +from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig +from tests.unit.scripts.compiler_differential._helpers.coverage._test_types import ( + CaptureProblemsTestCase, + RenderCaptureKindsTestCase, + RenderEncodedInputsTestCase, + RequiredKindsTestCase, +) +from tests.unit.scripts.compiler_differential._helpers.coverage.helpers import ( + audit_input, + empty_render_capture, + granted_usage, + private_usage, + reference, + usage, +) + +_HOOK: str = "sqlbuild.compiler.discovery.models" +_TARGET: str = "sqlbuild.compiler.compile.types:AttachedAuditTargetKind" +_TEST_MODE: str = "sqlbuild.compiler.compile.types:SqlTestMode" +_SCOPE: str = "sqlbuild.compiler.scopes.types:ScopeKind" +_VALUE_KIND: str = "sqlbuild.sql_values.types:SqlValueKind" +_SHARED_MACRO_FILE: str = ( + "import os\n\n\n" + "def round_it(ctx, value):\n return f\"ROUND({value}) + {ctx.vars['x']}\"\n\n\n" + "def region_tag():\n return os.environ.get('REGION', 'east')\n" +) +_TYPED_MACRO_FILE: str = ( + "from sqlbuild.refs import SqlResourceRef\n\n\n" + "def union_ids(relations: list[SqlResourceRef]) -> str:\n" + ' return " UNION ALL ".join(f"SELECT id FROM {item}" for item in relations)\n' +) + + +@pytest.mark.parametrize( + "test_case", + [ + RenderCaptureKindsTestCase( + description="macro_calls_rendered_in_a_model", + capture=empty_render_capture( + model_inputs=[ + { + "macro_source_sql": ( + 'SELECT @round_it(@double("amount")) AS amount\n' + 'FROM @union_ids([__ref("orders"), __ref("returns")])\n' + "JOIN @legacy_customers() c ON TRUE" + ), + "query_sql": ( + "SELECT ROUND(amount * 2) AS amount\n" + 'FROM (SELECT id FROM __ref("orders") UNION ALL ' + 'SELECT id FROM __ref("returns"))\n' + 'JOIN __ref("customers") c ON TRUE' + ), + "references": [ + reference(kind="REF", name="orders"), + reference(kind="REF", name="returns"), + reference(kind="REF", name="customers"), + ], + "macro_usages": [ + usage(kind="MACRO", name="round_it"), + usage(kind="MACRO", name="union_ids"), + ], + } + ], + loaded_macros={ + "round_it": { + "name": "round_it", + "raw_source": _SHARED_MACRO_FILE, + "dependencies": [{"kind": "macro", "name": "base"}], + }, + "region_tag": { + "name": "region_tag", + "raw_source": _SHARED_MACRO_FILE, + "dependencies": [{"kind": "macro", "name": "base"}], + }, + "union_ids": { + "name": "union_ids", + "raw_source": _TYPED_MACRO_FILE, + "dependencies": [], + }, + }, + ), + expected_present=frozenset( + { + "model_inputs", + "loaded_macros", + "macro_in_model", + "nested_macro_call", + "typed_reference_argument", + "macro_generated_reference", + "cross_file_macro_import", + "macro_reads_vars", + "reference_ref", + } + ), + expected_absent=frozenset({"macro_reads_environment", "macro_in_test"}), + ), + RenderCaptureKindsTestCase( + description="unused_or_untyped_macros_prove_nothing", + capture=empty_render_capture( + model_inputs=[ + { + "macro_source_sql": 'SELECT * FROM @round_it(__ref("orders"))', + "query_sql": 'SELECT * FROM ROUND(__ref("orders"))', + "references": [reference(kind="REF", name="orders")], + } + ], + loaded_macros={ + "round_it": { + "name": "round_it", + "raw_source": _SHARED_MACRO_FILE, + "dependencies": [{"kind": "macro", "name": "base"}], + }, + }, + ), + expected_present=frozenset({"model_inputs", "reference_ref"}), + expected_absent=frozenset( + { + "typed_reference_argument", + "cross_file_macro_import", + "macro_reads_vars", + "macro_reads_environment", + } + ), + ), + RenderCaptureKindsTestCase( + description="unexpanded_macro_text_proves_nothing", + capture=empty_render_capture( + model_inputs=[ + { + "macro_source_sql": 'SELECT @round_it(@double("amount")) FROM t', + "query_sql": 'SELECT @round_it(@double("amount")) FROM t', + } + ] + ), + expected_present=frozenset({"model_inputs"}), + expected_absent=frozenset( + {"nested_macro_call", "typed_reference_argument", "macro_generated_reference"} + ), + ), + RenderCaptureKindsTestCase( + description="interpolated_templates", + capture=empty_render_capture( + model_inputs=[ + { + "model_file": { + "query_sql": "SELECT '@@site' AS site, '@@ENV:CHANNEL' AS channel" + }, + "query_sql": ( + "SELECT 'north' AS site, 'web' AS channel\n" + "WHERE d >= __cursor_start() AND d >= @@@window_start" + ), + "config": { + "values": { + "materialized": "incremental", + "cursor": "d", + "placeholders": {"window_start": "'2026-01-01'"}, + } + }, + "schema_entry": { + "name": "orders", + "audits": [], + "columns": [ + { + "name": "amount", + "audits": [ + {"definition_name": "floor", "arguments": {"floor": -17}} + ], + } + ], + }, + } + ], + source_inputs=[ + {"source_entry": {"name": "orders", "expression": "(SELECT 0.5 AS rate)"}} + ], + audit_inputs=[ + audit_input( + definition="floor", + template="SELECT * FROM t WHERE @column < @floor", + rendered="SELECT * FROM t WHERE amount < -17", + ) + ], + discovered_inputs={ + "source_files": [ + { + "source_entries": [ + {"name": "orders", "expression": "(SELECT @@rate AS rate)"} + ] + } + ] + }, + ), + expected_present=frozenset( + { + "project_variable", + "environment_variable", + "cursor_intrinsic", + "runtime_placeholder", + "source_expression_rendered", + "audit_arguments", + "model_column_audit", + } + ), + expected_absent=frozenset({"model_audit", "singular_audit"}), + ), + RenderCaptureKindsTestCase( + description="surviving_tokens_without_config_and_built_in_audit_tokens_prove_nothing", + capture=empty_render_capture( + model_inputs=[ + { + "query_sql": "SELECT __cursor_start() AS s, @@@window_start AS w", + "config": {"values": {"materialized": "view"}}, + "schema_entry": { + "name": "orders", + "audits": [], + "columns": [ + { + "name": "amount", + "audits": [{"definition_name": "not_null", "arguments": {}}], + } + ], + }, + } + ], + audit_inputs=[ + audit_input( + definition="not_null", + template="SELECT @column FROM @relation WHERE @column IS NULL", + rendered='SELECT amount FROM __ref("orders") WHERE amount IS NULL', + ) + ], + ), + expected_present=frozenset({"model_column_audit"}), + expected_absent=frozenset( + {"cursor_intrinsic", "runtime_placeholder", "audit_arguments"} + ), + ), + RenderCaptureKindsTestCase( + description="unsubstituted_variables_prove_nothing", + capture=empty_render_capture( + model_inputs=[ + { + "model_file": {"query_sql": "SELECT '@@site', '@@ENV:CHANNEL'"}, + "query_sql": "SELECT '@@site', '@@ENV:CHANNEL'", + } + ] + ), + expected_present=frozenset({"model_inputs"}), + expected_absent=frozenset({"project_variable", "environment_variable"}), + ), + RenderCaptureKindsTestCase( + description="hooks_rendered_from_their_authored_entries", + capture=empty_render_capture( + model_inputs=[ + { + "model_file": { + "header_values": { + "post_hooks": [ + { + "__type__": f"{_HOOK}:SqlHookEntry", + "statement": "SELECT @note('@@CTX:destination.qualified')", + }, + { + "__type__": f"{_HOOK}:NamedSqlHookEntry", + "kwargs": {"label": "orders"}, + }, + {"__type__": f"{_HOOK}:PythonHookEntry", "name": "log"}, + ] + } + }, + "config": { + "values": { + "post_hooks": [ + { + "__type__": f"{_HOOK}:SqlHookEntry", + "statement": "SELECT 'refreshed main.orders'", + "definition_sql": None, + }, + { + "__type__": f"{_HOOK}:SqlHookEntry", + "statement": "SELECT 'orders' AS label", + "definition_sql": "SELECT @'label' AS label", + "kwargs": {"label": "orders"}, + }, + {"__type__": f"{_HOOK}:PythonHookEntry", "name": "log"}, + ] + }, + "matched_path_default": "marts", + }, + } + ] + ), + expected_present=frozenset( + { + "inline_sql_hook", + "named_sql_hook", + "python_hook", + "hook_context_variable", + "macro_in_hook", + "named_hook_arguments", + "path_default", + } + ), + expected_absent=frozenset(), + ), + RenderCaptureKindsTestCase( + description="declarations_used_by_resources", + capture=empty_render_capture( + model_inputs=[ + { + "declaration_usages": [ + usage(kind="ENUM", name="status", enum_member="OPEN"), + usage(kind="CONSTANT", name="weights"), + private_usage(kind="CONSTANT", name="_floor", owner="orders"), + ], + "enum_columns": {"status": {"name": "status"}}, + } + ], + test_inputs=[ + { + "mode": {"__enum__": f"{_TEST_MODE}.MACRO"}, + "declaration_usages": [ + granted_usage(kind="MACRO", name="double", through="orders") + ], + "case_name": "small", + } + ], + public_constants={ + "weights": { + "name": "weights", + "value": {"logical_type": {"kind": {"__enum__": f"{_VALUE_KIND}.LIST"}}}, + } + }, + scope_index={ + "declarations": [ + { + "identity": usage(kind="ENUM", name="status")["declaration"], + "scope": {"__enum__": f"{_SCOPE}.INHERITED"}, + } + ] + }, + ), + expected_present=frozenset( + { + "enum_member", + "list_constant", + "scope_private_use", + "scope_inherited_use", + "enum_column_contract", + "macro_in_test", + "tested_macro", + "macro_test", + "expected_model_grant", + "parameterized_test_case", + } + ), + expected_absent=frozenset({"scope_local_use", "scope_global_use"}), + ), + RenderCaptureKindsTestCase( + description="encoded_empty_collections", + capture=empty_render_capture( + loaded_macros={}, + public_enums={"__mapping__": []}, + model_inputs=[{"enum_columns": {"__mapping__": []}}], + ), + expected_present=frozenset({"model_inputs"}), + expected_absent=frozenset({"loaded_macros", "public_enums", "enum_column_contract"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_render_capture_when_classifying_then_only_rendered_inputs_are_credited( + test_case: RenderCaptureKindsTestCase, +) -> None: + kinds: frozenset[str] = rendered_input_kinds(json.dumps(test_case.capture)) + + assert test_case.expected_present <= kinds, test_case.expected_present - kinds + assert not test_case.expected_absent & kinds, test_case.expected_absent & kinds + + +@pytest.mark.parametrize( + "test_case", + [ + RenderEncodedInputsTestCase( + description="empty_project", + inputs=CompileProjectInputs( + project_config=ProjectConfig(name="orders", adapter="duckdb"), + local_config=LocalConfig(), + discovered_inputs=DiscoveredProjectInputs( + project_config=ProjectConfig(name="orders", adapter="duckdb"), + local_config=LocalConfig(), + ), + sql_lexical_syntax=SqlLexicalSyntax(), + ), + expected_kinds=frozenset(), + ) + ], + ids=lambda case: case.description, +) +def test_given_encoded_compile_inputs_when_classifying_then_encoded_empty_values_prove_nothing( + test_case: RenderEncodedInputsTestCase, +) -> None: + capture: str = render_stage_capture(test_case.inputs) + + assert rendered_input_kinds(capture) == test_case.expected_kinds + assert project_render_kinds({"0-compile": {"002-compile_project_inputs.json": capture}}) == ( + test_case.expected_kinds + ) + + +@pytest.mark.parametrize( + "test_case", + [ + RequiredKindsTestCase( + description="every_rendered_collection", + expected_required=frozenset( + field.name for field in dataclasses.fields(CompileProjectInputs) + ) + - RENDER_NON_COLLECTION_FIELDS, + ) + ], + ids=lambda case: case.description, +) +def test_given_compile_project_inputs_when_listing_required_kinds_then_collections_are_required( + test_case: RequiredKindsTestCase, +) -> None: + assert test_case.expected_required <= set(required_render_kinds()) + + +@pytest.mark.parametrize( + "test_case", + [ + CaptureProblemsTestCase( + description="sound_capture", + capture=empty_render_capture( + loaded_macros={"double": {"function": {"__callable__": "macros.math:double"}}} + ), + expected_problem_count=0, + ), + CaptureProblemsTestCase( + description="opaque_value_and_unnamed_callable", + capture=empty_render_capture( + loaded_macros={"double": {"function": {"__opaque__": "macros.math:Doubler"}}} + ), + expected_problem_count=2, + ), + CaptureProblemsTestCase( + description="missing_field", + capture={"__type__": "CompileProjectInputs", "model_inputs": []}, + expected_problem_count=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_render_capture_when_checking_then_incomplete_or_opaque_parts_are_reported( + test_case: CaptureProblemsTestCase, +) -> None: + problems: tuple[str, ...] = render_capture_problems(json.dumps(test_case.capture)) + + assert len(problems) == test_case.expected_problem_count, problems + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py index 0249e11836..87744bb7f6 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py +++ b/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass from scripts.compiler_differential.models import ExpectedOutcome, ProjectComparison @@ -48,6 +49,7 @@ class CoverageReportTestCase: """Covered kinds and the coverage report lines the harness must print.""" description: str + formatter: Callable[..., str] covered: frozenset[str] required: tuple[str, ...] expected_lines: tuple[str, ...] @@ -59,6 +61,6 @@ class SummaryTestCase: description: str comparisons: list[ProjectComparison] - missing_coverage: tuple[str, ...] + missing_coverage: dict[str, tuple[str, ...]] expected_lines: tuple[str, ...] expected_absent: tuple[str, ...] diff --git a/tests/unit/scripts/compiler_differential/_helpers/running/test_report.py b/tests/unit/scripts/compiler_differential/_helpers/running/test_report.py index 1662c3efb9..16dc259015 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/running/test_report.py +++ b/tests/unit/scripts/compiler_differential/_helpers/running/test_report.py @@ -6,6 +6,7 @@ from scripts.compiler_differential._helpers.running.report import ( format_discovery_coverage, + format_render_coverage, format_summary, ) from scripts.compiler_differential.models import Difference, ProjectComparison @@ -31,6 +32,7 @@ [ CoverageReportTestCase( description="complete", + formatter=format_discovery_coverage, covered=frozenset({"model_files", "crlf"}), required=("model_files", "crlf"), expected_lines=( @@ -40,6 +42,7 @@ ), CoverageReportTestCase( description="missing_kinds_are_named_in_required_order", + formatter=format_discovery_coverage, covered=frozenset({"model_files"}), required=("providers", "model_files", "crlf"), expected_lines=( @@ -47,13 +50,26 @@ " missing: providers, crlf", ), ), + CoverageReportTestCase( + description="render_names_indirect_and_missing_kinds", + formatter=format_render_coverage, + covered=frozenset({"model_inputs", "macro_reads_vars"}), + required=("model_inputs", "macro_reads_vars", "typed_reference_argument"), + expected_lines=( + "Render coverage: 2 of 3 input kinds exercised by the seed corpus", + " config-only kinds: none", + "indirect kinds: macro_reads_vars, macro_reads_constants, macro_reads_enums,", + "(credited because the used macro's own function source reads it", + " missing: typed_reference_argument", + ), + ), ], ids=lambda case: case.description, ) def test_given_covered_kinds_when_reporting_then_missing_kinds_are_named( test_case: CoverageReportTestCase, ) -> None: - report: str = format_discovery_coverage(covered=test_case.covered, required=test_case.required) + report: str = test_case.formatter(covered=test_case.covered, required=test_case.required) assert all(line in report for line in test_case.expected_lines), report @@ -64,24 +80,25 @@ def test_given_covered_kinds_when_reporting_then_missing_kinds_are_named( SummaryTestCase( description="identical_and_covered", comparisons=[_IDENTICAL], - missing_coverage=(), + missing_coverage={"discovery": (), "render": ()}, expected_lines=("Compiler differential passed: 1 projects identical",), expected_absent=("FAILED",), ), SummaryTestCase( description="identical_but_coverage_missing", comparisons=[_IDENTICAL], - missing_coverage=("providers", "crlf"), + missing_coverage={"discovery": ("providers", "crlf"), "render": ("path_default",)}, expected_lines=( "Compiler differential FAILED: 0 of 1 projects differ", "Required discovery coverage missing: providers, crlf", + "Required render coverage missing: path_default", ), expected_absent=("passed",), ), SummaryTestCase( description="differences_and_coverage_missing", comparisons=[_IDENTICAL, _DIFFERING], - missing_coverage=("providers",), + missing_coverage={"discovery": ("providers",)}, expected_lines=( "Compiler differential FAILED: 1 of 2 projects differ", "First difference: seed/1: `compile` stdout at /models/0", diff --git a/tests/unit/scripts/compiler_differential/main/_test_types.py b/tests/unit/scripts/compiler_differential/main/_test_types.py index 9dedbb9673..66189fc444 100644 --- a/tests/unit/scripts/compiler_differential/main/_test_types.py +++ b/tests/unit/scripts/compiler_differential/main/_test_types.py @@ -66,3 +66,12 @@ class RareFeatureSeedTestCase: expected_error_code: str expected_succeeding_commands: tuple[str, ...] other_seeds: tuple[int, ...] + + +@dataclass(frozen=True) +class CoverageFlagUsageTestCase: + """A coverage requirement given without what it needs, and the usage error it reports.""" + + description: str + arguments: tuple[str, ...] + expected_message: str diff --git a/tests/unit/scripts/compiler_differential/main/test_differential_usage.py b/tests/unit/scripts/compiler_differential/main/test_differential_usage.py new file mode 100644 index 0000000000..d88235215d --- /dev/null +++ b/tests/unit/scripts/compiler_differential/main/test_differential_usage.py @@ -0,0 +1,47 @@ +"""Coverage requirements are rejected unless the run captures stages for the seed corpus.""" + +from __future__ import annotations + +import pytest + +from scripts.compiler_differential.main.differential import run_compiler_differential +from tests.unit.scripts.compiler_differential.main._test_types import CoverageFlagUsageTestCase + +_USAGE_ERROR_EXIT_CODE: int = 2 + + +@pytest.mark.parametrize( + "test_case", + [ + CoverageFlagUsageTestCase( + description="render_coverage_without_captures", + arguments=("--corpus", "seeds", "--require-render-coverage"), + expected_message="--require-render-coverage needs --stage-captures and the seeds corpus", + ), + CoverageFlagUsageTestCase( + description="render_coverage_without_seeds", + arguments=("--corpus", "fixtures", "--stage-captures", "--require-render-coverage"), + expected_message="--require-render-coverage needs --stage-captures and the seeds corpus", + ), + CoverageFlagUsageTestCase( + description="discovery_coverage_without_captures", + arguments=("--corpus", "seeds", "--require-discovery-coverage"), + expected_message=( + "--require-discovery-coverage needs --stage-captures and the seeds corpus" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_coverage_requirement_without_captured_seeds_when_parsing_then_usage_error( + test_case: CoverageFlagUsageTestCase, capsys: pytest.CaptureFixture[str] +) -> None: + with pytest.raises(SystemExit) as raised: + _ = run_compiler_differential(list(test_case.arguments)) + + assert raised.value.code == _USAGE_ERROR_EXIT_CODE + assert test_case.expected_message in capsys.readouterr().err + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From f12e6423a8368d977b82c9e7af03f175d138ca11 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 08:57:20 +0100 Subject: [PATCH 3/4] test(compiler): add a render failure corpus Add one failing project per reachable render, scope and attachment code (P005, P007, P008, P009, S006, S007, S008, S009, K012, C214 and the C216 warning) and P001 cases for macro, interpolation, reference and hook argument errors, each checked against its first error message. render_error_codes derives the code set from the compiler; the end-to-end test requires every code to be reached unless it is listed as unreachable with a reason. Share the base-project case builder across the corpus modules. --- .../_helpers/comparing/comparison.py | 20 +- .../_helpers/corpus/case_builder.py | 49 +++ .../corpus/discovery_failure_cases.py | 96 +++-- .../_helpers/corpus/emitted_codes.py | 77 +++- .../_helpers/corpus/failure_cases.py | 199 +++++----- .../_helpers/corpus/render_failure_cases.py | 354 ++++++++++++++++++ scripts/compiler_differential/constants.py | 36 ++ scripts/compiler_differential/models.py | 2 + .../compiler_differential/_test_types.py | 3 + .../test_failure_corpus_codes.py | 34 +- .../_helpers/corpus/_test_types.py | 11 + .../_helpers/corpus/test_render_codes.py | 45 +++ 12 files changed, 763 insertions(+), 163 deletions(-) create mode 100644 scripts/compiler_differential/_helpers/corpus/case_builder.py create mode 100644 scripts/compiler_differential/_helpers/corpus/render_failure_cases.py create mode 100644 tests/unit/scripts/compiler_differential/_helpers/corpus/test_render_codes.py diff --git a/scripts/compiler_differential/_helpers/comparing/comparison.py b/scripts/compiler_differential/_helpers/comparing/comparison.py index 71c3ae0851..97dc2c2eea 100644 --- a/scripts/compiler_differential/_helpers/comparing/comparison.py +++ b/scripts/compiler_differential/_helpers/comparing/comparison.py @@ -72,14 +72,28 @@ def compare_engine_runs(*, project: str, left: EngineRun, right: EngineRun) -> l def diagnostic_codes(*, outcome: CommandOutcome, severity: str | None) -> tuple[str, ...]: """Return a JSON compile report's diagnostic codes in order, optionally of one severity.""" + return tuple( + str(diagnostic.get("code")) + for diagnostic in _diagnostics(outcome=outcome, severity=severity) + ) + + +def first_diagnostic_message(*, outcome: CommandOutcome, severity: str) -> str | None: + """Return the message of a JSON compile report's first diagnostic of one severity.""" + + diagnostics: list[dict[str, object]] = _diagnostics(outcome=outcome, severity=severity) + return str(diagnostics[0].get("message")) if diagnostics else None + + +def _diagnostics(*, outcome: CommandOutcome, severity: str | None) -> list[dict[str, object]]: payload: dict[str, object] = as_json_object(_loads(outcome.stdout)) or {} diagnostics: object = payload.get("diagnostics") - found: list[str] = [] + found: list[dict[str, object]] = [] for raw_diagnostic in diagnostics if isinstance(diagnostics, list) else (): diagnostic: dict[str, object] | None = as_json_object(raw_diagnostic) if diagnostic is not None and (severity is None or diagnostic.get("severity") == severity): - found.append(str(diagnostic.get("code"))) - return tuple(found) + found.append(diagnostic) + return found def _command_differences( diff --git a/scripts/compiler_differential/_helpers/corpus/case_builder.py b/scripts/compiler_differential/_helpers/corpus/case_builder.py new file mode 100644 index 0000000000..7e3bc2886e --- /dev/null +++ b/scripts/compiler_differential/_helpers/corpus/case_builder.py @@ -0,0 +1,49 @@ +"""Build failure cases as small edits of the shared base project.""" + +from scripts.compiler_differential.constants import ( + FAILURE_BASE_CONFIG, + FAILURE_BASE_FILES, + FAILURE_CONFIG_PATH, + FAILURE_MART_PATH, + FAILURE_STAGING_PATH, +) +from scripts.compiler_differential.models import FailureCase + +_MART_HEADER: str = 'MODEL (\n description "Order totals per customer",\n);\n\n' + + +def failure_case( + *, + name: str, + expected_code: str, + files: dict[str, str], + expected_warning_code: str | None = None, + expected_message: str | None = None, +) -> FailureCase: + """Return the base project with `files` added or replaced.""" + + return FailureCase( + files={**FAILURE_BASE_FILES, **files}, + name=name, + expected_code=expected_code, + expected_warning_code=expected_warning_code, + expected_message=expected_message, + ) + + +def staging_files(sql: str) -> dict[str, str]: + """Replace the staging model.""" + + return {FAILURE_STAGING_PATH: sql} + + +def mart_body_files(body: str) -> dict[str, str]: + """Replace the mart model's query, keeping its header.""" + + return {FAILURE_MART_PATH: _MART_HEADER + body} + + +def config_files(extra: str) -> dict[str, str]: + """Append `extra` to the base project configuration.""" + + return {FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + extra} diff --git a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py index 858e2864ee..09701b1305 100644 --- a/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/discovery_failure_cases.py @@ -1,5 +1,9 @@ """Minimal projects that fail inside project discovery, one per diagnostic family.""" +from scripts.compiler_differential._helpers.corpus.case_builder import ( + config_files, + failure_case, +) from scripts.compiler_differential.constants import ( FAILURE_BASE_CONFIG, FAILURE_BASE_FILES, @@ -33,16 +37,6 @@ ) -def _case(*, name: str, expected_code: str, files: dict[str, str]) -> FailureCase: - return FailureCase( - files={**FAILURE_BASE_FILES, **files}, name=name, expected_code=expected_code - ) - - -def _config(extra: str) -> dict[str, str]: - return {FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + extra} - - def discovery_failure_cases() -> tuple[FailureCase, ...]: """Return the discovery-time failure cases in a stable order.""" @@ -58,47 +52,47 @@ def discovery_failure_cases() -> tuple[FailureCase, ...]: def _config_cases() -> tuple[FailureCase, ...]: return ( - _case( + failure_case( name="config-bom", expected_code="D001", files={FAILURE_CONFIG_PATH: GENERATOR_BOM + FAILURE_BASE_CONFIG}, ), - _case( + failure_case( name="local-config-unknown-key", expected_code="D001", files={"sqlbuild_local.toml": "[settings]\nsql_analysiss = false\n"}, ), - _case( + failure_case( name="target-unknown-key", expected_code="D001", - files=_config('\n[targets.ci]\nschema = "ci"\nwarehouse_size = "large"\n'), + files=config_files('\n[targets.ci]\nschema = "ci"\nwarehouse_size = "large"\n'), ), - _case( + failure_case( name="dbt-unknown-key", expected_code="D001", - files=_config('\n[dbt]\nproject_dir = "dbt"\nmanifest = "dbt/manifest.json"\n'), + files=config_files('\n[dbt]\nproject_dir = "dbt"\nmanifest = "dbt/manifest.json"\n'), ), - _case( + failure_case( name="path-defaults-unmatched", expected_code="D007", - files=_config('\n[path_defaults."reporting/finance"]\nmaterialized = "table"\n'), + files=config_files('\n[path_defaults."reporting/finance"]\nmaterialized = "table"\n'), ), ) def _sql_file_cases() -> tuple[FailureCase, ...]: return ( - _case( + failure_case( name="model-bom", expected_code="D002", files={FAILURE_STAGING_PATH: GENERATOR_BOM + FAILURE_BASE_STAGING}, ), - _case( + failure_case( name="model-missing-header", expected_code="D002", files={FAILURE_STAGING_PATH: 'SELECT order_id\nFROM __source("raw_orders")\n'}, ), - _case( + failure_case( name="model-unknown-header-key", expected_code="D002", files={ @@ -107,12 +101,12 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="model-reserved-name", expected_code="D016", files={"models/marts/_chain_.sql": FAILURE_BASE_MART}, ), - _case( + failure_case( name="test-case-missing-parameter", expected_code="D003", files={ @@ -127,7 +121,7 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="generic-audit-header", expected_code="D004", files={ @@ -136,7 +130,7 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="list-constant-mixed-types", expected_code="D013", files={ @@ -145,7 +139,7 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="enum-duplicate-member", expected_code="D013", files={ @@ -154,7 +148,7 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="sql-hook-unknown-key", expected_code="D014", files={ @@ -163,7 +157,7 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="sql-function-header", expected_code="D002", files={ @@ -173,7 +167,7 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="column-nullable-not-null", expected_code="P002", files={ @@ -189,7 +183,7 @@ def _sql_file_cases() -> tuple[FailureCase, ...]: def _layout_cases() -> tuple[FailureCase, ...]: return tuple( - _case(name=name, expected_code="D013", files={path: contents}) + failure_case(name=name, expected_code="D013", files={path: contents}) for name, path, contents in ( ("scoped-root-at-project-root", "_enums/order_status.sql", ""), ("declaration-group-below-root", "models/_sqlbuild/macros/money.py", _MACRO_MODULE), @@ -205,7 +199,7 @@ def _layout_cases() -> tuple[FailureCase, ...]: def _yaml_file_cases() -> tuple[FailureCase, ...]: return ( - _case( + failure_case( name="schema-yml-model-metadata", expected_code="D005", files={ @@ -214,7 +208,7 @@ def _yaml_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="seed-yaml-extension", expected_code="D005", files={ @@ -222,7 +216,7 @@ def _yaml_file_cases() -> tuple[FailureCase, ...]: "seeds/order_channels.csv": "id,label\n1,web\n", }, ), - _case( + failure_case( name="source-unknown-key", expected_code="D006", files={ @@ -232,12 +226,12 @@ def _yaml_file_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="seed-declaration-without-csv", expected_code="D008", files={"seeds/order_channels.yml": _SEED_DECLARATION}, ), - _case( + failure_case( name="seed-header-mismatch", expected_code="D008", files={ @@ -250,12 +244,12 @@ def _yaml_file_cases() -> tuple[FailureCase, ...]: def _conflict_cases() -> tuple[FailureCase, ...]: return ( - _case( + failure_case( name="duplicate-source", expected_code="D007", files={"sources/more.yml": FAILURE_BASE_FILES[FAILURE_SOURCES_PATH]}, ), - _case( + failure_case( name="duplicate-scenario", expected_code="D007", files={ @@ -269,7 +263,7 @@ def _conflict_cases() -> tuple[FailureCase, ...]: for folder in ("north", "south") }, ), - _case( + failure_case( name="resource-name-collision", expected_code="D007", files={ @@ -277,7 +271,7 @@ def _conflict_cases() -> tuple[FailureCase, ...]: "seeds/stg_orders.csv": "id,label\n1,web\n", }, ), - _case( + failure_case( name="managed-source-without-loader", expected_code="D007", files={ @@ -287,7 +281,7 @@ def _conflict_cases() -> tuple[FailureCase, ...]: " columns:\n - name: id\n type: INTEGER\n" }, ), - _case( + failure_case( name="duplicate-python-node", expected_code="D007", files={ @@ -295,7 +289,7 @@ def _conflict_cases() -> tuple[FailureCase, ...]: "python/tasks/second.py": _TASK_MODULE, }, ), - _case( + failure_case( name="check-depends-on-check", expected_code="D007", files={ @@ -313,12 +307,12 @@ def _conflict_cases() -> tuple[FailureCase, ...]: def _python_cases() -> tuple[FailureCase, ...]: return ( - _case( + failure_case( name="python-node-import-error", expected_code="D011", files={"python/tasks/exports.py": "import missing_orders_client\n" + _TASK_MODULE}, ), - _case( + failure_case( name="duplicate-hook", expected_code="D011", files={ @@ -326,7 +320,7 @@ def _python_cases() -> tuple[FailureCase, ...]: "hooks/python/second.py": _HOOK_MODULE, }, ), - _case( + failure_case( name="provider-invalid-settings", expected_code="D012", files={ @@ -338,7 +332,7 @@ def _python_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="duplicate-provider", expected_code="D012", files={ @@ -351,12 +345,12 @@ def _python_cases() -> tuple[FailureCase, ...]: for module in ("first", "second") }, ), - _case( + failure_case( name="legacy-event-exporters-directory", expected_code="D015", files={"event_exporters/publish.py": "VALUE = 1\n"}, ), - _case( + failure_case( name="lifecycle-sink-default-parameter", expected_code="D015", files={ @@ -368,7 +362,7 @@ def _python_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="lifecycle-sink-unknown-provider", expected_code="D015", files={ @@ -380,7 +374,7 @@ def _python_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="command-output-sink-unknown-provider", expected_code="D015", files={ @@ -392,7 +386,7 @@ def _python_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="materialization-import-error", expected_code="D011", files={ @@ -403,7 +397,7 @@ def _python_cases() -> tuple[FailureCase, ...]: ), }, ), - _case( + failure_case( name="python-outside-extension-root", expected_code="D016", files={"helpers/formatting.py": "VALUE = 1\n"}, diff --git a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py index fdfb77c540..810e3f1480 100644 --- a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py +++ b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py @@ -2,16 +2,23 @@ from __future__ import annotations +import ast from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from scripts.compiler_differential._helpers.comparing.comparison import diagnostic_codes +from scripts.compiler_differential._helpers.comparing.comparison import ( + diagnostic_codes, + first_diagnostic_message, +) from scripts.compiler_differential._helpers.corpus.corpus import build_corpus from scripts.compiler_differential._helpers.running.execution import run_engine from scripts.compiler_differential.constants import ( CORPUS_FAILURES, + DIAGNOSTIC_CODE_PATTERN, ERROR_SEVERITY, EXPECT_SUCCESS, + RENDER_CODE_SCAN_EXCLUDED, + RENDER_RAISED_CODES, WARNING_SEVERITY, ) from scripts.compiler_differential.models import ( @@ -20,10 +27,15 @@ EmittedCodes, EngineRun, ) +from sqlbuild.compiler.auditing.constants import BUILT_IN_AUDIT_SHADOW_CODE +from sqlbuild.compiler.compile import constants as compile_constants +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.exceptions import DiscoveryError from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError +from sqlbuild.compiler.scopes.types import ScopeDiagnosticCode _SOURCE_DIRECTORY: str = "source" +_CODE_KEYWORD: str = "code" def emitted_failure_codes(*, options: DifferentialOptions) -> dict[str, EmittedCodes]: @@ -57,6 +69,66 @@ def discovery_error_codes() -> frozenset[str]: return frozenset(codes) +def render_error_codes() -> frozenset[str]: + """Return every code render, scope, and attachment can report, from compiler constants.""" + + compile_codes: set[str] = { + value + for name, value in vars(compile_constants).items() + if name.endswith("_CODE") + and isinstance(value, str) + and DIAGNOSTIC_CODE_PATTERN.fullmatch(value) + } + return frozenset( + { + CompileInputError.code, + BUILT_IN_AUDIT_SHADOW_CODE, + *compile_codes, + *(code.value for code in ScopeDiagnosticCode), + *RENDER_RAISED_CODES, + } + ) + + +def inline_compile_codes(compile_root: Path) -> dict[str, tuple[str, ...]]: + """Map each `code="X###"` literal in render-stage compile modules to where it is written.""" + + found: dict[str, list[str]] = {} + for path in sorted(compile_root.rglob("*.py")): + relative: str = path.relative_to(compile_root).as_posix() + if any(relative.startswith(f"{excluded}/") for excluded in RENDER_CODE_SCAN_EXCLUDED): + continue + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + code: str | None = _code_literal(node) + if code is not None: + found.setdefault(code, []).append(f"{relative}:{getattr(node, 'lineno', 0)}") + return {code: tuple(locations) for code, locations in found.items()} + + +def unaccounted_inline_codes( + *, compile_root: Path, accounted: frozenset[str] +) -> dict[str, tuple[str, ...]]: + """Return the inline render-stage codes that are neither required nor documented.""" + + return { + code: locations + for code, locations in inline_compile_codes(compile_root).items() + if code not in accounted + } + + +def _code_literal(node: ast.AST) -> str | None: + if not ( + isinstance(node, ast.keyword) + and node.arg == _CODE_KEYWORD + and isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + and DIAGNOSTIC_CODE_PATTERN.fullmatch(node.value.value) + ): + return None + return node.value.value + + def _emitted_codes(*, project: CorpusProject, options: DifferentialOptions) -> EmittedCodes: case_dir: Path = options.work_dir / project.name.replace("/", "__") source_dir: Path = case_dir / _SOURCE_DIRECTORY @@ -72,4 +144,7 @@ def _emitted_codes(*, project: CorpusProject, options: DifferentialOptions) -> E return EmittedCodes( errors=diagnostic_codes(outcome=run.outcomes[0], severity=ERROR_SEVERITY), warnings=diagnostic_codes(outcome=run.outcomes[0], severity=WARNING_SEVERITY), + first_error_message=first_diagnostic_message( + outcome=run.outcomes[0], severity=ERROR_SEVERITY + ), ) diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index 2cf0947ae8..c15ed2a7fc 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -1,46 +1,29 @@ """Minimal failing projects, each derived from the shared base project.""" +from scripts.compiler_differential._helpers.corpus.case_builder import ( + failure_case, + mart_body_files, + staging_files, +) from scripts.compiler_differential._helpers.corpus.discovery_failure_cases import ( discovery_failure_cases, ) +from scripts.compiler_differential._helpers.corpus.render_failure_cases import ( + render_failure_cases, +) from scripts.compiler_differential.constants import ( FAILURE_BASE_CONFIG, - FAILURE_BASE_FILES, FAILURE_BASE_MART, FAILURE_BASE_STAGING, FAILURE_CONFIG_PATH, FAILURE_MART_PATH, FAILURE_SOURCES_PATH, - FAILURE_STAGING_PATH, ) from scripts.compiler_differential.models import FailureCase _RULES_CONFIG: str = '\n[rules]\nselect = ["{codes}"]\n' -def _case( - *, - name: str, - expected_code: str, - files: dict[str, str], - expected_warning_code: str | None = None, -) -> FailureCase: - return FailureCase( - files={**FAILURE_BASE_FILES, **files}, - name=name, - expected_code=expected_code, - expected_warning_code=expected_warning_code, - ) - - -def _staging(sql: str) -> dict[str, str]: - return {FAILURE_STAGING_PATH: sql} - - -def _mart_body(body: str) -> dict[str, str]: - return {FAILURE_MART_PATH: 'MODEL (\n description "Order totals per customer",\n);\n\n' + body} - - def _rules(*codes: str) -> dict[str, str]: return { FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + _RULES_CONFIG.format(codes='", "'.join(codes)) @@ -50,82 +33,84 @@ def _rules(*codes: str) -> dict[str, str]: def all_failure_cases() -> tuple[FailureCase, ...]: """Return every failure case in a stable order.""" - return (*_compile_failure_cases(), *discovery_failure_cases()) + return (*_compile_failure_cases(), *discovery_failure_cases(), *render_failure_cases()) def _compile_failure_cases() -> tuple[FailureCase, ...]: return ( - _case( + failure_case( name="config-toml-syntax", expected_code="D001", files={ FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + "[defaults\nmaterialized = 'table'\n" }, ), - _case( + failure_case( name="config-unknown-default-key", expected_code="D001", files={ FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + '\n[defaults]\ndescription = "Shared"\n' }, ), - _case( + failure_case( name="config-unknown-adapter", expected_code="C601", files={FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG.replace('"duckdb"', '"warehouse9000"')}, ), - _case( + failure_case( name="model-header-syntax", expected_code="D002", - files=_staging( + files=staging_files( FAILURE_BASE_STAGING.replace( '"Staged orders",', '"Staged orders",\n materialized (,' ) ), ), - _case( + failure_case( name="model-sql-syntax", expected_code="P001", - files=_staging(FAILURE_BASE_STAGING.replace("SELECT order_id,", "SELECT order_id,, ")), + files=staging_files( + FAILURE_BASE_STAGING.replace("SELECT order_id,", "SELECT order_id,, ") + ), ), - _case( + failure_case( name="test-parse", expected_code="D003", files={"tests/unit/test_stg_orders.sql": "TEST(\n\nSELECT 1\n"}, ), - _case( + failure_case( name="audit-parse", expected_code="D004", files={ "models/marts/_sqlbuild/audits/singular/broken.sql": "AUDIT (name);\n\nSELECT 1\n" }, ), - _case( + failure_case( name="schema-unterminated-declaration", expected_code="D013", files={"models/staging/_sqlbuild/_schemas/orders.sql": "SCHEMA (\n name orders,\n"}, ), - _case( + failure_case( name="source-parse", expected_code="D006", files={FAILURE_SOURCES_PATH: "sources:\n - name: raw_orders\n columns: [\n"}, ), - _case( + failure_case( name="duplicate-model", expected_code="D007", files={"models/marts/stg_orders.sql": FAILURE_BASE_STAGING}, ), - _case( + failure_case( name="seed-without-header", expected_code="D008", files={"seeds/lookups.csv": ""}, ), - _case( + failure_case( name="scenario-parse", expected_code="D009", files={"tests/scenarios/orders.sql": "SCENARIO (\n\nSELECT 1\n"}, ), - _case( + failure_case( name="python-node-missing-description", expected_code="P010", files={ @@ -135,22 +120,22 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="enum-parse", expected_code="D013", files={"enums/order_status.sql": "ENUM (\n name order_status,\n members [,\n);\n"}, ), - _case( + failure_case( name="hook-without-header", expected_code="D014", files={"hooks/sql/record_refresh.sql": "SELECT 1\n"}, ), - _case( + failure_case( name="invalid-resource-name", expected_code="D016", files={"models/marts/order-totals.sql": FAILURE_BASE_MART}, ), - _case( + failure_case( name="built-in-audit-shadow", expected_code="S010", expected_warning_code="P003", @@ -160,7 +145,7 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="singular-audit-one-resource", expected_code="P004", files={ @@ -170,7 +155,7 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="macro-generated-reference", expected_code="P006", files={ @@ -179,40 +164,42 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ' """Return the staged orders relation."""\n' " return '__ref(\"stg_orders\")'\n" ), - **_mart_body( + **mart_body_files( "SELECT customer_id, SUM(amount) AS total_amount\n" "FROM @staged_orders()\nGROUP BY customer_id\n" ), }, ), - _case( + failure_case( name="missing-description", expected_code="P010", - files=_staging(FAILURE_BASE_STAGING.replace(' description "Staged orders",\n', "")), + files=staging_files( + FAILURE_BASE_STAGING.replace(' description "Staged orders",\n', "") + ), ), - _case( + failure_case( name="unknown-ref", expected_code="P001", - files=_mart_body( + files=mart_body_files( 'SELECT customer_id, SUM(amount) AS total_amount\nFROM __ref("stg_payments")\n' "GROUP BY customer_id\n" ), ), - _case( + failure_case( name="unknown-source", expected_code="P001", - files=_staging( + files=staging_files( FAILURE_BASE_STAGING.replace('__source("raw_orders")', '__source("raw_payments")') ), ), - _case( + failure_case( name="unknown-macro", expected_code="P001", - files=_staging( + files=staging_files( FAILURE_BASE_STAGING.replace("amount, status", '@cents("amount") AS amount, status') ), ), - _case( + failure_case( name="failing-macro", expected_code="P001", files={ @@ -221,41 +208,41 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ' """Refuse every input."""\n' ' raise ValueError(f"cannot convert {expression}")\n' ), - **_staging( + **staging_files( FAILURE_BASE_STAGING.replace( "amount, status", '@cents("amount") AS amount, status' ) ), }, ), - _case( + failure_case( name="unknown-enum-member", expected_code="P001", files={ "models/staging/_sqlbuild/_enums/order_status.sql": ( "ENUM (\n name order_status,\n members [PLACED, SHIPPED],\n);\n" ), - **_staging( + **staging_files( FAILURE_BASE_STAGING + 'WHERE status = @enum("order_status").RETURNED\n' ), }, ), - _case( + failure_case( name="unknown-constant", expected_code="P001", - files=_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + files=staging_files(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), ), - _case( + failure_case( name="invalid-decimal-constant", expected_code="D013", files={ "models/staging/_sqlbuild/_constants/limits.sql": ( 'CONSTANT (name minimum_amount, type decimal, value "ten");\n' ), - **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + **staging_files(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), }, ), - _case( + failure_case( name="duplicate-declaration", expected_code="P001", files={ @@ -263,28 +250,28 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: "CONSTANT (name minimum_amount, value 1);\n" "CONSTANT (name minimum_amount, value 2);\n" ), - **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + **staging_files(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), }, ), - _case( + failure_case( name="inaccessible-declaration", expected_code="P001", files={ "models/marts/_sqlbuild/_constants/limits.sql": ( "CONSTANT (name minimum_amount, value 1);\n" ), - **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + **staging_files(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), }, ), - _case( + failure_case( name="over-broad-global", expected_code="S024", files={ "constants/limits.sql": "CONSTANT (name minimum_amount, value 1);\n", - **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + **staging_files(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), }, ), - _case( + failure_case( name="unused-declaration", expected_code="S010", files={ @@ -293,7 +280,7 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="macro-import-cycle", expected_code="P001", files={ @@ -311,10 +298,10 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ), }, ), - _case( + failure_case( name="contract-missing-column", expected_code="K001", - files=_staging( + files=staging_files( 'MODEL (\n description "Staged orders",\n contract enforced,\n' " columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n" " amount (type DOUBLE),\n status (type VARCHAR),\n" @@ -323,51 +310,51 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' ), ), - _case( + failure_case( name="contract-type-mismatch", expected_code="K002", - files=_staging( + files=staging_files( 'MODEL (\n description "Staged orders",\n contract enforced,\n' " columns (\n order_id (type VARCHAR),\n customer_id (type INTEGER),\n" " amount (type DOUBLE),\n status (type VARCHAR),\n ),\n);\n\n" 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' ), ), - _case( + failure_case( name="set-operation-width", expected_code="B216", - files=_mart_body( + files=mart_body_files( 'SELECT customer_id, amount FROM __ref("stg_orders")\n' 'UNION ALL\nSELECT customer_id FROM __ref("stg_orders")\n' ), ), - _case( + failure_case( name="comparison-type", expected_code="B217", - files=_mart_body( + files=mart_body_files( "SELECT customer_id, SUM(amount) AS total_amount\n" "FROM __ref(\"stg_orders\")\nWHERE amount = DATE '2026-01-01'\n" "GROUP BY customer_id\n" ), ), - _case( + failure_case( name="unknown-column", expected_code="B002", - files=_mart_body( + files=mart_body_files( 'SELECT customer_id, SUM(discount) AS total_amount\nFROM __ref("stg_orders")\n' "GROUP BY customer_id\n" ), ), - _case( + failure_case( name="ungrouped-column", expected_code="B230", - files=_mart_body( + files=mart_body_files( "SELECT customer_id, status, SUM(amount) AS total_amount\n" 'FROM __ref("stg_orders")\n' "GROUP BY customer_id\n" ), ), - _case( + failure_case( name="test-expected-unknown-column", expected_code="B302", files={ @@ -380,19 +367,19 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="rule-unused-cte", expected_code="SQBRSQL005", files={ **_rules("SQBRSQL005"), - **_mart_body( + **mart_body_files( "WITH spare AS (SELECT 1 AS one)\n" "SELECT customer_id, SUM(amount) AS total_amount\n" 'FROM __ref("stg_orders")\nGROUP BY customer_id\n' ), }, ), - _case( + failure_case( name="rule-custom-finding", expected_code="XSQBRDIFF001", files={ @@ -410,11 +397,13 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ), }, ), - _case( + failure_case( name="multiple-errors-order", expected_code="P010", files={ - **_staging(FAILURE_BASE_STAGING.replace(' description "Staged orders",\n', "")), + **staging_files( + FAILURE_BASE_STAGING.replace(' description "Staged orders",\n', "") + ), FAILURE_MART_PATH: FAILURE_BASE_MART.replace( ' description "Order totals per customer",\n', "" ), @@ -424,29 +413,29 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ), }, ), - _case( + failure_case( name="contract-extra-column", expected_code="K005", - files=_staging( + files=staging_files( 'MODEL (\n description "Staged orders",\n contract enforced,\n' " columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n" " amount (type DOUBLE),\n ),\n);\n\n" 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' ), ), - _case( + failure_case( name="contract-unknown-type", expected_code="K002", - files=_staging( + files=staging_files( 'MODEL (\n description "Staged orders",\n' " columns (\n order_id (type MYSTERY_TYPE),\n ),\n);\n\n" 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' ), ), - _case( + failure_case( name="cursor-unknown-column", expected_code="B300", - files=_mart_body( + files=mart_body_files( "SELECT customer_id, SUM(amount) AS total_amount\n" 'FROM __ref("stg_orders")\nGROUP BY customer_id\n' ) @@ -461,7 +450,7 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="cursor-model-without-inputs", expected_code="P011", files={ @@ -473,22 +462,22 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) }, ), - _case( + failure_case( name="invalid-declaration-name", expected_code="D016", files={ "models/staging/_sqlbuild/_constants/limits.sql": ( "CONSTANT (name MinimumAmount, value 1);\n" ), - **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("MinimumAmount")\n'), + **staging_files(FAILURE_BASE_STAGING + 'WHERE amount > @const("MinimumAmount")\n'), }, ), - _case( + failure_case( name="python-node-syntax", expected_code="D011", files={"python/tasks/exports.py": "def broken(:\n pass\n"}, ), - _case( + failure_case( name="hook-unknown-argument", expected_code="P001", files={ @@ -501,20 +490,20 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ), }, ), - _case( + failure_case( name="model-private-enum-misuse", expected_code="P001", - files=_staging( + files=staging_files( 'MODEL (\n description "Staged orders",\n' " enums (\n _state [OPEN, CLOSED],\n ),\n);\n\n" "SELECT order_id, customer_id, amount, status\n" 'FROM __source("raw_orders")\nWHERE status = @enum("_state").PENDING\n' ), ), - _case( + failure_case( name="variable-not-defined", expected_code="P001", - files=_staging( + files=staging_files( FAILURE_BASE_STAGING.replace("status\n", "status, '@@region' AS region\n", 1) ), ), diff --git a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py new file mode 100644 index 0000000000..55db1432e5 --- /dev/null +++ b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py @@ -0,0 +1,354 @@ +"""Minimal projects that fail while rendering, scoping, or attaching compile inputs.""" + +from scripts.compiler_differential._helpers.corpus.case_builder import ( + config_files, + failure_case, + mart_body_files, + staging_files, +) +from scripts.compiler_differential.constants import ( + FAILURE_BASE_MART, + FAILURE_BASE_STAGING, + FAILURE_MART_PATH, +) +from scripts.compiler_differential.models import FailureCase + +_STAGING_HEADER: str = '"Staged orders",' +_MART_HEADER: str = '"Order totals per customer",' +_STAGING_COLUMNS: str = "amount, status" +_STAGING_MACROS: str = "models/staging/_sqlbuild/_macros/orders.py" +_MART_MACROS: str = "models/marts/_sqlbuild/_macros/orders.py" +_CENTS_MACRO: str = ( + 'def cents(expression: str) -> str:\n """Convert to cents."""\n' + ' return f"({expression} * 100)"\n' +) +_ORDERS_FOR: str = ( + "FUNCTION (\n" + ' description "Orders for one customer",\n' + " arguments (p_customer INTEGER),\n" + " returns table (\n order_id INTEGER\n ),\n);\n\n" + 'SELECT order_id\nFROM __ref("stg_orders")\nWHERE customer_id = p_customer\n' +) +_FLOOR_AUDIT: str = 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE amount < @minimum\n' +_RECORD_HOOK: str = "HOOK (\n description \"Record a refresh\"\n);\n\nSELECT @'label' AS label\n" +_TOTALS_READER: str = ( + 'AUDIT ();\n\nSELECT *\nFROM __ref("@model") o\n' + 'WHERE o.customer_id NOT IN (SELECT customer_id FROM __ref("customer_totals"))\n' +) +_HOOK_READING_TOTALS: str = ( + "from sqlbuild.hooks import hook\nfrom sqlbuild.refs import model\n\n\n" + '@hook(reads=[model("customer_totals")])\n' + 'def note_refresh(ctx):\n """Note a refresh."""\n return None\n' +) +_UPSTREAM_DBT_PROJECT: str = "name: upstream\nversion: '1.0.0'\nprofile: upstream\n" + + +def _staging_header(extra: str) -> dict[str, str]: + return staging_files( + FAILURE_BASE_STAGING.replace(_STAGING_HEADER, f"{_STAGING_HEADER}\n{extra}") + ) + + +def _staging_columns(*, columns: str, files: dict[str, str] | None = None) -> dict[str, str]: + return { + **(files or {}), + **staging_files(FAILURE_BASE_STAGING.replace(_STAGING_COLUMNS, columns, 1)), + } + + +def render_failure_cases() -> tuple[FailureCase, ...]: + """Return the render-time failure cases in a stable order.""" + + return (*_macro_cases(), *_interpolation_cases(), *_reference_cases(), *_attachment_cases()) + + +def _macro_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="macro-returns-non-string", + expected_code="P001", + expected_message="must return a SQL string", + files=_staging_columns( + columns="amount, status, @answer() AS answer", + files={ + _STAGING_MACROS: ( + 'def answer() -> int:\n """Return a number."""\n return 42\n' + ) + }, + ), + ), + failure_case( + name="macro-output-calls-macro", + expected_code="P001", + expected_message="returned SQLBuild call(s) @cents()", + files=_staging_columns( + columns='amount, status, @wrapped("amount") AS wrapped', + files={ + _STAGING_MACROS: _CENTS_MACRO + + '\n\ndef wrapped(expression: str) -> str:\n """Wrap a call."""\n' + + " return f'@cents(\"{expression}\")'\n" + }, + ), + ), + failure_case( + name="macro-non-literal-argument", + expected_code="P001", + expected_message="must use only Python literals", + files=_staging_columns( + columns="amount, status, @cents(amount) AS cents", + files={_STAGING_MACROS: _CENTS_MACRO}, + ), + ), + failure_case( + name="macro-in-sibling-scope", + expected_code="P001", + expected_message="is inaccessible", + files=_staging_columns( + columns='amount, status, @cents("amount") AS cents', + files={_MART_MACROS: _CENTS_MACRO}, + ), + ), + failure_case( + name="local-macro-needed-below-owner", + expected_code="S007", + files={ + _STAGING_MACROS: _CENTS_MACRO, + "models/staging/_sqlbuild/macros/wrapping.py": ( + "from models.staging._sqlbuild._macros.orders import cents\n\n\n" + 'def wrapped(expression: str) -> str:\n """Wrap cents."""\n' + " return cents(expression)\n" + ), + "models/staging/regional/stg_regional_orders.sql": ( + 'MODEL (\n description "Regional orders",\n);\n\n' + 'SELECT order_id, @wrapped("amount") AS amount_cents\n' + 'FROM __ref("stg_orders")\n' + ), + }, + ), + ) + + +def _interpolation_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="unset-environment-variable", + expected_code="P001", + expected_message="unknown environment variable '@@ENV:SQB_DIFFERENTIAL_UNSET'", + files=_staging_columns( + columns="amount, status, '@@ENV:SQB_DIFFERENTIAL_UNSET' AS channel" + ), + ), + failure_case( + name="context-variable-in-model-sql", + expected_code="P001", + expected_message="does not allow @@CTX templates", + files=_staging_columns( + columns="amount, status, '@@CTX:destination.qualified' AS destination" + ), + ), + failure_case( + name="runtime-placeholder-without-default", + expected_code="P001", + expected_message="@@@placeholders without default values", + files={ + "materializations/copy_table.py": ( + "from sqlbuild.executor.custom.models import MaterializationContext, " + "MaterializationResult\n\n\n" + "def materialize(ctx: MaterializationContext) -> MaterializationResult:\n" + " return MaterializationResult(relation=ctx.destination, audit_results=())\n" + ), + **staging_files( + FAILURE_BASE_STAGING.replace( + _STAGING_HEADER, f"{_STAGING_HEADER}\n materialized copy_table," + ).replace( + _STAGING_COLUMNS, + "amount, status, CAST(@@@window_start AS DATE) AS window_start", + ) + ), + }, + ), + failure_case( + name="cursor-intrinsic-outside-incremental", + expected_code="P001", + expected_message="uses cursor intrinsics but is not a built-in incremental model", + files=_staging_columns(columns="amount, status, __cursor_start() AS window_start"), + ), + failure_case( + name="named-hook-missing-argument", + expected_code="P001", + expected_message="is missing argument 'label'", + files={ + "hooks/sql/record_refresh.sql": _RECORD_HOOK, + FAILURE_MART_PATH: FAILURE_BASE_MART.replace( + _MART_HEADER, f'{_MART_HEADER}\n post_hooks [sql("record_refresh")],' + ), + }, + ), + failure_case( + name="generic-audit-missing-argument", + expected_code="P001", + expected_message="is missing argument 'minimum' for generic audit 'floor'", + files={ + "models/staging/_sqlbuild/_audits/generic/floor.sql": _FLOOR_AUDIT, + **_staging_header(" audits [floor],"), + }, + ), + ) + + +def _reference_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="table-function-arity", + expected_code="P001", + expected_message="expects 1 argument but received 2", + files={ + "functions/sql/table_fn__orders_for.sql": _ORDERS_FOR, + **mart_body_files( + "SELECT order_id AS customer_id, 1 AS total_amount\n" + 'FROM __table_fn("table_fn__orders_for")(1, 2)\n' + ), + }, + ), + failure_case( + name="unknown-table-function", + expected_code="P001", + expected_message="references unknown table function 'table_fn__missing'", + files=mart_body_files( + "SELECT order_id AS customer_id, 1 AS total_amount\n" + 'FROM __table_fn("table_fn__missing")(1)\n' + ), + ), + failure_case( + name="unknown-scalar-function", + expected_code="P001", + expected_message="references unknown SQL function 'is_large'", + files=_staging_columns(columns='amount, status, __udf("is_large")(amount) AS is_large'), + ), + failure_case( + name="test-expects-unknown-model", + expected_code="P001", + expected_message="expects unknown model 'stg_payments'", + files={ + "tests/unit/test_stg_payments.sql": ( + "TEST();\n\nWITH\n__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount," + " 'placed' AS status\n),\n" + "__expected__stg_payments AS (\n SELECT 1 AS order_id\n)\nSELECT 1\n" + ) + }, + ), + failure_case( + name="dbt-reference-without-manifest", + expected_code="C214", + files={ + **config_files('\n[dbt]\nproject_dir = "dbt"\n'), + "dbt/dbt_project.yml": _UPSTREAM_DBT_PROJECT, + **mart_body_files( + "SELECT customer_id, COUNT(*) AS order_count\n" + 'FROM __dbt_ref("upstream", "upstream_orders")\nGROUP BY customer_id\n' + ), + }, + ), + failure_case( + name="hard-coded-relation-in-python", + expected_code="P008", + files={ + "python/tasks/exports.py": ( + "from sqlbuild.tasks import task\n\n\n" + '@task\ndef export_orders(ctx):\n """Export orders."""\n' + ' return ctx.query("SELECT * FROM stg_orders")\n' + ) + }, + ), + ) + + +def _attachment_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="attached-audit-reads-dependant", + expected_code="P005", + files={ + "models/staging/_sqlbuild/_audits/generic/matches_totals.sql": _TOTALS_READER, + **_staging_header(" audits [matches_totals],"), + }, + ), + failure_case( + name="hook-reads-dependant", + expected_code="P007", + files={ + "models/staging/_sqlbuild/_hooks/python/notes.py": _HOOK_READING_TOTALS, + **_staging_header(' post_hooks [python("note_refresh")],'), + }, + ), + failure_case( + name="unneeded-sql-analysis-opt-out", + expected_code="P009", + files={ + **config_files("\n[settings]\nrequire_sql_analysis = true\n"), + **_staging_header(" sql_analysis false,"), + }, + ), + failure_case( + name="snapshot-schema-change-with-contract", + expected_code="K012", + files=staging_files( + 'MODEL (\n description "Staged orders",\n materialized snapshot,\n' + " unique_key [order_id],\n snapshot_strategy check,\n" + " check_columns [status],\n snapshot_schema_change append_new_columns,\n" + " contract enforced,\n columns (\n order_id (type INTEGER),\n" + " customer_id (type INTEGER),\n amount (type DOUBLE),\n" + " status (type VARCHAR),\n ),\n);\n\n" + 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' + ), + ), + failure_case( + name="schema-from-sibling-folder", + expected_code="S006", + files={ + "models/marts/_sqlbuild/_schemas/order_shape.sql": ( + 'SCHEMA (\n name order_shape,\n description "Order shape",\n' + " columns (\n order_id (type INTEGER),\n ),\n);\n" + ), + **_staging_header(" model_schema order_shape,"), + }, + ), + failure_case( + name="inherited-declaration-used-in-one-folder", + expected_code="S008", + files={ + "models/staging/_sqlbuild/constants/limits.sql": ( + "CONSTANT (name minimum_amount, value 1);\n" + ), + **staging_files(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + }, + ), + failure_case( + name="singular-audit-across-trees", + expected_code="S009", + files={ + "models/marts/_sqlbuild/audits/singular/totals_have_orders.sql": ( + 'AUDIT (name "totals_have_orders");\n\n' + 'SELECT t.customer_id\nFROM __ref("customer_totals") t\n' + 'LEFT JOIN __source("raw_orders") o ON t.customer_id = o.customer_id\n' + "WHERE o.customer_id IS NULL\n" + ) + }, + ), + failure_case( + name="orphan-audit-factory", + expected_code="S024", + expected_warning_code="C216", + files={ + "python/audits/quality.py": ( + "from sqlbuild.audits import AuditCase, audit_factory\n\n\n" + "@audit_factory\ndef order_quality():\n" + ' return [AuditCase(name="id_present", definition="not_null", ' + 'arguments={"column": "order_id"})]\n' + ), + "constants/limits.sql": "CONSTANT (name minimum_amount, value 1);\n", + **staging_files(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + }, + ), + ) diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 43d558123c..e9429d3521 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -457,3 +457,39 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: for kind in RENDER_MACRO_SOURCE_READS } RENDER_CONFIG_ONLY_KINDS: dict[str, str] = {} +DIAGNOSTIC_CODE_PATTERN: re.Pattern[str] = re.compile(r"[A-Z][0-9]{3}") +RENDER_RAISED_CODES: frozenset[str] = frozenset({"C214", "C216", "K012"}) +RENDER_CODE_SCAN_EXCLUDED: dict[str, str] = { + "_helpers/assembly": "semantic and metadata validation of the assembled CompiledProject (M4)", + "_helpers/analysis": "SQL analysis of rendered SQL, after the CompileProjectInputs frontier", +} +_SCOPE_QUERY_ONLY: str = "reported only by sqb scope queries, never by compile" +_TOLERANT_SCOPE_ONLY: str = ( + "reported only by the tolerant index sqb scope builds; compile discovery fails first " + "with a D-code" +) +_NEVER_EMITTED: str = "defined in ScopeDiagnosticCode but no compiler path reports it" +_FOLDED_INTO_P001: str = ( + "compile raises the strict scope index error, which reports it inside a P001 " + "'Invalid declaration scope index' message" +) +RENDER_UNREACHABLE_CODES: dict[str, str] = { + "S001": _SCOPE_QUERY_ONLY, + "S002": _SCOPE_QUERY_ONLY, + "S003": _FOLDED_INTO_P001, + "S004": "discovery rejects the identity first with D016", + "S005": "discovery rejects the identity first with D016", + "S011": _SCOPE_QUERY_ONLY, + "S012": _NEVER_EMITTED, + "S013": _SCOPE_QUERY_ONLY, + "S014": _SCOPE_QUERY_ONLY, + "S015": _NEVER_EMITTED, + "S016": _NEVER_EMITTED, + "S017": _NEVER_EMITTED, + "S018": "discovery rejects the duplicate resource first with D007", + "S019": _TOLERANT_SCOPE_ONLY, + "S020": _TOLERANT_SCOPE_ONLY, + "S021": _TOLERANT_SCOPE_ONLY, + "S022": _TOLERANT_SCOPE_ONLY, + "S023": _TOLERANT_SCOPE_ONLY, +} diff --git a/scripts/compiler_differential/models.py b/scripts/compiler_differential/models.py index f9c89633c9..9237e10793 100644 --- a/scripts/compiler_differential/models.py +++ b/scripts/compiler_differential/models.py @@ -138,6 +138,7 @@ class FailureCase(WritableProject): name: str expected_code: str expected_warning_code: str | None = None + expected_message: str | None = None @dataclass(frozen=True) @@ -174,6 +175,7 @@ class EmittedCodes: errors: tuple[str, ...] warnings: tuple[str, ...] + first_error_message: str | None = None @property def first_error(self) -> str | None: diff --git a/tests/e2e/scripts/compiler_differential/_test_types.py b/tests/e2e/scripts/compiler_differential/_test_types.py index e26a034b4b..8a4600042d 100644 --- a/tests/e2e/scripts/compiler_differential/_test_types.py +++ b/tests/e2e/scripts/compiler_differential/_test_types.py @@ -43,8 +43,11 @@ class FailureCorpusCodesTestCase: description: str expected_first_errors: dict[str, str] + expected_first_messages: dict[str, str] expected_warnings: dict[str, str] expected_discovery_codes: frozenset[str] + expected_render_codes: frozenset[str] + unreachable_render_codes: frozenset[str] @dataclass(frozen=True) diff --git a/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py b/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py index f4c96b1253..b0694a6d7b 100644 --- a/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py +++ b/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py @@ -1,8 +1,9 @@ -"""Every failure case emits its declared code as the first error, reaching every discovery code.""" +"""Every failure case emits its declared code first, reaching every discovery and render code.""" from __future__ import annotations import sys +from itertools import chain from pathlib import Path import pytest @@ -10,7 +11,9 @@ from scripts.compiler_differential._helpers.corpus.emitted_codes import ( discovery_error_codes, emitted_failure_codes, + render_error_codes, ) +from scripts.compiler_differential.constants import RENDER_UNREACHABLE_CODES from scripts.compiler_differential.main.failure_cases import failure_cases from scripts.compiler_differential.models import DifferentialOptions, EmittedCodes from tests.e2e.scripts.compiler_differential._test_types import FailureCorpusCodesTestCase @@ -24,13 +27,24 @@ expected_first_errors={ f"failure/{case.name}": case.expected_code for case in failure_cases() }, - expected_warnings={"failure/built-in-audit-shadow": "P003"}, + expected_first_messages={ + f"failure/{case.name}": case.expected_message + for case in failure_cases() + if case.expected_message is not None + }, + expected_warnings={ + f"failure/{case.name}": case.expected_warning_code + for case in failure_cases() + if case.expected_warning_code is not None + }, expected_discovery_codes=discovery_error_codes(), + expected_render_codes=render_error_codes() - frozenset(RENDER_UNREACHABLE_CODES), + unreachable_render_codes=frozenset(RENDER_UNREACHABLE_CODES), ) ], ids=lambda case: case.description, ) -def test_given_failure_corpus_when_compiling_then_emitted_first_errors_match_and_cover_discovery( +def test_given_failure_corpus_when_compiling_then_first_errors_match_and_cover_every_stage( test_case: FailureCorpusCodesTestCase, tmp_path: Path ) -> None: emitted: dict[str, EmittedCodes] = emitted_failure_codes( @@ -51,7 +65,21 @@ def test_given_failure_corpus_when_compiling_then_emitted_first_errors_match_and assert all( code in emitted[name].warnings for name, code in test_case.expected_warnings.items() ), emitted + first_messages: dict[str, str] = { + name: emitted[name].first_error_message or "" for name in test_case.expected_first_messages + } + assert all( + fragment in first_messages[name] + for name, fragment in test_case.expected_first_messages.items() + ), first_messages + reached: set[str | None] = { + *first_errors.values(), + *chain.from_iterable(codes.warnings for codes in emitted.values()), + } assert test_case.expected_discovery_codes <= set(first_errors.values()) + assert test_case.expected_render_codes <= reached, test_case.expected_render_codes - reached + assert not test_case.unreachable_render_codes & set(first_errors.values()) + assert test_case.unreachable_render_codes <= render_error_codes() if __name__ == "__main__": diff --git a/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py index eab6d1cbbc..48f6053e2b 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py +++ b/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import dataclass +from pathlib import Path from scripts.compiler_differential.models import ExpectedOutcome @@ -14,3 +15,13 @@ class CorpusExpectationTestCase: description: str corpora: tuple[str, ...] expected_outcomes: dict[str, ExpectedOutcome] + + +@dataclass(frozen=True) +class InlineCompileCodesTestCase: + """The compile package and the codes every inline render-stage literal must belong to.""" + + description: str + compile_root: Path + accounted_codes: frozenset[str] + expected_unaccounted: dict[str, tuple[str, ...]] diff --git a/tests/unit/scripts/compiler_differential/_helpers/corpus/test_render_codes.py b/tests/unit/scripts/compiler_differential/_helpers/corpus/test_render_codes.py new file mode 100644 index 0000000000..ba18491aff --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/corpus/test_render_codes.py @@ -0,0 +1,45 @@ +"""Every inline diagnostic code in render-stage compile modules is required or documented.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import sqlbuild.compiler.compile as compile_package +from scripts.compiler_differential._helpers.corpus.emitted_codes import ( + inline_compile_codes, + render_error_codes, + unaccounted_inline_codes, +) +from scripts.compiler_differential.constants import RENDER_UNREACHABLE_CODES +from tests.unit.scripts.compiler_differential._helpers.corpus._test_types import ( + InlineCompileCodesTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + InlineCompileCodesTestCase( + description="compile_package", + compile_root=Path(compile_package.__file__).parent, + accounted_codes=render_error_codes() | frozenset(RENDER_UNREACHABLE_CODES), + expected_unaccounted={}, + ) + ], + ids=lambda case: case.description, +) +def test_given_compile_modules_when_scanning_inline_codes_then_each_is_required_or_unreachable( + test_case: InlineCompileCodesTestCase, +) -> None: + unaccounted: dict[str, tuple[str, ...]] = unaccounted_inline_codes( + compile_root=test_case.compile_root, accounted=test_case.accounted_codes + ) + + assert inline_compile_codes(test_case.compile_root) + assert unaccounted == test_case.expected_unaccounted + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From a8dff6de479ae16af39347bb148474c3dcaf9938 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 7 Oct 2026 08:58:06 +0100 Subject: [PATCH 4/4] test(compiler): prove compile-input stage captures are complete Plan a generated project with every feature block and require its CompileProjectInputs capture to hold every field canonically, name every callable, and prove every required render kind. Perturb the native compile inputs to show the differential localises a render mismatch to the compile_project_inputs capture. --- .../compiler_differential/_test_types.py | 11 +++ .../scripts/compiler_differential/helpers.py | 21 ++++++ .../test_compiler_differential.py | 37 ++++++++++ .../test_render_capture.py | 68 +++++++++++++++++++ 4 files changed, 137 insertions(+) create mode 100644 tests/e2e/scripts/compiler_differential/test_render_capture.py diff --git a/tests/e2e/scripts/compiler_differential/_test_types.py b/tests/e2e/scripts/compiler_differential/_test_types.py index 8a4600042d..2056886b3d 100644 --- a/tests/e2e/scripts/compiler_differential/_test_types.py +++ b/tests/e2e/scripts/compiler_differential/_test_types.py @@ -59,3 +59,14 @@ class CoverageFailureTestCase: expected_exit_code: int expected_lines: tuple[str, ...] expected_absent: tuple[str, ...] + + +@dataclass(frozen=True) +class RenderCaptureTestCase: + """A generated project with every feature block and the render kinds its capture proves.""" + + description: str + seed: int + blocks: tuple[str, ...] + command: tuple[str, ...] + expected_kinds: frozenset[str] diff --git a/tests/e2e/scripts/compiler_differential/helpers.py b/tests/e2e/scripts/compiler_differential/helpers.py index e28a9accab..ea3af574c8 100644 --- a/tests/e2e/scripts/compiler_differential/helpers.py +++ b/tests/e2e/scripts/compiler_differential/helpers.py @@ -49,6 +49,27 @@ def _perturbed(*, until, **stages): return dataclasses.replace(result, model_files=tuple(reversed(result.model_files))) +frontier.native_frontier = _perturbed +""" + +RENDER_PERTURBATION: str = """ +import dataclasses + +import sqlbuild.compiler.frontier.main._compile_frontier as frontier +from sqlbuild.compiler.frontier.types import CompilerStage + +_original = frontier.native_frontier + + +def _perturbed(*, until, **stages): + result = _original(until=until, **stages) + if until is not CompilerStage.COMPILE_PROJECT_INPUTS or not result.model_inputs: + return result + first = result.model_inputs[0] + changed = dataclasses.replace(first, macro_deps=(*first.macro_deps, "perturbed_by_test")) + return dataclasses.replace(result, model_inputs=(changed, *result.model_inputs[1:])) + + frontier.native_frontier = _perturbed """ diff --git a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py index 0d80bd3002..ebf2ba845c 100644 --- a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py +++ b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py @@ -16,6 +16,7 @@ from tests.e2e.scripts.compiler_differential.helpers import ( DISCOVERY_PERTURBATION, NATIVE_ONLY_STDERR_LINE, + RENDER_PERTURBATION, harness_arguments, perturbation_arguments, write_broken_ref_project, @@ -129,6 +130,42 @@ def test_given_perturbed_native_discovery_when_comparing_then_discovery_capture_ assert all(re.search(pattern, output) for pattern in test_case.expected_patterns), output +@pytest.mark.parametrize( + "test_case", + [ + HarnessRunTestCase( + description="perturbed_native_compile_inputs", + extra_arguments=(), + expected_exit_code=1, + expected_lines=("DIFF project/waffle_shop",), + expected_patterns=( + r"- stage capture 0-compile/002-compile_project_inputs\.json at " + r"/model_inputs/0/macro_deps/", + r"- stage capture 2-plan/002-compile_project_inputs\.json at " + r"/model_inputs/0/macro_deps/", + ), + expected_absent=("001-discovered_project_inputs.json",), + ) + ], + ids=lambda case: case.description, +) +def test_given_perturbed_native_render_when_comparing_then_compile_inputs_capture_names_the_stage( + test_case: HarnessRunTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + exit_code: int = run_compiler_differential( + harness_arguments( + work_dir=tmp_path / "work", + extra=perturbation_arguments(tmp_path / "perturbation", source=RENDER_PERTURBATION), + ) + ) + + output: str = capsys.readouterr().out + assert exit_code == test_case.expected_exit_code, output + assert all(line in output for line in test_case.expected_lines), output + assert all(re.search(pattern, output) for pattern in test_case.expected_patterns), output + assert not any(text in output for text in test_case.expected_absent), output + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/e2e/scripts/compiler_differential/test_render_capture.py b/tests/e2e/scripts/compiler_differential/test_render_capture.py new file mode 100644 index 0000000000..ff23f217c2 --- /dev/null +++ b/tests/e2e/scripts/compiler_differential/test_render_capture.py @@ -0,0 +1,68 @@ +"""A real plan captures every compile-input field canonically and proves every render kind.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +from scripts.compiler_differential._helpers.coverage.render import ( + render_capture_problems, + rendered_input_kinds, + required_render_kinds, +) +from scripts.compiler_differential._helpers.running.execution import harness_environment +from scripts.compiler_differential.constants import ( + GENERATOR_FEATURE_BLOCKS, + RENDER_STAGE_CAPTURE_SUFFIX, + SQB_ENTRY, + STAGE_CAPTURE_ENV_VAR, +) +from scripts.compiler_differential.main.generate_project import generate_project +from tests.e2e.scripts.compiler_differential._test_types import RenderCaptureTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + RenderCaptureTestCase( + description="every_feature_block_planned", + seed=11, + blocks=GENERATOR_FEATURE_BLOCKS, + command=("plan", "--json"), + expected_kinds=frozenset(required_render_kinds()), + ) + ], + ids=lambda case: case.description, +) +def test_given_project_with_every_render_input_when_capturing_then_capture_is_complete( + test_case: RenderCaptureTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "project" + capture_dir: Path = tmp_path / "captures" + generate_project(seed=test_case.seed, blocks=test_case.blocks).write(project_dir) + + completed: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, "-c", SQB_ENTRY, "--no-color", *test_case.command], + cwd=project_dir, + env={**harness_environment(), STAGE_CAPTURE_ENV_VAR: str(capture_dir)}, + capture_output=True, + text=True, + check=False, + timeout=600, + ) + + assert completed.returncode == 0, completed.stdout + completed.stderr + text: str = next(capture_dir.glob(f"*{RENDER_STAGE_CAPTURE_SUFFIX}")).read_text( + encoding="utf-8" + ) + assert render_capture_problems(text) == () + assert test_case.expected_kinds <= rendered_input_kinds(text), ( + test_case.expected_kinds - rendered_input_kinds(text) + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"])