From d3be3d1a35f2d625e6977b16a39c6650ca3848de Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:58:00 +0100 Subject: [PATCH 1/7] fix(compiler)!: reject unknown on_schema_change, replay_on_change and dlt_sources keys Unknown on_schema_change and replay_on_change values fell back to the default policy, and dlt_sources groups and resources accepted any key. They now fail compilation: MODEL headers and [defaults]/[path_defaults] name the value, list the valid ones and show a valid header or TOML line; dlt keys get the nearest supported key as help. bounded- must carry a positive duration. The planner no longer maps unknown values to a default. BREAKING CHANGE: unknown on_schema_change or replay_on_change values and unknown dlt_sources keys now fail compilation. Refs: CHI-465 --- .../docs/concepts/models/configuration.md | 2 +- .../references/docs/integrations/dlt.md | 4 + .../authored_values/_helpers/change_policy.py | 64 +++++++++++++ .../main/_change_policy_header_help.py | 13 +++ .../main/_change_policy_problem.py | 11 +++ .../main/_change_policy_toml_help.py | 11 +++ .../_helpers/config/model_validation.py | 23 +++-- .../_helpers/integrations/loaders.py | 25 +++++ .../discovery/_helpers/yml/project.py | 26 ++++- .../planner/_helpers/changes/policy.py | 3 +- .../planner/_helpers/output/plan_entry.py | 11 +-- .../main/compile/test_unknown_config_keys.py | 54 ++++++++++- .../compiler/compile/_helpers/_test_types.py | 8 ++ .../_helpers/test_model_config_validation.py | 94 ++++++++++++++++++- .../discovery/_helpers/_test_types.py | 16 ++++ .../discovery/_helpers/test_yml_project.py | 60 ++++++++++++ .../discovery/_helpers/test_yml_sources.py | 81 ++++++++++++++++ .../planner/_helpers/changes/_test_types.py | 7 ++ .../planner/_helpers/changes/test_policy.py | 25 ++++- .../docs/concepts/models/configuration.mdx | 2 +- .../content/docs/docs/integrations/dlt.mdx | 4 + 21 files changed, 521 insertions(+), 23 deletions(-) create mode 100644 src/sqlbuild/compiler/authored_values/_helpers/change_policy.py create mode 100644 src/sqlbuild/compiler/authored_values/main/_change_policy_header_help.py create mode 100644 src/sqlbuild/compiler/authored_values/main/_change_policy_problem.py create mode 100644 src/sqlbuild/compiler/authored_values/main/_change_policy_toml_help.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index e7d71af19c..3dca51e3a8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -150,7 +150,7 @@ Table promotion mode is a project setting rather than a `MODEL()` field. Staged See [Incremental](../incremental.md) for full semantics. -Current compatibility behavior treats an unrecognized `on_schema_change` value as the default `append_new_columns` policy and an unrecognized `replay_on_change` value as `forward`. Use the documented values exactly; future validation may reject unknown values instead of applying these fallbacks. +Unknown `on_schema_change` or `replay_on_change` values fail compilation, in `MODEL()` headers, `[defaults]` and `[path_defaults]`. The error names the value and lists the valid ones. `bounded-` needs a positive duration such as `bounded-14d`. ## Snapshot fields diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md index 3bdc163c2a..71cbda289c 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md @@ -86,6 +86,7 @@ dlt_sources: bucket_url: "s3://my-bucket/events/" resources: - name: raw_events + reader: csv write_disposition: append ``` @@ -98,11 +99,14 @@ dlt_sources: | `name` | Source name SQLBuild exposes (referenced via `__source("name")`). Required. | | `table` | Source table to replicate (`sql_database` only). Required for that type. | | `endpoint` | Endpoint mapping (`rest_api` only). Required for that type. | +| `reader` | File reader: `csv`, `jsonl`, or `parquet` (`filesystem` only). Required for that type. | | `write_disposition` | dlt write disposition: `replace`, `append`, or `merge`. `merge` requires `primary_key`. | | `primary_key` / `merge_key` | dlt keys for `merge`. | | `incremental` | dlt incremental config mapping (e.g. cursor column and start value). | | `schema` | Per-resource schema override (falls back to the group `schema`). | +A `dlt_sources` entry accepts only `type`, `config`, `destination`, `schema`, and `resources`, and each resource accepts only the keys above for its source type. Any other key fails compilation with the nearest supported key as a hint. + Use SQLBuild's `write_disposition` (dlt's term), not `write_strategy`. `delete_insert` is not available declaratively; use a Python loader for that. Config values support SQLBuild's [interpolation](../concepts/interpolation.md): `${name}` for a project variable, `${ENV:NAME}` for an environment variable, and `${CTX:...}` for context, resolved at load time, so credentials stay out of the YAML. ### Destination diff --git a/src/sqlbuild/compiler/authored_values/_helpers/change_policy.py b/src/sqlbuild/compiler/authored_values/_helpers/change_policy.py new file mode 100644 index 0000000000..c87c9f902f --- /dev/null +++ b/src/sqlbuild/compiler/authored_values/_helpers/change_policy.py @@ -0,0 +1,64 @@ +"""Validation of authored on_schema_change and replay_on_change values.""" + +from __future__ import annotations + +from sqlbuild.compiler.planner.types import BackfillAction, OnSchemaChange +from sqlbuild.cursor_algebra.models import Duration +from sqlbuild.errors.setting_help.main.model_header_help import model_header_help +from sqlbuild.errors.setting_help.main.setting_help import setting_help + +_ON_SCHEMA_CHANGE_KEY: str = "on_schema_change" +_REPLAY_ON_CHANGE_KEY: str = "replay_on_change" +_BOUNDED_PREFIX: str = f"{BackfillAction.BOUNDED}-" +_VALID_VALUES: dict[str, tuple[str, ...]] = { + _ON_SCHEMA_CHANGE_KEY: tuple(sorted(policy.value for policy in OnSchemaChange)), + _REPLAY_ON_CHANGE_KEY: ( + BackfillAction.FORWARD_ONLY.value, + BackfillAction.FULL.value, + f"{_BOUNDED_PREFIX}", + ), +} +_EXAMPLE_VALUES: dict[str, str] = { + _ON_SCHEMA_CHANGE_KEY: OnSchemaChange.APPEND_NEW_COLUMNS.value, + _REPLAY_ON_CHANGE_KEY: f"{_BOUNDED_PREFIX}14d", +} + + +def change_policy_problem_impl(*, key: str, value: object) -> str | None: + """Describe why a change-policy value is invalid, or return None when it is valid.""" + + valid: str = ", ".join(_VALID_VALUES[key]) + if not isinstance(value, str): + return f"{key} must be a string; valid values: {valid}" + if key == _ON_SCHEMA_CHANGE_KEY and value in _VALID_VALUES[key]: + return None + if key == _REPLAY_ON_CHANGE_KEY: + if value in (BackfillAction.FORWARD_ONLY, BackfillAction.FULL): + return None + if value.startswith(_BOUNDED_PREFIX): + duration: str = value.removeprefix(_BOUNDED_PREFIX).strip() + if Duration.parse(duration) is not None: + return None + return ( + f"{key} '{value}' has an invalid duration '{duration}'; use a positive duration " + "such as 14d, 12h or 1mo" + ) + return f"unknown {key} '{value}'; valid values: {valid}" + + +def change_policy_header_help_impl(*, key: str) -> str: + """Show a valid MODEL header entry for a change-policy key.""" + + return model_header_help(purpose=f"use a valid {key}", entry=f"{key} {_EXAMPLE_VALUES[key]}") + + +def change_policy_toml_help_impl(*, key: str, file_name: str, section: str) -> str: + """Show a valid TOML setting line for a change-policy key.""" + + return setting_help( + purpose=f"use a valid {key}", + file_name=file_name, + section=section, + key=key, + value=_EXAMPLE_VALUES[key], + ) diff --git a/src/sqlbuild/compiler/authored_values/main/_change_policy_header_help.py b/src/sqlbuild/compiler/authored_values/main/_change_policy_header_help.py new file mode 100644 index 0000000000..9d40a3bc96 --- /dev/null +++ b/src/sqlbuild/compiler/authored_values/main/_change_policy_header_help.py @@ -0,0 +1,13 @@ +"""Public MODEL-header help entrypoint for change-policy values.""" + +from __future__ import annotations + +from sqlbuild.compiler.authored_values._helpers.change_policy import ( + change_policy_header_help_impl, +) + + +def change_policy_header_help(*, key: str) -> str: + """Show a valid MODEL header entry for on_schema_change or replay_on_change.""" + + return change_policy_header_help_impl(key=key) diff --git a/src/sqlbuild/compiler/authored_values/main/_change_policy_problem.py b/src/sqlbuild/compiler/authored_values/main/_change_policy_problem.py new file mode 100644 index 0000000000..773eb94c2c --- /dev/null +++ b/src/sqlbuild/compiler/authored_values/main/_change_policy_problem.py @@ -0,0 +1,11 @@ +"""Public change-policy value validation entrypoint.""" + +from __future__ import annotations + +from sqlbuild.compiler.authored_values._helpers.change_policy import change_policy_problem_impl + + +def change_policy_problem(*, key: str, value: object) -> str | None: + """Describe why an on_schema_change or replay_on_change value is invalid, or return None.""" + + return change_policy_problem_impl(key=key, value=value) diff --git a/src/sqlbuild/compiler/authored_values/main/_change_policy_toml_help.py b/src/sqlbuild/compiler/authored_values/main/_change_policy_toml_help.py new file mode 100644 index 0000000000..814249ae11 --- /dev/null +++ b/src/sqlbuild/compiler/authored_values/main/_change_policy_toml_help.py @@ -0,0 +1,11 @@ +"""Public TOML help entrypoint for change-policy values.""" + +from __future__ import annotations + +from sqlbuild.compiler.authored_values._helpers.change_policy import change_policy_toml_help_impl + + +def change_policy_toml_help(*, key: str, file_name: str, section: str) -> str: + """Show a valid TOML setting line for on_schema_change or replay_on_change.""" + + return change_policy_toml_help_impl(key=key, file_name=file_name, section=section) diff --git a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py index 8020d1a704..b8eae7c29b 100644 --- a/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/config/model_validation.py @@ -7,6 +7,10 @@ from decimal import Decimal, InvalidOperation from typing import cast +from sqlbuild.compiler.authored_values.main._change_policy_header_help import ( + change_policy_header_help, +) +from sqlbuild.compiler.authored_values.main._change_policy_problem import change_policy_problem from sqlbuild.compiler.compile.constants import ( MICROBATCH_LIMIT_ACTION_KEY, MICROBATCH_LIMIT_MAX_BATCHES_KEY, @@ -160,7 +164,6 @@ class _IncrementalConfigValues: max_microbatches: object | None microbatch_limit: object | None cursor_grain: str | None - replay_on_change: object | None merge_exclude_columns: object | None full_refresh: object | None @@ -198,7 +201,7 @@ def validate_incremental_config( return values: _IncrementalConfigValues = _resolve_incremental_config_values(config=config) - _validate_incremental_core(model_name=model_name, values=values) + _validate_incremental_core(config=config, model_name=model_name, values=values) _validate_incremental_batching( model_name=model_name, ref_count=ref_count, @@ -218,9 +221,18 @@ def validate_incremental_config( ) -def _validate_incremental_core(*, model_name: str, values: _IncrementalConfigValues) -> None: - if values.replay_on_change is not None and not isinstance(values.replay_on_change, str): - raise CompileInputError(f"model '{model_name}': replay_on_change must be a string") +def _validate_incremental_core( + *, config: CompileModelConfig, model_name: str, values: _IncrementalConfigValues +) -> None: + key: str + for key in ("on_schema_change", "replay_on_change"): + value: object | None = config.values.get(key) + problem: str | None = None if value is None else change_policy_problem(key=key, value=value) + if problem is not None: + raise CompileInputError( + f"model '{model_name}': {problem}", + help=change_policy_header_help(key=key), + ) if values.strategy is None: raise CompileInputError( f"model '{model_name}': incremental materialization requires incremental_strategy" @@ -405,7 +417,6 @@ def _resolve_incremental_config_values(*, config: CompileModelConfig) -> _Increm max_microbatches=config.values.get("max_microbatches"), microbatch_limit=config.values.get("microbatch_limit"), cursor_grain=get_config_str(values=config.values, key="cursor_grain"), - replay_on_change=config.values.get("replay_on_change"), merge_exclude_columns=config.values.get("merge_exclude_columns"), full_refresh=config.values.get("full_refresh"), ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py b/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py index 57ff142659..c6f1d2e12b 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py +++ b/src/sqlbuild/compiler/discovery/_helpers/integrations/loaders.py @@ -46,6 +46,17 @@ "unique_key", } ) +_DLT_SOURCE_GROUP_KEYS: frozenset[str] = frozenset( + {"config", "destination", "resources", "schema", "type"} +) +_DLT_RESOURCE_COMMON_KEYS: frozenset[str] = frozenset( + {"incremental", "merge_key", "name", "primary_key", "schema", "write_disposition"} +) +_DLT_RESOURCE_TYPE_KEYS: dict[str, frozenset[str]] = { + DLT_SOURCE_TYPE_FILESYSTEM: frozenset({"reader"}), + DLT_SOURCE_TYPE_REST_API: frozenset({"endpoint"}), + DLT_SOURCE_TYPE_SQL_DATABASE: frozenset({"table"}), +} def integration_loader_name(*, kind: str, source_name: str) -> str: @@ -201,6 +212,13 @@ def _optional_ingestr_string_tuple( def _parse_dlt_source_group( *, group: dict[str, object], file_path: Path, group_index: int ) -> tuple[SourceEntry, ...]: + reject_unknown_mapping_keys( + mapping=group, + allowed=_DLT_SOURCE_GROUP_KEYS, + file_path=file_path, + label="dlt source", + error_class=SourceParseError, + ) source_type: str = require_non_empty_string( entry=group, key="type", @@ -275,6 +293,13 @@ def _parse_dlt_resource_entry( raise SourceParseError( f"{file_path} dlt resource '{name}' must use dlt write_disposition, not write_strategy" ) + reject_unknown_mapping_keys( + mapping=resource, + allowed=_DLT_RESOURCE_COMMON_KEYS | _DLT_RESOURCE_TYPE_KEYS[source_type], + file_path=file_path, + label=f"dlt {source_type} resource '{name}'", + error_class=SourceParseError, + ) dlt_name: str = _dlt_resource_name( source_type=source_type, resource=resource, file_path=file_path ) diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py index 4aad18539e..f05cf7df1f 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py @@ -14,7 +14,11 @@ from yaml import YAMLError from sqlbuild.compiler.auditing.types import AuditSeverity -from sqlbuild.compiler.compile.constants import MAX_MICROBATCHES_CONFIG_KEY +from sqlbuild.compiler.authored_values.main._change_policy_problem import change_policy_problem +from sqlbuild.compiler.authored_values.main._change_policy_toml_help import ( + change_policy_toml_help, +) +from sqlbuild.compiler.compile.constants import MAX_MICROBATCHES_CONFIG_KEY, TEMPLATE_OPEN_TOKEN from sqlbuild.compiler.discovery._helpers.validation.supported_keys import unsupported_keys_help from sqlbuild.compiler.discovery.constants import ( CONFIG_CONCURRENCY_KEY, @@ -932,6 +936,20 @@ def _validate_allowed_keys( ) +def _validate_change_policies(*, mapping: dict[str, object], section: str, file_path: Path) -> None: + key: str + for key in ("on_schema_change", "replay_on_change"): + value: object | None = mapping.get(key) + if value is None or (isinstance(value, str) and TEMPLATE_OPEN_TOKEN in value): + continue + problem: str | None = change_policy_problem(key=key, value=value) + if problem is not None: + raise ProjectConfigError( + f"{file_path} [{section}] {problem}", + help=change_policy_toml_help(key=key, file_name=file_path.name, section=section), + ) + + def _normalize_path_default_key(*, path_key: str, file_path: Path) -> str: normalized_key: str = path_key.strip() if not normalized_key: @@ -986,6 +1004,7 @@ def _load_defaults(*, payload: object, file_path: Path) -> DefaultsConfig: _validate_allowed_keys( mapping=mapping, allowed_keys=_DEFAULTS_KEYS, label="[defaults]", file_path=file_path ) + _validate_change_policies(mapping=mapping, section="defaults", file_path=file_path) row_diff_exclude_columns: tuple[str, ...] = tuple( _load_string_sequence( payload=mapping.get("row_diff_exclude_columns"), @@ -1085,6 +1104,11 @@ def _load_path_defaults(*, payload: object, file_path: Path) -> dict[str, dict[s label=f"path_defaults['{path_key}']", file_path=file_path, ) + _validate_change_policies( + mapping=path_dict, + section=f'path_defaults."{path_key}"', + file_path=file_path, + ) _validate_path_default_tags(path_dict=path_dict, path_key=path_key, file_path=file_path) normalized_path_key: str = _normalize_path_default_key( path_key=path_key, file_path=file_path diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/policy.py b/src/sqlbuild/compiler/planner/_helpers/changes/policy.py index 419fcab0f6..2195b5de55 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/policy.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/policy.py @@ -4,6 +4,7 @@ import re +from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import BackfillResult from sqlbuild.compiler.planner.types import BackfillAction @@ -44,4 +45,4 @@ def _resolve_replay_value(raw: str | None) -> BackfillResult: if match is not None: duration: str = match.group(1).strip() return BackfillResult(action=BackfillAction.BOUNDED, duration=duration) - return BackfillResult(action=BackfillAction.FORWARD_ONLY) + raise PlannerInputError(f"unknown replay_on_change '{raw}'") diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index 1a0602f3ae..b91db81652 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -1027,12 +1027,11 @@ def _get_on_schema_change(model: CompiledModel) -> OnSchemaChange | None: """Extract on_schema_change from model config.""" raw: object | None = model.config.values.get("on_schema_change") - if isinstance(raw, str): - try: - return OnSchemaChange(raw) - except ValueError: - pass - return None + if raw is None: + return None + if isinstance(raw, str) and raw in set(OnSchemaChange): + return OnSchemaChange(raw) + raise PlannerInputError(f"model '{model.name}': unknown on_schema_change '{raw}'") def extract_seed_columns(seed: CompiledSeed) -> tuple[ColumnInfo, ...]: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py index 52d9bcc933..3c267d704f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_unknown_config_keys.py @@ -1,4 +1,4 @@ -"""E2E coverage for rejecting unknown MODEL header, model config layer, and source keys.""" +"""E2E coverage for rejecting unknown MODEL header, model config layer, and source keys and values.""" from __future__ import annotations @@ -66,6 +66,58 @@ "did you mean 'type'?", ), ), + ModelHeaderKeyCompileCase( + description="unknown on_schema_change in MODEL header", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/fct_orders.sql": ( + "MODEL (\n materialized incremental,\n incremental_strategy append,\n" + " on_schema_change append_columns,\n);\n\nSELECT 1 AS order_id\n" + ), + }, + expected_fragments=( + "error[P001]", + "unknown on_schema_change 'append_columns'; valid values: append_new_columns, " + "fail, ignore, sync_all_columns", + "on_schema_change append_new_columns,", + ), + ), + ModelHeaderKeyCompileCase( + description="unknown replay_on_change in project defaults", + repo_files={ + "sqlbuild_project.toml": ( + _PROJECT_TOML + '\n[defaults]\nreplay_on_change = "everything"\n' + ), + "models/fct_orders.sql": ( + "MODEL (materialized incremental, incremental_strategy append);\n\n" + "SELECT 1 AS order_id\n" + ), + }, + expected_fragments=( + "error[D001]", + "[defaults] unknown replay_on_change 'everything'; valid values: forward, full, " + "bounded-", + 'replay_on_change = "bounded-14d"', + ), + ), + ModelHeaderKeyCompileCase( + description="misspelled dlt resource key", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "sources/raw.yml": ( + "dlt_sources:\n - type: sql_database\n config:\n" + " credentials: duckdb:///inputs.duckdb\n resources:\n" + " - name: raw_orders\n table: orders\n" + " primary_keys: order_id\n" + ), + "models/stg_orders.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", + }, + expected_fragments=( + "error[D006]", + "dlt sql_database resource 'raw_orders' has unknown keys: primary_keys", + "did you mean 'primary_key'?", + ), + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 6378766475..91a9c1f4eb 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -608,6 +608,14 @@ class IncrementalConfigErrorTestCase: expected_error_fragment: str +@dataclass(frozen=True) +class ChangePolicyHelpTestCase: + description: str + key: str + value: str + expected_help: str + + @dataclass(frozen=True) class ContractConfigValidTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py index 178450d3c5..ef3f2a1c28 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_model_config_validation.py @@ -21,6 +21,7 @@ from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention from sqlbuild.spec.contracts.types import TimeTravelRetentionSource from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( + ChangePolicyHelpTestCase, ContractConfigErrorTestCase, ContractConfigValidTestCase, CustomMaterializationConfigErrorTestCase, @@ -37,6 +38,8 @@ SnapshotConfigValidTestCase, ) +_SNIPPET_INDENT: str = " " * 12 + @pytest.mark.parametrize( "test_case", @@ -354,7 +357,43 @@ def test_given_valid_config_when_validating_then_passes( "replay_on_change": {"add_column": "bounded-30d"}, }, ref_count=1, - expected_error_fragment="replay_on_change must be a string", + expected_error_fragment="replay_on_change must be a string; valid values: forward", + ), + IncrementalConfigErrorTestCase( + description="unknown replay_on_change raises with valid values", + config_values={ + "materialized": "incremental", + "incremental_strategy": "append", + "replay_on_change": "bounded", + }, + ref_count=1, + expected_error_fragment=( + "unknown replay_on_change 'bounded'; valid values: forward, full, " + "bounded-" + ), + ), + IncrementalConfigErrorTestCase( + description="bounded replay_on_change without valid duration raises", + config_values={ + "materialized": "incremental", + "incremental_strategy": "append", + "replay_on_change": "bounded-two-weeks", + }, + ref_count=1, + expected_error_fragment="has an invalid duration 'two-weeks'", + ), + IncrementalConfigErrorTestCase( + description="unknown on_schema_change raises with valid values", + config_values={ + "materialized": "incremental", + "incremental_strategy": "append", + "on_schema_change": "append_columns", + }, + ref_count=1, + expected_error_fragment=( + "unknown on_schema_change 'append_columns'; valid values: append_new_columns, " + "fail, ignore, sync_all_columns" + ), ), IncrementalConfigErrorTestCase( description="cursor without cursor_type raises", @@ -780,6 +819,59 @@ def test_given_invalid_config_when_validating_then_raises( ) +@pytest.mark.parametrize( + "test_case", + [ + ChangePolicyHelpTestCase( + description="on_schema_change", + key="on_schema_change", + value="append", + expected_help=( + "use a valid on_schema_change, add this to the MODEL header:\n" + f"{_SNIPPET_INDENT}MODEL (\n" + f"{_SNIPPET_INDENT} on_schema_change append_new_columns,\n" + f"{_SNIPPET_INDENT} ...\n" + f"{_SNIPPET_INDENT});" + ), + ), + ChangePolicyHelpTestCase( + description="replay_on_change", + key="replay_on_change", + value="everything", + expected_help=( + "use a valid replay_on_change, add this to the MODEL header:\n" + f"{_SNIPPET_INDENT}MODEL (\n" + f"{_SNIPPET_INDENT} replay_on_change bounded-14d,\n" + f"{_SNIPPET_INDENT} ...\n" + f"{_SNIPPET_INDENT});" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_unknown_change_policy_when_validating_then_help_shows_valid_header_line( + test_case: ChangePolicyHelpTestCase, +) -> None: + config: CompileModelConfig = CompileModelConfig( + values={ + "materialized": "incremental", + "incremental_strategy": "append", + test_case.key: test_case.value, + } + ) + + with pytest.raises(CompileInputError) as error_info: + validate_incremental_config( + config=config, + model_name="test_model", + ref_count=1, + known_input_names=frozenset({"orders"}), + ) + + assert f"unknown {test_case.key} '{test_case.value}'" in str(error_info.value) + assert error_info.value.help == test_case.expected_help + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 91fe3a1b76..b8a1d47041 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -329,6 +329,14 @@ class LoadProjectConfigErrorTestCase: expected_error_fragment: str +@dataclass(frozen=True) +class LoadProjectConfigHelpTestCase: + description: str + project_file_contents: str + expected_error_fragment: str + expected_help: str + + @dataclass(frozen=True) class LoadProjectCostConfigTestCase: description: str @@ -722,6 +730,14 @@ class ParseSourcesYamlErrorTestCase: expected_error_fragment: str +@dataclass(frozen=True) +class ParseSourcesYamlHelpTestCase: + description: str + contents: str + expected_error_fragment: str + expected_help: str + + @dataclass(frozen=True) class ParseSourcesYamlIngestrTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py index fca7e0c412..9e1cd99b0d 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py @@ -31,6 +31,7 @@ LoadLocalConfigTestCase, LoadNamedConnectionsTestCase, LoadProjectConfigErrorTestCase, + LoadProjectConfigHelpTestCase, LoadProjectConfigTestCase, LoadProjectConstantsConfigErrorTestCase, LoadProjectConstantsConfigTestCase, @@ -1766,6 +1767,34 @@ def test_given_local_target_execution_limits_when_loading_then_limits_are_typed( """.strip(), expected_error_fragment=r"option\(s\) were removed: replay_on_change", ), + LoadProjectConfigErrorTestCase( + description="raises for unknown replay_on_change in defaults", + project_file_contents=""" +name = "demo" +adapter = "duckdb" + +[defaults] +replay_on_change = "always" +""".strip(), + expected_error_fragment=( + r"\[defaults\] unknown replay_on_change 'always'; valid values: forward, full, " + r"bounded-" + ), + ), + LoadProjectConfigErrorTestCase( + description="raises for unknown on_schema_change in path defaults", + project_file_contents=""" +name = "demo" +adapter = "duckdb" + +[path_defaults.marts] +on_schema_change = "sync" +""".strip(), + expected_error_fragment=( + r"\[path_defaults.\"marts\"\] unknown on_schema_change 'sync'; valid values: " + r"append_new_columns, fail, ignore, sync_all_columns" + ), + ), LoadProjectConfigErrorTestCase( description="raises targeted error for removed dbt production_ref reuse config", project_file_contents=""" @@ -1802,6 +1831,37 @@ def test_given_invalid_project_config_file_when_loading_project_config_then_it_r load_project_config(project_dir=tmp_path) +@pytest.mark.parametrize( + "test_case", + [ + LoadProjectConfigHelpTestCase( + description="bounded replay without a valid duration in defaults", + project_file_contents=( + 'name = "demo"\nadapter = "duckdb"\n\n[defaults]\nreplay_on_change = "bounded-x"\n' + ), + expected_error_fragment="replay_on_change 'bounded-x' has an invalid duration 'x'", + expected_help=( + "use a valid replay_on_change, set this in sqlbuild_project.toml:\n" + " [defaults]\n" + ' replay_on_change = "bounded-14d"' + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_unknown_change_policy_in_defaults_when_loading_then_help_shows_toml_line( + test_case: LoadProjectConfigHelpTestCase, tmp_path: Path +) -> None: + project_file: Path = tmp_path / "sqlbuild_project.toml" + project_file.write_text(test_case.project_file_contents, encoding="utf-8") + + with pytest.raises(ProjectConfigError) as error_info: + load_project_config(project_dir=tmp_path) + + assert test_case.expected_error_fragment in str(error_info.value) + assert error_info.value.help == test_case.expected_help + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py index 45d68ea8ce..8ff38e7023 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_sources.py @@ -6,6 +6,7 @@ import pytest from sqlbuild.compiler.discovery._helpers.yml.sources import parse_sources_yml +from sqlbuild.compiler.discovery.exceptions import SourceParseError from sqlbuild.integrations.dlt.models import DltSourceConfig from sqlbuild.integrations.ingestr.models import IngestrSourceConfig from sqlbuild.spec.contracts.models import SourceEntry @@ -13,6 +14,7 @@ from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( ParseSourcesYamlDltTestCase, ParseSourcesYamlErrorTestCase, + ParseSourcesYamlHelpTestCase, ParseSourcesYamlIngestrTestCase, ParseSourcesYamlTestCase, ) @@ -787,6 +789,50 @@ def test_given_dlt_sources_yaml_when_parsing_then_expands_managed_sources( """, expected_error_fragment="dlt source destination cannot define 'credentials'", ), + ParseSourcesYamlErrorTestCase( + description="raises when dlt source group has an unknown key", + contents=""" + dlt_sources: + - type: sql_database + confg: + credentials: duckdb:///source.duckdb + resources: + - name: raw_orders + table: orders + """, + expected_error_fragment="dlt source has unknown keys: confg", + ), + ParseSourcesYamlErrorTestCase( + description="raises when dlt resource has an unknown key", + contents=""" + dlt_sources: + - type: sql_database + config: + credentials: duckdb:///source.duckdb + resources: + - name: raw_orders + table: orders + primary_keys: id + """, + expected_error_fragment=( + "dlt sql_database resource 'raw_orders' has unknown keys: primary_keys" + ), + ), + ParseSourcesYamlErrorTestCase( + description="raises when dlt resource uses a key of another source type", + contents=""" + dlt_sources: + - type: sql_database + config: + credentials: duckdb:///source.duckdb + resources: + - name: raw_orders + table: orders + endpoint: + path: orders + """, + expected_error_fragment="dlt sql_database resource 'raw_orders' has unknown keys: endpoint", + ), ParseSourcesYamlErrorTestCase( description="raises when write strategy is unknown", contents=""" @@ -1290,3 +1336,38 @@ def test_given_invalid_sources_yaml_when_parsing_then_it_raises_clear_errors( ) -> None: with pytest.raises(ValueError, match=test_case.expected_error_fragment): parse_sources_yml(contents=test_case.contents, file_path=Path("sources/raw.yml")) + + +@pytest.mark.parametrize( + "test_case", + [ + ParseSourcesYamlHelpTestCase( + description="misspelled rest_api resource key", + contents=""" +dlt_sources: + - type: rest_api + config: + client: + base_url: https://api.example.com/ + resources: + - name: raw_orders + write_dispositon: append + endpoint: + path: orders +""", + expected_error_fragment=( + "dlt rest_api resource 'raw_orders' has unknown keys: write_dispositon" + ), + expected_help="did you mean 'write_disposition'?", + ), + ], + ids=lambda case: case.description, +) +def test_given_misspelled_dlt_resource_key_when_parsing_then_help_names_nearest_key( + test_case: ParseSourcesYamlHelpTestCase, +) -> None: + with pytest.raises(SourceParseError) as error_info: + parse_sources_yml(contents=test_case.contents, file_path=Path("sources/raw.yml")) + + assert test_case.expected_error_fragment in str(error_info.value) + assert error_info.value.help == test_case.expected_help diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py index 099e4f5d7c..67848cdf49 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/_test_types.py @@ -89,6 +89,13 @@ class ResolveBackfillTestCase: expected_result: BackfillResult +@dataclass(frozen=True) +class ResolveBackfillErrorTestCase: + description: str + raw_value: str + expected_error_fragment: str + + @dataclass(frozen=True) class DetectRenamedModelTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_policy.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_policy.py index 95f8a09ea5..2b12491a2d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_policy.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/changes/test_policy.py @@ -3,9 +3,11 @@ import pytest from sqlbuild.compiler.planner._helpers.changes.policy import resolve_replay_on_change +from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import BackfillResult from sqlbuild.compiler.planner.types import BackfillAction from tests.unit.src.sqlbuild.compiler.planner._helpers.changes._test_types import ( + ResolveBackfillErrorTestCase, ResolveBackfillTestCase, ) @@ -28,11 +30,6 @@ raw_value=None, expected_result=BackfillResult(action=BackfillAction.FORWARD_ONLY), ), - ResolveBackfillTestCase( - description="returns forward only for unrecognized policy value", - raw_value="unknown", - expected_result=BackfillResult(action=BackfillAction.FORWARD_ONLY), - ), ], ids=lambda case: case.description, ) @@ -44,3 +41,21 @@ def test_given_policy_when_resolving_replay_on_change_then_returns_expected( ) assert result == test_case.expected_result + + +@pytest.mark.parametrize( + "test_case", + [ + ResolveBackfillErrorTestCase( + description="raises for an unvalidated policy value", + raw_value="unknown", + expected_error_fragment="unknown replay_on_change 'unknown'", + ), + ], + ids=lambda case: case.description, +) +def test_given_unvalidated_policy_when_resolving_replay_on_change_then_raises( + test_case: ResolveBackfillErrorTestCase, +) -> None: + with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment): + resolve_replay_on_change(replay_on_change=test_case.raw_value) diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index 51b24356ae..010e0a66d8 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -137,7 +137,7 @@ Table promotion mode is a project setting rather than a `MODEL()` field. Staged See [Incremental](/docs/concepts/incremental) for full semantics. -Current compatibility behavior treats an unrecognized `on_schema_change` value as the default `append_new_columns` policy and an unrecognized `replay_on_change` value as `forward`. Use the documented values exactly; future validation may reject unknown values instead of applying these fallbacks. +Unknown `on_schema_change` or `replay_on_change` values fail compilation, in `MODEL()` headers, `[defaults]` and `[path_defaults]`. The error names the value and lists the valid ones. `bounded-` needs a positive duration such as `bounded-14d`. ## Snapshot fields diff --git a/website/src/content/docs/docs/integrations/dlt.mdx b/website/src/content/docs/docs/integrations/dlt.mdx index 7341376c11..2f8e03e7fa 100644 --- a/website/src/content/docs/docs/integrations/dlt.mdx +++ b/website/src/content/docs/docs/integrations/dlt.mdx @@ -82,6 +82,7 @@ dlt_sources: bucket_url: "s3://my-bucket/events/" resources: - name: raw_events + reader: csv write_disposition: append ``` @@ -94,11 +95,14 @@ dlt_sources: | `name` | Source name SQLBuild exposes (referenced via `__source("name")`). Required. | | `table` | Source table to replicate (`sql_database` only). Required for that type. | | `endpoint` | Endpoint mapping (`rest_api` only). Required for that type. | +| `reader` | File reader: `csv`, `jsonl`, or `parquet` (`filesystem` only). Required for that type. | | `write_disposition` | dlt write disposition: `replace`, `append`, or `merge`. `merge` requires `primary_key`. | | `primary_key` / `merge_key` | dlt keys for `merge`. | | `incremental` | dlt incremental config mapping (e.g. cursor column and start value). | | `schema` | Per-resource schema override (falls back to the group `schema`). | +A `dlt_sources` entry accepts only `type`, `config`, `destination`, `schema`, and `resources`, and each resource accepts only the keys above for its source type. Any other key fails compilation with the nearest supported key as a hint. + Use SQLBuild's `write_disposition` (dlt's term), not `write_strategy`. `delete_insert` is not available declaratively; use a Python loader for that. Config values support SQLBuild's [interpolation](/docs/concepts/interpolation): `${name}` for a project variable, `${ENV:NAME}` for an environment variable, and `${CTX:...}` for context, resolved at load time, so credentials stay out of the YAML. ### Destination From eceab9920799d84a8a457fc10842a776dab89e87 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:50:51 +0100 Subject: [PATCH 2/7] fix(compiler): follow adapter lexical rules in compile-step SQL scanners Reference extraction, SQL test and scenario CTE scanning, assertion target detection, scope relationships and native SQL test planning now honour the active adapter's lexical syntax (backslash escapes, escape and raw string prefixes, triple quotes, dialect line comments and nested block comments) instead of generic SQL rules. Cache keys include the lexical syntax. Refs: CHI-466 --- .../_helpers/sql_tests/helper_scope.rs | 2 +- .../compiler/_helpers/sql_tests/markers.rs | 39 ++- .../compiler/_helpers/sql_tests/planning.rs | 49 ++-- .../src/compiler/tests/helpers.rs | 24 ++ .../src/sql_scan/_helpers/dialect.rs | 141 ++++++++++ .../src/sql_scan/_helpers/dollar.rs | 31 +++ .../src/sql_scan/_helpers/mod.rs | 3 + .../src/sql_scan/_helpers/parens.rs | 31 +++ .../src/sql_scan/constants.rs | 8 + .../sql_scan/main/dialect_matching_paren.rs | 13 + .../sql_scan/main/dialect_non_code_ranges.rs | 9 + .../src/sql_scan/main/matching_paren.rs | 22 +- .../src/sql_scan/main/mod.rs | 2 + .../src/sql_scan/main/non_code_end.rs | 29 +- .../sqlbuild-rules-native/src/sql_scan/mod.rs | 2 + .../src/sql_scan/models.rs | 27 ++ .../src/sql_scan/tests.rs | 4 + .../src/sql_scan/tests/helpers.rs | 43 +++ .../src/sql_scan/tests/test_dialect_scan.rs | 119 ++++++++ .../src/sql_scan/tests/test_types.rs | 14 + .../references/docs/concepts/testing.md | 7 + .../compile/_helpers/assembly/project.py | 25 +- .../compile/_helpers/attachment/audits.py | 30 +- .../compile/_helpers/attachment/core.py | 13 +- .../_helpers/attachment/declaration_scope.py | 3 + .../compile/_helpers/attachment/functions.py | 6 +- .../attachment/scope_relationships.py | 27 +- .../compile/_helpers/attachment/sql_tests.py | 52 +++- .../explicit_references/macro_arguments.py | 11 +- .../compiler/compile/_helpers/refs/cache.py | 24 +- .../compile/_helpers/refs/references.py | 74 ++--- .../compile/_helpers/scenarios/core.py | 18 +- .../compile/_helpers/sql_tests/core.py | 259 ++++++++++++------ .../compile/_helpers/sql_tests/native.py | 41 ++- .../compile/classes/sql_test_cte_extractor.py | 7 +- .../compile/main/_build_compile_inputs.py | 13 +- .../main/_scope_relationship_grants.py | 12 +- .../compile/main/sql_expansion_context.py | 14 +- src/sqlbuild/compiler/compile/models.py | 7 +- src/sqlbuild/compiler/compile/types.py | 2 + .../pipeline/main/compiled_project.py | 1 + .../_helpers/sql_tests/fixture_formatting.py | 5 + .../_helpers/sql_tests/native_planning.py | 14 + .../planner/classes/fixture_null_autofix.py | 3 + .../compiler/scopes/_helpers/builder.py | 6 +- .../scopes/main/load_or_build_scope_index.py | 8 +- .../sql_analysis/_helpers/scanning.py | 28 +- .../compiler/sql_analysis/constants.py | 1 + .../sql_analysis/main/_find_matching_paren.py | 12 +- .../sql_analysis/main/_iter_code_positions.py | 9 +- src/sqlbuild/compiler/sql_analysis/models.py | 44 ++- src/sqlbuild/lint/main/run_format.py | 22 +- .../main/scenario/test_authored_check_sql.py | 18 ++ .../main/test/test_dialect_string_escapes.py | 65 +++++ .../sqlbuild/cli/commands/main/test_rules.py | 5 + .../src/sqlbuild/compiler/pipeline/helpers.py | 1 + .../src/sqlbuild/integrations/dbt/helpers.py | 1 + .../compiler/compile/_helpers/_test_types.py | 18 ++ .../compiler/compile/_helpers/helpers.py | 2 + ...st_direct_logic_sql_test_classification.py | 7 +- .../test_expected_projection_scanning.py | 17 +- .../test_native_sql_test_extraction.py | 145 +++++++++- .../_helpers/test_rebind_declaration_scope.py | 9 +- .../compile/_helpers/test_scenarios.py | 123 ++++++++- .../test_set_operation_expected_ctes.py | 9 +- .../compile/_helpers/test_sql_references.py | 101 ++++++- .../compiler/compile/_helpers/test_tests.py | 12 +- .../compiler/planner/_helpers/helpers.py | 5 +- .../sql_analysis/_helpers/_test_types.py | 20 ++ .../sql_analysis/_helpers/test_scanning.py | 98 +++++++ .../src/sqlbuild/integrations/dbt/helpers.py | 6 +- tests/unit/src/sqlbuild/lint/_test_types.py | 10 + .../lint/test_adapter_lexical_syntax.py | 78 ++++++ .../content/docs/docs/concepts/testing.mdx | 7 + 74 files changed, 1832 insertions(+), 335 deletions(-) create mode 100644 crates/sqlbuild-rules-native/src/sql_scan/_helpers/dialect.rs create mode 100644 crates/sqlbuild-rules-native/src/sql_scan/_helpers/dollar.rs create mode 100644 crates/sqlbuild-rules-native/src/sql_scan/_helpers/mod.rs create mode 100644 crates/sqlbuild-rules-native/src/sql_scan/_helpers/parens.rs create mode 100644 crates/sqlbuild-rules-native/src/sql_scan/constants.rs create mode 100644 crates/sqlbuild-rules-native/src/sql_scan/main/dialect_matching_paren.rs create mode 100644 crates/sqlbuild-rules-native/src/sql_scan/main/dialect_non_code_ranges.rs create mode 100644 crates/sqlbuild-rules-native/src/sql_scan/tests/helpers.rs create mode 100644 crates/sqlbuild-rules-native/src/sql_scan/tests/test_dialect_scan.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py create mode 100644 tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/helper_scope.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/helper_scope.rs index 4d59240106..f26728038a 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/helper_scope.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/helper_scope.rs @@ -242,7 +242,7 @@ fn is_generated_cte_name(name: &str) -> bool { /// Distinct lowercase identifiers outside strings and comments, in first-appearance order. fn identifier_tokens(sql: &str, patterns: &SqlTestPatterns) -> Vec { - let protected = protected_ranges(&patterns.protected, sql); + let protected = protected_ranges(&patterns.lexical, sql); let mut seen: HashSet = HashSet::new(); let mut tokens: Vec = Vec::new(); let mut push = |token: &str| { diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs index ba1cda0325..de59e2f7ad 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/markers.rs @@ -5,21 +5,21 @@ use std::collections::HashSet; use regex::{Captures, Regex}; use crate::compiler::_helpers::sql_tests::planning::compile_error; -use crate::sql_scan::main::matching_paren::matching_paren; +use crate::sql_scan::main::dialect_matching_paren::dialect_matching_paren; +use crate::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; use crate::sql_scan::main::skip_whitespace::skip_whitespace; -use crate::sql_scan::models::QuotePolicy; -use crate::sql_scan::models::Unclosed; +use crate::sql_scan::models::{LexicalSyntax, Unclosed}; pub(crate) fn replace_callable_markers( sql: &str, pattern: &Regex, - protected_pattern: &Regex, + syntax: &LexicalSyntax, mut replacement: F, ) -> Result<(String, HashSet), String> where F: FnMut(&str, &str) -> Option<(String, bool)>, { - let protected = protected_ranges(protected_pattern, sql); + let protected = protected_ranges(syntax, sql); let mut output = String::with_capacity(sql.len()); let mut reached: HashSet = HashSet::new(); let mut cursor = 0; @@ -34,7 +34,7 @@ where if !sql[suffix_start..].starts_with('(') { continue; } - let suffix_end = matching_paren_end(sql, suffix_start)?; + let suffix_end = matching_paren_end(sql, suffix_start, syntax)?; let Some(name) = captures.get(1).map(|value| value.as_str()) else { continue; }; @@ -53,8 +53,8 @@ where Ok((output, reached)) } -fn matching_paren_end(sql: &str, open: usize) -> Result { - matching_paren(sql.as_bytes(), open, QuotePolicy::COMPILER) +fn matching_paren_end(sql: &str, open: usize, syntax: &LexicalSyntax) -> Result { + dialect_matching_paren(sql.as_bytes(), open, syntax) .map(|close| close + 1) .map_err(|error| { compile_error(match error { @@ -68,13 +68,13 @@ fn matching_paren_end(sql: &str, open: usize) -> Result { pub(crate) fn replace_named_markers( sql: &str, pattern: &Regex, - protected_pattern: &Regex, + syntax: &LexicalSyntax, mut replacement: F, ) -> String where F: FnMut(&str) -> Option, { - replace_marker_captures(sql, pattern, protected_pattern, |captures| { + replace_marker_captures(sql, pattern, syntax, |captures| { replacement(captures.get(1)?.as_str()) }) } @@ -82,13 +82,13 @@ where pub(crate) fn replace_dbt_ref_markers( sql: &str, pattern: &Regex, - protected_pattern: &Regex, + syntax: &LexicalSyntax, mut replacement: F, ) -> String where F: FnMut(&str) -> Option, { - replace_marker_captures(sql, pattern, protected_pattern, |captures| { + replace_marker_captures(sql, pattern, syntax, |captures| { let first = captures.get(1)?.as_str(); let name = captures.get(2).map_or_else( || first.to_string(), @@ -101,13 +101,13 @@ where fn replace_marker_captures( sql: &str, pattern: &Regex, - protected_pattern: &Regex, + syntax: &LexicalSyntax, mut replacement: F, ) -> String where F: FnMut(&Captures<'_>) -> Option, { - let protected = protected_ranges(protected_pattern, sql); + let protected = protected_ranges(syntax, sql); let mut output = String::with_capacity(sql.len()); let mut cursor = 0; for captures in pattern.captures_iter(sql) { @@ -128,8 +128,8 @@ where output } -pub(crate) fn marker_names(pattern: &Regex, protected_pattern: &Regex, sql: &str) -> Vec { - let protected = protected_ranges(protected_pattern, sql); +pub(crate) fn marker_names(pattern: &Regex, syntax: &LexicalSyntax, sql: &str) -> Vec { + let protected = protected_ranges(syntax, sql); let mut names: Vec = Vec::new(); for captures in pattern.captures_iter(sql) { let Some(full) = captures.get(0) else { @@ -145,11 +145,8 @@ pub(crate) fn marker_names(pattern: &Regex, protected_pattern: &Regex, sql: &str names } -pub(crate) fn protected_ranges(pattern: &Regex, sql: &str) -> Vec<(usize, usize)> { - pattern - .find_iter(sql) - .map(|value| (value.start(), value.end())) - .collect() +pub(crate) fn protected_ranges(syntax: &LexicalSyntax, sql: &str) -> Vec<(usize, usize)> { + dialect_non_code_ranges(sql, syntax) } pub(crate) fn in_protected_range(index: usize, ranges: &[(usize, usize)]) -> bool { diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs index d0203ec5aa..7ca1946254 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/planning.rs @@ -27,6 +27,7 @@ use crate::compiler::_helpers::sql_tests::rendering::{ rendered_chain_steps, }; use crate::constants::{TABLE_FUNCTION_TEST_MODE, UDF_TEST_MODE}; +use crate::sql_scan::models::LexicalSyntax; const DEFAULT_WORKERS: usize = 4; const MAX_WORKERS: usize = 4; @@ -73,6 +74,7 @@ struct PlanBatchRequest { render_sql: bool, #[serde(default = "default_true")] include_plan: bool, + lexical_syntax: LexicalSyntax, } #[derive(Debug, Deserialize)] @@ -80,6 +82,7 @@ struct PlanBatchRequest { struct ChainBatchRequest { models: Vec, tests: Vec, + lexical_syntax: LexicalSyntax, } /// Chain ordering reads only declared dependencies, so chain requests omit model SQL. @@ -219,12 +222,12 @@ pub(crate) struct SqlTestPatterns { table_function: Regex, dbt_reference: Regex, test_reference: Regex, - pub(crate) protected: Regex, + pub(crate) lexical: LexicalSyntax, pub(crate) identifier: Regex, } impl SqlTestPatterns { - fn new() -> Result { + fn new(lexical: LexicalSyntax) -> Result { Ok(Self { reference: compile_pattern(r#"(?i)__ref\(\"([^\"]+)\"\)"#)?, source: compile_pattern(r#"(?i)__source\(\"([^\"]+)\"\)"#)?, @@ -235,9 +238,7 @@ impl SqlTestPatterns { r#"(?i)__dbt_ref\(\s*\"([^\"]+)\"\s*(?:,\s*\"([^\"]+)\"\s*)?\)"#, )?, test_reference: compile_pattern(r#"(?i)__(?:ref|source|seed|dbt_ref|table_fn)\("#)?, - protected: compile_pattern( - r#"(?s)'(?:''|[^'])*'|\"(?:\"\"|[^\"])*\"|`(?:``|[^`])*`|\$\$.*?\$\$|--[^\n]*|/\*.*?\*/"#, - )?, + lexical, identifier: compile_pattern(r"[A-Za-z_][A-Za-z0-9_$]*")?, }) } @@ -532,7 +533,7 @@ impl GeneratedCteState { pub(crate) fn resolve_chains_json(request_json: &str) -> Result { let request: ChainBatchRequest = serde_json::from_str(request_json).map_err(|error| error.to_string())?; - let patterns = SqlTestPatterns::new()?; + let patterns = SqlTestPatterns::new(request.lexical_syntax)?; let models: HashMap = request .models .into_iter() @@ -616,7 +617,7 @@ pub(crate) fn plan_and_render_json(request_json: &str) -> Result set_difference_operator: request.set_difference_operator, requires_derived_table_aliases: request.requires_derived_table_aliases, analysis_templates: Arc::new(Mutex::new(HashMap::new())), - patterns: SqlTestPatterns::new()?, + patterns: SqlTestPatterns::new(request.lexical_syntax)?, render_dialect, rejects_nested_with, }; @@ -1053,7 +1054,7 @@ fn resolve_assertion_textual_sql( let mut seen: HashSet = HashSet::new(); for name in marker_names( &request.patterns.reference, - &request.patterns.protected, + &request.patterns.lexical, request.assertion_sql, ) { if !seen.insert(name.clone()) { @@ -1165,7 +1166,7 @@ fn topo_sort_model_chain(request: TopoSortRequest<'_>) -> Vec { } if let Some(model) = models.get(name) { let mut dependencies = if let Some(override_sql) = overrides.get(name) { - marker_names(&patterns.reference, &patterns.protected, override_sql) + marker_names(&patterns.reference, &patterns.lexical, override_sql) } else { model.model_dependencies.clone() }; @@ -1348,7 +1349,7 @@ fn resolve_textual_sql(request: TextualResolutionRequest<'_>) -> Result) -> Result) -> Result) -> Result) -> Vec }); } }; - for name in marker_names(&patterns.reference, &patterns.protected, sql) { + for name in marker_names(&patterns.reference, &patterns.lexical, sql) { let message = format!( "test '{test_name}': model '{model_name}' references __ref('{name}') which has no mock and is not in the expected chain" ); @@ -1629,14 +1630,14 @@ fn unresolved_reference_warnings(request: UnresolvedReferenceRequest<'_>) -> Vec (&patterns.source, SOURCE_FUNCTION), (&patterns.seed, SEED_FUNCTION), ] { - for name in marker_names(pattern, &patterns.protected, sql) { + for name in marker_names(pattern, &patterns.lexical, sql) { let message = format!( "test '{test_name}': model '{model_name}' references {function_name}('{name}') which has no mock" ); warn(function_name, name, message); } } - let protected = protected_ranges(&patterns.protected, sql); + let protected = protected_ranges(&patterns.lexical, sql); for captures in patterns.dbt_reference.captures_iter(sql) { let Some(full) = captures.get(0) else { continue; @@ -1698,7 +1699,7 @@ fn assertion_ref_targets( ) -> Vec { let mut targets: Vec = Vec::new(); for (_, sql) in assertions { - targets.extend(marker_names(&patterns.reference, &patterns.protected, sql)); + targets.extend(marker_names(&patterns.reference, &patterns.lexical, sql)); } dedupe(targets) } @@ -1707,9 +1708,9 @@ fn has_unresolved_test_reference(sql: &str, patterns: &SqlTestPatterns) -> bool if !patterns.test_reference.is_match(sql) { return false; } - !marker_names(&patterns.reference, &patterns.protected, sql).is_empty() - || !marker_names(&patterns.source, &patterns.protected, sql).is_empty() - || !marker_names(&patterns.seed, &patterns.protected, sql).is_empty() + !marker_names(&patterns.reference, &patterns.lexical, sql).is_empty() + || !marker_names(&patterns.source, &patterns.lexical, sql).is_empty() + || !marker_names(&patterns.seed, &patterns.lexical, sql).is_empty() || patterns.dbt_reference.is_match(sql) || patterns.table_function.is_match(sql) } diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs b/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs index e2dc0549c6..fcfaeb3e5a 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs @@ -393,6 +393,7 @@ pub(crate) fn model_test_batch_returns_ordered_artifact() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [ { "name": "stg_orders", @@ -455,6 +456,7 @@ pub(crate) fn unicode_cte_after_leading_with_preserves_identifier() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [{ "name": "orders", "querySql": "WITH \"订单行\" AS (SELECT order_id FROM __source(\"raw_orders\")) SELECT order_id FROM \"订单行\"", @@ -499,6 +501,7 @@ pub(crate) fn unresolved_reference_fast_rejection_preserves_warning() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [{ "name": "orders", "querySql": "SELECT * FROM __SOURCE(\"missing_orders\")", @@ -588,6 +591,7 @@ pub(crate) fn shared_textual_chain_renders_each_model_once() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": models, "functions": [], "tests": [{ @@ -670,6 +674,7 @@ fn plan_deep_diamond_with_missing_mock(sql_analysis_enabled: bool) -> Value { serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": deep_diamond_models(LAYERS), "tests": [{ "name": "orders_totals", @@ -718,6 +723,7 @@ pub(crate) fn plan_without_rendering_returns_executable_steps() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [ { "name": "stg_orders", @@ -797,6 +803,7 @@ pub(crate) fn textual_assertion_with_clause_merges_lifted_ctes() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [ { "name": "stg_orders", @@ -859,6 +866,7 @@ pub(crate) fn textual_assertion_with_clause_merges_lifted_ctes() -> bool { pub(crate) fn upstream_fallback_resolves() -> bool { let request = json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [ {"name": "stg_orders", "querySql": "SELECT __udf(\"identity_value\")(order_id) AS order_id FROM __source(\"raw_orders\")"}, {"name": "orders", "querySql": "SELECT * FROM __ref(\"stg_orders\")", "modelDependencies": ["stg_orders"]} @@ -911,6 +919,7 @@ pub(crate) fn chain_resolution_orders_unmocked_models() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_chain_resolution::resolve_chains_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": deep_diamond_models(2), "tests": [ { @@ -1034,6 +1043,7 @@ pub(crate) fn long_chain_plan_output_stays_linear() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": models, "tests": [{ "name": "orders_tip", @@ -1193,6 +1203,7 @@ pub(crate) fn snowflake_plan_keeps_quoted_expected_columns() -> bool { let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [{ "name": "orders", "querySql": "SELECT order_id AS \"Order Id\", status, amount FROM __source(\"raw_orders\")", @@ -1240,6 +1251,7 @@ fn plan_helper_scope_request( ) -> Result { crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [ { "name": "orders", @@ -1368,6 +1380,7 @@ pub(crate) fn mock_read_through_helper_brings_its_mock_dependencies_into_scope() let response: Value = serde_json::from_str( &crate::compiler::main::sql_test_planning::plan_and_render_json( &json!({ + "lexicalSyntax": generic_lexical_syntax(), "models": [ {"name": "raw_orders", "querySql": "SELECT 1 AS order_id", "modelDependencies": []}, {"name": "stg_orders", "querySql": "SELECT order_id FROM __ref(\"raw_orders\")", "modelDependencies": ["raw_orders"]}, @@ -1633,3 +1646,14 @@ pub(super) fn snowflake_function_synonyms_stay_as_authored() -> bool { && !sql.contains("STARTS_WITH") && !sql.contains("SUBSTRING") } + +fn generic_lexical_syntax() -> Value { + json!({ + "backslashEscapeQuotes": [], + "escapeStringPrefix": false, + "rawStringPrefix": false, + "tripleQuotedStrings": false, + "nestedBlockComments": false, + "lineCommentPrefixes": ["--"] + }) +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/_helpers/dialect.rs b/crates/sqlbuild-rules-native/src/sql_scan/_helpers/dialect.rs new file mode 100644 index 0000000000..4d4f6ee5bb --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/_helpers/dialect.rs @@ -0,0 +1,141 @@ +//! Dialect-aware comment and quoted-text boundaries, mirroring Python `dialect_non_code_end`. + +use crate::sql_scan::_helpers::dollar::{continues_dollar_word, dollar_quote_end}; +use crate::sql_scan::_helpers::parens::matching_paren_with; +use crate::sql_scan::constants::{ + STRING_PREFIX_CHARACTERS, STRING_PREFIX_MAX_LENGTH, TRIPLE_QUOTE_LENGTH, +}; +use crate::sql_scan::models::{LexicalSyntax, Unclosed}; + +/// Return the end of a comment or quoted text starting at `index` under the dialect's rules. +pub(crate) fn non_code_end( + sql: &[u8], + index: usize, + syntax: &LexicalSyntax, +) -> Result, Unclosed> { + if sql[index..].starts_with(b"/*") { + return block_comment_end(sql, index, syntax).map(Some); + } + if syntax + .line_comment_prefixes + .iter() + .any(|prefix| sql[index..].starts_with(prefix.as_bytes())) + { + return Ok(Some( + sql[index..] + .iter() + .position(|byte| *byte == b'\n') + .map_or(sql.len(), |offset| index + offset + 1), + )); + } + match sql.get(index) { + Some(b'$') => dollar_quote_end(sql, index), + Some(b'\'' | b'"' | b'`') => quoted_text_end(sql, index, syntax).map(Some), + _ => Ok(None), + } +} + +/// Return every comment and quoted-text range in `sql`; an unclosed one extends to the end. +pub(crate) fn non_code_ranges(sql: &str, syntax: &LexicalSyntax) -> Vec<(usize, usize)> { + let bytes = sql.as_bytes(); + let mut ranges: Vec<(usize, usize)> = Vec::new(); + let mut index = 0; + while index < bytes.len() { + match non_code_end(bytes, index, syntax) { + Ok(Some(end)) => { + ranges.push((index, end)); + index = end; + } + Ok(None) => index += 1, + Err(_) => { + ranges.push((index, bytes.len())); + break; + } + } + } + ranges +} + +/// Return the index of the parenthesis closing the one at `open` under the dialect's rules. +pub(crate) fn matching_paren( + sql: &[u8], + open: usize, + syntax: &LexicalSyntax, +) -> Result { + matching_paren_with(sql, open, |sql, index| non_code_end(sql, index, syntax)) +} + +fn block_comment_end(sql: &[u8], start: usize, syntax: &LexicalSyntax) -> Result { + let mut depth = 1usize; + let mut index = start + 2; + while depth > 0 { + let close = find(sql, b"*/", index).ok_or(Unclosed::BlockComment)?; + let open = if syntax.nested_block_comments { + find(&sql[..close], b"/*", index) + } else { + None + }; + if let Some(open) = open { + depth += 1; + index = open + 2; + } else { + depth -= 1; + index = close + 2; + } + } + Ok(index) +} + +fn quoted_text_end(sql: &[u8], start: usize, syntax: &LexicalSyntax) -> Result { + let quote = sql[start]; + let prefix = string_literal_prefix(sql, start); + let mut backslash_escapes = syntax.backslash_escapes(quote) + || (syntax.escape_string_prefix && quote == b'\'' && prefix == b"e"); + if syntax.raw_string_prefix && prefix.contains(&b'r') { + backslash_escapes = false; + } + let triple = [quote; TRIPLE_QUOTE_LENGTH]; + let delimiter: &[u8] = if syntax.triple_quoted_strings && sql[start..].starts_with(&triple) { + &triple + } else { + &triple[..1] + }; + let mut index = start + delimiter.len(); + while index < sql.len() { + if backslash_escapes && sql[index] == b'\\' { + index += 2; + continue; + } + if sql[index..].starts_with(delimiter) { + if delimiter.len() == 1 && sql.get(index + 1) == Some("e) { + index += 2; + continue; + } + return Ok(index + delimiter.len()); + } + index += 1; + } + Err(Unclosed::Quote) +} + +fn string_literal_prefix(sql: &[u8], start: usize) -> Vec { + let mut prefix_start = start; + while prefix_start > 0 + && start - prefix_start < STRING_PREFIX_MAX_LENGTH + && STRING_PREFIX_CHARACTERS.contains(&sql[prefix_start - 1]) + { + prefix_start -= 1; + } + if prefix_start > 0 && continues_dollar_word(sql[prefix_start - 1]) { + return Vec::new(); + } + sql[prefix_start..start].to_ascii_lowercase() +} + +fn find(haystack: &[u8], needle: &[u8], from: usize) -> Option { + haystack + .get(from..)? + .windows(needle.len()) + .position(|window| window == needle) + .map(|offset| from + offset) +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/_helpers/dollar.rs b/crates/sqlbuild-rules-native/src/sql_scan/_helpers/dollar.rs new file mode 100644 index 0000000000..601205157d --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/_helpers/dollar.rs @@ -0,0 +1,31 @@ +//! Dollar-quoted literal boundaries shared by every scanning policy. + +use crate::sql_scan::models::Unclosed; + +/// Return the end of the `$$` or `$tag$` literal opening at `start`, if one opens there. +pub(crate) fn dollar_quote_end(sql: &[u8], start: usize) -> Result, Unclosed> { + if start + .checked_sub(1) + .is_some_and(|previous| continues_dollar_word(sql[previous])) + { + return Ok(None); + } + let tag_end = sql[start + 1..] + .iter() + .position(|byte| !(byte.is_ascii_alphanumeric() || *byte == b'_')) + .map_or(sql.len(), |offset| start + 1 + offset); + let tag = &sql[start + 1..tag_end]; + if sql.get(tag_end) != Some(&b'$') || tag.first().is_some_and(u8::is_ascii_digit) { + return Ok(None); + } + let delimiter = &sql[start..=tag_end]; + sql[tag_end + 1..] + .windows(delimiter.len()) + .position(|window| window == delimiter) + .map(|offset| Some(tag_end + 1 + offset + delimiter.len())) + .ok_or(Unclosed::Quote) +} + +pub(crate) fn continues_dollar_word(byte: u8) -> bool { + !byte.is_ascii() || byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'$') +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/sql_scan/_helpers/mod.rs new file mode 100644 index 0000000000..efa92dc9fc --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod dialect; +pub(crate) mod dollar; +pub(crate) mod parens; diff --git a/crates/sqlbuild-rules-native/src/sql_scan/_helpers/parens.rs b/crates/sqlbuild-rules-native/src/sql_scan/_helpers/parens.rs new file mode 100644 index 0000000000..525b49e34f --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/_helpers/parens.rs @@ -0,0 +1,31 @@ +//! Parenthesis matching shared by every scanning policy. + +use crate::sql_scan::models::Unclosed; + +/// Return the index of the parenthesis closing the one at `open`, skipping non-code spans. +pub(crate) fn matching_paren_with( + sql: &[u8], + open: usize, + non_code_end: impl Fn(&[u8], usize) -> Result, Unclosed>, +) -> Result { + let mut depth = 0isize; + let mut index = open; + while index < sql.len() { + if let Some(end) = non_code_end(sql, index)? { + index = end; + continue; + } + match sql[index] { + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + return Ok(index); + } + } + _ => {} + } + index += 1; + } + Err(Unclosed::Parenthesis) +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/constants.rs b/crates/sqlbuild-rules-native/src/sql_scan/constants.rs new file mode 100644 index 0000000000..b9bec2ab1a --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/constants.rs @@ -0,0 +1,8 @@ +//! Lexical vocabulary shared by SQL scanning. + +/// Characters that may prefix a string literal, such as `E'...'` or `r'...'`. +pub(crate) const STRING_PREFIX_CHARACTERS: &[u8] = b"bBeErR"; +/// Longest string-literal prefix, such as `rb`. +pub(crate) const STRING_PREFIX_MAX_LENGTH: usize = 2; +/// Length of a triple-quote string delimiter. +pub(crate) const TRIPLE_QUOTE_LENGTH: usize = 3; diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/dialect_matching_paren.rs b/crates/sqlbuild-rules-native/src/sql_scan/main/dialect_matching_paren.rs new file mode 100644 index 0000000000..687a007d86 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/main/dialect_matching_paren.rs @@ -0,0 +1,13 @@ +//! Dialect-aware parenthesis matching. + +use crate::sql_scan::_helpers::dialect::matching_paren; +use crate::sql_scan::models::{LexicalSyntax, Unclosed}; + +/// Return the index of the parenthesis closing the one at `open` under the dialect's rules. +pub(crate) fn dialect_matching_paren( + sql: &[u8], + open: usize, + syntax: &LexicalSyntax, +) -> Result { + matching_paren(sql, open, syntax) +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/dialect_non_code_ranges.rs b/crates/sqlbuild-rules-native/src/sql_scan/main/dialect_non_code_ranges.rs new file mode 100644 index 0000000000..321597f648 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/main/dialect_non_code_ranges.rs @@ -0,0 +1,9 @@ +//! Dialect-aware comment and quoted-text ranges of one SQL text. + +use crate::sql_scan::_helpers::dialect::non_code_ranges; +use crate::sql_scan::models::LexicalSyntax; + +/// Return every comment and quoted-text range in `sql`; an unclosed one extends to the end. +pub(crate) fn dialect_non_code_ranges(sql: &str, syntax: &LexicalSyntax) -> Vec<(usize, usize)> { + non_code_ranges(sql, syntax) +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/matching_paren.rs b/crates/sqlbuild-rules-native/src/sql_scan/main/matching_paren.rs index 89ee9b98e3..363e245e82 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/main/matching_paren.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/main/matching_paren.rs @@ -1,5 +1,6 @@ //! Quote- and comment-aware parenthesis matching. +use crate::sql_scan::_helpers::parens::matching_paren_with; use crate::sql_scan::main::non_code_end::non_code_end; use crate::sql_scan::models::{QuotePolicy, Unclosed}; @@ -9,24 +10,5 @@ pub(crate) fn matching_paren( open: usize, policy: QuotePolicy, ) -> Result { - let mut depth = 0isize; - let mut index = open; - while index < sql.len() { - if let Some(end) = non_code_end(sql, index, policy)? { - index = end; - continue; - } - match sql[index] { - b'(' => depth += 1, - b')' => { - depth -= 1; - if depth == 0 { - return Ok(index); - } - } - _ => {} - } - index += 1; - } - Err(Unclosed::Parenthesis) + matching_paren_with(sql, open, |sql, index| non_code_end(sql, index, policy)) } diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/mod.rs b/crates/sqlbuild-rules-native/src/sql_scan/main/mod.rs index 4bf480ac79..2d272199c3 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/main/mod.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/main/mod.rs @@ -1,4 +1,6 @@ pub(crate) mod comment_end; +pub(crate) mod dialect_matching_paren; +pub(crate) mod dialect_non_code_ranges; pub(crate) mod matching_paren; pub(crate) mod non_code_end; pub(crate) mod quote_end; diff --git a/crates/sqlbuild-rules-native/src/sql_scan/main/non_code_end.rs b/crates/sqlbuild-rules-native/src/sql_scan/main/non_code_end.rs index bf57e1b0aa..8fbd9ba3d7 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/main/non_code_end.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/main/non_code_end.rs @@ -1,5 +1,6 @@ //! Boundaries of the comment or quoted text starting at one offset. +use crate::sql_scan::_helpers::dollar::dollar_quote_end; use crate::sql_scan::main::comment_end::comment_end; use crate::sql_scan::main::quote_end::quote_end; use crate::sql_scan::models::{QuotePolicy, Unclosed}; @@ -19,31 +20,3 @@ pub(crate) fn non_code_end( _ => Ok(None), } } - -/// Return the end of the `$$` or `$tag$` literal opening at `start`, if one opens there. -fn dollar_quote_end(sql: &[u8], start: usize) -> Result, Unclosed> { - if start - .checked_sub(1) - .is_some_and(|previous| continues_dollar_word(sql[previous])) - { - return Ok(None); - } - let tag_end = sql[start + 1..] - .iter() - .position(|byte| !(byte.is_ascii_alphanumeric() || *byte == b'_')) - .map_or(sql.len(), |offset| start + 1 + offset); - let tag = &sql[start + 1..tag_end]; - if sql.get(tag_end) != Some(&b'$') || tag.first().is_some_and(u8::is_ascii_digit) { - return Ok(None); - } - let delimiter = &sql[start..=tag_end]; - sql[tag_end + 1..] - .windows(delimiter.len()) - .position(|window| window == delimiter) - .map(|offset| Some(tag_end + 1 + offset + delimiter.len())) - .ok_or(Unclosed::Quote) -} - -fn continues_dollar_word(byte: u8) -> bool { - !byte.is_ascii() || byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'$') -} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/mod.rs b/crates/sqlbuild-rules-native/src/sql_scan/mod.rs index 650595395f..2ffdf440f3 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/mod.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/mod.rs @@ -1,3 +1,5 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; pub(crate) mod main; pub(crate) mod models; diff --git a/crates/sqlbuild-rules-native/src/sql_scan/models.rs b/crates/sqlbuild-rules-native/src/sql_scan/models.rs index ee967e3b9c..fa699b1e68 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/models.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/models.rs @@ -64,3 +64,30 @@ impl QuotePolicy { } } } + +/// One adapter's rules for where quoted text and comments end, mirroring Python `SqlLexicalSyntax`. +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub(crate) struct LexicalSyntax { + /// Quote characters inside which a backslash escapes the next byte. + pub(crate) backslash_escape_quotes: Vec, + /// `E'...'` strings honour backslash escapes. + pub(crate) escape_string_prefix: bool, + /// `r'...'` strings disable backslash escapes. + pub(crate) raw_string_prefix: bool, + /// `'''...'''` and `"""..."""` delimit strings. + pub(crate) triple_quoted_strings: bool, + /// Block comments nest. + pub(crate) nested_block_comments: bool, + /// Prefixes that start a line comment, such as `--`, `#` or `//`. + pub(crate) line_comment_prefixes: Vec, +} + +impl LexicalSyntax { + /// Return whether a backslash escapes the next byte inside text opened by `quote`. + pub(crate) fn backslash_escapes(&self, quote: u8) -> bool { + self.backslash_escape_quotes + .iter() + .any(|value| value.as_bytes() == [quote]) + } +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/tests.rs b/crates/sqlbuild-rules-native/src/sql_scan/tests.rs index 0b2156d36c..ab9f7b24dc 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/tests.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/tests.rs @@ -1,3 +1,7 @@ +#[path = "tests/test_dialect_scan.rs"] +mod dialect_scan; +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_sql_scan.rs"] mod sql_scan; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-rules-native/src/sql_scan/tests/helpers.rs b/crates/sqlbuild-rules-native/src/sql_scan/tests/helpers.rs new file mode 100644 index 0000000000..3a9c34da1a --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/tests/helpers.rs @@ -0,0 +1,43 @@ +//! Shared lexical syntaxes for SQL scanning tests. + +use crate::sql_scan::models::LexicalSyntax; + +pub(crate) fn generic() -> LexicalSyntax { + LexicalSyntax { + line_comment_prefixes: vec!["--".to_string()], + ..LexicalSyntax::default() + } +} + +pub(crate) fn single_quote_backslashes() -> LexicalSyntax { + LexicalSyntax { + backslash_escape_quotes: vec!["'".to_string()], + line_comment_prefixes: vec!["--".to_string(), "//".to_string()], + ..LexicalSyntax::default() + } +} + +pub(crate) fn escape_strings() -> LexicalSyntax { + LexicalSyntax { + escape_string_prefix: true, + nested_block_comments: true, + ..generic() + } +} + +pub(crate) fn triple_quotes() -> LexicalSyntax { + LexicalSyntax { + backslash_escape_quotes: vec!["'".to_string(), "\"".to_string()], + triple_quoted_strings: true, + line_comment_prefixes: vec!["--".to_string(), "#".to_string()], + ..LexicalSyntax::default() + } +} + +pub(crate) fn raw_strings() -> LexicalSyntax { + LexicalSyntax { + backslash_escape_quotes: vec!["'".to_string()], + raw_string_prefix: true, + ..generic() + } +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/tests/test_dialect_scan.rs b/crates/sqlbuild-rules-native/src/sql_scan/tests/test_dialect_scan.rs new file mode 100644 index 0000000000..64291fbdd4 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/sql_scan/tests/test_dialect_scan.rs @@ -0,0 +1,119 @@ +use crate::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; +use crate::sql_scan::tests::helpers::{ + escape_strings, generic, raw_strings, single_quote_backslashes, triple_quotes, +}; +use crate::sql_scan::tests::test_types::{ + DialectFirstNonCodeRangeTestCase, DialectNonCodeRangesTestCase, +}; + +#[test] +fn given_dialect_lexical_syntax_when_scanning_non_code_then_dialect_decides_boundaries() { + let test_cases = [ + DialectFirstNonCodeRangeTestCase { + description: "generic sql ends a quote at a backslash-escaped quote", + sql: r"'O\'Brien' x", + syntax: generic(), + expected_first_range: Some((0, 4)), + }, + DialectFirstNonCodeRangeTestCase { + description: "backslash escapes keep the quote open", + sql: r"'O\'Brien' x", + syntax: single_quote_backslashes(), + expected_first_range: Some((0, 10)), + }, + DialectFirstNonCodeRangeTestCase { + description: "escape strings honour backslashes", + sql: r"E'O\'Brien' x", + syntax: escape_strings(), + expected_first_range: Some((1, 11)), + }, + DialectFirstNonCodeRangeTestCase { + description: "plain strings keep backslashes literal under escape-string rules", + sql: r"'C:\' x", + syntax: escape_strings(), + expected_first_range: Some((0, 5)), + }, + DialectFirstNonCodeRangeTestCase { + description: "raw strings disable backslash escapes", + sql: r"r'C:\' x", + syntax: raw_strings(), + expected_first_range: Some((1, 6)), + }, + DialectFirstNonCodeRangeTestCase { + description: "triple quotes contain single quotes", + sql: "'''it's''' x", + syntax: triple_quotes(), + expected_first_range: Some((0, 10)), + }, + DialectFirstNonCodeRangeTestCase { + description: "hash starts a line comment", + sql: "# note\nx", + syntax: triple_quotes(), + expected_first_range: Some((0, 7)), + }, + DialectFirstNonCodeRangeTestCase { + description: "slash pair starts a line comment", + sql: "// note\nx", + syntax: single_quote_backslashes(), + expected_first_range: Some((0, 8)), + }, + DialectFirstNonCodeRangeTestCase { + description: "nested block comments end at the outer close", + sql: "/* a /* b */ c */ x", + syntax: escape_strings(), + expected_first_range: Some((0, 17)), + }, + DialectFirstNonCodeRangeTestCase { + description: "flat block comments end at the first close", + sql: "/* a /* b */ c */ x", + syntax: generic(), + expected_first_range: Some((0, 12)), + }, + DialectFirstNonCodeRangeTestCase { + description: "an unclosed backslash-escaped quote extends to the end", + sql: r"'O\'", + syntax: single_quote_backslashes(), + expected_first_range: Some((0, 4)), + }, + ]; + for test_case in test_cases { + let ranges: Vec<(usize, usize)> = dialect_non_code_ranges(test_case.sql, &test_case.syntax); + assert_eq!( + ranges.first().copied(), + test_case.expected_first_range, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_quoted_text_before_a_marker_when_collecting_non_code_ranges_then_dialect_decides_coverage() +{ + let test_cases = [ + DialectNonCodeRangesTestCase { + description: "backslash escapes keep a later marker in code", + sql: r#"SELECT 'O\'Brien' AS name FROM __ref("orders")"#, + syntax: single_quote_backslashes(), + expected_marker_protected: false, + }, + DialectNonCodeRangesTestCase { + description: "generic sql reads a later marker as quoted text", + sql: r#"SELECT 'O\'Brien' AS name FROM __ref("orders")"#, + syntax: generic(), + expected_marker_protected: true, + }, + ]; + for test_case in test_cases { + let ranges: Vec<(usize, usize)> = dialect_non_code_ranges(test_case.sql, &test_case.syntax); + let marker: usize = test_case.sql.find("__ref").unwrap_or_default(); + let protected: bool = ranges + .iter() + .any(|(start, end)| marker >= *start && marker < *end); + assert_eq!( + protected, test_case.expected_marker_protected, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules-native/src/sql_scan/tests/test_types.rs b/crates/sqlbuild-rules-native/src/sql_scan/tests/test_types.rs index 0951ec3e0e..ba72c5cbfb 100644 --- a/crates/sqlbuild-rules-native/src/sql_scan/tests/test_types.rs +++ b/crates/sqlbuild-rules-native/src/sql_scan/tests/test_types.rs @@ -14,3 +14,17 @@ pub(crate) struct NonCodeEndTestCase { pub(crate) policy: QuotePolicy, pub(crate) expected_end: Result, Unclosed>, } + +pub(crate) struct DialectFirstNonCodeRangeTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) syntax: crate::sql_scan::models::LexicalSyntax, + pub(crate) expected_first_range: Option<(usize, usize)>, +} + +pub(crate) struct DialectNonCodeRangesTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) syntax: crate::sql_scan::models::LexicalSyntax, + pub(crate) expected_marker_protected: bool, +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md index e91ebf9a39..8bd778c8c2 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/testing.md @@ -101,6 +101,13 @@ The trailing `SELECT 1` is required as a ceremonial closing statement. Any CTE without one of these prefixes is treated as a helper CTE, available to mock, model, `__expected__`, and `__assert__` SQL in the test. +SQLBuild splits test files into CTEs and finds `__ref`, `__source`, `__seed`, `__dbt_ref`, +`__udf` and `__table_fn` calls outside comments and quoted text, in tests, scenarios, models, +functions, audits and hooks alike. Quoted text and comments follow the project adapter's rules: +backslash escapes on Snowflake, BigQuery and Databricks, `E'...'` strings on DuckDB and Postgres, +nested block comments on DuckDB, Postgres, Databricks and SQL Server, `#` line comments on BigQuery, +and `//` line comments on Snowflake. + An `__expected__` CTE compares only the columns it lists, matched by name, so column order does not matter and unlisted model columns are ignored. Row counts are always compared. Listing a column the model does not output is a test error. When the expected CTE's columns are not explicit diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index 063daf4ac3..bd2634be06 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -97,7 +97,6 @@ ) from sqlbuild.compiler.compile._helpers.render.templating import expand_template_data from sqlbuild.compiler.compile._helpers.sharing.binding import shareable_binding_prekeys -from sqlbuild.compiler.compile._helpers.sql_tests.core import extract_assertion_target_model_names from sqlbuild.compiler.compile._helpers.sql_tests.identity import ( build_sql_test_case_fingerprint, ) @@ -375,6 +374,7 @@ def assemble_compiled_project( effective_target_database=_str_or_none(effective_target_values.get("database")), effective_target_schema=_str_or_none(effective_target_values.get("schema")), sql_analysis_dialect=profile.sql_analysis_dialect, + sql_lexical_syntax=inputs.sql_lexical_syntax, compile_cache_dir=inputs.compile_cache_dir, settings=inputs.effective_settings, scenario=resolve_effective_scenario_config( @@ -1737,6 +1737,7 @@ def _assemble_compiled_sql_test( test_name: str = _resolve_test_name(test_input) compiled_payload: CompiledModelSqlTestPayload | CompiledDirectLogicSqlTestPayload scope_deps: tuple[CompiledObjectKey, ...] + target_model_names: tuple[str, ...] = () if isinstance(test_input.payload, CompileDirectLogicSqlTestInputPayload): if test_input.payload.mode == SqlTestMode.MACRO: scope_deps = _macro_sql_test_scope_deps( @@ -1761,14 +1762,12 @@ def _assemble_compiled_sql_test( ) else: model_payload: CompileModelSqlTestInputPayload = test_input.payload - assertion_target_model_names: tuple[str, ...] = extract_assertion_target_model_names( - assertion_sql=tuple(cte.sql_body for cte in model_payload.assertion_ctes) - ) - scope_deps = sql_test_scope_deps( - expected_model_names=tuple( - dict.fromkeys((*model_payload.expected_model_names, *assertion_target_model_names)) + target_model_names = tuple( + dict.fromkeys( + (*model_payload.expected_model_names, *model_payload.assertion_target_model_names) ) ) + scope_deps = sql_test_scope_deps(expected_model_names=target_model_names) compiled_payload = CompiledModelSqlTestPayload( authored_ctes=model_payload.authored_ctes, macro_mocks=model_payload.macro_mocks, @@ -1839,19 +1838,11 @@ def _assemble_compiled_sql_test( else () ), assertion_target_model_names=( - assertion_target_model_names - if isinstance(test_input.payload, CompileModelSqlTestInputPayload) - else () - ), - target_model_names=( - tuple( - dict.fromkeys( - (*test_input.payload.expected_model_names, *assertion_target_model_names) - ) - ) + test_input.payload.assertion_target_model_names if isinstance(test_input.payload, CompileModelSqlTestInputPayload) else () ), + target_model_names=target_model_names, tested_resources=tested_resources, ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py index bd184f1235..32fc8f463c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py @@ -50,6 +50,7 @@ DeclarationExpansionContext, LoadedMacro, MacroContext, + ModelInputBuildContext, ) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, @@ -69,6 +70,7 @@ UsageRecord, ) from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import ( SchemaAuditInstance, SchemaColumn, @@ -101,6 +103,7 @@ class _AuditAttachmentContext: effective_vars: dict[str, object] macro_context: MacroContext declaration_expansion: DeclarationExpansionContext + sql_lexical_syntax: SqlLexicalSyntax scoped_declarations: dict[tuple[Path, DeclarationIdentity], DeclarationExpansionContext] = ( field(default_factory=dict, compare=False, repr=False) ) @@ -119,19 +122,21 @@ def remember_scoped_declarations( def build_audit_inputs( *, discovered_inputs: DiscoveredProjectInputs, - effective_settings: SettingsConfig, + context: ModelInputBuildContext, model_inputs: tuple[CompileModelInput, ...], source_inputs: tuple[CompileSourceInput, ...], - effective_vars: dict[str, object], - macro_context: MacroContext, - loaded_macros: dict[str, LoadedMacro], - declaration_expansion: DeclarationExpansionContext, generic_audit_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] | None = None, seed_inputs: tuple[CompileSeedInput, ...] = (), ) -> tuple[CompileAuditInput, ...]: """Build compile-time audit inputs from discovered SQL audit blocks.""" + effective_settings: SettingsConfig = context.effective_settings + effective_vars: dict[str, object] = context.effective_vars + macro_context: MacroContext = context.macro_context + loaded_macros: dict[str, LoadedMacro] = context.loaded_macros + declaration_expansion: DeclarationExpansionContext = context.declaration_expansion + sql_lexical_syntax: SqlLexicalSyntax = context.sql_lexical_syntax known_model_names: set[str] = build_known_ref_names(discovered_inputs) known_seed_names: set[str] = build_known_seed_names(discovered_inputs) known_source_names: set[str] = build_known_source_names(discovered_inputs) @@ -150,6 +155,7 @@ def build_audit_inputs( effective_vars=effective_vars, macro_context=macro_context, declaration_expansion=declaration_expansion, + sql_lexical_syntax=sql_lexical_syntax, ) audit_inputs: list[CompileAuditInput] = [] audit_file: DiscoveredAuditFile @@ -204,7 +210,9 @@ def build_audit_inputs( context=f"Audit '{audit_block.name or audit_file.file_path.stem}' evidence", ) references: tuple[CompileSqlReference, ...] = merge_call_site_references( - references=_combined_references(expanded_sql_body, expanded_evidence_sql), + references=_combined_references( + expanded_sql_body, expanded_evidence_sql, syntax=sql_lexical_syntax + ), argument_references=_argument_references(expansion, evidence_expansion), ) validate_audit_references( @@ -629,7 +637,9 @@ def build_attached_audit_input( context=f"Audit '{audit_instance.definition_name}' evidence", ) references: tuple[CompileSqlReference, ...] = merge_call_site_references( - references=_combined_references(expanded_sql_body, expanded_evidence_sql), + references=_combined_references( + expanded_sql_body, expanded_evidence_sql, syntax=context.sql_lexical_syntax + ), argument_references=_argument_references(expansion, evidence_expansion), ) validate_audit_references( @@ -951,11 +961,13 @@ def _argument_references( return tuple(references) -def _combined_references(*sql_values: str | None) -> tuple[CompileSqlReference, ...]: +def _combined_references( + *sql_values: str | None, syntax: SqlLexicalSyntax +) -> tuple[CompileSqlReference, ...]: """Extract references from independently compiled measurement/evidence queries.""" references: list[CompileSqlReference] = [] for sql in sql_values: if sql is not None: - references.extend(extract_sql_references(sql)) + references.extend(extract_sql_references(sql=sql, syntax=syntax)) return tuple(dict.fromkeys(references)) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index 57ac4780ff..c2b273a7ae 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -153,6 +153,7 @@ UsageKind, VisibilityReason, ) +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.spec.contracts.models import ( DefaultsConfig, @@ -261,6 +262,7 @@ class _HookExpansionContext: sql_hook_definitions: dict[str, DiscoveredSqlHookFile] consumer: ResourceIdentity | DeclarationIdentity facts: _HookExpansionFacts + sql_lexical_syntax: SqlLexicalSyntax @dataclass @@ -398,7 +400,9 @@ def build_model_inputs( legacy_schema_files: tuple[DiscoveredSchemaFile, ...] = tuple( schema_file for schema_file in discovered_inputs.schema_files if schema_file.model_entries ) - with cached_sql_reference_extractor(root=reference_cache_dir) as extract_references: + with cached_sql_reference_extractor( + root=reference_cache_dir, syntax=context.sql_lexical_syntax + ) as extract_references: return _build_model_inputs( discovered_inputs=discovered_inputs, context=context, @@ -629,6 +633,7 @@ def _build_model_inputs( collection_rendering=context.collection_rendering, resolver=context.declaration_resolver, ), + sql_lexical_syntax=context.sql_lexical_syntax, sql_hook_definitions=sql_hook_definitions, consumer=model_identity, ) @@ -1233,6 +1238,7 @@ def expand_model_hook_macros_result( loaded_macros: dict[str, LoadedMacro], macro_context: MacroContext, declaration_expansion: DeclarationExpansionContext, + sql_lexical_syntax: SqlLexicalSyntax, sql_hook_definitions: dict[str, DiscoveredSqlHookFile] | None = None, consumer: ResourceIdentity | None = None, ) -> HookExpansionResult: @@ -1257,6 +1263,7 @@ def expand_model_hook_macros_result( sql_hook_definitions=sql_hook_definitions or {}, consumer=consumer or ResourceIdentity(ResourceKind.MODEL, file_path.stem), facts=facts, + sql_lexical_syntax=sql_lexical_syntax, ), hook_key=hook_key, ) @@ -1494,7 +1501,9 @@ def expand_sql_macros_in_value( facts.add(expansion.usages) facts.add_references( merge_call_site_references( - references=extract_sql_references(expansion.sql), + references=extract_sql_references( + sql=expansion.sql, syntax=context.sql_lexical_syntax + ), argument_references=expansion.argument_references, ) ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py index ff3c048668..53ffdb65c0 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py @@ -26,12 +26,14 @@ from sqlbuild.compiler.scopes.main._validate_scope_index import validate_scope_index from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax def build_declaration_scope( *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: dict[str, LoadedMacro], + sql_lexical_syntax: SqlLexicalSyntax, compile_cache_dir: Path | None = None, ) -> DeclarationScopeBuild: """Build one canonical index and validate it before SQL expansion.""" @@ -53,6 +55,7 @@ def build_declaration_scope( build_scope_relationship_grants( discovered_inputs=discovered_inputs, index=index, + sql_lexical_syntax=sql_lexical_syntax, compile_cache_dir=compile_cache_dir, ) if has_scoped_relationship_declarations diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py index 309c614fc8..014b743b52 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/functions.py @@ -58,6 +58,7 @@ from sqlbuild.compiler.scopes.models import ResourceIdentity from sqlbuild.compiler.scopes.types import ResourceKind from sqlbuild.compiler.sql_analysis.main.import_polyglot import import_polyglot +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import ( DefaultsConfig, SettingsConfig, @@ -87,15 +88,16 @@ def build_sql_function_inputs( effective_vars: dict[str, object], effective_settings: SettingsConfig, target_config: TargetConfig | None, - adapter_name: str, macro_context: MacroContext, loaded_macros: dict[str, LoadedMacro], declaration_expansion: DeclarationExpansionContext, + sql_lexical_syntax: SqlLexicalSyntax, no_sql_validation: bool = False, python_functions_inherit_default_namespace: bool = True, ) -> tuple[CompileSqlFunctionInput, ...]: """Attach and validate SQL function metadata.""" + adapter_name: str = macro_context.adapter_name known_model_names: set[str] = build_known_ref_names(discovered_inputs) known_seed_names: set[str] = build_known_seed_names(discovered_inputs) known_source_names: set[str] = build_known_source_names(discovered_inputs) @@ -226,7 +228,7 @@ def build_sql_function_inputs( function_file=function_file, ) references: tuple[CompileSqlReference, ...] = merge_call_site_references( - references=extract_sql_references(expanded_body_sql), + references=extract_sql_references(sql=expanded_body_sql, syntax=sql_lexical_syntax), argument_references=expansion.argument_references, ) validate_function_references( diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py index 32818872f0..1593bb20b8 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py @@ -45,6 +45,7 @@ VisibilityResolution, ) from sqlbuild.compiler.scopes.types import DeclarationKind, GrantKind, ResourceKind, ScopeKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax type _SharedDeclarations = dict[tuple[str, str], tuple[DeclarationRecord, ...]] @@ -53,6 +54,7 @@ def build_scope_relationship_grants( *, discovered_inputs: DiscoveredProjectInputs, index: ScopeIndex, + sql_lexical_syntax: SqlLexicalSyntax, compile_cache_dir: Path | None = None, ) -> ScopeRelationshipBuild: """Return expected-model grants while retaining independent extraction faults.""" @@ -69,11 +71,13 @@ def build_scope_relationship_grants( lookup=lookup, shared_declarations=shared_declarations, fact_cache=fact_cache, + syntax=sql_lexical_syntax, ) scenario_grants, scenario_faults = _scenario_relationship_grants( discovered_inputs=discovered_inputs, lookup=lookup, shared_declarations=shared_declarations, + syntax=sql_lexical_syntax, ) return ScopeRelationshipBuild( grants=tuple(dict.fromkeys((*test_grants, *scenario_grants))), @@ -87,12 +91,15 @@ def _test_relationship_grants( lookup: ScopeLookup, shared_declarations: _SharedDeclarations, fact_cache: FactCacheStore, + syntax: SqlLexicalSyntax, ) -> tuple[tuple[GrantRecord, ...], tuple[ScopeRelationshipFault, ...]]: grants: list[GrantRecord] = [] faults: list[ScopeRelationshipFault] = [] cache_keys: dict[int, str] = ( { - file_index: _expected_models_fact_key(test_file=test_file, fact_cache=fact_cache) + file_index: _expected_models_fact_key( + test_file=test_file, fact_cache=fact_cache, syntax=syntax + ) for file_index, test_file in enumerate(discovered_inputs.test_files) } if fact_cache.enabled @@ -119,6 +126,7 @@ def _test_relationship_grants( else extract_sql_test_expected_model_names( sql=block.sql_body, file_label=str(test_file.relative_path), + syntax=syntax, mode=block.mode, ) ) @@ -143,6 +151,7 @@ def _test_relationship_grants( ), sql=block.sql_body, file_label=str(test_file.relative_path), + syntax=syntax, ) ) except Exception as error: @@ -161,9 +170,9 @@ def _test_relationship_grants( def _expected_models_fact_key( - *, test_file: DiscoveredSqlTestFile, fact_cache: FactCacheStore + *, test_file: DiscoveredSqlTestFile, fact_cache: FactCacheStore, syntax: SqlLexicalSyntax ) -> str: - parts: list[str] = [str(test_file.relative_path)] + parts: list[str] = [str(test_file.relative_path), syntax.cache_key] for block in test_file.blocks: parts.extend((block.sql_body, block.mode.value)) return fact_cache.key(*parts) @@ -189,13 +198,14 @@ def _scenario_relationship_grants( discovered_inputs: DiscoveredProjectInputs, lookup: ScopeLookup, shared_declarations: _SharedDeclarations, + syntax: SqlLexicalSyntax, ) -> tuple[tuple[GrantRecord, ...], tuple[ScopeRelationshipFault, ...]]: grants: list[GrantRecord] = [] faults: list[ScopeRelationshipFault] = [] for scenario in discovered_inputs.scenario_files: try: expected_names: tuple[str, ...] = extract_sql_scenario_expected_model_names( - sql=scenario.sql_body, file_label=str(scenario.relative_path) + sql=scenario.sql_body, file_label=str(scenario.relative_path), syntax=syntax ) grants.extend( _expected_model_grants( @@ -288,10 +298,15 @@ def _resolve_shared_declarations( def _tested_macro_grants( - *, lookup: ScopeLookup, resource: ResourceIdentity, sql: str, file_label: str + *, + lookup: ScopeLookup, + resource: ResourceIdentity, + sql: str, + file_label: str, + syntax: SqlLexicalSyntax, ) -> list[GrantRecord]: test_ctes: tuple[CompileSqlTestCte, ...] = extract_unclassified_sql_test_ctes( - sql=sql, file_label=file_label + sql=sql, file_label=file_label, syntax=syntax ) actual_cte: CompileSqlTestCte | None = next( (cte for cte in test_ctes if cte.name == MACRO_ACTUAL_TEST_CTE_NAME), None diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py index 6482191a90..f7dc043b2d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py @@ -76,6 +76,7 @@ VisibilityRecord, ) from sqlbuild.compiler.scopes.types import ResourceKind, ScopeKind, UsageKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @dataclass(frozen=True) @@ -100,6 +101,7 @@ def build_test_inputs_with_cache( external_sql_reference_resolver: ExternalSqlReferenceResolver | None, sql_function_inputs: tuple[CompileSqlFunctionInput, ...], compile_cache_dir: Path | None, + sql_lexical_syntax: SqlLexicalSyntax, ) -> tuple[CompileSqlTestInput, ...]: """Build SQL test inputs inside the compile timing boundary.""" @@ -113,6 +115,7 @@ def build_test_inputs_with_cache( external_sql_reference_resolver=external_sql_reference_resolver, sql_function_inputs=sql_function_inputs, compile_cache_dir=compile_cache_dir, + sql_lexical_syntax=sql_lexical_syntax, ) @@ -123,6 +126,7 @@ def build_test_inputs( macro_context: MacroContext, loaded_macros: dict[str, LoadedMacro], declaration_expansion: DeclarationExpansionContext, + sql_lexical_syntax: SqlLexicalSyntax, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, sql_function_inputs: tuple[CompileSqlFunctionInput, ...] = (), compile_cache_dir: Path | None = None, @@ -230,6 +234,7 @@ def build_test_inputs( test_block=expanded_test_block, test_file=test_file, test_mode=test_mode, + syntax=sql_lexical_syntax, ) tested_resource_names = _infer_tested_direct_logic_resource_names( raw_test_ctes=raw_test_ctes, @@ -238,6 +243,7 @@ def build_test_inputs( known_function_names=known_function_names, known_table_function_names=known_table_function_names, table_function_argument_counts=table_function_argument_counts, + syntax=sql_lexical_syntax, ) expansion: AuthoredSqlExpansionResult = expand_authored_sql_result( sql=parameter_sql, @@ -284,8 +290,17 @@ def build_test_inputs( (test.sql_body, str(test.test_file.relative_path), test.mode) for test in expanded_tests ), cache_root=compile_cache_dir, + syntax=sql_lexical_syntax, ) for test, test_ctes in zip(expanded_tests, test_ctes_batch, strict=True): + assertion_target_model_names: tuple[str, ...] = ( + extract_assertion_target_model_names( + assertion_sql=tuple(cte.sql_body for cte in test_ctes.payload.assertion_ctes), + syntax=sql_lexical_syntax, + ) + if isinstance(test_ctes.payload, CompileModelSqlTestCtes) + else () + ) validate_test_ctes( test_ctes=test_ctes, test_file=test.test_file, @@ -294,6 +309,7 @@ def build_test_inputs( known_source_names=known_source_names, known_table_function_names=known_table_function_names, loaded_macros=loaded_macros, + assertion_target_model_names=assertion_target_model_names, ) test_inputs.append( CompileSqlTestInput( @@ -304,6 +320,7 @@ def build_test_inputs( payload=_build_test_input_payload( test_ctes=test_ctes, tested_resource_names=test.tested_resource_names, + assertion_target_model_names=assertion_target_model_names, ), declaration_usages=test.declaration_usages, parent_name=test.parent_name, @@ -361,7 +378,10 @@ def _macro_test_declaration_usages( def _build_test_input_payload( - *, test_ctes: CompileSqlTestCtes, tested_resource_names: tuple[str, ...] + *, + test_ctes: CompileSqlTestCtes, + tested_resource_names: tuple[str, ...], + assertion_target_model_names: tuple[str, ...], ) -> CompileModelSqlTestInputPayload | CompileDirectLogicSqlTestInputPayload: match test_ctes.payload: case CompileModelSqlTestCtes() as model_payload: @@ -377,6 +397,7 @@ def _build_test_input_payload( expected_model_names=model_payload.expected_model_names, assertion_ctes=model_payload.assertion_ctes, assertion_names=model_payload.assertion_names, + assertion_target_model_names=assertion_target_model_names, ) case CompileDirectLogicSqlTestCtes() as direct_logic_payload: return CompileDirectLogicSqlTestInputPayload( @@ -393,10 +414,12 @@ def _validate_raw_direct_logic_test_ctes( test_block: DiscoveredSqlTestBlock, test_file: DiscoveredSqlTestFile, test_mode: SqlTestMode, + syntax: SqlLexicalSyntax, ) -> CompileSqlTestCtes: return extract_sql_test_ctes( sql=test_block.sql_body, file_label=str(test_file.relative_path), + syntax=syntax, mode=test_mode, ) @@ -411,6 +434,7 @@ def _infer_tested_direct_logic_resource_names( known_function_names: set[str], known_table_function_names: set[str], table_function_argument_counts: dict[str, int], + syntax: SqlLexicalSyntax, ) -> tuple[str, ...]: if not isinstance(raw_test_ctes.payload, CompileDirectLogicSqlTestCtes): raise CompileInputError( @@ -423,6 +447,7 @@ def _infer_tested_direct_logic_resource_names( test_file=test_file, known_function_names=known_function_names, known_table_function_names=known_table_function_names, + syntax=syntax, ) if raw_test_ctes.mode == SqlTestMode.TABLE_FN: return _infer_tested_table_function_names( @@ -431,6 +456,7 @@ def _infer_tested_direct_logic_resource_names( known_function_names=known_function_names, known_table_function_names=known_table_function_names, table_function_argument_counts=table_function_argument_counts, + syntax=syntax, ) tested_macro_names: tuple[str, ...] = find_macro_call_names( raw_test_ctes.payload.actual_cte.sql_body @@ -456,6 +482,7 @@ def _infer_tested_udf_names( test_file: DiscoveredSqlTestFile, known_function_names: set[str], known_table_function_names: set[str], + syntax: SqlLexicalSyntax, ) -> tuple[str, ...]: if not isinstance(raw_test_ctes.payload, CompileDirectLogicSqlTestCtes): raise CompileInputError( @@ -463,7 +490,7 @@ def _infer_tested_udf_names( "__udf_actual__ CTE and exactly one __udf_expected__ CTE" ) references: tuple[CompileSqlReference, ...] = extract_sql_references( - raw_test_ctes.payload.actual_cte.sql_body + sql=raw_test_ctes.payload.actual_cte.sql_body, syntax=syntax ) tested_udf_names: tuple[str, ...] = tuple( dict.fromkeys( @@ -500,6 +527,7 @@ def _infer_tested_table_function_names( known_function_names: set[str], known_table_function_names: set[str], table_function_argument_counts: dict[str, int], + syntax: SqlLexicalSyntax, ) -> tuple[str, ...]: if not isinstance(raw_test_ctes.payload, CompileDirectLogicSqlTestCtes): raise CompileInputError( @@ -507,7 +535,7 @@ def _infer_tested_table_function_names( "__table_fn_actual__ CTE and exactly one __table_fn_expected__ CTE" ) references: tuple[CompileSqlReference, ...] = extract_sql_references( - raw_test_ctes.payload.actual_cte.sql_body + sql=raw_test_ctes.payload.actual_cte.sql_body, syntax=syntax ) validate_table_function_reference_arities( references=references, @@ -550,6 +578,7 @@ def build_scenario_inputs( macro_context: MacroContext, loaded_macros: dict[str, LoadedMacro], declaration_expansion: DeclarationExpansionContext, + sql_lexical_syntax: SqlLexicalSyntax, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, ) -> tuple[CompileSqlScenarioInput, ...]: """Build compile-time scenario inputs from discovered SQL-native scenario files.""" @@ -590,14 +619,17 @@ def build_scenario_inputs( scenario_ctes: CompileSqlScenarioCtes = extract_sql_scenario_ctes( sql=expanded_sql_body, file_label=str(scenario_file.relative_path), + syntax=sql_lexical_syntax, ) _validate_scenario_source_references( scenario_ctes=scenario_ctes, scenario_file=scenario_file, known_source_names=known_source_names, + syntax=sql_lexical_syntax, ) assertion_target_model_names: tuple[str, ...] = extract_assertion_target_model_names( - assertion_sql=tuple(cte.sql_body for cte in scenario_ctes.assertion_ctes) + assertion_sql=tuple(cte.sql_body for cte in scenario_ctes.assertion_ctes), + syntax=sql_lexical_syntax, ) scenario_inputs.append( CompileSqlScenarioInput( @@ -629,10 +661,13 @@ def _validate_scenario_source_references( scenario_ctes: CompileSqlScenarioCtes, scenario_file: DiscoveredSqlScenarioFile, known_source_names: set[str], + syntax: SqlLexicalSyntax, ) -> None: cte: CompileSqlScenarioCte for cte in (*scenario_ctes.expected_ctes, *scenario_ctes.assertion_ctes): - references: tuple[CompileSqlReference, ...] = extract_sql_references(cte.sql_body) + references: tuple[CompileSqlReference, ...] = extract_sql_references( + sql=cte.sql_body, syntax=syntax + ) reference: CompileSqlReference for reference in references: if reference.ref_kind != SqlReferenceKind.SOURCE: @@ -645,7 +680,7 @@ def _validate_scenario_source_references( ) for cte in scenario_ctes.authored_ctes: - references = extract_sql_references(cte.sql_body) + references = extract_sql_references(sql=cte.sql_body, syntax=syntax) for reference in references: if reference.ref_kind != SqlReferenceKind.SOURCE: continue @@ -666,6 +701,7 @@ def validate_test_ctes( known_source_names: set[str], known_table_function_names: set[str], loaded_macros: dict[str, LoadedMacro], + assertion_target_model_names: tuple[str, ...], ) -> None: """Validate SQL-native test CTE targets against discovered inputs.""" @@ -714,9 +750,7 @@ def validate_test_ctes( ) assertion_target_name: str - for assertion_target_name in extract_assertion_target_model_names( - assertion_sql=tuple(cte.sql_body for cte in model_payload.assertion_ctes) - ): + for assertion_target_name in assertion_target_model_names: if assertion_target_name not in known_model_names: raise CompileInputError( f"SQL test file {test_file.relative_path} assertion references unknown model " diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py index dfa069eb6c..6965afd73c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py +++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/macro_arguments.py @@ -22,6 +22,7 @@ from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.scopes.models import DeclarationIdentity, ResourceIdentity from sqlbuild.compiler.scopes.types import ResourceKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind @@ -49,6 +50,8 @@ rf"{_RELATION_PLACEHOLDER_PREFIX}(\d+)__" ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + def relation_placeholder_text(index: int) -> str: """Return the in-expansion placeholder text for the typed reference at ``index``.""" @@ -80,7 +83,9 @@ def call_site_sql_references( if not refs: return () - return extract_sql_references(" ".join(reference_call_text(ref) for ref in refs)) + return extract_sql_references( + sql=" ".join(reference_call_text(ref) for ref in refs), syntax=_GENERIC_SQL_SYNTAX + ) def resource_references( @@ -165,7 +170,9 @@ def reject_macro_generated_references( def _generated_references(sql: str) -> tuple[SqlResourceRef, ...]: try: - references: tuple[CompileSqlReference, ...] = extract_sql_references(sql) + references: tuple[CompileSqlReference, ...] = extract_sql_references( + sql=sql, syntax=_GENERIC_SQL_SYNTAX + ) except CompileInputError: return () generated: dict[SqlResourceRef, None] = {} diff --git a/src/sqlbuild/compiler/compile/_helpers/refs/cache.py b/src/sqlbuild/compiler/compile/_helpers/refs/cache.py index 29c10e8a3f..c6241d5e72 100644 --- a/src/sqlbuild/compiler/compile/_helpers/refs/cache.py +++ b/src/sqlbuild/compiler/compile/_helpers/refs/cache.py @@ -16,6 +16,7 @@ from sqlbuild.compiler.compile.exceptions import AnalysisCacheEntryError from sqlbuild.compiler.compile.models import CompileSqlReference from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax _REFERENCE_CACHE_VERSION: int = 2 _CACHE_DATABASE_NAME: str = "sql-references.sqlite3" @@ -33,8 +34,10 @@ class _SqlReferenceCache: """Reuse exact SQL reference facts while preserving safe scanner fallback.""" - def __init__(self, *, root: Path | None) -> None: + def __init__(self, *, root: Path | None, syntax: SqlLexicalSyntax) -> None: self._root: Path | None = root + self._syntax: SqlLexicalSyntax = syntax + self._syntax_key: str = syntax.cache_key self._database_path: Path | None = None self._connection: sqlite3.Connection | None = None self._pending_contents_by_key: dict[str, str] = {} @@ -83,7 +86,7 @@ def __exit__( def references(self, sql: str) -> tuple[CompileSqlReference, ...]: """Return cached references or scan and record this exact expanded SQL.""" - cache_key: str = _reference_cache_key(sql) + cache_key: str = _reference_cache_key(sql=sql, syntax_key=self._syntax_key) pending_contents: str | None = self._pending_contents_by_key.get(cache_key) if pending_contents is not None: pending_references: tuple[CompileSqlReference, ...] | None = _references_from_contents( @@ -109,7 +112,9 @@ def references(self, sql: str) -> tuple[CompileSqlReference, ...]: self._disable() connection = None - references: tuple[CompileSqlReference, ...] = extract_sql_references(sql) + references: tuple[CompileSqlReference, ...] = extract_sql_references( + sql=sql, syntax=self._syntax + ) if self._database_path is not None: try: contents: str = _reference_contents( @@ -150,8 +155,11 @@ def _disable(self) -> None: pass -def _reference_cache_key(sql: str) -> str: - return hashlib.sha256(sql.encode()).hexdigest() +def _reference_cache_key(*, sql: str, syntax_key: str) -> str: + digest: Any = hashlib.sha256(syntax_key.encode()) + digest.update(b"\0") + digest.update(sql.encode()) + return digest.hexdigest() def _reference_contents(*, cache_key: str, references: tuple[CompileSqlReference, ...]) -> str: @@ -247,9 +255,9 @@ def _cache_entry_digest(*, cache_key: str, serialized_payload: str) -> str: @contextmanager def cached_sql_reference_extractor( - *, root: Path | None + *, root: Path | None, syntax: SqlLexicalSyntax ) -> Iterator[Callable[[str], tuple[CompileSqlReference, ...]]]: - """Yield an exact cached reference extractor for one compile invocation.""" + """Yield an exact cached reference extractor for one compile invocation and dialect.""" - with _SqlReferenceCache(root=root) as cache: + with _SqlReferenceCache(root=root, syntax=syntax) as cache: yield cache.references diff --git a/src/sqlbuild/compiler/compile/_helpers/refs/references.py b/src/sqlbuild/compiler/compile/_helpers/refs/references.py index 255f6b19c9..9671450f92 100644 --- a/src/sqlbuild/compiler/compile/_helpers/refs/references.py +++ b/src/sqlbuild/compiler/compile/_helpers/refs/references.py @@ -16,9 +16,8 @@ from sqlbuild.compiler.compile.models import CompileSqlReference from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.main._find_matching_paren import find_matching_paren -from sqlbuild.compiler.sql_analysis.main._skip_block_comment import skip_block_comment -from sqlbuild.compiler.sql_analysis.main._skip_line_comment import skip_line_comment -from sqlbuild.compiler.sql_analysis.main._skip_quoted_text import skip_quoted_text +from sqlbuild.compiler.sql_analysis.main._skip_dialect_non_code import dialect_non_code_end +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax _CONTEXT: str = "SQL reference" _DOUBLE_QUOTE_TOKEN: str = '"' @@ -34,12 +33,16 @@ _REFERENCE_PREFIX_BY_KIND: dict[SqlReferenceKind, str] = { ref_kind: prefix[:-1] for prefix, ref_kind in _REFERENCE_PREFIXES } -_REFERENCE_SCAN_PATTERN: re.Pattern[str] = re.compile(r"__|--|/\*|'|\"|`|\$") +_REFERENCE_SCAN_PATTERN: re.Pattern[str] = re.compile(r"__|--|/\*|//|#|'|\"|`|\$") -def extract_sql_references(sql: str) -> tuple[CompileSqlReference, ...]: - """Return logical SQL refs found outside comments and quoted text.""" +def extract_sql_references( + *, sql: str, syntax: SqlLexicalSyntax +) -> tuple[CompileSqlReference, ...]: + """Return logical SQL refs found outside comments and quoted text under the dialect's rules.""" + if syntax.reads_differently_from_generic(sql): + return _extract_sql_references_with_python(sql=sql, syntax=syntax) native_references: list[tuple[str, str, str | None, int | None]] | None = ( _native.extract_static_sql_references(sql) ) @@ -53,10 +56,12 @@ def extract_sql_references(sql: str) -> tuple[CompileSqlReference, ...]: ) for kind, name, package, call_argument_count in native_references ) - return _extract_sql_references_with_python(sql) + return _extract_sql_references_with_python(sql=sql, syntax=syntax) -def _extract_sql_references_with_python(sql: str) -> tuple[CompileSqlReference, ...]: +def _extract_sql_references_with_python( + *, sql: str, syntax: SqlLexicalSyntax +) -> tuple[CompileSqlReference, ...]: """Return logical SQL refs through the authoritative general scanner.""" references: list[CompileSqlReference] = [] @@ -66,19 +71,17 @@ def _extract_sql_references_with_python(sql: str) -> tuple[CompileSqlReference, index = _next_reference_scan_position(sql=sql, start=index) if index >= length: break - if sql.startswith("--", index): - index = skip_line_comment(sql=sql, start=index) - continue - if sql.startswith("/*", index): - index = skip_block_comment(sql=sql, start=index, context=_CONTEXT) - continue - if sql[index] in SQL_QUOTE_TOKENS: - index = skip_quoted_text(sql=sql, start=index, context=_CONTEXT) + non_code_end: int | None = dialect_non_code_end( + sql=sql, start=index, syntax=syntax, context=_CONTEXT + ) + if non_code_end is not None: + index = non_code_end continue parsed_reference: tuple[CompileSqlReference, int] | None = _parse_reference_at( sql=sql, start=index, + syntax=syntax, ) if parsed_reference is None: index += 1 @@ -93,7 +96,9 @@ def _next_reference_scan_position(*, sql: str, start: int) -> int: return match.start() if match is not None else len(sql) -def _parse_reference_at(*, sql: str, start: int) -> tuple[CompileSqlReference, int] | None: +def _parse_reference_at( + *, sql: str, start: int, syntax: SqlLexicalSyntax +) -> tuple[CompileSqlReference, int] | None: ref_kind: SqlReferenceKind | None = None prefix: str for prefix, candidate_kind in _REFERENCE_PREFIXES: @@ -105,10 +110,12 @@ def _parse_reference_at(*, sql: str, start: int) -> tuple[CompileSqlReference, i open_paren_index: int = start + len(_REFERENCE_PREFIX_BY_KIND[ref_kind]) closing_paren_index: int = find_matching_paren( - sql=sql, open_paren_index=open_paren_index, context=_CONTEXT + sql=sql, open_paren_index=open_paren_index, context=_CONTEXT, syntax=syntax ) raw_arguments: str = sql[open_paren_index + 1 : closing_paren_index] - argument_values: tuple[str, ...] = _split_top_level_arguments(raw_arguments) + argument_values: tuple[str, ...] = _split_top_level_arguments( + raw_arguments=raw_arguments, syntax=syntax + ) two_argument_count: int = 2 if ref_kind == SqlReferenceKind.DBT_REF: if len(argument_values) not in {1, two_argument_count}: @@ -137,9 +144,12 @@ def _parse_reference_at(*, sql: str, start: int) -> tuple[CompileSqlReference, i sql=sql, open_paren_index=call_suffix_start, context="SQL table function call", + syntax=syntax, ) call_argument_count = len( - _split_top_level_arguments(sql[call_suffix_start + 1 : call_suffix_end]) + _split_top_level_arguments( + raw_arguments=sql[call_suffix_start + 1 : call_suffix_end], syntax=syntax + ) ) return ( CompileSqlReference( @@ -163,27 +173,25 @@ def _skip_whitespace(*, sql: str, start: int) -> int: return index -def _split_top_level_arguments(raw_arguments: str) -> tuple[str, ...]: +def _split_top_level_arguments(*, raw_arguments: str, syntax: SqlLexicalSyntax) -> tuple[str, ...]: arguments: list[str] = [] current: list[str] = [] depth: int = 0 index: int = 0 saw_separator: bool = False while index < len(raw_arguments): - if raw_arguments.startswith("--", index): - index = skip_line_comment(sql=raw_arguments, start=index) - current.append(" ") - continue - if raw_arguments.startswith("/*", index): - index = skip_block_comment(sql=raw_arguments, start=index, context=_CONTEXT) - current.append(" ") + non_code_end: int | None = dialect_non_code_end( + sql=raw_arguments, start=index, syntax=syntax, context=_CONTEXT + ) + if non_code_end is not None: + current.append( + raw_arguments[index:non_code_end] + if raw_arguments[index] in SQL_QUOTE_TOKENS + else " " + ) + index = non_code_end continue character: str = raw_arguments[index] - if character in SQL_QUOTE_TOKENS: - quoted_end: int = skip_quoted_text(sql=raw_arguments, start=index, context=_CONTEXT) - current.append(raw_arguments[index:quoted_end]) - index = quoted_end - continue if character == SQL_OPEN_PAREN_TOKEN: depth += 1 elif character == SQL_CLOSE_PAREN_TOKEN: diff --git a/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py b/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py index 111313f063..144cb43faf 100644 --- a/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/scenarios/core.py @@ -27,6 +27,7 @@ CompileSqlScenarioCte, CompileSqlScenarioCtes, ) +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax _CONTEXT: str = "SQL scenario" _WITH_REQUIREMENT: str = ( @@ -34,7 +35,9 @@ ) -def extract_sql_scenario_ctes(*, sql: str, file_label: str) -> CompileSqlScenarioCtes: +def extract_sql_scenario_ctes( + *, sql: str, file_label: str, syntax: SqlLexicalSyntax +) -> CompileSqlScenarioCtes: """Extract top-level SQL-native scenario fixture, expected, and assertion CTEs.""" try: @@ -44,6 +47,7 @@ def extract_sql_scenario_ctes(*, sql: str, file_label: str) -> CompileSqlScenari context_label=_CONTEXT, with_requirement=_WITH_REQUIREMENT, cte_type=CompileSqlScenarioCte, + syntax=syntax, ) except CompileInputError as scanner_error: cte_values: tuple[tuple[str, str], ...] | None = extract_top_level_ctes_with_sql_analysis( @@ -54,13 +58,15 @@ def extract_sql_scenario_ctes(*, sql: str, file_label: str) -> CompileSqlScenari if cte_values is None: raise scanner_error from None ctes = tuple(CompileSqlScenarioCte(name=name, sql_body=body) for name, body in cte_values) - return _classify_sql_scenario_ctes(ctes=ctes, file_label=file_label) + return _classify_sql_scenario_ctes(ctes=ctes, file_label=file_label, syntax=syntax) -def extract_sql_scenario_expected_model_names(*, sql: str, file_label: str) -> tuple[str, ...]: +def extract_sql_scenario_expected_model_names( + *, sql: str, file_label: str, syntax: SqlLexicalSyntax +) -> tuple[str, ...]: """Extract explicit expected-model relationships without inspecting CTE bodies.""" - start: int = _skip_ignorable(sql=sql, start=0, context_label=_CONTEXT) + start: int = _skip_ignorable(sql=sql, start=0, context_label=_CONTEXT, syntax=syntax) if _try_consume_keyword(sql=sql, start=start, keyword=SQL_WITH_KEYWORD) is None: return () ctes: tuple[CompileSqlScenarioCte, ...] = extract_top_level_ctes_with_scanner( @@ -69,6 +75,7 @@ def extract_sql_scenario_expected_model_names(*, sql: str, file_label: str) -> t context_label=_CONTEXT, with_requirement=_WITH_REQUIREMENT, cte_type=CompileSqlScenarioCte, + syntax=syntax, ) return tuple( _require_prefixed_name( @@ -84,7 +91,7 @@ def extract_sql_scenario_expected_model_names(*, sql: str, file_label: str) -> t def _classify_sql_scenario_ctes( - *, ctes: tuple[CompileSqlScenarioCte, ...], file_label: str + *, ctes: tuple[CompileSqlScenarioCte, ...], file_label: str, syntax: SqlLexicalSyntax ) -> CompileSqlScenarioCtes: validate_independent_expected_and_assertion_ctes( ctes=ctes, @@ -92,6 +99,7 @@ def _classify_sql_scenario_ctes( assertion_prefix=ASSERT_SCENARIO_CTE_PREFIX, file_label=file_label, context_label="SQL scenario", + syntax=syntax, ) authored_ctes: list[CompileSqlScenarioCte] = [] expected_ctes: list[CompileSqlScenarioCte] = [] diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py index 687955d01b..045bc918f2 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/core.py @@ -31,6 +31,7 @@ SQL_ARGUMENT_SEPARATOR_TOKEN, SQL_CEREMONIAL_SELECT_VALUE, SQL_OPEN_PAREN_TOKEN, + SQL_QUOTE_TOKENS, SQL_SINGLE_QUOTE_TOKEN, SQL_STATEMENT_TERMINATOR_TOKEN, SQL_WILDCARD_TOKEN, @@ -58,21 +59,17 @@ ) from sqlbuild.compiler.sql_analysis.main._is_identifier_start import is_identifier_start from sqlbuild.compiler.sql_analysis.main._iter_code_positions import iter_code_positions -from sqlbuild.compiler.sql_analysis.main._skip_block_comment import skip_block_comment -from sqlbuild.compiler.sql_analysis.main._skip_line_comment import skip_line_comment -from sqlbuild.compiler.sql_analysis.main._skip_quoted_text import ( - skip_quoted_text, -) +from sqlbuild.compiler.sql_analysis.main._skip_dialect_non_code import dialect_non_code_end from sqlbuild.compiler.sql_analysis.main._split_set_operation_branches import ( split_set_operation_branches, ) from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax _CONTEXT: str = "SQL test" _SQL_TEST_WITH_REQUIREMENT: str = "mock CTEs and one __expected__ CTE" _DIRECT_DEPENDENCY_PATH_LENGTH: int = 2 _SQL_IDENTIFIER_QUOTE_TOKENS: frozenset[str] = frozenset({'"', "`"}) -_SQL_STRING_QUOTE_TOKENS: frozenset[str] = frozenset({SQL_SINGLE_QUOTE_TOKEN, "$"}) @dataclass(frozen=True) @@ -121,19 +118,22 @@ class _DirectLogicModeSpec: def extract_sql_test_ctes( - *, sql: str, file_label: str, mode: SqlTestMode = DEFAULT_SQL_TEST_MODE + *, + sql: str, + file_label: str, + syntax: SqlLexicalSyntax, + mode: SqlTestMode = DEFAULT_SQL_TEST_MODE, ) -> CompileSqlTestCtes: """Extract top-level SQL-native test mock and expected CTEs.""" ctes: tuple[CompileSqlTestCte, ...] = extract_unclassified_sql_test_ctes( - sql=sql, - file_label=file_label, + sql=sql, file_label=file_label, syntax=syntax ) - return classify_sql_test_ctes(ctes=ctes, file_label=file_label, mode=mode) + return classify_sql_test_ctes(ctes=ctes, file_label=file_label, mode=mode, syntax=syntax) def extract_unclassified_sql_test_ctes( - *, sql: str, file_label: str + *, sql: str, file_label: str, syntax: SqlLexicalSyntax ) -> tuple[CompileSqlTestCte, ...]: """Extract raw top-level CTEs before mode-specific classification.""" @@ -144,6 +144,7 @@ def extract_unclassified_sql_test_ctes( context_label=_CONTEXT, with_requirement=_SQL_TEST_WITH_REQUIREMENT, cte_type=CompileSqlTestCte, + syntax=syntax, ) except CompileInputError as scanner_error: cte_values: tuple[tuple[str, str], ...] | None = extract_top_level_ctes_with_sql_analysis( @@ -158,21 +159,29 @@ def extract_unclassified_sql_test_ctes( def classify_sql_test_ctes( - *, ctes: tuple[CompileSqlTestCte, ...], file_label: str, mode: SqlTestMode + *, + ctes: tuple[CompileSqlTestCte, ...], + file_label: str, + mode: SqlTestMode, + syntax: SqlLexicalSyntax, ) -> CompileSqlTestCtes: """Apply mode-specific validation to extracted SQL test CTEs.""" - return _classify_sql_test_ctes(ctes=ctes, file_label=file_label, mode=mode) + return _classify_sql_test_ctes(ctes=ctes, file_label=file_label, mode=mode, syntax=syntax) def extract_sql_test_expected_model_names( - *, sql: str, file_label: str, mode: SqlTestMode = DEFAULT_SQL_TEST_MODE + *, + sql: str, + file_label: str, + syntax: SqlLexicalSyntax, + mode: SqlTestMode = DEFAULT_SQL_TEST_MODE, ) -> tuple[str, ...]: """Extract explicit expected-model relationships without inspecting CTE bodies.""" if mode is not SqlTestMode.MODEL: return () - start: int = _skip_ignorable(sql=sql, start=0) + start: int = _skip_ignorable(sql=sql, start=0, syntax=syntax) if _try_consume_keyword(sql=sql, start=start, keyword=SQL_WITH_KEYWORD) is None: return () ctes: tuple[CompileSqlTestCte, ...] = extract_top_level_ctes_with_scanner( @@ -181,6 +190,7 @@ def extract_sql_test_expected_model_names( context_label=_CONTEXT, with_requirement=_SQL_TEST_WITH_REQUIREMENT, cte_type=CompileSqlTestCte, + syntax=syntax, ) return tuple( _require_prefixed_name( @@ -194,14 +204,16 @@ def extract_sql_test_expected_model_names( ) -def extract_assertion_target_model_names(*, assertion_sql: tuple[str, ...]) -> tuple[str, ...]: +def extract_assertion_target_model_names( + *, assertion_sql: tuple[str, ...], syntax: SqlLexicalSyntax +) -> tuple[str, ...]: """Extract assertion model targets in authored order using canonical references.""" targets: list[str] = [] for sql in assertion_sql: targets.extend( reference.ref_name - for reference in extract_sql_references(sql) + for reference in extract_sql_references(sql=sql, syntax=syntax) if reference.ref_kind == SqlReferenceKind.REF ) return tuple(dict.fromkeys(targets)) @@ -215,22 +227,27 @@ def extract_top_level_ctes_with_scanner[CteT]( context_label: str, with_requirement: str, cte_type: Callable[..., CteT], + syntax: SqlLexicalSyntax, ) -> tuple[CteT, ...]: """Scan top-level `WITH` CTEs followed by the ceremonial `SELECT 1` of a test file.""" with_end: int | None = _try_consume_keyword( sql=sql, - start=_skip_ignorable(sql=sql, start=0, context_label=context_label), + start=_skip_ignorable(sql=sql, start=0, context_label=context_label, syntax=syntax), keyword=SQL_WITH_KEYWORD, ) if with_end is None: raise CompileInputError( f"{context_label} '{file_label}' must declare {with_requirement} before `SELECT 1`" ) - index: int = _skip_ignorable(sql=sql, start=with_end, context_label=context_label) + index: int = _skip_ignorable( + sql=sql, start=with_end, context_label=context_label, syntax=syntax + ) recursive_end: int | None = _try_consume_keyword(sql=sql, start=index, keyword="RECURSIVE") if recursive_end is not None: - index = _skip_ignorable(sql=sql, start=recursive_end, context_label=context_label) + index = _skip_ignorable( + sql=sql, start=recursive_end, context_label=context_label, syntax=syntax + ) ctes: list[CteT] = [] seen_cte_names: set[str] = set() @@ -244,10 +261,17 @@ def extract_top_level_ctes_with_scanner[CteT]( ) seen_cte_names.add(cte_name) - index = _skip_ignorable(sql=sql, start=index, context_label=context_label) + index = _skip_ignorable(sql=sql, start=index, context_label=context_label, syntax=syntax) if index < len(sql) and sql[index] == SQL_OPEN_PAREN_TOKEN: - index = find_matching_paren(sql=sql, open_paren_index=index, context=context_label) + 1 - index = _skip_ignorable(sql=sql, start=index, context_label=context_label) + index = ( + find_matching_paren( + sql=sql, open_paren_index=index, context=context_label, syntax=syntax + ) + + 1 + ) + index = _skip_ignorable( + sql=sql, start=index, context_label=context_label, syntax=syntax + ) index = _consume_keyword( sql=sql, start=index, @@ -255,19 +279,23 @@ def extract_top_level_ctes_with_scanner[CteT]( file_label=file_label, context_label=context_label, ) - index = _skip_ignorable(sql=sql, start=index, context_label=context_label) + index = _skip_ignorable(sql=sql, start=index, context_label=context_label, syntax=syntax) if index >= len(sql) or sql[index] != SQL_OPEN_PAREN_TOKEN: raise CompileInputError( f"{context_label} '{file_label}' CTE '{cte_name}' must use AS (...)" ) cte_body_start: int = index + 1 cte_body_end: int = find_matching_paren( - sql=sql, open_paren_index=index, context=context_label + sql=sql, open_paren_index=index, context=context_label, syntax=syntax ) ctes.append(cte_type(name=cte_name, sql_body=sql[cte_body_start:cte_body_end].strip())) - index = _skip_ignorable(sql=sql, start=cte_body_end + 1, context_label=context_label) + index = _skip_ignorable( + sql=sql, start=cte_body_end + 1, context_label=context_label, syntax=syntax + ) if index < len(sql) and sql[index] == SQL_ARGUMENT_SEPARATOR_TOKEN: - index = _skip_ignorable(sql=sql, start=index + 1, context_label=context_label) + index = _skip_ignorable( + sql=sql, start=index + 1, context_label=context_label, syntax=syntax + ) continue break @@ -276,26 +304,38 @@ def extract_top_level_ctes_with_scanner[CteT]( start=index, file_label=file_label, context_label=context_label, + syntax=syntax, ) return tuple(ctes) def _classify_sql_test_ctes( - *, ctes: tuple[CompileSqlTestCte, ...], file_label: str, mode: SqlTestMode + *, + ctes: tuple[CompileSqlTestCte, ...], + file_label: str, + mode: SqlTestMode, + syntax: SqlLexicalSyntax, ) -> CompileSqlTestCtes: match mode: case SqlTestMode.MODEL: - return _classify_model_sql_test_ctes(ctes=ctes, file_label=file_label) + return _classify_model_sql_test_ctes(ctes=ctes, file_label=file_label, syntax=syntax) case SqlTestMode.MACRO | SqlTestMode.UDF | SqlTestMode.TABLE_FN: return _classify_direct_logic_sql_test_ctes( - ctes=ctes, file_label=file_label, spec=_DIRECT_LOGIC_MODE_SPECS[mode] + ctes=ctes, + file_label=file_label, + spec=_DIRECT_LOGIC_MODE_SPECS[mode], + syntax=syntax, ) case _: raise CompileInputError(f"SQL test '{file_label}' has unsupported mode '{mode}'") def _classify_direct_logic_sql_test_ctes( - *, ctes: tuple[CompileSqlTestCte, ...], file_label: str, spec: _DirectLogicModeSpec + *, + ctes: tuple[CompileSqlTestCte, ...], + file_label: str, + spec: _DirectLogicModeSpec, + syntax: SqlLexicalSyntax, ) -> CompileSqlTestCtes: mode: str = spec.mode.value authored_ctes: list[CompileSqlTestCte] = [] @@ -318,7 +358,9 @@ def _classify_direct_logic_sql_test_ctes( f"SQL test '{file_label}' mode '{mode}' must define exactly one " f"{spec.expected_cte_name} CTE" ) - _validate_expected_cte_query(cte=cte, file_label=file_label, label=cte.name) + _validate_expected_cte_query( + cte=cte, file_label=file_label, label=cte.name, syntax=syntax + ) expected_cte = cte continue if _is_model_mode_cte(cte.name): @@ -356,6 +398,7 @@ def _classify_direct_logic_sql_test_ctes( file_label=file_label, mode=spec.mode, actual_cte_name=spec.actual_cte_name, + syntax=syntax, ) return CompileSqlTestCtes( mode=spec.mode, @@ -369,7 +412,7 @@ def _classify_direct_logic_sql_test_ctes( def _classify_model_sql_test_ctes( - *, ctes: tuple[CompileSqlTestCte, ...], file_label: str + *, ctes: tuple[CompileSqlTestCte, ...], file_label: str, syntax: SqlLexicalSyntax ) -> CompileSqlTestCtes: validate_independent_expected_and_assertion_ctes( ctes=ctes, @@ -377,6 +420,7 @@ def _classify_model_sql_test_ctes( assertion_prefix=ASSERT_TEST_CTE_PREFIX, file_label=file_label, context_label="SQL test", + syntax=syntax, ) authored_ctes: list[CompileSqlTestCte] = [] macro_mocks: dict[str, str] = {} @@ -414,7 +458,9 @@ def _classify_model_sql_test_ctes( label="__macro__", file_label=file_label, ) - macro_mocks[macro_name] = _extract_macro_mock_value(cte=cte, file_label=file_label) + macro_mocks[macro_name] = _extract_macro_mock_value( + cte=cte, file_label=file_label, syntax=syntax + ) continue if cte.name.startswith(REF_TEST_CTE_PREFIX): mock_model_names.append( @@ -484,6 +530,7 @@ def _classify_model_sql_test_ctes( cte=cte, file_label=file_label, label=cte.name, + syntax=syntax, allow_empty_fixture=True, ) expected_ctes.append(cte) @@ -583,6 +630,7 @@ def _validate_call_free_direct_logic_bodies( file_label: str, mode: SqlTestMode, actual_cte_name: str, + syntax: SqlLexicalSyntax, ) -> None: helper_cte: CompileSqlTestCte for helper_cte in helper_ctes: @@ -592,6 +640,7 @@ def _validate_call_free_direct_logic_bodies( mode=mode, cte_label=f"helper CTE '{helper_cte.name}'", allowed_location=actual_cte_name, + syntax=syntax, ) _validate_no_direct_logic_calls( sql=expected_cte.sql_body, @@ -599,18 +648,25 @@ def _validate_call_free_direct_logic_bodies( mode=mode, cte_label=f"CTE {expected_cte.name}", allowed_location=actual_cte_name, + syntax=syntax, ) def _validate_no_direct_logic_calls( - *, sql: str, file_label: str, mode: SqlTestMode, cte_label: str, allowed_location: str + *, + sql: str, + file_label: str, + mode: SqlTestMode, + cte_label: str, + allowed_location: str, + syntax: SqlLexicalSyntax, ) -> None: macro_names: tuple[str, ...] = find_macro_call_names(sql) if macro_names: raise CompileInputError( f"SQL test '{file_label}' mode '{mode.value}' {cte_label} must not call macros" ) - references: tuple[CompileSqlReference, ...] = extract_sql_references(sql) + references: tuple[CompileSqlReference, ...] = extract_sql_references(sql=sql, syntax=syntax) reference: CompileSqlReference | None = next( ( item @@ -627,13 +683,15 @@ def _validate_no_direct_logic_calls( ) -def _extract_macro_mock_value(*, cte: CompileSqlTestCte, file_label: str) -> str: +def _extract_macro_mock_value( + *, cte: CompileSqlTestCte, file_label: str, syntax: SqlLexicalSyntax +) -> str: """Extract the single SQL string literal value from a __macro__ CTE.""" body: str = cte.sql_body.strip() - index: int = _skip_ignorable(sql=body, start=0) + index: int = _skip_ignorable(sql=body, start=0, syntax=syntax) index = _consume_keyword(sql=body, start=index, keyword="SELECT", file_label=file_label) - index = _skip_ignorable(sql=body, start=index) + index = _skip_ignorable(sql=body, start=index, syntax=syntax) if index >= len(body) or body[index] != SQL_SINGLE_QUOTE_TOKEN: raise CompileInputError( f"SQL test '{file_label}' macro mock '{cte.name}' must be a single SELECT string " @@ -641,9 +699,9 @@ def _extract_macro_mock_value(*, cte: CompileSqlTestCte, file_label: str) -> str ) value: str value, index = _read_sql_string_literal(sql=body, start=index) - index = _skip_ignorable(sql=body, start=index) + index = _skip_ignorable(sql=body, start=index, syntax=syntax) if index < len(body) and body[index] == SQL_STATEMENT_TERMINATOR_TOKEN: - index = _skip_ignorable(sql=body, start=index + 1) + index = _skip_ignorable(sql=body, start=index + 1, syntax=syntax) if index != len(body): raise CompileInputError( f"SQL test '{file_label}' macro mock '{cte.name}' must be a single SELECT string " @@ -674,17 +732,19 @@ def _validate_expected_cte_query( *, cte: CompileSqlTestCte, file_label: str, + syntax: SqlLexicalSyntax, label: str = "__expected__", allow_empty_fixture: bool = False, ) -> None: if allow_empty_fixture and is_empty_fixture_query(cte.sql_body): return - if _contains_select_star(cte.sql_body): + if _contains_select_star(sql=cte.sql_body, syntax=syntax): raise CompileInputError(f"SQL test '{file_label}' must not use SELECT * in {label} CTEs") branch_column_names: tuple[tuple[str, ...], ...] = _extract_expected_branch_column_names( sql=cte.sql_body, file_label=file_label, label=label, + syntax=syntax, ) first_branch_column_names: tuple[str, ...] = branch_column_names[0] branch_index: int @@ -698,7 +758,7 @@ def _validate_expected_cte_query( def _extract_expected_branch_column_names( - *, sql: str, file_label: str, label: str + *, sql: str, file_label: str, label: str, syntax: SqlLexicalSyntax ) -> tuple[tuple[str, ...], ...]: sql_analysis_column_names: tuple[tuple[str, ...], ...] | None = ( extract_expected_branch_column_names_with_sql_analysis( @@ -709,38 +769,42 @@ def _extract_expected_branch_column_names( return sql_analysis_column_names branches: tuple[str, ...] = split_set_operation_branches(sql=sql, context=_CONTEXT) return tuple( - _extract_expected_select_column_names(branch_sql=branch, file_label=file_label, label=label) + _extract_expected_select_column_names( + branch_sql=branch, file_label=file_label, label=label, syntax=syntax + ) for branch in branches ) def _extract_expected_select_column_names( - *, branch_sql: str, file_label: str, label: str + *, branch_sql: str, file_label: str, label: str, syntax: SqlLexicalSyntax ) -> tuple[str, ...]: - index: int = _skip_ignorable(sql=branch_sql, start=0) + index: int = _skip_ignorable(sql=branch_sql, start=0, syntax=syntax) select_end: int | None = _try_consume_keyword(sql=branch_sql, start=index, keyword="SELECT") if select_end is None: raise CompileInputError( f"SQL test '{file_label}' must define each {label} set-operation " "branch as a SELECT query" ) - select_list_end: int = _find_select_list_end(sql=branch_sql, start=select_end) + select_list_end: int = _find_select_list_end(sql=branch_sql, start=select_end, syntax=syntax) raw_select_list: str = branch_sql[select_end:select_list_end] - expressions: tuple[str, ...] = _split_top_level_commas(raw_select_list) + expressions: tuple[str, ...] = _split_top_level_commas(raw_value=raw_select_list, syntax=syntax) if not expressions: raise CompileInputError( f"SQL test '{file_label}' must project at least one column in {label}" ) return tuple( - _extract_expected_projection_name(expression=expression, file_label=file_label, label=label) + _extract_expected_projection_name( + expression=expression, file_label=file_label, label=label, syntax=syntax + ) for expression in expressions ) -def _find_select_list_end(*, sql: str, start: int) -> int: +def _find_select_list_end(*, sql: str, start: int, syntax: SqlLexicalSyntax) -> int: index: int depth: int - for index, depth in iter_code_positions(sql=sql[start:], context=_CONTEXT): + for index, depth in iter_code_positions(sql=sql[start:], context=_CONTEXT, syntax=syntax): if ( depth == 0 and _try_consume_keyword(sql=sql, start=start + index, keyword="FROM") is not None @@ -749,12 +813,12 @@ def _find_select_list_end(*, sql: str, start: int) -> int: return len(sql) -def _split_top_level_commas(raw_value: str) -> tuple[str, ...]: +def _split_top_level_commas(*, raw_value: str, syntax: SqlLexicalSyntax) -> tuple[str, ...]: values: list[str] = [] value_start: int = 0 index: int depth: int - for index, depth in iter_code_positions(sql=raw_value, context=_CONTEXT): + for index, depth in iter_code_positions(sql=raw_value, context=_CONTEXT, syntax=syntax): if depth == 0 and raw_value[index] == SQL_ARGUMENT_SEPARATOR_TOKEN: item: str = raw_value[value_start:index].strip() if item: @@ -767,8 +831,10 @@ def _split_top_level_commas(raw_value: str) -> tuple[str, ...]: return tuple(values) -def _extract_expected_projection_name(*, expression: str, file_label: str, label: str) -> str: - alias_name: str | None = _extract_as_alias(expression) +def _extract_expected_projection_name( + *, expression: str, file_label: str, label: str, syntax: SqlLexicalSyntax +) -> str: + alias_name: str | None = _extract_as_alias(expression=expression, syntax=syntax) if alias_name is not None: return alias_name stripped_expression: str = expression.strip() @@ -779,17 +845,17 @@ def _extract_expected_projection_name(*, expression: str, file_label: str, label ) -def _extract_as_alias(expression: str) -> str | None: +def _extract_as_alias(*, expression: str, syntax: SqlLexicalSyntax) -> str | None: last_alias_name: str | None = None index: int depth: int - for index, depth in iter_code_positions(sql=expression, context=_CONTEXT): + for index, depth in iter_code_positions(sql=expression, context=_CONTEXT, syntax=syntax): if depth != 0: continue as_end: int | None = _try_consume_keyword(sql=expression, start=index, keyword="AS") if as_end is None: continue - alias_index: int = _skip_ignorable(sql=expression, start=as_end) + alias_index: int = _skip_ignorable(sql=expression, start=as_end, syntax=syntax) if alias_index < len(expression) and is_identifier_start(expression[alias_index]): alias_name, alias_end = _read_identifier( sql=expression, @@ -807,13 +873,13 @@ def _is_simple_identifier(value: str) -> bool: return all(is_identifier_character(character) for character in value[1:]) -def _contains_select_star(sql: str) -> bool: +def _contains_select_star(*, sql: str, syntax: SqlLexicalSyntax) -> bool: index: int - for index, _depth in iter_code_positions(sql=sql, context=_CONTEXT): + for index, _depth in iter_code_positions(sql=sql, context=_CONTEXT, syntax=syntax): select_end: int | None = _try_consume_keyword(sql=sql, start=index, keyword="SELECT") if select_end is None: continue - value_index: int = _skip_ignorable(sql=sql, start=select_end) + value_index: int = _skip_ignorable(sql=sql, start=select_end, syntax=syntax) if value_index < len(sql) and sql[value_index] == SQL_WILDCARD_TOKEN: return True return False @@ -833,9 +899,12 @@ def _validate_ceremonial_select( sql: str, start: int, file_label: str, + syntax: SqlLexicalSyntax, context_label: str = _CONTEXT, ) -> None: - if _is_ceremonial_select_statement(sql=sql, start=start, context_label=context_label): + if _is_ceremonial_select_statement( + sql=sql, start=start, context_label=context_label, syntax=syntax + ): return raise CompileInputError( f"{context_label} '{file_label}' must end with a ceremonial top-level `SELECT 1` " @@ -843,22 +912,28 @@ def _validate_ceremonial_select( ) -def _is_ceremonial_select_statement(*, sql: str, start: int, context_label: str = _CONTEXT) -> bool: - index: int = _skip_ignorable(sql=sql, start=start, context_label=context_label) +def _is_ceremonial_select_statement( + *, sql: str, start: int, syntax: SqlLexicalSyntax, context_label: str = _CONTEXT +) -> bool: + index: int = _skip_ignorable(sql=sql, start=start, context_label=context_label, syntax=syntax) select_end: int | None = _try_consume_keyword(sql=sql, start=index, keyword="SELECT") if select_end is None: return False - index = _skip_ignorable(sql=sql, start=select_end, context_label=context_label) + index = _skip_ignorable(sql=sql, start=select_end, context_label=context_label, syntax=syntax) if index >= len(sql) or sql[index] != SQL_CEREMONIAL_SELECT_VALUE: return False - index = _skip_ignorable(sql=sql, start=index + 1, context_label=context_label) - return _is_statement_end(sql=sql, start=index, context_label=context_label) + index = _skip_ignorable(sql=sql, start=index + 1, context_label=context_label, syntax=syntax) + return _is_statement_end(sql=sql, start=index, context_label=context_label, syntax=syntax) -def _is_statement_end(*, sql: str, start: int, context_label: str = _CONTEXT) -> bool: +def _is_statement_end( + *, sql: str, start: int, syntax: SqlLexicalSyntax, context_label: str = _CONTEXT +) -> bool: index: int = start if index < len(sql) and sql[index] == SQL_STATEMENT_TERMINATOR_TOKEN: - index = _skip_ignorable(sql=sql, start=index + 1, context_label=context_label) + index = _skip_ignorable( + sql=sql, start=index + 1, context_label=context_label, syntax=syntax + ) return index == len(sql) @@ -869,6 +944,7 @@ def validate_independent_expected_and_assertion_ctes( assertion_prefix: str, file_label: str, context_label: str, + syntax: SqlLexicalSyntax, ) -> None: """Reject direct and transitive dependencies between expected and assertion checks.""" @@ -915,6 +991,7 @@ def validate_independent_expected_and_assertion_ctes( cte.name.casefold(): _known_cte_references( sql=cte.sql_body, names_by_key=names_by_key, + syntax=syntax, ) for cte in ctes } @@ -974,7 +1051,9 @@ def collect(value: Any) -> tuple[str, ...]: return collect(parsed.to_dict()) -def _known_cte_references(*, sql: str, names_by_key: dict[str, str]) -> tuple[str, ...]: +def _known_cte_references( + *, sql: str, names_by_key: dict[str, str], syntax: SqlLexicalSyntax +) -> tuple[str, ...]: folded_sql: str = sql.casefold() if not any(name in folded_sql for name in names_by_key): return () @@ -982,7 +1061,7 @@ def _known_cte_references(*, sql: str, names_by_key: dict[str, str]) -> tuple[st try: parsed: Any = polyglot_module.parse_one(sql, dialect="generic") except polyglot_module.PolyglotError: - return _known_cte_identifier_references(sql=sql, names_by_key=names_by_key) + return _known_cte_identifier_references(sql=sql, names_by_key=names_by_key, syntax=syntax) references: list[str] = [] for table in parsed.find_all("table"): key: str = str(getattr(table, "name", "") or "").casefold() @@ -991,22 +1070,21 @@ def _known_cte_references(*, sql: str, names_by_key: dict[str, str]) -> tuple[st return tuple(references) -def _known_cte_identifier_references(*, sql: str, names_by_key: dict[str, str]) -> tuple[str, ...]: +def _known_cte_identifier_references( + *, sql: str, names_by_key: dict[str, str], syntax: SqlLexicalSyntax +) -> tuple[str, ...]: references: list[str] = [] index: int = 0 while index < len(sql): - if sql.startswith("--", index): - index = skip_line_comment(sql=sql, start=index) - continue - if sql.startswith("/*", index): - index = skip_block_comment(sql=sql, start=index, context=_CONTEXT) - continue - if sql[index] in _SQL_STRING_QUOTE_TOKENS: - index = skip_quoted_text(sql=sql, start=index, context=_CONTEXT) + non_code_end: int | None = dialect_non_code_end( + sql=sql, start=index, syntax=syntax, context=_CONTEXT + ) + if non_code_end is not None and sql[index] not in _SQL_IDENTIFIER_QUOTE_TOKENS: + index = non_code_end continue - if sql[index] in _SQL_IDENTIFIER_QUOTE_TOKENS: + if non_code_end is not None: quote: str = sql[index] - end: int = skip_quoted_text(sql=sql, start=index, context=_CONTEXT) + end: int = non_code_end name: str = sql[index + 1 : end - 1].replace(quote * 2, quote) key: str = name.casefold() if key in names_by_key and key not in references: @@ -1075,17 +1153,20 @@ def _read_identifier( return sql[start:index], index -def _skip_ignorable(*, sql: str, start: int, context_label: str = _CONTEXT) -> int: +def _skip_ignorable( + *, sql: str, start: int, syntax: SqlLexicalSyntax, context_label: str = _CONTEXT +) -> int: index: int = start while index < len(sql): if sql[index].isspace(): index += 1 continue - if sql.startswith("--", index): - index = skip_line_comment(sql=sql, start=index) - continue - if sql.startswith("/*", index): - index = skip_block_comment(sql=sql, start=index, context=context_label) - continue + if sql[index] not in SQL_QUOTE_TOKENS: + comment_end: int | None = dialect_non_code_end( + sql=sql, start=index, syntax=syntax, context=context_label + ) + if comment_end is not None: + index = comment_end + continue return index return index diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py index 648fb30043..8792025f14 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py @@ -2,12 +2,14 @@ from __future__ import annotations +from collections.abc import Iterator from pathlib import Path from typing import Protocol, cast import orjson import sqlbuild._native as _native +from sqlbuild.compiler.compile._helpers.sql_tests.core import extract_sql_test_ctes from sqlbuild.compiler.compile.constants import ( SQL_TEST_FACT_CACHE_ALGORITHM, SQL_TEST_FACT_CACHE_NAMESPACE, @@ -21,6 +23,7 @@ ) from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax class _NativeSqlTestModule(Protocol): @@ -36,6 +39,7 @@ def extract_expanded_sql_tests_cached( *, tests: tuple[tuple[str, str, SqlTestMode], ...], cache_root: Path | None, + syntax: SqlLexicalSyntax, ) -> tuple[CompileSqlTestCtes, ...]: """Reuse exact per-file extraction facts and extract only tests of changed files natively.""" @@ -45,13 +49,15 @@ def extract_expanded_sql_tests_cached( algorithm=SQL_TEST_FACT_CACHE_ALGORITHM, ) as fact_cache: if not fact_cache.enabled: - return extract_expanded_sql_tests(tests) + return extract_expanded_sql_tests(tests=tests, syntax=syntax) indexes_by_file: dict[str, list[int]] = {} for index, (_sql, file_label, _mode) in enumerate(tests): indexes_by_file.setdefault(file_label, []).append(index) keys_by_file: dict[str, str] = { file_label: fact_cache.key( - file_label, *_file_test_key_parts(tests=tests, indexes=indexes) + file_label, + syntax.cache_key, + *_file_test_key_parts(tests=tests, indexes=indexes), ) for file_label, indexes in indexes_by_file.items() } @@ -73,7 +79,7 @@ def extract_expanded_sql_tests_cached( missing_indexes.extend(indexes_by_file[file_label]) if missing_indexes: extracted: tuple[CompileSqlTestCtes, ...] = extract_expanded_sql_tests( - tuple(tests[index] for index in missing_indexes) + tests=tuple(tests[index] for index in missing_indexes), syntax=syntax ) for index, test_ctes in zip(missing_indexes, extracted, strict=True): results[index] = test_ctes @@ -108,10 +114,37 @@ def _cached_file_tests( def extract_expanded_sql_tests( + *, tests: tuple[tuple[str, str, SqlTestMode], ...], + syntax: SqlLexicalSyntax, ) -> tuple[CompileSqlTestCtes, ...]: - """Extract and classify expanded tests in one authoritative native call.""" + """Extract and classify expanded tests, natively unless the dialect reads a test differently.""" + + dialect_indexes: frozenset[int] = frozenset( + index + for index, (sql, _file_label, _mode) in enumerate(tests) + if syntax.reads_differently_from_generic(sql) + ) + if not dialect_indexes: + return _extract_expanded_sql_tests_natively(tests) + native_results: Iterator[CompileSqlTestCtes] = iter( + _extract_expanded_sql_tests_natively( + tuple(test for index, test in enumerate(tests) if index not in dialect_indexes) + ) + ) + return tuple( + extract_sql_test_ctes(sql=sql, file_label=file_label, syntax=syntax, mode=mode) + if index in dialect_indexes + else next(native_results) + for index, (sql, file_label, mode) in enumerate(tests) + ) + +def _extract_expanded_sql_tests_natively( + tests: tuple[tuple[str, str, SqlTestMode], ...], +) -> tuple[CompileSqlTestCtes, ...]: + if not tests: + return () request_json: str = orjson.dumps( { "tests": [ diff --git a/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py b/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py index 53edda5aa0..8211248490 100644 --- a/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py +++ b/src/sqlbuild/compiler/compile/classes/sql_test_cte_extractor.py @@ -6,13 +6,16 @@ extract_unclassified_sql_test_ctes, ) from sqlbuild.compiler.compile.models import CompileSqlTestCte +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax class SqlTestCteExtractor: """Expose authored SQL-test CTE parsing to compiler consumers.""" @staticmethod - def extract(*, sql: str, file_label: str) -> tuple[CompileSqlTestCte, ...]: + def extract( + *, sql: str, file_label: str, syntax: SqlLexicalSyntax + ) -> tuple[CompileSqlTestCte, ...]: """Extract top-level SQL-test CTEs without classifying their roles.""" - return extract_unclassified_sql_test_ctes(sql=sql, file_label=file_label) + return extract_unclassified_sql_test_ctes(sql=sql, file_label=file_label, syntax=syntax) diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index a40fe5f8c1..534c86793f 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -127,6 +127,7 @@ def build_compile_inputs( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, compile_cache_dir=compile_cache_dir, + sql_lexical_syntax=adapter_context.sql_lexical_syntax, ) model_context: ModelInputBuildContext = ModelInputBuildContext( effective_vars=effective_vars, @@ -138,6 +139,7 @@ def build_compile_inputs( loaded_macros=declaration_scope.loaded_macros, value_renderer=adapter_context.value_renderer, collection_rendering=adapter_context.collection_rendering, + sql_lexical_syntax=adapter_context.sql_lexical_syntax, declaration_resolver=declaration_scope.resolver, ) model_build: ModelInputScopeBuild = _build_models_with_declarations( @@ -178,6 +180,7 @@ def build_compile_inputs( external_sql_reference_resolver=external_sql_reference_resolver, sql_function_inputs=sql_function_inputs, compile_cache_dir=compile_cache_dir, + sql_lexical_syntax=adapter_context.sql_lexical_syntax, ) scenario_inputs: tuple[CompileSqlScenarioInput, ...] = build_scenario_inputs( discovered_inputs=discovered_inputs, @@ -186,6 +189,7 @@ def build_compile_inputs( loaded_macros=declaration_scope.loaded_macros, declaration_expansion=model_context.declaration_expansion, external_sql_reference_resolver=external_sql_reference_resolver, + sql_lexical_syntax=adapter_context.sql_lexical_syntax, ) project_audit_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] = ( index_generic_audit_definitions(discovered_inputs.audit_files) @@ -198,13 +202,9 @@ def build_compile_inputs( diagnostics = (*diagnostics, *model_build.diagnostics) audit_inputs: tuple[CompileAuditInput, ...] = build_audit_inputs( discovered_inputs=discovered_inputs, - effective_settings=effective_settings, + context=model_context, model_inputs=model_build.inputs, source_inputs=source_inputs, - effective_vars=effective_vars, - macro_context=macro_context, - loaded_macros=declaration_scope.loaded_macros, - declaration_expansion=model_context.declaration_expansion, generic_audit_definitions=generic_audit_definitions, seed_inputs=seed_inputs, ) @@ -212,6 +212,7 @@ def build_compile_inputs( project_config=discovered_inputs.project_config, local_config=discovered_inputs.local_config, discovered_inputs=discovered_inputs, + sql_lexical_syntax=adapter_context.sql_lexical_syntax, run_id=resolved_run_id, effective_target_name=effective_target_name, effective_target=effective_target, @@ -261,12 +262,12 @@ def _build_sql_functions( effective_vars=context.effective_vars, effective_settings=context.effective_settings, target_config=context.target_config, - adapter_name=context.macro_context.adapter_name, macro_context=context.macro_context, loaded_macros=context.loaded_macros, declaration_expansion=declaration_expansion, no_sql_validation=no_sql_validation, python_functions_inherit_default_namespace=python_functions_inherit_default_namespace, + sql_lexical_syntax=context.sql_lexical_syntax, ) validate_table_function_call_arities( model_inputs=model_inputs, diff --git a/src/sqlbuild/compiler/compile/main/_scope_relationship_grants.py b/src/sqlbuild/compiler/compile/main/_scope_relationship_grants.py index 69d12227a7..f6d7255467 100644 --- a/src/sqlbuild/compiler/compile/main/_scope_relationship_grants.py +++ b/src/sqlbuild/compiler/compile/main/_scope_relationship_grants.py @@ -8,11 +8,19 @@ from sqlbuild.compiler.compile.models import ScopeRelationshipBuild from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.scopes.models import ScopeIndex +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax def build_scope_relationship_grants( - *, discovered_inputs: DiscoveredProjectInputs, index: ScopeIndex + *, + discovered_inputs: DiscoveredProjectInputs, + index: ScopeIndex, + sql_lexical_syntax: SqlLexicalSyntax, ) -> ScopeRelationshipBuild: """Return expected-model grants while retaining independent extraction faults.""" - return _build_scope_relationship_grants(discovered_inputs=discovered_inputs, index=index) + return _build_scope_relationship_grants( + discovered_inputs=discovered_inputs, + index=index, + sql_lexical_syntax=sql_lexical_syntax, + ) diff --git a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py index dfc0042d0d..06e375ed70 100644 --- a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py +++ b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py @@ -26,6 +26,7 @@ from sqlbuild.compiler.compile.types import TypedSqlValueRenderer from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( resolve_effective_adapter_name, ) @@ -57,7 +58,9 @@ def build_sql_expansion_context( cli_vars={} if cli_vars is None else cli_vars, ) scope: DeclarationScopeBuild = declaration_scope or _build_declaration_scope( - discovered_inputs=effective_discovered_inputs, static_scope=static_declaration_scope + discovered_inputs=effective_discovered_inputs, + static_scope=static_declaration_scope, + sql_lexical_syntax=value_renderer.sql_lexical_syntax, ) loaded_macros: dict[str, LoadedMacro] = scope.loaded_macros enums: dict[str, EnumDeclaration] @@ -107,7 +110,10 @@ def build_sql_expansion_context( def _build_declaration_scope( - *, discovered_inputs: DiscoveredProjectInputs, static_scope: DeclarationScopeBuild | None + *, + discovered_inputs: DiscoveredProjectInputs, + static_scope: DeclarationScopeBuild | None, + sql_lexical_syntax: SqlLexicalSyntax, ) -> DeclarationScopeBuild: loaded_macros: dict[str, LoadedMacro] = load_project_macros(discovered_inputs.macro_files) rebound: DeclarationScopeBuild | None = ( @@ -118,5 +124,7 @@ def _build_declaration_scope( ) ) return rebound or build_declaration_scope( - discovered_inputs=discovered_inputs, loaded_macros=loaded_macros + discovered_inputs=discovered_inputs, + loaded_macros=loaded_macros, + sql_lexical_syntax=sql_lexical_syntax, ) diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index dcd3fbfef1..ca50211cf0 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -61,7 +61,7 @@ UsageRecord, VisibilityRecord, ) -from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic +from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic, SqlLexicalSyntax from sqlbuild.spec.contracts.models import ( DefaultsConfig, LocalConfig, @@ -439,6 +439,7 @@ class CompileAdapterContext: value_renderer: TypedSqlValueRenderer collection_rendering: CollectionRendering python_functions_inherit_default_namespace: bool + sql_lexical_syntax: SqlLexicalSyntax @dataclass(frozen=True) @@ -454,6 +455,7 @@ class ModelInputBuildContext: loaded_macros: dict[str, LoadedMacro] value_renderer: TypedSqlValueRenderer collection_rendering: CollectionRendering + sql_lexical_syntax: SqlLexicalSyntax public_enums: dict[str, EnumDeclaration] = field(default_factory=dict) public_constants: dict[str, ConstantDeclaration] = field(default_factory=dict) public_model_schemas: dict[str, ModelSchemaDeclaration] = field(default_factory=dict) @@ -846,6 +848,7 @@ class CompileProjectInputs: project_config: ProjectConfig local_config: LocalConfig discovered_inputs: DiscoveredProjectInputs + sql_lexical_syntax: SqlLexicalSyntax run_id: str = "" effective_target_name: str | None = None effective_target: TargetConfig | None = None @@ -1048,6 +1051,7 @@ class CompiledProject: effective_target_database: str | None = None effective_target_schema: str | None = None sql_analysis_dialect: str | None = None + sql_lexical_syntax: SqlLexicalSyntax = field(default_factory=SqlLexicalSyntax) compile_cache_dir: Path | None = None settings: SettingsConfig = field(default_factory=SettingsConfig) scenario: ScenarioConfig = field(default_factory=ScenarioConfig) @@ -1133,6 +1137,7 @@ class CompileModelSqlTestInputPayload: expected_model_names: tuple[str, ...] = field(default_factory=tuple) assertion_ctes: tuple[CompileSqlTestCte, ...] = field(default_factory=tuple) assertion_names: tuple[str, ...] = field(default_factory=tuple) + assertion_target_model_names: tuple[str, ...] = field(default_factory=tuple) @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index ce19323a59..9820257fd2 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -9,12 +9,14 @@ if TYPE_CHECKING: from sqlbuild.compiler.compile.models import CompactBatchPreparation, CompileProjectInputs + from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax class TypedSqlValueRenderer(Protocol): """Adapter rendering operations consumed by authored SQL expansion.""" adapter_name: str + sql_lexical_syntax: SqlLexicalSyntax def render_typed_scalar(self, *, value: SqlValue) -> str: ... diff --git a/src/sqlbuild/compiler/pipeline/main/compiled_project.py b/src/sqlbuild/compiler/pipeline/main/compiled_project.py index 84d343e16e..499eda8df0 100644 --- a/src/sqlbuild/compiler/pipeline/main/compiled_project.py +++ b/src/sqlbuild/compiler/pipeline/main/compiled_project.py @@ -77,6 +77,7 @@ def build_compiled_project( python_functions_inherit_default_namespace=( adapter.python_functions_inherit_default_namespace() ), + sql_lexical_syntax=adapter.sql_lexical_syntax, ), selected_target=selected_target, no_sql_validation=no_sql_validation, diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py index f2f591c8e4..64dacfb4f4 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/fixture_formatting.py @@ -28,6 +28,7 @@ FixtureRelationMetadata, ) from sqlbuild.compiler.planner.types import ContractPolicy, FixtureKey +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import SchemaColumn, SchemaSeedEntry, SourceEntry _TYPED_NULL_LINE_PATTERN: re.Pattern[str] = re.compile( @@ -72,6 +73,7 @@ def format_redundant_fixture_nulls( files: dict[Path, str], project_dir: Path, discovered_inputs: DiscoveredProjectInputs, + sql_lexical_syntax: SqlLexicalSyntax, ) -> dict[Path, str]: """Remove typed-null fixture projections that planner completion can reproduce.""" @@ -100,6 +102,7 @@ def format_redundant_fixture_nulls( block=block, file_label=str(test_file.relative_path), relations=relations, + syntax=sql_lexical_syntax, ) if fixed_block == block.sql_body: continue @@ -117,6 +120,7 @@ def _format_test_block( block: DiscoveredSqlTestBlock, file_label: str, relations: dict[FixtureKey, FixtureRelationMetadata], + syntax: SqlLexicalSyntax, ) -> str: if ( block.mode is not SqlTestMode.MODEL @@ -128,6 +132,7 @@ def _format_test_block( authored_ctes: tuple[CompileSqlTestCte, ...] = SqlTestCteExtractor.extract( sql=block.sql_body, file_label=file_label, + syntax=syntax, ) except CompileInputError: return block.sql_body diff --git a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py b/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py index d47bbfb665..87c199d9ab 100644 --- a/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py +++ b/src/sqlbuild/compiler/planner/_helpers/sql_tests/native_planning.py @@ -35,6 +35,7 @@ from sqlbuild.compiler.planner.types import WarningSeverity from sqlbuild.compiler.profiling.classes.context import CompileTimingContext from sqlbuild.compiler.profiling.models import CompileTimingCollector +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.executor.testing.types import NativeSqlTestRenderingModule _CALL_SUFFIX_SENTINEL: str = "__SQLBUILD_CALL_SUFFIX__" @@ -103,6 +104,7 @@ def plan_sql_tests_natively( "sqlAnalysisDialect": adapter.sql_analysis_dialect(), "setDifferenceOperator": adapter.render_set_difference_operator(), "requiresDerivedTableAliases": adapter.requires_derived_table_aliases(), + "lexicalSyntax": _lexical_syntax_request(project.sql_lexical_syntax), "workers": _NATIVE_WORKERS, "renderSql": render_sql, "includePlan": include_plan, @@ -159,6 +161,7 @@ def resolve_sql_test_model_chains( request: dict[str, object] = { "models": _chain_model_requests(project=project), "tests": [_test_request(test=test) for test in tests], + "lexicalSyntax": _lexical_syntax_request(project.sql_lexical_syntax), } try: native_response: str = cast( @@ -457,3 +460,14 @@ def _render_call_template(*, rendered: str) -> tuple[str, str]: if not separator or _CALL_SUFFIX_SENTINEL in suffix: raise NativeSqlTestPlanningError("adapter SQL function call template is not deterministic") return prefix, suffix + + +def _lexical_syntax_request(syntax: SqlLexicalSyntax) -> dict[str, object]: + return { + "backslashEscapeQuotes": sorted(syntax.backslash_escape_quotes), + "escapeStringPrefix": syntax.escape_string_prefix, + "rawStringPrefix": syntax.raw_string_prefix, + "tripleQuotedStrings": syntax.triple_quoted_strings, + "nestedBlockComments": syntax.nested_block_comments, + "lineCommentPrefixes": sorted(syntax.line_comment_prefixes), + } diff --git a/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py b/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py index 97cf4178ec..16cfc725de 100644 --- a/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py +++ b/src/sqlbuild/compiler/planner/classes/fixture_null_autofix.py @@ -8,6 +8,7 @@ from sqlbuild.compiler.planner._helpers.sql_tests.fixture_formatting import ( format_redundant_fixture_nulls, ) +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax class FixtureNullAutofix: @@ -19,6 +20,7 @@ def apply( files: dict[Path, str], project_dir: Path, discovered_inputs: DiscoveredProjectInputs, + sql_lexical_syntax: SqlLexicalSyntax, ) -> dict[Path, str]: """Return contract-safe fixture source rewrites.""" @@ -26,4 +28,5 @@ def apply( files=files, project_dir=project_dir, discovered_inputs=discovered_inputs, + sql_lexical_syntax=sql_lexical_syntax, ) diff --git a/src/sqlbuild/compiler/scopes/_helpers/builder.py b/src/sqlbuild/compiler/scopes/_helpers/builder.py index 1da3d2e6ce..95a8235ef7 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/builder.py +++ b/src/sqlbuild/compiler/scopes/_helpers/builder.py @@ -61,6 +61,7 @@ ScopeKind, UsageKind, ) +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.sql_values.types import SqlValueKind _ROOTS: dict[ResourceKind, str] = { @@ -254,7 +255,9 @@ def _macro_dependency_usages(*, declarations: list[DeclarationRecord]) -> tuple[ return tuple(dict.fromkeys(usages)) -def build_tolerant_scope_index(*, project_dir: Path) -> ScopeIndex: +def build_tolerant_scope_index( + *, project_dir: Path, sql_lexical_syntax: SqlLexicalSyntax +) -> ScopeIndex: """Build a partial static index from published tolerant compiler facts.""" snapshot: TolerantScopeDiscovery = discover_scope_snapshot(project_dir=project_dir) @@ -269,6 +272,7 @@ def build_tolerant_scope_index(*, project_dir: Path) -> ScopeIndex: relationships: ScopeRelationshipBuild = build_scope_relationship_grants( discovered_inputs=snapshot.discovered_inputs, index=index, + sql_lexical_syntax=sql_lexical_syntax, ) diagnostics: tuple[ScopeDiagnostic, ...] = _tolerant_diagnostics( snapshot=snapshot, diff --git a/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py b/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py index f25bbfe920..2bf34d2a19 100644 --- a/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py +++ b/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py @@ -20,6 +20,7 @@ write_cached_scope_index, ) from sqlbuild.compiler.scopes.models import ScopeIndex +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( resolve_effective_adapter_name, ) @@ -44,6 +45,7 @@ def load_or_build_scope_index(*, project_dir: Path, no_cache: bool = False) -> S if cached is not None: return cached + sql_lexical_syntax: SqlLexicalSyntax = SqlLexicalSyntax() try: discovered: DiscoveredProjectInputs = discover_project_inputs( project_dir=resolved_project_dir, @@ -54,6 +56,7 @@ def load_or_build_scope_index(*, project_dir: Path, no_cache: bool = False) -> S local_config=discovered.local_config, ) adapter: BaseAdapter = resolve_adapter(adapter_name=adapter_name) + sql_lexical_syntax = adapter.sql_lexical_syntax compile_inputs: CompileProjectInputs = build_compile_inputs( discovered_inputs=discovered, adapter_context=CompileAdapterContext( @@ -65,6 +68,7 @@ def load_or_build_scope_index(*, project_dir: Path, no_cache: bool = False) -> S python_functions_inherit_default_namespace=( adapter.python_functions_inherit_default_namespace() ), + sql_lexical_syntax=adapter.sql_lexical_syntax, ), resolved_connection={}, no_sql_validation=True, @@ -73,7 +77,9 @@ def load_or_build_scope_index(*, project_dir: Path, no_cache: bool = False) -> S ) index: ScopeIndex = scope_index_with_compile_usages(inputs=compile_inputs) except (OSError, UnicodeError, ValueError, ImportError): - index = build_tolerant_scope_index(project_dir=resolved_project_dir) + index = build_tolerant_scope_index( + project_dir=resolved_project_dir, sql_lexical_syntax=sql_lexical_syntax + ) if not no_cache and fingerprint is not None: write_cached_scope_index( diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/scanning.py b/src/sqlbuild/compiler/sql_analysis/_helpers/scanning.py index 9352b90638..ee018e4914 100644 --- a/src/sqlbuild/compiler/sql_analysis/_helpers/scanning.py +++ b/src/sqlbuild/compiler/sql_analysis/_helpers/scanning.py @@ -25,7 +25,7 @@ ) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -_PAREN_SCAN_SPECIAL: re.Pattern[str] = re.compile(r"[-/'\"`$()]") +_PAREN_SCAN_SPECIAL: re.Pattern[str] = re.compile(r"[-/#'\"`$()]") _DOLLAR_QUOTE_DELIMITER: re.Pattern[str] = re.compile(r"\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$") @@ -175,7 +175,13 @@ def skip_block_comment_impl(*, sql: str, start: int, context: str = "SQL") -> in return closing_index + 2 -def find_matching_paren_impl(*, sql: str, open_paren_index: int, context: str = "SQL") -> int: +def find_matching_paren_impl( + *, + sql: str, + open_paren_index: int, + context: str = "SQL", + syntax: SqlLexicalSyntax | None = None, +) -> int: """Find the closing parenthesis matching an opening parenthesis.""" depth: int = 1 @@ -185,7 +191,9 @@ def find_matching_paren_impl(*, sql: str, open_paren_index: int, context: str = if special is None: break index = special.start() - non_code_end: int | None = _non_code_end(sql=sql, index=index, context=context) + non_code_end: int | None = _non_code_end( + sql=sql, index=index, context=context, syntax=syntax + ) if non_code_end is not None: index = non_code_end continue @@ -199,13 +207,17 @@ def find_matching_paren_impl(*, sql: str, open_paren_index: int, context: str = raise CompileInputError(f"{context} contains an unclosed parenthesis") -def iter_code_positions_impl(*, sql: str, context: str = "SQL") -> Iterator[tuple[int, int]]: +def iter_code_positions_impl( + *, sql: str, context: str = "SQL", syntax: SqlLexicalSyntax | None = None +) -> Iterator[tuple[int, int]]: """Yield code offsets outside comments, quotes, and parentheses with their nesting depth.""" depth: int = 0 index: int = 0 while index < len(sql): - non_code_end: int | None = _non_code_end(sql=sql, index=index, context=context) + non_code_end: int | None = _non_code_end( + sql=sql, index=index, context=context, syntax=syntax + ) if non_code_end is not None: index = non_code_end continue @@ -219,7 +231,11 @@ def iter_code_positions_impl(*, sql: str, context: str = "SQL") -> Iterator[tupl index += 1 -def _non_code_end(*, sql: str, index: int, context: str) -> int | None: +def _non_code_end( + *, sql: str, index: int, context: str, syntax: SqlLexicalSyntax | None +) -> int | None: + if syntax is not None: + return dialect_non_code_end_impl(sql=sql, start=index, syntax=syntax, context=context) if sql.startswith("--", index): return skip_line_comment_impl(sql=sql, start=index) if sql.startswith("/*", index): diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index bec3ddd0bf..cfeba4649c 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -16,6 +16,7 @@ SQL_STRING_PREFIX_CHARACTERS: frozenset[str] = frozenset("bBeErR") SQL_STRING_PREFIX_MAX_LENGTH: int = 2 SQL_TRIPLE_QUOTE_LENGTH: int = 3 +SQL_LINE_COMMENT_PREFIX: str = "--" SQL_BLOCK_COMMENT_OPEN: str = "/*" SQL_BLOCK_COMMENT_CLOSE: str = "*/" SQL_DOLLAR_QUOTE_DELIMITER: str = "$$" diff --git a/src/sqlbuild/compiler/sql_analysis/main/_find_matching_paren.py b/src/sqlbuild/compiler/sql_analysis/main/_find_matching_paren.py index 8599e4b19a..a5b31ae45c 100644 --- a/src/sqlbuild/compiler/sql_analysis/main/_find_matching_paren.py +++ b/src/sqlbuild/compiler/sql_analysis/main/_find_matching_paren.py @@ -1,13 +1,21 @@ """SQL parenthesis matching entrypoint.""" from sqlbuild.compiler.sql_analysis._helpers.scanning import find_matching_paren_impl +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -def find_matching_paren(*, sql: str, open_paren_index: int, context: str = "SQL") -> int: - """Find the closing parenthesis matching an opening parenthesis.""" +def find_matching_paren( + *, + sql: str, + open_paren_index: int, + context: str = "SQL", + syntax: SqlLexicalSyntax | None = None, +) -> int: + """Find the closing parenthesis, ending quoted text and comments per the optional dialect.""" return find_matching_paren_impl( sql=sql, open_paren_index=open_paren_index, context=context, + syntax=syntax, ) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_iter_code_positions.py b/src/sqlbuild/compiler/sql_analysis/main/_iter_code_positions.py index b2a491333d..649dbd11d2 100644 --- a/src/sqlbuild/compiler/sql_analysis/main/_iter_code_positions.py +++ b/src/sqlbuild/compiler/sql_analysis/main/_iter_code_positions.py @@ -3,9 +3,12 @@ from collections.abc import Iterator from sqlbuild.compiler.sql_analysis._helpers.scanning import iter_code_positions_impl +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -def iter_code_positions(*, sql: str, context: str = "SQL") -> Iterator[tuple[int, int]]: - """Yield each code character offset outside comments and quotes with its parenthesis depth.""" +def iter_code_positions( + *, sql: str, context: str = "SQL", syntax: SqlLexicalSyntax | None = None +) -> Iterator[tuple[int, int]]: + """Yield each code offset outside comments and quotes with its parenthesis depth.""" - return iter_code_positions_impl(sql=sql, context=context) + return iter_code_positions_impl(sql=sql, context=context, syntax=syntax) diff --git a/src/sqlbuild/compiler/sql_analysis/models.py b/src/sqlbuild/compiler/sql_analysis/models.py index 2a2a74dafd..7ddbd859e5 100644 --- a/src/sqlbuild/compiler/sql_analysis/models.py +++ b/src/sqlbuild/compiler/sql_analysis/models.py @@ -6,6 +6,14 @@ from dataclasses import dataclass, field from typing import Any +from sqlbuild.compiler.sql_analysis.constants import ( + SQL_BLOCK_COMMENT_OPEN, + SQL_ESCAPABLE_QUOTE_CHARACTERS, + SQL_ESCAPE_CHARACTER, + SQL_LINE_COMMENT_PREFIX, + SQL_TRIPLE_QUOTE_LENGTH, +) + @dataclass(frozen=True) class SqlBindingDiagnostic: @@ -49,4 +57,38 @@ class SqlLexicalSyntax: raw_string_prefix: bool = False triple_quoted_strings: bool = False nested_block_comments: bool = False - line_comment_prefixes: frozenset[str] = frozenset({"--"}) + line_comment_prefixes: frozenset[str] = frozenset({SQL_LINE_COMMENT_PREFIX}) + + @property + def cache_key(self) -> str: + """Return a process-stable text key for these lexical rules.""" + + return "|".join( + ( + ",".join(sorted(self.backslash_escape_quotes)), + str(self.escape_string_prefix), + str(self.raw_string_prefix), + str(self.triple_quoted_strings), + str(self.nested_block_comments), + ",".join(sorted(self.line_comment_prefixes)), + ) + ) + + def reads_differently_from_generic(self, sql: str) -> bool: + """Return whether these rules can read the SQL differently from generic SQL.""" + + if SQL_ESCAPE_CHARACTER in sql and ( + self.backslash_escape_quotes or self.escape_string_prefix + ): + return True + if self.triple_quoted_strings and any( + quote * SQL_TRIPLE_QUOTE_LENGTH in sql for quote in SQL_ESCAPABLE_QUOTE_CHARACTERS + ): + return True + if self.nested_block_comments and sql.count(SQL_BLOCK_COMMENT_OPEN) > 1: + return True + return any( + prefix in sql + for prefix in self.line_comment_prefixes + if prefix != SQL_LINE_COMMENT_PREFIX + ) diff --git a/src/sqlbuild/lint/main/run_format.py b/src/sqlbuild/lint/main/run_format.py index ea8a98be5e..91075f5b48 100644 --- a/src/sqlbuild/lint/main/run_format.py +++ b/src/sqlbuild/lint/main/run_format.py @@ -7,6 +7,7 @@ from sqlbuild.compiler.compile.types import TypedSqlValueRenderer from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.planner.classes.fixture_null_autofix import FixtureNullAutofix +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.lint._helpers.fixes import finalize_fix_reports, persist_changes, plan_rule_fixes from sqlbuild.lint._helpers.headers import scan_headers from sqlbuild.lint._helpers.native import ( @@ -20,6 +21,7 @@ from sqlbuild.lint._helpers.project_files import collect_project_files, sort_violations from sqlbuild.lint._helpers.suppressions import apply_suppressions from sqlbuild.lint.constants import VIOLATION_SEVERITY_WARNING +from sqlbuild.lint.exceptions import LintError from sqlbuild.lint.models import ( FormatChange, HeaderSpan, @@ -42,6 +44,8 @@ def run_format( ) -> LintRunResult: """Format all DSL files in place and report the violations that remain.""" + if discovered_inputs is not None and value_renderer is None: + raise LintError("fixture-null formatting requires the project adapter") files: dict[Path, str] = collect_project_files( project_dir=project_dir, selected_paths=selected_paths ) @@ -58,7 +62,11 @@ def run_format( files=safe_files, config=config, project_dir=project_dir, - discovered_inputs=discovered_inputs, + fixture_inputs=( + None + if discovered_inputs is None or value_renderer is None + else (discovered_inputs, value_renderer.sql_lexical_syntax) + ), fixtures_only=fixtures_only, ) format_faults.extend(rule_faults) @@ -105,7 +113,7 @@ def _apply_fixes( files: dict[Path, str], config: LintConfig, project_dir: Path, - discovered_inputs: DiscoveredProjectInputs | None, + fixture_inputs: tuple[DiscoveredProjectInputs, SqlLexicalSyntax] | None, fixtures_only: bool, ) -> tuple[dict[Path, str], list[LintViolation]]: """Return contents after native header and supported SQL body fixes.""" @@ -114,9 +122,10 @@ def _apply_fixes( FixtureNullAutofix.apply( files=files, project_dir=project_dir, - discovered_inputs=discovered_inputs, + discovered_inputs=fixture_inputs[0], + sql_lexical_syntax=fixture_inputs[1], ) - if discovered_inputs is not None + if fixture_inputs is not None else {} ) if fixtures_only: @@ -160,9 +169,10 @@ def _apply_fixes( FixtureNullAutofix.apply( files=post_native_files, project_dir=project_dir, - discovered_inputs=discovered_inputs, + discovered_inputs=fixture_inputs[0], + sql_lexical_syntax=fixture_inputs[1], ) - if discovered_inputs is not None + if fixture_inputs is not None else {} ) if fixture_formatted: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_authored_check_sql.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_authored_check_sql.py index 6d734d3b2a..7c5aa935ef 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_authored_check_sql.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_authored_check_sql.py @@ -63,6 +63,24 @@ ), expected_stdout_fragment="PASS=1 FAIL=0", ), + ScenarioAuthoredCheckSqlE2ETestCase( + description="DuckDB escape string before the first relation marker", + scenario_sql=( + "SCENARIO ();\n\n" + "WITH\n" + "__source__raw_orders AS (\n" + " SELECT 1 AS id, 10 AS amount\n" + " UNION ALL\n" + " SELECT 2 AS id, 5 AS amount\n" + "),\n" + "__assert__no_named_totals AS (\n" + " SELECT E'O\\'Brien' AS customer_name, total_amount\n" + ' FROM __ref("order_totals") WHERE total_amount <> 15\n' + ")\n" + "SELECT 1\n" + ), + expected_stdout_fragment="PASS=1 FAIL=0", + ), ], ids=lambda case: case.description, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py new file mode 100644 index 0000000000..994276ecfa --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_dialect_string_escapes.py @@ -0,0 +1,65 @@ +"""E2E coverage that compile-step SQL scanning follows DuckDB escape-string rules.""" + +from __future__ import annotations + +from pathlib import Path +from subprocess import CompletedProcess + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import SqlTestE2ETestCase +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "escape_strings"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "escape_strings.duckdb"\n' + ), + "models/stg_orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS order_id\n", + "models/order_names.sql": ( + "MODEL (materialized table);\n\n" + "SELECT E'O\\'Brien' AS customer_name, order_id\n" + 'FROM __ref("stg_orders")\n' + ), + "tests/unit/test_order_names.sql": ( + "TEST ();\n\n" + "WITH\n" + "__ref__stg_orders AS (SELECT 1 AS order_id),\n" + "__assert__named_orders AS (\n" + " SELECT E'O\\'Brien), (' AS note, customer_name\n" + ' FROM __ref("order_names")\n' + " WHERE customer_name <> 'O''Brien'\n" + ")\n" + "SELECT 1\n" + ), +} + + +@pytest.mark.parametrize( + "test_case", + ( + SqlTestE2ETestCase( + description="escape strings before model references and test markers", + expected_exit_code=0, + expected_stdout_fragment="PASS=1 FAIL=0", + ), + ), + ids=lambda case: case.description, +) +def test_given_duckdb_escape_strings_before_references_when_building_and_testing_then_succeeds( + test_case: SqlTestE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="escape_strings", repo_files=_PROJECT_FILES + ) + + built: CompletedProcess[str] = run_sqb(command=("--no-color", "build"), project_dir=project_dir) + tested: CompletedProcess[str] = run_sqb(command=("--no-color", "test"), project_dir=project_dir) + + assert built.returncode == test_case.expected_exit_code, built.stdout + built.stderr + assert tested.returncode == test_case.expected_exit_code, tested.stdout + tested.stderr + assert test_case.expected_stdout_fragment in tested.stdout, tested.stdout + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index c4773e46ec..139deb6db3 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -19,6 +19,7 @@ from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.compiler.compile.models import CompiledProject, DeclarationScopeBuild, LoadedMacro from sqlbuild.compiler.discovery.models import DiscoveredMacroFile, DiscoveredProjectInputs +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.rule_engine.models import Rule from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( DynamicPivotRulesIntegrationTestCase, @@ -735,12 +736,14 @@ def counting_build( *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: dict[str, LoadedMacro], + sql_lexical_syntax: SqlLexicalSyntax, compile_cache_dir: Path | None = None, ) -> DeclarationScopeBuild: scope_builds.append(tmp_path) return original_build( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, + sql_lexical_syntax=sql_lexical_syntax, compile_cache_dir=compile_cache_dir, ) @@ -795,12 +798,14 @@ def counting_build( *, discovered_inputs: DiscoveredProjectInputs, loaded_macros: dict[str, LoadedMacro], + sql_lexical_syntax: SqlLexicalSyntax, compile_cache_dir: Path | None = None, ) -> DeclarationScopeBuild: scope_builds.append(tmp_path) return original_build( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, + sql_lexical_syntax=sql_lexical_syntax, compile_cache_dir=compile_cache_dir, ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 8edeea1fde..debd19f9d3 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -42,6 +42,7 @@ value_renderer=DuckDbAdapter(), collection_rendering=CollectionRendering.VALUE_LIST, python_functions_inherit_default_namespace=True, + sql_lexical_syntax=DuckDbAdapter.sql_lexical_syntax, ) diff --git a/tests/integration/src/sqlbuild/integrations/dbt/helpers.py b/tests/integration/src/sqlbuild/integrations/dbt/helpers.py index fdf66c1377..95931ffd37 100644 --- a/tests/integration/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/integration/src/sqlbuild/integrations/dbt/helpers.py @@ -14,6 +14,7 @@ value_renderer=DuckDbAdapter(), collection_rendering=CollectionRendering.VALUE_LIST, python_functions_inherit_default_namespace=True, + sql_lexical_syntax=DuckDbAdapter.sql_lexical_syntax, ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 91a9c1f4eb..cae1745351 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -18,6 +18,7 @@ ) from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.scopes.models import UsageRecord +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import ResolvedTimeTravelRetention from sqlbuild.sql_values.models import SqlValue @@ -35,6 +36,23 @@ class SqlReferenceExtractionTestCase: expected_references: tuple[tuple[SqlReferenceKind, str, str | None], ...] +@dataclass(frozen=True) +class DialectCteScanTestCase: + description: str + syntax: SqlLexicalSyntax + sql: str + expected_cte_names: tuple[str, ...] + expected_assertion_targets: tuple[str, ...] + + +@dataclass(frozen=True) +class DialectSqlScanTestCase: + description: str + syntax: SqlLexicalSyntax + sql: str + expected_names: tuple[str, ...] + + @dataclass(frozen=True) class SqlReferenceExtractionErrorTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index e6a967255d..52e442f5dd 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -69,11 +69,13 @@ value_renderer=DuckDbAdapter(), collection_rendering=CollectionRendering.VALUE_LIST, python_functions_inherit_default_namespace=True, + sql_lexical_syntax=DuckDbAdapter.sql_lexical_syntax, ) DUCKDB_ARRAY_COMPILE_ADAPTER_CONTEXT: CompileAdapterContext = CompileAdapterContext( value_renderer=DUCKDB_COMPILE_ADAPTER_CONTEXT.value_renderer, collection_rendering=CollectionRendering.ARRAY, python_functions_inherit_default_namespace=True, + sql_lexical_syntax=DuckDbAdapter.sql_lexical_syntax, ) DUCKDB_DECLARATION_EXPANSION_CONTEXT: DeclarationExpansionContext = DeclarationExpansionContext( declarations=DeclarationResolutionContext(), diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_direct_logic_sql_test_classification.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_direct_logic_sql_test_classification.py index 78559b4c50..ed93ad2b24 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_direct_logic_sql_test_classification.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_direct_logic_sql_test_classification.py @@ -8,10 +8,13 @@ from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileSqlTestCte from sqlbuild.compiler.compile.types import SqlTestMode +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ClassifyDirectLogicSqlTestCtesErrorTestCase, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + _FILE: str = "tests/unit/orders.sql" @@ -281,6 +284,8 @@ def test_given_invalid_direct_logic_ctes_when_classifying_then_raises_mode_speci ) with pytest.raises(CompileInputError) as error_info: - _ = classify_sql_test_ctes(ctes=ctes, file_label=_FILE, mode=test_case.mode) + _ = classify_sql_test_ctes( + ctes=ctes, file_label=_FILE, mode=test_case.mode, syntax=_GENERIC_SQL_SYNTAX + ) assert str(error_info.value) == test_case.expected_message diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py index 1377c29cfb..6daaf4791a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_expected_projection_scanning.py @@ -8,10 +8,13 @@ from sqlbuild.compiler.sql_analysis.main._split_set_operation_branches import ( split_set_operation_branches, ) +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( ExpectedProjectionScanTestCase, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + @pytest.mark.parametrize( "test_case", @@ -112,11 +115,17 @@ def test_given_expected_cte_sql_when_scanning_top_level_then_results_are_charact == test_case.expected_branches ) assert ( - sql_test_core._find_select_list_end(sql=test_case.sql, start=0) + sql_test_core._find_select_list_end(sql=test_case.sql, start=0, syntax=_GENERIC_SQL_SYNTAX) == test_case.expected_select_list_end ) - assert sql_test_core._split_top_level_commas(test_case.sql) == test_case.expected_commas - assert sql_test_core._extract_as_alias(test_case.sql) == test_case.expected_alias - assert sql_test_core._contains_select_star(test_case.sql) is ( + assert ( + sql_test_core._split_top_level_commas(raw_value=test_case.sql, syntax=_GENERIC_SQL_SYNTAX) + == test_case.expected_commas + ) + assert ( + sql_test_core._extract_as_alias(expression=test_case.sql, syntax=_GENERIC_SQL_SYNTAX) + == test_case.expected_alias + ) + assert sql_test_core._contains_select_star(sql=test_case.sql, syntax=_GENERIC_SQL_SYNTAX) is ( test_case.expected_contains_select_star ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_native_sql_test_extraction.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_native_sql_test_extraction.py index 85ef3116ac..9a0672b680 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_native_sql_test_extraction.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_native_sql_test_extraction.py @@ -2,15 +2,27 @@ import pytest -from sqlbuild.compiler.compile._helpers.sql_tests.core import extract_sql_test_ctes +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.compiler.compile._helpers.sql_tests.core import ( + extract_assertion_target_model_names, + extract_sql_test_ctes, +) from sqlbuild.compiler.compile._helpers.sql_tests.native import extract_expanded_sql_tests -from sqlbuild.compiler.compile.models import CompileSqlTestCtes +from sqlbuild.compiler.compile.models import CompileModelSqlTestCtes, CompileSqlTestCtes from sqlbuild.compiler.compile.types import SqlTestMode +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( + DialectCteScanTestCase, ExpectedBooleanTestCase, NativeSqlTestExtractionParityTestCase, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + @pytest.mark.parametrize( "test_case", @@ -89,13 +101,15 @@ def test_given_expanded_sql_tests_when_native_batch_extracting_then_matches_refe test_case: NativeSqlTestExtractionParityTestCase, ) -> None: expected: CompileSqlTestCtes = extract_sql_test_ctes( + syntax=_GENERIC_SQL_SYNTAX, sql=test_case.sql, file_label="tests/unit/example.sql", mode=test_case.mode, ) actual: tuple[CompileSqlTestCtes, ...] = extract_expanded_sql_tests( - ((test_case.sql, "tests/unit/example.sql", test_case.mode),) + tests=((test_case.sql, "tests/unit/example.sql", test_case.mode),), + syntax=_GENERIC_SQL_SYNTAX, ) assert (actual == (expected,)) is test_case.expected_matches @@ -117,13 +131,17 @@ def test_given_cross_check_dependency_when_native_batch_extracting_then_matches_ ) with pytest.raises(ValueError) as reference_error: extract_sql_test_ctes( + syntax=_GENERIC_SQL_SYNTAX, sql=sql, file_label="tests/unit/example.sql", mode=SqlTestMode.MODEL, ) with pytest.raises(ValueError) as native_error: - extract_expanded_sql_tests(((sql, "tests/unit/example.sql", SqlTestMode.MODEL),)) + extract_expanded_sql_tests( + tests=((sql, "tests/unit/example.sql", SqlTestMode.MODEL),), + syntax=_GENERIC_SQL_SYNTAX, + ) assert (str(native_error.value) == str(reference_error.value)) is test_case.expected_result @@ -150,12 +168,129 @@ def test_given_comma_separated_cross_check_dependency_when_extracting_then_nativ ) with pytest.raises(ValueError) as reference_error: extract_sql_test_ctes( + syntax=_GENERIC_SQL_SYNTAX, sql=sql, file_label="tests/unit/example.sql", mode=SqlTestMode.MODEL, ) with pytest.raises(ValueError) as native_error: - extract_expanded_sql_tests(((sql, "tests/unit/example.sql", SqlTestMode.MODEL),)) + extract_expanded_sql_tests( + tests=((sql, "tests/unit/example.sql", SqlTestMode.MODEL),), + syntax=_GENERIC_SQL_SYNTAX, + ) assert (str(native_error.value) == str(reference_error.value)) is test_case.expected_result + + +@pytest.mark.parametrize( + "test_case", + [ + DialectCteScanTestCase( + description="snowflake backslash-escaped quote before a marker", + syntax=SnowflakeAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__ref__stg_orders AS (SELECT 'O\\'Brien), (' AS customer_name),\n" + "__assert__named AS (\n" + " SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" + ")\n" + "SELECT 1" + ), + expected_cte_names=("__ref__stg_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + DialectCteScanTestCase( + description="bigquery hash comment and triple quotes around markers", + syntax=BigQueryAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__ref__stg_orders AS (SELECT '''it's ), (''' AS note), # ), (\n" + "__assert__named AS (\n" + ' SELECT * FROM __ref("order_totals") # __ref("ignored")\n' + ")\n" + "SELECT 1" + ), + expected_cte_names=("__ref__stg_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + DialectCteScanTestCase( + description="databricks backslash-escaped quote before a marker", + syntax=DatabricksAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__ref__stg_orders AS (SELECT 'O\\'Brien), (' AS customer_name),\n" + "__assert__named AS (\n" + " SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" + ")\n" + "SELECT 1" + ), + expected_cte_names=("__ref__stg_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + DialectCteScanTestCase( + description="duckdb escape string and nested comment before a marker", + syntax=DuckDbAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__ref__stg_orders AS (SELECT E'O\\'Brien), (' AS customer_name),\n" + "/* outer /* inner */ ), ( */\n" + "__assert__named AS (\n" + " SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" + ")\n" + "SELECT 1" + ), + expected_cte_names=("__ref__stg_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + DialectCteScanTestCase( + description="postgres escape string before a marker", + syntax=PostgresAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__ref__stg_orders AS (SELECT E'O\\'Brien), (' AS customer_name),\n" + "__assert__named AS (\n" + " SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" + ")\n" + "SELECT 1" + ), + expected_cte_names=("__ref__stg_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + ], + ids=lambda case: case.description, +) +def test_given_dialect_sql_test_when_batch_extracting_then_follows_dialect_lexical_rules( + test_case: DialectCteScanTestCase, +) -> None: + extracted: tuple[CompileSqlTestCtes, ...] = extract_expanded_sql_tests( + tests=( + (test_case.sql, "tests/unit/orders.sql", SqlTestMode.MODEL), + ( + "WITH __ref__a AS (SELECT 1 AS id), __expected__b AS (SELECT 1 AS id) SELECT 1", + "tests/unit/plain.sql", + SqlTestMode.MODEL, + ), + ), + syntax=test_case.syntax, + ) + + payload: object = extracted[0].payload + assert isinstance(payload, CompileModelSqlTestCtes) + assert ( + tuple(cte.name for cte in (*payload.authored_ctes, *payload.assertion_ctes)) + == test_case.expected_cte_names + ) + assert ( + extract_assertion_target_model_names( + assertion_sql=tuple(cte.sql_body for cte in payload.assertion_ctes), + syntax=test_case.syntax, + ) + == test_case.expected_assertion_targets + ) + assert isinstance(extracted[1].payload, CompileModelSqlTestCtes) + assert extracted[1].payload.expected_model_names == ("b",) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py index 3c31f88dd1..ff1fe2a463 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py @@ -17,6 +17,7 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.scopes.models import DeclarationIdentity from sqlbuild.compiler.scopes.types import DeclarationKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( RebindDeclarationScopeTestCase, ) @@ -24,6 +25,8 @@ write_scoped_macro_orders_project, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + @pytest.mark.parametrize( "test_case", @@ -44,6 +47,7 @@ def test_given_same_discovery_when_rebinding_private_macros_then_index_matches_a write_scoped_macro_orders_project(tmp_path) discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) shared: DeclarationScopeBuild = build_declaration_scope( + sql_lexical_syntax=_GENERIC_SQL_SYNTAX, discovered_inputs=discovered, loaded_macros=load_project_macros(discovered.macro_files), ) @@ -56,7 +60,9 @@ def test_given_same_discovery_when_rebinding_private_macros_then_index_matches_a assert (rebound is not None) is test_case.expected_rebound assert rebound is not None fresh: DeclarationScopeBuild = build_declaration_scope( - discovered_inputs=discovered, loaded_macros=private_macros + sql_lexical_syntax=_GENERIC_SQL_SYNTAX, + discovered_inputs=discovered, + loaded_macros=private_macros, ) identity: DeclarationIdentity = DeclarationIdentity(DeclarationKind.MACRO, test_case.macro_name) private_value: object = rebound.resolver.projection.declarations[identity] @@ -88,6 +94,7 @@ def test_given_different_private_macro_metadata_when_rebinding_then_full_build_i write_scoped_macro_orders_project(tmp_path) discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) shared: DeclarationScopeBuild = build_declaration_scope( + sql_lexical_syntax=_GENERIC_SQL_SYNTAX, discovered_inputs=discovered, loaded_macros=load_project_macros(discovered.macro_files), ) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scenarios.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scenarios.py index 3b191b7ba2..f24b85846a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scenarios.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_scenarios.py @@ -4,11 +4,17 @@ import pytest +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile._helpers.attachment.sql_tests import build_scenario_inputs from sqlbuild.compiler.compile._helpers.scenarios.core import ( extract_sql_scenario_ctes, extract_sql_scenario_expected_model_names, ) +from sqlbuild.compiler.compile._helpers.sql_tests.core import extract_assertion_target_model_names from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompileSqlScenarioCtes, @@ -20,11 +26,13 @@ DiscoveredSourceFile, DiscoveredSqlScenarioFile, ) +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceEntry from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( BuildScenarioInputsErrorTestCase, BuildScenarioInputsTestCase, CteScannerMessageTestCase, + DialectCteScanTestCase, ExtractSqlScenarioCtesErrorTestCase, ExtractSqlScenarioCtesTestCase, ) @@ -32,6 +40,8 @@ DUCKDB_DECLARATION_EXPANSION_CONTEXT, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + @pytest.mark.parametrize( "test_case", @@ -145,6 +155,7 @@ def test_given_sql_scenario_cte_variants_when_extracting_then_it_returns_expecte test_case: ExtractSqlScenarioCtesTestCase, ) -> None: extracted_ctes: CompileSqlScenarioCtes = extract_sql_scenario_ctes( + syntax=_GENERIC_SQL_SYNTAX, sql=test_case.sql, file_label="tests/scenarios/revenue__customer_refund.sql", ) @@ -275,6 +286,7 @@ def test_given_invalid_sql_scenario_ctes_when_extracting_then_it_raises_clear_er ) -> None: with pytest.raises(ValueError, match=test_case.expected_error_fragment): extract_sql_scenario_ctes( + syntax=_GENERIC_SQL_SYNTAX, sql=test_case.sql, file_label="tests/scenarios/revenue__customer_refund.sql", ) @@ -362,6 +374,7 @@ def test_given_discovered_scenario_when_building_scenario_inputs_then_it_attache ) scenario_inputs: tuple[CompileSqlScenarioInput, ...] = build_scenario_inputs( + sql_lexical_syntax=_GENERIC_SQL_SYNTAX, discovered_inputs=discovered_inputs, effective_vars=test_case.effective_vars, macro_context=MacroContext( @@ -436,6 +449,7 @@ def test_given_invalid_scenario_source_refs_when_building_inputs_then_it_raises_ with pytest.raises(ValueError, match=test_case.expected_error_fragment): build_scenario_inputs( + sql_lexical_syntax=_GENERIC_SQL_SYNTAX, discovered_inputs=discovered_inputs, effective_vars={}, macro_context=MacroContext( @@ -502,7 +516,9 @@ def test_given_malformed_scenario_ctes_when_extracting_then_messages_name_the_sc test_case: CteScannerMessageTestCase, ) -> None: with pytest.raises(CompileInputError) as error_info: - _ = extract_sql_scenario_ctes(sql=test_case.sql, file_label="tests/scenarios/orders.sql") + _ = extract_sql_scenario_ctes( + sql=test_case.sql, file_label="tests/scenarios/orders.sql", syntax=_GENERIC_SQL_SYNTAX + ) assert str(error_info.value) == test_case.expected_message @@ -544,7 +560,110 @@ def test_given_malformed_scenario_ctes_when_scanning_expected_models_then_names_ ) -> None: with pytest.raises(CompileInputError) as error_info: _ = extract_sql_scenario_expected_model_names( - sql=test_case.sql, file_label="tests/scenarios/orders.sql" + sql=test_case.sql, + file_label="tests/scenarios/orders.sql", + syntax=_GENERIC_SQL_SYNTAX, ) assert str(error_info.value) == test_case.expected_message + + +@pytest.mark.parametrize( + "test_case", + [ + DialectCteScanTestCase( + description="snowflake backslash-escaped quote before a marker", + syntax=SnowflakeAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__source__raw_orders AS (SELECT 'O\\'Brien), (' AS customer_name),\n" + "__assert__named AS (\n" + " SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" + ")\n" + "SELECT 1" + ), + expected_cte_names=("__source__raw_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + DialectCteScanTestCase( + description="bigquery hash comment and triple quotes around markers", + syntax=BigQueryAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__source__raw_orders AS (SELECT '''it's ), (''' AS note), # ), (\n" + "__assert__named AS (\n" + ' SELECT * FROM __ref("order_totals") # __ref("ignored")\n' + ")\n" + "SELECT 1" + ), + expected_cte_names=("__source__raw_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + DialectCteScanTestCase( + description="databricks backslash-escaped quote before a marker", + syntax=DatabricksAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__source__raw_orders AS (SELECT 'O\\'Brien), (' AS customer_name),\n" + "__assert__named AS (\n" + " SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" + ")\n" + "SELECT 1" + ), + expected_cte_names=("__source__raw_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + DialectCteScanTestCase( + description="duckdb escape string and nested comment before a marker", + syntax=DuckDbAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__source__raw_orders AS (SELECT E'O\\'Brien), (' AS customer_name),\n" + "/* outer /* inner */ ), ( */\n" + "__assert__named AS (\n" + " SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" + ")\n" + "SELECT 1" + ), + expected_cte_names=("__source__raw_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + DialectCteScanTestCase( + description="postgres escape string before a marker", + syntax=PostgresAdapter.sql_lexical_syntax, + sql=( + "WITH\n" + "__source__raw_orders AS (SELECT E'O\\'Brien), (' AS customer_name),\n" + "__assert__named AS (\n" + " SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"order_totals\")\n" + ")\n" + "SELECT 1" + ), + expected_cte_names=("__source__raw_orders", "__assert__named"), + expected_assertion_targets=("order_totals",), + ), + ], + ids=lambda case: case.description, +) +def test_given_dialect_scenario_sql_when_extracting_ctes_then_follows_dialect_lexical_rules( + test_case: DialectCteScanTestCase, +) -> None: + extracted: CompileSqlScenarioCtes = extract_sql_scenario_ctes( + sql=test_case.sql, file_label="tests/scenarios/orders.sql", syntax=test_case.syntax + ) + + assert ( + tuple(cte.name for cte in (*extracted.authored_ctes, *extracted.assertion_ctes)) + == test_case.expected_cte_names + ) + assert ( + extract_assertion_target_model_names( + assertion_sql=tuple(cte.sql_body for cte in extracted.assertion_ctes), + syntax=test_case.syntax, + ) + == test_case.expected_assertion_targets + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py index 123f8a652d..d81aede976 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_set_operation_expected_ctes.py @@ -10,11 +10,14 @@ from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileSqlTestCte, CompileSqlTestCtes from sqlbuild.compiler.compile.types import SqlTestMode +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( SetOperationExpectedKind, SetOperationExpectedTestCase, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + _FILE: str = "tests/unit/orders.sql" _MISMATCH: str = ( "SQL test '{file}' must use the same {cte} projection names and order in every " @@ -99,7 +102,7 @@ def test_given_set_operation_expected_cte_when_classifying_then_it_is_accepted( ) classified: CompileSqlTestCtes = classify_sql_test_ctes( - ctes=ctes, file_label=_FILE, mode=test_case.kind.mode + ctes=ctes, file_label=_FILE, mode=test_case.kind.mode, syntax=_GENERIC_SQL_SYNTAX ) assert type(classified.payload).__name__ == test_case.expected_payload_type @@ -130,7 +133,9 @@ def test_given_invalid_set_operation_branch_when_classifying_then_it_names_the_e ) with pytest.raises(CompileInputError) as error_info: - _ = classify_sql_test_ctes(ctes=ctes, file_label=_FILE, mode=test_case.kind.mode) + _ = classify_sql_test_ctes( + ctes=ctes, file_label=_FILE, mode=test_case.kind.mode, syntax=_GENERIC_SQL_SYNTAX + ) assert str(error_info.value) == test_case.expected_error_template.format( file=_FILE, cte=test_case.kind.expected_cte_name diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py index 4d13a8b42b..4148fbc475 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py @@ -4,15 +4,24 @@ import pytest +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileSqlReference from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( + DialectSqlScanTestCase, SqlReferenceExtractionErrorTestCase, SqlReferenceExtractionTestCase, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + @pytest.mark.parametrize( "test_case", @@ -51,7 +60,9 @@ def test_given_simple_references_when_extracting_then_returns_authored_order( test_case: SqlReferenceExtractionTestCase, ) -> None: - references: tuple[CompileSqlReference, ...] = extract_sql_references(test_case.sql) + references: tuple[CompileSqlReference, ...] = extract_sql_references( + sql=test_case.sql, syntax=_GENERIC_SQL_SYNTAX + ) assert ( tuple( @@ -97,4 +108,90 @@ def test_given_unsupported_reference_sql_when_extracting_then_preserves_python_d test_case: SqlReferenceExtractionErrorTestCase, ) -> None: with pytest.raises(CompileInputError, match=test_case.expected_error): - extract_sql_references(test_case.sql) + extract_sql_references(sql=test_case.sql, syntax=_GENERIC_SQL_SYNTAX) + + +@pytest.mark.parametrize( + "test_case", + [ + DialectSqlScanTestCase( + description="snowflake backslash-escaped quote before a reference", + syntax=SnowflakeAdapter.sql_lexical_syntax, + sql="SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"orders\")", + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="bigquery backslash-escaped quote before a reference", + syntax=BigQueryAdapter.sql_lexical_syntax, + sql="SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"orders\")", + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="databricks backslash-escaped quote before a reference", + syntax=DatabricksAdapter.sql_lexical_syntax, + sql="SELECT 'O\\'Brien' AS customer_name, * FROM __ref(\"orders\")", + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="duckdb escape string before a reference", + syntax=DuckDbAdapter.sql_lexical_syntax, + sql="SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"orders\")", + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="postgres escape string before a reference", + syntax=PostgresAdapter.sql_lexical_syntax, + sql="SELECT E'O\\'Brien' AS customer_name, * FROM __ref(\"orders\")", + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="duckdb plain string keeps a literal backslash", + syntax=DuckDbAdapter.sql_lexical_syntax, + sql="SELECT 'C:\\' AS folder, * FROM __ref(\"orders\")", + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="databricks raw string keeps a literal backslash", + syntax=DatabricksAdapter.sql_lexical_syntax, + sql="SELECT r'C:\\' AS folder, * FROM __ref(\"orders\")", + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="bigquery triple-quoted string hides a reference", + syntax=BigQueryAdapter.sql_lexical_syntax, + sql="SELECT '''it's __ref(\"ignored\")''' AS note FROM __ref(\"orders\")", + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="bigquery hash comment hides a reference", + syntax=BigQueryAdapter.sql_lexical_syntax, + sql='SELECT 1 # __ref("ignored")\nFROM __ref("orders")', + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="snowflake slash comment hides a reference", + syntax=SnowflakeAdapter.sql_lexical_syntax, + sql='SELECT 1 // __ref("ignored")\nFROM __ref("orders")', + expected_names=("orders",), + ), + DialectSqlScanTestCase( + description="duckdb nested block comment hides a reference", + syntax=DuckDbAdapter.sql_lexical_syntax, + sql='SELECT 1 /* outer /* inner */ __ref("ignored") */ FROM __ref("orders")', + expected_names=("orders",), + ), + ], + ids=lambda case: case.description, +) +def test_given_dialect_sql_when_extracting_references_then_follows_dialect_lexical_rules( + test_case: DialectSqlScanTestCase, +) -> None: + references: tuple[CompileSqlReference, ...] = extract_sql_references( + sql=test_case.sql, syntax=test_case.syntax + ) + + assert tuple(reference.ref_name for reference in references) == test_case.expected_names + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py index 4f2e2d17ba..8071264867 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_tests.py @@ -13,12 +13,15 @@ CompileModelSqlTestCtes, ) from sqlbuild.compiler.compile.types import SqlTestMode +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( CteScannerMessageTestCase, ExtractSqlTestCtesErrorTestCase, ExtractSqlTestCtesTestCase, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + @pytest.mark.parametrize( "test_case", @@ -336,6 +339,7 @@ def test_given_model_sql_test_cte_variants_when_extracting_then_it_returns_expec test_case: ExtractSqlTestCtesTestCase, ) -> None: extracted_ctes: CompileSqlTestCtes = extract_sql_test_ctes( + syntax=_GENERIC_SQL_SYNTAX, sql=test_case.sql, file_label="tests/unit/orders.sql", mode=test_case.mode, @@ -422,6 +426,7 @@ def test_given_direct_logic_sql_test_cte_variants_when_extracting_then_it_return test_case: ExtractSqlTestCtesTestCase, ) -> None: extracted_ctes: CompileSqlTestCtes = extract_sql_test_ctes( + syntax=_GENERIC_SQL_SYNTAX, sql=test_case.sql, file_label="tests/unit/orders.sql", mode=test_case.mode, @@ -832,6 +837,7 @@ def test_given_invalid_sql_test_cte_variants_when_extracting_then_it_raises_clea ) -> None: with pytest.raises(ValueError, match=test_case.expected_error_fragment): extract_sql_test_ctes( + syntax=_GENERIC_SQL_SYNTAX, sql=test_case.sql, file_label="tests/unit/orders.sql", mode=test_case.mode, @@ -873,7 +879,9 @@ def test_given_malformed_sql_test_ctes_when_extracting_then_messages_name_the_sq test_case: CteScannerMessageTestCase, ) -> None: with pytest.raises(CompileInputError) as error_info: - _ = extract_sql_test_ctes(sql=test_case.sql, file_label="tests/unit/orders.sql") + _ = extract_sql_test_ctes( + sql=test_case.sql, file_label="tests/unit/orders.sql", syntax=_GENERIC_SQL_SYNTAX + ) assert str(error_info.value) == test_case.expected_message @@ -904,7 +912,7 @@ def test_given_malformed_sql_test_ctes_when_scanning_expected_models_then_names_ ) -> None: with pytest.raises(CompileInputError) as error_info: _ = extract_sql_test_expected_model_names( - sql=test_case.sql, file_label="tests/unit/orders.sql" + sql=test_case.sql, file_label="tests/unit/orders.sql", syntax=_GENERIC_SQL_SYNTAX ) assert str(error_info.value) == test_case.expected_message diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 78881f8001..bfc3e44d74 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -72,6 +72,7 @@ GraphResourceKind, ) from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.spec.contracts.models import ( SchemaColumn, SchemaModelEntry, @@ -94,6 +95,8 @@ SourceCursorInputColumnsTestCase, ) +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + def microbatch_model_with_cursor_end( test_case: MicrobatchCursorEndPlanTestCase, @@ -586,7 +589,7 @@ def build_scenario_from_test_case( for model_name in test_case.expected_model_names ) assertion_target_model_names: tuple[str, ...] = extract_assertion_target_model_names( - assertion_sql=test_case.assertion_sql_bodies + assertion_sql=test_case.assertion_sql_bodies, syntax=_GENERIC_SQL_SYNTAX ) return CompiledSqlScenario( key=scenario_key, diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py index a7db4a553d..9303c99533 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/_test_types.py @@ -72,3 +72,23 @@ class DialectNonCodeErrorTestCase: syntax: SqlLexicalSyntax sql: str expected_error: str + + +@dataclass(frozen=True) +class RequiresDialectScanTestCase: + """Detection of SQL a dialect lexes differently from generic SQL.""" + + description: str + syntax: SqlLexicalSyntax + sql: str + expected_reads_differently: bool + + +@dataclass(frozen=True) +class DialectMatchingParenTestCase: + """Dialect-aware parenthesis matching case.""" + + description: str + syntax: SqlLexicalSyntax | None + sql: str + expected_index: int diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py index abe21d87b2..2d45c8472c 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/_helpers/test_scanning.py @@ -6,16 +6,19 @@ from sqlbuild.compiler.sql_analysis._helpers.scanning import ( dialect_non_code_end_impl, dialect_quoted_text_end_impl, + find_matching_paren_impl, iter_code_positions_impl, skip_quoted_text_impl, ) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.unit.src.sqlbuild.compiler.sql_analysis._helpers._test_types import ( + DialectMatchingParenTestCase, DialectNonCodeErrorTestCase, DialectNonCodeSuccessTestCase, DialectQuotedTextErrorTestCase, DialectQuotedTextSuccessTestCase, IterCodePositionsTestCase, + RequiresDialectScanTestCase, SkipQuotedTextErrorTestCase, SkipQuotedTextSuccessTestCase, ) @@ -359,3 +362,98 @@ def test_given_sql_when_iterating_code_positions_then_yields_code_offsets_with_d test_case: IterCodePositionsTestCase, ) -> None: assert tuple(iter_code_positions_impl(sql=test_case.sql)) == test_case.expected_positions + + +@pytest.mark.parametrize( + "test_case", + ( + RequiresDialectScanTestCase( + description="backslash under backslash escapes", + syntax=SINGLE_QUOTE_BACKSLASH_SYNTAX, + sql="SELECT 'O\\'Brien'", + expected_reads_differently=True, + ), + RequiresDialectScanTestCase( + description="backslash under escape-string prefixes", + syntax=ESCAPE_PREFIX_SYNTAX, + sql="SELECT E'O\\'Brien'", + expected_reads_differently=True, + ), + RequiresDialectScanTestCase( + description="backslash under generic sql", + syntax=ANSI_SYNTAX, + sql="SELECT 'C:\\'", + expected_reads_differently=False, + ), + RequiresDialectScanTestCase( + description="triple quotes", + syntax=TRIPLE_QUOTE_SYNTAX, + sql='SELECT """note"""', + expected_reads_differently=True, + ), + RequiresDialectScanTestCase( + description="second block comment opener under nested comments", + syntax=NESTED_COMMENT_SYNTAX, + sql="/* a /* b */ */ SELECT 1", + expected_reads_differently=True, + ), + RequiresDialectScanTestCase( + description="single block comment under nested comments", + syntax=NESTED_COMMENT_SYNTAX, + sql="/* a */ SELECT 1", + expected_reads_differently=False, + ), + RequiresDialectScanTestCase( + description="dialect line comment prefix", + syntax=HASH_COMMENT_SYNTAX, + sql="SELECT 1 # note", + expected_reads_differently=True, + ), + RequiresDialectScanTestCase( + description="plain sql under a dialect", + syntax=SINGLE_QUOTE_BACKSLASH_SYNTAX, + sql="SELECT 'customer''s order' -- note", + expected_reads_differently=False, + ), + ), + ids=lambda case: case.description, +) +def test_given_sql_when_checking_dialect_sensitivity_then_reports_differences( + test_case: RequiresDialectScanTestCase, +) -> None: + reads_differently: bool = test_case.syntax.reads_differently_from_generic(test_case.sql) + + assert reads_differently is test_case.expected_reads_differently + + +@pytest.mark.parametrize( + "test_case", + ( + DialectMatchingParenTestCase( + description="backslash-escaped quote hides a parenthesis", + syntax=SINGLE_QUOTE_BACKSLASH_SYNTAX, + sql="('O\\') ' AS name) tail", + expected_index=len("('O\\') ' AS name"), + ), + DialectMatchingParenTestCase( + description="hash comment hides a parenthesis", + syntax=HASH_COMMENT_SYNTAX, + sql="(1 # )\n) tail", + expected_index=len("(1 # )\n"), + ), + DialectMatchingParenTestCase( + description="generic scanning keeps the backslash literal", + syntax=None, + sql="('C:\\') tail", + expected_index=len("('C:\\'"), + ), + ), + ids=lambda case: case.description, +) +def test_given_dialect_sql_when_matching_parenthesis_then_follows_dialect_rules( + test_case: DialectMatchingParenTestCase, +) -> None: + assert ( + find_matching_paren_impl(sql=test_case.sql, open_paren_index=0, syntax=test_case.syntax) + == test_case.expected_index + ) diff --git a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py index de504d1df0..efcfe9c8f6 100644 --- a/tests/unit/src/sqlbuild/integrations/dbt/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dbt/helpers.py @@ -29,6 +29,7 @@ PlanAction, PlanReason, ) +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.integrations.dbt._helpers.cli.runner import build_dbt_ls_argv from sqlbuild.integrations.dbt._helpers.graph.core import ( dbt_model_graph_key, @@ -48,6 +49,8 @@ ) from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig +_GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() + def _build_present_field(name: str, value: object) -> dict[str, object]: return {name: value} @@ -435,13 +438,14 @@ def build_compiled_project_with_model_specs( ) ), query_sql=sql, - references=extract_sql_references(sql), + references=extract_sql_references(sql=sql, syntax=_GENERIC_SQL_SYNTAX), ) ) return assemble_compiled_project( inputs=CompileProjectInputs( project_config=ProjectConfig(name="demo", adapter="duckdb"), local_config=LocalConfig(), + sql_lexical_syntax=_GENERIC_SQL_SYNTAX, discovered_inputs=DiscoveredProjectInputs( project_config=ProjectConfig(name="demo", adapter="duckdb"), local_config=LocalConfig(), diff --git a/tests/unit/src/sqlbuild/lint/_test_types.py b/tests/unit/src/sqlbuild/lint/_test_types.py index 478a55d411..ebd928eea2 100644 --- a/tests/unit/src/sqlbuild/lint/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_test_types.py @@ -4,6 +4,8 @@ from dataclasses import dataclass, field +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax + @dataclass(frozen=True) class LintBehaviorTestCase: @@ -116,3 +118,11 @@ class ReservedKeywordFoldTestCase: dialect: str keyword: str expected_same_fingerprint: bool + + +@dataclass(frozen=True) +class PassedAdapterSyntaxTestCase: + """A caller-held lexical syntax that SQL scanning must use without rediscovery.""" + + description: str + expected_syntax: SqlLexicalSyntax diff --git a/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py new file mode 100644 index 0000000000..6c42dc005f --- /dev/null +++ b/tests/unit/src/sqlbuild/lint/test_adapter_lexical_syntax.py @@ -0,0 +1,78 @@ +"""Caller-held adapter lexical syntax reaches lint and format SQL scanning.""" + +from __future__ import annotations + +from pathlib import Path +from typing import ClassVar +from unittest.mock import Mock + +import pytest + +import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module +import sqlbuild.compiler.planner._helpers.sql_tests.fixture_formatting as fixture_formatting_module +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from tests.unit.src.sqlbuild.lint._test_types import PassedAdapterSyntaxTestCase +from tests.unit.src.sqlbuild.lint.helpers import write_fixture_format_project + +_CUSTOM_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax( + triple_quoted_strings=True, line_comment_prefixes=frozenset({"--", "#"}) +) + + +class _CustomSyntaxDuckDbAdapter(DuckDbAdapter): + sql_lexical_syntax: ClassVar[SqlLexicalSyntax] = _CUSTOM_SYNTAX + + +@pytest.mark.parametrize( + "test_case", + [PassedAdapterSyntaxTestCase("fixture formatting uses the passed syntax", _CUSTOM_SYNTAX)], + ids=lambda case: case.description, +) +def test_given_passed_syntax_when_formatting_fixture_nulls_then_scanner_uses_it( + test_case: PassedAdapterSyntaxTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + test_file: Path = write_fixture_format_project( + tmp_path=tmp_path, fixture_sql="SELECT 1 AS id, CAST(NULL AS BOOLEAN) AS flag" + ) + inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + extract: Mock = Mock(wraps=fixture_formatting_module.SqlTestCteExtractor.extract) + monkeypatch.setattr(fixture_formatting_module.SqlTestCteExtractor, "extract", extract) + + _ = fixture_formatting_module.format_redundant_fixture_nulls( + files={test_file.resolve(): test_file.read_text(encoding="utf-8")}, + project_dir=tmp_path, + discovered_inputs=inputs, + sql_lexical_syntax=test_case.expected_syntax, + ) + + assert extract.call_args.kwargs["syntax"] is test_case.expected_syntax + + +@pytest.mark.parametrize( + "test_case", + [PassedAdapterSyntaxTestCase("expansion scope uses the renderer syntax", _CUSTOM_SYNTAX)], + ids=lambda case: case.description, +) +def test_given_value_renderer_when_building_expansion_context_then_scope_uses_its_syntax( + test_case: PassedAdapterSyntaxTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _ = write_fixture_format_project(tmp_path=tmp_path, fixture_sql="SELECT 1 AS id, TRUE AS flag") + build_scope: Mock = Mock(wraps=expansion_context_module.build_declaration_scope) + monkeypatch.setattr(expansion_context_module, "build_declaration_scope", build_scope) + + _ = expansion_context_module.build_sql_expansion_context( + project_dir=tmp_path, value_renderer=_CustomSyntaxDuckDbAdapter() + ) + + assert build_scope.call_args.kwargs["sql_lexical_syntax"] is test_case.expected_syntax + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/concepts/testing.mdx b/website/src/content/docs/docs/concepts/testing.mdx index 4c3c64626a..74c2470ec1 100644 --- a/website/src/content/docs/docs/concepts/testing.mdx +++ b/website/src/content/docs/docs/concepts/testing.mdx @@ -77,6 +77,13 @@ The trailing `SELECT 1` is required as a ceremonial closing statement. Any CTE without one of these prefixes is treated as a helper CTE, available to mock, model, `__expected__`, and `__assert__` SQL in the test. +SQLBuild splits test files into CTEs and finds `__ref`, `__source`, `__seed`, `__dbt_ref`, +`__udf` and `__table_fn` calls outside comments and quoted text, in tests, scenarios, models, +functions, audits and hooks alike. Quoted text and comments follow the project adapter's rules: +backslash escapes on Snowflake, BigQuery and Databricks, `E'...'` strings on DuckDB and Postgres, +nested block comments on DuckDB, Postgres, Databricks and SQL Server, `#` line comments on BigQuery, +and `//` line comments on Snowflake. + An `__expected__` CTE compares only the columns it lists, matched by name, so column order does not matter and unlisted model columns are ignored. Row counts are always compared. Listing a column the model does not output is a test error. When the expected CTE's columns are not explicit From cc4cd52bb1593e4a0040e465040e656820950c93 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 2 Oct 2026 17:02:12 +0100 Subject: [PATCH 3/7] fix(compiler): stop double-encoding JOIN USING columns in exact-name binding When a relation carries exact (quoted) column names, such as a source table read from the warehouse, binding encodes every identifier. JOIN USING columns were encoded twice, once through the SELECT and once through the JOIN node, so valid joins failed with B002 "JOIN USING column must exist on both sides". The capped microbatch dependency fixtures read a real source table again. Refs: CHI-467 --- .../_helpers/identifiers.rs | 5 -- .../cli/commands/main/build/_test_types.py | 9 +++ .../cli/commands/main/build/helpers.py | 33 +++++++++ .../build/test_capped_microbatch_build.py | 48 +++++++++++++ .../src/sqlbuild/executor/build/helpers.py | 35 +++++++--- .../compiler/sql_analysis/main/_test_types.py | 8 +++ .../main/test_schema_validation_scopes.py | 69 +++++++++++++++++++ 7 files changed, 192 insertions(+), 15 deletions(-) diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/identifiers.rs b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/identifiers.rs index 0d15bb9956..ccce73474a 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/identifiers.rs +++ b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/identifiers.rs @@ -172,11 +172,6 @@ pub(super) fn transform_statements( if let Some(with) = &mut select.with { with.ctes = fold_ctes(std::mem::take(&mut with.ctes), &authored); } - for join in &mut select.joins { - for identifier in &mut join.using { - *identifier = fold(identifier.clone(), &authored); - } - } if let Some(windows) = &mut select.windows { for window in windows { window.name = fold(window.name.clone(), &authored); diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 259876f0bf..c367a31a7f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -98,6 +98,15 @@ class CappedMicrobatchScenarioE2ETestCase: expected_exit_code: int +@dataclass(frozen=True) +class CappedFilterJoinE2ETestCase: + """Expected rows from a consumer that filters a capped producer and joins its source table.""" + + description: str + limit_action: str + expected_ids: tuple[int, ...] + + @dataclass(frozen=True) class CappedWatermarkRejectionE2ETestCase: """Expected static rejection for a capped watermark producer.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 2f47746609..60c5479ca0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -131,6 +131,39 @@ def capped_watermark_consumer_project_files(*, limit_action: str) -> dict[str, s return repo_files +def capped_filter_join_consumer_project_files(*, limit_action: str) -> dict[str, str]: + """Build a project whose microbatch consumer filters a capped producer and joins its source.""" + + repo_files: dict[str, str] = capped_microbatch_project_files(limit_action=limit_action) + repo_files["models/downstream_events.sql"] = ( + dedent( + """ + MODEL ( + materialized incremental, + incremental_strategy delete_insert, + incremental_mode microbatch, + microbatch_strategy watermark, + cursor event_time, + cursor_type timestamp, + cursor_grain day, + cursor_start '2026-01-01', + cursor_watermark_mode all, + cursor_inputs ( + capped_events (column event_time, roles [filter]), + raw_events (column event_time, roles [watermark]), + ), + batch_size 1d, + ); + SELECT capped.id, capped.event_time + FROM __ref("capped_events") AS capped + JOIN __source("raw_events") AS raw_events USING (id) + """ + ).strip() + + "\n" + ) + return repo_files + + def prepare_defer_clone_project( *, tmp_path: Path, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py index a56879249b..766986e419 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_capped_microbatch_build.py @@ -8,11 +8,13 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( CalendarGrainCursorBoundaryE2ETestCase, + CappedFilterJoinE2ETestCase, CappedMicrobatchBuildE2ETestCase, CappedMicrobatchScenarioE2ETestCase, CappedWatermarkRejectionE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + capped_filter_join_consumer_project_files, capped_microbatch_project_files, capped_watermark_consumer_project_files, ) @@ -531,5 +533,51 @@ def test_given_capped_producer_as_watermark_when_building_then_plan_rejects_befo assert target_names.isdisjoint(test_case.expected_absent_relations) +@pytest.mark.parametrize( + "test_case", + ( + CappedFilterJoinE2ETestCase( + description="join using the shared id column of a real source table", + limit_action="cap_from_end", + expected_ids=(3, 4, 5), + ), + ), + ids=lambda case: case.description, +) +def test_given_capped_filter_consumer_joining_source_table_using_id_when_building_then_join_binds( + tmp_path: Path, test_case: CappedFilterJoinE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="capped_filter_join", + repo_files=capped_filter_join_consumer_project_files(limit_action=test_case.limit_action), + ) + db_path: Path = project_dir / "regression.duckdb" + + import duckdb + + connection: duckdb.DuckDBPyConnection = duckdb.connect(str(db_path)) + connection.execute("CREATE TABLE raw_events (id INTEGER, event_time TIMESTAMP)") + connection.execute( + "INSERT INTO raw_events VALUES " + "(1, '2026-01-01 12:00:00'), (2, '2026-01-02 12:00:00'), " + "(3, '2026-01-03 12:00:00'), (4, '2026-01-04 12:00:00'), " + "(5, '2026-01-05 12:00:00')" + ) + connection.close() + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "B002" not in result.stdout + result.stderr + rows: list[tuple[object, ...]] = query_duckdb( + db_path=db_path, + sql="SELECT id FROM main.downstream_events ORDER BY id", + ) + assert tuple(row[0] for row in rows) == test_case.expected_ids + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/executor/build/helpers.py b/tests/integration/src/sqlbuild/executor/build/helpers.py index d1e949528a..05e2bb5231 100644 --- a/tests/integration/src/sqlbuild/executor/build/helpers.py +++ b/tests/integration/src/sqlbuild/executor/build/helpers.py @@ -197,6 +197,29 @@ def capped_microbatch_intermediary_sql(*, input_role: str) -> str: ) +def capped_dependency_warehouse_files(*, project_dir: Path) -> dict[str, str]: + """Create the capped graph's raw source table and return project files that read it.""" + + database_path: Path = project_dir / "warehouse.duckdb" + connection: duckdb.DuckDBPyConnection = duckdb.connect(str(database_path)) + try: + connection.execute( + "CREATE TABLE main.raw_events AS " + "SELECT 1 AS id, TIMESTAMP '2026-01-01 00:00:00' AS event_time" + ) + finally: + connection.close() + return { + "sqlbuild_project.toml": ( + 'name = "capped_dependency"\nadapter = "duckdb"\n\n' + f"[connection]\ndatabase = {json.dumps(str(database_path))}\n" + ), + "sources/raw.yml": ( + "sources:\n - name: raw_events\n schema: main\n table: raw_events\n" + ), + } + + def compile_capped_microbatch_intermediary_project( *, project_dir: Path, adapter: DuckDbAdapter, input_role: str ) -> CompilePipelineResult: @@ -205,11 +228,7 @@ def compile_capped_microbatch_intermediary_project( write_build_project_files( project_dir=project_dir, project_files={ - "sqlbuild_project.toml": 'name = "capped_dependency"\nadapter = "duckdb"\n', - "sources/raw.yml": ( - "sources:\n - name: raw_events\n expression: " - "SELECT 1 AS id, TIMESTAMP '2026-01-01 00:00:00' AS event_time\n" - ), + **capped_dependency_warehouse_files(project_dir=project_dir), "models/capped_events.sql": capped_dependency_producer_sql( action=MicrobatchLimitAction.CAP_FROM_END ), @@ -239,11 +258,7 @@ def compile_capped_dependency_project( """Compile a capped-producer project with optional intermediary views.""" project_files: dict[str, str] = { - "sqlbuild_project.toml": 'name = "capped_dependency"\nadapter = "duckdb"\n', - "sources/raw.yml": ( - "sources:\n - name: raw_events\n expression: " - "SELECT 1 AS id, TIMESTAMP '2026-01-01 00:00:00' AS event_time\n" - ), + **capped_dependency_warehouse_files(project_dir=project_dir), "models/capped_events.sql": capped_dependency_producer_sql(action=action), "models/downstream_events.sql": consumer_sql, } diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/_test_types.py index b50c6ad60d..159458e9b0 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/_test_types.py @@ -18,6 +18,14 @@ class SchemaValidationScopeTestCase: expected_diagnostic_count: int +@dataclass(frozen=True) +class ExactColumnCatalogValidationTestCase: + description: str + query_sql: str + schema: dict[str, dict[str, str]] + expected_messages: tuple[str, ...] + + @dataclass(frozen=True) class PolyglotSqlNormalizationTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_schema_validation_scopes.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_schema_validation_scopes.py index d24d314a51..a2b52ef8f6 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_schema_validation_scopes.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_schema_validation_scopes.py @@ -2,9 +2,11 @@ import pytest +from sqlbuild.compiler.sql_analysis.classes.binding_catalog import BindingCatalog from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations from sqlbuild.compiler.sql_analysis.models import SqlBindingResult, SqlSchemaValidationRequest from tests.unit.src.sqlbuild.compiler.sql_analysis.main._test_types import ( + ExactColumnCatalogValidationTestCase, SchemaValidationScopeTestCase, ) @@ -200,3 +202,70 @@ def test_given_nested_query_scopes_when_validating_complete_schema_then_resolves assert len(results) == len(test_case.dialects) assert sum(len(result.diagnostics) for result in results) == test_case.expected_diagnostic_count + + +@pytest.mark.parametrize( + "test_case", + [ + ExactColumnCatalogValidationTestCase( + description="join using a column that both exact-name relations hold", + query_sql=( + "SELECT orders.order_id FROM orders JOIN raw_orders AS raw_orders USING (order_id)" + ), + schema={ + "orders": {"order_id": "INTEGER"}, + "raw_orders": {'"order_id"': "INTEGER", '"ordered_at"': "TIMESTAMP"}, + }, + expected_messages=(), + ), + ExactColumnCatalogValidationTestCase( + description="unqualified join using across two exact-name relations", + query_sql="SELECT order_id FROM orders JOIN raw_orders USING (order_id)", + schema={ + "orders": {'"order_id"': "INTEGER"}, + "raw_orders": {'"order_id"': "INTEGER"}, + }, + expected_messages=(), + ), + ExactColumnCatalogValidationTestCase( + description="join using a column one exact-name relation lacks", + query_sql="SELECT orders.order_id FROM orders JOIN raw_orders USING (customer_id)", + schema={ + "orders": {"order_id": "INTEGER", "customer_id": "INTEGER"}, + "raw_orders": {'"order_id"': "INTEGER"}, + }, + expected_messages=( + "JOIN USING column 'customer_id' must exist on both sides (context: JOIN USING)", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_exact_name_relations_when_validating_join_using_then_matches_authored_columns( + test_case: ExactColumnCatalogValidationTestCase, +) -> None: + known_types: tuple[str, ...] = ("INTEGER", "TIMESTAMP") + catalog: BindingCatalog = BindingCatalog( + dialect="duckdb", + quoted_ignore_case=False, + known_functions=(), + known_types=known_types, + relations={}, + ) + + results: tuple[SqlBindingResult, ...] = get_schema_validations( + requests=( + SqlSchemaValidationRequest( + sql=test_case.query_sql, + dialect="duckdb", + schema=test_case.schema, + known_types=known_types, + catalog=catalog, + ), + ) + ) + + assert ( + tuple(diagnostic.message for diagnostic in results[0].diagnostics) + == test_case.expected_messages + ) From 34592e97738d32c3e300e42b6235ab4933e12866 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 2 Oct 2026 17:08:27 +0100 Subject: [PATCH 4/7] fix(cli): print a JSON diagnostic report when compile --json stops on an error When compile raised an error, such as a header parse failure, --json wrote nothing to stdout. It now prints one JSON document with has_errors, stopped, and the error's code, message and help, matching the stderr text. The exit code is unchanged and progress stays on stderr. Refs: CHI-470 --- .../sqlbuild/references/docs/cli/compile.md | 5 ++ .../cli/commands/_helpers/compile/output.py | 27 +++++++- .../_helpers/entry/dispatch_errors.py | 19 +++++- .../cli/commands/_helpers/entry/errors.py | 7 +- .../cli/commands/main/build/_test_types.py | 11 ++++ .../main/build/test_compile_json_behavior.py | 61 ++++++++++++++++++ .../cli/commands/main/compile/_test_types.py | 2 +- .../cli/commands/main/compile/helpers.py | 9 ++- .../main/compile/test_require_sql_analysis.py | 6 +- .../sqlbuild/cli/commands/main/test_rules.py | 5 +- .../cli/commands/main/entry/_test_types.py | 9 +++ .../cli/commands/main/entry/test_main.py | 64 +++++++++++++++++++ website/src/content/docs/docs/cli/compile.mdx | 5 ++ 13 files changed, 219 insertions(+), 11 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index 03525419f1..edf93ffdb7 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -146,6 +146,11 @@ Returns a structured report including: - `lineage_mode` - which lineage mode was used - `artifacts` - paths to written files +If compile stops on an error before it can build the report, such as a model header that does not +parse, `--json` still prints one JSON document. It has `command`, `has_errors: true`, +`stopped: true`, and a `diagnostics` list with that error's `code`, `message`, and `help`, the same +text the error shows on stderr. The exit code is still 1, and progress stays on stderr. + ## Column lineage modes The `--lineage-mode` flag controls how column lineage is computed during compile: diff --git a/src/sqlbuild/cli/commands/_helpers/compile/output.py b/src/sqlbuild/cli/commands/_helpers/compile/output.py index 80ecaede2b..3451e6c91e 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/output.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/output.py @@ -31,7 +31,11 @@ CompilerDiagnostic, RelatedLocation, ) -from sqlbuild.compiler.compile.types import CompiledResourceType, DiagnosticSeverity +from sqlbuild.compiler.compile.types import ( + CompiledResourceType, + DiagnosticPhase, + DiagnosticSeverity, +) from sqlbuild.compiler.discovery.constants import SQL_HOOK_OUTPUT_FIELDS from sqlbuild.compiler.discovery.main.serialize_hook_entries import serialize_hook_entries from sqlbuild.compiler.lineage.models import ProjectColumnLineage @@ -214,6 +218,27 @@ def format_compile_json( return json.dumps(result, indent=2) +def format_compile_error_json(*, code: str, message: str, help_text: str | None) -> str: + """Serialize a compile that stopped on a raised error as one JSON diagnostic report.""" + + diagnostic: CompilerDiagnostic = CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code=code, + message=message, + help=help_text, + ) + result: dict[str, object] = { + "version": _sqlbuild_version(), + "command": "compile", + "offline": True, + "has_errors": True, + "stopped": True, + "diagnostics": [_diagnostic_to_json(diagnostic)], + } + return orjson.dumps(result, option=orjson.OPT_INDENT_2).decode() + + def _summary( *, graph: ProjectGraph, diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py index 07fea40c29..5a8cab06bf 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py @@ -4,7 +4,10 @@ import sys from pathlib import Path -from sqlbuild.cli.commands._helpers.entry.errors import format_expected_error +from sqlbuild.cli.commands._helpers.entry.errors import ( + expected_error_parts, + format_expected_error, +) from sqlbuild.cli.commands._helpers.skills.update import maintain_sqlbuild_skills from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.cli.commands.exceptions import CliUserError, QueryDiffExecutionError @@ -46,6 +49,7 @@ def dispatch_and_handle_errors( ) if output_error is not None: effective_error = output_error + _write_compile_machine_error(args=args, error=effective_error, fallback_code="C000") outcome_status: object | None = getattr(effective_error, "status", None) if outcome_status is not None: print(f"Query diff outcome {outcome_status}", file=sys.stderr) @@ -58,6 +62,7 @@ def dispatch_and_handle_errors( return 1 except LintError as error: logging.getLogger("sqlbuild.cli").exception("lint failed") + _write_compile_machine_error(args=args, error=error, fallback_code="L001") print( format_expected_error(error=error, fallback_code="L001", use_color=use_color), file=sys.stderr, @@ -67,6 +72,7 @@ def dispatch_and_handle_errors( if not isinstance(error, _expected_command_failure_types()): raise logging.getLogger("sqlbuild.cli").exception("command failed") + _write_compile_machine_error(args=args, error=error, fallback_code="E001") print( format_expected_error(error=error, fallback_code="E001", use_color=use_color), file=sys.stderr, @@ -118,6 +124,17 @@ def _report_skill_freshness(*, invocation: ParsedCliInvocation) -> None: print(result.message, file=sys.stderr, end="") +def _write_compile_machine_error( + *, args: CliNamespace, error: Exception, fallback_code: str +) -> None: + if args.command != CliCommand.COMPILE or not getattr(args, "json", False): + return + from sqlbuild.cli.commands._helpers.compile.output import format_compile_error_json + + code, message, help_text = expected_error_parts(error=error, fallback_code=fallback_code) + print(format_compile_error_json(code=code, message=message, help_text=help_text)) + + def _write_diff_machine_error( *, args: CliNamespace, error: CliUserError ) -> QueryDiffExecutionError | None: diff --git a/src/sqlbuild/cli/commands/_helpers/entry/errors.py b/src/sqlbuild/cli/commands/_helpers/entry/errors.py index d9ddbe03f3..e4e12a8537 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/errors.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/errors.py @@ -19,10 +19,15 @@ class ColorAwareSqlbuildArgumentParser(SqlbuildArgumentParser): return ColorAwareSqlbuildArgumentParser -def format_expected_error(*, error: Exception, fallback_code: str, use_color: bool) -> str: +def expected_error_parts(*, error: Exception, fallback_code: str) -> tuple[str, str, str | None]: code: str = str(getattr(error, "code", fallback_code)) message: str = str(getattr(error, "message", str(error))) help_text: str | None = getattr(error, "help", None) + return code, message, help_text + + +def format_expected_error(*, error: Exception, fallback_code: str, use_color: bool) -> str: + code, message, help_text = expected_error_parts(error=error, fallback_code=fallback_code) return ( format_coded_error(code=code, message=message, help=help_text, use_color=use_color) + "\n" ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index c367a31a7f..095d87af2d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -776,6 +776,17 @@ class AuditFailureBuildE2ETestCase: expected_retained_relation_fragment: str +@dataclass(frozen=True) +class CompileJsonRaisedErrorE2ETestCase: + """Expected JSON report for a compile that stops on a raised error.""" + + description: str + repo_files: dict[str, str] + expected_code: str + expected_message_fragment: str + expected_help_fragment: str | None + + @dataclass(frozen=True) class CompileJsonBuildE2ETestCase: """Test case for compile JSON behavior.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py index 736251b3b9..2cfca30829 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py @@ -11,6 +11,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( CompileJsonBuildE2ETestCase, + CompileJsonRaisedErrorE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( prepare_inline_project, @@ -221,3 +222,63 @@ def test_given_model_with_hooks_when_running_compile_json_then_it_reports_hook_m assert hooks_by_name["notify"]["relative_path"] == "hooks/python/notify.py" assert hooks_by_name["notify"]["definition_hash"] assert hooks_by_name["notify"]["version_hash"] + + +@pytest.mark.parametrize( + "test_case", + [ + CompileJsonRaisedErrorE2ETestCase( + description="unknown materialization stops compile", + repo_files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "models/orders.sql": "MODEL (materialized tablex);\n\nSELECT 1 AS order_id\n", + }, + expected_code="P001", + expected_message_fragment="unknown materialization 'tablex'", + expected_help_fragment=None, + ), + CompileJsonRaisedErrorE2ETestCase( + description="unknown on_schema_change stops compile with help", + repo_files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "models/orders.sql": ( + "MODEL (\n materialized incremental,\n incremental_strategy append,\n" + " on_schema_change append_columns,\n);\n\nSELECT 1 AS order_id\n" + ), + }, + expected_code="P001", + expected_message_fragment="unknown on_schema_change 'append_columns'", + expected_help_fragment="on_schema_change append_new_columns,", + ), + ], + ids=lambda case: case.description, +) +def test_given_raised_compile_error_when_running_compile_json_then_stdout_holds_one_diagnostic_report( + test_case: CompileJsonRaisedErrorE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="orders", repo_files=test_case.repo_files + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("compile", "--json"), project_dir=project_dir + ) + + assert result.returncode == 1, result.stdout + result.stderr + payload: dict[str, object] = json.loads(result.stdout) + assert payload["command"] == "compile" + assert payload["has_errors"] is True + assert payload["stopped"] is True + diagnostics: list[dict[str, object]] = cast(list[dict[str, object]], payload["diagnostics"]) + assert len(diagnostics) == 1 + diagnostic: dict[str, object] = diagnostics[0] + assert diagnostic["severity"] == "error" + assert diagnostic["code"] == test_case.expected_code + message: str = cast(str, diagnostic["message"]) + assert test_case.expected_message_fragment in message + assert f"error[{test_case.expected_code}]: {message}" in result.stderr + help_text: str = cast(str, diagnostic.get("help", "")) + assert ("help" in diagnostic) is (test_case.expected_help_fragment is not None) + assert (test_case.expected_help_fragment or "") in help_text + assert all(line.strip() in result.stderr for line in help_text.splitlines()) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 8c798b12bb..8d1d5a5a68 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -443,5 +443,5 @@ class RequireSqlAnalysisCase: repo_files: tuple[tuple[str, str], ...] command: tuple[str, ...] expected_returncode: int - expected_diagnostics: tuple[tuple[str, str, int], ...] + expected_diagnostics: tuple[tuple[str, str | None, int | None], ...] expected_text_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index bd7b20fb3c..a178d3dfa3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -2584,8 +2584,11 @@ def resource_sql_project_files(files: tuple[tuple[str, str], ...]) -> dict[str, def require_sql_analysis_output( result: subprocess.CompletedProcess[str], -) -> tuple[tuple[tuple[str, str, int], ...], str]: - """Return `(code, path, line)` per JSON diagnostic and all JSON help, note and stderr text.""" +) -> tuple[tuple[tuple[str, str | None, int | None], ...], str]: + """Return `(code, path, line)` per JSON diagnostic and all JSON help, note and stderr text. + + A compile that stops on a raised error reports it without a path or line. + """ payload: dict[str, Any] = json.loads(result.stdout or '{"diagnostics": []}') diagnostics: list[dict[str, Any]] = payload["diagnostics"] @@ -2595,6 +2598,6 @@ def require_sql_analysis_output( parts.extend(item.get("notes", ())) text: str = "\n".join(parts) return ( - tuple((item["code"], item["path"], item["line"]) for item in diagnostics), + tuple((item["code"], item.get("path"), item.get("line")) for item in diagnostics), text, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_require_sql_analysis.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_require_sql_analysis.py index c24177f4b4..c34877a32f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_require_sql_analysis.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_require_sql_analysis.py @@ -104,7 +104,7 @@ ), command=_COMPILE, expected_returncode=1, - expected_diagnostics=(), + expected_diagnostics=(("D001", None, None),), expected_text_fragments=( "error[D001]:", "turns SQL analysis off with [settings] sql_analysis = false, which this project " @@ -123,7 +123,7 @@ ), command=_COMPILE, expected_returncode=1, - expected_diagnostics=(), + expected_diagnostics=(("D001", None, None),), expected_text_fragments=( "cannot set [settings] require_sql_analysis; it is a shared project policy that " "only sqlbuild_project.toml can set", @@ -141,7 +141,7 @@ ), command=_COMPILE, expected_returncode=1, - expected_diagnostics=(), + expected_diagnostics=(("P001", None, None),), expected_text_fragments=( "error[P001]: SQL syntax error in model 'order_lookup'", "= help: if this SQL is valid for your warehouse, skip SQL analysis for this " diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 139deb6db3..9c25bc3db8 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -1466,7 +1466,10 @@ def test_given_assertion_expected_dependency_when_compiling_then_compiler_reject assert exit_code == test_case.expected_exit_code captured: CaptureResult[str] = capsys.readouterr() - assert captured.out == "" + payload: dict[str, object] = json.loads(captured.out) + diagnostics: list[dict[str, object]] = cast(list[dict[str, object]], payload["diagnostics"]) + assert len(diagnostics) == 1 + assert test_case.expected_code in str(diagnostics[0]["message"]) assert test_case.expected_code in captured.err diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py index 68593ec23b..eb7202473b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py @@ -109,6 +109,15 @@ class MainTestCase: expected_dbt_init_skip_dbt_debug: bool = False +@dataclass(frozen=True) +class CompileJsonErrorRenderingTestCase: + description: str + argv: list[str] + error_factory: Callable[[], Exception] + expected_stdout_diagnostics: tuple[dict[str, object], ...] + expected_stderr_fragment: str + + @dataclass(frozen=True) class MainErrorRenderingTestCase: description: str diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 2c0e47f625..98ea5b9414 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import sys from collections.abc import Callable from pathlib import Path @@ -43,6 +44,7 @@ from sqlbuild.spec.contracts.exceptions import SpecConfigError from tests.unit.src.sqlbuild.cli.commands.main.entry._test_types import ( AuditAggregateRenderingTestCase, + CompileJsonErrorRenderingTestCase, MainErrorRenderingTestCase, MainTestCase, SkillFreshnessNoticeTestCase, @@ -2336,6 +2338,68 @@ def run_query( assert rendered_stderr.endswith("\n\n") +@pytest.mark.parametrize( + "test_case", + [ + CompileJsonErrorRenderingTestCase( + description="compile input error with help", + argv=["--project-dir", "/tmp/demo", "compile", "--json"], + error_factory=lambda: CompileInputError( + "model 'orders': unknown on_schema_change 'append_columns'", + help="use a valid on_schema_change", + ), + expected_stdout_diagnostics=( + { + "phase": "compile", + "severity": "error", + "code": "P001", + "message": "model 'orders': unknown on_schema_change 'append_columns'", + "help": "use a valid on_schema_change", + }, + ), + expected_stderr_fragment=( + "error[P001]: model 'orders': unknown on_schema_change 'append_columns'" + ), + ), + CompileJsonErrorRenderingTestCase( + description="plain value error uses the fallback code", + argv=["--project-dir", "/tmp/demo", "compile", "--json"], + error_factory=lambda: ValueError("invalid compile request"), + expected_stdout_diagnostics=( + { + "phase": "compile", + "severity": "error", + "code": "E001", + "message": "invalid compile request", + }, + ), + expected_stderr_fragment="error[E001]: invalid compile request", + ), + ], + ids=lambda case: case.description, +) +def test_given_raised_compile_error_when_running_main_then_json_mode_reports_one_diagnostic_document( + test_case: CompileJsonErrorRenderingTestCase, + capsys: pytest.CaptureFixture[str], +) -> None: + def run_compile(request: CompileCommandRequest) -> int: + del request + raise test_case.error_factory() + + exit_code: int = _main_with_dependencies( + argv=test_case.argv, handlers=build_handlers(run_compile=run_compile) + ) + captured: CaptureResult[str] = capsys.readouterr() + payload: dict[str, object] = json.loads(captured.out) + + assert exit_code == 1 + assert test_case.expected_stderr_fragment in captured.err + assert payload["command"] == "compile" + assert payload["has_errors"] is True + assert payload["stopped"] is True + assert tuple(payload["diagnostics"]) == test_case.expected_stdout_diagnostics + + @pytest.mark.parametrize( "test_case", ( diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index d5d553bbfc..a7dd474af3 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -140,6 +140,11 @@ Returns a structured report including: - `lineage_mode` - which lineage mode was used - `artifacts` - paths to written files +If compile stops on an error before it can build the report, such as a model header that does not +parse, `--json` still prints one JSON document. It has `command`, `has_errors: true`, +`stopped: true`, and a `diagnostics` list with that error's `code`, `message`, and `help`, the same +text the error shows on stderr. The exit code is still 1, and progress stays on stderr. + ## Column lineage modes The `--lineage-mode` flag controls how column lineage is computed during compile: From 08ebe215b5a81abe155de68e8b85d45b11d44b50 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 2 Oct 2026 17:29:01 +0100 Subject: [PATCH 5/7] test(executor): make the microbatch overlap test wait for overlap instead of sleeping The concurrent microbatch test relied on a 50 ms sleep for three delta stagings to overlap and saw two under load. After each model's serial first batch, the tracking adapter now holds stagings until three are active, with a 30 s timeout that fails the test with a clear message. Refs: CHI-468 --- .../concurrent/test_microbatch_concurrency.py | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py index 9e10f3c489..98193b4a7e 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_microbatch_concurrency.py @@ -3,7 +3,6 @@ from __future__ import annotations import threading -import time from pathlib import Path from textwrap import dedent from typing import Any @@ -22,10 +21,16 @@ run_concurrent_build, ) +_OVERLAP_TIMEOUT_SECONDS: float = 30.0 + class _TrackingDuckDbAdapter(DuckDbAdapter): - def __init__(self) -> None: + def __init__(self, *, overlap_target: int) -> None: self.track_delta_staging = False + self.overlap_target = overlap_target + self.overlap_wait_timed_out = False + self._overlap_reached = threading.Event() + self._serial_first_batch_models: set[str] = set() self.active_delta_staging = 0 self.max_active_delta_staging = 0 self.active_delta_models: dict[str, int] = {} @@ -53,7 +58,15 @@ def _execute(self, *, connection: Any, sql: str) -> Any: self.max_active_delta_models, len(self.active_delta_models), ) - time.sleep(0.05) + if self.active_delta_staging >= self.overlap_target: + self._overlap_reached.set() + # Each model stages its first batch serially before fanning out. + gated: bool = model_name in self._serial_first_batch_models + self._serial_first_batch_models.add(model_name) + if gated and not self._overlap_reached.wait(timeout=_OVERLAP_TIMEOUT_SECONDS): + with self._tracking_lock: + self.overlap_wait_timed_out = True + self._overlap_reached.set() try: return super()._execute(connection=connection, sql=sql) finally: @@ -167,7 +180,9 @@ def test_given_three_batch_ceiling_when_incremental_runs_then_batches_overlap( destination.parent.mkdir(parents=True, exist_ok=True) destination.write_text(contents, encoding="utf-8") - adapter: _TrackingDuckDbAdapter = _TrackingDuckDbAdapter() + adapter: _TrackingDuckDbAdapter = _TrackingDuckDbAdapter( + overlap_target=test_case.expected_max_active_batches + ) db_path: Path = tmp_path / "test.duckdb" initial_project_files: dict[str, str] = { path: contents.replace("batch_concurrency 3", "batch_concurrency 1") @@ -232,6 +247,10 @@ def test_given_three_batch_ceiling_when_incremental_runs_then_batches_overlap( adapter=adapter, ) + assert not adapter.overlap_wait_timed_out, ( + f"only {adapter.max_active_delta_staging} delta stagings were ever active together; " + f"expected {test_case.expected_max_active_batches} within {_OVERLAP_TIMEOUT_SECONDS}s" + ) assert incremental_result.status == test_case.expected_status assert adapter.max_active_delta_staging == test_case.expected_max_active_batches assert adapter.max_active_delta_models == test_case.expected_max_active_models From 8ddb3f481fd0d85cd05819b83b5bd15ea717b985 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 2 Oct 2026 17:45:12 +0100 Subject: [PATCH 6/7] fix(rules): key the Rules cache on the native build, not only the version Local development builds shared one SQLBuild version, so the Rules cache under target/rules-cache reused results from older builds until it was cleared. A build script now hashes the native crate sources and lockfile, and the ruleset fingerprint, model and custom-rule cache identities, and the SQL rule cache use the version plus that hash. Released builds keep a stable identity. Refs: CHI-471 --- crates/sqlbuild-rules-native/Cargo.toml | 3 + crates/sqlbuild-rules-native/build.rs | 81 +++++++++++++++++++ .../src/bindings/_helpers/functions.rs | 3 +- crates/sqlbuild-rules-native/src/constants.rs | 5 ++ .../src/engine/_helpers/evaluation.rs | 9 ++- .../src/engine/tests/test_evaluation.rs | 20 +++++ .../src/engine/tests/test_types.rs | 6 ++ .../src/rules/_helpers/catalogue.rs | 5 +- pyproject.toml | 1 + .../concepts/rules/execution-and-caching.md | 4 +- src/sqlbuild/_native.pyi | 2 + .../rule_engine/_helpers/run/rules.py | 8 +- .../sqlbuild/cli/commands/main/test_rules.py | 46 +++++++++++ .../concepts/rules/execution-and-caching.mdx | 4 +- 14 files changed, 184 insertions(+), 13 deletions(-) create mode 100644 crates/sqlbuild-rules-native/build.rs diff --git a/crates/sqlbuild-rules-native/Cargo.toml b/crates/sqlbuild-rules-native/Cargo.toml index de782e7349..dccde11346 100644 --- a/crates/sqlbuild-rules-native/Cargo.toml +++ b/crates/sqlbuild-rules-native/Cargo.toml @@ -30,6 +30,9 @@ stacker.workspace = true sqlparser.workspace = true toml.workspace = true +[build-dependencies] +sha2.workspace = true + [dev-dependencies] tempfile.workspace = true diff --git a/crates/sqlbuild-rules-native/build.rs b/crates/sqlbuild-rules-native/build.rs new file mode 100644 index 0000000000..a95c309a28 --- /dev/null +++ b/crates/sqlbuild-rules-native/build.rs @@ -0,0 +1,81 @@ +//! Derive a content identity for this native build so local caches notice source changes. +//! +//! The identity hashes every file under the listed crate entries plus the workspace lockfile, +//! keyed by `/`-separated paths relative to the crate root so it matches across platforms. +//! Entries that do not exist (for example the lockfile outside a workspace checkout) are left +//! out of both the hash and the `rerun-if-changed` list, because Cargo reruns a build script on +//! every build when a watched path is missing. + +use sha2::{Digest, Sha256}; +use std::path::{Component, Path, PathBuf}; + +const HASHED_ENTRIES: [&str; 3] = ["Cargo.toml", "src", "function_names"]; +const WORKSPACE_LOCKFILE: &str = "../../Cargo.lock"; + +fn main() -> Result<(), String> { + let crate_dir = + PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").map_err(|error| error.to_string())?); + let mut files: Vec<(String, PathBuf)> = Vec::new(); + for entry in HASHED_ENTRIES { + let path = crate_dir.join(entry); + if path.exists() { + println!("cargo:rerun-if-changed={}", path.display()); + collect_files(&crate_dir, &path, &mut files)?; + } + } + files.sort(); + let lockfile = crate_dir.join(WORKSPACE_LOCKFILE); + if lockfile.is_file() { + println!("cargo:rerun-if-changed={}", lockfile.display()); + files.push((WORKSPACE_LOCKFILE.to_owned(), lockfile)); + } + let mut digest = Sha256::new(); + for (label, file) in &files { + digest.update(label.as_bytes()); + digest.update([0]); + digest.update(std::fs::read(file).map_err(|error| error.to_string())?); + digest.update([0]); + } + let identity = format!("{:x}", digest.finalize()); + println!( + "cargo:rustc-env=SQLBUILD_NATIVE_SOURCE_HASH={}", + &identity[..16] + ); + Ok(()) +} + +fn portable_label(crate_dir: &Path, path: &Path) -> Result { + let relative = path + .strip_prefix(crate_dir) + .map_err(|error| error.to_string())?; + let parts: Vec = relative + .components() + .map(|component| match component { + Component::Normal(part) => Ok(part.to_string_lossy().into_owned()), + other => Err(format!("unexpected path component {other:?}")), + }) + .collect::>()?; + Ok(parts.join("/")) +} + +fn collect_files( + crate_dir: &Path, + path: &Path, + files: &mut Vec<(String, PathBuf)>, +) -> Result<(), String> { + if path.is_file() { + files.push((portable_label(crate_dir, path)?, path.to_path_buf())); + return Ok(()); + } + if !path.is_dir() { + return Ok(()); + } + for entry in std::fs::read_dir(path).map_err(|error| error.to_string())? { + collect_files( + crate_dir, + &entry.map_err(|error| error.to_string())?.path(), + files, + )?; + } + Ok(()) +} diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs index 28598b4d26..696e9578db 100644 --- a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs +++ b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs @@ -7,7 +7,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; use crate::configuration::main::load; -use crate::constants::API_VERSION; +use crate::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::engine::main::evaluate; use crate::models::CatalogueResponse; use crate::rules::main::{catalogue, selected_codes}; @@ -469,5 +469,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(render_owned_skill, module)?)?; module.add_function(wrap_pyfunction!(skill_freshness, module)?)?; module.add("API_VERSION", API_VERSION)?; + module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; Ok(()) } diff --git a/crates/sqlbuild-rules-native/src/constants.rs b/crates/sqlbuild-rules-native/src/constants.rs index bbd77bbedb..1d2d0a3c31 100644 --- a/crates/sqlbuild-rules-native/src/constants.rs +++ b/crates/sqlbuild-rules-native/src/constants.rs @@ -1,4 +1,9 @@ pub(crate) const API_VERSION: u32 = 1; +pub(crate) const NATIVE_BUILD_IDENTITY: &str = concat!( + env!("CARGO_PKG_VERSION"), + "+", + env!("SQLBUILD_NATIVE_SOURCE_HASH") +); pub(crate) const SCOPE_METADATA_SCHEMA_VERSION: u32 = 3; pub(crate) const BUILT_IN_RULE_NAMESPACE: &str = "SQBR"; pub(crate) const PIPELINE_SUBJECT: &str = "pipeline"; diff --git a/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs index 8a894db89b..4fad5b6fc6 100644 --- a/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs @@ -1,7 +1,7 @@ use crate::configuration::main::validate as config; use crate::constants::{ - API_VERSION, CUSTOM_HOST_REQUIRED_ERROR, GIT_DIRECTORY, LOGS_DIRECTORY, PYTHON_EXTENSION, - RULES_DIRECTORY, TARGET_DIRECTORY, + API_VERSION, CUSTOM_HOST_REQUIRED_ERROR, GIT_DIRECTORY, LOGS_DIRECTORY, NATIVE_BUILD_IDENTITY, + PYTHON_EXTENSION, RULES_DIRECTORY, TARGET_DIRECTORY, }; use crate::engine::_helpers::cache::{Cache, RuleCacheBucket, RuleCacheEntry}; use crate::models::{ @@ -552,7 +552,7 @@ fn custom_model_fact_identities( let mut digest = Sha256::new(); digest.update(b"custom-rule-model-facts-v1"); digest.update(ANALYSIS_BATCH_SCHEMA_VERSION.to_le_bytes()); - digest.update(env!("CARGO_PKG_VERSION").as_bytes()); + digest.update(NATIVE_BUILD_IDENTITY.as_bytes()); digest.update(serde_json::to_vec(model).map_err(|error| error.to_string())?); Ok(( model.relative_path.clone(), @@ -575,6 +575,7 @@ fn custom_model_fact_identities( fn custom_rule_identity(rule: &RuleMetadata, request: &EvaluateRequest) -> Result { let mut digest = Sha256::new(); digest.update(b"custom-rule-identity-v1"); + digest.update(NATIVE_BUILD_IDENTITY.as_bytes()); digest.update(serde_json::to_vec(rule).map_err(|error| error.to_string())?); digest.update( serde_json::to_vec(&request.config.rule_options.get(&rule.code)) @@ -623,7 +624,7 @@ fn model_cache_identity( identity: RuntimeIdentity { producer: "sqlbuild-rules".to_owned(), fact_schema: "sqlbuild-rules-model-v1".to_owned(), - runtime: env!("CARGO_PKG_VERSION").to_owned(), + runtime: NATIVE_BUILD_IDENTITY.to_owned(), rule_pack: ruleset.to_owned(), configuration: project.unwrap_or("model-local").to_owned(), }, diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs index 09ec3df1ec..ded80cec80 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs @@ -1,6 +1,7 @@ use serde_json::{Value, json}; use tempfile::TempDir; +use crate::constants::NATIVE_BUILD_IDENTITY; use crate::engine::main::evaluate::evaluate_json; use crate::engine::tests::helpers; use crate::engine::tests::test_types; @@ -1343,3 +1344,22 @@ fn given_threshold_override_change_when_evaluating_then_ruleset_fingerprint_chan } Ok(()) } + +#[test] +fn given_native_build_when_reading_cache_identity_then_version_carries_source_hash() { + let test_cases = [test_types::BuildIdentityTestCase { + description: "cache identity is the package version plus a source content hash", + expected_prefix: concat!(env!("CARGO_PKG_VERSION"), "+"), + expected_hash_length: 16, + }]; + for test_case in &test_cases { + let hash = NATIVE_BUILD_IDENTITY.strip_prefix(test_case.expected_prefix); + + assert!( + hash.is_some_and(|hash| hash.len() == test_case.expected_hash_length + && hash.bytes().all(|byte| byte.is_ascii_hexdigit())), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs index bf1a75ccb9..f2410357e2 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_types.rs @@ -106,6 +106,12 @@ pub(crate) struct ThresholdEvaluationTestCase { pub(crate) expected_codes: &'static [&'static str], } +pub(crate) struct BuildIdentityTestCase { + pub(crate) description: &'static str, + pub(crate) expected_prefix: &'static str, + pub(crate) expected_hash_length: usize, +} + pub(crate) struct ThresholdFingerprintTestCase { pub(crate) description: &'static str, pub(crate) base_config: Value, diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs index 5e19697f5e..f5105ad6cd 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/catalogue.rs @@ -1,4 +1,4 @@ -use crate::constants::API_VERSION; +use crate::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::models::RulesCodeGrammar; use crate::models::{CustomRule, ResolveRulesRequest, RuleGuidance, RuleMetadata}; use fensu_policy::policy::errors::PolicyError; @@ -487,6 +487,7 @@ pub(crate) fn fingerprint( config: &crate::models::RulesConfig, dialect: &str, ) -> Result { - let payload = serde_json::to_vec(&(rules, config, dialect)).map_err(|e| e.to_string())?; + let payload = serde_json::to_vec(&(rules, config, dialect, NATIVE_BUILD_IDENTITY)) + .map_err(|e| e.to_string())?; Ok(format!("{:x}", Sha256::digest(payload))) } diff --git a/pyproject.toml b/pyproject.toml index 4b3587418c..96b517bf77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,6 +139,7 @@ cache-keys = [ { file = "pyproject.toml" }, { file = "Cargo.toml" }, { file = "crates/sqlbuild-rules-native/Cargo.toml" }, + { file = "crates/sqlbuild-rules-native/build.rs" }, { file = "crates/sqlbuild-rules-native/src/**/*.rs" }, { file = "src/sqlbuild/_native.pyi" }, ] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md index 27d0688515..fa461cda58 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/execution-and-caching.md @@ -28,7 +28,9 @@ normal compiler before treating the complete project as valid. Rules cache under `target/rules-cache`. A cache identity includes the Rule code, implementation and imported helpers, options, invocation subject, accessed compiler facts, dialect, tracked project -observations, and compatibility versions. +observations, and compatibility versions. The SQLBuild version in that identity also carries a hash of +the native engine's source, so a local development build never reuses results from a different build +of the same version. Model-subject Rules support fine-grained reuse. Editing one model invalidates affected model subjects rather than every Rule invocation. Project-subject Rules run once and are appropriate for diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 51cd178294..118f3d2f5d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2,6 +2,8 @@ from typing import TypedDict +BUILD_IDENTITY: str + class LintPreparationRequest(TypedDict): expanded: str before_expansion: str diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index 962190784c..4022e48b90 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -8,10 +8,10 @@ import time from concurrent.futures import Executor, Future, ThreadPoolExecutor from dataclasses import asdict, dataclass, replace -from importlib.metadata import version from pathlib import Path from typing import Any +import sqlbuild._native as _native from sqlbuild.compiler.compile.constants import MACRO_TOKEN from sqlbuild.compiler.compile.models import ( CompiledModel, @@ -71,7 +71,7 @@ ) _SQL_RULE_CACHE_VERSION: str = "sql-rules-v5" -_SQLBUILD_VERSION: str = version("sqlbuild") +_RULES_BUILD_IDENTITY: str = _native.BUILD_IDENTITY _SQL_RULE_SUPPRESSION_CODE: str = "SQBRSQL000" @@ -697,7 +697,7 @@ def _sql_model_rule_identities( def _sql_rule_identity(*, model: CompiledModel, codes: tuple[str, ...], dialect: str) -> str: digest: Any = hashlib.sha256() digest.update(_SQL_RULE_CACHE_VERSION.encode()) - digest.update(_SQLBUILD_VERSION.encode()) + digest.update(_RULES_BUILD_IDENTITY.encode()) digest.update(dialect.encode()) digest.update("\0".join(codes).encode()) digest.update(model.authored_sql.encode()) @@ -735,7 +735,7 @@ def _sql_file_rule_identity( for value in ( _SQL_RULE_CACHE_VERSION, "file", - _SQLBUILD_VERSION, + _RULES_BUILD_IDENTITY, dialect, "\0".join(codes), relative_path, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 9c25bc3db8..b6c5142b08 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -646,6 +646,52 @@ def test_given_selected_sql_rule_when_compiling_then_authored_diagnostic_blocks_ assert (tmp_path / "target" / "manifest.json").is_file() +@pytest.mark.parametrize( + "test_case", + [RulesIntegrationTestCase("SQL rule cache follows the native build", 1, "SQBRSQL004")], + ids=lambda case: case.description, +) +def test_given_sql_rule_cache_from_another_native_build_when_compiling_then_rules_rerun( + test_case: RulesIntegrationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["{test_case.expected_code}"]\n', + encoding="utf-8", + ) + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + model.write_text( + 'MODEL (description "Orders");\nSELECT order_id FROM orders LIMIT 1\n', + encoding="utf-8", + ) + command: list[str] = ["--project-dir", str(tmp_path), "compile", "--json"] + + cold_exit: int = main(command) + cold: dict[str, object] = json.loads(capsys.readouterr().out) + monkeypatch.setattr( + rules_module, + "_RULES_BUILD_IDENTITY", + f"{rules_module._RULES_BUILD_IDENTITY}-rebuilt", + ) + rebuilt_exit: int = main(command) + rebuilt: dict[str, object] = json.loads(capsys.readouterr().out) + warm_exit: int = main(command) + warm: dict[str, object] = json.loads(capsys.readouterr().out) + + assert cold_exit == rebuilt_exit == warm_exit == test_case.expected_exit_code + rebuilt_timings: dict[str, int] = cast(dict[str, int], rebuilt["compile_timings"]) + warm_timings: dict[str, int] = cast(dict[str, int], warm["compile_timings"]) + assert (rebuilt_timings["rule_cache_hits"], rebuilt_timings["rule_cache_misses"]) == (0, 1) + assert (warm_timings["rule_cache_hits"], warm_timings["rule_cache_misses"]) == (1, 0) + assert [item["code"] for item in cast(list[dict[str, object]], rebuilt["diagnostics"])] == [ + test_case.expected_code + ] + assert rebuilt["diagnostics"] == cold["diagnostics"] + + @pytest.mark.parametrize( "test_case", [RulesIntegrationTestCase("warm SQL rules track non-model file edits", 1, "SQBRSQL004")], diff --git a/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx b/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx index 0d180f6d43..dcb0247ba5 100644 --- a/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx +++ b/website/src/content/docs/docs/concepts/rules/execution-and-caching.mdx @@ -25,7 +25,9 @@ normal compiler before treating the complete project as valid. Rules cache under `target/rules-cache`. A cache identity includes the Rule code, implementation and imported helpers, options, invocation subject, accessed compiler facts, dialect, tracked project -observations, and compatibility versions. +observations, and compatibility versions. The SQLBuild version in that identity also carries a hash of +the native engine's source, so a local development build never reuses results from a different build +of the same version. Model-subject Rules support fine-grained reuse. Editing one model invalidates affected model subjects rather than every Rule invocation. Project-subject Rules run once and are appropriate for From 86790dfaeb79a472b5d628490d3d4ce6b66498a2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 2 Oct 2026 19:39:13 +0100 Subject: [PATCH 7/7] test(e2e): use the valid replay_on_change forward in microbatch lifecycle fixtures --- .../commands/main/build/test_microbatch_direct_lifecycle.py | 6 +++--- .../commands/main/build/test_microbatch_failure_windows.py | 6 +++--- .../main/build/test_microbatch_replay_backfill_lifecycle.py | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py index ec17edfe84..0d71df9d74 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_direct_lifecycle.py @@ -59,7 +59,7 @@ def test_given_serial_microbatch_when_building_then_state_table_is_not_used( "models/orders.sql": timestamp_microbatch_model_sql( value_expression="payload", batch_concurrency=1, - replay_policy="forward_only", + replay_policy="forward", ), }, ) @@ -217,7 +217,7 @@ def test_given_concurrent_ceiling_when_first_run_and_full_refresh_then_target_bo "models/orders.sql": timestamp_microbatch_model_sql( value_expression="payload", batch_concurrency=3, - replay_policy="forward_only", + replay_policy="forward", ), }, ) @@ -303,7 +303,7 @@ def test_given_recreated_target_when_building_then_old_completion_generation_is_ "models/orders.sql": timestamp_microbatch_model_sql( value_expression="payload", batch_concurrency=2, - replay_policy="forward_only", + replay_policy="forward", ), }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py index f124475940..757f96b935 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_failure_windows.py @@ -59,7 +59,7 @@ def test_given_completion_write_failure_during_rebuild_when_retried_then_live_ta "models/orders.sql": timestamp_microbatch_model_sql( value_expression="payload", batch_concurrency=3, - replay_policy="forward_only", + replay_policy="forward", ), "adapters/failing_microbatch_state_duckdb.py": ( "import os\n" @@ -166,7 +166,7 @@ def test_given_concurrent_delta_audit_failure_when_fixed_then_rejected_partition "models/orders.sql": timestamp_microbatch_model_sql( value_expression="CAST(payload AS INTEGER)", batch_concurrency=3, - replay_policy="forward_only", + replay_policy="forward", extra_config=( "audits [expression_is_true (" 'name "positive_value", expression "value > 0", severity error, ' @@ -255,7 +255,7 @@ def test_given_concurrent_batches_when_audit_and_hooks_wrap_model_then_aggregate model_sql: str = timestamp_microbatch_model_sql( value_expression="payload", batch_concurrency=3, - replay_policy="forward_only", + replay_policy="forward", extra_config=( "pre_hooks [inline_sql(\"INSERT INTO main.hook_log VALUES ('pre')\")], " "post_hooks [inline_sql(\"INSERT INTO main.hook_log VALUES ('post')\")]," diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_replay_backfill_lifecycle.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_replay_backfill_lifecycle.py index 1ce0fbeb69..d6ec40454b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_replay_backfill_lifecycle.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_microbatch_replay_backfill_lifecycle.py @@ -37,7 +37,7 @@ def test_given_ahead_of_frontier_partial_backfill_when_normal_run_resumes_then_l tmp_path=tmp_path, project_name="microbatch_future_backfill", database_name="future_backfill.duckdb", - replay_policy="forward_only", + replay_policy="forward", ) execute_duckdb( db_path=db_path, @@ -120,7 +120,7 @@ def test_given_failed_backfill_tail_when_source_tail_disappears_then_no_durable_ tmp_path=tmp_path, project_name="microbatch_backfill_tail", database_name="backfill_tail.duckdb", - replay_policy="forward_only", + replay_policy="forward", ) execute_duckdb( db_path=db_path,