diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs index 8392ecf81c..0051e448d1 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs @@ -1,6 +1,9 @@ //! Conservative native fast path for compile-wide scalar SQL variables. +use crate::sql_scan::main::non_code_end::non_code_end; +use crate::sql_scan::models::QuotePolicy; use std::collections::HashMap; +use std::ops::Range; pub(crate) const UNCHANGED: u8 = 0; pub(crate) const SUBSTITUTED: u8 = 1; @@ -35,48 +38,23 @@ fn substitute_one(sql: &str, variables: &HashMap<&str, &str>) -> (u8, Option = None; while index < bytes.len() { - if let Some(quote_byte) = quote { - if bytes[index] == quote_byte { - if matches!(quote_byte, b'\'' | b'"') - && bytes.get(index + 1).copied() == Some(quote_byte) - { - index += 2; - continue; - } - quote = None; - index += 1; - continue; - } - if bytes[index..].starts_with(b"@@") { - let Some((end, next_state)) = substitute_token(sql, index, variables, state) else { + let text_end = match non_code_end(bytes, index, QuotePolicy::COMPILER) { + Ok(Some(end)) => end, + Ok(None) => index, + Err(_) => return (FALLBACK, None), + }; + if text_end > index { + let comment = bytes[index..].starts_with(b"--") || bytes[index..].starts_with(b"/*"); + if !comment { + let Some(next_state) = + substitute_quoted_text(sql, index..text_end, variables, state) + else { return (FALLBACK, None); }; state = next_state; - index = end; - continue; } - index += 1; - continue; - } - - if bytes[index..].starts_with(b"--") { - index = sql[index + 2..] - .find('\n') - .map_or(bytes.len(), |offset| index + 3 + offset); - continue; - } - if bytes[index..].starts_with(b"/*") { - let Some(offset) = sql[index + 2..].find("*/") else { - return (FALLBACK, None); - }; - index += offset + 4; - continue; - } - if matches!(bytes[index], b'\'' | b'"' | b'`') { - quote = Some(bytes[index]); - index += 1; + index = text_end; continue; } if bytes[index..].starts_with(b"@@") { @@ -89,9 +67,6 @@ fn substitute_one(sql: &str, variables: &HashMap<&str, &str>) -> (u8, Option { rendered.push_str(&sql[state.copy_start..]); @@ -101,6 +76,25 @@ fn substitute_one(sql: &str, variables: &HashMap<&str, &str>) -> (u8, Option, + variables: &HashMap<&str, &str>, + mut state: SubstitutionState, +) -> Option { + let mut index = text.start; + while let Some(offset) = sql[index..text.end].find("@@") { + let (token_end, next_state) = substitute_token(sql, index + offset, variables, state)?; + if token_end > text.end { + return None; + } + state = next_state; + index = token_end; + } + Some(state) +} + fn substitute_token( sql: &str, start: usize, diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs index 9a177333e3..3e168ad8a3 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs @@ -45,6 +45,40 @@ pub(crate) fn dynamic_or_malformed_sql_requests_fallback() -> bool { == vec![(FALLBACK, None); sqls.len()] } +pub(crate) fn dollar_quoted_text_is_quoted_for_substitution() -> bool { + let sqls = vec![ + "SELECT $$--@@region$$ AS x".to_owned(), + "SELECT $$ /* $$ AS a, '@@region' AS b -- */".to_owned(), + "SELECT $tag$ it's $$ -- @@region $tag$, @@region".to_owned(), + "SELECT price$1$ -- @@region\n, $1 /* @@region */".to_owned(), + "SELECT a$$b$$ -- @@region".to_owned(), + ]; + substitute_batch(&sqls, &[("region".to_owned(), "north".to_owned())]) + == vec![ + (SUBSTITUTED, Some("SELECT $$--north$$ AS x".to_owned())), + ( + SUBSTITUTED, + Some("SELECT $$ /* $$ AS a, 'north' AS b -- */".to_owned()), + ), + ( + SUBSTITUTED, + Some("SELECT $tag$ it's $$ -- north $tag$, north".to_owned()), + ), + (UNCHANGED, None), + (UNCHANGED, None), + ] +} + +pub(crate) fn unclosed_dollar_quote_requests_fallback() -> bool { + let sqls = vec![ + "SELECT $tag$ @@region".to_owned(), + "SELECT @@region, $$ open".to_owned(), + "-- @@region\nSELECT $t$ closes with another tag $u$".to_owned(), + ]; + substitute_batch(&sqls, &[("region".to_owned(), "north".to_owned())]) + == vec![(FALLBACK, None); sqls.len()] +} + pub(crate) fn simple_references_preserve_authored_order() -> bool { extract( "SELECT * FROM __source(\"orders\") UNION ALL SELECT * FROM __dbt_ref(\"shop\" , \"customers\")", diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs index fc13455259..56c6e801f1 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs @@ -1,5 +1,6 @@ use crate::compiler::tests::helpers::{ - dynamic_or_malformed_sql_requests_fallback, scalar_variables_preserve_lexical_boundaries, + dollar_quoted_text_is_quoted_for_substitution, dynamic_or_malformed_sql_requests_fallback, + scalar_variables_preserve_lexical_boundaries, unclosed_dollar_quote_requests_fallback, }; use crate::compiler::tests::test_types::StaticSqlOperationTestCase; @@ -16,6 +17,16 @@ fn given_sql_interpolation_cases_when_substituting_then_expected_behavior_holds( run: dynamic_or_malformed_sql_requests_fallback, expected_success: true, }, + StaticSqlOperationTestCase { + description: "dollar-quoted text is quoted text, not comments or code", + run: dollar_quoted_text_is_quoted_for_substitution, + expected_success: true, + }, + StaticSqlOperationTestCase { + description: "unclosed dollar quotes request Python fallback", + run: unclosed_dollar_quote_requests_fallback, + expected_success: true, + }, ]; for test_case in test_cases { diff --git a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py index cfe2c9ebcd..907b5145dd 100644 --- a/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/render_failure_cases.py @@ -145,6 +145,15 @@ def _interpolation_cases() -> tuple[FailureCase, ...]: columns="amount, status, '@@ENV:SQB_DIFFERENTIAL_UNSET' AS channel" ), ), + failure_case( + name="unclosed-dollar-quote-with-project-variable", + expected_code="P001", + expected_message="SQL interpolation contains an unclosed quoted string", + files={ + **config_files('\n[vars]\nregion = "north"\n'), + **_staging_columns(columns="amount, status, $tag$ @@region AS region"), + }, + ), failure_case( name="context-variable-in-model-sql", expected_code="P001", diff --git a/scripts/compiler_differential/classes/render_features.py b/scripts/compiler_differential/classes/render_features.py index 8ab0026dfb..04d285b573 100644 --- a/scripts/compiler_differential/classes/render_features.py +++ b/scripts/compiler_differential/classes/render_features.py @@ -208,6 +208,7 @@ def _interpolation(self) -> None: body=( "SELECT id,\n" " '@@site_label' AS site,\n" + " $$--@@site_label /* $$ AS site_note,\n" f" '@@ENV:{GENERATOR_CHANNEL_ENV_VAR}' AS channel,\n" " amount * @@discount_rate AS discounted\n" f'FROM __ref("{base.name}")\n' diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py index 203a118006..4245866f05 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py @@ -21,7 +21,6 @@ from sqlbuild.compiler.compile.constants import ( COMPILE_INPUT_READS, SQL_CONTEXT_NAME_EXTRA_TOKENS, - SQL_DOLLAR_QUOTE_TOKEN, SQL_IDENTIFIER_EXTRA_TOKEN, SQL_INTERPOLATION_TOKEN, SQL_QUOTE_TOKENS, @@ -81,7 +80,7 @@ def expand_authored_sql_result( # noqa: PLR0913 native: bool = native_stage_enabled(NativeStage.ATTACHMENTS) prepared_sql: str | None = ( prepare_static_project_vars_batch(sqls=(sql,), effective_vars=effective_vars)[0] - if native and SQL_DOLLAR_QUOTE_TOKEN not in sql + if native else None ) interpolated_sql: str = ( diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 546639e827..ff40e10148 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -89,7 +89,6 @@ SQL_STATEMENT_TERMINATOR_TOKEN: str = ";" SQL_SINGLE_QUOTE_TOKEN: str = "'" SQL_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"', "`", "$"}) -SQL_DOLLAR_QUOTE_TOKEN: str = "$" SQL_QUALIFIER_SEPARATOR_TOKEN: str = "." SQL_REFERENCE_NAME_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"'}) SQL_IDENTIFIER_EXTRA_TOKEN: str = "_" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 2e82b7c868..d8bbf66125 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -993,3 +993,11 @@ class DeepCallNestingCompileCase: depth: int engines: tuple[str, ...] expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class DollarQuotedProjectVarCompileCase: + description: str + engine: str + model_sql: str + expected_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dollar_quoted_project_vars.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dollar_quoted_project_vars.py new file mode 100644 index 0000000000..3919c76bc8 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dollar_quoted_project_vars.py @@ -0,0 +1,98 @@ +"""E2E coverage for project variables around dollar-quoted text on every compiler engine.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + DollarQuotedProjectVarCompileCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_ENGINES: tuple[str, ...] = ("python", "native", "native-preview") +_PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n\n[vars]\nregion = "north"\n' +_MODEL_HEADER: str = "MODEL (description 'Order region labels.', materialized view);\n\n" +_COMPILED_MODEL: Path = Path("target/compiled/models/order_labels.sql") + + +@pytest.mark.parametrize( + "test_case", + [ + DollarQuotedProjectVarCompileCase( + description=f"comment_markers_inside_dollar_quotes_are_text_{engine}", + engine=engine, + model_sql=( + "SELECT $$--@@region$$ AS label, $tag$ it's /* $tag$ AS note, " + "$$ /* $$ AS a, '@@region' AS b -- */\n" + ), + expected_fragment=( + "SELECT $$--north$$ AS label, $tag$ it's /* $tag$ AS note, " + "$$ /* $$ AS a, 'north' AS b -- */" + ), + ) + for engine in _ENGINES + ], + ids=lambda case: case.description, +) +def test_given_dollar_quoted_model_sql_when_compiling_then_vars_substitute_inside_quotes( + test_case: DollarQuotedProjectVarCompileCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/order_labels.sql": _MODEL_HEADER + test_case.model_sql, + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "--compiler-engine", test_case.engine, "compile"), + project_dir=project_dir, + ) + + assert result.returncode == 0, result.stdout + result.stderr + compiled_sql: str = (project_dir / _COMPILED_MODEL).read_text(encoding="utf-8") + assert test_case.expected_fragment in compiled_sql + + +@pytest.mark.parametrize( + "test_case", + [ + DollarQuotedProjectVarCompileCase( + description=f"unclosed_dollar_quote_reports_interpolation_error_{engine}", + engine=engine, + model_sql="SELECT $tag$ @@region AS label\n", + expected_fragment="error[P001]: SQL interpolation contains an unclosed quoted string", + ) + for engine in _ENGINES + ], + ids=lambda case: case.description, +) +def test_given_unclosed_dollar_quote_with_var_when_compiling_then_interpolation_reports_it( + test_case: DollarQuotedProjectVarCompileCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/order_labels.sql": _MODEL_HEADER + test_case.model_sql, + }, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "--compiler-engine", test_case.engine, "compile"), + project_dir=project_dir, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == 1, output + assert test_case.expected_fragment in output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py b/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py index f9f49c6ebe..746dcf7d9c 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/_test_types.py @@ -1,5 +1,7 @@ from __future__ import annotations +import random +from collections.abc import Callable from dataclasses import dataclass @@ -10,6 +12,7 @@ class AuthoredSqlParityTestCase: description: str seed: int count: int + generate: Callable[[random.Random], str] expected_minimum_expanded: int expected_minimum_python_errors: int diff --git a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py index cde1aa80cd..7433042c98 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/helpers.py @@ -92,6 +92,40 @@ def generated_authored_sql(*, rng: random.Random) -> str: return " ".join(parts) +_DOLLAR_PIECES: tuple[str, ...] = ( + "$", + "$$", + "$t$", + "$tag$", + "a$", + "$1", + "$1$", + "--", + "/*", + "*/", + "'", + "\n", + " ", + "@@region", + "@@limit_rows", + "@@missing", + "@@ENV:SQB_ORDERS_REGION", + '@const("sales_cap")', + '@enum("order_status").PLACED', +) + + +def generated_dollar_authored_sql(*, rng: random.Random) -> str: + """Return SQL dense in dollar quotes around variables, enum and constant references.""" + + parts: list[str] = [ + generated_reference_sql(rng=rng), + *rng.choices(_DOLLAR_PIECES, k=rng.randint(1, 10)), + ] + rng.shuffle(parts) + return "".join(parts) + + def authored_outcome( *, sql: str, engine: CompilerEngine, monkeypatch: pytest.MonkeyPatch ) -> AuthoredSqlExpansionResult | str: @@ -310,7 +344,8 @@ def _python_rendering(audit: GeneratedAudit) -> tuple[object, ...] | str: "sources/events.yml": ( "sources:\n - name: order_events\n" f" description: \"Order feed ${{coalesce(ENV:{_MISSING_ENV}, 'events')}}\"\n" - " expression: \"(SELECT 1 AS id, @tidy_label('@@region') AS region)\"\n" + " expression: \"(SELECT 1 AS id, @tidy_label('@@region') AS region, " + '$$--@@region /* $$ AS note)"\n' " columns:\n - name: id\n type: INTEGER\n audits:\n" " - accepted_values:\n values: [1, 2]\n" " - name: region\n type: VARCHAR\n" @@ -323,7 +358,7 @@ def _python_rendering(audit: GeneratedAudit) -> tuple[object, ...] | str: ), "audits/generic/amount_floor.sql": ( 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE @column < @minimum ' - "AND @tidy_label(\"label\") <> @'label'\n" + "AND @tidy_label(\"label\") <> @'label' AND $tag$--@@region$tag$ <> @'label'\n" ), "models/orders.sql": ( 'MODEL (\n materialized table,\n description "Orders.",\n' @@ -333,7 +368,8 @@ def _python_rendering(audit: GeneratedAudit) -> tuple[object, ...] | str: ), "tests/unit/test_orders.sql": ( "TEST();\n\nWITH\n__seed__channel_codes AS (\n" - " SELECT 1 AS id, @tidy_label(\"' Web '\") AS label\n),\n" + " SELECT 1 AS id, @tidy_label(\"' Web '\") AS label, $$ /* $$ AS note, " + "'@@region' AS region -- */\n),\n" "__source__order_events AS (\n SELECT 1 AS id, 'north' AS region\n),\n" "__expected__orders AS (\n SELECT 1 AS id, 1.5 AS amount, 'web' AS label\n)\n" "SELECT 1\n" diff --git a/tests/integration/src/sqlbuild/compiler/attachments/test_native_authored_sql.py b/tests/integration/src/sqlbuild/compiler/attachments/test_native_authored_sql.py index 1d6ed4a94b..cc8da56ad4 100644 --- a/tests/integration/src/sqlbuild/compiler/attachments/test_native_authored_sql.py +++ b/tests/integration/src/sqlbuild/compiler/attachments/test_native_authored_sql.py @@ -14,6 +14,7 @@ from tests.integration.src.sqlbuild.compiler.attachments.helpers import ( authored_outcome, generated_authored_sql, + generated_dollar_authored_sql, ) from tests.integration.src.sqlbuild.compiler.helpers import mismatches @@ -25,9 +26,18 @@ description="seeded variables, enum and constant references, quotes and comments", seed=20261008, count=2000, + generate=lambda rng: generated_authored_sql(rng=rng), expected_minimum_expanded=600, expected_minimum_python_errors=300, ), + AuthoredSqlParityTestCase( + description="dollar quotes around variables, enum and constant references", + seed=550, + count=2000, + generate=lambda rng: generated_dollar_authored_sql(rng=rng), + expected_minimum_expanded=300, + expected_minimum_python_errors=300, + ), ], ids=lambda case: case.description, ) @@ -35,7 +45,7 @@ def test_given_generated_authored_sql_when_expanding_with_preview_then_python_ou test_case: AuthoredSqlParityTestCase, monkeypatch: pytest.MonkeyPatch ) -> None: rng: random.Random = random.Random(test_case.seed) - sqls: list[str] = [generated_authored_sql(rng=rng) for _ in range(test_case.count)] + sqls: list[str] = [test_case.generate(rng) for _ in range(test_case.count)] python: list[AuthoredSqlExpansionResult | str] = [ authored_outcome(sql=sql, engine=CompilerEngine.PYTHON, monkeypatch=monkeypatch) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 3416f6b2c9..22280c108a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -72,6 +72,15 @@ class StaticProjectVarBatchTestCase: expected_sqls: tuple[str | None, ...] +@dataclass(frozen=True) +class StaticProjectVarDifferentialTestCase: + description: str + seed: int + sql_count: int + effective_vars: dict[str, object] + expected_minimum_dollar_quote_substitutions: int + + @dataclass(frozen=True) class AuditFactoryAttachmentTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 20adf26aff..f5d4c219df 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -1,3 +1,4 @@ +import random import sqlite3 from collections.abc import Callable from dataclasses import dataclass, replace @@ -22,6 +23,11 @@ expand_sql_macros, load_project_macros, ) +from sqlbuild.compiler.compile._helpers.render.sql_vars import ( + prepare_static_project_vars_batch, + substitute_sql_vars, +) +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( @@ -815,3 +821,85 @@ def diagnostic_spans( (location.path.as_posix(), location.line, location.column, location.end_column) for location in locations ) + + +@dataclass(frozen=True) +class StaticProjectVarDifferential: + """Native static substitutions beside the pure-Python scanner's output for the same SQL.""" + + native: dict[str, str] + python: dict[str, str | None] + substituted_dollar_quotes: int + + +_LEXICAL_SQL_FRAGMENTS: tuple[str, ...] = ( + "$", + "$$", + "$t$", + "$tag$", + "a$", + "$1", + "$1$", + "--", + "/*", + "*/", + "'", + '"', + "`", + "@@revision", + "@@status", + "@@@window_start", + "@@", + "@", + " ", + "\n", + "x", + "_", + "\u00e9", +) +_LEXICAL_SQL_MAX_FRAGMENTS: int = 12 +_DIFFERENTIAL_FILE_PATH: Path = Path("models/generated.sql") + + +def generated_lexical_sqls(*, seed: int, count: int) -> tuple[str, ...]: + """Generate SQL-like strings mixing quotes, dollar quotes, comments and variable tokens.""" + + rng: random.Random = random.Random(seed) + sqls: list[str] = [] + for _ in range(count): + fragment_count: int = rng.randint(1, _LEXICAL_SQL_MAX_FRAGMENTS) + sqls.append("".join(rng.choices(_LEXICAL_SQL_FRAGMENTS, k=fragment_count))) + return tuple(sqls) + + +def native_static_project_var_differential( + *, sqls: tuple[str, ...], effective_vars: dict[str, object] +) -> StaticProjectVarDifferential: + """Pair every native static substitution with the pure-Python scanner's output.""" + + prepared: tuple[str | None, ...] = prepare_static_project_vars_batch( + sqls=sqls, effective_vars=effective_vars + ) + native: dict[str, str] = cast( + dict[str, str], + dict(filter(lambda pair: pair[1] is not None, zip(sqls, prepared, strict=True))), + ) + python: dict[str, str | None] = { + sql: _python_substituted_sql(sql=sql, effective_vars=effective_vars) for sql in native + } + return StaticProjectVarDifferential( + native=native, + python=python, + substituted_dollar_quotes=sum( + sql != rendered and "$$" in sql for sql, rendered in native.items() + ), + ) + + +def _python_substituted_sql(*, sql: str, effective_vars: dict[str, object]) -> str | None: + try: + return substitute_sql_vars( + sql=sql, file_path=_DIFFERENTIAL_FILE_PATH, effective_vars=effective_vars + ) + except CompileInputError: + return None diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py index 06783b1633..59ae9b53a1 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_vars.py @@ -13,9 +13,15 @@ from sqlbuild.compiler.compile.exceptions import CompileInputError from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( StaticProjectVarBatchTestCase, + StaticProjectVarDifferentialTestCase, SubstituteSqlVarsErrorTestCase, SubstituteSqlVarsTestCase, ) +from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( + StaticProjectVarDifferential, + generated_lexical_sqls, + native_static_project_var_differential, +) _FILE_PATH: Path = Path("models/test_model.sql") @@ -238,6 +244,26 @@ def test_given_missing_var_when_substituting_then_raises( None, ), ), + StaticProjectVarBatchTestCase( + description="dollar-quoted text is quoted text, not comments", + sqls=( + "SELECT $$--@@revision$$ AS x", + "SELECT $$ /* $$ AS a, '@@status' AS b -- */", + "SELECT $tag$ it's $$ -- @@status $tag$, price$1$, $1, @@revision", + ), + effective_vars={"revision": 7, "status": "ready"}, + expected_sqls=( + "SELECT $$--7$$ AS x", + "SELECT $$ /* $$ AS a, 'ready' AS b -- */", + "SELECT $tag$ it's $$ -- ready $tag$, price$1$, $1, 7", + ), + ), + StaticProjectVarBatchTestCase( + description="unclosed dollar quotes fall back to the Python scanner", + sqls=("SELECT $tag$ @@revision", "-- @@revision\nSELECT $$ open"), + effective_vars={"revision": 7}, + expected_sqls=(None, None), + ), StaticProjectVarBatchTestCase( description="structured project variables fall back", sqls=("SELECT @@grants",), @@ -256,3 +282,36 @@ def test_given_static_model_sql_when_batch_preparing_then_returns_safe_results_a ) assert result == test_case.expected_sqls + + +@pytest.mark.parametrize( + "test_case", + [ + StaticProjectVarDifferentialTestCase( + description="generated quotes, dollar quotes, comments and tokens", + seed=550, + sql_count=4000, + effective_vars={"revision": 7, "status": "ready"}, + expected_minimum_dollar_quote_substitutions=1, + ) + ], + ids=lambda case: case.description, +) +def test_given_generated_lexical_sql_when_batch_preparing_then_native_matches_python_scanner( + test_case: StaticProjectVarDifferentialTestCase, +) -> None: + sqls: tuple[str, ...] = generated_lexical_sqls(seed=test_case.seed, count=test_case.sql_count) + + differential: StaticProjectVarDifferential = native_static_project_var_differential( + sqls=sqls, effective_vars=test_case.effective_vars + ) + + assert differential.native == differential.python + assert ( + differential.substituted_dollar_quotes + >= test_case.expected_minimum_dollar_quote_substitutions + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"])