fix(compiler): treat dollar-quoted strings as quoted text in variable substitution

crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs

@@ -1,6 +1,9 @@
 //! Conservative native fast path for compile-wide scalar SQL variables.
 
+use crate::sql_scan::main::non_code_end::non_code_end;
+use crate::sql_scan::models::QuotePolicy;
 use std::collections::HashMap;
+use std::ops::Range;
 
 pub(crate) const UNCHANGED: u8 = 0;
 pub(crate) const SUBSTITUTED: u8 = 1;
@@ -35,48 +38,23 @@ fn substitute_one(sql: &str, variables: &HashMap<&str, &str>) -> (u8, Option<Str
         copy_start: 0,
     };
     let mut index = 0;
-    let mut quote: Option<u8> = None;
     while index < bytes.len() {
-        if let Some(quote_byte) = quote {
-            if bytes[index] == quote_byte {
-                if matches!(quote_byte, b'\'' | b'"')
-                    && bytes.get(index + 1).copied() == Some(quote_byte)
-                {
-                    index += 2;
-                    continue;
-                }
-                quote = None;
-                index += 1;
-                continue;
-            }
-            if bytes[index..].starts_with(b"@@") {
-                let Some((end, next_state)) = substitute_token(sql, index, variables, state) else {
+        let text_end = match non_code_end(bytes, index, QuotePolicy::COMPILER) {
+            Ok(Some(end)) => end,
+            Ok(None) => index,
+            Err(_) => return (FALLBACK, None),
+        };
+        if text_end > index {
+            let comment = bytes[index..].starts_with(b"--") || bytes[index..].starts_with(b"/*");
+            if !comment {
+                let Some(next_state) =
+                    substitute_quoted_text(sql, index..text_end, variables, state)
+                else {
                     return (FALLBACK, None);
                 };
                 state = next_state;
-                index = end;
-                continue;
             }
-            index += 1;
-            continue;
-        }
-
-        if bytes[index..].starts_with(b"--") {
-            index = sql[index + 2..]
-                .find('\n')
-                .map_or(bytes.len(), |offset| index + 3 + offset);
-            continue;
-        }
-        if bytes[index..].starts_with(b"/*") {
-            let Some(offset) = sql[index + 2..].find("*/") else {
-                return (FALLBACK, None);
-            };
-            index += offset + 4;
-            continue;
-        }
-        if matches!(bytes[index], b'\'' | b'"' | b'`') {
-            quote = Some(bytes[index]);
-            index += 1;
+            index = text_end;
             continue;
         }
         if bytes[index..].starts_with(b"@@") {
@@ -89,9 +67,6 @@ fn substitute_one(sql: &str, variables: &HashMap<&str, &str>) -> (u8, Option<Str
         }
         index += 1;
     }
-    if quote.is_some() {
-        return (FALLBACK, None);
-    }
     match state.output {
         Some(mut rendered) => {
             rendered.push_str(&sql[state.copy_start..]);
@@ -101,6 +76,25 @@ fn substitute_one(sql: &str, variables: &HashMap<&str, &str>) -> (u8, Option<Str
     }
 }
 
+/// Python's `_interpolate_sql_segment`: substitute every token inside one quoted literal.
+fn substitute_quoted_text(
+    sql: &str,
+    text: Range<usize>,
+    variables: &HashMap<&str, &str>,
+    mut state: SubstitutionState,
+) -> Option<SubstitutionState> {
+    let mut index = text.start;
+    while let Some(offset) = sql[index..text.end].find("@@") {
+        let (token_end, next_state) = substitute_token(sql, index + offset, variables, state)?;
+        if token_end > text.end {
+            return None;
+        }
+        state = next_state;
+        index = token_end;
+    }
+    Some(state)
+}
+
 fn substitute_token(
     sql: &str,
     start: usize,

crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs

@@ -45,6 +45,40 @@ pub(crate) fn dynamic_or_malformed_sql_requests_fallback() -> bool {
         == vec![(FALLBACK, None); sqls.len()]
 }
 
+pub(crate) fn dollar_quoted_text_is_quoted_for_substitution() -> bool {
+    let sqls = vec![
+        "SELECT $$--@@region$$ AS x".to_owned(),
+        "SELECT $$ /* $$ AS a, '@@region' AS b -- */".to_owned(),
+        "SELECT $tag$ it's $$ -- @@region $tag$, @@region".to_owned(),
+        "SELECT price$1$ -- @@region\n, $1 /* @@region */".to_owned(),
+        "SELECT a$$b$$ -- @@region".to_owned(),
+    ];
+    substitute_batch(&sqls, &[("region".to_owned(), "north".to_owned())])
+        == vec![
+            (SUBSTITUTED, Some("SELECT $$--north$$ AS x".to_owned())),
+            (
+                SUBSTITUTED,
+                Some("SELECT $$ /* $$ AS a, 'north' AS b -- */".to_owned()),
+            ),
+            (
+                SUBSTITUTED,
+                Some("SELECT $tag$ it's $$ -- north $tag$, north".to_owned()),
+            ),
+            (UNCHANGED, None),
+            (UNCHANGED, None),
+        ]
+}
+
+pub(crate) fn unclosed_dollar_quote_requests_fallback() -> bool {
+    let sqls = vec![
+        "SELECT $tag$ @@region".to_owned(),
+        "SELECT @@region, $$ open".to_owned(),
+        "-- @@region\nSELECT $t$ closes with another tag $u$".to_owned(),
+    ];
+    substitute_batch(&sqls, &[("region".to_owned(), "north".to_owned())])
+        == vec![(FALLBACK, None); sqls.len()]
+}
+
 pub(crate) fn simple_references_preserve_authored_order() -> bool {
     extract(
         "SELECT * FROM __source(\"orders\") UNION ALL SELECT * FROM __dbt_ref(\"shop\" , \"customers\")",

crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs

@@ -1,5 +1,6 @@
 use crate::compiler::tests::helpers::{
-    dynamic_or_malformed_sql_requests_fallback, scalar_variables_preserve_lexical_boundaries,
+    dollar_quoted_text_is_quoted_for_substitution, dynamic_or_malformed_sql_requests_fallback,
+    scalar_variables_preserve_lexical_boundaries, unclosed_dollar_quote_requests_fallback,
 };
 use crate::compiler::tests::test_types::StaticSqlOperationTestCase;
 
@@ -16,6 +17,16 @@ fn given_sql_interpolation_cases_when_substituting_then_expected_behavior_holds(
             run: dynamic_or_malformed_sql_requests_fallback,
             expected_success: true,
         },
+        StaticSqlOperationTestCase {
+            description: "dollar-quoted text is quoted text, not comments or code",
+            run: dollar_quoted_text_is_quoted_for_substitution,
+            expected_success: true,
+        },
+        StaticSqlOperationTestCase {
+            description: "unclosed dollar quotes request Python fallback",
+            run: unclosed_dollar_quote_requests_fallback,
+            expected_success: true,
+        },
     ];
 
     for test_case in test_cases {

scripts/compiler_differential/_helpers/corpus/render_failure_cases.py

@@ -145,6 +145,15 @@ def _interpolation_cases() -> tuple[FailureCase, ...]:
                 columns="amount, status, '@@ENV:SQB_DIFFERENTIAL_UNSET' AS channel"
             ),
         ),
+        failure_case(
+            name="unclosed-dollar-quote-with-project-variable",
+            expected_code="P001",
+            expected_message="SQL interpolation contains an unclosed quoted string",
+            files={
+                **config_files('\n[vars]\nregion = "north"\n'),
+                **_staging_columns(columns="amount, status, $tag$ @@region AS region"),
+            },
+        ),
         failure_case(
             name="context-variable-in-model-sql",
             expected_code="P001",

scripts/compiler_differential/classes/render_features.py

@@ -208,6 +208,7 @@ def _interpolation(self) -> None:
             body=(
                 "SELECT id,\n"
                 "  '@@site_label' AS site,\n"
+                "  $$--@@site_label /* $$ AS site_note,\n"
                 f"  '@@ENV:{GENERATOR_CHANNEL_ENV_VAR}' AS channel,\n"
                 "  amount * @@discount_rate AS discounted\n"
                 f'FROM __ref("{base.name}")\n'

src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py

@@ -21,7 +21,6 @@
 from sqlbuild.compiler.compile.constants import (
     COMPILE_INPUT_READS,
     SQL_CONTEXT_NAME_EXTRA_TOKENS,
-    SQL_DOLLAR_QUOTE_TOKEN,
     SQL_IDENTIFIER_EXTRA_TOKEN,
     SQL_INTERPOLATION_TOKEN,
     SQL_QUOTE_TOKENS,
@@ -81,7 +80,7 @@ def expand_authored_sql_result(  # noqa: PLR0913
     native: bool = native_stage_enabled(NativeStage.ATTACHMENTS)
     prepared_sql: str | None = (
         prepare_static_project_vars_batch(sqls=(sql,), effective_vars=effective_vars)[0]
-        if native and SQL_DOLLAR_QUOTE_TOKEN not in sql
+        if native
         else None
     )
     interpolated_sql: str = (

src/sqlbuild/compiler/compile/constants.py

@@ -89,7 +89,6 @@
 SQL_STATEMENT_TERMINATOR_TOKEN: str = ";"
 SQL_SINGLE_QUOTE_TOKEN: str = "'"
 SQL_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"', "`", "$"})
-SQL_DOLLAR_QUOTE_TOKEN: str = "$"
 SQL_QUALIFIER_SEPARATOR_TOKEN: str = "."
 SQL_REFERENCE_NAME_QUOTE_TOKENS: frozenset[str] = frozenset({"'", '"'})
 SQL_IDENTIFIER_EXTRA_TOKEN: str = "_"

tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py

@@ -993,3 +993,11 @@ class DeepCallNestingCompileCase:
     depth: int
     engines: tuple[str, ...]
     expected_fragments: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class DollarQuotedProjectVarCompileCase:
+    description: str
+    engine: str
+    model_sql: str
+    expected_fragment: str

tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dollar_quoted_project_vars.py

@@ -0,0 +1,98 @@
+"""E2E coverage for project variables around dollar-quoted text on every compiler engine."""
+
+from __future__ import annotations
+
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import (
+    DollarQuotedProjectVarCompileCase,
+)
+from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb
+
+_ENGINES: tuple[str, ...] = ("python", "native", "native-preview")
+_PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n\n[vars]\nregion = "north"\n'
+_MODEL_HEADER: str = "MODEL (description 'Order region labels.', materialized view);\n\n"
+_COMPILED_MODEL: Path = Path("target/compiled/models/order_labels.sql")
+
+
+@pytest.mark.parametrize(
+    "test_case",
+    [
+        DollarQuotedProjectVarCompileCase(
+            description=f"comment_markers_inside_dollar_quotes_are_text_{engine}",
+            engine=engine,
+            model_sql=(
+                "SELECT $$--@@region$$ AS label, $tag$ it's /* $tag$ AS note, "
+                "$$ /* $$ AS a, '@@region' AS b -- */\n"
+            ),
+            expected_fragment=(
+                "SELECT $$--north$$ AS label, $tag$ it's /* $tag$ AS note, "
+                "$$ /* $$ AS a, 'north' AS b -- */"
+            ),
+        )
+        for engine in _ENGINES
+    ],
+    ids=lambda case: case.description,
+)
+def test_given_dollar_quoted_model_sql_when_compiling_then_vars_substitute_inside_quotes(
+    test_case: DollarQuotedProjectVarCompileCase, tmp_path: Path
+) -> None:
+    project_dir: Path = prepare_inline_project(
+        tmp_path=tmp_path,
+        project_name="orders",
+        repo_files={
+            "sqlbuild_project.toml": _PROJECT_TOML,
+            "models/order_labels.sql": _MODEL_HEADER + test_case.model_sql,
+        },
+    )
+
+    result: subprocess.CompletedProcess[str] = run_sqb(
+        command=("--no-color", "--compiler-engine", test_case.engine, "compile"),
+        project_dir=project_dir,
+    )
+
+    assert result.returncode == 0, result.stdout + result.stderr
+    compiled_sql: str = (project_dir / _COMPILED_MODEL).read_text(encoding="utf-8")
+    assert test_case.expected_fragment in compiled_sql
+
+
+@pytest.mark.parametrize(
+    "test_case",
+    [
+        DollarQuotedProjectVarCompileCase(
+            description=f"unclosed_dollar_quote_reports_interpolation_error_{engine}",
+            engine=engine,
+            model_sql="SELECT $tag$ @@region AS label\n",
+            expected_fragment="error[P001]: SQL interpolation contains an unclosed quoted string",
+        )
+        for engine in _ENGINES
+    ],
+    ids=lambda case: case.description,
+)
+def test_given_unclosed_dollar_quote_with_var_when_compiling_then_interpolation_reports_it(
+    test_case: DollarQuotedProjectVarCompileCase, tmp_path: Path
+) -> None:
+    project_dir: Path = prepare_inline_project(
+        tmp_path=tmp_path,
+        project_name="orders",
+        repo_files={
+            "sqlbuild_project.toml": _PROJECT_TOML,
+            "models/order_labels.sql": _MODEL_HEADER + test_case.model_sql,
+        },
+    )
+
+    result: subprocess.CompletedProcess[str] = run_sqb(
+        command=("--no-color", "--compiler-engine", test_case.engine, "compile"),
+        project_dir=project_dir,
+    )
+
+    output: str = result.stdout + result.stderr
+    assert result.returncode == 1, output
+    assert test_case.expected_fragment in output
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-vv"])

tests/integration/src/sqlbuild/compiler/attachments/_test_types.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import random
+from collections.abc import Callable
 from dataclasses import dataclass
 
 
@@ -10,6 +12,7 @@ class AuthoredSqlParityTestCase:
     description: str
     seed: int
     count: int
+    generate: Callable[[random.Random], str]
     expected_minimum_expanded: int
     expected_minimum_python_errors: int