fix(native): match Python's backtick quoting and opaque argument equality

Author: kvlonge

crates/sqlbuild-attachments/src/audits/_helpers/rendering.rs

@@ -72,9 +72,9 @@ pub(crate) fn merged_arguments(
     Some(Ok(merged))
 }
 
-/// Python's `left != right`, or None for opaque values and numbers differing only in text.
+/// Python's `left != right`, or None for nested opaque values and numbers differing in text.
 fn python_unequal(left: &ArgumentValue, right: &ArgumentValue) -> Option<bool> {
-    if matches!(left, ArgumentValue::Opaque) || matches!(right, ArgumentValue::Opaque) {
+    if contains_opaque(left) || contains_opaque(right) {
         return None;
     }
     if left == right {
@@ -108,3 +108,12 @@ fn python_unequal(left: &ArgumentValue, right: &ArgumentValue) -> Option<bool> {
         _ => Some(true),
     }
 }
+
+/// Whether `value` holds an opaque value at any depth, which only Python can compare.
+fn contains_opaque(value: &ArgumentValue) -> bool {
+    match value {
+        ArgumentValue::Opaque => true,
+        ArgumentValue::List(items) => items.iter().any(contains_opaque),
+        _ => false,
+    }
+}

crates/sqlbuild-attachments/src/audits/tests/test_render_attached_audit.rs

@@ -56,6 +56,24 @@ fn given_attached_audits_when_rendering_then_python_rendering_is_returned() {
             },
             expected_rendering: None,
         },
+        AttachedAuditTestCase {
+            description: "an override holding a nested opaque value defers to Python's equality",
+            attachment: AuditAttachment {
+                implicit_arguments: vec![(
+                    "values".to_owned(),
+                    ArgumentValue::List(vec![ArgumentValue::Opaque]),
+                )],
+                ..attachment(
+                    vec![(
+                        "values".to_owned(),
+                        ArgumentValue::List(vec![ArgumentValue::Opaque]),
+                    )],
+                    None,
+                    None,
+                )
+            },
+            expected_rendering: None,
+        },
         AttachedAuditTestCase {
             description: "a missing argument is Python's error before the policies",
             attachment: attachment(Vec::new(), Some("fatal"), None),

crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs

@@ -58,7 +58,7 @@ fn substitute_one(sql: &str, variables: &HashMap<&str, &str>) -> (u8, Option<Str
     };
     let mut index = 0;
     while index < bytes.len() {
-        let text_end = match non_code_end(bytes, index, QuotePolicy::COMPILER) {
+        let text_end = match python_non_code_end(bytes, index) {
             Ok(Some(end)) => end,
             Ok(None) => index,
             Err(Unclosed::BlockComment) => return (UNCLOSED_BLOCK_COMMENT, None),
@@ -97,6 +97,18 @@ fn substitute_one(sql: &str, variables: &HashMap<&str, &str>) -> (u8, Option<Str
     }
 }
 
+/// Python's comment and quote end at `index`; only `'` and `"` double, so backticks never escape.
+fn python_non_code_end(bytes: &[u8], index: usize) -> Result<Option<usize>, Unclosed> {
+    if bytes[index] != b'`' {
+        return non_code_end(bytes, index, QuotePolicy::COMPILER);
+    }
+    bytes[index + 1..]
+        .iter()
+        .position(|byte| *byte == b'`')
+        .map(|offset| Some(index + 1 + offset + 1))
+        .ok_or(Unclosed::Quote)
+}
+
 /// Python's `_interpolate_sql_segment`: substitute every token inside one quoted literal.
 fn substitute_quoted_text(
     sql: &str,

crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs

@@ -336,3 +336,20 @@ pub(crate) fn parse_error_on_ordinary_thread(header: String) -> Option<String> {
 pub(crate) fn nesting_error_at(position: usize) -> String {
     format!("values nest deeper than 256 levels at position {position}")
 }
+
+pub(crate) fn doubled_backticks_close_one_segment_and_open_the_next() -> bool {
+    let sqls = vec![
+        "SELECT `@@zz``".to_owned(),
+        "SELECT `@@region``@@region`, '@@region''s'".to_owned(),
+        "SELECT `a``b` -- @@region".to_owned(),
+    ];
+    substitute_batch(&sqls, &[("region".to_owned(), "north".to_owned())])
+        == vec![
+            (UNKNOWN_VARIABLE, Some("zz".to_owned())),
+            (
+                SUBSTITUTED,
+                Some("SELECT `north``north`, 'north''s'".to_owned()),
+            ),
+            (UNCHANGED, None),
+        ]
+}

crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs

@@ -1,12 +1,19 @@
 use crate::compiler::tests::helpers::{
-    dollar_quoted_text_is_quoted_for_substitution, dynamic_or_malformed_sql_requests_fallback,
-    scalar_variables_preserve_lexical_boundaries, unclosed_dollar_quote_stops_as_unclosed_quote,
+    dollar_quoted_text_is_quoted_for_substitution,
+    doubled_backticks_close_one_segment_and_open_the_next,
+    dynamic_or_malformed_sql_requests_fallback, scalar_variables_preserve_lexical_boundaries,
+    unclosed_dollar_quote_stops_as_unclosed_quote,
 };
 use crate::compiler::tests::test_types::StaticSqlOperationTestCase;
 
 #[test]
 fn given_sql_interpolation_cases_when_substituting_then_expected_behavior_holds() {
     let test_cases = [
+        StaticSqlOperationTestCase {
+            description: "doubled backticks close one quoted segment and open the next",
+            run: doubled_backticks_close_one_segment_and_open_the_next,
+            expected_success: true,
+        },
         StaticSqlOperationTestCase {
             description: "scalar variables preserve lexical boundaries",
             run: scalar_variables_preserve_lexical_boundaries,

scripts/compiler_differential/_helpers/corpus/attachment_failure_cases.py

@@ -71,6 +71,16 @@ def attachment_failure_cases() -> tuple[FailureCase, ...]:
                 **_staging_audits("floor"),
             },
         ),
+        failure_case(
+            name="audit-variable-in-doubled-backticks",
+            expected_code="P001",
+            expected_message="unknown project variable '@@floor_amount'",
+            files={
+                **config_files('\n[vars]\nregion = "north"\n'),
+                _FLOOR_AUDIT_PATH: _FLOOR_AUDIT.replace("@minimum", "`@@floor_amount``"),
+                **_staging_audits("floor"),
+            },
+        ),
         failure_case(
             name="sql-function-missing-returns",
             expected_code="P001",

tests/e2e/src/sqlbuild/cli/commands/main/compile/attachment_errors/test_native_attachment_errors.py

@@ -67,6 +67,12 @@
             expected_macro_calls=(1, 1, 1),
             expected_error_types=_ERROR_TYPES,
         ),
+        NativeAttachmentErrorTestCase(
+            description="an unknown project variable inside doubled backticks in audit SQL",
+            case_name="audit-variable-in-doubled-backticks",
+            expected_macro_calls=(1, 1, 1),
+            expected_error_types=_ERROR_TYPES,
+        ),
         NativeAttachmentErrorTestCase(
             description="a SQL function without returns",
             case_name="sql-function-missing-returns",

tests/integration/src/sqlbuild/compiler/attachments/helpers.py

@@ -150,6 +150,11 @@ def generated_authored_sql(*, rng: random.Random) -> str:
     "@@ENV:SQB_ORDERS_REGION",
     '@const("sales_cap")',
     '@enum("order_status").PLACED',
+    "`",
+    "``",
+    "`@@missing``",
+    "`@@regions``",
+    "`@@CTX:run.target``",
 )
 
 

tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py

@@ -78,7 +78,9 @@ class StaticProjectVarDifferentialTestCase:
     seed: int
     sql_count: int
     effective_vars: dict[str, object]
+    fragments: tuple[str, ...]
     expected_minimum_dollar_quote_substitutions: int
+    expected_minimum_doubled_backtick_substitutions: int
 
 
 @dataclass(frozen=True)

tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py

@@ -833,9 +833,10 @@ class StaticProjectVarDifferential:
     native: dict[str, str]
     python: dict[str, str | None]
     substituted_dollar_quotes: int
+    substituted_doubled_backticks: int
 
 
-_LEXICAL_SQL_FRAGMENTS: tuple[str, ...] = (
+LEXICAL_SQL_FRAGMENTS: tuple[str, ...] = (
     "$",
     "$$",
     "$t$",
@@ -860,18 +861,32 @@ class StaticProjectVarDifferential:
     "_",
     "\u00e9",
 )
+BACKTICK_SQL_FRAGMENTS: tuple[str, ...] = (
+    "`",
+    "``",
+    "'",
+    "''",
+    '"',
+    "@@revision",
+    "@@status",
+    "@@",
+    " ",
+    "x",
+    "--",
+    "\n",
+)
 _LEXICAL_SQL_MAX_FRAGMENTS: int = 12
 _DIFFERENTIAL_FILE_PATH: Path = Path("models/generated.sql")
 
 
-def generated_lexical_sqls(*, seed: int, count: int) -> tuple[str, ...]:
+def generated_lexical_sqls(*, seed: int, count: int, fragments: tuple[str, ...]) -> tuple[str, ...]:
     """Generate SQL-like strings mixing quotes, dollar quotes, comments and variable tokens."""
 
     rng: random.Random = random.Random(seed)
     sqls: list[str] = []
     for _ in range(count):
         fragment_count: int = rng.randint(1, _LEXICAL_SQL_MAX_FRAGMENTS)
-        sqls.append("".join(rng.choices(_LEXICAL_SQL_FRAGMENTS, k=fragment_count)))
+        sqls.append("".join(rng.choices(fragments, k=fragment_count)))
     return tuple(sqls)
 
 
@@ -896,6 +911,9 @@ def native_static_project_var_differential(
         substituted_dollar_quotes=sum(
             sql != rendered and "$$" in sql for sql, rendered in native.items()
         ),
+        substituted_doubled_backticks=sum(
+            sql != rendered and "``" in sql for sql, rendered in native.items()
+        ),
     )