diff --git a/src/sqlbuild/lint/_helpers/native_format.py b/src/sqlbuild/lint/_helpers/native_format.py index 4c3a5820ac..991abe742f 100644 --- a/src/sqlbuild/lint/_helpers/native_format.py +++ b/src/sqlbuild/lint/_helpers/native_format.py @@ -68,7 +68,9 @@ def format_native_sql_bodies( body: str = contents[body_start:body_end] trailing: str = body[len(body.rstrip()) :] core: str = body[: len(body) - len(trailing)] if trailing else body - neutralized, sites = neutralize_interpolation(body=core, dialect=config.dialect) + neutralized, sites = neutralize_interpolation( + body=core, dialect=config.dialect, for_formatting=True + ) cache_key: tuple[str, str] = (neutralized, config.dialect) requests_by_key.setdefault( cache_key, diff --git a/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py b/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py index 4cbe45af07..624f9ab04f 100644 --- a/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py +++ b/src/sqlbuild/lint/_helpers/sqlbuild_tokens.py @@ -30,10 +30,7 @@ from sqlbuild.lint.exceptions import InterpolationRestorationError from sqlbuild.lint.models import InterpolationSite -_SQLBUILD_FUNCTION_NAMES: tuple[str, ...] = ( - "__cursor_start", - "__cursor_end", - "__empty_fixture", +_SQLBUILD_REFERENCE_FUNCTION_NAMES: tuple[str, ...] = ( "__dbt_ref", "__table_fn", "__source", @@ -41,14 +38,17 @@ "__udf", "__ref", ) +_SQLBUILD_FUNCTION_NAMES: tuple[str, ...] = ( + "__cursor_start", + "__cursor_end", + "__empty_fixture", + *_SQLBUILD_REFERENCE_FUNCTION_NAMES, +) _AUDIT_PARAMETER_SENTINEL_TEMPLATE: str = "__sqlbuild_audit_parameter_{index}__" _CONTEXT_SENTINEL_TEMPLATE: str = "__sqlbuild_context_parameter_{index}__" _LINE_COMMENT_START: str = "--" _BLOCK_COMMENT_START: str = "/*" _BLOCK_COMMENT_END: str = "*/" -_SQLBUILD_FUNCTION_PATTERN: str = "|".join( - re.escape(name) for name in sorted(_SQLBUILD_FUNCTION_NAMES) -) _NON_CODE_SCAN_PATTERN: str = ( r"--[^\n]*(?:\n|\Z)" r"|/\*[\s\S]*?(?:\*/|\Z)" @@ -56,13 +56,23 @@ r'|"(?:\\.|""|[^"\\])*(?:"|\Z)' ) _BACKTICK_SCAN_PATTERN: str = r"|`(?:``|[^`])*(?:`|\Z)" -_SITE_SCAN_PATTERN: str = rf"|(?P@@|\$\{{|@|(?:{_SQLBUILD_FUNCTION_PATTERN})\s*\()" -_INTERPOLATION_SCAN_PATTERN: re.Pattern[str] = re.compile( - _NON_CODE_SCAN_PATTERN + _SITE_SCAN_PATTERN, re.IGNORECASE -) -_BACKTICK_INTERPOLATION_SCAN_PATTERN: re.Pattern[str] = re.compile( - _NON_CODE_SCAN_PATTERN + _BACKTICK_SCAN_PATTERN + _SITE_SCAN_PATTERN, re.IGNORECASE -) + + +@cache +def _interpolation_scan_pattern( + *, backtick_identifiers: bool, for_formatting: bool +) -> re.Pattern[str]: + """Keep lint fallback lexical policy identical to the native ASCII fast path.""" + + names: tuple[str, ...] = ( + _SQLBUILD_FUNCTION_NAMES if for_formatting else _SQLBUILD_REFERENCE_FUNCTION_NAMES + ) + functions: str = "|".join(re.escape(name) for name in sorted(names)) + whitespace: str = r"\s*" if for_formatting else "" + sites: str = rf"|(?P@@|\$\{{|@|(?:{functions}){whitespace}\()" + backticks: str = _BACKTICK_SCAN_PATTERN if backtick_identifiers else "" + flags: re.RegexFlag = re.IGNORECASE if for_formatting else re.NOFLAG + return re.compile(_NON_CODE_SCAN_PATTERN + backticks + sites, flags) @cache @@ -142,17 +152,15 @@ def neutralize_generic_audit_parameters(*, body: str) -> tuple[str, tuple[Interp def neutralize_interpolation( - *, body: str, dialect: str + *, body: str, dialect: str, for_formatting: bool = False ) -> tuple[str, tuple[InterpolationSite, ...]]: - """Replace every interpolation site with a unique sentinel identifier.""" + """Neutralize references for lint, or protect every intrinsic spelling for formatting.""" if not _contains_interpolation_candidate(body=body): return body, () backtick_identifiers: bool = _backtick_identifiers(dialect) - scan_pattern: re.Pattern[str] = ( - _BACKTICK_INTERPOLATION_SCAN_PATTERN - if backtick_identifiers - else _INTERPOLATION_SCAN_PATTERN + scan_pattern: re.Pattern[str] = _interpolation_scan_pattern( + backtick_identifiers=backtick_identifiers, for_formatting=for_formatting ) sites: list[InterpolationSite] = [] pieces: list[str] = [] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/playground/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/playground/_test_types.py index e0f77512c2..0bb1fa7a53 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/playground/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/playground/_test_types.py @@ -21,3 +21,10 @@ class PythonNodesPlaygroundLifecycleTestCase: expected_plan_fragments: tuple[str, ...] expected_build_fragments: tuple[str, ...] expected_check_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class UnicodeEmptyFixtureTestCase: + description: str + status: str + expected_added_findings: int = 0 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_unicode_empty_fixture.py b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_unicode_empty_fixture.py new file mode 100644 index 0000000000..3a35161fe5 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_unicode_empty_fixture.py @@ -0,0 +1,81 @@ +"""Unicode fixture exemptions through the generated playground and real CLI.""" + +import json +import re +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.playground._test_types import ( + UnicodeEmptyFixtureTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + UnicodeEmptyFixtureTestCase(text, text) + for text in ("complété", "total €", "order 😀", "complété € 😀") + ], + ids=lambda case: case.description, +) +def test_given_unicode_playground_fixture_when_running_rules_and_format_then_empty_input_is_allowed( + test_case: UnicodeEmptyFixtureTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "orders_playground" + scaffold: subprocess.CompletedProcess[str] = run_sqb( + command=("playground", str(project)), project_dir=tmp_path + ) + assert scaffold.returncode == 0, scaffold.stdout + scaffold.stderr + fixture: Path = project / "tests" / "unit" / "test_fact_orders.sql" + sql: str = fixture.read_text(encoding="utf-8") + sql = re.sub( + r"__ref__stg_payments AS \(.*?\n\)", + "__ref__stg_payments AS (\n SELECT * FROM __EMPTY_FIXTURE()\n)", + sql, + flags=re.DOTALL, + ) + fixture.write_text(sql, encoding="utf-8") + baseline: subprocess.CompletedProcess[str] = run_sqb( + command=("rules", "--json", "run", "SQBRSQL021"), project_dir=project + ) + baseline_count: int = json.loads(baseline.stdout)["finding_count"] + fixture.write_text( + sql.replace("'completed' AS status", f"'{test_case.status}' AS status"), encoding="utf-8" + ) + before: subprocess.CompletedProcess[str] = run_sqb( + command=("rules", "--json", "run", "SQBRSQL021"), project_dir=project + ) + assert before.returncode == baseline.returncode, before.stdout + before.stderr + assert ( + json.loads(before.stdout)["finding_count"] - baseline_count + == test_case.expected_added_findings + ) + assert json.loads(before.stdout)["findings"] == json.loads(baseline.stdout)["findings"] + formatted_result: subprocess.CompletedProcess[str] = run_sqb( + command=("format", "--json", str(fixture)), project_dir=project + ) + assert formatted_result.returncode == 0, formatted_result.stdout + formatted_result.stderr + formatted: str = fixture.read_text(encoding="utf-8") + assert test_case.status in formatted + assert "__EMPTY_FIXTURE()" in formatted + after: subprocess.CompletedProcess[str] = run_sqb( + command=("rules", "--json", "run", "SQBRSQL021"), project_dir=project + ) + assert after.returncode == baseline.returncode, after.stdout + after.stderr + assert ( + json.loads(after.stdout)["finding_count"] - baseline_count + == test_case.expected_added_findings + ) + checked: subprocess.CompletedProcess[str] = run_sqb( + command=("format", "--check", "--json", str(fixture)), + project_dir=project, + ) + assert checked.returncode == 0, checked.stdout + checked.stderr + assert fixture.read_text(encoding="utf-8") == formatted + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 2f4593e01b..e1175b09d9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -697,3 +697,13 @@ class FormatterSyntaxTestCase: sql: str expected_fragments: tuple[str, ...] macro_source: str = "" + + +@dataclass(frozen=True) +class UnicodeRuleLocationTestCase: + """SQL rule diagnostics expressed in authored Unicode character coordinates.""" + + description: str + rule_code: str + sql: str + expected_anchors: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py index f6b1d7e8ac..35d87a189d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py @@ -239,5 +239,62 @@ def test_given_cte_macros_and_comments_when_formatting_then_layout_and_rules_sta assert payload["findings"] == [] +@pytest.mark.parametrize( + "test_case", + [ + FormatterSyntaxTestCase( + f"Unicode restoration after {prefix}", + f"WITH orders AS (SELECT '{prefix}' AS label, order_id, payload FROM __ref(\"raw_orders\")),\n" + "values_input AS (SELECT column1 AS order_flag FROM VALUES (1)),\n" + f'-- Stage {prefix} orders.\n@copy_orders("staged_orders", "orders"),\n' + "final AS (SELECT label, " + f"CAST(order_id AS NUMERIC(/* type {prefix} */ 10, 2)) AS order_id, " + "payload:customer.name::VARCHAR AS customer_name, __cursor_start() AS start_at, " + "__cursor_end() AS end_at, FIRST_VALUE(order_id IGNORE NULLS) OVER (ORDER BY order_id) AS first_id, " + f"POSITION('{prefix}' IN label) AS label_position FROM staged_orders s CROSS JOIN values_input) SELECT * FROM final", + ( + f"'{prefix}'", + f"-- Stage {prefix} orders.", + f"/* type {prefix} */", + "NUMERIC(", + "payload:customer.name::VARCHAR", + '__ref("raw_orders")', + "__cursor_start()", + "__cursor_end()", + "FROM VALUES (1)", + "FIRST_VALUE(order_id IGNORE NULLS)", + f"POSITION('{prefix}' IN label)", + '\n@copy_orders("staged_orders", "orders"),\n', + "\nfinal AS (", + ), + ) + for prefix in ("é", "€", "😀", "é € 😀") + ], + ids=lambda case: case.description, +) +def test_given_multibyte_text_before_restorations_when_formatting_then_output_is_preserved( + test_case: FormatterSyntaxTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "snowflake"\n') + macros: Path = tmp_path / "models" / "_macros" + macros.mkdir(parents=True) + (macros / "orders.py").write_text( + "def copy_orders(name: str, source: str) -> str:\n" + ' return f"{name} AS (SELECT * FROM {source})"\n' + ) + model: Path = tmp_path / "models" / "order_summary.sql" + model.write_text( + 'MODEL (description "Orders é € 😀");\n' + test_case.sql + "\n", encoding="utf-8" + ) + assert main(["--project-dir", str(tmp_path), "format", "--json"]) == 0 + payload: dict[str, object] = json.loads(capsys.readouterr().out) + assert payload["faults"] == 0 + formatted: str = model.read_text(encoding="utf-8") + assert all(fragment in formatted for fragment in test_case.expected_fragments) + assert 'description "Orders é € 😀"' in formatted + assert main(["--project-dir", str(tmp_path), "format", "--check", "--json"]) == 0 + assert model.read_text(encoding="utf-8") == formatted + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_unicode_lint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_unicode_lint.py new file mode 100644 index 0000000000..cba4218412 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_unicode_lint.py @@ -0,0 +1,133 @@ +"""Unicode code-point locations through the real compiler/Rules CLI boundary.""" + +import json +from itertools import product +from pathlib import Path +from typing import Any + +import pytest + +import sqlbuild._native as _native +from sqlbuild.cli.commands.main.entrypoint.entry import main +from tests.integration.src.sqlbuild.cli.commands.main._test_types import UnicodeRuleLocationTestCase + + +@pytest.mark.parametrize( + "test_case", + [ + UnicodeRuleLocationTestCase( + f"{case.description} after {prefix}", + case.rule_code, + case.sql.replace("LABEL", prefix), + case.expected_anchors, + ) + for prefix, case in product( + ("é", "€", "😀", "é € 😀"), + ( + UnicodeRuleLocationTestCase( + "uncontrolled star", + "SQBRSQL021", + "SELECT 'LABEL' AS label, * FROM orders", + ("* FROM",), + ), + UnicodeRuleLocationTestCase( + "controlled projection", "SQBRSQL021", "SELECT 'LABEL' AS label", () + ), + UnicodeRuleLocationTestCase( + "computed terminal", + "SQBRSQL035", + "WITH final AS (SELECT 'LABEL' AS label) SELECT label || 'x' AS label FROM final", + ("SELECT label ||",), + ), + UnicodeRuleLocationTestCase( + "plain terminal", + "SQBRSQL035", + "((WITH final AS (SELECT 'LABEL' AS label) SELECT label FROM final))", + (), + ), + UnicodeRuleLocationTestCase( + "nested CTE", + "SQBRSQL036", + "SELECT 'LABEL' AS label, (WITH nested_orders AS (SELECT 1 AS order_id) SELECT order_id FROM nested_orders) AS order_id", + ("WITH nested_orders",), + ), + UnicodeRuleLocationTestCase( + "root wrapper", + "SQBRSQL036", + "((WITH final AS (SELECT 'LABEL' AS label) SELECT label FROM final))", + (), + ), + UnicodeRuleLocationTestCase( + "computed join", + "SQBRSQL040", + "WITH orders AS (SELECT 'LABEL' AS label) SELECT o.label FROM orders AS o JOIN orders AS c ON LOWER(o.label) = c.label", + ("ON LOWER",), + ), + UnicodeRuleLocationTestCase( + "plain join", + "SQBRSQL040", + "WITH orders AS (SELECT 'LABEL' AS label) SELECT o.label FROM orders AS o JOIN orders AS c ON o.label = c.label", + (), + ), + UnicodeRuleLocationTestCase( + "wrong final name", + "SQBRSQL041", + "WITH marker AS (SELECT 'LABEL' AS label), orders AS (SELECT label FROM marker) SELECT label FROM orders", + ("orders AS",), + ), + UnicodeRuleLocationTestCase( + "valid final name", + "SQBRSQL041", + "((WITH marker AS (SELECT 'LABEL' AS label), final AS (SELECT label FROM marker) SELECT label FROM final))", + (), + ), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_unicode_before_sql_when_running_rules_then_findings_have_character_locations( + test_case: UnicodeRuleLocationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + contents: str = 'MODEL (description "Orders");\n' + test_case.sql + "\n" + model.write_text(contents, encoding="utf-8") + native_payload: dict[str, Any] = json.loads( + _native.lint_sql_json( + json.dumps( + { + "version": 1, + "sql": test_case.sql, + "dialect": "duckdb", + "enabled_rules": [test_case.rule_code], + } + ) + ) + ) + assert [(finding["start"], finding["end"]) for finding in native_payload["diagnostics"]] == [ + (test_case.sql.index(anchor), test_case.sql.index(anchor) + len(anchor.split()[0])) + for anchor in test_case.expected_anchors + ] + result: int = main( + ["--project-dir", str(tmp_path), "rules", "--json", "run", test_case.rule_code] + ) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + expected_locations: list[tuple[int, int]] = [ + ( + contents.count("\n", 0, contents.index(anchor)) + 1, + contents.index(anchor) - contents.rfind("\n", 0, contents.index(anchor)), + ) + for anchor in test_case.expected_anchors + ] + assert result == int(bool(test_case.expected_anchors)) + assert [ + (finding["line"], finding["column"]) for finding in payload["findings"] + ] == expected_locations + assert all(finding["code"] == test_case.rule_code for finding in payload["findings"]) + assert all(finding["path"] == "models/orders.sql" for finding in payload["findings"]) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py index c55bb96eac..3d70633f8c 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py @@ -175,3 +175,12 @@ class RestoreFailureTestCase: body: str fixed_neutralized: str expected_message_fragment: str + + +@dataclass(frozen=True) +class UnicodePreparationTestCase: + """A Unicode prefix followed by formatter-sensitive SQLBuild intrinsics.""" + + description: str + prefix: str + expected_lint_calls: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py b/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py index 7d4ed6267d..28eef1a300 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_sqlbuild_tokens.py @@ -16,9 +16,60 @@ NeutralizeInterpolationTestCase, RestoreFailureTestCase, RestoreInterpolationTestCase, + UnicodePreparationTestCase, ) +@pytest.mark.parametrize( + "test_case", + [ + UnicodePreparationTestCase(label, label, ('__ref("orders")',)) + for label in ("ASCII", "é", "€", "😀", "é € 😀") + ], + ids=lambda case: case.description, +) +def test_given_unicode_before_intrinsics_when_preparing_then_lint_preserves_semantic_calls( + test_case: UnicodePreparationTestCase, +) -> None: + sql: str = ( + f"SELECT '{test_case.prefix}' AS label, __cursor_start(), __cursor_end() " + 'FROM __EMPTY_FIXTURE(); SELECT * FROM __ref("orders")' + ) + lint_sql, lint_sites = neutralize_interpolation(body=sql, dialect="duckdb") + format_sql, format_sites = neutralize_interpolation( + body=sql, dialect="duckdb", for_formatting=True + ) + assert tuple(site.original_text for site in lint_sites) == test_case.expected_lint_calls + assert "__EMPTY_FIXTURE()" in lint_sql + assert "__cursor_start()" in lint_sql + assert "__cursor_end()" in lint_sql + assert lint_sites[0].original_start == sql.index('__ref("orders")') + assert ( + lint_sql[lint_sites[0].neutralized_start : lint_sites[0].neutralized_end] + == lint_sites[0].sentinel + ) + assert len(format_sites) == 4 + assert restore_interpolation(fixed=format_sql, sites=format_sites) == sql + assert all( + sql[site.original_start : site.original_end] == site.original_text for site in format_sites + ) + ascii_prefix: str = "a" * len(test_case.prefix) + ascii_sql: str = sql.replace(f"'{test_case.prefix}'", f"'{ascii_prefix}'", 1) + native: tuple[str, list[tuple[str, int, int, int, int, str]], list[str]] | None = ( + _native.prepare_lint_sql( + { + "expanded": ascii_sql, + "before_expansion": ascii_sql, + "prior_sites": [], + "dialect": "duckdb", + } + ) + ) + assert native is not None + assert native[0].replace(f"'{ascii_prefix}'", f"'{test_case.prefix}'", 1) == lint_sql + assert tuple(InterpolationSite(*site) for site in native[1]) == lint_sites + + @pytest.mark.parametrize( "test_case", [ @@ -132,6 +183,18 @@ expected_neutralized="SELECT __custom(value) FROM items", expected_original_texts=(), ), + NeutralizeInterpolationTestCase( + description="lint retains semantic intrinsics in either case", + body="SELECT __cursor_start(), __cursor_end() FROM __empty_fixture(); SELECT * FROM __EMPTY_FIXTURE()", + expected_neutralized="SELECT __cursor_start(), __cursor_end() FROM __empty_fixture(); SELECT * FROM __EMPTY_FIXTURE()", + expected_original_texts=(), + ), + NeutralizeInterpolationTestCase( + description="lint matches native reference spelling policy", + body='SELECT * FROM __REF("orders"); SELECT * FROM __ref ("orders")', + expected_neutralized='SELECT * FROM __REF("orders"); SELECT * FROM __ref ("orders")', + expected_original_texts=(), + ), NeutralizeInterpolationTestCase( description="body without interpolation is unchanged", body="SELECT 1 AS x FROM t",