From fd47b282afa69e437c79142fcdfcb9183ee9a6d9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:25:15 +0100 Subject: [PATCH 01/14] refactor: drop the unreachable located reference-scan path from Python Production only extracts references without an origin from Python (typed macro arguments); located scan errors and P012 locations are produced by the native model, function and hook loops. The Python origin mapping, its SqlReferenceOrigin/SqlReferenceSourceMap models and the refs.native wrapper are deleted; their located cases stay covered by the Rust locating tests and the E2E reference-error suites. --- python-boundary.toml | 2 +- .../compiler/compile/_helpers/refs/native.py | 66 ----- .../compile/_helpers/refs/references.py | 241 +++--------------- src/sqlbuild/compiler/compile/models.py | 20 -- .../compile/_helpers/refs/_test_types.py | 14 - .../compiler/compile/_helpers/refs/helpers.py | 13 - .../_helpers/refs/test_native_references.py | 162 +----------- .../compiler/compile/_helpers/_test_types.py | 3 +- .../compile/_helpers/test_sql_references.py | 62 +---- 9 files changed, 63 insertions(+), 520 deletions(-) delete mode 100644 src/sqlbuild/compiler/compile/_helpers/refs/native.py delete mode 100644 tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/helpers.py diff --git a/python-boundary.toml b/python-boundary.toml index 3bef235801..7114a28c41 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 130215 +remaining_lines = 129970 [[stays]] path = "executor" diff --git a/src/sqlbuild/compiler/compile/_helpers/refs/native.py b/src/sqlbuild/compiler/compile/_helpers/refs/native.py deleted file mode 100644 index bcb6cb5038..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/refs/native.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Logical SQL references extracted by the native scanner.""" - -from __future__ import annotations - -from functools import cache -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import ( - CompileSqlReference, - InvalidSqlReferenceCall, - SqlReferenceScan, -) -from sqlbuild.compiler.compile.types import SqlReferenceScanFailure -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - -type _NativeExtraction = ( - tuple[ - tuple[ - list[tuple[str, str, str | None, int | None]], - list[tuple[str, str, int, str, str, str]], - ], - None, - ] - | tuple[None, tuple[str, int]] -) - - -def extract_native_sql_references( - *, sql: str, syntax: SqlLexicalSyntax -) -> SqlReferenceScan | SqlReferenceScanFailure: - """Return the references and rejected calls, or the scan error and its start.""" - - extraction: _NativeExtraction = _scanner(syntax).extract(sql) - scanned, failure = extraction - if scanned is None: - return cast(SqlReferenceScanFailure, failure) - references, invalid_calls = scanned - return SqlReferenceScan( - references=tuple( - CompileSqlReference( - ref_kind=SqlReferenceKind(kind), - ref_name=name, - ref_package=package, - call_argument_count=call_argument_count, - ) - for kind, name, package, call_argument_count in references - ), - invalid_calls=tuple( - InvalidSqlReferenceCall( - ref_kind=SqlReferenceKind(kind), - call=call, - start=start, - message=call_message, - help=help_text, - corrected_call=corrected_call, - ) - for kind, call, start, call_message, help_text, corrected_call in invalid_calls - ), - ) - - -@cache -def _scanner(syntax: SqlLexicalSyntax) -> _native.SqlReferenceScanner: - return _native.SqlReferenceScanner(syntax.native_mapping) diff --git a/src/sqlbuild/compiler/compile/_helpers/refs/references.py b/src/sqlbuild/compiler/compile/_helpers/refs/references.py index ce170e1ba8..e241cd04f6 100644 --- a/src/sqlbuild/compiler/compile/_helpers/refs/references.py +++ b/src/sqlbuild/compiler/compile/_helpers/refs/references.py @@ -2,238 +2,79 @@ from __future__ import annotations -from pathlib import Path +from functools import cache +from typing import cast +import sqlbuild._native as _native from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic -from sqlbuild.compiler.compile._helpers.refs.native import extract_native_sql_references -from sqlbuild.compiler.compile._helpers.render.spans import map_through_passes -from sqlbuild.compiler.compile.constants import ( - REFERENCE_CALL_SYNTAX_CODE, -) +from sqlbuild.compiler.compile.constants import REFERENCE_CALL_SYNTAX_CODE from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( CompilerDiagnostic, CompileSqlReference, - ExpansionSpan, InvalidSqlReferenceCall, - MappedOffset, - SqlReferenceOrigin, SqlReferenceScan, - SqlReferenceSourceMap, ) from sqlbuild.compiler.compile.types import ( DiagnosticPhase, DiagnosticSeverity, SqlReferenceScanFailure, ) +from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import SourceLocation def extract_sql_references( - *, sql: str, syntax: SqlLexicalSyntax, origin: SqlReferenceOrigin | None = None + *, sql: str, syntax: SqlLexicalSyntax ) -> tuple[CompileSqlReference, ...]: """Return logical SQL refs outside comments and quotes, reporting rejected calls as P012.""" - scan: SqlReferenceScan = scan_sql_reference_calls(sql=sql, syntax=syntax, origin=origin) - report_invalid_reference_calls(invalid_calls=scan.invalid_calls, origin=origin, syntax=syntax) - return scan.references - - -def scan_sql_reference_calls( - *, sql: str, syntax: SqlLexicalSyntax, origin: SqlReferenceOrigin | None = None -) -> SqlReferenceScan: - """Return valid references and rejected calls; a scan error is located in `origin`.""" - - outcome: SqlReferenceScan | SqlReferenceScanFailure = _reference_scan_outcome( - sql=sql, syntax=syntax - ) - if isinstance(outcome, SqlReferenceScan): - return outcome - raise _located_scan_error(failure=outcome, sql=sql, origin=origin, syntax=syntax) - - -def _reference_scan_outcome( - *, sql: str, syntax: SqlLexicalSyntax -) -> SqlReferenceScan | SqlReferenceScanFailure: - outcome: SqlReferenceScan | SqlReferenceScanFailure = extract_native_sql_references( - sql=sql, syntax=syntax - ) - return outcome - - -def _located_scan_error( - *, - failure: SqlReferenceScanFailure, - sql: str, - origin: SqlReferenceOrigin | None, - syntax: SqlLexicalSyntax, -) -> CompileInputError: - """Locate a scan error in authored text; unmapped or expansion-shaped faults stay unlocated.""" - - message, start = failure - if origin is None: - return CompileInputError(message) - authored_offset: int | None = _authored_fault_offset( - failure=failure, sql=sql, origin=origin, syntax=syntax - ) - path: str = origin.relative_path.as_posix() - if authored_offset is None: - return CompileInputError(f"{path}: {message}") - contents: str = origin.contents - line: int = contents.count("\n", 0, authored_offset) + 1 - column: int = authored_offset - (contents.rfind("\n", 0, authored_offset) + 1) + 1 - return CompileInputError(f"{path}:{line}:{column}: {message}") - - -def _authored_fault_offset( - *, - failure: SqlReferenceScanFailure, - sql: str, - origin: SqlReferenceOrigin, - syntax: SqlLexicalSyntax, -) -> int | None: - """Map the fault to authored text unless expansion output produced or shaped the failure.""" - - source_map: SqlReferenceSourceMap | None = origin.source_map - start: int = failure[1] - body_start: int | None = source_map.body_start() if source_map is not None else None - if source_map is None or body_start is None or start >= len(sql): - return None - mapped: MappedOffset = map_through_passes(offset=start, passes=source_map.passes) - authored_offset: int = body_start + mapped.offset - if ( - mapped.generated - or authored_offset >= len(origin.contents) - or origin.contents[authored_offset] != sql[start] - ): - return None - blanked: list[str] = list(sql) - for output_start, output_end in _expansion_output_ranges(source_map.passes): - blanked[output_start:output_end] = " " * (output_end - output_start) - if _reference_scan_outcome(sql="".join(blanked), syntax=syntax) != failure: - return None - return authored_offset - - -def _expansion_output_ranges( - passes: tuple[tuple[ExpansionSpan, ...], ...], -) -> list[tuple[int, int]]: - """Return every expansion's output range in the coordinates of the final pass's output.""" - - ranges: list[tuple[int, int]] = [] - for index, spans in enumerate(passes): - for span in spans: - output_start: int = span.output_start - output_end: int = span.output_end - for later_spans in passes[index + 1 :]: - output_start = _later_pass_offset(offset=output_start, spans=later_spans, end=False) - output_end = _later_pass_offset(offset=output_end, spans=later_spans, end=True) - ranges.append((output_start, output_end)) - return ranges - - -def _later_pass_offset(*, offset: int, spans: tuple[ExpansionSpan, ...], end: bool) -> int: - """Carry a range boundary through one later pass, widening it over a span that covers it.""" - - shift: int = 0 - for span in spans: - if offset < span.source_start or (end and offset == span.source_start): - break - if offset < span.source_end or (end and offset == span.source_end): - return span.output_end if end else span.output_start - shift += (span.output_end - span.output_start) - (span.source_end - span.source_start) - return offset + shift - - -def report_invalid_reference_calls( - *, - invalid_calls: tuple[InvalidSqlReferenceCall, ...], - origin: SqlReferenceOrigin | None, - syntax: SqlLexicalSyntax, -) -> None: - """Report each rejected reference call at its authored location when one is known.""" - - if not invalid_calls: - return - authored_starts: dict[str, list[int]] = _authored_invalid_call_starts( - origin=origin, syntax=syntax - ) - seen: dict[str, int] = {} - for invalid_call in invalid_calls: - occurrence: int = seen.get(invalid_call.call, 0) - seen[invalid_call.call] = occurrence + 1 - starts: list[int] = authored_starts.get(invalid_call.call, []) - location: SourceLocation | None = ( - reference_call_location( - path=origin.relative_path, - text=origin.contents, - start=starts[occurrence], - call=invalid_call.call, - ) - if origin is not None and occurrence < len(starts) - else None - ) + scan: SqlReferenceScan = scan_sql_reference_calls(sql=sql, syntax=syntax) + for invalid_call in scan.invalid_calls: report_compile_diagnostic( - key=reference_call_syntax_key( - file_path=origin.file_path if origin is not None else None, - call=invalid_call.call, - location=location, - ), + key=(REFERENCE_CALL_SYNTAX_CODE, "", invalid_call.call), diagnostic=CompilerDiagnostic( phase=DiagnosticPhase.COMPILE, severity=DiagnosticSeverity.ERROR, code=REFERENCE_CALL_SYNTAX_CODE, message=invalid_call.message, - resource_type=origin.resource_type if origin is not None else None, - resource_name=origin.resource_name if origin is not None else None, - path=origin.relative_path if origin is not None and location is None else None, - location=location, help=invalid_call.help, ), ) + return scan.references -def reference_call_syntax_key( - *, file_path: Path | None, call: str, location: SourceLocation | None -) -> tuple[str, ...]: - """Return the collector key shared by every report of one rejected reference call.""" - - owner: str = file_path.as_posix() if file_path is not None else "" - if location is None: - return (REFERENCE_CALL_SYNTAX_CODE, owner, call) - return ( - REFERENCE_CALL_SYNTAX_CODE, - owner, - f"{location.line:09d}:{location.column:09d}", - call, - ) - - -def _authored_invalid_call_starts( - *, origin: SqlReferenceOrigin | None, syntax: SqlLexicalSyntax -) -> dict[str, list[int]]: - if origin is None: - return {} - authored: SqlReferenceScan | SqlReferenceScanFailure = _reference_scan_outcome( - sql=origin.contents, syntax=syntax +def scan_sql_reference_calls(*, sql: str, syntax: SqlLexicalSyntax) -> SqlReferenceScan: + """Return valid references and rejected calls; a scan error raises unlocated.""" + + scanned, failure = _scanner(syntax).extract(sql) + if scanned is None: + raise CompileInputError(cast(SqlReferenceScanFailure, failure)[0]) + references, invalid_calls = scanned + return SqlReferenceScan( + references=tuple( + CompileSqlReference( + ref_kind=SqlReferenceKind(kind), + ref_name=name, + ref_package=package, + call_argument_count=call_argument_count, + ) + for kind, name, package, call_argument_count in references + ), + invalid_calls=tuple( + InvalidSqlReferenceCall( + ref_kind=SqlReferenceKind(kind), + call=call, + start=start, + message=call_message, + help=help_text, + corrected_call=corrected_call, + ) + for kind, call, start, call_message, help_text, corrected_call in invalid_calls + ), ) - if not isinstance(authored, SqlReferenceScan): - return {} - starts: dict[str, list[int]] = {} - for invalid_call in authored.invalid_calls: - starts.setdefault(invalid_call.call, []).append(invalid_call.start) - return starts -def reference_call_location(*, path: Path, text: str, start: int, call: str) -> SourceLocation: - """Return the span of ``call`` starting at ``start`` in ``text`` as a source location.""" - - end: int = start + len(call) - return SourceLocation( - path=path, - line=text.count("\n", 0, start) + 1, - column=start - (text.rfind("\n", 0, start) + 1) + 1, - end_line=text.count("\n", 0, end) + 1, - end_column=end - (text.rfind("\n", 0, end) + 1) + 1, - ) +@cache +def _scanner(syntax: SqlLexicalSyntax) -> _native.SqlReferenceScanner: + return _native.SqlReferenceScanner(syntax.native_mapping) diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index e8218a86c1..53f665a792 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -498,26 +498,6 @@ class SqlReferenceScan: invalid_calls: tuple[InvalidSqlReferenceCall, ...] = () -@dataclass(frozen=True) -class SqlReferenceSourceMap: - """Where the authored body starts in its file and the expansion passes from it to the SQL.""" - - body_start: Callable[[], int | None] - passes: tuple[tuple[ExpansionSpan, ...], ...] - - -@dataclass(frozen=True) -class SqlReferenceOrigin: - """The authored file that SQL handed to reference extraction came from.""" - - file_path: Path - relative_path: Path - contents: str - resource_type: CompiledResourceType | None = None - resource_name: str | None = None - source_map: SqlReferenceSourceMap | None = None - - @dataclass(frozen=True) class CompiledLineageSourceFact: """Compact upstream column fact extracted during SQL analysis.""" diff --git a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/_test_types.py b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/_test_types.py index 208352ff9c..c6fc409c30 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/_test_types.py @@ -2,8 +2,6 @@ from dataclasses import dataclass -from sqlbuild.compiler.compile.models import SqlReferenceSourceMap - @dataclass(frozen=True) class CraftedReferenceTestCase: @@ -23,18 +21,6 @@ class ReferenceExtractionTestCase: sql: str expected_references: tuple[tuple[str, str, str | None, int | None], ...] = () expected_error: str = "" - expected_native_calls: int = 0 - - -@dataclass(frozen=True) -class NativeReferenceErrorTestCase: - """Expanded SQL whose native error is located through its source map without Python.""" - - description: str - sql: str - contents: str - source_map: SqlReferenceSourceMap | None - expected_message: str @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/helpers.py deleted file mode 100644 index 7169985f1c..0000000000 --- a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/helpers.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Source maps for located reference extraction errors.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import ExpansionSpan, SqlReferenceSourceMap - - -def source_map_at( - *, body_start: int, passes: tuple[tuple[ExpansionSpan, ...], ...] = () -) -> SqlReferenceSourceMap: - """Return a source map whose authored body starts at `body_start` in its file.""" - - return SqlReferenceSourceMap(body_start=lambda: body_start, passes=passes) diff --git a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py index 63dd69079f..9eb38d107e 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py +++ b/tests/integration/src/sqlbuild/compiler/compile/_helpers/refs/test_native_references.py @@ -1,29 +1,22 @@ -"""Native reference extraction: references, rejected calls, located errors and scan bounds.""" +"""Native reference extraction: references, rejected calls and scan bounds.""" from __future__ import annotations import time from collections.abc import Callable -from operator import itemgetter -from pathlib import Path from typing import Any import pytest -from sqlbuild.compiler.compile._helpers.refs import references -from sqlbuild.compiler.compile._helpers.refs.native import extract_native_sql_references +import sqlbuild._native as _native from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ExpansionSpan, SqlReferenceOrigin, SqlReferenceScan -from sqlbuild.compiler.compile.types import SqlReferenceScanFailure from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from tests.integration.src.sqlbuild.compiler.compile._helpers.refs._test_types import ( CraftedReferenceTestCase, - NativeReferenceErrorTestCase, ReferenceExtractionTestCase, ReferenceScanBoundTestCase, ) -from tests.integration.src.sqlbuild.compiler.compile._helpers.refs.helpers import source_map_at _GENERIC_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() _REFERENCE_SQLS: tuple[ @@ -57,13 +50,14 @@ '__table_fn("orders_for")(' + ", ".join(f"(c{index} /* , */)" for index in range(100_000)) + ")" ) _NESTED_CALLS: str = '__table_fn("orders_for")(' * 1_000 + "1" + ")" * 1_000 +_SCANNER: _native.SqlReferenceScanner = _native.SqlReferenceScanner(_GENERIC_SYNTAX.native_mapping) _OUTCOME_SUMMARIES: dict[type, Callable[[Any], int | str]] = { - SqlReferenceScan: lambda scan: len(scan.references) + len(scan.invalid_calls), - tuple: itemgetter(0), + tuple: lambda outcome: len(outcome[0][0]) + len(outcome[0][1]), + type(None): lambda outcome: outcome[1][0], } _REJECTED_AND_FAILED: dict[type, Callable[[Any], tuple[int, int]]] = { - SqlReferenceScan: lambda scan: (len(scan.invalid_calls), 0), - tuple: lambda _failure: (0, 1), + tuple: lambda outcome: (len(outcome[0][1]), 0), + type(None): lambda _outcome: (0, 1), } _MANY_REFERENCES: str = "SELECT 1 FROM " + " JOIN ".join( f'__ref("orders_{index}")' for index in range(50_000) @@ -188,11 +182,9 @@ def test_given_crafted_reference_sql_when_extracting_then_rejected_and_failed_counts_match( test_case: CraftedReferenceTestCase, ) -> None: - outcome: SqlReferenceScan | SqlReferenceScanFailure = extract_native_sql_references( - sql=test_case.sql, syntax=_GENERIC_SYNTAX - ) + outcome: Any = _SCANNER.extract(test_case.sql) - assert _REJECTED_AND_FAILED[type(outcome)](outcome) == ( + assert _REJECTED_AND_FAILED[type(outcome[0])](outcome) == ( test_case.expected_rejected, test_case.expected_failed, ) @@ -206,28 +198,14 @@ def test_given_crafted_reference_sql_when_extracting_then_rejected_and_failed_co sql=sql, expected_references=expected_references, expected_error=expected_error, - expected_native_calls=1, ) for name, sql, expected_references, expected_error in _REFERENCE_SQLS ], ids=lambda case: case.description, ) def test_given_reference_sql_when_extracting_references_then_native_scanner_runs( - test_case: ReferenceExtractionTestCase, monkeypatch: pytest.MonkeyPatch + test_case: ReferenceExtractionTestCase, ) -> None: - native_calls: list[str] = [] - native_extract: Callable[..., SqlReferenceScan | SqlReferenceScanFailure] = ( - references.extract_native_sql_references - ) - - def counting_extract( - *, sql: str, syntax: SqlLexicalSyntax - ) -> SqlReferenceScan | SqlReferenceScanFailure: - native_calls.append(sql) - return native_extract(sql=sql, syntax=syntax) - - monkeypatch.setattr(references, "extract_native_sql_references", counting_extract) - error: str = "" extracted: tuple[tuple[str, str, str | None, int | None], ...] = () try: @@ -238,119 +216,7 @@ def counting_extract( except CompileInputError as raised: error = str(raised) - assert (extracted, error, len(native_calls)) == ( - test_case.expected_references, - test_case.expected_error, - test_case.expected_native_calls, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - NativeReferenceErrorTestCase( - description="unclosed_reference_call_points_at_the_call_after_the_header", - sql='SELECT *\nFROM __ref("orders"', - contents='MODEL (description "Orders");\n\nSELECT *\nFROM __ref("orders"', - source_map=source_map_at(body_start=31), - expected_message="models/orders.sql:4:6: SQL reference contains an unclosed parenthesis", - ), - NativeReferenceErrorTestCase( - description="empty_table_function_argument_after_a_rejected_call", - sql='SELECT * FROM __ref(orders)\nJOIN __table_fn("orders_for")(1,,2)', - contents='SELECT * FROM __ref(orders)\nJOIN __table_fn("orders_for")(1,,2)', - source_map=source_map_at(body_start=0), - expected_message="models/orders.sql:2:6: SQL reference contains an empty argument", - ), - NativeReferenceErrorTestCase( - description="unclosed_quote_points_at_the_quote_in_code_points", - sql="SELECT 'é', __ref(\"orders\") WHERE note = 'open", - contents="-- é\nSELECT 'é', __ref(\"orders\") WHERE note = 'open", - source_map=source_map_at(body_start=5), - expected_message="models/orders.sql:2:42: SQL reference contains an unclosed quoted string", - ), - NativeReferenceErrorTestCase( - description="fault_after_neutral_expansions_in_two_passes_maps_through_both", - sql="SELECT 'open', 1 + 2\nFROM __ref(\"orders\"", - contents='SELECT @enum("s").OPEN, @m()\nFROM __ref("orders"', - source_map=source_map_at( - body_start=0, - passes=( - (ExpansionSpan(source_start=7, source_end=22, output_start=7, output_end=13),), - ( - ExpansionSpan( - source_start=15, source_end=19, output_start=15, output_end=20 - ), - ), - ), - ), - expected_message="models/orders.sql:2:6: SQL reference contains an unclosed parenthesis", - ), - NativeReferenceErrorTestCase( - description="fault_inside_macro_output_is_unlocated", - sql='SELECT * FROM __ref("orders") /* expanded', - contents='SELECT * FROM __ref("orders") @expand()', - source_map=source_map_at( - body_start=0, - passes=( - ( - ExpansionSpan( - source_start=30, source_end=39, output_start=30, output_end=41 - ), - ), - ), - ), - expected_message="models/orders.sql: SQL reference contains an unclosed block comment", - ), - NativeReferenceErrorTestCase( - description="authored_fault_shaped_by_an_unclosed_macro_quote_is_unlocated", - sql="SELECT 'x || ' || note || 'open", - contents="SELECT @q() || ' || note || 'open", - source_map=source_map_at( - body_start=0, - passes=( - (ExpansionSpan(source_start=7, source_end=11, output_start=7, output_end=9),), - ), - ), - expected_message="models/orders.sql: SQL reference contains an unclosed quoted string", - ), - NativeReferenceErrorTestCase( - description="without_a_source_map_the_file_alone_is_named", - sql='SELECT *\nFROM __ref("orders"', - contents='SELECT *\nFROM __ref("orders"', - source_map=None, - expected_message="models/orders.sql: SQL reference contains an unclosed parenthesis", - ), - ], - ids=lambda case: case.description, -) -def test_given_native_reference_error_when_extracting_then_raised_located( - test_case: NativeReferenceErrorTestCase, monkeypatch: pytest.MonkeyPatch -) -> None: - - with pytest.raises(CompileInputError) as raised: - _ = extract_sql_references( - sql=test_case.sql, - syntax=_GENERIC_SYNTAX, - origin=SqlReferenceOrigin( - file_path=Path("/project/models/orders.sql"), - relative_path=Path("models/orders.sql"), - contents=test_case.contents, - source_map=test_case.source_map, - ), - ) - - assert ( - type(raised.value), - str(raised.value), - raised.value.code, - raised.value.help, - ) == ( - CompileInputError, - test_case.expected_message, - "P001", - None, - ) + assert (extracted, error) == (test_case.expected_references, test_case.expected_error) @pytest.mark.parametrize( @@ -393,12 +259,10 @@ def test_given_worst_case_reference_sql_when_extracting_natively_then_finishes_q test_case: ReferenceScanBoundTestCase, ) -> None: started: float = time.thread_time() - outcome: SqlReferenceScan | SqlReferenceScanFailure = extract_native_sql_references( - sql=test_case.sql, syntax=_GENERIC_SYNTAX - ) + outcome: Any = _SCANNER.extract(test_case.sql) elapsed: float = time.thread_time() - started - summary: int | str = _OUTCOME_SUMMARIES[type(outcome)](outcome) + summary: int | str = _OUTCOME_SUMMARIES[type(outcome[0])](outcome) assert (summary, elapsed < test_case.expected_maximum_seconds) == ( test_case.expected_outcome, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 354ec57e68..972ab91505 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -849,9 +849,8 @@ class DescriptionInputClassificationCase: @dataclass(frozen=True) class CollectedReferenceCallSyntaxTestCase: description: str - contents: str sql: str - expected_diagnostics: tuple[tuple[str, str, Path, SourceLocation | None], ...] + expected_diagnostics: tuple[tuple[str, str, Path | None, SourceLocation | None], ...] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py index 8dd6bb50a1..ea9d99f453 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_references.py @@ -3,7 +3,6 @@ from __future__ import annotations import re -from pathlib import Path import pytest @@ -15,11 +14,9 @@ from sqlbuild.compiler.compile._helpers.diagnostics.collector import collect_compile_diagnostics from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import CompileSqlReference, SqlReferenceOrigin -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.models import CompileSqlReference from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import SourceLocation from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( CollectedReferenceCallSyntaxTestCase, DialectSqlScanTestCase, @@ -28,7 +25,6 @@ ) _GENERIC_SQL_SYNTAX: SqlLexicalSyntax = SqlLexicalSyntax() -_ORIGIN_PATH: Path = Path("models/marts/order_totals.sql") @pytest.mark.parametrize( @@ -213,69 +209,26 @@ def test_given_reference_call_compile_cannot_replace_when_extracting_then_raises "test_case", [ CollectedReferenceCallSyntaxTestCase( - description="every rejected call is reported at its authored location", - contents=( - 'MODEL (description "Order totals.");\n\n' - "SELECT * FROM __ref(stg_orders) o\n" - "JOIN __source('raw_payments') p USING (order_id)\n" - ), + description="every rejected call is reported once without a location", sql=( "SELECT * FROM __ref(stg_orders) o\n" "JOIN __source('raw_payments') p USING (order_id)\n" + "JOIN __ref(stg_orders) again USING (order_id)\n" ), expected_diagnostics=( - ( - "P012", - "__ref(stg_orders) is not a valid __ref() call", - _ORIGIN_PATH, - SourceLocation(path=_ORIGIN_PATH, line=3, column=15, end_line=3, end_column=32), - ), - ( - "P012", - "__source('raw_payments') is not a valid __source() call", - _ORIGIN_PATH, - SourceLocation(path=_ORIGIN_PATH, line=4, column=6, end_line=4, end_column=30), - ), - ), - ), - CollectedReferenceCallSyntaxTestCase( - description="a commented-out rejected call does not take the authored location", - contents=("-- was __ref(stg_orders)\nSELECT * FROM __ref(stg_orders)\n"), - sql="SELECT * FROM __ref(stg_orders)\n", - expected_diagnostics=( - ( - "P012", - "__ref(stg_orders) is not a valid __ref() call", - _ORIGIN_PATH, - SourceLocation(path=_ORIGIN_PATH, line=2, column=15, end_line=2, end_column=32), - ), - ), - ), - CollectedReferenceCallSyntaxTestCase( - description="a call missing from the authored text keeps the file path", - contents="SELECT * FROM @staged_orders()\n", - sql="SELECT * FROM __ref(stg_orders)\n", - expected_diagnostics=( - ("P012", "__ref(stg_orders) is not a valid __ref() call", _ORIGIN_PATH, None), + ("P012", "__ref(stg_orders) is not a valid __ref() call", None, None), + ("P012", "__source('raw_payments') is not a valid __source() call", None, None), ), ), ], ids=lambda case: case.description, ) -def test_given_rejected_calls_when_collecting_diagnostics_then_reports_each_with_location( +def test_given_rejected_calls_when_collecting_diagnostics_then_reports_each_unlocated( test_case: CollectedReferenceCallSyntaxTestCase, ) -> None: - origin: SqlReferenceOrigin = SqlReferenceOrigin( - file_path=Path("/project/models/marts/order_totals.sql"), - relative_path=_ORIGIN_PATH, - contents=test_case.contents, - resource_type=CompiledResourceType.MODEL, - resource_name="order_totals", - ) - with collect_compile_diagnostics() as collected: references: tuple[CompileSqlReference, ...] = extract_sql_references( - sql=test_case.sql, syntax=_GENERIC_SQL_SYNTAX, origin=origin + sql=test_case.sql, syntax=_GENERIC_SQL_SYNTAX ) assert references == () @@ -286,7 +239,6 @@ def test_given_rejected_calls_when_collecting_diagnostics_then_reports_each_with ) == test_case.expected_diagnostics ) - assert {diagnostic.resource_name for diagnostic in collected.diagnostics} == {"order_totals"} @pytest.mark.parametrize( From 7e1e8055eb6fbd3a1292fa76fc4f9f2a133bd1ca Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 01:44:12 +0100 Subject: [PATCH 02/14] refactor: read explicit-reference project facts natively The explicit-reference report and the planner's audit-gate and hook-read edges read models, sources, seeds, functions, audits and Python hooks straight from the project views in Rust, and the report is built as Python's compile models natively. Python keeps only the user-code owners (functions, loader definitions and their literal SQL). The JSON request builders and response decoding are deleted. --- .../_helpers/analysis/explicit_references.rs | 216 +++++++++++--- .../src/bindings/_helpers/analysis/mod.rs | 1 + .../_helpers/analysis/reference_facts.rs | 269 +++++++++++++++++ python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 9 +- .../_helpers/explicit_references/checks.py | 281 ++---------------- .../graph/_helpers/execution_gates.py | 82 +---- 7 files changed, 474 insertions(+), 386 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis/reference_facts.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index c578031f6e..4ed3a6b245 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -1,56 +1,185 @@ //! Compile's explicit-reference checks over plain project facts, run natively. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; -use serde_json::{Value, json}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::explicit_references::errors::ReferenceFailure; use sqlbuild_analysis::explicit_references::main::check_explicit_references::check_explicit_references; use sqlbuild_analysis::explicit_references::main::execution_gate_edges::execution_gate_edges; use sqlbuild_analysis::explicit_references::models::{ - ExecutionGateEdges, ExecutionGateRequest, GateEdge, HookEdge, HookType, ReferenceReport, - ReferenceRequest, + ExecutionGateEdges, ExecutionGateRequest, GateEdge, HookEdge, HookType, LiteralSql, + ReferenceReport, ReferenceRequest, SqlOwner, }; use sqlbuild_analysis::explicit_references::types::ResourceKey; use sqlbuild_core::panics::main::native_failure::native_failure; use sqlbuild_pysource::literal_sql::main::literal_sql_calls::literal_sql_calls; use sqlbuild_pysource::literal_sql::models::FunctionSource; +use crate::bindings::_helpers::analysis::reference_facts::{ProjectFacts, project_facts, ref_keys}; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::types::CompilerDetach; const EXPLICIT_REFERENCES_CONTEXT: &str = "native explicit-reference checks"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_INPUT_ERROR: &str = "CompileInputError"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const REFERENCE_MODELS_MODULE: &str = "sqlbuild.compiler.references.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const DIAGNOSTIC_PHASE: &str = "compile"; +const DIAGNOSTIC_SEVERITY: &str = "error"; + +/// One Python SQL owner as Python collects it: the owner, its path order and literal SQL calls. +type OwnerInput<'py> = (Bound<'py, PyAny>, usize, Vec<(String, String, usize)>); -/// Python's audit-gate, hook-read and Python-SQL checks: `{report}` or `{raise}` JSON. +/// The project-wide settings of one explicit-reference check. +#[derive(FromPyObject)] +#[pyo3(from_item_all)] +struct ReportSettings { + /// Whether the project keeps old-name views unless a model disables them. + retains_old_names: bool, + enforce_explicit: bool, + explicit_help: String, + /// The text Python substitutes for f-string values in literal SQL. + placeholder: String, +} + +/// Python's audit-gate, hook-read and Python-SQL checks as a `PythonSqlReferenceReport`. #[pyfunction] -fn check_native_explicit_references(py: Python<'_>, request_json: &str) -> PyResult { - let request: ReferenceRequest = serde_json::from_str(request_json).map_err(|error| { - compiler_error(native_failure( - EXPLICIT_REFERENCES_CONTEXT, - &error.to_string(), - )) - })?; - let outcome: Value = match py +fn explicit_reference_report<'py>( + py: Python<'py>, + project: &Bound<'py, PyAny>, + settings: ReportSettings, + owners: Vec>, +) -> PyResult> { + let facts: ProjectFacts = project_facts(project, settings.retains_old_names)?; + let request = ReferenceRequest { + enforce_explicit: settings.enforce_explicit, + dialect: project.getattr("sql_analysis_dialect")?.extract()?, + explicit_help: settings.explicit_help, + placeholder: settings.placeholder, + resources: facts.resources, + models: facts.models, + sources: facts.sources, + seeds: facts.seeds, + audits: facts.audits, + hooks: facts.hooks, + owners: owners + .iter() + .map(|(owner, path_order, literals)| sql_owner(owner, *path_order, literals)) + .collect::>()?, + }; + match py .compiler_detach(|| Ok(check_explicit_references(&request))) .map_err(compiler_error)? { - Ok(report) => json!({ "report": report_value(&report)? }), - Err(ReferenceFailure::CompileInput(message)) => json!({ "raise": message }), - Err(ReferenceFailure::UnknownDialect(message)) => return Err(compiler_error(message)), - Err(ReferenceFailure::Internal(reason)) => { - return Err(compiler_error(native_failure( - EXPLICIT_REFERENCES_CONTEXT, - &reason, - ))); - } - }; - serde_json::to_string(&outcome).map_err(|error| { - compiler_error(native_failure( + Ok(report) => python_report(py, &report, &owners), + Err(ReferenceFailure::CompileInput(message)) => Err(PyErr::from_value( + py.import(COMPILE_ERRORS_MODULE)? + .getattr(COMPILE_INPUT_ERROR)? + .call1((message,))?, + )), + Err(ReferenceFailure::UnknownDialect(message)) => Err(compiler_error(message)), + Err(ReferenceFailure::Internal(reason)) => Err(compiler_error(native_failure( EXPLICIT_REFERENCES_CONTEXT, - &error.to_string(), - )) + &reason, + ))), + } +} + +fn sql_owner( + owner: &Bound<'_, PyAny>, + path_order: usize, + literals: &[(String, String, usize)], +) -> PyResult { + Ok(SqlOwner { + label: owner.getattr("label")?.extract()?, + kind: owner.getattr("kind")?.str()?.extract()?, + path_order, + own_refs: ref_keys(&owner.getattr("own_refs")?)?, + upstream_loaders: owner + .getattr("upstream_loader_by_source")? + .call_method0("items")? + .try_iter()? + .map(|item| item?.extract()) + .collect::>()?, + literals: literals + .iter() + .map(|(method, sql, line)| LiteralSql { + method: method.clone(), + sql: sql.clone(), + line: *line, + }) + .collect(), }) } +/// The report's diagnostics and unmatched relations as Python's compile models. +fn python_report<'py>( + py: Python<'py>, + report: &ReferenceReport, + owners: &[OwnerInput<'py>], +) -> PyResult> { + let compile_models = py.import(COMPILE_MODELS_MODULE)?; + let reference_models = py.import(REFERENCE_MODELS_MODULE)?; + let source_location: Bound<'py, PyAny> = py + .import(CONTRACT_MODELS_MODULE)? + .getattr("SourceLocation")?; + let path_class: Bound<'py, PyAny> = py.import("pathlib")?.getattr("Path")?; + let compiler_diagnostic: Bound<'py, PyAny> = compile_models.getattr("CompilerDiagnostic")?; + let mut diagnostics: Vec> = Vec::with_capacity(report.diagnostics.len()); + for row in &report.diagnostics { + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("phase", DIAGNOSTIC_PHASE)?; + fields.set_item("severity", DIAGNOSTIC_SEVERITY)?; + fields.set_item("code", row.code)?; + fields.set_item("message", &row.message)?; + fields.set_item("resource_type", row.resource.as_ref().map(|key| &key.0))?; + fields.set_item("resource_name", row.resource.as_ref().map(|key| &key.1))?; + match &row.path { + Some(path) => fields.set_item("path", path_class.call1((path,))?)?, + None => fields.set_item("path", py.None())?, + } + match row.owner { + Some(owner) => { + let location: Bound<'py, PyDict> = PyDict::new(py); + location.set_item("path", owners[owner].0.getattr("relative_path")?)?; + location.set_item("line", row.line)?; + location.set_item("column", 1)?; + fields.set_item("location", source_location.call((), Some(&location))?)?; + } + None => fields.set_item("location", py.None())?, + } + fields.set_item("help", &row.help)?; + diagnostics.push(compiler_diagnostic.call((), Some(&fields))?); + } + let literal_relation: Bound<'py, PyAny> = reference_models.getattr("LiteralSqlRelation")?; + let relation_name: Bound<'py, PyAny> = reference_models.getattr("RelationName")?; + let mut unmatched: Vec> = Vec::with_capacity(report.unmatched.len()); + for row in &report.unmatched { + let owner: &Bound<'py, PyAny> = &owners[row.owner].0; + let relation: Bound<'py, PyDict> = PyDict::new(py); + relation.set_item("name", &row.relation.name)?; + relation.set_item("schema", &row.relation.schema)?; + relation.set_item("database", &row.relation.database)?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("owner_label", owner.getattr("label")?)?; + fields.set_item("owner_kind", owner.getattr("kind")?)?; + fields.set_item("relative_path", owner.getattr("relative_path")?)?; + fields.set_item("line", row.line)?; + fields.set_item("method", &row.method)?; + fields.set_item("relation", relation_name.call((), Some(&relation))?)?; + unmatched.push(literal_relation.call((), Some(&fields))?); + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("diagnostics", PyTuple::new(py, diagnostics)?)?; + fields.set_item("unmatched", PyTuple::new(py, unmatched)?)?; + compile_models + .getattr("PythonSqlReferenceReport")? + .call((), Some(&fields)) +} + /// `(method, sql, line)` for each literal SQL call to the context in a dedented function source. #[pyfunction] fn python_literal_sql_calls( @@ -82,16 +211,18 @@ type HookRow = ( Option, ); -/// The attached-audit gate edges and hook-read edges of one project's facts, in Python's order. +/// The attached-audit gate edges and hook-read edges of one project, in Python's order. #[pyfunction] -fn native_execution_gate_edges(request_json: &str) -> PyResult<(Vec, Vec)> { - let request: ExecutionGateRequest = serde_json::from_str(request_json).map_err(|error| { - compiler_error(native_failure( - EXPLICIT_REFERENCES_CONTEXT, - &error.to_string(), - )) - })?; - let edges: ExecutionGateEdges = execution_gate_edges(&request); +fn native_execution_gate_edges( + project: &Bound<'_, PyAny>, +) -> PyResult<(Vec, Vec)> { + let facts: ProjectFacts = project_facts(project, false)?; + let edges: ExecutionGateEdges = execution_gate_edges(&ExecutionGateRequest { + resources: facts.resources, + models: facts.models, + audits: facts.audits, + hooks: facts.hooks, + }); Ok(( edges.audit_gates.into_iter().map(gate_row).collect(), edges.hook_reads.into_iter().map(hook_row).collect(), @@ -111,17 +242,8 @@ fn hook_row(edge: HookEdge) -> HookRow { (edge.hook_name, edge.gated, edge.read, hook_type, edge.path) } -fn report_value(report: &ReferenceReport) -> PyResult { - serde_json::to_value(report).map_err(|error| { - compiler_error(native_failure( - EXPLICIT_REFERENCES_CONTEXT, - &error.to_string(), - )) - }) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(check_native_explicit_references, module)?)?; + module.add_function(wrap_pyfunction!(explicit_reference_report, module)?)?; module.add_function(wrap_pyfunction!(python_literal_sql_calls, module)?)?; module.add_function(wrap_pyfunction!(native_execution_gate_edges, module)?)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs index 62d4a083e3..0177fd8277 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod normalization; pub(crate) mod normalization_results; mod positions; pub(crate) mod queries; +pub(crate) mod reference_facts; pub(crate) mod relation_names; mod required_columns; pub(crate) mod sql_tests; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/reference_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/reference_facts.rs new file mode 100644 index 0000000000..d03eb26660 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/reference_facts.rs @@ -0,0 +1,269 @@ +//! The explicit-reference facts of a compiled project, read from its Python views. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyList, PyTuple}; +use sqlbuild_analysis::explicit_references::models::{ + AuditGate, GraphResource, ModelReferences, NamedRelation, PythonHook, Relation, SourceRelation, + SqlHook, +}; +use sqlbuild_analysis::explicit_references::types::ResourceKey; + +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const PYTHON_HOOK_ENTRY: &str = "PythonHookEntry"; +const SQL_HOOK_ENTRY: &str = "SqlHookEntry"; +const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; +const MIGRATE_FROM_KEY: &str = "migrate_from"; +const OLD_NAME_VIEW_KEY: &str = "old_name_view"; + +/// The project facts the explicit-reference checks and execution-gate edges read. +pub(crate) struct ProjectFacts { + /// Models, then sources, seeds and functions, with their dependencies. + pub(crate) resources: Vec, + pub(crate) models: Vec, + pub(crate) sources: Vec, + pub(crate) seeds: Vec, + pub(crate) audits: Vec, + pub(crate) hooks: Vec, +} + +/// The facts of `project`; `retains_old_names` is whether the project keeps old-name views. +pub(crate) fn project_facts( + project: &Bound<'_, PyAny>, + retains_old_names: bool, +) -> PyResult { + let py: Python<'_> = project.py(); + let entries: HookEntries<'_> = HookEntries::imported(py)?; + let mut resources: Vec = Vec::new(); + let mut models: Vec = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + resources.push(GraphResource { + key: model.getattr("key_row")?.extract()?, + deps: model.getattr("dep_rows")?.extract()?, + }); + models.push(model_references(&model, &entries, retains_old_names)?); + } + let mut sources: Vec = Vec::new(); + for source in project.getattr("sources")?.try_iter()? { + let source: Bound<'_, PyAny> = source?; + resources.push(graph_resource(&source)?); + let entry: Bound<'_, PyAny> = source.getattr("source_entry")?; + sources.push(SourceRelation { + name: source.getattr("name")?.extract()?, + table: entry.getattr("table")?.extract()?, + schema: entry.getattr("schema")?.extract()?, + database: entry.getattr("database")?.extract()?, + expression: !entry.getattr("expression")?.is_none(), + }); + } + let mut seeds: Vec = Vec::new(); + for seed in project.getattr("seeds")?.try_iter()? { + let seed: Bound<'_, PyAny> = seed?; + resources.push(graph_resource(&seed)?); + let destination: Bound<'_, PyAny> = seed.getattr("destination")?; + seeds.push(NamedRelation { + name: seed.getattr("name")?.extract()?, + relation: Relation { + name: destination.getattr("name")?.extract()?, + schema: destination.getattr("schema")?.extract()?, + database: destination.getattr("database")?.extract()?, + }, + }); + } + for function in project.getattr("functions")?.try_iter()? { + resources.push(graph_resource(&function?)?); + } + let mut audits: Vec = Vec::new(); + for audit in project.getattr("audits")?.try_iter()? { + audits.push(audit_gate(&audit?)?); + } + let mut hooks: Vec = Vec::new(); + for hook in project.getattr("hook_functions")?.try_iter()? { + let hook: Bound<'_, PyAny> = hook?; + hooks.push(PythonHook { + name: hook.getattr("name")?.extract()?, + path: hook + .getattr("relative_path")? + .call_method0("as_posix")? + .extract()?, + reads: ref_keys(&hook.getattr("reads")?)?, + }); + } + Ok(ProjectFacts { + resources, + models, + sources, + seeds, + audits, + hooks, + }) +} + +/// `PythonHookEntry` and `SqlHookEntry`, which model hook entries are instances of. +struct HookEntries<'py> { + python: Bound<'py, PyAny>, + sql: Bound<'py, PyAny>, +} + +impl<'py> HookEntries<'py> { + fn imported(py: Python<'py>) -> PyResult { + let module = py.import(DISCOVERY_MODELS_MODULE)?; + Ok(Self { + python: module.getattr(PYTHON_HOOK_ENTRY)?, + sql: module.getattr(SQL_HOOK_ENTRY)?, + }) + } +} + +fn model_references( + model: &Bound<'_, PyAny>, + entries: &HookEntries<'_>, + retains_old_names: bool, +) -> PyResult { + let values: Bound<'_, PyAny> = model.getattr("config")?.getattr("values")?; + let (database, schema, name): (Option, Option, String) = + model.getattr("destination_row")?.extract()?; + let mut sql_hooks: Vec = Vec::new(); + for hook_key in MODEL_HOOK_KEYS { + for (index, entry) in hook_items(&values.call_method1("get", (hook_key,))?)? + .into_iter() + .enumerate() + { + if !entry.is_instance(&entries.sql)? || !entry.getattr("reads")?.is_truthy()? { + continue; + } + let path: Bound<'_, PyAny> = entry.getattr("relative_path")?; + sql_hooks.push(SqlHook { + hook_key: hook_key.to_owned(), + index, + name: entry.getattr("name")?.extract()?, + path: if path.is_none() { + None + } else { + Some(path.str()?.extract()?) + }, + reads: ref_keys(&entry.getattr("reads")?)?, + }); + } + } + Ok(ModelReferences { + name: model.getattr("name")?.extract()?, + path: model.getattr("relative_path")?.str()?.extract()?, + destination: Relation { + name, + schema, + database, + }, + python_hooks: python_hook_names(&values, entries)?, + sql_hooks, + migrate_from: migrate_from(&values, retains_old_names)?, + }) +} + +/// `model_python_hook_names`: the model's Python hooks in lifecycle order, each once. +fn python_hook_names( + values: &Bound<'_, PyAny>, + entries: &HookEntries<'_>, +) -> PyResult> { + let mut names: Vec = Vec::new(); + for hook_key in MODEL_HOOK_KEYS { + let value: Bound<'_, PyAny> = values.call_method1("get", (hook_key,))?; + let items: Vec> = if value.is_instance(&entries.python)? { + vec![value] + } else if is_sequence(&value) { + value.try_iter()?.collect::>()? + } else { + Vec::new() + }; + for item in items { + if !item.is_instance(&entries.python)? { + continue; + } + let name: String = item.getattr("name")?.extract()?; + if !names.contains(&name) { + names.push(name); + } + } + } + Ok(names) +} + +/// The entries of one hook key: a list or tuple's items, or the value itself. +fn hook_items<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { + if is_sequence(value) { + return value.try_iter()?.collect(); + } + Ok(vec![value.clone()]) +} + +fn is_sequence(value: &Bound<'_, PyAny>) -> bool { + value.is_instance_of::() || value.is_instance_of::() +} + +/// The `migrate_from` string when the model keeps an old-name view, as Python resolves it. +fn migrate_from(values: &Bound<'_, PyAny>, retains_old_names: bool) -> PyResult> { + let raw: Bound<'_, PyAny> = values.call_method1("get", (MIGRATE_FROM_KEY,))?; + let Ok(text) = raw.extract::() else { + return Ok(None); + }; + let retained: bool = if values.contains(OLD_NAME_VIEW_KEY)? { + !values + .get_item(OLD_NAME_VIEW_KEY)? + .is(PyBool::new(values.py(), false)) + } else { + retains_old_names + }; + Ok(retained.then_some(text)) +} + +fn audit_gate(audit: &Bound<'_, PyAny>) -> PyResult { + let kind: Bound<'_, PyAny> = audit.getattr("attached_target_kind")?; + let name: Bound<'_, PyAny> = audit.getattr("attached_target_name")?; + let target: Option = if kind.is_none() || name.is_none() { + None + } else { + Some((kind.str()?.extract()?, name.extract()?)) + }; + Ok(AuditGate { + name: audit.getattr("name")?.extract()?, + path: audit + .getattr("audit_file")? + .getattr("relative_path")? + .str()? + .extract()?, + target, + scope_deps: object_keys(&audit.getattr("scope_deps")?)?, + }) +} + +fn graph_resource(resource: &Bound<'_, PyAny>) -> PyResult { + Ok(GraphResource { + key: object_key(&resource.getattr("key")?)?, + deps: object_keys(&resource.getattr("deps")?)?, + }) +} + +/// A `CompiledObjectKey` as its `(resource type, name)` pair. +fn object_key(key: &Bound<'_, PyAny>) -> PyResult { + Ok(( + key.getattr("resource_type")?.str()?.extract()?, + key.getattr("name")?.extract()?, + )) +} + +fn object_keys(keys: &Bound<'_, PyAny>) -> PyResult> { + keys.try_iter()?.map(|key| object_key(&key?)).collect() +} + +/// `SqlResourceRef`s as `(kind, name)` pairs. +pub(crate) fn ref_keys(refs: &Bound<'_, PyAny>) -> PyResult> { + refs.try_iter()? + .map(|reference| { + let reference: Bound<'_, PyAny> = reference?; + Ok(( + reference.getattr("kind")?.str()?.extract()?, + reference.getattr("name")?.extract()?, + )) + }) + .collect() +} diff --git a/python-boundary.toml b/python-boundary.toml index 7114a28c41..7c0969b143 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 129970 +remaining_lines = 129661 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 6b8d8c4dd1..c6e93de0bc 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1909,9 +1909,14 @@ def assemble_project_resource_facts( list[bool], tuple[str, int, str, str, str | None, int | None] | None, ]: ... -def check_native_explicit_references(request_json: str, /) -> str: ... +def explicit_reference_report( + project: Any, + settings: dict[str, object], + owners: list[tuple[Any, int, list[tuple[str, str, int]]]], + /, +) -> Any: ... def native_execution_gate_edges( - request_json: str, / + project: Any, / ) -> tuple[ list[tuple[str, tuple[str, str], tuple[str, str], tuple[str, str]]], list[tuple[str, tuple[str, str], tuple[str, str], str, str | None]], diff --git a/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py b/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py index 8c65c7639c..d4c081a7bc 100644 --- a/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py +++ b/src/sqlbuild/compiler/compile/_helpers/explicit_references/checks.py @@ -1,4 +1,4 @@ -"""Compile's explicit-reference checks: Python collects the facts, native decides.""" +"""Compile's explicit-reference checks: Python collects user-code owners, native decides.""" from __future__ import annotations @@ -6,46 +6,19 @@ import textwrap from collections.abc import Callable, Mapping from dataclasses import dataclass, field -from itertools import chain from pathlib import Path -from typing import Any - -import orjson import sqlbuild._native as _native -from sqlbuild.compiler.compile.constants import MIGRATE_FROM_CONFIG_KEY -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledRelationLocation, - CompilerDiagnostic, - PythonSqlReferenceReport, -) -from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, - CompiledAuditView, - CompiledModelView, - CompiledProjectView, - CompiledSourceView, - DiagnosticPhase, - DiagnosticSeverity, -) +from sqlbuild.compiler.compile.models import PythonSqlReferenceReport +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.main.explicit_references_help import explicit_references_help -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, SqlHookEntry +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.graph.main._model_python_hook_names import model_python_hook_names -from sqlbuild.compiler.migrations.main._resolve_old_name_view_retention import ( - resolve_old_name_view_retention, -) -from sqlbuild.compiler.references.models import LiteralSqlRelation, RelationName from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind from sqlbuild.python_nodes.main.read_loader_definition import read_loader_definition from sqlbuild.python_nodes.models import LoaderDefinition, SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind -from sqlbuild.spec.contracts.models import SourceEntry, SourceLocation -_MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") -_RAISE_KEY: str = "raise" -_REPORT_KEY: str = "report" _VALUE_PLACEHOLDER: str = "__sqlbuild_python_value_" @@ -59,13 +32,6 @@ class _PythonSqlOwner: upstream_loader_by_source: Mapping[str, str] = field(default_factory=dict) -@dataclass(frozen=True) -class _LiteralSql: - method: str - sql: str - line: int - - def explicit_reference_report( *, project: CompiledProjectView, @@ -77,222 +43,26 @@ def explicit_reference_report( owners: tuple[_PythonSqlOwner, ...] = ( _owners(project=project, discovered_inputs=discovered_inputs) if enforce_explicit else () ) - literals: tuple[tuple[_LiteralSql, ...], ...] = tuple( - _literal_sql_calls(owner.function) for owner in owners - ) paths: list[Path] = sorted({owner.relative_path for owner in owners}) path_order: dict[Path, int] = {path: index for index, path in enumerate(paths)} - outcome: dict[str, Any] = orjson.loads( - _native.check_native_explicit_references( - orjson.dumps( - { - "enforce_explicit": enforce_explicit, - "dialect": project.sql_analysis_dialect, - "explicit_help": ( - explicit_references_help(allowed="hard-coded relation names") - if enforce_explicit - else "" - ), - "placeholder": _VALUE_PLACEHOLDER, - "resources": _resource_rows(project), - "models": [ - _model_row( - model=model, - old_name_retention=( - discovered_inputs.project_config.migrations.old_name_views - ), - ) - for model in project.models - ], - "sources": [_source_row(source) for source in project.sources], - "seeds": [ - { - "name": seed.name, - "relation": _location_row(seed.destination), - } - for seed in project.seeds - ], - "audits": [_audit_row(audit) for audit in project.audits], - "hooks": [ - { - "name": hook.name, - "path": hook.relative_path.as_posix(), - "reads": _ref_rows(hook.reads), - } - for hook in project.hook_functions - ], - "owners": [ - _owner_row( - owner=owner, - literals=owner_literals, - path_order=path_order[owner.relative_path], - ) - for owner, owner_literals in zip(owners, literals, strict=True) - ], - } - ).decode() - ) - ) - if _RAISE_KEY in outcome: - raise CompileInputError(outcome[_RAISE_KEY]) - report: dict[str, Any] = outcome[_REPORT_KEY] - return PythonSqlReferenceReport( - diagnostics=tuple(_diagnostic(row=row, owners=owners) for row in report["diagnostics"]), - unmatched=tuple( - LiteralSqlRelation( - owner_label=owners[row["owner"]].label, - owner_kind=owners[row["owner"]].kind, - relative_path=owners[row["owner"]].relative_path, - line=row["line"], - method=row["method"], - relation=RelationName( - name=row["relation"]["name"], - schema=row["relation"]["schema"], - database=row["relation"]["database"], - ), - ) - for row in report["unmatched"] - ), - ) - - -def _diagnostic(*, row: dict[str, Any], owners: tuple[_PythonSqlOwner, ...]) -> CompilerDiagnostic: - owner: int | None = row["owner"] - resource: list[str] | None = row["resource"] - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=row["code"], - message=row["message"], - resource_type=None if resource is None else resource[0], - resource_name=None if resource is None else resource[1], - path=None if row["path"] is None else Path(row["path"]), - location=( - None - if owner is None - else SourceLocation(path=owners[owner].relative_path, line=row["line"], column=1) - ), - help=row["help"], - ) - - -def _owner_row( - *, owner: _PythonSqlOwner, literals: tuple[_LiteralSql, ...], path_order: int -) -> dict[str, object]: - return { - "label": owner.label, - "kind": str(owner.kind), - "path_order": path_order, - "own_refs": _ref_rows(tuple(owner.own_refs)), - "upstream_loaders": list(owner.upstream_loader_by_source.items()), - "literals": [ - {"method": literal.method, "sql": literal.sql, "line": literal.line} - for literal in literals - ], - } - - -def _resource_rows(project: CompiledProjectView) -> list[dict[str, object]]: - return [ - *(_graph_row(key=model.key_row, deps=model.dep_rows) for model in project.models), - *( - _graph_row(key=_key_row(resource.key), deps=_key_rows(resource.deps)) - for resource in (*project.sources, *project.seeds, *project.functions) - ), - ] - - -def _graph_row(*, key: tuple[str, str], deps: list[tuple[str, str]]) -> dict[str, object]: - return {"key": key, "deps": deps} - - -def _key_row(key: CompiledObjectKey) -> tuple[str, str]: - return (str(key.resource_type), key.name) - - -def _key_rows(keys: tuple[CompiledObjectKey, ...]) -> list[tuple[str, str]]: - return [_key_row(key) for key in keys] - - -def _model_row(*, model: CompiledModelView, old_name_retention: str | None) -> dict[str, object]: - raw: object = model.config.values.get(MIGRATE_FROM_CONFIG_KEY) - retained: bool = isinstance(raw, str) and ( - resolve_old_name_view_retention( - config_values=model.config.values, project_retention=old_name_retention - ) - is not None - ) - return { - "name": model.name, - "path": str(model.relative_path), - "destination": _destination_row(model.destination_row), - "python_hooks": list(model_python_hook_names(model=model)), - "sql_hooks": list( - chain.from_iterable( - _sql_hook_rows(hook_key=hook_key, entries=model.config.values.get(hook_key)) - for hook_key in _MODEL_HOOK_KEYS - ) - ), - "migrate_from": raw if retained else None, - } - - -def _sql_hook_rows(*, hook_key: str, entries: object) -> list[dict[str, object]]: - """SQL hook entries with declared reads, at their position among the key's entries.""" - - items: tuple[object, ...] = tuple(entries) if isinstance(entries, list | tuple) else (entries,) - return [ + return _native.explicit_reference_report( + project, { - "hook_key": hook_key, - "index": index, - "name": entry.name, - "path": None if entry.relative_path is None else str(entry.relative_path), - "reads": _ref_rows(entry.reads), - } - for index, entry in enumerate(items) - if isinstance(entry, SqlHookEntry) and entry.reads - ] - - -def _source_row(source: CompiledSourceView) -> dict[str, object]: - entry: SourceEntry = source.source_entry - return { - "name": source.name, - "table": entry.table, - "schema": entry.schema, - "database": entry.database, - "expression": entry.expression is not None, - } - - -def _audit_row(audit: CompiledAuditView) -> dict[str, object]: - target: tuple[str, str] | None = ( - ( - str(AttachedAuditTargetKind(audit.attached_target_kind).resource_type), - audit.attached_target_name, - ) - if audit.attached_target_kind is not None and audit.attached_target_name is not None - else None + "retains_old_names": discovered_inputs.project_config.migrations.old_name_views + is not None, + "enforce_explicit": enforce_explicit, + "explicit_help": ( + explicit_references_help(allowed="hard-coded relation names") + if enforce_explicit + else "" + ), + "placeholder": _VALUE_PLACEHOLDER, + }, + [ + (owner, path_order[owner.relative_path], _literal_sql_calls(owner.function)) + for owner in owners + ], ) - return { - "name": audit.name, - "path": str(audit.audit_file.relative_path), - "target": target, - "scope_deps": _key_rows(audit.scope_deps), - } - - -def _location_row(location: CompiledRelationLocation) -> dict[str, str | None]: - return _destination_row((location.database, location.schema, location.name)) - - -def _destination_row(row: tuple[str | None, str | None, str]) -> dict[str, str | None]: - database, schema, name = row - return {"name": name, "schema": schema, "database": database} - - -def _ref_rows(refs: tuple[SqlResourceRef, ...]) -> list[tuple[str, str]]: - return [(str(ref.kind), ref.name) for ref in refs] def _owners( @@ -369,16 +139,13 @@ def _loader_owners( return tuple(owners) -def _literal_sql_calls(function: Callable[..., object]) -> tuple[_LiteralSql, ...]: +def _literal_sql_calls(function: Callable[..., object]) -> list[tuple[str, str, int]]: try: lines: list[str] start_line: int lines, start_line = inspect.getsourcelines(function) except (OSError, TypeError): - return () - return tuple( - _LiteralSql(method=method, sql=sql, line=line) - for method, sql, line in _native.python_literal_sql_calls( - textwrap.dedent("".join(lines)), start_line, _VALUE_PLACEHOLDER - ) + return [] + return _native.python_literal_sql_calls( + textwrap.dedent("".join(lines)), start_line, _VALUE_PLACEHOLDER ) diff --git a/src/sqlbuild/compiler/graph/_helpers/execution_gates.py b/src/sqlbuild/compiler/graph/_helpers/execution_gates.py index d2180b5c69..8df4b01988 100644 --- a/src/sqlbuild/compiler/graph/_helpers/execution_gates.py +++ b/src/sqlbuild/compiler/graph/_helpers/execution_gates.py @@ -2,24 +2,18 @@ from __future__ import annotations -from itertools import chain from pathlib import Path -import orjson - import sqlbuild._native as _native from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import ( - AttachedAuditTargetKind, - CompiledAuditView, CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.discovery.models import PythonHookEntry, SqlHookEntry +from sqlbuild.compiler.discovery.models import PythonHookEntry from sqlbuild.compiler.graph.models import AttachedAuditGateEdge, HookReadEdge from sqlbuild.compiler.graph.types import HookReadType, NativeKey -from sqlbuild.python_nodes.models import SqlResourceRef _MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") @@ -42,7 +36,7 @@ def attached_audit_gate_edges_impl( ) -> tuple[AttachedAuditGateEdge, ...]: """Return edges from each extra audit read to its model or seed target, or source triggers.""" - gates, _ = _native.native_execution_gate_edges(_request(project)) + gates, _ = _native.native_execution_gate_edges(project) return tuple( AttachedAuditGateEdge( audit_name=audit_name, target=_key(target), gated=_key(gated), read=_key(read) @@ -54,7 +48,7 @@ def attached_audit_gate_edges_impl( def hook_read_edges_impl(*, project: CompiledProjectView) -> tuple[HookReadEdge, ...]: """Return edges from each hook read to the model that runs the hook.""" - _, hooks = _native.native_execution_gate_edges(_request(project)) + _, hooks = _native.native_execution_gate_edges(project) return tuple( HookReadEdge( hook_name=hook_name, @@ -69,73 +63,3 @@ def hook_read_edges_impl(*, project: CompiledProjectView) -> tuple[HookReadEdge, def _key(pair: NativeKey) -> CompiledObjectKey: return CompiledObjectKey(resource_type=CompiledResourceType(pair[0]), name=pair[1]) - - -def _request(project: CompiledProjectView) -> str: - return orjson.dumps( - { - "resources": [ - {"key": model.key_row, "deps": model.dep_rows} for model in project.models - ], - "models": [_model_row(model) for model in project.models], - "audits": [_audit_row(audit) for audit in project.audits], - "hooks": [ - { - "name": hook.name, - "path": hook.relative_path.as_posix(), - "reads": _ref_rows(hook.reads), - } - for hook in project.hook_functions - ], - } - ).decode() - - -def _model_row(model: CompiledModelView) -> dict[str, object]: - return { - "name": model.name, - "path": str(model.relative_path), - "python_hooks": list(model_python_hook_names_impl(model)), - "sql_hooks": list( - chain.from_iterable( - _sql_hook_rows(hook_key=hook_key, entries=model.config.values.get(hook_key)) - for hook_key in _MODEL_HOOK_KEYS - ) - ), - } - - -def _sql_hook_rows(*, hook_key: str, entries: object) -> list[dict[str, object]]: - items: tuple[object, ...] = tuple(entries) if isinstance(entries, list | tuple) else (entries,) - return [ - { - "hook_key": hook_key, - "index": index, - "name": entry.name, - "path": None if entry.relative_path is None else str(entry.relative_path), - "reads": _ref_rows(entry.reads), - } - for index, entry in enumerate(items) - if isinstance(entry, SqlHookEntry) and entry.reads - ] - - -def _audit_row(audit: CompiledAuditView) -> dict[str, object]: - target: tuple[str, str] | None = ( - ( - str(AttachedAuditTargetKind(audit.attached_target_kind).resource_type), - audit.attached_target_name, - ) - if audit.attached_target_kind is not None and audit.attached_target_name is not None - else None - ) - return { - "name": audit.name, - "path": "", - "target": target, - "scope_deps": [(str(key.resource_type), key.name) for key in audit.scope_deps], - } - - -def _ref_rows(refs: tuple[SqlResourceRef, ...]) -> list[tuple[str, str]]: - return [(str(ref.kind), ref.name) for ref in refs] From 5f40f68e0b9cdea14b8c4eba4a5018fd07ade004 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:25:25 +0100 Subject: [PATCH 03/14] refactor: read resource SQL check inputs natively The audit, SQL test and SQL hook SQL checks read their inputs from the project views and build the CompilerDiagnostic results in Rust; get_resource_sql_diagnostics is a one-call wrapper. --- .../_helpers/semantic_checks/resource_sql.rs | 452 ++++++++++++------ python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 11 +- .../_helpers/diagnostics/resource_sql.py | 150 +----- 4 files changed, 311 insertions(+), 304 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index 3ef89e711a..5fb50715ed 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -3,8 +3,9 @@ use std::sync::Arc; use pyo3::prelude::{ - Bound, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyRef, PyResult, Python, }; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::semantic_checks::main::resource_sql_diagnostics::resource_sql_diagnostics; use sqlbuild_analysis::semantic_checks::models::{ @@ -17,156 +18,315 @@ use crate::bindings::models::ProjectCatalog; use crate::bindings::types::CompilerDetach; const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const SQL_ANALYSIS_CONFIG_KEY: &str = "sql_analysis"; +const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; +const BUILT_IN_AUDIT_ROOT: &str = ""; +const UDF_KIND: &str = "udf"; +const TABLE_FUNCTION_KIND: &str = "table_fn"; +const TABLE_FUNCTION_STUB_PREFIX: &str = "__sqlbuild_table_function_"; -type Shape = (String, Vec<(String, String)>); -/// `(name, definition, (target kind, target), column, built-in, path, contents, bodies, opts out, relations)`. -type AuditRow = ( - String, - String, - Option<(String, String)>, - Option, - bool, - String, - String, - Vec<(Option, Option)>, - bool, - Vec, -); -/// `(name, expected models, path, contents, compiled body, authored body, opts out)`. -type TestRow = (String, Vec, String, String, String, String, bool); -/// `(key, index, statement, hook path, definition, relations)`. -type HookRow = ( - String, - usize, - String, - Option, - Option, - Vec, -); -/// `(name, path, authored SQL, analysis disabled, hooks)`. -type ModelRow = (String, String, String, bool, Vec); -/// `(dialect, quoted ignore case, require analysis, shapes, audits, tests, models, hook files)`. -type RequestInput = ( - Option, - bool, - bool, - Vec, - Vec, - Vec, - Vec, - Vec<(String, String)>, -); -/// `(kind, name, is error, code, message, path, span, note, help)`. -type DiagnosticRow = ( - &'static str, - String, - bool, - String, - String, - String, - (i64, i64, Option, Option), - Option, - Option, -); - -fn audit(row: AuditRow) -> AuditSqlInput { - let ( - name, - definition_name, - attached_target, - attached_column, - built_in, - path, - contents, - bodies, - opts_out, - relations, - ) = row; - AuditSqlInput { - name, - definition_name, +/// The compile models a resource SQL diagnostic is built from. +struct DiagnosticClasses<'py> { + compiler_diagnostic: Bound<'py, PyAny>, + source_location: Bound<'py, PyAny>, + path: Bound<'py, PyAny>, +} + +/// Bind the project's audit, SQL test and SQL hook bodies on `catalog` against `shapes`. +#[pyfunction] +fn resource_sql_diagnostics_of<'py>( + catalog: PyRef<'_, ProjectCatalog>, + project: &Bound<'py, PyAny>, + shapes: &Bound<'py, PyDict>, + profile: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let request: ResourceSqlRequest = request(project, shapes, profile)?; + let catalog = &catalog.inner; + let outcome = py + .compiler_detach(|| Ok(resource_sql_diagnostics(&request, catalog))) + .map_err(compiler_error)?; + let diagnostics: Vec = match outcome { + Ok(diagnostics) => diagnostics, + Err(failure) => return Err(failure_error(py, failure)?), + }; + let classes = DiagnosticClasses { + compiler_diagnostic: py + .import(COMPILE_MODELS_MODULE)? + .getattr("CompilerDiagnostic")?, + source_location: py + .import(CONTRACT_MODELS_MODULE)? + .getattr("SourceLocation")?, + path: py.import("pathlib")?.getattr("Path")?, + }; + let built: Vec> = diagnostics + .into_iter() + .map(|diagnostic| python_diagnostic(py, &classes, diagnostic)) + .collect::>()?; + PyTuple::new(py, built) +} + +fn request( + project: &Bound<'_, PyAny>, + shapes: &Bound<'_, PyDict>, + profile: &Bound<'_, PyAny>, +) -> PyResult { + let py: Python<'_> = project.py(); + let reference_class: Bound<'_, PyAny> = py + .import(COMPILE_MODELS_MODULE)? + .getattr("CompileSqlReference")?; + let mut audits: Vec = Vec::new(); + for audit in project.getattr("audits")?.try_iter()? { + audits.push(audit_input(&audit?, &reference_class)?); + } + let mut sql_tests: Vec = Vec::new(); + for test in project.getattr("sql_tests")?.try_iter()? { + sql_tests.push(sql_test_input(&test?)?); + } + let mut models: Vec = Vec::new(); + for model in project.getattr("models")?.try_iter()? { + models.push(model_input(&model?, &reference_class)?); + } + let mut hook_files: Vec<(String, Arc)> = Vec::new(); + for file in project.getattr("sql_hook_files")?.try_iter()? { + let file: Bound<'_, PyAny> = file?; + hook_files.push(( + file.getattr("relative_path")?.str()?.extract()?, + Arc::from(file.getattr("contents")?.extract::()?), + )); + } + Ok(ResourceSqlRequest { + dialect: profile.getattr("sql_analysis_dialect")?.extract()?, + quoted_ignore_case: profile + .getattr("quoted_identifiers_ignore_case")? + .extract()?, + require_sql_analysis: project + .getattr("settings")? + .getattr("require_sql_analysis")? + .extract()?, + shapes: shapes + .iter() + .map(|(name, columns)| shape(&name, &columns)) + .collect::>()?, + audits, + sql_tests, + models, + hook_files, + }) +} + +/// One closed relation shape: its name and `(column, type)` pairs in order. +fn shape( + name: &Bound<'_, PyAny>, + columns: &Bound<'_, PyAny>, +) -> PyResult<(String, Vec<(String, String)>)> { + let mut pairs: Vec<(String, String)> = Vec::new(); + for pair in columns.call_method0("items")?.try_iter()? { + pairs.push(pair?.extract()?); + } + Ok((name.extract()?, pairs)) +} + +fn audit_input( + audit: &Bound<'_, PyAny>, + reference_class: &Bound<'_, PyAny>, +) -> PyResult { + let file: Bound<'_, PyAny> = audit.getattr("audit_file")?; + let block: Bound<'_, PyAny> = audit.getattr("audit_block")?; + let target_name: Bound<'_, PyAny> = audit.getattr("attached_target_name")?; + let attached_target: Option<(String, String)> = if target_name.is_none() { + None + } else { + Some(( + audit.getattr("attached_target_kind")?.str()?.extract()?, + target_name.extract()?, + )) + }; + let first_part: Option = file + .getattr("file_path")? + .getattr("parts")? + .try_iter()? + .next() + .transpose()? + .map(|part| part.extract()) + .transpose()?; + let mut bodies: Vec<(Option, Option)> = Vec::new(); + for attribute in ["sql_body", "measure_sql", "evidence_sql"] { + bodies.push(( + audit.getattr(attribute)?.extract()?, + block.getattr(attribute)?.extract()?, + )); + } + Ok(AuditSqlInput { + name: audit.getattr("name")?.extract()?, + definition_name: audit.getattr("definition_name")?.extract()?, attached_target, - attached_column, - built_in, - path, - contents: Arc::from(contents), + attached_column: audit.getattr("attached_column_name")?.extract()?, + built_in: first_part.as_deref() == Some(BUILT_IN_AUDIT_ROOT), + path: file.getattr("relative_path")?.str()?.extract()?, + contents: Arc::from(file.getattr("contents")?.extract::()?), bodies, - opts_out, - relations, + opts_out: opts_out(&block.getattr("header_values")?)?, + relations: relations(&audit.getattr("references")?, reference_class)?, + }) +} + +fn sql_test_input(test: &Bound<'_, PyAny>) -> PyResult { + let file: Bound<'_, PyAny> = test.getattr("test_file")?; + let block: Bound<'_, PyAny> = test.getattr("test_block")?; + Ok(SqlTestSqlInput { + name: test.getattr("name")?.extract()?, + expected_model_names: test + .getattr("expected_model_names")? + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?, + path: file.getattr("relative_path")?.str()?.extract()?, + contents: Arc::from(file.getattr("contents")?.extract::()?), + sql_body: test.getattr("sql_body")?.extract()?, + authored_body: block.getattr("sql_body")?.extract()?, + opts_out: opts_out(&block.getattr("header_values")?)?, + }) +} + +fn model_input( + model: &Bound<'_, PyAny>, + reference_class: &Bound<'_, PyAny>, +) -> PyResult { + let values: Bound<'_, PyAny> = model.getattr("config")?.getattr("values")?; + let mut hooks: Vec = Vec::new(); + for key in MODEL_HOOK_KEYS { + let entries: Bound<'_, PyAny> = values.call_method1("get", (key,))?; + if !(entries.is_instance_of::() || entries.is_instance_of::()) { + continue; + } + for (index, entry) in entries.try_iter()?.enumerate() { + let entry: Bound<'_, PyAny> = entry?; + let Some(statement) = text(&optional_attribute(&entry, "statement")?)? else { + continue; + }; + let hook_path: Bound<'_, PyAny> = optional_attribute(&entry, "relative_path")?; + let reads: Bound<'_, PyAny> = optional_attribute(&entry, "reads")?; + hooks.push(HookSqlInput { + key: key.to_owned(), + index, + statement, + hook_path: if hook_path.is_none() { + None + } else { + Some(hook_path.str()?.extract()?) + }, + definition: text(&optional_attribute(&entry, "definition_sql")?)?, + relations: if reads.is_truthy()? { + relations(&reads, reference_class)? + } else { + Vec::new() + }, + }); + } } + Ok(HookModelInput { + name: model.getattr("name")?.extract()?, + path: model.getattr("relative_path")?.str()?.extract()?, + authored_sql: Arc::from(model.getattr("authored_sql")?.extract::()?), + analysis_disabled: opts_out(&values)?, + hooks, + }) } -fn sql_test(row: TestRow) -> SqlTestSqlInput { - let (name, expected_model_names, path, contents, sql_body, authored_body, opts_out) = row; - SqlTestSqlInput { - name, - expected_model_names, - path, - contents: Arc::from(contents), - sql_body, - authored_body, - opts_out, +/// The value when it is a `str`, otherwise `None`. +fn text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_instance_of::() { + return value.extract().map(Some); } + Ok(None) } -fn model(row: ModelRow) -> HookModelInput { - let (name, path, authored_sql, analysis_disabled, hooks) = row; - HookModelInput { - name, - path, - authored_sql: Arc::from(authored_sql), - analysis_disabled, - hooks: hooks - .into_iter() - .map( - |(key, index, statement, hook_path, definition, relations)| HookSqlInput { - key, - index, - statement, - hook_path, - definition, - relations, - }, - ) - .collect(), +/// `getattr(value, name, None)`. +fn optional_attribute<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult> { + if value.hasattr(name)? { + value.getattr(name) + } else { + Ok(value.py().None().into_bound(value.py())) } } -fn request(input: RequestInput) -> ResourceSqlRequest { - let (dialect, quoted_ignore_case, require_sql_analysis, shapes, audits, tests, models, files) = - input; - ResourceSqlRequest { - dialect, - quoted_ignore_case, - require_sql_analysis, - shapes, - audits: audits.into_iter().map(audit).collect(), - sql_tests: tests.into_iter().map(sql_test).collect(), - models: models.into_iter().map(model).collect(), - hook_files: files - .into_iter() - .map(|(path, contents)| (path, Arc::from(contents))) - .collect(), +/// Whether the values set `sql_analysis = false`. +fn opts_out(values: &Bound<'_, PyAny>) -> PyResult { + Ok(values + .call_method1("get", (SQL_ANALYSIS_CONFIG_KEY,))? + .is(PyBool::new(values.py(), false))) +} + +/// `sorted(binding_relation_names(...))` of the `CompileSqlReference`s among `references`. +fn relations( + references: &Bound<'_, PyAny>, + reference_class: &Bound<'_, PyAny>, +) -> PyResult> { + let mut names: Vec = Vec::new(); + for reference in references.try_iter()? { + let reference: Bound<'_, PyAny> = reference?; + if !reference.is_instance(reference_class)? { + continue; + } + let kind: String = reference.getattr("ref_kind")?.str()?.extract()?; + let name: String = reference.getattr("ref_name")?.extract()?; + if kind == UDF_KIND { + continue; + } + names.push(if kind == TABLE_FUNCTION_KIND { + format!("{TABLE_FUNCTION_STUB_PREFIX}{name}") + } else { + name + }); } + names.sort_unstable(); + names.dedup(); + Ok(names) } -fn row(diagnostic: ResourceSqlDiagnostic) -> DiagnosticRow { - let kind = match diagnostic.kind { - ResourceSqlKind::Audit => "audit", - ResourceSqlKind::SqlTest => "sql_test", - ResourceSqlKind::Model => "model", - }; - ( - kind, - diagnostic.resource_name, - diagnostic.is_error, - diagnostic.code, - diagnostic.message, - diagnostic.path, - diagnostic.span, - diagnostic.note, - diagnostic.help, - ) +fn python_diagnostic<'py>( + py: Python<'py>, + classes: &DiagnosticClasses<'py>, + diagnostic: ResourceSqlDiagnostic, +) -> PyResult> { + let (line, column, end_line, end_column) = diagnostic.span; + let location: Bound<'py, PyDict> = PyDict::new(py); + location.set_item("path", classes.path.call1((&diagnostic.path,))?)?; + location.set_item("line", line)?; + location.set_item("column", column)?; + location.set_item("end_line", end_line)?; + location.set_item("end_column", end_column)?; + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("phase", "compile")?; + fields.set_item( + "severity", + if diagnostic.is_error { + "error" + } else { + "warning" + }, + )?; + fields.set_item("code", diagnostic.code)?; + fields.set_item("message", diagnostic.message)?; + fields.set_item( + "resource_type", + match diagnostic.kind { + ResourceSqlKind::Audit => "audit", + ResourceSqlKind::SqlTest => "sql_test", + ResourceSqlKind::Model => "model", + }, + )?; + fields.set_item("resource_name", diagnostic.resource_name)?; + fields.set_item( + "location", + classes.source_location.call((), Some(&location))?, + )?; + fields.set_item("notes", PyTuple::new(py, diagnostic.note)?)?; + fields.set_item("help", diagnostic.help)?; + classes.compiler_diagnostic.call((), Some(&fields)) } fn failure_error(py: Python<'_>, failure: ResourceSqlFailure) -> PyResult { @@ -182,24 +342,6 @@ fn failure_error(py: Python<'_>, failure: ResourceSqlFailure) -> PyResult }) } -/// Bind audit, SQL test and SQL hook bodies on `catalog` as Python's resource SQL checks do. -#[pyfunction] -fn resource_sql_diagnostic_rows( - py: Python<'_>, - catalog: PyRef<'_, ProjectCatalog>, - input: RequestInput, -) -> PyResult> { - let request = request(input); - let catalog = &catalog.inner; - let outcome = py - .compiler_detach(|| Ok(resource_sql_diagnostics(&request, catalog))) - .map_err(compiler_error)?; - match outcome { - Ok(diagnostics) => Ok(diagnostics.into_iter().map(row).collect()), - Err(failure) => Err(failure_error(py, failure)?), - } -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(resource_sql_diagnostic_rows, module)?) + module.add_function(wrap_pyfunction!(resource_sql_diagnostics_of, module)?) } diff --git a/python-boundary.toml b/python-boundary.toml index 7c0969b143..2018b0dbb9 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 129661 +remaining_lines = 129519 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index c6e93de0bc..46de9b86a3 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -44,6 +44,7 @@ from sqlbuild.compiler.compile.models import ( InferredColumn, LoadedMacro, MacroContext, + PythonSqlReferenceReport, StaticMacroExport, StaticMacroFault, ) @@ -1308,7 +1309,13 @@ def unknown_mapping_keys( keys: list[tuple[str, bool]], allowed: list[str], / ) -> tuple[str, str] | None: ... def unsupported_keys_help(keys: list[str], supported_keys: list[str], /) -> str: ... -def resource_sql_diagnostic_rows(catalog: object, request: object, /) -> list[object]: ... +def resource_sql_diagnostics_of( + catalog: object, + project: Any, + shapes: dict[str, dict[str, str]], + profile: ExpressionInferenceProfile, + /, +) -> tuple[CompilerDiagnostic, ...]: ... def compiler_query_hash(sql: str, dialect: str | None, /) -> str: ... def function_definition_hash(sql: str, language: str, dialect: str | None, /) -> str: ... def normalize_query_text(sql: str, /) -> str: ... @@ -1914,7 +1921,7 @@ def explicit_reference_report( settings: dict[str, object], owners: list[tuple[Any, int, list[tuple[str, str, int]]]], /, -) -> Any: ... +) -> PythonSqlReferenceReport: ... def native_execution_gate_edges( project: Any, / ) -> tuple[ diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py index 9712e43569..c04e76824c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/resource_sql.py @@ -2,34 +2,12 @@ from __future__ import annotations -from pathlib import Path from typing import Any, cast import sqlbuild._native as _native from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import binding_relation_names -from sqlbuild.compiler.compile.models import CompilerDiagnostic, CompileSqlReference -from sqlbuild.compiler.compile.types import ( - CompiledAuditView, - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - DiagnosticPhase, - DiagnosticSeverity, -) -from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_CONFIG_KEY -from sqlbuild.spec.contracts.models import SourceLocation - -_MODEL_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") -_BUILT_IN_AUDIT_ROOT: str = "" -_RESOURCE_TYPES: dict[str, CompiledResourceType] = { - "audit": CompiledResourceType.AUDIT, - "sql_test": CompiledResourceType.SQL_TEST, - "model": CompiledResourceType.MODEL, -} - -type _Span = tuple[int, int, int | None, int | None] -type _Row = tuple[str, str, bool, str, str, str, _Span, str | None, str | None] +from sqlbuild.compiler.compile.models import CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledProjectView def get_resource_sql_diagnostics( @@ -40,126 +18,6 @@ def get_resource_sql_diagnostics( ) -> tuple[CompilerDiagnostic, ...]: """Bind audits, SQL tests and SQL hooks against the same closed shapes as models.""" - request: Any = ( - profile.sql_analysis_dialect, - profile.quoted_identifiers_ignore_case, - project.settings.require_sql_analysis, - [(name, list(columns.items())) for name, columns in shapes.items()], - [_audit_row(audit) for audit in project.audits], - [ - ( - test.name, - list(test.expected_model_names), - str(test.test_file.relative_path), - test.test_file.contents, - test.sql_body, - test.test_block.sql_body, - test.test_block.header_values.get(SQL_ANALYSIS_CONFIG_KEY) is False, - ) - for test in project.sql_tests - ], - [_model_row(model) for model in project.models], - [ - (str(hook_file.relative_path), hook_file.contents) - for hook_file in project.sql_hook_files - ], - ) - rows: list[_Row] = cast( - list[_Row], - _native.resource_sql_diagnostic_rows(cast(Any, project.binding_catalog).native, request), - ) - paths: dict[str, Path] = {str(path): path for path in _paths(project)} - return tuple(_diagnostic(row=row, paths=paths) for row in rows) - - -def _paths(project: CompiledProjectView) -> list[Path]: - """Every path a resource SQL diagnostic can name, so rows map back to the authored `Path`.""" - - paths: list[Path] = [audit.audit_file.relative_path for audit in project.audits] - paths.extend(test.test_file.relative_path for test in project.sql_tests) - paths.extend(model.relative_path for model in project.models) - paths.extend(hook_file.relative_path for hook_file in project.sql_hook_files) - return paths - - -def _relations(references: tuple[object, ...]) -> list[str]: - return sorted( - binding_relation_names( - tuple( - reference for reference in references if isinstance(reference, CompileSqlReference) - ) - ) - ) - - -def _audit_row(audit: CompiledAuditView) -> tuple[object, ...]: - target: tuple[str, str] | None = ( - None - if audit.attached_target_name is None - else (str(audit.attached_target_kind), audit.attached_target_name) - ) - return ( - audit.name, - audit.definition_name, - target, - audit.attached_column_name, - audit.audit_file.file_path.parts[:1] == (_BUILT_IN_AUDIT_ROOT,), - str(audit.audit_file.relative_path), - audit.audit_file.contents, - [ - (audit.sql_body, audit.audit_block.sql_body), - (audit.measure_sql, audit.audit_block.measure_sql), - (audit.evidence_sql, audit.audit_block.evidence_sql), - ], - audit.audit_block.header_values.get(SQL_ANALYSIS_CONFIG_KEY) is False, - _relations(audit.references), - ) - - -def _model_row(model: CompiledModelView) -> tuple[object, ...]: - hooks: list[tuple[object, ...]] = [] - for hook_key in _MODEL_HOOK_KEYS: - entries: object = model.config.values.get(hook_key) - if not isinstance(entries, list | tuple): - continue - for hook_index, entry in enumerate(cast(list[object] | tuple[object, ...], entries)): - statement: object = getattr(entry, "statement", None) - if not isinstance(statement, str): - continue - hook_path: Path | None = getattr(entry, "relative_path", None) - definition: object = getattr(entry, "definition_sql", None) - hooks.append( - ( - hook_key, - hook_index, - statement, - None if hook_path is None else str(hook_path), - definition if isinstance(definition, str) else None, - _relations(tuple(getattr(entry, "reads", ()) or ())), - ) - ) - return ( - model.name, - str(model.relative_path), - model.authored_sql, - model.config.values.get(SQL_ANALYSIS_CONFIG_KEY) is False, - hooks, - ) - - -def _diagnostic(*, row: _Row, paths: dict[str, Path]) -> CompilerDiagnostic: - kind, name, is_error, code, message, path, span, note, help_text = row - line, column, end_line, end_column = span - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR if is_error else DiagnosticSeverity.WARNING, - code=code, - message=message, - resource_type=_RESOURCE_TYPES[kind], - resource_name=name, - location=SourceLocation( - path=paths[path], line=line, column=column, end_line=end_line, end_column=end_column - ), - notes=() if note is None else (note,), - help=help_text, + return _native.resource_sql_diagnostics_of( + cast(Any, project.binding_catalog).native, project, shapes, profile ) From 39919c2c0f6a9d93d6194f1fb2c17de84bab3002 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:51:45 +0100 Subject: [PATCH 04/14] refactor: report scope index errors and serve the semantic help catalogue natively The compile driver reports each error-severity scope finding through the diagnostic collector from Rust, with the same key and CompilerDiagnostic Python built; diagnostics/scope.py is deleted. semantic_help_catalogue reads the native catalogue the diagnostics already use, replacing the duplicated Python table. model_schema_enum_declarations had no caller, and context_templates' expand_config_templates and build_run_context_values were single-use pass-throughs. --- .../src/semantic_checks/main/mod.rs | 1 + .../main/semantic_help_catalogue.rs | 9 ++ .../compile_driver/assembly_project.rs | 7 +- .../src/bindings/_helpers/scopes/mod.rs | 1 + .../bindings/_helpers/scopes/scope_errors.rs | 99 +++++++++++++++++++ .../semantic_checks/help_catalogue.rs | 15 +++ .../bindings/_helpers/semantic_checks/mod.rs | 1 + .../_helpers/semantic_checks/registration.rs | 3 +- python-boundary.toml | 2 +- .../_helpers/corpus/emitted_codes.py | 4 +- src/sqlbuild/_native.pyi | 1 + .../compile/_helpers/diagnostics/help.py | 32 ------ .../compile/_helpers/diagnostics/scope.py | 69 ------------- .../named_declarations/model_schemas.py | 21 ---- .../_helpers/render/context_templates.py | 44 +-------- .../pipeline/test_semantic_diagnostic_ux.py | 7 +- 16 files changed, 144 insertions(+), 172 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/semantic_checks/main/semantic_help_catalogue.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/help_catalogue.rs delete mode 100644 src/sqlbuild/compiler/compile/_helpers/diagnostics/help.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/diagnostics/scope.py diff --git a/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs b/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs index 95cdc4c65b..b04c2c7848 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/main/mod.rs @@ -3,3 +3,4 @@ pub mod complete_semantic_diagnostics; pub mod finish_type_recovery; pub mod plan_type_recovery; pub mod resource_sql_diagnostics; +pub mod semantic_help_catalogue; diff --git a/crates/sqlbuild-analysis/src/semantic_checks/main/semantic_help_catalogue.rs b/crates/sqlbuild-analysis/src/semantic_checks/main/semantic_help_catalogue.rs new file mode 100644 index 0000000000..0ae8e0741d --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/main/semantic_help_catalogue.rs @@ -0,0 +1,9 @@ +//! The code-owned remediation of each semantic diagnostic code. + +use crate::semantic_checks::constants::SEMANTIC_HELP; + +/// Each semantic diagnostic code and its help, in catalogue order. +#[must_use] +pub fn semantic_help_catalogue() -> &'static [(&'static str, &'static str)] { + SEMANTIC_HELP +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs index 376a3f1339..5f71574f50 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs @@ -16,10 +16,10 @@ use crate::bindings::_helpers::compile_driver::assembly_models::{ binding_rows, compiled_models, project_binding_diagnostics, }; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported, keywords}; +use crate::bindings::_helpers::scopes::scope_errors::report_scope_index_errors; const NATIVE_MODULE: &str = "sqlbuild._native"; const SCOPE_USAGES_MODULE: &str = "sqlbuild.compiler.compile.main._scope_index_with_compile_usages"; -const SCOPE_DIAGNOSTICS_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.scope"; const CONTEXT_TEMPLATES_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.context_templates"; pub(crate) const ADAPTER_NAME_MODULE: &str = @@ -110,10 +110,7 @@ pub(crate) fn assembled_project<'py>( &imported(py, SCOPE_USAGES_MODULE, "scope_index_with_compile_usages")?, &[("inputs", inputs.clone())], )?; - call_with( - &imported(py, SCOPE_DIAGNOSTICS_MODULE, "report_scope_index_errors")?, - &[("index", scope_index.clone())], - )?; + report_scope_index_errors(&scope_index)?; let target_values: Bound<'py, PyAny> = effective_target_values(inputs)?; let models: Bound<'py, PyTuple> = compiled_models(inputs, &context, (&native.getattr("models")?, names))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs index 88f94269f2..c4023fd1a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/mod.rs @@ -6,6 +6,7 @@ pub(crate) mod registration; pub(crate) mod relationship_names; pub(crate) mod relationship_scan; pub(crate) mod scope_cache; +pub(crate) mod scope_errors; pub(crate) mod scope_index; pub(crate) mod scope_lookup; pub(crate) mod scope_objects; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs new file mode 100644 index 0000000000..40b2b87e1c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_errors.rs @@ -0,0 +1,99 @@ +//! Report a scope index's error findings as individual compile diagnostics. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; + +const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const ERROR_SEVERITY: &str = "error"; + +/// `report_scope_index_errors`: each error-severity finding of `index`, keyed for the collector. +pub(crate) fn report_scope_index_errors(index: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = index.py(); + let report = py + .import(COLLECTOR_MODULE)? + .getattr("report_compile_diagnostic")?; + let compiler_diagnostic = py + .import(COMPILE_MODELS_MODULE)? + .getattr("CompilerDiagnostic")?; + let path_class = py.import("pathlib")?.getattr("Path")?; + for item in index.getattr("diagnostics")?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + if item.getattr("severity")?.str()?.extract::()? != ERROR_SEVERITY { + continue; + } + let code: Bound<'_, PyAny> = item.getattr("code")?.getattr("value")?; + let message: Bound<'_, PyAny> = item.getattr("message")?; + let path: Bound<'_, PyAny> = item.getattr("path")?; + let declaration: Bound<'_, PyAny> = item.getattr("declaration")?; + let resource: Bound<'_, PyAny> = item.getattr("resource")?; + let key: Bound<'_, PyTuple> = PyTuple::new( + py, + [ + code.clone(), + or_empty(&path)?, + or_empty(&declaration)?.str()?.into_any(), + or_empty(&resource)?.str()?.into_any(), + message.clone(), + ], + )?; + let (resource_type, resource_name) = owning_resource(&resource, &declaration)?; + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("phase", "compile")?; + fields.set_item("severity", ERROR_SEVERITY)?; + fields.set_item("code", code)?; + fields.set_item("message", message)?; + fields.set_item("resource_type", resource_type)?; + fields.set_item("resource_name", resource_name)?; + if path.is_none() { + fields.set_item("path", py.None())?; + } else { + fields.set_item("path", path_class.call1((path,))?)?; + } + fields.set_item("line", item.getattr("line")?)?; + fields.set_item("column", item.getattr("column")?)?; + let diagnostic: Bound<'_, PyAny> = compiler_diagnostic.call((), Some(&fields))?; + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("key", key)?; + arguments.set_item("diagnostic", diagnostic)?; + report.call((), Some(&arguments))?; + } + Ok(()) +} + +/// `value or ""`. +fn or_empty<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if value.is_truthy()? { + return Ok(value.clone()); + } + Ok(PyString::new(value.py(), "").into_any()) +} + +/// The compiled resource a finding names: its resource, else an audit declaration. +fn owning_resource( + resource: &Bound<'_, PyAny>, + declaration: &Bound<'_, PyAny>, +) -> PyResult<(Option<&'static str>, Option)> { + if !resource.is_none() { + let kind: String = resource.getattr("kind")?.str()?.extract()?; + let resource_type: Option<&'static str> = match kind.as_str() { + "model" => Some("model"), + "source" => Some("source"), + "seed" => Some("seed"), + "function" => Some("udf"), + "test" => Some("sql_test"), + "scenario" => Some("sql_scenario"), + _ => None, + }; + if resource_type.is_some() { + return Ok((resource_type, Some(resource.getattr("name")?.extract()?))); + } + } + if !declaration.is_none() { + let kind: String = declaration.getattr("kind")?.str()?.extract()?; + if matches!(kind.as_str(), "audit" | "singular_audit") { + return Ok((Some("audit"), Some(declaration.getattr("name")?.extract()?))); + } + } + Ok((None, None)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/help_catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/help_catalogue.rs new file mode 100644 index 0000000000..8061ca065b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/help_catalogue.rs @@ -0,0 +1,15 @@ +//! The semantic diagnostic help catalogue, for Python's tests and corpus tooling. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::semantic_checks::main::semantic_help_catalogue::semantic_help_catalogue as catalogue; + +/// Each semantic diagnostic code and its help. +#[pyfunction] +fn semantic_help_catalogue() -> Vec<(&'static str, &'static str)> { + catalogue().to_vec() +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(semantic_help_catalogue, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs index 9ce91706e6..c8ec68a68d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod completion; pub(crate) mod failures; +pub(crate) mod help_catalogue; pub(crate) mod metadata; pub(crate) mod registration; pub(crate) mod resource_sql; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs index 10b484a3e1..9e5dc46d5c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs @@ -3,12 +3,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::semantic_checks::{ - completion, metadata, resource_sql, type_recovery, + completion, help_catalogue, metadata, resource_sql, type_recovery, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { type_recovery::register(module)?; metadata::register(module)?; resource_sql::register(module)?; + help_catalogue::register(module)?; completion::register(module) } diff --git a/python-boundary.toml b/python-boundary.toml index 2018b0dbb9..3b16e285b7 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 129519 +remaining_lines = 129363 [[stays]] path = "executor" diff --git a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py index 5ea544538f..fcdfecef9c 100644 --- a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py +++ b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py @@ -6,6 +6,7 @@ from concurrent.futures import ThreadPoolExecutor from pathlib import Path +import sqlbuild._native as _native from scripts.compiler_differential._helpers.comparing.comparison import ( diagnostic_codes, first_diagnostic, @@ -39,7 +40,6 @@ ) from sqlbuild.compiler.auditing.constants import BUILT_IN_AUDIT_SHADOW_CODE from sqlbuild.compiler.compile import constants as compile_constants -from sqlbuild.compiler.compile._helpers.diagnostics.help import semantic_help_catalogue from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.exceptions import DiscoveryError from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError @@ -118,7 +118,7 @@ def render_error_codes() -> frozenset[str]: def analysis_error_codes(compiler_root: Path) -> frozenset[str]: """Return the help-catalogue codes plus every code in the analysis-stage modules and crates.""" - found: set[str] = set(semantic_help_catalogue()) + found: set[str] = {code for code, _ in _native.semantic_help_catalogue()} for root in ANALYSIS_CODE_SCAN_ROOTS: path: Path = compiler_root / root for module in sorted(path.rglob("*.py")) if path.is_dir() else (path,): diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 46de9b86a3..193db0e92e 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1309,6 +1309,7 @@ def unknown_mapping_keys( keys: list[tuple[str, bool]], allowed: list[str], / ) -> tuple[str, str] | None: ... def unsupported_keys_help(keys: list[str], supported_keys: list[str], /) -> str: ... +def semantic_help_catalogue() -> list[tuple[str, str]]: ... def resource_sql_diagnostics_of( catalog: object, project: Any, diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/help.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/help.py deleted file mode 100644 index 1ccfd4bf25..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/help.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Code-owned semantic remediation; no catch-all help for unrelated diagnostics.""" - -_SEMANTIC_HELP: dict[str, str] = { - "B003": "qualify this column with the intended input alias", - "B004": "give each referenced relation a visible alias in this scope", - "B005": "match the relation alias list to its available output columns", - "B101": "check the function spelling or declare the project function before using it", - "B102": "pass the number of arguments required by this function's signature", - "B210": "use an expression whose inferred type satisfies this SQL construct", - "B211": "use a Boolean predicate in this condition", - "B212": "use operands supported by this arithmetic operator, or convert them explicitly", - "B213": "supply argument types supported by this function or conditional expression", - "B214": "make the assigned expression compatible with the destination type", - "B215": "align the corresponding types in each set-operation branch", - "B216": "return the same number of columns on both sides of this operation", - "B217": "compare compatible types using a correctly typed literal or explicit conversion", - "B218": "choose a supported source-to-target cast for this dialect", - "B219": "declare or cast this expression's unknown type so it can be checked", - "B230": "add this expression to GROUP BY or use it inside an aggregate", - "B231": "move this aggregate to SELECT or HAVING, or compute it in an input CTE", - "B232": "use this window function with a valid OVER clause and window context", - "B233": "use distinct names for relations or CTEs within this scope", - "B234": "use a valid constant bound for LIMIT or OFFSET", - "B300": "change the metadata column name to an existing output column", - "B301": "align this declared metadata or audit value with the column or argument type", - "B302": "align the SQL-test fixture or expected columns with the tested resource's output", -} - - -def semantic_help_catalogue() -> dict[str, str]: - """Return the code-owned remediation catalogue.""" - return dict(_SEMANTIC_HELP) diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/scope.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/scope.py deleted file mode 100644 index cc2e0e7731..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/scope.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Report declaration-scope index errors as individual compile diagnostics.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import ( - CompiledResourceType, - DiagnosticPhase, - DiagnosticSeverity, -) -from sqlbuild.compiler.scopes.models import ScopeDiagnostic, ScopeIndex -from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind -from sqlbuild.compiler.scopes.types import DiagnosticSeverity as ScopeSeverity - -_RESOURCE_TYPES: dict[ResourceKind, CompiledResourceType] = { - ResourceKind.MODEL: CompiledResourceType.MODEL, - ResourceKind.SOURCE: CompiledResourceType.SOURCE, - ResourceKind.SEED: CompiledResourceType.SEED, - ResourceKind.FUNCTION: CompiledResourceType.UDF, - ResourceKind.TEST: CompiledResourceType.SQL_TEST, - ResourceKind.SCENARIO: CompiledResourceType.SQL_SCENARIO, -} -_DECLARATION_RESOURCE_TYPES: dict[DeclarationKind, CompiledResourceType] = { - DeclarationKind.AUDIT: CompiledResourceType.AUDIT, - DeclarationKind.SINGULAR_AUDIT: CompiledResourceType.AUDIT, -} - - -def report_scope_index_errors(*, index: ScopeIndex) -> None: - """Report each error-severity scope finding as its own compile diagnostic.""" - - for item in index.diagnostics: - if item.severity is not ScopeSeverity.ERROR: - continue - report_compile_diagnostic( - key=( - item.code.value, - item.path or "", - str(item.declaration or ""), - str(item.resource or ""), - item.message, - ), - diagnostic=_compile_diagnostic(item), - ) - - -def _compile_diagnostic(item: ScopeDiagnostic) -> CompilerDiagnostic: - resource_type: CompiledResourceType | None = None - resource_name: str | None = None - if item.resource is not None and item.resource.kind in _RESOURCE_TYPES: - resource_type = _RESOURCE_TYPES[item.resource.kind] - resource_name = item.resource.name - elif item.declaration is not None and item.declaration.kind in _DECLARATION_RESOURCE_TYPES: - resource_type = _DECLARATION_RESOURCE_TYPES[item.declaration.kind] - resource_name = item.declaration.name - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=item.code.value, - message=item.message, - resource_type=resource_type, - resource_name=resource_name, - path=Path(item.path) if item.path is not None else None, - line=item.line, - column=item.column, - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py b/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py index ad4f50f1d8..0d73b79f17 100644 --- a/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py +++ b/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py @@ -11,12 +11,10 @@ from sqlbuild.compiler.compile.types import DiagnosticPhase, DiagnosticSeverity from sqlbuild.compiler.discovery.models import ( DiscoveredModelSchemaFile, - EnumDeclaration, ModelSchemaDeclaration, ) from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, DeclarationRecord, ResourceIdentity, UsageRecord, @@ -93,22 +91,3 @@ def _record_row( record.ownership_root.path, ), ) - - -def model_schema_enum_declarations( - *, schema: ModelSchemaDeclaration, resolver: DeclarationScopeResolver | None -) -> dict[str, EnumDeclaration]: - """Return enum column types resolved from the schema's own location.""" - - if resolver is None: - return {} - enums: dict[str, EnumDeclaration] = {} - for column in schema.columns: - if column.type is None: - continue - value: object = resolver.projection.declarations.get( - DeclarationIdentity(DeclarationKind.ENUM, column.type) - ) - if isinstance(value, EnumDeclaration): - enums[column.type] = value - return enums diff --git a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py index 5160e57304..4f6baacade 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/context_templates.py @@ -21,13 +21,13 @@ def resolve_early_model_templates( return cast( dict[str, object], - expand_config_templates( + expand_template_data( value=values, variables=effective_vars, - context_values=build_run_context_values( - effective_target_name=effective_target_name, - run_id=run_id, - ), + context_values={ + CompileContextKey.RUN_ID: run_id, + CompileContextKey.RUN_TARGET: effective_target_name, + }, context_label="model config", allow_context=True, preserve_context_tokens=False, @@ -36,17 +36,6 @@ def resolve_early_model_templates( ) -def build_run_context_values( - *, effective_target_name: str | None, run_id: str -) -> dict[str, str | None]: - """Build the compile-time CTX values known before resource-specific resolution.""" - - return { - CompileContextKey.RUN_ID: run_id, - CompileContextKey.RUN_TARGET: effective_target_name, - } - - def resolve_run_id(*, selected_run_id: str | None) -> str: """Resolve a stable compile invocation id.""" @@ -55,26 +44,3 @@ def resolve_run_id(*, selected_run_id: str | None) -> str: timestamp_prefix: str = datetime.now(tz=UTC).strftime("%Y%m%dT%H%M%SZ") unique_suffix: str = uuid4().hex[:12] return f"{timestamp_prefix}_{unique_suffix}" - - -def expand_config_templates( - *, - value: object, - variables: dict[str, object], - context_values: dict[str, str | None], - context_label: str, - allow_context: bool, - preserve_context_tokens: bool, - preserve_unknown_context: bool, -) -> object: - """Expand `${...}` templates in a config value natively, recording what it reads.""" - - return expand_template_data( - value=value, - variables=variables, - context_values=context_values, - context_label=context_label, - allow_context=allow_context, - preserve_context_tokens=preserve_context_tokens, - preserve_unknown_context=preserve_unknown_context, - ) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py index 4b1ebb1190..236ab18434 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py @@ -9,8 +9,8 @@ import pytest from _pytest.capture import CaptureResult +import sqlbuild._native as _native from sqlbuild.cli.commands.main.entrypoint.entry import main -from sqlbuild.compiler.compile._helpers.diagnostics.help import semantic_help_catalogue from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( DiagnosticUxCase, ) @@ -200,7 +200,10 @@ def test_given_semantic_failure_when_compiling_cold_and_warm_then_reports_root_e def test_given_semantic_codes_when_explaining_then_each_code_owns_distinct_help( test_case: DiagnosticUxCase, ) -> None: - help_by_code: dict[str, str] = {**semantic_help_catalogue(), "B002": "did you mean 'quantity'?"} + help_by_code: dict[str, str] = { + **dict(_native.semantic_help_catalogue()), + "B002": "did you mean 'quantity'?", + } assert len(help_by_code) - len(set(help_by_code.values())) == test_case.expected_errors assert all( "review the SQL expression, input types, and authoritative schema" not in value From def1fe7e1e2c8896f9945e638273ecb67204e232 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:08:43 +0100 Subject: [PATCH 05/14] refactor: share the hard-coded relation remedy and old-name message natively The runtime relation guard and the planner's old-name checks call the same native remedy and P008 old-name message the compile-time explicit-reference check uses; the Python duplicates are deleted. --- .../_helpers/python_sql.rs | 52 ++++--------------- .../src/explicit_references/main/mod.rs | 2 + .../main/old_name_message.rs | 15 ++++++ .../main/relation_remedy.rs | 34 ++++++++++++ .../_helpers/analysis/explicit_references.rs | 25 +++++++++ python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 6 +++ .../compiler/references/_helpers/remedies.py | 38 -------------- .../main/_old_name_reference_message.py | 8 ++- .../main/hard_coded_relation_remedy.py | 12 ++--- 10 files changed, 100 insertions(+), 94 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/explicit_references/main/old_name_message.rs create mode 100644 crates/sqlbuild-analysis/src/explicit_references/main/relation_remedy.rs delete mode 100644 src/sqlbuild/compiler/references/_helpers/remedies.py diff --git a/crates/sqlbuild-analysis/src/explicit_references/_helpers/python_sql.rs b/crates/sqlbuild-analysis/src/explicit_references/_helpers/python_sql.rs index 382c3a3af7..97c479d155 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/_helpers/python_sql.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/_helpers/python_sql.rs @@ -5,10 +5,10 @@ use std::collections::HashSet; use crate::explicit_references::_helpers::relations::{ ProjectRelation, matched, project_relations, }; -use crate::explicit_references::constants::{ - HARD_CODED_RELATION_CODE, HOOK_OWNER, LOADER_OWNER, SOURCE_TYPE, -}; +use crate::explicit_references::constants::HARD_CODED_RELATION_CODE; use crate::explicit_references::errors::ReferenceFailure; +use crate::explicit_references::main::old_name_message::old_name_message; +use crate::explicit_references::main::relation_remedy::relation_remedy; use crate::explicit_references::models::{ LiteralSql, ReferenceDiagnostic, ReferenceReport, ReferenceRequest, Relation, SqlOwner, UnmatchedRelation, @@ -154,12 +154,7 @@ impl OwnerScan<'_> { fn message(&self, literal: &LiteralSql, written: &str, found: &ProjectRelation) -> String { match &found.compatibility_for { - Some(model) => format!( - "{} names '{written}' in SQL passed to ctx.{}(); it is the old name of \ - model:{model}, kept only as a compatibility view for consumers outside the \ - project", - self.owner.label, literal.method - ), + Some(model) => old_name_message(&self.owner.label, written, &literal.method, model), None => format!( "{} names {}:{} as '{written}' in SQL passed to ctx.{}()", self.owner.label, found.target.0, found.target.1, literal.method @@ -178,43 +173,14 @@ impl OwnerScan<'_> { }; format!( "{}{qualify}\n = help: {}", - self.remedy(&found.target), + relation_remedy( + &self.owner.kind, + &found.target, + &self.owner.upstream_loaders + ), self.request.explicit_help ) } - - /// `hard_coded_relation_remedy_impl`. - fn remedy(&self, (kind, name): &ResourceKey) -> String { - let typed: String = format!("{kind}(\"{name}\")"); - if self.owner.kind == LOADER_OWNER { - if kind != SOURCE_TYPE { - return format!( - "loaders run before models and seeds and cannot read them; read \ - {kind}:{name} from a task or asset instead" - ); - } - return match self - .owner - .upstream_loaders - .iter() - .find(|(source, _)| source == name) - { - Some((_, loader)) => { - format!("use ctx.loader({loader}) instead of the relation name") - } - None => format!("use ctx.source(\"{name}\") instead of the relation name"), - }; - } - let declaration: String = if self.owner.kind == HOOK_OWNER { - format!("@hook(reads={typed})") - } else { - format!("depends_on={typed}") - }; - format!( - "declare it with {declaration} and use ctx.relation({typed}) instead of the relation \ - name" - ) - } } fn extracted( diff --git a/crates/sqlbuild-analysis/src/explicit_references/main/mod.rs b/crates/sqlbuild-analysis/src/explicit_references/main/mod.rs index 9e7e305311..3b3871c48a 100644 --- a/crates/sqlbuild-analysis/src/explicit_references/main/mod.rs +++ b/crates/sqlbuild-analysis/src/explicit_references/main/mod.rs @@ -1,2 +1,4 @@ pub mod check_explicit_references; pub mod execution_gate_edges; +pub mod old_name_message; +pub mod relation_remedy; diff --git a/crates/sqlbuild-analysis/src/explicit_references/main/old_name_message.rs b/crates/sqlbuild-analysis/src/explicit_references/main/old_name_message.rs new file mode 100644 index 0000000000..fcdb49f902 --- /dev/null +++ b/crates/sqlbuild-analysis/src/explicit_references/main/old_name_message.rs @@ -0,0 +1,15 @@ +//! The P008 message for SQL naming a relation kept only as a compatibility view. + +/// `old_name_reference_message`: `owner_label` wrote `written` in SQL passed to `ctx.()`. +#[must_use] +pub fn old_name_message( + owner_label: &str, + written: &str, + method: &str, + model_name: &str, +) -> String { + format!( + "{owner_label} names '{written}' in SQL passed to ctx.{method}(); it is the old name of \ + model:{model_name}, kept only as a compatibility view for consumers outside the project" + ) +} diff --git a/crates/sqlbuild-analysis/src/explicit_references/main/relation_remedy.rs b/crates/sqlbuild-analysis/src/explicit_references/main/relation_remedy.rs new file mode 100644 index 0000000000..43db711343 --- /dev/null +++ b/crates/sqlbuild-analysis/src/explicit_references/main/relation_remedy.rs @@ -0,0 +1,34 @@ +//! How Python code should read a project resource instead of naming its relation in SQL. + +use crate::explicit_references::constants::{HOOK_OWNER, LOADER_OWNER, SOURCE_TYPE}; +use crate::explicit_references::types::ResourceKey; + +/// `hard_coded_relation_remedy`; `upstream_loaders` maps sources to the loaders writing them. +#[must_use] +pub fn relation_remedy( + owner_kind: &str, + (kind, name): &ResourceKey, + upstream_loaders: &[(String, String)], +) -> String { + let typed: String = format!("{kind}(\"{name}\")"); + if owner_kind == LOADER_OWNER { + if kind != SOURCE_TYPE { + return format!( + "loaders run before models and seeds and cannot read them; read \ + {kind}:{name} from a task or asset instead" + ); + } + return match upstream_loaders.iter().find(|(source, _)| source == name) { + Some((_, loader)) => format!("use ctx.loader({loader}) instead of the relation name"), + None => format!("use ctx.source(\"{name}\") instead of the relation name"), + }; + } + let declaration: String = if owner_kind == HOOK_OWNER { + format!("@hook(reads={typed})") + } else { + format!("depends_on={typed}") + }; + format!( + "declare it with {declaration} and use ctx.relation({typed}) instead of the relation name" + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index 4ed3a6b245..503d409342 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -8,6 +8,8 @@ use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::explicit_references::errors::ReferenceFailure; use sqlbuild_analysis::explicit_references::main::check_explicit_references::check_explicit_references; use sqlbuild_analysis::explicit_references::main::execution_gate_edges::execution_gate_edges; +use sqlbuild_analysis::explicit_references::main::old_name_message::old_name_message; +use sqlbuild_analysis::explicit_references::main::relation_remedy::relation_remedy; use sqlbuild_analysis::explicit_references::models::{ ExecutionGateEdges, ExecutionGateRequest, GateEdge, HookEdge, HookType, LiteralSql, ReferenceReport, ReferenceRequest, SqlOwner, @@ -200,6 +202,27 @@ fn python_literal_sql_calls( .collect() } +/// How an owner of `owner_kind` should read `target` instead of naming its relation in SQL. +#[pyfunction] +fn hard_coded_relation_remedy( + owner_kind: &str, + target: ResourceKey, + upstream_loaders: Vec<(String, String)>, +) -> String { + relation_remedy(owner_kind, &target, &upstream_loaders) +} + +/// The P008 message for SQL naming a relation kept only as a compatibility view. +#[pyfunction] +fn old_name_reference_message( + owner_label: &str, + written: &str, + method: &str, + model_name: &str, +) -> String { + old_name_message(owner_label, written, method, model_name) +} + /// `(audit name, target, gated, read)` for one attached-audit ordering edge. type GateRow = (String, ResourceKey, ResourceKey, ResourceKey); /// `(hook name, gated, read, HookReadType value, path)` for one hook-read ordering edge. @@ -246,5 +269,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(explicit_reference_report, module)?)?; module.add_function(wrap_pyfunction!(python_literal_sql_calls, module)?)?; module.add_function(wrap_pyfunction!(native_execution_gate_edges, module)?)?; + module.add_function(wrap_pyfunction!(hard_coded_relation_remedy, module)?)?; + module.add_function(wrap_pyfunction!(old_name_reference_message, module)?)?; Ok(()) } diff --git a/python-boundary.toml b/python-boundary.toml index 3b16e285b7..5da976d316 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 129363 +remaining_lines = 129321 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 193db0e92e..395fea96ec 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1923,6 +1923,12 @@ def explicit_reference_report( owners: list[tuple[Any, int, list[tuple[str, str, int]]]], /, ) -> PythonSqlReferenceReport: ... +def hard_coded_relation_remedy( + owner_kind: str, target: tuple[str, str], upstream_loaders: list[tuple[str, str]], / +) -> str: ... +def old_name_reference_message( + owner_label: str, written: str, method: str, model_name: str, / +) -> str: ... def native_execution_gate_edges( project: Any, / ) -> tuple[ diff --git a/src/sqlbuild/compiler/references/_helpers/remedies.py b/src/sqlbuild/compiler/references/_helpers/remedies.py deleted file mode 100644 index 3e5944d18e..0000000000 --- a/src/sqlbuild/compiler/references/_helpers/remedies.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Remedies for project relation names hard-coded in Python SQL.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.python_nodes.types import SqlResourceRefKind - - -def hard_coded_relation_remedy_impl( - *, - owner_kind: HardCodedRelationOwnerKind, - ref: SqlResourceRef, - upstream_loader_by_source: Mapping[str, str], -) -> str: - """Return how the owner should read ``ref`` instead of naming it in SQL.""" - - typed: str = f'{ref.kind.value}("{ref.name}")' - if owner_kind is HardCodedRelationOwnerKind.LOADER: - if ref.kind is not SqlResourceRefKind.SOURCE: - return ( - "loaders run before models and seeds and cannot read them; read " - f"{ref.kind.value}:{ref.name} from a task or asset instead" - ) - loader_name: str | None = upstream_loader_by_source.get(ref.name) - if loader_name is not None: - return f"use ctx.loader({loader_name}) instead of the relation name" - return f'use ctx.source("{ref.name}") instead of the relation name' - declaration: str = ( - f"@hook(reads={typed})" - if owner_kind is HardCodedRelationOwnerKind.HOOK - else f"depends_on={typed}" - ) - return ( - f"declare it with {declaration} and use ctx.relation({typed}) instead of the relation name" - ) diff --git a/src/sqlbuild/compiler/references/main/_old_name_reference_message.py b/src/sqlbuild/compiler/references/main/_old_name_reference_message.py index ea7d3df620..5fde835797 100644 --- a/src/sqlbuild/compiler/references/main/_old_name_reference_message.py +++ b/src/sqlbuild/compiler/references/main/_old_name_reference_message.py @@ -2,14 +2,12 @@ from __future__ import annotations +import sqlbuild._native as _native + def old_name_reference_message( *, owner_label: str, written: str, method: str, model_name: str ) -> str: """Return the P008 message for SQL naming a relation kept only as a compatibility view.""" - return ( - f"{owner_label} names '{written}' in SQL passed to ctx.{method}(); it is the old name " - f"of model:{model_name}, kept only as a compatibility view for consumers outside the " - "project" - ) + return _native.old_name_reference_message(owner_label, written, method, model_name) diff --git a/src/sqlbuild/compiler/references/main/hard_coded_relation_remedy.py b/src/sqlbuild/compiler/references/main/hard_coded_relation_remedy.py index 94ee894819..9a72167c23 100644 --- a/src/sqlbuild/compiler/references/main/hard_coded_relation_remedy.py +++ b/src/sqlbuild/compiler/references/main/hard_coded_relation_remedy.py @@ -4,7 +4,7 @@ from collections.abc import Mapping -from sqlbuild.compiler.references._helpers.remedies import hard_coded_relation_remedy_impl +import sqlbuild._native as _native from sqlbuild.compiler.references.types import HardCodedRelationOwnerKind from sqlbuild.python_nodes.models import SqlResourceRef @@ -17,10 +17,8 @@ def hard_coded_relation_remedy( ) -> str: """Return how Python code should read ``ref`` instead of naming it in SQL.""" - return hard_coded_relation_remedy_impl( - owner_kind=owner_kind, - ref=ref, - upstream_loader_by_source={} - if upstream_loader_by_source is None - else upstream_loader_by_source, + return _native.hard_coded_relation_remedy( + owner_kind.value, + (ref.kind.value, ref.name), + list((upstream_loader_by_source or {}).items()), ) From 82e24e0187db83eb94231ca4e79742fa7db8a222 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:18:59 +0100 Subject: [PATCH 06/14] refactor: read model schema scope usages natively The compile usages read schema declarations and the resolver's schema and enum records from Python objects in Rust, report S006 through the collector and build the UsageRecords; the named_declarations package is deleted and its caller uses the native function directly. --- .../_helpers/scopes/compile_usages.rs | 143 +++++++++++++++++- python-boundary.toml | 2 +- src/sqlbuild/_native.pyi | 10 +- .../named_declarations/model_schemas.py | 93 ------------ .../main/_scope_index_with_compile_usages.py | 12 +- 5 files changed, 151 insertions(+), 109 deletions(-) delete mode 100644 src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs index 7773c54f72..33705c8763 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -3,7 +3,10 @@ use std::collections::HashMap; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_scopes::compile_usages::main::distinct_positions::distinct_positions; use sqlbuild_scopes::compile_usages::main::merged_dependencies::merged_dependencies; @@ -17,6 +20,13 @@ use sqlbuild_scopes::scope_index::models::{ use crate::bindings::_helpers::compiled_project::model_loop_named::scope_path_message; +const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const INACCESSIBLE_DECLARATION_CODE: &str = "S006"; +const SCHEMA_USAGE_KINDS: [&str; 2] = ["schema", "enum"]; + type IdentityRow = (String, String); /// A declaration's kind, name and private owner; a resource's kind and name with no owner. type QualifiedRow = (String, String, Option); @@ -75,8 +85,135 @@ fn diagnostic_row(diagnostic: InaccessibleDiagnostic) -> DiagnosticRow { ) } -/// Each schema's parent and enum usages over `records` by `(kind, name)` handle, with S006s. +/// `model_schema_scope_usages`: each schema's parent and enum `UsageRecord`s; S006s are reported. #[pyfunction] +fn model_schema_scope_usages<'py>( + schema_files: &Bound<'py, PyAny>, + resolver: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = schema_files.py(); + if resolver.is_none() { + return Ok(PyTuple::empty(py)); + } + let mut records: Vec> = Vec::new(); + let mut record_rows: Vec<(IdentityRow, RecordRow)> = Vec::new(); + let declarations = resolver.getattr("lookup")?.getattr("declarations")?; + for item in declarations.call_method0("items")?.try_iter()? { + let (identity, found): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + if !found.is_truthy()? + || !identity.getattr("owner")?.is_none() + || !SCHEMA_USAGE_KINDS.contains( + &identity + .getattr("kind")? + .getattr("value")? + .extract::()? + .as_str(), + ) + { + continue; + } + let record: Bound<'py, PyAny> = found.get_item(0)?; + let recorded: Bound<'py, PyAny> = record.getattr("identity")?; + record_rows.push(( + ( + recorded.getattr("kind")?.getattr("value")?.extract()?, + recorded.getattr("name")?.extract()?, + ), + record_row(&record)?, + )); + records.push(record); + } + let mut schemas: Vec = Vec::new(); + for file in schema_files.try_iter()? { + for declaration in file?.getattr("declarations")?.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + let mut columns: Vec<(String, Option)> = Vec::new(); + for column in declaration.getattr("columns")?.try_iter()? { + let column: Bound<'py, PyAny> = column?; + columns.push(( + column.getattr("name")?.extract()?, + column.getattr("type")?.extract()?, + )); + } + schemas.push(( + declaration.getattr("name")?.extract()?, + declaration.getattr("extends")?.extract()?, + columns, + )); + } + } + let (usages, diagnostics, error) = scope_schema_usages(schemas, record_rows)?; + if let Some(message) = error { + return Err(PyErr::from_value( + py.import(SCOPE_ERRORS_MODULE)? + .getattr("InvalidScopePathError")? + .call1((message,))?, + )); + } + report_inaccessible(py, diagnostics)?; + let usage_record = py.import(SCOPE_MODELS_MODULE)?.getattr("UsageRecord")?; + let mut built: Vec> = Vec::with_capacity(usages.len()); + for (consumer, used) in usages { + let fields: Bound<'py, PyDict> = PyDict::new(py); + fields.set_item("consumer", records[consumer].getattr("identity")?)?; + fields.set_item("declaration", records[used].getattr("identity")?)?; + built.push(usage_record.call((), Some(&fields))?); + } + PyTuple::new(py, built) +} + +/// A declaration record's path, scope, private owner, owning path and ownership root path. +fn record_row(record: &Bound<'_, PyAny>) -> PyResult { + let identity: Bound<'_, PyAny> = record.getattr("identity")?; + let owner: Bound<'_, PyAny> = identity.getattr("owner")?; + Ok(( + record.getattr("path")?.extract()?, + record.getattr("scope")?.getattr("value")?.extract()?, + if owner.is_none() { + None + } else { + Some(( + owner.getattr("kind")?.getattr("value")?.extract()?, + owner.getattr("name")?.extract()?, + )) + }, + record.getattr("owning_path")?.extract()?, + record + .getattr("ownership_root")? + .getattr("path")? + .extract()?, + )) +} + +/// Report each S006 through the active compile diagnostic collector. +fn report_inaccessible(py: Python<'_>, diagnostics: Vec) -> PyResult<()> { + if diagnostics.is_empty() { + return Ok(()); + } + let report = py + .import(COLLECTOR_MODULE)? + .getattr("report_compile_diagnostic")?; + let compiler_diagnostic = py + .import(COMPILE_MODELS_MODULE)? + .getattr("CompilerDiagnostic")?; + let path_class = py.import("pathlib")?.getattr("Path")?; + for (key, message, path, help) in diagnostics { + let fields: Bound<'_, PyDict> = PyDict::new(py); + fields.set_item("phase", "compile")?; + fields.set_item("severity", "error")?; + fields.set_item("code", INACCESSIBLE_DECLARATION_CODE)?; + fields.set_item("message", message)?; + fields.set_item("path", path_class.call1((path,))?)?; + fields.set_item("help", help)?; + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item("key", PyTuple::new(py, key)?)?; + arguments.set_item("diagnostic", compiler_diagnostic.call((), Some(&fields))?)?; + report.call((), Some(&arguments))?; + } + Ok(()) +} + +/// Each schema's parent and enum usages over `records` by `(kind, name)` handle, with S006s. fn scope_schema_usages( schemas: Vec, records: Vec<(IdentityRow, RecordRow)>, @@ -157,6 +294,6 @@ pub(crate) fn merged_usage_rows( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(scope_schema_usages, module)?)?; + module.add_function(wrap_pyfunction!(model_schema_scope_usages, module)?)?; Ok(()) } diff --git a/python-boundary.toml b/python-boundary.toml index 5da976d316..65a4bd970f 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 129321 +remaining_lines = 129224 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 395fea96ec..24e6fa5e0d 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -38,6 +38,7 @@ from sqlbuild.compiler.compile.models import ( CompileSqlScenarioInput, CompileSqlTestInput, DeclarationScopeBuild, + DeclarationScopeResolver, DynamicColumnContractProof, FunctionArgument, FunctionReturnColumn, @@ -1385,10 +1386,11 @@ def read_scope_index_cache(path: Path, fingerprint: str, /) -> ScopeIndex | None def write_scope_index_cache( location: tuple[Path, str], fingerprint: str, index: ScopeIndex, / ) -> bool: ... -def scope_schema_usages( - schemas: list[tuple[str, str | None, list[tuple[str, str | None]]]], - records: list[tuple[tuple[str, str], tuple[str, str, tuple[str, str] | None, str | None, str]]], -) -> tuple[list[tuple[int, int]], list[tuple[list[str], str, str, str]], str | None]: ... +def model_schema_scope_usages( + schema_files: Sequence[DiscoveredModelSchemaFile], + resolver: DeclarationScopeResolver | None, + /, +) -> tuple[UsageRecord, ...]: ... class NativeConfigError: class_name: str diff --git a/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py b/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py deleted file mode 100644 index 0d73b79f17..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/named_declarations/model_schemas.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Scope facts for reusable model schemas: parent schemas and enum column types.""" - -from __future__ import annotations - -from itertools import chain -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic -from sqlbuild.compiler.compile.models import CompilerDiagnostic, DeclarationScopeResolver -from sqlbuild.compiler.compile.types import DiagnosticPhase, DiagnosticSeverity -from sqlbuild.compiler.discovery.models import ( - DiscoveredModelSchemaFile, - ModelSchemaDeclaration, -) -from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError -from sqlbuild.compiler.scopes.models import ( - DeclarationRecord, - ResourceIdentity, - UsageRecord, -) -from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeDiagnosticCode - -_SCHEMA_USAGE_KINDS: frozenset[DeclarationKind] = frozenset( - {DeclarationKind.SCHEMA, DeclarationKind.ENUM} -) - - -def model_schema_scope_usages( - *, - schema_files: tuple[DiscoveredModelSchemaFile, ...], - resolver: DeclarationScopeResolver | None, -) -> tuple[UsageRecord, ...]: - """Require parent schemas and enum column types to be visible from each schema's folder.""" - - if resolver is None: - return () - records: list[DeclarationRecord] = [ - found[0] - for identity, found in resolver.lookup.declarations.items() - if found and identity.owner is None and identity.kind in _SCHEMA_USAGE_KINDS - ] - usages: list[tuple[int, int]] - diagnostics: list[tuple[list[str], str, str, str]] - error: str | None - usages, diagnostics, error = _native.scope_schema_usages( - list(map(_schema_row, chain.from_iterable(item.declarations for item in schema_files))), - list(map(_record_row, records)), - ) - if error is not None: - raise InvalidScopePathError(error) - for key, message, path, help_text in diagnostics: - report_compile_diagnostic( - key=tuple(key), - diagnostic=CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=ScopeDiagnosticCode.INACCESSIBLE_DECLARATION.value, - message=message, - path=Path(path), - help=help_text, - ), - ) - return tuple( - UsageRecord(consumer=records[consumer].identity, declaration=records[used].identity) - for consumer, used in usages - ) - - -def _schema_row( - declaration: ModelSchemaDeclaration, -) -> tuple[str, str | None, list[tuple[str, str | None]]]: - return ( - declaration.name, - declaration.extends, - [(column.name, column.type) for column in declaration.columns], - ) - - -def _record_row( - record: DeclarationRecord, -) -> tuple[tuple[str, str], tuple[str, str, tuple[str, str] | None, str | None, str]]: - owner: ResourceIdentity | None = record.identity.owner - return ( - (record.identity.kind.value, record.identity.name), - ( - record.path, - record.scope.value, - None if owner is None else (owner.kind.value, owner.name), - record.owning_path, - record.ownership_root.path, - ), - ) diff --git a/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py b/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py index 60e92e5e21..8ccdcc776a 100644 --- a/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py +++ b/src/sqlbuild/compiler/compile/main/_scope_index_with_compile_usages.py @@ -2,9 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.compile._helpers.named_declarations.model_schemas import ( - model_schema_scope_usages, -) +import sqlbuild._native as _native from sqlbuild.compiler.compile.types import CompileInputsView from sqlbuild.compiler.scopes.main._get_placement_validated_scope_index import ( get_placement_validated_scope_index, @@ -16,11 +14,9 @@ def scope_index_with_compile_usages(*, inputs: CompileInputsView) -> ScopeIndex: """Return compile inputs' canonical index with all usage and placement facts.""" collected_usages: list[UsageRecord] = list( - model_schema_scope_usages( - schema_files=inputs.discovered_inputs.model_schema_files, - resolver=( - inputs.declaration_scope.resolver if inputs.declaration_scope is not None else None - ), + _native.model_schema_scope_usages( + inputs.discovered_inputs.model_schema_files, + inputs.declaration_scope.resolver if inputs.declaration_scope is not None else None, ) ) for model_input in inputs.model_inputs: From bb142d2824b8a5bad6f119b7e24a421c69016db2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:27:22 +0100 Subject: [PATCH 07/14] refactor: attach compile inputs natively Compile-input attachment runs in native bindings: effective config, target and cache context, the declaration scope, the model input loop with its class dictionary, resource and check attachment, and the compiled record and update helpers. The Python attachment and compiled_project modules are removed; the compile driver and callers use the native functions. The inspection of user Python resources for missing-description diagnostics stays in Python. --- .../compile_attachment/declaration_scope.rs | 202 ++++++ .../compile_attachment/effective_config.rs | 105 +++ .../_helpers/compile_attachment/mod.rs | 6 + .../compile_attachment/model_config.rs | 96 +++ .../compile_attachment/model_inputs.rs | 318 +++++++++ .../compile_attachment/registration.rs | 45 ++ .../_helpers/compile_attachment/target.rs | 247 +++++++ .../_helpers/compile_driver/assembly.rs | 5 +- .../_helpers/compile_driver/inputs.rs | 13 +- .../_helpers/compile_driver/project.rs | 2 +- .../_helpers/compiled_project/attachment.rs | 637 ++++++++++++++++++ .../_helpers/compiled_project/loop_classes.rs | 488 ++++++++++++++ .../bindings/_helpers/compiled_project/mod.rs | 3 + .../_helpers/compiled_project/project.rs | 23 +- .../_helpers/compiled_project/project_view.rs | 2 +- .../_helpers/compiled_project/records.rs | 619 +++++++++++++++++ .../_helpers/compiled_project/registration.rs | 4 +- .../compiled_project/resource_diagnostics.rs | 2 +- .../src/bindings/_helpers/mod.rs | 1 + .../src/bindings/main/register.rs | 9 +- fensu.toml | 5 - scripts/compiler_differential/constants.py | 1 - src/sqlbuild/_native.pyi | 123 ++++ .../compile/_helpers/attachment/core.py | 311 --------- .../_helpers/attachment/declaration_scope.py | 131 ---- .../_helpers/attachment/model_config.py | 153 ----- .../compile/_helpers/attachment/target.py | 141 ---- .../_helpers/diagnostics/python_resources.py} | 19 +- .../_helpers/macro_bridge/call_store.py | 4 +- .../compiler/compile/main/effective_config.py | 2 +- .../compile/main/effective_runtime.py | 2 +- .../compile/main/effective_settings.py | 4 +- .../main/effective_target_namespace.py | 2 +- .../compile/main/sql_expansion_context.py | 6 +- src/sqlbuild/compiler/compile/models.py | 17 + src/sqlbuild/compiler/compile/types.py | 3 +- .../compiler/compiled_project/__init__.py | 1 - .../compiled_project/_helpers/__init__.py | 1 - .../_helpers/assurance_attachment.py | 212 ------ .../compiled_project/_helpers/loop_inputs.py | 100 --- .../_helpers/model_updates.py | 220 ------ .../_helpers/resource_attachment.py | 119 ---- .../compiled_project/_helpers/rows.py | 196 ------ .../compiler/compiled_project/constants.py | 9 - .../compiled_project/main/__init__.py | 1 - .../compiled_project/main/_attach_checks.py | 61 -- .../main/_attach_resources.py | 40 -- .../main/_compile_inputs_view.py | 15 - .../main/_model_loop_classes.py | 135 ---- .../main/_native_config_classes.py | 27 - .../main/_record_model_analyses.py | 13 - .../main/_with_binding_diagnostics.py | 13 - .../main/_with_target_defaults.py | 20 - .../main/_with_type_recovery.py | 16 - .../main/compiled_audit_record.py | 11 - .../main/compiled_function_record.py | 12 - .../main/compiled_model_record.py | 11 - .../main/compiled_project_facts.py | 13 - .../main/compiled_project_record.py | 15 - .../main/compiled_seed_record.py | 11 - .../main/compiled_source_record.py | 11 - .../compiled_project/main/declaration_rows.py | 14 - .../compiled_project/main/inputs_with.py | 12 - .../main/plain_typed_value.py | 10 - .../compiled_project/main/project_with.py | 12 - .../compiler/compiled_project/types.py | 9 - .../contracts/_helpers/native_contracts.py | 4 +- .../contracts/_helpers/native_promotion.py | 4 +- .../pipeline/_helpers/target_defaults.py | 11 +- .../planner/_helpers/scenario/relations.py | 4 +- .../semantic_checks/_helpers/completion.py | 8 +- .../semantic_checks/_helpers/stage.py | 4 +- .../semantic_checks/_helpers/type_recovery.py | 6 +- .../_helpers/engine/custom_rules.py | 4 +- .../_helpers/host/custom_host_pool.py | 4 +- .../rule_engine/_helpers/run/native_rows.py | 8 +- .../rule_engine/_helpers/run/rules.py | 4 +- .../rule_engine/classes/audit_facts.py | 4 +- .../rule_engine/classes/project_facts.py | 10 +- .../sqlbuild/cli/commands/main/test_rules.py | 13 +- .../compile/native_model_config/helpers.py | 27 +- .../compiler/compiled_project/helpers.py | 54 +- .../sqlbuild/compiler/contracts/helpers.py | 9 +- .../src/sqlbuild/compiler/lineage/helpers.py | 10 +- .../lineage/test_native_fast_lineage.py | 8 +- .../pipeline/test_expression_catalog_cache.py | 4 +- .../compiler/planner/_helpers/helpers.py | 6 +- .../_helpers/test_warehouse_snapshot.py | 6 +- .../sqlbuild/compiler/planner/main/helpers.py | 9 +- .../planner/main/test_build_execution_plan.py | 6 +- .../compiler/project_assembly/helpers.py | 8 +- .../src/sqlbuild/compiler/scopes/helpers.py | 10 +- .../compiler/semantic_checks/helpers.py | 7 +- .../test_native_semantic_checks.py | 4 +- .../src/sqlbuild/lint/main/helpers.py | 9 +- .../host_payload/test_host_payload_facts.py | 6 +- .../cli/commands/main/compile/helpers.py | 4 +- .../main/compile/test_compile_output.py | 8 +- .../compile/test_compile_target_writer.py | 6 +- .../compile/_helpers/config/helpers.py | 56 +- .../_helpers/test_rebind_declaration_scope.py | 4 +- .../sqlbuild/compiler/dag/main/test_build.py | 2 +- .../sqlbuild/compiler/lineage/test_columns.py | 6 +- .../sqlbuild/compiler/manifest/test_main.py | 4 +- .../compiler/planner/_helpers/helpers.py | 6 +- .../sql_test_assembly/test_plan_test.py | 4 +- .../_helpers/test_fixture_planning_work.py | 6 +- .../_helpers/test_scenario_relations.py | 4 +- .../_helpers/test_selection_staleness.py | 4 +- .../_helpers/test_source_load_nodes.py | 4 +- .../compiler/planner/main/identity/helpers.py | 4 +- .../compiler/planner/main/test_execution.py | 4 +- .../main/test_static_command_planning.py | 6 +- .../rule_engine/_helpers/engine/helpers.py | 4 +- .../_helpers/engine/test_custom_rules.py | 6 +- .../_helpers/engine/test_native_facts.py | 4 +- .../_helpers/engine/test_typed_constants.py | 4 +- .../rule_engine/_helpers/host/helpers.py | 4 +- .../classes/audit_facts/test_audit_facts.py | 4 +- .../model_test_facts/test_model_test_facts.py | 4 +- .../classes/shared_fact_encoder/helpers.py | 4 +- .../evaluate/test_cache_and_suppressions.py | 6 +- 122 files changed, 3246 insertions(+), 2288 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs delete mode 100644 src/sqlbuild/compiler/compile/_helpers/attachment/core.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/attachment/target.py rename src/sqlbuild/compiler/{compiled_project/_helpers/resource_rows.py => compile/_helpers/diagnostics/python_resources.py} (85%) delete mode 100644 src/sqlbuild/compiler/compiled_project/__init__.py delete mode 100644 src/sqlbuild/compiler/compiled_project/_helpers/__init__.py delete mode 100644 src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py delete mode 100644 src/sqlbuild/compiler/compiled_project/_helpers/loop_inputs.py delete mode 100644 src/sqlbuild/compiler/compiled_project/_helpers/model_updates.py delete mode 100644 src/sqlbuild/compiler/compiled_project/_helpers/resource_attachment.py delete mode 100644 src/sqlbuild/compiler/compiled_project/_helpers/rows.py delete mode 100644 src/sqlbuild/compiler/compiled_project/constants.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/__init__.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/_attach_checks.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/_attach_resources.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/_compile_inputs_view.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/_model_loop_classes.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/_native_config_classes.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/_record_model_analyses.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/_with_binding_diagnostics.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/_with_target_defaults.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/_with_type_recovery.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/compiled_audit_record.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/compiled_function_record.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/compiled_model_record.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/compiled_project_facts.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/compiled_project_record.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/compiled_seed_record.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/compiled_source_record.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/declaration_rows.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/inputs_with.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/plain_typed_value.py delete mode 100644 src/sqlbuild/compiler/compiled_project/main/project_with.py delete mode 100644 src/sqlbuild/compiler/compiled_project/types.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs new file mode 100644 index 0000000000..cb2e2c978e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/declaration_scope.rs @@ -0,0 +1,202 @@ +//! The process-local declaration scope of one compile invocation. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::PyTuple; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const SCAN_CACHE_MODULE: &str = "sqlbuild.compiler.compile.classes.sql_test_scan_cache"; +const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; +const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const OPEN_INDEX_MODULE: &str = "sqlbuild.compiler.scopes.main._open_native_scope_index"; +const VALIDATE_INDEX_MODULE: &str = "sqlbuild.compiler.scopes.main._validate_scope_index"; + +fn compile_error(py: Python<'_>, message: Bound<'_, PyAny>) -> PyResult { + Ok(PyErr::from_value( + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?.call1((message,))?, + )) +} + +/// The declaration scope of `index`, resolved for `loaded_macros`. +fn scope_build<'py>( + discovered: &Bound<'py, PyAny>, + loaded_macros: &Bound<'py, PyAny>, + (index, lookup, scan_cache): (Bound<'py, PyAny>, Bound<'py, PyAny>, Bound<'py, PyAny>), +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let resolver: Bound<'py, PyAny> = call_with( + &imported(py, DECLARATIONS_MODULE, "build_declaration_scope_resolver")?, + &[ + ("discovered_inputs", discovered.clone()), + ("scope_index", index.clone()), + ("loaded_macros", loaded_macros.clone()), + ("lookup", lookup), + ], + )?; + call_with( + &imported(py, COMPILE_MODELS_MODULE, "DeclarationScopeBuild")?, + &[ + ("loaded_macros", loaded_macros.clone()), + ("index", index), + ("resolver", resolver), + ("sql_test_scans", scan_cache), + ], + ) +} + +/// Build one canonical index and validate it before SQL expansion. +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, loaded_macros, sql_lexical_syntax, compile_cache_dir=None))] +pub(crate) fn build_declaration_scope<'py>( + discovered_inputs: &Bound<'py, PyAny>, + loaded_macros: &Bound<'py, PyAny>, + sql_lexical_syntax: &Bound<'py, PyAny>, + compile_cache_dir: Option>, +) -> PyResult> { + let py: Python<'py> = discovered_inputs.py(); + let scan_cache: Bound<'py, PyAny> = call_with( + &imported(py, SCAN_CACHE_MODULE, "SqlTestScanCache")?, + &[( + "cache_dir", + compile_cache_dir.unwrap_or_else(|| py.None().into_bound(py)), + )], + )?; + let test_files: Bound<'py, PyAny> = discovered_inputs.getattr("test_files")?; + let scenario_files: Bound<'py, PyAny> = discovered_inputs.getattr("scenario_files")?; + imported(py, NATIVE_MODULE, "validate_authored_sql_test_ctes")?.call1(( + &test_files, + &scenario_files, + sql_lexical_syntax.getattr("native_mapping")?, + ))?; + let native: Bound<'py, PyAny> = call_with( + &imported(py, OPEN_INDEX_MODULE, "open_native_scope_index")?, + &[ + ("discovered_inputs", discovered_inputs.clone()), + ("loaded_macros", loaded_macros.clone()), + ], + )?; + let validated: PyResult> = call_with( + &imported(py, VALIDATE_INDEX_MODULE, "validate_scope_index")?, + &[("index", native.getattr("index")?)], + ); + if let Err(error) = validated { + if !error.is_instance( + py, + &imported(py, SCOPE_ERRORS_MODULE, "ScopeValidationError")?, + ) { + return Err(error); + } + let raised: PyErr = compile_error(py, error.value(py).str()?.into_any())?; + raised.set_cause(py, Some(error)); + return Err(raised); + } + if native + .getattr("has_scoped_relationship_declarations")? + .is_truthy()? + && (test_files.is_truthy()? || scenario_files.is_truthy()?) + { + let faults: Bound<'py, PyAny> = call_with( + &native.getattr("grant_relationships")?, + &[ + ("discovered_inputs", discovered_inputs.clone()), + ("syntax", sql_lexical_syntax.clone()), + ( + "scan_store", + PyTuple::new( + py, + [ + scan_cache.call_method0("opened")?, + imported( + py, + COMPILE_CONSTANTS_MODULE, + "SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM", + )?, + ], + )? + .into_any(), + ), + ], + )?; + if faults.is_truthy()? { + return Err(compile_error(py, faults.get_item(0)?.get_item(1)?)?); + } + } + let index: Bound<'py, PyAny> = native.call_method0("index_with_relationships")?; + let lookup: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "build_scope_lookup")?.call1((&index,))?; + scope_build( + discovered_inputs, + loaded_macros, + (index, lookup, scan_cache), + ) +} + +/// Each loaded macro's metadata that the built index recorded. +fn indexed_macro_metadata<'py>(loaded_macros: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = loaded_macros.py(); + let code_type: Bound<'py, PyAny> = imported(py, "types", "CodeType")?; + let signature: Bound<'py, PyAny> = imported(py, "inspect", "signature")?; + let mut rows: Vec> = Vec::new(); + for pair in loaded_macros.call_method0("items")?.try_iter()? { + let (key, loaded): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; + let function: Bound<'py, PyAny> = loaded.getattr("function")?; + let code: Bound<'py, PyAny> = + imported(py, "builtins", "getattr")?.call1((&function, "__code__", py.None()))?; + let first_line: Bound<'py, PyAny> = if code.is_instance(&code_type)? { + code.getattr("co_firstlineno")? + } else { + py.None().into_bound(py) + }; + let parameters: Bound<'py, PyTuple> = PyTuple::new( + py, + signature + .call1((&function,))? + .getattr("parameters")? + .try_iter()? + .collect::>>()?, + )?; + rows.push(PyTuple::new( + py, + [ + key, + loaded.getattr("name")?, + loaded.getattr("relative_path")?, + loaded.getattr("raw_source")?, + loaded.getattr("dependencies")?, + first_line, + parameters.into_any(), + ], + )?); + } + PyTuple::new(py, rows) +} + +/// Pair a built index with private macro instances, or return None if their metadata differs. +#[pyfunction] +#[pyo3(signature = (*, scope, discovered_inputs, loaded_macros))] +pub(crate) fn rebind_declaration_scope<'py>( + scope: &Bound<'py, PyAny>, + discovered_inputs: &Bound<'py, PyAny>, + loaded_macros: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = scope.py(); + let current: Bound<'py, PyTuple> = indexed_macro_metadata(loaded_macros)?; + let indexed: Bound<'py, PyTuple> = indexed_macro_metadata(&scope.getattr("loaded_macros")?)?; + if current.as_any().ne(indexed.as_any())? { + return Ok(py.None().into_bound(py)); + } + scope_build( + discovered_inputs, + loaded_macros, + ( + scope.getattr("index")?, + scope.getattr("resolver")?.getattr("lookup")?, + scope.getattr("sql_test_scans")?, + ), + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs new file mode 100644 index 0000000000..bff1441a51 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/effective_config.rs @@ -0,0 +1,105 @@ +//! The effective connection, settings and vars of a project, its local config and target. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyBool, PyDict, PyList, PyString}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const TEMPLATING_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.templating"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONNECTION_LABEL: &str = "effective connection"; + +/// Merge base project connection with the selected environment overrides. +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config, target_config, effective_vars))] +pub(crate) fn build_effective_connection<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, + effective_vars: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_config.py(); + let (connection, error): (Bound<'py, PyAny>, Option) = + imported(py, NATIVE_MODULE, "effective_config_connection")? + .call1((project_config, local_config, target_config))? + .extract()?; + if connection.is_none() { + let raised: Bound<'py, PyAny> = imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")? + .call1((error.unwrap_or_default(),))?; + return Err(PyErr::from_value(raised)); + } + let falsy = || PyBool::new(py, false).to_owned().into_any(); + call_with( + &imported(py, TEMPLATING_MODULE, "expand_template_data")?, + &[ + ("value", connection), + ("variables", effective_vars.clone()), + ("context_values", PyDict::new(py).into_any()), + ( + "context_label", + PyString::new(py, CONNECTION_LABEL).into_any(), + ), + ("allow_context", falsy()), + ("preserve_context_tokens", falsy()), + ("preserve_unknown_context", falsy()), + ], + ) +} + +/// Merge project settings with local developer overrides. +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config))] +pub(crate) fn build_effective_settings<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_config.py(); + let settings_class: Bound<'py, PyAny> = imported(py, CONTRACT_MODELS_MODULE, "SettingsConfig")?; + let names: Vec> = imported(py, "dataclasses", "fields")? + .call1((&settings_class,))? + .try_iter()? + .map(|field| field?.getattr("name")) + .collect::>()?; + let values: Bound<'py, PyDict> = imported(py, NATIVE_MODULE, "effective_config_settings")? + .call1(( + ( + project_config.getattr("settings")?, + local_config.getattr("settings")?, + ), + local_config.getattr("setting_overrides")?, + PyList::new(py, names)?, + ))? + .downcast_into()?; + settings_class.call((), Some(&values)) +} + +/// Merge effective vars using the locked precedence order. +#[pyfunction] +#[pyo3(signature = (*, project_config, local_config, target_config, cli_vars))] +pub(crate) fn build_effective_vars<'py>( + project_config: &Bound<'py, PyAny>, + local_config: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, + cli_vars: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project_config.py(); + let merged: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "effective_config_vars")? + .call1(((project_config, local_config), target_config, cli_vars))?; + imported(py, TEMPLATING_MODULE, "expand_effective_vars")?.call1((merged,)) +} + +/// `build_effective_vars` for a discovered project and target, without CLI vars. +pub(crate) fn project_vars<'py>( + discovered: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, +) -> PyResult> { + build_effective_vars( + &discovered.getattr("project_config")?, + &discovered.getattr("local_config")?, + target_config, + PyDict::new(discovered.py()).as_any(), + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs new file mode 100644 index 0000000000..bb3a0161fb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod declaration_scope; +pub(crate) mod effective_config; +pub(crate) mod model_config; +pub(crate) mod model_inputs; +pub(crate) mod registration; +pub(crate) mod target; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs new file mode 100644 index 0000000000..8e98074ca2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_config.rs @@ -0,0 +1,96 @@ +//! The native config builder, validator and inherited storage policies of one compile. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::compiled_project::loop_classes::table_backed; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; +const RETENTION_MODULE: &str = "sqlbuild.compiler.compile._helpers.config.retention"; +const TABLE_TYPE_MODULE: &str = "sqlbuild.compiler.compile._helpers.config.table_type"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; + +/// Capture the project inputs every model's native config build and validation read. +#[pyfunction] +#[pyo3(signature = (*, inputs, validator))] +pub(crate) fn native_model_config_session<'py>( + inputs: &Bound<'py, PyAny>, + validator: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = inputs.py(); + let project_config: Bound<'py, PyAny> = inputs.getattr("project_config")?; + let target: Bound<'py, PyAny> = inputs.getattr("target_config")?; + let declared: Bound<'py, PyAny> = project_config.getattr("materialization_defaults")?; + let materialization_defaults: Bound<'py, PyAny> = if declared.is_truthy()? { + declared + } else { + imported(py, CONTRACT_MODELS_MODULE, "MaterializationDefaultsConfig")?.call0()? + }; + let hook_entries: Bound<'py, PyTuple> = PyTuple::new( + py, + ["SqlHookEntry", "NamedSqlHookEntry", "PythonHookEntry"] + .iter() + .map(|name| imported(py, DISCOVERY_MODELS_MODULE, name)) + .collect::>>()?, + )?; + let target_namespace: Bound<'py, PyAny> = if target.is_none() { + target.clone() + } else { + PyTuple::new(py, [target.getattr("database")?, target.getattr("schema")?])?.into_any() + }; + let builder: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "NativeModelConfigBuilder")? + .call1(( + ( + project_config.getattr("defaults")?, + project_config.getattr("path_defaults")?, + hook_entries, + ), + ( + inputs.getattr("effective_vars")?, + imported(py, UNICODE_ENVIRONMENT_MODULE, "UnicodeEnvironment")?.call0()?, + ), + ( + ( + inputs.getattr("effective_target_name")?, + inputs.getattr("run_id")?, + ), + target_namespace, + ), + ))?; + let inherited: Bound<'py, PyDict> = PyDict::new(py); + let empty_name: Bound<'py, PyAny> = PyString::new(py, "").into_any(); + let mut materializations: Vec> = vec![py.None().into_bound(py)]; + materializations.extend(table_backed(py)?); + for materialized in materializations { + let policy = |module: &str, name: &str| -> PyResult> { + call_with( + &imported(py, module, name)?, + &[ + ("materialized", materialized.clone()), + ("model_value", py.None().into_bound(py)), + ("materialization_defaults", materialization_defaults.clone()), + ("target_config", target.clone()), + ("model_name", empty_name.clone()), + ], + ) + }; + let retention: Bound<'py, PyAny> = + policy(RETENTION_MODULE, "resolve_time_travel_retention")?; + let table_type: Bound<'py, PyAny> = policy(TABLE_TYPE_MODULE, "resolve_table_type")?; + inherited.set_item(&materialized, (retention, table_type))?; + } + call_with( + &imported(py, COMPILE_MODELS_MODULE, "NativeModelConfigSession")?, + &[ + ("path_defaults", project_config.getattr("path_defaults")?), + ("builder", builder), + ("validator", validator.clone()), + ("inherited_storage", inherited.into_any()), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs new file mode 100644 index 0000000000..a150be18c6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -0,0 +1,318 @@ +//! Every discovered model file attached as a model input by the native model loop. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; + +use crate::bindings::_helpers::compile_attachment::model_config::native_model_config_session; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::compiled_project::loop_classes::model_loop_classes; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const VALIDATION_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.validation"; +const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; +const CURSOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; +const DECLARATIONS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.declarations"; +const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.macros"; +const TEMPLATING_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.templating"; +const UNICODE_ENVIRONMENT_MODULE: &str = "sqlbuild.compiler.compile.classes.unicode_environment"; +const PROJECT_VARS_MODULE: &str = "sqlbuild.compiler.authored_values.main._project_var_values"; +const MACRO_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; +const CONTRACT_CLASSES_MODULE: &str = + "sqlbuild.compiler.model_config.main._native_contract_classes"; +const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; +const PYTHON_NODE_MODELS_MODULE: &str = "sqlbuild.python_nodes.models"; + +fn dict<'py>( + py: Python<'py>, + entries: Vec<(&str, Bound<'py, PyAny>)>, +) -> PyResult> { + let built: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in entries { + built.set_item(key, value)?; + } + Ok(built) +} + +/// `sql_analysis_opt_out_help(model_owned=...)`. +fn opt_out_help<'py>(py: Python<'py>, model_owned: bool) -> PyResult> { + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("model_owned", model_owned)?; + imported(py, VALIDATION_MODULE, "sql_analysis_opt_out_help")?.call((), Some(&keywords)) +} + +/// The host the loop calls back into: `constant_text(*, declaration, file_path)`. +fn loop_host<'py>(context: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = context.py(); + let context: Py = context.clone().unbind(); + let constant_text = move |arguments: &Bound<'_, PyTuple>, + keywords: Option<&Bound<'_, PyDict>>| + -> PyResult> { + let py: Python<'_> = arguments.py(); + let context: &Bound<'_, PyAny> = context.bind(py); + let passed: Bound<'_, PyDict> = + keywords.map_or_else(|| PyDict::new(py), |keywords| keywords.clone()); + call_with( + &imported(py, DECLARATIONS_MODULE, "render_constant_declaration")?, + &[ + ("declaration", passed.as_any().get_item("declaration")?), + ("value_renderer", context.getattr("value_renderer")?), + ( + "collection_rendering", + context.getattr("collection_rendering")?, + ), + ("file_path", passed.as_any().get_item("file_path")?), + ], + ) + .map(Bound::unbind) + }; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item( + "constant_text", + PyCFunction::new_closure(py, None, None, constant_text)?, + )?; + imported(py, "types", "SimpleNamespace")?.call((), Some(&keywords)) +} + +fn macro_inputs<'py>(context: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = context.py(); + dict( + py, + vec![ + ("loaded_macros", context.getattr("loaded_macros")?), + ("macro_context", context.getattr("macro_context")?), + ( + "bridge", + imported(py, MACRO_BRIDGE_MODULE, "active_macro_bridge")?.call0()?, + ), + ( + "scoped", + pyo3::types::PyBool::new(py, !context.getattr("declaration_resolver")?.is_none()) + .to_owned() + .into_any(), + ), + ( + "declaration_context", + imported(py, COMPILE_MODELS_MODULE, "DeclarationResolutionContext")?, + ), + ( + "macro_argument_error", + imported(py, COMPILE_ERRORS_MODULE, "MacroArgumentError")?, + ), + ( + "expansion", + imported(py, MACROS_MODULE, "macro_expansion_classes")?.call0()?, + ), + ( + "generic_syntax", + imported(py, SQL_ANALYSIS_MODELS_MODULE, "SqlLexicalSyntax")? + .call0()? + .getattr("native_mapping")?, + ), + ], + ) +} + +fn sql_hook_inputs<'py>( + discovered: &Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = context.py(); + dict( + py, + vec![ + ( + "named_sql_hook_entry", + imported(py, DISCOVERY_MODELS_MODULE, "NamedSqlHookEntry")?, + ), + ( + "python_hook_entry", + imported(py, DISCOVERY_MODELS_MODULE, "PythonHookEntry")?, + ), + ("hook_files", discovered.getattr("sql_hook_files")?), + ("effective_vars", context.getattr("effective_vars")?), + ( + "environment", + imported(py, UNICODE_ENVIRONMENT_MODULE, "UnicodeEnvironment")?.call0()?, + ), + ( + "render_project_var", + imported(py, PROJECT_VARS_MODULE, "render_project_var_text")?, + ), + ( + "sql_resource_ref", + imported(py, PYTHON_NODE_MODELS_MODULE, "SqlResourceRef")?, + ), + ( + "explicit_kinds", + imported(py, COMPILE_CONSTANTS_MODULE, "EXPLICIT_REFERENCE_KINDS")?, + ), + ( + "declaration_identity", + imported(py, SCOPE_MODELS_MODULE, "DeclarationIdentity")?, + ), + ( + "sql_hook_kind", + imported(py, SCOPE_TYPES_MODULE, "DeclarationKind")?.getattr("SQL_HOOK")?, + ), + ], + ) +} + +/// Attach schema metadata to discovered model files. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_model_inputs<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_model_inputs", + options, + ( + &["discovered_inputs", "context", "native_project"], + &[ + "no_sql_validation", + "defer_model_sql_validation", + "external_sql_reference_resolver", + ], + ), + )?; + let discovered_inputs: &Bound<'py, PyAny> = &arguments.value("discovered_inputs")?; + let context: &Bound<'py, PyAny> = &arguments.value("context")?; + let native_project: &Bound<'py, PyAny> = &arguments.value("native_project")?; + let no_sql_validation: bool = arguments.flag("no_sql_validation")?; + let defer_model_sql_validation: bool = arguments.flag("defer_model_sql_validation")?; + let external: Bound<'py, PyAny> = arguments.value("external_sql_reference_resolver")?; + let settings: Bound<'py, PyAny> = context.getattr("effective_settings")?; + let validator: Bound<'py, PyAny> = imported(py, NATIVE_MODULE, "NativeModelValidator")? + .call_method1( + "from_inputs", + ( + discovered_inputs, + settings.getattr("microbatch_concurrency")?, + ), + )?; + if !external.is_none() { + call_with( + &external.getattr("validate_model_names")?, + &[("known_model_names", validator.call_method0("model_names")?)], + )?; + } + let project_dir: Bound<'py, PyAny> = discovered_inputs.getattr("project_dir")?; + let root: Bound<'py, PyAny> = if project_dir.is_truthy()? { + project_dir + } else { + imported(py, "pathlib", "Path")?.call0()? + }; + let project_config_path: Bound<'py, PyAny> = root.div(imported( + py, + DISCOVERY_CONSTANTS_MODULE, + "PROJECT_CONFIG_FILENAME", + )?)?; + let config_inputs: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "NativeModelConfigInputs")?, + &[ + ( + "project_config", + discovered_inputs.getattr("project_config")?, + ), + ("target_config", context.getattr("target_config")?), + ("effective_vars", context.getattr("effective_vars")?), + ( + "effective_target_name", + context.getattr("effective_target_name")?, + ), + ("run_id", context.getattr("run_id")?), + ( + "microbatch_concurrency", + settings.getattr("microbatch_concurrency")?, + ), + ], + )?; + let session: Bound<'py, PyAny> = native_model_config_session(&config_inputs, &validator)?; + let provider_names: Vec> = discovered_inputs + .getattr("providers")? + .try_iter()? + .map(|provider| provider?.getattr("name")) + .collect::>()?; + let host_inputs: Bound<'py, PyDict> = dict( + py, + vec![ + ( + "report_diagnostic", + imported(py, COLLECTOR_MODULE, "report_compile_diagnostic")?, + ), + ( + "record_template_reads", + imported(py, TEMPLATING_MODULE, "record_template_reads")?, + ), + ( + "visible_declarations", + imported(py, COMPILE_MODELS_MODULE, "VisibleModelDeclarations")?, + ), + ("config_session", session), + ("syntax_help", opt_out_help(py, true)?), + ("source_syntax_help", opt_out_help(py, false)?), + ("macros", macro_inputs(context)?.into_any()), + ( + "sql_hooks", + sql_hook_inputs(discovered_inputs, context)?.into_any(), + ), + ( + "hook_functions", + discovered_inputs.getattr("hook_functions")?, + ), + ( + "provider_names", + PyFrozenSet::new(py, &provider_names)?.into_any(), + ), + ], + )?; + let classes: Bound<'py, PyDict> = model_loop_classes( + &discovered_inputs.getattr("audit_factories")?, + context, + &external, + &host_inputs, + )?; + let syntax: Bound<'py, PyAny> = context.getattr("sql_lexical_syntax")?; + let native_loop: Bound<'py, PyAny> = + imported(py, NATIVE_MODULE, "NativeModelLoop")?.call1(( + ( + validator, + imported(py, CONTRACT_CLASSES_MODULE, "native_contract_classes")?.call0()?, + classes, + ), + syntax.getattr("native_mapping")?, + imported(py, CURSOR_MODULE, "cursor_intrinsic_semantics")?.call0()?, + ( + ( + settings.getattr("sql_analysis")?, + no_sql_validation, + defer_model_sql_validation, + settings.getattr("require_sql_analysis")?, + ), + project_config_path.str()?, + ), + ))?; + let files: Bound<'py, PyList> = PyList::new( + py, + discovered_inputs + .getattr("model_files")? + .try_iter()? + .collect::>>()?, + )?; + let inputs: Bound<'py, PyAny> = + native_loop.call_method1("run", (native_project, loop_host(context)?, files))?; + PyTuple::new(py, inputs.try_iter()?.collect::>>()?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs new file mode 100644 index 0000000000..ba7d88e1d6 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/registration.rs @@ -0,0 +1,45 @@ +//! Register the compile-input attachment functions. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::wrap_pyfunction; + +use crate::bindings::_helpers::compile_attachment::{ + declaration_scope, effective_config, model_config, model_inputs, target, +}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!( + effective_config::build_effective_connection, + module + )?)?; + module.add_function(wrap_pyfunction!( + effective_config::build_effective_settings, + module + )?)?; + module.add_function(wrap_pyfunction!( + effective_config::build_effective_vars, + module + )?)?; + module.add_function(wrap_pyfunction!( + model_config::native_model_config_session, + module + )?)?; + module.add_function(wrap_pyfunction!(model_inputs::build_model_inputs, module)?)?; + module.add_function(wrap_pyfunction!( + target::build_compile_target_context, + module + )?)?; + module.add_function(wrap_pyfunction!( + target::build_effective_target_namespace, + module + )?)?; + module.add_function(wrap_pyfunction!( + declaration_scope::build_declaration_scope, + module + )?)?; + module.add_function(wrap_pyfunction!( + declaration_scope::rebind_declaration_scope, + module + )?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs new file mode 100644 index 0000000000..cabc68551f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/target.rs @@ -0,0 +1,247 @@ +//! The effective target, its namespace, and the shared project-local compile-cache directory. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pyfunction; +use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; + +use crate::bindings::_helpers::compile_attachment::effective_config::project_vars; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; + +const TARGET_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_name"; +const TARGET_CONFIG_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_target_config"; +const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; +const FRONTIER_CONSTANTS_MODULE: &str = "sqlbuild.compiler.frontier.constants"; +const CACHE_DIRECTORY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiler_cache_directory"; +const CONTEXT_TEMPLATES_MODULE: &str = + "sqlbuild.compiler.compile._helpers.render.context_templates"; +const DATABASE_KEY: &str = "database"; +const SCHEMA_KEY: &str = "schema"; + +/// The selected target's name and merged config, or `(None, None)`. +fn resolved_target<'py>( + discovered: &Bound<'py, PyAny>, + selected_target: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { + let py: Python<'py> = discovered.py(); + let project_config: Bound<'py, PyAny> = discovered.getattr("project_config")?; + let local_config: Bound<'py, PyAny> = discovered.getattr("local_config")?; + let target_name: Bound<'py, PyAny> = call_with( + &imported(py, TARGET_NAME_MODULE, "resolve_target_name")?, + &[ + ("project_config", project_config.clone()), + ("local_config", local_config.clone()), + ("selected_target", selected_target.clone()), + ], + )?; + if target_name.is_none() { + return Ok((target_name, py.None().into_bound(py))); + } + let target_config: Bound<'py, PyAny> = call_with( + &imported(py, TARGET_CONFIG_MODULE, "resolve_target_config")?, + &[ + ("project_config", project_config), + ("local_config", local_config), + ("target_name", target_name.clone()), + ], + )?; + Ok((target_name, target_config)) +} + +/// Resolve the effective target and shared project-local compile-cache directory. +#[pyfunction] +#[pyo3(signature = (*, discovered_inputs, selected_target, no_cache))] +pub(crate) fn build_compile_target_context<'py>( + discovered_inputs: &Bound<'py, PyAny>, + selected_target: &Bound<'py, PyAny>, + no_cache: bool, +) -> PyResult> { + let py: Python<'py> = discovered_inputs.py(); + let (target_name, target_config) = resolved_target(discovered_inputs, selected_target)?; + let cache_dir: Bound<'py, PyAny> = compile_cache_root( + &discovered_inputs.getattr("project_dir")?, + &target_config, + no_cache, + )?; + PyTuple::new(py, [target_name, target_config, cache_dir]) +} + +/// The shared compile-cache directory, or `None` when caching is disabled. +fn compile_cache_root<'py>( + project_dir: &Bound<'py, PyAny>, + target_config: &Bound<'py, PyAny>, + no_cache: bool, +) -> PyResult> { + let py: Python<'py> = project_dir.py(); + let constant = |name: &str| imported(py, COMPILE_CONSTANTS_MODULE, name); + let disabled_value: String = constant("COMPILE_CACHE_DISABLE_VALUE")?.extract()?; + let disable_env: String = constant("COMPILE_CACHE_DISABLE_ENV_VAR")?.extract()?; + let disabled: bool = py + .import("os")? + .getattr("environ")? + .call_method1("get", (disable_env,))? + .extract::>()? + .is_some_and(|value| value == disabled_value); + if no_cache + || project_dir.is_none() + || (!target_config.is_none() + && target_config + .getattr("compile_cache")? + .is(pyo3::types::PyBool::new(py, false))) + || disabled + { + return Ok(py.None().into_bound(py)); + } + let root: Bound<'py, PyAny> = + imported(py, CACHE_DIRECTORY_MODULE, "compiler_cache_directory")?.call1((project_dir,))?; + remove_retired_caches(&root)?; + Ok(root) +} + +/// Delete the fact and reference caches older releases kept under each retired cache root. +fn remove_retired_caches(root: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = root.py(); + let directory_name: String = imported( + py, + FRONTIER_CONSTANTS_MODULE, + "COMPILER_CACHE_DIRECTORY_NAME", + )? + .extract()?; + let names: [Bound<'_, PyAny>; 2] = [ + imported( + py, + COMPILE_CONSTANTS_MODULE, + "RETIRED_FACT_CACHE_DIRECTORY_NAME", + )?, + imported( + py, + COMPILE_CONSTANTS_MODULE, + "RETIRED_REFERENCE_CACHE_DIRECTORY_NAME", + )?, + ]; + let rmtree: Bound<'_, PyAny> = imported(py, "shutil", "rmtree")?; + let ignore: Bound<'_, PyDict> = PyDict::new(py); + ignore.set_item("ignore_errors", true)?; + for suffix in imported( + py, + FRONTIER_CONSTANTS_MODULE, + "RETIRED_CACHE_NAMESPACE_SUFFIXES", + )? + .try_iter()? + { + let suffix: String = suffix?.extract()?; + for name in &names { + let retired: Bound<'_, PyAny> = root + .getattr("parent")? + .div(format!("{directory_name}{suffix}"))? + .div(name)?; + if retired.call_method0("is_dir")?.is_truthy()? { + rmtree.call((retired,), Some(&ignore))?; + } + } + } + Ok(()) +} + +/// `value if isinstance(value, str) and value else None`. +fn optional_string<'py>( + value: Option>, + py: Python<'py>, +) -> PyResult> { + Ok(match value { + Some(value) if value.is_instance_of::() && value.is_truthy()? => value, + _ => py.None().into_bound(py), + }) +} + +/// `first or second or third`. +fn first_truthy<'py>(values: [Bound<'py, PyAny>; 3]) -> PyResult> { + let [first, second, third] = values; + if first.is_truthy()? { + return Ok(first); + } + if second.is_truthy()? { + return Ok(second); + } + Ok(third) +} + +/// Resolve the target and its effective database/schema without compiling resources. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn build_effective_target_namespace<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "build_effective_target_namespace", + options, + ( + &[ + "discovered_inputs", + "selected_target", + "connection_database", + "connection_schema", + "default_database", + "default_schema", + ], + &[], + ), + )?; + let discovered_inputs: &Bound<'py, PyAny> = &arguments.value("discovered_inputs")?; + let selected_target: &Bound<'py, PyAny> = &arguments.value("selected_target")?; + let connection_database: Bound<'py, PyAny> = arguments.value("connection_database")?; + let connection_schema: Bound<'py, PyAny> = arguments.value("connection_schema")?; + let default_database: Bound<'py, PyAny> = arguments.value("default_database")?; + let default_schema: Bound<'py, PyAny> = arguments.value("default_schema")?; + let (target_name, target_config) = resolved_target(discovered_inputs, selected_target)?; + let effective_vars: Bound<'py, PyAny> = project_vars(discovered_inputs, &target_config)?; + let target_value = |key: &str| -> PyResult> { + if target_config.is_none() { + return Ok(py.None().into_bound(py)); + } + target_config.getattr(key) + }; + let values: Bound<'py, PyDict> = PyDict::new(py); + values.set_item( + DATABASE_KEY, + first_truthy([ + target_value(DATABASE_KEY)?, + connection_database, + default_database, + ])?, + )?; + values.set_item( + SCHEMA_KEY, + first_truthy([target_value(SCHEMA_KEY)?, connection_schema, default_schema])?, + )?; + let run_id: Bound<'py, PyAny> = call_with( + &imported(py, CONTEXT_TEMPLATES_MODULE, "resolve_run_id")?, + &[("selected_run_id", py.None().into_bound(py))], + )?; + let resolved: Bound<'py, PyDict> = call_with( + &imported( + py, + CONTEXT_TEMPLATES_MODULE, + "resolve_early_model_templates", + )?, + &[ + ("values", values.into_any()), + ("effective_vars", effective_vars), + ("effective_target_name", target_name.clone()), + ("run_id", run_id), + ], + )? + .downcast_into()?; + PyTuple::new( + py, + [ + target_name, + target_config, + optional_string(resolved.get_item(DATABASE_KEY)?, py)?, + optional_string(resolved.get_item(SCHEMA_KEY)?, py)?, + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs index 29a3595ccb..807c9a2003 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly.rs @@ -11,9 +11,8 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; -const PROJECT_WITH_MODULE: &str = "sqlbuild.compiler.compiled_project.main.project_with"; -const RECORD_ANALYSES_MODULE: &str = - "sqlbuild.compiler.compiled_project.main._record_model_analyses"; +const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; +const RECORD_ANALYSES_MODULE: &str = "sqlbuild._native"; const EXPLICIT_REFERENCES_MODULE: &str = "sqlbuild.compiler.compile._helpers.explicit_references.checks"; const ACTIVE_BRIDGE_MODULE: &str = "sqlbuild.compiler.macro_bridge.main.active_macro_bridge"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs index fc76e5b89e..ceb29e8a99 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/inputs.rs @@ -9,18 +9,17 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; const CALL_STORE_MODULE: &str = "sqlbuild.compiler.compile._helpers.macro_bridge.call_store"; -const ATTACHMENT_MODULE: &str = "sqlbuild.compiler.compile._helpers.attachment.core"; -const DECLARATION_SCOPE_MODULE: &str = - "sqlbuild.compiler.compile._helpers.attachment.declaration_scope"; +const ATTACHMENT_MODULE: &str = "sqlbuild._native"; +const DECLARATION_SCOPE_MODULE: &str = "sqlbuild._native"; const COLLECTOR_MODULE: &str = "sqlbuild.compiler.compile._helpers.diagnostics.collector"; const CONTEXT_TEMPLATES_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.context_templates"; const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.macros"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const ATTACH_CHECKS_MODULE: &str = "sqlbuild.compiler.compiled_project.main._attach_checks"; -const ATTACH_RESOURCES_MODULE: &str = "sqlbuild.compiler.compiled_project.main._attach_resources"; -const INPUTS_VIEW_MODULE: &str = "sqlbuild.compiler.compiled_project.main._compile_inputs_view"; -const INPUTS_WITH_MODULE: &str = "sqlbuild.compiler.compiled_project.main.inputs_with"; +const ATTACH_CHECKS_MODULE: &str = "sqlbuild._native"; +const ATTACH_RESOURCES_MODULE: &str = "sqlbuild._native"; +const INPUTS_VIEW_MODULE: &str = "sqlbuild._native"; +const INPUTS_WITH_MODULE: &str = "sqlbuild._native"; const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; const NATIVE_MODULE: &str = "sqlbuild._native"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index 797e9b18bc..4ed96a7291 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -14,7 +14,7 @@ const FRONTIER_TYPES_MODULE: &str = "sqlbuild.compiler.frontier.types"; const BUILD_INPUTS_MODULE: &str = "sqlbuild._native"; const ASSEMBLE_MODULE: &str = "sqlbuild._native"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; -const PROJECT_WITH_MODULE: &str = "sqlbuild.compiler.compiled_project.main.project_with"; +const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; const PROMOTION_MODULE: &str = "sqlbuild.compiler.contracts.main.promotion_conflicts"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs new file mode 100644 index 0000000000..4ac75aaedf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs @@ -0,0 +1,637 @@ +//! Seeds, functions, sources, SQL tests, scenarios and audits attached from discovered inputs. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use std::collections::HashMap; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, within, +}; +use crate::bindings::_helpers::compiled_project::audit_loop::{AuditBlockRow, AuditFileRow}; +use crate::bindings::_helpers::compiled_project::function_loop::FunctionRow; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::resource_diagnostics::{ + DiagnosticSettings, LoaderRow, resource_diagnostics, +}; +use crate::bindings::_helpers::compiled_project::scenario_loop::ScenarioRow; +use crate::bindings::_helpers::compiled_project::source_loop::SourceRow; +use crate::bindings::_helpers::compiled_project::sql_test_loop::{ + TestBlockRow, TestCaseRow, TestFileRow, +}; + +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; +const BUILTIN_AUDITS_MODULE: &str = "sqlbuild.compiler.auditing.main._builtins"; +const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main.record"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const SQL_VALUE_ERRORS_MODULE: &str = "sqlbuild.sql_values.exceptions"; +const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const PYTHON_RESOURCES_MODULE: &str = + "sqlbuild.compiler.compile._helpers.diagnostics.python_resources"; +const PATHLIB_MODULE: &str = "pathlib"; +const TEST_TIMING_KEY: &str = "test_input_compile_ms"; +const DESCRIPTION_KEY: &str = "description"; +const MEASUREMENT_MODE: &str = "MEASUREMENT"; + +/// `getattr(value, name)` extracted. +fn attr<'py, T: pyo3::FromPyObject<'py>>(value: &Bound<'py, PyAny>, name: &str) -> PyResult { + value.getattr(name)?.extract() +} + +/// `str(value.)`. +fn text(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + Ok(value.getattr(name)?.str()?.to_string()) +} + +/// `value..stem`. +fn stem(value: &Bound<'_, PyAny>, name: &str) -> PyResult { + attr(&value.getattr(name)?, "stem") +} + +/// Each item of the iterable `value.`. +fn items<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult>> { + value.getattr(name)?.try_iter()?.collect() +} + +fn enum_member<'py>( + py: Python<'py>, + (module, class, member): (&str, &str, &str), +) -> PyResult> { + imported(py, module, class)?.getattr(member) +} + +/// `{member.value: member for member in enum_class}`. +fn by_value<'py>(py: Python<'py>, module: &str, class: &str) -> PyResult> { + let members: Bound<'py, PyDict> = PyDict::new(py); + for member in imported(py, module, class)?.try_iter()? { + let member: Bound<'py, PyAny> = member?; + members.set_item(member.getattr("value")?, member)?; + } + Ok(members) +} + +fn seed_declarations<'py>(discovered: &Bound<'py, PyAny>) -> PyResult, Py)>> { + let mut declarations: Vec<(Py, Py)> = Vec::new(); + for schema_file in items(discovered, "schema_files")? { + for entry in items(&schema_file, "seed_entries")? { + declarations.push((entry.unbind(), schema_file.clone().unbind())); + } + } + Ok(declarations) +} + +/// The function rows of `field`; Python function files have no authored SQL contents. +fn function_rows( + discovered: &Bound<'_, PyAny>, + (field, body): (&str, &str), + python: bool, +) -> PyResult> { + items(discovered, field)? + .into_iter() + .map(|file| { + Ok(( + text(&file, "file_path")?, + text(&file, "relative_path")?, + stem(&file, "file_path")?, + file.clone().unbind(), + attr(&file, "header_values")?, + attr(&file, body)?, + if python { + None + } else { + Some(attr(&file, "contents")?) + }, + )) + }) + .collect() +} + +/// `override if override is not None else fallback`. +fn either<'py>( + defaults: &Bound<'py, PyAny>, + (preferred, fallback): (&str, &str), +) -> PyResult> { + let value: Bound<'py, PyAny> = defaults.getattr(preferred)?; + if value.is_none() { + return attr(defaults, fallback); + } + value.extract() +} + +/// `(database, schema)` of a target config, or `None`. +fn target_namespace( + target: &Bound<'_, PyAny>, +) -> PyResult, Option)>> { + if target.is_none() { + return Ok(None); + } + Ok(Some((attr(target, "database")?, attr(target, "schema")?))) +} + +/// Pair every seed, then attach every function and source, in that order. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn attach_resources<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "attach_resources", + options, + ( + &[ + "discovered_inputs", + "native_project", + "settings", + "no_sql_validation", + "python_functions_inherit_default_namespace", + ], + &[], + ), + )?; + let discovered_inputs: &Bound<'py, PyAny> = &arguments.value("discovered_inputs")?; + let native_project: &Bound<'py, NativeCompiledProject> = + &arguments.value("native_project")?.downcast_into()?; + let (effective_settings, target_config, adapter_name): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + String, + ) = arguments.value("settings")?.extract()?; + let no_sql_validation: bool = arguments.flag("no_sql_validation")?; + let python_functions_inherit_default_namespace: bool = + arguments.flag("python_functions_inherit_default_namespace")?; + let validate: bool = attr::(&effective_settings, "sql_analysis")? && !no_sql_validation; + let seeds = NativeCompiledProject::attach_seeds( + native_project, + seed_declarations(discovered_inputs)?, + items(discovered_inputs, "seed_files")? + .into_iter() + .map(Bound::unbind) + .collect(), + &imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?, + )?; + let defaults: Bound<'py, PyAny> = discovered_inputs + .getattr("project_config")? + .getattr("defaults")?; + let function_kind: Bound<'py, PyAny> = + enum_member(py, (SCOPE_TYPES_MODULE, "ResourceKind", "FUNCTION"))?; + let functions = NativeCompiledProject::attach_functions( + native_project, + ( + function_rows(discovered_inputs, ("sql_function_files", "body_sql"), false)?, + function_rows( + discovered_inputs, + ("python_function_files", "body_python"), + true, + )?, + ), + ( + adapter_name, + validate, + ( + either(&defaults, ("function_database", "database"))?, + either(&defaults, ("function_schema", "schema"))?, + ), + target_namespace(&target_config)?, + python_functions_inherit_default_namespace, + function_kind.unbind(), + ), + )?; + let mut source_rows: Vec = Vec::new(); + for source_file in items(discovered_inputs, "source_files")? { + let file_path: String = text(&source_file, "file_path")?; + for entry in items(&source_file, "source_entries")? { + source_rows.push(( + file_path.clone(), + source_file.clone().unbind(), + entry.unbind(), + )); + } + } + let sources = NativeCompiledProject::attach_sources( + native_project, + source_rows, + ( + attr(&defaults, "contract")?, + validate, + enum_member(py, (SCOPE_TYPES_MODULE, "ResourceKind", "SOURCE"))?.unbind(), + ), + )?; + PyTuple::new( + py, + [ + PyTuple::new(py, seeds)?.into_any(), + PyTuple::new(py, functions)?.into_any(), + PyTuple::new(py, sources)?.into_any(), + ], + ) +} + +fn test_case_row(case: Bound<'_, PyAny>) -> PyResult { + Ok(( + case.clone().unbind(), + attr(&case, "name")?, + items(&case, "values")? + .into_iter() + .map(|pair| pair.extract()) + .collect::>()?, + )) +} + +fn test_block_row(block: Bound<'_, PyAny>) -> PyResult { + Ok(( + block.clone().unbind(), + attr(&block, "name")?, + attr(&block, "sql_body")?, + attr(&block.getattr("mode")?, "value")?, + items(&block, "parameters")? + .iter() + .map(|parameter| attr(parameter, "name")) + .collect::>()?, + items(&block, "cases")? + .into_iter() + .map(test_case_row) + .collect::>()?, + attr(&block, "sql_body_span")?, + )) +} + +fn test_file_row(file: Bound<'_, PyAny>) -> PyResult { + Ok(( + text(&file, "file_path")?, + text(&file, "relative_path")?, + stem(&file, "relative_path")?, + file.clone().unbind(), + attr(&file, "contents")?, + items(&file, "blocks")? + .into_iter() + .map(test_block_row) + .collect::>()?, + )) +} + +fn scenario_row(scenario: Bound<'_, PyAny>) -> PyResult { + Ok(( + text(&scenario, "file_path")?, + text(&scenario, "relative_path")?, + scenario.clone().unbind(), + attr(&scenario, "contents")?, + attr(&scenario, "sql_body")?, + attr(&scenario, "name")?, + )) +} + +/// The adapter's typed scalar renderer as `(text, None)`, or `(None, error)` for a bad value. +fn parameter_renderer<'py>(value_renderer: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value_renderer.py(); + let renderer: Py = value_renderer.clone().unbind(); + let errors: Py = PyTuple::new( + py, + [ + imported(py, SQL_VALUE_ERRORS_MODULE, "SqlValueRenderingError")?, + imported(py, SQL_VALUE_ERRORS_MODULE, "SqlValueValidationError")?, + ], + )? + .into_any() + .unbind(); + let render = move |arguments: &Bound<'_, PyTuple>, + _: Option<&Bound<'_, PyDict>>| + -> PyResult<(Option, Option)> { + let py: Python<'_> = arguments.py(); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("value", arguments.get_item(0)?)?; + match renderer + .bind(py) + .call_method("render_typed_scalar", (), Some(&keywords)) + { + Ok(rendered) => Ok((Some(rendered.extract()?), None)), + Err(error) if error.matches(py, errors.bind(py))? => { + Ok((None, Some(error.value(py).str()?.to_string()))) + } + Err(error) => Err(error), + } + }; + Ok(PyCFunction::new_closure(py, None, None, render)?.into_any()) +} + +fn sql_test_settings<'py>( + py: Python<'py>, + (value_renderer, syntax, scan_cache): ( + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + &Bound<'py, PyAny>, + ), +) -> PyResult> { + let settings: Bound<'py, PyDict> = PyDict::new(py); + for (key, class) in [ + ("test_input", "CompileSqlTestInput"), + ("model_payload", "CompileModelSqlTestInputPayload"), + ("direct_payload", "CompileDirectLogicSqlTestInputPayload"), + ("test_cte", "CompileSqlTestCte"), + ("scenario_input", "CompileSqlScenarioInput"), + ("scenario_cte", "CompileSqlScenarioCte"), + ] { + settings.set_item(key, imported(py, COMPILE_MODELS_MODULE, class)?)?; + } + settings.set_item("modes", by_value(py, COMPILE_TYPES_MODULE, "SqlTestMode")?)?; + settings.set_item( + "test_kind", + enum_member(py, (SCOPE_TYPES_MODULE, "ResourceKind", "TEST"))?, + )?; + settings.set_item( + "scenario_kind", + enum_member(py, (SCOPE_TYPES_MODULE, "ResourceKind", "SCENARIO"))?, + )?; + settings.set_item( + "runtime_usage", + enum_member(py, (SCOPE_TYPES_MODULE, "UsageKind", "RUNTIME"))?, + )?; + settings.set_item("render_parameter", parameter_renderer(value_renderer)?)?; + settings.set_item("adapter_name", value_renderer.getattr("adapter_name")?)?; + settings.set_item( + "source_location", + imported(py, CONTRACT_MODELS_MODULE, "SourceLocation")?, + )?; + settings.set_item("scan_store", scan_cache.call_method0("opened")?)?; + settings.set_item( + "scan_key", + ( + imported(py, COMPILE_CONSTANTS_MODULE, "SQL_TEST_CTE_SCAN_ALGORITHM")?, + syntax.getattr("cache_key")?, + ), + )?; + Ok(settings) +} + +fn audit_block_row(block: Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = block.py(); + let measurement: Bound<'_, PyAny> = enum_member( + py, + ( + AUDITING_TYPES_MODULE, + "AuditEvaluationMode", + MEASUREMENT_MODE, + ), + )?; + Ok(( + block.clone().unbind(), + attr(&block, "name")?, + attr(&block, "sql_body")?, + attr(&block, "evidence_sql")?, + block.getattr("evaluation_mode")?.eq(measurement)?, + attr(&block, "header_values")?, + )) +} + +fn audit_file_row(file: Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = file.py(); + let generic: Bound<'_, PyAny> = + enum_member(py, (SCOPE_TYPES_MODULE, "DeclarationKind", "AUDIT"))?; + Ok(( + text(&file, "file_path")?, + text(&file, "relative_path")?, + stem(&file, "file_path")?, + file.clone().unbind(), + file.getattr("declaration_kind")?.is(&generic), + attr(&file, "contents")?, + items(&file, "blocks")? + .into_iter() + .map(audit_block_row) + .collect::>()?, + )) +} + +fn audit_settings<'py>( + py: Python<'py>, + effective_settings: &Bound<'py, PyAny>, +) -> PyResult> { + let settings: Bound<'py, PyDict> = PyDict::new(py); + let severity: Bound<'py, PyAny> = effective_settings.getattr("default_audit_severity")?; + settings.set_item( + "default_severity", + if severity.is_none() { + None + } else { + Some(severity.str()?) + }, + )?; + settings.set_item( + "default_run_scope", + effective_settings.getattr("default_audit_run_scope")?, + )?; + let builtins: Vec = + imported(py, BUILTIN_AUDITS_MODULE, "builtin_audit_definitions")? + .call0()? + .try_iter()? + .map(|file| audit_file_row(file?)) + .collect::>()?; + settings.set_item("builtins", builtins)?; + settings.set_item( + "declaration_identity", + imported(py, SCOPE_MODELS_MODULE, "DeclarationIdentity")?, + )?; + settings.set_item( + "singular_kind", + enum_member( + py, + (SCOPE_TYPES_MODULE, "DeclarationKind", "SINGULAR_AUDIT"), + )?, + )?; + settings.set_item( + "audit_kind", + enum_member(py, (SCOPE_TYPES_MODULE, "DeclarationKind", "AUDIT"))?, + )?; + settings.set_item( + "resource_kinds", + by_value(py, SCOPE_TYPES_MODULE, "ResourceKind")?, + )?; + Ok(settings) +} + +/// Each loader's description, relative path and whether that path is a YAML file. +fn loader_rows<'py>(discovered: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = discovered.py(); + let suffixes: Bound<'py, PyAny> = + imported(py, DISCOVERY_CONSTANTS_MODULE, "YAML_FILE_SUFFIXES")?; + let mut rows: HashMap = HashMap::new(); + for loader in items(discovered, "loader_functions")? { + let relative_path: Bound<'py, PyAny> = loader.getattr("relative_path")?; + rows.insert( + attr(&loader, "name")?, + ( + attr(&loader, DESCRIPTION_KEY)?, + relative_path.str()?.to_string(), + suffixes.contains(relative_path.getattr("suffix")?)?, + ), + ); + } + Ok(rows) +} + +fn diagnostic_settings<'py>( + discovered: &Bound<'py, PyAny>, + source_inputs: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = discovered.py(); + let settings: Bound<'py, PyDict> = PyDict::new(py); + settings.set_item( + "compiler_diagnostic", + imported(py, COMPILE_MODELS_MODULE, "CompilerDiagnostic")?, + )?; + settings.set_item( + "compile_error", + imported(py, COMPILE_ERRORS_MODULE, "CompileInputError")?, + )?; + settings.set_item( + "source_location", + imported(py, CONTRACT_MODELS_MODULE, "SourceLocation")?, + )?; + settings.set_item("path_class", imported(py, PATHLIB_MODULE, "Path")?)?; + settings.set_item("model_files", items(discovered, "model_files")?)?; + let factories: Vec<(String, String, usize)> = items(discovered, "audit_factories")? + .iter() + .map(|factory| { + Ok(( + attr(factory, "name")?, + text(factory, "relative_path")?, + attr(factory, "line")?, + )) + }) + .collect::>()?; + settings.set_item("audit_factories", factories)?; + let scenarios: Vec> = items(discovered, "scenario_files")? + .iter() + .map(|scenario| { + PyTuple::new( + py, + [ + scenario.getattr("name")?, + text(scenario, "relative_path")?.into_bound_py_any(py)?, + scenario.getattr("contents")?, + scenario + .getattr("header_values")? + .call_method1("get", (DESCRIPTION_KEY,))?, + ], + ) + }) + .collect::>()?; + settings.set_item("scenarios", scenarios)?; + let hooks: Vec> = items(discovered, "sql_hook_files")? + .iter() + .map(|hook| { + PyTuple::new( + py, + [ + hook.getattr("name")?, + text(hook, "relative_path")?.into_bound_py_any(py)?, + hook.getattr("contents")?, + hook.getattr(DESCRIPTION_KEY)?, + ], + ) + }) + .collect::>()?; + settings.set_item("sql_hooks", hooks)?; + settings.set_item("loaders", loader_rows(discovered)?)?; + settings.set_item( + "python_resources", + call_with( + &imported(py, PYTHON_RESOURCES_MODULE, "python_resource_rows")?, + &[ + ("discovered_inputs", discovered.clone()), + ("source_inputs", source_inputs.clone()), + ], + )?, + )?; + Ok(settings) +} + +/// Attach every SQL test case, scenario and audit, then report every resource diagnostic. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn attach_checks<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "attach_checks", + options, + ( + &[ + "discovered_inputs", + "native_project", + "settings", + "scan_cache", + "source_inputs", + ], + &[], + ), + )?; + let discovered_inputs: &Bound<'py, PyAny> = &arguments.value("discovered_inputs")?; + let native_project: &Bound<'py, NativeCompiledProject> = + &arguments.value("native_project")?.downcast_into()?; + let (effective_settings, value_renderer, syntax): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = arguments.value("settings")?.extract()?; + let scan_cache: &Bound<'py, PyAny> = &arguments.value("scan_cache")?; + let source_inputs: &Bound<'py, PyAny> = &arguments.value("source_inputs")?; + let timing: Bound<'py, PyAny> = + imported(py, PROFILING_MODULE, "record_compile_timing")?.call1((TEST_TIMING_KEY,))?; + let (tests, scenarios) = within(&timing, |_| { + let attached = NativeCompiledProject::attach_sql_tests( + native_project, + items(discovered_inputs, "test_files")? + .into_iter() + .map(test_file_row) + .collect::>()?, + items(discovered_inputs, "scenario_files")? + .into_iter() + .map(scenario_row) + .collect::>()?, + sql_test_settings(py, (&value_renderer, &syntax, scan_cache))?.extract()?, + )?; + scan_cache.call_method0("save")?; + Ok(attached) + })?; + let (audits, audit_diagnostics) = NativeCompiledProject::attach_audits( + native_project, + items(discovered_inputs, "audit_files")? + .into_iter() + .map(audit_file_row) + .collect::>()?, + audit_settings(py, &effective_settings)?.extract()?, + )?; + let settings: DiagnosticSettings = + diagnostic_settings(discovered_inputs, source_inputs)?.extract()?; + let (orphans, descriptions, cursors) = + resource_diagnostics(&native_project.borrow(), py, &settings)?; + let diagnostics: Vec> = audit_diagnostics + .into_iter() + .chain(orphans) + .chain(descriptions) + .chain(cursors) + .collect(); + PyTuple::new( + py, + [ + PyTuple::new(py, tests)?.into_any(), + PyTuple::new(py, scenarios)?.into_any(), + PyTuple::new(py, audits)?.into_any(), + PyTuple::new(py, diagnostics)?.into_any(), + ], + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(attach_resources, module)?)?; + module.add_function(wrap_pyfunction!(attach_checks, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs new file mode 100644 index 0000000000..219cf1d045 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/loop_classes.rs @@ -0,0 +1,488 @@ +//! The Python classes, project declarations and callbacks the native model loop builds with. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const COMPILE_ERRORS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; +const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; +const DISCOVERY_MODELS_MODULE: &str = "sqlbuild.compiler.discovery.models"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const CONFIG_ERROR_MODULE: &str = "sqlbuild.compiler.model_config.main._native_config_error"; +const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; +const SCOPE_ERRORS_MODULE: &str = "sqlbuild.compiler.scopes.exceptions"; +const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; +const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const CONTRACT_TYPES_MODULE: &str = "sqlbuild.spec.contracts.types"; +const NATIVE_MODULE: &str = "sqlbuild._native"; +/// The materializations a model's storage policies are inherited for, besides none. +const TABLE_BACKED_MATERIALIZATIONS: [&str; 3] = ["TABLE", "INCREMENTAL", "SNAPSHOT"]; +/// The declaration kinds whose project-owned records the loop names. +const NAMED_KINDS: [&str; 4] = ["AUDIT", "SCHEMA", "SQL_HOOK", "PYTHON_HOOK"]; + +fn class<'py>(py: Python<'py>, module: &str, name: &str) -> PyResult> { + imported(py, module, name) +} + +fn member<'py>( + py: Python<'py>, + module: &str, + (enum_class, name): (&str, &str), +) -> PyResult> { + imported(py, module, enum_class)?.getattr(name) +} + +fn dict<'py>( + py: Python<'py>, + entries: Vec<(&str, Bound<'py, PyAny>)>, +) -> PyResult> { + let built: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in entries { + built.set_item(key, value)?; + } + Ok(built) +} + +/// The values of a table-backed `MaterializationType`, as text. +pub(crate) fn table_backed<'py>(py: Python<'py>) -> PyResult>> { + TABLE_BACKED_MATERIALIZATIONS + .iter() + .map(|name| member(py, PLANNER_TYPES_MODULE, ("MaterializationType", name))) + .collect() +} + +/// The builder, storage defaults and classes a model's `CompileModelConfig` is made of. +pub(crate) fn native_config_classes<'py>( + session: &Bound<'py, PyAny>, + record_template_reads: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = session.py(); + let table_backed: Vec> = table_backed(py)? + .iter() + .map(|materialization| Ok(materialization.str()?.into_any())) + .collect::>()?; + dict( + py, + vec![ + ("builder", session.getattr("builder")?), + ("inherited_storage", session.getattr("inherited_storage")?), + ("table_backed", PyTuple::new(py, table_backed)?.into_any()), + ( + "compile_model_config", + class(py, COMPILE_MODELS_MODULE, "CompileModelConfig")?, + ), + ( + "resolved_retention", + class(py, CONTRACT_MODELS_MODULE, "ResolvedTimeTravelRetention")?, + ), + ( + "retention_source", + member( + py, + CONTRACT_TYPES_MODULE, + ("TimeTravelRetentionSource", "MODEL"), + )?, + ), + ( + "resolved_table_type", + class(py, CONTRACT_MODELS_MODULE, "ResolvedTableType")?, + ), + ("table_type", class(py, CONTRACT_TYPES_MODULE, "TableType")?), + ( + "table_type_source", + member(py, CONTRACT_TYPES_MODULE, ("TableTypeSource", "MODEL"))?, + ), + ("record_template_reads", record_template_reads.clone()), + ], + ) +} + +/// Each project-owned generic audit's, schema's and hook's first record, by `(kind, name)`. +fn named_records<'py>(resolver: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = resolver.py(); + let records: Bound<'py, PyDict> = PyDict::new(py); + if resolver.is_none() { + return Ok(records); + } + let kinds: Vec> = NAMED_KINDS + .iter() + .map(|name| member(py, SCOPE_TYPES_MODULE, ("DeclarationKind", name))) + .collect::>()?; + let declarations: Bound<'py, PyAny> = resolver.getattr("lookup")?.getattr("declarations")?; + for pair in declarations.call_method0("items")?.try_iter()? { + let (identity, records_of): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; + let kind: Bound<'py, PyAny> = identity.getattr("kind")?; + if kinds.iter().any(|named| kind.is(named)) + && identity.getattr("owner")?.is_none() + && records_of.is_truthy()? + { + records.set_item( + (kind.getattr("value")?, identity.getattr("name")?), + records_of.get_item(0)?, + )?; + } + } + Ok(records) +} + +/// Project-owned enums by name, as `model_schema_enum_declarations` resolves column types. +fn schema_enums<'py>(resolver: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = resolver.py(); + let enums: Bound<'py, PyDict> = PyDict::new(py); + if resolver.is_none() { + return Ok(enums); + } + let enum_kind: Bound<'py, PyAny> = member(py, SCOPE_TYPES_MODULE, ("DeclarationKind", "ENUM"))?; + let enum_class: Bound<'py, PyAny> = class(py, DISCOVERY_MODELS_MODULE, "EnumDeclaration")?; + let declarations: Bound<'py, PyAny> = + resolver.getattr("projection")?.getattr("declarations")?; + for pair in declarations.call_method0("items")?.try_iter()? { + let (identity, value): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; + if identity.getattr("kind")?.is(&enum_kind) + && identity.getattr("owner")?.is_none() + && value.is_instance(&enum_class)? + { + enums.set_item(identity.getattr("name")?, value)?; + } + } + Ok(enums) +} + +/// A hook function's parameters as `inspect.signature` reads them: name, kind, default. +#[pyfunction] +fn hook_signature<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + let inspect: Bound<'py, PyAny> = py.import("inspect")?.into_any(); + let empty: Bound<'py, PyAny> = inspect.getattr("Parameter")?.getattr("empty")?; + let mut parameters: Vec<(String, i64, bool)> = Vec::new(); + for parameter in inspect + .call_method1("signature", (function,))? + .getattr("parameters")? + .call_method0("values")? + .try_iter()? + { + let parameter: Bound<'py, PyAny> = parameter?; + parameters.push(( + parameter.getattr("name")?.extract()?, + parameter + .getattr("kind")? + .call_method0("__int__")? + .extract()?, + !parameter.getattr("default")?.is(&empty), + )); + } + PyList::new(py, parameters) +} + +/// What the loop resolves each model's visible declarations from. +fn visibility_classes<'py>( + context: &Bound<'py, PyAny>, + visible_declarations: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = context.py(); + let resolver: Bound<'py, PyAny> = context.getattr("declaration_resolver")?; + let (projects, resource_specific) = if resolver.is_none() { + (false, PyFrozenSet::empty(py)?) + } else { + let lookup: Bound<'py, PyAny> = resolver.getattr("lookup")?; + let declarations: Bound<'py, PyAny> = lookup.getattr("index")?.getattr("declarations")?; + let private: Bound<'py, PyAny> = member(py, SCOPE_TYPES_MODULE, ("ScopeKind", "PRIVATE"))?; + let mut owners: Vec> = Vec::new(); + for declaration in declarations.try_iter()? { + let declaration: Bound<'py, PyAny> = declaration?; + let owner: Bound<'py, PyAny> = declaration.getattr("identity")?.getattr("owner")?; + if declaration.getattr("scope")?.is(&private) && !owner.is_none() { + owners.push(owner); + } + } + for resource in lookup.getattr("grants_by_resource")?.try_iter()? { + owners.push(resource?); + } + (declarations.is_truthy()?, PyFrozenSet::new(py, &owners)?) + }; + dict( + py, + vec![ + ("resolver", resolver), + ("projects", projects.into_bound_py_any(py)?), + ("resource_specific", resource_specific.into_any()), + ("public_enums", context.getattr("public_enums")?), + ("public_constants", context.getattr("public_constants")?), + ("visible_declarations", visible_declarations.clone()), + ( + "resource_identity", + class(py, SCOPE_MODELS_MODULE, "ResourceIdentity")?, + ), + ( + "declaration_identity", + class(py, SCOPE_MODELS_MODULE, "DeclarationIdentity")?, + ), + ( + "model_kind", + member(py, SCOPE_TYPES_MODULE, ("ResourceKind", "MODEL"))?, + ), + ( + "enum_kind", + member(py, SCOPE_TYPES_MODULE, ("DeclarationKind", "ENUM"))?, + ), + ( + "constant_kind", + member(py, SCOPE_TYPES_MODULE, ("DeclarationKind", "CONSTANT"))?, + ), + ( + "private_owner", + member( + py, + SCOPE_TYPES_MODULE, + ("VisibilityReason", "PRIVATE_OWNER"), + )?, + ), + ( + "visibility_record", + class(py, SCOPE_MODELS_MODULE, "VisibilityRecord")?, + ), + ], + ) +} + +fn input_view_classes<'py>(py: Python<'py>) -> PyResult> { + let compiled: Vec<(&str, Bound<'py, PyAny>)> = vec![ + ( + "object_key", + class(py, COMPILE_MODELS_MODULE, "CompiledObjectKey")?, + ), + ( + "relation_location", + class(py, COMPILE_MODELS_MODULE, "CompiledRelationLocation")?, + ), + ( + "inferred_column", + class(py, COMPILE_MODELS_MODULE, "InferredColumn")?, + ), + ( + "nullability", + class(py, LINEAGE_TYPES_MODULE, "InferredNullability")?, + ), + ( + "compiler_diagnostic", + class(py, COMPILE_MODELS_MODULE, "CompilerDiagnostic")?, + ), + ( + "source_location", + class(py, CONTRACT_MODELS_MODULE, "SourceLocation")?, + ), + ( + "compact_lineage_facts", + class(py, COMPILE_MODELS_MODULE, "CompactLineageFacts")?, + ), + ( + "lineage_column_fact", + class(py, COMPILE_MODELS_MODULE, "CompiledLineageColumnFact")?, + ), + ( + "lineage_source_fact", + class(py, COMPILE_MODELS_MODULE, "CompiledLineageSourceFact")?, + ), + ( + "contract_proof", + class(py, COMPILE_MODELS_MODULE, "DynamicColumnContractProof")?, + ), + ( + "family_proof", + class(py, COMPILE_MODELS_MODULE, "DynamicColumnFamilyProof")?, + ), + ]; + dict( + py, + vec![ + ( + "sql_reference", + class(py, COMPILE_MODELS_MODULE, "CompileSqlReference")?, + ), + ( + "reference_kind", + class(py, REFERENCE_TYPES_MODULE, "SqlReferenceKind")?, + ), + ( + "schema_model_entry", + class(py, CONTRACT_MODELS_MODULE, "SchemaModelEntry")?, + ), + ( + "sql_expansion", + class(py, COMPILE_MODELS_MODULE, "CompiledSqlExpansion")?, + ), + ( + "expansion_span", + class(py, COMPILE_MODELS_MODULE, "ExpansionSpan")?, + ), + ( + "audits", + dict( + py, + vec![ + ( + "audit_severity", + class(py, AUDITING_TYPES_MODULE, "AuditSeverity")?, + ), + ( + "attached_target_kind", + class(py, COMPILE_TYPES_MODULE, "AttachedAuditTargetKind")?, + ), + ], + )? + .into_any(), + ), + ( + "functions", + dict( + py, + vec![ + ( + "function_argument", + class(py, COMPILE_MODELS_MODULE, "FunctionArgument")?, + ), + ( + "function_return_column", + class(py, COMPILE_MODELS_MODULE, "FunctionReturnColumn")?, + ), + ( + "function_language", + class(py, COMPILE_TYPES_MODULE, "FunctionLanguage")?, + ), + ], + )? + .into_any(), + ), + ("compiled", dict(py, compiled)?.into_any()), + ], + ) +} + +/// `{item.: item for item in items}`. +fn by_attribute<'py>(items: &Bound<'py, PyAny>, attribute: &str) -> PyResult> { + let keyed: Bound<'py, PyDict> = PyDict::new(items.py()); + for item in items.try_iter()? { + let item: Bound<'py, PyAny> = item?; + keyed.set_item(item.getattr(attribute)?, item)?; + } + Ok(keyed) +} + +/// The classes, error factories and project declarations the model loop reads. +pub(crate) fn model_loop_classes<'py>( + audit_factories: &Bound<'py, PyAny>, + context: &Bound<'py, PyAny>, + external_resolver: &Bound<'py, PyAny>, + host_inputs: &Bound<'py, PyDict>, +) -> PyResult> { + let py: Python<'py> = context.py(); + let host = |key: &str| -> PyResult> { host_inputs.as_any().get_item(key) }; + let resolver: Bound<'py, PyAny> = context.getattr("declaration_resolver")?; + let mut provider_names: Vec = host("provider_names")? + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?; + provider_names.sort(); + let hooks: Bound<'py, PyDict> = dict( + py, + vec![ + ( + "functions", + by_attribute(&host("hook_functions")?, "name")?.into_any(), + ), + ("provider_names", provider_names.into_bound_py_any(py)?), + ( + "python_hook_entry", + class(py, DISCOVERY_MODELS_MODULE, "PythonHookEntry")?, + ), + ( + "signature", + wrap_pyfunction!(hook_signature, py)?.into_any(), + ), + ], + )?; + let record: Bound<'py, PyDict> = dict( + py, + vec![ + ("replace", imported(py, "dataclasses", "replace")?), + ( + "declaration_rows", + imported(py, NATIVE_MODULE, "declaration_rows")?, + ), + ], + )?; + dict( + py, + vec![ + ( + "usage_record", + class(py, SCOPE_MODELS_MODULE, "UsageRecord")?, + ), + ( + "compile_error", + class(py, COMPILE_ERRORS_MODULE, "CompileInputError")?, + ), + ("syntax_help", host("syntax_help")?), + ("source_syntax_help", host("source_syntax_help")?), + ("macros", host("macros")?), + ("sql_hooks", host("sql_hooks")?), + ( + "scope_path_error", + class(py, SCOPE_ERRORS_MODULE, "InvalidScopePathError")?, + ), + ( + "sql_hook_entry", + class(py, DISCOVERY_MODELS_MODULE, "SqlHookEntry")?, + ), + ( + "config_error", + imported(py, CONFIG_ERROR_MODULE, "native_config_error")?, + ), + ( + "audit_factories", + by_attribute(audit_factories, "name")?.into_any(), + ), + ( + "generated_usage", + member(py, SCOPE_TYPES_MODULE, ("UsageKind", "GENERATED"))?, + ), + ( + "contract_enforced", + member(py, PLANNER_TYPES_MODULE, ("ContractPolicy", "ENFORCED"))?, + ), + ("model_schemas", context.getattr("public_model_schemas")?), + ("named_records", named_records(&resolver)?.into_any()), + ("schema_enums", schema_enums(&resolver)?.into_any()), + ( + "compiler_diagnostic", + class(py, COMPILE_MODELS_MODULE, "CompilerDiagnostic")?, + ), + ("report_diagnostic", host("report_diagnostic")?), + ("path_class", imported(py, "pathlib", "Path")?), + ("external_resolver", external_resolver.clone()), + ( + "reference_kind", + class(py, REFERENCE_TYPES_MODULE, "SqlReferenceKind")?, + ), + ("partial", imported(py, "functools", "partial")?), + ( + "config", + native_config_classes(&host("config_session")?, &host("record_template_reads")?)? + .into_any(), + ), + ("hooks", hooks.into_any()), + ( + "visibility", + visibility_classes(context, &host("visible_declarations")?)?.into_any(), + ), + ("record", record.into_any()), + ("input_view", input_view_classes(py)?.into_any()), + ], + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs index aaf6b3a972..29c1c178b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod attachment; pub(crate) mod audit_loop; pub(crate) mod audit_views; pub(crate) mod compiled_model_analysis; @@ -5,6 +6,7 @@ pub(crate) mod compiled_model_view; pub(crate) mod function_loop; pub(crate) mod function_views; pub(crate) mod inputs_view; +pub(crate) mod loop_classes; pub(crate) mod model_input_view; pub(crate) mod model_loop; pub(crate) mod model_loop_calls; @@ -23,6 +25,7 @@ pub(crate) mod model_loop_schema; pub(crate) mod model_loop_visibility; pub(crate) mod project; pub(crate) mod project_view; +pub(crate) mod records; pub(crate) mod registration; pub(crate) mod resource_diagnostics; pub(crate) mod scenario_loop; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index bd519a1933..e74ac57b62 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -334,7 +334,7 @@ impl NativeCompiledProject { } /// Retain one model's compile-input facts; a name recorded again is replaced in place. - fn record_model(&mut self, model: ModelInputPy<'_>) -> PyResult<()> { + pub(crate) fn record_model(&mut self, model: ModelInputPy<'_>) -> PyResult<()> { compiler_guard(|| { let facts: CompiledModelFacts = model_facts(model)?; record_model(&mut self.facts, facts); @@ -343,7 +343,7 @@ impl NativeCompiledProject { } /// Attach a recorded model's typed analysis; `False` when that model was never recorded. - fn record_model_analysis( + pub(crate) fn record_model_analysis( &mut self, name: &str, columns: AnalysisPy, @@ -373,7 +373,7 @@ impl NativeCompiledProject { } /// Record each assembled native model view's final typed analysis; return how many. - fn record_view_analyses( + pub(crate) fn record_view_analyses( slf: &Bound<'_, Self>, views: Vec>, ) -> PyResult { @@ -391,12 +391,15 @@ impl NativeCompiledProject { } /// The compile inputs view of this owner's model inputs with these input fields. - fn inputs_view(slf: Bound<'_, Self>, fields: Bound<'_, PyDict>) -> PyResult { + pub(crate) fn inputs_view( + slf: Bound<'_, Self>, + fields: Bound<'_, PyDict>, + ) -> PyResult { NativeInputsView::new(slf.unbind(), &fields) } /// The compiled project view of this owner's models with these project fields. - fn project_view( + pub(crate) fn project_view( slf: Bound<'_, Self>, fields: Bound<'_, PyDict>, ) -> PyResult { @@ -404,7 +407,7 @@ impl NativeCompiledProject { } /// Pair seed declarations with their CSV files and keep them; return each seed's view. - fn attach_seeds( + pub(crate) fn attach_seeds( slf: &Bound<'_, Self>, declarations: Vec<(Py, Py)>, seed_files: Vec>, @@ -430,7 +433,7 @@ impl NativeCompiledProject { } /// Attach every discovered source in order with the kept loop; return each source's view. - fn attach_sources( + pub(crate) fn attach_sources( slf: &Bound<'_, Self>, rows: Vec, settings: SourceSettings, @@ -447,7 +450,7 @@ impl NativeCompiledProject { } /// Attach every SQL, then Python, function with the kept loop; return each one's view. - fn attach_functions( + pub(crate) fn attach_functions( slf: &Bound<'_, Self>, rows: (Vec, Vec), settings: FunctionSettings, @@ -495,7 +498,7 @@ impl NativeCompiledProject { } /// Attach every SQL test case, then every scenario, with the kept loop. - fn attach_sql_tests( + pub(crate) fn attach_sql_tests( slf: &Bound<'_, Self>, files: Vec, scenarios: Vec, @@ -524,7 +527,7 @@ impl NativeCompiledProject { } /// Attach singular, then resource-attached, audits; return views and override warnings. - fn attach_audits<'py>( + pub(crate) fn attach_audits<'py>( slf: &Bound<'py, Self>, files: Vec, settings: AuditSettings, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs index 8d7cfae0eb..88bc987e5c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs @@ -274,7 +274,7 @@ impl NativeProjectView { /// Every field by name, for an explicit `CompiledProject` projection. #[pyo3(signature = (include_sql_expansions = true))] - fn fields<'py>( + pub(crate) fn fields<'py>( &self, py: Python<'py>, include_sql_expansions: bool, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs new file mode 100644 index 0000000000..86cb146d1e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs @@ -0,0 +1,619 @@ +//! Compiled records and views: projecting native views as records, and recording later stages. + +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyFrozenSetMethods, PyList, PyTuple}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::compiled_project::inputs_view::NativeInputsView; +use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; +use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const DECIMAL_KIND: &str = "decimal"; +const COLLECTION_KINDS: [&str; 2] = ["list", "set"]; +const SCALAR_KINDS: [&str; 5] = ["string", "integer", "boolean", "float", "null"]; + +fn replace<'py>( + value: &Bound<'py, PyAny>, + changes: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let py: Python<'py> = value.py(); + let keywords: Bound<'py, PyDict> = PyDict::new(py); + for (name, change) in changes { + keywords.set_item(*name, change)?; + } + imported(py, DATACLASSES_MODULE, "replace")?.call((value,), Some(&keywords)) +} + +/// `value` when it already is a `class` record, else `class(**{field: getattr(value, field)})`. +fn record<'py>(value: &Bound<'py, PyAny>, class: &str) -> PyResult> { + let py: Python<'py> = value.py(); + let class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS_MODULE, class)?; + if value.is_instance(&class)? { + return Ok(value.clone()); + } + let fields: Bound<'py, PyDict> = PyDict::new(py); + for field in imported(py, DATACLASSES_MODULE, "fields")? + .call1((&class,))? + .try_iter()? + { + let name: Bound<'py, PyAny> = field?.getattr("name")?; + fields.set_item( + &name, + value.getattr(name.downcast::()?)?, + )?; + } + class.call((), Some(&fields)) +} + +/// The model as a `CompiledModel`, projected from its owner when it is a native view. +#[pyfunction] +pub(crate) fn compiled_model_record<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + record(model, "CompiledModel") +} + +/// The audit as a `CompiledAudit`, projected from its owner when it is a native view. +#[pyfunction] +pub(crate) fn compiled_audit_record<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + record(audit, "CompiledAudit") +} + +/// The function as a `CompiledFunction`, projected from its owner when it is a native view. +#[pyfunction] +pub(crate) fn compiled_function_record<'py>( + function: &Bound<'py, PyAny>, +) -> PyResult> { + record(function, "CompiledFunction") +} + +/// The source as a `CompiledSource`, projected from its owner when it is a native view. +#[pyfunction] +pub(crate) fn compiled_source_record<'py>( + source: &Bound<'py, PyAny>, +) -> PyResult> { + record(source, "CompiledSource") +} + +/// The seed as a `CompiledSeed`, projected from its owner when it is a native view. +#[pyfunction] +pub(crate) fn compiled_seed_record<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + record(seed, "CompiledSeed") +} + +/// Each item of `fields[name]` as a `class` record. +fn records_of<'py>(fields: &Bound<'py, PyDict>, (name, class): (&str, &str)) -> PyResult<()> { + let py: Python<'py> = fields.py(); + let items: Vec> = fields + .as_any() + .get_item(name)? + .try_iter()? + .map(|item| record(&item?, class)) + .collect::>()?; + fields.set_item(name, PyTuple::new(py, items)?) +} + +/// The project as a `CompiledProject`, projected from its owner when it is a native view. +#[pyfunction] +#[pyo3(signature = (project, *, include_sql_expansions=true))] +pub(crate) fn compiled_project_record<'py>( + project: &Bound<'py, PyAny>, + include_sql_expansions: bool, +) -> PyResult> { + let py: Python<'py> = project.py(); + let class: Bound<'py, PyAny> = imported(py, COMPILE_MODELS_MODULE, "CompiledProject")?; + if project.is_instance(&class)? { + if include_sql_expansions { + return Ok(project.clone()); + } + return replace(project, &[("sql_expansions", PyDict::new(py).into_any())]); + } + let view: PyRef<'py, NativeProjectView> = project.extract()?; + let fields: Bound<'py, PyDict> = view.fields(py, include_sql_expansions)?; + drop(view); + for pair in [ + ("seeds", "CompiledSeed"), + ("sources", "CompiledSource"), + ("functions", "CompiledFunction"), + ("audits", "CompiledAudit"), + ] { + records_of(&fields, pair)?; + } + class.call((), Some(&fields)) +} + +/// The model with these final binding diagnostics. +#[pyfunction] +#[pyo3(signature = (*, model, diagnostics))] +pub(crate) fn with_binding_diagnostics<'py>( + model: &Bound<'py, PyAny>, + diagnostics: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = model.py(); + if model.is_instance(&imported(py, COMPILE_MODELS_MODULE, "CompiledModel")?)? { + return replace(model, &[("binding_diagnostics", diagnostics.clone())]); + } + model.call_method1("with_binding_diagnostics", (diagnostics,)) +} + +/// The model with its recovered bindings and its poisoned outputs' types forgotten. +#[pyfunction] +#[pyo3(signature = (*, model, diagnostics, poisoned))] +pub(crate) fn with_type_recovery<'py>( + model: &Bound<'py, PyAny>, + diagnostics: &Bound<'py, PyAny>, + poisoned: &Bound<'py, PyFrozenSet>, +) -> PyResult> { + let py: Python<'py> = model.py(); + let name: String = model.getattr("name")?.extract()?; + let mut unchecked: Vec = Vec::new(); + for pair in poisoned.iter() { + let (model_name, column): (String, String) = pair.extract()?; + if model_name == name { + unchecked.push(column); + } + } + if !model.is_instance(&imported(py, COMPILE_MODELS_MODULE, "CompiledModel")?)? { + return model.call_method1("with_type_recovery", (diagnostics, unchecked)); + } + let inferred: Bound<'py, PyAny> = model.getattr("inferred_columns")?; + let columns: Bound<'py, PyAny> = if inferred.is_none() { + inferred + } else { + let mut columns: Vec> = Vec::new(); + for column in inferred.try_iter()? { + let column: Bound<'py, PyAny> = column?; + let key = (name.as_str(), column.getattr("name")?); + columns.push(if poisoned.contains(key)? { + replace(&column, &[("type", py.None().into_bound(py))])? + } else { + column + }); + } + PyTuple::new(py, columns)?.into_any() + }; + replace( + model, + &[ + ("binding_diagnostics", diagnostics.clone()), + ("inferred_columns", columns), + ( + "unchecked_output_columns", + PyFrozenSet::new(py, &unchecked)?.into_any(), + ), + ], + ) +} + +/// The project with `changes`; a native view shares its owner's models. +#[pyfunction] +#[pyo3(signature = (*, project, **changes))] +pub(crate) fn project_with<'py>( + project: &Bound<'py, PyAny>, + changes: Option>, +) -> PyResult> { + with_changes(project, "CompiledProject", changes) +} + +/// The compile inputs with `changes`; a native view shares its owner's model inputs. +#[pyfunction] +#[pyo3(signature = (*, inputs, **changes))] +pub(crate) fn inputs_with<'py>( + inputs: &Bound<'py, PyAny>, + changes: Option>, +) -> PyResult> { + with_changes(inputs, "CompileProjectInputs", changes) +} + +/// `replace(value, **changes)` for a record, else the native view's `with_changes(**changes)`. +fn with_changes<'py>( + value: &Bound<'py, PyAny>, + class: &str, + changes: Option>, +) -> PyResult> { + let py: Python<'py> = value.py(); + let changes: Bound<'py, PyDict> = changes.unwrap_or_else(|| PyDict::new(py)); + if value.is_instance(&imported(py, COMPILE_MODELS_MODULE, class)?)? { + return imported(py, DATACLASSES_MODULE, "replace")?.call((value,), Some(&changes)); + } + value.call_method("with_changes", (), Some(&changes)) +} + +/// The native owner's inputs view when it holds every model input; else a record. +#[pyfunction] +#[pyo3(signature = (*, fields, native_project))] +pub(crate) fn compile_inputs_view<'py>( + fields: &Bound<'py, PyDict>, + native_project: Option>, +) -> PyResult> { + let py: Python<'py> = fields.py(); + if let Some(project) = &native_project { + let mut native: bool = true; + for model in fields.as_any().get_item("model_inputs")?.try_iter()? { + if !model?.is_instance_of::() { + native = false; + break; + } + } + if native { + let view: NativeInputsView = + NativeCompiledProject::inputs_view(project.clone(), fields.clone())?; + return view.into_bound_py_any(py); + } + } + let keywords: Bound<'py, PyDict> = fields.copy()?; + keywords.set_item("native_project", native_project)?; + imported(py, COMPILE_MODELS_MODULE, "CompileProjectInputs")?.call((), Some(&keywords)) +} + +/// A typed SQL value as the plain value native consumers read. +#[pyfunction] +pub(crate) fn plain_typed_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + let kind: String = value.getattr("kind")?.getattr("value")?.extract()?; + let inner: Bound<'py, PyAny> = value.getattr("value")?; + if kind == DECIMAL_KIND { + return Ok(inner.str()?.into_any()); + } + if SCALAR_KINDS.contains(&kind.as_str()) { + return Ok(inner); + } + if COLLECTION_KINDS.contains(&kind.as_str()) { + let items: Vec> = inner + .try_iter()? + .map(|item| plain_typed_value(&item?)) + .collect::>()?; + return Ok(PyList::new(py, items)?.into_any()); + } + let mapping: Bound<'py, PyDict> = PyDict::new(py); + for pair in inner.try_iter()? { + let (key, item): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; + mapping.set_item(key, plain_typed_value(&item)?)?; + } + Ok(mapping.into_any()) +} + +/// An enum declaration as the native project row. +fn enum_row<'py>(declaration: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = declaration.py(); + let members: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = declaration + .getattr("members")? + .try_iter()? + .map(|member| { + let member: Bound<'py, PyAny> = member?; + Ok((member.getattr("name")?, member.getattr("value")?)) + }) + .collect::>()?; + PyTuple::new( + py, + [ + declaration.getattr("name")?, + declaration + .getattr("relative_path")? + .call_method0("as_posix")?, + members.into_bound_py_any(py)?, + py.None().into_bound(py), + py.None().into_bound(py), + py.None().into_bound(py), + ], + ) +} + +/// A constant declaration as the native project row, its value made JSON-safe. +fn constant_row<'py>(declaration: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = declaration.py(); + let render_as: Bound<'py, PyAny> = declaration.getattr("render_as")?; + PyTuple::new( + py, + [ + declaration.getattr("name")?, + declaration + .getattr("relative_path")? + .call_method0("as_posix")?, + PyList::empty(py).into_any(), + plain_typed_value(&declaration.getattr("value")?)?, + declaration + .getattr("logical_type")? + .getattr("display_name")?, + if render_as.is_none() { + render_as + } else { + render_as.getattr("value")? + }, + ], + ) +} + +/// Enum and constant declarations as native rows, in the given order. +#[pyfunction] +#[pyo3(signature = (*, enums, constants))] +pub(crate) fn declaration_rows<'py>( + enums: &Bound<'py, PyAny>, + constants: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyList>, Bound<'py, PyList>)> { + let py: Python<'py> = enums.py(); + let enum_rows: Vec> = enums + .try_iter()? + .map(|declaration| enum_row(&declaration?)) + .collect::>()?; + let constant_rows: Vec> = constants + .try_iter()? + .map(|declaration| constant_row(&declaration?)) + .collect::>()?; + Ok((PyList::new(py, enum_rows)?, PyList::new(py, constant_rows)?)) +} + +/// `(name, type)` of each dynamic family; `casefold` keys the family by its folded name. +fn family_types<'py>( + families: &Bound<'py, PyAny>, + casefold: bool, +) -> PyResult)>> { + families + .try_iter()? + .map(|family| { + let family: Bound<'py, PyAny> = family?; + let name: Bound<'py, PyAny> = family.getattr("name")?; + let name: String = if casefold { + name.call_method0("casefold")?.extract()? + } else { + name.extract()? + }; + Ok((name, family.getattr("inferred_type")?.extract()?)) + }) + .collect() +} + +/// Transfer each assembled model's typed analysis to its retained facts; return how many. +#[pyfunction] +#[pyo3(signature = (*, project, models))] +pub(crate) fn record_model_analyses<'py>( + project: &Bound<'py, NativeCompiledProject>, + models: &Bound<'py, PyAny>, +) -> PyResult { + let py: Python<'py> = project.py(); + let models: Vec> = models.try_iter()?.collect::>()?; + let views: Vec> = models + .iter() + .filter(|model| model.is_instance_of::()) + .map(|model| model.extract::>()) + .collect::>()?; + let mut transferred: usize = NativeCompiledProject::record_view_analyses(project, views)?; + let nullable: Bound<'py, PyAny> = + imported(py, LINEAGE_TYPES_MODULE, "InferredNullability")?.getattr("NULLABLE")?; + for model in &models { + if model.is_instance_of::() { + continue; + } + let inferred: Bound<'py, PyAny> = model.getattr("inferred_rows")?; + let rows: Vec<(String, Option, Bound<'py, PyAny>)> = if inferred.is_none() { + Vec::new() + } else { + inferred.extract()? + }; + let proof: Bound<'py, PyAny> = model.getattr("dynamic_column_contract")?; + let proof = if proof.is_none() { + None + } else { + let families: Bound<'py, PyAny> = proof.getattr("families")?; + Some(( + proof.getattr("output_proven")?.extract()?, + proof.getattr("bare_dynamic_pivot")?.extract()?, + family_types(&families, true)?, + proof.getattr("failure_reason")?.extract()?, + family_types(&families, false)?, + )) + }; + let analysis = ( + !inferred.is_none(), + rows.iter() + .map(|(name, column_type, _)| (name.clone(), column_type.clone())) + .collect(), + rows.iter() + .map(|(_, _, nullability)| nullability.eq(&nullable)) + .collect::>()?, + model.getattr("fast_lineage_has_star")?.is_truthy()?, + model + .getattr("unchecked_output_columns")? + .try_iter()? + .map(|column| column?.extract()) + .collect::>()?, + ); + let name: String = model.getattr("name")?.extract()?; + transferred += usize::from( + project + .borrow_mut() + .record_model_analysis(&name, analysis, proof), + ); + } + Ok(transferred) +} + +/// A native project for a `CompiledProject` built without the compile-input stage. +fn hand_built_project<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let native: Bound<'py, NativeCompiledProject> = + Bound::new(py, NativeCompiledProject::default())?; + let models: Bound<'py, PyAny> = project.getattr("models")?; + for model in models.try_iter()? { + let model: Bound<'py, PyAny> = model?; + native + .borrow_mut() + .record_model(model_row(&model)?.as_any().extract()?)?; + } + let _ = record_model_analyses(&native, &models)?; + Ok(native) +} + +/// The compile's retained project, or one recorded from a hand-built project. +#[pyfunction] +pub(crate) fn compiled_project_facts<'py>( + project: &Bound<'py, PyAny>, +) -> PyResult> { + let retained: Bound<'py, PyAny> = project.getattr("native_project")?; + if !retained.is_none() { + return Ok(retained.downcast_into()?); + } + hand_built_project(project) +} + +fn model_row<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + let relative_path: Bound<'py, PyAny> = model.getattr("relative_path")?; + let config: Bound<'py, PyAny> = model.getattr("config")?; + let references: Vec> = model + .getattr("references")? + .try_iter()? + .map(|reference| { + let reference: Bound<'py, PyAny> = reference?; + PyTuple::new( + py, + [ + reference.getattr("ref_kind")?.str()?.into_any(), + reference.getattr("ref_name")?, + reference.getattr("ref_package")?, + ], + ) + }) + .collect::>()?; + let (enums, constants) = declaration_rows( + &model.getattr("enum_declarations")?, + &model.getattr("constant_declarations")?, + )?; + PyTuple::new( + py, + [ + PyTuple::new( + py, + [ + model.getattr("name")?, + relative_path.call_method0("as_posix")?, + model.getattr("query_sql")?, + model.getattr("authored_query_sql")?, + model.getattr("authored_sql")?, + ], + )? + .into_any(), + PyTuple::new( + py, + [ + config.getattr("values")?, + PyList::new( + py, + config + .getattr("model_header_keys")? + .try_iter()? + .collect::>>()?, + )? + .into_any(), + config.getattr("layer_schema")?, + ], + )? + .into_any(), + PyList::new(py, references)?.into_any(), + schema_row(&model.getattr("schema_entry")?, &relative_path)?, + PyList::new( + py, + model + .getattr("enum_columns")? + .try_iter()? + .collect::>>()?, + )? + .into_any(), + PyTuple::new(py, [enums.into_any(), constants.into_any()])?.into_any(), + ], + ) +} + +fn schema_row<'py>( + entry: &Bound<'py, PyAny>, + relative_path: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = entry.py(); + if entry.is_none() { + return Ok(entry.clone()); + } + let columns: Vec> = entry + .getattr("columns")? + .try_iter()? + .map(|column| { + let column: Bound<'py, PyAny> = column?; + let location: Bound<'py, PyAny> = column.getattr("location")?; + let elsewhere: bool = + !location.is_none() && location.getattr("path")?.ne(relative_path)?; + PyTuple::new( + py, + [ + column.getattr("name")?, + column.getattr("type")?, + column.getattr("nullable")?, + column.getattr("audits")?.len()?.into_bound_py_any(py)?, + elsewhere.into_bound_py_any(py)?, + ], + ) + }) + .collect::>()?; + let families: Vec> = entry + .getattr("dynamic_columns")? + .try_iter()? + .map(|family| { + let family: Bound<'py, PyAny> = family?; + PyTuple::new( + py, + [ + family.getattr("name")?.call_method0("casefold")?, + family.getattr("name")?, + family.getattr("pivot_column")?, + family.getattr("value_column")?, + family.getattr("aggregate")?, + family.getattr("type")?, + family.getattr("name_pattern")?, + ], + ) + }) + .collect::>()?; + Ok(PyTuple::new( + py, + [ + entry.getattr("audits")?.len()?.into_bound_py_any(py)?, + PyList::new(py, columns)?.into_any(), + PyList::new(py, families)?.into_any(), + ( + entry.getattr("type_enforcement")?.is_truthy()?, + !entry.getattr("model_schema")?.is_none(), + ) + .into_bound_py_any(py)?, + ], + )? + .into_any()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + for function in [ + wrap_pyfunction!(compiled_model_record, module)?, + wrap_pyfunction!(compiled_audit_record, module)?, + wrap_pyfunction!(compiled_function_record, module)?, + wrap_pyfunction!(compiled_source_record, module)?, + wrap_pyfunction!(compiled_seed_record, module)?, + wrap_pyfunction!(compiled_project_record, module)?, + wrap_pyfunction!(with_binding_diagnostics, module)?, + wrap_pyfunction!(with_type_recovery, module)?, + wrap_pyfunction!(project_with, module)?, + wrap_pyfunction!(inputs_with, module)?, + wrap_pyfunction!(compile_inputs_view, module)?, + wrap_pyfunction!(plain_typed_value, module)?, + wrap_pyfunction!(declaration_rows, module)?, + wrap_pyfunction!(record_model_analyses, module)?, + wrap_pyfunction!(compiled_project_facts, module)?, + ] { + module.add_function(function)?; + } + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs index 4b36f9b0eb..93b6bc7006 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/registration.rs @@ -2,5 +2,7 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::compiled_project::project::register(module)?; - crate::bindings::_helpers::compiled_project::model_loop::register(module) + crate::bindings::_helpers::compiled_project::model_loop::register(module)?; + crate::bindings::_helpers::compiled_project::attachment::register(module)?; + crate::bindings::_helpers::compiled_project::records::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs index c7f9412a8a..4fa99fa92f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs @@ -32,7 +32,7 @@ const CURSOR_INPUTS_KEY: &str = "cursor_inputs"; const PYTHON_UDF_DECORATOR: &str = "udf"; /// A loader's description, relative path and whether that path is a YAML file. -type LoaderRow = (Option, String, bool); +pub(crate) type LoaderRow = (Option, String, bool); /// A Python resource's kind, name, description (a `str` that may hold surrogates), location, help. type PythonResourceRow = (String, String, Option>, Py, String); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 8e36750bba..cb760a85cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod analysis_session; pub(crate) mod attachments; pub(crate) mod boundary; pub(crate) mod cache; +pub(crate) mod compile_attachment; pub(crate) mod compile_driver; pub(crate) mod compile_outputs; pub(crate) mod compiled_project; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 86bb63b8e9..b1e9de0921 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -4,10 +4,10 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ - analysis, analysis_session, attachments, boundary, cache, compile_driver, compile_outputs, - compiled_project, contracts, dag_artifact, discovery, fingerprints, graph, lineage, manifest, - model_config, project_assembly, refactor, render, rules, scopes, semantic_checks, - sql_test_glue, sqltext, type_system, + analysis, analysis_session, attachments, boundary, cache, compile_attachment, compile_driver, + compile_outputs, compiled_project, contracts, dag_artifact, discovery, fingerprints, graph, + lineage, manifest, model_config, project_assembly, refactor, render, rules, scopes, + semantic_checks, sql_test_glue, sqltext, type_system, }; use crate::bindings::models; @@ -58,6 +58,7 @@ fn register_analysis_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { compile_outputs::registration::register(module)?; graph::registration::register(module)?; compiled_project::registration::register(module)?; + compile_attachment::registration::register(module)?; compile_driver::registration::register(module)?; dag_artifact::registration::register(module)?; manifest::registration::register(module)?; diff --git a/fensu.toml b/fensu.toml index a4d7b2e642..998041d176 100644 --- a/fensu.toml +++ b/fensu.toml @@ -23,11 +23,6 @@ exclude = ["tests/e2e/fixtures/**"] [targets.python.thresholds] max_positional_args = 1 -[[targets.python.rule_exceptions]] -rule = "FFS120" -path = "src/sqlbuild/compiler/compiled_project/main/compiled_project_record.py" -reason = "Preserve the existing positional project argument while adding an optional keyword-only projection flag for custom-rule hosts." - [[targets.python.rule_exceptions]] rule = "FFR001" path = "src/sqlbuild/cli/commands/models.py" diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 208b7c1233..42d2504d8b 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -661,7 +661,6 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: ANALYSIS_CODE_SCAN_ROOTS: tuple[str, ...] = ( "compile/_helpers/assembly", "compile/_helpers/analysis", - "compile/_helpers/attachment/model_config.py", "compile/_helpers/diagnostics", "contracts", "pipeline/_helpers/target_validation.py", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 24e6fa5e0d..521dc996e7 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -15,6 +15,7 @@ from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt from sqlbuild.cli.entry.models import CompileCommandRequest, CompileProfileFlags from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity +from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompileAnalysisSelection, @@ -22,7 +23,9 @@ from sqlbuild.compiler.compile.models import ( CompiledAudit, CompiledFunction, CompiledLineageColumnFact, + CompiledModel, CompiledObjectKey, + CompiledProject, CompiledRelationLocation, CompiledSeed, CompiledSource, @@ -45,16 +48,29 @@ from sqlbuild.compiler.compile.models import ( InferredColumn, LoadedMacro, MacroContext, + ModelInputBuildContext, + NativeModelConfigInputs, + NativeModelConfigSession, PythonSqlReferenceReport, StaticMacroExport, StaticMacroFault, ) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, + AuditInput, + CompiledAuditView, + CompiledFunctionView, CompiledModelView, CompiledProjectView, + CompiledSeedView, + CompiledSourceView, CompileInputsView, + FunctionInput, FunctionLanguage, + ModelInput, + SeedInput, + SourceInput, + TypedSqlValueRenderer, ) from sqlbuild.compiler.discovery.models import ( ConstantDeclaration, @@ -142,6 +158,7 @@ from sqlbuild.spec.contracts.models import ( SourceLocation, TargetConfig, ) +from sqlbuild.sql_values.models import SqlValue BUILD_IDENTITY: str @@ -770,6 +787,112 @@ class NativeCompiledProject: def graph_builds(self) -> int: ... def __eq__(self, other: object) -> bool: ... +def compiled_model_record(model: CompiledModelView) -> CompiledModel: ... +def compiled_audit_record(audit: CompiledAuditView) -> CompiledAudit: ... +def compiled_function_record(function: CompiledFunctionView) -> CompiledFunction: ... +def compiled_source_record(source: CompiledSourceView) -> CompiledSource: ... +def compiled_seed_record(seed: CompiledSeedView) -> CompiledSeed: ... +def compiled_project_record( + project: CompiledProjectView, *, include_sql_expansions: bool = True +) -> CompiledProject: ... +def with_binding_diagnostics( + *, model: CompiledModelView, diagnostics: tuple[CompilerDiagnostic, ...] +) -> CompiledModelView: ... +def with_type_recovery( + *, + model: CompiledModelView, + diagnostics: tuple[CompilerDiagnostic, ...], + poisoned: frozenset[tuple[str, str]], +) -> CompiledModelView: ... +def project_with(*, project: CompiledProjectView, **changes: Any) -> CompiledProjectView: ... +def inputs_with(*, inputs: CompileInputsView, **changes: Any) -> CompileInputsView: ... +def compile_inputs_view( + *, fields: dict[str, Any], native_project: NativeCompiledProject | None +) -> CompileInputsView: ... +def plain_typed_value(value: SqlValue) -> object: ... +def declaration_rows( + *, enums: Sequence[EnumDeclaration], constants: Sequence[ConstantDeclaration] +) -> tuple[list[tuple[object, ...]], list[tuple[object, ...]]]: ... +def record_model_analyses( + *, project: NativeCompiledProject, models: Sequence[CompiledModelView] +) -> int: ... +def compiled_project_facts(project: CompiledProjectView) -> NativeCompiledProject: ... +def attach_resources( + *, + discovered_inputs: DiscoveredProjectInputs, + native_project: NativeCompiledProject, + settings: tuple[SettingsConfig, TargetConfig | None, str], + no_sql_validation: bool, + python_functions_inherit_default_namespace: bool, +) -> tuple[tuple[SeedInput, ...], tuple[FunctionInput, ...], tuple[SourceInput, ...]]: ... +def attach_checks( + *, + discovered_inputs: DiscoveredProjectInputs, + native_project: NativeCompiledProject, + settings: tuple[SettingsConfig, TypedSqlValueRenderer, SqlLexicalSyntax], + scan_cache: SqlTestScanCache, + source_inputs: tuple[SourceInput, ...], +) -> tuple[ + tuple[CompileSqlTestInput, ...], + tuple[CompileSqlScenarioInput, ...], + tuple[AuditInput, ...], + tuple[CompilerDiagnostic, ...], +]: ... +def build_effective_connection( + *, + project_config: ProjectConfig, + local_config: LocalConfig, + target_config: TargetConfig | None, + effective_vars: dict[str, object], +) -> dict[str, object]: ... +def build_effective_settings( + *, project_config: ProjectConfig, local_config: LocalConfig +) -> SettingsConfig: ... +def build_effective_vars( + *, + project_config: ProjectConfig, + local_config: LocalConfig, + target_config: TargetConfig | None, + cli_vars: dict[str, object], +) -> dict[str, object]: ... +def native_model_config_session( + *, inputs: NativeModelConfigInputs, validator: NativeModelValidator +) -> NativeModelConfigSession: ... +def build_model_inputs( + *, + discovered_inputs: DiscoveredProjectInputs, + context: ModelInputBuildContext, + no_sql_validation: bool = False, + defer_model_sql_validation: bool = False, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, + native_project: NativeCompiledProject, +) -> tuple[ModelInput, ...]: ... +def build_compile_target_context( + *, discovered_inputs: DiscoveredProjectInputs, selected_target: str | None, no_cache: bool +) -> tuple[str | None, TargetConfig | None, Path | None]: ... +def build_effective_target_namespace( + *, + discovered_inputs: DiscoveredProjectInputs, + selected_target: str | None, + connection_database: object | None, + connection_schema: object | None, + default_database: str | None, + default_schema: str | None, +) -> tuple[str | None, TargetConfig | None, str | None, str | None]: ... +def build_declaration_scope( + *, + discovered_inputs: DiscoveredProjectInputs, + loaded_macros: dict[str, LoadedMacro], + sql_lexical_syntax: SqlLexicalSyntax, + compile_cache_dir: Path | None = None, +) -> DeclarationScopeBuild: ... +def rebind_declaration_scope( + *, + scope: DeclarationScopeBuild, + discovered_inputs: DiscoveredProjectInputs, + loaded_macros: dict[str, LoadedMacro], +) -> DeclarationScopeBuild | None: ... + class NativeMacroExpansion: def __init__( self, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py deleted file mode 100644 index 665ff09619..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ /dev/null @@ -1,311 +0,0 @@ -"""Attachment helpers for building pre-semantic compile inputs.""" - -from __future__ import annotations - -from dataclasses import dataclass, fields -from pathlib import Path -from typing import Any, cast - -import sqlbuild._native as _native -from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text -from sqlbuild.compiler.compile._helpers.analysis.validation import sql_analysis_opt_out_help -from sqlbuild.compiler.compile._helpers.attachment.model_config import ( - native_model_config_session, -) -from sqlbuild.compiler.compile._helpers.diagnostics.collector import report_compile_diagnostic -from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( - cursor_intrinsic_semantics, -) -from sqlbuild.compiler.compile._helpers.render.declarations import ( - render_constant_declaration, -) -from sqlbuild.compiler.compile._helpers.render.macros import ( - macro_expansion_classes, -) -from sqlbuild.compiler.compile._helpers.render.templating import ( - expand_effective_vars, - expand_template_data, - record_template_reads, -) -from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment -from sqlbuild.compiler.compile.constants import EXPLICIT_REFERENCE_KINDS -from sqlbuild.compiler.compile.exceptions import CompileInputError, MacroArgumentError -from sqlbuild.compiler.compile.models import ( - DeclarationResolutionContext, - LoadedMacro, - ModelInputBuildContext, - ModelValidatorContext, - NativeModelConfigInputs, -) -from sqlbuild.compiler.compile.types import ModelInput -from sqlbuild.compiler.compiled_project.main._model_loop_classes import model_loop_classes -from sqlbuild.compiler.discovery.constants import PROJECT_CONFIG_FILENAME -from sqlbuild.compiler.discovery.models import ( - ConstantDeclaration, - DiscoveredProjectInputs, - DiscoveredSqlModelFile, - EnumDeclaration, - NamedSqlHookEntry, - PythonHookEntry, -) -from sqlbuild.compiler.macro_bridge.main.active_macro_bridge import active_macro_bridge -from sqlbuild.compiler.model_config.main._native_contract_classes import native_contract_classes -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - VisibilityRecord, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, -) -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - SettingsConfig, - TargetConfig, -) - - -@dataclass(frozen=True) -class _VisibleModelDeclarations: - local_enums: dict[str, EnumDeclaration] - local_constants: dict[str, ConstantDeclaration] - enums: dict[str, EnumDeclaration] - constants: dict[str, ConstantDeclaration] - inaccessible_enums: dict[str, DeclarationRecord] - inaccessible_constants: dict[str, DeclarationRecord] - enum_visibility: dict[str, tuple[VisibilityRecord, ...]] - constant_visibility: dict[str, tuple[VisibilityRecord, ...]] - macros: dict[str, LoadedMacro] - macro_records: dict[str, DeclarationRecord] - inaccessible_macros: dict[str, DeclarationRecord] - - -@dataclass(frozen=True) -class _ModelValidationContext: - effective_settings: SettingsConfig - no_sql_validation: bool - project_config_path: Path - defer_model_sql_validation: bool - external_sql_reference_resolver: ExternalSqlReferenceResolver | None - sql_lexical_syntax: SqlLexicalSyntax - validators: ModelValidatorContext - - -@dataclass(frozen=True) -class _ModelInputLoop: - discovered_inputs: DiscoveredProjectInputs - context: ModelInputBuildContext - validation_context: _ModelValidationContext - - -@dataclass -class _ModelLoopHost: - """The attachment steps the native model loop calls back into, one model index at a time.""" - - loop: _ModelInputLoop - - def constant_text(self, *, declaration: ConstantDeclaration, file_path: Path) -> str: - return render_constant_declaration( - declaration=declaration, - value_renderer=self.loop.context.value_renderer, - collection_rendering=self.loop.context.collection_rendering, - file_path=file_path, - ) - - -def build_model_inputs( - *, - discovered_inputs: DiscoveredProjectInputs, - context: ModelInputBuildContext, - no_sql_validation: bool = False, - defer_model_sql_validation: bool = False, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, - native_project: _native.NativeCompiledProject, -) -> tuple[ModelInput, ...]: - """Attach schema metadata to discovered model files.""" - - return _build_model_inputs( - discovered_inputs=discovered_inputs, - context=context, - no_sql_validation=no_sql_validation, - defer_model_sql_validation=defer_model_sql_validation, - external_sql_reference_resolver=external_sql_reference_resolver, - native_project=native_project, - ) - - -def _build_model_inputs( - *, - discovered_inputs: DiscoveredProjectInputs, - context: ModelInputBuildContext, - no_sql_validation: bool, - defer_model_sql_validation: bool, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None, - native_project: _native.NativeCompiledProject, -) -> tuple[ModelInput, ...]: - effective_vars: dict[str, object] = context.effective_vars - effective_settings: SettingsConfig = context.effective_settings - validator: _native.NativeModelValidator = _native.NativeModelValidator.from_inputs( - discovered_inputs, effective_settings.microbatch_concurrency - ) - if external_sql_reference_resolver is not None: - external_sql_reference_resolver.validate_model_names( - known_model_names=validator.model_names() - ) - validation_context: _ModelValidationContext = _ModelValidationContext( - effective_settings=effective_settings, - no_sql_validation=no_sql_validation, - project_config_path=(discovered_inputs.project_dir or Path()) / PROJECT_CONFIG_FILENAME, - defer_model_sql_validation=defer_model_sql_validation, - external_sql_reference_resolver=external_sql_reference_resolver, - sql_lexical_syntax=context.sql_lexical_syntax, - validators=ModelValidatorContext( - settings=effective_settings, - external_sql_reference_resolver=external_sql_reference_resolver, - native_config=native_model_config_session( - inputs=NativeModelConfigInputs( - project_config=discovered_inputs.project_config, - target_config=context.target_config, - effective_vars=effective_vars, - effective_target_name=context.effective_target_name, - run_id=context.run_id, - microbatch_concurrency=effective_settings.microbatch_concurrency, - ), - validator=validator, - ), - ), - ) - loop: _ModelInputLoop = _ModelInputLoop( - discovered_inputs=discovered_inputs, - context=context, - validation_context=validation_context, - ) - return tuple(_run_native_model_loop(loop=loop, native_project=native_project)) - - -def _run_native_model_loop( - *, loop: _ModelInputLoop, native_project: _native.NativeCompiledProject -) -> list[ModelInput]: - """Run the model loop natively; models it cannot finish yet call back into the attachment.""" - - files: tuple[DiscoveredSqlModelFile, ...] = loop.discovered_inputs.model_files - settings: SettingsConfig = loop.validation_context.effective_settings - native_loop: _native.NativeModelLoop = _native.NativeModelLoop( - ( - loop.validation_context.validators.native_config.validator, - native_contract_classes(), - model_loop_classes( - audit_factories=loop.discovered_inputs.audit_factories, - context=loop.context, - external_resolver=loop.validation_context.external_sql_reference_resolver, - host_inputs={ - "report_diagnostic": report_compile_diagnostic, - "record_template_reads": record_template_reads, - "visible_declarations": _VisibleModelDeclarations, - "config_session": loop.validation_context.validators.native_config, - "syntax_help": sql_analysis_opt_out_help(model_owned=True), - "source_syntax_help": sql_analysis_opt_out_help(model_owned=False), - "macros": { - "loaded_macros": loop.context.loaded_macros, - "macro_context": loop.context.macro_context, - "bridge": active_macro_bridge(), - "scoped": loop.context.declaration_resolver is not None, - "declaration_context": DeclarationResolutionContext, - "macro_argument_error": MacroArgumentError, - "expansion": macro_expansion_classes(), - "generic_syntax": SqlLexicalSyntax().native_mapping, - }, - "sql_hooks": { - "named_sql_hook_entry": NamedSqlHookEntry, - "python_hook_entry": PythonHookEntry, - "hook_files": loop.discovered_inputs.sql_hook_files, - "effective_vars": loop.context.effective_vars, - "environment": UnicodeEnvironment(), - "render_project_var": render_project_var_text, - "sql_resource_ref": SqlResourceRef, - "explicit_kinds": EXPLICIT_REFERENCE_KINDS, - "declaration_identity": DeclarationIdentity, - "sql_hook_kind": DeclarationKind.SQL_HOOK, - }, - "hook_functions": loop.discovered_inputs.hook_functions, - "provider_names": frozenset( - provider.name for provider in loop.discovered_inputs.providers - ), - }, - ), - ), - loop.context.sql_lexical_syntax.native_mapping, - cursor_intrinsic_semantics(), - ( - ( - settings.sql_analysis, - loop.validation_context.no_sql_validation, - loop.validation_context.defer_model_sql_validation, - settings.require_sql_analysis, - ), - str(loop.validation_context.project_config_path), - ), - ) - inputs: list[_native.NativeModelInput] = native_loop.run( - native_project, _ModelLoopHost(loop=loop), list(files) - ) - return list(inputs) - - -def build_effective_connection( - *, - project_config: ProjectConfig, - local_config: LocalConfig, - target_config: TargetConfig | None, - effective_vars: dict[str, object], -) -> dict[str, object]: - """Merge base project connection with the selected environment overrides.""" - - connection, error = _native.effective_config_connection( - project_config, local_config, target_config - ) - if connection is None: - raise CompileInputError(error or "") - return cast( - dict[str, object], - expand_template_data( - value=connection, - variables=effective_vars, - context_values={}, - context_label="effective connection", - allow_context=False, - preserve_context_tokens=False, - preserve_unknown_context=False, - ), - ) - - -def build_effective_settings( - *, project_config: ProjectConfig, local_config: LocalConfig -) -> SettingsConfig: - """Merge project settings with local developer overrides.""" - - values: dict[str, object] = _native.effective_config_settings( - (project_config.settings, local_config.settings), - local_config.setting_overrides, - [field.name for field in fields(SettingsConfig)], - ) - return SettingsConfig(**cast(dict[str, Any], values)) - - -def build_effective_vars( - *, - project_config: ProjectConfig, - local_config: LocalConfig, - target_config: TargetConfig | None, - cli_vars: dict[str, object], -) -> dict[str, object]: - """Merge effective vars using the locked precedence order.""" - - return expand_effective_vars( - _native.effective_config_vars((project_config, local_config), target_config, cli_vars) - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py deleted file mode 100644 index 6874fa1934..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Build process-local declaration scope state for one compile invocation.""" - -from __future__ import annotations - -import inspect -from collections.abc import Mapping -from pathlib import Path -from types import CodeType - -import sqlbuild._native as _native -from sqlbuild.compiler.compile._helpers.render.declarations import ( - build_declaration_scope_resolver, -) -from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache -from sqlbuild.compiler.compile.constants import SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - DeclarationScopeBuild, - LoadedMacro, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.scopes.exceptions import ScopeValidationError -from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index -from sqlbuild.compiler.scopes.main._validate_scope_index import validate_scope_index -from sqlbuild.compiler.scopes.models import ScopeIndex, ScopeLookup -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax - - -def build_declaration_scope( - *, - discovered_inputs: DiscoveredProjectInputs, - loaded_macros: dict[str, LoadedMacro], - sql_lexical_syntax: SqlLexicalSyntax, - compile_cache_dir: Path | None = None, -) -> DeclarationScopeBuild: - """Build one canonical index and validate it before SQL expansion.""" - - scan_cache: SqlTestScanCache = SqlTestScanCache(cache_dir=compile_cache_dir) - - _native.validate_authored_sql_test_ctes( - discovered_inputs.test_files, - discovered_inputs.scenario_files, - sql_lexical_syntax.native_mapping, - ) - return _build_native_declaration_scope( - discovered_inputs=discovered_inputs, - loaded_macros=loaded_macros, - sql_lexical_syntax=sql_lexical_syntax, - scan_cache=scan_cache, - ) - - -def _build_native_declaration_scope( - *, - discovered_inputs: DiscoveredProjectInputs, - loaded_macros: dict[str, LoadedMacro], - sql_lexical_syntax: SqlLexicalSyntax, - scan_cache: SqlTestScanCache, -) -> DeclarationScopeBuild: - """Build and validate the scope index natively, with its relationship grants and lookup.""" - - native: _native.NativeScopeIndex = open_native_scope_index( - discovered_inputs=discovered_inputs, loaded_macros=loaded_macros - ) - try: - validate_scope_index(index=native.index) - except ScopeValidationError as error: - raise CompileInputError(str(error)) from error - if native.has_scoped_relationship_declarations and ( - discovered_inputs.test_files or discovered_inputs.scenario_files - ): - faults: tuple[tuple[Path, str], ...] = native.grant_relationships( - discovered_inputs=discovered_inputs, - syntax=sql_lexical_syntax, - scan_store=(scan_cache.opened(), SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM), - ) - if faults: - raise CompileInputError(faults[0][1]) - index: ScopeIndex = native.index_with_relationships() - lookup: ScopeLookup = _native.build_scope_lookup(index) - return DeclarationScopeBuild( - loaded_macros=loaded_macros, - index=index, - resolver=build_declaration_scope_resolver( - discovered_inputs=discovered_inputs, - scope_index=index, - loaded_macros=loaded_macros, - lookup=lookup, - ), - sql_test_scans=scan_cache, - ) - - -def rebind_declaration_scope( - *, - scope: DeclarationScopeBuild, - discovered_inputs: DiscoveredProjectInputs, - loaded_macros: dict[str, LoadedMacro], -) -> DeclarationScopeBuild | None: - """Pair a built index with private macro instances, or return None if their metadata differs.""" - - if _indexed_macro_metadata(loaded_macros) != _indexed_macro_metadata(scope.loaded_macros): - return None - return DeclarationScopeBuild( - loaded_macros=loaded_macros, - index=scope.index, - resolver=build_declaration_scope_resolver( - discovered_inputs=discovered_inputs, - scope_index=scope.index, - loaded_macros=loaded_macros, - lookup=scope.resolver.lookup, - ), - sql_test_scans=scope.sql_test_scans, - ) - - -def _indexed_macro_metadata(loaded_macros: Mapping[str, LoadedMacro]) -> tuple[object, ...]: - return tuple( - ( - key, - macro.name, - macro.relative_path, - macro.raw_source, - macro.dependencies, - code.co_firstlineno - if isinstance(code := getattr(macro.function, "__code__", None), CodeType) - else None, - tuple(inspect.signature(macro.function).parameters), - ) - for key, macro in loaded_macros.items() - ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py deleted file mode 100644 index f29b2e7697..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Attachment helpers for building pre-semantic compile inputs.""" - -from __future__ import annotations - -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.compile._helpers.config.retention import resolve_time_travel_retention -from sqlbuild.compiler.compile._helpers.config.table_type import resolve_table_type -from sqlbuild.compiler.compile._helpers.render.templating import ( - record_template_reads, -) -from sqlbuild.compiler.compile.classes.unicode_environment import UnicodeEnvironment -from sqlbuild.compiler.compile.models import ( - CompileModelConfig, - ModelValidationRequest, - NativeModelConfigInputs, - NativeModelConfigSession, -) -from sqlbuild.compiler.compiled_project.main._native_config_classes import native_config_classes -from sqlbuild.compiler.discovery.models import ( - DiscoveredSqlModelFile, - NamedSqlHookEntry, - PythonHookEntry, - SqlHookEntry, -) -from sqlbuild.compiler.model_config.main._native_config_error import native_config_error -from sqlbuild.compiler.planner.types import MaterializationType -from sqlbuild.spec.contracts.models import ( - MaterializationDefaultsConfig, - ProjectConfig, - TargetConfig, -) - -_MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) -_TABLE_BACKED_MATERIALIZATIONS: tuple[str, ...] = ( - MaterializationType.TABLE, - MaterializationType.INCREMENTAL, - MaterializationType.SNAPSHOT, -) -type _BuiltNamespace = tuple[str | None, bool, str | None] -type _StorageOverrides = tuple[tuple[int | None, bool] | None, str | None] -type _BuiltConfig = tuple[ - dict[str, object], - tuple[str, ...], - _BuiltNamespace, - _StorageOverrides, - list[tuple[str, str]], -] -_MODEL_DESCRIPTION_KEY: str = "description" -_MODEL_SCHEMA_KEY: str = "model_schema" - - -def native_model_config_session( - *, inputs: NativeModelConfigInputs, validator: _native.NativeModelValidator -) -> NativeModelConfigSession: - """Capture the project inputs every model's native config build and validation read.""" - - project_config: ProjectConfig = inputs.project_config - target: TargetConfig | None = inputs.target_config - materialization_defaults: MaterializationDefaultsConfig = ( - project_config.materialization_defaults or MaterializationDefaultsConfig() - ) - return NativeModelConfigSession( - path_defaults=project_config.path_defaults, - builder=_native.NativeModelConfigBuilder( - ( - project_config.defaults, - project_config.path_defaults, - (SqlHookEntry, NamedSqlHookEntry, PythonHookEntry), - ), - (inputs.effective_vars, UnicodeEnvironment()), - ( - (inputs.effective_target_name, inputs.run_id), - None if target is None else (target.database, target.schema), - ), - ), - validator=validator, - inherited_storage={ - materialized: ( - resolve_time_travel_retention( - materialized=materialized, - model_value=None, - materialization_defaults=materialization_defaults, - target_config=target, - model_name="", - ), - resolve_table_type( - materialized=materialized, - model_value=None, - materialization_defaults=materialization_defaults, - target_config=target, - model_name="", - ), - ) - for materialized in (None, *_TABLE_BACKED_MATERIALIZATIONS) - }, - ) - - -def build_native_model_config( - *, - session: NativeModelConfigSession, - model_file: DiscoveredSqlModelFile, - matched_path_default: str | None, -) -> CompileModelConfig: - """Build a model's effective config natively, raising its first config error.""" - - built: object = session.builder.model_config( - model_file.header_values, - (matched_path_default, model_file.file_path.stem), - native_config_classes(session=session, record_template_reads=record_template_reads), - ) - if isinstance(built, _native.NativeConfigError): - raise native_config_error(error=built) - return cast(CompileModelConfig, built) - - -def native_model_validation( - *, session: NativeModelConfigSession, request: ModelValidationRequest, rejected: list[bool] -) -> int | _native.NativeConfigError | None: - """Validate natively: None, the first error, or the index of a rejected dbt reference.""" - - config: CompileModelConfig = request.config - return session.validator.validate( - config.values, - ( - request.model_file.file_path.stem, - request.query_sql, - str(request.model_file.relative_path), - ), - ( - [ - (str(reference.ref_kind), reference.ref_name, reference_rejected) - for reference, reference_rejected in zip(request.references, rejected, strict=True) - ], - ( - None - if request.declared_columns is None - else [column.name for column in request.declared_columns] - ), - config.time_travel_retention.unmanaged, - config.table_type.declared, - ), - ) - - -def native_validation_error( - *, error: _native.NativeConfigError, values: dict[str, object] -) -> Exception: - """Return a native validator error, unmarked because validators read macro-expanded SQL.""" - - return native_config_error(error=error, values=values) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/target.py b/src/sqlbuild/compiler/compile/_helpers/attachment/target.py deleted file mode 100644 index 99b2964754..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/target.py +++ /dev/null @@ -1,141 +0,0 @@ -"""Effective target and compile-cache resolution helpers.""" - -from __future__ import annotations - -import os -import shutil -from pathlib import Path - -from sqlbuild.compiler.compile._helpers.attachment.core import build_effective_vars -from sqlbuild.compiler.compile._helpers.render.context_templates import ( - resolve_early_model_templates, - resolve_run_id, -) -from sqlbuild.compiler.compile.constants import ( - COMPILE_CACHE_DISABLE_ENV_VAR, - COMPILE_CACHE_DISABLE_VALUE, - RETIRED_FACT_CACHE_DIRECTORY_NAME, - RETIRED_REFERENCE_CACHE_DIRECTORY_NAME, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.frontier.constants import ( - COMPILER_CACHE_DIRECTORY_NAME, - RETIRED_CACHE_NAMESPACE_SUFFIXES, -) -from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.spec.contracts.models import TargetConfig - - -def build_compile_target_context( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None, - no_cache: bool, -) -> tuple[str | None, TargetConfig | None, Path | None]: - """Resolve the effective target and shared project-local compile-cache directory.""" - - target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=selected_target, - ) - target_config: TargetConfig | None = ( - resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ) - if target_name is not None - else None - ) - cache_dir: Path | None = compile_cache_root( - project_dir=discovered_inputs.project_dir, - target_config=target_config, - no_cache=no_cache, - ) - return target_name, target_config, cache_dir - - -def compile_cache_root( - *, project_dir: Path | None, target_config: TargetConfig | None, no_cache: bool -) -> Path | None: - """Return the shared compile-cache directory, or None when caching is disabled.""" - - if ( - no_cache - or project_dir is None - or (target_config is not None and target_config.compile_cache is False) - or os.environ.get(COMPILE_CACHE_DISABLE_ENV_VAR) == COMPILE_CACHE_DISABLE_VALUE - ): - return None - root: Path = compiler_cache_directory(project_dir) - _remove_retired_caches(root=root) - return root - - -def _remove_retired_caches(*, root: Path) -> None: - """Delete the fact and reference caches older releases kept under each retired cache root.""" - - for suffix in RETIRED_CACHE_NAMESPACE_SUFFIXES: - for name in (RETIRED_FACT_CACHE_DIRECTORY_NAME, RETIRED_REFERENCE_CACHE_DIRECTORY_NAME): - retired: Path = root.parent / f"{COMPILER_CACHE_DIRECTORY_NAME}{suffix}" / name - if retired.is_dir(): - shutil.rmtree(retired, ignore_errors=True) - - -def build_effective_target_namespace( - *, - discovered_inputs: DiscoveredProjectInputs, - selected_target: str | None, - connection_database: object | None, - connection_schema: object | None, - default_database: str | None, - default_schema: str | None, -) -> tuple[str | None, TargetConfig | None, str | None, str | None]: - """Resolve the target and its effective database/schema without compiling resources.""" - - target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=selected_target, - ) - target_config: TargetConfig | None = ( - resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ) - if target_name is not None - else None - ) - effective_vars: dict[str, object] = build_effective_vars( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_config=target_config, - cli_vars={}, - ) - target_values: dict[str, object] = resolve_early_model_templates( - values={ - "database": (target_config.database if target_config is not None else None) - or connection_database - or default_database, - "schema": (target_config.schema if target_config is not None else None) - or connection_schema - or default_schema, - }, - effective_vars=effective_vars, - effective_target_name=target_name, - run_id=resolve_run_id(selected_run_id=None), - ) - return ( - target_name, - target_config, - _optional_string(target_values.get("database")), - _optional_string(target_values.get("schema")), - ) - - -def _optional_string(value: object | None) -> str | None: - return value if isinstance(value, str) and value else None diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/resource_rows.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py similarity index 85% rename from src/sqlbuild/compiler/compiled_project/_helpers/resource_rows.py rename to src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py index d596ecbb38..18f136d0d9 100644 --- a/src/sqlbuild/compiler/compiled_project/_helpers/resource_rows.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/python_resources.py @@ -1,4 +1,4 @@ -"""Rows the native resource diagnostics read: loaders and user Python resources.""" +"""User Python resources as the rows the native missing-description diagnostics read.""" from __future__ import annotations @@ -7,27 +7,12 @@ from functools import partial from pathlib import Path -from sqlbuild.compiler.compile.types import SourceInput -from sqlbuild.compiler.compiled_project.types import PythonResourceRow -from sqlbuild.compiler.discovery.constants import YAML_FILE_SUFFIXES +from sqlbuild.compiler.compile.types import PythonResourceRow, SourceInput from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredProvider from sqlbuild.errors.setting_help.main.snippet_help import snippet_help from sqlbuild.spec.contracts.models import SourceLocation -def loader_rows(discovered_inputs: DiscoveredProjectInputs) -> dict[str, tuple[object, ...]]: - """Each loader's description, relative path and whether that path is a YAML file.""" - - return { - loader.name: ( - loader.description, - str(loader.relative_path), - loader.relative_path.suffix in YAML_FILE_SUFFIXES, - ) - for loader in discovered_inputs.loader_functions - } - - def python_resource_rows( *, discovered_inputs: DiscoveredProjectInputs, source_inputs: tuple[SourceInput, ...] ) -> list[PythonResourceRow]: diff --git a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py index 1901cba38e..48af17b0f7 100644 --- a/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py +++ b/src/sqlbuild/compiler/compile/_helpers/macro_bridge/call_store.py @@ -4,7 +4,9 @@ from pathlib import Path -from sqlbuild.compiler.compile._helpers.attachment.target import build_compile_target_context +from sqlbuild._native import ( + build_compile_target_context, +) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.macro_bridge.classes.macro_bridge import MacroBridge from sqlbuild.compiler.macro_bridge.main._claim_first_compile import claim_first_compile diff --git a/src/sqlbuild/compiler/compile/main/effective_config.py b/src/sqlbuild/compiler/compile/main/effective_config.py index 5e48d0741d..5bb306fe43 100644 --- a/src/sqlbuild/compiler/compile/main/effective_config.py +++ b/src/sqlbuild/compiler/compile/main/effective_config.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.compile._helpers.attachment.core import ( +from sqlbuild._native import ( build_effective_connection, build_effective_vars, ) diff --git a/src/sqlbuild/compiler/compile/main/effective_runtime.py b/src/sqlbuild/compiler/compile/main/effective_runtime.py index f0a4a4e6db..719c6713cc 100644 --- a/src/sqlbuild/compiler/compile/main/effective_runtime.py +++ b/src/sqlbuild/compiler/compile/main/effective_runtime.py @@ -2,7 +2,7 @@ from __future__ import annotations -from sqlbuild.compiler.compile._helpers.attachment.core import ( +from sqlbuild._native import ( build_effective_vars, ) from sqlbuild.compiler.compile._helpers.render.context_templates import resolve_run_id diff --git a/src/sqlbuild/compiler/compile/main/effective_settings.py b/src/sqlbuild/compiler/compile/main/effective_settings.py index e33f7860a1..dbe8e63a9b 100644 --- a/src/sqlbuild/compiler/compile/main/effective_settings.py +++ b/src/sqlbuild/compiler/compile/main/effective_settings.py @@ -2,7 +2,9 @@ from __future__ import annotations -from sqlbuild.compiler.compile._helpers.attachment.core import build_effective_settings +from sqlbuild._native import ( + build_effective_settings, +) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.spec.contracts.models import SettingsConfig diff --git a/src/sqlbuild/compiler/compile/main/effective_target_namespace.py b/src/sqlbuild/compiler/compile/main/effective_target_namespace.py index 2e4f8937fb..87184dda4a 100644 --- a/src/sqlbuild/compiler/compile/main/effective_target_namespace.py +++ b/src/sqlbuild/compiler/compile/main/effective_target_namespace.py @@ -1,6 +1,6 @@ """Public effective target namespace operation.""" -from sqlbuild.compiler.compile._helpers.attachment.target import ( +from sqlbuild._native import ( build_effective_target_namespace as _build_effective_target_namespace, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs diff --git a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py index ebc498b355..d3b65cd047 100644 --- a/src/sqlbuild/compiler/compile/main/sql_expansion_context.py +++ b/src/sqlbuild/compiler/compile/main/sql_expansion_context.py @@ -5,10 +5,10 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild._native import discover_project_inputs -from sqlbuild.compiler.compile._helpers.attachment.core import build_effective_vars -from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import ( +from sqlbuild._native import ( build_declaration_scope, + build_effective_vars, + discover_project_inputs, rebind_declaration_scope, ) from sqlbuild.compiler.compile._helpers.render.declarations import ( diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 53f665a792..59439b520e 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -384,6 +384,23 @@ class DeclarationExpansionResult: usages: tuple[UsageRecord, ...] = field(default_factory=tuple) +@dataclass(frozen=True) +class VisibleModelDeclarations: + """One model's visible declarations and macros, as the native model loop resolves them.""" + + local_enums: dict[str, EnumDeclaration] + local_constants: dict[str, ConstantDeclaration] + enums: dict[str, EnumDeclaration] + constants: dict[str, ConstantDeclaration] + inaccessible_enums: dict[str, DeclarationRecord] + inaccessible_constants: dict[str, DeclarationRecord] + enum_visibility: dict[str, tuple[VisibilityRecord, ...]] + constant_visibility: dict[str, tuple[VisibilityRecord, ...]] + macros: dict[str, LoadedMacro] + macro_records: dict[str, DeclarationRecord] + inaccessible_macros: dict[str, DeclarationRecord] + + @dataclass(frozen=True) class DeclarationResolutionContext: """Declarations visible during one authored SQL expansion.""" diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index 44f5090117..01c2391b5d 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -8,7 +8,7 @@ from sqlbuild.sql_values.models import SqlValue if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Callable, Sequence from pathlib import Path import sqlbuild._native as _native @@ -673,3 +673,4 @@ class SqlTestMode(StrEnum): type SqlReferenceScanFailure = tuple[str, int] +type PythonResourceRow = tuple[str, str, str | None, Callable[[], SourceLocation], str] diff --git a/src/sqlbuild/compiler/compiled_project/__init__.py b/src/sqlbuild/compiler/compiled_project/__init__.py deleted file mode 100644 index 31c048c1f1..0000000000 --- a/src/sqlbuild/compiler/compiled_project/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""The compile's natively owned project and the recorders producers retain facts with.""" diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/__init__.py b/src/sqlbuild/compiler/compiled_project/_helpers/__init__.py deleted file mode 100644 index d6aacb385a..0000000000 --- a/src/sqlbuild/compiler/compiled_project/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compiled-project recording helpers.""" diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py b/src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py deleted file mode 100644 index c652443256..0000000000 --- a/src/sqlbuild/compiler/compiled_project/_helpers/assurance_attachment.py +++ /dev/null @@ -1,212 +0,0 @@ -"""SQL tests, scenarios and audits attached natively by the compile's model loop.""" - -from __future__ import annotations - -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.auditing.main._builtins import builtin_audit_definitions -from sqlbuild.compiler.auditing.types import AuditEvaluationMode -from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache -from sqlbuild.compiler.compile.constants import SQL_TEST_CTE_SCAN_ALGORITHM -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompileDirectLogicSqlTestInputPayload, - CompileModelSqlTestInputPayload, - CompilerDiagnostic, - CompileSqlScenarioCte, - CompileSqlScenarioInput, - CompileSqlTestCte, - CompileSqlTestInput, -) -from sqlbuild.compiler.compile.types import ( - AuditInput, - SourceInput, - SqlTestMode, - TypedSqlValueRenderer, -) -from sqlbuild.compiler.compiled_project._helpers.resource_rows import ( - loader_rows, - python_resource_rows, -) -from sqlbuild.compiler.discovery.models import ( - DiscoveredAuditFile, - DiscoveredProjectInputs, - DiscoveredSqlScenarioFile, - DiscoveredSqlTestBlock, - DiscoveredSqlTestFile, -) -from sqlbuild.compiler.profiling.main.record import record_compile_timing -from sqlbuild.compiler.scopes.models import DeclarationIdentity -from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind, UsageKind -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import SettingsConfig, SourceLocation -from sqlbuild.sql_values.exceptions import SqlValueRenderingError, SqlValueValidationError -from sqlbuild.sql_values.models import SqlValue - - -def attach_sql_tests_impl( - *, - discovered_inputs: DiscoveredProjectInputs, - native_project: _native.NativeCompiledProject, - value_renderer: TypedSqlValueRenderer, - syntax: SqlLexicalSyntax, - scan_cache: SqlTestScanCache, -) -> tuple[tuple[CompileSqlTestInput, ...], tuple[CompileSqlScenarioInput, ...]]: - """Attach every SQL test case, then every scenario, saving the scan store afterwards.""" - - with record_compile_timing("test_input_compile_ms"): - tests, scenarios = native_project.attach_sql_tests( - [_test_file_row(test_file) for test_file in discovered_inputs.test_files], - [_scenario_row(scenario) for scenario in discovered_inputs.scenario_files], - { - "test_input": CompileSqlTestInput, - "model_payload": CompileModelSqlTestInputPayload, - "direct_payload": CompileDirectLogicSqlTestInputPayload, - "test_cte": CompileSqlTestCte, - "scenario_input": CompileSqlScenarioInput, - "scenario_cte": CompileSqlScenarioCte, - "modes": {mode.value: mode for mode in SqlTestMode}, - "test_kind": ResourceKind.TEST, - "scenario_kind": ResourceKind.SCENARIO, - "runtime_usage": UsageKind.RUNTIME, - "render_parameter": _parameter_renderer(value_renderer), - "adapter_name": value_renderer.adapter_name, - "source_location": SourceLocation, - "scan_store": scan_cache.opened(), - "scan_key": (SQL_TEST_CTE_SCAN_ALGORITHM, syntax.cache_key), - }, - ) - scan_cache.save() - return tuple(tests), tuple(scenarios) - - -def attach_audits_impl( - *, - discovered_inputs: DiscoveredProjectInputs, - native_project: _native.NativeCompiledProject, - effective_settings: SettingsConfig, -) -> tuple[tuple[AuditInput, ...], tuple[CompilerDiagnostic, ...]]: - """Attach every singular audit, then each model's, source's and seed's generic audits.""" - - default_severity: object | None = effective_settings.default_audit_severity - audits, diagnostics = native_project.attach_audits( - [_audit_file_row(audit_file) for audit_file in discovered_inputs.audit_files], - { - "default_severity": None if default_severity is None else str(default_severity), - "default_run_scope": effective_settings.default_audit_run_scope, - "builtins": [_audit_file_row(audit_file) for audit_file in builtin_audit_definitions()], - "declaration_identity": DeclarationIdentity, - "singular_kind": DeclarationKind.SINGULAR_AUDIT, - "audit_kind": DeclarationKind.AUDIT, - "resource_kinds": {kind.value: kind for kind in ResourceKind}, - }, - ) - return tuple(audits), tuple(diagnostics) - - -def _parameter_renderer(value_renderer: TypedSqlValueRenderer) -> object: - def render(value: SqlValue) -> tuple[str | None, str | None]: - try: - return value_renderer.render_typed_scalar(value=value), None - except (SqlValueRenderingError, SqlValueValidationError) as error: - return None, str(error) - - return render - - -def _test_file_row(test_file: DiscoveredSqlTestFile) -> tuple[object, ...]: - return ( - str(test_file.file_path), - str(test_file.relative_path), - test_file.relative_path.stem, - test_file, - test_file.contents, - [_test_block_row(block) for block in test_file.blocks], - ) - - -def _test_block_row(block: DiscoveredSqlTestBlock) -> tuple[object, ...]: - return ( - block, - block.name, - block.sql_body, - block.mode.value, - [parameter.name for parameter in block.parameters], - [(case, case.name, list(case.values)) for case in block.cases], - block.sql_body_span, - ) - - -def _scenario_row(scenario: DiscoveredSqlScenarioFile) -> tuple[object, ...]: - return ( - str(scenario.file_path), - str(scenario.relative_path), - scenario, - scenario.contents, - scenario.sql_body, - scenario.name, - ) - - -def _audit_file_row(audit_file: DiscoveredAuditFile) -> tuple[object, ...]: - return ( - str(audit_file.file_path), - str(audit_file.relative_path), - audit_file.file_path.stem, - audit_file, - audit_file.declaration_kind is DeclarationKind.AUDIT, - audit_file.contents, - [ - ( - block, - block.name, - block.sql_body, - block.evidence_sql, - block.evaluation_mode == AuditEvaluationMode.MEASUREMENT, - block.header_values, - ) - for block in audit_file.blocks - ], - ) - - -def resource_diagnostics_impl( - *, - discovered_inputs: DiscoveredProjectInputs, - native_project: _native.NativeCompiledProject, - source_inputs: tuple[SourceInput, ...], -) -> tuple[CompilerDiagnostic, ...]: - """Orphaned audit factory warnings, then missing descriptions, then cursor-input errors.""" - - orphans, descriptions, cursors = native_project.resource_diagnostics( - { - "compiler_diagnostic": CompilerDiagnostic, - "compile_error": CompileInputError, - "source_location": SourceLocation, - "path_class": Path, - "model_files": list(discovered_inputs.model_files), - "audit_factories": [ - (factory.name, str(factory.relative_path), factory.line) - for factory in discovered_inputs.audit_factories - ], - "scenarios": [ - ( - scenario.name, - str(scenario.relative_path), - scenario.contents, - scenario.header_values.get("description"), - ) - for scenario in discovered_inputs.scenario_files - ], - "sql_hooks": [ - (hook.name, str(hook.relative_path), hook.contents, hook.description) - for hook in discovered_inputs.sql_hook_files - ], - "loaders": loader_rows(discovered_inputs), - "python_resources": python_resource_rows( - discovered_inputs=discovered_inputs, source_inputs=source_inputs - ), - } - ) - return (*orphans, *descriptions, *cursors) diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/loop_inputs.py b/src/sqlbuild/compiler/compiled_project/_helpers/loop_inputs.py deleted file mode 100644 index c107089052..0000000000 --- a/src/sqlbuild/compiler/compiled_project/_helpers/loop_inputs.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Project declarations and scope classes the native model loop resolves models with.""" - -import inspect -from collections.abc import Callable -from inspect import Parameter - -from sqlbuild.compiler.compile.models import DeclarationScopeResolver, ModelInputBuildContext -from sqlbuild.compiler.discovery.models import EnumDeclaration -from sqlbuild.compiler.scopes.models import ( - DeclarationIdentity, - DeclarationRecord, - ResourceIdentity, - VisibilityRecord, -) -from sqlbuild.compiler.scopes.types import ( - DeclarationKind, - ResourceKind, - ScopeKind, - VisibilityReason, -) - -_NAMED_KINDS: frozenset[DeclarationKind] = frozenset( - { - DeclarationKind.AUDIT, - DeclarationKind.SCHEMA, - DeclarationKind.SQL_HOOK, - DeclarationKind.PYTHON_HOOK, - } -) - - -def named_records( - resolver: DeclarationScopeResolver | None, -) -> dict[tuple[str, str], DeclarationRecord]: - """Each project-owned generic audit's, schema's and hook's first record, by `(kind, name)`.""" - - if resolver is None: - return {} - records: dict[tuple[str, str], DeclarationRecord] = {} - for identity, declarations in resolver.lookup.declarations.items(): - if identity.kind in _NAMED_KINDS and identity.owner is None and declarations: - records[identity.kind.value, identity.name] = declarations[0] - return records - - -def hook_signature(function: Callable[..., object]) -> list[tuple[str, int, bool]]: - """A hook function's parameters as `inspect.signature` reads them: name, kind, default.""" - - return [ - (parameter.name, int(parameter.kind), parameter.default is not Parameter.empty) - for parameter in inspect.signature(function).parameters.values() - ] - - -def schema_enums(resolver: DeclarationScopeResolver | None) -> dict[str, EnumDeclaration]: - """Project-owned enums by name, as `model_schema_enum_declarations` resolves column types.""" - - if resolver is None: - return {} - enums: dict[str, EnumDeclaration] = {} - for identity, value in resolver.projection.declarations.items(): - if ( - identity.kind is DeclarationKind.ENUM - and identity.owner is None - and isinstance(value, EnumDeclaration) - ): - enums[identity.name] = value - return enums - - -def visibility_classes( - *, context: ModelInputBuildContext, visible_declarations: object -) -> dict[str, object]: - """What the loop resolves each model's visible declarations from.""" - - resolver: DeclarationScopeResolver | None = context.declaration_resolver - return { - "resolver": resolver, - "projects": resolver is not None and bool(resolver.lookup.index.declarations), - "resource_specific": ( - frozenset( - declaration.identity.owner - for declaration in resolver.lookup.index.declarations - if declaration.scope is ScopeKind.PRIVATE and declaration.identity.owner is not None - ) - | frozenset(resolver.lookup.grants_by_resource) - if resolver is not None - else frozenset() - ), - "public_enums": context.public_enums, - "public_constants": context.public_constants, - "visible_declarations": visible_declarations, - "resource_identity": ResourceIdentity, - "declaration_identity": DeclarationIdentity, - "model_kind": ResourceKind.MODEL, - "enum_kind": DeclarationKind.ENUM, - "constant_kind": DeclarationKind.CONSTANT, - "private_owner": VisibilityReason.PRIVATE_OWNER, - "visibility_record": VisibilityRecord, - } diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/model_updates.py b/src/sqlbuild/compiler/compiled_project/_helpers/model_updates.py deleted file mode 100644 index b346300292..0000000000 --- a/src/sqlbuild/compiler/compiled_project/_helpers/model_updates.py +++ /dev/null @@ -1,220 +0,0 @@ -"""Record what later compile stages finish for a model, natively when its owner holds it.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import fields, replace -from typing import Any, cast - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import ( - CompiledAudit, - CompiledFunction, - CompiledModel, - CompiledProject, - CompiledSeed, - CompiledSource, - CompileProjectInputs, - CompilerDiagnostic, - InferredColumn, -) -from sqlbuild.compiler.compile.types import ( - CompiledAuditView, - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledSeedView, - CompiledSourceView, - CompileInputsView, -) - - -def compiled_model_record_impl(model: CompiledModelView) -> CompiledModel: - """The model as a `CompiledModel`, projected from its owner when it is a native view.""" - - if isinstance(model, CompiledModel): - return model - return CompiledModel( - **{field.name: getattr(model, field.name) for field in fields(CompiledModel)} - ) - - -def with_binding_diagnostics_impl( - *, model: CompiledModelView, diagnostics: tuple[CompilerDiagnostic, ...] -) -> CompiledModelView: - """The model with these final binding diagnostics.""" - - if isinstance(model, CompiledModel): - return replace(model, binding_diagnostics=diagnostics) - return cast(_native.NativeCompiledModel, model).with_binding_diagnostics(diagnostics) - - -def with_type_recovery_impl( - *, - model: CompiledModelView, - diagnostics: tuple[CompilerDiagnostic, ...], - poisoned: frozenset[tuple[str, str]], -) -> CompiledModelView: - """The model with its recovered bindings and its poisoned outputs' types forgotten.""" - - if not isinstance(model, CompiledModel): - native: _native.NativeCompiledModel = cast(_native.NativeCompiledModel, model) - return native.with_type_recovery( - diagnostics, [column for name, column in poisoned if name == native.name] - ) - columns: tuple[InferredColumn, ...] | None = None - if model.inferred_columns is not None: - columns = tuple( - replace(column, type=None) if (model.name, column.name) in poisoned else column - for column in model.inferred_columns - ) - unchecked: frozenset[str] = frozenset(column for name, column in poisoned if name == model.name) - return replace( - model, - binding_diagnostics=diagnostics, - inferred_columns=columns, - unchecked_output_columns=unchecked, - ) - - -def with_target_defaults_impl( - *, - model: _native.NativeCompiledModel, - defaults: tuple[str | None, str | None], - render_qualified_name: Callable[..., str | None], -) -> CompiledModelView: - """The model with the adapter's `(schema, database)` defaults filled into its destination.""" - - return model.with_target_defaults(defaults, render_qualified_name) - - -def compiled_project_record_impl( - *, project: CompiledProjectView, include_sql_expansions: bool = True -) -> CompiledProject: - """The project as a `CompiledProject`, projected from its owner when it is a native view.""" - - if isinstance(project, CompiledProject): - return project if include_sql_expansions else replace(project, sql_expansions={}) - fields: dict[str, Any] = cast(_native.NativeProjectView, project).fields(include_sql_expansions) - fields["seeds"] = tuple(map(compiled_seed_record_impl, fields["seeds"])) - fields["sources"] = tuple(map(compiled_source_record_impl, fields["sources"])) - fields["functions"] = tuple(map(compiled_function_record_impl, fields["functions"])) - fields["audits"] = tuple(map(compiled_audit_record_impl, fields["audits"])) - return CompiledProject(**fields) - - -def compiled_audit_record_impl(audit: CompiledAuditView) -> CompiledAudit: - """The audit as a `CompiledAudit`, projected from its owner when it is a native view.""" - - if isinstance(audit, CompiledAudit): - return audit - return CompiledAudit( - key=audit.key, - scope_deps=audit.scope_deps, - name=audit.name, - definition_name=audit.definition_name, - audit_file=audit.audit_file, - audit_block=audit.audit_block, - sql_body=audit.sql_body, - evaluation_mode=audit.evaluation_mode, - measurement_contract=audit.measurement_contract, - thresholds=audit.thresholds, - minimum_samples=audit.minimum_samples, - measure_sql=audit.measure_sql, - evidence_sql=audit.evidence_sql, - evidence_limit=audit.evidence_limit, - references=audit.references, - attached_target_kind=audit.attached_target_kind, - attached_target_name=audit.attached_target_name, - attached_column_name=audit.attached_column_name, - severity=audit.severity, - run_scope=audit.run_scope, - always_run=audit.always_run, - description=audit.description, - ) - - -def compiled_function_record_impl(function: CompiledFunctionView) -> CompiledFunction: - """The function as a `CompiledFunction`, projected from its owner when it is a native view.""" - - if isinstance(function, CompiledFunction): - return function - return CompiledFunction( - key=function.key, - deps=function.deps, - name=function.name, - relative_path=function.relative_path, - arguments=function.arguments, - returns=function.returns, - body_sql=function.body_sql, - destination=function.destination, - fingerprint_destination=function.fingerprint_destination, - return_columns=function.return_columns, - references=function.references, - language=function.language, - source_file_path=function.source_file_path, - runtime_version=function.runtime_version, - entry_point=function.entry_point, - packages=function.packages, - tags=function.tags, - description=function.description, - ) - - -def compiled_source_record_impl(source: CompiledSourceView) -> CompiledSource: - """The source as a `CompiledSource`, projected from its owner when it is a native view.""" - - if isinstance(source, CompiledSource): - return source - return CompiledSource( - key=source.key, - deps=source.deps, - name=source.name, - source_entry=source.source_entry, - source_file=source.source_file, - ) - - -def compiled_seed_record_impl(seed: CompiledSeedView) -> CompiledSeed: - """The seed as a `CompiledSeed`, projected from its owner when it is a native view.""" - - if isinstance(seed, CompiledSeed): - return seed - return CompiledSeed( - key=seed.key, - deps=seed.deps, - name=seed.name, - seed_file=seed.seed_file, - schema_entry=seed.schema_entry, - schema_file=seed.schema_file, - destination=seed.destination, - external=seed.external, - ) - - -def project_with_impl(*, project: CompiledProjectView, **changes: Any) -> CompiledProjectView: - """The project with `changes`; a native view shares its owner's models.""" - - if isinstance(project, CompiledProject): - return replace(project, **changes) - return cast(_native.NativeProjectView, project).with_changes(**changes) - - -def inputs_with_impl(*, inputs: CompileInputsView, **changes: Any) -> CompileInputsView: - """The compile inputs with `changes`; a native view shares its owner's model inputs.""" - - if isinstance(inputs, CompileProjectInputs): - return replace(inputs, **changes) - return cast(_native.NativeInputsView, inputs).with_changes(**changes) - - -def compile_inputs_view_impl( - *, fields: dict[str, Any], native_project: _native.NativeCompiledProject | None -) -> CompileInputsView: - """The native owner's inputs view when it holds every model input; else a record.""" - - if native_project is not None and all( - isinstance(model, _native.NativeModelInput) for model in fields["model_inputs"] - ): - return native_project.inputs_view(fields) - return CompileProjectInputs(**fields, native_project=native_project) diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/resource_attachment.py b/src/sqlbuild/compiler/compiled_project/_helpers/resource_attachment.py deleted file mode 100644 index 0b27e7e4b1..0000000000 --- a/src/sqlbuild/compiler/compiled_project/_helpers/resource_attachment.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Seeds, functions and sources attached natively and held by the compile's native project.""" - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.types import FunctionInput, SeedInput, SourceInput -from sqlbuild.compiler.discovery.models import ( - DiscoveredProjectInputs, - DiscoveredSchemaFile, - DiscoveredSourceFile, -) -from sqlbuild.compiler.scopes.types import ResourceKind -from sqlbuild.spec.contracts.models import ( - DefaultsConfig, - SchemaSeedEntry, - SettingsConfig, - SourceEntry, - TargetConfig, -) - - -def attach_seeds_impl( - *, - discovered_inputs: DiscoveredProjectInputs, - native_project: _native.NativeCompiledProject, -) -> tuple[SeedInput, ...]: - """Pair every declared seed with its CSV file; raise for a declaration without one.""" - - declarations: list[tuple[SchemaSeedEntry, DiscoveredSchemaFile]] = [] - for schema_file in discovered_inputs.schema_files: - declarations.extend((seed_entry, schema_file) for seed_entry in schema_file.seed_entries) - return tuple( - native_project.attach_seeds( - declarations, list(discovered_inputs.seed_files), CompileInputError - ) - ) - - -def attach_functions_impl( - *, - discovered_inputs: DiscoveredProjectInputs, - native_project: _native.NativeCompiledProject, - settings: tuple[SettingsConfig, TargetConfig | None, str], - no_sql_validation: bool, - python_functions_inherit_default_namespace: bool, -) -> tuple[FunctionInput, ...]: - """Attach and validate every SQL, then Python, function in discovery order.""" - - effective_settings, target_config, adapter_name = settings - defaults: DefaultsConfig = discovered_inputs.project_config.defaults - sql_rows: list[tuple[object, ...]] = [ - ( - str(function_file.file_path), - str(function_file.relative_path), - function_file.file_path.stem, - function_file, - function_file.header_values, - function_file.body_sql, - function_file.contents, - ) - for function_file in discovered_inputs.sql_function_files - ] - python_rows: list[tuple[object, ...]] = [ - ( - str(function_file.file_path), - str(function_file.relative_path), - function_file.file_path.stem, - function_file, - function_file.header_values, - function_file.body_python, - None, - ) - for function_file in discovered_inputs.python_function_files - ] - return tuple( - native_project.attach_functions( - (sql_rows, python_rows), - ( - adapter_name, - effective_settings.sql_analysis and not no_sql_validation, - ( - defaults.function_database - if defaults.function_database is not None - else defaults.database, - defaults.function_schema - if defaults.function_schema is not None - else defaults.schema, - ), - None if target_config is None else (target_config.database, target_config.schema), - python_functions_inherit_default_namespace, - ResourceKind.FUNCTION, - ), - ) - ) - - -def attach_sources_impl( - *, - discovered_inputs: DiscoveredProjectInputs, - native_project: _native.NativeCompiledProject, - effective_settings: SettingsConfig, - no_sql_validation: bool, -) -> tuple[SourceInput, ...]: - """Expand and validate every discovered source in declaration order.""" - - rows: list[tuple[str, DiscoveredSourceFile, SourceEntry]] = [] - for source_file in discovered_inputs.source_files: - rows.extend( - (str(source_file.file_path), source_file, entry) for entry in source_file.source_entries - ) - return tuple( - native_project.attach_sources( - rows, - ( - discovered_inputs.project_config.defaults.contract, - effective_settings.sql_analysis and not no_sql_validation, - ResourceKind.SOURCE, - ), - ) - ) diff --git a/src/sqlbuild/compiler/compiled_project/_helpers/rows.py b/src/sqlbuild/compiler/compiled_project/_helpers/rows.py deleted file mode 100644 index 8e069f74f7..0000000000 --- a/src/sqlbuild/compiler/compiled_project/_helpers/rows.py +++ /dev/null @@ -1,196 +0,0 @@ -"""Rows producers hand the native compiled project when they retain a resource's facts.""" - -from __future__ import annotations - -from dataclasses import dataclass -from decimal import Decimal -from pathlib import Path -from typing import cast - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.models import ( - CompileModelConfig, - CompileSqlReference, - DynamicColumnContractProof, -) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.models import SchemaModelEntry -from sqlbuild.sql_values.models import SqlValue -from sqlbuild.sql_values.types import SqlValueKind - - -@dataclass(frozen=True) -class _ModelDeclarations: - enum_columns: tuple[str, ...] - enums: tuple[EnumDeclaration, ...] - constants: tuple[ConstantDeclaration, ...] - - -def record_model_analyses_impl( - *, project: _native.NativeCompiledProject, models: tuple[CompiledModelView, ...] -) -> int: - """Transfer each assembled model's typed analysis to its retained facts; return how many.""" - - transferred: int = project.record_view_analyses( - [model for model in models if isinstance(model, _native.NativeCompiledModel)] - ) - for model in models: - if isinstance(model, _native.NativeCompiledModel): - continue - proof: DynamicColumnContractProof | None = model.dynamic_column_contract - inferred: list[tuple[str, str | None, str]] | None = model.inferred_rows - columns: list[tuple[str, str | None, str]] = inferred or [] - transferred += project.record_model_analysis( - model.name, - ( - inferred is not None, - [(name, column_type) for name, column_type, _ in columns], - [nullability == InferredNullability.NULLABLE for _, _, nullability in columns], - bool(model.fast_lineage_has_star), - list(model.unchecked_output_columns), - ), - None - if proof is None - else ( - proof.output_proven, - proof.bare_dynamic_pivot, - [(family.name.casefold(), family.inferred_type) for family in proof.families], - proof.failure_reason, - [(family.name, family.inferred_type) for family in proof.families], - ), - ) - return transferred - - -def hand_built_project_impl(project: CompiledProjectView) -> _native.NativeCompiledProject: - """A native project for a `CompiledProject` built without the compile-input stage.""" - - native: _native.NativeCompiledProject = _native.NativeCompiledProject() - for model in project.models: - native.record_model( - _model_row( - identity=( - model.name, - model.relative_path.as_posix(), - model.query_sql, - model.authored_query_sql, - model.authored_sql, - ), - config=model.config, - references=model.references, - schema_entry=model.schema_entry, - relative_path=model.relative_path, - declarations=_ModelDeclarations( - enum_columns=tuple(model.enum_columns), - enums=model.enum_declarations, - constants=model.constant_declarations, - ), - ) - ) - _ = record_model_analyses_impl(project=native, models=project.models) - return native - - -def enum_row(declaration: EnumDeclaration) -> tuple[object, ...]: - """An enum declaration as the native project row.""" - - return ( - declaration.name, - declaration.relative_path.as_posix(), - [(member.name, member.value) for member in declaration.members], - None, - None, - None, - ) - - -def constant_row(declaration: ConstantDeclaration) -> tuple[object, ...]: - """A constant declaration as the native project row, its value made JSON-safe.""" - - return ( - declaration.name, - declaration.relative_path.as_posix(), - [], - typed_value_payload(declaration.value), - declaration.logical_type.display_name, - declaration.render_as.value if declaration.render_as is not None else None, - ) - - -def typed_value_payload(value: SqlValue) -> object: - """A typed SQL value as the plain value native consumers read.""" - - if value.kind == SqlValueKind.DECIMAL: - return str(cast(Decimal, value.value)) - if value.kind in { - SqlValueKind.STRING, - SqlValueKind.INTEGER, - SqlValueKind.BOOLEAN, - SqlValueKind.FLOAT, - SqlValueKind.NULL, - }: - return value.value - if value.kind in {SqlValueKind.LIST, SqlValueKind.SET}: - return [typed_value_payload(item) for item in cast(tuple[SqlValue, ...], value.value)] - return { - key: typed_value_payload(item) - for key, item in cast(tuple[tuple[str, SqlValue], ...], value.value) - } - - -def _model_row( - *, - identity: tuple[str, str, str, str, str], - config: CompileModelConfig, - references: tuple[CompileSqlReference, ...], - schema_entry: SchemaModelEntry | None, - relative_path: Path, - declarations: _ModelDeclarations, -) -> tuple[object, ...]: - return ( - identity, - (config.values, list(config.model_header_keys), config.layer_schema), - [ - (str(reference.ref_kind), reference.ref_name, reference.ref_package) - for reference in references - ], - _schema_row(entry=schema_entry, relative_path=relative_path), - list(declarations.enum_columns), - ( - [enum_row(declaration) for declaration in declarations.enums], - [constant_row(declaration) for declaration in declarations.constants], - ), - ) - - -def _schema_row(*, entry: SchemaModelEntry | None, relative_path: Path) -> object: - if entry is None: - return None - return ( - len(entry.audits), - [ - ( - column.name, - column.type, - column.nullable, - len(column.audits), - column.location is not None and column.location.path != relative_path, - ) - for column in entry.columns - ], - [ - ( - family.name.casefold(), - family.name, - family.pivot_column, - family.value_column, - family.aggregate, - family.type, - family.name_pattern, - ) - for family in entry.dynamic_columns - ], - (bool(entry.type_enforcement), entry.model_schema is not None), - ) diff --git a/src/sqlbuild/compiler/compiled_project/constants.py b/src/sqlbuild/compiler/compiled_project/constants.py deleted file mode 100644 index 5208508531..0000000000 --- a/src/sqlbuild/compiler/compiled_project/constants.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Constants of the native compiled project and its model loop.""" - -from sqlbuild.compiler.planner.types import MaterializationType - -TABLE_BACKED_MATERIALIZATIONS: tuple[MaterializationType, ...] = ( - MaterializationType.TABLE, - MaterializationType.INCREMENTAL, - MaterializationType.SNAPSHOT, -) diff --git a/src/sqlbuild/compiler/compiled_project/main/__init__.py b/src/sqlbuild/compiler/compiled_project/main/__init__.py deleted file mode 100644 index 5de9ede1eb..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compiled-project entrypoints.""" diff --git a/src/sqlbuild/compiler/compiled_project/main/_attach_checks.py b/src/sqlbuild/compiler/compiled_project/main/_attach_checks.py deleted file mode 100644 index 80c703c218..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_attach_checks.py +++ /dev/null @@ -1,61 +0,0 @@ -"""SQL tests, scenarios and audits attached natively and held by the compile's native project.""" - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache -from sqlbuild.compiler.compile.models import ( - CompilerDiagnostic, - CompileSqlScenarioInput, - CompileSqlTestInput, -) -from sqlbuild.compiler.compile.types import AuditInput, SourceInput, TypedSqlValueRenderer -from sqlbuild.compiler.compiled_project._helpers.assurance_attachment import ( - attach_audits_impl, - attach_sql_tests_impl, - resource_diagnostics_impl, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax -from sqlbuild.spec.contracts.models import SettingsConfig - - -def attach_checks( - *, - discovered_inputs: DiscoveredProjectInputs, - native_project: _native.NativeCompiledProject, - settings: tuple[SettingsConfig, TypedSqlValueRenderer, SqlLexicalSyntax], - scan_cache: SqlTestScanCache, - source_inputs: tuple[SourceInput, ...], -) -> tuple[ - tuple[CompileSqlTestInput, ...], - tuple[CompileSqlScenarioInput, ...], - tuple[AuditInput, ...], - tuple[CompilerDiagnostic, ...], -]: - """Attach every SQL test case, scenario and audit, then report every resource diagnostic.""" - - effective_settings, value_renderer, syntax = settings - tests, scenarios = attach_sql_tests_impl( - discovered_inputs=discovered_inputs, - native_project=native_project, - value_renderer=value_renderer, - syntax=syntax, - scan_cache=scan_cache, - ) - audits, diagnostics = attach_audits_impl( - discovered_inputs=discovered_inputs, - native_project=native_project, - effective_settings=effective_settings, - ) - return ( - tests, - scenarios, - audits, - ( - *diagnostics, - *resource_diagnostics_impl( - discovered_inputs=discovered_inputs, - native_project=native_project, - source_inputs=source_inputs, - ), - ), - ) diff --git a/src/sqlbuild/compiler/compiled_project/main/_attach_resources.py b/src/sqlbuild/compiler/compiled_project/main/_attach_resources.py deleted file mode 100644 index 02e9d66513..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_attach_resources.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Seeds, functions and sources attached natively and held by the compile's native project.""" - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import FunctionInput, SeedInput, SourceInput -from sqlbuild.compiler.compiled_project._helpers.resource_attachment import ( - attach_functions_impl, - attach_seeds_impl, - attach_sources_impl, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import SettingsConfig, TargetConfig - - -def attach_resources( - *, - discovered_inputs: DiscoveredProjectInputs, - native_project: _native.NativeCompiledProject, - settings: tuple[SettingsConfig, TargetConfig | None, str], - no_sql_validation: bool, - python_functions_inherit_default_namespace: bool, -) -> tuple[tuple[SeedInput, ...], tuple[FunctionInput, ...], tuple[SourceInput, ...]]: - """Pair every seed, then attach every function and source, in that order.""" - - seeds: tuple[SeedInput, ...] = attach_seeds_impl( - discovered_inputs=discovered_inputs, native_project=native_project - ) - functions: tuple[FunctionInput, ...] = attach_functions_impl( - discovered_inputs=discovered_inputs, - native_project=native_project, - settings=settings, - no_sql_validation=no_sql_validation, - python_functions_inherit_default_namespace=python_functions_inherit_default_namespace, - ) - sources: tuple[SourceInput, ...] = attach_sources_impl( - discovered_inputs=discovered_inputs, - native_project=native_project, - effective_settings=settings[0], - no_sql_validation=no_sql_validation, - ) - return seeds, functions, sources diff --git a/src/sqlbuild/compiler/compiled_project/main/_compile_inputs_view.py b/src/sqlbuild/compiler/compiled_project/main/_compile_inputs_view.py deleted file mode 100644 index 1b3cba941f..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_compile_inputs_view.py +++ /dev/null @@ -1,15 +0,0 @@ -"""A compile's attached inputs, held by its native owner when it finished every model.""" - -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.compiled_project._helpers.model_updates import compile_inputs_view_impl - - -def compile_inputs_view( - *, fields: dict[str, Any], native_project: _native.NativeCompiledProject | None -) -> CompileInputsView: - """The native owner's inputs view when it holds every model input; else a record.""" - - return compile_inputs_view_impl(fields=fields, native_project=native_project) diff --git a/src/sqlbuild/compiler/compiled_project/main/_model_loop_classes.py b/src/sqlbuild/compiler/compiled_project/main/_model_loop_classes.py deleted file mode 100644 index 48c83f1684..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_model_loop_classes.py +++ /dev/null @@ -1,135 +0,0 @@ -"""The Python classes and project inputs the native model loop builds models with.""" - -from dataclasses import replace -from functools import partial -from pathlib import Path -from typing import cast - -from sqlbuild.compiler.auditing.types import AuditSeverity -from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.models import ( - CompactLineageFacts, - CompiledLineageColumnFact, - CompiledLineageSourceFact, - CompiledObjectKey, - CompiledRelationLocation, - CompiledSqlExpansion, - CompilerDiagnostic, - CompileSqlReference, - DeclarationScopeResolver, - DynamicColumnContractProof, - DynamicColumnFamilyProof, - ExpansionSpan, - FunctionArgument, - FunctionReturnColumn, - InferredColumn, - ModelInputBuildContext, - NativeModelConfigSession, -) -from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, FunctionLanguage -from sqlbuild.compiler.compiled_project._helpers.loop_inputs import ( - hook_signature, - named_records, - schema_enums, - visibility_classes, -) -from sqlbuild.compiler.compiled_project.main._native_config_classes import native_config_classes -from sqlbuild.compiler.compiled_project.main.declaration_rows import declaration_rows -from sqlbuild.compiler.discovery.models import ( - DiscoveredAuditFactory, - DiscoveredHookFunction, - PythonHookEntry, - SqlHookEntry, -) -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.model_config.main._native_config_error import native_config_error -from sqlbuild.compiler.planner.types import ContractPolicy -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver, SqlReferenceKind -from sqlbuild.compiler.scopes.exceptions import InvalidScopePathError -from sqlbuild.compiler.scopes.models import UsageRecord -from sqlbuild.compiler.scopes.types import UsageKind -from sqlbuild.spec.contracts.models import SchemaModelEntry, SourceLocation - - -def model_loop_classes( - *, - audit_factories: tuple[DiscoveredAuditFactory, ...], - context: ModelInputBuildContext, - external_resolver: ExternalSqlReferenceResolver | None, - host_inputs: dict[str, object], -) -> dict[str, object]: - """Return the classes, error factories and project declarations the model loop reads.""" - - resolver: DeclarationScopeResolver | None = context.declaration_resolver - hook_functions: tuple[DiscoveredHookFunction, ...] = cast( - tuple[DiscoveredHookFunction, ...], host_inputs["hook_functions"] - ) - provider_names: frozenset[str] = cast(frozenset[str], host_inputs["provider_names"]) - session: NativeModelConfigSession = cast( - NativeModelConfigSession, host_inputs["config_session"] - ) - return { - "usage_record": UsageRecord, - "compile_error": CompileInputError, - "syntax_help": host_inputs["syntax_help"], - "source_syntax_help": host_inputs["source_syntax_help"], - "macros": host_inputs["macros"], - "sql_hooks": host_inputs["sql_hooks"], - "scope_path_error": InvalidScopePathError, - "sql_hook_entry": SqlHookEntry, - "config_error": native_config_error, - "audit_factories": {factory.name: factory for factory in audit_factories}, - "generated_usage": UsageKind.GENERATED, - "contract_enforced": ContractPolicy.ENFORCED, - "model_schemas": context.public_model_schemas, - "named_records": named_records(resolver), - "schema_enums": schema_enums(resolver), - "compiler_diagnostic": CompilerDiagnostic, - "report_diagnostic": host_inputs["report_diagnostic"], - "path_class": Path, - "external_resolver": external_resolver, - "reference_kind": SqlReferenceKind, - "partial": partial, - "config": native_config_classes( - session=session, record_template_reads=host_inputs["record_template_reads"] - ), - "hooks": { - "functions": {function.name: function for function in hook_functions}, - "provider_names": sorted(provider_names), - "python_hook_entry": PythonHookEntry, - "signature": hook_signature, - }, - "visibility": visibility_classes( - context=context, visible_declarations=host_inputs["visible_declarations"] - ), - "record": {"replace": replace, "declaration_rows": declaration_rows}, - "input_view": { - "sql_reference": CompileSqlReference, - "reference_kind": SqlReferenceKind, - "schema_model_entry": SchemaModelEntry, - "sql_expansion": CompiledSqlExpansion, - "expansion_span": ExpansionSpan, - "audits": { - "audit_severity": AuditSeverity, - "attached_target_kind": AttachedAuditTargetKind, - }, - "functions": { - "function_argument": FunctionArgument, - "function_return_column": FunctionReturnColumn, - "function_language": FunctionLanguage, - }, - "compiled": { - "object_key": CompiledObjectKey, - "relation_location": CompiledRelationLocation, - "inferred_column": InferredColumn, - "nullability": InferredNullability, - "compiler_diagnostic": CompilerDiagnostic, - "source_location": SourceLocation, - "compact_lineage_facts": CompactLineageFacts, - "lineage_column_fact": CompiledLineageColumnFact, - "lineage_source_fact": CompiledLineageSourceFact, - "contract_proof": DynamicColumnContractProof, - "family_proof": DynamicColumnFamilyProof, - }, - }, - } diff --git a/src/sqlbuild/compiler/compiled_project/main/_native_config_classes.py b/src/sqlbuild/compiler/compiled_project/main/_native_config_classes.py deleted file mode 100644 index 630cfcc029..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_native_config_classes.py +++ /dev/null @@ -1,27 +0,0 @@ -"""The builder, storage defaults and classes a model's `CompileModelConfig` is made of.""" - -from sqlbuild.compiler.compile.models import CompileModelConfig, NativeModelConfigSession -from sqlbuild.compiler.compiled_project.constants import TABLE_BACKED_MATERIALIZATIONS -from sqlbuild.spec.contracts.models import ResolvedTableType, ResolvedTimeTravelRetention -from sqlbuild.spec.contracts.types import TableType, TableTypeSource, TimeTravelRetentionSource - - -def native_config_classes( - *, session: NativeModelConfigSession, record_template_reads: object -) -> dict[str, object]: - """The builder, storage defaults and classes a model's `CompileModelConfig` is made of.""" - - return { - "builder": session.builder, - "inherited_storage": session.inherited_storage, - "table_backed": tuple( - str(materialization) for materialization in TABLE_BACKED_MATERIALIZATIONS - ), - "compile_model_config": CompileModelConfig, - "resolved_retention": ResolvedTimeTravelRetention, - "retention_source": TimeTravelRetentionSource.MODEL, - "resolved_table_type": ResolvedTableType, - "table_type": TableType, - "table_type_source": TableTypeSource.MODEL, - "record_template_reads": record_template_reads, - } diff --git a/src/sqlbuild/compiler/compiled_project/main/_record_model_analyses.py b/src/sqlbuild/compiler/compiled_project/main/_record_model_analyses.py deleted file mode 100644 index 437f3562ab..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_record_model_analyses.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Transfer assembled models' typed analysis results to the native compiled project.""" - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.compiled_project._helpers.rows import record_model_analyses_impl - - -def record_model_analyses( - *, project: _native.NativeCompiledProject, models: tuple[CompiledModelView, ...] -) -> int: - """Attach each model's inferred columns and dynamic pivot proof to its retained facts.""" - - return record_model_analyses_impl(project=project, models=models) diff --git a/src/sqlbuild/compiler/compiled_project/main/_with_binding_diagnostics.py b/src/sqlbuild/compiler/compiled_project/main/_with_binding_diagnostics.py deleted file mode 100644 index 481393f0a7..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_with_binding_diagnostics.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Record a compiled model's final binding diagnostics.""" - -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.compiled_project._helpers.model_updates import with_binding_diagnostics_impl - - -def with_binding_diagnostics( - *, model: CompiledModelView, diagnostics: tuple[CompilerDiagnostic, ...] -) -> CompiledModelView: - """The model with these final binding diagnostics.""" - - return with_binding_diagnostics_impl(model=model, diagnostics=diagnostics) diff --git a/src/sqlbuild/compiler/compiled_project/main/_with_target_defaults.py b/src/sqlbuild/compiler/compiled_project/main/_with_target_defaults.py deleted file mode 100644 index b449833d34..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_with_target_defaults.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Fill an adapter's default schema and database into a native compiled model's destination.""" - -from collections.abc import Callable - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.compiled_project._helpers.model_updates import with_target_defaults_impl - - -def with_target_defaults( - *, - model: _native.NativeCompiledModel, - defaults: tuple[str | None, str | None], - render_qualified_name: Callable[..., str | None], -) -> CompiledModelView: - """The model with the adapter's `(schema, database)` defaults filled into its destination.""" - - return with_target_defaults_impl( - model=model, defaults=defaults, render_qualified_name=render_qualified_name - ) diff --git a/src/sqlbuild/compiler/compiled_project/main/_with_type_recovery.py b/src/sqlbuild/compiler/compiled_project/main/_with_type_recovery.py deleted file mode 100644 index 860c502cc7..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/_with_type_recovery.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Record a compiled model's semantic type recovery.""" - -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.compiled_project._helpers.model_updates import with_type_recovery_impl - - -def with_type_recovery( - *, - model: CompiledModelView, - diagnostics: tuple[CompilerDiagnostic, ...], - poisoned: frozenset[tuple[str, str]], -) -> CompiledModelView: - """The model with its recovered bindings and its poisoned outputs' types forgotten.""" - - return with_type_recovery_impl(model=model, diagnostics=diagnostics, poisoned=poisoned) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_audit_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_audit_record.py deleted file mode 100644 index bd551710db..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/compiled_audit_record.py +++ /dev/null @@ -1,11 +0,0 @@ -"""A compiled audit as a `CompiledAudit` record, for callers that copy or expose it.""" - -from sqlbuild.compiler.compile.models import CompiledAudit -from sqlbuild.compiler.compile.types import CompiledAuditView -from sqlbuild.compiler.compiled_project._helpers.model_updates import compiled_audit_record_impl - - -def compiled_audit_record(audit: CompiledAuditView) -> CompiledAudit: - """The audit as a `CompiledAudit`, projected from its owner when it is a native view.""" - - return compiled_audit_record_impl(audit) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_function_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_function_record.py deleted file mode 100644 index c988babf6e..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/compiled_function_record.py +++ /dev/null @@ -1,12 +0,0 @@ -"""A compiled function as a `CompiledFunction` record, for callers that copy or expose it.""" - -from sqlbuild.compiler.compile.types import CompiledFunctionView -from sqlbuild.compiler.compiled_project._helpers.model_updates import ( - compiled_function_record_impl, -) - - -def compiled_function_record(function: CompiledFunctionView) -> CompiledFunctionView: - """The function as a `CompiledFunction`, projected from its owner when it is a native view.""" - - return compiled_function_record_impl(function) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_model_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_model_record.py deleted file mode 100644 index bcdc2613bb..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/compiled_model_record.py +++ /dev/null @@ -1,11 +0,0 @@ -"""A compiled model as a `CompiledModel` record, for callers that copy or ship it.""" - -from sqlbuild.compiler.compile.models import CompiledModel -from sqlbuild.compiler.compile.types import CompiledModelView -from sqlbuild.compiler.compiled_project._helpers.model_updates import compiled_model_record_impl - - -def compiled_model_record(model: CompiledModelView) -> CompiledModel: - """The model as a `CompiledModel`, projected from its owner when it is a native view.""" - - return compiled_model_record_impl(model) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_project_facts.py b/src/sqlbuild/compiler/compiled_project/main/compiled_project_facts.py deleted file mode 100644 index 1c1b65fcca..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/compiled_project_facts.py +++ /dev/null @@ -1,13 +0,0 @@ -"""The native compiled project a consumer reads for one `CompiledProject`.""" - -import sqlbuild._native as _native -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project._helpers.rows import hand_built_project_impl - - -def compiled_project_facts(project: CompiledProjectView) -> _native.NativeCompiledProject: - """The compile's retained project, or one recorded from a hand-built project.""" - - if project.native_project is not None: - return project.native_project - return hand_built_project_impl(project) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_project_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_project_record.py deleted file mode 100644 index d698d9c082..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/compiled_project_record.py +++ /dev/null @@ -1,15 +0,0 @@ -"""A compiled project as a `CompiledProject` record, for callers that copy or ship it.""" - -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project._helpers.model_updates import compiled_project_record_impl - - -def compiled_project_record( - project: CompiledProjectView, *, include_sql_expansions: bool = True -) -> CompiledProject: - """The project as a `CompiledProject`, projected from its owner when it is a native view.""" - - return compiled_project_record_impl( - project=project, include_sql_expansions=include_sql_expansions - ) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_seed_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_seed_record.py deleted file mode 100644 index 33f6724975..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/compiled_seed_record.py +++ /dev/null @@ -1,11 +0,0 @@ -"""A compiled seed as a `CompiledSeed` record, for callers that copy or expose it.""" - -from sqlbuild.compiler.compile.models import CompiledSeed -from sqlbuild.compiler.compile.types import CompiledSeedView -from sqlbuild.compiler.compiled_project._helpers.model_updates import compiled_seed_record_impl - - -def compiled_seed_record(seed: CompiledSeedView) -> CompiledSeed: - """The seed as a `CompiledSeed`, projected from its owner when it is a native view.""" - - return compiled_seed_record_impl(seed) diff --git a/src/sqlbuild/compiler/compiled_project/main/compiled_source_record.py b/src/sqlbuild/compiler/compiled_project/main/compiled_source_record.py deleted file mode 100644 index 51c0c910e3..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/compiled_source_record.py +++ /dev/null @@ -1,11 +0,0 @@ -"""A compiled source as a `CompiledSource` record, for callers that copy or expose it.""" - -from sqlbuild.compiler.compile.models import CompiledSource -from sqlbuild.compiler.compile.types import CompiledSourceView -from sqlbuild.compiler.compiled_project._helpers.model_updates import compiled_source_record_impl - - -def compiled_source_record(source: CompiledSourceView) -> CompiledSource: - """The source as a `CompiledSource`, projected from its owner when it is a native view.""" - - return compiled_source_record_impl(source) diff --git a/src/sqlbuild/compiler/compiled_project/main/declaration_rows.py b/src/sqlbuild/compiler/compiled_project/main/declaration_rows.py deleted file mode 100644 index d1c346b4a1..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/declaration_rows.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Declaration rows native consumers read.""" - -from sqlbuild.compiler.compiled_project._helpers.rows import constant_row, enum_row -from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration - - -def declaration_rows( - *, enums: tuple[EnumDeclaration, ...], constants: tuple[ConstantDeclaration, ...] -) -> tuple[list[tuple[object, ...]], list[tuple[object, ...]]]: - """Return enum and constant declarations as native rows, in the given order.""" - - return [enum_row(declaration) for declaration in enums], [ - constant_row(declaration) for declaration in constants - ] diff --git a/src/sqlbuild/compiler/compiled_project/main/inputs_with.py b/src/sqlbuild/compiler/compiled_project/main/inputs_with.py deleted file mode 100644 index f05e7123af..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/inputs_with.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Compile inputs with some fields changed, natively when their owner holds them.""" - -from typing import Any - -from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.compiled_project._helpers.model_updates import inputs_with_impl - - -def inputs_with(*, inputs: CompileInputsView, **changes: Any) -> CompileInputsView: - """The compile inputs with `changes`; a native view shares its owner's model inputs.""" - - return inputs_with_impl(inputs=inputs, **changes) diff --git a/src/sqlbuild/compiler/compiled_project/main/plain_typed_value.py b/src/sqlbuild/compiler/compiled_project/main/plain_typed_value.py deleted file mode 100644 index 2f3fe093dc..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/plain_typed_value.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Typed SQL values as the plain values native consumers read.""" - -from sqlbuild.compiler.compiled_project._helpers.rows import typed_value_payload -from sqlbuild.sql_values.models import SqlValue - - -def plain_typed_value(value: SqlValue) -> object: - """Return the value JSON-safe: decimals as text, collections and mappings recursively.""" - - return typed_value_payload(value) diff --git a/src/sqlbuild/compiler/compiled_project/main/project_with.py b/src/sqlbuild/compiler/compiled_project/main/project_with.py deleted file mode 100644 index bb011f16cc..0000000000 --- a/src/sqlbuild/compiler/compiled_project/main/project_with.py +++ /dev/null @@ -1,12 +0,0 @@ -"""A compiled project with some fields changed, natively when its owner holds it.""" - -from typing import Any - -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project._helpers.model_updates import project_with_impl - - -def project_with(*, project: CompiledProjectView, **changes: Any) -> CompiledProjectView: - """The project with `changes`; a native view shares its owner's models.""" - - return project_with_impl(project=project, **changes) diff --git a/src/sqlbuild/compiler/compiled_project/types.py b/src/sqlbuild/compiler/compiled_project/types.py deleted file mode 100644 index 68f523dc8a..0000000000 --- a/src/sqlbuild/compiler/compiled_project/types.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Row shapes the native model loop hands its Python host.""" - -from collections.abc import Callable - -from sqlbuild.spec.contracts.models import SourceLocation - -type ReferenceRow = tuple[str, str, str | None, int | None] -type SpanRow = tuple[int, int, int, int] -type PythonResourceRow = tuple[str, str, str | None, Callable[[], SourceLocation], str] diff --git a/src/sqlbuild/compiler/contracts/_helpers/native_contracts.py b/src/sqlbuild/compiler/contracts/_helpers/native_contracts.py index 46aa691589..f157e9fd84 100644 --- a/src/sqlbuild/compiler/contracts/_helpers/native_contracts.py +++ b/src/sqlbuild/compiler/contracts/_helpers/native_contracts.py @@ -3,6 +3,9 @@ from __future__ import annotations import sqlbuild._native as _native +from sqlbuild._native import ( + compiled_project_facts, +) from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.compiler.compile.models import ( CompilerDiagnostic, @@ -15,7 +18,6 @@ DiagnosticPhase, DiagnosticSeverity, ) -from sqlbuild.compiler.compiled_project.main.compiled_project_facts import compiled_project_facts from sqlbuild.compiler.contracts._helpers.output_locations import ( output_column_location, output_related_locations, diff --git a/src/sqlbuild/compiler/contracts/_helpers/native_promotion.py b/src/sqlbuild/compiler/contracts/_helpers/native_promotion.py index 45cbab049f..c077905736 100644 --- a/src/sqlbuild/compiler/contracts/_helpers/native_promotion.py +++ b/src/sqlbuild/compiler/contracts/_helpers/native_promotion.py @@ -5,6 +5,9 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild._native import ( + compiled_project_facts, +) from sqlbuild.adapter.contract.types import TablePromotionMode from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import ( @@ -13,7 +16,6 @@ DiagnosticPhase, DiagnosticSeverity, ) -from sqlbuild.compiler.compiled_project.main.compiled_project_facts import compiled_project_facts def native_promotion_conflicts( diff --git a/src/sqlbuild/compiler/pipeline/_helpers/target_defaults.py b/src/sqlbuild/compiler/pipeline/_helpers/target_defaults.py index a4d9953c01..1c3f4505a5 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/target_defaults.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/target_defaults.py @@ -7,6 +7,9 @@ from typing import cast import sqlbuild._native as _native +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledRelationLocation, @@ -18,8 +21,6 @@ CompiledProjectView, CompiledSeedView, ) -from sqlbuild.compiler.compiled_project.main._with_target_defaults import with_target_defaults -from sqlbuild.compiler.compiled_project.main.project_with import project_with def apply_target_defaults( @@ -76,10 +77,8 @@ def _model_with_defaults( render_qualified_name: Callable[..., str | None], ) -> CompiledModelView: if not isinstance(model, CompiledModel): - return with_target_defaults( - model=cast(_native.NativeCompiledModel, model), - defaults=(default_schema, default_database), - render_qualified_name=render_qualified_name, + return cast(_native.NativeCompiledModel, model).with_target_defaults( + (default_schema, default_database), render_qualified_name ) return replace( model, diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py index b5d053ff31..4f7b8e4e5a 100644 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py +++ b/src/sqlbuild/compiler/planner/_helpers/scenario/relations.py @@ -6,6 +6,9 @@ from collections.abc import Callable from dataclasses import replace +from sqlbuild._native import ( + compiled_model_record, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.compiler.compile.constants import ( @@ -28,7 +31,6 @@ CompiledResourceType, CompiledSeedView, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.planner._helpers.fixtures.completion import ( build_relation_fixture_completion, ) diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py b/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py index f623cf26ea..d4bc4e255a 100644 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py @@ -6,6 +6,10 @@ from typing import Any import sqlbuild._native as _native +from sqlbuild._native import ( + project_with, + with_binding_diagnostics, +) from sqlbuild.compiler.compile.classes.semantic_completion_inputs import SemanticCompletionInputs from sqlbuild.compiler.compile.constants import UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE from sqlbuild.compiler.compile.models import CompilerDiagnostic @@ -16,10 +20,6 @@ DiagnosticPhase, DiagnosticSeverity, ) -from sqlbuild.compiler.compiled_project.main._with_binding_diagnostics import ( - with_binding_diagnostics, -) -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.semantic_checks._helpers.payloads import diagnostic_ids, lineage_payload from sqlbuild.spec.contracts.models import SourceLocation diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py b/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py index d0b498e3c9..69b50053a0 100644 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py @@ -5,11 +5,13 @@ from typing import Any import sqlbuild._native as _native +from sqlbuild._native import ( + project_with, +) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.types import DiagnosticRow from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.semantic_checks._helpers.completion import complete_native_diagnostics from sqlbuild.compiler.semantic_checks._helpers.metadata import native_metadata_diagnostics from sqlbuild.compiler.semantic_checks._helpers.payloads import session_fact_models diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py b/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py index 582011908b..32440c2c31 100644 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py +++ b/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py @@ -6,6 +6,10 @@ from typing import Any import sqlbuild._native as _native +from sqlbuild._native import ( + project_with, + with_type_recovery, +) from sqlbuild.compiler.analysis_session.types import DiagnosticRow from sqlbuild.compiler.compile.classes.semantic_completion_inputs import SemanticCompletionInputs from sqlbuild.compiler.compile.models import ( @@ -16,8 +20,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.compiled_project.main._with_type_recovery import with_type_recovery -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.semantic_checks._helpers.payloads import ( diagnostic_ids, has_blocking_binding, diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py index 2299101a90..e6805a87d2 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -13,9 +13,11 @@ import orjson +from sqlbuild._native import ( + compiled_project_record, +) from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_project_record import compiled_project_record from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( diff --git a/src/sqlbuild/rule_engine/_helpers/host/custom_host_pool.py b/src/sqlbuild/rule_engine/_helpers/host/custom_host_pool.py index 654e0851cc..3dbf3ea5d2 100644 --- a/src/sqlbuild/rule_engine/_helpers/host/custom_host_pool.py +++ b/src/sqlbuild/rule_engine/_helpers/host/custom_host_pool.py @@ -17,9 +17,11 @@ import orjson import sqlbuild._native as _native +from sqlbuild._native import ( + compiled_project_record, +) from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_project_record import compiled_project_record from sqlbuild.rule_engine._helpers.host.host_capacity import available_cores from sqlbuild.rule_engine.constants import ( CUSTOM_HOST_ABANDONED_SUFFIX, diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py index ae4eaef3ea..5f67c8da7e 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/native_rows.py +++ b/src/sqlbuild/rule_engine/_helpers/run/native_rows.py @@ -9,6 +9,11 @@ import orjson import sqlbuild._native as _native +from sqlbuild._native import ( + compiled_project_facts, + declaration_rows, + plain_typed_value, +) from sqlbuild.adapter.type_system.main.types_equal import types_equal from sqlbuild.compiler.compile.models import ( CompiledModelSqlTestPayload, @@ -17,9 +22,6 @@ CompileSqlTestCte, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_project_facts import compiled_project_facts -from sqlbuild.compiler.compiled_project.main.declaration_rows import declaration_rows -from sqlbuild.compiler.compiled_project.main.plain_typed_value import plain_typed_value from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.compiler.scopes.main.scope_metadata import scope_metadata_projection from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index 8d046b6218..9f3710fcd6 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -14,6 +14,9 @@ from typing import Any import sqlbuild._native as _native +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.constants import MACRO_TOKEN from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -28,7 +31,6 @@ CompileInputsView, ModelInput, ) -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.lint.classes.stop_context import LintStopContext diff --git a/src/sqlbuild/rule_engine/classes/audit_facts.py b/src/sqlbuild/rule_engine/classes/audit_facts.py index 8a9c0ba1cc..361b2dce77 100644 --- a/src/sqlbuild/rule_engine/classes/audit_facts.py +++ b/src/sqlbuild/rule_engine/classes/audit_facts.py @@ -2,9 +2,11 @@ from functools import cached_property +from sqlbuild._native import ( + compiled_audit_record, +) from sqlbuild.compiler.compile.models import CompiledAudit from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_audit_record import compiled_audit_record from sqlbuild.rule_engine.models import Model diff --git a/src/sqlbuild/rule_engine/classes/project_facts.py b/src/sqlbuild/rule_engine/classes/project_facts.py index 595de795c9..b07de8a219 100644 --- a/src/sqlbuild/rule_engine/classes/project_facts.py +++ b/src/sqlbuild/rule_engine/classes/project_facts.py @@ -2,13 +2,13 @@ from pathlib import Path -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_audit_record import compiled_audit_record -from sqlbuild.compiler.compiled_project.main.compiled_function_record import ( +from sqlbuild._native import ( + compiled_audit_record, compiled_function_record, + compiled_seed_record, + compiled_source_record, ) -from sqlbuild.compiler.compiled_project.main.compiled_seed_record import compiled_seed_record -from sqlbuild.compiler.compiled_project.main.compiled_source_record import compiled_source_record +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine.classes.project_tree import ProjectTree, public_model from sqlbuild.rule_engine.models import Model diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 2bfb3c12c1..3d2284cab8 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -12,7 +12,6 @@ from _pytest.capture import CaptureResult import sqlbuild._native as native_module -import sqlbuild.compiler.compile._helpers.attachment.declaration_scope as declaration_scope_module import sqlbuild.compiler.compile._helpers.render.macros as macros_module import sqlbuild.compiler.compile.main.sql_expansion_context as expansion_context_module import sqlbuild.rule_engine._helpers.host.custom_host_pool as custom_host_pool_module @@ -791,9 +790,7 @@ def test_given_warm_scoped_macro_sql_test_when_compiling_then_rules_reuse_compil macro_finding: str = f"tests/unit/test_macro_orders.sql:{test_case.expected_code}" cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) scope_builds: list[Path] = [] - original_build: Callable[..., DeclarationScopeBuild] = ( - declaration_scope_module.build_declaration_scope - ) + original_build: Callable[..., DeclarationScopeBuild] = native_module.build_declaration_scope def counting_build( *, @@ -810,7 +807,7 @@ def counting_build( compile_cache_dir=compile_cache_dir, ) - monkeypatch.setattr(declaration_scope_module, "build_declaration_scope", counting_build) + monkeypatch.setattr(native_module, "build_declaration_scope", counting_build) monkeypatch.setattr(expansion_context_module, "build_declaration_scope", counting_build) warm: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) @@ -850,9 +847,7 @@ def test_given_cold_scoped_macro_sql_test_when_compiling_then_early_lint_uses_pr ) scope_builds: list[Path] = [] macro_loads: list[dict[str, LoadedMacro]] = [] - original_build: Callable[..., DeclarationScopeBuild] = ( - declaration_scope_module.build_declaration_scope - ) + original_build: Callable[..., DeclarationScopeBuild] = native_module.build_declaration_scope original_load: Callable[..., dict[str, LoadedMacro]] = ( expansion_context_module.load_project_macros ) @@ -877,7 +872,7 @@ def recording_load(macro_files: tuple[DiscoveredMacroFile, ...]) -> dict[str, Lo macro_loads.append(loaded) return loaded - monkeypatch.setattr(declaration_scope_module, "build_declaration_scope", counting_build) + monkeypatch.setattr(native_module, "build_declaration_scope", counting_build) monkeypatch.setattr(expansion_context_module, "build_declaration_scope", counting_build) monkeypatch.setattr(macros_module, "load_project_macros", recording_load) monkeypatch.setattr(expansion_context_module, "load_project_macros", recording_load) diff --git a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py index cab46a6b74..144f892d98 100644 --- a/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compile/native_model_config/helpers.py @@ -6,11 +6,7 @@ from typing import cast import sqlbuild._native as _native -from sqlbuild.compiler.compile._helpers.attachment.model_config import ( - native_model_config_session, - native_model_validation, - native_validation_error, -) +from sqlbuild._native import native_model_config_session from sqlbuild.compiler.compile.models import ( CompileModelConfig, CompileSqlReference, @@ -19,6 +15,7 @@ NativeModelConfigSession, ) from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.model_config.main._native_config_error import native_config_error from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.spec.contracts.models import ProjectConfig @@ -66,12 +63,28 @@ def validation_outcome(*, values: dict[str, object]) -> object: ) outcome: _native.NativeConfigError | None = cast( _native.NativeConfigError | None, - native_model_validation(session=session, request=request, rejected=[False]), + session.validator.validate( + request.config.values, + ( + request.model_file.file_path.stem, + request.query_sql, + str(request.model_file.relative_path), + ), + ( + [ + (str(reference.ref_kind), reference.ref_name, False) + for reference in request.references + ], + None, + request.config.time_travel_retention.unmanaged, + request.config.table_type.declared, + ), + ), ) return [ _ACCEPTED, *( - raised_error_shape(native_validation_error(error=error, values=values)) + raised_error_shape(native_config_error(error=error, values=values)) for error in filter(None, (outcome,)) ), ][-1] diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 7b2dfaa4e3..21fe8a2f83 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -8,7 +8,8 @@ from collections.abc import Callable, Iterator from dataclasses import replace from pathlib import Path -from typing import Any +from types import ModuleType +from typing import Any, cast import pytest @@ -23,7 +24,6 @@ from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.auditing.models import MeasurementThresholdBound, MeasurementThresholds from sqlbuild.compiler.compile import models as compile_models -from sqlbuild.compiler.compile._helpers.attachment import core as attachment_core from sqlbuild.compiler.compile._helpers.refs.references import extract_sql_references from sqlbuild.compiler.compile.models import ( CompileAdapterContext, @@ -38,10 +38,13 @@ CompiledModelView, CompiledProjectView, CompileInputsView, + ModelInput, ) -from sqlbuild.compiler.compiled_project.main import compiled_project_facts as facts_module +from sqlbuild.compiler.contracts._helpers import native_contracts as native_contracts_module +from sqlbuild.compiler.contracts._helpers import native_promotion as native_promotion_module from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax +from sqlbuild.rule_engine._helpers.run import native_rows as native_rows_module from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SourceColumnEntry from sqlbuild.sql_values.types import CollectionRendering from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -66,13 +69,23 @@ } +_FACTS_CONSUMERS: tuple[ModuleType, ...] = ( + native_rows_module, + native_contracts_module, + native_promotion_module, +) + + def forbid_hand_built_projects(*, monkeypatch: pytest.MonkeyPatch) -> None: """Fail if a consumer records a project from `CompiledProject` instead of the compile's.""" - def refuse(project: object) -> object: - raise AssertionError(f"rules rebuilt project facts from {type(project).__name__}") + def retained_only(project: CompiledProjectView) -> _native.NativeCompiledProject: + retained: _native.NativeCompiledProject | None = project.native_project + assert retained is not None, f"rules rebuilt project facts from {type(project).__name__}" + return retained - monkeypatch.setattr(facts_module, "hand_built_project_impl", refuse) + for module in _FACTS_CONSUMERS: + monkeypatch.setattr(module, "compiled_project_facts", retained_only) def skip_model_retention(*, monkeypatch: pytest.MonkeyPatch) -> None: @@ -82,22 +95,21 @@ def skip_model_retention(*, monkeypatch: pytest.MonkeyPatch) -> None: attaches its other resources. """ - native_loop: Callable[..., object] = attachment_core._run_native_model_loop - - def discarded(**kwargs: Any) -> object: - loop: Any = kwargs["loop"] - _ = native_loop( - **{ - **kwargs, - "loop": replace( - loop, discovered_inputs=replace(loop.discovered_inputs, model_files=()) - ), - } - ) - kwargs["native_project"] = _native.NativeCompiledProject() - return native_loop(**kwargs) + build_inputs: Callable[..., tuple[ModelInput, ...]] = cast( + Callable[..., tuple[ModelInput, ...]], _native.build_model_inputs + ) - monkeypatch.setattr(attachment_core, "_run_native_model_loop", discarded) + def discarded(**kwargs: Any) -> tuple[ModelInput, ...]: + discovered: DiscoveredProjectInputs = kwargs["discovered_inputs"] + without_models: dict[str, Any] = { + **kwargs, + "discovered_inputs": replace(discovered, model_files=()), + } + _ = build_inputs(**without_models) + throwaway: dict[str, Any] = {**kwargs, "native_project": _native.NativeCompiledProject()} + return build_inputs(**throwaway) + + monkeypatch.setattr(_native, "build_model_inputs", discarded) LEGACY_PROJECT_TYPES: tuple[str, ...] = ( diff --git a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py index 56de61f101..189adaf725 100644 --- a/tests/integration/src/sqlbuild/compiler/contracts/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/contracts/helpers.py @@ -12,7 +12,12 @@ import pytest import sqlbuild.compiler.contracts.main._evaluate_native_model_contracts as contract_entry -from sqlbuild._native import build_project_graph, discover_project_inputs +from sqlbuild._native import ( + build_project_graph, + compiled_model_record, + compiled_project_record, + discover_project_inputs, +) from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -22,8 +27,6 @@ InferredColumn, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.compiled_project_record import compiled_project_record from sqlbuild.compiler.contracts.main.validate import evaluate_model_contracts from sqlbuild.compiler.contracts.models import ContractValidationResult from sqlbuild.compiler.lineage.types import InferredNullability diff --git a/tests/integration/src/sqlbuild/compiler/lineage/helpers.py b/tests/integration/src/sqlbuild/compiler/lineage/helpers.py index df42624aac..924b2d468b 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/helpers.py @@ -10,11 +10,15 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import ProjectColumnLineage, build_project_graph, discover_project_inputs +from sqlbuild._native import ( + ProjectColumnLineage, + build_project_graph, + compiled_model_record, + discover_project_inputs, + project_with, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.lineage.types import ColumnLineageMode _PROJECT_TOML: str = ( diff --git a/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py b/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py index 44fddb43c8..e9c5cec76c 100644 --- a/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py +++ b/tests/integration/src/sqlbuild/compiler/lineage/test_native_fast_lineage.py @@ -12,10 +12,12 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import ProjectColumnLineage +from sqlbuild._native import ( + ProjectColumnLineage, + compiled_model_record, + project_with, +) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from tests.integration.src.sqlbuild.compiler.helpers import ( GoldenEntry, golden_differences, diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py index b14f5b45d7..9775f7f77e 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py @@ -6,10 +6,12 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.project_with import project_with from tests.integration.src.sqlbuild.compiler.lineage.helpers import compiled_project from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( ExpressionBatchCase, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py index 14ced6d8fc..f48e3a7d86 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -6,6 +6,10 @@ from pathlib import Path from typing import Any +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -25,8 +29,6 @@ CompiledSeedView, CompiledSourceView, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import ( DiscoveredSchemaFile, DiscoveredSeedFile, diff --git a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py index c658ac2fc1..592eafe31a 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/planner/_helpers/test_warehouse_snapshot.py @@ -6,6 +6,10 @@ import pytest +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -17,8 +21,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.fingerprints.main.write import write_fingerprint from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.warehouse.snapshot import gather_warehouse_snapshot diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py index 7c839aea91..617a777127 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/helpers.py @@ -12,7 +12,12 @@ from types import MappingProxyType from typing import Any, cast -from sqlbuild._native import discover_project_inputs, run_compile_pipeline +from sqlbuild._native import ( + compiled_model_record, + discover_project_inputs, + project_with, + run_compile_pipeline, +) from sqlbuild.adapter.contract.models import RelationInfo, RelationReadProbe from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import ( @@ -36,8 +41,6 @@ CompiledSourceView, FunctionLanguage, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import ( DiscoveredProjectInputs, DiscoveredSchemaFile, diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py index c59d3bccce..893d4b43bb 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py @@ -7,12 +7,14 @@ import pytest +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.output.main.plan import format_plan from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.planner._helpers.changes import detect as change_detection from sqlbuild.compiler.planner._helpers.planning import identities as identity_planning from sqlbuild.compiler.planner.exceptions import PlannerInputError diff --git a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py index 5b7ea297e8..2b3f192183 100644 --- a/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/project_assembly/helpers.py @@ -13,14 +13,18 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs +from sqlbuild._native import ( + assemble_project, + build_compile_inputs, + discover_project_inputs, + inputs_with, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS from sqlbuild.compiler.compile.models import ( CompileAdapterContext, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.compiled_project.main.inputs_with import inputs_with from sqlbuild.compiler.project_assembly.types import ProjectRequestRow from sqlbuild.sql_values.types import CollectionRendering diff --git a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py index fd1c905a1e..ebaa9d828e 100644 --- a/tests/integration/src/sqlbuild/compiler/scopes/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/scopes/helpers.py @@ -13,10 +13,11 @@ import pytest import sqlbuild._native as _native -from sqlbuild._native import discover_project_inputs +from sqlbuild._native import ( + build_declaration_scope, + discover_project_inputs, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile._helpers.attachment import declaration_scope -from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import build_declaration_scope from sqlbuild.compiler.compile._helpers.render.declarations import resolve_declaration_context from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -32,6 +33,7 @@ DiscoveredProjectInputs, EnumDeclaration, ) +from sqlbuild.compiler.scopes.main import _open_native_scope_index as open_index_module from sqlbuild.compiler.scopes.main._declaration_lexical_path import declaration_lexical_path from sqlbuild.compiler.scopes.main._open_native_scope_index import open_native_scope_index from sqlbuild.compiler.scopes.main._resolve_scope_declaration_visibility import ( @@ -328,7 +330,7 @@ def counted( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros ) - monkeypatch.setattr(declaration_scope, "open_native_scope_index", counted) + monkeypatch.setattr(open_index_module, "open_native_scope_index", counted) outcome: ScopeOutcome = _scope_outcome( discovered=discovered, macros=macros, diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index 578c6a74b6..e441922471 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -15,7 +15,11 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.diagnostics.recovery as semantic_recovery import sqlbuild.compiler.semantic_checks._helpers.stage as semantic_stage -from sqlbuild._native import build_project_graph, discover_project_inputs +from sqlbuild._native import ( + build_project_graph, + discover_project_inputs, + project_with, +) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.analysis_session.types import DiagnosticRow @@ -24,7 +28,6 @@ ) from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.semantic_checks._helpers.metadata import native_metadata_diagnostics from sqlbuild.compiler.semantic_checks._helpers.payloads import session_fact_models from sqlbuild.compiler.semantic_checks.main._complete_native_semantic_diagnostics import ( diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py index e23e6e4e18..9af50230f5 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py @@ -10,8 +10,10 @@ import pytest import sqlbuild._native as native_module +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.project_with import project_with from tests.integration.src.sqlbuild.compiler.helpers import ( GoldenEntry, golden_differences, diff --git a/tests/integration/src/sqlbuild/lint/main/helpers.py b/tests/integration/src/sqlbuild/lint/main/helpers.py index 7bbe835f06..78e74f1825 100644 --- a/tests/integration/src/sqlbuild/lint/main/helpers.py +++ b/tests/integration/src/sqlbuild/lint/main/helpers.py @@ -5,11 +5,14 @@ from dataclasses import replace from pathlib import Path -from sqlbuild._native import build_compiled_project, discover_project_inputs +from sqlbuild._native import ( + build_compiled_project, + compiled_model_record, + discover_project_inputs, + project_with, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with _PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' diff --git a/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py b/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py index ebec9b591a..b6ad426c3f 100644 --- a/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py +++ b/tests/integration/src/sqlbuild/rule_engine/host_payload/test_host_payload_facts.py @@ -4,11 +4,11 @@ import pytest -from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_project_record import ( +from sqlbuild._native import ( compiled_project_record, ) +from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject +from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.rule_engine._helpers.engine.custom_rules import host_project from sqlbuild.rule_engine._helpers.host.custom_host_pool import host_payload_project from sqlbuild.rule_engine.classes.project_tree import public_model diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index 19c92acac4..c12849f906 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -6,6 +6,9 @@ from pathlib import Path from typing import Any +from sqlbuild._native import ( + project_with, +) from sqlbuild.adapter.contract.models import LifeCycleEvent, QueryResult from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -29,7 +32,6 @@ CompiledResourceType, FunctionLanguage, ) -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.planner.models import ( AuditPlanEntry, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py index bfc8b5a41f..6751d0e54a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py @@ -6,7 +6,11 @@ import pytest -from sqlbuild._native import ProjectColumnLineage +from sqlbuild._native import ( + ProjectColumnLineage, + compiled_model_record, + project_with, +) from sqlbuild.cli.commands._helpers.compile.output import ( format_compile_json, format_compile_text, @@ -17,8 +21,6 @@ ) from sqlbuild.compiler.compile.models import CompiledObjectKey, CompileModelConfig from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import SqlHookEntry from sqlbuild.compiler.pipeline.models import ProjectGraph from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 8382d5bb4b..0911791559 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -8,6 +8,10 @@ import pytest import sqlbuild._native as _native +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.commands._helpers.compile import target_writer as target_writer_module from sqlbuild.cli.commands._helpers.compile.target_writer import ( @@ -24,8 +28,6 @@ CompiledResourceType, DiagnosticPhase, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py index 5abc2b166c..0fbec2669a 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/config/helpers.py @@ -1,28 +1,32 @@ """Helpers for model configuration reuse and native validation tests.""" from pathlib import Path +from typing import cast import sqlbuild._native as _native -from sqlbuild._native import build_compile_inputs, discover_project_inputs -from sqlbuild.compiler.compile._helpers.attachment.model_config import ( - build_native_model_config, +from sqlbuild._native import ( + build_compile_inputs, + discover_project_inputs, native_model_config_session, - native_validation_error, ) +from sqlbuild.compiler.compile._helpers.render.templating import record_template_reads from sqlbuild.compiler.compile.models import ( CompileModelConfig, NativeModelConfigInputs, NativeModelConfigSession, ) from sqlbuild.compiler.compile.types import CompileInputsView -from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.model_config.main._native_config_error import native_config_error from sqlbuild.spec.contracts.models import ( DefaultsConfig, MaterializationDefaultsConfig, ProjectConfig, + ResolvedTableType, + ResolvedTimeTravelRetention, SchemaColumn, TargetConfig, ) +from sqlbuild.spec.contracts.types import TableType, TableTypeSource, TimeTravelRetentionSource from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( DUCKDB_COMPILE_ADAPTER_CONTEXT, ) @@ -83,7 +87,7 @@ def validate_natively( # noqa: PLR0913 ) assert not isinstance(outcome, int) for error in filter(None, (outcome,)): - raise native_validation_error(error=error, values=config.values) + raise native_config_error(error=error, values=config.values) def build_config_natively( # noqa: PLR0913 @@ -117,16 +121,32 @@ def build_config_natively( # noqa: PLR0913 ({"orders"}, set(), set(), set(), set()), set(), False ), ) - return build_native_model_config( - session=session, - model_file=DiscoveredSqlModelFile( - file_path=Path("models/marts/orders.sql"), - relative_path=Path("models/marts/orders.sql"), - contents="SELECT 1", - header_values=model_header_values, - header_column_locations={}, - output_column_locations={}, - query_sql="SELECT 1", - ), - matched_path_default=matched_path_default, + built: object = session.builder.model_config( + model_header_values, + (matched_path_default, "orders"), + native_config_classes(session=session), ) + for error in filter(_is_config_error, (built,)): + raise native_config_error(error=cast(_native.NativeConfigError, error)) + return cast(CompileModelConfig, built) + + +def native_config_classes(*, session: NativeModelConfigSession) -> dict[str, object]: + """The builder, storage defaults and classes a model's `CompileModelConfig` is made of.""" + + return { + "builder": session.builder, + "inherited_storage": session.inherited_storage, + "table_backed": ("table", "incremental", "snapshot"), + "compile_model_config": CompileModelConfig, + "resolved_retention": ResolvedTimeTravelRetention, + "retention_source": TimeTravelRetentionSource.MODEL, + "resolved_table_type": ResolvedTableType, + "table_type": TableType, + "table_type_source": TableTypeSource.MODEL, + "record_template_reads": record_template_reads, + } + + +def _is_config_error(value: object) -> bool: + return isinstance(value, _native.NativeConfigError) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py index 2df8110bb7..935e5a6369 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_rebind_declaration_scope.py @@ -7,9 +7,9 @@ import pytest -from sqlbuild._native import discover_project_inputs -from sqlbuild.compiler.compile._helpers.attachment.declaration_scope import ( +from sqlbuild._native import ( build_declaration_scope, + discover_project_inputs, rebind_declaration_scope, ) from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py index 621b130840..5daf7e3da1 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/test_build.py @@ -13,12 +13,12 @@ build_compile_inputs, build_dag_json, discover_project_inputs, + project_with, ) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.dag.types import NodeKind from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.graph.main.project_lineage_views import project_lineage_views diff --git a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py index 76def5dffc..9349c6c40f 100644 --- a/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py +++ b/tests/unit/src/sqlbuild/compiler/lineage/test_columns.py @@ -4,7 +4,10 @@ import pytest -from sqlbuild._native import ProjectColumnLineage +from sqlbuild._native import ( + ProjectColumnLineage, + compiled_model_record, +) from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, CompiledLineageSourceFact, @@ -14,7 +17,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.lineage.models import ColumnLineage, ModelColumnLineage from sqlbuild.compiler.lineage.types import ( ColumnLineageConfidence, diff --git a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py index 7a549c4bce..bbacfc9788 100644 --- a/tests/unit/src/sqlbuild/compiler/manifest/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/manifest/test_main.py @@ -8,6 +8,9 @@ import pytest import sqlbuild._native as _native +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, LoadedMacro, @@ -18,7 +21,6 @@ CompiledSeedView, CompiledSourceView, ) -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredSqlHookFile, SqlHookEntry from sqlbuild.compiler.manifest.main.build import build_manifest from sqlbuild.compiler.planner.models import AuditPlanEntry, ChainStep, PlanOutput, SqlTestPlanEntry diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 9453017c0c..094cc05f91 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -12,6 +12,10 @@ from typing import ClassVar import sqlbuild._native as _native +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.compiler.auditing.types import AuditSeverity @@ -46,8 +50,6 @@ CompiledSourceView, SqlTestMode, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import ( DiscoveredAuditBlock, DiscoveredAuditFile, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py index 769f17f68d..318f7f9605 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/sql_test_assembly/test_plan_test.py @@ -5,6 +5,9 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter @@ -20,7 +23,6 @@ CompileSqlTestCte, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile from sqlbuild.compiler.planner._helpers.sql_tests.native_planning import ( plan_compiled_sql_test_artifacts, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py index 9e44554ac2..a789428dd2 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_fixture_planning_work.py @@ -5,6 +5,10 @@ import pytest +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledSqlTest, @@ -15,8 +19,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.planner._helpers.fixtures import completion as fixture_completion from sqlbuild.compiler.planner._helpers.output.plan_output import build_selected_test_entries from sqlbuild.compiler.planner._helpers.sql_tests import assembly as sql_test_assembly diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py index 208e7ebed7..587ed8102f 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_relations.py @@ -6,6 +6,9 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter @@ -17,7 +20,6 @@ CompileSqlScenarioCte, ) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.compiler.planner._helpers.scenario.relations import ( build_scenario_execution_plan, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index 69fef260e4..a6dbdd897b 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -6,13 +6,15 @@ import pytest +from sqlbuild._native import ( + compiled_model_record, +) from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject from sqlbuild.compiler.compile.types import ( CompiledModelView, CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.fingerprints.constants import NODE_TYPE_SEED from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner._helpers.planning.scopes import resolve_planner_scopes diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py index 0773889e39..b8f8830034 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_source_load_nodes.py @@ -7,6 +7,9 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -19,7 +22,6 @@ CompiledProjectView, CompiledResourceType, ) -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.planner._helpers.graph.loader_dag import ( build_intermediate_source_map, expand_selected_loader_dependencies, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py index 57ca0ac495..cd00e48e5d 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/identity/helpers.py @@ -1,6 +1,9 @@ from dataclasses import replace from pathlib import Path +from sqlbuild._native import ( + compiled_model_record, +) from sqlbuild.compiler.compile.models import ( CompiledFunction, CompiledModel, @@ -10,7 +13,6 @@ FunctionReturnColumn, ) from sqlbuild.compiler.compile.types import CompiledModelView, CompiledResourceType -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record from sqlbuild.compiler.planner._helpers.graph.core import build_downstream_deps from sqlbuild.compiler.planner._helpers.identity.direct import ( build_direct_model_version_identities, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py index 3224a08207..890873ae51 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_execution.py @@ -5,10 +5,12 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.models import ( diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py index 21a78e3899..b9b34cd0d0 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/test_static_command_planning.py @@ -9,10 +9,12 @@ import pytest +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.compiler.compile.models import CompileModelConfig from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledSeedView -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredSeedFile from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.main.commands._relations import resolve_static_relation_context diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py index e4378bf8b7..20fb1f390c 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/helpers.py @@ -8,6 +8,9 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.models import ( CompiledDirectLogicSqlTestPayload, CompiledModelSqlTestPayload, @@ -17,7 +20,6 @@ CompileSqlTestCte, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredSqlTestBlock, DiscoveredSqlTestFile from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine import native diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py index 9c013f18a7..43d9d99bfa 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_custom_rules.py @@ -5,10 +5,12 @@ import pytest +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.compiler.compile.models import CompactLineageFacts from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main._evaluate import evaluate from sqlbuild.rule_engine.models import ( diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py index 80fe76fc77..51c428b117 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_native_facts.py @@ -3,13 +3,15 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.models import ( CompiledObjectKey, CompiledSqlScenario, CompiledSqlTest, ) from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType, SqlTestMode -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import ( DiscoveredSqlScenarioFile, ) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py index fb8d88879b..81b9d91e6d 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_typed_constants.py @@ -3,7 +3,9 @@ import pytest -from sqlbuild.compiler.compiled_project.main.declaration_rows import declaration_rows +from sqlbuild._native import ( + declaration_rows, +) from sqlbuild.compiler.discovery.models import ConstantDeclaration from sqlbuild.sql_values.main.normalize import normalize_sql_value from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py index e805376121..daa5f077eb 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py @@ -13,8 +13,10 @@ import pytest import sqlbuild._native as native_module +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.rule_engine._helpers.engine.catalogue import build_catalogue from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached from sqlbuild.rule_engine._helpers.host import custom_host_pool diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py index 04ba55c983..f48e681c8e 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/audit_facts/test_audit_facts.py @@ -4,8 +4,10 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.types import CompiledAuditView, CompiledProjectView -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.rule_engine.classes.audit_facts import AuditFacts from sqlbuild.rule_engine.models import Model from tests.unit.src.sqlbuild.rule_engine.classes.audit_facts._test_types import ( diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py b/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py index d26e3ba301..2db8ccba43 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/model_test_facts/test_model_test_facts.py @@ -5,9 +5,11 @@ import pytest +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.models import CompiledSqlTest from sqlbuild.compiler.compile.types import CompiledProjectView, SqlTestMode -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.rule_engine.classes.model_test_facts import TestFacts as ModelTestFacts from sqlbuild.rule_engine.models import Model from tests.unit.src.sqlbuild.rule_engine._helpers.engine.helpers import ( diff --git a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py index 4b6084dc9b..1fe8015782 100644 --- a/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/classes/shared_fact_encoder/helpers.py @@ -1,12 +1,14 @@ from pathlib import Path +from sqlbuild._native import ( + project_with, +) from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledSource from sqlbuild.compiler.compile.types import ( CompiledProjectView, CompiledResourceType, CompiledSourceView, ) -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.compiler.discovery.models import DiscoveredSourceFile from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views from sqlbuild.spec.contracts.models import SourceEntry diff --git a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py index df0e6a0334..e94a7aa128 100644 --- a/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py +++ b/tests/unit/src/sqlbuild/rule_engine/main/evaluate/test_cache_and_suppressions.py @@ -5,9 +5,11 @@ import pytest +from sqlbuild._native import ( + compiled_model_record, + project_with, +) from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.compiled_project.main.compiled_model_record import compiled_model_record -from sqlbuild.compiler.compiled_project.main.project_with import project_with from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main._evaluate import evaluate from sqlbuild.rule_engine.models import RuleExemption, RuleIgnore, RulesConfig, RulesResult From 6592d252dbf493b5fe440990280b269e30833afe Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:37:03 +0100 Subject: [PATCH 08/14] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 65a4bd970f..c3c8a307b5 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 129224 +remaining_lines = 127172 [[stays]] path = "executor" From bcd581b9988b303eb4acfb211df92b80fc1a7170 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:52:42 +0100 Subject: [PATCH 09/14] refactor: run the semantic completion stage natively --- .../_helpers/analysis_session/session.rs | 2 +- .../_helpers/boundary/keyword_request.rs | 58 ++ .../src/bindings/_helpers/boundary/mod.rs | 1 + .../bindings/_helpers/lineage/tree_render.rs | 45 +- .../_helpers/semantic_checks/completion.rs | 363 +++++---- .../_helpers/semantic_checks/metadata.rs | 710 +++++++++++++----- .../bindings/_helpers/semantic_checks/mod.rs | 2 + .../_helpers/semantic_checks/registration.rs | 8 +- .../_helpers/semantic_checks/stage.rs | 121 +++ .../_helpers/semantic_checks/stage_objects.rs | 135 ++++ .../_helpers/semantic_checks/type_recovery.rs | 413 ++++++---- src/sqlbuild/_native.pyi | 115 +-- .../compile/_helpers/diagnostics/recovery.py | 6 +- .../compiler/semantic_checks/__init__.py | 1 - .../semantic_checks/_helpers/__init__.py | 0 .../semantic_checks/_helpers/completion.py | 175 ----- .../semantic_checks/_helpers/metadata.py | 301 -------- .../semantic_checks/_helpers/payloads.py | 79 -- .../semantic_checks/_helpers/stage.py | 54 -- .../semantic_checks/_helpers/type_recovery.py | 164 ---- .../compiler/semantic_checks/constants.py | 18 - .../compiler/semantic_checks/main/__init__.py | 1 - .../_complete_native_semantic_diagnostics.py | 29 - .../cli/commands/main/compile/_test_types.py | 9 - .../cli/commands/main/compile/helpers.py | 25 - .../compile/test_dense_compile_performance.py | 35 - .../compiler/semantic_checks/helpers.py | 103 +-- .../test_native_metadata_checks.py | 14 +- .../test_native_semantic_checks.py | 11 +- 29 files changed, 1396 insertions(+), 1602 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/boundary/keyword_request.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs delete mode 100644 src/sqlbuild/compiler/semantic_checks/__init__.py delete mode 100644 src/sqlbuild/compiler/semantic_checks/_helpers/__init__.py delete mode 100644 src/sqlbuild/compiler/semantic_checks/_helpers/completion.py delete mode 100644 src/sqlbuild/compiler/semantic_checks/_helpers/metadata.py delete mode 100644 src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py delete mode 100644 src/sqlbuild/compiler/semantic_checks/_helpers/stage.py delete mode 100644 src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py delete mode 100644 src/sqlbuild/compiler/semantic_checks/constants.py delete mode 100644 src/sqlbuild/compiler/semantic_checks/main/__init__.py delete mode 100644 src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index d7cbfd8d38..226d293a29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -164,7 +164,7 @@ fn internal_error(reason: &str) -> PyErr { impl NativeModelAnalysisSession { /// Models whose output names and lineage the finished session keeps for later stages. #[getter] - fn fact_models(&self) -> Vec { + pub(crate) fn fact_models(&self) -> Vec { self.finished .as_ref() .map(finished_fact_models) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/keyword_request.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/keyword_request.rs new file mode 100644 index 0000000000..944c60fcac --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/keyword_request.rs @@ -0,0 +1,58 @@ +//! Keyword-only native function arguments read from `**request`. + +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; + +/// Keyword arguments of one call, each required. +pub(crate) struct Request<'a, 'py> { + function: &'static str, + arguments: Option<&'a Bound<'py, PyDict>>, + py: Python<'py>, +} + +impl<'a, 'py> Request<'a, 'py> { + pub(crate) fn new( + function: &'static str, + arguments: Option<&'a Bound<'py, PyDict>>, + ) -> PyResult { + let py: Python<'py> = match arguments { + Some(arguments) => arguments.py(), + None => { + return Err(PyTypeError::new_err(format!( + "{function}() missing required keyword-only arguments" + ))); + } + }; + Ok(Self { + function, + arguments, + py, + }) + } + + pub(crate) fn py(&self) -> Python<'py> { + self.py + } + + pub(crate) fn get(&self, name: &str) -> PyResult> { + let value: Option> = match self.arguments { + Some(arguments) => arguments.get_item(name)?, + None => None, + }; + value.ok_or_else(|| { + PyTypeError::new_err(format!( + "{}() missing 1 required keyword-only argument: '{name}'", + self.function + )) + }) + } + + /// The argument, or `None` when the caller left it out. + pub(crate) fn optional(&self, name: &str) -> PyResult>> { + match self.arguments { + Some(arguments) => arguments.get_item(name), + None => Ok(None), + } + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs index 0a21932dc9..7c0e808fc6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs @@ -1,4 +1,5 @@ mod allocator; pub(crate) mod config_values; +pub(crate) mod keyword_request; pub(crate) mod oracles; pub(crate) mod panics; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs index 6007889188..724458e623 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/tree_render.rs @@ -2,7 +2,6 @@ use std::collections::HashSet; -use pyo3::exceptions::PyTypeError; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -13,6 +12,8 @@ use sqlbuild_analysis::lineage_graph::main::tree_lines::tree_lines; use sqlbuild_analysis::lineage_graph::models::TreeLine; use sqlbuild_analysis::lineage_graph::types::TreeVisitor; +use crate::bindings::_helpers::boundary::keyword_request::Request; + const BUILTINS_MODULE: &str = "builtins"; /// Box-drawing dependency tree lines for one focus node, expanding each node once. @@ -111,48 +112,6 @@ fn serialize_column_edge<'py>( Ok(payload) } -/// Keyword arguments of one call, each required. -struct Request<'a, 'py> { - function: &'static str, - arguments: Option<&'a Bound<'py, PyDict>>, - py: Python<'py>, -} - -impl<'a, 'py> Request<'a, 'py> { - fn new(function: &'static str, arguments: Option<&'a Bound<'py, PyDict>>) -> PyResult { - let py: Python<'py> = match arguments { - Some(arguments) => arguments.py(), - None => { - return Err(PyTypeError::new_err(format!( - "{function}() missing required keyword-only arguments" - ))); - } - }; - Ok(Self { - function, - arguments, - py, - }) - } - - fn py(&self) -> Python<'py> { - self.py - } - - fn get(&self, name: &str) -> PyResult> { - let value: Option> = match self.arguments { - Some(arguments) => arguments.get_item(name)?, - None => None, - }; - value.ok_or_else(|| { - PyTypeError::new_err(format!( - "{}() missing 1 required keyword-only argument: '{name}'", - self.function - )) - }) - } -} - /// `str(value)`, as an f-string renders a callback's text. fn text(value: &Bound<'_, PyAny>) -> PyResult { value.str()?.extract() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs index 8829c70ce4..64e06a54a3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/completion.rs @@ -1,189 +1,250 @@ -//! Native diagnostic recovery, explanations and opt-out rejection. +//! Native diagnostic recovery, explanations and opt-out rejection after the metadata checks. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyString, PyTuple}; use sqlbuild_analysis::semantic_checks::main::complete_semantic_diagnostics::complete_semantic_diagnostics; use sqlbuild_analysis::semantic_checks::models::{ - CompletedDiagnostic, CompletionModel, CompletionRequest, FinalDiagnostic, SemanticDiagnostic, - SemanticLocation, + CompletedDiagnostic, CompletionModel, CompletionRequest, SemanticDiagnostic, SemanticLocation, }; -use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::semantic_checks::failures::semantic_error; +use crate::bindings::_helpers::semantic_checks::metadata::{ShapeRows, shape_rows}; use crate::bindings::_helpers::semantic_checks::session_facts::{ column_names, session_model_facts, }; -use crate::bindings::_helpers::semantic_checks::type_recovery::{LineageInput, lineage_outputs}; -use crate::bindings::models::ProjectCatalog; +use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; +use crate::bindings::_helpers::semantic_checks::type_recovery::{ + LineageInput, StageShared, diagnostic_ids, id_of, inferred_names, items, lineage_outputs, + lineage_payload, +}; use crate::bindings::types::CompilerDetach; -/// `(line, column, end_line, end_column)`. -type LocationRow = (i64, i64, Option, Option); -/// `(id, code, message, resource_type, resource_name, line, column, location, help, notes)`. -type DiagnosticInput = ( - usize, - String, - String, - Option, - Option, - Option, - Option, - Option, - Option, - Vec, -); -/// `(name, query_sql, authored_sql, inferred names, lineage, binding codes, opt-out file)`. -type ModelInput = ( - String, - String, - String, - Option>, - Option>, - Vec, - Option, -); -/// `(dialect, diagnostics, models, shapes)`. -type RequestInput = ( - Option, - Vec, - Vec, - Vec<(String, Vec<(String, String)>)>, -); -/// `(source, message, help, notes, location, line, column, changed)`. -type CompletedRow = ( - usize, - String, - Option, - Vec, - Option, - Option, - Option, - bool, -); -/// `(completed position, (model, message, note, help))`: exactly one side is set. -type OrderRow = (Option, Option<(usize, String, String, String)>); -/// `(completed, model binding positions, order)`. -type OutcomeRow = (Vec, Option>>, Vec); +const RECOVERED_CODE: &str = "B002"; -/// Recover, explain and reject opt-outs, reading facts the payload omits from `session`. -#[pyfunction] -#[pyo3(signature = (catalog, request, session=None))] -fn complete_semantic_checks( - py: Python<'_>, - catalog: PyRef<'_, ProjectCatalog>, - request: RequestInput, - session: Option>, -) -> PyResult { - let request = completion_request(request, session.as_deref()).map_err(compiler_error)?; - let catalog = &catalog.inner; - let (diagnostics, model_bindings, order) = py - .compiler_detach(|| Ok(complete_semantic_diagnostics(&request, catalog))) +/// The recovered, explained and opt-out-checked project; a native internal failure raises. +pub(crate) fn completed_diagnostics<'py>( + classes: &StageClasses<'py>, + project: &Bound<'py, PyAny>, + shared: &StageShared<'_>, +) -> PyResult> { + let py: Python<'py> = classes.py(); + let models: Vec> = items(&project.getattr("models")?)?; + let ids: Bound<'py, PyDict> = diagnostic_ids(project, &models)?; + let diagnostics: Vec> = items(&project.getattr("diagnostics")?)?; + let mut recovers: bool = false; + for item in &diagnostics { + if item.getattr("code")?.eq(RECOVERED_CODE)? { + recovers = true; + break; + } + } + let shapes: ShapeRows = if diagnostics.is_empty() { + Vec::new() + } else { + shape_rows(&classes.inputs.call_method1("shapes", (project,))?)? + }; + let mut semantic: Vec = Vec::with_capacity(diagnostics.len()); + for item in &diagnostics { + semantic.push(semantic_diagnostic(item, &ids)?); + } + let mut completion_models: Vec = Vec::with_capacity(models.len()); + for model in &models { + completion_models.push(completion_model(model, recovers, shared)?); + } + let request = CompletionRequest { + dialect: project.getattr("sql_analysis_dialect")?.extract()?, + diagnostics: semantic, + models: completion_models, + shapes, + }; + let (completed, model_bindings, order) = py + .compiler_detach(|| Ok(complete_semantic_diagnostics(&request, shared.catalog))) .map_err(compiler_error)? .map_err(|failure| semantic_error(py, &failure))? .into_parts(); - Ok(( - diagnostics.into_iter().map(completed_row).collect(), - model_bindings, - order.into_iter().map(order_row).collect(), - )) -} - -fn location(row: LocationRow) -> SemanticLocation { - SemanticLocation { - line: row.0, - column: row.1, - end_line: row.2, - end_column: row.3, + let mut rebuilt: Vec> = Vec::with_capacity(completed.len()); + for row in completed { + let original: Bound<'py, PyAny> = diagnostics[row.source].clone(); + rebuilt.push(completed_diagnostic(classes, original, row)?); } -} - -fn location_row(location: SemanticLocation) -> LocationRow { - ( - location.line, - location.column, - location.end_line, - location.end_column, + let models_value: Bound<'py, PyAny> = match model_bindings { + None => project.getattr("models")?, + Some(model_bindings) => { + let mut updated: Vec> = Vec::with_capacity(models.len()); + for (model, positions) in models.iter().zip(model_bindings) { + let selected: Vec> = positions + .into_iter() + .map(|position| rebuilt[position].clone()) + .collect(); + updated.push(classes.call( + &classes.with_binding_diagnostics, + &[ + ("model", model), + ("diagnostics", &PyTuple::new(py, selected)?.into_any()), + ], + )?); + } + PyTuple::new(py, updated)?.into_any() + } + }; + let mut ordered: Vec> = Vec::with_capacity(order.len()); + for entry in order { + let (position, opt_out) = entry.into_parts(); + if let Some(position) = position { + ordered.push(rebuilt[position].clone()); + } else if let Some(opt_out) = opt_out { + let model: &Bound<'py, PyAny> = &models[opt_out.model]; + let notes: Bound<'py, PyAny> = PyTuple::new(py, [opt_out.note])?.into_any(); + ordered.push(classes.call( + &classes.diagnostic, + &[ + ("phase", &classes.compile_phase), + ("severity", &classes.error), + ("code", &classes.opt_out_code), + ("message", &PyString::new(py, &opt_out.message).into_any()), + ("resource_type", &classes.model_resource), + ("resource_name", &model.getattr("name")?), + ("location", &model.getattr("rejected_sql_analysis_opt_out")?), + ("notes", ¬es), + ("help", &PyString::new(py, &opt_out.help).into_any()), + ], + )?); + } + } + classes.project_with( + project, + &[ + ("models", &models_value), + ("diagnostics", &PyTuple::new(py, ordered)?.into_any()), + ], ) } -fn completion_request( - (dialect, diagnostics, models, shapes): RequestInput, - session: Option<&NativeModelAnalysisSession>, -) -> Result { - Ok(CompletionRequest { - dialect, - diagnostics: diagnostics.into_iter().map(semantic_diagnostic).collect(), - models: models - .into_iter() - .map(|model| completion_model(model, session)) - .collect::>()?, - shapes, - }) +fn optional_location(location: &Bound<'_, PyAny>) -> PyResult> { + if location.is_none() { + return Ok(None); + } + Ok(Some(SemanticLocation { + line: location.getattr("line")?.extract()?, + column: location.getattr("column")?.extract()?, + end_line: location.getattr("end_line")?.extract()?, + end_column: location.getattr("end_column")?.extract()?, + })) } fn semantic_diagnostic( - (id, code, message, resource_type, resource_name, line, column, place, help, notes): DiagnosticInput, -) -> SemanticDiagnostic { - SemanticDiagnostic { - id, - code, - message, - resource_type, - resource_name, - line, - column, - location: place.map(location), - help, - notes, - } + item: &Bound<'_, PyAny>, + ids: &Bound<'_, PyDict>, +) -> PyResult { + let resource_type: Bound<'_, PyAny> = item.getattr("resource_type")?; + Ok(SemanticDiagnostic { + id: id_of(ids, item)?, + code: item.getattr("code")?.extract()?, + message: item.getattr("message")?.extract()?, + resource_type: if resource_type.is_none() { + None + } else { + Some(resource_type.str()?.extract()?) + }, + resource_name: item.getattr("resource_name")?.extract()?, + line: item.getattr("line")?.extract()?, + column: item.getattr("column")?.extract()?, + location: optional_location(&item.getattr("location")?)?, + help: item.getattr("help")?.extract()?, + notes: items(&item.getattr("notes")?)? + .iter() + .map(PyAnyMethods::extract) + .collect::>()?, + }) } fn completion_model( - (name, query_sql, authored_sql, inferred_columns, lineage, binding_codes, opt_out): ModelInput, - session: Option<&NativeModelAnalysisSession>, -) -> Result { - let inferred_columns: Vec = match inferred_columns { - Some(names) => names, - None => column_names(session_model_facts(session, &name)?).unwrap_or_default(), + model: &Bound<'_, PyAny>, + recovers: bool, + shared: &StageShared<'_>, +) -> PyResult { + let name: String = model.getattr("name")?.extract()?; + let from_session: bool = shared.session_models.contains(&name); + let inferred_columns: Vec = if from_session { + column_names(session_model_facts(shared.session, &name).map_err(compiler_error)?) + .unwrap_or_default() + } else { + inferred_names(model)?.unwrap_or_default() }; - let lineage: Vec = match lineage { - Some(lineage) => lineage, - None => session_model_facts(session, &name)?.lineage.clone(), + let lineage: Vec = if !recovers { + Vec::new() + } else if from_session { + session_model_facts(shared.session, &name) + .map_err(compiler_error)? + .lineage + .clone() + } else { + lineage_payload(model)? }; + let opt_out: Bound<'_, PyAny> = model.getattr("rejected_sql_analysis_opt_out")?; + let mut binding_codes: Vec = Vec::new(); + for item in model.getattr("binding_diagnostics")?.try_iter()? { + binding_codes.push(item?.getattr("code")?.extract()?); + } Ok(CompletionModel { name, - query_sql, - authored_sql, + query_sql: model.getattr("query_sql")?.extract()?, + authored_sql: model.getattr("authored_sql")?.extract()?, inferred_columns, lineage: lineage_outputs(lineage), binding_codes, - rejected_opt_out_file: opt_out, + rejected_opt_out_file: if opt_out.is_none() { + None + } else { + Some(opt_out.getattr("path")?.getattr("name")?.extract()?) + }, }) } -fn completed_row(diagnostic: CompletedDiagnostic) -> CompletedRow { - ( - diagnostic.source, - diagnostic.message, - diagnostic.help, - diagnostic.notes, - diagnostic.location.map(location_row), - diagnostic.line, - diagnostic.column, - diagnostic.changed, - ) -} - -fn order_row(entry: FinalDiagnostic) -> OrderRow { - let (position, opt_out) = entry.into_parts(); - ( - position, - opt_out.map(|opt_out| (opt_out.model, opt_out.message, opt_out.note, opt_out.help)), +fn completed_diagnostic<'py>( + classes: &StageClasses<'py>, + original: Bound<'py, PyAny>, + row: CompletedDiagnostic, +) -> PyResult> { + if !row.changed { + return Ok(original); + } + let py: Python<'py> = classes.py(); + let mut current: Bound<'py, PyAny> = original.getattr("location")?; + if let (Some(existing), Some(location)) = (optional_location(¤t)?, row.location.as_ref()) + && ( + existing.line, + existing.column, + existing.end_line, + existing.end_column, + ) != ( + location.line, + location.column, + location.end_line, + location.end_column, + ) + { + current = classes.replaced( + ¤t, + &[ + ("line", &location.line.into_bound_py_any(py)?), + ("column", &location.column.into_bound_py_any(py)?), + ("end_line", &location.end_line.into_bound_py_any(py)?), + ("end_column", &location.end_column.into_bound_py_any(py)?), + ], + )?; + } + classes.replaced( + &original, + &[ + ("message", &PyString::new(py, &row.message).into_any()), + ("help", &row.help.into_bound_py_any(py)?), + ("notes", &PyTuple::new(py, row.notes)?.into_any()), + ("location", ¤t), + ("line", &row.line.into_bound_py_any(py)?), + ("column", &row.column.into_bound_py_any(py)?), + ], ) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(complete_semantic_checks, module)?)?; - Ok(()) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index b6f265a8da..d155831cc8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -1,225 +1,557 @@ //! Native semantic metadata checks: config references, cursors, function calls and SQL tests. -use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; -use pyo3::{pyfunction, wrap_pyfunction}; +use std::sync::Arc; + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyString, PyTuple}; use sqlbuild_analysis::semantic_checks::main::check_semantic_metadata::check_semantic_metadata; use sqlbuild_analysis::semantic_checks::models::{ AuditValue, MetadataAudit, MetadataFinding, MetadataFunction, MetadataModel, MetadataRequest, MetadataSource, MetadataSqlTest, ModelMetadataFindings, }; -use std::sync::Arc; +use sqlbuild_analysis::semantic_validation::models as validation; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::semantic_checks::failures::semantic_error; -use crate::bindings::models::ProjectCatalog; +use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; +use crate::bindings::_helpers::semantic_checks::type_recovery::items; use crate::bindings::types::CompilerDetach; +const METADATA_CONFIG_KEYS: [&str; 4] = [ + "unique_key", + "cursor", + "partition_column", + "row_diff_exclude_columns", +]; + +/// `[(relation, [(column, value)])]` in Python's order. +pub(crate) type ShapeRows = Vec<(String, Vec<(String, String)>)>; /// `(key, names)` pairs in Python's order. -type NamesInput = Vec<(String, Vec)>; -/// `(column, definition, expression, to, field, values as (kind, text))`. -type AuditInput = ( - Option, - String, - Option, - String, - String, - Option>, -); -/// `(name, query, authored, checked, functions, references, cursor inputs, cursor, type, audits)`. -type ModelInput = ( - String, - String, - String, - bool, - bool, - NamesInput, - NamesInput, - Option, - Option, - Vec, -); -/// `(dialect, quoted case, return types, functions, shapes, models, files, sources, SQL tests)`. -type RequestInput = ( - Option, - bool, - Vec<(String, String)>, - Vec<(String, Vec<(String, String)>)>, - Vec<(String, Vec<(String, String)>)>, - Vec, - Vec, - Vec<(String, Option, usize)>, - Vec<(usize, NamesInput)>, -); -/// `(code, message, line, column)`. -type ErrorRow = (&'static str, String, i64, i64); -/// `(code, message, line, column, is error)`. -type AuditRow = (String, String, i64, i64, bool); -/// `([(function, audit, reference findings)], source, SQL test findings, fallback types)`. -type OutcomeRow = ( - Vec<(Vec, Vec, Vec)>, - Vec<(usize, ErrorRow)>, - Vec<(usize, ErrorRow, i64)>, - Vec, -); - -/// Run the metadata checks on `catalog`'s analysis pool. -#[pyfunction] -fn check_semantic_metadata_rows( - py: Python<'_>, - catalog: PyRef<'_, ProjectCatalog>, - request: RequestInput, -) -> PyResult { - let request = metadata_request(request)?; - let catalog = &catalog.inner; +type NamesRow = (String, Vec); + +/// The relations the metadata request reads, kept for building its diagnostics. +struct MetadataInputs<'py> { + shapes: Bound<'py, PyAny>, + models: Vec>, + sources: Vec>, + tests: Vec>, +} + +/// The project's semantic metadata diagnostics, then its resource SQL diagnostics when asked. +pub(crate) fn metadata_diagnostics<'py>( + classes: &StageClasses<'py>, + (project, profile): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + resource_sql_analysis: bool, + catalog: &validation::ProjectCatalog, +) -> PyResult>> { + let py: Python<'py> = classes.py(); + if !project + .getattr("settings")? + .getattr("sql_analysis")? + .is_truthy()? + { + return Ok(Vec::new()); + } + let (inputs, request) = metadata_request(classes, project, profile)?; let outcome = py .compiler_detach(|| Ok(check_semantic_metadata(&request, catalog))) .map_err(compiler_error)? .map_err(|failure| semantic_error(py, &failure))?; - Ok(( - outcome.models.into_iter().map(model_rows).collect(), - outcome - .sources - .into_iter() - .map(|(index, error)| (index, error_row(error))) - .collect(), - outcome - .sql_tests - .into_iter() - .map(|(index, error, end)| (index, error_row(error), end)) - .collect(), - outcome.fallback_types, - )) -} - -fn error_row(error: MetadataFinding) -> ErrorRow { - (error.code, error.message, error.line, error.column) -} - -fn model_rows(errors: ModelMetadataFindings) -> (Vec, Vec, Vec) { - ( - errors.functions.into_iter().map(error_row).collect(), - errors - .audits - .into_iter() - .map(|finding| { - ( - finding.code, - finding.message, - finding.line, - finding.column, - finding.is_error, - ) - }) - .collect(), - errors.references.into_iter().map(error_row).collect(), - ) + let dialect: Bound<'py, PyAny> = profile.getattr("sql_analysis_dialect")?; + for type_sql in &outcome.fallback_types { + let _ = classes.call( + &classes.normalize_type, + &[ + ("type_sql", &PyString::new(py, type_sql).into_any()), + ("dialect", &dialect), + ], + )?; + } + let mut diagnostics: Vec> = Vec::new(); + for (model, findings) in inputs.models.iter().zip(outcome.models) { + if checked(classes, model)? { + diagnostics.extend(model_diagnostics(classes, model, findings)?); + } + } + for (index, finding) in outcome.sources { + let source: &Bound<'py, PyAny> = &inputs.sources[index]; + let path: Bound<'py, PyAny> = source.getattr("source_file")?.getattr("relative_path")?; + diagnostics.push(diagnostic( + classes, + finding, + (&classes.source_resource, &source.getattr("name")?), + (&path, None), + )?); + } + for (index, finding, end_column) in outcome.sql_tests { + diagnostics.push(sql_test_diagnostic( + classes, + &inputs.tests[index], + finding, + end_column, + )?); + } + if resource_sql_analysis { + let resource: Bound<'py, PyAny> = classes.call( + &classes.inputs.getattr("resource_sql_diagnostics")?, + &[ + ("project", project), + ("shapes", &inputs.shapes), + ("profile", profile), + ], + )?; + diagnostics.extend(items(&resource)?); + } + Ok(diagnostics) } -fn metadata_request( - ( - dialect, - quoted_ignore_case, - return_types, +fn metadata_request<'py>( + classes: &StageClasses<'py>, + project: &Bound<'py, PyAny>, + profile: &Bound<'py, PyAny>, +) -> PyResult<(MetadataInputs<'py>, MetadataRequest)> { + let shapes: Bound<'py, PyAny> = classes.call( + &classes.inputs.getattr("metadata_shapes")?, + &[("project", project), ("profile", profile)], + )?; + let mut tests: Vec> = Vec::new(); + for test in project.getattr("sql_tests")?.try_iter()? { + let test: Bound<'py, PyAny> = test?; + if test + .getattr("payload")? + .is_instance(&classes.model_test_payload)? + { + tests.push(test); + } + } + let columns_by_sql: Bound<'py, PyAny> = classes.call( + &classes.inputs.getattr("sql_test_columns")?, + &[("project", project), ("profile", profile)], + )?; + let functions: Vec = metadata_functions(project)?; + let sources: Vec> = items(&project.getattr("sources")?)?; + let (files, contents) = file_contents(classes.py(), &sources, &tests)?; + let models: Vec> = items(&project.getattr("models")?)?; + let mut metadata_models: Vec = Vec::with_capacity(models.len()); + for model in &models { + metadata_models.push(metadata_model(classes, model)?); + } + let mut metadata_sources: Vec = Vec::with_capacity(sources.len()); + for source in &sources { + let cursor: Bound<'py, PyAny> = source.getattr("source_entry")?.getattr("cursor_column")?; + let index: usize = file_index(&files, &source.getattr("source_file")?)?; + metadata_sources.push(MetadataSource { + name: source.getattr("name")?.extract()?, + cursor_column: if cursor.is_truthy()? { + Some(cursor.extract()?) + } else { + None + }, + contents: contents[index].clone(), + }); + } + let mut metadata_tests: Vec = Vec::with_capacity(tests.len()); + for test in &tests { + let index: usize = file_index(&files, &test.getattr("test_file")?)?; + metadata_tests.push(MetadataSqlTest { + contents: contents[index].clone(), + ctes: test_ctes(&test.getattr("payload")?, &columns_by_sql)?, + }); + } + let request = MetadataRequest { + dialect: profile.getattr("sql_analysis_dialect")?.extract()?, + quoted_ignore_case: profile + .getattr("quoted_identifiers_ignore_case")? + .extract()?, + return_types: pairs(&profile.getattr("function_return_types")?)?, functions, + shapes: shape_rows(&shapes)?, + models: metadata_models, + sources: metadata_sources, + sql_tests: metadata_tests, + }; + let inputs = MetadataInputs { shapes, models, - files, sources, - sql_tests, - ): RequestInput, -) -> PyResult { - let files: Vec> = files.into_iter().map(Arc::from).collect(); - let file = |index: usize| -> PyResult> { - files - .get(index) - .cloned() - .ok_or_else(|| PyValueError::new_err("a metadata row names no file text")) + tests, }; - Ok(MetadataRequest { - dialect, - quoted_ignore_case, - return_types, - functions: functions - .into_iter() - .map(|(key, arguments)| MetadataFunction { key, arguments }) - .collect(), - shapes, - models: models.into_iter().map(metadata_model).collect(), - sources: sources - .into_iter() - .map(|(name, cursor_column, index)| { - Ok(MetadataSource { - name, - cursor_column, - contents: file(index)?, - }) - }) - .collect::>()?, - sql_tests: sql_tests - .into_iter() - .map(|(index, ctes)| { - Ok(MetadataSqlTest { - contents: file(index)?, - ctes, - }) - }) - .collect::>()?, - }) + Ok((inputs, request)) } -fn metadata_model( - ( - name, - query_sql, - authored_sql, - checked, - calls_functions, - references, - cursor_inputs, - cursor, - cursor_type, - audits, - ): ModelInput, -) -> MetadataModel { - MetadataModel { - name, - query_sql, - authored_sql, - checked, - calls_functions, - references, - cursor_inputs, - cursor, - cursor_type, - audits: audits.into_iter().map(metadata_audit).collect(), +/// Declared functions by `name.casefold()`, the last declaration winning in the first's place. +fn metadata_functions(project: &Bound<'_, PyAny>) -> PyResult> { + let functions: Bound<'_, PyDict> = PyDict::new(project.py()); + for function in project.getattr("functions")?.try_iter()? { + let function: Bound<'_, PyAny> = function?; + functions.set_item( + function.getattr("name")?.call_method0("casefold")?, + &function, + )?; } + let mut metadata: Vec = Vec::with_capacity(functions.len()); + for (key, function) in functions.iter() { + let mut arguments: Vec<(String, String)> = Vec::new(); + for argument in function.getattr("arguments")?.try_iter()? { + let argument: Bound<'_, PyAny> = argument?; + arguments.push(( + argument.getattr("name")?.extract()?, + argument.getattr("type")?.extract()?, + )); + } + metadata.push(MetadataFunction { + key: key.extract()?, + arguments, + }); + } + Ok(metadata) } -fn audit_value((kind, text): (String, String)) -> AuditValue { - match kind.as_str() { - "null" => AuditValue::Null, - "bool" => AuditValue::Bool(!text.is_empty()), - "number" => AuditValue::Number(text), - "text" => AuditValue::Text(text), - _ => AuditValue::Other, +/// Each distinct file path's index and the text its first resource carries. +fn file_contents<'py>( + py: Python<'py>, + sources: &[Bound<'py, PyAny>], + tests: &[Bound<'py, PyAny>], +) -> PyResult<(Bound<'py, PyDict>, Vec>)> { + let file_texts: Bound<'py, PyDict> = PyDict::new(py); + for file in sources + .iter() + .map(|source| source.getattr("source_file")) + .chain(tests.iter().map(|test| test.getattr("test_file"))) + { + let file: Bound<'py, PyAny> = file?; + let path: Bound<'py, PyAny> = file.getattr("file_path")?; + if !file_texts.contains(&path)? { + file_texts.set_item(path, file.getattr("contents")?)?; + } + } + let files: Bound<'py, PyDict> = PyDict::new(py); + let mut contents: Vec> = Vec::with_capacity(file_texts.len()); + for (index, (path, text)) in file_texts.iter().enumerate() { + files.set_item(path, index)?; + contents.push(Arc::from(text.extract::()?)); } + Ok((files, contents)) +} + +fn file_index(files: &Bound<'_, PyDict>, file: &Bound<'_, PyAny>) -> PyResult { + files + .as_any() + .get_item(file.getattr("file_path")?)? + .extract() +} + +/// Each authored and expected CTE's name with the columns its body infers. +fn test_ctes( + payload: &Bound<'_, PyAny>, + columns_by_sql: &Bound<'_, PyAny>, +) -> PyResult> { + let mut ctes: Vec = Vec::new(); + for cte in payload + .getattr("authored_ctes")? + .try_iter()? + .chain(payload.getattr("expected_ctes")?.try_iter()?) + { + let cte: Bound<'_, PyAny> = cte?; + ctes.push(( + cte.getattr("name")?.extract()?, + columns_by_sql + .get_item(cte.getattr("sql_body")?)? + .try_iter()? + .map(|name| name?.extract()) + .collect::>()?, + )); + } + Ok(ctes) +} + +/// A checked model's function errors, then its audit findings, then its reference errors. +fn model_diagnostics<'py>( + classes: &StageClasses<'py>, + model: &Bound<'py, PyAny>, + findings: ModelMetadataFindings, +) -> PyResult>> { + let mut diagnostics: Vec> = Vec::new(); + for finding in findings.functions { + diagnostics.push(model_error( + classes, + model, + &finding_row(&finding), + &classes.error, + )?); + } + for audit in findings.audits { + let severity: &Bound<'py, PyAny> = if audit.is_error { + &classes.error + } else { + &classes.warning + }; + let row: FindingRow<'_> = (&audit.code, &audit.message, audit.line, audit.column); + diagnostics.push(model_error(classes, model, &row, severity)?); + } + for finding in findings.references { + diagnostics.push(model_error( + classes, + model, + &finding_row(&finding), + &classes.error, + )?); + } + Ok(diagnostics) +} + +fn sql_test_diagnostic<'py>( + classes: &StageClasses<'py>, + test: &Bound<'py, PyAny>, + finding: MetadataFinding, + end_column: i64, +) -> PyResult> { + let py: Python<'py> = classes.py(); + let path: Bound<'py, PyAny> = test.getattr("test_file")?.getattr("relative_path")?; + let location: Bound<'py, PyAny> = classes.call( + &classes.location, + &[ + ("path", &path), + ("line", &finding.line.into_bound_py_any(py)?), + ("column", &finding.column.into_bound_py_any(py)?), + ("end_line", &finding.line.into_bound_py_any(py)?), + ("end_column", &end_column.into_bound_py_any(py)?), + ], + )?; + diagnostic( + classes, + finding, + (&classes.sql_test_resource, &test.getattr("name")?), + (&path, Some(&location)), + ) } -fn metadata_audit( - (column, definition, expression, target, field, values): AuditInput, -) -> MetadataAudit { - MetadataAudit { - column, - definition, - expression, - target, - field, - values: values.map(|values| values.into_iter().map(audit_value).collect()), +/// `{name: {column: value}}` as ordered rows. +pub(crate) fn shape_rows(shapes: &Bound<'_, PyAny>) -> PyResult { + let mut rows: ShapeRows = Vec::new(); + for item in shapes.call_method0("items")?.try_iter()? { + let (name, columns): (String, Bound<'_, PyAny>) = item?.extract()?; + rows.push((name, pairs(&columns)?)); } + Ok(rows) } -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(check_semantic_metadata_rows, module)?)?; - Ok(()) +/// A `{key: value}` text mapping's items in order. +fn pairs(mapping: &Bound<'_, PyAny>) -> PyResult> { + mapping + .call_method0("items")? + .try_iter()? + .map(|item| item?.extract()) + .collect() +} + +/// `(code, message, line, column)`. +type FindingRow<'a> = (&'a str, &'a str, i64, i64); + +fn finding_row(finding: &MetadataFinding) -> FindingRow<'_> { + (finding.code, &finding.message, finding.line, finding.column) +} + +fn model_error<'py>( + classes: &StageClasses<'py>, + model: &Bound<'py, PyAny>, + &(code, message, line, column): &FindingRow<'_>, + severity: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = classes.py(); + classes.call( + &classes.diagnostic, + &[ + ("phase", &classes.compile_phase), + ("severity", severity), + ("code", &PyString::new(py, code).into_any()), + ("message", &PyString::new(py, message).into_any()), + ("resource_type", &classes.model_resource), + ("resource_name", &model.getattr("name")?), + ("path", &model.getattr("relative_path")?), + ("line", &line.into_bound_py_any(py)?), + ("column", &column.into_bound_py_any(py)?), + ], + ) +} + +fn diagnostic<'py>( + classes: &StageClasses<'py>, + finding: MetadataFinding, + (resource_type, resource_name): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (path, location): (&Bound<'py, PyAny>, Option<&Bound<'py, PyAny>>), +) -> PyResult> { + let py: Python<'py> = classes.py(); + let code: Bound<'py, PyAny> = PyString::new(py, finding.code).into_any(); + let message: Bound<'py, PyAny> = PyString::new(py, &finding.message).into_any(); + let line: Bound<'py, PyAny> = finding.line.into_bound_py_any(py)?; + let column: Bound<'py, PyAny> = finding.column.into_bound_py_any(py)?; + let mut arguments: Vec<(&str, &Bound<'py, PyAny>)> = vec![ + ("phase", &classes.compile_phase), + ("severity", &classes.error), + ("code", &code), + ("message", &message), + ("resource_type", resource_type), + ("resource_name", resource_name), + ("path", path), + ("line", &line), + ("column", &column), + ]; + if let Some(location) = location { + arguments.push(("location", location)); + } + classes.call(&classes.diagnostic, &arguments) +} + +/// Python's `config.get(sql_analysis) is not False and model.binding_validated`. +fn checked(classes: &StageClasses<'_>, model: &Bound<'_, PyAny>) -> PyResult { + let opted: Bound<'_, PyAny> = model + .getattr("config")? + .getattr("values")? + .call_method1("get", (&classes.sql_analysis_key,))?; + if opted.is(PyBool::new(model.py(), false)) { + return Ok(false); + } + model.getattr("binding_validated")?.is_truthy() +} + +fn text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_instance_of::() { + return Ok(Some(value.extract()?)); + } + Ok(None) +} + +fn column_names(classes: &StageClasses<'_>, value: &Bound<'_, PyAny>) -> PyResult> { + classes + .inputs + .call_method1("column_names", (value,))? + .try_iter()? + .map(|name| name?.extract()) + .collect() +} + +fn metadata_model(classes: &StageClasses<'_>, model: &Bound<'_, PyAny>) -> PyResult { + let values: Bound<'_, PyAny> = model.getattr("config")?.getattr("values")?; + let get = |key: &str| values.call_method1("get", (key,)); + let mut references: Vec = Vec::new(); + for key in METADATA_CONFIG_KEYS { + references.push((key.to_owned(), column_names(classes, &get(key)?)?)); + } + let custom: Bound<'_, PyAny> = get("config")?; + if custom.is_instance_of::() { + let partition: Bound<'_, PyAny> = custom.call_method1("get", ("partition_column",))?; + references.push(( + "partition_column".to_owned(), + column_names(classes, &partition)?, + )); + } + let tolerances: Bound<'_, PyAny> = get("row_diff_tolerances")?; + if tolerances.is_instance_of::() { + let by_column: Bound<'_, PyAny> = tolerances.call_method1("get", ("by_column",))?; + if by_column.is_instance_of::() { + let names: Vec = by_column + .try_iter()? + .map(|name| name?.str()?.extract()) + .collect::>()?; + references.push(("row_diff_tolerances".to_owned(), names)); + } + } + let cursor_inputs: Bound<'_, PyAny> = get("cursor_inputs")?; + let mut upstreams: Vec = Vec::new(); + if cursor_inputs.is_instance_of::() { + for item in cursor_inputs.call_method0("items")?.try_iter()? { + let (upstream, columns): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + upstreams.push((upstream.str()?.extract()?, column_names(classes, &columns)?)); + } + } + let mut calls_functions: bool = false; + for reference in model.getattr("references")?.try_iter()? { + let kind: Bound<'_, PyAny> = reference?.getattr("ref_kind")?; + if kind.eq(&classes.udf_kind)? || kind.eq(&classes.table_function_kind)? { + calls_functions = true; + break; + } + } + Ok(MetadataModel { + name: model.getattr("name")?.extract()?, + query_sql: model.getattr("query_sql")?.extract()?, + authored_sql: model.getattr("authored_sql")?.extract()?, + checked: checked(classes, model)?, + calls_functions, + references, + cursor_inputs: upstreams, + cursor: text(&get("cursor")?)?, + cursor_type: text(&get("cursor_type")?)?, + audits: metadata_audits(model)?, + }) +} + +fn metadata_audits(model: &Bound<'_, PyAny>) -> PyResult> { + let entry: Bound<'_, PyAny> = model.getattr("schema_entry")?; + if entry.is_none() { + return Ok(Vec::new()); + } + let py: Python<'_> = model.py(); + let mut attached: Vec<(Option, Bound<'_, PyAny>)> = Vec::new(); + for audit in entry.getattr("audits")?.try_iter()? { + attached.push((None, audit?)); + } + for column in entry.getattr("columns")?.try_iter()? { + let column: Bound<'_, PyAny> = column?; + let name: String = column.getattr("name")?.extract()?; + for audit in column.getattr("audits")?.try_iter()? { + attached.push((Some(name.clone()), audit?)); + } + } + let empty: Bound<'_, PyAny> = PyString::new(py, "").into_any(); + let mut audits: Vec = Vec::with_capacity(attached.len()); + for (column, audit) in attached { + let arguments: Bound<'_, PyAny> = audit.getattr("arguments")?; + let values: Bound<'_, PyAny> = arguments.call_method1("get", ("values",))?; + let values: Option> = + if values.is_instance_of::() || values.is_instance_of::() { + Some( + values + .try_iter()? + .map(|value| audit_value(&value?)) + .collect::>()?, + ) + } else { + None + }; + audits.push(MetadataAudit { + column, + definition: audit.getattr("definition_name")?.extract()?, + expression: text(&arguments.call_method1("get", ("expression",))?)?, + target: arguments + .call_method1("get", ("to", &empty))? + .str()? + .extract()?, + field: arguments + .call_method1("get", ("field", &empty))? + .str()? + .extract()?, + values, + }); + } + Ok(audits) +} + +/// An accepted value by Python type: null, bool, number, text or other. +fn audit_value(value: &Bound<'_, PyAny>) -> PyResult { + if value.is_none() { + return Ok(AuditValue::Null); + } + if value.is_instance_of::() { + return Ok(AuditValue::Bool(value.is_truthy()?)); + } + if value.is_instance_of::() || value.is_instance_of::() { + return Ok(AuditValue::Number(value.str()?.extract()?)); + } + if value.is_instance_of::() { + return Ok(AuditValue::Text(value.extract()?)); + } + Ok(AuditValue::Other) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs index c8ec68a68d..c5c67b1a80 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs @@ -5,4 +5,6 @@ pub(crate) mod metadata; pub(crate) mod registration; pub(crate) mod resource_sql; pub(crate) mod session_facts; +pub(crate) mod stage; +pub(crate) mod stage_objects; pub(crate) mod type_recovery; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs index 9e5dc46d5c..487d03d3ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs @@ -2,14 +2,10 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::semantic_checks::{ - completion, help_catalogue, metadata, resource_sql, type_recovery, -}; +use crate::bindings::_helpers::semantic_checks::{help_catalogue, resource_sql, stage}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - type_recovery::register(module)?; - metadata::register(module)?; resource_sql::register(module)?; help_catalogue::register(module)?; - completion::register(module) + stage::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs new file mode 100644 index 0000000000..9cd3ca1d7b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage.rs @@ -0,0 +1,121 @@ +//! The native semantic completion stage: type recovery, metadata checks, then completion. + +use std::collections::HashSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult}; +use pyo3::types::{PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::_helpers::boundary::panics::NativeCompilerError; +use crate::bindings::_helpers::semantic_checks::completion::completed_diagnostics; +use crate::bindings::_helpers::semantic_checks::metadata::metadata_diagnostics; +use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; +use crate::bindings::_helpers::semantic_checks::type_recovery::{ + StageShared, items, recovered_types, +}; +use crate::bindings::models::ProjectCatalog; + +const NO_CATALOG_MESSAGE: &str = + "NativeCompilerError: native semantic completion: the project has no analysis catalog"; + +/// The completed project from keyword arguments; an internal failure raises `NativeCompilerError`. +#[pyfunction] +#[pyo3(signature = (**request))] +fn complete_semantic_project<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("complete_semantic_project", request)?; + let project: Bound<'py, PyAny> = request.get("project")?; + let profile: Bound<'py, PyAny> = request.get("profile")?; + let binding_results: Bound<'py, PyAny> = request.get("binding_results")?; + let resource_sql_analysis: bool = request.get("resource_sql_analysis")?.extract()?; + let session: Option> = + match request.optional("session")? { + Some(session) if !session.is_none() => Some(session.extract()?), + _ => None, + }; + let catalog: PyRef<'py, ProjectCatalog> = project_catalog(&project)?; + let classes = StageClasses::new(request.py())?; + let shared = StageShared { + catalog: &catalog.inner, + session: session.as_deref(), + session_models: session_fact_models(&project, session.as_deref())?, + }; + let recovered: Bound<'py, PyAny> = + recovered_types(&classes, &project, &binding_results, &shared)?; + let metadata: Vec> = metadata_diagnostics( + &classes, + (&recovered, &profile), + resource_sql_analysis, + &catalog.inner, + )?; + let mut diagnostics: Vec> = items(&recovered.getattr("diagnostics")?)?; + diagnostics.extend(metadata); + let combined: Bound<'py, PyAny> = classes.project_with( + &recovered, + &[( + "diagnostics", + &PyTuple::new(request.py(), diagnostics)?.into_any(), + )], + )?; + completed_diagnostics(&classes, &combined, &shared) +} + +/// The metadata and resource SQL diagnostics alone, as the stage appends them. +#[pyfunction] +#[pyo3(signature = (*, project, profile, resource_sql_analysis))] +fn semantic_metadata_diagnostics<'py>( + project: &Bound<'py, PyAny>, + profile: &Bound<'py, PyAny>, + resource_sql_analysis: bool, +) -> PyResult> { + let catalog: PyRef<'py, ProjectCatalog> = project_catalog(project)?; + let classes = StageClasses::new(project.py())?; + let diagnostics: Vec> = metadata_diagnostics( + &classes, + (project, profile), + resource_sql_analysis, + &catalog.inner, + )?; + PyTuple::new(project.py(), diagnostics) +} + +fn project_catalog<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let binding_catalog: Bound<'py, PyAny> = project.getattr("binding_catalog")?; + if binding_catalog.is_none() { + return Err(NativeCompilerError::new_err(NO_CATALOG_MESSAGE)); + } + binding_catalog.getattr("native")?.extract() +} + +/// Models whose output names and lineage are the finished session's, so need no payload. +fn session_fact_models( + project: &Bound<'_, PyAny>, + session: Option<&NativeModelAnalysisSession>, +) -> PyResult> { + let Some(session) = session else { + return Ok(HashSet::new()); + }; + let kept: HashSet = session.fact_models().into_iter().collect(); + let mut models: HashSet = HashSet::new(); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'_, PyAny> = model?; + let name: String = model.getattr("name")?.extract()?; + if !kept.contains(&name) { + continue; + } + let proof: Bound<'_, PyAny> = model.getattr("dynamic_column_contract")?; + if !proof.is_none() && proof.getattr("output_proven")?.is_truthy()? { + continue; + } + models.insert(name); + } + Ok(models) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(complete_semantic_project, module)?)?; + module.add_function(wrap_pyfunction!(semantic_metadata_diagnostics, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs new file mode 100644 index 0000000000..c766c9b4b8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs @@ -0,0 +1,135 @@ +//! The Python models, enums and compile helpers the semantic completion stage reads and builds. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods}; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; +const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const PROJECT_WITH_MODULE: &str = "sqlbuild.compiler.compiled_project.main.project_with"; +const TYPE_RECOVERY_MODULE: &str = "sqlbuild.compiler.compiled_project.main._with_type_recovery"; +const BINDING_DIAGNOSTICS_MODULE: &str = + "sqlbuild.compiler.compiled_project.main._with_binding_diagnostics"; +const COMPLETION_INPUTS_MODULE: &str = + "sqlbuild.compiler.compile.classes.semantic_completion_inputs"; +const SCHEMA_VALIDATION_MODULE: &str = "sqlbuild.compiler.sql_analysis.main._schema_validation"; +const NORMALIZE_TYPE_MODULE: &str = "sqlbuild.adapter.type_system.main.normalize_type"; +const DATACLASSES_MODULE: &str = "dataclasses"; +const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const REFERENCE_TYPES_MODULE: &str = "sqlbuild.compiler.references.types"; + +/// Imported once per stage run. +pub(crate) struct StageClasses<'py> { + pub(crate) diagnostic: Bound<'py, PyAny>, + pub(crate) location: Bound<'py, PyAny>, + pub(crate) model_test_payload: Bound<'py, PyAny>, + pub(crate) compile_phase: Bound<'py, PyAny>, + pub(crate) error: Bound<'py, PyAny>, + pub(crate) warning: Bound<'py, PyAny>, + pub(crate) model_resource: Bound<'py, PyAny>, + pub(crate) source_resource: Bound<'py, PyAny>, + pub(crate) sql_test_resource: Bound<'py, PyAny>, + pub(crate) opt_out_code: Bound<'py, PyAny>, + pub(crate) replace: Bound<'py, PyAny>, + pub(crate) project_with: Bound<'py, PyAny>, + pub(crate) with_type_recovery: Bound<'py, PyAny>, + pub(crate) with_binding_diagnostics: Bound<'py, PyAny>, + pub(crate) inputs: Bound<'py, PyAny>, + pub(crate) schema_validations: Bound<'py, PyAny>, + pub(crate) normalize_type: Bound<'py, PyAny>, + pub(crate) sql_analysis_key: Bound<'py, PyAny>, + pub(crate) udf_kind: Bound<'py, PyAny>, + pub(crate) table_function_kind: Bound<'py, PyAny>, +} + +impl<'py> StageClasses<'py> { + pub(crate) fn new(py: Python<'py>) -> PyResult { + let models = py.import(COMPILE_MODELS_MODULE)?; + let types = py.import(COMPILE_TYPES_MODULE)?; + let severity = types.getattr("DiagnosticSeverity")?; + let resource = types.getattr("CompiledResourceType")?; + let kinds = py + .import(REFERENCE_TYPES_MODULE)? + .getattr("SqlReferenceKind")?; + Ok(Self { + diagnostic: models.getattr("CompilerDiagnostic")?, + location: py + .import(CONTRACT_MODELS_MODULE)? + .getattr("SourceLocation")?, + model_test_payload: models.getattr("CompiledModelSqlTestPayload")?, + compile_phase: types.getattr("DiagnosticPhase")?.getattr("COMPILE")?, + error: severity.getattr("ERROR")?, + warning: severity.getattr("WARNING")?, + model_resource: resource.getattr("MODEL")?, + source_resource: resource.getattr("SOURCE")?, + sql_test_resource: resource.getattr("SQL_TEST")?, + opt_out_code: py + .import(COMPILE_CONSTANTS_MODULE)? + .getattr("UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE")?, + replace: py.import(DATACLASSES_MODULE)?.getattr("replace")?, + project_with: py.import(PROJECT_WITH_MODULE)?.getattr("project_with")?, + with_type_recovery: py + .import(TYPE_RECOVERY_MODULE)? + .getattr("with_type_recovery")?, + with_binding_diagnostics: py + .import(BINDING_DIAGNOSTICS_MODULE)? + .getattr("with_binding_diagnostics")?, + inputs: py + .import(COMPLETION_INPUTS_MODULE)? + .getattr("SemanticCompletionInputs")?, + schema_validations: py + .import(SCHEMA_VALIDATION_MODULE)? + .getattr("get_schema_validations")?, + normalize_type: py + .import(NORMALIZE_TYPE_MODULE)? + .getattr("normalize_type")?, + sql_analysis_key: py + .import(DISCOVERY_CONSTANTS_MODULE)? + .getattr("SQL_ANALYSIS_CONFIG_KEY")?, + udf_kind: kinds.getattr("UDF")?, + table_function_kind: kinds.getattr("TABLE_FUNCTION")?, + }) + } + + pub(crate) fn py(&self) -> Python<'py> { + self.diagnostic.py() + } + + /// `callable(**pairs)`. + pub(crate) fn call( + &self, + callable: &Bound<'py, PyAny>, + pairs: &[(&str, &Bound<'py, PyAny>)], + ) -> PyResult> { + let kwargs: Bound<'py, PyDict> = PyDict::new(self.py()); + for (name, value) in pairs { + kwargs.set_item(*name, *value)?; + } + callable.call((), Some(&kwargs)) + } + + /// `dataclasses.replace(value, **pairs)`. + pub(crate) fn replaced( + &self, + value: &Bound<'py, PyAny>, + pairs: &[(&str, &Bound<'py, PyAny>)], + ) -> PyResult> { + let kwargs: Bound<'py, PyDict> = PyDict::new(self.py()); + for (name, value) in pairs { + kwargs.set_item(*name, *value)?; + } + self.replace.call((value,), Some(&kwargs)) + } + + /// `project_with(project=project, **pairs)`. + pub(crate) fn project_with( + &self, + project: &Bound<'py, PyAny>, + pairs: &[(&str, &Bound<'py, PyAny>)], + ) -> PyResult> { + let mut all: Vec<(&str, &Bound<'py, PyAny>)> = vec![("project", project)]; + all.extend_from_slice(pairs); + self.call(&self.project_with, &all) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs index c95f6030ed..85df3f79b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/type_recovery.rs @@ -1,14 +1,16 @@ -//! Native output-type recovery for failed models, with Python's revalidation in between. +//! Native output-type recovery for failed models, revalidating planned models in Python. -use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyRef, PyResult, Python}; -use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use std::collections::HashSet; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple}; use sqlbuild_analysis::semantic_checks::main::finish_type_recovery::finish_type_recovery; use sqlbuild_analysis::semantic_checks::main::plan_type_recovery::plan_type_recovery; use sqlbuild_analysis::semantic_checks::models::{ - DiagnosticOwner, LineageOutput, ModelBinding, RawBinding, RecoveryModel, TypeRecoveryPlan, - TypeRecoveryRequest, TypeRecoveryStep, + DiagnosticOwner, LineageOutput, ModelBinding, RawBinding, RecoveryModel, TypeRecoveryRequest, }; use sqlbuild_analysis::semantic_checks::types::RevisedBinding; +use sqlbuild_analysis::semantic_validation::models as validation; use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; use crate::bindings::_helpers::boundary::panics::compiler_error; @@ -16,91 +18,170 @@ use crate::bindings::_helpers::semantic_checks::failures::{internal_error, seman use crate::bindings::_helpers::semantic_checks::session_facts::{ column_names, session_model_facts, }; -use crate::bindings::models::ProjectCatalog; +use crate::bindings::_helpers::semantic_checks::stage_objects::StageClasses; use crate::bindings::types::CompilerDetach; +const UNCHANGED: &str = "unchanged"; +const UNKNOWN_RECOVERED_TYPE: &str = "UNKNOWN"; + /// `(output_column, [(resource_name, column_name)])`. pub(crate) type LineageInput = (String, Vec<(String, String)>); -/// `(name, query_sql, inferred names, reference names, lineage, bindings, raw bindings)`. -type ModelInput = ( - String, - String, - Option>, - Vec, - Option>, - Vec<(usize, String, String, bool)>, - Vec<(usize, String, String, Option, Option)>, -); -/// `(dialect, models, [(id, code, is_model, resource_name)])`. -type RequestInput = ( - Option, - Vec, - Vec<(usize, String, bool, Option)>, -); -/// `(kept [(diagnostic index, appended note)], model binding positions)`. -type OutcomeRow = (Vec<(usize, Option)>, Vec>); - -/// One planned type recovery, held between the native plan and Python's revalidation. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct SemanticTypeRecovery { - request: TypeRecoveryRequest, - step: TypeRecoveryStep, -} -#[pymethods] -impl SemanticTypeRecovery { - /// `unchanged` or `planned`. - #[getter] - fn status(&self) -> &'static str { - self.step.status() - } - - /// Poisoned `(model, column)` outputs in Python's dict order. - #[getter] - fn poisoned(&self) -> Vec<(String, String)> { - self.step - .plan() - .map(TypeRecoveryPlan::poisoned_outputs) - .unwrap_or_default() - } - - /// Model indexes Python revalidates with unknown poisoned types, in order. - #[getter] - fn revalidated(&self) -> Vec { - self.step - .plan() - .map(TypeRecoveryPlan::revalidated_models) - .unwrap_or_default() - } - - /// Retained diagnostics and model binding positions; an internal failure raises. - fn finish(&self, revised: Vec>) -> PyResult { - let plan = self - .step - .plan() - .ok_or_else(|| internal_error("finishing a type recovery that has no plan"))?; - finish_type_recovery(&self.request, plan, &revised) - .map(|outcome| (outcome.kept, outcome.model_bindings)) - .map_err(|reason| internal_error(&reason)) - } +/// The analysis state every semantic completion step shares. +pub(crate) struct StageShared<'a> { + pub(crate) catalog: &'a validation::ProjectCatalog, + pub(crate) session: Option<&'a NativeModelAnalysisSession>, + /// Models whose output names and lineage the finished session supplies. + pub(crate) session_models: HashSet, } -/// Plan type recovery, reading names and lineage from `session` for models with no lineage. -#[pyfunction] -#[pyo3(signature = (catalog, request, session=None))] -fn plan_semantic_type_recovery( - py: Python<'_>, - catalog: PyRef<'_, ProjectCatalog>, - request: RequestInput, - session: Option>, -) -> PyResult { - let request = recovery_request(request, session.as_deref()).map_err(compiler_error)?; - let catalog = &catalog.inner; +/// The project with recovered output types; unchanged when no model has a blocking binding. +pub(crate) fn recovered_types<'py>( + classes: &StageClasses<'py>, + project: &Bound<'py, PyAny>, + binding_results: &Bound<'py, PyAny>, + shared: &StageShared<'_>, +) -> PyResult> { + let py: Python<'py> = classes.py(); + let models: Vec> = items(&project.getattr("models")?)?; + if !blocking_binding_found(&models)? { + return Ok(project.clone()); + } + let ids: Bound<'py, PyDict> = diagnostic_ids(project, &models)?; + let raw_ids: Bound<'py, PyDict> = PyDict::new(py); + for rows in binding_results.call_method0("values")?.try_iter()? { + for row in rows?.try_iter()? { + let row: Bound<'py, PyAny> = row?; + if !raw_ids.contains(&row)? { + raw_ids.set_item(&row, raw_ids.len())?; + } + } + } + let diagnostics: Vec> = items(&project.getattr("diagnostics")?)?; + let mut owners: Vec = Vec::with_capacity(diagnostics.len()); + for item in &diagnostics { + owners.push(DiagnosticOwner { + id: id_of(&ids, item)?, + code: item.getattr("code")?.extract()?, + is_model: item.getattr("resource_type")?.eq(&classes.model_resource)?, + resource_name: item.getattr("resource_name")?.extract()?, + }); + } + let mut recovery_models: Vec = Vec::with_capacity(models.len()); + for model in &models { + recovery_models.push(recovery_model( + model, + binding_results, + (&ids, &raw_ids), + shared, + )?); + } + let request = TypeRecoveryRequest { + dialect: project.getattr("sql_analysis_dialect")?.extract()?, + models: recovery_models, + diagnostics: owners, + }; let step = py - .compiler_detach(|| Ok(plan_type_recovery(&request, catalog))) + .compiler_detach(|| Ok(plan_type_recovery(&request, shared.catalog))) .map_err(compiler_error)? .map_err(|failure| semantic_error(py, &failure))?; - Ok(SemanticTypeRecovery { request, step }) + if step.status() == UNCHANGED { + return Ok(project.clone()); + } + let plan = step + .plan() + .ok_or_else(|| internal_error("finishing a type recovery that has no plan"))?; + let poisoned: Vec<(String, String)> = plan.poisoned_outputs(); + let revised: Vec> = + revalidated(classes, project, &plan.revalidated_models(), &poisoned)?; + let outcome = + finish_type_recovery(&request, plan, &revised).map_err(|reason| internal_error(&reason))?; + let mut kept: Vec> = Vec::with_capacity(outcome.kept.len()); + for (index, note) in outcome.kept { + let original: &Bound<'py, PyAny> = &diagnostics[index]; + let mut notes: Vec> = items(&original.getattr("notes")?)?; + if let Some(note) = note { + notes.push(PyString::new(py, ¬e).into_any()); + } + kept.push(classes.replaced(original, &[("notes", &PyTuple::new(py, notes)?.into_any())])?); + } + let poisoned_outputs: Bound<'py, PyAny> = PyFrozenSet::new(py, &poisoned)?.into_any(); + let mut recovered: Vec> = Vec::with_capacity(models.len()); + for (model, positions) in models.iter().zip(outcome.model_bindings) { + let selected: Vec> = positions + .into_iter() + .map(|position| kept[position].clone()) + .collect(); + recovered.push(classes.call( + &classes.with_type_recovery, + &[ + ("model", model), + ("diagnostics", &PyTuple::new(py, selected)?.into_any()), + ("poisoned", &poisoned_outputs), + ], + )?); + } + classes.project_with( + project, + &[ + ("models", &PyTuple::new(py, recovered)?.into_any()), + ("diagnostics", &PyTuple::new(py, kept)?.into_any()), + ], + ) +} + +/// Every item of a Python iterable. +pub(crate) fn items<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { + value.try_iter()?.collect() +} + +/// One id per distinct diagnostic value, project diagnostics first, as Python's dicts see them. +pub(crate) fn diagnostic_ids<'py>( + project: &Bound<'py, PyAny>, + models: &[Bound<'py, PyAny>], +) -> PyResult> { + let ids: Bound<'py, PyDict> = PyDict::new(project.py()); + for item in project.getattr("diagnostics")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + if !ids.contains(&item)? { + ids.set_item(&item, ids.len())?; + } + } + for model in models { + for item in model.getattr("binding_diagnostics")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + if !ids.contains(&item)? { + ids.set_item(&item, ids.len())?; + } + } + } + Ok(ids) +} + +pub(crate) fn id_of(ids: &Bound<'_, PyDict>, item: &Bound<'_, PyAny>) -> PyResult { + ids.as_any().get_item(item)?.extract() +} + +/// Each output column with the `(resource, column)` pairs it reads. +pub(crate) fn lineage_payload(model: &Bound<'_, PyAny>) -> PyResult> { + let columns: Bound<'_, PyAny> = model.getattr("fast_lineage_columns")?; + if !columns.is_truthy()? { + return Ok(Vec::new()); + } + let mut lineage: Vec = Vec::new(); + for output in columns.try_iter()? { + let output: Bound<'_, PyAny> = output?; + let mut upstream: Vec<(String, String)> = Vec::new(); + for source in output.getattr("upstream_columns")?.try_iter()? { + let source: Bound<'_, PyAny> = source?; + upstream.push(( + source.getattr("resource_name")?.extract()?, + source.getattr("column_name")?.extract()?, + )); + } + lineage.push((output.getattr("output_column")?.extract()?, upstream)); + } + Ok(lineage) } pub(crate) fn lineage_outputs(lineage: Vec) -> Vec { @@ -113,69 +194,137 @@ pub(crate) fn lineage_outputs(lineage: Vec) -> Vec .collect() } -fn recovery_request( - (dialect, models, diagnostics): RequestInput, - session: Option<&NativeModelAnalysisSession>, -) -> Result { - Ok(TypeRecoveryRequest { - dialect, - models: models - .into_iter() - .map(|model| recovery_model(model, session)) - .collect::>()?, - diagnostics: diagnostics - .into_iter() - .map(|(id, code, is_model, resource_name)| DiagnosticOwner { - id, - code, - is_model, - resource_name, - }) - .collect(), - }) +/// The names of a model's inferred columns, or `None` when its shape is unknown. +pub(crate) fn inferred_names(model: &Bound<'_, PyAny>) -> PyResult>> { + let columns: Bound<'_, PyAny> = model.getattr("inferred_columns")?; + if columns.is_none() { + return Ok(None); + } + columns + .try_iter()? + .map(|column| column?.getattr("name")?.extract()) + .collect::>() + .map(Some) +} + +fn blocking_binding_found(models: &[Bound<'_, PyAny>]) -> PyResult { + for model in models { + for item in model.getattr("binding_diagnostics")?.try_iter()? { + if item?.getattr("is_error")?.is_truthy()? { + return Ok(true); + } + } + } + Ok(false) } fn recovery_model( - (name, query_sql, inferred_columns, references, lineage, bindings, raw_bindings): ModelInput, - session: Option<&NativeModelAnalysisSession>, -) -> Result { - let (inferred_columns, lineage) = match lineage { - Some(lineage) => (inferred_columns, lineage), - None => { - let facts = session_model_facts(session, &name)?; - (column_names(facts), facts.lineage.clone()) + model: &Bound<'_, PyAny>, + binding_results: &Bound<'_, PyAny>, + (ids, raw_ids): (&Bound<'_, PyDict>, &Bound<'_, PyDict>), + shared: &StageShared<'_>, +) -> PyResult { + let name: String = model.getattr("name")?.extract()?; + let from_session: bool = shared.session_models.contains(&name); + let bindings: Vec> = items(&model.getattr("binding_diagnostics")?)?; + let mut model_bindings: Vec = Vec::with_capacity(bindings.len()); + for item in &bindings { + model_bindings.push(ModelBinding { + id: id_of(ids, item)?, + code: item.getattr("code")?.extract()?, + message: item.getattr("message")?.extract()?, + is_error: item.getattr("is_error")?.extract()?, + }); + } + let mut raw_bindings: Vec = Vec::new(); + if !bindings.is_empty() { + let raws: Bound<'_, PyAny> = + binding_results.call_method1("get", (&name, PyList::empty(model.py())))?; + for raw in raws.try_iter()? { + let raw: Bound<'_, PyAny> = raw?; + raw_bindings.push(RawBinding { + id: id_of(raw_ids, &raw)?, + code: raw.get_item(0)?.extract()?, + message: raw.get_item(1)?.extract()?, + start: raw.get_item(4)?.extract()?, + end: raw.get_item(5)?.extract()?, + }); } + } + let (inferred_columns, lineage) = if from_session { + let facts = session_model_facts(shared.session, &name).map_err(compiler_error)?; + (column_names(facts), facts.lineage.clone()) + } else { + (inferred_names(model)?, lineage_payload(model)?) }; + let mut references: Vec = Vec::new(); + for reference in model.getattr("references")?.try_iter()? { + references.push(reference?.getattr("ref_name")?.extract()?); + } Ok(RecoveryModel { name, - query_sql, + query_sql: model.getattr("query_sql")?.extract()?, inferred_columns, references, lineage: lineage_outputs(lineage), - bindings: bindings - .into_iter() - .map(|(id, code, message, is_error)| ModelBinding { - id, - code, - message, - is_error, - }) - .collect(), - raw_bindings: raw_bindings - .into_iter() - .map(|(id, code, message, start, end)| RawBinding { - id, - code, - message, - start, - end, - }) - .collect(), + bindings: model_bindings, + raw_bindings, }) } -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_class::()?; - module.add_function(wrap_pyfunction!(plan_semantic_type_recovery, module)?)?; - Ok(()) +/// Python's revalidation of each planned model with unknown types for poisoned outputs. +fn revalidated<'py>( + classes: &StageClasses<'py>, + project: &Bound<'py, PyAny>, + models: &[usize], + poisoned: &[(String, String)], +) -> PyResult>> { + if models.is_empty() { + return Ok(Vec::new()); + } + let py: Python<'py> = classes.py(); + let shapes: Bound<'py, PyAny> = classes.inputs.call_method1("shapes", (project,))?; + let unknown: Bound<'py, PyDict> = PyDict::new(py); + for item in shapes.call_method0("items")?.try_iter()? { + let (name, columns): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let copied: Bound<'py, PyDict> = PyDict::new(py); + copied.update(columns.downcast::()?.as_mapping())?; + unknown.set_item(name, copied)?; + } + for (name, column) in poisoned { + if let Some(columns) = unknown.get_item(name)? + && columns.contains(column)? + { + columns.set_item(column, UNKNOWN_RECOVERED_TYPE)?; + } + } + let project_models: Bound<'py, PyAny> = project.getattr("models")?; + let mut revised: Vec> = Vec::with_capacity(models.len()); + for index in models { + let request: Bound<'py, PyAny> = classes.call( + &classes.inputs.getattr("revalidation_request")?, + &[ + ("project", project), + ("model", &project_models.get_item(*index)?), + ("shapes", &unknown.clone().into_any()), + ], + )?; + let result: Bound<'py, PyAny> = classes + .call( + &classes.schema_validations, + &[("requests", &PyTuple::new(py, [request])?.into_any())], + )? + .get_item(0)?; + let mut rows: Vec = Vec::new(); + for item in result.getattr("diagnostics")?.try_iter()? { + let item: Bound<'py, PyAny> = item?; + rows.push(( + item.getattr("code")?.extract()?, + item.getattr("start")?.extract()?, + item.getattr("end")?.extract()?, + )); + } + revised.push(rows); + } + Ok(revised) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 521dc996e7..5c0f0e376c 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1782,112 +1782,17 @@ def _oracle_cte_fact_recovery( ) -> tuple[str | None, list[tuple[str, str]], list[tuple[str, str]], list[str], list[str]]: ... # Native analysis: semantic completion. -class SemanticTypeRecovery: - @property - def status(self) -> str: ... - @property - def poisoned(self) -> list[tuple[str, str]]: ... - @property - def revalidated(self) -> list[int]: ... - def finish( - self, revised: list[list[tuple[str, int | None, int | None]]], / - ) -> tuple[list[tuple[int, str | None]], list[list[int]]]: ... - -def plan_semantic_type_recovery( - catalog: object, - request: tuple[ - str | None, - list[ - tuple[ - str, - str, - list[str] | None, - list[str], - list[tuple[str, list[tuple[str, str]]]] | None, - list[tuple[int, str, str, bool]], - list[tuple[int, str, str, int | None, int | None]], - ] - ], - list[tuple[int, str, bool, str | None]], - ], - session: NativeModelAnalysisSession | None = None, - /, -) -> SemanticTypeRecovery: ... -def check_semantic_metadata_rows( - catalog: object, - request: tuple[ - str | None, - bool, - list[tuple[str, str]], - list[tuple[str, list[tuple[str, str]]]], - list[tuple[str, list[tuple[str, str]]]], - list[tuple[object, ...]], - list[str], - list[tuple[str, str | None, int]], - list[tuple[int, list[tuple[str, list[str]]]]], - ], - /, -) -> tuple[ - list[ - tuple[ - list[tuple[str, str, int, int]], - list[tuple[str, str, int, int, bool]], - list[tuple[str, str, int, int]], - ] - ], - list[tuple[int, tuple[str, str, int, int]]], - list[tuple[int, tuple[str, str, int, int], int]], - list[str], -]: ... -def complete_semantic_checks( - catalog: object, - request: tuple[ - str | None, - list[ - tuple[ - int, - str, - str, - str | None, - str | None, - int | None, - int | None, - tuple[int, int, int | None, int | None] | None, - str | None, - list[str], - ] - ], - list[ - tuple[ - str, - str, - str, - list[str] | None, - list[tuple[str, list[tuple[str, str]]]] | None, - list[str], - str | None, - ] - ], - list[tuple[str, list[tuple[str, str]]]], - ], +def complete_semantic_project( + *, + project: Any, + profile: Any, + binding_results: dict[str, list[tuple[str, str, int | None, int | None, int | None, int | None, str]]], + resource_sql_analysis: bool, session: NativeModelAnalysisSession | None = None, - /, -) -> tuple[ - list[ - tuple[ - int, - str, - str | None, - list[str], - tuple[int, int, int | None, int | None] | None, - int | None, - int | None, - bool, - ] - ], - list[list[int]] | None, - list[tuple[int | None, tuple[int, str, str, str] | None]], -]: ... +) -> Any: ... +def semantic_metadata_diagnostics( + *, project: Any, profile: Any, resource_sql_analysis: bool +) -> tuple[Any, ...]: ... # Native analysis: contracts. type _ContractDiagnosticRow = tuple[ diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py index d62407d202..7aba6a4b26 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py @@ -4,12 +4,10 @@ from typing import Any +import sqlbuild._native as _native from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.types import DiagnosticRow from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.semantic_checks.main._complete_native_semantic_diagnostics import ( - complete_native_semantic_diagnostics, -) def complete_semantic_diagnostics( @@ -22,7 +20,7 @@ def complete_semantic_diagnostics( ) -> CompiledProjectView: """Recover type facts before metadata checks, then explain retained root diagnostics.""" - return complete_native_semantic_diagnostics( + return _native.complete_semantic_project( project=project, profile=profile, binding_results=binding_results, diff --git a/src/sqlbuild/compiler/semantic_checks/__init__.py b/src/sqlbuild/compiler/semantic_checks/__init__.py deleted file mode 100644 index 6796d94ab5..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native semantic completion.""" diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/__init__.py b/src/sqlbuild/compiler/semantic_checks/_helpers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py b/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py deleted file mode 100644 index d4bc4e255a..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/completion.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Native diagnostic recovery, explanations and opt-out rejection after the metadata checks.""" - -from __future__ import annotations - -from dataclasses import replace -from typing import Any - -import sqlbuild._native as _native -from sqlbuild._native import ( - project_with, - with_binding_diagnostics, -) -from sqlbuild.compiler.compile.classes.semantic_completion_inputs import SemanticCompletionInputs -from sqlbuild.compiler.compile.constants import UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - DiagnosticPhase, - DiagnosticSeverity, -) -from sqlbuild.compiler.semantic_checks._helpers.payloads import diagnostic_ids, lineage_payload -from sqlbuild.spec.contracts.models import SourceLocation - -_RECOVERED_CODE: str = "B002" - -type _Location = tuple[int, int, int | None, int | None] -type _Completed = tuple[ - int, str, str | None, list[str], _Location | None, int | None, int | None, bool -] - - -def complete_native_diagnostics( - *, - project: CompiledProjectView, - catalog: object, - session: Any | None, - session_models: frozenset[str], -) -> CompiledProjectView: - """The recovered, explained and opt-out-checked project; a native internal failure raises.""" - - ids: dict[CompilerDiagnostic, int] = diagnostic_ids(project) - recovers: bool = any(item.code == _RECOVERED_CODE for item in project.diagnostics) - shapes: dict[str, dict[str, str]] = ( - SemanticCompletionInputs.shapes(project) if project.diagnostics else {} - ) - request: Any = ( - project.sql_analysis_dialect, - [_diagnostic_payload(item=item, ids=ids) for item in project.diagnostics], - [ - _model_payload(model=model, lineage=recovers, from_session=model.name in session_models) - for model in project.models - ], - [(name, list(columns.items())) for name, columns in shapes.items()], - ) - completed, model_bindings, order = _native.complete_semantic_checks(catalog, request, session) - diagnostics: list[CompilerDiagnostic] = [ - _completed(original=project.diagnostics[row[0]], row=row) for row in completed - ] - models: tuple[CompiledModelView, ...] = project.models - if model_bindings is not None: - models = tuple( - with_binding_diagnostics( - model=model, diagnostics=_positions(diagnostics=diagnostics, positions=positions) - ) - for model, positions in zip(project.models, model_bindings, strict=True) - ) - final: list[CompilerDiagnostic] = [] - for position, opt_out in order: - if position is not None: - final.append(diagnostics[position]) - elif opt_out is not None: - index, message, note, help_text = opt_out - final.append( - _opt_out( - model=project.models[index], message=message, note=note, help_text=help_text - ) - ) - return project_with(project=project, models=models, diagnostics=tuple(final)) - - -def _positions( - *, diagnostics: list[CompilerDiagnostic], positions: list[int] -) -> tuple[CompilerDiagnostic, ...]: - return tuple(diagnostics[position] for position in positions) - - -def _diagnostic_payload( - *, item: CompilerDiagnostic, ids: dict[CompilerDiagnostic, int] -) -> tuple[Any, ...]: - location: SourceLocation | None = item.location - return ( - ids[item], - item.code, - item.message, - None if item.resource_type is None else str(item.resource_type), - item.resource_name, - item.line, - item.column, - None - if location is None - else (location.line, location.column, location.end_line, location.end_column), - item.help, - list(item.notes), - ) - - -def _model_payload( - *, model: CompiledModelView, lineage: bool, from_session: bool -) -> tuple[Any, ...]: - opt_out: SourceLocation | None = model.rejected_sql_analysis_opt_out - return ( - model.name, - model.query_sql, - model.authored_sql, - None if from_session else [column.name for column in model.inferred_columns or ()], - _lineage(model=model, lineage=lineage, from_session=from_session), - [item.code for item in model.binding_diagnostics], - None if opt_out is None else opt_out.path.name, - ) - - -def _completed(*, original: CompilerDiagnostic, row: _Completed) -> CompilerDiagnostic: - _, message, help_text, notes, location, line, column, changed = row - if not changed: - return original - current: SourceLocation | None = original.location - if current is not None and location is not None: - line_number, column_number, end_line, end_column = location - if (current.line, current.column, current.end_line, current.end_column) != location: - current = replace( - current, - line=line_number, - column=column_number, - end_line=end_line, - end_column=end_column, - ) - return replace( - original, - message=message, - help=help_text, - notes=tuple(notes), - location=current, - line=line, - column=column, - ) - - -def _opt_out( - *, model: CompiledModelView, message: str, note: str, help_text: str -) -> CompilerDiagnostic: - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=UNNEEDED_SQL_ANALYSIS_OPT_OUT_CODE, - message=message, - resource_type=CompiledResourceType.MODEL, - resource_name=model.name, - location=model.rejected_sql_analysis_opt_out, - notes=(note,), - help=help_text, - ) - - -def _lineage( - *, model: CompiledModelView, lineage: bool, from_session: bool -) -> list[tuple[str, list[tuple[str, str]]]] | None: - """The model's lineage rows; None asks for the session's, and only recovery reads them.""" - - if not lineage: - return [] - if from_session: - return None - return lineage_payload(model) diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/metadata.py b/src/sqlbuild/compiler/semantic_checks/_helpers/metadata.py deleted file mode 100644 index 5c96d1a84f..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/metadata.py +++ /dev/null @@ -1,301 +0,0 @@ -"""Native semantic metadata checks, with Python's audit and resource SQL checks in order.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any, cast - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.adapter.type_system.main.normalize_type import normalize_type -from sqlbuild.compiler.compile.classes.semantic_completion_inputs import SemanticCompletionInputs -from sqlbuild.compiler.compile.models import ( - CompiledModelSqlTestPayload, - CompiledSqlTest, - CompilerDiagnostic, -) -from sqlbuild.compiler.compile.types import ( - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - CompiledSourceView, - DiagnosticPhase, - DiagnosticSeverity, -) -from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_CONFIG_KEY -from sqlbuild.compiler.semantic_checks.constants import ( - METADATA_CONFIG_KEYS, - METADATA_FUNCTION_REFERENCE_KINDS, -) -from sqlbuild.spec.contracts.models import SchemaAuditInstance, SourceLocation - -type _ErrorRow = tuple[str, str, int, int] -type _NamesRow = tuple[str, list[str]] - - -def native_metadata_diagnostics( - *, - project: CompiledProjectView, - profile: ExpressionInferenceProfile, - resource_sql_analysis: bool, - catalog: object, -) -> tuple[CompilerDiagnostic, ...]: - """The project's semantic metadata diagnostics; a native internal failure raises.""" - - if not project.settings.sql_analysis: - return () - shapes: dict[str, dict[str, str]] = SemanticCompletionInputs.metadata_shapes( - project=project, profile=profile - ) - tests: list[CompiledSqlTest] = [ - test for test in project.sql_tests if isinstance(test.payload, CompiledModelSqlTestPayload) - ] - columns_by_sql: dict[str, tuple[str, ...]] = SemanticCompletionInputs.sql_test_columns( - project=project, profile=profile - ) - functions: dict[str, CompiledFunctionView] = { - function.name.casefold(): function for function in project.functions - } - file_texts: dict[Path, str] = {} - for source in project.sources: - _ = file_texts.setdefault(source.source_file.file_path, source.source_file.contents) - for test in tests: - _ = file_texts.setdefault(test.test_file.file_path, test.test_file.contents) - files: dict[Path, int] = {path: index for index, path in enumerate(file_texts)} - request: Any = ( - profile.sql_analysis_dialect, - profile.quoted_identifiers_ignore_case, - list(profile.function_return_types.items()), - [_function_payload(key=key, function=function) for key, function in functions.items()], - [(name, list(columns.items())) for name, columns in shapes.items()], - [_model_payload(model) for model in project.models], - list(file_texts.values()), - [_source_payload(source=source, files=files) for source in project.sources], - [ - _sql_test_payload(test=test, columns_by_sql=columns_by_sql, files=files) - for test in tests - ], - ) - model_rows, source_rows, test_rows, fallback_types = _native.check_semantic_metadata_rows( - catalog, request - ) - for type_sql in fallback_types: - _ = normalize_type(type_sql=type_sql, dialect=profile.sql_analysis_dialect) - diagnostics: list[CompilerDiagnostic] = [] - for model, (function_rows, audit_rows, reference_rows) in zip( - project.models, model_rows, strict=True - ): - if not _checked(model): - continue - diagnostics.extend(_model_error(model=model, row=row) for row in function_rows) - diagnostics.extend(_audit_error(model=model, row=row) for row in audit_rows) - diagnostics.extend(_model_error(model=model, row=row) for row in reference_rows) - diagnostics.extend( - _source_error(source=project.sources[index], row=row) for index, row in source_rows - ) - diagnostics.extend( - _sql_test_error(test=tests[index], row=row, end_column=end_column) - for index, row, end_column in test_rows - ) - if resource_sql_analysis: - diagnostics.extend( - SemanticCompletionInputs.resource_sql_diagnostics( - project=project, shapes=shapes, profile=profile - ) - ) - return tuple(diagnostics) - - -def _function_payload( - *, key: str, function: CompiledFunctionView -) -> tuple[str, list[tuple[str, str]]]: - return (key, [(argument.name, argument.type) for argument in function.arguments]) - - -def _checked(model: CompiledModelView) -> bool: - return model.config.values.get(SQL_ANALYSIS_CONFIG_KEY) is not False and model.binding_validated - - -def _string(value: object) -> str | None: - return value if isinstance(value, str) else None - - -def _model_payload(model: CompiledModelView) -> tuple[Any, ...]: - values: dict[str, object] = model.config.values - references: list[_NamesRow] = [ - (key, list(SemanticCompletionInputs.column_names(values.get(key)))) - for key in METADATA_CONFIG_KEYS - ] - custom: object = values.get("config") - if isinstance(custom, dict): - partition: object = cast(dict[str, object], custom).get("partition_column") - references.append( - ("partition_column", list(SemanticCompletionInputs.column_names(partition))) - ) - tolerances: object = values.get("row_diff_tolerances") - by_column: object = ( - cast(dict[str, object], tolerances).get("by_column") - if isinstance(tolerances, dict) - else None - ) - if isinstance(by_column, dict): - references.append(("row_diff_tolerances", [str(name) for name in by_column])) - cursor_inputs: object = values.get("cursor_inputs") - upstreams: list[_NamesRow] = [] - if isinstance(cursor_inputs, dict): - upstreams = [ - (str(upstream), list(SemanticCompletionInputs.column_names(columns))) - for upstream, columns in cursor_inputs.items() - ] - return ( - model.name, - model.query_sql, - model.authored_sql, - _checked(model), - any( - reference.ref_kind in METADATA_FUNCTION_REFERENCE_KINDS - for reference in model.references - ), - references, - upstreams, - _string(values.get("cursor")), - _string(values.get("cursor_type")), - _audit_payloads(model), - ) - - -def _audit_payloads(model: CompiledModelView) -> list[tuple[object, ...]]: - if model.schema_entry is None: - return [] - attached: list[tuple[str | None, SchemaAuditInstance]] = [ - (None, audit) for audit in model.schema_entry.audits - ] - for schema_column in model.schema_entry.columns: - attached.extend((schema_column.name, audit) for audit in schema_column.audits) - payloads: list[tuple[object, ...]] = [] - for column, audit in attached: - arguments: dict[str, object] = audit.arguments - expression: object = arguments.get("expression") - values: object = arguments.get("values") - payloads.append( - ( - column, - audit.definition_name, - expression if isinstance(expression, str) else None, - str(arguments.get("to", "")), - str(arguments.get("field", "")), - [_audit_value(value) for value in values] - if isinstance(values, (list, tuple)) - else None, - ) - ) - return payloads - - -def _audit_value(value: object) -> tuple[str, str]: - """An accepted value by Python type: null, bool, number, text or other.""" - - if value is None: - return ("null", "") - if isinstance(value, bool): - return ("bool", "true" if value else "") - if isinstance(value, int | float): - return ("number", str(value)) - if isinstance(value, str): - return ("text", value) - return ("other", "") - - -def _source_payload( - *, source: CompiledSourceView, files: dict[Path, int] -) -> tuple[str, str | None, int]: - return ( - source.name, - source.source_entry.cursor_column or None, - files[source.source_file.file_path], - ) - - -def _sql_test_payload( - *, test: CompiledSqlTest, columns_by_sql: dict[str, tuple[str, ...]], files: dict[Path, int] -) -> tuple[int, list[_NamesRow]]: - payload: CompiledModelSqlTestPayload = cast(CompiledModelSqlTestPayload, test.payload) - return ( - files[test.test_file.file_path], - [ - (cte.name, list(columns_by_sql[cte.sql_body])) - for cte in (*payload.authored_ctes, *payload.expected_ctes) - ], - ) - - -def _model_error(*, model: CompiledModelView, row: _ErrorRow) -> CompilerDiagnostic: - code, message, line, column = row - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=code, - message=message, - resource_type=CompiledResourceType.MODEL, - resource_name=model.name, - path=model.relative_path, - line=line, - column=column, - ) - - -def _audit_error( - *, model: CompiledModelView, row: tuple[str, str, int, int, bool] -) -> CompilerDiagnostic: - code, message, line, column, is_error = row - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR if is_error else DiagnosticSeverity.WARNING, - code=code, - message=message, - resource_type=CompiledResourceType.MODEL, - resource_name=model.name, - path=model.relative_path, - line=line, - column=column, - ) - - -def _source_error(*, source: CompiledSourceView, row: _ErrorRow) -> CompilerDiagnostic: - code, message, line, column = row - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=code, - message=message, - resource_type=CompiledResourceType.SOURCE, - resource_name=source.name, - path=source.source_file.relative_path, - line=line, - column=column, - ) - - -def _sql_test_error( - *, test: CompiledSqlTest, row: _ErrorRow, end_column: int -) -> CompilerDiagnostic: - code, message, line, column = row - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=code, - message=message, - resource_type=CompiledResourceType.SQL_TEST, - resource_name=test.name, - path=test.test_file.relative_path, - line=line, - column=column, - location=SourceLocation( - path=test.test_file.relative_path, - line=line, - column=column, - end_line=line, - end_column=end_column, - ), - ) diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py b/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py deleted file mode 100644 index 38ba30d904..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/payloads.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Plain-data payloads of compiled facts for the native semantic completion requests.""" - -from __future__ import annotations - -from typing import Any, cast - -from sqlbuild.compiler.analysis_session.types import DiagnosticRow -from sqlbuild.compiler.compile.models import ( - CompiledLineageColumnFact, - CompilerDiagnostic, - DynamicColumnContractProof, -) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView - - -def lineage_payload(model: CompiledModelView) -> list[tuple[str, list[tuple[str, str]]]]: - """Each output column with the `(resource, column)` pairs it reads.""" - - return [_output_payload(output) for output in model.fast_lineage_columns or ()] - - -def session_fact_models(*, project: CompiledProjectView, session: Any | None) -> frozenset[str]: - """Models whose output names and lineage are the finished session's, so need no payload.""" - - if session is None: - return frozenset() - kept: frozenset[str] = frozenset(cast(list[str], session.fact_models)) - return frozenset( - model.name - for model in project.models - if model.name in kept and not _output_proven(model.dynamic_column_contract) - ) - - -def _output_proven(proof: DynamicColumnContractProof | None) -> bool: - """Whether a dynamic proof replaced the analysed output names.""" - - return proof is not None and proof.output_proven - - -def _output_payload(output: CompiledLineageColumnFact) -> tuple[str, list[tuple[str, str]]]: - return ( - output.output_column, - [(source.resource_name, source.column_name) for source in output.upstream_columns], - ) - - -def diagnostic_ids(project: CompiledProjectView) -> dict[CompilerDiagnostic, int]: - """One id per distinct diagnostic value, as Python's value-keyed dicts and sets see them.""" - - ids: dict[CompilerDiagnostic, int] = {} - for item in project.diagnostics: - _ = ids.setdefault(item, len(ids)) - for model in project.models: - for item in model.binding_diagnostics: - _ = ids.setdefault(item, len(ids)) - return ids - - -def raw_binding_ids( - binding_results: dict[str, list[DiagnosticRow]], -) -> dict[DiagnosticRow, int]: - """One id per distinct raw binding row value.""" - - ids: dict[DiagnosticRow, int] = {} - for rows in binding_results.values(): - for row in rows: - _ = ids.setdefault(row, len(ids)) - return ids - - -def has_blocking_binding(project: CompiledProjectView) -> bool: - """Whether any model owns a binding error, without which type recovery changes nothing.""" - - for model in project.models: - for item in model.binding_diagnostics: - if item.is_error: - return True - return False diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py b/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py deleted file mode 100644 index 69b50053a0..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/stage.py +++ /dev/null @@ -1,54 +0,0 @@ -"""The native semantic completion stage: type recovery, metadata checks, then completion.""" - -from __future__ import annotations - -from typing import Any - -import sqlbuild._native as _native -from sqlbuild._native import ( - project_with, -) -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.types import DiagnosticRow -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.semantic_checks._helpers.completion import complete_native_diagnostics -from sqlbuild.compiler.semantic_checks._helpers.metadata import native_metadata_diagnostics -from sqlbuild.compiler.semantic_checks._helpers.payloads import session_fact_models -from sqlbuild.compiler.semantic_checks._helpers.type_recovery import recover_native_output_types -from sqlbuild.compiler.semantic_checks.constants import NATIVE_SEMANTIC_NO_CATALOG - - -def completed_semantic_project( - *, - project: CompiledProjectView, - profile: ExpressionInferenceProfile, - binding_results: dict[str, list[DiagnosticRow]], - resource_sql_analysis: bool, - session: Any | None, -) -> CompiledProjectView: - """The completed project; a native internal failure raises `NativeCompilerError`.""" - - if project.binding_catalog is None: - raise _native.NativeCompilerError(NATIVE_SEMANTIC_NO_CATALOG) - catalog: object = project.binding_catalog.native - session_models: frozenset[str] = session_fact_models(project=project, session=session) - recovered: CompiledProjectView = recover_native_output_types( - project=project, - binding_results=binding_results, - catalog=catalog, - session=session, - session_models=session_models, - ) - metadata: tuple[CompilerDiagnostic, ...] = native_metadata_diagnostics( - project=recovered, - profile=profile, - resource_sql_analysis=resource_sql_analysis, - catalog=catalog, - ) - return complete_native_diagnostics( - project=project_with(project=recovered, diagnostics=(*recovered.diagnostics, *metadata)), - catalog=catalog, - session=session, - session_models=session_models, - ) diff --git a/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py b/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py deleted file mode 100644 index 32440c2c31..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/_helpers/type_recovery.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Native output-type recovery, with Python's revalidation on the compile's binding catalog.""" - -from __future__ import annotations - -from dataclasses import replace -from typing import Any - -import sqlbuild._native as _native -from sqlbuild._native import ( - project_with, - with_type_recovery, -) -from sqlbuild.compiler.analysis_session.types import DiagnosticRow -from sqlbuild.compiler.compile.classes.semantic_completion_inputs import SemanticCompletionInputs -from sqlbuild.compiler.compile.models import ( - CompilerDiagnostic, -) -from sqlbuild.compiler.compile.types import ( - CompiledModelView, - CompiledProjectView, - CompiledResourceType, -) -from sqlbuild.compiler.semantic_checks._helpers.payloads import ( - diagnostic_ids, - has_blocking_binding, - lineage_payload, - raw_binding_ids, -) -from sqlbuild.compiler.semantic_checks.constants import ( - NATIVE_TYPE_RECOVERY_UNCHANGED, - UNKNOWN_RECOVERED_TYPE, -) -from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations -from sqlbuild.compiler.sql_analysis.models import SqlBindingResult - -type _Revised = list[tuple[str, int | None, int | None]] - - -def recover_native_output_types( - *, - project: CompiledProjectView, - binding_results: dict[str, list[DiagnosticRow]], - catalog: object, - session: Any | None, - session_models: frozenset[str], -) -> CompiledProjectView: - """The project with recovered output types; a native internal failure raises.""" - - if not has_blocking_binding(project): - return project - ids: dict[CompilerDiagnostic, int] = diagnostic_ids(project) - raw_ids: dict[DiagnosticRow, int] = raw_binding_ids(binding_results) - request: Any = ( - project.sql_analysis_dialect, - [ - _model_payload( - model=model, - binding_results=binding_results, - ids=ids, - raw_ids=raw_ids, - from_session=model.name in session_models, - ) - for model in project.models - ], - [ - ( - ids[item], - item.code, - item.resource_type == CompiledResourceType.MODEL, - item.resource_name, - ) - for item in project.diagnostics - ], - ) - recovery: _native.SemanticTypeRecovery = _native.plan_semantic_type_recovery( - catalog, request, session - ) - if recovery.status == NATIVE_TYPE_RECOVERY_UNCHANGED: - return project - poisoned: list[tuple[str, str]] = recovery.poisoned - outcome: tuple[list[tuple[int, str | None]], list[list[int]]] = recovery.finish( - _revalidated(project=project, revalidated=recovery.revalidated, poisoned=poisoned) - ) - kept, model_bindings = outcome - diagnostics: tuple[CompilerDiagnostic, ...] = tuple( - _with_note(diagnostic=project.diagnostics[index], note=note) for index, note in kept - ) - poisoned_outputs: frozenset[tuple[str, str]] = frozenset(poisoned) - models: tuple[CompiledModelView, ...] = tuple( - with_type_recovery( - model=model, - diagnostics=_positions(diagnostics=diagnostics, positions=positions), - poisoned=poisoned_outputs, - ) - for model, positions in zip(project.models, model_bindings, strict=True) - ) - return project_with(project=project, models=models, diagnostics=diagnostics) - - -def _revalidated( - *, project: CompiledProjectView, revalidated: list[int], poisoned: list[tuple[str, str]] -) -> list[_Revised]: - """Python's revalidation of each planned model with unknown types for poisoned outputs.""" - - if not revalidated: - return [] - shapes: dict[str, dict[str, str]] = _unknown_shapes( - shapes=SemanticCompletionInputs.shapes(project), poisoned=poisoned - ) - revised: list[_Revised] = [] - for index in revalidated: - result: SqlBindingResult = get_schema_validations( - requests=( - SemanticCompletionInputs.revalidation_request( - project=project, model=project.models[index], shapes=shapes - ), - ) - )[0] - revised.append([(item.code, item.start, item.end) for item in result.diagnostics]) - return revised - - -def _model_payload( - *, - model: CompiledModelView, - binding_results: dict[str, list[DiagnosticRow]], - ids: dict[CompilerDiagnostic, int], - raw_ids: dict[DiagnosticRow, int], - from_session: bool, -) -> tuple[Any, ...]: - raws: list[DiagnosticRow] = ( - binding_results.get(model.name, []) if model.binding_diagnostics else [] - ) - return ( - model.name, - model.query_sql, - None - if from_session or model.inferred_columns is None - else [column.name for column in model.inferred_columns], - [reference.ref_name for reference in model.references], - None if from_session else lineage_payload(model), - [(ids[item], item.code, item.message, item.is_error) for item in model.binding_diagnostics], - [(raw_ids[raw], raw[0], raw[1], raw[4], raw[5]) for raw in raws], - ) - - -def _with_note(*, diagnostic: CompilerDiagnostic, note: str | None) -> CompilerDiagnostic: - return replace(diagnostic, notes=(*diagnostic.notes, *((note,) if note is not None else ()))) - - -def _positions( - *, diagnostics: tuple[CompilerDiagnostic, ...], positions: list[int] -) -> tuple[CompilerDiagnostic, ...]: - return tuple(diagnostics[position] for position in positions) - - -def _unknown_shapes( - *, shapes: dict[str, dict[str, str]], poisoned: list[tuple[str, str]] -) -> dict[str, dict[str, str]]: - unknown: dict[str, dict[str, str]] = {name: dict(columns) for name, columns in shapes.items()} - for name, column in poisoned: - if name in unknown and column in unknown[name]: - unknown[name][column] = UNKNOWN_RECOVERED_TYPE - return unknown diff --git a/src/sqlbuild/compiler/semantic_checks/constants.py b/src/sqlbuild/compiler/semantic_checks/constants.py deleted file mode 100644 index 853738663f..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/constants.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Native semantic completion statuses and inputs.""" - -from sqlbuild.compiler.references.types import SqlReferenceKind - -NATIVE_TYPE_RECOVERY_UNCHANGED: str = "unchanged" -NATIVE_SEMANTIC_NO_CATALOG: str = ( - "NativeCompilerError: native semantic completion: the project has no analysis catalog" -) -UNKNOWN_RECOVERED_TYPE: str = "UNKNOWN" -METADATA_CONFIG_KEYS: tuple[str, ...] = ( - "unique_key", - "cursor", - "partition_column", - "row_diff_exclude_columns", -) -METADATA_FUNCTION_REFERENCE_KINDS: frozenset[SqlReferenceKind] = frozenset( - {SqlReferenceKind.UDF, SqlReferenceKind.TABLE_FUNCTION} -) diff --git a/src/sqlbuild/compiler/semantic_checks/main/__init__.py b/src/sqlbuild/compiler/semantic_checks/main/__init__.py deleted file mode 100644 index b7664e28d3..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native semantic completion entry points.""" diff --git a/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py b/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py deleted file mode 100644 index dc14400120..0000000000 --- a/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Complete semantic diagnostics natively.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.types import DiagnosticRow -from sqlbuild.compiler.compile.types import CompiledProjectView -from sqlbuild.compiler.semantic_checks._helpers.stage import completed_semantic_project - - -def complete_native_semantic_diagnostics( - *, - project: CompiledProjectView, - profile: ExpressionInferenceProfile, - binding_results: dict[str, list[DiagnosticRow]], - resource_sql_analysis: bool, - session: Any | None = None, -) -> CompiledProjectView: - """The completed project; a native internal failure raises `NativeCompilerError`.""" - - return completed_semantic_project( - project=project, - profile=profile, - binding_results=binding_results, - resource_sql_analysis=resource_sql_analysis, - session=session, - ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index da417a095e..dfa12101fa 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -248,15 +248,6 @@ class DenseCompileGuardTestCase: expected_fingerprint: str -@dataclass(frozen=True) -class DenseMetadataTextGuardTestCase: - """A dense project whose semantic metadata request may carry each file's text only once.""" - - description: str - model_count: int - expected_requests: int = 1 - - @dataclass(frozen=True) class DenseWarmEditCompileGuardTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 49f6b58631..c5e922f912 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -3510,31 +3510,6 @@ def compare_incremental_compile(*, project_dir: Path) -> IncrementalEditComparis return IncrementalEditComparison(incremental=incremental, reference=reference) -def record_metadata_text_characters(monkeypatch: pytest.MonkeyPatch) -> list[int]: - """Record the file-text characters each native semantic metadata request carries.""" - - sizes: list[int] = [] - check: Callable[..., object] = cast( - Callable[..., object], native_module.check_semantic_metadata_rows - ) - - def recorded(catalog: object, request: tuple[object, ...]) -> object: - sizes.append(sum(len(text) for text in cast(list[str], request[5]))) - return check(catalog, request) - - monkeypatch.setattr(native_module, "check_semantic_metadata_rows", recorded) - return sizes - - -def project_text_characters(project_dir: Path) -> int: - """Characters in every file of a project that has not compiled yet.""" - - return sum( - len(path.read_text(encoding="utf-8")) - for path in filter(Path.is_file, project_dir.rglob("*")) - ) - - def in_process_reuse_run( *, project_dir: Path, capsys: pytest.CaptureFixture[str], args: tuple[str, ...] = () ) -> CompileReuseRun: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index a3c7bff6fe..dec0bf7ea8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -19,18 +19,13 @@ from sqlbuild.rule_engine.models import Rule, RulesConfig from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( DenseCompileGuardTestCase, - DenseMetadataTextGuardTestCase, DenseWarmEditCompileGuardTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( - CompileReuseRun, FreshProcessCompileBenchmarkResult, _run_fresh_process_compile_benchmark, fresh_process_compile_cache_metrics, - in_process_reuse_run, measure_model_sql_bytes, - project_text_characters, - record_metadata_text_characters, run_dense_warm_edit_benchmark, ) @@ -136,36 +131,6 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ assert result.peak_rss_bytes < test_case.expected_max_rss_bytes -@pytest.mark.performance -@pytest.mark.cold_compile_performance -@pytest.mark.parametrize( - "test_case", - [ - DenseMetadataTextGuardTestCase(f"dense_models_{models}_metadata_file_texts", models) - for models in (1000, 3000, 5000, 10000) - ], - ids=lambda case: case.description, -) -def test_given_dense_project_when_checking_semantic_metadata_then_each_file_text_crosses_once( - test_case: DenseMetadataTextGuardTestCase, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - project_dir: Path = tmp_path / "dense_orders" - write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count) - budget: int = project_text_characters(project_dir) - sizes: list[int] = record_metadata_text_characters(monkeypatch) - - run: CompileReuseRun = in_process_reuse_run( - project_dir=project_dir, capsys=capsys, args=("--no-cache",) - ) - - assert run.returncode == 0, run.stderr - assert len(sizes) == test_case.expected_requests - assert 0 < sizes[0] <= budget, (sizes, budget) - - @pytest.mark.performance @pytest.mark.cold_compile_performance @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py index e441922471..c17d00de94 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/helpers.py @@ -14,12 +14,7 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.diagnostics.recovery as semantic_recovery -import sqlbuild.compiler.semantic_checks._helpers.stage as semantic_stage -from sqlbuild._native import ( - build_project_graph, - discover_project_inputs, - project_with, -) +from sqlbuild._native import build_project_graph, discover_project_inputs, project_with from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.compiler.analysis_session.types import DiagnosticRow @@ -28,11 +23,6 @@ ) from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView -from sqlbuild.compiler.semantic_checks._helpers.metadata import native_metadata_diagnostics -from sqlbuild.compiler.semantic_checks._helpers.payloads import session_fact_models -from sqlbuild.compiler.semantic_checks.main._complete_native_semantic_diagnostics import ( - complete_native_semantic_diagnostics, -) from tests.integration.src.sqlbuild.compiler.analysis_session.helpers import ( generated_analysis_files, ) @@ -248,7 +238,7 @@ def note_kinds(project: CompiledProjectView, kinds: tuple[str, ...]) -> Counter[ def native_completion(inputs: SemanticInputs) -> CompiledProjectView: """The native stage's completed project.""" - return complete_native_semantic_diagnostics( + return native_module.complete_semantic_project( project=inputs.project, profile=inputs.profile, binding_results=inputs.binding_results, @@ -284,18 +274,27 @@ def without_session(inputs: SemanticInputs) -> SemanticInputs: return replace(inputs, native_session=None) -def record_session_models(*, monkeypatch: pytest.MonkeyPatch) -> list[frozenset[str]]: - """Keep each completion's models whose output names and lineage the session supplies.""" +def session_fact_models(inputs: SemanticInputs) -> frozenset[str]: + """Models whose output names and lineage the finished session supplies to the stage.""" + + kept: frozenset[str] = frozenset(getattr(inputs.native_session, "fact_models", ())) + proven: frozenset[str] = frozenset( + model.name for model in filter(_output_proven, inputs.project.models) + ) + return frozenset(model.name for model in inputs.project.models) & kept - proven + - selections: list[frozenset[str]] = [] +def _output_proven(model: CompiledModelView) -> bool: + return model.dynamic_column_contract is not None and model.dynamic_column_contract.output_proven - def recorded(**arguments: Any) -> frozenset[str]: - selected: frozenset[str] = session_fact_models(**arguments) - selections.append(selected) - return selected - monkeypatch.setattr(semantic_stage, "session_fact_models", recorded) - return selections +def has_blocking_binding(project: CompiledProjectView) -> bool: + """Whether any model owns a binding error, so the stage plans a type recovery.""" + + bindings: list[tuple[CompilerDiagnostic, ...]] = [ + model.binding_diagnostics for model in project.models + ] + return any(item.is_error for item in chain.from_iterable(bindings)) def completion_view(project: CompiledProjectView) -> tuple[object, ...]: @@ -316,29 +315,6 @@ def _model_view(model: CompiledModelView) -> tuple[object, ...]: ) -def record_native_statuses(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: - """Count native type recovery plans, completions and completed diagnostics.""" - - statuses: Counter[str] = Counter() - plan: Callable[..., Any] = native_module.plan_semantic_type_recovery - complete: Callable[..., Any] = native_module.complete_semantic_checks - - def counted_plan(*arguments: Any) -> Any: - recovery: Any = plan(*arguments) - statuses[f"type_recovery_{recovery.status}"] += 1 - return recovery - - def counted_complete(*arguments: Any) -> Any: - outcome: Any = complete(*arguments) - statuses["completion_native"] += 1 - statuses["diagnostics_native"] += len(outcome[0]) - return outcome - - monkeypatch.setattr(native_module, "plan_semantic_type_recovery", counted_plan) - monkeypatch.setattr(native_module, "complete_semantic_checks", counted_complete) - return statuses - - def non_ascii_variant(*, files: dict[str, str], rng: random.Random) -> dict[str, str]: """The project with some column names spelt beyond ASCII and a comment after each model header holding non-ASCII text, digits and Python-only line separators.""" @@ -548,37 +524,26 @@ def generated_metadata_files(*, rng: random.Random, model_count: int) -> dict[st def native_metadata(inputs: SemanticInputs) -> tuple[CompilerDiagnostic, ...]: """The native stage's metadata diagnostics.""" - catalog: Any = cast(Any, inputs.project.binding_catalog) - return native_metadata_diagnostics( + return native_module.semantic_metadata_diagnostics( project=inputs.project, profile=inputs.profile, resource_sql_analysis=inputs.resource_sql_analysis, - catalog=catalog.native, ) -def _family_counts(outcome: Any) -> Counter[str]: - counts: Counter[str] = Counter() - for function_rows, reference_rows, schema_audit_rows in outcome[0]: - counts.update(f"function {row[0]}" for row in function_rows) - counts.update(f"reference {row[0]}" for row in reference_rows) - counts.update(f"schema_audit {row[0]}" for row in schema_audit_rows) - counts.update(f"source {row[0]}" for _, row in outcome[1]) - counts.update(f"sql_test {row[0]}" for _, row, _ in outcome[2]) - return counts - - -def record_metadata_families(*, monkeypatch: pytest.MonkeyPatch) -> Counter[str]: - """Count native metadata runs and errors by family and code.""" +def _completed(outcome: tuple[str, object]) -> bool: + return outcome[0] == "completed" - counts: Counter[str] = Counter() - check: Callable[..., Any] = native_module.check_semantic_metadata_rows - def counted(*arguments: Any) -> Any: - outcome: Any = check(*arguments) - counts["native"] += 1 - counts.update(_family_counts(outcome)) - return outcome +def metadata_families(views: dict[str, object]) -> Counter[str]: + """Completed metadata runs and their diagnostics counted by resource type and code.""" - monkeypatch.setattr(native_module, "check_semantic_metadata_rows", counted) - return counts + completed: list[tuple[CompilerDiagnostic, ...]] = [ + cast(tuple[CompilerDiagnostic, ...], value) + for _, value in filter(_completed, cast(list[tuple[str, object]], list(views.values()))) + ] + families: Counter[str] = Counter( + f"{item.resource_type} {item.code}" for item in chain.from_iterable(completed) + ) + families["native"] = len(completed) + return families diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py index cd3675054c..dd452c37c5 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_metadata_checks.py @@ -24,8 +24,8 @@ captured_semantic_inputs, files_with_non_ascii, generated_metadata_files, + metadata_families, native_metadata, - record_metadata_families, semantic_outcome, with_dialect, ) @@ -44,10 +44,9 @@ golden_prefix="metadata_generated", expected_minimum_native=32, expected_minimum_families={ - "function B102": 100, - "function B301": 100, - "reference B300": 50, - "reference B301": 16, + "model B102": 100, + "model B300": 50, + "model B301": 116, "source B300": 4, "sql_test B302": 16, }, @@ -61,7 +60,7 @@ non_ascii=True, golden_prefix="metadata_formerly_deferred", expected_minimum_native=12, - expected_minimum_families={"function B301": 10, "reference B300": 4}, + expected_minimum_families={"model B300": 4, "model B301": 10}, ), ], ids=lambda case: case.description, @@ -72,7 +71,7 @@ def test_given_generated_projects_when_checking_metadata_natively_then_matches_r monkeypatch: pytest.MonkeyPatch, ) -> None: rng: random.Random = random.Random(test_case.seed) - families: Counter[str] = record_metadata_families(monkeypatch=monkeypatch) + families: Counter[str] = Counter() golden: list[GoldenEntry] = [] for index in range(test_case.count): project_dir: Path = tmp_path / f"project_{index}" @@ -91,6 +90,7 @@ def test_given_generated_projects_when_checking_metadata_natively_then_matches_r ) for dialect in test_case.dialects } + families.update(metadata_families(views)) golden.append(golden_entry("metadata", views, masked=(str(project_dir),))) assert ( diff --git a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py index 9af50230f5..1ec117546b 100644 --- a/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py +++ b/tests/integration/src/sqlbuild/compiler/semantic_checks/test_native_semantic_checks.py @@ -34,14 +34,14 @@ completion_view, files_with_non_ascii, generated_semantic_files, + has_blocking_binding, native_completion, note_kinds, outcome_label, proven_output_count, - record_native_statuses, - record_session_models, semantic_outcome, session_corpus_files, + session_fact_models, with_dialect, without_session, ) @@ -80,7 +80,7 @@ def test_given_generated_failing_projects_when_completing_natively_then_matches_ monkeypatch: pytest.MonkeyPatch, ) -> None: rng: random.Random = random.Random(test_case.seed) - statuses: Counter[str] = record_native_statuses(monkeypatch=monkeypatch) + statuses: Counter[str] = Counter() codes: Counter[str] = Counter() golden: list[GoldenEntry] = [] for index in range(test_case.count): @@ -95,6 +95,9 @@ def test_given_generated_failing_projects_when_completing_natively_then_matches_ views: dict[str, object] = {} for dialect in test_case.dialects: native: CompiledProjectView = completed_natively(with_dialect(captured, dialect)) + statuses["completion_native"] += 1 + statuses["diagnostics_native"] += len(native.diagnostics) + statuses["type_recovery_planned"] += has_blocking_binding(captured.project) codes.update(item.code for item in native.diagnostics) codes.update(note_kinds(native, _NOTE_KINDS)) views[dialect] = completion_view(native) @@ -263,7 +266,7 @@ def test_given_compiles_when_completing_from_the_session_then_matches_payload_an ) for index in range(test_case.count) ] - selections: list[frozenset[str]] = record_session_models(monkeypatch=monkeypatch) + selections: list[frozenset[str]] = [session_fact_models(inputs) for inputs in captured] codes: Counter[str] = Counter() golden: list[GoldenEntry] = [] payload_views: list[tuple[object, ...]] = [] From 03dff4b9faf2c9bf1f292fc8f603085de49d0123 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:06:38 +0100 Subject: [PATCH 10/14] refactor: infer query columns, expression shapes and pivot proofs natively --- .../bindings/_helpers/analysis_session/mod.rs | 2 + .../analysis_session/profile_inputs.rs | 158 ++++++++ .../analysis_session/query_columns.rs | 7 +- .../_helpers/analysis_session/registration.rs | 5 +- .../_helpers/analysis_session/session.rs | 14 +- .../analysis_session/stage_functions.rs | 363 ++++++++++++++++++ .../compile_driver/assembly_models.rs | 4 +- src/sqlbuild/_native.pyi | 46 +-- .../_helpers/dynamic_contracts.py | 47 --- .../_helpers/expression_shapes.py | 58 --- .../_helpers/query_columns.py | 146 ------- .../_infer_native_expression_source_shapes.py | 16 - .../main/_infer_native_query_column_rows.py | 27 -- .../main/_infer_native_query_columns.py | 27 -- .../main/_prove_native_dynamic_contracts.py | 21 - .../_helpers/analysis/fixture_columns.py | 4 +- .../_helpers/assembly/metadata_validation.py | 4 +- .../_helpers/assembly/semantic_shapes.py | 4 +- .../_helpers/assembly/source_bindings.py | 4 +- .../_helpers/native_stages/assembly.py | 7 +- .../compiler/analysis_session/helpers.py | 13 +- .../test_native_query_columns.py | 4 +- .../src/sqlbuild/compiler/pipeline/helpers.py | 34 +- .../_helpers/analysis/test_dynamic_pivot.py | 6 +- .../compiler/compile/_helpers/helpers.py | 10 +- .../_helpers/render/test_cursor_intrinsics.py | 4 +- .../_helpers/test_sql_analysis_columns.py | 4 +- 27 files changed, 603 insertions(+), 436 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs delete mode 100644 src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py delete mode 100644 src/sqlbuild/compiler/analysis_session/_helpers/expression_shapes.py delete mode 100644 src/sqlbuild/compiler/analysis_session/_helpers/query_columns.py delete mode 100644 src/sqlbuild/compiler/analysis_session/main/_infer_native_expression_source_shapes.py delete mode 100644 src/sqlbuild/compiler/analysis_session/main/_infer_native_query_column_rows.py delete mode 100644 src/sqlbuild/compiler/analysis_session/main/_infer_native_query_columns.py delete mode 100644 src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs index 11eaaf57b5..3e2bb419c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs @@ -1,5 +1,7 @@ pub(crate) mod nullability_rules; pub(crate) mod oracle; +pub(crate) mod profile_inputs; pub(crate) mod query_columns; pub(crate) mod registration; pub(crate) mod session; +pub(crate) mod stage_functions; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs new file mode 100644 index 0000000000..f9ca93d7c1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs @@ -0,0 +1,158 @@ +//! Inference-profile settings crossing into native analysis: rule ids, shapes and the catalog. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; +use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; + +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const BINDING_CATALOG_MODULE: &str = "sqlbuild.compiler.sql_analysis.main._binding_catalog"; +const SQL_ANALYSIS_CONSTANTS_MODULE: &str = "sqlbuild.compiler.sql_analysis.constants"; +const GENERIC_DIALECT: &str = "generic"; +const ADAPTER_RULE_ID: &str = "python"; +/// `(module, function, native rule id)` of the nullability rules the native analysis knows. +const KNOWN_RULES: [(&str, &str, &str); 3] = [ + ( + "sqlbuild.adapter.type_system.main.first_arg_nullability", + "first_arg_nullability", + "first_arg", + ), + ( + "sqlbuild.adapter.type_system.main.conditional_result_nullability", + "conditional_result_nullability", + "conditional_result", + ), + ( + "sqlbuild.adapter.type_system.main.safe_cast_nullability", + "safe_cast_nullability", + "safe_cast", + ), +]; + +/// The adapter's own nullability rules and `InferredNullability`, which native analysis calls. +pub(crate) type AdapterRules = Option<(Py, Py)>; + +/// The adapter nullability rules as `(name, rule id)` rows, and the adapter's own rules. +pub(crate) fn nullability_rules(profile: &Bound<'_, PyAny>) -> PyResult<(Pairs, AdapterRules)> { + let py: Python<'_> = profile.py(); + let known: Bound<'_, PyDict> = PyDict::new(py); + for (module, function, id) in KNOWN_RULES { + known.set_item(py.import(module)?.getattr(function)?, id)?; + } + let adapter: Bound<'_, PyDict> = PyDict::new(py); + let mut rows: Pairs = Vec::new(); + for item in profile + .getattr("function_nullability_rules")? + .call_method0("items")? + .try_iter()? + { + let (name, rule): (Bound<'_, PyAny>, Bound<'_, PyAny>) = item?.extract()?; + let id: String = match known.get_item(&rule)? { + Some(id) => id.extract()?, + None => { + adapter.set_item(&name, &rule)?; + ADAPTER_RULE_ID.to_owned() + } + }; + rows.push((name.extract()?, id)); + } + if adapter.is_empty() { + return Ok((rows, None)); + } + let nullability: Bound<'_, PyAny> = py + .import(LINEAGE_TYPES_MODULE)? + .getattr("InferredNullability")?; + Ok((rows, Some((adapter.unbind(), nullability.unbind())))) +} + +/// `{relation: {column: value}}` as ordered rows, each value as `str(value)`. +pub(crate) fn shape_rows(shapes: &Bound<'_, PyAny>) -> PyResult { + let mut rows: Shapes = Vec::new(); + for item in shapes.call_method0("items")?.try_iter()? { + let (name, shape): (String, Bound<'_, PyAny>) = item?.extract()?; + let mut pairs: Pairs = Vec::new(); + for column in shape.call_method0("items")?.try_iter()? { + let (column, value): (String, Bound<'_, PyAny>) = column?.extract()?; + pairs.push((column, value.str()?.extract()?)); + } + rows.push((name, pairs)); + } + Ok(rows) +} + +/// A `{key: value}` text mapping's items in order. +pub(crate) fn pairs(mapping: &Bound<'_, PyAny>) -> PyResult { + mapping + .call_method0("items")? + .try_iter()? + .map(|item| item?.extract()) + .collect() +} + +/// `profile.sql_analysis_dialect or "generic"`. +pub(crate) fn profile_dialect(profile: &Bound<'_, PyAny>) -> PyResult { + dialect_or_generic(&profile.getattr("sql_analysis_dialect")?) +} + +/// `dialect or "generic"`. +pub(crate) fn dialect_or_generic(dialect: &Bound<'_, PyAny>) -> PyResult { + if dialect.is_truthy()? { + return dialect.extract(); + } + Ok(GENERIC_DIALECT.to_owned()) +} + +/// Python's `NATIVE_DIALECT_ALIASES.get(dialect, dialect)`. +pub(crate) fn aliased_dialect(py: Python<'_>, dialect: String) -> PyResult { + let aliases: Bound<'_, PyAny> = py + .import(SQL_ANALYSIS_CONSTANTS_MODULE)? + .getattr("NATIVE_DIALECT_ALIASES")?; + let key: Bound<'_, PyString> = PyString::new(py, &dialect); + aliases.call_method1("get", (&key, &key))?.extract() +} + +/// Python's `inferred_binding_shape` test for keeping authored identifier quoting. +pub(crate) fn case_sensitive_shapes( + profile: &Bound<'_, PyAny>, + dialect: &Bound<'_, PyAny>, +) -> PyResult { + if profile + .getattr("quoted_identifiers_ignore_case")? + .is_truthy()? + { + return Ok(false); + } + profile + .py() + .import(SQL_ANALYSIS_CONSTANTS_MODULE)? + .getattr("CASE_SENSITIVE_BINDING_DIALECTS")? + .contains(dialect) +} + +/// `profile.binding_catalog or create_binding_catalog(...)` with no functions, types or relations. +pub(crate) fn profile_catalog<'py>( + profile: &Bound<'py, PyAny>, + dialect: &str, +) -> PyResult> { + let catalog: Bound<'py, PyAny> = profile.getattr("binding_catalog")?; + if catalog.is_truthy()? { + return Ok(catalog); + } + let py: Python<'py> = profile.py(); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("dialect", dialect)?; + arguments.set_item( + "quoted_ignore_case", + profile.getattr("quoted_identifiers_ignore_case")?, + )?; + arguments.set_item("known_functions", ())?; + arguments.set_item("known_types", ())?; + arguments.set_item("relations", PyDict::new(py))?; + py.import(BINDING_CATALOG_MODULE)? + .getattr("create_binding_catalog")? + .call((), Some(&arguments)) +} + +/// `profile.function_return_types` items in order. +pub(crate) fn return_types(profile: &Bound<'_, PyAny>) -> PyResult { + pairs(&profile.getattr("function_return_types")?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs index 2bd572c62a..13b2eecf79 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs @@ -24,7 +24,7 @@ const QUERY_COLUMNS_CONTEXT: &str = "native query column analysis"; const PROJECTION_FACTS_CONTEXT: &str = "native fixture projection facts"; /// `(sql, placeholders, analysis names, lineage references, recover CTE facts, mode)`. -type QueryRow = ( +pub(crate) type QueryRow = ( String, Pairs, Vec, @@ -37,12 +37,12 @@ type RequestRow = (String, Pairs, Pairs, Shapes, Shapes, Vec, String); /// `([(name, type)], NULL literal names, quoted names)`. type ProjectionFactsRow = (Vec<(String, String)>, Vec, Vec); /// `(succeeded, [(name, type, nullability)] or None, has star)`. -type ColumnsRow = (bool, Option, String)>>, bool); +pub(crate) type ColumnsRow = (bool, Option, String)>>, bool); /// Each query's columns, as Python's analysis outside the session inferred them. #[pyfunction] #[pyo3(signature = (catalog, request, adapter_rules=None))] -fn infer_query_columns( +pub(crate) fn infer_query_columns( py: Python<'_>, catalog: Option>, request: RequestRow, @@ -141,7 +141,6 @@ fn fixture_projection_facts( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(infer_query_columns, module)?)?; module.add_function(wrap_pyfunction!(fixture_projection_facts, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs index 6e72b49d20..c58f280a4c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs @@ -2,10 +2,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::analysis_session::{oracle, query_columns, session}; +use crate::bindings::_helpers::analysis_session::{ + oracle, query_columns, session, stage_functions, +}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { session::register(module)?; query_columns::register(module)?; + stage_functions::register(module)?; oracle::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index 226d293a29..df8973e549 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -46,7 +46,7 @@ type ColumnRow = (String, Option, String); /// `(output column, transform code, confidence code, [(resource type, resource, column)])`. type LineageItem = (String, u8, u8, Vec<(String, String, String)>); /// `(name, pivot column, value column, aggregate, type, name pattern)`. -type FamilyRow = (String, String, String, String, String, Option); +pub(crate) type FamilyRow = (String, String, String, String, String, Option); /// A model's name, SQL, placeholders, references, lineage, required names, flags and pivot. type ModelRow = ( String, @@ -95,7 +95,7 @@ type ProofRow = ( bool, ); /// One model's pivot SQL and declared families. -type PivotModelRow = (String, Vec); +pub(crate) type PivotModelRow = (String, Vec); /// The dialect, relation facts and families by table every proof reads, and the models. type PivotRequestRow = ( String, @@ -106,7 +106,7 @@ type PivotRequestRow = ( Vec, ); /// `("absent" | "deferred" | "proof", proof)`. -type ContractRow = (&'static str, Option); +pub(crate) type ContractRow = (&'static str, Option); /// `(hits, misses, stored, why the store was not read or saved)`. type CacheStatsRow = (usize, usize, usize, Option); /// `(catalog schema additions, analysis-shape names, whether each model's analysis succeeded)`. @@ -241,7 +241,7 @@ impl NativeModelAnalysisSession { } /// Each model's dynamic pivot proof from the finished session's tables. - fn prove_dynamic_contracts( + pub(crate) fn prove_dynamic_contracts( &self, py: Python<'_>, models: Vec, @@ -307,7 +307,7 @@ fn start_model_analysis_session( /// Each expression's shape, `None` where it has none; adapter rules run as in the session. #[pyfunction] #[pyo3(signature = (catalog, request, adapter_rules=None))] -fn infer_expression_source_shapes( +pub(crate) fn infer_expression_source_shapes( py: Python<'_>, catalog: PyRef<'_, ProjectCatalog>, request: (String, bool, Pairs, Option, Vec), @@ -482,7 +482,7 @@ fn outcome_row(outcome: ModelOutcome, cleaned_sql: String) -> OutcomeRow { /// Each model's dynamic pivot proof outside any session. #[pyfunction] -fn prove_dynamic_column_contracts( +pub(crate) fn prove_dynamic_column_contracts( py: Python<'_>, request: PivotRequestRow, ) -> PyResult> { @@ -515,7 +515,5 @@ fn pivot_model(row: PivotModelRow) -> PivotModel { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_function(wrap_pyfunction!(start_model_analysis_session, module)?)?; - module.add_function(wrap_pyfunction!(infer_expression_source_shapes, module)?)?; - module.add_function(wrap_pyfunction!(prove_dynamic_column_contracts, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs new file mode 100644 index 0000000000..0414970f55 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs @@ -0,0 +1,363 @@ +//! Query columns, expression-source shapes and dynamic pivot proofs read from Python objects. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyListMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::assembly::analysis_session::types::Pairs; + +use crate::bindings::_helpers::analysis_session::profile_inputs::{ + aliased_dialect, case_sensitive_shapes, dialect_or_generic, nullability_rules, profile_catalog, + profile_dialect, return_types, shape_rows, +}; +use crate::bindings::_helpers::analysis_session::query_columns::{ + ColumnsRow, QueryRow, infer_query_columns, +}; +use crate::bindings::_helpers::analysis_session::session::{ + ContractRow, FamilyRow, NativeModelAnalysisSession, PivotModelRow, + infer_expression_source_shapes, prove_dynamic_column_contracts, +}; +use crate::bindings::_helpers::boundary::keyword_request::Request; +use crate::bindings::models::ProjectCatalog; + +const BATCH_MODE: &str = "batch"; +const TABLE_FUNCTION_KIND: &str = "table_fn"; +const TABLE_FUNCTION_ANALYSIS_PREFIX: &str = "__sqlbuild_table_function_"; +/// Python's `_LINEAGE_RESOURCE_TYPES`: reference kind to compiled resource type value. +const LINEAGE_RESOURCE_TYPES: [(&str, &str); 4] = [ + ("ref", "model"), + ("source", "source"), + ("seed", "seed"), + ("table_fn", "table_fn"), +]; +const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; + +/// Each query's `(succeeded, column rows, has star)`; a native internal failure raises. +#[pyfunction] +#[pyo3(signature = (**request))] +fn infer_native_query_column_rows<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("infer_native_query_column_rows", request)?; + let empty: Bound<'py, PyAny> = PyDict::new(request.py()).into_any(); + let nullability: Bound<'py, PyAny> = + or_empty(request.optional("column_nullability_by_table")?, &empty)?; + query_column_rows( + &request.get("queries")?, + &request.get("profile")?, + (&empty, &nullability), + ) +} + +/// Each query's columns as `NativeQueryColumns`; a native internal failure raises. +#[pyfunction] +#[pyo3(signature = (**request))] +fn infer_native_query_columns<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("infer_native_query_columns", request)?; + let py: Python<'py> = request.py(); + let empty: Bound<'py, PyAny> = PyDict::new(py).into_any(); + let types: Bound<'py, PyAny> = or_empty(request.optional("column_types_by_table")?, &empty)?; + let nullability: Bound<'py, PyAny> = + or_empty(request.optional("column_nullability_by_table")?, &empty)?; + let rows: Vec = query_column_rows( + &request.get("queries")?, + &request.get("profile")?, + (&types, &nullability), + )?; + let query_columns: Bound<'py, PyAny> = py + .import(ANALYSIS_MODELS_MODULE)? + .getattr("NativeQueryColumns")?; + let column_class: Bound<'py, PyAny> = py + .import(COMPILE_MODELS_MODULE)? + .getattr("InferredColumn")?; + let nullability_class: Bound<'py, PyAny> = py + .import(LINEAGE_TYPES_MODULE)? + .getattr("InferredNullability")?; + let mut analyses: Vec> = Vec::with_capacity(rows.len()); + for (succeeded, columns, has_star) in rows { + let columns: Bound<'py, PyAny> = match columns { + None => py.None().into_bound(py), + Some(columns) => { + let mut built: Vec> = Vec::with_capacity(columns.len()); + for (name, data_type, nullability) in columns { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("name", name)?; + arguments.set_item("type", data_type)?; + arguments.set_item("nullability", nullability_class.call1((nullability,))?)?; + built.push(column_class.call((), Some(&arguments))?); + } + PyTuple::new(py, built)?.into_any() + } + }; + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("succeeded", succeeded)?; + arguments.set_item("columns", columns)?; + arguments.set_item("has_star", has_star)?; + analyses.push(query_columns.call((), Some(&arguments))?); + } + PyTuple::new(py, analyses) +} + +/// One shape per expression, cached on the profile's binding catalog; a failure raises. +#[pyfunction] +#[pyo3(signature = (**request))] +fn infer_native_expression_source_shapes<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("infer_native_expression_source_shapes", request)?; + let py: Python<'py> = request.py(); + let expressions: Bound<'py, PyAny> = request.get("expressions")?; + let profile: Bound<'py, PyAny> = request.get("profile")?; + let dialect: String = aliased_dialect(py, profile_dialect(&profile)?)?; + let catalog: Bound<'py, PyAny> = profile_catalog(&profile, &dialect)?; + let cache: Bound<'py, PyAny> = catalog.getattr("expression_shapes")?; + let pending: Bound<'py, PyDict> = PyDict::new(py); + for expression in expressions.try_iter()? { + let expression: Bound<'py, PyAny> = expression?; + if !cache.contains(&expression)? { + pending.set_item(expression, py.None())?; + } + } + if !pending.is_empty() { + let (rules, adapter_rules) = nullability_rules(&profile)?; + let native: PyRef<'py, ProjectCatalog> = catalog.getattr("native")?.extract()?; + let pending_names: Vec = pending + .keys() + .iter() + .map(|name| name.extract()) + .collect::>()?; + let case_sensitive: bool = + case_sensitive_shapes(&profile, &dialect.clone().into_bound_py_any(py)?)?; + let shapes: Vec> = infer_expression_source_shapes( + py, + native, + ( + dialect, + case_sensitive, + return_types(&profile)?, + Some(rules), + pending_names, + ), + adapter_rules, + )?; + for (expression, shape) in pending.keys().iter().zip(shapes) { + let value: Bound<'py, PyAny> = match shape { + Some(shape) => shape.into_py_dict_ordered(py)?, + None => py.None().into_bound(py), + }; + cache.set_item(expression, value)?; + } + } + let mut shapes: Vec> = Vec::new(); + for expression in expressions.try_iter()? { + let cached: Bound<'py, PyAny> = cache.get_item(expression?)?; + shapes.push(if cached.is_none() { + cached + } else { + let copied: Bound<'py, PyDict> = PyDict::new(py); + copied.update(cached.downcast::()?.as_mapping())?; + copied.into_any() + }); + } + PyTuple::new(py, shapes) +} + +/// The proof row per `(sql, families)` model from the session or `tables`, else None. +#[pyfunction] +#[pyo3(signature = (**request))] +fn prove_native_dynamic_contracts<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("prove_native_dynamic_contracts", request)?; + let py: Python<'py> = request.py(); + let session: Bound<'py, PyAny> = request.get("session")?; + let tables: Bound<'py, PyAny> = request.get("tables")?; + let models: Bound<'py, PyAny> = request.get("models")?; + if !models.is_truthy()? { + return Ok(PyTuple::empty(py)); + } + let mut rows: Vec = Vec::new(); + for model in models.try_iter()? { + let (sql, families): (String, Bound<'py, PyAny>) = model?.extract()?; + rows.push((sql, family_rows(&families)?)); + } + let contracts: Vec = if session.is_none() { + let by_table: Bound<'py, PyAny> = tables.getattr("dynamic_families_by_table")?; + let mut families: Vec<(String, Vec)> = Vec::new(); + for item in by_table.call_method0("items")?.try_iter()? { + let (name, declared): (String, Bound<'py, PyAny>) = item?.extract()?; + families.push((name, family_rows(&declared)?)); + } + prove_dynamic_column_contracts( + py, + ( + dialect_or_generic(&tables.getattr("dialect")?)?, + shape_rows(&tables.getattr("column_types_by_table")?)?, + shape_rows(&tables.getattr("authoritative_column_types_by_table")?)?, + shape_rows(&tables.getattr("column_nullability_by_table")?)?, + families, + rows, + ), + )? + } else { + let session: PyRef<'py, NativeModelAnalysisSession> = session.extract()?; + session.prove_dynamic_contracts(py, rows)? + }; + let mut proofs: Vec> = Vec::with_capacity(contracts.len()); + for (_, proof) in contracts { + proofs.push(proof.into_bound_py_any(py)?); + } + PyTuple::new(py, proofs) +} + +/// Declared dynamic families as native request rows. +pub(crate) fn family_rows(families: &Bound<'_, PyAny>) -> PyResult> { + let mut rows: Vec = Vec::new(); + for family in families.try_iter()? { + let family: Bound<'_, PyAny> = family?; + rows.push(( + family.getattr("name")?.extract()?, + family.getattr("pivot_column")?.extract()?, + family.getattr("value_column")?.extract()?, + family.getattr("aggregate")?.extract()?, + family.getattr("type")?.extract()?, + family.getattr("name_pattern")?.extract()?, + )); + } + Ok(rows) +} + +fn or_empty<'py>( + value: Option>, + empty: &Bound<'py, PyAny>, +) -> PyResult> { + match value { + Some(value) if value.is_truthy()? => Ok(value), + _ => Ok(empty.clone()), + } +} + +fn query_column_rows( + queries: &Bound<'_, PyAny>, + profile: &Bound<'_, PyAny>, + (types, nullability): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), +) -> PyResult> { + let py: Python<'_> = profile.py(); + if !queries.is_truthy()? { + return Ok(Vec::new()); + } + let queries: Vec> = queries.try_iter()?.collect::>()?; + let normalization_dialect: String = profile_dialect(profile)?; + let dialect: String = aliased_dialect(py, normalization_dialect.clone())?; + let mut batch: bool = false; + for query in &queries { + if query.getattr("mode")?.eq(BATCH_MODE)? { + batch = true; + break; + } + } + let catalog: Option> = if batch { + Some(profile_catalog(profile, &dialect)?.getattr("native")?) + } else { + None + }; + let (rules, adapter_rules) = nullability_rules(profile)?; + let mut rows: Vec = Vec::with_capacity(queries.len()); + for query in &queries { + rows.push(query_row(query)?); + } + let catalog: Option> = + catalog.map(|native| native.extract()).transpose()?; + infer_query_columns( + py, + catalog, + ( + dialect, + return_types(profile)?, + rules, + shape_rows(types)?, + shape_rows(nullability)?, + rows, + normalization_dialect, + ), + adapter_rules, + ) +} + +fn query_row(query: &Bound<'_, PyAny>) -> PyResult { + let placeholders: Bound<'_, PyAny> = query.getattr("placeholders")?; + let placeholders: Pairs = if placeholders.is_truthy()? { + placeholders + .call_method0("items")? + .try_iter()? + .map(|item| item?.extract()) + .collect::>()? + } else { + Vec::new() + }; + let mut analysis_names: Vec = Vec::new(); + let mut lineage: Vec<(String, String, String)> = Vec::new(); + for reference in query.getattr("references")?.try_iter()? { + let reference: Bound<'_, PyAny> = reference?; + let kind: String = reference.getattr("ref_kind")?.extract()?; + let name: String = reference.getattr("ref_name")?.extract()?; + let analysis_name: String = if kind == TABLE_FUNCTION_KIND { + format!("{TABLE_FUNCTION_ANALYSIS_PREFIX}{name}") + } else { + name.clone() + }; + analysis_names.push(analysis_name.clone()); + if let Some((_, resource_type)) = LINEAGE_RESOURCE_TYPES + .iter() + .find(|(reference_kind, _)| *reference_kind == kind) + { + let entry = (analysis_name, (*resource_type).to_owned(), name); + match lineage + .iter_mut() + .find(|(existing, _, _)| *existing == entry.0) + { + Some(existing) => *existing = entry, + None => lineage.push(entry), + } + } + } + Ok(( + query.getattr("sql")?.extract()?, + placeholders, + analysis_names, + lineage, + query.getattr("recover_cte_facts")?.extract()?, + query.getattr("mode")?.extract()?, + )) +} + +trait OrderedDict { + fn into_py_dict_ordered<'py>(self, py: Python<'py>) -> PyResult>; +} + +impl OrderedDict for Pairs { + fn into_py_dict_ordered<'py>(self, py: Python<'py>) -> PyResult> { + let dict: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in self { + dict.set_item(key, value)?; + } + Ok(dict.into_any()) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(infer_native_query_column_rows, module)?)?; + module.add_function(wrap_pyfunction!(infer_native_query_columns, module)?)?; + module.add_function(wrap_pyfunction!( + infer_native_expression_source_shapes, + module + )?)?; + module.add_function(wrap_pyfunction!(prove_native_dynamic_contracts, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index 96aa7a4797..582675569c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -14,8 +14,6 @@ const SYNTAX_CHECKS_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis. const CURSOR_INTRINSICS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.cursor_intrinsics"; const MACROS_MODULE: &str = "sqlbuild.compiler.compile._helpers.render.macros"; -const INFER_ROWS_MODULE: &str = - "sqlbuild.compiler.analysis_session.main._infer_native_query_column_rows"; const HAND_BUILT_MODULE: &str = "sqlbuild.compiler.compile._helpers.assembly.hand_built_models"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; @@ -218,7 +216,7 @@ fn fallback_columns<'py>( ], )?; call_with( - &imported(py, INFER_ROWS_MODULE, "infer_native_query_column_rows")?, + &imported(py, NATIVE_MODULE, "infer_native_query_column_rows")?, &[ ("queries", PyTuple::new(py, [query])?.into_any()), ("profile", context.profile.clone()), diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5c0f0e376c..5c45c46c11 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1738,35 +1738,25 @@ def start_model_analysis_session( cache: tuple[str, str] | None = None, adapter_rules: tuple[Mapping[str, Callable[..., object]], type] | None = None, ) -> NativeModelAnalysisSession: ... -def prove_dynamic_column_contracts( - request: tuple[ - str, - list[tuple[str, list[tuple[str, str]]]], - list[tuple[str, list[tuple[str, str]]]], - list[tuple[str, list[tuple[str, str]]]], - list[tuple[str, list[tuple[str, str, str, str, str, str | None]]]], - list[tuple[str, list[tuple[str, str, str, str, str, str | None]]]], - ], - /, -) -> list[_PivotContractRow]: ... -def infer_query_columns( - catalog: object | None, - request: tuple[ - str, - list[tuple[str, str]], - list[tuple[str, str]], - list[tuple[str, list[tuple[str, str]]]], - list[tuple[str, list[tuple[str, str]]]], - list[tuple[object, ...]], - str, - ], - adapter_rules: tuple[Mapping[str, Callable[..., object]], type] | None = None, +def infer_native_query_column_rows( + *, + queries: Sequence[Any], + profile: Any, + column_nullability_by_table: Mapping[str, Mapping[str, Any]] | None = None, ) -> list[tuple[bool, list[tuple[str, str | None, str]] | None, bool]]: ... -def infer_expression_source_shapes( - catalog: object, - request: tuple[str, bool, list[tuple[str, str]], list[tuple[str, str]], list[str]], - adapter_rules: tuple[Mapping[str, Callable[..., object]], type] | None = None, -) -> list[list[tuple[str, str]] | None]: ... +def infer_native_query_columns( + *, + queries: Sequence[Any], + profile: Any, + column_types_by_table: Mapping[str, Mapping[str, str]] | None = None, + column_nullability_by_table: Mapping[str, Mapping[str, Any]] | None = None, +) -> tuple[Any, ...]: ... +def infer_native_expression_source_shapes( + *, expressions: tuple[str, ...], profile: Any +) -> tuple[dict[str, str] | None, ...]: ... +def prove_native_dynamic_contracts( + *, session: NativeModelAnalysisSession | None, tables: Any, models: tuple[tuple[str, tuple[Any, ...]], ...] +) -> tuple[Any, ...]: ... def _oracle_names_set_operation(sql: str, /) -> bool: ... def _oracle_cte_fact_recovery( request: tuple[ diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py b/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py deleted file mode 100644 index 6699417d25..0000000000 --- a/src/sqlbuild/compiler/analysis_session/_helpers/dynamic_contracts.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Prove dynamic column contracts natively for models outside the analysis session.""" - -from __future__ import annotations - -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.compiler.analysis_session._helpers.profile_rows import shape_rows -from sqlbuild.compiler.analysis_session._helpers.session_rows import family_rows -from sqlbuild.compiler.analysis_session.models import NativePivotTables -from sqlbuild.compiler.analysis_session.types import ContractRow, ProofRow -from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - -type _ModelRow = tuple[str, list[tuple[str, str, str, str, str, str | None]]] - - -def native_dynamic_contracts( - *, - session: Any | None, - tables: NativePivotTables, - models: tuple[tuple[str, tuple[SchemaDynamicColumnFamily, ...]], ...], -) -> tuple[ProofRow | None, ...]: - """Each model's proof row in order; a native internal failure raises.""" - - if not models: - return () - rows: list[_ModelRow] = [_model_row(sql=sql, families=families) for sql, families in models] - contracts: list[ContractRow] = ( - session.prove_dynamic_contracts(rows) - if session is not None - else _native.prove_dynamic_column_contracts( - ( - tables.dialect or "generic", - shape_rows(tables.column_types_by_table), - shape_rows(tables.authoritative_column_types_by_table), - shape_rows(tables.column_nullability_by_table), - family_rows(tables.dynamic_families_by_table), - rows, - ) - ) - ) - return tuple(proof for _, proof in contracts) - - -def _model_row(*, sql: str, families: tuple[SchemaDynamicColumnFamily, ...]) -> _ModelRow: - return (sql, list(map(PythonModelAnalysis.family_row, families))) diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/expression_shapes.py b/src/sqlbuild/compiler/analysis_session/_helpers/expression_shapes.py deleted file mode 100644 index 880ce5d100..0000000000 --- a/src/sqlbuild/compiler/analysis_session/_helpers/expression_shapes.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Infer expression-source shapes natively, filling the binding catalog's shape cache.""" - -from __future__ import annotations - -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.adapter.contract.types import FunctionNullabilityRule -from sqlbuild.compiler.analysis_session._helpers.profile_rows import ( - adapter_nullability_rules, - case_sensitive_shapes, - nullability_rule_rows, -) -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.sql_analysis.constants import NATIVE_DIALECT_ALIASES -from sqlbuild.compiler.sql_analysis.main._binding_catalog import create_binding_catalog - - -def native_expression_source_shapes( - *, expressions: tuple[str, ...], profile: ExpressionInferenceProfile -) -> tuple[dict[str, str] | None, ...]: - """One shape per expression; a native internal failure raises `NativeCompilerError`.""" - - dialect: str = profile.sql_analysis_dialect or "generic" - dialect = NATIVE_DIALECT_ALIASES.get(dialect, dialect) - catalog: Any = profile.binding_catalog or create_binding_catalog( - dialect=dialect, - quoted_ignore_case=profile.quoted_identifiers_ignore_case, - known_functions=(), - known_types=(), - relations={}, - ) - cache: dict[str, dict[str, str] | None] = catalog.expression_shapes - pending: tuple[str, ...] = tuple( - dict.fromkeys(expression for expression in expressions if expression not in cache) - ) - if pending: - adapter_rules: dict[str, FunctionNullabilityRule] = adapter_nullability_rules(profile) - shapes: list[list[tuple[str, str]] | None] = _native.infer_expression_source_shapes( - catalog.native, - ( - dialect, - case_sensitive_shapes(profile=profile, dialect=dialect), - list(profile.function_return_types.items()), - nullability_rule_rows(profile), - list(pending), - ), - (adapter_rules, InferredNullability) if adapter_rules else None, - ) - cache.update( - (expression, dict(shape) if shape is not None else None) - for expression, shape in zip(pending, shapes, strict=True) - ) - return tuple( - None if cache[expression] is None else dict(cache[expression] or {}) - for expression in expressions - ) diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/query_columns.py b/src/sqlbuild/compiler/analysis_session/_helpers/query_columns.py deleted file mode 100644 index 25d3b946e1..0000000000 --- a/src/sqlbuild/compiler/analysis_session/_helpers/query_columns.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Python's per-query column analysis outside the session, run natively.""" - -from __future__ import annotations - -from collections.abc import Mapping, Sequence -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.adapter.contract.types import FunctionNullabilityRule -from sqlbuild.compiler.analysis_session._helpers.profile_rows import ( - adapter_nullability_rules, - nullability_rule_rows, - shape_rows, -) -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns -from sqlbuild.compiler.analysis_session.types import ColumnRow -from sqlbuild.compiler.compile.models import CompileSqlReference, InferredColumn -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.sql_analysis.constants import ( - NATIVE_DIALECT_ALIASES, - TABLE_FUNCTION_ANALYSIS_PREFIX, -) -from sqlbuild.compiler.sql_analysis.main._binding_catalog import create_binding_catalog - -_BATCH_MODE: str = "batch" -_LINEAGE_RESOURCE_TYPES: dict[str, CompiledResourceType] = { - SqlReferenceKind.REF: CompiledResourceType.MODEL, - SqlReferenceKind.SOURCE: CompiledResourceType.SOURCE, - SqlReferenceKind.SEED: CompiledResourceType.SEED, - SqlReferenceKind.TABLE_FUNCTION: CompiledResourceType.TABLE_FN, -} - - -def native_query_columns( - *, - queries: Sequence[NativeColumnQuery], - profile: ExpressionInferenceProfile, - column_types_by_table: Mapping[str, Mapping[str, str]], - column_nullability_by_table: Mapping[str, Mapping[str, InferredNullability]], -) -> tuple[NativeQueryColumns, ...]: - """Each query's columns; a native internal failure raises `NativeCompilerError`.""" - - return tuple( - NativeQueryColumns( - succeeded=succeeded, - columns=None if columns is None else _inferred_columns(columns), - has_star=has_star, - ) - for succeeded, columns, has_star in native_query_column_rows( - queries=queries, - profile=profile, - column_types_by_table=column_types_by_table, - column_nullability_by_table=column_nullability_by_table, - ) - ) - - -def native_query_column_rows( - *, - queries: Sequence[NativeColumnQuery], - profile: ExpressionInferenceProfile, - column_types_by_table: Mapping[str, Mapping[str, str]], - column_nullability_by_table: Mapping[str, Mapping[str, InferredNullability]], -) -> list[tuple[bool, list[ColumnRow] | None, bool]]: - """Each query's `(succeeded, column rows, has star)` as the native analysis returns it.""" - - if not queries: - return [] - dialect: str = profile.sql_analysis_dialect or "generic" - dialect = NATIVE_DIALECT_ALIASES.get(dialect, dialect) - catalog: Any | None = ( - ( - profile.binding_catalog - or create_binding_catalog( - dialect=dialect, - quoted_ignore_case=profile.quoted_identifiers_ignore_case, - known_functions=(), - known_types=(), - relations={}, - ) - ) - if any(query.mode == _BATCH_MODE for query in queries) - else None - ) - adapter_rules: dict[str, FunctionNullabilityRule] = adapter_nullability_rules(profile) - return _native.infer_query_columns( - None if catalog is None else catalog.native, - ( - dialect, - list(profile.function_return_types.items()), - nullability_rule_rows(profile), - shape_rows(column_types_by_table), - shape_rows(column_nullability_by_table), - [_query_row(query) for query in queries], - profile.sql_analysis_dialect or "generic", - ), - (adapter_rules, InferredNullability) if adapter_rules else None, - ) - - -def _inferred_columns( - columns: list[tuple[str, str | None, str]], -) -> tuple[InferredColumn, ...]: - return tuple( - InferredColumn(name=name, type=data_type, nullability=InferredNullability(nullability)) - for name, data_type, nullability in columns - ) - - -def _query_row(query: NativeColumnQuery) -> tuple[object, ...]: - references: tuple[CompileSqlReference, ...] = query.references - return ( - query.sql, - list((query.placeholders or {}).items()), - [_analysis_name(reference) for reference in references], - [ - (name, resource_type, resource_name) - for name, (resource_type, resource_name) in _lineage_references(references).items() - ], - query.recover_cte_facts, - query.mode, - ) - - -def _lineage_references( - references: tuple[CompileSqlReference, ...], -) -> dict[str, tuple[str, str]]: - """Each lineage reference's analysis name, resource type and name; the last wins in place.""" - - return { - _analysis_name(reference): ( - _LINEAGE_RESOURCE_TYPES[reference.ref_kind].value, - reference.ref_name, - ) - for reference in references - if reference.ref_kind in _LINEAGE_RESOURCE_TYPES - } - - -def _analysis_name(reference: CompileSqlReference) -> str: - if reference.ref_kind == SqlReferenceKind.TABLE_FUNCTION: - return f"{TABLE_FUNCTION_ANALYSIS_PREFIX}{reference.ref_name}" - return reference.ref_name diff --git a/src/sqlbuild/compiler/analysis_session/main/_infer_native_expression_source_shapes.py b/src/sqlbuild/compiler/analysis_session/main/_infer_native_expression_source_shapes.py deleted file mode 100644 index 7509f3c4f5..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/_infer_native_expression_source_shapes.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Infer expression-source shapes natively.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session._helpers.expression_shapes import ( - native_expression_source_shapes, -) - - -def infer_native_expression_source_shapes( - *, expressions: tuple[str, ...], profile: ExpressionInferenceProfile -) -> tuple[dict[str, str] | None, ...]: - """One shape per expression; a native internal failure raises `NativeCompilerError`.""" - - return native_expression_source_shapes(expressions=expressions, profile=profile) diff --git a/src/sqlbuild/compiler/analysis_session/main/_infer_native_query_column_rows.py b/src/sqlbuild/compiler/analysis_session/main/_infer_native_query_column_rows.py deleted file mode 100644 index 2d44728cbf..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/_infer_native_query_column_rows.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Infer query output column rows natively outside the analysis session.""" - -from __future__ import annotations - -from collections.abc import Mapping, Sequence - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session._helpers.query_columns import native_query_column_rows -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery -from sqlbuild.compiler.analysis_session.types import ColumnRow -from sqlbuild.compiler.lineage.types import InferredNullability - - -def infer_native_query_column_rows( - *, - queries: Sequence[NativeColumnQuery], - profile: ExpressionInferenceProfile, - column_nullability_by_table: Mapping[str, Mapping[str, InferredNullability]] | None = None, -) -> list[tuple[bool, list[ColumnRow] | None, bool]]: - """Each query's `(succeeded, column rows, has star)`; a native internal failure raises.""" - - return native_query_column_rows( - queries=queries, - profile=profile, - column_types_by_table={}, - column_nullability_by_table=column_nullability_by_table or {}, - ) diff --git a/src/sqlbuild/compiler/analysis_session/main/_infer_native_query_columns.py b/src/sqlbuild/compiler/analysis_session/main/_infer_native_query_columns.py deleted file mode 100644 index b4837e9689..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/_infer_native_query_columns.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Infer the output columns of queries outside the native analysis session.""" - -from __future__ import annotations - -from collections.abc import Mapping, Sequence - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session._helpers.query_columns import native_query_columns -from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns -from sqlbuild.compiler.lineage.types import InferredNullability - - -def infer_native_query_columns( - *, - queries: Sequence[NativeColumnQuery], - profile: ExpressionInferenceProfile, - column_types_by_table: Mapping[str, Mapping[str, str]] | None = None, - column_nullability_by_table: Mapping[str, Mapping[str, InferredNullability]] | None = None, -) -> tuple[NativeQueryColumns, ...]: - """Each query's columns; a native internal failure raises `NativeCompilerError`.""" - - return native_query_columns( - queries=queries, - profile=profile, - column_types_by_table=column_types_by_table or {}, - column_nullability_by_table=column_nullability_by_table or {}, - ) diff --git a/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py b/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py deleted file mode 100644 index 81c0e362ff..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/_prove_native_dynamic_contracts.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Prove dynamic column contracts natively.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.compiler.analysis_session._helpers.dynamic_contracts import native_dynamic_contracts -from sqlbuild.compiler.analysis_session.models import NativePivotTables -from sqlbuild.compiler.analysis_session.types import ProofRow -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -def prove_native_dynamic_contracts( - *, - session: Any | None, - tables: NativePivotTables, - models: tuple[tuple[str, tuple[SchemaDynamicColumnFamily, ...]], ...], -) -> tuple[ProofRow | None, ...]: - """The proof row per `(sql, families)` model from the session or `tables`, else None.""" - - return native_dynamic_contracts(session=session, tables=tables, models=models) diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py b/src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py index 05b747a64d..1b7373d28d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/fixture_columns.py @@ -3,10 +3,8 @@ from __future__ import annotations import sqlbuild._native as _native +from sqlbuild._native import infer_native_query_columns from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( - infer_native_query_columns, -) from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns from sqlbuild.compiler.compile.models import ( FixtureColumnInference, diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py b/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py index 6c496aef8d..96987854ef 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py @@ -2,10 +2,8 @@ from __future__ import annotations +from sqlbuild._native import infer_native_query_columns from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( - infer_native_query_columns, -) from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns from sqlbuild.compiler.compile.models import CompiledModelSqlTestPayload from sqlbuild.compiler.compile.types import CompiledProjectView diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py index 77c5a9d064..31f71d945a 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py @@ -5,10 +5,8 @@ from dataclasses import replace from typing import Any +from sqlbuild._native import infer_native_expression_source_shapes from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.main._infer_native_expression_source_shapes import ( - infer_native_expression_source_shapes, -) from sqlbuild.compiler.compile._helpers.analysis.reference_names import table_function_analysis_name from sqlbuild.compiler.compile.models import ( CompileSqlReference, diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py b/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py index eb3c17763c..495f234713 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py @@ -6,10 +6,8 @@ from graphlib import TopologicalSorter from typing import Any +from sqlbuild._native import infer_native_query_columns from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( - infer_native_query_columns, -) from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns from sqlbuild.compiler.compile._helpers.analysis.binding_requests import ( get_complete_schema_binding_request, diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py index 9121966adb..f669defe84 100644 --- a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py +++ b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py @@ -4,16 +4,11 @@ from typing import Any +from sqlbuild._native import infer_native_expression_source_shapes, prove_native_dynamic_contracts from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.main._analyze_native_model_sql import ( analyze_native_model_sql, ) -from sqlbuild.compiler.analysis_session.main._infer_native_expression_source_shapes import ( - infer_native_expression_source_shapes, -) -from sqlbuild.compiler.analysis_session.main._prove_native_dynamic_contracts import ( - prove_native_dynamic_contracts, -) from sqlbuild.compiler.analysis_session.models import ( NativeModelAnalyses, NativeModelAnalysisRequest, diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index b919d9b2d3..80c1f701cb 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -16,7 +16,12 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.diagnostics.recovery as semantic_recovery import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly -from sqlbuild._native import build_compile_inputs, discover_project_inputs +from sqlbuild._native import ( + build_compile_inputs, + discover_project_inputs, + infer_native_expression_source_shapes, + prove_native_dynamic_contracts, +) from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main @@ -27,12 +32,6 @@ from sqlbuild.compiler.analysis_session.main._dynamic_contract_proof import ( dynamic_contract_proof as contract_proof, ) -from sqlbuild.compiler.analysis_session.main._infer_native_expression_source_shapes import ( - infer_native_expression_source_shapes, -) -from sqlbuild.compiler.analysis_session.main._prove_native_dynamic_contracts import ( - prove_native_dynamic_contracts, -) from sqlbuild.compiler.analysis_session.main._session_outcome_analysis import ( session_outcome_analysis, ) diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py index 3e507ee1c6..82852ea94d 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_query_columns.py @@ -13,10 +13,8 @@ import sqlbuild._native as native_module import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly +from sqlbuild._native import infer_native_query_columns from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( - infer_native_query_columns, -) from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeModelAnalyses from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index afbeadcaeb..90d2179bbd 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -1,5 +1,6 @@ from __future__ import annotations +import functools import json from collections.abc import Callable from pathlib import Path @@ -7,7 +8,8 @@ import pytest -import sqlbuild._native as native_module +import sqlbuild.compiler.compile._helpers.assembly.semantic_shapes as semantic_shapes_module +import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly from sqlbuild._native import ( build_compile_inputs, build_compiled_project, @@ -15,6 +17,7 @@ run_compile_pipeline, ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes @@ -212,17 +215,32 @@ def apply_warehouse_setup_sql(*, project_dir: Path, statements: tuple[str, ...]) def traced_shape_batches(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, ...]]: - """Record the expressions of every native expression-source shape batch.""" + """Record each shape request's expressions the binding catalog's shape cache misses.""" - original: Callable[..., Any] = native_module.infer_expression_source_shapes batches: list[tuple[str, ...]] = [] + for module in (native_stage_assembly, semantic_shapes_module): + original: Callable[..., Any] = module.infer_native_expression_source_shapes + monkeypatch.setattr( + module, + "infer_native_expression_source_shapes", + functools.partial(_traced_shapes, original=original, batches=batches), + ) + return batches - def traced(catalog: object, request: tuple[Any, ...], *rules: Any) -> Any: - batches.append(tuple(request[4])) - return original(catalog, request, *rules) - monkeypatch.setattr(native_module, "infer_expression_source_shapes", traced) - return batches +def _traced_shapes( + *, + original: Callable[..., Any], + batches: list[tuple[str, ...]], + expressions: tuple[str, ...], + profile: ExpressionInferenceProfile, +) -> tuple[dict[str, str] | None, ...]: + cache: dict[str, object] = getattr(profile.binding_catalog, "expression_shapes", {}) + pending: tuple[str, ...] = tuple( + dict.fromkeys(filter(lambda item: item not in cache, expressions)) + ) + batches.extend((pending,) * bool(pending)) + return original(expressions=expressions, profile=profile) _OFFLINE_POSTGRES_CONNECTION: dict[str, object] = { diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py index 2eccb09ea6..ca6728c312 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py @@ -2,9 +2,7 @@ import pytest -from sqlbuild.compiler.analysis_session._helpers.dynamic_contracts import ( - native_dynamic_contracts, -) +from sqlbuild._native import prove_native_dynamic_contracts from sqlbuild.compiler.analysis_session.main._dynamic_contract_proof import ( dynamic_contract_proof as contract_proof, ) @@ -151,7 +149,7 @@ def test_given_dynamic_family_when_analyzing_output_then_requires_supported_closed_shape( test_case: DynamicPivotAnalysisTestCase, ) -> None: - (row,) = native_dynamic_contracts( + (row,) = prove_native_dynamic_contracts( session=None, tables=NativePivotTables( dialect=test_case.dialect, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index e63daf8241..3c9327d4be 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -5,7 +5,12 @@ from types import MappingProxyType from typing import cast -from sqlbuild._native import assemble_project, build_compile_inputs, discover_project_inputs +from sqlbuild._native import ( + assemble_project, + build_compile_inputs, + discover_project_inputs, + infer_native_query_columns, +) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter @@ -13,9 +18,6 @@ from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( - infer_native_query_columns, -) from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( collect_compile_diagnostics, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py index d0eb22c4da..8520ec0c33 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_cursor_intrinsics.py @@ -2,10 +2,8 @@ import pytest +from sqlbuild._native import infer_native_query_columns from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( - infer_native_query_columns, -) from sqlbuild.compiler.analysis_session.models import NativeColumnQuery from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( cursor_intrinsics_analysis_sql, diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py index 9b54407c9e..d729ea72fb 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py @@ -2,10 +2,8 @@ import pytest +from sqlbuild._native import infer_native_query_columns from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.main._infer_native_query_columns import ( - infer_native_query_columns, -) from sqlbuild.compiler.analysis_session.models import NativeColumnQuery, NativeQueryColumns from sqlbuild.compiler.compile._helpers.analysis.reference_names import ( substitute_placeholder_defaults, From 79f8c37ce8584c5bf22343dc2ab8a03995367d1d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:23:14 +0100 Subject: [PATCH 11/14] refactor: drive the model analysis session natively --- .../bindings/_helpers/analysis_session/mod.rs | 2 + .../analysis_session/model_analyses.rs | 318 ++++++++++++++++++ .../analysis_session/outcome_objects.rs | 295 ++++++++++++++++ .../_helpers/analysis_session/registration.rs | 4 +- fensu.toml | 5 + src/sqlbuild/_native.pyi | 5 + .../analysis_session/_helpers/__init__.py | 1 - .../_helpers/model_analysis.py | 65 ---- .../analysis_session/_helpers/profile_rows.py | 64 ---- .../analysis_session/_helpers/session_rows.py | 188 ----------- .../analysis_session/classes/__init__.py | 1 - .../classes/native_model_analysis.py | 116 ------- .../compiler/analysis_session/constants.py | 8 - .../analysis_session/main/__init__.py | 1 - .../main/_analyze_native_model_sql.py | 15 - .../main/_dynamic_contract_proof.py | 13 - .../main/_session_outcome_analysis.py | 20 -- .../_helpers/assembly/hand_built_models.py | 5 +- .../compile/_helpers/diagnostics/recovery.py | 4 +- .../_helpers/native_stages/assembly.py | 7 +- .../cli/commands/main/compile/helpers.py | 2 +- .../compile/test_native_analysis_seams.py | 6 +- .../analysis_session/_test_helpers.py | 18 +- .../compiler/analysis_session/helpers.py | 12 +- .../test_native_model_analysis.py | 5 +- .../_helpers/analysis/test_dynamic_pivot.py | 4 +- 26 files changed, 660 insertions(+), 524 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs delete mode 100644 src/sqlbuild/compiler/analysis_session/_helpers/__init__.py delete mode 100644 src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py delete mode 100644 src/sqlbuild/compiler/analysis_session/_helpers/profile_rows.py delete mode 100644 src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py delete mode 100644 src/sqlbuild/compiler/analysis_session/classes/__init__.py delete mode 100644 src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py delete mode 100644 src/sqlbuild/compiler/analysis_session/main/__init__.py delete mode 100644 src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py delete mode 100644 src/sqlbuild/compiler/analysis_session/main/_dynamic_contract_proof.py delete mode 100644 src/sqlbuild/compiler/analysis_session/main/_session_outcome_analysis.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs index 3e2bb419c9..d212ed073a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs @@ -1,5 +1,7 @@ +pub(crate) mod model_analyses; pub(crate) mod nullability_rules; pub(crate) mod oracle; +pub(crate) mod outcome_objects; pub(crate) mod profile_inputs; pub(crate) mod query_columns; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs new file mode 100644 index 0000000000..965951e4aa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -0,0 +1,318 @@ +//! One compile's native model analysis session, with Python's catalog and metric bookkeeping. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyString, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::analysis_session::profile_inputs::{ + case_sensitive_shapes, nullability_rules, profile_dialect, return_types, shape_rows, +}; +use crate::bindings::_helpers::analysis_session::stage_functions::family_rows; +use crate::bindings::_helpers::boundary::keyword_request::Request; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; +const PYTHON_ANALYSIS_MODULE: &str = "sqlbuild.compiler.compile.classes.python_model_analysis"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const CODE_IDENTITY_MODULE: &str = "sqlbuild.compiler.frontier.main.compiled_code_identity"; +const LOG_DEBUG_MODULE: &str = "sqlbuild.diagnostics.main.log_debug_event"; +const LOGGING_MODULE: &str = "logging"; +const DEBUG_LOGGER: &str = "sqlbuild.compile"; +const ANALYSIS_FAILURE_MESSAGE: &str = "native compact query analysis failed"; +const STORE_FAILURE_MESSAGE: &str = "native model analysis cache not read or saved"; +const STORE_FILE_NAME: &str = "model-analyses.bin"; +const STORE_VERSION: &str = "native-model-analysis-store-v1"; + +/// Each model's analysis by name and the finished session; a native internal failure raises. +#[pyfunction] +#[pyo3(signature = (**request))] +fn analyze_native_model_sql<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("analyze_native_model_sql", request)?; + let py: Python<'py> = request.py(); + let request: Bound<'py, PyAny> = request.get("request")?; + let analyses_class: Bound<'py, PyAny> = py + .import(ANALYSIS_MODELS_MODULE)? + .getattr("NativeModelAnalyses")?; + let model_inputs: Bound<'py, PyAny> = request.getattr("model_inputs")?; + if !model_inputs.is_truthy()? { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("positions", PyDict::new(py))?; + arguments.set_item("succeeded", PyFrozenSet::empty(py)?)?; + arguments.set_item("session", py.None())?; + return analyses_class.call((), Some(&arguments)); + } + let profile: Bound<'py, PyAny> = request.getattr("inference_profile")?; + let catalog: Bound<'py, PyAny> = profile.getattr("binding_catalog")?; + let python: Bound<'py, PyAny> = { + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("model_inputs", &model_inputs)?; + arguments.set_item("inference_profile", &profile)?; + arguments.set_item( + "complete_binding_schemas", + request.getattr("complete_binding_schemas")?, + )?; + py.import(PYTHON_ANALYSIS_MODULE)? + .getattr("PythonModelAnalysis")? + .call((), Some(&arguments))? + }; + let (rules, adapter_rules) = nullability_rules(&profile)?; + let session_request: Bound<'py, PyAny> = session_request(&request, &python, &catalog, rules)?; + let (cache, adapter): (Bound<'py, PyAny>, Bound<'py, PyAny>) = match adapter_rules { + Some((adapter, nullability)) => ( + py.None().into_bound(py), + PyTuple::new( + py, + [ + adapter.into_any().into_bound(py), + nullability.into_bound(py), + ], + )? + .into_any(), + ), + None => (analysis_store(&request)?, py.None().into_bound(py)), + }; + let session: Bound<'py, PyAny> = py + .import(NATIVE_MODULE)? + .getattr("start_model_analysis_session")? + .call1((catalog.getattr("native")?, session_request, cache, adapter))?; + run_session(&request, (&python, &catalog), &session, &analyses_class) +} + +/// The session request row, in the order the native session reads it. +fn session_request<'py>( + request: &Bound<'py, PyAny>, + python: &Bound<'py, PyAny>, + catalog: &Bound<'py, PyAny>, + rules: Vec<(String, String)>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let profile: Bound<'py, PyAny> = request.getattr("inference_profile")?; + let mut families: Vec<(String, Vec<_>)> = Vec::new(); + for item in request + .getattr("dynamic_families_by_table")? + .call_method0("items")? + .try_iter()? + { + let (name, declared): (String, Bound<'py, PyAny>) = item?.extract()?; + families.push((name, family_rows(&declared)?)); + } + let items: Vec> = vec![ + profile_dialect(&profile)?.into_bound_py_any(py)?, + case_sensitive_shapes(&profile, &profile.getattr("sql_analysis_dialect")?)? + .into_bound_py_any(py)?, + return_types(&profile)?.into_bound_py_any(py)?, + rules.into_bound_py_any(py)?, + request.getattr("rich_type_inference")?, + shape_rows(&request.getattr("column_types_by_table")?)?.into_bound_py_any(py)?, + shape_rows(&request.getattr("column_nullability_by_table")?)?.into_bound_py_any(py)?, + shape_rows(&request.getattr("complete_binding_schemas")?)?.into_bound_py_any(py)?, + shape_rows(&catalog.getattr("schemas")?)?.into_bound_py_any(py)?, + families.into_bound_py_any(py)?, + python.call_method0("model_rows")?, + ]; + Ok(PyTuple::new(py, items)?.into_any()) +} + +/// The analysis store's path and environment, or None where Python bypasses its cache. +fn analysis_store<'py>(request: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = request.py(); + let context: Bound<'py, PyAny> = request.getattr("analysis_cache")?; + if context.is_none() { + return Ok(py.None().into_bound(py)); + } + let path: Bound<'py, PyAny> = context + .getattr("root")? + .div(STORE_FILE_NAME)? + .str()? + .into_any(); + let identity: Bound<'py, PyAny> = py + .import(CODE_IDENTITY_MODULE)? + .getattr("compiled_code_identity")? + .call0()?; + let parts: Bound<'py, PyList> = PyList::new( + py, + [ + PyString::new(py, STORE_VERSION).into_any(), + identity, + context.getattr("shared_fingerprint")?, + context.getattr("signature_namespace")?, + ], + )?; + let environment: Bound<'py, PyAny> = py + .import(NATIVE_MODULE)? + .getattr("content_digest")? + .call1((parts,))?; + Ok(PyTuple::new(py, [path, environment])?.into_any()) +} + +/// Run and finish the session, leaving Python's catalog and metrics as its analysis did. +fn run_session<'py>( + request: &Bound<'py, PyAny>, + (python, catalog): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + session: &Bound<'py, PyAny>, + analyses_class: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = request.py(); + let types: Bound<'py, PyDict> = copied(&request.getattr("column_types_by_table")?)?; + let nullability: Bound<'py, PyDict> = copied(&request.getattr("column_nullability_by_table")?)?; + let (publications, failures): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + session.call_method0("run")?.extract()?; + for failure in failures.try_iter()? { + log_debug(py, ANALYSIS_FAILURE_MESSAGE, &failure?)?; + } + let unknown: Bound<'py, PyAny> = py + .import(LINEAGE_TYPES_MODULE)? + .getattr("InferredNullability")? + .getattr("UNKNOWN")?; + for item in publications.try_iter()? { + let (name, shape): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + let published: Bound<'py, PyAny> = dict_of(&shape)?; + types.call_method1("setdefault", (&name, &published))?; + let defaults: Bound<'py, PyAny> = PyDict::new(py) + .into_any() + .getattr("fromkeys")? + .call1((&published, &unknown))?; + nullability.call_method1("setdefault", (&name, defaults))?; + } + let (additions, analysis_names, succeeded): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = session.call_method0("finish")?.extract()?; + record_catalog_changes(catalog, &additions, &analysis_names, (&types, &nullability))?; + record_cache(python, session)?; + let positions: Bound<'py, PyDict> = PyDict::new(py); + let mut names: Vec> = Vec::new(); + for analysed in python.getattr("requests")?.try_iter()? { + let name: Bound<'py, PyAny> = analysed? + .getattr("model_input")? + .getattr("model_file")? + .getattr("file_path")? + .getattr("stem")?; + positions.set_item(&name, names.len())?; + names.push(name); + } + let succeeded: Vec = succeeded.extract()?; + let succeeded_names: Vec<&Bound<'py, PyAny>> = names + .iter() + .zip(&succeeded) + .filter_map(|(name, success)| success.then_some(name)) + .collect(); + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("positions", positions)?; + arguments.set_item("succeeded", PyFrozenSet::new(py, succeeded_names)?)?; + arguments.set_item("session", session)?; + analyses_class.call((), Some(&arguments)) +} + +/// Leave Python's binding catalog as its own analysis would have. +fn record_catalog_changes<'py>( + catalog: &Bound<'py, PyAny>, + schema_additions: &Bound<'py, PyAny>, + analysis_names: &Bound<'py, PyAny>, + (types, nullability): (&Bound<'py, PyDict>, &Bound<'py, PyDict>), +) -> PyResult<()> { + let py: Python<'py> = catalog.py(); + let schemas: Bound<'py, PyAny> = catalog.getattr("schemas")?; + let additions: Bound<'py, PyDict> = PyDict::new(py); + for item in schema_additions.try_iter()? { + let (name, shape): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + if !schemas.contains(&name)? { + additions.set_item(name, dict_of(&shape)?)?; + } + } + schemas.call_method1("update", (&additions,))?; + if !additions.is_empty() { + catalog + .getattr("native")? + .call_method1("update_relations", (&additions,))?; + } + let selected_types: Bound<'py, PyDict> = PyDict::new(py); + let selected_nullability: Bound<'py, PyDict> = PyDict::new(py); + let names: Vec> = analysis_names.try_iter()?.collect::>()?; + for name in &names { + if let Some(shape) = types.get_item(name)? { + selected_types.set_item(name, shape)?; + } + } + for name in &names { + if let Some(shape) = nullability.get_item(name)? { + selected_nullability.set_item(name, shape)?; + } + } + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("types", selected_types)?; + arguments.set_item("nullability", selected_nullability)?; + catalog + .getattr("prepare_analysis")? + .call((), Some(&arguments))?; + Ok(()) +} + +fn record_cache(python: &Bound<'_, PyAny>, session: &Bound<'_, PyAny>) -> PyResult<()> { + let stats: Bound<'_, PyAny> = session.getattr("cache_stats")?; + if stats.is_none() { + python.call_method0("record_uncached")?; + return Ok(()); + } + let (hits, misses, _, failure): ( + Bound<'_, PyAny>, + Bound<'_, PyAny>, + Bound<'_, PyAny>, + Bound<'_, PyAny>, + ) = stats.extract()?; + if !failure.is_none() { + log_debug(python.py(), STORE_FAILURE_MESSAGE, &failure)?; + } + if hits.add(&misses)?.eq(0)? { + python.call_method0("record_uncached")?; + return Ok(()); + } + let arguments: Bound<'_, PyDict> = PyDict::new(python.py()); + arguments.set_item("hits", hits)?; + arguments.set_item("misses", misses)?; + python + .getattr("record_cached")? + .call((), Some(&arguments))?; + Ok(()) +} + +fn log_debug(py: Python<'_>, message: &str, failure: &Bound<'_, PyAny>) -> PyResult<()> { + let arguments: Bound<'_, PyDict> = PyDict::new(py); + arguments.set_item( + "logger", + py.import(LOGGING_MODULE)? + .getattr("getLogger")? + .call1((DEBUG_LOGGER,))?, + )?; + arguments.set_item("message", message)?; + arguments.set_item("sqlbuild_error", failure)?; + py.import(LOG_DEBUG_MODULE)? + .getattr("log_debug_event")? + .call((), Some(&arguments))?; + Ok(()) +} + +/// `dict(mapping)`. +fn copied<'py>(mapping: &Bound<'py, PyAny>) -> PyResult> { + let copy: Bound<'py, PyDict> = PyDict::new(mapping.py()); + copy.update(mapping.downcast()?)?; + Ok(copy) +} + +/// `dict(pairs)`. +fn dict_of<'py>(pairs: &Bound<'py, PyAny>) -> PyResult> { + let dict: Bound<'py, PyDict> = PyDict::new(pairs.py()); + for item in pairs.try_iter()? { + let (key, value): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item?.extract()?; + dict.set_item(key, value)?; + } + Ok(dict.into_any()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(analyze_native_model_sql, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs new file mode 100644 index 0000000000..b3d9bb9962 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs @@ -0,0 +1,295 @@ +//! Finished session outcomes and pivot proof rows as Python's analysis objects. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; + +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const SQL_ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.sql_analysis.models"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const LINEAGE_NATIVE: &str = "native"; + +type Object<'py> = Bound<'py, PyAny>; +/// `(succeeded, columns, lineage source, lineage rows, star, resolved, bindings, validated, SQL)`. +type OutcomeObjects<'py> = ( + Object<'py>, + Object<'py>, + String, + Object<'py>, + Object<'py>, + Object<'py>, + Object<'py>, + Object<'py>, + Object<'py>, +); +/// `(output proven, fixed columns, families, input relations, failure reason, bare pivot)`. +type ProofObjects<'py> = ( + Object<'py>, + Object<'py>, + Object<'py>, + Object<'py>, + Object<'py>, + Object<'py>, +); + +/// The outcome at `position` of a finished session as `ModelSqlAnalysis`, for hand-built inputs. +#[pyfunction] +#[pyo3(signature = (**request))] +fn session_outcome_analysis<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("session_outcome_analysis", request)?; + let py: Python<'py> = request.py(); + let (outcome, contract): (Bound<'py, PyAny>, Bound<'py, PyAny>) = request + .get("session")? + .call_method1("outcome_rows", (request.get("position")?,))? + .extract()?; + let (succeeded, columns, source, rows, has_star, star_resolved, raw, validated, cleaned): OutcomeObjects<'py> = outcome.extract()?; + let models = py.import(COMPILE_MODELS_MODULE)?; + let lineage: Bound<'py, PyAny> = if source != LINEAGE_NATIVE { + lineage_facts(&rows)? + } else if succeeded.is_truthy()? { + compact_lineage(&rows)? + } else { + PyTuple::empty(py).into_any() + }; + let binding_class: Bound<'py, PyAny> = py + .import(SQL_ANALYSIS_MODELS_MODULE)? + .getattr("SqlBindingDiagnostic")?; + let mut bindings: Vec> = Vec::new(); + for row in raw.try_iter()? { + let row: Bound<'py, PyTuple> = + PyTuple::new(py, row?.try_iter()?.collect::>>()?)?; + bindings.push(binding_class.call1(row)?); + } + let result: Bound<'py, PyAny> = call( + &models.getattr("PolyglotAnalysisResult")?, + &[ + ("analysis_succeeded", succeeded), + ("columns", inferred_columns(&columns)?), + ("lineage_columns", lineage), + ("has_star", has_star), + ("star_resolved", star_resolved), + ( + "binding_diagnostics", + PyTuple::new(py, bindings)?.into_any(), + ), + ("binding_validated", validated), + ], + )?; + call( + &models.getattr("ModelSqlAnalysis")?, + &[ + ("polyglot_analysis", result), + ("placeholders", request.get("placeholders")?), + ("fused_binding_validated", true.into_bound_py_any(py)?), + ("cleaned_sql", cleaned), + ("dynamic_column_contract", proof(&contract.get_item(1)?)?), + ], + ) +} + +/// A native dynamic pivot proof row as Python's proof, None when absent. +#[pyfunction] +fn dynamic_contract_proof<'py>(row: &Bound<'py, PyAny>) -> PyResult> { + proof(row) +} + +fn proof<'py>(row: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = row.py(); + if row.is_none() { + return Ok(row.clone()); + } + let (output_proven, columns, families, inputs, failure_reason, bare): ProofObjects<'py> = + row.extract()?; + let models = py.import(COMPILE_MODELS_MODULE)?; + let family_class: Bound<'py, PyAny> = models.getattr("DynamicColumnFamilyProof")?; + let mut proofs: Vec> = Vec::new(); + for family in families.try_iter()? { + let (name, inferred_type): (Bound<'py, PyAny>, Bound<'py, PyAny>) = family?.extract()?; + proofs.push(call( + &family_class, + &[("name", name), ("inferred_type", inferred_type)], + )?); + } + let fixed: Bound<'py, PyAny> = inferred_columns(&columns)?; + let fixed: Bound<'py, PyAny> = if fixed.is_truthy()? { + fixed + } else { + PyTuple::empty(py).into_any() + }; + call( + &models.getattr("DynamicColumnContractProof")?, + &[ + ("output_proven", output_proven), + ("fixed_columns", fixed), + ("families", PyTuple::new(py, proofs)?.into_any()), + ("input_relations", tuple_of(&inputs)?), + ("failure_reason", failure_reason), + ("bare_dynamic_pivot", bare), + ], + ) +} + +/// `(name, type, nullability)` rows as `InferredColumn`s, None for None. +fn inferred_columns<'py>(rows: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = rows.py(); + if rows.is_none() { + return Ok(rows.clone()); + } + let column_class: Bound<'py, PyAny> = py + .import(COMPILE_MODELS_MODULE)? + .getattr("InferredColumn")?; + let nullability_class: Bound<'py, PyAny> = py + .import(LINEAGE_TYPES_MODULE)? + .getattr("InferredNullability")?; + let mut columns: Vec> = Vec::new(); + for row in rows.try_iter()? { + let (name, data_type, nullability): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = row?.extract()?; + columns.push(call( + &column_class, + &[ + ("name", name), + ("type", data_type), + ("nullability", nullability_class.call1((nullability,))?), + ], + )?); + } + Ok(PyTuple::new(py, columns)?.into_any()) +} + +/// Native lineage rows as Python's compact facts over a per-model string pool. +fn compact_lineage<'py>(rows: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = rows.py(); + let pool: Bound<'py, PyDict> = PyDict::new(py); + let index = |value: Bound<'py, PyAny>| -> PyResult> { + if let Some(existing) = pool.get_item(&value)? { + return Ok(existing); + } + let position = pool.len().into_bound_py_any(py)?; + pool.set_item(&value, &position)?; + Ok(position) + }; + let mut compact: Vec> = Vec::new(); + for row in rows.try_iter()? { + let (output_column, transform_code, confidence_code, sources): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = row?.extract()?; + let mut source_indexes: Vec> = Vec::new(); + for source in sources.try_iter()? { + let (resource_type, resource_name, column_name): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = source?.extract()?; + let indexes = [ + index(resource_type)?, + index(resource_name)?, + index(column_name)?, + ]; + source_indexes.push(PyTuple::new(py, indexes)?.into_any()); + } + let output: Bound<'py, PyAny> = index(output_column)?; + compact.push( + PyTuple::new( + py, + [ + output, + transform_code, + confidence_code, + PyTuple::new(py, source_indexes)?.into_any(), + ], + )? + .into_any(), + ); + } + let keys: Bound<'py, PyAny> = PyTuple::new(py, pool.keys())?.into_any(); + call( + &py.import(COMPILE_MODELS_MODULE)? + .getattr("CompactLineageFacts")?, + &[ + ("string_pool", keys), + ("rows", PyTuple::new(py, compact)?.into_any()), + ], + ) +} + +/// Native lineage rows as the plain facts Python's re-analysis returns. +fn lineage_facts<'py>(rows: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = rows.py(); + let models = py.import(COMPILE_MODELS_MODULE)?; + let column_class: Bound<'py, PyAny> = models.getattr("CompiledLineageColumnFact")?; + let source_class: Bound<'py, PyAny> = models.getattr("CompiledLineageSourceFact")?; + let compact: Bound<'py, PyAny> = models.getattr("CompactLineageFacts")?; + let mut facts: Vec> = Vec::new(); + for row in rows.try_iter()? { + let (output_column, transform_code, confidence_code, sources): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = row?.extract()?; + let mut upstream: Vec> = Vec::new(); + for source in sources.try_iter()? { + let (resource_type, resource_name, column_name): ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + ) = source?.extract()?; + upstream.push(call( + &source_class, + &[ + ("resource_type", resource_type), + ("resource_name", resource_name), + ("column_name", column_name), + ], + )?); + } + facts.push(call( + &column_class, + &[ + ("output_column", output_column), + ("upstream_columns", PyTuple::new(py, upstream)?.into_any()), + ( + "transform_kind", + compact.call_method1("transform_kind", (transform_code,))?, + ), + ( + "confidence", + compact.call_method1("confidence", (confidence_code,))?, + ), + ], + )?); + } + Ok(PyTuple::new(py, facts)?.into_any()) +} + +fn tuple_of<'py>(items: &Bound<'py, PyAny>) -> PyResult> { + Ok(PyTuple::new(items.py(), items.try_iter()?.collect::>>()?)?.into_any()) +} + +fn call<'py>( + callable: &Bound<'py, PyAny>, + pairs: &[(&str, Bound<'py, PyAny>)], +) -> PyResult> { + let arguments: Bound<'py, PyDict> = PyDict::new(callable.py()); + for (name, value) in pairs { + arguments.set_item(*name, value)?; + } + callable.call((), Some(&arguments)) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(session_outcome_analysis, module)?)?; + module.add_function(wrap_pyfunction!(dynamic_contract_proof, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs index c58f280a4c..b0a64e9832 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs @@ -3,11 +3,13 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::analysis_session::{ - oracle, query_columns, session, stage_functions, + model_analyses, oracle, outcome_objects, query_columns, session, stage_functions, }; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { session::register(module)?; + model_analyses::register(module)?; + outcome_objects::register(module)?; query_columns::register(module)?; stage_functions::register(module)?; oracle::register(module) diff --git a/fensu.toml b/fensu.toml index 998041d176..58169a47a0 100644 --- a/fensu.toml +++ b/fensu.toml @@ -73,6 +73,11 @@ rule = "FFR309" path = "src/sqlbuild/compiler/lineage/__init__.py" reason = "Column lineage is built natively; the package keeps only the lineage models, enums and errors the native graph and its callers share." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/compiler/analysis_session/__init__.py" +reason = "Model analysis runs natively; the package keeps only the request and result models, row types and constants the native session and its callers share." + [[targets.python.rule_exceptions]] rule = "FFR601" path = "src/sqlbuild/adapter/contract/classes/base_adapter.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 5c45c46c11..871d864114 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1738,6 +1738,11 @@ def start_model_analysis_session( cache: tuple[str, str] | None = None, adapter_rules: tuple[Mapping[str, Callable[..., object]], type] | None = None, ) -> NativeModelAnalysisSession: ... +def analyze_native_model_sql(*, request: Any) -> Any: ... +def session_outcome_analysis( + *, session: Any, position: int, placeholders: dict[str, str] | None +) -> Any: ... +def dynamic_contract_proof(row: Any, /) -> Any: ... def infer_native_query_column_rows( *, queries: Sequence[Any], diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/__init__.py b/src/sqlbuild/compiler/analysis_session/_helpers/__init__.py deleted file mode 100644 index 64fb62aa62..0000000000 --- a/src/sqlbuild/compiler/analysis_session/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native model analysis session helpers.""" diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py b/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py deleted file mode 100644 index a6d659465f..0000000000 --- a/src/sqlbuild/compiler/analysis_session/_helpers/model_analysis.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Run the native model analysis session.""" - -from __future__ import annotations - -from typing import Any, cast - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.types import FunctionNullabilityRule -from sqlbuild.compiler.analysis_session._helpers.profile_rows import adapter_nullability_rules -from sqlbuild.compiler.analysis_session._helpers.session_rows import session_request -from sqlbuild.compiler.analysis_session.classes.native_model_analysis import NativeModelAnalysis -from sqlbuild.compiler.analysis_session.constants import ( - NATIVE_ANALYSIS_STORE_FILE_NAME, - NATIVE_ANALYSIS_STORE_VERSION, -) -from sqlbuild.compiler.analysis_session.models import ( - NativeModelAnalyses, - NativeModelAnalysisRequest, -) -from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis -from sqlbuild.compiler.compile.models import AnalysisCacheContext -from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity -from sqlbuild.compiler.lineage.types import InferredNullability - - -def native_model_analyses(*, request: NativeModelAnalysisRequest) -> NativeModelAnalyses: - """Each model's analysis by name and the finished session; a native internal failure raises.""" - - if not request.model_inputs: - return NativeModelAnalyses(positions={}, succeeded=frozenset(), session=None) - catalog: Any = cast(Any, request.inference_profile.binding_catalog) - python: PythonModelAnalysis = PythonModelAnalysis( - model_inputs=request.model_inputs, - inference_profile=request.inference_profile, - complete_binding_schemas=request.complete_binding_schemas, - ) - adapter_rules: dict[str, FunctionNullabilityRule] = adapter_nullability_rules( - request.inference_profile - ) - session: _native.NativeModelAnalysisSession = _native.start_model_analysis_session( - catalog.native, - session_request(request=request, python=python, schemas=catalog.schemas), - None if adapter_rules else _analysis_store(request), - (adapter_rules, InferredNullability) if adapter_rules else None, - ) - return NativeModelAnalysis(request=request, python=python, catalog=catalog).analyses(session) - - -def _analysis_store(request: NativeModelAnalysisRequest) -> tuple[str, str] | None: - """The analysis store's path and environment, or None where Python bypasses its cache.""" - - context: AnalysisCacheContext | None = request.analysis_cache - if context is None: - return None - return ( - str(context.root / NATIVE_ANALYSIS_STORE_FILE_NAME), - _native.content_digest( - [ - NATIVE_ANALYSIS_STORE_VERSION, - compiled_code_identity(), - context.shared_fingerprint, - context.signature_namespace, - ] - ), - ) diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/profile_rows.py b/src/sqlbuild/compiler/analysis_session/_helpers/profile_rows.py deleted file mode 100644 index e4cb8eb2a5..0000000000 --- a/src/sqlbuild/compiler/analysis_session/_helpers/profile_rows.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Inference-profile settings crossing into native analysis: rule ids and shape case.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.adapter.contract.types import FunctionNullabilityRule -from sqlbuild.adapter.type_system.main.conditional_result_nullability import ( - conditional_result_nullability, -) -from sqlbuild.adapter.type_system.main.first_arg_nullability import first_arg_nullability -from sqlbuild.adapter.type_system.main.safe_cast_nullability import safe_cast_nullability -from sqlbuild.compiler.analysis_session.constants import ( - NULLABILITY_RULE_ADAPTER, - NULLABILITY_RULE_CONDITIONAL_RESULT, - NULLABILITY_RULE_FIRST_ARG, - NULLABILITY_RULE_SAFE_CAST, -) -from sqlbuild.compiler.analysis_session.types import ShapeRows -from sqlbuild.compiler.sql_analysis.constants import CASE_SENSITIVE_BINDING_DIALECTS - -_NULLABILITY_RULE_IDS: dict[FunctionNullabilityRule, str] = { - first_arg_nullability: NULLABILITY_RULE_FIRST_ARG, - conditional_result_nullability: NULLABILITY_RULE_CONDITIONAL_RESULT, - safe_cast_nullability: NULLABILITY_RULE_SAFE_CAST, -} - - -def nullability_rule_rows(profile: ExpressionInferenceProfile) -> list[tuple[str, str]]: - """Adapter nullability rules as `(name, rule id)`; an adapter's own rule is `python`.""" - - return [ - (name, _NULLABILITY_RULE_IDS.get(rule, NULLABILITY_RULE_ADAPTER)) - for name, rule in profile.function_nullability_rules.items() - ] - - -def adapter_nullability_rules( - profile: ExpressionInferenceProfile, -) -> dict[str, FunctionNullabilityRule]: - """The adapter's own nullability rules, which the native session calls back into.""" - - return { - name: rule - for name, rule in profile.function_nullability_rules.items() - if rule not in _NULLABILITY_RULE_IDS - } - - -def case_sensitive_shapes(*, profile: ExpressionInferenceProfile, dialect: str | None) -> bool: - """Python's `inferred_binding_shape` test for keeping authored identifier quoting.""" - - return not profile.quoted_identifiers_ignore_case and dialect in CASE_SENSITIVE_BINDING_DIALECTS - - -def shape_rows(shapes: Mapping[str, Mapping[str, str]]) -> ShapeRows: - """`{relation: {column: value}}` as ordered rows; discovery rejects non-text names and types.""" - - return [(name, _shape_pairs(shape)) for name, shape in shapes.items()] - - -def _shape_pairs(shape: Mapping[str, str]) -> list[tuple[str, str]]: - return [(column, str(value)) for column, value in shape.items()] diff --git a/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py b/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py deleted file mode 100644 index c0566c871f..0000000000 --- a/src/sqlbuild/compiler/analysis_session/_helpers/session_rows.py +++ /dev/null @@ -1,188 +0,0 @@ -"""Plain rows crossing into and out of the native model analysis session.""" - -from __future__ import annotations - -from collections.abc import Mapping, Sequence - -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session._helpers.profile_rows import ( - case_sensitive_shapes, - nullability_rule_rows, - shape_rows, -) -from sqlbuild.compiler.analysis_session.constants import LINEAGE_NATIVE -from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest -from sqlbuild.compiler.analysis_session.types import ( - ColumnRow, - DiagnosticRow, - FamilyRow, - LineageItem, - OutcomeRow, - ProofRow, - ShapeRows, -) -from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis -from sqlbuild.compiler.compile.models import ( - CompactLineageFacts, - CompiledLineageColumnFact, - CompiledLineageSourceFact, - DynamicColumnContractProof, - DynamicColumnFamilyProof, - InferredColumn, - ModelSqlAnalysis, - PolyglotAnalysisResult, -) -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic -from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily - - -def session_request( - *, - request: NativeModelAnalysisRequest, - python: PythonModelAnalysis, - schemas: Mapping[str, Mapping[str, str]], -) -> tuple[object, ...]: - """The session request.""" - - profile: ExpressionInferenceProfile = request.inference_profile - shapes: tuple[ShapeRows, ...] = ( - shape_rows(request.column_types_by_table), - shape_rows(request.column_nullability_by_table), - shape_rows(request.complete_binding_schemas), - shape_rows(schemas), - ) - return ( - profile.sql_analysis_dialect or "generic", - case_sensitive_shapes(profile=profile, dialect=profile.sql_analysis_dialect), - list(profile.function_return_types.items()), - nullability_rule_rows(profile), - request.rich_type_inference, - *shapes, - family_rows(request.dynamic_families_by_table), - python.model_rows(), - ) - - -def outcome_analysis( - *, outcome: OutcomeRow, contract: ProofRow | None, placeholders: dict[str, str] | None -) -> ModelSqlAnalysis: - """One finished session outcome as the `ModelSqlAnalysis` it replaced.""" - - succeeded, columns, source, rows, has_star, star_resolved, raw, validated, cleaned = outcome - return ModelSqlAnalysis( - polyglot_analysis=PolyglotAnalysisResult( - analysis_succeeded=succeeded, - columns=inferred_columns(columns), - lineage_columns=_lineage(succeeded=succeeded, source=source, rows=rows), - has_star=has_star, - star_resolved=star_resolved, - binding_diagnostics=binding_diagnostics(raw), - binding_validated=validated, - ), - placeholders=placeholders, - fused_binding_validated=True, - cleaned_sql=cleaned, - dynamic_column_contract=contract_proof(contract), - ) - - -def _lineage( - *, succeeded: bool, source: str, rows: list[LineageItem] -) -> Sequence[CompiledLineageColumnFact]: - if source != LINEAGE_NATIVE: - return lineage_facts(rows) - return compact_lineage(rows) if succeeded else () - - -def contract_proof(row: ProofRow | None) -> DynamicColumnContractProof | None: - """A native dynamic pivot proof as Python's, None when absent.""" - - if row is None: - return None - output_proven, columns, families, input_relations, failure_reason, bare = row - return DynamicColumnContractProof( - output_proven=output_proven, - fixed_columns=inferred_columns(columns) or (), - families=tuple( - DynamicColumnFamilyProof(name=name, inferred_type=inferred_type) - for name, inferred_type in families - ), - input_relations=tuple(input_relations), - failure_reason=failure_reason, - bare_dynamic_pivot=bare, - ) - - -def inferred_columns(rows: list[ColumnRow] | None) -> tuple[InferredColumn, ...] | None: - if rows is None: - return None - return tuple( - InferredColumn(name=name, type=data_type, nullability=InferredNullability(nullability)) - for name, data_type, nullability in rows - ) - - -def binding_diagnostics(rows: list[DiagnosticRow]) -> tuple[SqlBindingDiagnostic, ...]: - return tuple(SqlBindingDiagnostic(*row) for row in rows) - - -def compact_lineage(rows: list[LineageItem]) -> CompactLineageFacts: - """Native lineage rows as Python's compact facts over a per-model string pool.""" - - pool: dict[str, int] = {} - compact_rows: list[tuple[int, int, int, tuple[tuple[int, int, int], ...]]] = [] - for output_column, transform_code, confidence_code, sources in rows: - source_indexes: list[tuple[int, int, int]] = [] - for resource_type, resource_name, column_name in sources: - source_indexes.append( - ( - pool.setdefault(resource_type, len(pool)), - pool.setdefault(resource_name, len(pool)), - pool.setdefault(column_name, len(pool)), - ) - ) - compact_rows.append( - ( - pool.setdefault(output_column, len(pool)), - transform_code, - confidence_code, - tuple(source_indexes), - ) - ) - return CompactLineageFacts(string_pool=tuple(pool), rows=tuple(compact_rows)) - - -def lineage_facts(rows: list[LineageItem]) -> tuple[CompiledLineageColumnFact, ...]: - """Native lineage rows as the plain facts Python's re-analysis returns.""" - - return tuple( - CompiledLineageColumnFact( - output_column=output_column, - upstream_columns=_source_facts(sources), - transform_kind=CompactLineageFacts.transform_kind(transform_code), - confidence=CompactLineageFacts.confidence(confidence_code), - ) - for output_column, transform_code, confidence_code, sources in rows - ) - - -def _source_facts(sources: list[tuple[str, str, str]]) -> tuple[CompiledLineageSourceFact, ...]: - return tuple( - CompiledLineageSourceFact( - resource_type=resource_type, resource_name=resource_name, column_name=column_name - ) - for resource_type, resource_name, column_name in sources - ) - - -def family_rows( - families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], -) -> list[tuple[str, list[FamilyRow]]]: - """Declared dynamic families by relation, as native request rows.""" - - return [(name, _family_rows(families)) for name, families in families_by_table.items()] - - -def _family_rows(families: tuple[SchemaDynamicColumnFamily, ...]) -> list[FamilyRow]: - return [PythonModelAnalysis.family_row(family) for family in families] diff --git a/src/sqlbuild/compiler/analysis_session/classes/__init__.py b/src/sqlbuild/compiler/analysis_session/classes/__init__.py deleted file mode 100644 index 74909fb82d..0000000000 --- a/src/sqlbuild/compiler/analysis_session/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native model analysis session classes.""" diff --git a/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py b/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py deleted file mode 100644 index 6fdec2125d..0000000000 --- a/src/sqlbuild/compiler/analysis_session/classes/native_model_analysis.py +++ /dev/null @@ -1,116 +0,0 @@ -"""One compile's native model analysis session, with Python's catalog and metric bookkeeping.""" - -from __future__ import annotations - -import logging -from typing import Any - -import sqlbuild._native as _native -from sqlbuild.compiler.analysis_session.constants import ( - NATIVE_ANALYSIS_FAILURE_MESSAGE, - NATIVE_ANALYSIS_STORE_FAILURE_MESSAGE, -) -from sqlbuild.compiler.analysis_session.models import ( - NativeModelAnalyses, - NativeModelAnalysisRequest, -) -from sqlbuild.compiler.analysis_session.types import FinishRow, ShapeRows, StepRow -from sqlbuild.compiler.compile.classes.python_model_analysis import PythonModelAnalysis -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.diagnostics.main.log_debug_event import log_debug_event - -_DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.compile") - - -class NativeModelAnalysis: - """Drive one native session; its outcomes stay native for compiled model assembly.""" - - def __init__( - self, - *, - request: NativeModelAnalysisRequest, - python: PythonModelAnalysis, - catalog: Any, - ) -> None: - self._python: PythonModelAnalysis = python - self._catalog: Any = catalog - self._types: dict[str, dict[str, str]] = dict(request.column_types_by_table) - self._nullability: dict[str, dict[str, InferredNullability]] = dict( - request.column_nullability_by_table - ) - - def analyses(self, session: _native.NativeModelAnalysisSession) -> NativeModelAnalyses: - """Every model's session position by name; a native internal failure raises.""" - - step: StepRow = session.run() - publications, failures = step - for failure in failures: - log_debug_event( - logger=_DEBUG_LOGGER, - message=NATIVE_ANALYSIS_FAILURE_MESSAGE, - sqlbuild_error=failure, - ) - self._publish(publications) - finished: FinishRow = session.finish() - schema_additions, analysis_names, succeeded = finished - self._record_catalog_changes( - schema_additions=schema_additions, analysis_names=analysis_names - ) - self._record_cache(session) - names: tuple[str, ...] = tuple( - request.model_input.model_file.file_path.stem for request in self._python.requests - ) - return NativeModelAnalyses( - positions={name: position for position, name in enumerate(names)}, - succeeded=frozenset( - name for name, success in zip(names, succeeded, strict=True) if success - ), - session=session, - ) - - def _record_cache(self, session: _native.NativeModelAnalysisSession) -> None: - stats: tuple[int, int, int, str | None] | None = session.cache_stats - if stats is None: - self._python.record_uncached() - return - hits, misses, _, failure = stats - if failure is not None: - log_debug_event( - logger=_DEBUG_LOGGER, - message=NATIVE_ANALYSIS_STORE_FAILURE_MESSAGE, - sqlbuild_error=failure, - ) - if hits + misses == 0: - self._python.record_uncached() - return - self._python.record_cached(hits=hits, misses=misses) - - def _publish(self, publications: ShapeRows) -> None: - for name, shape in publications: - published: dict[str, str] = dict(shape) - self._types.setdefault(name, published) - self._nullability.setdefault( - name, dict.fromkeys(published, InferredNullability.UNKNOWN) - ) - - def _record_catalog_changes( - self, *, schema_additions: ShapeRows, analysis_names: list[str] - ) -> None: - """Leave Python's binding catalog as its own analysis would have.""" - - additions: dict[str, dict[str, str]] = { - name: dict(shape) - for name, shape in schema_additions - if name not in self._catalog.schemas - } - self._catalog.schemas.update(additions) - if additions: - self._catalog.native.update_relations(additions) - self._catalog.prepare_analysis( - types={name: self._types[name] for name in analysis_names if name in self._types}, - nullability={ - name: self._nullability[name] - for name in analysis_names - if name in self._nullability - }, - ) diff --git a/src/sqlbuild/compiler/analysis_session/constants.py b/src/sqlbuild/compiler/analysis_session/constants.py index f785ed67fe..cc6670f3f8 100644 --- a/src/sqlbuild/compiler/analysis_session/constants.py +++ b/src/sqlbuild/compiler/analysis_session/constants.py @@ -1,12 +1,4 @@ """Native model analysis session constants.""" LINEAGE_NATIVE: str = "native" -LINEAGE_FACTS: str = "facts" -NATIVE_ANALYSIS_FAILURE_MESSAGE: str = "native compact query analysis failed" -NULLABILITY_RULE_FIRST_ARG: str = "first_arg" -NULLABILITY_RULE_CONDITIONAL_RESULT: str = "conditional_result" -NULLABILITY_RULE_SAFE_CAST: str = "safe_cast" -NULLABILITY_RULE_ADAPTER: str = "python" NATIVE_ANALYSIS_STORE_FILE_NAME: str = "model-analyses.bin" -NATIVE_ANALYSIS_STORE_VERSION: str = "native-model-analysis-store-v1" -NATIVE_ANALYSIS_STORE_FAILURE_MESSAGE: str = "native model analysis cache not read or saved" diff --git a/src/sqlbuild/compiler/analysis_session/main/__init__.py b/src/sqlbuild/compiler/analysis_session/main/__init__.py deleted file mode 100644 index 355f68e13c..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Native model analysis session entry points.""" diff --git a/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py b/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py deleted file mode 100644 index 5db2708717..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Analyze model SQL natively.""" - -from __future__ import annotations - -from sqlbuild.compiler.analysis_session._helpers.model_analysis import native_model_analyses -from sqlbuild.compiler.analysis_session.models import ( - NativeModelAnalyses, - NativeModelAnalysisRequest, -) - - -def analyze_native_model_sql(*, request: NativeModelAnalysisRequest) -> NativeModelAnalyses: - """Return each model's analysis (and pivot proof) and the session.""" - - return native_model_analyses(request=request) diff --git a/src/sqlbuild/compiler/analysis_session/main/_dynamic_contract_proof.py b/src/sqlbuild/compiler/analysis_session/main/_dynamic_contract_proof.py deleted file mode 100644 index 2e058baa95..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/_dynamic_contract_proof.py +++ /dev/null @@ -1,13 +0,0 @@ -"""A native dynamic pivot proof row as Python's proof, for hand-built model inputs only.""" - -from __future__ import annotations - -from sqlbuild.compiler.analysis_session._helpers.session_rows import contract_proof -from sqlbuild.compiler.analysis_session.types import ProofRow -from sqlbuild.compiler.compile.models import DynamicColumnContractProof - - -def dynamic_contract_proof(row: ProofRow | None) -> DynamicColumnContractProof | None: - """The proof, or None when the row is absent.""" - - return contract_proof(row) diff --git a/src/sqlbuild/compiler/analysis_session/main/_session_outcome_analysis.py b/src/sqlbuild/compiler/analysis_session/main/_session_outcome_analysis.py deleted file mode 100644 index 74342b9b6a..0000000000 --- a/src/sqlbuild/compiler/analysis_session/main/_session_outcome_analysis.py +++ /dev/null @@ -1,20 +0,0 @@ -"""A finished session outcome as Python's analysis, for hand-built model inputs only.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.compiler.analysis_session._helpers.session_rows import outcome_analysis -from sqlbuild.compiler.analysis_session.types import ContractRow, OutcomeRow -from sqlbuild.compiler.compile.models import ModelSqlAnalysis - - -def session_outcome_analysis( - *, session: Any, position: int, placeholders: dict[str, str] | None -) -> ModelSqlAnalysis: - """The outcome at `position` of a finished session as an explicit boundary conversion.""" - - outcome: OutcomeRow - contract: ContractRow - outcome, contract = session.outcome_rows(position) - return outcome_analysis(outcome=outcome, contract=contract[1], placeholders=placeholders) diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/hand_built_models.py b/src/sqlbuild/compiler/compile/_helpers/assembly/hand_built_models.py index 0bfa744cc6..7c80f25bcf 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/hand_built_models.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/hand_built_models.py @@ -6,10 +6,7 @@ from pathlib import Path import sqlbuild._native as _native -from sqlbuild.compiler.analysis_session.main._dynamic_contract_proof import dynamic_contract_proof -from sqlbuild.compiler.analysis_session.main._session_outcome_analysis import ( - session_outcome_analysis, -) +from sqlbuild._native import dynamic_contract_proof, session_outcome_analysis from sqlbuild.compiler.compile._helpers.assembly.targets import build_model_relation_target from sqlbuild.compiler.compile.models import ( CompiledLineageColumnFact, diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py index 7aba6a4b26..907ed49485 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py @@ -4,7 +4,7 @@ from typing import Any -import sqlbuild._native as _native +from sqlbuild._native import complete_semantic_project from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.types import DiagnosticRow from sqlbuild.compiler.compile.types import CompiledProjectView @@ -20,7 +20,7 @@ def complete_semantic_diagnostics( ) -> CompiledProjectView: """Recover type facts before metadata checks, then explain retained root diagnostics.""" - return _native.complete_semantic_project( + return complete_semantic_project( project=project, profile=profile, binding_results=binding_results, diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py index f669defe84..444018a284 100644 --- a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py +++ b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py @@ -4,11 +4,12 @@ from typing import Any -from sqlbuild._native import infer_native_expression_source_shapes, prove_native_dynamic_contracts -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.analysis_session.main._analyze_native_model_sql import ( +from sqlbuild._native import ( analyze_native_model_sql, + infer_native_expression_source_shapes, + prove_native_dynamic_contracts, ) +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.analysis_session.models import ( NativeModelAnalyses, NativeModelAnalysisRequest, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index c5e922f912..81336c5963 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -4815,7 +4815,7 @@ def full_macro_call_runs( (native_stages, "infer_native_expression_source_shapes"), (native_stages, "analyze_native_model_sql"), (native_sql_test_stage, "assemble_native_sql_tests"), - (diagnostic_recovery, "complete_native_semantic_diagnostics"), + (diagnostic_recovery, "complete_semantic_project"), (contract_validation, "evaluate_native_model_contracts"), (promotion_conflicts, "native_promotion_conflict_diagnostics"), (compile_lineage, "build_compile_lineage"), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index e8f977cece..b710e6cb6c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -33,7 +33,7 @@ "infer_native_expression_source_shapes": [ANY], "analyze_native_model_sql": [ANY], "assemble_native_sql_tests": [ANY], - "complete_native_semantic_diagnostics": [ANY], + "complete_semantic_project": [ANY], "evaluate_native_model_contracts": [ANY], "native_promotion_conflict_diagnostics": [()], "build_compile_lineage": [ANY], @@ -76,9 +76,7 @@ def test_given_project_when_compiling_then_native_seams_answer( assert planned_artifacts assert all(isinstance(artifact, NativeSqlTestArtifact) for artifact in planned_artifacts) planned_counts.append(len(planned_artifacts)) - assert isinstance( - seams["complete_native_semantic_diagnostics"][0], native_module.NativeProjectView - ) + assert isinstance(seams["complete_semantic_project"][0], native_module.NativeProjectView) assert isinstance(seams["evaluate_native_model_contracts"][0], ContractValidationResult) assert [ len(call.args[0].tests) for call in native_sql_test_planning.call_args_list diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_helpers.py index 6db0951552..70165aa640 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/_test_helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/_test_helpers.py @@ -14,13 +14,29 @@ conditional_result_nullability, ) from sqlbuild.adapter.type_system.main.first_arg_nullability import first_arg_nullability +from sqlbuild.adapter.type_system.main.safe_cast_nullability import safe_cast_nullability from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter -from sqlbuild.compiler.analysis_session._helpers.profile_rows import nullability_rule_rows + +_NULLABILITY_RULE_IDS: dict[object, str] = { + first_arg_nullability: "first_arg", + conditional_result_nullability: "conditional_result", + safe_cast_nullability: "safe_cast", +} + + +def nullability_rule_rows(profile: ExpressionInferenceProfile) -> list[tuple[str, str]]: + """The profile's nullability rules as the native `(name, rule id)` rows.""" + + return [ + (name, _NULLABILITY_RULE_IDS.get(rule, "python")) + for name, rule in profile.function_nullability_rules.items() + ] + type Generate = Callable[[random.Random, list[str], int], str] type CteFactView = ( diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py index 80c1f701cb..10ecbee4ec 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/helpers.py @@ -17,24 +17,18 @@ import sqlbuild.compiler.compile._helpers.diagnostics.recovery as semantic_recovery import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stage_assembly from sqlbuild._native import ( + analyze_native_model_sql, build_compile_inputs, discover_project_inputs, infer_native_expression_source_shapes, prove_native_dynamic_contracts, + session_outcome_analysis, ) +from sqlbuild._native import dynamic_contract_proof as contract_proof from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.entry.main.entry import main from sqlbuild.compiler.analysis_session.constants import LINEAGE_NATIVE -from sqlbuild.compiler.analysis_session.main._analyze_native_model_sql import ( - analyze_native_model_sql, -) -from sqlbuild.compiler.analysis_session.main._dynamic_contract_proof import ( - dynamic_contract_proof as contract_proof, -) -from sqlbuild.compiler.analysis_session.main._session_outcome_analysis import ( - session_outcome_analysis, -) from sqlbuild.compiler.analysis_session.models import ( NativeModelAnalyses, NativeModelAnalysisRequest, diff --git a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py index 5590867d80..77a6e7efc4 100644 --- a/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py +++ b/tests/integration/src/sqlbuild/compiler/analysis_session/test_native_model_analysis.py @@ -11,13 +11,10 @@ import pytest import sqlbuild._native as native_module -from sqlbuild._native import assemble_project +from sqlbuild._native import analyze_native_model_sql, assemble_project from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.compiler.analysis_session.main._analyze_native_model_sql import ( - analyze_native_model_sql, -) from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest from sqlbuild.compiler.compile.types import CompiledProjectView, CompileInputsView from sqlbuild.compiler.lineage.types import ColumnLineageMode, InferredNullability diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py index ca6728c312..06c6fd07b7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/analysis/test_dynamic_pivot.py @@ -2,10 +2,8 @@ import pytest +from sqlbuild._native import dynamic_contract_proof as contract_proof from sqlbuild._native import prove_native_dynamic_contracts -from sqlbuild.compiler.analysis_session.main._dynamic_contract_proof import ( - dynamic_contract_proof as contract_proof, -) from sqlbuild.compiler.analysis_session.models import NativePivotTables from sqlbuild.compiler.compile.models import DynamicColumnContractProof from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily From dc0fb467772b39ab73cb7ff7366dc309c993f8e1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:40:33 +0100 Subject: [PATCH 12/14] style: format the native stubs --- src/sqlbuild/_native.pyi | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 871d864114..899d0a65d9 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -1760,7 +1760,10 @@ def infer_native_expression_source_shapes( *, expressions: tuple[str, ...], profile: Any ) -> tuple[dict[str, str] | None, ...]: ... def prove_native_dynamic_contracts( - *, session: NativeModelAnalysisSession | None, tables: Any, models: tuple[tuple[str, tuple[Any, ...]], ...] + *, + session: NativeModelAnalysisSession | None, + tables: Any, + models: tuple[tuple[str, tuple[Any, ...]], ...], ) -> tuple[Any, ...]: ... def _oracle_names_set_operation(sql: str, /) -> bool: ... def _oracle_cte_fact_recovery( @@ -1781,7 +1784,9 @@ def complete_semantic_project( *, project: Any, profile: Any, - binding_results: dict[str, list[tuple[str, str, int | None, int | None, int | None, int | None, str]]], + binding_results: dict[ + str, list[tuple[str, str, int | None, int | None, int | None, int | None, str]] + ], resource_sql_analysis: bool, session: NativeModelAnalysisSession | None = None, ) -> Any: ... From 6a1d282daeb50b1ead3f29e4e9c2d0948c820287 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 03:50:20 +0100 Subject: [PATCH 13/14] refactor: point semantic stage objects at the native project helpers --- .../src/bindings/_helpers/semantic_checks/stage_objects.rs | 7 +++---- python-boundary.toml | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs index c766c9b4b8..1fd931636b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/stage_objects.rs @@ -7,10 +7,9 @@ const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; -const PROJECT_WITH_MODULE: &str = "sqlbuild.compiler.compiled_project.main.project_with"; -const TYPE_RECOVERY_MODULE: &str = "sqlbuild.compiler.compiled_project.main._with_type_recovery"; -const BINDING_DIAGNOSTICS_MODULE: &str = - "sqlbuild.compiler.compiled_project.main._with_binding_diagnostics"; +const PROJECT_WITH_MODULE: &str = "sqlbuild._native"; +const TYPE_RECOVERY_MODULE: &str = "sqlbuild._native"; +const BINDING_DIAGNOSTICS_MODULE: &str = "sqlbuild._native"; const COMPLETION_INPUTS_MODULE: &str = "sqlbuild.compiler.compile.classes.semantic_completion_inputs"; const SCHEMA_VALIDATION_MODULE: &str = "sqlbuild.compiler.sql_analysis.main._schema_validation"; diff --git a/python-boundary.toml b/python-boundary.toml index c3c8a307b5..b7e53da950 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 127172 +remaining_lines = 125499 [[stays]] path = "executor" From cc8d6f95c9fecbe13474e33649c4c421c3d350f7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 04:40:07 +0100 Subject: [PATCH 14/14] build: add an incremental fast profile for local native rebuilds --- AGENTS.md | 2 +- Cargo.toml | 8 ++++++++ Makefile | 4 ++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 7776917bb7..3831a4f3ef 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -204,7 +204,7 @@ ## Rust and Native Builds - Python imports the native extension built from `crates/sqlbuild-python`. After changing any - Rust, rebuild it incrementally with `uv run --no-sync maturin develop --release --skip-install` + Rust, rebuild it incrementally with `make fast-native` (release optimisation, incremental) before running Python tests. Use one profile consistently within a worktree. - Always run Python commands with `uv run --no-sync`. A syncing `uv run` or `uv sync` can replace the locally built extension with a packaged one; if that happens, rebuild with maturin rather diff --git a/Cargo.toml b/Cargo.toml index b4f5016969..cb65e46fc8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,3 +126,11 @@ debug = false [profile.release] strip = "symbols" + +# Local iteration: release-level optimisation with incremental builds, so rebuilding after a +# focused change does not recompile the whole binding crate. CI and wheels use `release`. +[profile.fast] +inherits = "release" +incremental = true +codegen-units = 256 +strip = "none" diff --git a/Makefile b/Makefile index 52be76b8d4..ce32a1d665 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,10 @@ type: dev-native: uv run --no-sync maturin develop --profile dev --skip-install +.PHONY: fast-native +fast-native: + uv run --no-sync maturin develop --profile fast --skip-install + rust-check: rust-lint uv run fensu check cargo test --workspace --all-features