From a5f1799cde45940a86091b13e649c7003c0a1ae1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 06:57:48 +0100 Subject: [PATCH 1/2] feat(native): register the analysis stages behind the preview engine --- crates/sqlbuild-analysis/src/assembly/mod.rs | 3 + .../src/assembly/project/mod.rs | 1 + crates/sqlbuild-analysis/src/contracts/mod.rs | 1 + crates/sqlbuild-analysis/src/lib.rs | 5 + crates/sqlbuild-analysis/src/lineage/mod.rs | 1 + .../src/semantic_checks/mod.rs | 1 + .../sqlbuild-analysis/src/type_system/mod.rs | 1 + .../bindings/_helpers/analysis_session/mod.rs | 1 + .../_helpers/analysis_session/registration.rs | 7 + .../src/bindings/_helpers/contracts/mod.rs | 1 + .../_helpers/contracts/registration.rs | 7 + .../src/bindings/_helpers/lineage/mod.rs | 1 + .../bindings/_helpers/lineage/registration.rs | 7 + .../src/bindings/_helpers/mod.rs | 7 + .../bindings/_helpers/project_assembly/mod.rs | 1 + .../_helpers/project_assembly/registration.rs | 7 + .../bindings/_helpers/semantic_checks/mod.rs | 1 + .../_helpers/semantic_checks/registration.rs | 7 + .../bindings/_helpers/sql_test_glue/mod.rs | 1 + .../_helpers/sql_test_glue/registration.rs | 7 + .../src/bindings/_helpers/type_system/mod.rs | 1 + .../_helpers/type_system/registration.rs | 7 + .../src/bindings/main/register.rs | 16 ++- src/sqlbuild/_native.pyi | 14 ++ .../main/_native_normalize_type.py | 14 ++ .../_helpers/compile/target_writer.py | 26 +++- .../compiler/analysis_session/__init__.py | 1 + .../analysis_session/main/__init__.py | 1 + .../main/_analyze_native_model_sql.py | 29 ++++ .../compile/_helpers/assembly/project.py | 89 +++---------- .../_helpers/assembly/semantic_shapes.py | 68 +++++++++- .../compile/_helpers/diagnostics/recovery.py | 14 ++ .../_helpers/native_stages/assembly.py | 55 ++++++++ src/sqlbuild/compiler/compile/models.py | 1 + .../main/_evaluate_native_model_contracts.py | 15 +++ .../compiler/contracts/main/validate.py | 11 ++ src/sqlbuild/compiler/frontier/constants.py | 7 + src/sqlbuild/compiler/frontier/types.py | 7 + .../main/_build_native_column_lineage.py | 14 ++ src/sqlbuild/compiler/lineage/main/columns.py | 11 ++ .../compiler/project_assembly/__init__.py | 1 + .../project_assembly/main/__init__.py | 1 + .../main/_assemble_native_project.py | 23 ++++ .../compiler/semantic_checks/__init__.py | 1 + .../compiler/semantic_checks/main/__init__.py | 1 + .../_complete_native_semantic_diagnostics.py | 19 +++ .../compiler/sql_test_glue/__init__.py | 1 + .../compiler/sql_test_glue/main/__init__.py | 1 + .../main/plan_native_sql_test_artifacts.py | 19 +++ .../cli/commands/main/compile/_test_types.py | 8 ++ .../cli/commands/main/compile/helpers.py | 60 ++++++++- .../compile/test_native_analysis_seams.py | 78 +++++++++++ .../adapter/type_system/_test_types.py | 8 ++ .../type_system/test_native_normalize_type.py | 47 +++++++ .../frontier/test_native_stage_tiers.py | 126 ++++++++++++++++++ 55 files changed, 788 insertions(+), 75 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/assembly/mod.rs create mode 100644 crates/sqlbuild-analysis/src/assembly/project/mod.rs create mode 100644 crates/sqlbuild-analysis/src/contracts/mod.rs create mode 100644 crates/sqlbuild-analysis/src/lineage/mod.rs create mode 100644 crates/sqlbuild-analysis/src/semantic_checks/mod.rs create mode 100644 crates/sqlbuild-analysis/src/type_system/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs create mode 100644 src/sqlbuild/adapter/type_system/main/_native_normalize_type.py create mode 100644 src/sqlbuild/compiler/analysis_session/__init__.py create mode 100644 src/sqlbuild/compiler/analysis_session/main/__init__.py create mode 100644 src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py create mode 100644 src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py create mode 100644 src/sqlbuild/compiler/contracts/main/_evaluate_native_model_contracts.py create mode 100644 src/sqlbuild/compiler/lineage/main/_build_native_column_lineage.py create mode 100644 src/sqlbuild/compiler/project_assembly/__init__.py create mode 100644 src/sqlbuild/compiler/project_assembly/main/__init__.py create mode 100644 src/sqlbuild/compiler/project_assembly/main/_assemble_native_project.py create mode 100644 src/sqlbuild/compiler/semantic_checks/__init__.py create mode 100644 src/sqlbuild/compiler/semantic_checks/main/__init__.py create mode 100644 src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py create mode 100644 src/sqlbuild/compiler/sql_test_glue/__init__.py create mode 100644 src/sqlbuild/compiler/sql_test_glue/main/__init__.py create mode 100644 src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py create mode 100644 tests/unit/src/sqlbuild/adapter/type_system/test_native_normalize_type.py diff --git a/crates/sqlbuild-analysis/src/assembly/mod.rs b/crates/sqlbuild-analysis/src/assembly/mod.rs new file mode 100644 index 0000000000..079a1bcfca --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/mod.rs @@ -0,0 +1,3 @@ +//! The native model analysis session: catalog, dataflow scheduling, projection and shape publication. + +pub mod project; diff --git a/crates/sqlbuild-analysis/src/assembly/project/mod.rs b/crates/sqlbuild-analysis/src/assembly/project/mod.rs new file mode 100644 index 0000000000..2c6b3134d3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/mod.rs @@ -0,0 +1 @@ +//! Compiled project assembly: resources, dependencies, destinations and their diagnostics. diff --git a/crates/sqlbuild-analysis/src/contracts/mod.rs b/crates/sqlbuild-analysis/src/contracts/mod.rs new file mode 100644 index 0000000000..cf143bb851 --- /dev/null +++ b/crates/sqlbuild-analysis/src/contracts/mod.rs @@ -0,0 +1 @@ +//! Model column contracts and promotion conflicts. diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 74d7298030..2c6aee23ac 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -2,10 +2,15 @@ #![forbid(unsafe_code)] +pub mod assembly; pub mod column_references; pub mod compiler; mod constants; +pub mod contracts; +pub mod lineage; pub mod query_analysis; +pub mod semantic_checks; pub mod semantic_usage; pub mod semantic_validation; pub mod sql_tokens; +pub mod type_system; diff --git a/crates/sqlbuild-analysis/src/lineage/mod.rs b/crates/sqlbuild-analysis/src/lineage/mod.rs new file mode 100644 index 0000000000..9819e31334 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/mod.rs @@ -0,0 +1 @@ +//! Fast column lineage facts for compiled models. diff --git a/crates/sqlbuild-analysis/src/semantic_checks/mod.rs b/crates/sqlbuild-analysis/src/semantic_checks/mod.rs new file mode 100644 index 0000000000..85e0e654a1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/mod.rs @@ -0,0 +1 @@ +//! Semantic completion: type recovery, metadata checks, diagnostic recovery and explanations. diff --git a/crates/sqlbuild-analysis/src/type_system/mod.rs b/crates/sqlbuild-analysis/src/type_system/mod.rs new file mode 100644 index 0000000000..b053fa0aea --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/mod.rs @@ -0,0 +1 @@ +//! Type normalization and type equality shared by every analysis stage. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs new file mode 100644 index 0000000000..7444bd0374 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs @@ -0,0 +1 @@ +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs new file mode 100644 index 0000000000..bbd9c13b6b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs @@ -0,0 +1,7 @@ +//! Register the native model analysis session functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs new file mode 100644 index 0000000000..7444bd0374 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs @@ -0,0 +1 @@ +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs new file mode 100644 index 0000000000..a821930745 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs @@ -0,0 +1,7 @@ +//! Register the native contract functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs new file mode 100644 index 0000000000..7444bd0374 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs @@ -0,0 +1 @@ +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs new file mode 100644 index 0000000000..e747ba06c2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs @@ -0,0 +1,7 @@ +//! Register the native column lineage functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 4ad9b8950e..148a78e9a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -1,10 +1,17 @@ pub(crate) mod analysis; +pub(crate) mod analysis_session; pub(crate) mod attachments; pub(crate) mod boundary; pub(crate) mod cache; +pub(crate) mod contracts; pub(crate) mod discovery; +pub(crate) mod lineage; pub(crate) mod model_config; +pub(crate) mod project_assembly; pub(crate) mod render; pub(crate) mod rules; pub(crate) mod scopes; +pub(crate) mod semantic_checks; +pub(crate) mod sql_test_glue; pub(crate) mod sqltext; +pub(crate) mod type_system; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs new file mode 100644 index 0000000000..7444bd0374 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs @@ -0,0 +1 @@ +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs new file mode 100644 index 0000000000..63be94a863 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs @@ -0,0 +1,7 @@ +//! Register the native project assembly functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs new file mode 100644 index 0000000000..7444bd0374 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs @@ -0,0 +1 @@ +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs new file mode 100644 index 0000000000..7ee7c2c7f0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs @@ -0,0 +1,7 @@ +//! Register the native semantic completion functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs new file mode 100644 index 0000000000..7444bd0374 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -0,0 +1 @@ +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs new file mode 100644 index 0000000000..641f84817b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs @@ -0,0 +1,7 @@ +//! Register the native SQL test planning functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs new file mode 100644 index 0000000000..7444bd0374 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs @@ -0,0 +1 @@ +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs new file mode 100644 index 0000000000..cd60143ae3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs @@ -0,0 +1,7 @@ +//! Register the native type system functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index cde73bc8c0..37e0e09255 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -4,7 +4,9 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ - analysis, attachments, boundary, cache, discovery, model_config, render, rules, scopes, sqltext, + analysis, analysis_session, attachments, boundary, cache, contracts, discovery, lineage, + model_config, project_assembly, render, rules, scopes, semantic_checks, sql_test_glue, sqltext, + type_system, }; use crate::bindings::models; @@ -32,7 +34,19 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { render::macro_calls::register(module)?; cache::native_store::register(module)?; attachments::registration::register(module)?; + register_analysis_stages(module)?; module.add("API_VERSION", API_VERSION)?; module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; Ok(()) } + +fn register_analysis_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { + type_system::registration::register(module)?; + analysis_session::registration::register(module)?; + semantic_checks::registration::register(module)?; + contracts::registration::register(module)?; + lineage::registration::register(module)?; + sql_test_glue::registration::register(module)?; + project_assembly::registration::register(module)?; + Ok(()) +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 854d26027b..cc0cc43c2f 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -261,6 +261,20 @@ def content_digest(parts: list[str]) -> str: ... def fingerprint_project_files(root: str, excluded_files: list[str]) -> str: ... def digest_files(paths: list[str]) -> list[str | None]: ... +# Native analysis: type system. + +# Native analysis: model analysis session. + +# Native analysis: semantic completion. + +# Native analysis: contracts. + +# Native analysis: column lineage facts. + +# Native analysis: SQL test planning glue. + +# Native analysis: compiled project assembly. + # Internal oracle hooks for tests that compare native foundations with Python; not an API. def _oracle_json_dumps(dialect_json: str, value_json: str) -> str: ... def _oracle_text_positions( diff --git a/src/sqlbuild/adapter/type_system/main/_native_normalize_type.py b/src/sqlbuild/adapter/type_system/main/_native_normalize_type.py new file mode 100644 index 0000000000..37efa7eb36 --- /dev/null +++ b/src/sqlbuild/adapter/type_system/main/_native_normalize_type.py @@ -0,0 +1,14 @@ +"""Native type normalization for the preview compiler engine.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.models import NormalizedType +from sqlbuild.adapter.contract.types import TypeDialect + + +def normalize_native_type( + *, type_sql: str, dialect: TypeDialect | str | None +) -> NormalizedType | None: + """Return the native normalization, or None where Python must normalize the type.""" + + return None diff --git a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py index 7c2a99bf20..da8b849949 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py @@ -47,6 +47,8 @@ DiagnosticSeverity, FunctionLanguage, ) +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.main.execution.sql_test_artifacts import ( plan_and_render_sql_test_artifacts, @@ -56,6 +58,9 @@ ) from sqlbuild.compiler.planner.models import AuditPlanEntry, NativeSqlTestArtifact, PlanOutput from sqlbuild.compiler.profiling.main.record import record_compile_timing +from sqlbuild.compiler.sql_test_glue.main.plan_native_sql_test_artifacts import ( + plan_native_sql_test_artifacts, +) from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -569,12 +574,23 @@ def _plan_static_test_artifacts( with OperationLifecycle( operation_kind="project", operation_name="sql_test_planning" ) as lifecycle: - native_artifacts: tuple[NativeSqlTestArtifact, ...] = plan_and_render_sql_test_artifacts( - project=project, - tests=tests, - adapter=adapter, - sql_analysis_enabled=project.settings.sql_analysis, + native_artifacts: tuple[NativeSqlTestArtifact, ...] | None = ( + plan_native_sql_test_artifacts( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=project.settings.sql_analysis, + ) + if native_stage_enabled(NativeStage.SQL_TEST_GLUE) + else None ) + if native_artifacts is None: + native_artifacts = plan_and_render_sql_test_artifacts( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=project.settings.sql_analysis, + ) lifecycle.completed(metadata={"item_count": len(native_artifacts)}) return native_artifacts diff --git a/src/sqlbuild/compiler/analysis_session/__init__.py b/src/sqlbuild/compiler/analysis_session/__init__.py new file mode 100644 index 0000000000..d0736bf682 --- /dev/null +++ b/src/sqlbuild/compiler/analysis_session/__init__.py @@ -0,0 +1 @@ +"""The native model analysis session for the preview compiler engine.""" diff --git a/src/sqlbuild/compiler/analysis_session/main/__init__.py b/src/sqlbuild/compiler/analysis_session/main/__init__.py new file mode 100644 index 0000000000..355f68e13c --- /dev/null +++ b/src/sqlbuild/compiler/analysis_session/main/__init__.py @@ -0,0 +1 @@ +"""Native model analysis session entry points.""" diff --git a/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py b/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py new file mode 100644 index 0000000000..e4f0121194 --- /dev/null +++ b/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py @@ -0,0 +1,29 @@ +"""Analyze model SQL natively for the preview compiler engine.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile.models import ( + CompileModelInput, + ModelAnalysisCaching, + ModelSqlAnalysis, +) +from sqlbuild.compiler.lineage.types import InferredNullability + + +def analyze_native_model_sql( + *, + known_functions: tuple[str, ...], + known_types: tuple[str, ...], + model_inputs: tuple[CompileModelInput, ...], + column_nullability_by_table: dict[str, dict[str, InferredNullability]], + column_types_by_table: dict[str, dict[str, str]], + inference_profile: ExpressionInferenceProfile, + allow_compact_analysis: bool, + rich_type_inference: bool, + analysis_caching: ModelAnalysisCaching | None, + complete_binding_schemas: dict[str, dict[str, str]], +) -> dict[str, ModelSqlAnalysis] | None: + """Return each model's analysis by name, or None where Python must analyze the models.""" + + return None diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index 0d8e1b8c9d..e3933ad94b 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -21,10 +21,7 @@ read_compact_analysis_cache_candidate, record_analysis_cache_metrics, ) -from sqlbuild.compiler.compile._helpers.analysis.columns import ( - substitute_placeholder_defaults, - table_function_analysis_name, -) +from sqlbuild.compiler.compile._helpers.analysis.columns import substitute_placeholder_defaults from sqlbuild.compiler.compile._helpers.analysis.compact import ( NativeCompactAnalysis, analyze_columns_and_lineage_with_polyglot, @@ -54,6 +51,7 @@ from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( binding_required_names, binding_schema_for_model, + build_column_nullability_by_table, build_complete_binding_schemas, get_expression_source_shapes, published_model_shape, @@ -79,6 +77,10 @@ ) from sqlbuild.compiler.compile._helpers.diagnostics.recovery import complete_semantic_diagnostics from sqlbuild.compiler.compile._helpers.diagnostics.scope import report_scope_index_errors +from sqlbuild.compiler.compile._helpers.native_stages.assembly import ( + analyze_model_sql_by_engine, + assemble_project_by_engine, +) from sqlbuild.compiler.compile._helpers.render.context_templates import ( resolve_early_model_templates, ) @@ -98,7 +100,6 @@ ) from sqlbuild.compiler.compile._helpers.sql_tests.identity import build_sql_test_case_fingerprint from sqlbuild.compiler.compile.classes.stored_model_analyses import StoredModelAnalyses -from sqlbuild.compiler.compile.constants import NOT_NULL_AUDIT_NAME from sqlbuild.compiler.compile.main._scope_index_with_compile_usages import ( scope_index_with_compile_usages, ) @@ -182,10 +183,7 @@ ) from sqlbuild.spec.contracts.models import ( DefaultsConfig, - SchemaAuditInstance, - SchemaColumn, SchemaDynamicColumnFamily, - SourceColumnEntry, SourceEntry, SourceLocation, TargetConfig, @@ -213,6 +211,16 @@ def assemble_compiled_project( ) -> CompiledProject: """Convert attached compile inputs into the planner-ready project view.""" + native_project: CompiledProject | None = assemble_project_by_engine( + inputs=inputs, + inference_profile=inference_profile, + skip_column_inference=skip_column_inference, + column_lineage_mode=column_lineage_mode, + analysis_cache_dir=analysis_cache_dir, + analysis_model_names=analysis_model_names, + ) + if native_project is not None: + return native_project sql_analysis_enabled: bool = ( inputs.effective_settings.sql_analysis and not skip_column_inference ) @@ -220,7 +228,7 @@ def assemble_compiled_project( seed_input.schema_entry.name for seed_input in inputs.seed_inputs ) column_nullability_by_table: dict[str, dict[str, InferredNullability]] = ( - _build_column_nullability_by_table(inputs) + build_column_nullability_by_table(inputs) ) column_types_by_table: dict[str, dict[str, str]] = _build_column_types_by_table(inputs) dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] = ( @@ -321,7 +329,8 @@ def assemble_compiled_project( model_sql_analysis_by_name: dict[str, _ModelSqlAnalysis] = {} if sql_analysis_enabled: with record_compile_timing("model_analysis_ms"): - model_sql_analysis_by_name = _analyze_model_sql_in_parallel( + python_analysis: partial[dict[str, _ModelSqlAnalysis]] = partial( + _analyze_model_sql_in_parallel, known_functions=known_function_names(inputs.sql_function_inputs), known_types=known_declared_types( functions=inputs.sql_function_inputs, column_types=column_types_by_table @@ -350,6 +359,9 @@ def assemble_compiled_project( ), complete_binding_schemas=complete_binding_schemas, ) + model_sql_analysis_by_name = analyze_model_sql_by_engine( + python_analysis=python_analysis + ) scope_index: ScopeIndex = scope_index_with_compile_usages(inputs=inputs) report_scope_index_errors(index=scope_index) effective_target_values: dict[str, object] = resolve_early_model_templates( @@ -1369,33 +1381,6 @@ def _model_name(model_input: CompileModelInput) -> str: return model_input.model_file.file_path.stem -def _build_column_nullability_by_table( - inputs: CompileProjectInputs, -) -> dict[str, dict[str, InferredNullability]]: - facts: dict[str, dict[str, InferredNullability]] = {} - for model_input in inputs.model_inputs: - if model_input.schema_entry is None: - continue - facts[model_input.schema_entry.name] = _schema_column_nullability( - model_input.schema_entry.columns - ) - for seed_input in inputs.seed_inputs: - facts[seed_input.schema_entry.name] = _schema_column_nullability( - seed_input.schema_entry.columns - ) - for source_input in inputs.source_inputs: - facts[source_input.source_entry.name] = _source_column_nullability( - source_input.source_entry.columns - ) - for function_input in inputs.sql_function_inputs: - if not function_input.return_columns: - continue - facts[table_function_analysis_name(function_input.name)] = { - column.name: InferredNullability.UNKNOWN for column in function_input.return_columns - } - return facts - - def _build_dynamic_families_by_table( inputs: CompileProjectInputs, ) -> dict[str, tuple[SchemaDynamicColumnFamily, ...]]: @@ -1501,36 +1486,6 @@ def _binding_compiler_diagnostics( return tuple(result) -def _schema_column_nullability( - columns: tuple[SchemaColumn, ...], -) -> dict[str, InferredNullability]: - return { - column.name: _declared_column_nullability(nullable=column.nullable, audits=column.audits) - for column in columns - } - - -def _source_column_nullability( - columns: tuple[SourceColumnEntry, ...], -) -> dict[str, InferredNullability]: - return { - column.name: _declared_column_nullability(nullable=column.nullable, audits=column.audits) - for column in columns - } - - -def _declared_column_nullability( - *, - nullable: bool | None, - audits: tuple[SchemaAuditInstance, ...], -) -> InferredNullability: - if nullable is False: - return InferredNullability.NON_NULL - if any(audit.definition_name == NOT_NULL_AUDIT_NAME for audit in audits): - return InferredNullability.NON_NULL - return InferredNullability.UNKNOWN - - def _assemble_compiled_source( *, source_input: CompileSourceInput, diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py index 1fb6963cb3..85f3713d60 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py @@ -12,6 +12,7 @@ analyze_columns_and_lineage_with_polyglot, analyze_queries_with_compact_polyglot_batch, ) +from sqlbuild.compiler.compile.constants import NOT_NULL_AUDIT_NAME from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledProject, @@ -20,6 +21,7 @@ CompileSqlReference, PolyglotAnalysisResult, ) +from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.planner.constants import ( SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, SNAPSHOT_DEFAULT_VALID_TO_COLUMN, @@ -32,7 +34,12 @@ ) from sqlbuild.compiler.sql_analysis.main._binding_catalog import create_binding_catalog from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql -from sqlbuild.spec.contracts.models import SourceEntry +from sqlbuild.spec.contracts.models import ( + SchemaAuditInstance, + SchemaColumn, + SourceColumnEntry, + SourceEntry, +) def binding_relation_names(references: tuple[CompileSqlReference, ...]) -> frozenset[str]: @@ -131,6 +138,65 @@ def build_declared_column_types(inputs: CompileProjectInputs) -> dict[str, dict[ return facts +def build_column_nullability_by_table( + inputs: CompileProjectInputs, +) -> dict[str, dict[str, InferredNullability]]: + """Return what each model, seed, source and table function declares about column nulls.""" + + facts: dict[str, dict[str, InferredNullability]] = {} + for model_input in inputs.model_inputs: + if model_input.schema_entry is None: + continue + facts[model_input.schema_entry.name] = _schema_column_nullability( + model_input.schema_entry.columns + ) + for seed_input in inputs.seed_inputs: + facts[seed_input.schema_entry.name] = _schema_column_nullability( + seed_input.schema_entry.columns + ) + for source_input in inputs.source_inputs: + facts[source_input.source_entry.name] = _source_column_nullability( + source_input.source_entry.columns + ) + for function_input in inputs.sql_function_inputs: + if not function_input.return_columns: + continue + facts[table_function_analysis_name(function_input.name)] = { + column.name: InferredNullability.UNKNOWN for column in function_input.return_columns + } + return facts + + +def _schema_column_nullability( + columns: tuple[SchemaColumn, ...], +) -> dict[str, InferredNullability]: + return { + column.name: _declared_column_nullability(nullable=column.nullable, audits=column.audits) + for column in columns + } + + +def _source_column_nullability( + columns: tuple[SourceColumnEntry, ...], +) -> dict[str, InferredNullability]: + return { + column.name: _declared_column_nullability(nullable=column.nullable, audits=column.audits) + for column in columns + } + + +def _declared_column_nullability( + *, + nullable: bool | None, + audits: tuple[SchemaAuditInstance, ...], +) -> InferredNullability: + if nullable is False: + return InferredNullability.NON_NULL + if any(audit.definition_name == NOT_NULL_AUDIT_NAME for audit in audits): + return InferredNullability.NON_NULL + return InferredNullability.UNKNOWN + + def build_complete_binding_schemas(inputs: CompileProjectInputs) -> dict[str, dict[str, str]]: """Collect authoritative declared interfaces before SQL inference.""" schemas: dict[str, dict[str, str]] = {} diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py index cb363cf8e0..521430a59e 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py @@ -27,6 +27,11 @@ CompiledProject, CompilerDiagnostic, ) +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.semantic_checks.main._complete_native_semantic_diagnostics import ( + complete_native_semantic_diagnostics, +) from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic _MISSING_CODE: str = "B002" @@ -41,6 +46,15 @@ def complete_semantic_diagnostics( resource_sql_analysis: bool = True, ) -> CompiledProject: """Recover type facts before metadata checks, then explain retained root diagnostics.""" + if native_stage_enabled(NativeStage.SEMANTIC_CHECKS): + native_project: CompiledProject | None = complete_native_semantic_diagnostics( + project=project, + profile=profile, + binding_results=binding_results, + resource_sql_analysis=resource_sql_analysis, + ) + if native_project is not None: + return native_project project = recover_output_types(project=project, binding_results=binding_results) project = replace( project, diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py new file mode 100644 index 0000000000..49bfdd90a1 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py @@ -0,0 +1,55 @@ +"""Hand compiled-project assembly and model analysis to their native stages when enabled.""" + +from __future__ import annotations + +from functools import partial +from pathlib import Path + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.analysis_session.main._analyze_native_model_sql import ( + analyze_native_model_sql, +) +from sqlbuild.compiler.compile.models import CompiledProject, CompileProjectInputs, ModelSqlAnalysis +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.lineage.types import ColumnLineageMode +from sqlbuild.compiler.project_assembly.main._assemble_native_project import ( + assemble_native_project, +) + + +def assemble_project_by_engine( + *, + inputs: CompileProjectInputs, + inference_profile: ExpressionInferenceProfile | None, + skip_column_inference: bool, + column_lineage_mode: ColumnLineageMode, + analysis_cache_dir: Path | None, + analysis_model_names: frozenset[str] | None, +) -> CompiledProject | None: + """Return the natively assembled project, or None where Python must assemble it.""" + + if not native_stage_enabled(NativeStage.PROJECT_ASSEMBLY): + return None + return assemble_native_project( + inputs=inputs, + inference_profile=inference_profile, + skip_column_inference=skip_column_inference, + column_lineage_mode=column_lineage_mode, + analysis_cache_dir=analysis_cache_dir, + analysis_model_names=analysis_model_names, + ) + + +def analyze_model_sql_by_engine( + *, python_analysis: partial[dict[str, ModelSqlAnalysis]] +) -> dict[str, ModelSqlAnalysis]: + """Analyze models natively with the Python analysis's arguments, or run the Python analysis.""" + + if native_stage_enabled(NativeStage.MODEL_ANALYSIS): + native_analyses: dict[str, ModelSqlAnalysis] | None = analyze_native_model_sql( + **python_analysis.keywords + ) + if native_analyses is not None: + return native_analyses + return python_analysis() diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 91315721e6..e06674ac31 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -1184,6 +1184,7 @@ class CompiledProject: sql_expansions: dict[Path, CompiledSqlExpansion] = field( default_factory=dict, compare=False, repr=False ) + native_session: Any | None = field(default=None, compare=False, repr=False) @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/contracts/main/_evaluate_native_model_contracts.py b/src/sqlbuild/compiler/contracts/main/_evaluate_native_model_contracts.py new file mode 100644 index 0000000000..bb823aa8c2 --- /dev/null +++ b/src/sqlbuild/compiler/contracts/main/_evaluate_native_model_contracts.py @@ -0,0 +1,15 @@ +"""Evaluate model contracts natively for the preview compiler engine.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.types import TypeDialect +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.contracts.models import ContractValidationResult + + +def evaluate_native_model_contracts( + *, project: CompiledProject, dialect: TypeDialect | str | None +) -> ContractValidationResult | None: + """Return the contract diagnostics, or None where Python must evaluate the contracts.""" + + return None diff --git a/src/sqlbuild/compiler/contracts/main/validate.py b/src/sqlbuild/compiler/contracts/main/validate.py index 9f453ebd0a..20049a5419 100644 --- a/src/sqlbuild/compiler/contracts/main/validate.py +++ b/src/sqlbuild/compiler/contracts/main/validate.py @@ -5,7 +5,12 @@ from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject, CompilerDiagnostic from sqlbuild.compiler.contracts._helpers.columns import collect_model_column_contract_diagnostics +from sqlbuild.compiler.contracts.main._evaluate_native_model_contracts import ( + evaluate_native_model_contracts, +) from sqlbuild.compiler.contracts.models import ContractValidationResult +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.planner.types import ContractPolicy from sqlbuild.spec.contracts.types import ColumnContractMode @@ -17,6 +22,12 @@ def evaluate_model_contracts( ) -> ContractValidationResult: """Evaluate model header column contracts against inferred output columns.""" + if native_stage_enabled(NativeStage.CONTRACTS): + native_result: ContractValidationResult | None = evaluate_native_model_contracts( + project=project, dialect=dialect + ) + if native_result is not None: + return native_result if not any( _requires_contract_evaluation(model=model, mode=project.settings.column_contract_mode) for model in project.models diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index 572857ecf4..6da274d310 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -25,6 +25,13 @@ NativeStage.MACRO_CALLS: NativeStageTier.PREVIEW, NativeStage.MACRO_CALL_STORE: NativeStageTier.PREVIEW, NativeStage.ATTACHMENTS: NativeStageTier.PREVIEW, + NativeStage.TYPE_SYSTEM: NativeStageTier.PREVIEW, + NativeStage.MODEL_ANALYSIS: NativeStageTier.PREVIEW, + NativeStage.SEMANTIC_CHECKS: NativeStageTier.PREVIEW, + NativeStage.CONTRACTS: NativeStageTier.PREVIEW, + NativeStage.LINEAGE_FACTS: NativeStageTier.PREVIEW, + NativeStage.SQL_TEST_GLUE: NativeStageTier.PREVIEW, + NativeStage.PROJECT_ASSEMBLY: NativeStageTier.PREVIEW, } ENGINE_NATIVE_STAGE_TIERS: dict[CompilerEngine, frozenset[NativeStageTier]] = { CompilerEngine.PYTHON: frozenset(), diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index 70b4d146c3..17b611e46f 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -23,6 +23,13 @@ class NativeStage(StrEnum): MACRO_CALLS = "macro_calls" MACRO_CALL_STORE = "macro_call_store" ATTACHMENTS = "attachments" + TYPE_SYSTEM = "type_system" + MODEL_ANALYSIS = "model_analysis" + SEMANTIC_CHECKS = "semantic_checks" + CONTRACTS = "contracts" + LINEAGE_FACTS = "lineage_facts" + SQL_TEST_GLUE = "sql_test_glue" + PROJECT_ASSEMBLY = "project_assembly" class NativeStageTier(StrEnum): diff --git a/src/sqlbuild/compiler/lineage/main/_build_native_column_lineage.py b/src/sqlbuild/compiler/lineage/main/_build_native_column_lineage.py new file mode 100644 index 0000000000..d19bf5b536 --- /dev/null +++ b/src/sqlbuild/compiler/lineage/main/_build_native_column_lineage.py @@ -0,0 +1,14 @@ +"""Build fast column lineage facts natively for the preview compiler engine.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.lineage.models import ProjectColumnLineage + + +def build_native_column_lineage( + *, project: CompiledProject, dialect: str | None, model_names: frozenset[str] | None +) -> ProjectColumnLineage | None: + """Return the fast lineage graph, or None where Python must build it.""" + + return None diff --git a/src/sqlbuild/compiler/lineage/main/columns.py b/src/sqlbuild/compiler/lineage/main/columns.py index a881acc4e7..fc44eab5bc 100644 --- a/src/sqlbuild/compiler/lineage/main/columns.py +++ b/src/sqlbuild/compiler/lineage/main/columns.py @@ -3,12 +3,17 @@ from __future__ import annotations from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.lineage._helpers.fast_columns import ( build_fast_project_column_lineage, ) from sqlbuild.compiler.lineage._helpers.rich_columns import ( build_rich_project_column_lineage, ) +from sqlbuild.compiler.lineage.main._build_native_column_lineage import ( + build_native_column_lineage, +) from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.lineage.types import ColumnLineageMode @@ -24,6 +29,12 @@ def build_project_column_lineage( match mode: case ColumnLineageMode.FAST: + if native_stage_enabled(NativeStage.LINEAGE_FACTS): + native_lineage: ProjectColumnLineage | None = build_native_column_lineage( + project=project, dialect=dialect, model_names=model_names + ) + if native_lineage is not None: + return native_lineage return build_fast_project_column_lineage( project=project, dialect=dialect, diff --git a/src/sqlbuild/compiler/project_assembly/__init__.py b/src/sqlbuild/compiler/project_assembly/__init__.py new file mode 100644 index 0000000000..16a18c5b6e --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/__init__.py @@ -0,0 +1 @@ +"""Native compiled project assembly for the preview compiler engine.""" diff --git a/src/sqlbuild/compiler/project_assembly/main/__init__.py b/src/sqlbuild/compiler/project_assembly/main/__init__.py new file mode 100644 index 0000000000..646fd16ff4 --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/main/__init__.py @@ -0,0 +1 @@ +"""Native compiled project assembly entry points.""" diff --git a/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project.py b/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project.py new file mode 100644 index 0000000000..74ba3e90fb --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project.py @@ -0,0 +1,23 @@ +"""Assemble the compiled project natively for the preview compiler engine.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile.models import CompiledProject, CompileProjectInputs +from sqlbuild.compiler.lineage.types import ColumnLineageMode + + +def assemble_native_project( + *, + inputs: CompileProjectInputs, + inference_profile: ExpressionInferenceProfile | None, + skip_column_inference: bool, + column_lineage_mode: ColumnLineageMode, + analysis_cache_dir: Path | None, + analysis_model_names: frozenset[str] | None, +) -> CompiledProject | None: + """Return the compiled project, or None where Python must assemble it.""" + + return None diff --git a/src/sqlbuild/compiler/semantic_checks/__init__.py b/src/sqlbuild/compiler/semantic_checks/__init__.py new file mode 100644 index 0000000000..d0defadb2d --- /dev/null +++ b/src/sqlbuild/compiler/semantic_checks/__init__.py @@ -0,0 +1 @@ +"""Native semantic completion for the preview compiler engine.""" diff --git a/src/sqlbuild/compiler/semantic_checks/main/__init__.py b/src/sqlbuild/compiler/semantic_checks/main/__init__.py new file mode 100644 index 0000000000..b7664e28d3 --- /dev/null +++ b/src/sqlbuild/compiler/semantic_checks/main/__init__.py @@ -0,0 +1 @@ +"""Native semantic completion entry points.""" diff --git a/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py b/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py new file mode 100644 index 0000000000..ba3914b53f --- /dev/null +++ b/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py @@ -0,0 +1,19 @@ +"""Complete semantic diagnostics natively for the preview compiler engine.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic + + +def complete_native_semantic_diagnostics( + *, + project: CompiledProject, + profile: ExpressionInferenceProfile, + binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], + resource_sql_analysis: bool, +) -> CompiledProject | None: + """Return the completed project, or None where Python must complete its diagnostics.""" + + return None diff --git a/src/sqlbuild/compiler/sql_test_glue/__init__.py b/src/sqlbuild/compiler/sql_test_glue/__init__.py new file mode 100644 index 0000000000..6e5850e4af --- /dev/null +++ b/src/sqlbuild/compiler/sql_test_glue/__init__.py @@ -0,0 +1 @@ +"""Native SQL test planning glue for the preview compiler engine.""" diff --git a/src/sqlbuild/compiler/sql_test_glue/main/__init__.py b/src/sqlbuild/compiler/sql_test_glue/main/__init__.py new file mode 100644 index 0000000000..e1bc28b0f1 --- /dev/null +++ b/src/sqlbuild/compiler/sql_test_glue/main/__init__.py @@ -0,0 +1 @@ +"""Native SQL test planning glue entry points.""" diff --git a/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py b/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py new file mode 100644 index 0000000000..7b5861fead --- /dev/null +++ b/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py @@ -0,0 +1,19 @@ +"""Plan and render SQL test artifacts natively for the preview compiler engine.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.compiler.compile.models import CompiledProject, CompiledSqlTest +from sqlbuild.compiler.planner.models import NativeSqlTestArtifact + + +def plan_native_sql_test_artifacts( + *, + project: CompiledProject, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + sql_analysis_enabled: bool, +) -> tuple[NativeSqlTestArtifact, ...] | None: + """Return one artifact per test, or None where Python must plan the tests.""" + + return None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index d8bbf66125..14a36598f9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -812,6 +812,14 @@ class CompilerEngineParityTestCase: expected_exit_codes: tuple[int, int] +@dataclass(frozen=True) +class NativeAnalysisSeamTestCase: + """Every engine compiling a copy of one project while the analysis stage seams are recorded.""" + + description: str + expected_preview_returns: dict[str, list[object]] + + @dataclass(frozen=True) class CompilerEngineMacroParityTestCase: """Two engines compiling separate copies of one macro-heavy project.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 7efeec91b1..10b4e6cae4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -21,12 +21,13 @@ from dataclasses import replace from io import StringIO from pathlib import Path -from types import FrameType +from types import FrameType, ModuleType from typing import Any, NamedTuple, cast import duckdb import pytest +import sqlbuild.cli.commands._helpers.compile.target_writer as target_writer import sqlbuild.cli.commands.main.project._compile as compile_command_module import sqlbuild.cli.compile_render_reuse._helpers.load_notice as render_load_notice import sqlbuild.cli.compile_render_reuse.main._render_reuse_session as render_reuse_main @@ -35,10 +36,14 @@ import sqlbuild.cli.compile_reuse._helpers.store as reuse_store import sqlbuild.compiler.compile._helpers.assembly.binding_waves as binding_waves import sqlbuild.compiler.compile._helpers.assembly.project as project_assembly +import sqlbuild.compiler.compile._helpers.diagnostics.recovery as diagnostic_recovery import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module +import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stages import sqlbuild.compiler.compile.classes.binding_dataflow as binding_dataflow import sqlbuild.compiler.compile.classes.render_reuse_session as render_reuse_session import sqlbuild.compiler.compile.classes.stored_model_analyses as stored_model_analyses +import sqlbuild.compiler.contracts.main.validate as contract_validation +import sqlbuild.compiler.lineage.main.columns as column_lineage import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class from scripts.cold_compile_performance.main.read_compile_measurement import read_compile_measurement from scripts.cold_compile_performance.main.semantic_compile_fingerprint import ( @@ -4741,3 +4746,56 @@ def engine_macro_call_runs( ) ), ) + + +_ANALYSIS_SEAMS: tuple[tuple[ModuleType, str], ...] = ( + (native_stages, "assemble_native_project"), + (native_stages, "analyze_native_model_sql"), + (diagnostic_recovery, "complete_native_semantic_diagnostics"), + (contract_validation, "evaluate_native_model_contracts"), + (column_lineage, "build_native_column_lineage"), + (target_writer, "plan_native_sql_test_artifacts"), +) + + +def record_analysis_seams(*, monkeypatch: pytest.MonkeyPatch) -> dict[str, list[object]]: + """Record what each native analysis stage seam returns, keyed by its entry name.""" + + results: dict[str, list[object]] = {} + for module, name in _ANALYSIS_SEAMS: + monkeypatch.setattr( + module, name, _recorded_seam(entry=getattr(module, name), name=name, results=results) + ) + return results + + +def _recorded_seam( + *, entry: Callable[..., object], name: str, results: dict[str, list[object]] +) -> Callable[..., object]: + def recorded(**arguments: object) -> object: + result: object = entry(**arguments) + results.setdefault(name, []).append(result) + return result + + return recorded + + +def engine_in_process_compile( + *, + project_dir: Path, + engine: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> tuple[CompileReuseRun, dict[str, list[object]]]: + """Compile in this process under `engine`; return the run and what each seam returned.""" + + with monkeypatch.context() as patch: + for name, value in { + **COMPILE_REUSE_ENV, + COMPILER_ENGINE_ENV_VAR: engine, + REUSE_DISABLE_ENV_VAR: "1", + }.items(): + patch.setenv(name, value) + seams: dict[str, list[object]] = record_analysis_seams(monkeypatch=patch) + run: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) + return run, seams diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py new file mode 100644 index 0000000000..16f87da078 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -0,0 +1,78 @@ +"""The analysis stage seams defer to Python, so every engine compiles a project identically.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + NativeAnalysisSeamTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + CompileReuseRun, + copy_compile_project, + engine_in_process_compile, + prepare_compile_reuse_project, + report_without_engine, +) + +_ENGINES: tuple[str, ...] = ("python", "native", "native-preview") + + +@pytest.mark.parametrize( + "test_case", + [ + NativeAnalysisSeamTestCase( + description="models_contracts_lineage_and_sql_tests", + expected_preview_returns={ + "assemble_native_project": [None], + "analyze_native_model_sql": [None], + "complete_native_semantic_diagnostics": [None], + "evaluate_native_model_contracts": [None], + "build_native_column_lineage": [None], + "plan_native_sql_test_artifacts": [None], + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_project_when_compiling_with_each_engine_then_seams_defer_and_outputs_match( + test_case: NativeAnalysisSeamTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + prepared_project: Path = tmp_path / "orders" + prepare_compile_reuse_project(project_dir=prepared_project) + outcomes: dict[str, tuple[CompileReuseRun, dict[str, list[object]]]] = { + engine: engine_in_process_compile( + project_dir=copy_compile_project( + source=prepared_project, destination=tmp_path / engine + ), + engine=engine, + monkeypatch=monkeypatch, + capsys=capsys, + ) + for engine in _ENGINES + } + python_run, python_seams = outcomes["python"] + native_run, native_seams = outcomes["native"] + preview_run, preview_seams = outcomes["native-preview"] + + assert python_run.returncode == 0, python_run.stderr + assert python_run.compiled + assert (python_seams, native_seams, preview_seams) == ( + {}, + {}, + test_case.expected_preview_returns, + ) + assert (native_run.returncode, preview_run.returncode) == (0, 0) + assert report_without_engine(native_run) == report_without_engine(python_run) + assert report_without_engine(preview_run) == report_without_engine(python_run) + assert native_run.compiled == python_run.compiled + assert preview_run.compiled == python_run.compiled + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/adapter/type_system/_test_types.py b/tests/unit/src/sqlbuild/adapter/type_system/_test_types.py index ce055965dc..8b5e8e423f 100644 --- a/tests/unit/src/sqlbuild/adapter/type_system/_test_types.py +++ b/tests/unit/src/sqlbuild/adapter/type_system/_test_types.py @@ -26,3 +26,11 @@ class NumericFamilyTestCase: dialect: str | None raw_type: str expected_family: str | None + + +@dataclass(frozen=True) +class NativeTypeNormalizationTestCase: + description: str + dialect: str | None + raw_type: str + expected_type: NormalizedType | None diff --git a/tests/unit/src/sqlbuild/adapter/type_system/test_native_normalize_type.py b/tests/unit/src/sqlbuild/adapter/type_system/test_native_normalize_type.py new file mode 100644 index 0000000000..c7da19a0d3 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapter/type_system/test_native_normalize_type.py @@ -0,0 +1,47 @@ +"""Native type normalization defers every type to Python until the native type system lands.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.adapter.type_system.main._native_normalize_type import normalize_native_type +from tests.unit.src.sqlbuild.adapter.type_system._test_types import ( + NativeTypeNormalizationTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeTypeNormalizationTestCase( + description="snowflake number", + dialect="snowflake", + raw_type="NUMBER(38,0)", + expected_type=None, + ), + NativeTypeNormalizationTestCase( + description="duckdb sized varchar", + dialect="duckdb", + raw_type="VARCHAR(20)", + expected_type=None, + ), + NativeTypeNormalizationTestCase( + description="no dialect", + dialect=None, + raw_type="TIMESTAMP WITH TIME ZONE", + expected_type=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_type_string_when_normalizing_natively_then_it_defers_to_python( + test_case: NativeTypeNormalizationTestCase, +) -> None: + assert ( + normalize_native_type(type_sql=test_case.raw_type, dialect=test_case.dialect) + == test_case.expected_type + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py index 5a9b4953ad..8e6f7793a2 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py @@ -157,6 +157,132 @@ stage=NativeStage.ATTACHMENTS, expected_enabled=True, ), + NativeStageTierTestCase( + description="python_type_system", + engine=CompilerEngine.PYTHON, + stage=NativeStage.TYPE_SYSTEM, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_type_system", + engine=CompilerEngine.NATIVE, + stage=NativeStage.TYPE_SYSTEM, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_type_system", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.TYPE_SYSTEM, + expected_enabled=True, + ), + NativeStageTierTestCase( + description="python_model_analysis", + engine=CompilerEngine.PYTHON, + stage=NativeStage.MODEL_ANALYSIS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_model_analysis", + engine=CompilerEngine.NATIVE, + stage=NativeStage.MODEL_ANALYSIS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_model_analysis", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.MODEL_ANALYSIS, + expected_enabled=True, + ), + NativeStageTierTestCase( + description="python_semantic_checks", + engine=CompilerEngine.PYTHON, + stage=NativeStage.SEMANTIC_CHECKS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_semantic_checks", + engine=CompilerEngine.NATIVE, + stage=NativeStage.SEMANTIC_CHECKS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_semantic_checks", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.SEMANTIC_CHECKS, + expected_enabled=True, + ), + NativeStageTierTestCase( + description="python_contracts", + engine=CompilerEngine.PYTHON, + stage=NativeStage.CONTRACTS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_contracts", + engine=CompilerEngine.NATIVE, + stage=NativeStage.CONTRACTS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_contracts", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.CONTRACTS, + expected_enabled=True, + ), + NativeStageTierTestCase( + description="python_lineage_facts", + engine=CompilerEngine.PYTHON, + stage=NativeStage.LINEAGE_FACTS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_lineage_facts", + engine=CompilerEngine.NATIVE, + stage=NativeStage.LINEAGE_FACTS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_lineage_facts", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.LINEAGE_FACTS, + expected_enabled=True, + ), + NativeStageTierTestCase( + description="python_sql_test_glue", + engine=CompilerEngine.PYTHON, + stage=NativeStage.SQL_TEST_GLUE, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_sql_test_glue", + engine=CompilerEngine.NATIVE, + stage=NativeStage.SQL_TEST_GLUE, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_sql_test_glue", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.SQL_TEST_GLUE, + expected_enabled=True, + ), + NativeStageTierTestCase( + description="python_project_assembly", + engine=CompilerEngine.PYTHON, + stage=NativeStage.PROJECT_ASSEMBLY, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_project_assembly", + engine=CompilerEngine.NATIVE, + stage=NativeStage.PROJECT_ASSEMBLY, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_project_assembly", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.PROJECT_ASSEMBLY, + expected_enabled=True, + ), ], ids=lambda case: case.description, ) From 18d6a2be3bea2d6655a74d3e3445a23f1cebb449 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:55:35 +0100 Subject: [PATCH 2/2] feat(native): add the dynamic pivot, expression-source and SQL test seams Route the per-model dynamic pivot proof through the model-analysis dispatch: ModelSqlAnalysis gains an optional dynamic column contract proof that replaces the Python proof when the native analysis supplies one, and the native analysis request now carries the dynamic families. Infer expression-source shapes through a MODEL_ANALYSIS stub and assemble compiled SQL test facts through a SQL_TEST_GLUE stub, each gated once per compile, so the M4d and M4h lanes no longer need to edit project.py. Drop the vacuous native type normalization test. --- .../main/_analyze_native_model_sql.py | 23 +--- .../_infer_native_expression_source_shapes.py | 13 +++ .../compiler/analysis_session/models.py | 27 +++++ .../compile/_helpers/assembly/project.py | 73 +++++++------ .../_helpers/native_stages/assembly.py | 49 ++++++++- .../_helpers/native_stages/sql_tests.py | 30 ++++++ src/sqlbuild/compiler/compile/models.py | 1 + .../main/_assemble_native_sql_tests.py | 13 +++ .../cli/commands/main/compile/helpers.py | 3 + .../compile/test_native_analysis_seams.py | 2 + .../adapter/type_system/_test_types.py | 8 -- .../type_system/test_native_normalize_type.py | 47 -------- .../_helpers/native_stages/_test_types.py | 15 +++ .../test_dynamic_column_contract_by_engine.py | 101 ++++++++++++++++++ 14 files changed, 297 insertions(+), 108 deletions(-) create mode 100644 src/sqlbuild/compiler/analysis_session/main/_infer_native_expression_source_shapes.py create mode 100644 src/sqlbuild/compiler/analysis_session/models.py create mode 100644 src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py create mode 100644 src/sqlbuild/compiler/sql_test_glue/main/_assemble_native_sql_tests.py delete mode 100644 tests/unit/src/sqlbuild/adapter/type_system/test_native_normalize_type.py create mode 100644 tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/_test_types.py create mode 100644 tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/test_dynamic_column_contract_by_engine.py diff --git a/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py b/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py index e4f0121194..ea5cdadba4 100644 --- a/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py +++ b/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py @@ -2,28 +2,13 @@ from __future__ import annotations -from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import ( - CompileModelInput, - ModelAnalysisCaching, - ModelSqlAnalysis, -) -from sqlbuild.compiler.lineage.types import InferredNullability +from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest +from sqlbuild.compiler.compile.models import ModelSqlAnalysis def analyze_native_model_sql( - *, - known_functions: tuple[str, ...], - known_types: tuple[str, ...], - model_inputs: tuple[CompileModelInput, ...], - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - column_types_by_table: dict[str, dict[str, str]], - inference_profile: ExpressionInferenceProfile, - allow_compact_analysis: bool, - rich_type_inference: bool, - analysis_caching: ModelAnalysisCaching | None, - complete_binding_schemas: dict[str, dict[str, str]], + *, request: NativeModelAnalysisRequest ) -> dict[str, ModelSqlAnalysis] | None: - """Return each model's analysis by name, or None where Python must analyze the models.""" + """Return each model's analysis (and any pivot proof) by name, or None to defer to Python.""" return None diff --git a/src/sqlbuild/compiler/analysis_session/main/_infer_native_expression_source_shapes.py b/src/sqlbuild/compiler/analysis_session/main/_infer_native_expression_source_shapes.py new file mode 100644 index 0000000000..a39c7a6609 --- /dev/null +++ b/src/sqlbuild/compiler/analysis_session/main/_infer_native_expression_source_shapes.py @@ -0,0 +1,13 @@ +"""Infer expression-source shapes natively for the preview compiler engine.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile + + +def infer_native_expression_source_shapes( + *, expressions: tuple[str, ...], profile: ExpressionInferenceProfile +) -> tuple[dict[str, str] | None, ...] | None: + """Return one shape per expression, or None where Python must infer the shapes.""" + + return None diff --git a/src/sqlbuild/compiler/analysis_session/models.py b/src/sqlbuild/compiler/analysis_session/models.py new file mode 100644 index 0000000000..fedb337ce2 --- /dev/null +++ b/src/sqlbuild/compiler/analysis_session/models.py @@ -0,0 +1,27 @@ +"""Inputs handed to the native model analysis session.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile.models import CompileModelInput, ModelAnalysisCaching +from sqlbuild.compiler.lineage.types import InferredNullability +from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily + + +@dataclass(frozen=True) +class NativeModelAnalysisRequest: + """The Python model analysis's arguments plus the facts dynamic pivot proofs need.""" + + known_functions: tuple[str, ...] + known_types: tuple[str, ...] + model_inputs: tuple[CompileModelInput, ...] + column_nullability_by_table: dict[str, dict[str, InferredNullability]] + column_types_by_table: dict[str, dict[str, str]] + inference_profile: ExpressionInferenceProfile + allow_compact_analysis: bool + rich_type_inference: bool + analysis_caching: ModelAnalysisCaching | None + complete_binding_schemas: dict[str, dict[str, str]] + dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index e3933ad94b..abfc37621c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -53,7 +53,6 @@ binding_schema_for_model, build_column_nullability_by_table, build_complete_binding_schemas, - get_expression_source_shapes, published_model_shape, ) from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( @@ -80,6 +79,11 @@ from sqlbuild.compiler.compile._helpers.native_stages.assembly import ( analyze_model_sql_by_engine, assemble_project_by_engine, + dynamic_column_contract_by_engine, + expression_source_shapes_by_engine, +) +from sqlbuild.compiler.compile._helpers.native_stages.sql_tests import ( + assemble_sql_tests_by_engine, ) from sqlbuild.compiler.compile._helpers.render.context_templates import ( resolve_early_model_templates, @@ -271,7 +275,7 @@ def assemble_compiled_project( for source_input in inputs.source_inputs if source_input.source_entry.expression ) - expression_shapes: tuple[dict[str, str] | None, ...] = get_expression_source_shapes( + expression_shapes: tuple[dict[str, str] | None, ...] = expression_source_shapes_by_engine( expressions=tuple( cast(str, source_input.source_entry.expression) for source_input in expression_sources @@ -360,7 +364,8 @@ def assemble_compiled_project( complete_binding_schemas=complete_binding_schemas, ) model_sql_analysis_by_name = analyze_model_sql_by_engine( - python_analysis=python_analysis + python_analysis=python_analysis, + dynamic_families_by_table=dynamic_families_by_table, ) scope_index: ScopeIndex = scope_index_with_compile_usages(inputs=inputs) report_scope_index_errors(index=scope_index) @@ -446,11 +451,11 @@ def assemble_compiled_project( for function_input in inputs.sql_function_inputs ), audits=tuple(_assemble_compiled_audit(audit_input) for audit_input in inputs.audit_inputs), - sql_tests=tuple( - _assemble_compiled_sql_test( + sql_tests=assemble_sql_tests_by_engine( + inputs=inputs, + assemble_python_test=lambda test_input: _assemble_compiled_sql_test( test_input=test_input, model_inputs=inputs.model_inputs, inputs=inputs - ) - for test_input in inputs.test_inputs + ), ), sql_scenarios=tuple( _assemble_compiled_sql_scenario(scenario_input) @@ -578,30 +583,36 @@ def _assemble_compiled_model( placeholders=placeholders, dialect=profile.sql_analysis_dialect, ) - dynamic_column_contract: DynamicColumnContractProof | None = analyze_dynamic_column_contract( - query_sql=( - substitute_placeholder_defaults( - query_sql=analysis_query_sql, - placeholders=placeholders, - ) - if placeholders - else analysis_query_sql - ), - dialect=profile.sql_analysis_dialect, - families=( - model_input.schema_entry.dynamic_columns if model_input.schema_entry is not None else () - ), - column_types_by_table=column_types_by_table or {}, - authoritative_column_types_by_table=( - dynamic_contract_analysis_inputs.authoritative_column_types_by_table - if dynamic_contract_analysis_inputs is not None - else {} - ), - column_nullability_by_table=column_nullability_by_table or {}, - dynamic_families_by_table=( - dynamic_contract_analysis_inputs.families_by_table - if dynamic_contract_analysis_inputs is not None - else {} + dynamic_column_contract: DynamicColumnContractProof | None = dynamic_column_contract_by_engine( + sql_analysis=sql_analysis, + python_proof=partial( + analyze_dynamic_column_contract, + query_sql=( + substitute_placeholder_defaults( + query_sql=analysis_query_sql, + placeholders=placeholders, + ) + if placeholders + else analysis_query_sql + ), + dialect=profile.sql_analysis_dialect, + families=( + model_input.schema_entry.dynamic_columns + if model_input.schema_entry is not None + else () + ), + column_types_by_table=column_types_by_table or {}, + authoritative_column_types_by_table=( + dynamic_contract_analysis_inputs.authoritative_column_types_by_table + if dynamic_contract_analysis_inputs is not None + else {} + ), + column_nullability_by_table=column_nullability_by_table or {}, + dynamic_families_by_table=( + dynamic_contract_analysis_inputs.families_by_table + if dynamic_contract_analysis_inputs is not None + else {} + ), ), ) if dynamic_column_contract is not None and dynamic_column_contract.output_proven: diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py index 49bfdd90a1..fc4fe25baa 100644 --- a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py +++ b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py @@ -9,13 +9,26 @@ from sqlbuild.compiler.analysis_session.main._analyze_native_model_sql import ( analyze_native_model_sql, ) -from sqlbuild.compiler.compile.models import CompiledProject, CompileProjectInputs, ModelSqlAnalysis +from sqlbuild.compiler.analysis_session.main._infer_native_expression_source_shapes import ( + infer_native_expression_source_shapes, +) +from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest +from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( + get_expression_source_shapes, +) +from sqlbuild.compiler.compile.models import ( + CompiledProject, + CompileProjectInputs, + DynamicColumnContractProof, + ModelSqlAnalysis, +) from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.compiler.project_assembly.main._assemble_native_project import ( assemble_native_project, ) +from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily def assemble_project_by_engine( @@ -41,15 +54,45 @@ def assemble_project_by_engine( ) +def expression_source_shapes_by_engine( + *, expressions: tuple[str, ...], profile: ExpressionInferenceProfile +) -> tuple[dict[str, str] | None, ...]: + """Infer one shape per expression source natively, or with Python where native defers.""" + + if native_stage_enabled(NativeStage.MODEL_ANALYSIS): + native_shapes: tuple[dict[str, str] | None, ...] | None = ( + infer_native_expression_source_shapes(expressions=expressions, profile=profile) + ) + if native_shapes is not None: + return native_shapes + return get_expression_source_shapes(expressions=expressions, profile=profile) + + def analyze_model_sql_by_engine( - *, python_analysis: partial[dict[str, ModelSqlAnalysis]] + *, + python_analysis: partial[dict[str, ModelSqlAnalysis]], + dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], ) -> dict[str, ModelSqlAnalysis]: """Analyze models natively with the Python analysis's arguments, or run the Python analysis.""" if native_stage_enabled(NativeStage.MODEL_ANALYSIS): native_analyses: dict[str, ModelSqlAnalysis] | None = analyze_native_model_sql( - **python_analysis.keywords + request=NativeModelAnalysisRequest( + **python_analysis.keywords, dynamic_families_by_table=dynamic_families_by_table + ) ) if native_analyses is not None: return native_analyses return python_analysis() + + +def dynamic_column_contract_by_engine( + *, + sql_analysis: ModelSqlAnalysis | None, + python_proof: partial[DynamicColumnContractProof | None], +) -> DynamicColumnContractProof | None: + """Return the proof the model's native analysis carries, or prove the contract in Python.""" + + if sql_analysis is not None and sql_analysis.dynamic_column_contract is not None: + return sql_analysis.dynamic_column_contract + return python_proof() diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py new file mode 100644 index 0000000000..fde4b1cf87 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py @@ -0,0 +1,30 @@ +"""Hand compiled SQL test assembly to its native stage when enabled.""" + +from __future__ import annotations + +from collections.abc import Callable + +from sqlbuild.compiler.compile.models import ( + CompiledSqlTest, + CompileProjectInputs, + CompileSqlTestInput, +) +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.sql_test_glue.main._assemble_native_sql_tests import ( + assemble_native_sql_tests, +) + + +def assemble_sql_tests_by_engine( + *, + inputs: CompileProjectInputs, + assemble_python_test: Callable[[CompileSqlTestInput], CompiledSqlTest], +) -> tuple[CompiledSqlTest, ...]: + """Assemble every SQL test natively, or each test in Python where native defers.""" + + if native_stage_enabled(NativeStage.SQL_TEST_GLUE): + native_tests: tuple[CompiledSqlTest, ...] | None = assemble_native_sql_tests(inputs=inputs) + if native_tests is not None: + return native_tests + return tuple(assemble_python_test(test_input) for test_input in inputs.test_inputs) diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index e06674ac31..96be6e448c 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -823,6 +823,7 @@ class ModelSqlAnalysis: fused_binding_validated: bool = False cleaned_sql: str | None = None validated_schema: dict[str, dict[str, str]] | None = None + dynamic_column_contract: DynamicColumnContractProof | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/sql_test_glue/main/_assemble_native_sql_tests.py b/src/sqlbuild/compiler/sql_test_glue/main/_assemble_native_sql_tests.py new file mode 100644 index 0000000000..b732f71039 --- /dev/null +++ b/src/sqlbuild/compiler/sql_test_glue/main/_assemble_native_sql_tests.py @@ -0,0 +1,13 @@ +"""Assemble compiled SQL test facts natively for the preview compiler engine.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile.models import CompiledSqlTest, CompileProjectInputs + + +def assemble_native_sql_tests( + *, inputs: CompileProjectInputs +) -> tuple[CompiledSqlTest, ...] | None: + """Return one compiled test per test input, or None where Python must assemble the tests.""" + + return None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 10b4e6cae4..665e118250 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -39,6 +39,7 @@ import sqlbuild.compiler.compile._helpers.diagnostics.recovery as diagnostic_recovery import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stages +import sqlbuild.compiler.compile._helpers.native_stages.sql_tests as native_sql_test_stage import sqlbuild.compiler.compile.classes.binding_dataflow as binding_dataflow import sqlbuild.compiler.compile.classes.render_reuse_session as render_reuse_session import sqlbuild.compiler.compile.classes.stored_model_analyses as stored_model_analyses @@ -4750,7 +4751,9 @@ def engine_macro_call_runs( _ANALYSIS_SEAMS: tuple[tuple[ModuleType, str], ...] = ( (native_stages, "assemble_native_project"), + (native_stages, "infer_native_expression_source_shapes"), (native_stages, "analyze_native_model_sql"), + (native_sql_test_stage, "assemble_native_sql_tests"), (diagnostic_recovery, "complete_native_semantic_diagnostics"), (contract_validation, "evaluate_native_model_contracts"), (column_lineage, "build_native_column_lineage"), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index 16f87da078..8cd7acd8a6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -27,7 +27,9 @@ description="models_contracts_lineage_and_sql_tests", expected_preview_returns={ "assemble_native_project": [None], + "infer_native_expression_source_shapes": [None], "analyze_native_model_sql": [None], + "assemble_native_sql_tests": [None], "complete_native_semantic_diagnostics": [None], "evaluate_native_model_contracts": [None], "build_native_column_lineage": [None], diff --git a/tests/unit/src/sqlbuild/adapter/type_system/_test_types.py b/tests/unit/src/sqlbuild/adapter/type_system/_test_types.py index 8b5e8e423f..ce055965dc 100644 --- a/tests/unit/src/sqlbuild/adapter/type_system/_test_types.py +++ b/tests/unit/src/sqlbuild/adapter/type_system/_test_types.py @@ -26,11 +26,3 @@ class NumericFamilyTestCase: dialect: str | None raw_type: str expected_family: str | None - - -@dataclass(frozen=True) -class NativeTypeNormalizationTestCase: - description: str - dialect: str | None - raw_type: str - expected_type: NormalizedType | None diff --git a/tests/unit/src/sqlbuild/adapter/type_system/test_native_normalize_type.py b/tests/unit/src/sqlbuild/adapter/type_system/test_native_normalize_type.py deleted file mode 100644 index c7da19a0d3..0000000000 --- a/tests/unit/src/sqlbuild/adapter/type_system/test_native_normalize_type.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Native type normalization defers every type to Python until the native type system lands.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.adapter.type_system.main._native_normalize_type import normalize_native_type -from tests.unit.src.sqlbuild.adapter.type_system._test_types import ( - NativeTypeNormalizationTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - NativeTypeNormalizationTestCase( - description="snowflake number", - dialect="snowflake", - raw_type="NUMBER(38,0)", - expected_type=None, - ), - NativeTypeNormalizationTestCase( - description="duckdb sized varchar", - dialect="duckdb", - raw_type="VARCHAR(20)", - expected_type=None, - ), - NativeTypeNormalizationTestCase( - description="no dialect", - dialect=None, - raw_type="TIMESTAMP WITH TIME ZONE", - expected_type=None, - ), - ], - ids=lambda case: case.description, -) -def test_given_type_string_when_normalizing_natively_then_it_defers_to_python( - test_case: NativeTypeNormalizationTestCase, -) -> None: - assert ( - normalize_native_type(type_sql=test_case.raw_type, dialect=test_case.dialect) - == test_case.expected_type - ) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/_test_types.py new file mode 100644 index 0000000000..a49bf87317 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/_test_types.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild.compiler.compile.models import DynamicColumnContractProof, ModelSqlAnalysis +from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily + + +@dataclass(frozen=True) +class DynamicColumnContractDispatchTestCase: + description: str + sql_analysis: ModelSqlAnalysis | None + dialect: str + families: tuple[SchemaDynamicColumnFamily, ...] + expected_proof: DynamicColumnContractProof | None diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/test_dynamic_column_contract_by_engine.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/test_dynamic_column_contract_by_engine.py new file mode 100644 index 0000000000..92ff874493 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/test_dynamic_column_contract_by_engine.py @@ -0,0 +1,101 @@ +"""A model's native analysis may carry its dynamic pivot proof; otherwise Python proves it.""" + +from __future__ import annotations + +from functools import partial + +import pytest + +from sqlbuild.compiler.compile._helpers.analysis.dynamic_pivot import ( + analyze_dynamic_column_contract, +) +from sqlbuild.compiler.compile._helpers.native_stages.assembly import ( + dynamic_column_contract_by_engine, +) +from sqlbuild.compiler.compile.models import ( + DynamicColumnContractProof, + ModelSqlAnalysis, + PolyglotAnalysisResult, +) +from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily +from tests.unit.src.sqlbuild.compiler.compile._helpers.native_stages._test_types import ( + DynamicColumnContractDispatchTestCase, +) + +_NATIVE_PROOF: DynamicColumnContractProof = DynamicColumnContractProof( + output_proven=False, failure_reason="native session proof" +) +_PYTHON_UNSUPPORTED_PROOF: DynamicColumnContractProof = DynamicColumnContractProof( + output_proven=False, + failure_reason=("adapter dialect 'bigquery' does not support compiler-proven dynamic pivots"), +) +_FAMILY: SchemaDynamicColumnFamily = SchemaDynamicColumnFamily( + name="category_amounts", + pivot_column="category", + value_column="amount", + aggregate="MAX", + type="DECIMAL(12,2)", +) +_ANALYSIS: PolyglotAnalysisResult = PolyglotAnalysisResult(analysis_succeeded=True) + + +@pytest.mark.parametrize( + "test_case", + ( + DynamicColumnContractDispatchTestCase( + description="native analysis proof replaces the python proof", + sql_analysis=ModelSqlAnalysis( + polyglot_analysis=_ANALYSIS, + placeholders=None, + dynamic_column_contract=_NATIVE_PROOF, + ), + dialect="bigquery", + families=(_FAMILY,), + expected_proof=_NATIVE_PROOF, + ), + DynamicColumnContractDispatchTestCase( + description="analysis without a proof falls back to the python proof", + sql_analysis=ModelSqlAnalysis(polyglot_analysis=_ANALYSIS, placeholders=None), + dialect="bigquery", + families=(_FAMILY,), + expected_proof=_PYTHON_UNSUPPORTED_PROOF, + ), + DynamicColumnContractDispatchTestCase( + description="unanalyzed model falls back to the python proof", + sql_analysis=None, + dialect="bigquery", + families=(_FAMILY,), + expected_proof=_PYTHON_UNSUPPORTED_PROOF, + ), + DynamicColumnContractDispatchTestCase( + description="model without dynamic families has no proof", + sql_analysis=ModelSqlAnalysis(polyglot_analysis=_ANALYSIS, placeholders=None), + dialect="snowflake", + families=(), + expected_proof=None, + ), + ), + ids=lambda case: case.description, +) +def test_given_model_analysis_when_proving_dynamic_contract_then_native_proof_takes_precedence( + test_case: DynamicColumnContractDispatchTestCase, +) -> None: + proof: DynamicColumnContractProof | None = dynamic_column_contract_by_engine( + sql_analysis=test_case.sql_analysis, + python_proof=partial( + analyze_dynamic_column_contract, + query_sql="SELECT * FROM order_amounts", + dialect=test_case.dialect, + families=test_case.families, + column_types_by_table={}, + authoritative_column_types_by_table={}, + column_nullability_by_table={}, + dynamic_families_by_table={}, + ), + ) + + assert proof == test_case.expected_proof + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"])