diff --git a/crates/sqlbuild-analysis/src/assembly/mod.rs b/crates/sqlbuild-analysis/src/assembly/mod.rs new file mode 100644 index 0000000000..079a1bcfca --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/mod.rs @@ -0,0 +1,3 @@ +//! The native model analysis session: catalog, dataflow scheduling, projection and shape publication. + +pub mod project; diff --git a/crates/sqlbuild-analysis/src/assembly/project/mod.rs b/crates/sqlbuild-analysis/src/assembly/project/mod.rs new file mode 100644 index 0000000000..2c6b3134d3 --- /dev/null +++ b/crates/sqlbuild-analysis/src/assembly/project/mod.rs @@ -0,0 +1 @@ +//! Compiled project assembly: resources, dependencies, destinations and their diagnostics. diff --git a/crates/sqlbuild-analysis/src/contracts/mod.rs b/crates/sqlbuild-analysis/src/contracts/mod.rs new file mode 100644 index 0000000000..cf143bb851 --- /dev/null +++ b/crates/sqlbuild-analysis/src/contracts/mod.rs @@ -0,0 +1 @@ +//! Model column contracts and promotion conflicts. diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 74d7298030..2c6aee23ac 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -2,10 +2,15 @@ #![forbid(unsafe_code)] +pub mod assembly; pub mod column_references; pub mod compiler; mod constants; +pub mod contracts; +pub mod lineage; pub mod query_analysis; +pub mod semantic_checks; pub mod semantic_usage; pub mod semantic_validation; pub mod sql_tokens; +pub mod type_system; diff --git a/crates/sqlbuild-analysis/src/lineage/mod.rs b/crates/sqlbuild-analysis/src/lineage/mod.rs new file mode 100644 index 0000000000..9819e31334 --- /dev/null +++ b/crates/sqlbuild-analysis/src/lineage/mod.rs @@ -0,0 +1 @@ +//! Fast column lineage facts for compiled models. diff --git a/crates/sqlbuild-analysis/src/semantic_checks/mod.rs b/crates/sqlbuild-analysis/src/semantic_checks/mod.rs new file mode 100644 index 0000000000..85e0e654a1 --- /dev/null +++ b/crates/sqlbuild-analysis/src/semantic_checks/mod.rs @@ -0,0 +1 @@ +//! Semantic completion: type recovery, metadata checks, diagnostic recovery and explanations. diff --git a/crates/sqlbuild-analysis/src/type_system/mod.rs b/crates/sqlbuild-analysis/src/type_system/mod.rs new file mode 100644 index 0000000000..b053fa0aea --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/mod.rs @@ -0,0 +1 @@ +//! Type normalization and type equality shared by every analysis stage. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs new file mode 100644 index 0000000000..7444bd0374 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/mod.rs @@ -0,0 +1 @@ +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs new file mode 100644 index 0000000000..bbd9c13b6b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/registration.rs @@ -0,0 +1,7 @@ +//! Register the native model analysis session functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs new file mode 100644 index 0000000000..7444bd0374 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/mod.rs @@ -0,0 +1 @@ +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs new file mode 100644 index 0000000000..a821930745 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/registration.rs @@ -0,0 +1,7 @@ +//! Register the native contract functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs new file mode 100644 index 0000000000..7444bd0374 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/mod.rs @@ -0,0 +1 @@ +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs new file mode 100644 index 0000000000..e747ba06c2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/registration.rs @@ -0,0 +1,7 @@ +//! Register the native column lineage functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 4ad9b8950e..148a78e9a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -1,10 +1,17 @@ pub(crate) mod analysis; +pub(crate) mod analysis_session; pub(crate) mod attachments; pub(crate) mod boundary; pub(crate) mod cache; +pub(crate) mod contracts; pub(crate) mod discovery; +pub(crate) mod lineage; pub(crate) mod model_config; +pub(crate) mod project_assembly; pub(crate) mod render; pub(crate) mod rules; pub(crate) mod scopes; +pub(crate) mod semantic_checks; +pub(crate) mod sql_test_glue; pub(crate) mod sqltext; +pub(crate) mod type_system; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs new file mode 100644 index 0000000000..7444bd0374 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/mod.rs @@ -0,0 +1 @@ +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs new file mode 100644 index 0000000000..63be94a863 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/registration.rs @@ -0,0 +1,7 @@ +//! Register the native project assembly functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs new file mode 100644 index 0000000000..7444bd0374 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/mod.rs @@ -0,0 +1 @@ +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs new file mode 100644 index 0000000000..7ee7c2c7f0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/registration.rs @@ -0,0 +1,7 @@ +//! Register the native semantic completion functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs new file mode 100644 index 0000000000..7444bd0374 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/mod.rs @@ -0,0 +1 @@ +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs new file mode 100644 index 0000000000..641f84817b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/registration.rs @@ -0,0 +1,7 @@ +//! Register the native SQL test planning functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs new file mode 100644 index 0000000000..7444bd0374 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs @@ -0,0 +1 @@ +pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs new file mode 100644 index 0000000000..cd60143ae3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs @@ -0,0 +1,7 @@ +//! Register the native type system functions. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index cde73bc8c0..37e0e09255 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -4,7 +4,9 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ - analysis, attachments, boundary, cache, discovery, model_config, render, rules, scopes, sqltext, + analysis, analysis_session, attachments, boundary, cache, contracts, discovery, lineage, + model_config, project_assembly, render, rules, scopes, semantic_checks, sql_test_glue, sqltext, + type_system, }; use crate::bindings::models; @@ -32,7 +34,19 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { render::macro_calls::register(module)?; cache::native_store::register(module)?; attachments::registration::register(module)?; + register_analysis_stages(module)?; module.add("API_VERSION", API_VERSION)?; module.add("BUILD_IDENTITY", NATIVE_BUILD_IDENTITY)?; Ok(()) } + +fn register_analysis_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { + type_system::registration::register(module)?; + analysis_session::registration::register(module)?; + semantic_checks::registration::register(module)?; + contracts::registration::register(module)?; + lineage::registration::register(module)?; + sql_test_glue::registration::register(module)?; + project_assembly::registration::register(module)?; + Ok(()) +} diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 854d26027b..cc0cc43c2f 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -261,6 +261,20 @@ def content_digest(parts: list[str]) -> str: ... def fingerprint_project_files(root: str, excluded_files: list[str]) -> str: ... def digest_files(paths: list[str]) -> list[str | None]: ... +# Native analysis: type system. + +# Native analysis: model analysis session. + +# Native analysis: semantic completion. + +# Native analysis: contracts. + +# Native analysis: column lineage facts. + +# Native analysis: SQL test planning glue. + +# Native analysis: compiled project assembly. + # Internal oracle hooks for tests that compare native foundations with Python; not an API. def _oracle_json_dumps(dialect_json: str, value_json: str) -> str: ... def _oracle_text_positions( diff --git a/src/sqlbuild/adapter/type_system/main/_native_normalize_type.py b/src/sqlbuild/adapter/type_system/main/_native_normalize_type.py new file mode 100644 index 0000000000..37efa7eb36 --- /dev/null +++ b/src/sqlbuild/adapter/type_system/main/_native_normalize_type.py @@ -0,0 +1,14 @@ +"""Native type normalization for the preview compiler engine.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.models import NormalizedType +from sqlbuild.adapter.contract.types import TypeDialect + + +def normalize_native_type( + *, type_sql: str, dialect: TypeDialect | str | None +) -> NormalizedType | None: + """Return the native normalization, or None where Python must normalize the type.""" + + return None diff --git a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py index 7c2a99bf20..da8b849949 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py @@ -47,6 +47,8 @@ DiagnosticSeverity, FunctionLanguage, ) +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.planner.exceptions import PlannerInputError from sqlbuild.compiler.planner.main.execution.sql_test_artifacts import ( plan_and_render_sql_test_artifacts, @@ -56,6 +58,9 @@ ) from sqlbuild.compiler.planner.models import AuditPlanEntry, NativeSqlTestArtifact, PlanOutput from sqlbuild.compiler.profiling.main.record import record_compile_timing +from sqlbuild.compiler.sql_test_glue.main.plan_native_sql_test_artifacts import ( + plan_native_sql_test_artifacts, +) from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -569,12 +574,23 @@ def _plan_static_test_artifacts( with OperationLifecycle( operation_kind="project", operation_name="sql_test_planning" ) as lifecycle: - native_artifacts: tuple[NativeSqlTestArtifact, ...] = plan_and_render_sql_test_artifacts( - project=project, - tests=tests, - adapter=adapter, - sql_analysis_enabled=project.settings.sql_analysis, + native_artifacts: tuple[NativeSqlTestArtifact, ...] | None = ( + plan_native_sql_test_artifacts( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=project.settings.sql_analysis, + ) + if native_stage_enabled(NativeStage.SQL_TEST_GLUE) + else None ) + if native_artifacts is None: + native_artifacts = plan_and_render_sql_test_artifacts( + project=project, + tests=tests, + adapter=adapter, + sql_analysis_enabled=project.settings.sql_analysis, + ) lifecycle.completed(metadata={"item_count": len(native_artifacts)}) return native_artifacts diff --git a/src/sqlbuild/compiler/analysis_session/__init__.py b/src/sqlbuild/compiler/analysis_session/__init__.py new file mode 100644 index 0000000000..d0736bf682 --- /dev/null +++ b/src/sqlbuild/compiler/analysis_session/__init__.py @@ -0,0 +1 @@ +"""The native model analysis session for the preview compiler engine.""" diff --git a/src/sqlbuild/compiler/analysis_session/main/__init__.py b/src/sqlbuild/compiler/analysis_session/main/__init__.py new file mode 100644 index 0000000000..355f68e13c --- /dev/null +++ b/src/sqlbuild/compiler/analysis_session/main/__init__.py @@ -0,0 +1 @@ +"""Native model analysis session entry points.""" diff --git a/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py b/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py new file mode 100644 index 0000000000..ea5cdadba4 --- /dev/null +++ b/src/sqlbuild/compiler/analysis_session/main/_analyze_native_model_sql.py @@ -0,0 +1,14 @@ +"""Analyze model SQL natively for the preview compiler engine.""" + +from __future__ import annotations + +from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest +from sqlbuild.compiler.compile.models import ModelSqlAnalysis + + +def analyze_native_model_sql( + *, request: NativeModelAnalysisRequest +) -> dict[str, ModelSqlAnalysis] | None: + """Return each model's analysis (and any pivot proof) by name, or None to defer to Python.""" + + return None diff --git a/src/sqlbuild/compiler/analysis_session/main/_infer_native_expression_source_shapes.py b/src/sqlbuild/compiler/analysis_session/main/_infer_native_expression_source_shapes.py new file mode 100644 index 0000000000..a39c7a6609 --- /dev/null +++ b/src/sqlbuild/compiler/analysis_session/main/_infer_native_expression_source_shapes.py @@ -0,0 +1,13 @@ +"""Infer expression-source shapes natively for the preview compiler engine.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile + + +def infer_native_expression_source_shapes( + *, expressions: tuple[str, ...], profile: ExpressionInferenceProfile +) -> tuple[dict[str, str] | None, ...] | None: + """Return one shape per expression, or None where Python must infer the shapes.""" + + return None diff --git a/src/sqlbuild/compiler/analysis_session/models.py b/src/sqlbuild/compiler/analysis_session/models.py new file mode 100644 index 0000000000..fedb337ce2 --- /dev/null +++ b/src/sqlbuild/compiler/analysis_session/models.py @@ -0,0 +1,27 @@ +"""Inputs handed to the native model analysis session.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile.models import CompileModelInput, ModelAnalysisCaching +from sqlbuild.compiler.lineage.types import InferredNullability +from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily + + +@dataclass(frozen=True) +class NativeModelAnalysisRequest: + """The Python model analysis's arguments plus the facts dynamic pivot proofs need.""" + + known_functions: tuple[str, ...] + known_types: tuple[str, ...] + model_inputs: tuple[CompileModelInput, ...] + column_nullability_by_table: dict[str, dict[str, InferredNullability]] + column_types_by_table: dict[str, dict[str, str]] + inference_profile: ExpressionInferenceProfile + allow_compact_analysis: bool + rich_type_inference: bool + analysis_caching: ModelAnalysisCaching | None + complete_binding_schemas: dict[str, dict[str, str]] + dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index 0d8e1b8c9d..abfc37621c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -21,10 +21,7 @@ read_compact_analysis_cache_candidate, record_analysis_cache_metrics, ) -from sqlbuild.compiler.compile._helpers.analysis.columns import ( - substitute_placeholder_defaults, - table_function_analysis_name, -) +from sqlbuild.compiler.compile._helpers.analysis.columns import substitute_placeholder_defaults from sqlbuild.compiler.compile._helpers.analysis.compact import ( NativeCompactAnalysis, analyze_columns_and_lineage_with_polyglot, @@ -54,8 +51,8 @@ from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( binding_required_names, binding_schema_for_model, + build_column_nullability_by_table, build_complete_binding_schemas, - get_expression_source_shapes, published_model_shape, ) from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( @@ -79,6 +76,15 @@ ) from sqlbuild.compiler.compile._helpers.diagnostics.recovery import complete_semantic_diagnostics from sqlbuild.compiler.compile._helpers.diagnostics.scope import report_scope_index_errors +from sqlbuild.compiler.compile._helpers.native_stages.assembly import ( + analyze_model_sql_by_engine, + assemble_project_by_engine, + dynamic_column_contract_by_engine, + expression_source_shapes_by_engine, +) +from sqlbuild.compiler.compile._helpers.native_stages.sql_tests import ( + assemble_sql_tests_by_engine, +) from sqlbuild.compiler.compile._helpers.render.context_templates import ( resolve_early_model_templates, ) @@ -98,7 +104,6 @@ ) from sqlbuild.compiler.compile._helpers.sql_tests.identity import build_sql_test_case_fingerprint from sqlbuild.compiler.compile.classes.stored_model_analyses import StoredModelAnalyses -from sqlbuild.compiler.compile.constants import NOT_NULL_AUDIT_NAME from sqlbuild.compiler.compile.main._scope_index_with_compile_usages import ( scope_index_with_compile_usages, ) @@ -182,10 +187,7 @@ ) from sqlbuild.spec.contracts.models import ( DefaultsConfig, - SchemaAuditInstance, - SchemaColumn, SchemaDynamicColumnFamily, - SourceColumnEntry, SourceEntry, SourceLocation, TargetConfig, @@ -213,6 +215,16 @@ def assemble_compiled_project( ) -> CompiledProject: """Convert attached compile inputs into the planner-ready project view.""" + native_project: CompiledProject | None = assemble_project_by_engine( + inputs=inputs, + inference_profile=inference_profile, + skip_column_inference=skip_column_inference, + column_lineage_mode=column_lineage_mode, + analysis_cache_dir=analysis_cache_dir, + analysis_model_names=analysis_model_names, + ) + if native_project is not None: + return native_project sql_analysis_enabled: bool = ( inputs.effective_settings.sql_analysis and not skip_column_inference ) @@ -220,7 +232,7 @@ def assemble_compiled_project( seed_input.schema_entry.name for seed_input in inputs.seed_inputs ) column_nullability_by_table: dict[str, dict[str, InferredNullability]] = ( - _build_column_nullability_by_table(inputs) + build_column_nullability_by_table(inputs) ) column_types_by_table: dict[str, dict[str, str]] = _build_column_types_by_table(inputs) dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] = ( @@ -263,7 +275,7 @@ def assemble_compiled_project( for source_input in inputs.source_inputs if source_input.source_entry.expression ) - expression_shapes: tuple[dict[str, str] | None, ...] = get_expression_source_shapes( + expression_shapes: tuple[dict[str, str] | None, ...] = expression_source_shapes_by_engine( expressions=tuple( cast(str, source_input.source_entry.expression) for source_input in expression_sources @@ -321,7 +333,8 @@ def assemble_compiled_project( model_sql_analysis_by_name: dict[str, _ModelSqlAnalysis] = {} if sql_analysis_enabled: with record_compile_timing("model_analysis_ms"): - model_sql_analysis_by_name = _analyze_model_sql_in_parallel( + python_analysis: partial[dict[str, _ModelSqlAnalysis]] = partial( + _analyze_model_sql_in_parallel, known_functions=known_function_names(inputs.sql_function_inputs), known_types=known_declared_types( functions=inputs.sql_function_inputs, column_types=column_types_by_table @@ -350,6 +363,10 @@ def assemble_compiled_project( ), complete_binding_schemas=complete_binding_schemas, ) + model_sql_analysis_by_name = analyze_model_sql_by_engine( + python_analysis=python_analysis, + dynamic_families_by_table=dynamic_families_by_table, + ) scope_index: ScopeIndex = scope_index_with_compile_usages(inputs=inputs) report_scope_index_errors(index=scope_index) effective_target_values: dict[str, object] = resolve_early_model_templates( @@ -434,11 +451,11 @@ def assemble_compiled_project( for function_input in inputs.sql_function_inputs ), audits=tuple(_assemble_compiled_audit(audit_input) for audit_input in inputs.audit_inputs), - sql_tests=tuple( - _assemble_compiled_sql_test( + sql_tests=assemble_sql_tests_by_engine( + inputs=inputs, + assemble_python_test=lambda test_input: _assemble_compiled_sql_test( test_input=test_input, model_inputs=inputs.model_inputs, inputs=inputs - ) - for test_input in inputs.test_inputs + ), ), sql_scenarios=tuple( _assemble_compiled_sql_scenario(scenario_input) @@ -566,30 +583,36 @@ def _assemble_compiled_model( placeholders=placeholders, dialect=profile.sql_analysis_dialect, ) - dynamic_column_contract: DynamicColumnContractProof | None = analyze_dynamic_column_contract( - query_sql=( - substitute_placeholder_defaults( - query_sql=analysis_query_sql, - placeholders=placeholders, - ) - if placeholders - else analysis_query_sql - ), - dialect=profile.sql_analysis_dialect, - families=( - model_input.schema_entry.dynamic_columns if model_input.schema_entry is not None else () - ), - column_types_by_table=column_types_by_table or {}, - authoritative_column_types_by_table=( - dynamic_contract_analysis_inputs.authoritative_column_types_by_table - if dynamic_contract_analysis_inputs is not None - else {} - ), - column_nullability_by_table=column_nullability_by_table or {}, - dynamic_families_by_table=( - dynamic_contract_analysis_inputs.families_by_table - if dynamic_contract_analysis_inputs is not None - else {} + dynamic_column_contract: DynamicColumnContractProof | None = dynamic_column_contract_by_engine( + sql_analysis=sql_analysis, + python_proof=partial( + analyze_dynamic_column_contract, + query_sql=( + substitute_placeholder_defaults( + query_sql=analysis_query_sql, + placeholders=placeholders, + ) + if placeholders + else analysis_query_sql + ), + dialect=profile.sql_analysis_dialect, + families=( + model_input.schema_entry.dynamic_columns + if model_input.schema_entry is not None + else () + ), + column_types_by_table=column_types_by_table or {}, + authoritative_column_types_by_table=( + dynamic_contract_analysis_inputs.authoritative_column_types_by_table + if dynamic_contract_analysis_inputs is not None + else {} + ), + column_nullability_by_table=column_nullability_by_table or {}, + dynamic_families_by_table=( + dynamic_contract_analysis_inputs.families_by_table + if dynamic_contract_analysis_inputs is not None + else {} + ), ), ) if dynamic_column_contract is not None and dynamic_column_contract.output_proven: @@ -1369,33 +1392,6 @@ def _model_name(model_input: CompileModelInput) -> str: return model_input.model_file.file_path.stem -def _build_column_nullability_by_table( - inputs: CompileProjectInputs, -) -> dict[str, dict[str, InferredNullability]]: - facts: dict[str, dict[str, InferredNullability]] = {} - for model_input in inputs.model_inputs: - if model_input.schema_entry is None: - continue - facts[model_input.schema_entry.name] = _schema_column_nullability( - model_input.schema_entry.columns - ) - for seed_input in inputs.seed_inputs: - facts[seed_input.schema_entry.name] = _schema_column_nullability( - seed_input.schema_entry.columns - ) - for source_input in inputs.source_inputs: - facts[source_input.source_entry.name] = _source_column_nullability( - source_input.source_entry.columns - ) - for function_input in inputs.sql_function_inputs: - if not function_input.return_columns: - continue - facts[table_function_analysis_name(function_input.name)] = { - column.name: InferredNullability.UNKNOWN for column in function_input.return_columns - } - return facts - - def _build_dynamic_families_by_table( inputs: CompileProjectInputs, ) -> dict[str, tuple[SchemaDynamicColumnFamily, ...]]: @@ -1501,36 +1497,6 @@ def _binding_compiler_diagnostics( return tuple(result) -def _schema_column_nullability( - columns: tuple[SchemaColumn, ...], -) -> dict[str, InferredNullability]: - return { - column.name: _declared_column_nullability(nullable=column.nullable, audits=column.audits) - for column in columns - } - - -def _source_column_nullability( - columns: tuple[SourceColumnEntry, ...], -) -> dict[str, InferredNullability]: - return { - column.name: _declared_column_nullability(nullable=column.nullable, audits=column.audits) - for column in columns - } - - -def _declared_column_nullability( - *, - nullable: bool | None, - audits: tuple[SchemaAuditInstance, ...], -) -> InferredNullability: - if nullable is False: - return InferredNullability.NON_NULL - if any(audit.definition_name == NOT_NULL_AUDIT_NAME for audit in audits): - return InferredNullability.NON_NULL - return InferredNullability.UNKNOWN - - def _assemble_compiled_source( *, source_input: CompileSourceInput, diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py index 1fb6963cb3..85f3713d60 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py @@ -12,6 +12,7 @@ analyze_columns_and_lineage_with_polyglot, analyze_queries_with_compact_polyglot_batch, ) +from sqlbuild.compiler.compile.constants import NOT_NULL_AUDIT_NAME from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledProject, @@ -20,6 +21,7 @@ CompileSqlReference, PolyglotAnalysisResult, ) +from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.compiler.planner.constants import ( SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, SNAPSHOT_DEFAULT_VALID_TO_COLUMN, @@ -32,7 +34,12 @@ ) from sqlbuild.compiler.sql_analysis.main._binding_catalog import create_binding_catalog from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql -from sqlbuild.spec.contracts.models import SourceEntry +from sqlbuild.spec.contracts.models import ( + SchemaAuditInstance, + SchemaColumn, + SourceColumnEntry, + SourceEntry, +) def binding_relation_names(references: tuple[CompileSqlReference, ...]) -> frozenset[str]: @@ -131,6 +138,65 @@ def build_declared_column_types(inputs: CompileProjectInputs) -> dict[str, dict[ return facts +def build_column_nullability_by_table( + inputs: CompileProjectInputs, +) -> dict[str, dict[str, InferredNullability]]: + """Return what each model, seed, source and table function declares about column nulls.""" + + facts: dict[str, dict[str, InferredNullability]] = {} + for model_input in inputs.model_inputs: + if model_input.schema_entry is None: + continue + facts[model_input.schema_entry.name] = _schema_column_nullability( + model_input.schema_entry.columns + ) + for seed_input in inputs.seed_inputs: + facts[seed_input.schema_entry.name] = _schema_column_nullability( + seed_input.schema_entry.columns + ) + for source_input in inputs.source_inputs: + facts[source_input.source_entry.name] = _source_column_nullability( + source_input.source_entry.columns + ) + for function_input in inputs.sql_function_inputs: + if not function_input.return_columns: + continue + facts[table_function_analysis_name(function_input.name)] = { + column.name: InferredNullability.UNKNOWN for column in function_input.return_columns + } + return facts + + +def _schema_column_nullability( + columns: tuple[SchemaColumn, ...], +) -> dict[str, InferredNullability]: + return { + column.name: _declared_column_nullability(nullable=column.nullable, audits=column.audits) + for column in columns + } + + +def _source_column_nullability( + columns: tuple[SourceColumnEntry, ...], +) -> dict[str, InferredNullability]: + return { + column.name: _declared_column_nullability(nullable=column.nullable, audits=column.audits) + for column in columns + } + + +def _declared_column_nullability( + *, + nullable: bool | None, + audits: tuple[SchemaAuditInstance, ...], +) -> InferredNullability: + if nullable is False: + return InferredNullability.NON_NULL + if any(audit.definition_name == NOT_NULL_AUDIT_NAME for audit in audits): + return InferredNullability.NON_NULL + return InferredNullability.UNKNOWN + + def build_complete_binding_schemas(inputs: CompileProjectInputs) -> dict[str, dict[str, str]]: """Collect authoritative declared interfaces before SQL inference.""" schemas: dict[str, dict[str, str]] = {} diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py index cb363cf8e0..521430a59e 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py @@ -27,6 +27,11 @@ CompiledProject, CompilerDiagnostic, ) +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.semantic_checks.main._complete_native_semantic_diagnostics import ( + complete_native_semantic_diagnostics, +) from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic _MISSING_CODE: str = "B002" @@ -41,6 +46,15 @@ def complete_semantic_diagnostics( resource_sql_analysis: bool = True, ) -> CompiledProject: """Recover type facts before metadata checks, then explain retained root diagnostics.""" + if native_stage_enabled(NativeStage.SEMANTIC_CHECKS): + native_project: CompiledProject | None = complete_native_semantic_diagnostics( + project=project, + profile=profile, + binding_results=binding_results, + resource_sql_analysis=resource_sql_analysis, + ) + if native_project is not None: + return native_project project = recover_output_types(project=project, binding_results=binding_results) project = replace( project, diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py new file mode 100644 index 0000000000..fc4fe25baa --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/native_stages/assembly.py @@ -0,0 +1,98 @@ +"""Hand compiled-project assembly and model analysis to their native stages when enabled.""" + +from __future__ import annotations + +from functools import partial +from pathlib import Path + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.analysis_session.main._analyze_native_model_sql import ( + analyze_native_model_sql, +) +from sqlbuild.compiler.analysis_session.main._infer_native_expression_source_shapes import ( + infer_native_expression_source_shapes, +) +from sqlbuild.compiler.analysis_session.models import NativeModelAnalysisRequest +from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( + get_expression_source_shapes, +) +from sqlbuild.compiler.compile.models import ( + CompiledProject, + CompileProjectInputs, + DynamicColumnContractProof, + ModelSqlAnalysis, +) +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.lineage.types import ColumnLineageMode +from sqlbuild.compiler.project_assembly.main._assemble_native_project import ( + assemble_native_project, +) +from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily + + +def assemble_project_by_engine( + *, + inputs: CompileProjectInputs, + inference_profile: ExpressionInferenceProfile | None, + skip_column_inference: bool, + column_lineage_mode: ColumnLineageMode, + analysis_cache_dir: Path | None, + analysis_model_names: frozenset[str] | None, +) -> CompiledProject | None: + """Return the natively assembled project, or None where Python must assemble it.""" + + if not native_stage_enabled(NativeStage.PROJECT_ASSEMBLY): + return None + return assemble_native_project( + inputs=inputs, + inference_profile=inference_profile, + skip_column_inference=skip_column_inference, + column_lineage_mode=column_lineage_mode, + analysis_cache_dir=analysis_cache_dir, + analysis_model_names=analysis_model_names, + ) + + +def expression_source_shapes_by_engine( + *, expressions: tuple[str, ...], profile: ExpressionInferenceProfile +) -> tuple[dict[str, str] | None, ...]: + """Infer one shape per expression source natively, or with Python where native defers.""" + + if native_stage_enabled(NativeStage.MODEL_ANALYSIS): + native_shapes: tuple[dict[str, str] | None, ...] | None = ( + infer_native_expression_source_shapes(expressions=expressions, profile=profile) + ) + if native_shapes is not None: + return native_shapes + return get_expression_source_shapes(expressions=expressions, profile=profile) + + +def analyze_model_sql_by_engine( + *, + python_analysis: partial[dict[str, ModelSqlAnalysis]], + dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]], +) -> dict[str, ModelSqlAnalysis]: + """Analyze models natively with the Python analysis's arguments, or run the Python analysis.""" + + if native_stage_enabled(NativeStage.MODEL_ANALYSIS): + native_analyses: dict[str, ModelSqlAnalysis] | None = analyze_native_model_sql( + request=NativeModelAnalysisRequest( + **python_analysis.keywords, dynamic_families_by_table=dynamic_families_by_table + ) + ) + if native_analyses is not None: + return native_analyses + return python_analysis() + + +def dynamic_column_contract_by_engine( + *, + sql_analysis: ModelSqlAnalysis | None, + python_proof: partial[DynamicColumnContractProof | None], +) -> DynamicColumnContractProof | None: + """Return the proof the model's native analysis carries, or prove the contract in Python.""" + + if sql_analysis is not None and sql_analysis.dynamic_column_contract is not None: + return sql_analysis.dynamic_column_contract + return python_proof() diff --git a/src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py new file mode 100644 index 0000000000..fde4b1cf87 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/native_stages/sql_tests.py @@ -0,0 +1,30 @@ +"""Hand compiled SQL test assembly to its native stage when enabled.""" + +from __future__ import annotations + +from collections.abc import Callable + +from sqlbuild.compiler.compile.models import ( + CompiledSqlTest, + CompileProjectInputs, + CompileSqlTestInput, +) +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage +from sqlbuild.compiler.sql_test_glue.main._assemble_native_sql_tests import ( + assemble_native_sql_tests, +) + + +def assemble_sql_tests_by_engine( + *, + inputs: CompileProjectInputs, + assemble_python_test: Callable[[CompileSqlTestInput], CompiledSqlTest], +) -> tuple[CompiledSqlTest, ...]: + """Assemble every SQL test natively, or each test in Python where native defers.""" + + if native_stage_enabled(NativeStage.SQL_TEST_GLUE): + native_tests: tuple[CompiledSqlTest, ...] | None = assemble_native_sql_tests(inputs=inputs) + if native_tests is not None: + return native_tests + return tuple(assemble_python_test(test_input) for test_input in inputs.test_inputs) diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 91315721e6..96be6e448c 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -823,6 +823,7 @@ class ModelSqlAnalysis: fused_binding_validated: bool = False cleaned_sql: str | None = None validated_schema: dict[str, dict[str, str]] | None = None + dynamic_column_contract: DynamicColumnContractProof | None = None @dataclass(frozen=True) @@ -1184,6 +1185,7 @@ class CompiledProject: sql_expansions: dict[Path, CompiledSqlExpansion] = field( default_factory=dict, compare=False, repr=False ) + native_session: Any | None = field(default=None, compare=False, repr=False) @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/contracts/main/_evaluate_native_model_contracts.py b/src/sqlbuild/compiler/contracts/main/_evaluate_native_model_contracts.py new file mode 100644 index 0000000000..bb823aa8c2 --- /dev/null +++ b/src/sqlbuild/compiler/contracts/main/_evaluate_native_model_contracts.py @@ -0,0 +1,15 @@ +"""Evaluate model contracts natively for the preview compiler engine.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.types import TypeDialect +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.contracts.models import ContractValidationResult + + +def evaluate_native_model_contracts( + *, project: CompiledProject, dialect: TypeDialect | str | None +) -> ContractValidationResult | None: + """Return the contract diagnostics, or None where Python must evaluate the contracts.""" + + return None diff --git a/src/sqlbuild/compiler/contracts/main/validate.py b/src/sqlbuild/compiler/contracts/main/validate.py index 9f453ebd0a..20049a5419 100644 --- a/src/sqlbuild/compiler/contracts/main/validate.py +++ b/src/sqlbuild/compiler/contracts/main/validate.py @@ -5,7 +5,12 @@ from sqlbuild.adapter.contract.types import TypeDialect from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject, CompilerDiagnostic from sqlbuild.compiler.contracts._helpers.columns import collect_model_column_contract_diagnostics +from sqlbuild.compiler.contracts.main._evaluate_native_model_contracts import ( + evaluate_native_model_contracts, +) from sqlbuild.compiler.contracts.models import ContractValidationResult +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.planner.types import ContractPolicy from sqlbuild.spec.contracts.types import ColumnContractMode @@ -17,6 +22,12 @@ def evaluate_model_contracts( ) -> ContractValidationResult: """Evaluate model header column contracts against inferred output columns.""" + if native_stage_enabled(NativeStage.CONTRACTS): + native_result: ContractValidationResult | None = evaluate_native_model_contracts( + project=project, dialect=dialect + ) + if native_result is not None: + return native_result if not any( _requires_contract_evaluation(model=model, mode=project.settings.column_contract_mode) for model in project.models diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index 572857ecf4..6da274d310 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -25,6 +25,13 @@ NativeStage.MACRO_CALLS: NativeStageTier.PREVIEW, NativeStage.MACRO_CALL_STORE: NativeStageTier.PREVIEW, NativeStage.ATTACHMENTS: NativeStageTier.PREVIEW, + NativeStage.TYPE_SYSTEM: NativeStageTier.PREVIEW, + NativeStage.MODEL_ANALYSIS: NativeStageTier.PREVIEW, + NativeStage.SEMANTIC_CHECKS: NativeStageTier.PREVIEW, + NativeStage.CONTRACTS: NativeStageTier.PREVIEW, + NativeStage.LINEAGE_FACTS: NativeStageTier.PREVIEW, + NativeStage.SQL_TEST_GLUE: NativeStageTier.PREVIEW, + NativeStage.PROJECT_ASSEMBLY: NativeStageTier.PREVIEW, } ENGINE_NATIVE_STAGE_TIERS: dict[CompilerEngine, frozenset[NativeStageTier]] = { CompilerEngine.PYTHON: frozenset(), diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py index 70b4d146c3..17b611e46f 100644 --- a/src/sqlbuild/compiler/frontier/types.py +++ b/src/sqlbuild/compiler/frontier/types.py @@ -23,6 +23,13 @@ class NativeStage(StrEnum): MACRO_CALLS = "macro_calls" MACRO_CALL_STORE = "macro_call_store" ATTACHMENTS = "attachments" + TYPE_SYSTEM = "type_system" + MODEL_ANALYSIS = "model_analysis" + SEMANTIC_CHECKS = "semantic_checks" + CONTRACTS = "contracts" + LINEAGE_FACTS = "lineage_facts" + SQL_TEST_GLUE = "sql_test_glue" + PROJECT_ASSEMBLY = "project_assembly" class NativeStageTier(StrEnum): diff --git a/src/sqlbuild/compiler/lineage/main/_build_native_column_lineage.py b/src/sqlbuild/compiler/lineage/main/_build_native_column_lineage.py new file mode 100644 index 0000000000..d19bf5b536 --- /dev/null +++ b/src/sqlbuild/compiler/lineage/main/_build_native_column_lineage.py @@ -0,0 +1,14 @@ +"""Build fast column lineage facts natively for the preview compiler engine.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.lineage.models import ProjectColumnLineage + + +def build_native_column_lineage( + *, project: CompiledProject, dialect: str | None, model_names: frozenset[str] | None +) -> ProjectColumnLineage | None: + """Return the fast lineage graph, or None where Python must build it.""" + + return None diff --git a/src/sqlbuild/compiler/lineage/main/columns.py b/src/sqlbuild/compiler/lineage/main/columns.py index a881acc4e7..fc44eab5bc 100644 --- a/src/sqlbuild/compiler/lineage/main/columns.py +++ b/src/sqlbuild/compiler/lineage/main/columns.py @@ -3,12 +3,17 @@ from __future__ import annotations from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.lineage._helpers.fast_columns import ( build_fast_project_column_lineage, ) from sqlbuild.compiler.lineage._helpers.rich_columns import ( build_rich_project_column_lineage, ) +from sqlbuild.compiler.lineage.main._build_native_column_lineage import ( + build_native_column_lineage, +) from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.lineage.types import ColumnLineageMode @@ -24,6 +29,12 @@ def build_project_column_lineage( match mode: case ColumnLineageMode.FAST: + if native_stage_enabled(NativeStage.LINEAGE_FACTS): + native_lineage: ProjectColumnLineage | None = build_native_column_lineage( + project=project, dialect=dialect, model_names=model_names + ) + if native_lineage is not None: + return native_lineage return build_fast_project_column_lineage( project=project, dialect=dialect, diff --git a/src/sqlbuild/compiler/project_assembly/__init__.py b/src/sqlbuild/compiler/project_assembly/__init__.py new file mode 100644 index 0000000000..16a18c5b6e --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/__init__.py @@ -0,0 +1 @@ +"""Native compiled project assembly for the preview compiler engine.""" diff --git a/src/sqlbuild/compiler/project_assembly/main/__init__.py b/src/sqlbuild/compiler/project_assembly/main/__init__.py new file mode 100644 index 0000000000..646fd16ff4 --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/main/__init__.py @@ -0,0 +1 @@ +"""Native compiled project assembly entry points.""" diff --git a/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project.py b/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project.py new file mode 100644 index 0000000000..74ba3e90fb --- /dev/null +++ b/src/sqlbuild/compiler/project_assembly/main/_assemble_native_project.py @@ -0,0 +1,23 @@ +"""Assemble the compiled project natively for the preview compiler engine.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile.models import CompiledProject, CompileProjectInputs +from sqlbuild.compiler.lineage.types import ColumnLineageMode + + +def assemble_native_project( + *, + inputs: CompileProjectInputs, + inference_profile: ExpressionInferenceProfile | None, + skip_column_inference: bool, + column_lineage_mode: ColumnLineageMode, + analysis_cache_dir: Path | None, + analysis_model_names: frozenset[str] | None, +) -> CompiledProject | None: + """Return the compiled project, or None where Python must assemble it.""" + + return None diff --git a/src/sqlbuild/compiler/semantic_checks/__init__.py b/src/sqlbuild/compiler/semantic_checks/__init__.py new file mode 100644 index 0000000000..d0defadb2d --- /dev/null +++ b/src/sqlbuild/compiler/semantic_checks/__init__.py @@ -0,0 +1 @@ +"""Native semantic completion for the preview compiler engine.""" diff --git a/src/sqlbuild/compiler/semantic_checks/main/__init__.py b/src/sqlbuild/compiler/semantic_checks/main/__init__.py new file mode 100644 index 0000000000..b7664e28d3 --- /dev/null +++ b/src/sqlbuild/compiler/semantic_checks/main/__init__.py @@ -0,0 +1 @@ +"""Native semantic completion entry points.""" diff --git a/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py b/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py new file mode 100644 index 0000000000..ba3914b53f --- /dev/null +++ b/src/sqlbuild/compiler/semantic_checks/main/_complete_native_semantic_diagnostics.py @@ -0,0 +1,19 @@ +"""Complete semantic diagnostics natively for the preview compiler engine.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic + + +def complete_native_semantic_diagnostics( + *, + project: CompiledProject, + profile: ExpressionInferenceProfile, + binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], + resource_sql_analysis: bool, +) -> CompiledProject | None: + """Return the completed project, or None where Python must complete its diagnostics.""" + + return None diff --git a/src/sqlbuild/compiler/sql_test_glue/__init__.py b/src/sqlbuild/compiler/sql_test_glue/__init__.py new file mode 100644 index 0000000000..6e5850e4af --- /dev/null +++ b/src/sqlbuild/compiler/sql_test_glue/__init__.py @@ -0,0 +1 @@ +"""Native SQL test planning glue for the preview compiler engine.""" diff --git a/src/sqlbuild/compiler/sql_test_glue/main/__init__.py b/src/sqlbuild/compiler/sql_test_glue/main/__init__.py new file mode 100644 index 0000000000..e1bc28b0f1 --- /dev/null +++ b/src/sqlbuild/compiler/sql_test_glue/main/__init__.py @@ -0,0 +1 @@ +"""Native SQL test planning glue entry points.""" diff --git a/src/sqlbuild/compiler/sql_test_glue/main/_assemble_native_sql_tests.py b/src/sqlbuild/compiler/sql_test_glue/main/_assemble_native_sql_tests.py new file mode 100644 index 0000000000..b732f71039 --- /dev/null +++ b/src/sqlbuild/compiler/sql_test_glue/main/_assemble_native_sql_tests.py @@ -0,0 +1,13 @@ +"""Assemble compiled SQL test facts natively for the preview compiler engine.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile.models import CompiledSqlTest, CompileProjectInputs + + +def assemble_native_sql_tests( + *, inputs: CompileProjectInputs +) -> tuple[CompiledSqlTest, ...] | None: + """Return one compiled test per test input, or None where Python must assemble the tests.""" + + return None diff --git a/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py b/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py new file mode 100644 index 0000000000..7b5861fead --- /dev/null +++ b/src/sqlbuild/compiler/sql_test_glue/main/plan_native_sql_test_artifacts.py @@ -0,0 +1,19 @@ +"""Plan and render SQL test artifacts natively for the preview compiler engine.""" + +from __future__ import annotations + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.compiler.compile.models import CompiledProject, CompiledSqlTest +from sqlbuild.compiler.planner.models import NativeSqlTestArtifact + + +def plan_native_sql_test_artifacts( + *, + project: CompiledProject, + tests: tuple[CompiledSqlTest, ...], + adapter: BaseAdapter, + sql_analysis_enabled: bool, +) -> tuple[NativeSqlTestArtifact, ...] | None: + """Return one artifact per test, or None where Python must plan the tests.""" + + return None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index d8bbf66125..14a36598f9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -812,6 +812,14 @@ class CompilerEngineParityTestCase: expected_exit_codes: tuple[int, int] +@dataclass(frozen=True) +class NativeAnalysisSeamTestCase: + """Every engine compiling a copy of one project while the analysis stage seams are recorded.""" + + description: str + expected_preview_returns: dict[str, list[object]] + + @dataclass(frozen=True) class CompilerEngineMacroParityTestCase: """Two engines compiling separate copies of one macro-heavy project.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 7efeec91b1..665e118250 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -21,12 +21,13 @@ from dataclasses import replace from io import StringIO from pathlib import Path -from types import FrameType +from types import FrameType, ModuleType from typing import Any, NamedTuple, cast import duckdb import pytest +import sqlbuild.cli.commands._helpers.compile.target_writer as target_writer import sqlbuild.cli.commands.main.project._compile as compile_command_module import sqlbuild.cli.compile_render_reuse._helpers.load_notice as render_load_notice import sqlbuild.cli.compile_render_reuse.main._render_reuse_session as render_reuse_main @@ -35,10 +36,15 @@ import sqlbuild.cli.compile_reuse._helpers.store as reuse_store import sqlbuild.compiler.compile._helpers.assembly.binding_waves as binding_waves import sqlbuild.compiler.compile._helpers.assembly.project as project_assembly +import sqlbuild.compiler.compile._helpers.diagnostics.recovery as diagnostic_recovery import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module +import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stages +import sqlbuild.compiler.compile._helpers.native_stages.sql_tests as native_sql_test_stage import sqlbuild.compiler.compile.classes.binding_dataflow as binding_dataflow import sqlbuild.compiler.compile.classes.render_reuse_session as render_reuse_session import sqlbuild.compiler.compile.classes.stored_model_analyses as stored_model_analyses +import sqlbuild.compiler.contracts.main.validate as contract_validation +import sqlbuild.compiler.lineage.main.columns as column_lineage import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class from scripts.cold_compile_performance.main.read_compile_measurement import read_compile_measurement from scripts.cold_compile_performance.main.semantic_compile_fingerprint import ( @@ -4741,3 +4747,58 @@ def engine_macro_call_runs( ) ), ) + + +_ANALYSIS_SEAMS: tuple[tuple[ModuleType, str], ...] = ( + (native_stages, "assemble_native_project"), + (native_stages, "infer_native_expression_source_shapes"), + (native_stages, "analyze_native_model_sql"), + (native_sql_test_stage, "assemble_native_sql_tests"), + (diagnostic_recovery, "complete_native_semantic_diagnostics"), + (contract_validation, "evaluate_native_model_contracts"), + (column_lineage, "build_native_column_lineage"), + (target_writer, "plan_native_sql_test_artifacts"), +) + + +def record_analysis_seams(*, monkeypatch: pytest.MonkeyPatch) -> dict[str, list[object]]: + """Record what each native analysis stage seam returns, keyed by its entry name.""" + + results: dict[str, list[object]] = {} + for module, name in _ANALYSIS_SEAMS: + monkeypatch.setattr( + module, name, _recorded_seam(entry=getattr(module, name), name=name, results=results) + ) + return results + + +def _recorded_seam( + *, entry: Callable[..., object], name: str, results: dict[str, list[object]] +) -> Callable[..., object]: + def recorded(**arguments: object) -> object: + result: object = entry(**arguments) + results.setdefault(name, []).append(result) + return result + + return recorded + + +def engine_in_process_compile( + *, + project_dir: Path, + engine: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> tuple[CompileReuseRun, dict[str, list[object]]]: + """Compile in this process under `engine`; return the run and what each seam returned.""" + + with monkeypatch.context() as patch: + for name, value in { + **COMPILE_REUSE_ENV, + COMPILER_ENGINE_ENV_VAR: engine, + REUSE_DISABLE_ENV_VAR: "1", + }.items(): + patch.setenv(name, value) + seams: dict[str, list[object]] = record_analysis_seams(monkeypatch=patch) + run: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) + return run, seams diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py new file mode 100644 index 0000000000..8cd7acd8a6 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -0,0 +1,80 @@ +"""The analysis stage seams defer to Python, so every engine compiles a project identically.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + NativeAnalysisSeamTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + CompileReuseRun, + copy_compile_project, + engine_in_process_compile, + prepare_compile_reuse_project, + report_without_engine, +) + +_ENGINES: tuple[str, ...] = ("python", "native", "native-preview") + + +@pytest.mark.parametrize( + "test_case", + [ + NativeAnalysisSeamTestCase( + description="models_contracts_lineage_and_sql_tests", + expected_preview_returns={ + "assemble_native_project": [None], + "infer_native_expression_source_shapes": [None], + "analyze_native_model_sql": [None], + "assemble_native_sql_tests": [None], + "complete_native_semantic_diagnostics": [None], + "evaluate_native_model_contracts": [None], + "build_native_column_lineage": [None], + "plan_native_sql_test_artifacts": [None], + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_project_when_compiling_with_each_engine_then_seams_defer_and_outputs_match( + test_case: NativeAnalysisSeamTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + prepared_project: Path = tmp_path / "orders" + prepare_compile_reuse_project(project_dir=prepared_project) + outcomes: dict[str, tuple[CompileReuseRun, dict[str, list[object]]]] = { + engine: engine_in_process_compile( + project_dir=copy_compile_project( + source=prepared_project, destination=tmp_path / engine + ), + engine=engine, + monkeypatch=monkeypatch, + capsys=capsys, + ) + for engine in _ENGINES + } + python_run, python_seams = outcomes["python"] + native_run, native_seams = outcomes["native"] + preview_run, preview_seams = outcomes["native-preview"] + + assert python_run.returncode == 0, python_run.stderr + assert python_run.compiled + assert (python_seams, native_seams, preview_seams) == ( + {}, + {}, + test_case.expected_preview_returns, + ) + assert (native_run.returncode, preview_run.returncode) == (0, 0) + assert report_without_engine(native_run) == report_without_engine(python_run) + assert report_without_engine(preview_run) == report_without_engine(python_run) + assert native_run.compiled == python_run.compiled + assert preview_run.compiled == python_run.compiled + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/_test_types.py new file mode 100644 index 0000000000..a49bf87317 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/_test_types.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild.compiler.compile.models import DynamicColumnContractProof, ModelSqlAnalysis +from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily + + +@dataclass(frozen=True) +class DynamicColumnContractDispatchTestCase: + description: str + sql_analysis: ModelSqlAnalysis | None + dialect: str + families: tuple[SchemaDynamicColumnFamily, ...] + expected_proof: DynamicColumnContractProof | None diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/test_dynamic_column_contract_by_engine.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/test_dynamic_column_contract_by_engine.py new file mode 100644 index 0000000000..92ff874493 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/native_stages/test_dynamic_column_contract_by_engine.py @@ -0,0 +1,101 @@ +"""A model's native analysis may carry its dynamic pivot proof; otherwise Python proves it.""" + +from __future__ import annotations + +from functools import partial + +import pytest + +from sqlbuild.compiler.compile._helpers.analysis.dynamic_pivot import ( + analyze_dynamic_column_contract, +) +from sqlbuild.compiler.compile._helpers.native_stages.assembly import ( + dynamic_column_contract_by_engine, +) +from sqlbuild.compiler.compile.models import ( + DynamicColumnContractProof, + ModelSqlAnalysis, + PolyglotAnalysisResult, +) +from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily +from tests.unit.src.sqlbuild.compiler.compile._helpers.native_stages._test_types import ( + DynamicColumnContractDispatchTestCase, +) + +_NATIVE_PROOF: DynamicColumnContractProof = DynamicColumnContractProof( + output_proven=False, failure_reason="native session proof" +) +_PYTHON_UNSUPPORTED_PROOF: DynamicColumnContractProof = DynamicColumnContractProof( + output_proven=False, + failure_reason=("adapter dialect 'bigquery' does not support compiler-proven dynamic pivots"), +) +_FAMILY: SchemaDynamicColumnFamily = SchemaDynamicColumnFamily( + name="category_amounts", + pivot_column="category", + value_column="amount", + aggregate="MAX", + type="DECIMAL(12,2)", +) +_ANALYSIS: PolyglotAnalysisResult = PolyglotAnalysisResult(analysis_succeeded=True) + + +@pytest.mark.parametrize( + "test_case", + ( + DynamicColumnContractDispatchTestCase( + description="native analysis proof replaces the python proof", + sql_analysis=ModelSqlAnalysis( + polyglot_analysis=_ANALYSIS, + placeholders=None, + dynamic_column_contract=_NATIVE_PROOF, + ), + dialect="bigquery", + families=(_FAMILY,), + expected_proof=_NATIVE_PROOF, + ), + DynamicColumnContractDispatchTestCase( + description="analysis without a proof falls back to the python proof", + sql_analysis=ModelSqlAnalysis(polyglot_analysis=_ANALYSIS, placeholders=None), + dialect="bigquery", + families=(_FAMILY,), + expected_proof=_PYTHON_UNSUPPORTED_PROOF, + ), + DynamicColumnContractDispatchTestCase( + description="unanalyzed model falls back to the python proof", + sql_analysis=None, + dialect="bigquery", + families=(_FAMILY,), + expected_proof=_PYTHON_UNSUPPORTED_PROOF, + ), + DynamicColumnContractDispatchTestCase( + description="model without dynamic families has no proof", + sql_analysis=ModelSqlAnalysis(polyglot_analysis=_ANALYSIS, placeholders=None), + dialect="snowflake", + families=(), + expected_proof=None, + ), + ), + ids=lambda case: case.description, +) +def test_given_model_analysis_when_proving_dynamic_contract_then_native_proof_takes_precedence( + test_case: DynamicColumnContractDispatchTestCase, +) -> None: + proof: DynamicColumnContractProof | None = dynamic_column_contract_by_engine( + sql_analysis=test_case.sql_analysis, + python_proof=partial( + analyze_dynamic_column_contract, + query_sql="SELECT * FROM order_amounts", + dialect=test_case.dialect, + families=test_case.families, + column_types_by_table={}, + authoritative_column_types_by_table={}, + column_nullability_by_table={}, + dynamic_families_by_table={}, + ), + ) + + assert proof == test_case.expected_proof + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py index 5a9b4953ad..8e6f7793a2 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_native_stage_tiers.py @@ -157,6 +157,132 @@ stage=NativeStage.ATTACHMENTS, expected_enabled=True, ), + NativeStageTierTestCase( + description="python_type_system", + engine=CompilerEngine.PYTHON, + stage=NativeStage.TYPE_SYSTEM, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_type_system", + engine=CompilerEngine.NATIVE, + stage=NativeStage.TYPE_SYSTEM, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_type_system", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.TYPE_SYSTEM, + expected_enabled=True, + ), + NativeStageTierTestCase( + description="python_model_analysis", + engine=CompilerEngine.PYTHON, + stage=NativeStage.MODEL_ANALYSIS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_model_analysis", + engine=CompilerEngine.NATIVE, + stage=NativeStage.MODEL_ANALYSIS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_model_analysis", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.MODEL_ANALYSIS, + expected_enabled=True, + ), + NativeStageTierTestCase( + description="python_semantic_checks", + engine=CompilerEngine.PYTHON, + stage=NativeStage.SEMANTIC_CHECKS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_semantic_checks", + engine=CompilerEngine.NATIVE, + stage=NativeStage.SEMANTIC_CHECKS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_semantic_checks", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.SEMANTIC_CHECKS, + expected_enabled=True, + ), + NativeStageTierTestCase( + description="python_contracts", + engine=CompilerEngine.PYTHON, + stage=NativeStage.CONTRACTS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_contracts", + engine=CompilerEngine.NATIVE, + stage=NativeStage.CONTRACTS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_contracts", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.CONTRACTS, + expected_enabled=True, + ), + NativeStageTierTestCase( + description="python_lineage_facts", + engine=CompilerEngine.PYTHON, + stage=NativeStage.LINEAGE_FACTS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_lineage_facts", + engine=CompilerEngine.NATIVE, + stage=NativeStage.LINEAGE_FACTS, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_lineage_facts", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.LINEAGE_FACTS, + expected_enabled=True, + ), + NativeStageTierTestCase( + description="python_sql_test_glue", + engine=CompilerEngine.PYTHON, + stage=NativeStage.SQL_TEST_GLUE, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_sql_test_glue", + engine=CompilerEngine.NATIVE, + stage=NativeStage.SQL_TEST_GLUE, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_sql_test_glue", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.SQL_TEST_GLUE, + expected_enabled=True, + ), + NativeStageTierTestCase( + description="python_project_assembly", + engine=CompilerEngine.PYTHON, + stage=NativeStage.PROJECT_ASSEMBLY, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_project_assembly", + engine=CompilerEngine.NATIVE, + stage=NativeStage.PROJECT_ASSEMBLY, + expected_enabled=False, + ), + NativeStageTierTestCase( + description="native_preview_project_assembly", + engine=CompilerEngine.NATIVE_PREVIEW, + stage=NativeStage.PROJECT_ASSEMBLY, + expected_enabled=True, + ), ], ids=lambda case: case.description, )