fix: use upstream Polyglot parser

pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
   "duckdb>=1.3.2",
   "filelock>=3.20.3",
   "orjson>=3.10.0",
-  "polyglot-sql-chio==0.9.3",
+  "polyglot-sql==0.11.0",
   "pydantic-settings>=2,<3",
   "PyYAML>=6.0.2",
   "rich>=15.0.0",

scripts/rules_benchmark/_helpers/workflow.py

@@ -1099,7 +1099,7 @@ def _hardware_payload() -> dict[str, object]:
 def _version_payload() -> dict[str, str]:
     return {
         "sqlbuild": importlib.metadata.version("sqlbuild"),
-        "polyglot_sql": importlib.metadata.version("polyglot-sql-chio"),
+        "polyglot_sql": importlib.metadata.version("polyglot-sql"),
     }
 
 

src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py

@@ -42,7 +42,7 @@ def build_sql_test_artifact_identity_context(
         {
             "algorithm": _ALGORITHM_FINGERPRINT,
             "sqlbuild_version": _package_version("sqlbuild"),
-            "polyglot_version": _package_version("polyglot-sql-chio"),
+            "polyglot_version": _package_version("polyglot-sql"),
             "python_version": platform.python_version_tuple()[:2],
             "adapter_class": f"{type(adapter).__module__}.{type(adapter).__qualname__}",
             "set_difference_operator": adapter.render_set_difference_operator(),

src/sqlbuild/compiler/compile/_helpers/analysis/cache.py

@@ -85,7 +85,7 @@ def build_analysis_cache_context(
         "algorithm": _ANALYSIS_ALGORITHM_FINGERPRINT,
         "cache_version": _ANALYSIS_CACHE_VERSION,
         "sqlbuild_version": _package_version("sqlbuild"),
-        "polyglot_version": _package_version("polyglot-sql-chio"),
+        "polyglot_version": _package_version("polyglot-sql"),
         "python_version": platform.python_version_tuple()[:2],
         "allow_compact_analysis": allow_compact_analysis,
         "inference_profile": profile_payload,

src/sqlbuild/compiler/compile/_helpers/analysis/columns.py

@@ -10,8 +10,8 @@
 from sqlbuild.adapter.contract.models import ExpressionInferenceProfile
 from sqlbuild.adapter.contract.types import FunctionNullabilityRule
 from sqlbuild.compiler.compile._helpers.analysis.cte_facts import (
+    _polyglot_cte_passthrough_facts,
     _polyglot_cte_passthrough_nullability_from_parsed,
-    _polyglot_cte_passthrough_type_facts,
     _polyglot_cte_passthrough_types_from_parsed,
 )
 from sqlbuild.compiler.compile.constants import (
@@ -27,6 +27,7 @@
     CompiledLineageColumnFact,
     CompiledLineageSourceFact,
     CompileSqlReference,
+    CteFactResolvers,
     InferredColumn,
     PolyglotAnalysisResult,
 )
@@ -233,6 +234,9 @@
     TIMESTAMP_WITH_TIME_ZONE_SQL_TYPE_NAME as _TIMESTAMP_WITH_TIME_ZONE_SQL_TYPE_NAME,
 )
 from sqlbuild.compiler.sql_analysis.main._find_matching_paren import find_matching_paren
+from sqlbuild.compiler.sql_analysis.main._normalize_for_polyglot import (
+    normalize_sql_for_polyglot,
+)
 from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations
 from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql
 from sqlbuild.compiler.sql_analysis.models import (
@@ -268,7 +272,10 @@ def infer_columns_with_sql_analysis(
 
     profile: ExpressionInferenceProfile = inference_profile or ExpressionInferenceProfile()
 
-    cleaned_sql: str = _replace_refs_with_stubs(query_sql)
+    cleaned_sql: str = _replace_refs_with_stubs(
+        query_sql=query_sql,
+        dialect=profile.sql_analysis_dialect,
+    )
     if placeholders:
         cleaned_sql = substitute_placeholder_defaults(
             query_sql=cleaned_sql, placeholders=placeholders
@@ -295,7 +302,10 @@ def analyze_columns_with_polyglot(
     """Infer columns with one Polyglot parse, returning False when unavailable."""
 
     profile: ExpressionInferenceProfile = inference_profile or ExpressionInferenceProfile()
-    cleaned_sql: str = _replace_refs_with_stubs(query_sql)
+    cleaned_sql: str = _replace_refs_with_stubs(
+        query_sql=query_sql,
+        dialect=profile.sql_analysis_dialect,
+    )
     if placeholders:
         cleaned_sql = substitute_placeholder_defaults(
             query_sql=cleaned_sql, placeholders=placeholders
@@ -324,7 +334,10 @@ def analyze_columns_and_lineage_with_polyglot(
 
     polyglot_module: Any = import_polyglot_sql()
     profile: ExpressionInferenceProfile = inference_profile or ExpressionInferenceProfile()
-    cleaned_sql: str = _replace_refs_with_stubs(query_sql)
+    cleaned_sql: str = _replace_refs_with_stubs(
+        query_sql=query_sql,
+        dialect=profile.sql_analysis_dialect,
+    )
     if placeholders:
         cleaned_sql = substitute_placeholder_defaults(
             query_sql=cleaned_sql, placeholders=placeholders
@@ -398,7 +411,7 @@ def get_complete_schema_binding_request(
 ) -> SqlSchemaValidationRequest:
     """Build one stable native schema-validation request."""
 
-    cleaned_sql: str = _replace_refs_with_stubs(query_sql)
+    cleaned_sql: str = _replace_refs_with_stubs(query_sql=query_sql, dialect=dialect)
     if placeholders:
         cleaned_sql = substitute_placeholder_defaults(
             query_sql=cleaned_sql,
@@ -470,18 +483,24 @@ def _analyze_columns_and_lineage_with_compact_polyglot(
         return None
     reference_map: dict[str, tuple[CompiledResourceType, str]] = _lineage_reference_map(references)
     relation_alias_by_name: dict[str, str | None] = _compact_relation_alias_by_name(analysis)
-    cte_passthrough_types, direct_cte_outputs = (
-        _polyglot_cte_passthrough_type_facts(
+    cte_passthrough_types, cte_passthrough_nullability, direct_cte_outputs = (
+        _polyglot_cte_passthrough_facts(
             polyglot_module=polyglot_module,
             cleaned_sql=cleaned_sql,
             dialect=dialect,
             column_types_by_table=column_types_by_table,
+            column_nullability_by_table=column_nullability_by_table,
             inference_profile=inference_profile,
             analysis=analysis,
-            expression_type_resolver=_polyglot_expression_type,
+            resolvers=CteFactResolvers(
+                expression_type=_polyglot_expression_type,
+                nullability=_infer_polyglot_nullability,
+                shallow_nullability=_infer_polyglot_shallow_nullability,
+                alias_nullability=_polyglot_alias_nullability_from_select,
+            ),
         )
         if recover_cte_facts
-        else ({}, frozenset())
+        else ({}, {}, frozenset())
     )
     columns: list[InferredColumn] = []
     lineage_columns: list[CompiledLineageColumnFact] = []
@@ -507,9 +526,13 @@ def _analyze_columns_and_lineage_with_compact_polyglot(
                         projection=projection, inference_profile=inference_profile
                     )
                 ),
-                nullability=_compact_projection_nullability(
-                    projection=projection,
-                    infer_nullability=infer_nullability,
+                nullability=(
+                    cte_passthrough_nullability.get(output_column)
+                    if output_column in direct_cte_outputs
+                    else None
+                )
+                or _compact_projection_nullability(
+                    projection=projection, infer_nullability=infer_nullability
                 ),
             )
         )
@@ -1578,7 +1601,7 @@ def _replacer(match: re.Match[str]) -> str:
     return _PLACEHOLDER_PATTERN.sub(_replacer, query_sql)
 
 
-def _replace_refs_with_stubs(query_sql: str) -> str:
+def _replace_refs_with_stubs(*, query_sql: str, dialect: str | None = None) -> str:
     """Replace SQLBuild marker calls with parseable SQL stubs."""
 
     result: str = _REF_PATTERN.sub(r"\1", query_sql)
@@ -1587,7 +1610,7 @@ def _replace_refs_with_stubs(query_sql: str) -> str:
     result = _DBT_REF_PATTERN.sub(r"\1", result)
     result = _UDF_PATTERN.sub(r"__sqlbuild_udf_\1", result)
     result = _replace_table_function_calls_with_stubs(result)
-    return result
+    return normalize_sql_for_polyglot(sql=result, dialect=dialect)
 
 
 def _replace_table_function_calls_with_stubs(query_sql: str) -> str:

src/sqlbuild/compiler/compile/_helpers/analysis/cte_facts.py

@@ -9,6 +9,7 @@
 from sqlbuild.adapter.type_system.main.normalize_type import normalize_type
 from sqlbuild.adapter.type_system.main.types_equal import types_equal
 from sqlbuild.compiler.compile.constants import SQL_WILDCARD_TOKEN
+from sqlbuild.compiler.compile.models import CteFactResolvers
 from sqlbuild.compiler.lineage.types import InferredNullability
 from sqlbuild.compiler.sql_analysis.constants import (
     POLYGLOT_BINARY_OPERAND_COUNT as _POLYGLOT_BINARY_OPERAND_COUNT,
@@ -97,36 +98,57 @@ def __call__(
     ) -> dict[str, InferredNullability]: ...
 
 
-def _polyglot_cte_passthrough_type_facts(
+def _polyglot_cte_passthrough_facts(
     *,
     polyglot_module: Any,
     cleaned_sql: str,
     dialect: str | None,
     column_types_by_table: dict[str, dict[str, str]],
+    column_nullability_by_table: dict[str, dict[str, InferredNullability]],
     inference_profile: ExpressionInferenceProfile,
     analysis: dict[str, Any],
-    expression_type_resolver: _ExpressionTypeResolver,
-) -> tuple[dict[str, str], frozenset[str]]:
-    """Recover types that compact analysis incorrectly skips across direct CTE reads."""
+    resolvers: CteFactResolvers,
+) -> tuple[dict[str, str], dict[str, InferredNullability], frozenset[str]]:
+    """Recover facts that compact analysis skips across direct CTE reads."""
 
-    if not column_types_by_table or not analysis.get("cteFacts"):
-        return {}, frozenset()
+    if not analysis.get("cteFacts") or not (column_types_by_table or column_nullability_by_table):
+        return {}, {}, frozenset()
     try:
         parsed: Any = polyglot_module.parse_one(cleaned_sql, dialect=dialect or "generic")
     except polyglot_module.PolyglotError:
-        return {}, frozenset()
+        return {}, {}, frozenset()
     ctes: tuple[tuple[str, Any, bool], ...] = _polyglot_top_level_ctes(parsed)
     if not ctes:
-        return {}, frozenset()
-    inferred_types: dict[str, str] = _polyglot_cte_passthrough_types_from_parsed(
-        parsed=parsed,
-        column_types_by_table=column_types_by_table,
-        inference_profile=inference_profile,
-        expression_type_resolver=expression_type_resolver,
+        return {}, {}, frozenset()
+    inferred_types: dict[str, str] = (
+        _polyglot_cte_passthrough_types_from_parsed(
+            parsed=parsed,
+            column_types_by_table=column_types_by_table,
+            inference_profile=inference_profile,
+            expression_type_resolver=resolvers.expression_type,
+        )
+        if column_types_by_table
+        else {}
     )
-    return inferred_types, _polyglot_direct_cte_output_names(
-        parsed=parsed,
-        cte_names=frozenset(name for name, _, _ in ctes),
+    inferred_nullability: dict[str, InferredNullability] = (
+        _polyglot_cte_passthrough_nullability_from_parsed(
+            parsed=parsed,
+            column_nullability_by_table=column_nullability_by_table,
+            inference_profile=inference_profile,
+            nullability_resolver=resolvers.nullability,
+            shallow_nullability_resolver=resolvers.shallow_nullability,
+            alias_nullability_resolver=resolvers.alias_nullability,
+        )
+        if column_nullability_by_table
+        else {}
+    )
+    return (
+        inferred_types,
+        inferred_nullability,
+        _polyglot_direct_cte_output_names(
+            parsed=parsed,
+            cte_names=frozenset(name for name, _, _ in ctes),
+        ),
     )
 
 

src/sqlbuild/compiler/compile/_helpers/analysis/required_columns.py

@@ -71,7 +71,7 @@ def resolve_required_external_columns(
     polyglot_module: Any = import_polyglot_sql()
     try:
         parsed: Any = polyglot_module.parse_one(
-            _replace_refs_with_stubs(query_sql),
+            _replace_refs_with_stubs(query_sql=query_sql, dialect=dialect),
             dialect=dialect or "generic",
         )
     except polyglot_module.PolyglotError:

src/sqlbuild/compiler/compile/_helpers/analysis/validation.py

@@ -24,7 +24,7 @@ def validate_sql_syntax(
 ) -> None:
     """Validate that the model query SQL is parseable by Polyglot."""
 
-    cleaned_sql: str = _replace_refs_with_stubs(query_sql)
+    cleaned_sql: str = _replace_refs_with_stubs(query_sql=query_sql, dialect=dialect)
     if placeholders:
         cleaned_sql = substitute_placeholder_defaults(
             query_sql=cleaned_sql, placeholders=placeholders
@@ -171,6 +171,7 @@ def _validate_sql_syntax_with_message(
     cleaned_sql: str = _clean_sql_for_validation(
         query_sql=query_sql,
         placeholders=placeholders,
+        dialect=dialect,
     )
     polyglot_module: Any = import_polyglot_sql()
     try:
@@ -194,14 +195,20 @@ def _validate_hook_sql_with_message(
     cleaned_sql: str = _clean_sql_for_validation(
         query_sql=query_sql,
         placeholders=placeholders,
+        dialect=dialect,
     )
     error_message: str | None = _validate_sql_with_polyglot(sql=cleaned_sql, dialect=dialect)
     if error_message is not None:
         _raise_sql_validation_error(error_prefix=error_prefix, error_message=error_message)
 
 
-def _clean_sql_for_validation(*, query_sql: str, placeholders: dict[str, str] | None) -> str:
-    cleaned_sql: str = _replace_refs_with_stubs(query_sql)
+def _clean_sql_for_validation(
+    *,
+    query_sql: str,
+    placeholders: dict[str, str] | None,
+    dialect: str | None,
+) -> str:
+    cleaned_sql: str = _replace_refs_with_stubs(query_sql=query_sql, dialect=dialect)
     if placeholders:
         return substitute_placeholder_defaults(
             query_sql=cleaned_sql,

src/sqlbuild/compiler/compile/models.py

@@ -5,6 +5,7 @@
 from collections.abc import Callable, Mapping
 from dataclasses import dataclass, field
 from pathlib import Path
+from typing import Any
 
 from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds
 from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity
@@ -143,6 +144,16 @@ class InferredColumn:
     nullability: InferredNullability = InferredNullability.UNKNOWN
 
 
+@dataclass(frozen=True)
+class CteFactResolvers:
+    """Expression resolvers used for conservative CTE fact recovery."""
+
+    expression_type: Callable[..., Any]
+    nullability: Callable[..., Any]
+    shallow_nullability: Callable[..., Any]
+    alias_nullability: Callable[..., Any]
+
+
 @dataclass(frozen=True)
 class CompileModelConfig:
     """Pre-semantic effective config layers attached to a model input."""