perf: speed up warm compiles and upgrade Polyglot to 0.12.2

Cargo.toml

@@ -15,7 +15,7 @@ fensu-policy = "=0.22.0"
 globset = "0.4"
 proc-macro2 = { version = "1", features = ["span-locations"] }
 pyo3 = { version = "0.26", features = ["abi3-py312"] }
-polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "cd599007a1ba6c4cc1f2d99938fc0e6d3b376a9b", default-features = false, features = [
+polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "dd7154035566933a0f9acae17a2c102afb4dbfde", default-features = false, features = [
     "ast-tools",
     "builder",
     "dialect-bigquery",

crates/sqlbuild-rules-native/src/query_analysis/_helpers/engine.rs

@@ -122,6 +122,8 @@ struct ProjectAnalysis {
     has_star: bool,
     #[serde(skip)]
     requires_legacy_fallback: bool,
+    #[serde(skip)]
+    preserve_fallback_lineage: bool,
 }
 
 struct ProjectAnalysisInputs<'a> {
@@ -611,19 +613,18 @@ fn try_borrowed_query(
                 upstream_columns,
             });
         }
+        let requires_legacy_fallback = columns.iter().any(|column| column.data_type.is_none());
         projections.push((
             *index,
             Ok(ProjectAnalysis {
                 columns,
                 lineage_columns,
                 has_star: false,
-                requires_legacy_fallback: false,
+                requires_legacy_fallback,
+                preserve_fallback_lineage: true,
             }),
         ));
     }
-    if projections.iter().any(|(_, result)| matches!(result, Ok(analysis) if analysis.columns.iter().any(|column| column.data_type.is_none()))) {
-        return Err(Box::new(work));
-    }
     Ok(CompiledQueryWorkResult {
         projections,
         validation: if work.query.binding_schema.is_some() {
@@ -933,6 +934,7 @@ struct CompactProjectBatch {
 #[serde(untagged)]
 enum CompactProjectResponse {
     Success((Vec<usize>, bool)),
+    LegacyTypeRecovery((Vec<usize>, bool, &'static str)),
     Failure(String),
 }
 
@@ -985,7 +987,7 @@ impl CompactProjectAccumulator {
                 return Ok(());
             }
         };
-        if analysis.requires_legacy_fallback {
+        if analysis.requires_legacy_fallback && !analysis.preserve_fallback_lineage {
             self.set_template(
                 projection_index,
                 CompactProjectResponse::Failure(
@@ -1024,7 +1026,15 @@ impl CompactProjectAccumulator {
         }
         self.set_template(
             projection_index,
-            CompactProjectResponse::Success((columns, analysis.has_star)),
+            if analysis.requires_legacy_fallback {
+                CompactProjectResponse::LegacyTypeRecovery((
+                    columns,
+                    analysis.has_star,
+                    "native project type recovery requires legacy fallback",
+                ))
+            } else {
+                CompactProjectResponse::Success((columns, analysis.has_star))
+            },
         )
     }
 
@@ -1194,6 +1204,7 @@ fn project_analysis(inputs: ProjectAnalysisInputs<'_>) -> ProjectAnalysis {
         lineage_columns,
         has_star: inputs.analysis.has_root_star,
         requires_legacy_fallback,
+        preserve_fallback_lineage: false,
     }
 }
 

crates/sqlbuild-rules-native/src/query_analysis/tests/helpers.rs

@@ -164,6 +164,38 @@ pub(crate) fn binding_warnings_preserve_declared_relationship_evidence() -> bool
     true
 }
 
+pub(crate) fn bound_lineage_survives_type_recovery() -> bool {
+    for operator in ["UNION ALL", "UNION ALL BY NAME"] {
+        let response: Value = serde_json::from_str(
+            &analyze_project_compact_json(&json!({
+                "queries": [{
+                    "sql": format!("WITH combined AS (SELECT CAST(amount AS DOUBLE) AS amount FROM orders {operator} SELECT list_extract([amount], 1) AS amount FROM refunds) SELECT amount FROM combined"),
+                    "dialect": "duckdb",
+                    "schema": {"strict": true, "tables": [
+                        {"name": "orders", "columns": [{"name": "amount", "type": "DOUBLE"}]},
+                        {"name": "refunds", "columns": [{"name": "amount", "type": "DOUBLE"}]}
+                    ]}
+                }],
+                "templates": [{"queryIndex": 0, "recoverCteFacts": true, "richTypeInference": false, "references": {
+                    "orders": {"resourceType": "model", "resourceName": "orders"},
+                    "refunds": {"resourceType": "model", "resourceName": "refunds"}
+                }}],
+                "projections": [{"templateIndex": 0}]
+            }).to_string()).expect("query should analyze"),
+        ).expect("response should decode");
+        let index = response["templates"][0][0][0]
+            .as_u64()
+            .expect("successful fact") as usize;
+        let fact = &response["facts"][index];
+        assert_eq!(
+            response["templates"][0][2],
+            "native project type recovery requires legacy fallback"
+        );
+        assert_eq!(fact[5].as_array().expect("upstream columns").len(), 2);
+    }
+    true
+}
+
 pub(crate) fn widening_aggregates_require_compatibility_recovery() -> bool {
     for expression in [
         "AVG(order_id)",

crates/sqlbuild-rules-native/src/query_analysis/tests/test_query_analysis.rs

@@ -4,7 +4,7 @@ use crate::query_analysis::tests::helpers::{
     binding_warnings_preserve_declared_relationship_evidence,
     borrowed_facts_preserve_named_outputs_and_terminal_sources,
     borrowed_facts_preserve_union_dependencies_and_join_nullability,
-    canonical_queries_reuse_semantics_and_project_resources,
+    bound_lineage_survives_type_recovery, canonical_queries_reuse_semantics_and_project_resources,
     combined_queries_preserve_standalone_binding,
     compact_project_query_interns_repeated_lineage_strings,
     interleaved_query_templates_preserve_template_order,
@@ -56,6 +56,11 @@ fn given_compact_query_cases_when_analyzing_projects_then_expected_behavior_hold
             run: widening_aggregates_require_compatibility_recovery,
             expected_success: true,
         },
+        CompactQueryAnalysisTestCase {
+            description: "resolved bound lineage survives type recovery",
+            run: bound_lineage_survives_type_recovery,
+            expected_success: true,
+        },
         CompactQueryAnalysisTestCase {
             description: "compact project queries intern repeated lineage strings",
             run: compact_project_query_interns_repeated_lineage_strings,

pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
   "duckdb>=1.3.2",
   "filelock>=3.20.3",
   "orjson>=3.10.0",
-  "polyglot-sql-chio==0.12.1",
+  "polyglot-sql-chio==0.12.2",
   "pydantic-settings>=2,<3",
   "PyYAML>=6.0.2",
   "rich>=15.0.0",

src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py

@@ -10,6 +10,8 @@
 from pathlib import Path
 from typing import cast
 
+import orjson
+
 from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter
 from sqlbuild.cli.compile.models import (
     SqlTestArtifactCacheRecord,
@@ -25,7 +27,7 @@
 )
 
 _CACHE_VERSION: int = 3
-_ALGORITHM_FINGERPRINT: str = "sql-test-artifact-v3-polyglot-scope-fix"
+_ALGORITHM_FINGERPRINT: str = "sql-test-artifact-v4-orjson-identity"
 _CACHE_FILE_NAME: str = "sql-test-artifacts.json"
 _MAX_CACHE_BYTES: int = 10_000_000
 _MAX_CACHE_RECORDS: int = 100_000
@@ -276,8 +278,7 @@ def _is_safe_relative_sql_path(path: Path) -> bool:
 
 
 def _payload_digest(payload: object) -> str:
-    encoded: str = json.dumps(payload, ensure_ascii=True, separators=(",", ":"), sort_keys=True)
-    return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+    return hashlib.sha256(orjson.dumps(payload, option=orjson.OPT_SORT_KEYS)).hexdigest()
 
 
 def _package_version(package_name: str) -> str:

src/sqlbuild/cli/commands/_helpers/compile/target_writer.py

@@ -499,20 +499,20 @@ def _remove_stale_compiled_files(*, target_dir: Path, managed_paths: set[Path])
     compiled_dir: Path = target_dir / _COMPILED_DIR
     if not compiled_dir.is_dir():
         return
+    managed_names: set[str] = {os.fspath(path) for path in managed_paths}
     for root, directories, filenames in os.walk(compiled_dir, topdown=False):
-        root_path: Path = Path(root)
         for filename in filenames:
-            path: Path = root_path / filename
-            if path not in managed_paths:
-                path.unlink()
+            path: str = os.path.join(root, filename)
+            if path not in managed_names:
+                os.unlink(path)
         for name in directories:
-            _remove_empty_directory(root_path / name)
+            _remove_empty_directory(os.path.join(root, name))
     _remove_empty_directory(compiled_dir)
 
 
-def _remove_empty_directory(directory: Path) -> None:
+def _remove_empty_directory(directory: str | Path) -> None:
     try:
-        directory.rmdir()
+        os.rmdir(directory)
     except OSError:
         pass