perf: cut serial compile work and duplicated parsing on dense projects

crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs

@@ -165,6 +165,7 @@ impl SessionCatalog {
         let mut shared: Vec<Option<SharedQuery>> = self.shared_queries(batch, &cleaned)?;
         self.shared_members += shared.iter().flatten().count();
         let mut query_digests: HashMap<&str, MemoKey> = HashMap::new();
+        let batch_digest: MemoKey = batch_key_digest(batch);
         let keys: Vec<Option<MemoKey>> = batch
             .members
             .iter()
@@ -174,7 +175,12 @@ impl SessionCatalog {
                 let query: MemoKey = *query_digests
                     .entry(share.key.as_str())
                     .or_insert_with(|| Sha256::digest(share.key.as_bytes()).into());
-                Some(shared_result_key(batch, member, share, &query))
+                Some(shared_result_key(
+                    batch,
+                    member,
+                    share,
+                    (&query, &batch_digest),
+                ))
             })
             .collect();
         for (member, sql) in batch.members.iter().zip(&cleaned) {
@@ -609,21 +615,23 @@ impl SessionCatalog {
                 }
             };
             query_indexes.push(query_index);
-            let template: Value = json!({
-                "queryIndex": query_index,
-                "references": lineage_resources(member, share.as_ref()),
-                "functionReturnTypes": function_return_types,
-                "declaredColumnOrder": declared_column_order(member, batch.nullability),
-                "recoverCteFacts": member.recover_cte_facts,
-                "richTypeInference": batch.rich_type_inference,
-            });
-            let template_index: usize =
-                *template_keys
-                    .entry(template.to_string())
-                    .or_insert_with(|| {
-                        templates.push(template);
-                        templates.len() - 1
-                    });
+            let references: Map<String, Value> = lineage_resources(member, share.as_ref());
+            let declared: Value = json!(declared_column_order(member, batch.nullability));
+            let member_template_identity: String =
+                json!([query_index, references, declared, member.recover_cte_facts]).to_string();
+            let template_index: usize = *template_keys
+                .entry(member_template_identity)
+                .or_insert_with(|| {
+                    templates.push(json!({
+                        "queryIndex": query_index,
+                        "references": references,
+                        "functionReturnTypes": function_return_types,
+                        "declaredColumnOrder": declared,
+                        "recoverCteFacts": member.recover_cte_facts,
+                        "richTypeInference": batch.rich_type_inference,
+                    }));
+                    templates.len() - 1
+                });
             let resource_names: Map<String, Value> = member
                 .lineage_references
                 .iter()
@@ -897,12 +905,19 @@ fn named_sources(
         .collect()
 }
 
-/// Digest of Python's `shared_result_keys` entry, given its shared query's digest.
+/// Digest of the settings every member of `batch` shares, hashed once per batch.
+fn batch_key_digest(batch: &Batch<'_>) -> MemoKey {
+    let shared: String =
+        json!([batch.rich_type_inference, batch.function_return_types]).to_string();
+    Sha256::digest(shared.as_bytes()).into()
+}
+
+/// Digest of Python's `shared_result_keys` entry, given its shared query's and batch's digests.
 fn shared_result_key(
     batch: &Batch<'_>,
     member: &BatchMember<'_>,
     share: &SharedQuery,
-    query: &MemoKey,
+    (query, batch_digest): (&MemoKey, &MemoKey),
 ) -> MemoKey {
     let reference_types: Vec<(&str, &str)> = member
         .lineage_references
@@ -911,14 +926,13 @@ fn shared_result_key(
         .collect();
     let rest: String = json!([
         member.recover_cte_facts,
-        batch.rich_type_inference,
-        batch.function_return_types,
         declared_column_order(member, batch.nullability),
         reference_types,
     ])
     .to_string();
     let mut hasher: Sha256 = Sha256::new();
     hasher.update(query);
+    hasher.update(batch_digest);
     hasher.update(rest.as_bytes());
     hasher.finalize().into()
 }

crates/sqlbuild-analysis/src/compiled_project/models.rs

@@ -1,6 +1,7 @@
 //! Facts a compile's producers retain about each resource, in production order.
 
 use std::collections::{BTreeMap, HashMap};
+use std::sync::Arc;
 
 use serde_json::Value;
 
@@ -17,11 +18,11 @@ pub struct CompiledModelFacts {
     pub name: String,
     pub relative_path: String,
     /// SQL after variable, declaration and macro expansion.
-    pub query_sql: String,
+    pub query_sql: Arc<str>,
     /// The authored query below the `MODEL (...)` header.
-    pub authored_query_sql: String,
+    pub authored_query_sql: Arc<str>,
     /// The whole authored file.
-    pub authored_sql: String,
+    pub authored_sql: Arc<str>,
     pub config: ModelConfigFacts,
     pub references: Vec<ModelReferenceFact>,
     pub schema: Option<ModelSchemaFact>,
@@ -210,9 +211,9 @@ pub struct SplicedSql {
 pub struct PlainModel {
     pub name: String,
     pub relative_path: String,
-    pub query_sql: String,
-    pub authored_query_sql: String,
-    pub authored_sql: String,
+    pub query_sql: Arc<str>,
+    pub authored_query_sql: Arc<str>,
+    pub authored_sql: Arc<str>,
     pub config: ModelConfigFacts,
     /// The attached schema entry; present when the header declares description, columns or audits.
     pub schema: Option<ModelSchemaFact>,

crates/sqlbuild-analysis/src/compiled_project/tests/helpers.rs

@@ -3,7 +3,7 @@ use crate::compiled_project::models::{CompiledModelFacts, ModelAnalysisFact};
 pub(super) fn model(name: &str, query_sql: &str) -> CompiledModelFacts {
     CompiledModelFacts {
         name: name.to_owned(),
-        query_sql: query_sql.to_owned(),
+        query_sql: query_sql.into(),
         ..CompiledModelFacts::default()
     }
 }

crates/sqlbuild-analysis/src/compiled_project/tests/test_store.rs

@@ -51,7 +51,7 @@ fn given_recorded_models_when_reading_then_order_and_latest_facts_are_kept() {
             .map(|facts| facts.name.as_str())
             .collect();
         let query_sql: Option<&str> =
-            model_facts(&project, "orders").map(|facts| facts.query_sql.as_str());
+            model_facts(&project, "orders").map(|facts| &*facts.query_sql);
 
         assert_eq!(order, test_case.expected_order, "{}", test_case.description);
         assert_eq!(

crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/constants.rs

@@ -0,0 +1,2 @@
+pub(crate) const SIMPLE_TEMPLATE_DIALECTS: [&str; 2] = ["duckdb", "snowflake"];
+pub(crate) const MAX_TEMPLATE_DEPTH: usize = 32;

crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs

@@ -2,6 +2,7 @@ pub mod assembly;
 pub mod authored_macro_calls;
 pub mod case_identity;
 pub(crate) mod check_independence;
+pub(crate) mod constants;
 pub mod cte_rename;
 pub mod cte_slices;
 pub mod cte_sql;
@@ -19,3 +20,4 @@ pub mod reference_calls;
 pub mod relation_markers;
 pub mod rendering;
 pub mod scenario_extraction;
+pub(crate) mod simple_template;