perf(snowflake): cut per-model statement overhead, and write a Rules-failed compile to target/compiled-failed

crates/sqlbuild-cache/src/project_reuse/_helpers/target.rs

@@ -4,6 +4,7 @@ use std::collections::{BTreeMap, HashMap};
 use std::path::{Path, PathBuf};
 
 use crate::project_reuse::_helpers::files::file_digest;
+use crate::project_reuse::constants::FAILED_COMPILED_DIRECTORY_NAME;
 use crate::project_reuse::models::{CompileRecord, RecordedArtifact, ReuseRules};
 use crate::project_snapshot::main::stamp_path::stamp_path;
 use crate::project_snapshot::models::PathStamp;
@@ -20,10 +21,18 @@ fn compiled_root(project_dir: &Path, rules: &ReuseRules) -> PathBuf {
     project_dir.join(&rules.compiled_root)
 }
 
-/// Stat every compiled artifact plus `extra_paths`, keyed by path.
+/// The failed-compile directory beside the compiled directory.
+fn failed_root(project_dir: &Path, rules: &ReuseRules) -> PathBuf {
+    compiled_root(project_dir, rules).with_file_name(FAILED_COMPILED_DIRECTORY_NAME)
+}
+
+/// Stat every compiled and failed-compile artifact plus `extra_paths`, keyed by path.
 fn snapshot_target_files(project_dir: &Path, extra_paths: &[&str], rules: &ReuseRules) -> Stamps {
     let mut stamps: Stamps = BTreeMap::new();
-    let mut pending: Vec<PathBuf> = vec![compiled_root(project_dir, rules)];
+    let mut pending: Vec<PathBuf> = vec![
+        compiled_root(project_dir, rules),
+        failed_root(project_dir, rules),
+    ];
     while let Some(directory) = pending.pop() {
         let Ok(entries) = std::fs::read_dir(&directory) else {
             continue;
@@ -58,21 +67,29 @@ fn target_stamps(project_dir: &Path, paths: &[&str], tree: bool, rules: &ReuseRu
     if !tree {
         return paths.iter().copied().filter_map(keyed_stamp).collect();
     }
-    let prefix: String = compiled_prefix(project_dir, rules);
+    let prefixes: [String; 2] = tree_prefixes(project_dir, rules);
     let extra: Vec<&str> = paths
         .iter()
         .copied()
-        .filter(|path| !path.starts_with(prefix.as_str()))
+        .filter(|path| !in_trees(path, &prefixes))
         .collect();
     snapshot_target_files(project_dir, &extra, rules)
 }
 
-fn compiled_prefix(project_dir: &Path, rules: &ReuseRules) -> String {
-    format!(
-        "{}{}",
-        compiled_root(project_dir, rules).to_string_lossy(),
-        std::path::MAIN_SEPARATOR
-    )
+/// The compiled and failed-compile directories, each with a trailing separator.
+fn tree_prefixes(project_dir: &Path, rules: &ReuseRules) -> [String; 2] {
+    [
+        compiled_root(project_dir, rules),
+        failed_root(project_dir, rules),
+    ]
+    .map(|root| format!("{}{}", root.to_string_lossy(), std::path::MAIN_SEPARATOR))
+}
+
+/// Whether `path` is under one of the `prefixes` directories.
+fn in_trees(path: &str, prefixes: &[String]) -> bool {
+    prefixes
+        .iter()
+        .any(|prefix| path.starts_with(prefix.as_str()))
 }
 
 /// Whether `stamp` shows what this compile left there, and whether that read its content.
@@ -123,13 +140,13 @@ pub(crate) fn verified_target_files(
     since_ns: i64,
     rules: &ReuseRules,
 ) -> Option<Vec<PathStamp>> {
-    let prefix: String = compiled_prefix(project_dir, rules);
+    let prefixes: [String; 2] = tree_prefixes(project_dir, rules);
     let dag_path: Option<&str> = record.dag_path.as_deref();
     let written: bool = record.artifacts_written;
     let expected: BTreeMap<&str, &RecordedArtifact> = record
         .artifacts
         .iter()
-        .filter(|(path, _)| path.starts_with(prefix.as_str()) || Some(path.as_str()) == dag_path)
+        .filter(|(path, _)| in_trees(path, &prefixes) || Some(path.as_str()) == dag_path)
         .map(|(path, artifact)| (path.as_str(), artifact))
         .collect();
     let paths: Vec<&str> = expected.keys().copied().collect();

crates/sqlbuild-cache/src/project_reuse/constants.rs

@@ -8,3 +8,5 @@ pub const REUSE_IMAGE_SUFFIX: &str = "image";
 pub const REUSE_MAX_STORED_IMAGES: usize = 2;
 /// Prefix of digests computed natively, so they never equal a Python digest.
 pub const NATIVE_DIGEST_PREFIX: &str = "b3:";
+/// The sibling of the compiled directory that holds a compile whose Rules failed.
+pub const FAILED_COMPILED_DIRECTORY_NAME: &str = "compiled-failed";

crates/sqlbuild-core/src/cli_text/main/format_compile_text.rs

@@ -83,6 +83,9 @@ pub fn format_compile_text(report: &CompileTextReport) -> String {
         "Project compiled",
         &summary,
     ));
+    for path in &report.failed_model_sql {
+        lines.push(format!("  {} {path}", style.muted("Failed SQL:")));
+    }
     lines.push(format!(
         "  {} {}/",
         style.muted("Wrote:"),

crates/sqlbuild-core/src/cli_text/models.rs

@@ -232,6 +232,8 @@ pub struct CompileTextReport {
     pub warning_count: usize,
     /// The compiled SQL directory relative to `target/`'s parent, without a trailing slash.
     pub compiled_dir: String,
+    /// Each failing model's SQL file under `compiled_dir` when Rules failed, relative like it.
+    pub failed_model_sql: Vec<String>,
     pub manifest: bool,
     pub lineage: LineageNote,
 }

crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs

@@ -104,27 +104,52 @@ fn given_partial_reasons_when_building_notice_then_groups_by_reason() {
 
 #[test]
 fn given_report_when_formatting_then_lists_models_and_summary() {
-    let test_cases = [CompileTextTestCase {
-        description: "one plain model with a manifest and unavailable lineage",
-        report: CompileTextReport {
-            style: CliStyle { use_color: false },
-            models: vec![ModelLine {
-                name: "orders".to_owned(),
-                failed: false,
-                column_count: 2,
-            }],
-            semantic_notice: None,
-            diagnostics: Vec::new(),
-            seed_count: 1,
-            function_count: 0,
-            error_count: 0,
-            warning_count: 0,
-            compiled_dir: "target/compiled".to_owned(),
-            manifest: true,
-            lineage: LineageNote::Unavailable,
+    let test_cases = [
+        CompileTextTestCase {
+            description: "one plain model with a manifest and unavailable lineage",
+            report: CompileTextReport {
+                style: CliStyle { use_color: false },
+                models: vec![ModelLine {
+                    name: "orders".to_owned(),
+                    failed: false,
+                    column_count: 2,
+                }],
+                semantic_notice: None,
+                diagnostics: Vec::new(),
+                seed_count: 1,
+                function_count: 0,
+                error_count: 0,
+                warning_count: 0,
+                compiled_dir: "target/compiled".to_owned(),
+                failed_model_sql: Vec::new(),
+                manifest: true,
+                lineage: LineageNote::Unavailable,
+            },
+            expected_text: "\nCompile ready  1 model\n\n\u{2514}\u{2500}\u{2500} orders                   OK   2 columns\n\n\u{2713} Project compiled  1 model, 1 seed, 0 functions, 0 errors, 0 warnings\n  Wrote: target/compiled/\n  Wrote: target/manifest.json\n  Column lineage: unavailable\n",
         },
-        expected_text: "\nCompile ready  1 model\n\n\u{2514}\u{2500}\u{2500} orders                   OK   2 columns\n\n\u{2713} Project compiled  1 model, 1 seed, 0 functions, 0 errors, 0 warnings\n  Wrote: target/compiled/\n  Wrote: target/manifest.json\n  Column lineage: unavailable\n",
-    }];
+        CompileTextTestCase {
+            description: "a compile whose Rules failed names each failing model's SQL",
+            report: CompileTextReport {
+                style: CliStyle { use_color: false },
+                models: vec![ModelLine {
+                    name: "orders".to_owned(),
+                    failed: true,
+                    column_count: 2,
+                }],
+                semantic_notice: None,
+                diagnostics: Vec::new(),
+                seed_count: 0,
+                function_count: 0,
+                error_count: 1,
+                warning_count: 0,
+                compiled_dir: "target/compiled-failed".to_owned(),
+                failed_model_sql: vec!["target/compiled-failed/models/orders.sql".to_owned()],
+                manifest: false,
+                lineage: LineageNote::Silent,
+            },
+            expected_text: "\nCompile ready  1 model\n\n\u{2514}\u{2500}\u{2500} orders                   FAIL 2 columns\n\n\u{2717} Project compiled  1 model, 0 seeds, 0 functions, 1 error, 0 warnings\n  Failed SQL: target/compiled-failed/models/orders.sql\n  Wrote: target/compiled-failed/\n",
+        },
+    ];
 
     for test_case in test_cases {
         assert_eq!(

crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs

@@ -5,6 +5,7 @@ use pyo3::types::{
     PyDict, PyDictMethods, PyList, PySet, PySetMethods, PyString, PyTuple, PyTupleMethods,
 };
 use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction};
+use sqlbuild_cache::project_reuse::constants::FAILED_COMPILED_DIRECTORY_NAME;
 use sqlbuild_core::cli_text::main::format_compile_text::format_compile_text as compile_text;
 use sqlbuild_core::cli_text::main::semantic_coverage_notice::semantic_coverage_notice as notice_text;
 use sqlbuild_core::cli_text::models::{
@@ -59,6 +60,7 @@ struct ReportInputs<'py> {
     graph: Bound<'py, PyAny>,
     project: Bound<'py, PyAny>,
     written: Bound<'py, PyAny>,
+    /// A manifest was requested and written, which a compile whose Rules failed never does.
     manifest: bool,
     lineage: Bound<'py, PyAny>,
     diagnostics: Vec<Bound<'py, PyAny>>,
@@ -71,11 +73,13 @@ impl<'py> ReportInputs<'py> {
     fn read(arguments: &KeywordArguments<'py>) -> PyResult<Self> {
         let graph: Bound<'py, PyAny> = arguments.value("graph")?;
         let lineage_mode: Bound<'py, PyAny> = arguments.value("lineage_mode")?;
+        let written: Bound<'py, PyAny> = arguments.value("written")?;
+        let failed_compile: bool = native_field(&written, "failed_compile")?.is_truthy()?;
         Ok(Self {
             project: native_field(&graph, "project")?,
             graph,
-            written: arguments.value("written")?,
-            manifest: arguments.flag("manifest")?,
+            manifest: arguments.flag("manifest")? && !failed_compile,
+            written,
             lineage: arguments.value("lineage")?,
             diagnostics: arguments
                 .value("diagnostics")?
@@ -182,11 +186,15 @@ impl<'py> ReportInputs<'py> {
         coverage_notice(&self.semantic_coverage()?, reader)
     }
 
-    /// `written.target_dir / "compiled"` relative to its `target` directory, POSIX style.
+    /// `written.target_dir / "compiled"`, or `"compiled-failed"` when Rules failed, relative to its `target` directory, POSIX style.
     fn compiled_dir(&self) -> PyResult<Bound<'py, PyAny>> {
         let py: Python<'py> = self.py();
-        let path: Bound<'py, PyAny> =
-            native_field(&self.written, "target_dir")?.div(COMPILED_DIRECTORY)?;
+        let directory: &str = if native_field(&self.written, "failed_compile")?.is_truthy()? {
+            FAILED_COMPILED_DIRECTORY_NAME
+        } else {
+            COMPILED_DIRECTORY
+        };
+        let path: Bound<'py, PyAny> = native_field(&self.written, "target_dir")?.div(directory)?;
         let parts: Bound<'py, PyTuple> = native_field(&path, "parts")?.downcast_into()?;
         for (index, part) in parts.iter().enumerate() {
             if part.eq(TARGET_DIRECTORY)? {
@@ -198,6 +206,16 @@ impl<'py> ReportInputs<'py> {
         }
         path.call_method0("as_posix")
     }
+
+    /// Each failing model's SQL file under `compiled_dir`, relative like it, when Rules failed.
+    fn failed_model_sql(&self) -> PyResult<Vec<String>> {
+        let directory: String = self.compiled_dir()?.extract()?;
+        Ok(native_field(&self.written, "failed_model_sql")?
+            .extract::<Vec<String>>()?
+            .into_iter()
+            .map(|relative| format!("{directory}/{relative}"))
+            .collect())
+    }
 }
 
 /// `int(value)` of a location field, or `None`.
@@ -392,6 +410,7 @@ pub(crate) fn format_compile_text<'py>(
         error_count: inputs.severity_count(ERROR_SEVERITY)?,
         warning_count: inputs.severity_count(WARNING_SEVERITY)?,
         compiled_dir: reader.text(&inputs.compiled_dir()?)?,
+        failed_model_sql: inputs.failed_model_sql()?,
         manifest: inputs.manifest,
         lineage,
     };
@@ -1036,6 +1055,12 @@ fn compile_json<'py>(
             ),
         ],
     )?;
+    if native_field(&inputs.written, "failed_compile")?.is_truthy()? {
+        artifacts.set_item(
+            "failed_model_sql",
+            PyList::new(py, inputs.failed_model_sql()?)?,
+        )?;
+    }
     let report: Bound<'_, PyDict> = dict(
         py,
         vec![

crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_attempt.rs

@@ -8,6 +8,7 @@ use pyo3::prelude::{
 };
 use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyList, PyString};
 use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction};
+use sqlbuild_cache::project_reuse::constants::FAILED_COMPILED_DIRECTORY_NAME;
 use sqlbuild_cli::replay::main::write_launch_record::write_launch_record as write_record_file;
 use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV;
 
@@ -554,6 +555,7 @@ fn compiled_with_delta<'py>(
             .join(TARGET_DIRECTORY_NAME)
             .join(COMPILED_DIRECTORY_NAME),
     )?;
+    let failed_root: PathBuf = compiled_root.with_file_name(FAILED_COMPILED_DIRECTORY_NAME);
     match delta {
         DeltaOutcome::Finished(finished) if !verifying => {
             keep_delta_claim();
@@ -595,10 +597,13 @@ fn compiled_with_delta<'py>(
             raised: None,
             compiled: verify::compiled_digests(&compiled_root),
             compiled_root: compiled_root.to_string_lossy().into_owned(),
+            failed: verify::compiled_digests(&failed_root),
+            failed_root: failed_root.to_string_lossy().into_owned(),
         }),
         DeltaOutcome::Raised(error) => verify::hold_expected(verify::Expected {
             raised: Some(error.to_string()),
             compiled_root: compiled_root.to_string_lossy().into_owned(),
+            failed_root: failed_root.to_string_lossy().into_owned(),
             ..verify::Expected::default()
         }),
     }