From 9769e777068a1fbc4bc9ff8d6963cc40c8be5382 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 06:30:25 +0100 Subject: [PATCH 1/4] refactor: write compile output, artifacts and reuse records natively Port the sqb compile report (text, JSON and stopped-compile JSON), the semantic-check notice, compile lineage selection, the static and planned target writers, the SQL-test artifact cache, background artifact preparation and staged publication, and the full-compile reuse recording to Rust. The compile driver calls them directly; the artifact-write recorder is passed explicitly instead of through a global registry. The Python implementations are removed. --- .../src/artifacts/_helpers/mod.rs | 1 + .../artifacts/_helpers/sql_test_records.rs | 70 ++ .../sqlbuild-cache/src/artifacts/constants.rs | 3 + .../sqlbuild-cache/src/artifacts/main/mod.rs | 3 + .../src/artifacts/main/new_sql_test_record.rs | 34 + .../main/reusable_sql_test_record.rs | 37 + .../artifacts/main/stored_sql_test_record.rs | 24 + crates/sqlbuild-cache/src/artifacts/models.rs | 36 + crates/sqlbuild-cache/src/digest/main/mod.rs | 1 + .../src/digest/main/sha256_hex.rs | 12 + .../src/cli_text/_helpers/diagnostics.rs | 123 ++ .../src/cli_text/_helpers/layout.rs | 70 ++ .../src/cli_text/_helpers/mod.rs | 2 + .../sqlbuild-core/src/cli_text/constants.rs | 31 + .../src/cli_text/main/format_compile_text.rs | 108 ++ crates/sqlbuild-core/src/cli_text/main/mod.rs | 2 + .../cli_text/main/semantic_coverage_notice.rs | 83 ++ crates/sqlbuild-core/src/cli_text/mod.rs | 7 + crates/sqlbuild-core/src/cli_text/models.rs | 156 +++ crates/sqlbuild-core/src/cli_text/tests.rs | 6 + .../src/cli_text/tests/helpers.rs | 31 + .../src/cli_text/tests/test_cli_text.rs | 124 ++ .../src/cli_text/tests/test_types.rs | 20 + crates/sqlbuild-core/src/lib.rs | 1 + crates/sqlbuild-core/src/text/main/mod.rs | 1 + .../src/text/main/python_splitlines.rs | 40 + .../src/text/tests/test_python_str.rs | 36 +- .../src/text/tests/test_types.rs | 6 + .../compile_command/artifact_batch.rs | 81 ++ .../bindings/_helpers/compile_command/mod.rs | 8 + .../compile_command/prepared_artifacts.rs | 591 +++++++++ .../_helpers/compile_command/registration.rs | 14 + .../_helpers/compile_command/report.rs | 1089 +++++++++++++++++ .../compile_command/reuse_recording.rs | 577 +++++++++ .../_helpers/compile_command/surrogates.rs | 84 ++ .../_helpers/compile_command/target_writer.rs | 812 ++++++++++++ .../compile_command/test_artifact_cache.rs | 504 ++++++++ .../_helpers/compile_driver/analysis.rs | 55 +- .../compile_driver/assembly_analysis.rs | 2 +- .../_helpers/compile_driver/command.rs | 128 +- .../_helpers/compile_driver/phases.rs | 76 +- .../_helpers/compile_driver/project.rs | 2 +- .../_helpers/compile_outputs/artifacts.rs | 24 +- .../_helpers/compile_outputs/json_report.rs | 36 +- .../_helpers/compile_outputs/registration.rs | 3 +- .../_helpers/compile_outputs/reuse.rs | 6 +- .../_helpers/compiled_project/attachment.rs | 2 +- .../src/bindings/_helpers/mod.rs | 1 + .../src/bindings/main/register.rs | 9 +- src/sqlbuild/_native.pyi | 112 +- .../_helpers/build_planning/compile_target.py | 2 +- .../cli/commands/_helpers/compile/__init__.py | 1 - .../cli/commands/_helpers/compile/lineage.py | 34 - .../cli/commands/_helpers/compile/output.py | 726 ----------- .../_helpers/compile/semantic_notice.py | 82 -- .../compile/sql_test_artifact_cache.py | 303 ----- .../_helpers/compile/target_writer.py | 660 ---------- .../_helpers/entry/dispatch_errors.py | 2 +- .../_helpers/refactor/compile_facts.py | 5 +- .../commands/classes/native_artifact_batch.py | 37 - .../classes/prepared_compile_artifacts.py | 260 ---- src/sqlbuild/cli/compile/models.py | 22 - .../compile_reuse/_helpers/native_reuse.py | 143 --- .../cli/compile_reuse/_helpers/store.py | 49 - .../cli/compile_reuse/classes/__init__.py | 0 .../compile_artifact_write_registry.py | 66 - .../classes/compile_artifact_writes.py | 53 - .../classes/recorded_compile_output.py | 43 - src/sqlbuild/cli/compile_reuse/constants.py | 4 - .../main/_compile_reuse_timings.py | 12 - .../main/_write_reusable_compile.py | 38 - src/sqlbuild/cli/compile_reuse/models.py | 9 - ...ntic_coverage.py => _semantic_coverage.py} | 0 ..._paths.py => _macro_store_module_paths.py} | 0 .../manifest/main/_build_graph_manifest.py | 2 +- .../planner/_helpers/changes/detect.py | 2 +- .../planner/_helpers/identity/direct.py | 2 +- .../planner/_helpers/migrations/discovery.py | 2 +- .../profiling/main/{record.py => _record.py} | 0 ...hook_identities.py => _hook_identities.py} | 0 .../cli/commands/main/compile/helpers.py | 19 +- .../compile/test_native_analysis_seams.py | 4 +- .../commands/main/compile_outputs/helpers.py | 15 +- .../test_native_json_report.py | 3 - .../src/sqlbuild/cli/commands/main/helpers.py | 52 +- .../main/test_compile_prepared_artifacts.py | 20 +- .../main/test_startup_import_footprint.py | 1 - .../sqlbuild/compiler/pipeline/test_main.py | 8 +- .../cli/commands/main/compile/_test_types.py | 2 +- .../cli/commands/main/compile/helpers.py | 38 +- .../main/compile/test_compile_command.py | 83 +- .../main/compile/test_compile_output.py | 4 +- .../compile/test_compile_target_writer.py | 49 +- .../test_staged_artifact_publication.py | 82 +- .../main/test_attempt_import_footprint.py | 1 - 95 files changed, 5321 insertions(+), 2903 deletions(-) create mode 100644 crates/sqlbuild-cache/src/artifacts/_helpers/sql_test_records.rs create mode 100644 crates/sqlbuild-cache/src/artifacts/main/new_sql_test_record.rs create mode 100644 crates/sqlbuild-cache/src/artifacts/main/reusable_sql_test_record.rs create mode 100644 crates/sqlbuild-cache/src/artifacts/main/stored_sql_test_record.rs create mode 100644 crates/sqlbuild-cache/src/digest/main/sha256_hex.rs create mode 100644 crates/sqlbuild-core/src/cli_text/_helpers/diagnostics.rs create mode 100644 crates/sqlbuild-core/src/cli_text/_helpers/layout.rs create mode 100644 crates/sqlbuild-core/src/cli_text/_helpers/mod.rs create mode 100644 crates/sqlbuild-core/src/cli_text/constants.rs create mode 100644 crates/sqlbuild-core/src/cli_text/main/format_compile_text.rs create mode 100644 crates/sqlbuild-core/src/cli_text/main/mod.rs create mode 100644 crates/sqlbuild-core/src/cli_text/main/semantic_coverage_notice.rs create mode 100644 crates/sqlbuild-core/src/cli_text/mod.rs create mode 100644 crates/sqlbuild-core/src/cli_text/models.rs create mode 100644 crates/sqlbuild-core/src/cli_text/tests.rs create mode 100644 crates/sqlbuild-core/src/cli_text/tests/helpers.rs create mode 100644 crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs create mode 100644 crates/sqlbuild-core/src/cli_text/tests/test_types.rs create mode 100644 crates/sqlbuild-core/src/text/main/python_splitlines.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_command/surrogates.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs delete mode 100644 src/sqlbuild/cli/commands/_helpers/compile/__init__.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/compile/lineage.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/compile/output.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/compile/semantic_notice.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py delete mode 100644 src/sqlbuild/cli/commands/_helpers/compile/target_writer.py delete mode 100644 src/sqlbuild/cli/commands/classes/native_artifact_batch.py delete mode 100644 src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py delete mode 100644 src/sqlbuild/cli/compile_reuse/_helpers/store.py delete mode 100644 src/sqlbuild/cli/compile_reuse/classes/__init__.py delete mode 100644 src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py delete mode 100644 src/sqlbuild/cli/compile_reuse/classes/compile_artifact_writes.py delete mode 100644 src/sqlbuild/cli/compile_reuse/classes/recorded_compile_output.py delete mode 100644 src/sqlbuild/cli/compile_reuse/main/_compile_reuse_timings.py delete mode 100644 src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py rename src/sqlbuild/compiler/compile/main/{semantic_coverage.py => _semantic_coverage.py} (100%) rename src/sqlbuild/compiler/macro_bridge/main/{macro_store_module_paths.py => _macro_store_module_paths.py} (100%) rename src/sqlbuild/compiler/profiling/main/{record.py => _record.py} (100%) rename src/sqlbuild/compiler/python_nodes/main/{hook_identities.py => _hook_identities.py} (100%) diff --git a/crates/sqlbuild-cache/src/artifacts/_helpers/mod.rs b/crates/sqlbuild-cache/src/artifacts/_helpers/mod.rs index c0f9955931..c72064b61f 100644 --- a/crates/sqlbuild-cache/src/artifacts/_helpers/mod.rs +++ b/crates/sqlbuild-cache/src/artifacts/_helpers/mod.rs @@ -1 +1,2 @@ pub(crate) mod files; +pub(crate) mod sql_test_records; diff --git a/crates/sqlbuild-cache/src/artifacts/_helpers/sql_test_records.rs b/crates/sqlbuild-cache/src/artifacts/_helpers/sql_test_records.rs new file mode 100644 index 0000000000..c9ca26c990 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/_helpers/sql_test_records.rs @@ -0,0 +1,70 @@ +//! File facts of compiled SQL test artifacts: mtimes, content digests and safe paths. + +use std::fs::Metadata; +use std::path::{Component, Path, PathBuf}; + +#[cfg(unix)] +use crate::artifacts::constants::NANOS_PER_SECOND; +use crate::artifacts::constants::SQL_EXTENSION; +use crate::digest::main::sha256_hex::sha256_hex; + +/// Python's `os.stat_result.st_mtime_ns`. +#[cfg(unix)] +pub(crate) fn mtime_ns(metadata: &Metadata) -> Option { + use std::os::unix::fs::MetadataExt; + metadata + .mtime() + .checked_mul(NANOS_PER_SECOND)? + .checked_add(metadata.mtime_nsec()) +} + +/// Python's `os.stat_result.st_mtime_ns`. +#[cfg(not(unix))] +pub(crate) fn mtime_ns(metadata: &Metadata) -> Option { + let since = match metadata.modified() { + Ok(modified) => match modified.duration_since(std::time::UNIX_EPOCH) { + Ok(since) => since, + Err(_) => return None, + }, + Err(_) => return None, + }; + let Ok(nanos) = i64::try_from(since.as_nanos()) else { + return None; + }; + Some(nanos) +} + +/// The file's SHA-256 text, or `None` when it cannot be read, as Python's `OSError` is skipped. +pub(crate) fn file_sha256(path: &Path) -> Option { + match std::fs::read(path) { + Ok(bytes) => Some(sha256_hex(&bytes)), + Err(_) => None, + } +} + +/// The file's metadata, or `None` when it cannot be read. +pub(crate) fn metadata(path: &Path) -> Option { + let Ok(metadata) = std::fs::metadata(path) else { + return None; + }; + Some(metadata) +} + +/// The path as Python's `PurePath(path)` holds it: without `.` components. +pub(crate) fn normalized_relative_path(path: &Path) -> PathBuf { + path.components() + .filter(|component| !matches!(component, Component::CurDir)) + .collect() +} + +/// A relative `.sql` path that cannot leave the tests directory. +pub(crate) fn is_safe_relative_sql_path(path: &Path) -> bool { + !path.is_absolute() + && !path.has_root() + && !path + .components() + .any(|component| matches!(component, Component::ParentDir)) + && path + .extension() + .is_some_and(|extension| extension == SQL_EXTENSION) +} diff --git a/crates/sqlbuild-cache/src/artifacts/constants.rs b/crates/sqlbuild-cache/src/artifacts/constants.rs index dd298d6931..6625926d4d 100644 --- a/crates/sqlbuild-cache/src/artifacts/constants.rs +++ b/crates/sqlbuild-cache/src/artifacts/constants.rs @@ -1,2 +1,5 @@ /// The mode new artifacts are created with before the process umask, as Python's `os.open`. pub const NEW_FILE_MODE: u32 = 0o666; +/// The extension every compiled SQL test artifact carries. +pub(crate) const SQL_EXTENSION: &str = "sql"; +pub(crate) const NANOS_PER_SECOND: i64 = 1_000_000_000; diff --git a/crates/sqlbuild-cache/src/artifacts/main/mod.rs b/crates/sqlbuild-cache/src/artifacts/main/mod.rs index c922563fbf..090d08595b 100644 --- a/crates/sqlbuild-cache/src/artifacts/main/mod.rs +++ b/crates/sqlbuild-cache/src/artifacts/main/mod.rs @@ -1,4 +1,7 @@ +pub mod new_sql_test_record; pub mod publish_staged; pub mod remove_stale; +pub mod reusable_sql_test_record; pub mod staged_files; +pub mod stored_sql_test_record; pub mod write_artifacts; diff --git a/crates/sqlbuild-cache/src/artifacts/main/new_sql_test_record.rs b/crates/sqlbuild-cache/src/artifacts/main/new_sql_test_record.rs new file mode 100644 index 0000000000..b123c7fa7a --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/main/new_sql_test_record.rs @@ -0,0 +1,34 @@ +//! The cache record of a newly written SQL test artifact. + +use std::fs::Metadata; +use std::path::{Path, PathBuf}; + +use crate::artifacts::_helpers::sql_test_records::{ + file_sha256, is_safe_relative_sql_path, metadata, mtime_ns, normalized_relative_path, +}; +use crate::artifacts::models::SqlTestArtifactRecord; + +/// The artifact's record, or `None` when it cannot be read or reused safely. +#[must_use] +pub fn new_sql_test_record( + tests_root: &Path, + artifact_path: &Path, + identity: &str, +) -> Option { + let relative_path: PathBuf = match artifact_path.strip_prefix(tests_root) { + Ok(relative) => normalized_relative_path(relative), + Err(_) => return None, + }; + let metadata: Metadata = metadata(artifact_path)?; + let content_sha256: String = file_sha256(artifact_path)?; + if !is_safe_relative_sql_path(&relative_path) { + return None; + } + Some(SqlTestArtifactRecord { + identity: identity.to_owned(), + relative_path, + size: metadata.len(), + mtime_ns: mtime_ns(&metadata)?, + content_sha256, + }) +} diff --git a/crates/sqlbuild-cache/src/artifacts/main/reusable_sql_test_record.rs b/crates/sqlbuild-cache/src/artifacts/main/reusable_sql_test_record.rs new file mode 100644 index 0000000000..73d58f8154 --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/main/reusable_sql_test_record.rs @@ -0,0 +1,37 @@ +//! Whether a cached SQL test artifact is unchanged and can be kept. + +use std::fs::Metadata; +use std::path::{Path, PathBuf}; + +use crate::artifacts::_helpers::sql_test_records::{ + file_sha256, is_safe_relative_sql_path, metadata, mtime_ns, +}; +use crate::artifacts::models::SqlTestArtifactRecord; + +/// The record with its artifact's current mtime, when the artifact is unchanged. +#[must_use] +pub fn reusable_sql_test_record( + tests_root: &Path, + record: &SqlTestArtifactRecord, + identity: &str, +) -> Option { + if record.identity != identity || !is_safe_relative_sql_path(&record.relative_path) { + return None; + } + let artifact_path: PathBuf = tests_root.join(&record.relative_path); + let metadata: Metadata = metadata(&artifact_path)?; + if !metadata.is_file() || metadata.len() != record.size { + return None; + } + let current_mtime_ns: i64 = mtime_ns(&metadata)?; + if current_mtime_ns == record.mtime_ns { + return Some(record.clone()); + } + if file_sha256(&artifact_path)? != record.content_sha256 { + return None; + } + Some(SqlTestArtifactRecord { + mtime_ns: current_mtime_ns, + ..record.clone() + }) +} diff --git a/crates/sqlbuild-cache/src/artifacts/main/stored_sql_test_record.rs b/crates/sqlbuild-cache/src/artifacts/main/stored_sql_test_record.rs new file mode 100644 index 0000000000..896d5a812d --- /dev/null +++ b/crates/sqlbuild-cache/src/artifacts/main/stored_sql_test_record.rs @@ -0,0 +1,24 @@ +//! A SQL test artifact record read back from the cache file. + +use std::path::{Path, PathBuf}; + +use crate::artifacts::_helpers::sql_test_records::{ + is_safe_relative_sql_path, normalized_relative_path, +}; +use crate::artifacts::models::{SqlTestArtifactRecord, StoredSqlTestRecord}; + +/// The stored record, or `None` when its path could leave the tests directory. +#[must_use] +pub fn stored_sql_test_record(stored: StoredSqlTestRecord) -> Option { + let relative_path: PathBuf = normalized_relative_path(Path::new(&stored.relative_path)); + if !is_safe_relative_sql_path(&relative_path) { + return None; + } + Some(SqlTestArtifactRecord { + identity: stored.identity, + relative_path, + size: stored.size, + mtime_ns: stored.mtime_ns, + content_sha256: stored.content_sha256, + }) +} diff --git a/crates/sqlbuild-cache/src/artifacts/models.rs b/crates/sqlbuild-cache/src/artifacts/models.rs index 3e66d6eaca..d3eb9963e5 100644 --- a/crates/sqlbuild-cache/src/artifacts/models.rs +++ b/crates/sqlbuild-cache/src/artifacts/models.rs @@ -19,3 +19,39 @@ pub enum Publication { /// Each staged file was published at its compiled path, in this order. Files(Vec<(PathBuf, PathBuf)>), } + +/// What reusing one compiled SQL test artifact requires: its identity, place and content. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SqlTestArtifactRecord { + /// The digest of everything the artifact was rendered from. + pub identity: String, + /// The artifact's path below the compiled tests directory. + pub relative_path: PathBuf, + pub size: u64, + pub mtime_ns: i64, + /// The lowercase hex SHA-256 of the artifact's bytes. + pub content_sha256: String, +} + +/// The fields of one cache file record, already checked to be strings and non-negative integers. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StoredSqlTestRecord { + pub identity: String, + /// The `relative_path` text as stored. + pub relative_path: String, + pub size: u64, + pub mtime_ns: i64, + pub content_sha256: String, +} + +impl SqlTestArtifactRecord { + /// The relative path's `as_posix()` text, as the cache file stores it. + #[must_use] + pub fn relative_posix(&self) -> String { + self.relative_path + .components() + .map(|component| component.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/") + } +} diff --git a/crates/sqlbuild-cache/src/digest/main/mod.rs b/crates/sqlbuild-cache/src/digest/main/mod.rs index 335187a89e..fb6e1e2b02 100644 --- a/crates/sqlbuild-cache/src/digest/main/mod.rs +++ b/crates/sqlbuild-cache/src/digest/main/mod.rs @@ -5,3 +5,4 @@ pub mod digest_files; pub mod fingerprint_project_files; pub mod hex_digest; pub mod installed_code_identity; +pub mod sha256_hex; diff --git a/crates/sqlbuild-cache/src/digest/main/sha256_hex.rs b/crates/sqlbuild-cache/src/digest/main/sha256_hex.rs new file mode 100644 index 0000000000..641142ecf3 --- /dev/null +++ b/crates/sqlbuild-cache/src/digest/main/sha256_hex.rs @@ -0,0 +1,12 @@ +//! The SHA-256 text Python's `hashlib.sha256(data).hexdigest()` gives. + +use sha2::{Digest, Sha256}; + +/// Lowercase hexadecimal SHA-256 of `bytes`. +#[must_use] +pub fn sha256_hex(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/diagnostics.rs b/crates/sqlbuild-core/src/cli_text/_helpers/diagnostics.rs new file mode 100644 index 0000000000..7c7c303f8d --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/_helpers/diagnostics.rs @@ -0,0 +1,123 @@ +//! A diagnostic's human text: header, resource, located source with carets, notes and help. + +use crate::cli_text::models::{CliStyle, DiagnosticText, LocationText}; +use crate::text::main::python_splitlines::python_splitlines; + +const ERROR_SEVERITY: &str = "error"; +const WARNING_SEVERITY: &str = "warning"; +const MODEL_RESOURCE: &str = "model"; + +/// Every shown diagnostic, separated by blank lines, with folded warning counts. +pub(crate) fn diagnostics_text(diagnostics: &[DiagnosticText], style: CliStyle) -> String { + let mut lines: Vec = Vec::new(); + for diagnostic in diagnostics { + lines.extend(diagnostic_lines(diagnostic, style)); + if diagnostic.occurrences > 1 { + lines.push(format!( + " = {} occurrences; all retained in JSON and the warning count", + diagnostic.occurrences + )); + } + lines.push(String::new()); + } + let _ = lines.pop(); + lines.join("\n") +} + +fn diagnostic_lines(diagnostic: &DiagnosticText, style: CliStyle) -> Vec { + let header: String = format!( + "{}[{}]: {}", + diagnostic.severity, diagnostic.code, diagnostic.message + ); + let mut lines: Vec = vec![match diagnostic.severity.as_str() { + ERROR_SEVERITY => style.error(&header), + WARNING_SEVERITY => style.warning(&header), + _ => style.muted(&header), + }]; + if let Some(resource) = &diagnostic.resource_name { + let label: String = match &diagnostic.resource_type { + Some(kind) if kind != MODEL_RESOURCE => kind.replace('_', " "), + _ => MODEL_RESOURCE.to_owned(), + }; + lines.push(format!(" {label}: {}", style.object_name(resource))); + } + if let Some(location) = &diagnostic.location { + lines.extend(location_block(location, None)); + } else if let Some(path) = &diagnostic.path { + lines.push(format!(" --> {path}")); + } + for related in &diagnostic.related { + lines.push(String::new()); + lines.push(format!(" {}:", related.label)); + lines.extend(location_block( + &related.location, + related.message.as_deref(), + )); + } + lines.extend( + diagnostic + .notes + .iter() + .map(|note| format!(" {} {note}", style.muted("= note:"))), + ); + if !diagnostic.affected_rules.is_empty() { + lines.push(format!( + " {} {}", + style.muted("= affected Rules:"), + diagnostic.affected_rules.join(", ") + )); + } + if let Some(help) = &diagnostic.help { + lines.push(format!(" {} {help}", style.muted("= help:"))); + } + lines +} + +/// ` --> path:line:column`, then the source line with carets when the line exists. +fn location_block(location: &LocationText, message: Option<&str>) -> Vec { + let mut lines: Vec = vec![format!( + " --> {}:{}:{}", + location.path, location.line, location.column + )]; + let Some(source) = &location.source else { + return lines; + }; + let source_lines: Vec<&str> = python_splitlines(source); + let Ok(line_index) = usize::try_from(location.line) else { + return lines; + }; + if !(1..=source_lines.len()).contains(&line_index) { + return lines; + } + let source_line: &str = source_lines[line_index - 1]; + let line_number: String = location.line.to_string(); + let gutter: String = " ".repeat(line_number.chars().count()); + lines.push(format!(" {gutter} |")); + lines.push(format!(" {line_number} | {source_line}")); + lines.push(format!( + " {gutter} | {}", + caret_line(location, source_line, message) + )); + lines +} + +fn caret_line(location: &LocationText, source_line: &str, message: Option<&str>) -> String { + let start_column: i64 = location.column.max(1); + let mut end_column: i64 = match location.end_column { + Some(column) if column != 0 => column, + _ => start_column + 1, + }; + if location + .end_line + .is_some_and(|end_line| end_line != location.line) + { + end_column = i64::try_from(source_line.chars().count()).unwrap_or(i64::MAX) + 1; + } + let caret_count: i64 = (end_column - start_column).max(1); + let suffix: String = message.map_or_else(String::new, |message| format!(" {message}")); + format!( + "{}{}{suffix}", + " ".repeat(usize::try_from(start_column - 1).unwrap_or_default()), + "^".repeat(usize::try_from(caret_count).unwrap_or_default()) + ) +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/layout.rs b/crates/sqlbuild-core/src/cli_text/_helpers/layout.rs new file mode 100644 index 0000000000..b0fb44a96b --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/_helpers/layout.rs @@ -0,0 +1,70 @@ +//! The shared layout pieces of human CLI output, as `sqlbuild.presentation` renders them. + +use crate::cli_text::constants::{ + ELLIPSIS, PHASE_FAIL_GLYPH, PHASE_OK_GLYPH, TREE_BRANCH_GLYPH, TREE_LAST_GLYPH, +}; +use crate::cli_text::models::CliStyle; + +/// `format_surface_header`: accent title plus dim context. +pub(crate) fn surface_header(style: CliStyle, title: &str, context: &str) -> String { + let rendered: String = style.title(title); + if context.is_empty() { + return rendered; + } + format!("{rendered} {}", style.muted(context)) +} + +/// `format_phase_line`: state glyph, label, dim summary. +pub(crate) fn phase_line(style: CliStyle, ok: bool, label: &str, summary: &str) -> String { + let glyph: String = if ok { + style.success(PHASE_OK_GLYPH) + } else { + style.error(PHASE_FAIL_GLYPH) + }; + let rendered_label: String = if ok { + label.to_owned() + } else { + style.error_strong(label) + }; + let rendered: String = format!("{glyph} {rendered_label}"); + if summary.is_empty() { + return rendered; + } + format!("{rendered} {}", style.muted(summary)) +} + +/// `format_status_cell`: the styled status padded on its plain text to `width`. +pub(crate) fn status_cell(style: CliStyle, status: &str, width: usize) -> String { + let padding: String = " ".repeat(width.saturating_sub(status.chars().count())); + format!("{}{padding}", style.status(status)) +} + +/// `tree_connector`: a dim branch, or the last branch of a group. +pub(crate) fn tree_connector(style: CliStyle, last: bool) -> String { + style.muted(if last { + TREE_LAST_GLYPH + } else { + TREE_BRANCH_GLYPH + }) +} + +/// `"{count} {singular}"`, pluralized with `s` unless the count is one. +pub(crate) fn count_label(count: usize, singular: &str) -> String { + if count == 1 { + return format!("{count} {singular}"); + } + format!("{count} {singular}s") +} + +/// The text cut to `width` characters with `...`, or left-justified to `width`. +pub(crate) fn fit(text: &str, width: usize) -> String { + let length: usize = text.chars().count(); + if length > width { + if width <= ELLIPSIS.len() { + return ".".repeat(width); + } + let kept: String = text.chars().take(width - ELLIPSIS.len()).collect(); + return format!("{kept}{ELLIPSIS}"); + } + format!("{text}{}", " ".repeat(width - length)) +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs b/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs new file mode 100644 index 0000000000..6bc7e5db96 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod diagnostics; +pub(crate) mod layout; diff --git a/crates/sqlbuild-core/src/cli_text/constants.rs b/crates/sqlbuild-core/src/cli_text/constants.rs new file mode 100644 index 0000000000..5f26c08244 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/constants.rs @@ -0,0 +1,31 @@ +//! The ANSI styles, glyphs and limits of human CLI output. + +pub(crate) const BOLD: &str = "\x1b[1m"; +pub(crate) const DIM: &str = "\x1b[2m"; +pub(crate) const RED: &str = "\x1b[38;5;167m"; +pub(crate) const GREEN: &str = "\x1b[32m"; +pub(crate) const YELLOW: &str = "\x1b[33m"; +pub(crate) const BLUE: &str = "\x1b[34m"; +pub(crate) const RESET: &str = "\x1b[0m"; +pub(crate) const SUCCESS_STATUS_LABELS: [&str; 3] = ["OK", "PASS", "SUCCESS"]; +pub(crate) const WARNING_STATUS_LABELS: [&str; 2] = ["WARN", "WARNING"]; +pub(crate) const ERROR_STATUS_LABELS: [&str; 3] = ["ERROR", "FAIL", "FAILED"]; +pub(crate) const SKIPPED_STATUS_LABELS: [&str; 2] = ["SKIP", "SKIPPED"]; +pub(crate) const PHASE_OK_GLYPH: &str = "\u{2713}"; +pub(crate) const PHASE_FAIL_GLYPH: &str = "\u{2717}"; +pub(crate) const TREE_BRANCH_GLYPH: &str = "\u{251c}\u{2500}\u{2500}"; +pub(crate) const TREE_LAST_GLYPH: &str = "\u{2514}\u{2500}\u{2500}"; +/// The human report lists at most this many models. +pub(crate) const HUMAN_MODEL_LIMIT: usize = 100; +pub(crate) const MIN_MODEL_NAME_WIDTH: usize = 24; +pub(crate) const MAX_MODEL_NAME_WIDTH: usize = 48; +pub(crate) const STATUS_WIDTH: usize = 4; +/// The semantic notice shows at most this many reasons, and this many models per reason. +pub(crate) const NOTICE_DISPLAY_LIMIT: usize = 10; +/// What `fit` puts in place of the cut tail of a long name. +pub(crate) const ELLIPSIS: &str = "..."; +pub(crate) const OPEN_SOURCE: &str = "open source: "; +pub(crate) const OPEN_INPUT: &str = "open input: "; +pub(crate) const INVALID_UPSTREAM: &str = "depends on invalid model "; +pub(crate) const DEFAULT_ACTION: &str = + "declare the input shape, then run `sqb compile --no-cache`"; diff --git a/crates/sqlbuild-core/src/cli_text/main/format_compile_text.rs b/crates/sqlbuild-core/src/cli_text/main/format_compile_text.rs new file mode 100644 index 0000000000..87f9e1d43e --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_compile_text.rs @@ -0,0 +1,108 @@ +//! The human `sqb compile` report. + +use crate::cli_text::_helpers::diagnostics::diagnostics_text; +use crate::cli_text::_helpers::layout::{ + count_label, fit, phase_line, status_cell, surface_header, tree_connector, +}; +use crate::cli_text::constants::{ + HUMAN_MODEL_LIMIT, MAX_MODEL_NAME_WIDTH, MIN_MODEL_NAME_WIDTH, STATUS_WIDTH, +}; +use crate::cli_text::models::{CompileTextReport, LineageNote, ModelLine}; + +/// The report exactly as `format_compile_text` printed it, with its leading and final newline. +#[must_use] +pub fn format_compile_text(report: &CompileTextReport) -> String { + let style = report.style; + let mut lines: Vec = vec![ + surface_header( + style, + "Compile ready", + &count_label(report.models.len(), "model"), + ), + String::new(), + ]; + let visible: &[ModelLine] = &report.models[..report.models.len().min(HUMAN_MODEL_LIMIT)]; + let name_width: usize = visible + .iter() + .map(|model| model.name.chars().count()) + .max() + .unwrap_or(0) + .clamp(MIN_MODEL_NAME_WIDTH, MAX_MODEL_NAME_WIDTH); + for (index, model) in visible.iter().enumerate() { + let connector: String = tree_connector(style, index + 1 == visible.len()); + lines.push(format!( + "{connector} {} {} {}", + style.object_name(&fit(&model.name, name_width)), + status_cell( + style, + if model.failed { "FAIL" } else { "OK" }, + STATUS_WIDTH + ), + style.muted(&format!("{} columns", model.column_count)) + )); + } + if report.models.len() > visible.len() { + lines.push(String::new()); + lines.push(format!( + " {}", + style.muted(&format!( + "Showing {} of {} models.", + visible.len(), + report.models.len() + )) + )); + lines.push(format!( + " {}", + style.muted("Use --json for the full compile report.") + )); + } + lines.push(String::new()); + if let Some(notice) = report + .semantic_notice + .as_deref() + .filter(|notice| !notice.is_empty()) + { + lines.push(notice.to_owned()); + lines.push(String::new()); + } + if !report.diagnostics.is_empty() { + lines.push(diagnostics_text(&report.diagnostics, style)); + lines.push(String::new()); + } + let summary: String = [ + count_label(report.models.len(), "model"), + count_label(report.seed_count, "seed"), + count_label(report.function_count, "function"), + count_label(report.error_count, "error"), + count_label(report.warning_count, "warning"), + ] + .join(", "); + lines.push(phase_line( + style, + report.error_count == 0, + "Project compiled", + &summary, + )); + lines.push(format!( + " {} {}/", + style.muted("Wrote:"), + report.compiled_dir + )); + if report.manifest { + lines.push(format!(" {} target/manifest.json", style.muted("Wrote:"))); + } + match report.lineage { + LineageNote::Rich => lines.push(format!( + " {} rich analysis complete; inspect with sqb lineage [.]", + style.muted("Column lineage:") + )), + LineageNote::Unavailable => { + lines.push(format!( + " {} unavailable", + style.warning("Column lineage:") + )); + } + LineageNote::Silent => {} + } + format!("\n{}\n", lines.join("\n")) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/mod.rs b/crates/sqlbuild-core/src/cli_text/main/mod.rs new file mode 100644 index 0000000000..75b612fceb --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/mod.rs @@ -0,0 +1,2 @@ +pub mod format_compile_text; +pub mod semantic_coverage_notice; diff --git a/crates/sqlbuild-core/src/cli_text/main/semantic_coverage_notice.rs b/crates/sqlbuild-core/src/cli_text/main/semantic_coverage_notice.rs new file mode 100644 index 0000000000..e8cfe029d2 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/semantic_coverage_notice.rs @@ -0,0 +1,83 @@ +//! One actionable notice summarizing partial semantic checks, without counting as a warning. + +use std::collections::BTreeMap; + +use crate::cli_text::constants::{ + DEFAULT_ACTION, INVALID_UPSTREAM, NOTICE_DISPLAY_LIMIT, OPEN_INPUT, OPEN_SOURCE, +}; + +/// The action that clears one partial-check reason. +fn action(reason: &str) -> String { + if let Some(source) = reason.strip_prefix(OPEN_SOURCE) { + return format!( + "run `sqb contract generate --from --select source:{source} --write` and \ + enforce the complete source contract" + ); + } + if reason.starts_with(INVALID_UPSTREAM) { + return "fix the upstream error, then run `sqb compile --no-cache`".to_owned(); + } + match reason { + "some output types are unknown" => { + "declare output types, then run `sqb compile --no-cache`" + } + "output shape could not be inferred" => { + "declare the output schema; run `sqb compile --no-cache`" + } + "unresolved star over open inputs" => { + "declare complete input contracts, then run `sqb compile --no-cache`" + } + "SQL analysis disabled" => "enable sql_analysis, then run `sqb compile --no-cache`", + "type checking awaits dialect coercion support" => { + "use a supported dialect for type checks; run `sqb compile --json` to inspect coverage" + } + _ => DEFAULT_ACTION, + } + .to_owned() +} + +/// The notice for each model's partial-check reasons, or `None` when every check was complete. +#[must_use] +pub fn semantic_coverage_notice(reasons: &[(String, Vec)]) -> Option { + if reasons.is_empty() { + return None; + } + let mut by_model: Vec<&(String, Vec)> = reasons.iter().collect(); + by_model.sort_by(|left, right| left.0.cmp(&right.0)); + let mut groups: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); + for (model, values) in by_model { + for reason in values { + groups + .entry(reason.as_str()) + .or_default() + .push(model.as_str()); + } + } + let mut lines: Vec = vec![format!( + "Semantic checks were partial for {} models:", + reasons.len() + )]; + for (reason, models) in groups.iter().take(NOTICE_DISPLAY_LIMIT) { + let suffix: String = if models.len() > NOTICE_DISPLAY_LIMIT { + format!(", and {} more", models.len() - NOTICE_DISPLAY_LIMIT) + } else { + String::new() + }; + let detail: String = reason + .replace(OPEN_SOURCE, "reads open source ") + .replace(OPEN_INPUT, "reads open input "); + let shown: Vec<&str> = models.iter().take(NOTICE_DISPLAY_LIMIT).copied().collect(); + lines.push(format!( + " {}{suffix}: {detail}; {}.", + shown.join(", "), + action(reason) + )); + } + if groups.len() > NOTICE_DISPLAY_LIMIT { + lines.push(format!( + " and {} more reasons; run `sqb compile --json` for the full model/reason list.", + groups.len() - NOTICE_DISPLAY_LIMIT + )); + } + Some(lines.join("\n")) +} diff --git a/crates/sqlbuild-core/src/cli_text/mod.rs b/crates/sqlbuild-core/src/cli_text/mod.rs new file mode 100644 index 0000000000..1e64f3d116 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs new file mode 100644 index 0000000000..3054461df0 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -0,0 +1,156 @@ +//! Plain data of the human compile report, and the semantic styling it is rendered with. + +use crate::cli_text::constants::{ + BLUE, BOLD, DIM, ERROR_STATUS_LABELS, GREEN, RED, RESET, SKIPPED_STATUS_LABELS, + SUCCESS_STATUS_LABELS, WARNING_STATUS_LABELS, YELLOW, +}; + +/// The default CLI theme, applied only when color is enabled. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CliStyle { + pub use_color: bool, +} + +impl CliStyle { + /// `TextStyle.apply`: the text wrapped in `prefix` and a reset, unless color or text is off. + fn apply(self, prefix: &str, text: &str) -> String { + if !self.use_color || prefix.is_empty() || text.is_empty() { + return text.to_owned(); + } + format!("{prefix}{text}{RESET}") + } + + #[must_use] + pub fn title(self, text: &str) -> String { + self.apply(&format!("{BLUE}{BOLD}"), text) + } + + #[must_use] + pub fn muted(self, text: &str) -> String { + self.apply(DIM, text) + } + + #[must_use] + pub fn object_name(self, text: &str) -> String { + self.apply("", text) + } + + #[must_use] + pub fn success(self, text: &str) -> String { + self.apply(GREEN, text) + } + + #[must_use] + pub fn warning(self, text: &str) -> String { + self.apply(YELLOW, text) + } + + #[must_use] + pub fn error(self, text: &str) -> String { + self.apply(RED, text) + } + + #[must_use] + pub fn error_strong(self, text: &str) -> String { + self.apply(&format!("{RED}{BOLD}"), text) + } + + /// `CliStyle.status`: the status word styled by what it reports. + #[must_use] + pub fn status(self, status: &str) -> String { + let normalized: String = status.to_uppercase(); + if SUCCESS_STATUS_LABELS.contains(&normalized.as_str()) { + return self.success(status); + } + if WARNING_STATUS_LABELS.contains(&normalized.as_str()) { + return self.warning(status); + } + if ERROR_STATUS_LABELS.contains(&normalized.as_str()) { + return self.error(status); + } + if SKIPPED_STATUS_LABELS.contains(&normalized.as_str()) { + return self.muted(status); + } + status.to_owned() + } +} + +/// One source location, with the authored text of its file when the report has it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LocationText { + /// `str(location.path)`. + pub path: String, + pub line: i64, + pub column: i64, + pub end_line: Option, + pub end_column: Option, + /// The authored text of `path`, when the report holds it. + pub source: Option, +} + +/// A related location of a diagnostic: its label, location and optional caret message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RelatedText { + pub label: String, + pub location: LocationText, + pub message: Option, +} + +/// One diagnostic as the human report renders it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DiagnosticText { + /// `str(diagnostic.severity)`: `error`, `warning` or `info`. + pub severity: String, + pub code: String, + pub message: String, + /// `str(diagnostic.resource_type)`, when set. + pub resource_type: Option, + pub resource_name: Option, + pub location: Option, + /// `str(diagnostic.path)`, shown only when there is no location. + pub path: Option, + pub related: Vec, + pub notes: Vec, + pub affected_rules: Vec, + pub help: Option, + /// How many equal warnings this one stands for; errors and notes are always `1`. + pub occurrences: usize, +} + +/// What the human report says about column lineage. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LineageNote { + /// Rich lineage completed. + Rich, + /// Lineage was expected but is missing. + Unavailable, + /// Nothing to say. + Silent, +} + +/// One listed model: its name, whether an error names it, and its output column count. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ModelLine { + pub name: String, + pub failed: bool, + pub column_count: i64, +} + +/// Everything the human compile report renders. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CompileTextReport { + pub style: CliStyle, + /// Every selected model, in project order. + pub models: Vec, + pub semantic_notice: Option, + /// The diagnostics shown, with repeated equal warnings already folded. + pub diagnostics: Vec, + pub seed_count: usize, + pub function_count: usize, + pub error_count: usize, + pub warning_count: usize, + /// The compiled SQL directory relative to `target/`'s parent, without a trailing slash. + pub compiled_dir: String, + pub manifest: bool, + pub lineage: LineageNote, +} diff --git a/crates/sqlbuild-core/src/cli_text/tests.rs b/crates/sqlbuild-core/src/cli_text/tests.rs new file mode 100644 index 0000000000..df2cc7a5a6 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_cli_text.rs"] +mod cli_text; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-core/src/cli_text/tests/helpers.rs b/crates/sqlbuild-core/src/cli_text/tests/helpers.rs new file mode 100644 index 0000000000..1b9a8dbf19 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/tests/helpers.rs @@ -0,0 +1,31 @@ +use crate::cli_text::models::{DiagnosticText, LocationText}; + +/// A location in `models/a.sql`, whose text has a CRLF line, a non-ASCII name and a vertical tab. +pub(super) fn location(line: i64, column: i64, end: (Option, Option)) -> LocationText { + LocationText { + path: "models/a.sql".to_owned(), + line, + column, + end_line: end.0, + end_column: end.1, + source: Some("SELECT 1\r\nFROM orders\u{e9} x\u{0b}third".to_owned()), + } +} + +/// A diagnostic with only its severity, code and message set. +pub(super) fn diagnostic(severity: &str, code: &str, message: &str) -> DiagnosticText { + DiagnosticText { + severity: severity.to_owned(), + code: code.to_owned(), + message: message.to_owned(), + resource_type: None, + resource_name: None, + location: None, + path: None, + related: Vec::new(), + notes: Vec::new(), + affected_rules: Vec::new(), + help: None, + occurrences: 1, + } +} diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs new file mode 100644 index 0000000000..2f47337f66 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -0,0 +1,124 @@ +use crate::cli_text::_helpers::diagnostics::diagnostics_text; +use crate::cli_text::main::format_compile_text::format_compile_text; +use crate::cli_text::main::semantic_coverage_notice::semantic_coverage_notice; +use crate::cli_text::models::{ + CliStyle, CompileTextReport, DiagnosticText, LineageNote, ModelLine, RelatedText, +}; +use crate::cli_text::tests::helpers::{diagnostic, location}; +use crate::cli_text::tests::test_types::{ + CompileTextTestCase, DiagnosticsTextTestCase, NoticeTestCase, +}; + +/// Expectations recorded from the retired Python formatter. +#[test] +fn given_diagnostics_when_rendering_then_matches_python_text() { + let test_cases = [DiagnosticsTextTestCase { + description: "located error with related location and a repeated warning", + diagnostics: vec![ + DiagnosticText { + resource_type: Some("sql_test".to_owned()), + resource_name: Some("orders_test".to_owned()), + location: Some(location(2, 3, (Some(2), Some(7)))), + related: vec![RelatedText { + label: "declared here".to_owned(), + location: location(1, 1, (Some(3), Some(1))), + message: Some("here".to_owned()), + }], + notes: vec!["n1".to_owned()], + affected_rules: vec!["R1".to_owned(), "R2".to_owned()], + help: Some("fix it".to_owned()), + ..diagnostic("error", "C001", "bad column") + }, + DiagnosticText { + path: Some("models/b.sql".to_owned()), + occurrences: 2, + ..diagnostic("warning", "W001", "careful") + }, + ], + style: CliStyle { use_color: true }, + expected_text: "\x1b[38;5;167merror[C001]: bad column\x1b[0m\n sql test: orders_test\n --> models/a.sql:2:3\n |\n 2 | FROM orders\u{e9} x\n | ^^^^\n\n declared here:\n --> models/a.sql:1:1\n |\n 1 | SELECT 1\n | ^^^^^^^^ here\n \x1b[2m= note:\x1b[0m n1\n \x1b[2m= affected Rules:\x1b[0m R1, R2\n \x1b[2m= help:\x1b[0m fix it\n\n\x1b[33mwarning[W001]: careful\x1b[0m\n --> models/b.sql\n = 2 occurrences; all retained in JSON and the warning count", + }]; + + for test_case in test_cases { + assert_eq!( + diagnostics_text(&test_case.diagnostics, test_case.style), + test_case.expected_text, + "{}", + test_case.description + ); + } +} + +/// Expectations recorded from the retired Python notice. +#[test] +fn given_partial_reasons_when_building_notice_then_groups_by_reason() { + let test_cases = [ + NoticeTestCase { + description: "reasons grouped across models with known and unknown actions", + reasons: vec![ + ( + "zeta".to_owned(), + vec![ + "open source: raw_orders".to_owned(), + "SQL analysis disabled".to_owned(), + ], + ), + ( + "alpha".to_owned(), + vec!["open source: raw_orders".to_owned(), "weird".to_owned()], + ), + ], + expected_notice: Some( + "Semantic checks were partial for 2 models:\n zeta: SQL analysis disabled; enable sql_analysis, then run `sqb compile --no-cache`.\n alpha, zeta: reads open source raw_orders; run `sqb contract generate --from --select source:raw_orders --write` and enforce the complete source contract.\n alpha: weird; declare the input shape, then run `sqb compile --no-cache`.", + ), + }, + NoticeTestCase { + description: "complete checks print no notice", + reasons: Vec::new(), + expected_notice: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + semantic_coverage_notice(&test_case.reasons).as_deref(), + test_case.expected_notice, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_report_when_formatting_then_lists_models_and_summary() { + let test_cases = [CompileTextTestCase { + description: "one plain model with a manifest and unavailable lineage", + report: CompileTextReport { + style: CliStyle { use_color: false }, + models: vec![ModelLine { + name: "orders".to_owned(), + failed: false, + column_count: 2, + }], + semantic_notice: None, + diagnostics: Vec::new(), + seed_count: 1, + function_count: 0, + error_count: 0, + warning_count: 0, + compiled_dir: "target/compiled".to_owned(), + manifest: true, + lineage: LineageNote::Unavailable, + }, + expected_text: "\nCompile ready 1 model\n\n\u{2514}\u{2500}\u{2500} orders OK 2 columns\n\n\u{2713} Project compiled 1 model, 1 seed, 0 functions, 0 errors, 0 warnings\n Wrote: target/compiled/\n Wrote: target/manifest.json\n Column lineage: unavailable\n", + }]; + + for test_case in test_cases { + assert_eq!( + format_compile_text(&test_case.report), + test_case.expected_text, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_types.rs b/crates/sqlbuild-core/src/cli_text/tests/test_types.rs new file mode 100644 index 0000000000..9f4c59aac0 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/tests/test_types.rs @@ -0,0 +1,20 @@ +use crate::cli_text::models::{CliStyle, CompileTextReport, DiagnosticText}; + +pub(super) struct DiagnosticsTextTestCase { + pub(super) description: &'static str, + pub(super) diagnostics: Vec, + pub(super) style: CliStyle, + pub(super) expected_text: &'static str, +} + +pub(super) struct NoticeTestCase { + pub(super) description: &'static str, + pub(super) reasons: Vec<(String, Vec)>, + pub(super) expected_notice: Option<&'static str>, +} + +pub(super) struct CompileTextTestCase { + pub(super) description: &'static str, + pub(super) report: CompileTextReport, + pub(super) expected_text: &'static str, +} diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 10e15f1ed3..2f85e79618 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -2,6 +2,7 @@ #![forbid(unsafe_code)] +pub mod cli_text; pub mod constants; pub mod json; pub mod models; diff --git a/crates/sqlbuild-core/src/text/main/mod.rs b/crates/sqlbuild-core/src/text/main/mod.rs index d439d53a75..a92cd0d2e8 100644 --- a/crates/sqlbuild-core/src/text/main/mod.rs +++ b/crates/sqlbuild-core/src/text/main/mod.rs @@ -12,5 +12,6 @@ pub mod python_cleandoc; pub mod python_isalnum; pub mod python_isdigit; pub mod python_isspace; +pub mod python_splitlines; pub mod python_str_repr; pub mod python_strip; diff --git a/crates/sqlbuild-core/src/text/main/python_splitlines.rs b/crates/sqlbuild-core/src/text/main/python_splitlines.rs new file mode 100644 index 0000000000..70d938de77 --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/python_splitlines.rs @@ -0,0 +1,40 @@ +//! Python's `str.splitlines()` with no arguments. + +/// Whether Python's `str.splitlines()` ends a line at `character`. +fn is_line_boundary(character: char) -> bool { + matches!( + character, + '\n' | '\r' + | '\u{0b}' + | '\u{0c}' + | '\u{1c}' + | '\u{1d}' + | '\u{1e}' + | '\u{85}' + | '\u{2028}' + | '\u{2029}' + ) +} + +/// Python's `text.splitlines()`: lines without their endings; `\r\n` ends one line. +#[must_use] +pub fn python_splitlines(text: &str) -> Vec<&str> { + let mut lines: Vec<&str> = Vec::new(); + let mut start: usize = 0; + let mut characters = text.char_indices().peekable(); + while let Some((index, character)) = characters.next() { + if !is_line_boundary(character) { + continue; + } + lines.push(&text[start..index]); + start = index + character.len_utf8(); + if character == '\r' && characters.peek().is_some_and(|(_, next)| *next == '\n') { + let _ = characters.next(); + start += 1; + } + } + if start < text.len() { + lines.push(&text[start..]); + } + lines +} diff --git a/crates/sqlbuild-core/src/text/tests/test_python_str.rs b/crates/sqlbuild-core/src/text/tests/test_python_str.rs index cd5a678ed8..d17002cb3a 100644 --- a/crates/sqlbuild-core/src/text/tests/test_python_str.rs +++ b/crates/sqlbuild-core/src/text/tests/test_python_str.rs @@ -2,7 +2,10 @@ use crate::text::main::python_casefold::python_casefold; use crate::text::main::python_isalnum::python_isalnum; use crate::text::main::python_isdigit::python_isdigit; use crate::text::main::python_isspace::python_isspace; -use crate::text::tests::test_types::{CasefoldTestCase, CharacterClassTestCase}; +use crate::text::main::python_splitlines::python_splitlines; +use crate::text::tests::test_types::{ + CasefoldTestCase, CharacterClassTestCase, SplitlinesTestCase, +}; /// Expectations recorded from CPython 3.12. #[test] @@ -124,3 +127,34 @@ fn given_characters_when_classifying_then_matches_python_str_predicates() { ); } } + +/// Expectations recorded from CPython 3.12. +#[test] +fn given_text_when_splitting_lines_then_matches_python_splitlines() { + let test_cases = [ + SplitlinesTestCase { + description: "every Python line boundary, with CRLF as one", + text: "a\r\nb\rc\u{1c}d\u{2028}\n", + expected_lines: &["a", "b", "c", "d", ""], + }, + SplitlinesTestCase { + description: "empty text has no lines", + text: "", + expected_lines: &[], + }, + SplitlinesTestCase { + description: "a final newline ends the last line", + text: "one\n", + expected_lines: &["one"], + }, + ]; + + for test_case in test_cases { + assert_eq!( + python_splitlines(test_case.text), + test_case.expected_lines, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/text/tests/test_types.rs b/crates/sqlbuild-core/src/text/tests/test_types.rs index 3eb58cfe4b..a5fe685bbe 100644 --- a/crates/sqlbuild-core/src/text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/text/tests/test_types.rs @@ -59,3 +59,9 @@ pub(super) struct CharacterClassTestCase { pub(super) expected_digit: bool, pub(super) expected_space: bool, } + +pub(super) struct SplitlinesTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_lines: &'static [&'static str], +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs new file mode 100644 index 0000000000..a9d7ad48f4 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/artifact_batch.rs @@ -0,0 +1,81 @@ +//! Compiled artifacts queued for one native write, keeping unchanged files when asked. + +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; + +use crate::bindings::_helpers::compile_command::reuse_recording::{ + ArtifactRecorder, artifact_written, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{imported, within}; +use crate::bindings::_helpers::compile_outputs::artifacts::write_compiled_artifacts; + +const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._record"; + +/// Run `body` inside `record_compile_timing(phase)`. +pub(crate) fn timed<'py, T>( + py: Python<'py>, + phase: &str, + body: impl FnOnce() -> PyResult, +) -> PyResult { + within( + &imported(py, PROFILING_MODULE, "record_compile_timing")?.call1((phase,))?, + |_| body(), + ) +} + +/// The artifacts of one write group, written together by `flush`. +pub(crate) struct ArtifactBatch<'a> { + check_existing: bool, + recorder: ArtifactRecorder<'a>, + files: Vec<(PathBuf, Vec)>, +} + +impl<'a> ArtifactBatch<'a> { + pub(crate) fn new(check_existing: bool, recorder: ArtifactRecorder<'a>) -> Self { + Self { + check_existing, + recorder, + files: Vec::new(), + } + } + + /// Queue one artifact with platform line endings, noting its bytes for compile reuse. + pub(crate) fn queue(&mut self, path: &Bound<'_, PyAny>, contents: Vec) -> PyResult<()> { + let contents: Vec = if cfg!(windows) { + contents + .iter() + .flat_map(|byte| { + if *byte == b'\n' { + vec![b'\r', b'\n'] + } else { + vec![*byte] + } + }) + .collect() + } else { + contents + }; + artifact_written(self.recorder, path, &contents)?; + self.files.push((path.extract()?, contents)); + Ok(()) + } + + /// Queue `sql` with trailing whitespace replaced by one newline. + pub(crate) fn queue_sql(&mut self, path: &Bound<'_, PyAny>, sql: &str) -> PyResult<()> { + let trimmed: &str = + sql.trim_end_matches(sqlbuild_core::text::main::python_isspace::python_isspace); + self.queue(path, format!("{trimmed}\n").into_bytes()) + } + + /// Write every queued artifact natively and empty the queue. + pub(crate) fn flush(&mut self, py: Python<'_>) -> PyResult<()> { + if self.files.is_empty() { + return Ok(()); + } + let files: Vec<(PathBuf, Vec)> = std::mem::take(&mut self.files); + timed(py, "physical_write_ms", || { + write_compiled_artifacts(py, files, self.check_existing).map(|_| ()) + }) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs new file mode 100644 index 0000000000..eb1ab460a8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod artifact_batch; +pub(crate) mod prepared_artifacts; +pub(crate) mod registration; +pub(crate) mod report; +pub(crate) mod reuse_recording; +pub(crate) mod surrogates; +pub(crate) mod target_writer; +pub(crate) mod test_artifact_cache; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs new file mode 100644 index 0000000000..a5141150a8 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/prepared_artifacts.rs @@ -0,0 +1,591 @@ +//! Prepare compile artifacts in the background and publish them only at the normal write phase. + +use pyo3::exceptions::PyOSError; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRef, PyRefMut, PyResult, Python, +}; +use pyo3::types::{PyCFunction, PyDict, PyFrozenSet, PyTuple}; +use pyo3::{IntoPyObjectExt, pyclass, pymethods}; + +use crate::bindings::_helpers::compile_command::reuse_recording::CompileArtifactWrites; +use crate::bindings::_helpers::compile_command::target_writer::{ + Prepared, planning_diagnostics, publish_staged_target, staging_changed, write_static_target, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, +}; +use crate::bindings::_helpers::compile_outputs::artifacts::staged_artifact_files; + +const MIN_PREPARED_ARTIFACT_MODELS: usize = 128; +const MAX_PREPARED_ARTIFACT_MODELS: usize = 5000; +const STAGING_PREFIX: &str = ".sqlbuild-staging-"; +const LOCK_SUFFIX: &str = ".lock"; +const OWNER_MARKER: &str = ".owner"; +const COMPILED_DIR: &str = "compiled"; +/// The logger the retired Python class warned through, kept so log filters still apply. +const LOGGER_NAME: &str = "sqlbuild.cli.commands.classes.prepared_compile_artifacts"; +const FILELOCK_MODULE: &str = "filelock"; +const ARTIFACT_THREADS: &str = "sqlbuild-artifacts"; +const TEST_THREADS: &str = "sqlbuild-tests"; + +/// One background preparation task: staged artifacts or planned SQL tests only. +#[pyclass(module = "sqlbuild._native", subclass)] +pub(crate) struct PreparedCompileArtifacts { + enabled: bool, + /// Where the compile notes the artifacts this task writes or keeps. + artifact_writes: Option>, + staging_dir: Option>, + owner_lock: Option>, + created_target_dir: Option>, + executor: Option>, + future: Option>, + planned_tests: Option>, + planning_inputs: Option<(Py, Py, Py)>, +} + +fn owner_lock_path<'py>(staging_dir: &Bound<'py, PyAny>) -> PyResult> { + let name: String = staging_dir.getattr("name")?.extract()?; + staging_dir.call_method1("with_name", (format!("{name}{LOCK_SUFFIX}"),)) +} + +fn owns_staging_directory(staging_dir: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = staging_dir.py(); + let read: PyResult> = staging_dir.div(OWNER_MARKER)?.call_method( + "read_text", + (), + Some(&keywords( + py, + &[("encoding", "utf-8".into_bound_py_any(py)?)], + )?), + ); + match read { + Ok(text) => text.eq(staging_dir.getattr("name")?), + Err(error) if error.is_instance_of::(py) => Ok(false), + Err(error) => Err(error), + } +} + +/// Release the lock after deleting its file, ignoring a file that cannot be deleted. +fn release_and_unlink(lock: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = lock.py(); + let unlinked: PyResult> = imported(py, "pathlib", "Path")? + .call1((lock.getattr("lock_file")?,))? + .call_method( + "unlink", + (), + Some(&keywords( + py, + &[("missing_ok", true.into_bound_py_any(py)?)], + )?), + ); + if let Err(error) = unlinked + && !error.is_instance_of::(py) + { + return Err(error); + } + let _ = lock.call_method0("release")?; + Ok(()) +} + +fn warn( + py: Python<'_>, + message: &str, + arguments: (Bound<'_, PyAny>, Bound<'_, PyAny>), +) -> PyResult<()> { + let _ = imported(py, "logging", "getLogger")? + .call1((LOGGER_NAME,))? + .call_method1("warning", (message, arguments.0, arguments.1))?; + Ok(()) +} + +/// Remove one ownerless staging directory while holding its lock; `false` stops cleanup. +fn remove_abandoned_directory(directory: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = directory.py(); + let lock: Bound<'_, PyAny> = imported(py, FILELOCK_MODULE, "FileLock")?.call( + (owner_lock_path(directory)?,), + Some(&keywords(py, &[("timeout", 0_i64.into_bound_py_any(py)?)])?), + )?; + if let Err(error) = lock.call_method0("acquire") { + if error.is_instance(py, &imported(py, FILELOCK_MODULE, "Timeout")?) { + return Ok(true); + } + if error.is_instance_of::(py) { + warn( + py, + "skipped abandoned staging cleanup in %s: %s", + ( + directory.getattr("parent")?, + error.value(py).clone().into_any(), + ), + )?; + return Ok(false); + } + return Err(error); + } + let removed: PyResult<()> = (|| { + if directory.call_method0("is_dir")?.is_truthy()? { + let _ = imported(py, "shutil", "rmtree")?.call1((directory,))?; + } + Ok(()) + })(); + if let Err(error) = removed { + if !error.is_instance_of::(py) { + return Err(error); + } + let _ = lock.call_method0("release")?; + warn( + py, + "stopped abandoned staging cleanup at %s: %s", + (directory.clone(), error.value(py).clone().into_any()), + )?; + return Ok(false); + } + release_and_unlink(&lock)?; + Ok(true) +} + +/// `ThreadPoolExecutor(max_workers=1, thread_name_prefix=prefix)`. +fn single_thread_executor<'py>(py: Python<'py>, prefix: &str) -> PyResult> { + call_with( + &imported(py, "concurrent.futures", "ThreadPoolExecutor")?, + &[ + ("max_workers", 1_i64.into_bound_py_any(py)?), + ("thread_name_prefix", prefix.into_bound_py_any(py)?), + ], + ) +} + +/// Submit `task` to run inside a copy of the current context, as the caller's work would. +fn submit_in_context<'py>( + executor: &Bound<'py, PyAny>, + task: Bound<'py, PyCFunction>, +) -> PyResult> { + let py: Python<'py> = executor.py(); + let context: Bound<'py, PyAny> = imported(py, "contextvars", "copy_context")?.call0()?; + executor.call_method1("submit", (context.getattr("run")?, task)) +} + +fn is_os_error(py: Python<'_>, error: &pyo3::PyErr) -> bool { + error.is_instance_of::(py) +} + +/// The background test plan's `result`, with its project, adapter and target directory. +type Planning<'py> = (Bound<'py, PyAny>, [Bound<'py, PyAny>; 3]); + +impl PreparedCompileArtifacts { + /// Where the compile notes this task's artifact writes. + fn recorder(slf: &Bound<'_, Self>) -> Option> { + slf.borrow() + .artifact_writes + .as_ref() + .map(|recorder| recorder.clone_ref(slf.py())) + } + + /// Whether staging pays off: uncached, mid-sized projects whose memory stays within budget. + fn stages_artifacts(slf: &Bound<'_, Self>, project: &Bound<'_, PyAny>) -> PyResult { + let models: usize = project.getattr("models")?.len()?; + let minimum: usize = slf.getattr("min_prepared_models")?.extract()?; + let maximum: usize = slf.getattr("max_prepared_models")?.extract()?; + Ok(project.getattr("compile_cache_dir")?.is_none() + && minimum <= models + && models <= maximum) + } + + /// Stage artifacts beside `target_dir` so publication can move them instead of copying. + fn start_staging<'py>( + slf: &Bound<'py, Self>, + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + target_dir: &Bound<'py, PyAny>, + ) -> PyResult { + let py: Python<'py> = slf.py(); + let hex: String = imported(py, "uuid", "uuid4")? + .call0()? + .getattr("hex")? + .extract()?; + let staging_dir: Bound<'py, PyAny> = target_dir.div(format!("{STAGING_PREFIX}{hex}"))?; + let prepared: PyResult<()> = (|| { + if !target_dir.call_method0("is_dir")?.is_truthy()? { + let _ = target_dir.call_method( + "mkdir", + (), + Some(&keywords(py, &[("parents", true.into_bound_py_any(py)?)])?), + )?; + slf.borrow_mut().created_target_dir = Some(target_dir.clone().unbind()); + } + let lock: Bound<'py, PyAny> = imported(py, FILELOCK_MODULE, "FileLock")? + .call1((owner_lock_path(&staging_dir)?,))?; + slf.borrow_mut().owner_lock = Some(lock.clone().unbind()); + let _ = lock.call_method0("acquire")?; + let _ = slf.call_method1("create_staging_directory", (&staging_dir,))?; + Ok(()) + })(); + if let Err(error) = prepared { + if is_os_error(py, &error) { + return Ok(false); + } + return Err(error); + } + let executor: Bound<'py, PyAny> = single_thread_executor(py, ARTIFACT_THREADS)?; + let (project, adapter, staged) = ( + project.clone().unbind(), + adapter.clone().unbind(), + staging_dir.clone().unbind(), + ); + let recorder: Option> = Self::recorder(slf); + let prepare = move |arguments: &Bound<'_, PyTuple>, + _: Option<&Bound<'_, PyDict>>| + -> PyResult> { + let py: Python<'_> = arguments.py(); + let staging_dir: &Bound<'_, PyAny> = staged.bind(py); + let written: Bound<'_, PyAny> = write_static_target( + staging_dir, + (adapter.bind(py), project.bind(py)), + &py.None().into_bound(py), + Prepared { + planned_tests: None, + recorder: recorder.as_ref(), + }, + )?; + let files: Vec = + staged_artifact_files(py, staging_dir.div(COMPILED_DIR)?.extract()?)?; + Ok( + PyTuple::new(py, [written, PyFrozenSet::new(py, &files)?.into_any()])? + .into_any() + .unbind(), + ) + }; + let future: Bound<'py, PyAny> = submit_in_context( + &executor, + PyCFunction::new_closure(py, None, None, prepare)?, + )?; + let mut state: PyRefMut<'py, Self> = slf.borrow_mut(); + state.staging_dir = Some(staging_dir.unbind()); + state.executor = Some(executor.unbind()); + state.future = Some(future.unbind()); + Ok(true) + } + + /// Plan SQL tests against the published target while the compile continues. + fn start_test_planning<'py>( + slf: &Bound<'py, Self>, + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + target_dir: &Bound<'py, PyAny>, + ) -> PyResult<()> { + let py: Python<'py> = slf.py(); + let executor: Bound<'py, PyAny> = single_thread_executor(py, TEST_THREADS)?; + let (planned_project, planned_adapter, planned_target) = ( + project.clone().unbind(), + adapter.clone().unbind(), + target_dir.clone().unbind(), + ); + let recorder: Option> = Self::recorder(slf); + let plan = move |arguments: &Bound<'_, PyTuple>, + _: Option<&Bound<'_, PyDict>>| + -> PyResult> { + let py: Python<'_> = arguments.py(); + Ok(call_with( + &imported(py, "sqlbuild._native", "plan_static_sql_tests")?, + &[ + ("target_dir", planned_target.bind(py).clone()), + ("adapter", planned_adapter.bind(py).clone()), + ("project", planned_project.bind(py).clone()), + ( + "artifact_writes", + recorder.as_ref().map_or_else( + || py.None().into_bound(py), + |recorder| recorder.bind(py).clone().into_any(), + ), + ), + ], + )? + .unbind()) + }; + let future: Bound<'py, PyAny> = + submit_in_context(&executor, PyCFunction::new_closure(py, None, None, plan)?)?; + let mut state: PyRefMut<'py, Self> = slf.borrow_mut(); + state.planning_inputs = Some(( + project.clone().unbind(), + adapter.clone().unbind(), + target_dir.clone().unbind(), + )); + state.executor = Some(executor.unbind()); + state.planned_tests = Some(future.unbind()); + Ok(()) + } + + /// The background test plan's `result` and its project, adapter and target, when planning. + fn planning<'py>(slf: &Bound<'py, Self>) -> PyResult>> { + let py: Python<'py> = slf.py(); + let state: PyRef<'py, Self> = slf.borrow(); + let (Some(planned), Some((project, adapter, target_dir))) = + (&state.planned_tests, &state.planning_inputs) + else { + return Ok(None); + }; + Ok(Some(( + planned.bind(py).getattr("result")?, + [ + project.bind(py).clone(), + adapter.bind(py).clone(), + target_dir.bind(py).clone(), + ], + ))) + } + + /// The staged preparation's `(written, files)`, or `None` when staging failed with `OSError`. + fn staged_result<'py>(future: &Bound<'py, PyAny>) -> PyResult>> { + match future.call_method0("result") { + Ok(result) => Ok(Some(result)), + Err(error) if is_os_error(future.py(), &error) => Ok(None), + Err(error) => Err(error), + } + } +} + +#[pymethods] +impl PreparedCompileArtifacts { + #[new] + #[pyo3(signature = (*, enabled, artifact_writes=None))] + fn new(enabled: bool, artifact_writes: Option>) -> Self { + Self { + enabled, + artifact_writes, + staging_dir: None, + owner_lock: None, + created_target_dir: None, + executor: None, + future: None, + planned_tests: None, + planning_inputs: None, + } + } + + #[classattr] + fn min_prepared_models() -> usize { + MIN_PREPARED_ARTIFACT_MODELS + } + + #[classattr] + fn max_prepared_models() -> usize { + MAX_PREPARED_ARTIFACT_MODELS + } + + #[getter] + fn enabled(&self) -> bool { + self.enabled + } + + fn __enter__(slf: Bound<'_, Self>) -> Bound<'_, Self> { + slf + } + + #[pyo3(signature = (*_exception))] + fn __exit__(slf: &Bound<'_, Self>, _exception: &Bound<'_, PyTuple>) -> PyResult<()> { + let py: Python<'_> = slf.py(); + let (executor, staging_dir, owner_lock, created) = { + let state: PyRef<'_, Self> = slf.borrow(); + ( + state.executor.as_ref().map(|value| value.clone_ref(py)), + state.staging_dir.as_ref().map(|value| value.clone_ref(py)), + state.owner_lock.as_ref().map(|value| value.clone_ref(py)), + state + .created_target_dir + .as_ref() + .map(|value| value.clone_ref(py)), + ) + }; + let shutdown: PyResult<()> = match executor { + Some(executor) => executor + .bind(py) + .call_method( + "shutdown", + (), + Some(&keywords( + py, + &[ + ("wait", true.into_bound_py_any(py)?), + ("cancel_futures", true.into_bound_py_any(py)?), + ], + )?), + ) + .map(|_| ()), + None => Ok(()), + }; + let cleanup: PyResult<()> = (|| { + if let Some(staging_dir) = staging_dir { + let _ = imported(py, "shutil", "rmtree")?.call( + (staging_dir,), + Some(&keywords( + py, + &[("ignore_errors", true.into_bound_py_any(py)?)], + )?), + )?; + } + if let Some(lock) = owner_lock { + release_and_unlink(lock.bind(py))?; + } + if let Some(directory) = created + && let Err(error) = directory.bind(py).call_method0("rmdir") + && !is_os_error(py, &error) + { + return Err(error); + } + Ok(()) + })(); + cleanup?; + shutdown + } + + /// Delete staging directories whose owner lock this compile can hold for the removal. + #[pyo3(signature = (*, target_dir))] + fn remove_abandoned_staging(&self, target_dir: &Bound<'_, PyAny>) -> PyResult<()> { + if !self.enabled { + return Ok(()); + } + let py: Python<'_> = target_dir.py(); + let directories: Bound<'_, PyAny> = py + .import("builtins")? + .getattr("sorted")? + .call1((target_dir.call_method1("glob", (format!("{STAGING_PREFIX}*"),))?,))?; + for directory in directories.try_iter()? { + let directory: Bound<'_, PyAny> = directory?; + if directory.getattr("suffix")?.eq(LOCK_SUFFIX)? { + continue; + } + if !remove_abandoned_directory(&directory)? { + return Ok(()); + } + } + Ok(()) + } + + /// Create the staging directory and mark it with its own name as the owner. + fn create_staging_directory(&self, staging_dir: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = staging_dir.py(); + let _ = staging_dir.call_method0("mkdir")?; + let _ = staging_dir.div(OWNER_MARKER)?.call_method( + "write_text", + (staging_dir.getattr("name")?,), + Some(&keywords( + py, + &[("encoding", "utf-8".into_bound_py_any(py)?)], + )?), + )?; + Ok(()) + } + + /// Stage artifacts beside `target_dir` when worthwhile, otherwise plan SQL tests ahead. + #[pyo3(signature = (**options))] + fn start(slf: &Bound<'_, Self>, options: Option>) -> PyResult<()> { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + slf.py(), + "start", + options, + ( + &["project", "adapter", "target_dir", "stage_artifacts"], + &[], + ), + )?; + if !slf.borrow().enabled { + return Ok(()); + } + let (project, adapter, target_dir) = ( + &arguments.value("project")?, + &arguments.value("adapter")?, + &arguments.value("target_dir")?, + ); + if arguments.flag("stage_artifacts")? + && Self::stages_artifacts(slf, project)? + && Self::start_staging(slf, project, adapter, target_dir)? + { + return Ok(()); + } + if project.getattr("sql_tests")?.is_truthy()? { + Self::start_test_planning(slf, project, adapter, target_dir)?; + } + Ok(()) + } + + /// Publish what was prepared, or `None` when nothing was staged. + #[pyo3(signature = (*, target_dir, manifest))] + fn publish<'py>( + slf: &Bound<'py, Self>, + target_dir: &Bound<'py, PyAny>, + manifest: &Bound<'py, PyAny>, + ) -> PyResult>> { + let py: Python<'py> = slf.py(); + if let Some((result, [project, adapter, _])) = Self::planning(slf)? { + let recorder: Option> = Self::recorder(slf); + let prepared: Prepared<'_, 'py> = Prepared { + planned_tests: Some(&result), + recorder: recorder.as_ref(), + }; + return write_static_target(target_dir, (&adapter, &project), manifest, prepared) + .map(Some); + } + let (future, staging_dir) = { + let state: PyRef<'py, Self> = slf.borrow(); + match (&state.future, &state.staging_dir) { + (Some(future), Some(staging_dir)) => { + (future.bind(py).clone(), staging_dir.bind(py).clone()) + } + _ => return Ok(None), + } + }; + let Some(result) = Self::staged_result(&future)? else { + return Ok(None); + }; + let (prepared, expected_files): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + result.extract()?; + if !owns_staging_directory(&staging_dir)? { + let name: String = staging_dir.getattr("name")?.extract()?; + return Err(staging_changed( + py, + &format!( + "staging directory '{name}' was removed or replaced before publication; \ + no artifact was published" + ), + )?); + } + let recorder: Option> = Self::recorder(slf); + publish_staged_target( + (&prepared, &expected_files), + target_dir, + manifest, + recorder.as_ref(), + ) + .map(Some) + } + + /// Background SQL test planning diagnostics, without publishing any artifact. + fn planning_diagnostics<'py>(slf: &Bound<'py, Self>) -> PyResult>> { + let py: Python<'py> = slf.py(); + if let Some((result, [project, adapter, target_dir])) = Self::planning(slf)? { + let recorder: Option> = Self::recorder(slf); + let prepared: Prepared<'_, 'py> = Prepared { + planned_tests: Some(&result), + recorder: recorder.as_ref(), + }; + return planning_diagnostics(&target_dir, (&adapter, &project), prepared) + .map(|diagnostics| Some(diagnostics.into_any())); + } + let future: Option> = slf + .borrow() + .future + .as_ref() + .map(|future| future.bind(py).clone()); + let Some(future) = future else { + return Ok(None); + }; + match Self::staged_result(&future)? { + Some(result) => Ok(Some(result.get_item(0)?.getattr("diagnostics")?)), + None => Ok(None), + } + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs new file mode 100644 index 0000000000..afaea3f886 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/registration.rs @@ -0,0 +1,14 @@ +//! Register the `sqb compile` report, artifact writing and reuse recording. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::compile_command::{ + prepared_artifacts, report, reuse_recording, target_writer, +}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + report::register(module)?; + reuse_recording::register(module)?; + target_writer::register(module)?; + prepared_artifacts::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs new file mode 100644 index 0000000000..4865d9f632 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/report.rs @@ -0,0 +1,1089 @@ +//! The `sqb compile` report: human text, the JSON report, and the JSON report of a stopped compile. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{ + PyDict, PyDictMethods, PyList, PySet, PySetMethods, PyString, PyTuple, PyTupleMethods, +}; +use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::main::format_compile_text::format_compile_text as compile_text; +use sqlbuild_core::cli_text::main::semantic_coverage_notice::semantic_coverage_notice as notice_text; +use sqlbuild_core::cli_text::models::{ + CliStyle, CompileTextReport, DiagnosticText, LineageNote, LocationText, ModelLine, RelatedText, +}; + +use crate::bindings::_helpers::compile_command::surrogates::SurrogateText; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::compile_outputs::json_report::{ + emit_json_report, emit_orjson_report, +}; + +const SEMANTIC_COVERAGE_MODULE: &str = "sqlbuild.compiler.compile.main._semantic_coverage"; +const HOOK_ENTRIES_MODULE: &str = "sqlbuild.compiler.discovery.main.serialize_hook_entries"; +const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; +const HOOK_IDENTITIES_MODULE: &str = "sqlbuild.compiler.python_nodes.main._hook_identities"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const METADATA_MODULE: &str = "importlib.metadata"; +const TARGET_DIRECTORY: &str = "target"; +const COMPILED_DIRECTORY: &str = "compiled"; +const MANIFEST_ARTIFACT: &str = "target/manifest.json"; +const PACKAGE_NAME: &str = "sqlbuild"; +const UNKNOWN_VERSION: &str = "unknown"; +const MODEL_TYPE: &str = "model"; +const SEED_TYPE: &str = "seed"; +const FUNCTION_TYPES: [&str; 2] = ["udf", "table_fn"]; +const SQL_TEST_TYPE: &str = "sql_test"; +const ERROR_SEVERITY: &str = "error"; +const WARNING_SEVERITY: &str = "warning"; +const RICH_LINEAGE: &str = "rich"; +const NO_LINEAGE: &str = "none"; +const DECIMAL_KIND: &str = "decimal"; +const REPORT_OPTIONS: (&[&str], &[&str]) = ( + &["graph", "written", "manifest", "lineage", "diagnostics"], + &[ + "lineage_mode", + "selected_keys", + "sql_validation_enabled", + "use_color", + "timings_ms", + ], +); +const HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; + +/// What both compile reports are built from. +struct ReportInputs<'py> { + graph: Bound<'py, PyAny>, + project: Bound<'py, PyAny>, + written: Bound<'py, PyAny>, + manifest: bool, + lineage: Bound<'py, PyAny>, + diagnostics: Vec>, + lineage_mode: String, + selected_keys: Bound<'py, PyAny>, + sql_validation_enabled: bool, +} + +impl<'py> ReportInputs<'py> { + fn read(arguments: &KeywordArguments<'py>) -> PyResult { + let graph: Bound<'py, PyAny> = arguments.value("graph")?; + let lineage_mode: Bound<'py, PyAny> = arguments.value("lineage_mode")?; + Ok(Self { + project: graph.getattr("project")?, + graph, + written: arguments.value("written")?, + manifest: arguments.flag("manifest")?, + lineage: arguments.value("lineage")?, + diagnostics: arguments + .value("diagnostics")? + .try_iter()? + .collect::>()?, + lineage_mode: if lineage_mode.is_none() { + "fast".to_owned() + } else { + lineage_mode.str()?.to_string() + }, + selected_keys: arguments.value("selected_keys")?, + sql_validation_enabled: !arguments.passed("sql_validation_enabled")? + || arguments.flag("sql_validation_enabled")?, + }) + } + + fn py(&self) -> Python<'py> { + self.graph.py() + } + + /// The selected keys' names of `resource_type`, or `None` when nothing was deselected. + fn selected_names(&self, resource_type: &str) -> PyResult>> { + if self.selected_keys.is_none() { + return Ok(None); + } + let names: Bound<'py, PySet> = PySet::empty(self.py())?; + for key in self.selected_keys.try_iter()? { + let key: Bound<'py, PyAny> = key?; + if key.getattr("resource_type")?.eq(resource_type)? { + names.add(key.getattr("name")?)?; + } + } + Ok(Some(names)) + } + + /// The project's models, kept to the selected ones. + fn selected_models(&self) -> PyResult>> { + let models: Vec> = self + .project + .getattr("models")? + .try_iter()? + .collect::>()?; + let Some(names) = self.selected_names(MODEL_TYPE)? else { + return Ok(models); + }; + let mut selected: Vec> = Vec::new(); + for model in models { + if names.contains(model.getattr("name")?)? { + selected.push(model); + } + } + Ok(selected) + } + + /// How many graph or selected resources have one of `types`. + fn selected_count(&self, types: &[&str]) -> PyResult { + let mut count: usize = 0; + if self.selected_keys.is_none() { + for key in self + .graph + .getattr("native")? + .call_method0("keys")? + .try_iter()? + { + let (kind, _): (String, Bound<'py, PyAny>) = key?.extract()?; + count += usize::from(types.contains(&kind.as_str())); + } + return Ok(count); + } + for key in self.selected_keys.try_iter()? { + let kind: String = key?.getattr("resource_type")?.str()?.to_string(); + count += usize::from(types.contains(&kind.as_str())); + } + Ok(count) + } + + fn severity_count(&self, severity: &str) -> PyResult { + let mut count: usize = 0; + for diagnostic in &self.diagnostics { + count += usize::from(diagnostic.getattr("severity")?.eq(severity)?); + } + Ok(count) + } + + /// `selected_semantic_coverage`: each selected model's partial-check reasons. + fn semantic_coverage(&self) -> PyResult> { + let py: Python<'py> = self.py(); + if !self.sql_validation_enabled { + return Ok(PyDict::new(py)); + } + let reasons: Bound<'py, PyDict> = + imported(py, SEMANTIC_COVERAGE_MODULE, "get_semantic_coverage")? + .call1((&self.project,))? + .downcast_into()?; + let Some(names) = self.selected_names(MODEL_TYPE)? else { + return Ok(reasons); + }; + let selected: Bound<'py, PyDict> = PyDict::new(py); + for (name, values) in reasons.iter() { + if names.contains(&name)? { + selected.set_item(name, values)?; + } + } + Ok(selected) + } + + fn semantic_notice(&self, reader: &SurrogateText) -> PyResult> { + let mut reasons: Vec<(String, Vec)> = Vec::new(); + for (name, values) in self.semantic_coverage()?.iter() { + let values: Vec = values + .try_iter()? + .map(|value| reader.text(&value?)) + .collect::>()?; + reasons.push((reader.text(&name)?, values)); + } + Ok(notice_text(&reasons)) + } + + /// `written.target_dir / "compiled"` relative to its `target` directory, POSIX style. + fn compiled_dir(&self) -> PyResult> { + let py: Python<'py> = self.py(); + let path: Bound<'py, PyAny> = self + .written + .getattr("target_dir")? + .div(COMPILED_DIRECTORY)?; + let parts: Bound<'py, PyTuple> = path.getattr("parts")?.downcast_into()?; + for (index, part) in parts.iter().enumerate() { + if part.eq(TARGET_DIRECTORY)? { + let kept: Bound<'py, PyTuple> = parts.get_slice(index, parts.len()); + return imported(py, "pathlib", "Path")? + .call1(kept)? + .call_method0("as_posix"); + } + } + path.call_method0("as_posix") + } +} + +/// `int(value)` of a location field, or `None`. +fn optional_number(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + value.extract().map(Some) +} + +/// Each authored file's text by path: models with SQL, then sources, then SQL tests. +fn source_texts<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let texts: Bound<'py, PyDict> = PyDict::new(project.py()); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let authored: Bound<'py, PyAny> = model.getattr("authored_sql")?; + if authored.is_truthy()? { + texts.set_item(model.getattr("relative_path")?, authored)?; + } + } + for (collection, file) in [("sources", "source_file"), ("sql_tests", "test_file")] { + for resource in project.getattr(collection)?.try_iter()? { + let file: Bound<'py, PyAny> = resource?.getattr(file)?; + texts.set_item(file.getattr("relative_path")?, file.getattr("contents")?)?; + } + } + Ok(texts) +} + +fn location_text( + reader: &SurrogateText, + location: &Bound<'_, PyAny>, + texts: &Bound<'_, PyDict>, +) -> PyResult { + let path: Bound<'_, PyAny> = location.getattr("path")?; + Ok(LocationText { + path: reader.text(&path)?, + line: location.getattr("line")?.extract()?, + column: location.getattr("column")?.extract()?, + end_line: optional_number(&location.getattr("end_line")?)?, + end_column: optional_number(&location.getattr("end_column")?)?, + source: texts + .get_item(path)? + .map(|text| reader.text(&text)) + .transpose()?, + }) +} + +fn texts_of(reader: &SurrogateText, value: &Bound<'_, PyAny>) -> PyResult> { + value.try_iter()?.map(|item| reader.text(&item?)).collect() +} + +fn diagnostic_text( + reader: &SurrogateText, + diagnostic: &Bound<'_, PyAny>, + texts: &Bound<'_, PyDict>, + occurrences: usize, +) -> PyResult { + let location: Bound<'_, PyAny> = diagnostic.getattr("location")?; + let mut related: Vec = Vec::new(); + for item in diagnostic.getattr("related_locations")?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + related.push(RelatedText { + label: reader.text(&item.getattr("label")?)?, + location: location_text(reader, &item.getattr("location")?, texts)?, + message: reader.optional(&item.getattr("message")?)?, + }); + } + Ok(DiagnosticText { + severity: reader.text(&diagnostic.getattr("severity")?)?, + code: reader.text(&diagnostic.getattr("code")?)?, + message: reader.text(&diagnostic.getattr("message")?)?, + resource_type: reader.optional(&diagnostic.getattr("resource_type")?)?, + resource_name: reader.optional(&diagnostic.getattr("resource_name")?)?, + location: if location.is_none() { + None + } else { + Some(location_text(reader, &location, texts)?) + }, + path: reader.optional(&diagnostic.getattr("path")?)?, + related, + notes: texts_of(reader, &diagnostic.getattr("notes")?)?, + affected_rules: texts_of(reader, &diagnostic.getattr("affected_rules")?)?, + help: reader.optional(&diagnostic.getattr("help")?)?, + occurrences, + }) +} + +/// The diagnostics the human report shows, each repeated warning folded into its first. +fn shown_diagnostics( + inputs: &ReportInputs<'_>, + reader: &SurrogateText, +) -> PyResult> { + let py: Python<'_> = inputs.py(); + if inputs.diagnostics.is_empty() { + return Ok(Vec::new()); + } + let texts: Bound<'_, PyDict> = source_texts(&inputs.project)?; + let warning_counts: Bound<'_, PyDict> = PyDict::new(py); + for diagnostic in &inputs.diagnostics { + if diagnostic.getattr("severity")?.eq(WARNING_SEVERITY)? { + let count: usize = warning_counts + .get_item(diagnostic)? + .map_or(Ok(0), |count| count.extract())?; + warning_counts.set_item(diagnostic, count + 1)?; + } + } + let shown_warnings: Bound<'_, PyDict> = PyDict::new(py); + let mut shown: Vec = Vec::new(); + for diagnostic in &inputs.diagnostics { + if shown_warnings.contains(diagnostic)? { + continue; + } + if diagnostic.getattr("severity")?.eq(WARNING_SEVERITY)? { + shown_warnings.set_item(diagnostic, true)?; + } + let occurrences: usize = warning_counts + .get_item(diagnostic)? + .map_or(Ok(1), |count| count.extract())?; + shown.push(diagnostic_text(reader, diagnostic, &texts, occurrences)?); + } + Ok(shown) +} + +/// Each model an error diagnostic names, other than SQL tests. +fn error_models<'py>(inputs: &ReportInputs<'py>) -> PyResult> { + let names: Bound<'py, PySet> = PySet::empty(inputs.py())?; + for diagnostic in &inputs.diagnostics { + let name: Bound<'_, PyAny> = diagnostic.getattr("resource_name")?; + if diagnostic.getattr("is_error")?.is_truthy()? + && !name.is_none() + && !diagnostic.getattr("resource_type")?.eq(SQL_TEST_TYPE)? + { + names.add(name)?; + } + } + Ok(names) +} + +/// The human-readable compile report. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn format_compile_text<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "format_compile_text", options, REPORT_OPTIONS)?; + let inputs: ReportInputs<'py> = ReportInputs::read(&arguments)?; + let reader: SurrogateText = SurrogateText::default(); + let failed: Bound<'py, PySet> = error_models(&inputs)?; + let mut models: Vec = Vec::new(); + for model in inputs.selected_models()? { + let name: Bound<'py, PyAny> = model.getattr("name")?; + models.push(ModelLine { + failed: failed.contains(&name)?, + name: reader.text(&name)?, + column_count: model.getattr("output_column_count")?.extract()?, + }); + } + let lineage: LineageNote = if !inputs.lineage.is_none() && inputs.lineage_mode == RICH_LINEAGE { + LineageNote::Rich + } else if inputs.lineage.is_none() + && inputs + .project + .getattr("settings")? + .getattr("sql_analysis")? + .is_truthy()? + && inputs.lineage_mode != NO_LINEAGE + { + LineageNote::Unavailable + } else { + LineageNote::Silent + }; + let report: CompileTextReport = CompileTextReport { + style: CliStyle { + use_color: arguments.flag("use_color")?, + }, + models, + semantic_notice: inputs.semantic_notice(&reader)?, + diagnostics: shown_diagnostics(&inputs, &reader)?, + seed_count: inputs.selected_count(&[SEED_TYPE])?, + function_count: inputs.selected_count(&FUNCTION_TYPES)?, + error_count: inputs.severity_count(ERROR_SEVERITY)?, + warning_count: inputs.severity_count(WARNING_SEVERITY)?, + compiled_dir: reader.text(&inputs.compiled_dir()?)?, + manifest: inputs.manifest, + lineage, + }; + reader.python(py, &compile_text(&report)) +} + +/// The partial semantic-check notice of the selected models, printed beside `--json` output. +pub(crate) fn json_semantic_notice<'py>( + graph: &Bound<'py, PyAny>, + selected_keys: &Bound<'py, PyAny>, + sql_validation_enabled: bool, +) -> PyResult>> { + let py: Python<'py> = graph.py(); + let reader: SurrogateText = SurrogateText::default(); + let notice: Option = ReportInputs { + project: graph.getattr("project")?, + graph: graph.clone(), + written: py.None().into_bound(py), + manifest: false, + lineage: py.None().into_bound(py), + diagnostics: Vec::new(), + lineage_mode: String::new(), + selected_keys: selected_keys.clone(), + sql_validation_enabled, + } + .semantic_notice(&reader)?; + notice.map(|notice| reader.python(py, ¬ice)).transpose() +} + +fn dict<'py>( + py: Python<'py>, + entries: Vec<(&str, Bound<'py, PyAny>)>, +) -> PyResult> { + let built: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in entries { + built.set_item(key, value)?; + } + Ok(built) +} + +fn any<'py, T: IntoPyObjectExt<'py>>(py: Python<'py>, value: T) -> PyResult> { + value.into_bound_py_any(py) +} + +/// The installed SQLBuild version, or `unknown`. +fn sqlbuild_version(py: Python<'_>) -> PyResult> { + match imported(py, METADATA_MODULE, "version")?.call1((PACKAGE_NAME,)) { + Ok(version) => Ok(version), + Err(error) + if error.is_instance(py, &imported(py, METADATA_MODULE, "PackageNotFoundError")?) => + { + any(py, UNKNOWN_VERSION) + } + Err(error) => Err(error), + } +} + +fn location_json<'py>(location: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = location.py(); + let payload: Bound<'py, PyDict> = dict( + py, + vec![ + ("path", location.getattr("path")?.str()?.into_any()), + ("line", location.getattr("line")?), + ("column", location.getattr("column")?), + ], + )?; + for key in ["end_line", "end_column"] { + let value: Bound<'py, PyAny> = location.getattr(key)?; + if !value.is_none() { + payload.set_item(key, value)?; + } + } + Ok(payload) +} + +/// `_diagnostic_to_json`: one diagnostic's JSON fields, omitting the unset ones. +fn diagnostic_json<'py>(diagnostic: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = diagnostic.py(); + let payload: Bound<'py, PyDict> = dict( + py, + vec![ + ("phase", diagnostic.getattr("phase")?.str()?.into_any()), + ( + "severity", + diagnostic.getattr("severity")?.str()?.into_any(), + ), + ("code", diagnostic.getattr("code")?), + ("message", diagnostic.getattr("message")?), + ], + )?; + let resource_type: Bound<'py, PyAny> = diagnostic.getattr("resource_type")?; + if !resource_type.is_none() { + payload.set_item("resource_type", resource_type.str()?)?; + } + for key in ["resource_name", "column_name"] { + let value: Bound<'py, PyAny> = diagnostic.getattr(key)?; + if !value.is_none() { + payload.set_item(key, value)?; + } + } + let path: Bound<'py, PyAny> = diagnostic.getattr("path")?; + if !path.is_none() { + payload.set_item("path", path.str()?)?; + } + for key in ["line", "column"] { + let value: Bound<'py, PyAny> = diagnostic.getattr(key)?; + if !value.is_none() { + payload.set_item(key, value)?; + } + } + let location: Bound<'py, PyAny> = diagnostic.getattr("location")?; + if !location.is_none() { + payload.set_item("location", location_json(&location)?)?; + } + let related: Bound<'py, PyAny> = diagnostic.getattr("related_locations")?; + if related.is_truthy()? { + let mut items: Vec> = Vec::new(); + for item in related.try_iter()? { + let item: Bound<'py, PyAny> = item?; + let entry: Bound<'py, PyDict> = dict( + py, + vec![ + ("label", item.getattr("label")?), + ( + "location", + location_json(&item.getattr("location")?)?.into_any(), + ), + ], + )?; + let message: Bound<'py, PyAny> = item.getattr("message")?; + if !message.is_none() { + entry.set_item("message", message)?; + } + items.push(entry); + } + payload.set_item("related_locations", PyList::new(py, items)?)?; + } + let help: Bound<'py, PyAny> = diagnostic.getattr("help")?; + if !help.is_none() { + payload.set_item("help", help)?; + } + for key in ["notes", "affected_rules"] { + let value: Bound<'py, PyAny> = diagnostic.getattr(key)?; + if value.is_truthy()? { + payload.set_item( + key, + PyList::new(py, value.try_iter()?.collect::>>()?)?, + )?; + } + } + Ok(payload) +} + +/// `item[key] = value.qualified_name` when the destination names a relation. +fn qualified_name(item: &Bound<'_, PyDict>, resource: &Bound<'_, PyAny>) -> PyResult<()> { + let name: Bound<'_, PyAny> = resource.getattr("destination")?.getattr("qualified_name")?; + if !name.is_none() { + item.set_item("qualified_name", name)?; + } + Ok(()) +} + +fn lineage_summary<'py>( + lineage: &Bound<'py, PyAny>, + model: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = model.py(); + let name: Bound<'py, PyAny> = model.getattr("name")?; + if lineage.is_none() || !lineage.call_method1("has_model", (&name,))?.is_truthy()? { + return dict(py, vec![("available", any(py, false)?)]); + } + dict( + py, + vec![ + ("available", any(py, true)?), + ("column_count", model.getattr("output_column_count")?), + ( + "edge_count", + lineage.call_method1("edge_count_targeting", (&name,))?, + ), + ( + "has_star", + lineage.call_method1("model_has_star", (&name,))?, + ), + ], + ) +} + +fn model_resource<'py>( + inputs: &ReportInputs<'py>, + model: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = model.py(); + let key: Bound<'py, PyAny> = model.getattr("key")?; + let mut depends_on: Vec> = Vec::new(); + for upstream in inputs + .graph + .getattr("native")? + .call_method1( + "upstream", + ((key.getattr("resource_type")?.str()?, key.getattr("name")?),), + )? + .try_iter()? + { + let (kind, name): (Bound<'py, PyAny>, Bound<'py, PyAny>) = upstream?.extract()?; + depends_on.push(dict(py, vec![("resource_type", kind), ("name", name)])?); + } + let values: Bound<'py, PyAny> = model.getattr("config")?.getattr("values")?; + let item: Bound<'py, PyDict> = dict( + py, + vec![ + ("name", model.getattr("name")?), + ( + "relative_path", + model.getattr("relative_path")?.str()?.into_any(), + ), + ( + "materialized", + values + .call_method1("get", ("materialized", "view"))? + .str()? + .into_any(), + ), + ("column_count", model.getattr("output_column_count")?), + ("depends_on", PyList::new(py, depends_on)?.into_any()), + ( + "lineage", + lineage_summary(&inputs.lineage, model)?.into_any(), + ), + ("query_sql", model.getattr("query_sql")?), + ], + )?; + qualified_name(&item, model)?; + for hook_key in HOOK_KEYS { + let hooks: Bound<'py, PyAny> = values.call_method1("get", (hook_key,))?; + if !hooks.is_none() { + item.set_item( + hook_key, + call_with( + &imported(py, HOOK_ENTRIES_MODULE, "serialize_hook_entries")?, + &[ + ("value", hooks), + ( + "sql_fields", + imported(py, DISCOVERY_CONSTANTS_MODULE, "SQL_HOOK_OUTPUT_FIELDS")?, + ), + ], + )?, + )?; + } + } + Ok(item) +} + +fn hook_definitions<'py>(project: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = project.py(); + let mut definitions: Vec> = Vec::new(); + for hook in project.getattr("sql_hook_files")?.try_iter()? { + let hook: Bound<'py, PyAny> = hook?; + definitions.push(dict( + py, + vec![ + ("type", any(py, "sql")?), + ("name", hook.getattr("name")?), + ( + "relative_path", + hook.getattr("relative_path")?.call_method0("as_posix")?, + ), + ("description", hook.getattr("description")?), + ("sql", hook.getattr("sql_body")?), + ], + )?); + } + let functions: Bound<'py, PyAny> = project.getattr("hook_functions")?; + let identities: Bound<'py, PyAny> = + imported(py, HOOK_IDENTITIES_MODULE, "build_hook_identities")?.call1((&functions,))?; + for hook in functions.try_iter()? { + let hook: Bound<'py, PyAny> = hook?; + let identity: Bound<'py, PyAny> = identities.get_item(hook.getattr("name")?)?; + definitions.push(dict( + py, + vec![ + ("type", any(py, "python")?), + ("name", hook.getattr("name")?), + ( + "relative_path", + hook.getattr("relative_path")?.call_method0("as_posix")?, + ), + ("description", hook.getattr("description")?), + ("definition_hash", identity.getattr("definition_hash")?), + ("version_hash", identity.getattr("version_hash")?), + ], + )?); + } + PyList::new(py, definitions) +} + +fn function_resource<'py>(function: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = function.py(); + let return_columns: Bound<'py, PyAny> = function.getattr("return_columns")?; + let mut columns: Vec> = Vec::new(); + for column in return_columns.try_iter()? { + let column: Bound<'py, PyAny> = column?; + columns.push(dict( + py, + vec![ + ("name", column.getattr("name")?), + ("type", column.getattr("type")?), + ], + )?); + } + let item: Bound<'py, PyDict> = dict( + py, + vec![ + ("name", function.getattr("name")?), + ( + "relative_path", + function.getattr("relative_path")?.str()?.into_any(), + ), + ("language", function.getattr("language")?.getattr("value")?), + ("description", function.getattr("description")?), + ( + "return_kind", + any( + py, + if return_columns.is_truthy()? { + "table" + } else { + "scalar" + }, + )?, + ), + ("returns", function.getattr("returns")?), + ("return_columns", PyList::new(py, columns)?.into_any()), + ], + )?; + qualified_name(&item, function)?; + Ok(item) +} + +fn audit_resource<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + let item: Bound<'py, PyDict> = dict( + py, + vec![ + ("name", audit.getattr("name")?), + ( + "relative_path", + audit + .getattr("audit_file")? + .getattr("relative_path")? + .str()? + .into_any(), + ), + ], + )?; + let kind: Bound<'py, PyAny> = audit.getattr("attached_target_kind")?; + let target: Bound<'py, PyAny> = audit.getattr("attached_target_name")?; + if !kind.is_none() && !target.is_none() { + let attached: Bound<'py, PyDict> = dict( + py, + vec![("resource_type", kind.getattr("value")?), ("name", target)], + )?; + let column: Bound<'py, PyAny> = audit.getattr("attached_column_name")?; + if !column.is_none() { + attached.set_item("column_name", column)?; + } + item.set_item("attached_to", attached)?; + } + Ok(item) +} + +fn test_resource<'py>(test: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = test.py(); + let payload: Bound<'py, PyAny> = test.getattr("payload")?; + let expected_models: Bound<'py, PyList> = if payload.is_instance(&imported( + py, + COMPILE_MODELS_MODULE, + "CompiledModelSqlTestPayload", + )?)? { + PyList::new( + py, + payload + .getattr("expected_model_names")? + .try_iter()? + .collect::>>()?, + )? + } else { + PyList::empty(py) + }; + let test_file_path: Bound<'py, PyAny> = test.getattr("test_file")?.getattr("relative_path")?; + let item: Bound<'py, PyDict> = dict( + py, + vec![ + ("name", test.getattr("name")?), + ("relative_path", test_file_path.str()?.into_any()), + ("expected_models", expected_models.into_any()), + ], + )?; + let case_name: Bound<'py, PyAny> = test.getattr("case_name")?; + if case_name.is_none() { + return Ok(item); + } + let schema: Vec> = test + .getattr("parameter_schema")? + .try_iter()? + .collect::>()?; + let types: Bound<'py, PyDict> = PyDict::new(py); + let mut parameter_schema: Vec> = Vec::new(); + for parameter in &schema { + let kind: Bound<'py, PyAny> = parameter.getattr("value_type")?.getattr("value")?; + types.set_item(parameter.getattr("name")?, &kind)?; + } + for parameter in &schema { + parameter_schema.push(dict( + py, + vec![ + ("name", parameter.getattr("name")?), + ("type", parameter.getattr("value_type")?.getattr("value")?), + ("nullable", parameter.getattr("nullable")?), + ], + )?); + } + let mut parameters: Vec> = Vec::new(); + for pair in test.getattr("parameter_values")?.try_iter()? { + let (name, value): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair?.extract()?; + let inner: Bound<'py, PyAny> = value.getattr("value")?; + let shown: Bound<'py, PyAny> = if value.getattr("kind")?.eq(DECIMAL_KIND)? { + inner.str()?.into_any() + } else { + inner + }; + let kind: Bound<'py, PyAny> = types.as_any().get_item(&name)?; + parameters.push(dict( + py, + vec![("name", name), ("type", kind), ("value", shown)], + )?); + } + let source_path: Bound<'py, PyAny> = test.getattr("source_path")?; + let source_path: Bound<'py, PyAny> = if source_path.is_truthy()? { + source_path + } else { + test_file_path + }; + for (key, value) in [ + ("source_path", source_path.call_method0("as_posix")?), + ("block_index", test.getattr("block_index")?), + ("parent_name", test.getattr("parent_name")?), + ("case_name", case_name), + ("case_index", test.getattr("case_index")?), + ("case_fingerprint", test.getattr("case_fingerprint")?), + ( + "parameter_schema", + PyList::new(py, parameter_schema)?.into_any(), + ), + ("parameters", PyList::new(py, parameters)?.into_any()), + ] { + item.set_item(key, value)?; + } + Ok(item) +} + +/// Each item of `project.` through `resource`. +fn resources<'py>( + project: &Bound<'py, PyAny>, + collection: &str, + resource: impl Fn(&Bound<'py, PyAny>) -> PyResult>, +) -> PyResult> { + let items: Vec> = project + .getattr(collection)? + .try_iter()? + .map(|item| resource(&item?)) + .collect::>()?; + Ok(PyList::new(project.py(), items)?.into_any()) +} + +fn source_resource<'py>(source: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = source.py(); + dict( + py, + vec![ + ("name", source.getattr("name")?), + ( + "relative_path", + source + .getattr("source_file")? + .getattr("relative_path")? + .str()? + .into_any(), + ), + ( + "column_count", + any( + py, + source.getattr("source_entry")?.getattr("columns")?.len()?, + )?, + ), + ], + ) +} + +fn seed_resource<'py>(seed: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = seed.py(); + let item: Bound<'py, PyDict> = dict( + py, + vec![ + ("name", seed.getattr("name")?), + ( + "relative_path", + seed.getattr("seed_file")? + .getattr("relative_path")? + .str()? + .into_any(), + ), + ( + "column_count", + any(py, seed.getattr("schema_entry")?.getattr("columns")?.len()?)?, + ), + ], + )?; + qualified_name(&item, seed)?; + Ok(item) +} + +/// The machine-readable offline compile report. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn format_compile_json( + py: Python<'_>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "format_compile_json", options, REPORT_OPTIONS)?; + let inputs: ReportInputs<'_> = ReportInputs::read(&arguments)?; + let project: &Bound<'_, PyAny> = &inputs.project; + let mut has_errors: bool = false; + let mut diagnostics: Vec> = Vec::new(); + for diagnostic in &inputs.diagnostics { + has_errors = has_errors || diagnostic.getattr("is_error")?.is_truthy()?; + diagnostics.push(diagnostic_json(diagnostic)?); + } + let length = |collection: &str| -> PyResult { project.getattr(collection)?.len() }; + let summary: Bound<'_, PyDict> = dict( + py, + vec![ + ("models", any(py, length("models")?)?), + ("selected_models", any(py, inputs.selected_models()?.len())?), + ("sources", any(py, length("sources")?)?), + ("seeds", any(py, length("seeds")?)?), + ( + "selected_seeds", + any(py, inputs.selected_count(&[SEED_TYPE])?)?, + ), + ("functions", any(py, length("functions")?)?), + ( + "selected_functions", + any(py, inputs.selected_count(&FUNCTION_TYPES)?)?, + ), + ("audits", any(py, length("audits")?)?), + ("tests", any(py, length("sql_tests")?)?), + ( + "hooks", + any(py, length("sql_hook_files")? + length("hook_functions")?)?, + ), + ( + "execution_layers", + inputs + .graph + .getattr("native")? + .call_method0("model_layer_count")?, + ), + ("errors", any(py, inputs.severity_count(ERROR_SEVERITY)?)?), + ( + "warnings", + any(py, inputs.severity_count(WARNING_SEVERITY)?)?, + ), + ], + )?; + let resource_lists: Bound<'_, PyDict> = dict( + py, + vec![ + ( + "models", + resources(project, "models", |model| model_resource(&inputs, model))?, + ), + ("sources", resources(project, "sources", source_resource)?), + ("seeds", resources(project, "seeds", seed_resource)?), + ( + "functions", + resources(project, "functions", function_resource)?, + ), + ("audits", resources(project, "audits", audit_resource)?), + ("tests", resources(project, "sql_tests", test_resource)?), + ("hooks", hook_definitions(project)?.into_any()), + ], + )?; + let artifacts: Bound<'_, PyDict> = dict( + py, + vec![ + ( + "compiled_sql_dir", + inputs.compiled_dir()?.call_method1("__add__", ("/",))?, + ), + ( + "manifest", + if inputs.manifest { + any(py, MANIFEST_ARTIFACT)? + } else { + py.None().into_bound(py) + }, + ), + ], + )?; + let report: Bound<'_, PyDict> = dict( + py, + vec![ + ("version", sqlbuild_version(py)?), + ("command", any(py, "compile")?), + ("offline", any(py, true)?), + ("has_errors", any(py, has_errors)?), + ("summary", summary.into_any()), + ("diagnostics", PyList::new(py, diagnostics)?.into_any()), + ( + "semantic_checks_partial", + inputs.semantic_coverage()?.into_any(), + ), + ("compile_timings", arguments.value("timings_ms")?), + ("lineage_mode", any(py, inputs.lineage_mode.as_str())?), + ("resources", resource_lists.into_any()), + ("artifacts", artifacts.into_any()), + ], + )?; + emit_json_report(report.as_any()) +} + +/// A compile that stopped on a raised error, as one JSON diagnostic report. +#[pyfunction] +#[pyo3(signature = (*, code, message, help_text, location=None))] +pub(crate) fn format_compile_error_json<'py>( + code: Bound<'py, PyAny>, + message: Bound<'py, PyAny>, + help_text: Bound<'py, PyAny>, + location: Option>, +) -> PyResult { + let py: Python<'py> = code.py(); + let location: Bound<'py, PyAny> = location.unwrap_or_else(|| py.None().into_bound(py)); + let (path, line, column) = if location.is_none() { + (location.clone(), location.clone(), location.clone()) + } else { + ( + location.getattr("path")?, + location.getattr("line")?, + location.getattr("column")?, + ) + }; + let diagnostic: Bound<'py, PyAny> = call_with( + &imported(py, COMPILE_MODELS_MODULE, "CompilerDiagnostic")?, + &[ + ("phase", any(py, "compile")?), + ("severity", any(py, ERROR_SEVERITY)?), + ("code", code), + ("message", message), + ("path", path), + ("line", line), + ("column", column), + ("location", location), + ("help", help_text), + ], + )?; + let report: Bound<'py, PyDict> = dict( + py, + vec![ + ("version", sqlbuild_version(py)?), + ("command", any(py, "compile")?), + ("offline", any(py, true)?), + ("has_errors", any(py, true)?), + ("stopped", any(py, true)?), + ( + "diagnostics", + PyList::new(py, [diagnostic_json(&diagnostic)?])?.into_any(), + ), + ], + )?; + emit_orjson_report(report.as_any()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_compile_text, module)?)?; + module.add_function(wrap_pyfunction!(format_compile_json, module)?)?; + module.add_function(wrap_pyfunction!(format_compile_error_json, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs new file mode 100644 index 0000000000..aa64f0de47 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/reuse_recording.rs @@ -0,0 +1,577 @@ +//! What one full compile printed and left in each artifact, and storing it for compile reuse. + +use std::collections::HashMap; +use std::sync::{Mutex, PoisonError}; + +use pyo3::exceptions::{PyException, PyOSError}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; +use pyo3::{IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, +}; +use crate::bindings::_helpers::compile_outputs::reuse::artifact_digest; + +const REUSE_CONSTANTS_MODULE: &str = "sqlbuild.cli.compile_reuse.constants"; +const NATIVE_REUSE_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.native_reuse"; +const PROVIDER_SETTINGS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.provider_settings"; +const LAUNCH_RECORDS_MODULE: &str = "sqlbuild.cli.compile_reuse._helpers.launch_records"; +const MACRO_STORE_PATHS_MODULE: &str = + "sqlbuild.compiler.macro_bridge.main._macro_store_module_paths"; +const MISS_OUTCOME: &str = "miss"; +const HIT_OUTCOME: &str = "hit"; +const BYPASS_OUTCOME: &str = "bypass"; +const STORED: &str = "stored"; +const SKIPPED: &str = "skipped"; + +/// What a compile left at one artifact path: its digest, or the size and mtime it kept. +#[derive(Clone, Debug, PartialEq, Eq)] +enum RecordedArtifact { + Written(String), + Kept { size: u64, mtime_ns: i64 }, +} + +/// Artifacts one compile wrote or kept, by absolute path, following staged moves. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct CompileArtifactWrites { + artifacts: Mutex>, +} + +impl CompileArtifactWrites { + fn locked(&self) -> std::sync::MutexGuard<'_, HashMap> { + self.artifacts + .lock() + .unwrap_or_else(PoisonError::into_inner) + } + + fn record(&self, path: String, artifact: RecordedArtifact) { + let _ = self.locked().insert(path, artifact); + } + + fn move_one(&self, source: &str, destination: String) { + let mut artifacts = self.locked(); + if let Some(artifact) = artifacts.remove(source) { + let _ = artifacts.insert(destination, artifact); + } + } + + fn move_tree(&self, source: &str, destination: &str) { + let prefix: String = format!("{source}{}", std::path::MAIN_SEPARATOR); + let mut artifacts = self.locked(); + let moved: Vec<(String, RecordedArtifact)> = artifacts + .iter() + .filter(|(path, _)| path.starts_with(&prefix)) + .map(|(path, artifact)| { + ( + format!("{destination}{}", &path[source.len()..]), + artifact.clone(), + ) + }) + .collect(); + artifacts.retain(|path, _| !path.starts_with(&prefix)); + artifacts.extend(moved); + } +} + +#[pymethods] +impl CompileArtifactWrites { + /// Every recorded artifact by absolute path, as `(digest, size, mtime_ns)`. + #[getter] + fn artifacts<'py>(&self, py: Python<'py>) -> PyResult> { + let rows: Bound<'py, PyDict> = PyDict::new(py); + for (path, artifact) in self.locked().iter() { + match artifact { + RecordedArtifact::Written(digest) => { + rows.set_item(path, (digest, py.None(), py.None()))?; + } + RecordedArtifact::Kept { size, mtime_ns } => { + rows.set_item(path, (py.None(), size, mtime_ns))?; + } + } + } + Ok(rows) + } +} + +/// Where a compile notes the artifacts it writes; `None` outside a recorded compile. +pub(crate) type ArtifactRecorder<'a> = Option<&'a Py>; + +/// `os.path.abspath(path)`. +fn absolute(path: &Bound<'_, PyAny>) -> PyResult { + imported(path.py(), "os.path", "abspath")? + .call1((path,))? + .extract() +} + +/// A new, empty recorder of one compile's artifact writes. +pub(crate) fn new_artifact_recorder(py: Python<'_>) -> PyResult> { + Py::new( + py, + CompileArtifactWrites { + artifacts: Mutex::new(HashMap::new()), + }, + ) +} + +/// Note the exact bytes an artifact holds after a write or an unchanged-file check. +pub(crate) fn artifact_written( + recorder: ArtifactRecorder<'_>, + path: &Bound<'_, PyAny>, + contents: &[u8], +) -> PyResult<()> { + let Some(recorder) = recorder else { + return Ok(()); + }; + recorder.get().record( + absolute(path)?, + RecordedArtifact::Written(artifact_digest(contents)), + ); + Ok(()) +} + +/// Note an earlier artifact kept as-is after its size and mtime were validated. +pub(crate) fn artifact_kept( + recorder: ArtifactRecorder<'_>, + path: &Bound<'_, PyAny>, + (size, mtime_ns): (u64, i64), +) -> PyResult<()> { + let Some(recorder) = recorder else { + return Ok(()); + }; + recorder + .get() + .record(absolute(path)?, RecordedArtifact::Kept { size, mtime_ns }); + Ok(()) +} + +/// Note that one staged artifact was published at another path. +pub(crate) fn artifact_moved( + recorder: ArtifactRecorder<'_>, + source: &Bound<'_, PyAny>, + destination: &Bound<'_, PyAny>, +) -> PyResult<()> { + let Some(recorder) = recorder else { + return Ok(()); + }; + recorder + .get() + .move_one(&absolute(source)?, absolute(destination)?); + Ok(()) +} + +/// Note that a staged artifact directory was published at another directory. +pub(crate) fn artifact_tree_moved( + recorder: ArtifactRecorder<'_>, + source: &Bound<'_, PyAny>, + destination: &Bound<'_, PyAny>, +) -> PyResult<()> { + let Some(recorder) = recorder else { + return Ok(()); + }; + recorder + .get() + .move_tree(&absolute(source)?, &absolute(destination)?); + Ok(()) +} + +/// The stderr lines and stdout report one compile printed, kept as the Python text it printed. +type PrintedOutput = (Vec>, Option>); + +/// Compile output printed at once (notes) or after storing (the report), keeping the text. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct RecordedCompileOutput { + output: Mutex, +} + +impl RecordedCompileOutput { + pub(crate) fn new() -> Self { + Self { + output: Mutex::new((Vec::new(), None)), + } + } + + fn locked(&self) -> std::sync::MutexGuard<'_, PrintedOutput> { + self.output.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +/// `print(*values, file=sys.)`. +pub(crate) fn print_to<'py>( + py: Python<'py>, + stream: &str, + text: impl pyo3::IntoPyObject<'py>, +) -> PyResult<()> { + let keywords: Bound<'_, PyDict> = + keywords(py, &[("file", py.import("sys")?.getattr(stream)?)])?; + let _ = py + .import("builtins")? + .getattr("print")? + .call((text,), Some(&keywords))?; + Ok(()) +} + +#[pymethods] +impl RecordedCompileOutput { + /// Print one stderr result line, such as a note, and record it. + pub(crate) fn print_stderr_line(&self, line: Bound<'_, PyString>) -> PyResult<()> { + print_to(line.py(), "stderr", &line)?; + self.locked().0.push(line.unbind()); + Ok(()) + } + + /// Record the compile report; it is printed once the compile has been stored. + pub(crate) fn record_stdout(&self, text: &Bound<'_, PyString>) -> PyResult<()> { + let recorded: Bound<'_, PyString> = text.add("\n")?.downcast_into()?; + self.locked().1 = Some(recorded.unbind()); + Ok(()) + } + + /// Print the recorded compile report, if any, exactly as it is stored. + pub(crate) fn print_stdout(&self, py: Python<'_>) -> PyResult<()> { + let Some(text) = self.locked().1.as_ref().map(|text| text.clone_ref(py)) else { + return Ok(()); + }; + let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; + let _ = stdout.call_method1("write", (text,))?; + let _ = stdout.call_method0("flush")?; + Ok(()) + } + + #[getter] + fn stderr_lines(&self, py: Python<'_>) -> Vec> { + self.locked() + .0 + .iter() + .map(|line| line.clone_ref(py)) + .collect() + } + + #[getter] + fn stdout(&self, py: Python<'_>) -> Option> { + self.locked().1.as_ref().map(|text| text.clone_ref(py)) + } +} + +/// The hit, miss and bypass counters plus the reuse check time, keyed as `compile_timings` is. +pub(crate) fn compile_reuse_timings<'py>( + attempt: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = attempt.py(); + let outcome: String = attempt.getattr("outcome")?.str()?.to_string(); + let timings: Bound<'py, PyDict> = PyDict::new(py); + for (name, counted) in [ + ("REUSE_HIT_TIMING", HIT_OUTCOME), + ("REUSE_MISS_TIMING", MISS_OUTCOME), + ("REUSE_BYPASS_TIMING", BYPASS_OUTCOME), + ] { + timings.set_item( + imported(py, REUSE_CONSTANTS_MODULE, name)?, + i64::from(outcome == counted), + )?; + } + timings.set_item( + imported(py, REUSE_CONSTANTS_MODULE, "REUSE_CHECK_TIMING")?, + attempt.getattr("check_ms")?, + )?; + Ok(timings) +} + +fn reuse_logger(py: Python<'_>) -> PyResult> { + imported(py, "logging", "getLogger")?.call1((imported( + py, + REUSE_CONSTANTS_MODULE, + "REUSE_LOGGER_NAME", + )?,)) +} + +/// Loaded module files, then the macro store's extra module paths, as Python lists them. +fn module_paths(py: Python<'_>) -> PyResult> { + let paths: Bound<'_, PyList> = PyList::empty(py); + let modules: Bound<'_, PyAny> = py.import("builtins")?.getattr("tuple")?.call1((py + .import("sys")? + .getattr("modules")? + .call_method0("values")?,))?; + let getattr: Bound<'_, PyAny> = py.import("builtins")?.getattr("getattr")?; + for module in modules.try_iter()? { + let path: Bound<'_, PyAny> = getattr.call1((module?, "__file__", py.None()))?; + if path.is_instance_of::() { + paths.append(path)?; + } + } + let extra: Bound<'_, PyAny> = py.import("builtins")?.getattr("sorted")?.call1((imported( + py, + MACRO_STORE_PATHS_MODULE, + "macro_store_module_paths", + )? + .call0()?,))?; + for path in extra.try_iter()? { + let path: Bound<'_, PyAny> = path?; + if path.is_instance_of::() { + paths.append(path)?; + } + } + Ok(paths) +} + +/// The notice a long reuse record prints first, with when it started. +fn start_notice(py: Python<'_>, native: &Bound<'_, PyAny>) -> PyResult> { + let pending: i64 = native.call_method0("pending_digest_bytes")?.extract()?; + let paths: i64 = native.call_method0("snapshot_paths")?.extract()?; + let constant = |name: &str| imported(py, REUSE_CONSTANTS_MODULE, name); + if pending < constant("REUSE_STORE_NOTICE_BYTES")?.extract::()? + && paths < constant("REUSE_STORE_NOTICE_PATHS")?.extract::()? + { + return Ok(None); + } + let mebibytes: Bound<'_, PyAny> = pending + .into_bound_py_any(py)? + .call_method1("__truediv__", (constant("BYTES_PER_MEBIBYTE")?,))?; + let message: Bound<'_, PyAny> = constant("REUSE_STORE_START_MESSAGE")?.call_method( + "format", + (), + Some(&keywords( + py, + &[ + ("paths", paths.into_bound_py_any(py)?), + ("mebibytes", mebibytes), + ], + )?), + )?; + print_to(py, "stderr", &message)?; + Ok(Some(imported(py, "time", "monotonic")?.call0()?.extract()?)) +} + +fn finish_notice(py: Python<'_>, started: Option, message: &str) -> PyResult<()> { + let Some(started) = started else { + return Ok(()); + }; + let now: f64 = imported(py, "time", "monotonic")?.call0()?.extract()?; + let text: Bound<'_, PyAny> = imported(py, REUSE_CONSTANTS_MODULE, message)?.call_method( + "format", + (), + Some(&keywords( + py, + &[("seconds", (now - started).into_bound_py_any(py)?)], + )?), + )?; + print_to(py, "stderr", &text) +} + +/// What `write_reusable_compile` records about one full compile. +struct CompileFacts<'py> { + native: Bound<'py, PyAny>, + output: Bound<'py, PyAny>, + exit_code: Bound<'py, PyAny>, + input_reads: Bound<'py, PyAny>, + artifact_writes: Bound<'py, PyAny>, + compile_cache_enabled: bool, + artifacts_written: bool, + dag_artifact_path: Bound<'py, PyAny>, + json_output: Bound<'py, PyAny>, +} + +/// Store a reusable compile natively; returns whether it was stored. +fn record<'py>(facts: &CompileFacts<'py>) -> PyResult { + let py: Python<'py> = facts.native.py(); + let settings: Bound<'_, PyAny> = call_with( + &imported(py, PROVIDER_SETTINGS_MODULE, "provider_settings_inputs")?, + &[( + "settings_classes", + facts.input_reads.getattr("settings_classes")?, + )], + )?; + let unsupported: Bound<'_, PyAny> = settings.getattr("unsupported_reason")?; + if !unsupported.is_none() { + let _ = reuse_logger(py)?.call_method1( + "debug", + ("Compile reuse is off for this project: %s", &unsupported), + )?; + } + let read_run_id: Bound<'_, PyAny> = facts.input_reads.getattr("read_run_id")?; + let stdout: Bound<'_, PyAny> = facts.output.getattr("stdout")?; + let eligible: bool = !stdout.is_none() + && facts.compile_cache_enabled + && !read_run_id.is_truthy()? + && unsupported.is_none(); + let notice_started: Option = if eligible { + start_notice(py, &facts.native)? + } else { + None + }; + let settings_inputs: Bound<'_, PyAny> = if unsupported.is_none() { + let rows: Bound<'_, PyList> = PyList::empty(py); + let list = |value: Bound<'py, PyAny>| -> PyResult> { + Ok(PyList::new(py, value.try_iter()?.collect::>>()?)?.into_any()) + }; + for item in settings.getattr("inputs")?.try_iter()? { + let item: Bound<'_, PyAny> = item?; + rows.append(( + item.getattr("case_sensitive")?, + list(item.getattr("names")?)?, + list(item.getattr("prefixes")?)?, + list(item.getattr("env_files")?)?, + list(item.getattr("secrets_dirs")?)?, + ))?; + } + rows.into_any() + } else { + py.None().into_bound(py) + }; + let dag_path: Bound<'_, PyAny> = if facts.dag_artifact_path.is_none() { + py.None().into_bound(py) + } else { + imported(py, "os.path", "abspath")?.call1((&facts.dag_artifact_path,))? + }; + let payload: Bound<'_, PyDict> = keywords( + py, + &[ + ("stdout", stdout), + ( + "stderr_lines", + PyList::new( + py, + facts + .output + .getattr("stderr_lines")? + .try_iter()? + .collect::>>()?, + )? + .into_any(), + ), + ("exit_code", facts.exit_code.clone()), + ( + "compile_cache_enabled", + facts.compile_cache_enabled.into_bound_py_any(py)?, + ), + ("read_run_id", read_run_id), + ("settings_inputs", settings_inputs), + ( + "template_environment_names", + PyList::new( + py, + facts + .input_reads + .getattr("environment_names")? + .try_iter()? + .collect::>>()?, + )? + .into_any(), + ), + ("module_paths", module_paths(py)?.into_any()), + ("artifacts", facts.artifact_writes.getattr("artifacts")?), + ( + "artifacts_written", + facts.artifacts_written.into_bound_py_any(py)?, + ), + ("dag_path", dag_path), + ("json_output", facts.json_output.clone()), + ( + "rules", + imported(py, NATIVE_REUSE_MODULE, "reuse_rules")?.call0()?, + ), + ], + )?; + let (outcome, digested): (String, Bound<'_, PyAny>) = + facts.native.call_method1("record", (payload,))?.extract()?; + let _ = reuse_logger(py)?.call_method1( + "debug", + ("Compile reuse record digested %d project files", digested), + )?; + finish_notice( + py, + notice_started, + if outcome == SKIPPED { + "REUSE_STORE_SKIPPED_MESSAGE" + } else { + "REUSE_STORE_DONE_MESSAGE" + }, + )?; + Ok(outcome == STORED) +} + +/// Store a reusable full compile, or drop the stored one when it cannot be reused. +#[pyfunction] +#[pyo3(signature = (**options))] +pub(crate) fn write_reusable_compile( + py: Python<'_>, + options: Option>, +) -> PyResult<()> { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "write_reusable_compile", + options, + ( + &[ + "attempt", + "output", + "exit_code", + "input_reads", + "artifact_writes", + "compile_cache_enabled", + "artifacts_written", + "dag_artifact_path", + "json_output", + ], + &[], + ), + )?; + let attempt: Bound<'_, PyAny> = arguments.value("attempt")?; + let entry_path: Bound<'_, PyAny> = attempt.getattr("entry_path")?; + let native: Bound<'_, PyAny> = attempt.getattr("native")?; + if !attempt.getattr("outcome")?.eq(MISS_OUTCOME)? || entry_path.is_none() || native.is_none() { + return Ok(()); + } + let facts: CompileFacts<'_> = CompileFacts { + native, + output: arguments.value("output")?, + exit_code: arguments.value("exit_code")?, + input_reads: arguments.value("input_reads")?, + artifact_writes: arguments.value("artifact_writes")?, + compile_cache_enabled: arguments.flag("compile_cache_enabled")?, + artifacts_written: arguments.flag("artifacts_written")?, + dag_artifact_path: arguments.value("dag_artifact_path")?, + json_output: arguments.value("json_output")?, + }; + let stored: bool = match record(&facts) { + Ok(stored) => stored, + Err(error) if error.is_instance_of::(py) => { + let _ = reuse_logger(py)?.call_method( + "debug", + ("Compile reuse did not store this compile",), + Some(&keywords( + py, + &[("exc_info", error.value(py).clone().into_any())], + )?), + )?; + let unlinked: PyResult> = entry_path.call_method( + "unlink", + (), + Some(&keywords( + py, + &[("missing_ok", true.into_bound_py_any(py)?)], + )?), + ); + if let Err(error) = unlinked + && !error.is_instance_of::(py) + { + return Err(error); + } + false + } + Err(error) => return Err(error), + }; + if stored { + let _ = call_with( + &imported(py, LAUNCH_RECORDS_MODULE, "write_launch_record")?, + &[("attempt", attempt)], + )?; + } + Ok(()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(write_reusable_compile, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/surrogates.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/surrogates.rs new file mode 100644 index 0000000000..9feba30b5e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/surrogates.rs @@ -0,0 +1,84 @@ +//! Python text with lone surrogates carried through Rust formatting and handed back unchanged. + +use std::cell::Cell; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBytes, PyString, PyStringMethods}; + +/// Lone surrogates travel as Supplementary Private Use Area-A characters, one per surrogate. +const CARRIER_BASE: u32 = 0xF_0000; +const SURROGATE_BASE: u32 = 0xD800; +const SURROGATE_COUNT: u32 = 0x800; +const UTF16: &str = "utf-16-le"; +const SURROGATE_PASS: &str = "surrogatepass"; + +/// Reads Python text for Rust formatting, remembering whether any lone surrogate was carried. +#[derive(Default)] +pub(crate) struct SurrogateText { + carried: Cell, +} + +impl SurrogateText { + /// `str(value)` as Rust text; each lone surrogate becomes its carrier character. + pub(crate) fn text(&self, value: &Bound<'_, PyAny>) -> PyResult { + let text: Bound<'_, PyString> = value.str()?; + if let Ok(plain) = text.to_str() { + return Ok(plain.to_owned()); + } + self.carried.set(true); + let encoded: Vec = text + .call_method1("encode", (UTF16, SURROGATE_PASS))? + .extract()?; + let units = encoded + .as_chunks::<2>() + .0 + .iter() + .map(|pair| u16::from_le_bytes(*pair)); + Ok(char::decode_utf16(units) + .map(|decoded| { + decoded.unwrap_or_else(|error| { + char::from_u32( + CARRIER_BASE + u32::from(error.unpaired_surrogate()) - SURROGATE_BASE, + ) + .unwrap_or(char::REPLACEMENT_CHARACTER) + }) + }) + .collect()) + } + + /// `str(value)`, or `None` for `None`. + pub(crate) fn optional(&self, value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + self.text(value).map(Some) + } + + /// Formatted text as a Python string, with every carried surrogate restored. + pub(crate) fn python<'py>( + &self, + py: Python<'py>, + text: &str, + ) -> PyResult> { + if !self.carried.get() { + return Ok(PyString::new(py, text)); + } + let mut encoded: Vec = Vec::with_capacity(text.len() * 2); + for character in text.chars() { + let code: u32 = u32::from(character); + if (CARRIER_BASE..CARRIER_BASE + SURROGATE_COUNT).contains(&code) { + let unit: u16 = + u16::try_from(code - CARRIER_BASE + SURROGATE_BASE).unwrap_or(u16::MAX); + encoded.extend(unit.to_le_bytes()); + continue; + } + let mut units: [u16; 2] = [0; 2]; + for unit in character.encode_utf16(&mut units) { + encoded.extend(unit.to_le_bytes()); + } + } + Ok(PyBytes::new(py, &encoded) + .call_method1("decode", (UTF16, SURROGATE_PASS))? + .downcast_into()?) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs new file mode 100644 index 0000000000..72ee87dd30 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -0,0 +1,812 @@ +//! Write compiled project output to `target/`: models, functions, audits, SQL tests, manifest. + +use std::collections::HashMap; +use std::path::PathBuf; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{ + Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple, PyTupleMethods}; +use pyo3::{IntoPyObjectExt, pyclass, pyfunction, wrap_pyfunction}; +use sqlbuild_cache::artifacts::main::new_sql_test_record::new_sql_test_record; +use sqlbuild_cache::artifacts::main::reusable_sql_test_record::reusable_sql_test_record; +use sqlbuild_cache::artifacts::models::SqlTestArtifactRecord; + +use crate::bindings::_helpers::compile_command::artifact_batch::{ArtifactBatch, timed}; +use crate::bindings::_helpers::compile_command::reuse_recording::{ + ArtifactRecorder, CompileArtifactWrites, artifact_kept, artifact_moved, artifact_tree_moved, +}; +use crate::bindings::_helpers::compile_command::test_artifact_cache::{ + IdentityContext, artifact_identity, artifact_record_key, identity_context, read_cache, + write_cache, +}; +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, keywords, within, +}; +use crate::bindings::_helpers::compile_outputs::artifacts::{ + publish_staged_artifacts, remove_stale_artifacts, +}; + +const COMPILED_DIR: &str = "compiled"; +const MODELS_DIR: &str = "models"; +const FUNCTIONS_DIR: &str = "functions"; +const AUDITS_DIR: &str = "audits"; +const GENERIC_DIR: &str = "generic"; +const SINGULAR_DIR: &str = "singular"; +const TESTS_DIR: &str = "tests"; +const MANIFEST_FILE: &str = "manifest.json"; +const SQL_FILE_SUFFIX: &str = ".sql"; +/// `functions/` path components that already lead a function's path. +const LANGUAGE_DIR_DEPTH: usize = 2; +const PUBLISHED_STAGING_CHANGED: &str = "changed"; +const PUBLISHED_TREE: &str = "tree"; +const OUTPUT_MODELS_MODULE: &str = "sqlbuild.cli.output.models"; +const CLI_EXCEPTIONS_MODULE: &str = "sqlbuild.cli.commands.exceptions"; +const COMPILED_TEST_PATH_MODULE: &str = "sqlbuild.cli.paths.main._compiled_sql_test_output_path"; +const TEST_PATH_MODULE: &str = "sqlbuild.cli.paths.main._sql_test_output_path"; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; +const MODEL_CHAIN_MODULE: &str = "sqlbuild.compiler.planner.main.execution.sql_test_model_chain"; +const PLAN_ARTIFACTS_MODULE: &str = + "sqlbuild.compiler.sql_test_glue.main.plan_native_sql_test_artifacts"; +const COMPARISON_SQL_MODULE: &str = "sqlbuild.executor.testing.main.comparison_sql"; +const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; +const TARGET_OPTIONS: (&[&str], &[&str]) = (&["target_dir", "adapter", "project"], &["manifest"]); +const STAGING_CHANGED_HELP: &str = + "Rerun the compile; avoid deleting target/ while a compile is running."; + +/// `base / part / ...`. +fn joined<'py>(base: &Bound<'py, PyAny>, parts: &[&str]) -> PyResult> { + let mut path: Bound<'py, PyAny> = base.clone(); + for part in parts { + path = path.div(*part)?; + } + Ok(path) +} + +fn path_class(py: Python<'_>) -> PyResult> { + imported(py, "pathlib", "Path") +} + +/// `relative_path` under `models/`, unless it already starts there. +fn model_output_path<'py>(relative_path: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = relative_path.py(); + let parts: Bound<'py, PyTuple> = relative_path.getattr("parts")?.downcast_into()?; + if !parts.is_empty() && parts.get_item(0)?.eq(MODELS_DIR)? { + return path_class(py)?.call1(parts); + } + path_class(py)?.call1((MODELS_DIR,))?.div(relative_path) +} + +/// `relative_path` under `functions//`, as a `.sql` file. +fn function_output_path<'py>( + relative_path: &Bound<'py, PyAny>, + language: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = relative_path.py(); + let parts: Bound<'py, PyTuple> = relative_path.getattr("parts")?.downcast_into()?; + let language_dir: Bound<'py, PyAny> = language.getattr("value")?; + let path: Bound<'py, PyAny> = if parts.len() >= LANGUAGE_DIR_DEPTH + && parts.get_item(0)?.eq(FUNCTIONS_DIR)? + && parts.get_item(1)?.eq(&language_dir)? + { + path_class(py)?.call1(parts)? + } else { + path_class(py)? + .call1((FUNCTIONS_DIR,))? + .div(language_dir)? + .div(relative_path)? + }; + path.call_method1("with_suffix", (SQL_FILE_SUFFIX,)) +} + +/// The audit's folder and file name: generic audits under their target, others singular. +fn audit_output_path<'py>(audit: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = audit.py(); + let target: Bound<'py, PyAny> = audit.getattr("attached_target_name")?; + let column: Bound<'py, PyAny> = audit.getattr("attached_column_name")?; + let name: String = audit.getattr("name")?.str()?.to_string(); + let folder: Bound<'py, PyAny> = if target.is_none() { + path_class(py)?.call1((SINGULAR_DIR,))? + } else { + path_class(py)?.call1((GENERIC_DIR,))?.div(&target)? + }; + let file_name: String = if !target.is_none() && !column.is_none() { + format!("{name}__{}{SQL_FILE_SUFFIX}", column.str()?) + } else { + format!("{name}{SQL_FILE_SUFFIX}") + }; + folder.div(file_name) +} + +/// The function's `CREATE FUNCTION` statements joined into one artifact. +fn function_sql(adapter: &Bound<'_, PyAny>, function: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = adapter.py(); + let mut arguments: Vec<(&str, Bound<'_, PyAny>)> = vec![( + "destination", + function.getattr("destination")?.getattr("qualified_name")?, + )]; + for field in [ + "arguments", + "returns", + "body_sql", + "return_columns", + "language", + "runtime_version", + "entry_point", + "packages", + ] { + arguments.push((field, function.getattr(field)?)); + } + let statements: Vec = adapter + .getattr("render_create_function")? + .call((), Some(&keywords(py, &arguments)?))? + .extract()?; + Ok(statements.join(";\n\n")) +} + +/// Delete compiled files no write group manages. +fn remove_stale(target_dir: &Bound<'_, PyAny>, managed: &[Bound<'_, PyAny>]) -> PyResult<()> { + let py: Python<'_> = target_dir.py(); + let compiled: PathBuf = joined(target_dir, &[COMPILED_DIR])?.extract()?; + let managed: Vec = managed + .iter() + .map(|path| path.extract()) + .collect::>()?; + timed(py, "stale_traversal_ms", || { + remove_stale_artifacts(py, compiled, managed).map(|_| ()) + }) +} + +fn write_manifest( + target_dir: &Bound<'_, PyAny>, + manifest: &Bound<'_, PyAny>, + recorder: ArtifactRecorder<'_>, +) -> PyResult<()> { + if manifest.is_none() { + return Ok(()); + } + let mut batch: ArtifactBatch = ArtifactBatch::new(true, recorder); + batch.queue( + &joined(target_dir, &[MANIFEST_FILE])?, + manifest.call_method0("contents")?.extract()?, + )?; + batch.flush(target_dir.py()) +} + +/// Whether `target/compiled` exists, after making sure `target/` does. +fn prepared_target(target_dir: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = target_dir.py(); + let existed: bool = joined(target_dir, &[COMPILED_DIR])? + .call_method0("is_dir")? + .is_truthy()?; + let _ = target_dir.call_method( + "mkdir", + (), + Some(&keywords( + py, + &[ + ("parents", true.into_bound_py_any(py)?), + ("exist_ok", true.into_bound_py_any(py)?), + ], + )?), + )?; + Ok(existed) +} + +fn written_target<'py>( + target_dir: &Bound<'py, PyAny>, + counts: [usize; 5], + diagnostics: Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = target_dir.py(); + let [models, seeds, functions, audits, tests] = counts; + call_with( + &imported(py, OUTPUT_MODELS_MODULE, "WrittenTarget")?, + &[ + ("model_count", models.into_bound_py_any(py)?), + ("seed_count", seeds.into_bound_py_any(py)?), + ("function_count", functions.into_bound_py_any(py)?), + ("audit_count", audits.into_bound_py_any(py)?), + ("test_count", tests.into_bound_py_any(py)?), + ("target_dir", target_dir.clone()), + ("diagnostics", diagnostics), + ], + ) +} + +fn length(value: &Bound<'_, PyAny>, field: &str) -> PyResult { + value.getattr(field)?.len() +} + +/// Write the planner's compiled output under `target_dir`. +fn write_planned_target<'py>( + target_dir: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, + plan_output: &Bound<'py, PyAny>, + manifest: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = target_dir.py(); + let remove_stale_files: bool = prepared_target(target_dir)?; + let mut batch: ArtifactBatch = ArtifactBatch::new(remove_stale_files, None); + let mut managed: Vec> = Vec::new(); + let compiled: Bound<'py, PyAny> = joined(target_dir, &[COMPILED_DIR])?; + for entry in plan_output.getattr("model_entries")?.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + let path: Bound<'py, PyAny> = + compiled.div(model_output_path(&entry.getattr("relative_path")?)?)?; + batch.queue_sql(&path, &entry.getattr("resolved_sql")?.extract::()?)?; + managed.push(path); + } + batch.flush(py)?; + for entry in plan_output.getattr("function_entries")?.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + if entry + .getattr("destination")? + .getattr("qualified_name")? + .is_none() + { + continue; + } + let sql: String = function_sql(adapter, &entry)?; + let path: Bound<'py, PyAny> = compiled.div(function_output_path( + &entry.getattr("relative_path")?, + &entry.getattr("language")?, + )?)?; + batch.queue_sql(&path, &sql)?; + managed.push(path); + } + batch.flush(py)?; + for entry in plan_output.getattr("audit_entries")?.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + let path: Bound<'py, PyAny> = compiled.div(AUDITS_DIR)?.div(audit_output_path(&entry)?)?; + batch.queue_sql(&path, &entry.getattr("resolved_sql")?.extract::()?)?; + managed.push(path); + } + batch.flush(py)?; + for entry in plan_output.getattr("test_entries")?.try_iter()? { + let entry: Bound<'py, PyAny> = entry?; + let path: Bound<'py, PyAny> = compiled + .div(TESTS_DIR)? + .div(imported(py, TEST_PATH_MODULE, "sql_test_output_path")?.call1((&entry,))?)?; + let comparison_sql: String = timed(py, "comparison_render_ms", || { + call_with( + &imported(py, COMPARISON_SQL_MODULE, "build_sql_test_comparison_sql")?, + &[ + ("test_entry", entry.clone()), + ( + "set_difference_operator", + adapter.call_method0("render_set_difference_operator")?, + ), + ( + "sql_analysis_dialect", + adapter.call_method0("sql_analysis_dialect")?, + ), + ], + )? + .extract() + })?; + batch.queue_sql(&path, &comparison_sql)?; + managed.push(path); + } + batch.flush(py)?; + if remove_stale_files { + remove_stale(target_dir, &managed)?; + } + write_manifest(target_dir, manifest, None)?; + written_target( + target_dir, + [ + length(plan_output, "model_entries")?, + length(plan_output, "seed_entries")?, + length(plan_output, "function_entries")?, + length(plan_output, "audit_entries")?, + length(plan_output, "test_entries")?, + ], + PyTuple::empty(py).into_any(), + ) +} + +/// SQL tests planned ahead of the write phase, and where the compile notes its artifact writes. +#[derive(Clone, Copy, Default)] +pub(crate) struct Prepared<'a, 'py> { + /// `planned_tests()` hands over the background plan, when one was started. + pub(crate) planned_tests: Option<&'a Bound<'py, PyAny>>, + pub(crate) recorder: ArtifactRecorder<'a>, +} + +/// A test needing a fresh artifact, with its cache slot and artifact identity when caching. +type PendingTest = (Py, Option, Option); + +/// The managed SQL test artifact paths and the planning diagnostics of one write. +type WrittenTests<'py> = (Vec>, Vec>); + +/// SQL tests whose cached artifacts cannot be reused, planned, beside the reusable ones. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct PlannedStaticSqlTests { + tests_root: Py, + /// Each pending test with its cache slot and artifact identity, when caching. + pending: Vec, + /// One planned `NativeSqlTestArtifact` per pending test. + artifacts: Vec>, + cached_paths: Vec>, + cached_records: Vec<(String, SqlTestArtifactRecord)>, +} + +/// `zip(pending, artifacts, strict=True)`. +fn zipped( + planned: &PlannedStaticSqlTests, +) -> PyResult)>> { + if planned.pending.len() != planned.artifacts.len() { + let relation: &str = if planned.artifacts.len() < planned.pending.len() { + "shorter" + } else { + "longer" + }; + return Err(PyValueError::new_err(format!( + "zip() argument 2 is {relation} than argument 1" + ))); + } + Ok(planned.pending.iter().zip(planned.artifacts.iter())) +} + +/// Plan the tests natively inside the SQL-test planning lifecycle. +fn planned_artifacts<'py>( + adapter: &Bound<'py, PyAny>, + project: &Bound<'py, PyAny>, + tests: Vec>, +) -> PyResult>> { + let py: Python<'py> = project.py(); + if tests.is_empty() { + return Ok(Vec::new()); + } + let lifecycle: Bound<'py, PyAny> = call_with( + &imported(py, LIFECYCLE_MODULE, "OperationLifecycle")?, + &[ + ("operation_kind", "project".into_bound_py_any(py)?), + ("operation_name", "sql_test_planning".into_bound_py_any(py)?), + ], + )?; + within(&lifecycle, |entered| { + let artifacts: Vec> = call_with( + &imported(py, PLAN_ARTIFACTS_MODULE, "plan_native_sql_test_artifacts")?, + &[ + ("project", project.clone()), + ("tests", PyTuple::new(py, tests)?.into_any()), + ("adapter", adapter.clone()), + ( + "sql_analysis_enabled", + project.getattr("settings")?.getattr("sql_analysis")?, + ), + ], + )? + .try_iter()? + .map(|artifact| artifact.map(Bound::unbind)) + .collect::>()?; + let metadata: Bound<'py, PyDict> = PyDict::new(py); + metadata.set_item("item_count", artifacts.len())?; + let _ = call_with( + &entered.getattr("completed")?, + &[("metadata", metadata.into_any())], + )?; + Ok(artifacts) + }) +} + +/// Plan every SQL test whose cached artifact under `target_dir` cannot be reused. +fn plan_tests<'py>( + target_dir: &Bound<'py, PyAny>, + (adapter, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + recorder: ArtifactRecorder<'_>, +) -> PyResult { + let py: Python<'py> = project.py(); + let tests_root: Bound<'py, PyAny> = joined(target_dir, &[COMPILED_DIR, TESTS_DIR])?; + let tests_root_path: PathBuf = tests_root.extract()?; + let cache_dir: Bound<'py, PyAny> = project.getattr("compile_cache_dir")?; + let cached: HashMap = read_cache(&cache_dir)?; + let tests: Bound<'py, PyAny> = project.getattr("sql_tests")?; + let caching: Option<(IdentityContext<'py>, Bound<'py, PyAny>)> = if cache_dir.is_none() { + None + } else { + let context: IdentityContext<'py> = identity_context(project, adapter)?; + let chains: Bound<'py, PyAny> = call_with( + &imported(py, MODEL_CHAIN_MODULE, "sql_test_model_chain_names_by_key")?, + &[("project", project.clone()), ("tests", tests.clone())], + )?; + Some((context, chains)) + }; + let mut pending: Vec = Vec::new(); + let mut cached_paths: Vec> = Vec::new(); + let mut cached_records: Vec<(String, SqlTestArtifactRecord)> = Vec::new(); + let empty: Bound<'py, PyTuple> = PyTuple::empty(py); + for test in tests.try_iter()? { + let test: Bound<'py, PyAny> = test?; + let Some((context, chains)) = &caching else { + pending.push((test.unbind(), None, None)); + continue; + }; + let record_key: String = artifact_record_key(&test)?; + let chain: Bound<'py, PyAny> = + chains.call_method1("get", (test.getattr("key")?, &empty))?; + let identity: String = artifact_identity(&test, &chain, context)?; + let reusable: Option = cached + .get(&record_key) + .and_then(|record| reusable_sql_test_record(&tests_root_path, record, &identity)); + if let Some(record) = reusable { + let path: Bound<'py, PyAny> = tests_root.div(&record.relative_path)?; + artifact_kept(recorder, &path, (record.size, record.mtime_ns))?; + cached_paths.push(path.unbind()); + cached_records.push((record_key, record)); + continue; + } + pending.push((test.unbind(), Some(record_key), Some(identity))); + } + let pending_tests: Vec> = pending + .iter() + .map(|(test, _, _)| test.bind(py).clone()) + .collect(); + let artifacts: Vec> = planned_artifacts(adapter, project, pending_tests)?; + Ok(PlannedStaticSqlTests { + tests_root: tests_root.unbind(), + pending, + artifacts, + cached_paths, + cached_records, + }) +} + +/// The planned tests handed over by `planned_tests()`, when they target `target_dir`. +fn resolved_plan<'py>( + target_dir: &Bound<'py, PyAny>, + (adapter, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + prepared: Prepared<'_, 'py>, +) -> PyResult> { + let py: Python<'py> = project.py(); + if let Some(planned_tests) = prepared.planned_tests { + let planned: Bound<'py, PyAny> = planned_tests.call0()?; + if !planned.is_none() { + let planned: Bound<'py, PlannedStaticSqlTests> = planned.downcast_into()?; + if planned + .get() + .tests_root + .bind(py) + .eq(joined(target_dir, &[COMPILED_DIR, TESTS_DIR])?)? + { + return Ok(planned); + } + } + } + Bound::new( + py, + plan_tests(target_dir, (adapter, project), prepared.recorder)?, + ) +} + +/// The planning errors of one test's artifact, as SQL test diagnostics. +fn artifact_diagnostics<'py>( + test: &Bound<'py, PyAny>, + artifact: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = test.py(); + let mut diagnostics: Vec> = Vec::new(); + for message in artifact.getattr("error_messages")?.try_iter()? { + diagnostics.push(call_with( + &imported(py, COMPILE_MODELS_MODULE, "CompilerDiagnostic")?, + &[ + ("phase", "test".into_bound_py_any(py)?), + ("severity", "error".into_bound_py_any(py)?), + ( + "code", + imported(py, PLANNER_EXCEPTIONS_MODULE, "PlannerInputError")? + .getattr("code")?, + ), + ("message", message?), + ("resource_type", "sql_test".into_bound_py_any(py)?), + ("resource_name", test.getattr("name")?), + ("path", test.getattr("source_path")?), + ], + )?); + } + Ok(diagnostics) +} + +/// Write planned SQL test artifacts and the cache; returns managed paths and planning errors. +fn write_static_tests<'py>( + target_dir: &Bound<'py, PyAny>, + (adapter, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + prepared: Prepared<'_, 'py>, + batch: &mut ArtifactBatch, +) -> PyResult> { + let py: Python<'py> = project.py(); + let planned: Bound<'py, PlannedStaticSqlTests> = + resolved_plan(target_dir, (adapter, project), prepared)?; + let planned: &PlannedStaticSqlTests = planned.get(); + let tests_root: &Bound<'py, PyAny> = planned.tests_root.bind(py); + let tests_root_path: PathBuf = tests_root.extract()?; + let mut managed: Vec> = planned + .cached_paths + .iter() + .map(|path| path.bind(py).clone()) + .collect(); + let mut records: Vec<(String, SqlTestArtifactRecord)> = planned.cached_records.clone(); + let mut test_paths: Vec> = Vec::new(); + for ((test, _, _), artifact) in zipped(planned)? { + let artifact: &Bound<'py, PyAny> = artifact.bind(py); + let path: Bound<'py, PyAny> = tests_root.div(call_with( + &imported( + py, + COMPILED_TEST_PATH_MODULE, + "compiled_sql_test_output_path", + )?, + &[ + ("test", test.bind(py).clone()), + ("model_names", artifact.getattr("model_names")?), + ], + )?)?; + batch.queue_sql(&path, &artifact.getattr("sql")?.extract::()?)?; + managed.push(path.clone()); + test_paths.push(path); + } + batch.flush(py)?; + let mut diagnostics: Vec> = Vec::new(); + for (((test, record_key, identity), artifact), path) in zipped(planned)?.zip(&test_paths) { + let found: Vec> = artifact_diagnostics(test.bind(py), artifact.bind(py))?; + if !found.is_empty() { + diagnostics.extend(found); + continue; + } + if let (Some(record_key), Some(identity)) = (record_key, identity) { + let artifact_path: PathBuf = path.extract()?; + if let Some(record) = new_sql_test_record(&tests_root_path, &artifact_path, identity) { + records.retain(|(key, _)| key != record_key); + records.push((record_key.clone(), record)); + } + } + } + timed(py, "cache_publication_ms", || { + write_cache(&project.getattr("compile_cache_dir")?, &records) + })?; + Ok((managed, diagnostics)) +} + +/// Write offline compiled output under `target_dir`, reusing SQL tests planned ahead. +pub(crate) fn write_static_target<'py>( + target_dir: &Bound<'py, PyAny>, + (adapter, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + manifest: &Bound<'py, PyAny>, + prepared: Prepared<'_, 'py>, +) -> PyResult> { + let py: Python<'py> = target_dir.py(); + let remove_stale_files: bool = prepared_target(target_dir)?; + let mut batch: ArtifactBatch = ArtifactBatch::new(remove_stale_files, prepared.recorder); + let mut managed: Vec> = Vec::new(); + let compiled: Bound<'py, PyAny> = joined(target_dir, &[COMPILED_DIR])?; + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let path: Bound<'py, PyAny> = + compiled.div(model_output_path(&model.getattr("relative_path")?)?)?; + batch.queue_sql(&path, &model.getattr("query_sql")?.extract::()?)?; + managed.push(path); + } + batch.flush(py)?; + for function in project.getattr("functions")?.try_iter()? { + let function: Bound<'py, PyAny> = function?; + if function + .getattr("destination")? + .getattr("qualified_name")? + .is_none() + { + continue; + } + let sql: String = function_sql(adapter, &function)?; + let path: Bound<'py, PyAny> = compiled.div(function_output_path( + &function.getattr("relative_path")?, + &function.getattr("language")?, + )?)?; + batch.queue_sql(&path, &sql)?; + managed.push(path); + } + batch.flush(py)?; + for audit in project.getattr("audits")?.try_iter()? { + let audit: Bound<'py, PyAny> = audit?; + let path: Bound<'py, PyAny> = compiled.div(AUDITS_DIR)?.div(audit_output_path(&audit)?)?; + batch.queue_sql(&path, &audit.getattr("sql_body")?.extract::()?)?; + managed.push(path); + } + batch.flush(py)?; + let (test_paths, diagnostics) = + write_static_tests(target_dir, (adapter, project), prepared, &mut batch)?; + managed.extend(test_paths); + if remove_stale_files { + remove_stale(target_dir, &managed)?; + } + write_manifest(target_dir, manifest, prepared.recorder)?; + written_target( + target_dir, + [ + length(project, "models")?, + length(project, "seeds")?, + length(project, "functions")?, + length(project, "audits")?, + length(project, "sql_tests")?, + ], + PyTuple::new(py, diagnostics)?.into_any(), + ) +} + +/// Plan uncached SQL tests and report their errors without writing artifacts or cache. +pub(crate) fn planning_diagnostics<'py>( + target_dir: &Bound<'py, PyAny>, + (adapter, project): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + prepared: Prepared<'_, 'py>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let planned: Bound<'py, PlannedStaticSqlTests> = + resolved_plan(target_dir, (adapter, project), prepared)?; + let mut diagnostics: Vec> = Vec::new(); + for ((test, _, _), artifact) in zipped(planned.get())? { + diagnostics.extend(artifact_diagnostics(test.bind(py), artifact.bind(py))?); + } + PyTuple::new(py, diagnostics) +} + +/// Publish staged files with the same unchanged-file and stale-file semantics as a write. +pub(crate) fn publish_staged_target<'py>( + (prepared, expected_files): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + target_dir: &Bound<'py, PyAny>, + manifest: &Bound<'py, PyAny>, + recorder: ArtifactRecorder<'_>, +) -> PyResult> { + let py: Python<'py> = target_dir.py(); + let staged_dir: Bound<'py, PyAny> = joined(&prepared.getattr("target_dir")?, &[COMPILED_DIR])?; + let compiled_dir: Bound<'py, PyAny> = joined(target_dir, &[COMPILED_DIR])?; + let existed: bool = compiled_dir.call_method0("is_dir")?.is_truthy()?; + let expected: Vec = py + .import("builtins")? + .getattr("sorted")? + .call1((expected_files,))? + .extract()?; + let (staged, compiled): (PathBuf, PathBuf) = (staged_dir.extract()?, compiled_dir.extract()?); + let (kind, published) = timed(py, "physical_write_ms", || { + publish_staged_artifacts(py, staged, compiled, expected) + })?; + if kind == PUBLISHED_STAGING_CHANGED { + return Err(staging_changed( + py, + "staged compile artifacts changed before publication; no artifact was published", + )?); + } + if kind == PUBLISHED_TREE { + artifact_tree_moved(recorder, &staged_dir, &compiled_dir)?; + } else { + let mut managed: Vec> = Vec::new(); + for (source, path) in &published { + artifact_moved( + recorder, + &source.into_bound_py_any(py)?, + &path.into_bound_py_any(py)?, + )?; + managed.push(path_class(py)?.call1((path,))?); + } + if existed { + remove_stale(target_dir, &managed)?; + } + } + write_manifest(target_dir, manifest, recorder)?; + written_target( + target_dir, + [ + prepared.getattr("model_count")?.extract()?, + prepared.getattr("seed_count")?.extract()?, + prepared.getattr("function_count")?.extract()?, + prepared.getattr("audit_count")?.extract()?, + prepared.getattr("test_count")?.extract()?, + ], + prepared.getattr("diagnostics")?, + ) +} + +/// `StagedArtifactsChangedError(message, help=...)`. +pub(crate) fn staging_changed(py: Python<'_>, message: &str) -> PyResult { + let error: Bound<'_, PyAny> = + imported(py, CLI_EXCEPTIONS_MODULE, "StagedArtifactsChangedError")?.call( + (message,), + Some(&keywords( + py, + &[("help", STAGING_CHANGED_HELP.into_bound_py_any(py)?)], + )?), + )?; + Ok(PyErr::from_value(error)) +} + +/// Write offline compiled output files under `target_dir`. +#[pyfunction] +#[pyo3(signature = (**options))] +fn write_static_compile_target<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = + KeywordArguments::read(py, "write_static_compile_target", options, TARGET_OPTIONS)?; + write_static_target( + &arguments.value("target_dir")?, + (&arguments.value("adapter")?, &arguments.value("project")?), + &arguments.value("manifest")?, + Prepared::default(), + ) +} + +/// Write the planner's compiled output files under `target_dir`. +#[pyfunction] +#[pyo3(signature = (**options))] +fn write_compile_target<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "write_compile_target", + options, + (&["target_dir", "adapter", "plan_output"], &["manifest"]), + )?; + write_planned_target( + &arguments.value("target_dir")?, + &arguments.value("adapter")?, + &arguments.value("plan_output")?, + &arguments.value("manifest")?, + ) +} + +/// Plan uncached SQL tests and report their errors without writing artifacts or cache. +#[pyfunction] +#[pyo3(signature = (**options))] +fn static_sql_test_planning_diagnostics<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult> { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "static_sql_test_planning_diagnostics", + options, + (&["target_dir", "adapter", "project"], &[]), + )?; + planning_diagnostics( + &arguments.value("target_dir")?, + (&arguments.value("adapter")?, &arguments.value("project")?), + Prepared::default(), + ) +} + +/// Plan every SQL test whose cached artifact under `target_dir` cannot be reused. +#[pyfunction] +#[pyo3(signature = (**options))] +fn plan_static_sql_tests<'py>( + py: Python<'py>, + options: Option>, +) -> PyResult { + let arguments: KeywordArguments<'py> = KeywordArguments::read( + py, + "plan_static_sql_tests", + options, + (&["target_dir", "adapter", "project"], &["artifact_writes"]), + )?; + let recorder: Option> = + arguments.value("artifact_writes")?.extract()?; + plan_tests( + &arguments.value("target_dir")?, + (&arguments.value("adapter")?, &arguments.value("project")?), + recorder.as_ref(), + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + module.add_function(wrap_pyfunction!(write_static_compile_target, module)?)?; + module.add_function(wrap_pyfunction!(write_compile_target, module)?)?; + module.add_function(wrap_pyfunction!( + static_sql_test_planning_diagnostics, + module + )?)?; + module.add_function(wrap_pyfunction!(plan_static_sql_tests, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs new file mode 100644 index 0000000000..da85adc00a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/test_artifact_cache.rs @@ -0,0 +1,504 @@ +//! Persistent identities of compiled SQL test artifacts, so unchanged tests keep their files. + +use std::collections::HashMap; +use std::path::PathBuf; + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyList, PyListMethods, PyString, PyTuple}; +use sqlbuild_cache::artifacts::main::stored_sql_test_record::stored_sql_test_record; +use sqlbuild_cache::artifacts::models::{SqlTestArtifactRecord, StoredSqlTestRecord}; +use sqlbuild_cache::digest::main::sha256_hex::sha256_hex; + +use crate::bindings::_helpers::compile_driver::python_calls::imported; +use crate::bindings::_helpers::compile_outputs::json_report::{compact_sorted_json, orjson_sorted}; + +const CACHE_VERSION: i64 = 4; +const ALGORITHM_FINGERPRINT: &str = "sql-test-artifact-v4-orjson-identity"; +const CACHE_FILE_NAME: &str = "sql-test-artifacts.json"; +const MAX_CACHE_BYTES: u64 = 10_000_000; +const MAX_CACHE_RECORDS: usize = 100_000; +const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; +const METADATA_MODULE: &str = "importlib.metadata"; + +/// The digest shared by every test artifact identity, and each model's own identity. +pub(crate) struct IdentityContext<'py> { + common_identity: String, + model_identities: Bound<'py, PyDict>, +} + +/// `hashlib.sha256(orjson.dumps(payload, option=OPT_SORT_KEYS)).hexdigest()`. +fn payload_digest(payload: &Bound<'_, PyAny>) -> PyResult { + let encoded: String = orjson_sorted(payload)?; + Ok(sha256_hex(encoded.as_bytes())) +} + +fn dict<'py>( + py: Python<'py>, + entries: Vec<(&str, Bound<'py, PyAny>)>, +) -> PyResult> { + let built: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in entries { + built.set_item(key, value)?; + } + Ok(built) +} + +/// `sorted(items)`, as Python orders them. +fn sorted<'py>(items: Bound<'py, PyList>) -> PyResult> { + items + .py() + .import("builtins")? + .getattr("sorted")? + .call1((items,)) +} + +fn package_version<'py>(py: Python<'py>, package: &str) -> PyResult> { + match imported(py, METADATA_MODULE, "version")?.call1((package,)) { + Ok(version) => Ok(version), + Err(error) + if error.is_instance(py, &imported(py, METADATA_MODULE, "PackageNotFoundError")?) => + { + "unknown".into_bound_py_any(py) + } + Err(error) => Err(error), + } +} + +/// `sorted((str(dep.resource_type), dep.name) for dep in deps)`. +fn sorted_deps<'py>(deps: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = deps.py(); + let pairs: Bound<'py, PyList> = PyList::empty(py); + for dep in deps.try_iter()? { + let dep: Bound<'py, PyAny> = dep?; + pairs.append((dep.getattr("resource_type")?.str()?, dep.getattr("name")?))?; + } + sorted(pairs) +} + +fn model_identity(model: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = model.py(); + let values: Bound<'_, PyAny> = model.getattr("config")?.getattr("values")?; + payload_digest( + dict( + py, + vec![ + ("name", model.getattr("name")?), + ("query_sql", model.getattr("query_sql")?), + ("deps", sorted_deps(&model.getattr("deps")?)?), + ( + "cursor", + ( + values.call_method1("get", ("cursor_type",))?, + values.call_method1("get", ("cursor_grain",))?, + ) + .into_bound_py_any(py)?, + ), + ], + )? + .as_any(), + ) +} + +/// Hash adapter and model inputs once for every test artifact identity. +pub(crate) fn identity_context<'py>( + project: &Bound<'py, PyAny>, + adapter: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let adapter_type: Bound<'py, PyAny> = adapter.get_type().into_any(); + let functions: Bound<'py, PyList> = PyList::empty(py); + for function in project.getattr("functions")?.try_iter()? { + let function: Bound<'py, PyAny> = function?; + functions.append(( + function.getattr("name")?, + function.getattr("destination")?.getattr("qualified_name")?, + ))?; + } + let version_tuple: Bound<'py, PyAny> = py + .import("platform")? + .call_method0("python_version_tuple")? + .get_item(pyo3::types::PySlice::new(py, 0, 2, 1))?; + let common: Bound<'py, PyDict> = dict( + py, + vec![ + ("algorithm", ALGORITHM_FINGERPRINT.into_bound_py_any(py)?), + ("sqlbuild_version", package_version(py, "sqlbuild")?), + ( + "polyglot_version", + package_version(py, "polyglot-sql-chio")?, + ), + ("python_version", version_tuple), + ( + "adapter_class", + format!( + "{}.{}", + adapter_type.getattr("__module__")?.str()?, + adapter_type.getattr("__qualname__")?.str()? + ) + .into_bound_py_any(py)?, + ), + ( + "set_difference_operator", + adapter.call_method0("render_set_difference_operator")?, + ), + ( + "sql_analysis_dialect", + adapter.call_method0("sql_analysis_dialect")?, + ), + ( + "requires_derived_table_aliases", + adapter.call_method0("requires_derived_table_aliases")?, + ), + ( + "sql_analysis_enabled", + project.getattr("settings")?.getattr("sql_analysis")?, + ), + ( + "effective_target_name", + project.getattr("effective_target_name")?, + ), + ( + "effective_target_database", + project.getattr("effective_target_database")?, + ), + ( + "effective_target_schema", + project.getattr("effective_target_schema")?, + ), + ("function_locations", sorted(functions)?), + ], + )?; + let model_identities: Bound<'py, PyDict> = PyDict::new(py); + for model in project.getattr("models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + model_identities.set_item(model.getattr("name")?, model_identity(&model)?)?; + } + Ok(IdentityContext { + common_identity: payload_digest(common.as_any())?, + model_identities, + }) +} + +/// `[(cte.name, cte.sql_body) for cte in ctes]`. +fn cte_payload<'py>(ctes: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = ctes.py(); + let rows: Bound<'py, PyList> = PyList::empty(py); + for cte in ctes.try_iter()? { + let cte: Bound<'py, PyAny> = cte?; + rows.append((cte.getattr("name")?, cte.getattr("sql_body")?))?; + } + Ok(rows.into_any()) +} + +/// `sorted(mapping.items())`. +fn sorted_items<'py>(mapping: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = mapping.py(); + sorted(PyList::new( + py, + mapping + .call_method0("items")? + .try_iter()? + .collect::>>()?, + )?) +} + +fn test_payload<'py>(test: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = test.py(); + let payload: Bound<'py, PyDict> = dict( + py, + vec![ + ("case_fingerprint", test.getattr("case_fingerprint")?), + ("mode", test.getattr("mode")?.getattr("value")?), + ("sql_body", test.getattr("sql_body")?), + ("scope_deps", sorted_deps(&test.getattr("scope_deps")?)?), + ], + )?; + let test_payload: Bound<'py, PyAny> = test.getattr("payload")?; + if test_payload.is_instance(&imported( + py, + COMPILE_MODELS_MODULE, + "CompiledModelSqlTestPayload", + )?)? { + let block: Bound<'py, PyAny> = test.getattr("test_block")?; + payload.set_item( + "model", + dict( + py, + vec![ + ( + "authored_ctes", + cte_payload(&test_payload.getattr("authored_ctes")?)?, + ), + ( + "macro_mocks", + sorted_items(&test_payload.getattr("macro_mocks")?)?, + ), + ( + "model_query_overrides", + sorted_items(&test_payload.getattr("model_query_overrides")?)?, + ), + ( + "expected_model_names", + test_payload.getattr("expected_model_names")?, + ), + ( + "assertion_ctes", + cte_payload(&test_payload.getattr("assertion_ctes")?)?, + ), + ( + "cursor_window", + (block.getattr("cursor_start")?, block.getattr("cursor_end")?) + .into_bound_py_any(py)?, + ), + ], + )?, + )?; + } else if test_payload.is_instance(&imported( + py, + COMPILE_MODELS_MODULE, + "CompiledDirectLogicSqlTestPayload", + )?)? { + payload.set_item( + "direct", + dict( + py, + vec![ + ( + "helper_ctes", + cte_payload(&test_payload.getattr("helper_ctes")?)?, + ), + ( + "actual_cte", + cte_payload( + PyTuple::new(py, [test_payload.getattr("actual_cte")?])?.as_any(), + )?, + ), + ( + "expected_cte", + cte_payload( + PyTuple::new(py, [test_payload.getattr("expected_cte")?])?.as_any(), + )?, + ), + ( + "tested_resource_names", + test_payload.getattr("tested_resource_names")?, + ), + ], + )?, + )?; + } + Ok(payload) +} + +/// The complete identity of one rendered comparison artifact. +pub(crate) fn artifact_identity( + test: &Bound<'_, PyAny>, + model_chain_names: &Bound<'_, PyAny>, + context: &IdentityContext<'_>, +) -> PyResult { + let py: Python<'_> = test.py(); + let chain: Bound<'_, PyList> = PyList::empty(py); + for name in model_chain_names.try_iter()? { + let name: Bound<'_, PyAny> = name?; + let identity: Option> = context.model_identities.get_item(&name)?; + chain.append((name, identity))?; + } + payload_digest( + dict( + py, + vec![ + ( + "common_identity", + context.common_identity.as_str().into_bound_py_any(py)?, + ), + ("test", test_payload(test)?.into_any()), + ("model_chain", chain.into_any()), + ], + )? + .as_any(), + ) +} + +/// The stable cache slot of one source test case. +pub(crate) fn artifact_record_key(test: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = test.py(); + let source_path: Bound<'_, PyAny> = test.getattr("source_path")?; + payload_digest( + dict( + py, + vec![ + ( + "source_path", + if source_path.is_none() { + source_path + } else { + source_path.call_method0("as_posix")? + }, + ), + ("block_index", test.getattr("block_index")?), + ("name", test.getattr("name")?), + ("parent_name", test.getattr("parent_name")?), + ("case_name", test.getattr("case_name")?), + ("case_index", test.getattr("case_index")?), + ], + )? + .as_any(), + ) +} + +/// A string field of a stored record, or `None` when absent or not a string. +fn stored_text(fields: &Bound<'_, PyDict>, key: &str) -> PyResult> { + match fields.get_item(key)? { + Some(item) if item.is_instance_of::() => Ok(Some(item.extract()?)), + _ => Ok(None), + } +} + +/// A non-negative integer field of a stored record, or `None` when absent, negative or not an `int`. +fn stored_integer(fields: &Bound<'_, PyDict>, key: &str) -> PyResult> { + let Some(item) = fields.get_item(key)? else { + return Ok(None); + }; + if !item.is_instance_of::() || item.is_instance_of::() { + return Ok(None); + } + let Ok(number) = item.extract::() else { + return Ok(None); + }; + Ok(Some(number)) +} + +/// One stored record, or `None` when any field has the wrong type or an unsafe path. +fn stored_record(value: &Bound<'_, PyAny>) -> PyResult> { + let Ok(fields) = value.downcast::() else { + return Ok(None); + }; + let (Some(identity), Some(relative_path), Some(content_sha256)) = ( + stored_text(fields, "identity")?, + stored_text(fields, "relative_path")?, + stored_text(fields, "content_sha256")?, + ) else { + return Ok(None); + }; + let (Some(size), Some(mtime_ns)) = ( + stored_integer(fields, "size")?, + stored_integer(fields, "mtime_ns")?, + ) else { + return Ok(None); + }; + let Ok(mtime_ns) = i64::try_from(mtime_ns) else { + return Ok(None); + }; + Ok(stored_sql_test_record(StoredSqlTestRecord { + identity, + relative_path, + size, + mtime_ns, + content_sha256, + })) +} + +/// The valid cache records; a cache that cannot be read never fails compilation. +pub(crate) fn read_cache( + cache_dir: &Bound<'_, PyAny>, +) -> PyResult> { + let py: Python<'_> = cache_dir.py(); + if cache_dir.is_none() { + return Ok(HashMap::new()); + } + let cache_path: PathBuf = cache_dir.extract::()?.join(CACHE_FILE_NAME); + let Ok(metadata) = std::fs::metadata(&cache_path) else { + return Ok(HashMap::new()); + }; + if !metadata.is_file() || metadata.len() > MAX_CACHE_BYTES { + return Ok(HashMap::new()); + } + let Ok(contents) = std::fs::read(&cache_path) else { + return Ok(HashMap::new()); + }; + let Ok(text) = String::from_utf8(contents) else { + return Ok(HashMap::new()); + }; + let payload: Bound<'_, PyAny> = match imported(py, "json", "loads")?.call1((text,)) { + Ok(payload) => payload, + Err(error) if error.is_instance_of::(py) => { + return Ok(HashMap::new()); + } + Err(error) => return Err(error), + }; + let Ok(payload) = payload.downcast::() else { + return Ok(HashMap::new()); + }; + let version: Option> = payload.get_item("version")?; + if !version.is_some_and(|version| version.eq(CACHE_VERSION).unwrap_or(false)) { + return Ok(HashMap::new()); + } + let Some(raw_records) = payload.get_item("records")? else { + return Ok(HashMap::new()); + }; + let Ok(raw_records) = raw_records.downcast::() else { + return Ok(HashMap::new()); + }; + if raw_records.len() > MAX_CACHE_RECORDS { + return Ok(HashMap::new()); + } + let mut records: HashMap = HashMap::new(); + for (key, value) in raw_records.iter() { + if let Some(record) = stored_record(&value)? { + records.insert(key.extract()?, record); + } + } + Ok(records) +} + +/// Atomically replace the cache snapshot; a cache that cannot be written is ignored. +pub(crate) fn write_cache( + cache_dir: &Bound<'_, PyAny>, + records: &[(String, SqlTestArtifactRecord)], +) -> PyResult<()> { + let py: Python<'_> = cache_dir.py(); + if cache_dir.is_none() { + return Ok(()); + } + let rows: Bound<'_, PyDict> = PyDict::new(py); + for (key, record) in records { + rows.set_item( + key, + dict( + py, + vec![ + ("identity", record.identity.as_str().into_bound_py_any(py)?), + ( + "relative_path", + record.relative_posix().into_bound_py_any(py)?, + ), + ("size", record.size.into_bound_py_any(py)?), + ("mtime_ns", record.mtime_ns.into_bound_py_any(py)?), + ( + "content_sha256", + record.content_sha256.as_str().into_bound_py_any(py)?, + ), + ], + )?, + )?; + } + let payload: Bound<'_, PyDict> = dict( + py, + vec![ + ("version", CACHE_VERSION.into_bound_py_any(py)?), + ("records", rows.into_any()), + ], + )?; + let encoded: String = compact_sorted_json(payload.as_any())?; + let directory: PathBuf = cache_dir.extract()?; + let temporary: PathBuf = + directory.join(format!(".{CACHE_FILE_NAME}.{}.tmp", std::process::id())); + let written: std::io::Result<()> = py.detach(|| { + std::fs::create_dir_all(&directory)?; + std::fs::write(&temporary, encoded.as_bytes())?; + std::fs::rename(&temporary, directory.join(CACHE_FILE_NAME)) + }); + let _ = written; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs index 8d9c267653..750433cbec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis.rs @@ -16,7 +16,12 @@ const ADAPTERS_MODULE: &str = "sqlbuild.cli.commands._helpers.runtime.adapters"; const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation_lifecycle"; const SELECTED_GRAPH_MODULE: &str = "sqlbuild._native"; -const LINEAGE_MODULE: &str = "sqlbuild.cli.commands._helpers.compile.lineage"; +const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; +const RICH_LINEAGE: &str = "rich"; +const FAST_LINEAGE: &str = "fast"; +const RICH_COLUMN_LINEAGE: &str = "RICH"; +const FAST_COLUMN_LINEAGE: &str = "FAST"; + const CONTRACTS_MODULE: &str = "sqlbuild.compiler.contracts.main.validate"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.compiler.contracts.models"; const SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline.main.project_graph_selection"; @@ -32,6 +37,33 @@ const CLI_COMPILE_MODELS_MODULE: &str = "sqlbuild.cli.compile.models"; const GENERIC_DIALECT: &str = "generic"; const TARGET_DIRECTORY: &str = "target"; +/// `ColumnLineageMode.`. +fn column_lineage_mode<'py>(py: Python<'py>, member: &str) -> PyResult> { + imported(py, LINEAGE_TYPES_MODULE, "ColumnLineageMode")?.getattr(member) +} + +/// Column lineage in the compile's mode; `none` skips the analyzer. +fn compile_lineage<'py>( + graph: &Bound<'py, PyAny>, + dialect: &Bound<'py, PyAny>, + mode: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = graph.py(); + let member: &str = if mode.eq(RICH_LINEAGE)? { + RICH_COLUMN_LINEAGE + } else if mode.eq(FAST_LINEAGE)? { + FAST_COLUMN_LINEAGE + } else { + return Ok(py.None().into_bound(py)); + }; + imported(py, "sqlbuild._native", "build_project_column_lineage")?.call1(( + graph.getattr("project")?, + dialect, + column_lineage_mode(py, member)?, + py.None(), + )) +} + /// Milliseconds since `start`, truncated as Python's `int((monotonic() - start) * 1000)`. fn elapsed_ms(start: Instant) -> u64 { (start.elapsed().as_secs_f64() * 1000.0) as u64 @@ -335,9 +367,14 @@ impl<'py> Driver<'_, 'py> { (options.analysis_selection.clone(),), Some(&keywords(py, &[("on_inputs_ready", on_inputs_ready)])?), )?; - let lineage_mode: Bound<'py, PyAny> = - imported(py, LINEAGE_MODULE, "compile_analysis_lineage_mode")? - .call1((options.lineage_mode.clone(),))?; + let lineage_mode: Bound<'py, PyAny> = column_lineage_mode( + py, + if options.lineage_mode.eq(RICH_LINEAGE)? { + RICH_COLUMN_LINEAGE + } else { + FAST_COLUMN_LINEAGE + }, + )?; let graph: Bound<'py, PyAny> = within(&lifecycle, |_| { call_with( &imported( @@ -374,14 +411,8 @@ impl<'py> Driver<'_, 'py> { ) -> PyResult<(Bound<'py, PyAny>, u64)> { let started: Instant = Instant::now(); self.status.start("Analyzing column lineage...")?; - let lineage: Bound<'py, PyAny> = call_with( - &imported(self.py, LINEAGE_MODULE, "build_compile_lineage")?, - &[ - ("graph", graph.clone()), - ("dialect", dialect.clone()), - ("mode", self.options.lineage_mode.clone()), - ], - )?; + let lineage: Bound<'py, PyAny> = + compile_lineage(graph, dialect, &self.options.lineage_mode)?; let lineage_ms: u64 = elapsed_ms(started); self.status.complete(&format!( "Analyzed column lineage. ({}s)", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index 6172668169..059f75bbd8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -18,7 +18,7 @@ pub(crate) const NATIVE_STAGES_MODULE: &str = "sqlbuild.compiler.compile._helpers.native_stages.assembly"; const ANALYSIS_CACHE_MODULE: &str = "sqlbuild.compiler.compile._helpers.analysis.cache"; pub(crate) const ANALYSIS_MODELS_MODULE: &str = "sqlbuild.compiler.analysis_session.models"; -const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main.record"; +const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._record"; const PROFILING_CPU_MODULE: &str = "sqlbuild.compiler.profiling.main._record_cpu"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const PLANNER_TYPES_MODULE: &str = "sqlbuild.compiler.planner.types"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index f93804f240..f59594105c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -1,9 +1,15 @@ //! The `sqb compile` command: analysis, output phases, reuse recording and printed output. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyMapping, PyStringMethods, PyTuple}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyMapping, PyString, PyStringMethods, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use crate::bindings::_helpers::compile_command::report::{ + format_compile_json, format_compile_text, json_semantic_notice, +}; +use crate::bindings::_helpers::compile_command::reuse_recording::{ + CompileArtifactWrites, RecordedCompileOutput, compile_reuse_timings, new_artifact_recorder, +}; use crate::bindings::_helpers::compile_driver::analysis::analyze; use crate::bindings::_helpers::compile_driver::phases::{ elapsed_ms, manifest_payload, monotonic, resolved_dag_path, sql_test_planning_diagnostics, @@ -15,19 +21,13 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ const REUSE_MODELS_MODULE: &str = "sqlbuild.cli.compile_reuse.models"; const REUSE_TYPES_MODULE: &str = "sqlbuild.cli.compile_reuse.types"; -const RECORDED_OUTPUT_MODULE: &str = "sqlbuild.cli.compile_reuse.classes.recorded_compile_output"; -const REUSE_CONSTANTS_MODULE: &str = "sqlbuild.cli.compile_reuse.constants"; -const REUSE_TIMINGS_MODULE: &str = "sqlbuild.cli.compile_reuse.main._compile_reuse_timings"; -const WRITE_REUSABLE_MODULE: &str = "sqlbuild.cli.compile_reuse.main._write_reusable_compile"; +const NATIVE_MODULE: &str = "sqlbuild._native"; const STATUS_REPORTER_MODULE: &str = "sqlbuild.presentation.classes.transient_status_reporter"; const COMPILE_CONSTANTS_MODULE: &str = "sqlbuild.compiler.compile.constants"; const COMPILE_MODELS_MODULE: &str = "sqlbuild.compiler.compile.models"; const COMPILE_TYPES_MODULE: &str = "sqlbuild.compiler.compile.types"; const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._collect"; -const PREPARED_ARTIFACTS_MODULE: &str = "sqlbuild.cli.commands.classes.prepared_compile_artifacts"; const SKIPPED_RULES_MODULE: &str = "sqlbuild.rule_engine.main.render_skipped_rules"; -const SEMANTIC_NOTICE_MODULE: &str = "sqlbuild.cli.commands._helpers.compile.semantic_notice"; -const OUTPUT_MODULE: &str = "sqlbuild.cli.commands._helpers.compile.output"; const SUPPORTS_COLOR_MODULE: &str = "sqlbuild.presentation.main.supports_color"; /// What one full compile reports for reuse recording. @@ -42,7 +42,7 @@ struct CommandOutcome<'py> { struct Recorders<'py> { detailed_timings: Bound<'py, PyAny>, input_reads: Bound<'py, PyAny>, - artifact_writes: Bound<'py, PyAny>, + artifact_writes: Py, prepared_artifacts: Bound<'py, PyAny>, } @@ -74,8 +74,7 @@ fn run_compile_command<'py>( Some(attempt) => attempt, None => bypass_attempt(py, &effective_request, total_start)?, }; - let output: Bound<'py, PyAny> = - imported(py, RECORDED_OUTPUT_MODULE, "RecordedCompileOutput")?.call0()?; + let output: Bound<'py, PyAny> = Bound::new(py, RecordedCompileOutput::new())?.into_any(); let status: Status<'py> = Status(compile_status(&request)?); let gc: Bound<'py, PyModule> = py.import("gc")?; let was_enabled: bool = gc.call_method0("isenabled")?.is_truthy()?; @@ -167,41 +166,40 @@ fn recorded_compile<'py>( &imported(py, COMPILE_CONSTANTS_MODULE, "COMPILE_INPUT_READS")? .call_method0("recording")?, |input_reads| { - within( - &imported(py, REUSE_CONSTANTS_MODULE, "COMPILE_ARTIFACT_WRITES")? - .call_method0("recording")?, - |artifact_writes| { - let manager: Bound<'py, PyAny> = call_with( - &imported( - py, - PREPARED_ARTIFACTS_MODULE, - "PreparedCompileArtifacts", - )?, - &[( - "enabled", - PyBool::new(py, !skip_write).to_owned().into_any(), - )], - )?; - within(&manager, |prepared_artifacts| { - let recorders: Recorders<'py> = Recorders { - detailed_timings: detailed_timings.clone(), - input_reads: input_reads.clone(), - artifact_writes: artifact_writes.clone(), - prepared_artifacts: prepared_artifacts.clone(), - }; - let reuse_timings: Bound<'py, PyAny> = call_with( - &imported(py, REUSE_TIMINGS_MODULE, "compile_reuse_timings")?, - &[("attempt", attempt.clone())], - )?; - let outcome: CommandOutcome<'py> = compiled_with_status( - (request, &recorders), - (status, output), - (total_start, &reuse_timings), - )?; - Ok((outcome, recorders.input_reads, recorders.artifact_writes)) - }) - }, - ) + let artifact_writes: Py = new_artifact_recorder(py)?; + let manager: Bound<'py, PyAny> = call_with( + &imported(py, NATIVE_MODULE, "PreparedCompileArtifacts")?, + &[ + ( + "enabled", + PyBool::new(py, !skip_write).to_owned().into_any(), + ), + ( + "artifact_writes", + artifact_writes.bind(py).clone().into_any(), + ), + ], + )?; + within(&manager, |prepared_artifacts| { + let recorders: Recorders<'py> = Recorders { + detailed_timings: detailed_timings.clone(), + input_reads: input_reads.clone(), + artifact_writes: artifact_writes.clone_ref(py), + prepared_artifacts: prepared_artifacts.clone(), + }; + let reuse_timings: Bound<'py, PyAny> = + compile_reuse_timings(attempt)?.into_any(); + let outcome: CommandOutcome<'py> = compiled_with_status( + (request, &recorders), + (status, output), + (total_start, &reuse_timings), + )?; + Ok(( + outcome, + recorders.input_reads, + recorders.artifact_writes.into_bound(py).into_any(), + )) + }) }, ) }, @@ -209,7 +207,7 @@ fn recorded_compile<'py>( close_status(status)?; let (outcome, input_reads, artifact_writes) = compiled?; let stored: PyResult> = call_with( - &imported(py, WRITE_REUSABLE_MODULE, "write_reusable_compile")?, + &imported(py, NATIVE_MODULE, "write_reusable_compile")?, &[ ("attempt", attempt.clone()), ("output", output.clone()), @@ -273,11 +271,12 @@ fn compiled_with_status<'py>( } else { request.getattr("dag_path")? }; - write_dag_artifact((&dag_path, &project_dir), &analysis, status)?; + let recorder: Option<&Py> = Some(&recorders.artifact_writes); + write_dag_artifact((&dag_path, &project_dir), (&analysis, recorder), status)?; let (written, write_ms): (Bound<'py, PyAny>, i64) = write_artifacts( (skip_write || rules_failed, &project_dir), (&analysis, &manifest), - prepared_artifacts, + (prepared_artifacts, recorder), status, )?; let timings: Bound<'py, PyDict> = timings_ms( @@ -286,7 +285,7 @@ fn compiled_with_status<'py>( total_start, )?; let withheld: Bound<'py, PyAny> = if rules_failed && !skip_write { - sql_test_planning_diagnostics(&project_dir, &analysis, prepared_artifacts)? + sql_test_planning_diagnostics(&project_dir, &analysis, (prepared_artifacts, recorder))? } else { PyTuple::empty(py).into_any() }; @@ -450,24 +449,18 @@ fn record_output<'py>( ("selected_keys", selected_keys.clone()), ]; let mut arguments: Vec<(&str, Bound<'py, PyAny>)> = shared.to_vec(); - let rendered: Bound<'py, PyAny> = if request.getattr("json_output")?.is_truthy()? { - let notice: Bound<'py, PyAny> = call_with( - &imported(py, SEMANTIC_NOTICE_MODULE, "semantic_coverage_notice")?, - &[ - ("project", graph.getattr("project")?), - ("selected_keys", selected_keys), - ("sql_validation_enabled", validation_enabled.clone()), - ], - )?; - if notice.is_truthy()? { + let rendered: Bound<'py, PyString> = if request.getattr("json_output")?.is_truthy()? { + let notice: Option> = + json_semantic_notice(&graph, &selected_keys, validation_enabled.is_truthy()?)?; + if let Some(notice) = notice { output.call_method1("print_stderr_line", (notice,))?; } arguments.push(("timings_ms", timings.clone().into_any())); arguments.push(("sql_validation_enabled", validation_enabled)); - call_with( - &imported(py, OUTPUT_MODULE, "format_compile_json")?, - &arguments, - )? + PyString::new( + py, + &format_compile_json(py, Some(keywords(py, &arguments)?))?, + ) } else { let use_color: bool = !request.getattr("no_color")?.is_truthy()? && imported(py, SUPPORTS_COLOR_MODULE, "supports_color")? @@ -478,10 +471,7 @@ fn record_output<'py>( PyBool::new(py, use_color).to_owned().into_any(), )); arguments.push(("sql_validation_enabled", validation_enabled)); - call_with( - &imported(py, OUTPUT_MODULE, "format_compile_text")?, - &arguments, - )? + format_compile_text(py, Some(keywords(py, &arguments)?))? }; output.call_method1("record_stdout", (rendered,))?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs index a15255bdf8..8a3277ae13 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/phases.rs @@ -2,16 +2,19 @@ use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyBytes, PyTuple}; +use pyo3::types::PyTuple; +use crate::bindings::_helpers::compile_command::artifact_batch::ArtifactBatch; +use crate::bindings::_helpers::compile_command::reuse_recording::ArtifactRecorder; +use crate::bindings::_helpers::compile_command::target_writer::{ + Prepared, planning_diagnostics, write_static_target, +}; use crate::bindings::_helpers::compile_driver::python_calls::{Status, call_with, imported}; use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; const MANIFEST_MODULE: &str = "sqlbuild.compiler.manifest.main._build_graph_manifest"; const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; const PYTHON_NODES_MODULE: &str = "sqlbuild.compiler.python_nodes.main.graph"; -const ARTIFACT_BATCH_MODULE: &str = "sqlbuild.cli.commands.classes.native_artifact_batch"; -const TARGET_WRITER_MODULE: &str = "sqlbuild.cli.commands._helpers.compile.target_writer"; const OUTPUT_MODELS_MODULE: &str = "sqlbuild.cli.output.models"; const COMPILE_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.compile.exceptions"; const PLANNER_EXCEPTIONS_MODULE: &str = "sqlbuild.compiler.planner.exceptions"; @@ -101,7 +104,7 @@ pub(crate) fn resolved_dag_path<'py>( /// Write the DAG JSON artifact when a DAG path is requested. pub(crate) fn write_dag_artifact<'py>( (dag_path, project_dir): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - analysis: &Bound<'py, PyAny>, + (analysis, recorder): (&Bound<'py, PyAny>, ArtifactRecorder<'_>), status: &Status<'py>, ) -> PyResult<()> { if dag_path.is_none() { @@ -129,21 +132,9 @@ pub(crate) fn write_dag_artifact<'py>( &project_name, Some(&python_graph), )?; - let batch: Bound<'py, PyAny> = call_with( - &imported(py, ARTIFACT_BATCH_MODULE, "NativeArtifactBatch")?, - &[( - "check_existing", - PyBool::new(py, false).to_owned().into_any(), - )], - )?; - call_with( - &batch.getattr("queue")?, - &[ - ("path", path), - ("contents", PyBytes::new(py, dag_json.as_bytes()).into_any()), - ], - )?; - batch.call_method0("flush")?; + let mut batch: ArtifactBatch = ArtifactBatch::new(false, recorder); + batch.queue(&path, dag_json.into_bytes())?; + batch.flush(py)?; status.complete(&format!( "Wrote DAG artifact. ({}s)", seconds_since(py, started)? @@ -154,7 +145,7 @@ pub(crate) fn write_dag_artifact<'py>( pub(crate) fn sql_test_planning_diagnostics<'py>( project_dir: &Bound<'py, PyAny>, analysis: &Bound<'py, PyAny>, - prepared_artifacts: &Bound<'py, PyAny>, + (prepared_artifacts, recorder): (&Bound<'py, PyAny>, ArtifactRecorder<'_>), ) -> PyResult> { let py: Python<'py> = analysis.py(); let planned: PyResult> = (|| { @@ -165,18 +156,18 @@ pub(crate) fn sql_test_planning_diagnostics<'py>( return Ok(staged); } } - call_with( - &imported( - py, - TARGET_WRITER_MODULE, - "static_sql_test_planning_diagnostics", - )?, - &[ - ("target_dir", target_dir(project_dir)?), - ("adapter", analysis.getattr("adapter")?), - ("project", analysis.getattr("graph")?.getattr("project")?), - ], - ) + Ok(planning_diagnostics( + &target_dir(project_dir)?, + ( + &analysis.getattr("adapter")?, + &analysis.getattr("graph")?.getattr("project")?, + ), + Prepared { + planned_tests: None, + recorder, + }, + )? + .into_any()) })(); let error = match planned { Ok(diagnostics) => return Ok(diagnostics), @@ -221,7 +212,7 @@ pub(crate) fn sql_test_planning_diagnostics<'py>( pub(crate) fn write_artifacts<'py>( (skip_write, project_dir): (bool, &Bound<'py, PyAny>), (analysis, manifest): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - prepared_artifacts: &Bound<'py, PyAny>, + (prepared_artifacts, recorder): (&Bound<'py, PyAny>, ArtifactRecorder<'_>), status: &Status<'py>, ) -> PyResult<(Bound<'py, PyAny>, i64)> { let py: Python<'py> = analysis.py(); @@ -254,14 +245,17 @@ pub(crate) fn write_artifacts<'py>( )? }; if staged.is_none() { - call_with( - &imported(py, TARGET_WRITER_MODULE, "write_static_compile_target")?, - &[ - ("target_dir", target), - ("adapter", analysis.getattr("adapter")?), - ("project", analysis.getattr("graph")?.getattr("project")?), - ("manifest", manifest.clone()), - ], + write_static_target( + &target, + ( + &analysis.getattr("adapter")?, + &analysis.getattr("graph")?.getattr("project")?, + ), + manifest, + Prepared { + planned_tests: None, + recorder, + }, )? } else { staged diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs index cdd8a42d09..d4c88faade 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/project.rs @@ -27,7 +27,7 @@ const PROMOTION_MODULE: &str = "sqlbuild.compiler.contracts.main.promotion_confl const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const LINEAGE_TYPES_MODULE: &str = "sqlbuild.compiler.lineage.types"; const ANALYSIS_SELECTION_MODULE: &str = "sqlbuild.compiler.pipeline._helpers.analysis_selection"; -const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main.record"; +const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._record"; const ADAPTER_NAME_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_adapter_name"; const COLLECTION_RENDERING_MODULE: &str = "sqlbuild.spec.contracts.main.resolve_effective_collection_rendering"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs index 228e421fdf..b1718b837f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs @@ -1,12 +1,11 @@ -//! Write, clean and publish compiled artifacts natively for the Python target writer. +//! Write, clean and publish compiled artifacts natively, raising the errors Python raised. use std::collections::HashSet; use std::path::PathBuf; use pyo3::exceptions::PyOSError; -use pyo3::prelude::{Bound, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::prelude::{PyAnyMethods, PyErr, PyResult, Python}; use pyo3::types::PyBytes; -use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_cache::artifacts::errors::ArtifactError; use sqlbuild_cache::artifacts::main::publish_staged::publish_staged; use sqlbuild_cache::artifacts::main::remove_stale::remove_stale; @@ -19,8 +18,7 @@ use crate::bindings::_helpers::boundary::panics::compiler_guard; type PublishedRow = (String, Vec<(String, String)>); /// Write each `(path, contents)` artifact; returns how many were written and left unchanged. -#[pyfunction] -fn write_compiled_artifacts( +pub(crate) fn write_compiled_artifacts( py: Python<'_>, files: Vec<(PathBuf, Vec)>, check_existing: bool, @@ -34,8 +32,7 @@ fn write_compiled_artifacts( } /// Delete unmanaged compiled files and empty directories; returns how many files were removed. -#[pyfunction] -fn remove_stale_artifacts( +pub(crate) fn remove_stale_artifacts( py: Python<'_>, compiled_dir: PathBuf, managed: Vec, @@ -48,8 +45,7 @@ fn remove_stale_artifacts( } /// Publish staged artifacts: `("changed", [])`, `("tree", [])` or `("files", [(source, path)])`. -#[pyfunction] -fn publish_staged_artifacts( +pub(crate) fn publish_staged_artifacts( py: Python<'_>, staged_dir: PathBuf, compiled_dir: PathBuf, @@ -75,8 +71,7 @@ fn publish_staged_artifacts( } /// Every staged artifact's path relative to `staged_dir`. -#[pyfunction] -fn staged_artifact_files(py: Python<'_>, staged_dir: PathBuf) -> PyResult> { +pub(crate) fn staged_artifact_files(py: Python<'_>, staged_dir: PathBuf) -> PyResult> { compiler_guard(|| { let files: Vec = py.detach(|| staged_files(&staged_dir)); Ok(files.iter().map(|path| lossy(path)).collect()) @@ -130,10 +125,3 @@ fn existing_decode_error(py: Python<'_>, path: &PathBuf) -> PyErr { Ok(_) => PyOSError::new_err(format!("compiled artifact '{}' changed", path.display())), } } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(staged_artifact_files, module)?)?; - module.add_function(wrap_pyfunction!(write_compiled_artifacts, module)?)?; - module.add_function(wrap_pyfunction!(remove_stale_artifacts, module)?)?; - module.add_function(wrap_pyfunction!(publish_staged_artifacts, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs index c502f0aaef..b289578fe4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs @@ -45,7 +45,7 @@ struct Conversion { /// The report as `orjson.dumps(report, option=OPT_INDENT_2)`, raising orjson's `TypeError`. #[pyfunction] -fn emit_orjson_report(report: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn emit_orjson_report(report: &Bound<'_, PyAny>) -> PyResult { compiler_guard(|| { let mut conversion: Conversion = Conversion::default(); let value: JsonValue = conversion.json_value(report)?; @@ -58,7 +58,7 @@ fn emit_orjson_report(report: &Bound<'_, PyAny>) -> PyResult { /// The report as orjson writes it, or as `json.dumps(indent=2)` does when orjson rejects it. #[pyfunction] -fn emit_json_report(report: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn emit_json_report(report: &Bound<'_, PyAny>) -> PyResult { compiler_guard(|| { let mut conversion: Conversion = Conversion::default(); let value: JsonValue = conversion.json_value(report)?; @@ -75,6 +75,38 @@ fn emit_json_report(report: &Bound<'_, PyAny>) -> PyResult { }) } +/// `orjson.dumps(value, option=OPT_SORT_KEYS)`, raising orjson's `TypeError`. +pub(crate) fn orjson_sorted(value: &Bound<'_, PyAny>) -> PyResult { + let mut conversion: Conversion = Conversion::default(); + let converted: JsonValue = conversion.json_value(value)?; + if let Some(error) = conversion.orjson_error { + return Err(PyTypeError::new_err(error)); + } + emitted( + &converted, + &JsonDialect::Orjson(OrjsonOptions { + indent_2: false, + sort_keys: true, + }), + ) +} + +/// `json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True)`. +pub(crate) fn compact_sorted_json(value: &Bound<'_, PyAny>) -> PyResult { + let mut conversion: Conversion = Conversion::default(); + let converted: JsonValue = conversion.json_value(value)?; + if let Some(error) = conversion.stdlib_error { + return Err(error); + } + emitted( + &converted, + &JsonDialect::Stdlib(StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None).with_separators(",", ":") + }), + ) +} + fn emitted(value: &JsonValue, dialect: &JsonDialect) -> PyResult { dumps(value, dialect).map_err(|error| PyTypeError::new_err(format!("{error:?}"))) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs index 4db4e556c6..604ca7f3d2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/registration.rs @@ -2,10 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -use crate::bindings::_helpers::compile_outputs::{artifacts, json_report, reuse}; +use crate::bindings::_helpers::compile_outputs::{json_report, reuse}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - artifacts::register(module)?; json_report::register(module)?; reuse::register(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs index 0abd4df25e..2eec1a8b5b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs @@ -273,8 +273,7 @@ fn check_compile_reuse(py: Python<'_>, check: ReuseCheckPy) -> PyResult String { +pub(crate) fn artifact_digest(contents: &[u8]) -> String { native_digest(contents) } @@ -303,6 +302,5 @@ fn texts(values: &[OsString]) -> Vec { pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; - module.add_function(wrap_pyfunction!(check_compile_reuse, module)?)?; - module.add_function(wrap_pyfunction!(artifact_digest, module)?) + module.add_function(wrap_pyfunction!(check_compile_reuse, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs index 4ac75aaedf..2bb97a8e51 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs @@ -28,7 +28,7 @@ const SCOPE_TYPES_MODULE: &str = "sqlbuild.compiler.scopes.types"; const SCOPE_MODELS_MODULE: &str = "sqlbuild.compiler.scopes.models"; const AUDITING_TYPES_MODULE: &str = "sqlbuild.compiler.auditing.types"; const BUILTIN_AUDITS_MODULE: &str = "sqlbuild.compiler.auditing.main._builtins"; -const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main.record"; +const PROFILING_MODULE: &str = "sqlbuild.compiler.profiling.main._record"; const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; const SQL_VALUE_ERRORS_MODULE: &str = "sqlbuild.sql_values.exceptions"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 268eaac38c..86f280618a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod boundary; pub(crate) mod cache; pub(crate) mod cli; pub(crate) mod compile_attachment; +pub(crate) mod compile_command; pub(crate) mod compile_driver; pub(crate) mod compile_outputs; pub(crate) mod compiled_project; diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 96bf439dfa..bc99bd56b7 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -5,10 +5,10 @@ use sqlbuild_rules::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; use crate::bindings::_helpers::boundary::panics::NativeCompilerError; use crate::bindings::_helpers::{ analysis, analysis_session, attachments, boundary, cache, cli, compile_attachment, - compile_driver, compile_outputs, compiled_project, contracts, dag_artifact, discovery, - fingerprints, frontier, graph, lineage, macro_bridge, manifest, model_config, project_assembly, - refactor, render, rules, scopes, semantic_checks, semantic_shapes, sql_test_glue, sqltext, - type_system, + compile_command, compile_driver, compile_outputs, compiled_project, contracts, dag_artifact, + discovery, fingerprints, frontier, graph, lineage, macro_bridge, manifest, model_config, + project_assembly, refactor, render, rules, scopes, semantic_checks, semantic_shapes, + sql_test_glue, sqltext, type_system, }; use crate::bindings::models; @@ -62,6 +62,7 @@ fn register_analysis_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { graph::registration::register(module)?; compiled_project::registration::register(module)?; compile_attachment::registration::register(module)?; + compile_command::registration::register(module)?; compile_driver::registration::register(module)?; frontier::registration::register(module)?; macro_bridge::registration::register(module)?; diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index ba39d0eb51..c7720dfa57 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -8,13 +8,14 @@ from _typeshed import SupportsRichComparison from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ExpressionInferenceProfile, RelationInfo -from sqlbuild.cli.commands.classes.prepared_compile_artifacts import PreparedCompileArtifacts from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.compile.models import CompileAnalysis from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt from sqlbuild.cli.entry.models import CompileCommandRequest, CompileProfileFlags +from sqlbuild.cli.output.models import WrittenTarget from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity +from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.models import ( CompileAdapterContext, @@ -1007,15 +1008,106 @@ class NativeReuseAttempt: def record(self, record: dict[str, object]) -> tuple[str, int]: ... def check_compile_reuse(check: dict[str, object]) -> NativeReuseAttempt: ... -def artifact_digest(contents: bytes) -> str: ... -def write_compiled_artifacts( - files: list[tuple[Path, bytes]], check_existing: bool -) -> tuple[int, int]: ... -def staged_artifact_files(staged_dir: Path) -> list[str]: ... -def remove_stale_artifacts(compiled_dir: Path, managed: list[str]) -> int: ... -def publish_staged_artifacts( - staged_dir: Path, compiled_dir: Path, expected: list[str] -) -> tuple[str, list[tuple[str, str]]]: ... + +class CompileArtifactWrites: + @property + def artifacts(self) -> dict[str, tuple[str | None, int | None, int | None]]: ... + +class RecordedCompileOutput: + def print_stderr_line(self, line: str) -> None: ... + def record_stdout(self, text: str) -> None: ... + def print_stdout(self) -> None: ... + @property + def stderr_lines(self) -> list[str]: ... + @property + def stdout(self) -> str | None: ... + +def write_reusable_compile( + *, + attempt: CompileReuseAttempt, + output: RecordedCompileOutput, + exit_code: int, + input_reads: CompileInputReads, + artifact_writes: CompileArtifactWrites, + compile_cache_enabled: bool, + artifacts_written: bool, + dag_artifact_path: Path | None, + json_output: bool, +) -> None: ... +def format_compile_text( + *, + graph: ProjectGraph, + written: WrittenTarget, + manifest: bool, + lineage: ProjectColumnLineage | None, + diagnostics: tuple[CompilerDiagnostic, ...], + lineage_mode: CompileLineageMode = ..., + use_color: bool, + selected_keys: frozenset[CompiledObjectKey] | None = None, + sql_validation_enabled: bool = True, +) -> str: ... +def format_compile_json( + *, + graph: ProjectGraph, + written: WrittenTarget, + manifest: bool, + timings_ms: dict[str, int], + lineage: ProjectColumnLineage | None, + diagnostics: tuple[CompilerDiagnostic, ...], + lineage_mode: CompileLineageMode = ..., + selected_keys: frozenset[CompiledObjectKey] | None = None, + sql_validation_enabled: bool = True, +) -> str: ... +def format_compile_error_json( + *, code: str, message: str, help_text: str | None, location: SourceLocation | None = None +) -> str: ... + +class PlannedStaticSqlTests: ... + +def write_static_compile_target( + *, + target_dir: Path, + adapter: BaseAdapter, + project: CompiledProjectView, + manifest: NativeManifest | None = None, +) -> WrittenTarget: ... +def write_compile_target( + *, + target_dir: Path, + adapter: BaseAdapter, + plan_output: PlanOutput, + manifest: NativeManifest | None = None, +) -> WrittenTarget: ... +def static_sql_test_planning_diagnostics( + *, target_dir: Path, adapter: BaseAdapter, project: CompiledProjectView +) -> tuple[CompilerDiagnostic, ...]: ... +def plan_static_sql_tests( + *, target_dir: Path, adapter: BaseAdapter, project: CompiledProjectView +) -> PlannedStaticSqlTests: ... + +class PreparedCompileArtifacts: + min_prepared_models: int + max_prepared_models: int + def __init__(self, *, enabled: bool) -> None: ... + @property + def enabled(self) -> bool: ... + def __enter__(self) -> PreparedCompileArtifacts: ... + def __exit__(self, *exception: object) -> None: ... + def remove_abandoned_staging(self, *, target_dir: Path) -> None: ... + def create_staging_directory(self, staging_dir: Path) -> None: ... + def start( + self, + *, + project: CompiledProjectView, + adapter: BaseAdapter, + target_dir: Path, + stage_artifacts: bool, + ) -> None: ... + def publish( + self, *, target_dir: Path, manifest: NativeManifest | None + ) -> WrittenTarget | None: ... + def planning_diagnostics(self) -> tuple[CompilerDiagnostic, ...] | None: ... + def finalize_rule_findings_rows( request: dict[str, object], ) -> list[tuple[bool, str, str, int, int, str, str]]: ... diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/compile_target.py b/src/sqlbuild/cli/commands/_helpers/build_planning/compile_target.py index c9d62648a6..5209610623 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/compile_target.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/compile_target.py @@ -3,7 +3,7 @@ from __future__ import annotations import sqlbuild._native as _native -from sqlbuild.cli.commands._helpers.compile.target_writer import write_compile_target +from sqlbuild._native import write_compile_target from sqlbuild.cli.commands.models import ( BuildCommandRequest, BuildInvocation, diff --git a/src/sqlbuild/cli/commands/_helpers/compile/__init__.py b/src/sqlbuild/cli/commands/_helpers/compile/__init__.py deleted file mode 100644 index 6f523aefd0..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/compile/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Compile command helpers.""" diff --git a/src/sqlbuild/cli/commands/_helpers/compile/lineage.py b/src/sqlbuild/cli/commands/_helpers/compile/lineage.py deleted file mode 100644 index 0aaad2bd5b..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/compile/lineage.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Compile command lineage helpers.""" - -from __future__ import annotations - -import sqlbuild._native as _native -from sqlbuild._native import ProjectColumnLineage -from sqlbuild.cli.commands.types import CompileLineageMode -from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.compiler.pipeline.models import ProjectGraph - - -def build_compile_lineage( - *, - graph: ProjectGraph, - dialect: str | None, - mode: CompileLineageMode, -) -> ProjectColumnLineage | None: - match mode: - case CompileLineageMode.NONE: - return None - case CompileLineageMode.FAST: - return _native.build_project_column_lineage( - graph.project, dialect, ColumnLineageMode.FAST, None - ) - case CompileLineageMode.RICH: - return _native.build_project_column_lineage( - graph.project, dialect, ColumnLineageMode.RICH, None - ) - - -def compile_analysis_lineage_mode(mode: CompileLineageMode) -> ColumnLineageMode: - if mode == CompileLineageMode.RICH: - return ColumnLineageMode.RICH - return ColumnLineageMode.FAST diff --git a/src/sqlbuild/cli/commands/_helpers/compile/output.py b/src/sqlbuild/cli/commands/_helpers/compile/output.py deleted file mode 100644 index 08b08badab..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/compile/output.py +++ /dev/null @@ -1,726 +0,0 @@ -"""Compile command output formatting.""" - -from __future__ import annotations - -from collections import Counter -from importlib.metadata import PackageNotFoundError, version -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild._native import ProjectColumnLineage -from sqlbuild.cli.commands._helpers.compile.semantic_notice import ( - selected_semantic_coverage, - semantic_coverage_notice, -) -from sqlbuild.cli.commands.constants import TARGET_DIRECTORY_NAME -from sqlbuild.cli.commands.types import CompileLineageMode -from sqlbuild.cli.output.models import ( - WrittenTarget, -) -from sqlbuild.compiler.compile.models import ( - CompiledModelSqlTestPayload, - CompiledObjectKey, - CompiledSqlTest, - CompilerDiagnostic, - RelatedLocation, -) -from sqlbuild.compiler.compile.types import ( - CompiledAuditView, - CompiledFunctionView, - CompiledModelView, - CompiledProjectView, - CompiledResourceType, - CompiledSeedView, - CompiledSourceView, - DiagnosticPhase, - DiagnosticSeverity, -) -from sqlbuild.compiler.discovery.constants import SQL_HOOK_OUTPUT_FIELDS -from sqlbuild.compiler.discovery.main.serialize_hook_entries import serialize_hook_entries -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.phase_line import format_phase_line -from sqlbuild.presentation.main.status_cell import format_status_cell -from sqlbuild.presentation.main.surface_header import format_surface_header -from sqlbuild.presentation.main.tree_connector import tree_connector -from sqlbuild.spec.contracts.models import SourceLocation -from sqlbuild.sql_values.types import SqlValueKind - -_HUMAN_MODEL_LIMIT: int = 100 -_MIN_MODEL_NAME_WIDTH: int = 24 -_MAX_MODEL_NAME_WIDTH: int = 48 - - -def format_compile_text( - *, - graph: ProjectGraph, - written: WrittenTarget, - manifest: bool, - lineage: ProjectColumnLineage | None, - diagnostics: tuple[CompilerDiagnostic, ...], - lineage_mode: CompileLineageMode = CompileLineageMode.FAST, - use_color: bool, - selected_keys: frozenset[CompiledObjectKey] | None = None, - sql_validation_enabled: bool = True, -) -> str: - """Format human-readable compile output.""" - - style: CliStyle = CliStyle(use_color=use_color) - visible_project_models: tuple[CompiledModelView, ...] = _selected_models( - graph=graph, - selected_keys=selected_keys, - ) - lines: list[str] = [ - format_surface_header( - style=style, - title="Compile ready", - context=_count_label(count=len(visible_project_models), singular="model"), - ), - "", - ] - visible_models: tuple[CompiledModelView, ...] = visible_project_models[:_HUMAN_MODEL_LIMIT] - model_name_width: int = _model_name_width(visible_models) - error_models: frozenset[str] = _models_with_error_diagnostics(diagnostics) - model_index: int - for model_index, model in enumerate(visible_models): - model_name: str = _fit(text=model.name, width=model_name_width) - status: str = "FAIL" if model.name in error_models else "OK" - connector: str = tree_connector(style=style, last=model_index == len(visible_models) - 1) - lines.append( - f"{connector} {style.object_name(model_name)} " - f"{format_status_cell(style=style, status=status, width=4)} " - f"{style.muted(f'{model.output_column_count} columns')}" - ) - hidden_model_count: int = len(visible_project_models) - len(visible_models) - if hidden_model_count > 0: - lines.append("") - lines.append( - " " - + style.muted(f"Showing {len(visible_models)} of {len(visible_project_models)} models.") - ) - lines.append(" " + style.muted("Use --json for the full compile report.")) - lines.append("") - notice: str | None = semantic_coverage_notice( - project=graph.project, - selected_keys=selected_keys, - sql_validation_enabled=sql_validation_enabled, - ) - if notice: - lines.append(notice) - lines.append("") - if diagnostics: - lines.append( - _format_diagnostics_text( - diagnostics=diagnostics, - source_texts={ - **_model_source_texts(graph.project.models), - **{ - source.source_file.relative_path: source.source_file.contents - for source in graph.project.sources - }, - **{ - test.test_file.relative_path: test.test_file.contents - for test in graph.project.sql_tests - }, - }, - style=style, - ) - ) - lines.append("") - error_count: int = _diagnostic_count(diagnostics=diagnostics, severity=DiagnosticSeverity.ERROR) - warning_count: int = _diagnostic_count( - diagnostics=diagnostics, severity=DiagnosticSeverity.WARNING - ) - selected_seed_count: int = _selected_resource_count( - graph=graph, - selected_keys=selected_keys, - resource_types={CompiledResourceType.SEED}, - ) - selected_function_count: int = _selected_resource_count( - graph=graph, - selected_keys=selected_keys, - resource_types={CompiledResourceType.UDF, CompiledResourceType.TABLE_FN}, - ) - compiled_summary: str = ( - f"{_count_label(count=len(visible_project_models), singular='model')}, " - f"{_count_label(count=selected_seed_count, singular='seed')}, " - f"{_count_label(count=selected_function_count, singular='function')}, " - f"{_count_label(count=error_count, singular='error')}, " - f"{_count_label(count=warning_count, singular='warning')}" - ) - lines.append( - format_phase_line( - style=style, - ok=error_count == 0, - label="Project compiled", - summary=compiled_summary, - ) - ) - lines.append(f" {style.muted('Wrote:')} {_relative_target_path(_compiled_sql_dir(written))}/") - if manifest: - lines.append(f" {style.muted('Wrote:')} target/manifest.json") - if lineage is not None and lineage_mode == CompileLineageMode.RICH: - lines.append( - f" {style.muted('Column lineage:')} rich analysis complete; " - "inspect with sqb lineage [.]" - ) - elif ( - lineage is None - and graph.project.settings.sql_analysis - and lineage_mode != CompileLineageMode.NONE - ): - lines.append(f" {style.warning('Column lineage:')} unavailable") - return "\n" + "\n".join(lines) + "\n" - - -def format_compile_json( - *, - graph: ProjectGraph, - written: WrittenTarget, - manifest: bool, - timings_ms: dict[str, int], - lineage: ProjectColumnLineage | None, - diagnostics: tuple[CompilerDiagnostic, ...], - lineage_mode: CompileLineageMode = CompileLineageMode.FAST, - selected_keys: frozenset[CompiledObjectKey] | None = None, - sql_validation_enabled: bool = True, -) -> str: - """Serialize the offline compile report as JSON.""" - - result: dict[str, object] = { - "version": _sqlbuild_version(), - "command": "compile", - "offline": True, - "has_errors": any(diagnostic.is_error for diagnostic in diagnostics), - "summary": _summary( - graph=graph, - diagnostics=diagnostics, - selected_keys=selected_keys, - ), - "diagnostics": [_diagnostic_to_json(diagnostic) for diagnostic in diagnostics], - "semantic_checks_partial": selected_semantic_coverage( - project=graph.project, - selected_keys=selected_keys, - sql_validation_enabled=sql_validation_enabled, - ), - "compile_timings": timings_ms, - "lineage_mode": lineage_mode.value, - "resources": _resources(graph=graph, lineage=lineage), - "artifacts": _artifacts(written=written, manifest=manifest), - } - emitted: str = _native.emit_json_report(result) - return emitted - - -def format_compile_error_json( - *, code: str, message: str, help_text: str | None, location: SourceLocation | None = None -) -> str: - """Serialize a compile that stopped on a raised error as one JSON diagnostic report.""" - - diagnostic: CompilerDiagnostic = CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, - code=code, - message=message, - path=location.path if location is not None else None, - line=location.line if location is not None else None, - column=location.column if location is not None else None, - location=location, - help=help_text, - ) - result: dict[str, object] = { - "version": _sqlbuild_version(), - "command": "compile", - "offline": True, - "has_errors": True, - "stopped": True, - "diagnostics": [_diagnostic_to_json(diagnostic)], - } - emitted: str = _native.emit_orjson_report(result) - return emitted - - -def _summary( - *, - graph: ProjectGraph, - diagnostics: tuple[CompilerDiagnostic, ...], - selected_keys: frozenset[CompiledObjectKey] | None = None, -) -> dict[str, int]: - project: CompiledProjectView = graph.project - return { - "models": len(project.models), - "selected_models": len(_selected_models(graph=graph, selected_keys=selected_keys)), - "sources": len(project.sources), - "seeds": len(project.seeds), - "selected_seeds": _selected_resource_count( - graph=graph, - selected_keys=selected_keys, - resource_types={CompiledResourceType.SEED}, - ), - "functions": len(project.functions), - "selected_functions": _selected_resource_count( - graph=graph, - selected_keys=selected_keys, - resource_types={CompiledResourceType.UDF, CompiledResourceType.TABLE_FN}, - ), - "audits": len(project.audits), - "tests": len(project.sql_tests), - "hooks": len(project.sql_hook_files) + len(project.hook_functions), - "execution_layers": _execution_layer_count(graph), - "errors": _diagnostic_count(diagnostics=diagnostics, severity=DiagnosticSeverity.ERROR), - "warnings": _diagnostic_count(diagnostics=diagnostics, severity=DiagnosticSeverity.WARNING), - } - - -def _selected_models( - *, - graph: ProjectGraph, - selected_keys: frozenset[CompiledObjectKey] | None, -) -> tuple[CompiledModelView, ...]: - if selected_keys is None: - return graph.project.models - selected_names: frozenset[str] = frozenset( - key.name for key in selected_keys if key.resource_type == CompiledResourceType.MODEL - ) - return tuple(model for model in graph.project.models if model.name in selected_names) - - -def _selected_resource_count( - *, - graph: ProjectGraph, - selected_keys: frozenset[CompiledObjectKey] | None, - resource_types: set[CompiledResourceType], -) -> int: - if selected_keys is None: - type_names: frozenset[str] = frozenset(map(str, resource_types)) - return sum(kind in type_names for kind, _ in graph.native.keys()) - return sum(key.resource_type in resource_types for key in selected_keys) - - -def _resources(*, graph: ProjectGraph, lineage: ProjectColumnLineage | None) -> dict[str, object]: - project: CompiledProjectView = graph.project - return { - "models": [ - _model_resource(graph=graph, model=model, lineage=lineage) for model in project.models - ], - "sources": [_source_resource(source) for source in project.sources], - "seeds": [_seed_resource(seed) for seed in project.seeds], - "functions": [_function_resource(function) for function in project.functions], - "audits": [_audit_resource(audit) for audit in project.audits], - "tests": [_test_resource(test) for test in project.sql_tests], - "hooks": _hook_definitions(project), - } - - -def _model_resource( - *, - graph: ProjectGraph, - model: CompiledModelView, - lineage: ProjectColumnLineage | None, -) -> dict[str, object]: - item: dict[str, object] = { - "name": model.name, - "relative_path": str(model.relative_path), - "materialized": str(model.config.values.get("materialized", "view")), - "column_count": model.output_column_count, - "depends_on": [ - {"resource_type": kind, "name": name} - for kind, name in graph.native.upstream((str(model.key.resource_type), model.key.name)) - ], - "lineage": _lineage_summary(lineage=lineage, model=model), - "query_sql": model.query_sql, - } - if model.destination.qualified_name is not None: - item["qualified_name"] = model.destination.qualified_name - for hook_key in ("pre_hooks", "post_hooks"): - hooks: object = model.config.values.get(hook_key) - if hooks is not None: - item[hook_key] = _hook_resources(hooks=hooks) - return item - - -def _hook_resources(*, hooks: object) -> list[dict[str, object]]: - return serialize_hook_entries(value=hooks, sql_fields=SQL_HOOK_OUTPUT_FIELDS) - - -def _hook_definitions(project: CompiledProjectView) -> list[dict[str, object]]: - definitions: list[dict[str, object]] = [ - { - "type": "sql", - "name": hook.name, - "relative_path": hook.relative_path.as_posix(), - "description": hook.description, - "sql": hook.sql_body, - } - for hook in project.sql_hook_files - ] - python_identities: dict[str, PythonNodeIdentity] = build_hook_identities(project.hook_functions) - for hook in project.hook_functions: - python_definition: dict[str, object] = { - "type": "python", - "name": hook.name, - "relative_path": hook.relative_path.as_posix(), - "description": hook.description, - "definition_hash": python_identities[hook.name].definition_hash, - "version_hash": python_identities[hook.name].version_hash, - } - definitions.append(python_definition) - return definitions - - -def _source_resource(source: CompiledSourceView) -> dict[str, object]: - return { - "name": source.name, - "relative_path": str(source.source_file.relative_path), - "column_count": len(source.source_entry.columns), - } - - -def _seed_resource(seed: CompiledSeedView) -> dict[str, object]: - item: dict[str, object] = { - "name": seed.name, - "relative_path": str(seed.seed_file.relative_path), - "column_count": len(seed.schema_entry.columns), - } - if seed.destination.qualified_name is not None: - item["qualified_name"] = seed.destination.qualified_name - return item - - -def _function_resource(function: CompiledFunctionView) -> dict[str, object]: - item: dict[str, object] = { - "name": function.name, - "relative_path": str(function.relative_path), - "language": function.language.value, - "description": function.description, - "return_kind": "table" if function.return_columns else "scalar", - "returns": function.returns, - "return_columns": [ - {"name": column.name, "type": column.type} for column in function.return_columns - ], - } - if function.destination.qualified_name is not None: - item["qualified_name"] = function.destination.qualified_name - return item - - -def _audit_resource(audit: CompiledAuditView) -> dict[str, object]: - item: dict[str, object] = { - "name": audit.name, - "relative_path": str(audit.audit_file.relative_path), - } - if audit.attached_target_kind is not None and audit.attached_target_name is not None: - item["attached_to"] = { - "resource_type": audit.attached_target_kind.value, - "name": audit.attached_target_name, - } - if audit.attached_column_name is not None: - item["attached_to"]["column_name"] = audit.attached_column_name - return item - - -def _test_resource(test: CompiledSqlTest) -> dict[str, object]: - expected_models: list[str] = [] - if isinstance(test.payload, CompiledModelSqlTestPayload): - expected_models = list(test.payload.expected_model_names) - item: dict[str, object] = { - "name": test.name, - "relative_path": str(test.test_file.relative_path), - "expected_models": expected_models, - } - if test.case_name is not None: - parameter_types: dict[str, str] = { - parameter.name: parameter.value_type.value for parameter in test.parameter_schema - } - item.update( - { - "source_path": (test.source_path or test.test_file.relative_path).as_posix(), - "block_index": test.block_index, - "parent_name": test.parent_name, - "case_name": test.case_name, - "case_index": test.case_index, - "case_fingerprint": test.case_fingerprint, - "parameter_schema": [ - { - "name": parameter.name, - "type": parameter.value_type.value, - "nullable": parameter.nullable, - } - for parameter in test.parameter_schema - ], - "parameters": [ - { - "name": name, - "type": parameter_types[name], - "value": ( - str(value.value) if value.kind == SqlValueKind.DECIMAL else value.value - ), - } - for name, value in test.parameter_values - ], - } - ) - return item - - -def _lineage_summary( - *, lineage: ProjectColumnLineage | None, model: CompiledModelView -) -> dict[str, object]: - if lineage is None: - return {"available": False} - if not lineage.has_model(model.name): - return {"available": False} - return { - "available": True, - "column_count": model.output_column_count, - "edge_count": lineage.edge_count_targeting(model.name), - "has_star": lineage.model_has_star(model.name), - } - - -def _artifacts(*, written: WrittenTarget, manifest: bool) -> dict[str, object]: - return { - "compiled_sql_dir": _relative_target_path(_compiled_sql_dir(written)) + "/", - "manifest": "target/manifest.json" if manifest else None, - } - - -def _model_name_width(models: tuple[CompiledModelView, ...]) -> int: - longest: int = max((len(model.name) for model in models), default=0) - return min(max(longest, _MIN_MODEL_NAME_WIDTH), _MAX_MODEL_NAME_WIDTH) - - -def _fit(*, text: str, width: int) -> str: - if len(text) > width: - minimum_collapsible_width: int = 3 - if width <= minimum_collapsible_width: - return "." * width - return text[: width - 3] + "..." - return text.ljust(width) - - -def _count_label(*, count: int, singular: str) -> str: - if count == 1: - return f"{count} {singular}" - return f"{count} {singular}s" - - -def _diagnostic_count( - *, diagnostics: tuple[CompilerDiagnostic, ...], severity: DiagnosticSeverity -) -> int: - return sum(1 for diagnostic in diagnostics if diagnostic.severity == severity) - - -def _models_with_error_diagnostics( - diagnostics: tuple[CompilerDiagnostic, ...], -) -> frozenset[str]: - return frozenset( - diagnostic.resource_name - for diagnostic in diagnostics - if diagnostic.is_error - and diagnostic.resource_name is not None - and diagnostic.resource_type != CompiledResourceType.SQL_TEST - ) - - -def _diagnostic_to_json(diagnostic: CompilerDiagnostic) -> dict[str, object]: - payload: dict[str, object] = { - "phase": str(diagnostic.phase), - "severity": str(diagnostic.severity), - "code": diagnostic.code, - "message": diagnostic.message, - } - if diagnostic.resource_type is not None: - payload["resource_type"] = str(diagnostic.resource_type) - if diagnostic.resource_name is not None: - payload["resource_name"] = diagnostic.resource_name - if diagnostic.column_name is not None: - payload["column_name"] = diagnostic.column_name - if diagnostic.path is not None: - payload["path"] = str(diagnostic.path) - if diagnostic.line is not None: - payload["line"] = diagnostic.line - if diagnostic.column is not None: - payload["column"] = diagnostic.column - if diagnostic.location is not None: - payload["location"] = _location_to_json(diagnostic.location) - if diagnostic.related_locations: - payload["related_locations"] = [ - { - "label": related.label, - "location": _location_to_json(related.location), - **({"message": related.message} if related.message is not None else {}), - } - for related in diagnostic.related_locations - ] - if diagnostic.help is not None: - payload["help"] = diagnostic.help - if diagnostic.notes: - payload["notes"] = list(diagnostic.notes) - if diagnostic.affected_rules: - payload["affected_rules"] = list(diagnostic.affected_rules) - return payload - - -def _location_to_json(location: SourceLocation) -> dict[str, object]: - payload: dict[str, object] = { - "path": str(location.path), - "line": location.line, - "column": location.column, - } - if location.end_line is not None: - payload["end_line"] = location.end_line - if location.end_column is not None: - payload["end_column"] = location.end_column - return payload - - -def _format_diagnostics_text( - *, diagnostics: tuple[CompilerDiagnostic, ...], source_texts: dict[Path, str], style: CliStyle -) -> str: - lines: list[str] = [] - warning_counts: Counter[CompilerDiagnostic] = Counter( - diagnostic - for diagnostic in diagnostics - if diagnostic.severity == DiagnosticSeverity.WARNING - ) - shown_warnings: set[CompilerDiagnostic] = set() - for diagnostic in diagnostics: - if diagnostic in shown_warnings: - continue - if diagnostic.severity == DiagnosticSeverity.WARNING: - shown_warnings.add(diagnostic) - lines.extend( - _format_diagnostic_text( - diagnostic=diagnostic, - source_texts=source_texts, - style=style, - ) - ) - count: int = warning_counts[diagnostic] - if count > 1: - lines.append(f" = {count} occurrences; all retained in JSON and the warning count") - lines.append("") - if lines: - lines.pop() - return "\n".join(lines) - - -def _format_diagnostic_text( - *, diagnostic: CompilerDiagnostic, source_texts: dict[Path, str], style: CliStyle -) -> list[str]: - header: str = f"{diagnostic.severity}[{diagnostic.code}]: {diagnostic.message}" - lines: list[str] = [_style_diagnostic(text=header, diagnostic=diagnostic, style=style)] - if diagnostic.resource_name is not None: - label: str = "model" - resource: str = diagnostic.resource_name - if ( - diagnostic.resource_type is not None - and str(diagnostic.resource_type) != CompiledResourceType.MODEL - ): - label = str(diagnostic.resource_type).replace("_", " ") - lines.append(f" {label}: {style.object_name(resource)}") - if diagnostic.location is not None: - lines.extend( - _format_location_block( - location=diagnostic.location, - source_texts=source_texts, - message=None, - ) - ) - elif diagnostic.path is not None: - lines.append(f" --> {diagnostic.path}") - for related_location in diagnostic.related_locations: - lines.append("") - lines.append(f" {related_location.label}:") - lines.extend( - _format_related_location_block( - related_location=related_location, - source_texts=source_texts, - ) - ) - lines.extend(f" {style.muted('= note:')} {note}" for note in diagnostic.notes) - if diagnostic.affected_rules: - lines.append(f" {style.muted('= affected Rules:')} {', '.join(diagnostic.affected_rules)}") - if diagnostic.help is not None: - lines.append(f" {style.muted('= help:')} {diagnostic.help}") - return lines - - -def _format_related_location_block( - *, related_location: RelatedLocation, source_texts: dict[Path, str] -) -> list[str]: - return _format_location_block( - location=related_location.location, - source_texts=source_texts, - message=related_location.message, - ) - - -def _format_location_block( - *, location: SourceLocation, source_texts: dict[Path, str], message: str | None -) -> list[str]: - rendered_location: str = f"{location.path}:{location.line}:{location.column}" - lines: list[str] = [f" --> {rendered_location}"] - source_text: str | None = source_texts.get(location.path) - if source_text is None: - return lines - source_lines: list[str] = source_text.splitlines() - if location.line < 1 or location.line > len(source_lines): - return lines - source_line: str = source_lines[location.line - 1] - line_number: str = str(location.line) - gutter: str = " " * len(line_number) - lines.append(f" {gutter} |") - lines.append(f" {line_number} | {source_line}") - lines.append( - f" {gutter} | {_caret_line(location=location, source_line=source_line, message=message)}" - ) - return lines - - -def _caret_line(*, location: SourceLocation, source_line: str, message: str | None) -> str: - start_column: int = max(1, location.column) - end_column: int = location.end_column or start_column + 1 - if location.end_line is not None and location.end_line != location.line: - end_column = len(source_line) + 1 - caret_count: int = max(1, end_column - start_column) - suffix: str = f" {message}" if message is not None else "" - return " " * (start_column - 1) + "^" * caret_count + suffix - - -def _model_source_texts(models: tuple[CompiledModelView, ...]) -> dict[Path, str]: - return {model.relative_path: model.authored_sql for model in models if model.authored_sql} - - -def _style_diagnostic(*, text: str, diagnostic: CompilerDiagnostic, style: CliStyle) -> str: - if diagnostic.severity == DiagnosticSeverity.ERROR: - return style.error(text) - if diagnostic.severity == DiagnosticSeverity.WARNING: - return style.warning(text) - return style.muted(text) - - -def _execution_layer_count(graph: ProjectGraph) -> int: - return graph.native.model_layer_count() - - -def _relative_target_path(path: Path) -> str: - parts: tuple[str, ...] = path.parts - if TARGET_DIRECTORY_NAME in parts: - target_index: int = parts.index(TARGET_DIRECTORY_NAME) - return Path(*parts[target_index:]).as_posix() - return path.as_posix() - - -def _compiled_sql_dir(written: WrittenTarget) -> Path: - return written.target_dir / "compiled" - - -def _sqlbuild_version() -> str: - try: - return version("sqlbuild") - except PackageNotFoundError: - return "unknown" diff --git a/src/sqlbuild/cli/commands/_helpers/compile/semantic_notice.py b/src/sqlbuild/cli/commands/_helpers/compile/semantic_notice.py deleted file mode 100644 index 1710171b86..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/compile/semantic_notice.py +++ /dev/null @@ -1,82 +0,0 @@ -"""One actionable semantic-check notice for human and machine commands.""" - -from sqlbuild.compiler.compile.main.semantic_coverage import get_semantic_coverage -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledProjectView, CompiledResourceType - -_DISPLAY_LIMIT: int = 10 -_ACTIONS: dict[str, str] = { - "some output types are unknown": "declare output types, then run `sqb compile --no-cache`", - "output shape could not be inferred": "declare the output schema; run `sqb compile --no-cache`", - "unresolved star over open inputs": ( - "declare complete input contracts, then run `sqb compile --no-cache`" - ), - "SQL analysis disabled": "enable sql_analysis, then run `sqb compile --no-cache`", - "type checking awaits dialect coercion support": ( - "use a supported dialect for type checks; run `sqb compile --json` to inspect coverage" - ), -} - - -def selected_semantic_coverage( - *, - project: CompiledProjectView, - selected_keys: frozenset[CompiledObjectKey] | None, - sql_validation_enabled: bool = True, -) -> dict[str, tuple[str, ...]]: - """Report coverage for the compile selection rather than unanalyzed sibling models.""" - if not sql_validation_enabled: - return {} - reasons: dict[str, tuple[str, ...]] = get_semantic_coverage(project) - if selected_keys is None: - return reasons - names: set[str] = { - key.name for key in selected_keys if key.resource_type == CompiledResourceType.MODEL - } - return {name: values for name, values in reasons.items() if name in names} - - -def semantic_coverage_notice( - *, - project: CompiledProjectView, - selected_keys: frozenset[CompiledObjectKey] | None = None, - sql_validation_enabled: bool = True, -) -> str | None: - """Summarize partial checks without increasing the diagnostic warning count.""" - reasons: dict[str, tuple[str, ...]] = selected_semantic_coverage( - project=project, - selected_keys=selected_keys, - sql_validation_enabled=sql_validation_enabled, - ) - if not reasons: - return None - groups: dict[str, list[str]] = {} - for model, values in sorted(reasons.items()): - for reason in values: - groups.setdefault(reason, []).append(model) - lines: list[str] = [f"Semantic checks were partial for {len(reasons)} models:"] - for reason, models in sorted(groups.items())[:10]: - suffix: str = ( - f", and {len(models) - _DISPLAY_LIMIT} more" if len(models) > _DISPLAY_LIMIT else "" - ) - detail: str = reason.replace("open source: ", "reads open source ").replace( - "open input: ", "reads open input " - ) - action: str = _ACTIONS.get( - reason, "declare the input shape, then run `sqb compile --no-cache`" - ) - if reason.startswith("open source: "): - source: str = reason.removeprefix("open source: ") - action = ( - "run `sqb contract generate --from " - f"--select source:{source} --write` and enforce the complete source contract" - ) - elif reason.startswith("depends on invalid model "): - action = "fix the upstream error, then run `sqb compile --no-cache`" - lines.append(f" {', '.join(models[:10])}{suffix}: {detail}; {action}.") - if len(groups) > _DISPLAY_LIMIT: - lines.append( - f" and {len(groups) - 10} more reasons; " - "run `sqb compile --json` for the full model/reason list." - ) - return "\n".join(lines) diff --git a/src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py b/src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py deleted file mode 100644 index 5e18142ca8..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py +++ /dev/null @@ -1,303 +0,0 @@ -"""Persistent identities for unchanged compiled SQL test artifacts.""" - -from __future__ import annotations - -import hashlib -import json -import os -import platform -from dataclasses import replace -from importlib.metadata import PackageNotFoundError, version -from pathlib import Path -from typing import cast - -import orjson - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.compile.models import ( - SqlTestArtifactCacheRecord, - SqlTestArtifactIdentityContext, -) -from sqlbuild.compiler.compile.models import ( - CompiledDirectLogicSqlTestPayload, - CompiledModelSqlTestPayload, - CompiledSqlTest, - CompileSqlTestCte, -) -from sqlbuild.compiler.compile.types import CompiledModelView, CompiledProjectView - -_CACHE_VERSION: int = 4 -_ALGORITHM_FINGERPRINT: str = "sql-test-artifact-v4-orjson-identity" -_CACHE_FILE_NAME: str = "sql-test-artifacts.json" -_MAX_CACHE_BYTES: int = 10_000_000 -_MAX_CACHE_RECORDS: int = 100_000 -_PARENT_PATH_COMPONENT: str = ".." -_SQL_FILE_SUFFIX: str = ".sql" - - -def build_sql_test_artifact_identity_context( - *, project: CompiledProjectView, adapter: BaseAdapter -) -> SqlTestArtifactIdentityContext: - """Hash adapter and model inputs once for all test artifact identities.""" - - common_identity: str = _payload_digest( - { - "algorithm": _ALGORITHM_FINGERPRINT, - "sqlbuild_version": _package_version("sqlbuild"), - "polyglot_version": _package_version("polyglot-sql-chio"), - "python_version": platform.python_version_tuple()[:2], - "adapter_class": f"{type(adapter).__module__}.{type(adapter).__qualname__}", - "set_difference_operator": adapter.render_set_difference_operator(), - "sql_analysis_dialect": adapter.sql_analysis_dialect(), - "requires_derived_table_aliases": adapter.requires_derived_table_aliases(), - "sql_analysis_enabled": project.settings.sql_analysis, - "effective_target_name": project.effective_target_name, - "effective_target_database": project.effective_target_database, - "effective_target_schema": project.effective_target_schema, - "function_locations": sorted( - (function.name, function.destination.qualified_name) - for function in project.functions - ), - } - ) - return SqlTestArtifactIdentityContext( - common_identity=common_identity, - model_identities={model.name: _model_identity(model=model) for model in project.models}, - ) - - -def sql_test_artifact_identity( - *, - test: CompiledSqlTest, - model_chain_names: tuple[str, ...], - context: SqlTestArtifactIdentityContext, -) -> str: - """Return the complete identity of one rendered comparison artifact.""" - - return _payload_digest( - { - "common_identity": context.common_identity, - "test": _test_payload(test=test), - "model_chain": [ - (name, context.model_identities.get(name)) for name in model_chain_names - ], - } - ) - - -def sql_test_artifact_record_key(*, test: CompiledSqlTest) -> str: - """Return the stable cache slot for one source test case.""" - - return _payload_digest( - { - "source_path": None if test.source_path is None else test.source_path.as_posix(), - "block_index": test.block_index, - "name": test.name, - "parent_name": test.parent_name, - "case_name": test.case_name, - "case_index": test.case_index, - } - ) - - -def read_sql_test_artifact_cache( - *, cache_dir: Path | None -) -> dict[str, SqlTestArtifactCacheRecord]: - """Read valid cache records; cache failures never fail compilation.""" - - if cache_dir is None: - return {} - cache_path: Path = cache_dir / _CACHE_FILE_NAME - try: - if not cache_path.is_file() or cache_path.stat().st_size > _MAX_CACHE_BYTES: - return {} - payload: object = json.loads(cache_path.read_text(encoding="utf-8")) - if not isinstance(payload, dict) or payload.get("version") != _CACHE_VERSION: - return {} - raw_records: object = payload.get("records") - if not isinstance(raw_records, dict) or len(raw_records) > _MAX_CACHE_RECORDS: - return {} - records: dict[str, SqlTestArtifactCacheRecord] = {} - for key, value in raw_records.items(): - record: SqlTestArtifactCacheRecord | None = _record_from_payload(value) - if isinstance(key, str) and record is not None: - records[key] = record - return records - except (OSError, ValueError, TypeError, json.JSONDecodeError): - return {} - - -def write_sql_test_artifact_cache( - *, cache_dir: Path | None, records: dict[str, SqlTestArtifactCacheRecord] -) -> None: - """Atomically replace the cache snapshot; cache failures are non-fatal.""" - - if cache_dir is None: - return - payload: dict[str, object] = { - "version": _CACHE_VERSION, - "records": { - key: { - "identity": record.identity, - "relative_path": record.relative_path.as_posix(), - "size": record.size, - "mtime_ns": record.mtime_ns, - "content_sha256": record.content_sha256, - } - for key, record in records.items() - }, - } - try: - cache_dir.mkdir(parents=True, exist_ok=True) - cache_path: Path = cache_dir / _CACHE_FILE_NAME - temporary_path: Path = cache_dir / f".{_CACHE_FILE_NAME}.{os.getpid()}.tmp" - temporary_path.write_text( - json.dumps(payload, ensure_ascii=True, separators=(",", ":"), sort_keys=True), - encoding="utf-8", - ) - temporary_path.replace(cache_path) - except OSError: - return - - -def artifact_matches_cache_record( - *, tests_root: Path, record: SqlTestArtifactCacheRecord, identity: str -) -> SqlTestArtifactCacheRecord | None: - """Return the reusable artifact's record, verifying content when only its mtime moved.""" - - if record.identity != identity or not _is_safe_relative_sql_path(record.relative_path): - return None - artifact_path: Path = tests_root / record.relative_path - try: - stat: os.stat_result = artifact_path.stat() - if not artifact_path.is_file() or stat.st_size != record.size: - return None - if stat.st_mtime_ns == record.mtime_ns: - return record - if _file_sha256(artifact_path) != record.content_sha256: - return None - except OSError: - return None - return replace(record, mtime_ns=stat.st_mtime_ns) - - -def build_sql_test_artifact_cache_record( - *, tests_root: Path, artifact_path: Path, identity: str -) -> SqlTestArtifactCacheRecord | None: - """Capture the metadata needed to reuse one newly written artifact.""" - - try: - relative_path: Path = artifact_path.relative_to(tests_root) - stat: os.stat_result = artifact_path.stat() - content_sha256: str = _file_sha256(artifact_path) - except (OSError, ValueError): - return None - if not _is_safe_relative_sql_path(relative_path): - return None - return SqlTestArtifactCacheRecord( - identity=identity, - relative_path=relative_path, - size=stat.st_size, - mtime_ns=stat.st_mtime_ns, - content_sha256=content_sha256, - ) - - -def _model_identity(*, model: CompiledModelView) -> str: - return _payload_digest( - { - "name": model.name, - "query_sql": model.query_sql, - "deps": sorted((str(dep.resource_type), dep.name) for dep in model.deps), - "cursor": ( - model.config.values.get("cursor_type"), - model.config.values.get("cursor_grain"), - ), - } - ) - - -def _test_payload(*, test: CompiledSqlTest) -> dict[str, object]: - payload: dict[str, object] = { - "case_fingerprint": test.case_fingerprint, - "mode": test.mode.value, - "sql_body": test.sql_body, - "scope_deps": sorted((str(dep.resource_type), dep.name) for dep in test.scope_deps), - } - if isinstance(test.payload, CompiledModelSqlTestPayload): - payload["model"] = { - "authored_ctes": _cte_payload(test.payload.authored_ctes), - "macro_mocks": sorted(test.payload.macro_mocks.items()), - "model_query_overrides": sorted(test.payload.model_query_overrides.items()), - "expected_model_names": test.payload.expected_model_names, - "assertion_ctes": _cte_payload(test.payload.assertion_ctes), - "cursor_window": (test.test_block.cursor_start, test.test_block.cursor_end), - } - elif isinstance(test.payload, CompiledDirectLogicSqlTestPayload): - payload["direct"] = { - "helper_ctes": _cte_payload(test.payload.helper_ctes), - "actual_cte": _cte_payload((test.payload.actual_cte,)), - "expected_cte": _cte_payload((test.payload.expected_cte,)), - "tested_resource_names": test.payload.tested_resource_names, - } - return payload - - -def _cte_payload(ctes: tuple[CompileSqlTestCte, ...]) -> list[tuple[str, str]]: - return [(cte.name, cte.sql_body) for cte in ctes] - - -def _record_from_payload(payload: object) -> SqlTestArtifactCacheRecord | None: - if not isinstance(payload, dict): - return None - values: dict[str, object] = cast(dict[str, object], payload) - identity: object = values.get("identity") - relative_path: object = values.get("relative_path") - size: object = values.get("size") - mtime_ns: object = values.get("mtime_ns") - content_sha256: object = values.get("content_sha256") - if ( - not isinstance(identity, str) - or not isinstance(relative_path, str) - or not isinstance(size, int) - or isinstance(size, bool) - or size < 0 - or not isinstance(mtime_ns, int) - or isinstance(mtime_ns, bool) - or mtime_ns < 0 - or not isinstance(content_sha256, str) - ): - return None - path: Path = Path(relative_path) - if not _is_safe_relative_sql_path(path): - return None - return SqlTestArtifactCacheRecord( - identity=identity, - relative_path=path, - size=size, - mtime_ns=mtime_ns, - content_sha256=content_sha256, - ) - - -def _is_safe_relative_sql_path(path: Path) -> bool: - return ( - not path.is_absolute() - and _PARENT_PATH_COMPONENT not in path.parts - and path.suffix == _SQL_FILE_SUFFIX - ) - - -def _file_sha256(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() - - -def _payload_digest(payload: object) -> str: - return hashlib.sha256(orjson.dumps(payload, option=orjson.OPT_SORT_KEYS)).hexdigest() - - -def _package_version(package_name: str) -> str: - try: - return version(package_name) - except PackageNotFoundError: - return "unknown" diff --git a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py deleted file mode 100644 index 52ae0b17d3..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py +++ /dev/null @@ -1,660 +0,0 @@ -"""Write compiled project output to the target/ directory.""" - -from __future__ import annotations - -import os -from collections.abc import Callable -from functools import partial -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.compile.sql_test_artifact_cache import ( - artifact_matches_cache_record, - build_sql_test_artifact_cache_record, - build_sql_test_artifact_identity_context, - read_sql_test_artifact_cache, - sql_test_artifact_identity, - sql_test_artifact_record_key, - write_sql_test_artifact_cache, -) -from sqlbuild.cli.commands.classes.native_artifact_batch import NativeArtifactBatch -from sqlbuild.cli.commands.exceptions import StagedArtifactsChangedError -from sqlbuild.cli.compile.models import ( - PendingStaticSqlTest, - PlannedStaticSqlTests, - SqlTestArtifactCacheRecord, - SqlTestArtifactIdentityContext, -) -from sqlbuild.cli.compile_reuse.constants import COMPILE_ARTIFACT_WRITES -from sqlbuild.cli.output.models import ( - WrittenTarget, -) -from sqlbuild.cli.paths.main._compiled_sql_test_output_path import ( - compiled_sql_test_output_path, -) -from sqlbuild.cli.paths.main._sql_test_output_path import sql_test_output_path -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledSqlTest, - CompilerDiagnostic, -) -from sqlbuild.compiler.compile.types import ( - CompiledProjectView, - CompiledResourceType, - DiagnosticPhase, - DiagnosticSeverity, - FunctionLanguage, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.execution.sql_test_model_chain import ( - sql_test_model_chain_names_by_key, -) -from sqlbuild.compiler.planner.models import AuditPlanEntry, NativeSqlTestArtifact, PlanOutput -from sqlbuild.compiler.profiling.main.record import record_compile_timing -from sqlbuild.compiler.sql_test_glue.main.plan_native_sql_test_artifacts import ( - plan_native_sql_test_artifacts, -) -from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle - -_COMPILED_DIR: str = "compiled" -_MODELS_DIR: str = "models" -_FUNCTIONS_DIR: str = "functions" -_AUDITS_DIR: str = "audits" -_GENERIC_DIR: str = "generic" -_SINGULAR_DIR: str = "singular" -_TESTS_DIR: str = "tests" -_MANIFEST_FILE: str = "manifest.json" -_SQL_FILE_SUFFIX: str = ".sql" -_SQL_TEST_PLANNING_ERROR_CODE: str = PlannerInputError.code -_PUBLISHED_STAGING_CHANGED: str = "changed" -_PUBLISHED_TREE: str = "tree" - - -def write_compile_target( - *, - target_dir: Path, - adapter: BaseAdapter, - plan_output: PlanOutput, - manifest: _native.NativeManifest | None = None, -) -> WrittenTarget: - """Write compiled output files under target_dir.""" - - remove_stale_files: bool = (target_dir / _COMPILED_DIR).is_dir() - target_dir.mkdir(parents=True, exist_ok=True) - batch: NativeArtifactBatch = NativeArtifactBatch(check_existing=remove_stale_files) - managed_paths: set[Path] = set() - for write_group in ( - partial(_write_models, target_dir=target_dir, plan_output=plan_output), - partial(_write_functions, target_dir=target_dir, adapter=adapter, plan_output=plan_output), - partial(_write_audits, target_dir=target_dir, plan_output=plan_output), - partial(_write_tests, target_dir=target_dir, adapter=adapter, plan_output=plan_output), - ): - managed_paths.update(write_group(batch=batch)) - batch.flush() - if remove_stale_files: - with record_compile_timing("stale_traversal_ms"): - _remove_stale_compiled_files(target_dir=target_dir, managed_paths=managed_paths) - if manifest is not None: - _write_manifest(target_dir=target_dir, manifest=manifest) - - return WrittenTarget( - model_count=len(plan_output.model_entries), - seed_count=len(plan_output.seed_entries), - function_count=len(plan_output.function_entries), - audit_count=len(plan_output.audit_entries), - test_count=len(plan_output.test_entries), - target_dir=target_dir, - ) - - -def write_static_compile_target( - *, - target_dir: Path, - adapter: BaseAdapter, - project: CompiledProjectView, - manifest: _native.NativeManifest | None = None, - planned_tests: Callable[[], PlannedStaticSqlTests] | None = None, -) -> WrittenTarget: - """Write offline compiled output files under target_dir, reusing SQL tests planned ahead.""" - - remove_stale_files: bool = (target_dir / _COMPILED_DIR).is_dir() - target_dir.mkdir(parents=True, exist_ok=True) - batch: NativeArtifactBatch = NativeArtifactBatch(check_existing=remove_stale_files) - managed_paths: set[Path] = set() - for write_group in ( - partial(_write_static_models, target_dir=target_dir, project=project), - partial(_write_static_functions, target_dir=target_dir, adapter=adapter, project=project), - partial(_write_static_audits, target_dir=target_dir, project=project), - ): - managed_paths.update(write_group(batch=batch)) - batch.flush() - test_paths, test_diagnostics = _write_static_tests( - target_dir=target_dir, - adapter=adapter, - project=project, - planned_tests=planned_tests, - batch=batch, - ) - managed_paths.update(test_paths) - if remove_stale_files: - with record_compile_timing("stale_traversal_ms"): - _remove_stale_compiled_files(target_dir=target_dir, managed_paths=managed_paths) - if manifest is not None: - _write_manifest(target_dir=target_dir, manifest=manifest) - - return WrittenTarget( - model_count=len(project.models), - seed_count=len(project.seeds), - function_count=len(project.functions), - audit_count=len(project.audits), - test_count=len(project.sql_tests), - target_dir=target_dir, - diagnostics=test_diagnostics, - ) - - -def staged_artifact_files(*, target_dir: Path) -> frozenset[str]: - """Return the relative paths of every compiled artifact staged under target_dir.""" - - return frozenset(_native.staged_artifact_files(target_dir / _COMPILED_DIR)) - - -def publish_static_compile_target( - *, - prepared: WrittenTarget, - target_dir: Path, - manifest: _native.NativeManifest | None, - expected_files: frozenset[str], -) -> WrittenTarget: - """Publish staged files with the same unchanged-file and stale-file semantics.""" - - _publish_staged( - staged_dir=prepared.target_dir / _COMPILED_DIR, - target_dir=target_dir, - expected_files=expected_files, - ) - if manifest is not None: - _write_manifest(target_dir=target_dir, manifest=manifest) - return WrittenTarget( - model_count=prepared.model_count, - seed_count=prepared.seed_count, - function_count=prepared.function_count, - audit_count=prepared.audit_count, - test_count=prepared.test_count, - target_dir=target_dir, - diagnostics=prepared.diagnostics, - ) - - -def _publish_staged(*, staged_dir: Path, target_dir: Path, expected_files: frozenset[str]) -> None: - compiled_dir: Path = target_dir / _COMPILED_DIR - existed: bool = compiled_dir.is_dir() - with record_compile_timing("physical_write_ms"): - kind, published = _native.publish_staged_artifacts( - staged_dir, compiled_dir, sorted(expected_files) - ) - if kind == _PUBLISHED_STAGING_CHANGED: - raise StagedArtifactsChangedError( - "staged compile artifacts changed before publication; no artifact was published", - help="Rerun the compile; avoid deleting target/ while a compile is running.", - ) - if kind == _PUBLISHED_TREE: - COMPILE_ARTIFACT_WRITES.moved_tree(source=staged_dir, destination=compiled_dir) - return - for source, path in published: - COMPILE_ARTIFACT_WRITES.moved(source=source, destination=path) - if existed: - with record_compile_timing("stale_traversal_ms"): - _remove_stale_compiled_files( - target_dir=target_dir, - managed_paths={Path(path) for _, path in published}, - ) - - -def _write_models( - *, target_dir: Path, plan_output: PlanOutput, batch: NativeArtifactBatch -) -> set[Path]: - """Write model resolved SQL.""" - - managed_paths: set[Path] = set() - for entry in plan_output.model_entries: - compiled_path: Path = target_dir / _COMPILED_DIR / _model_output_path(entry.relative_path) - _write_sql(path=compiled_path, sql=entry.resolved_sql, batch=batch) - managed_paths.add(compiled_path) - return managed_paths - - -def _write_static_models( - *, - target_dir: Path, - project: CompiledProjectView, - batch: NativeArtifactBatch, -) -> set[Path]: - """Write offline model query SQL.""" - - managed_paths: set[Path] = set() - for model in project.models: - compiled_path: Path = target_dir / _COMPILED_DIR / _model_output_path(model.relative_path) - _write_sql(path=compiled_path, sql=model.query_sql, batch=batch) - managed_paths.add(compiled_path) - return managed_paths - - -def _write_functions( - *, - target_dir: Path, - adapter: BaseAdapter, - plan_output: PlanOutput, - batch: NativeArtifactBatch, -) -> set[Path]: - """Write executable SQL function DDL.""" - - managed_paths: set[Path] = set() - for entry in plan_output.function_entries: - if entry.destination.qualified_name is None: - continue - statements: tuple[str, ...] = adapter.render_create_function( - destination=entry.destination.qualified_name, - arguments=entry.arguments, - returns=entry.returns, - body_sql=entry.body_sql, - return_columns=entry.return_columns, - language=entry.language, - runtime_version=entry.runtime_version, - entry_point=entry.entry_point, - packages=entry.packages, - ) - function_path: Path = ( - target_dir - / _COMPILED_DIR - / _function_output_path(relative_path=entry.relative_path, language=entry.language) - ) - _write_sql( - path=function_path, - sql=";\n\n".join(statements), - batch=batch, - ) - managed_paths.add(function_path) - return managed_paths - - -def _write_static_functions( - *, - target_dir: Path, - adapter: BaseAdapter, - project: CompiledProjectView, - batch: NativeArtifactBatch, -) -> set[Path]: - """Write offline rendered SQL function DDL.""" - - managed_paths: set[Path] = set() - for function in project.functions: - if function.destination.qualified_name is None: - continue - statements: tuple[str, ...] = adapter.render_create_function( - destination=function.destination.qualified_name, - arguments=function.arguments, - returns=function.returns, - body_sql=function.body_sql, - return_columns=function.return_columns, - language=function.language, - runtime_version=function.runtime_version, - entry_point=function.entry_point, - packages=function.packages, - ) - function_path: Path = ( - target_dir - / _COMPILED_DIR - / _function_output_path( - relative_path=function.relative_path, language=function.language - ) - ) - _write_sql( - path=function_path, - sql=";\n\n".join(statements), - batch=batch, - ) - managed_paths.add(function_path) - return managed_paths - - -def _write_audits( - *, target_dir: Path, plan_output: PlanOutput, batch: NativeArtifactBatch -) -> set[Path]: - """Write resolved audit SQL.""" - - managed_paths: set[Path] = set() - for entry in plan_output.audit_entries: - folder: Path = _audit_folder(entry) - file_name: str = _audit_file_name(entry) - audit_path: Path = target_dir / _COMPILED_DIR / _AUDITS_DIR / folder / file_name - _write_sql(path=audit_path, sql=entry.resolved_sql, batch=batch) - managed_paths.add(audit_path) - return managed_paths - - -def _write_static_audits( - *, - target_dir: Path, - project: CompiledProjectView, - batch: NativeArtifactBatch, -) -> set[Path]: - """Write offline resolved audit SQL.""" - - managed_paths: set[Path] = set() - for audit in project.audits: - folder: Path = _static_audit_folder(attached_target_name=audit.attached_target_name) - file_name: str = _static_audit_file_name( - name=audit.name, - attached_target_name=audit.attached_target_name, - attached_column_name=audit.attached_column_name, - ) - audit_path: Path = target_dir / _COMPILED_DIR / _AUDITS_DIR / folder / file_name - _write_sql(path=audit_path, sql=audit.sql_body, batch=batch) - managed_paths.add(audit_path) - return managed_paths - - -def _write_tests( - *, - target_dir: Path, - adapter: BaseAdapter, - plan_output: PlanOutput, - batch: NativeArtifactBatch, -) -> set[Path]: - """Write resolved SQL-native test SQL.""" - - managed_paths: set[Path] = set() - for entry in plan_output.test_entries: - test_path: Path = target_dir / _COMPILED_DIR / _TESTS_DIR / sql_test_output_path(entry) - with record_compile_timing("comparison_render_ms"): - comparison_sql: str = build_sql_test_comparison_sql( - test_entry=entry, - set_difference_operator=adapter.render_set_difference_operator(), - sql_analysis_dialect=adapter.sql_analysis_dialect(), - ) - _write_sql(path=test_path, sql=comparison_sql, batch=batch) - managed_paths.add(test_path) - return managed_paths - - -def static_sql_test_planning_diagnostics( - *, - target_dir: Path, - adapter: BaseAdapter, - project: CompiledProjectView, - planned_tests: Callable[[], PlannedStaticSqlTests] | None = None, -) -> tuple[CompilerDiagnostic, ...]: - """Plan uncached SQL tests and report their errors without writing artifacts or cache.""" - - planned: PlannedStaticSqlTests = _resolve_planned_static_tests( - target_dir=target_dir, adapter=adapter, project=project, planned_tests=planned_tests - ) - diagnostics: list[CompilerDiagnostic] = [] - for pending, artifact in zip(planned.pending, planned.artifacts, strict=True): - diagnostics.extend(_sql_test_artifact_diagnostics(test=pending.test, artifact=artifact)) - return tuple(diagnostics) - - -def plan_static_sql_tests( - *, target_dir: Path, adapter: BaseAdapter, project: CompiledProjectView -) -> PlannedStaticSqlTests: - """Plan every SQL test whose cached artifact under target_dir cannot be reused.""" - - tests_root: Path = target_dir / _COMPILED_DIR / _TESTS_DIR - pending, cached_paths, cached_records = _partition_cached_static_tests( - tests_root=tests_root, adapter=adapter, project=project - ) - return PlannedStaticSqlTests( - tests_root=tests_root, - pending=pending, - artifacts=_plan_static_test_artifacts( - adapter=adapter, project=project, tests=tuple(item.test for item in pending) - ), - cached_paths=frozenset(cached_paths), - cached_records=cached_records, - ) - - -def _resolve_planned_static_tests( - *, - target_dir: Path, - adapter: BaseAdapter, - project: CompiledProjectView, - planned_tests: Callable[[], PlannedStaticSqlTests] | None, -) -> PlannedStaticSqlTests: - planned: PlannedStaticSqlTests | None = planned_tests() if planned_tests is not None else None - if planned is not None and planned.tests_root == target_dir / _COMPILED_DIR / _TESTS_DIR: - return planned - return plan_static_sql_tests(target_dir=target_dir, adapter=adapter, project=project) - - -def _partition_cached_static_tests( - *, tests_root: Path, adapter: BaseAdapter, project: CompiledProjectView -) -> tuple[ - tuple[PendingStaticSqlTest, ...], - set[Path], - dict[str, SqlTestArtifactCacheRecord], -]: - """Split SQL tests into pending work and reusable cached artifacts.""" - - managed_paths: set[Path] = set() - cached_records: dict[str, SqlTestArtifactCacheRecord] = read_sql_test_artifact_cache( - cache_dir=project.compile_cache_dir - ) - current_records: dict[str, SqlTestArtifactCacheRecord] = {} - identity_context: SqlTestArtifactIdentityContext | None = ( - build_sql_test_artifact_identity_context(project=project, adapter=adapter) - if project.compile_cache_dir is not None - else None - ) - model_chain_names_by_key: dict[CompiledObjectKey, tuple[str, ...]] = {} - if identity_context is not None: - model_chain_names_by_key = sql_test_model_chain_names_by_key( - project=project, - tests=project.sql_tests, - ) - pending: list[PendingStaticSqlTest] = [] - for test in project.sql_tests: - record_key: str | None = None - artifact_identity: str | None = None - if identity_context is not None: - record_key = sql_test_artifact_record_key(test=test) - artifact_identity = sql_test_artifact_identity( - test=test, - model_chain_names=model_chain_names_by_key.get(test.key, ()), - context=identity_context, - ) - cached_record: SqlTestArtifactCacheRecord | None = cached_records.get(record_key) - if cached_record is not None: - reusable_record: SqlTestArtifactCacheRecord | None = artifact_matches_cache_record( - tests_root=tests_root, - record=cached_record, - identity=artifact_identity, - ) - if reusable_record is not None: - cached_path: Path = tests_root / reusable_record.relative_path - COMPILE_ARTIFACT_WRITES.kept( - path=cached_path, - size=reusable_record.size, - mtime_ns=reusable_record.mtime_ns, - ) - managed_paths.add(cached_path) - current_records[record_key] = reusable_record - continue - pending.append( - PendingStaticSqlTest( - test=test, record_key=record_key, artifact_identity=artifact_identity - ) - ) - return tuple(pending), managed_paths, current_records - - -def _plan_static_test_artifacts( - *, adapter: BaseAdapter, project: CompiledProjectView, tests: tuple[CompiledSqlTest, ...] -) -> tuple[NativeSqlTestArtifact, ...]: - if not tests: - return () - with OperationLifecycle( - operation_kind="project", operation_name="sql_test_planning" - ) as lifecycle: - native_artifacts: tuple[NativeSqlTestArtifact, ...] = plan_native_sql_test_artifacts( - project=project, - tests=tests, - adapter=adapter, - sql_analysis_enabled=project.settings.sql_analysis, - ) - lifecycle.completed(metadata={"item_count": len(native_artifacts)}) - return native_artifacts - - -def _write_static_tests( - *, - target_dir: Path, - adapter: BaseAdapter, - project: CompiledProjectView, - planned_tests: Callable[[], PlannedStaticSqlTests] | None, - batch: NativeArtifactBatch, -) -> tuple[set[Path], tuple[CompilerDiagnostic, ...]]: - """Write offline SQL-native test SQL and report uncached planning errors.""" - - diagnostics: list[CompilerDiagnostic] = [] - planned: PlannedStaticSqlTests = _resolve_planned_static_tests( - target_dir=target_dir, adapter=adapter, project=project, planned_tests=planned_tests - ) - tests_root: Path = planned.tests_root - managed_paths: set[Path] = set(planned.cached_paths) - current_records: dict[str, SqlTestArtifactCacheRecord] = dict(planned.cached_records) - test_paths: list[Path] = [] - for pending, artifact in zip(planned.pending, planned.artifacts, strict=True): - test_path: Path = tests_root / compiled_sql_test_output_path( - test=pending.test, - model_names=artifact.model_names, - ) - _write_sql(path=test_path, sql=artifact.sql, batch=batch) - managed_paths.add(test_path) - test_paths.append(test_path) - batch.flush() - for pending, artifact, test_path in zip( - planned.pending, planned.artifacts, test_paths, strict=True - ): - test: CompiledSqlTest = pending.test - record_key: str | None = pending.record_key - artifact_identity: str | None = pending.artifact_identity - artifact_diagnostics: tuple[CompilerDiagnostic, ...] = _sql_test_artifact_diagnostics( - test=test, artifact=artifact - ) - if artifact_diagnostics: - diagnostics.extend(artifact_diagnostics) - continue - if record_key is not None and artifact_identity is not None: - record: SqlTestArtifactCacheRecord | None = build_sql_test_artifact_cache_record( - tests_root=tests_root, - artifact_path=test_path, - identity=artifact_identity, - ) - if record is not None: - current_records[record_key] = record - with record_compile_timing("cache_publication_ms"): - write_sql_test_artifact_cache( - cache_dir=project.compile_cache_dir, - records=current_records, - ) - return managed_paths, tuple(diagnostics) - - -def _sql_test_artifact_diagnostics( - *, test: CompiledSqlTest, artifact: NativeSqlTestArtifact -) -> tuple[CompilerDiagnostic, ...]: - return tuple( - CompilerDiagnostic( - phase=DiagnosticPhase.TEST, - severity=DiagnosticSeverity.ERROR, - code=_SQL_TEST_PLANNING_ERROR_CODE, - message=message, - resource_type=CompiledResourceType.SQL_TEST, - resource_name=test.name, - path=test.source_path, - ) - for message in artifact.error_messages - ) - - -def _write_manifest(*, target_dir: Path, manifest: _native.NativeManifest) -> None: - """Write manifest.json.""" - - manifest_path: Path = target_dir / _MANIFEST_FILE - batch: NativeArtifactBatch = NativeArtifactBatch(check_existing=True) - batch.queue(path=manifest_path, contents=manifest.contents()) - batch.flush() - - -def _write_sql(*, path: Path, sql: str, batch: NativeArtifactBatch) -> None: - """Queue one SQL file for the native writer.""" - - batch.queue(path=path, contents=(sql.rstrip() + "\n").encode("utf-8")) - - -def _remove_stale_compiled_files(*, target_dir: Path, managed_paths: set[Path]) -> None: - _ = _native.remove_stale_artifacts( - target_dir / _COMPILED_DIR, [os.fspath(path) for path in managed_paths] - ) - - -def _model_output_path(relative_path: Path) -> Path: - parts: tuple[str, ...] = relative_path.parts - if parts and parts[0] == _MODELS_DIR: - return Path(*parts) - return Path(_MODELS_DIR) / relative_path - - -def _function_output_path(*, relative_path: Path, language: FunctionLanguage) -> Path: - parts: tuple[str, ...] = relative_path.parts - language_dir: str = language.value - function_language_path_part_count: int = 2 - if ( - len(parts) >= function_language_path_part_count - and parts[0] == _FUNCTIONS_DIR - and parts[1] == language_dir - ): - return Path(*parts).with_suffix(_SQL_FILE_SUFFIX) - return (Path(_FUNCTIONS_DIR) / language_dir / relative_path).with_suffix(_SQL_FILE_SUFFIX) - - -def _audit_folder(entry: AuditPlanEntry) -> Path: - """Determine the audit output folder.""" - - if entry.attached_target_name is not None: - return Path(_GENERIC_DIR) / entry.attached_target_name - return Path(_SINGULAR_DIR) - - -def _static_audit_folder(*, attached_target_name: str | None) -> Path: - """Determine the offline audit output folder.""" - - if attached_target_name is not None: - return Path(_GENERIC_DIR) / attached_target_name - return Path(_SINGULAR_DIR) - - -def _audit_file_name(entry: AuditPlanEntry) -> str: - """Determine the audit output file name.""" - - if entry.attached_target_name is not None and entry.attached_column_name is not None: - return f"{entry.name}__{entry.attached_column_name}{_SQL_FILE_SUFFIX}" - return f"{entry.name}{_SQL_FILE_SUFFIX}" - - -def _static_audit_file_name( - *, - name: str, - attached_target_name: str | None, - attached_column_name: str | None, -) -> str: - """Determine the offline audit output file name.""" - - if attached_target_name is not None and attached_column_name is not None: - return f"{name}__{attached_column_name}{_SQL_FILE_SUFFIX}" - return f"{name}{_SQL_FILE_SUFFIX}" diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py index aa6062ccfd..8b540d9a75 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py @@ -145,7 +145,7 @@ def _write_compile_machine_error( ) -> None: if args.command != CliCommand.COMPILE or not getattr(args, "json", False): return - from sqlbuild.cli.commands._helpers.compile.output import format_compile_error_json + from sqlbuild._native import format_compile_error_json code, message, help_text = expected_error_parts(error=error, fallback_code=fallback_code) location: object = getattr(error, "location", None) diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py b/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py index 20c6166e52..ca61bd42d4 100644 --- a/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py +++ b/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py @@ -4,10 +4,7 @@ from pathlib import Path -from sqlbuild._native import analyze_compile_project -from sqlbuild.cli.commands._helpers.compile.target_writer import ( - static_sql_test_planning_diagnostics, -) +from sqlbuild._native import analyze_compile_project, static_sql_test_planning_diagnostics from sqlbuild.cli.commands.models import RefactorCompile from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.compile.models import CompileAnalysis diff --git a/src/sqlbuild/cli/commands/classes/native_artifact_batch.py b/src/sqlbuild/cli/commands/classes/native_artifact_batch.py deleted file mode 100644 index 1d6ae2a949..0000000000 --- a/src/sqlbuild/cli/commands/classes/native_artifact_batch.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Queue compiled artifacts and write each group in one native call.""" - -from __future__ import annotations - -import os -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.cli.compile_reuse.constants import COMPILE_ARTIFACT_WRITES -from sqlbuild.compiler.profiling.main.record import record_compile_timing - -_POSIX_LINE_SEPARATOR: str = "\n" - - -class NativeArtifactBatch: - """Compiled artifacts queued for the native writer, keeping unchanged files when asked.""" - - def __init__(self, *, check_existing: bool) -> None: - self.check_existing: bool = check_existing - self._files: list[tuple[Path, bytes]] = [] - - def queue(self, *, path: Path, contents: bytes) -> None: - """Queue one artifact with platform line endings, noting its bytes for compile reuse.""" - - if os.linesep != _POSIX_LINE_SEPARATOR: - contents = contents.replace(b"\n", os.linesep.encode()) - COMPILE_ARTIFACT_WRITES.written(path=path, contents=contents) - self._files.append((path, contents)) - - def flush(self) -> None: - """Write every queued artifact natively and empty the queue.""" - - if not self._files: - return - with record_compile_timing("physical_write_ms"): - _ = _native.write_compiled_artifacts(self._files, self.check_existing) - self._files = [] diff --git a/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py b/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py deleted file mode 100644 index eedd5ec7d3..0000000000 --- a/src/sqlbuild/cli/commands/classes/prepared_compile_artifacts.py +++ /dev/null @@ -1,260 +0,0 @@ -"""Prepare compile artifacts in the background and publish them only at the normal write phase.""" - -from __future__ import annotations - -import logging -import shutil -import uuid -from concurrent.futures import Future, ThreadPoolExecutor -from contextvars import Context, copy_context -from pathlib import Path -from types import TracebackType - -from filelock import FileLock, Timeout - -import sqlbuild._native as _native -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.compile.target_writer import ( - plan_static_sql_tests, - publish_static_compile_target, - staged_artifact_files, - static_sql_test_planning_diagnostics, - write_static_compile_target, -) -from sqlbuild.cli.commands.exceptions import StagedArtifactsChangedError -from sqlbuild.cli.compile.models import PlannedStaticSqlTests -from sqlbuild.cli.output.models import WrittenTarget -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.compile.types import CompiledProjectView - -_MIN_PREPARED_ARTIFACT_MODELS: int = 128 -_MAX_PREPARED_ARTIFACT_MODELS: int = 5000 -_STAGING_PREFIX: str = ".sqlbuild-staging-" -_LOCK_SUFFIX: str = ".lock" -_OWNER_MARKER: str = ".owner" -_LOGGER: logging.Logger = logging.getLogger(__name__) -type _PreparedResult = tuple[WrittenTarget, frozenset[str]] - - -class PreparedCompileArtifacts: - """Own one background preparation task, either staged artifacts or planned SQL tests only.""" - - def __init__(self, *, enabled: bool) -> None: - self.enabled: bool = enabled - self._staging_dir: Path | None = None - self._owner_lock: FileLock | None = None - self._created_target_dir: Path | None = None - self._executor: ThreadPoolExecutor | None = None - self._future: Future[_PreparedResult] | None = None - self._planned_tests: Future[PlannedStaticSqlTests] | None = None - self._planning_inputs: tuple[CompiledProjectView, BaseAdapter, Path] | None = None - - def __enter__(self) -> PreparedCompileArtifacts: - return self - - def __exit__( - self, - exception_type: type[BaseException] | None, - exception: BaseException | None, - traceback: TracebackType | None, - ) -> None: - try: - if self._executor is not None: - self._executor.shutdown(wait=True, cancel_futures=True) - finally: - if self._staging_dir is not None: - shutil.rmtree(self._staging_dir, ignore_errors=True) - if self._owner_lock is not None: - _release_and_unlink(self._owner_lock) - if self._created_target_dir is not None: - _remove_if_empty(self._created_target_dir) - - def remove_abandoned_staging(self, *, target_dir: Path) -> None: - """Delete staging directories whose owner lock this compile can hold for the removal.""" - - if not self.enabled: - return - for directory in sorted(target_dir.glob(f"{_STAGING_PREFIX}*")): - if directory.suffix == _LOCK_SUFFIX: - continue - if not _remove_abandoned_directory(directory): - return - - def start( - self, - *, - project: CompiledProjectView, - adapter: BaseAdapter, - target_dir: Path, - stage_artifacts: bool, - ) -> None: - """Stage artifacts beside target_dir when worthwhile, otherwise plan SQL tests ahead.""" - - if not self.enabled: - return - if ( - stage_artifacts - and _stages_artifacts(project) - and self._start_staging(project=project, adapter=adapter, target_dir=target_dir) - ): - return - if project.sql_tests: - self._start_test_planning(project=project, adapter=adapter, target_dir=target_dir) - - def _start_test_planning( - self, *, project: CompiledProjectView, adapter: BaseAdapter, target_dir: Path - ) -> None: - """Plan SQL tests against the published target while the compile continues.""" - - self._planning_inputs = (project, adapter, target_dir) - self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-tests") - context: Context = copy_context() - - def plan() -> PlannedStaticSqlTests: - return context.run( - plan_static_sql_tests, target_dir=target_dir, adapter=adapter, project=project - ) - - self._planned_tests = self._executor.submit(plan) - - def _start_staging( - self, *, project: CompiledProjectView, adapter: BaseAdapter, target_dir: Path - ) -> bool: - """Stage artifacts beside target_dir so publication can move them instead of copying.""" - - staging_dir: Path = target_dir / f"{_STAGING_PREFIX}{uuid.uuid4().hex}" - try: - if not target_dir.is_dir(): - target_dir.mkdir(parents=True) - self._created_target_dir = target_dir - self._owner_lock = FileLock(_owner_lock_path(staging_dir)) - self._owner_lock.acquire() - _create_staging_directory(staging_dir) - except OSError: - return False - self._staging_dir = staging_dir - self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="sqlbuild-artifacts") - context: Context = copy_context() - - def prepare() -> _PreparedResult: - written: WrittenTarget = context.run( - write_static_compile_target, - target_dir=staging_dir, - adapter=adapter, - project=project, - ) - return written, staged_artifact_files(target_dir=staging_dir) - - self._future = self._executor.submit(prepare) - return True - - def publish( - self, *, target_dir: Path, manifest: _native.NativeManifest | None - ) -> WrittenTarget | None: - if self._planned_tests is not None and self._planning_inputs is not None: - project, adapter, _ = self._planning_inputs - return write_static_compile_target( - target_dir=target_dir, - adapter=adapter, - project=project, - manifest=manifest, - planned_tests=self._planned_tests.result, - ) - if self._future is None or self._staging_dir is None: - return None - try: - prepared, expected_files = self._future.result() - except OSError: - return None - if not _owns_staging_directory(self._staging_dir): - raise StagedArtifactsChangedError( - f"staging directory '{self._staging_dir.name}' was removed or replaced before " - "publication; no artifact was published", - help="Rerun the compile; avoid deleting target/ while a compile is running.", - ) - return publish_static_compile_target( - prepared=prepared, - target_dir=target_dir, - manifest=manifest, - expected_files=expected_files, - ) - - def planning_diagnostics(self) -> tuple[CompilerDiagnostic, ...] | None: - """Return background SQL test planning diagnostics without publishing any artifact.""" - - if self._planned_tests is not None and self._planning_inputs is not None: - project, adapter, target_dir = self._planning_inputs - return static_sql_test_planning_diagnostics( - target_dir=target_dir, - adapter=adapter, - project=project, - planned_tests=self._planned_tests.result, - ) - if self._future is None: - return None - try: - return self._future.result()[0].diagnostics - except OSError: - return None - - -def _stages_artifacts(project: CompiledProjectView) -> bool: - """Staging pays off for uncached, mid-sized projects whose memory stays within budget.""" - - return ( - project.compile_cache_dir is None - and _MIN_PREPARED_ARTIFACT_MODELS <= len(project.models) <= _MAX_PREPARED_ARTIFACT_MODELS - ) - - -def _owner_lock_path(staging_dir: Path) -> Path: - return staging_dir.with_name(f"{staging_dir.name}{_LOCK_SUFFIX}") - - -def _create_staging_directory(staging_dir: Path) -> None: - staging_dir.mkdir() - (staging_dir / _OWNER_MARKER).write_text(staging_dir.name, encoding="utf-8") - - -def _owns_staging_directory(staging_dir: Path) -> bool: - try: - return (staging_dir / _OWNER_MARKER).read_text(encoding="utf-8") == staging_dir.name - except OSError: - return False - - -def _remove_abandoned_directory(directory: Path) -> bool: - """Remove one ownerless staging directory while holding its lock; False stops cleanup.""" - - lock: FileLock = FileLock(_owner_lock_path(directory), timeout=0) - try: - lock.acquire() - except Timeout: - return True - except OSError as error: - _LOGGER.warning("skipped abandoned staging cleanup in %s: %s", directory.parent, error) - return False - try: - if directory.is_dir(): - shutil.rmtree(directory) - except OSError as error: - lock.release() - _LOGGER.warning("stopped abandoned staging cleanup at %s: %s", directory, error) - return False - _release_and_unlink(lock) - return True - - -def _release_and_unlink(lock: FileLock) -> None: - try: - Path(lock.lock_file).unlink(missing_ok=True) - except OSError: - pass - lock.release() - - -def _remove_if_empty(directory: Path) -> None: - try: - directory.rmdir() - except OSError: - pass diff --git a/src/sqlbuild/cli/compile/models.py b/src/sqlbuild/cli/compile/models.py index fd437097ee..076731c0ba 100644 --- a/src/sqlbuild/cli/compile/models.py +++ b/src/sqlbuild/cli/compile/models.py @@ -10,12 +10,10 @@ from sqlbuild.cli.output.models import WrittenTarget from sqlbuild.compiler.compile.models import ( CompiledObjectKey, - CompiledSqlTest, CompilerDiagnostic, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.models import NativeSqlTestArtifact @dataclass(frozen=True) @@ -67,23 +65,3 @@ class SqlTestArtifactIdentityContext: common_identity: str model_identities: dict[str, str] - - -@dataclass(frozen=True) -class PendingStaticSqlTest: - """One SQL test that needs a freshly planned artifact, with its optional cache identity.""" - - test: CompiledSqlTest - record_key: str | None - artifact_identity: str | None - - -@dataclass(frozen=True) -class PlannedStaticSqlTests: - """SQL-test artifacts planned for one target directory before any file is written.""" - - tests_root: Path - pending: tuple[PendingStaticSqlTest, ...] - artifacts: tuple[NativeSqlTestArtifact, ...] - cached_paths: frozenset[Path] - cached_records: dict[str, SqlTestArtifactCacheRecord] diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py b/src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py index 5fcc2587c6..abb322f09a 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/native_reuse.py @@ -2,27 +2,21 @@ from __future__ import annotations -import contextlib import logging import os import sys -import time from dataclasses import replace from pathlib import Path import sqlbuild._native as _native from sqlbuild._native import NativeReuseAttempt -from sqlbuild.cli.compile_reuse._helpers.provider_settings import provider_settings_inputs from sqlbuild.cli.compile_reuse._helpers.runtime_identity import ( invocation_identity, redirected_output_files, runtime_identity, ) from sqlbuild.cli.compile_reuse._helpers.timings import elapsed_ms, reuse_timings -from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites -from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput from sqlbuild.cli.compile_reuse.constants import ( - BYTES_PER_MEBIBYTE, COMPILED_DIRECTORY_NAME, EXCLUDED_DIRECTORIES, EXCLUDED_ROOT_DIRECTORIES, @@ -42,11 +36,6 @@ REUSE_LOGGER_NAME, REUSE_MAX_ENTRY_BYTES, REUSE_MAX_STORED_ENTRIES, - REUSE_STORE_DONE_MESSAGE, - REUSE_STORE_NOTICE_BYTES, - REUSE_STORE_NOTICE_PATHS, - REUSE_STORE_SKIPPED_MESSAGE, - REUSE_STORE_START_MESSAGE, TARGET_DIRECTORY_NAME, TOTAL_TIMING, TRACKED_ENVIRONMENT_PREFIXES, @@ -55,18 +44,13 @@ from sqlbuild.cli.compile_reuse.models import ( CompileReuseAttempt, CompileReuseRequest, - SettingsInputsResult, ) from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome -from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads from sqlbuild.compiler.compile.constants import ( COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, ) from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory -from sqlbuild.compiler.macro_bridge.main.macro_store_module_paths import ( - macro_store_module_paths, -) from sqlbuild.presentation.main.supports_color import supports_color _LOGGER: logging.Logger = logging.getLogger(REUSE_LOGGER_NAME) @@ -152,103 +136,6 @@ def _replayed( return replace(missed, outcome=CompileReuseOutcome.HIT, check_ms=check_ms, exit_code=exit_code) -def record_native( - *, - store_path: Path, - native: NativeReuseAttempt, - output: RecordedCompileOutput, - exit_code: int, - input_reads: CompileInputReads, - artifact_writes: CompileArtifactWrites, - compile_cache_enabled: bool, - artifacts_written: bool, - dag_artifact_path: Path | None, - json_output: bool, -) -> bool: - """Store a reusable compile natively, or clear the slot; return whether it was stored.""" - - try: - return _record( - native=native, - output=output, - exit_code=exit_code, - input_reads=input_reads, - artifact_writes=artifact_writes, - compile_cache_enabled=compile_cache_enabled, - artifacts_written=artifacts_written, - dag_artifact_path=dag_artifact_path, - json_output=json_output, - ) - except Exception: - _LOGGER.debug("Compile reuse did not store this compile", exc_info=True) - with contextlib.suppress(OSError): - store_path.unlink(missing_ok=True) - return False - - -def _record( - *, - native: NativeReuseAttempt, - output: RecordedCompileOutput, - exit_code: int, - input_reads: CompileInputReads, - artifact_writes: CompileArtifactWrites, - compile_cache_enabled: bool, - artifacts_written: bool, - dag_artifact_path: Path | None, - json_output: bool, -) -> bool: - settings: SettingsInputsResult = provider_settings_inputs( - settings_classes=input_reads.settings_classes - ) - if settings.unsupported_reason is not None: - _LOGGER.debug("Compile reuse is off for this project: %s", settings.unsupported_reason) - eligible: bool = ( - output.stdout is not None - and compile_cache_enabled - and not input_reads.read_run_id - and settings.unsupported_reason is None - ) - notice_started: float | None = _start_notice(native=native) if eligible else None - outcome, digested_files = native.record( - { - "stdout": output.stdout, - "stderr_lines": list(output.stderr_lines), - "exit_code": exit_code, - "compile_cache_enabled": compile_cache_enabled, - "read_run_id": input_reads.read_run_id, - "settings_inputs": None - if settings.unsupported_reason is not None - else [ - ( - item.case_sensitive, - list(item.names), - list(item.prefixes), - list(item.env_files), - list(item.secrets_dirs), - ) - for item in settings.inputs - ], - "template_environment_names": list(input_reads.environment_names), - "module_paths": _module_paths(), - "artifacts": { - path: (artifact.digest, artifact.size, artifact.mtime_ns) - for path, artifact in artifact_writes.artifacts.items() - }, - "artifacts_written": artifacts_written, - "dag_path": None if dag_artifact_path is None else os.path.abspath(dag_artifact_path), - "json_output": json_output, - "rules": reuse_rules(), - } - ) - _LOGGER.debug("Compile reuse record digested %d project files", digested_files) - _finish_notice( - started=notice_started, - message=REUSE_STORE_SKIPPED_MESSAGE if outcome == _SKIPPED else REUSE_STORE_DONE_MESSAGE, - ) - return outcome == _STORED - - def reuse_rules() -> dict[str, object]: """The reuse constants the native check and record share with the Python implementation.""" @@ -274,33 +161,3 @@ def reuse_rules() -> dict[str, object]: "compiled_root": os.path.join(TARGET_DIRECTORY_NAME, COMPILED_DIRECTORY_NAME), "retired_directories": [RETIRED_REUSE_DIRECTORY_NAME], } - - -def _module_paths() -> list[str]: - """Loaded module files, then the macro store's extra module paths, as Python lists them.""" - - module_files: list[object] = [ - getattr(module, "__file__", None) for module in tuple(sys.modules.values()) - ] - return [ - path - for path in (*module_files, *sorted(macro_store_module_paths())) - if isinstance(path, str) - ] - - -def _start_notice(*, native: NativeReuseAttempt) -> float | None: - pending_bytes: int = native.pending_digest_bytes() - paths: int = native.snapshot_paths() - if pending_bytes < REUSE_STORE_NOTICE_BYTES and paths < REUSE_STORE_NOTICE_PATHS: - return None - print( - REUSE_STORE_START_MESSAGE.format(paths=paths, mebibytes=pending_bytes / BYTES_PER_MEBIBYTE), - file=sys.stderr, - ) - return time.monotonic() - - -def _finish_notice(*, started: float | None, message: str) -> None: - if started is not None: - print(message.format(seconds=time.monotonic() - started), file=sys.stderr) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/store.py b/src/sqlbuild/cli/compile_reuse/_helpers/store.py deleted file mode 100644 index 8bbbacb5fe..0000000000 --- a/src/sqlbuild/cli/compile_reuse/_helpers/store.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Record a completed full compile so an unchanged rerun can replay it.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.compile_reuse._helpers.launch_records import write_launch_record -from sqlbuild.cli.compile_reuse._helpers.native_reuse import record_native -from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites -from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt -from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome -from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads - - -def write_compile_entry( - *, - attempt: CompileReuseAttempt, - output: RecordedCompileOutput, - exit_code: int, - input_reads: CompileInputReads, - artifact_writes: CompileArtifactWrites, - compile_cache_enabled: bool, - artifacts_written: bool, - dag_artifact_path: Path | None, - json_output: bool, -) -> None: - """Store a reusable compile, or drop the stored one when this compile cannot be reused.""" - - if ( - attempt.outcome is not CompileReuseOutcome.MISS - or attempt.entry_path is None - or attempt.native is None - ): - return - stored: bool = record_native( - store_path=attempt.entry_path, - native=attempt.native, - output=output, - exit_code=exit_code, - input_reads=input_reads, - artifact_writes=artifact_writes, - compile_cache_enabled=compile_cache_enabled, - artifacts_written=artifacts_written, - dag_artifact_path=dag_artifact_path, - json_output=json_output, - ) - if stored: - write_launch_record(attempt=attempt) diff --git a/src/sqlbuild/cli/compile_reuse/classes/__init__.py b/src/sqlbuild/cli/compile_reuse/classes/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py b/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py deleted file mode 100644 index 4a895a8777..0000000000 --- a/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Process-wide delivery of compile artifact writes to the active compile recorders.""" - -from __future__ import annotations - -import os -import threading -from collections.abc import Iterator -from contextlib import contextmanager - -import sqlbuild._native as _native -from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites -from sqlbuild.cli.compile_reuse.models import RecordedArtifact - - -class CompileArtifactWriteRegistry: - """Deliver artifact writes from every thread, including background staging threads.""" - - def __init__(self) -> None: - self._lock: threading.Lock = threading.Lock() - self._recorders: tuple[CompileArtifactWrites, ...] = () - - @contextmanager - def recording(self) -> Iterator[CompileArtifactWrites]: - """Record artifact writes until the block exits.""" - - recorder: CompileArtifactWrites = CompileArtifactWrites() - with self._lock: - self._recorders = (*self._recorders, recorder) - try: - yield recorder - finally: - with self._lock: - self._recorders = tuple(item for item in self._recorders if item is not recorder) - - def written(self, *, path: str | os.PathLike[str], contents: bytes) -> None: - """Note the exact bytes an artifact holds after a write or an unchanged-file check.""" - - recorders: tuple[CompileArtifactWrites, ...] = self._recorders - if not recorders: - return - artifact: RecordedArtifact = RecordedArtifact(digest=_native.artifact_digest(contents)) - for recorder in recorders: - recorder.record(path=os.path.abspath(path), artifact=artifact) - - def kept(self, *, path: str | os.PathLike[str], size: int, mtime_ns: int) -> None: - """Note an earlier artifact kept as-is after its size and mtime were validated.""" - - artifact: RecordedArtifact = RecordedArtifact(digest=None, size=size, mtime_ns=mtime_ns) - for recorder in self._recorders: - recorder.record(path=os.path.abspath(path), artifact=artifact) - - def moved(self, *, source: str | os.PathLike[str], destination: str | os.PathLike[str]) -> None: - """Note that one staged artifact was published at another path.""" - - for recorder in self._recorders: - recorder.move(source=os.path.abspath(source), destination=os.path.abspath(destination)) - - def moved_tree( - self, *, source: str | os.PathLike[str], destination: str | os.PathLike[str] - ) -> None: - """Note that a staged artifact directory was published at another directory.""" - - for recorder in self._recorders: - recorder.move_tree( - source=os.path.abspath(source), destination=os.path.abspath(destination) - ) diff --git a/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_writes.py b/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_writes.py deleted file mode 100644 index 8f797a49c0..0000000000 --- a/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_writes.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Artifacts one compile wrote or kept, keyed by absolute path.""" - -from __future__ import annotations - -import os -import threading - -from sqlbuild.cli.compile_reuse.models import RecordedArtifact - - -class CompileArtifactWrites: - """Record what one compile left in each artifact path, following staged moves.""" - - def __init__(self) -> None: - self._lock: threading.Lock = threading.Lock() - self._artifacts: dict[str, RecordedArtifact] = {} - - def record(self, *, path: str, artifact: RecordedArtifact) -> None: - """Record the artifact this compile left at one path.""" - - with self._lock: - self._artifacts[path] = artifact - - def move(self, *, source: str, destination: str) -> None: - """Follow one staged artifact to its published path.""" - - with self._lock: - artifact: RecordedArtifact | None = self._artifacts.pop(source, None) - if artifact is not None: - self._artifacts[destination] = artifact - - def move_tree(self, *, source: str, destination: str) -> None: - """Follow every staged artifact under one directory to its published directory.""" - - prefix: str = source + os.sep - with self._lock: - moved: dict[str, RecordedArtifact] = { - destination + path[len(source) :]: artifact - for path, artifact in self._artifacts.items() - if path.startswith(prefix) - } - self._artifacts = { - path: artifact - for path, artifact in self._artifacts.items() - if not path.startswith(prefix) - } | moved - - @property - def artifacts(self) -> dict[str, RecordedArtifact]: - """Return the recorded artifacts by absolute path.""" - - with self._lock: - return dict(self._artifacts) diff --git a/src/sqlbuild/cli/compile_reuse/classes/recorded_compile_output.py b/src/sqlbuild/cli/compile_reuse/classes/recorded_compile_output.py deleted file mode 100644 index fec9748f59..0000000000 --- a/src/sqlbuild/cli/compile_reuse/classes/recorded_compile_output.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Compile command output that is printed and retained for later reuse.""" - -from __future__ import annotations - -import sys - - -class RecordedCompileOutput: - """Print compile notes at once and the report after storing, keeping the exact text.""" - - def __init__(self) -> None: - self._stderr_lines: list[str] = [] - self._stdout: str | None = None - - def print_stderr_line(self, line: str) -> None: - """Print one stderr result line, such as a note, and record it.""" - - print(line, file=sys.stderr) - self._stderr_lines.append(line) - - def record_stdout(self, text: str) -> None: - """Record the compile report; it is printed once the compile has been stored.""" - - self._stdout = f"{text}\n" - - def print_stdout(self) -> None: - """Print the recorded compile report, if any, exactly as it is stored.""" - - if self._stdout is not None: - _ = sys.stdout.write(self._stdout) - sys.stdout.flush() - - @property - def stderr_lines(self) -> tuple[str, ...]: - """Return the recorded stderr result lines in emission order.""" - - return tuple(self._stderr_lines) - - @property - def stdout(self) -> str | None: - """Return the recorded stdout text, or None when no report was printed.""" - - return self._stdout diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index dfdcdc65fd..0980a46ebe 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -2,9 +2,6 @@ from __future__ import annotations -from sqlbuild.cli.compile_reuse.classes.compile_artifact_write_registry import ( - CompileArtifactWriteRegistry, -) from sqlbuild.compiler.frontier.constants import ( COMPILE_REUSE_DISABLE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR, @@ -77,7 +74,6 @@ REUSE_STORE_SKIPPED_MESSAGE: str = ( "Did not record compile for reuse: files changed while it ran ({seconds:.1f} s)" ) -COMPILE_ARTIFACT_WRITES: CompileArtifactWriteRegistry = CompileArtifactWriteRegistry() LAUNCH_RECORD_FORMAT: int = 1 LAUNCH_RECORD_DIRECTORY: tuple[str, ...] = (TARGET_DIRECTORY_NAME, "cache", "launcher-v1") LAUNCH_RECORD_LIMIT: int = 32 diff --git a/src/sqlbuild/cli/compile_reuse/main/_compile_reuse_timings.py b/src/sqlbuild/cli/compile_reuse/main/_compile_reuse_timings.py deleted file mode 100644 index f98d3d077b..0000000000 --- a/src/sqlbuild/cli/compile_reuse/main/_compile_reuse_timings.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Public entry for the compile reuse counters reported in compile_timings.""" - -from __future__ import annotations - -from sqlbuild.cli.compile_reuse._helpers.timings import reuse_timings -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt - - -def compile_reuse_timings(*, attempt: CompileReuseAttempt) -> dict[str, int]: - """Return the hit, miss, and bypass counters plus the time spent checking for reuse.""" - - return reuse_timings(outcome=attempt.outcome, check_ms=attempt.check_ms) diff --git a/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py b/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py deleted file mode 100644 index 0736318972..0000000000 --- a/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Public entry for storing a full compile so an unchanged rerun can replay it.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.compile_reuse._helpers.store import write_compile_entry -from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites -from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt -from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads - - -def write_reusable_compile( - *, - attempt: CompileReuseAttempt, - output: RecordedCompileOutput, - exit_code: int, - input_reads: CompileInputReads, - artifact_writes: CompileArtifactWrites, - compile_cache_enabled: bool, - artifacts_written: bool, - dag_artifact_path: Path | None, - json_output: bool, -) -> None: - """Store a reusable full compile, or drop the stored one when it cannot be reused.""" - - write_compile_entry( - attempt=attempt, - output=output, - exit_code=exit_code, - input_reads=input_reads, - artifact_writes=artifact_writes, - compile_cache_enabled=compile_cache_enabled, - artifacts_written=artifacts_written, - dag_artifact_path=dag_artifact_path, - json_output=json_output, - ) diff --git a/src/sqlbuild/cli/compile_reuse/models.py b/src/sqlbuild/cli/compile_reuse/models.py index 0cf9624571..5ea1860000 100644 --- a/src/sqlbuild/cli/compile_reuse/models.py +++ b/src/sqlbuild/cli/compile_reuse/models.py @@ -49,15 +49,6 @@ class SettingsInputsResult: unsupported_reason: str | None = None -@dataclass(frozen=True) -class RecordedArtifact: - """What one compile wrote to an artifact: its content digest, or the stat it kept.""" - - digest: str | None - size: int | None = None - mtime_ns: int | None = None - - @dataclass(frozen=True) class CompileReuseAttempt: """Result of checking for a reusable compile, carried into a full compile on a miss.""" diff --git a/src/sqlbuild/compiler/compile/main/semantic_coverage.py b/src/sqlbuild/compiler/compile/main/_semantic_coverage.py similarity index 100% rename from src/sqlbuild/compiler/compile/main/semantic_coverage.py rename to src/sqlbuild/compiler/compile/main/_semantic_coverage.py diff --git a/src/sqlbuild/compiler/macro_bridge/main/macro_store_module_paths.py b/src/sqlbuild/compiler/macro_bridge/main/_macro_store_module_paths.py similarity index 100% rename from src/sqlbuild/compiler/macro_bridge/main/macro_store_module_paths.py rename to src/sqlbuild/compiler/macro_bridge/main/_macro_store_module_paths.py diff --git a/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py b/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py index 6a2386322b..2f032de89e 100644 --- a/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py +++ b/src/sqlbuild/compiler/manifest/main/_build_graph_manifest.py @@ -7,7 +7,7 @@ import sqlbuild._native as _native from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities +from sqlbuild.compiler.python_nodes.main._hook_identities import build_hook_identities def build_graph_manifest( diff --git a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py index 0ade165563..22ee64a1e5 100644 --- a/src/sqlbuild/compiler/planner/_helpers/changes/detect.py +++ b/src/sqlbuild/compiler/planner/_helpers/changes/detect.py @@ -55,7 +55,7 @@ WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import BackfillAction, ChangeKind -from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities +from sqlbuild.compiler.python_nodes.main._hook_identities import build_hook_identities from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.diagnostics.main.log_sql import log_sql from sqlbuild.spec.contracts.main.get_config_str import get_config_str diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/direct.py b/src/sqlbuild/compiler/planner/_helpers/identity/direct.py index ebbb499166..44b861581e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/identity/direct.py +++ b/src/sqlbuild/compiler/planner/_helpers/identity/direct.py @@ -32,7 +32,7 @@ PlannerScope, ) from sqlbuild.compiler.planner.types import GraphResourceKind -from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities +from sqlbuild.compiler.python_nodes.main._hook_identities import build_hook_identities def build_direct_model_version_identities( diff --git a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py index d560e13336..c7363377c7 100644 --- a/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py +++ b/src/sqlbuild/compiler/planner/_helpers/migrations/discovery.py @@ -41,7 +41,7 @@ WarehouseSnapshot, ) from sqlbuild.compiler.planner.types import MaterializationType, WarningSeverity -from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities +from sqlbuild.compiler.python_nodes.main._hook_identities import build_hook_identities from sqlbuild.spec.contracts.main.get_config_str import get_config_str _HISTORY_MATERIALIZATIONS: frozenset[str] = frozenset( diff --git a/src/sqlbuild/compiler/profiling/main/record.py b/src/sqlbuild/compiler/profiling/main/_record.py similarity index 100% rename from src/sqlbuild/compiler/profiling/main/record.py rename to src/sqlbuild/compiler/profiling/main/_record.py diff --git a/src/sqlbuild/compiler/python_nodes/main/hook_identities.py b/src/sqlbuild/compiler/python_nodes/main/_hook_identities.py similarity index 100% rename from src/sqlbuild/compiler/python_nodes/main/hook_identities.py rename to src/sqlbuild/compiler/python_nodes/main/_hook_identities.py diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 81336c5963..141bc06c3f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -31,10 +31,8 @@ import sqlbuild._native as native_module import sqlbuild.adapter.type_system._helpers.type_normalization as type_normalization -import sqlbuild.cli.commands._helpers.compile.lineage as compile_lineage -import sqlbuild.cli.commands._helpers.compile.target_writer as target_writer import sqlbuild.cli.compile_reuse._helpers.native_reuse as native_reuse -import sqlbuild.cli.compile_reuse.main._write_reusable_compile as write_reusable_module +import sqlbuild.cli.compile_reuse._helpers.provider_settings as provider_settings import sqlbuild.compiler.compile._helpers.diagnostics.recovery as diagnostic_recovery import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stages @@ -44,6 +42,7 @@ import sqlbuild.compiler.contracts.main.validate as contract_validation import sqlbuild.compiler.frontier.main.compiled_code_identity as compiled_code_identity_module import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class +import sqlbuild.compiler.sql_test_glue.main.plan_native_sql_test_artifacts as plan_artifacts_module from scripts.cold_compile_performance.main.read_compile_measurement import read_compile_measurement from scripts.cold_compile_performance.main.semantic_compile_fingerprint import ( semantic_compile_fingerprint, @@ -3355,7 +3354,7 @@ def write_before_reuse_store( ) -> None: """Let another writer change target/ once, after a compile wrote it and before it stores.""" - original: Callable[..., None] = write_reusable_module.write_reusable_compile + original: Callable[..., None] = native_module.write_reusable_compile pending: list[Callable[[Path], None]] = [write] def write_then_store(**kwargs: Any) -> None: @@ -3364,7 +3363,7 @@ def write_then_store(**kwargs: Any) -> None: pending.clear() original(**kwargs) - monkeypatch.setattr(write_reusable_module, "write_reusable_compile", write_then_store) + monkeypatch.setattr(native_module, "write_reusable_compile", write_then_store) def rewrite_compiled_model_unchanged(project_dir: Path) -> None: @@ -3458,7 +3457,7 @@ def fail_reuse_store(*, monkeypatch: pytest.MonkeyPatch, error: BaseException) - def raise_error(**_kwargs: Any) -> None: raise error - monkeypatch.setattr(native_reuse, "provider_settings_inputs", raise_error) + monkeypatch.setattr(provider_settings, "provider_settings_inputs", raise_error) def compile_in_process_output( @@ -4818,8 +4817,8 @@ def full_macro_call_runs( (diagnostic_recovery, "complete_semantic_project"), (contract_validation, "evaluate_native_model_contracts"), (promotion_conflicts, "native_promotion_conflict_diagnostics"), - (compile_lineage, "build_compile_lineage"), - (target_writer, "plan_native_sql_test_artifacts"), + (native_module, "build_project_column_lineage"), + (plan_artifacts_module, "plan_native_sql_test_artifacts"), ) @@ -4837,8 +4836,8 @@ def record_analysis_seams(*, monkeypatch: pytest.MonkeyPatch) -> dict[str, list[ def _recorded_seam( *, entry: Callable[..., object], name: str, results: dict[str, list[object]] ) -> Callable[..., object]: - def recorded(**arguments: object) -> object: - result: object = entry(**arguments) + def recorded(*positional: object, **arguments: object) -> object: + result: object = entry(*positional, **arguments) results.setdefault(name, []).append(result) return result diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py index b710e6cb6c..c9747f32a2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_analysis_seams.py @@ -36,7 +36,7 @@ "complete_semantic_project": [ANY], "evaluate_native_model_contracts": [ANY], "native_promotion_conflict_diagnostics": [()], - "build_compile_lineage": [ANY], + "build_project_column_lineage": [ANY], "plan_native_sql_test_artifacts": [ANY], }, ) @@ -62,7 +62,7 @@ def test_given_project_when_compiling_then_native_seams_answer( assert native_seams == test_case.expected_native_returns planned_counts: list[int] = [] for seams in (native_seams,): - assert isinstance(seams["build_compile_lineage"][0], ProjectColumnLineage) + assert isinstance(seams["build_project_column_lineage"][0], ProjectColumnLineage) assert isinstance(seams["assemble_native_project_resources"][0], NativeProjectResources) assert isinstance(seams["analyze_native_model_sql"][0], NativeModelAnalyses) assert seams["analyze_native_model_sql"][0].session is not None diff --git a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py index def9e03042..60733f0503 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/helpers.py @@ -11,9 +11,10 @@ import pytest import sqlbuild._native as native_module -from sqlbuild.cli.commands.classes import prepared_compile_artifacts +from sqlbuild._native import PreparedCompileArtifacts from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.cli.compile_reuse._helpers import native_reuse as native_reuse_module +from sqlbuild.cli.compile_reuse._helpers import provider_settings as provider_settings_module from sqlbuild.cli.compile_reuse._helpers.runtime_identity import runtime_identity from sqlbuild.cli.compile_reuse.constants import ( NATIVE_REUSE_DIRECTORY_NAME, @@ -24,6 +25,14 @@ from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory STAGING_PREFIX: str = ".sqlbuild-staging-" + + +class _StagingEveryProject(PreparedCompileArtifacts): + """Stage artifacts for a project of any size.""" + + min_prepared_models: int = 1 + + OUTPUT_MODEL_HEADER: str = ( "MODEL (description 'Orders {name}.', materialized table, contract enforced, " "columns (order_id (type INTEGER)));\n" @@ -99,7 +108,7 @@ def prepare_output_compiles(*, monkeypatch: pytest.MonkeyPatch, reuse_disabled: """Stage small projects natively and settle file stamps.""" monkeypatch.setenv(REUSE_DISABLE_ENV_VAR, reuse_disabled) - monkeypatch.setattr(prepared_compile_artifacts, "_MIN_PREPARED_ARTIFACT_MODELS", 1) + monkeypatch.setattr(native_module, "PreparedCompileArtifacts", _StagingEveryProject) monkeypatch.setattr(native_reuse_module, "RACY_WINDOW_NS", 0) @@ -226,7 +235,7 @@ def raise_error(**_kwargs: object) -> None: raise RuntimeError("settings went away") _edit(project_dir=project_dir) - monkeypatch.setattr(native_reuse_module, "provider_settings_inputs", raise_error) + monkeypatch.setattr(provider_settings_module, "provider_settings_inputs", raise_error) def set_tracked_variable(project_dir: Path, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/test_native_json_report.py b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/test_native_json_report.py index b5df07235e..529e6cd6f0 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/test_native_json_report.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/compile_outputs/test_native_json_report.py @@ -9,7 +9,6 @@ import pytest import sqlbuild._native as native_module -from sqlbuild.cli.commands._helpers.compile import output as output_module from tests.integration.src.sqlbuild.cli.commands.main.compile_outputs._test_types import ( CliJsonReportTestCase, JsonReportErrorTestCase, @@ -183,8 +182,6 @@ def test_given_lone_surrogate_in_full_report_when_compiling_json_then_native_wri assert test_case.expected_fragment in text assert text.isascii() assert text == json.dumps(json.loads(text), indent=2) + "\n" - assert not hasattr(output_module, "json") - assert not hasattr(output_module, "orjson") @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index f2774c5e89..98a778b8d3 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -14,6 +14,7 @@ import pytest import sqlbuild._native as native_module +from sqlbuild._native import PlannedStaticSqlTests, PreparedCompileArtifacts from sqlbuild.adapter.contract.exceptions import AdapterUserError from sqlbuild.adapter.contract.models import ( RenderedRetentionChange, @@ -22,16 +23,15 @@ ) from sqlbuild.adapter.contract.types import RetentionChangePhase from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.commands._helpers.compile import target_writer -from sqlbuild.cli.commands.classes import prepared_compile_artifacts -from sqlbuild.cli.commands.classes.prepared_compile_artifacts import PreparedCompileArtifacts from sqlbuild.cli.commands.main.entrypoint.entry import main -from sqlbuild.cli.compile.models import PlannedStaticSqlTests from sqlbuild.compiler.analysis_session.models import NativeQueryColumns from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_SUFFIX from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.planner.exceptions import NativeSqlTestPlanningError from sqlbuild.compiler.planner.models import NativeSqlTestArtifact +from sqlbuild.compiler.sql_test_glue.main import ( + plan_native_sql_test_artifacts as plan_artifacts_module, +) from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( RepeatedJsonParseTestCase, ) @@ -170,26 +170,62 @@ def fail_native_planner(monkeypatch: pytest.MonkeyPatch) -> None: def fail(**kwargs: object) -> tuple[NativeSqlTestArtifact, ...]: raise NativeSqlTestPlanningError("native SQL-test planning failed: worker stopped") - monkeypatch.setattr(target_writer, "plan_native_sql_test_artifacts", fail) + monkeypatch.setattr(plan_artifacts_module, "plan_native_sql_test_artifacts", fail) def record_test_planning_threads(monkeypatch: pytest.MonkeyPatch) -> list[str]: """Record the thread-name prefix of every background SQL-test planning task.""" threads: list[str] = [] - plan: Callable[..., PlannedStaticSqlTests] = prepared_compile_artifacts.plan_static_sql_tests + plan: Callable[..., PlannedStaticSqlTests] = native_module.plan_static_sql_tests def recorded_plan(**kwargs: Any) -> PlannedStaticSqlTests: threads.append(threading.current_thread().name.rpartition("_")[0]) return plan(**kwargs) - monkeypatch.setattr(prepared_compile_artifacts, "plan_static_sql_tests", recorded_plan) + monkeypatch.setattr(native_module, "plan_static_sql_tests", recorded_plan) return threads -def skip_artifact_preparation(self: PreparedCompileArtifacts, **kwargs: object) -> None: +def use_prepared_artifacts( + monkeypatch: pytest.MonkeyPatch, *, model_range: tuple[int, int] +) -> None: + """Compile with background preparation staging projects of this model range.""" + + class Prepared(PreparedCompileArtifacts): + min_prepared_models: int = model_range[0] + max_prepared_models: int = model_range[1] + + monkeypatch.setattr(native_module, "PreparedCompileArtifacts", Prepared) + + +def use_staging_directory_factory( + monkeypatch: pytest.MonkeyPatch, *, create_staging: Callable[[Path], None] +) -> None: + """Compile with background staging that creates its directory through create_staging.""" + + class Prepared(PreparedCompileArtifacts): + def create_staging_directory(self, staging_dir: Path) -> None: + create_staging(staging_dir) + + monkeypatch.setattr(native_module, "PreparedCompileArtifacts", Prepared) + + +def skip_artifact_preparation(monkeypatch: pytest.MonkeyPatch) -> None: """Disable background preparation so the write phase plans SQL tests inline.""" + class Prepared(PreparedCompileArtifacts): + def start(self, **kwargs: Any) -> None: + """Prepare nothing.""" + + monkeypatch.setattr(native_module, "PreparedCompileArtifacts", Prepared) + + +def create_owned_staging_directory(staging_dir: Path) -> None: + """Create a staging directory the way a compile does.""" + + PreparedCompileArtifacts(enabled=True).create_staging_directory(staging_dir) + def has_second_filesystem(root: Path) -> bool: """Return whether root is a writable directory on another device than the temp directory.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py index 8d98c44dd4..f39fe91b8e 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py @@ -11,8 +11,6 @@ import pytest from filelock import FileLock -from sqlbuild.cli.commands.classes import prepared_compile_artifacts -from sqlbuild.cli.commands.classes.prepared_compile_artifacts import PreparedCompileArtifacts from sqlbuild.cli.commands.main.entrypoint.entry import main from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( AbandonedStagingCompileTestCase, @@ -27,6 +25,7 @@ compile_project_json, compile_then_edit_one_test, compiled_files, + create_owned_staging_directory, fail_native_planner, has_second_filesystem, keep_native_planner, @@ -35,6 +34,8 @@ skip_artifact_preparation, staging_directories, unavailable_staging_directory, + use_prepared_artifacts, + use_staging_directory_factory, write_prepared_artifacts_project, write_staging_directory, write_test_planning_project, @@ -84,10 +85,7 @@ def test_given_compile_mode_when_preparing_artifacts_in_background_then_output_m capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(prepared_compile_artifacts, "_MIN_PREPARED_ARTIFACT_MODELS", 1) - monkeypatch.setattr( - prepared_compile_artifacts, "_MAX_PREPARED_ARTIFACT_MODELS", test_case.max_prepared_models - ) + use_prepared_artifacts(monkeypatch, model_range=(1, test_case.max_prepared_models)) background: Path = tmp_path / "background" write_test_planning_project( project_dir=background, @@ -103,7 +101,7 @@ def test_given_compile_mode_when_preparing_artifacts_in_background_then_output_m project_dir=background, compile_args=test_case.compile_args, capsys=capsys ) with monkeypatch.context() as inline_patch: - inline_patch.setattr(PreparedCompileArtifacts, "start", skip_artifact_preparation) + skip_artifact_preparation(inline_patch) inline_result: tuple[int, dict[str, object]] = compile_project_json( project_dir=inline, compile_args=test_case.compile_args, capsys=capsys ) @@ -160,7 +158,7 @@ def test_given_background_planning_fails_when_compiling_then_error_matches_inlin project_dir=background, compile_args=test_case.compile_args, capsys=capsys ) with monkeypatch.context() as inline_patch: - inline_patch.setattr(PreparedCompileArtifacts, "start", skip_artifact_preparation) + skip_artifact_preparation(inline_patch) inline_result: tuple[int, dict[str, object]] = compile_project_json( project_dir=inline, compile_args=test_case.compile_args, capsys=capsys ) @@ -178,7 +176,7 @@ def test_given_background_planning_fails_when_compiling_then_error_matches_inlin "test_case", ( PreparedArtifactsCompileTestCase( - "cold staged artifacts", 128, prepared_compile_artifacts._create_staging_directory + "cold staged artifacts", 128, create_owned_staging_directory ), PreparedArtifactsCompileTestCase( "unavailable staging storage", 128, unavailable_staging_directory @@ -192,9 +190,7 @@ def test_given_existing_artifacts_when_rules_fail_then_staged_changes_are_not_pu capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr( - prepared_compile_artifacts, "_create_staging_directory", test_case.staging_directory_factory - ) + use_staging_directory_factory(monkeypatch, create_staging=test_case.staging_directory_factory) config_path: Path = tmp_path / "sqlbuild_project.toml" config: str = 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = ["SQBRSQL021"]\n' config_path.write_text(config, encoding="utf-8") diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py index bd04eddc48..c170be2c02 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -66,7 +66,6 @@ "polyglot_sql", "pydantic", "sqlbuild.adapter.contract.classes.base_adapter", - "sqlbuild.cli.commands._helpers.compile.output", "sqlbuild.cli.compile.models", "sqlbuild.compiler.compile.models", "sqlbuild.compiler.pipeline", diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index 315866eb17..047e73d94f 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -7,10 +7,14 @@ import pytest -from sqlbuild._native import build_compiled_project, build_project_graph, discover_project_inputs +from sqlbuild._native import ( + build_compiled_project, + build_project_graph, + discover_project_inputs, + write_compile_target, +) from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter -from sqlbuild.cli.commands._helpers.compile.target_writer import write_compile_target from sqlbuild.compiler.compile.models import InferredColumn from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py index 2512e7ec3c..4999e39e4e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -169,7 +169,7 @@ class RuleGatedTestPlanningTestCase: @dataclass(frozen=True) -class CrossDeviceStagedPublishTestCase: +class StagedPublishTestCase: description: str existing_files: dict[str, str] expected_files: dict[str, str] diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py index c12849f906..c48c7dfd15 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -1,17 +1,17 @@ from __future__ import annotations -import errno -import os from collections.abc import Callable from pathlib import Path from typing import Any from sqlbuild._native import ( + PreparedCompileArtifacts, project_with, ) from sqlbuild.adapter.contract.models import LifeCycleEvent, QueryResult from sqlbuild.adapter.contract.types import LifeCycleEventKind from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.cli.output.models import WrittenTarget from sqlbuild.compiler.auditing.types import ( AuditAttachmentKind, AuditRunScope, @@ -582,24 +582,24 @@ def write_relative_files(*, root: Path, files: dict[str, str]) -> None: path.write_text(contents, encoding="utf-8") -def cross_device_replace( - *, staging_root: Path, real_replace: Callable[[Any, Any], None] -) -> Callable[[Any, Any], None]: - """Return an os.replace that rejects moves out of staging_root like a second filesystem.""" +class StagingEveryProject(PreparedCompileArtifacts): + """Stage artifacts for a project of any size.""" - staging_prefix: str = os.fspath(staging_root) - replacements: dict[bool, Callable[[Any, Any], None]] = { - True: cross_device_rename, - False: real_replace, - } - - def replace_file(source: Any, destination: Any) -> None: - replacements[os.fspath(source).startswith(staging_prefix)](source, destination) + min_prepared_models: int = 1 - return replace_file +def staged_then_published( + *, target: Path, before_publish: Callable[[Path], None] = lambda _: None +) -> WrittenTarget | None: + """Stage the project's artifacts beside target, run before_publish, then publish them.""" -def cross_device_rename(source: Any, destination: Any) -> None: - """Reject a move as if source and destination were on different filesystems.""" - - raise OSError(errno.EXDEV, "Invalid cross-device link") + with StagingEveryProject(enabled=True) as artifacts: + artifacts.start( + project=build_static_target_writer_project(), + adapter=DuckDbAdapter(), + target_dir=target, + stage_artifacts=True, + ) + _ = artifacts.planning_diagnostics() + before_publish(target) + return artifacts.publish(target_dir=target, manifest=None) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 16f9d7d814..03dc9c925c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -3,23 +3,22 @@ import gc import json import sys +import threading from collections.abc import Callable from pathlib import Path -from typing import Any, cast +from typing import Any import pytest import sqlbuild._native as discovery_entry -from sqlbuild.cli.commands._helpers.compile import lineage as compile_lineage -from sqlbuild.cli.commands._helpers.compile import target_writer from sqlbuild.cli.commands._helpers.runtime import adapters as runtime_adapters -from sqlbuild.cli.commands.classes import prepared_compile_artifacts from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.compile.main.run import run_compile from sqlbuild.cli.entry.models import CompileCommandRequest -from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.compiler.pipeline.models import ProjectGraph +from sqlbuild.compiler.sql_test_glue.main import ( + plan_native_sql_test_artifacts as plan_artifacts_module, +) from sqlbuild.presentation.classes import transient_status_reporter from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileColumnContractModeTestCase, @@ -512,39 +511,42 @@ def test_given_explicit_column_contract_mode_when_compiling_column_audit_then_au lineage_mode=CompileLineageMode.RICH, expected_lineage_mode_values=(ColumnLineageMode.RICH.value,), ), + CompileLineageModeTestCase( + description="skips lineage when disabled", + lineage_mode=CompileLineageMode.NONE, + expected_lineage_mode_values=(), + ), ), ids=lambda case: case.description, ) -def test_given_lineage_mode_when_resolving_compile_analysis_then_returns_expected_mode( +def test_given_lineage_mode_when_compiling_then_lineage_runs_in_that_mode( test_case: CompileLineageModeTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], ) -> None: - mode: ColumnLineageMode = compile_lineage.compile_analysis_lineage_mode(test_case.lineage_mode) + project_dir: Path = prepare_static_compile_project(tmp_path) + modes: list[str] = [] + build_lineage: Callable[..., object] = discovery_entry.build_project_column_lineage - assert (mode.value,) == test_case.expected_lineage_mode_values + def recorded_lineage(*args: Any) -> object: + modes.append(ColumnLineageMode(args[2]).value) + return build_lineage(*args) + monkeypatch.setattr(discovery_entry, "build_project_column_lineage", recorded_lineage) -@pytest.mark.parametrize( - "test_case", - [ - CompileLineageModeTestCase( - description="skips lineage when disabled", - lineage_mode=CompileLineageMode.NONE, - expected_lineage_mode_values=(), + exit_code: int = run_compile( + CompileCommandRequest( + project_dir=project_dir, + json_output=True, + no_cache=True, + lineage_mode=test_case.lineage_mode, ) - ], - ids=lambda case: case.description, -) -def test_given_lineage_disabled_when_building_compile_lineage_then_skips_analyzer( - test_case: CompileLineageModeTestCase, -) -> None: - lineage: object = compile_lineage.build_compile_lineage( - graph=cast(ProjectGraph, object()), - dialect=None, - mode=test_case.lineage_mode, ) + _ = capsys.readouterr() - assert lineage is None - assert test_case.expected_lineage_mode_values == () + assert exit_code == 0 + assert tuple(modes) == test_case.expected_lineage_mode_values @pytest.mark.parametrize( @@ -594,26 +596,23 @@ def test_given_rule_error_when_compiling_json_then_withheld_test_planning_errors project_dir: Path = prepare_rule_gated_compile_project( tmp_path, test_header=test_case.test_header ) - planning_calls: list[Path] = [] - static_planning: Callable[..., tuple[CompilerDiagnostic, ...]] = ( - target_writer.static_sql_test_planning_diagnostics - ) + planning_on_main_thread: list[bool] = [] + plan_artifacts: Callable[..., object] = plan_artifacts_module.plan_native_sql_test_artifacts - def counted_static_planning(**kwargs: Any) -> tuple[CompilerDiagnostic, ...]: - planning_calls.append(kwargs["target_dir"]) - return static_planning(**kwargs) + def counted_inline_planning(**kwargs: Any) -> object: + planning_on_main_thread.append(threading.current_thread() is threading.main_thread()) + return plan_artifacts(**kwargs) + + class Prepared(discovery_entry.PreparedCompileArtifacts): + min_prepared_models: int = test_case.min_prepared_models monkeypatch.setattr( runtime_adapters, "resolve_adapter", lambda *args, **kwargs: NoConnectDuckDbAdapter() ) monkeypatch.setattr( - target_writer, "static_sql_test_planning_diagnostics", counted_static_planning - ) - monkeypatch.setattr( - prepared_compile_artifacts, - "_MIN_PREPARED_ARTIFACT_MODELS", - test_case.min_prepared_models, + plan_artifacts_module, "plan_native_sql_test_artifacts", counted_inline_planning ) + monkeypatch.setattr(discovery_entry, "PreparedCompileArtifacts", Prepared) exit_code: int = run_compile( CompileCommandRequest(project_dir=project_dir, json_output=True, no_cache=True) @@ -626,7 +625,7 @@ def counted_static_planning(**kwargs: Any) -> tuple[CompilerDiagnostic, ...]: ) assert test_case.expected_message_fragment in payload["diagnostics"][-1]["message"] assert payload["summary"]["errors"] == len(test_case.expected_codes) - assert len(planning_calls) == test_case.expected_inline_planning_calls + assert planning_on_main_thread.count(True) == test_case.expected_inline_planning_calls assert not (project_dir / "target" / "compiled" / "tests").exists() diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py index 6751d0e54a..81860c21b6 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_output.py @@ -9,11 +9,9 @@ from sqlbuild._native import ( ProjectColumnLineage, compiled_model_record, - project_with, -) -from sqlbuild.cli.commands._helpers.compile.output import ( format_compile_json, format_compile_text, + project_with, ) from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.output.models import ( diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py index 0911791559..1a7db737c3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_target_writer.py @@ -11,13 +11,10 @@ from sqlbuild._native import ( compiled_model_record, project_with, -) -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.commands._helpers.compile import target_writer as target_writer_module -from sqlbuild.cli.commands._helpers.compile.target_writer import ( write_compile_target, write_static_compile_target, ) +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.cli.output.models import ( WrittenTarget, ) @@ -29,7 +26,11 @@ DiagnosticPhase, ) from sqlbuild.compiler.manifest.main.build import build_manifest +from sqlbuild.compiler.planner.main.execution import sql_test_model_chain as model_chain_module from sqlbuild.compiler.planner.models import ChainStep, PlanOutput, SqlTestPlanEntry +from sqlbuild.compiler.sql_test_glue.main import ( + plan_native_sql_test_artifacts as plan_artifacts_module, +) from sqlbuild.executor.testing.main.comparison_sql import build_sql_test_comparison_sql from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( ExpectedMessageTestCase, @@ -353,8 +354,8 @@ def test_given_unchanged_test_artifact_when_writing_again_then_skips_test_plan_r ) artifact_path: Path = next((target_dir / "compiled" / "tests").rglob("*.sql")) original_mtime_ns: int = artifact_path.stat().st_mtime_ns - planner_spy: Mock = Mock(wraps=target_writer_module.plan_native_sql_test_artifacts) - monkeypatch.setattr(target_writer_module, "plan_native_sql_test_artifacts", planner_spy) + planner_spy: Mock = Mock(wraps=plan_artifacts_module.plan_native_sql_test_artifacts) + monkeypatch.setattr(plan_artifacts_module, "plan_native_sql_test_artifacts", planner_spy) _ = write_static_compile_target( target_dir=target_dir, @@ -390,8 +391,8 @@ def test_given_artifact_with_moved_mtime_and_same_bytes_when_writing_then_reuses artifact_path: Path = next((target_dir / "compiled" / "tests").rglob("*.sql")) moved_mtime_ns: int = artifact_path.stat().st_mtime_ns + 1_000_000_000 os.utime(artifact_path, ns=(moved_mtime_ns, moved_mtime_ns)) - planner_spy: Mock = Mock(wraps=target_writer_module.plan_native_sql_test_artifacts) - monkeypatch.setattr(target_writer_module, "plan_native_sql_test_artifacts", planner_spy) + planner_spy: Mock = Mock(wraps=plan_artifacts_module.plan_native_sql_test_artifacts) + monkeypatch.setattr(plan_artifacts_module, "plan_native_sql_test_artifacts", planner_spy) _ = write_static_compile_target( target_dir=target_dir, @@ -433,8 +434,8 @@ def test_given_test_with_planning_error_when_writing_twice_then_error_is_not_cac adapter=DuckDbAdapter(), project=project, ) - planner_spy: Mock = Mock(wraps=target_writer_module.plan_native_sql_test_artifacts) - monkeypatch.setattr(target_writer_module, "plan_native_sql_test_artifacts", planner_spy) + planner_spy: Mock = Mock(wraps=plan_artifacts_module.plan_native_sql_test_artifacts) + monkeypatch.setattr(plan_artifacts_module, "plan_native_sql_test_artifacts", planner_spy) second: WrittenTarget = write_static_compile_target( target_dir=target_dir, @@ -474,8 +475,8 @@ def test_given_changed_model_in_test_closure_when_writing_then_rebuilds_test_art adapter=DuckDbAdapter(), project=project, ) - planner_spy: Mock = Mock(wraps=target_writer_module.plan_native_sql_test_artifacts) - monkeypatch.setattr(target_writer_module, "plan_native_sql_test_artifacts", planner_spy) + planner_spy: Mock = Mock(wraps=plan_artifacts_module.plan_native_sql_test_artifacts) + monkeypatch.setattr(plan_artifacts_module, "plan_native_sql_test_artifacts", planner_spy) changed_project: CompiledProjectView = project_with( project=project, models=( @@ -520,8 +521,8 @@ def test_given_changed_unrelated_model_when_writing_then_reuses_test_artifact( project=project, models=(*project.models, unrelated_model), ) - planner_spy: Mock = Mock(wraps=target_writer_module.plan_native_sql_test_artifacts) - monkeypatch.setattr(target_writer_module, "plan_native_sql_test_artifacts", planner_spy) + planner_spy: Mock = Mock(wraps=plan_artifacts_module.plan_native_sql_test_artifacts) + monkeypatch.setattr(plan_artifacts_module, "plan_native_sql_test_artifacts", planner_spy) _ = write_static_compile_target( target_dir=target_dir, @@ -548,8 +549,8 @@ def test_given_changed_test_sql_when_writing_then_rebuilds_test_artifact( adapter=DuckDbAdapter(), project=project, ) - planner_spy: Mock = Mock(wraps=target_writer_module.plan_native_sql_test_artifacts) - monkeypatch.setattr(target_writer_module, "plan_native_sql_test_artifacts", planner_spy) + planner_spy: Mock = Mock(wraps=plan_artifacts_module.plan_native_sql_test_artifacts) + monkeypatch.setattr(plan_artifacts_module, "plan_native_sql_test_artifacts", planner_spy) changed_test: CompiledSqlTest = replace( project.sql_tests[0], sql_body=project.sql_tests[0].sql_body + "\n-- edit" ) @@ -584,8 +585,8 @@ def test_given_changed_compile_target_when_writing_then_rebuilds_test_artifact( adapter=DuckDbAdapter(), project=project, ) - planner_spy: Mock = Mock(wraps=target_writer_module.plan_native_sql_test_artifacts) - monkeypatch.setattr(target_writer_module, "plan_native_sql_test_artifacts", planner_spy) + planner_spy: Mock = Mock(wraps=plan_artifacts_module.plan_native_sql_test_artifacts) + monkeypatch.setattr(plan_artifacts_module, "plan_native_sql_test_artifacts", planner_spy) _ = write_static_compile_target( target_dir=target_dir, @@ -611,11 +612,11 @@ def test_given_compile_cache_disabled_when_writing_twice_then_rebuilds_test_arti project=build_cached_target_writer_project(target_dir=target_dir), compile_cache_dir=None, ) - planner_spy: Mock = Mock(wraps=target_writer_module.plan_native_sql_test_artifacts) - monkeypatch.setattr(target_writer_module, "plan_native_sql_test_artifacts", planner_spy) - closure_index_spy: Mock = Mock(wraps=target_writer_module.sql_test_model_chain_names_by_key) + planner_spy: Mock = Mock(wraps=plan_artifacts_module.plan_native_sql_test_artifacts) + monkeypatch.setattr(plan_artifacts_module, "plan_native_sql_test_artifacts", planner_spy) + closure_index_spy: Mock = Mock(wraps=model_chain_module.sql_test_model_chain_names_by_key) monkeypatch.setattr( - target_writer_module, + model_chain_module, "sql_test_model_chain_names_by_key", closure_index_spy, ) @@ -652,8 +653,8 @@ def test_given_modified_test_artifact_when_writing_then_rebuilds_expected_sql( artifact_path: Path = next((target_dir / "compiled" / "tests").rglob("*.sql")) expected_sql: str = artifact_path.read_text(encoding="utf-8") artifact_path.write_text("SELECT 'tampered'\n", encoding="utf-8") - planner_spy: Mock = Mock(wraps=target_writer_module.plan_native_sql_test_artifacts) - monkeypatch.setattr(target_writer_module, "plan_native_sql_test_artifacts", planner_spy) + planner_spy: Mock = Mock(wraps=plan_artifacts_module.plan_native_sql_test_artifacts) + monkeypatch.setattr(plan_artifacts_module, "plan_native_sql_test_artifacts", planner_spy) _ = write_static_compile_target( target_dir=target_dir, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_staged_artifact_publication.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_staged_artifact_publication.py index 1cc116e81b..ced212f287 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_staged_artifact_publication.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_staged_artifact_publication.py @@ -1,4 +1,4 @@ -"""Staged cold-compile artifacts publish safely across filesystems and concurrent cleanup.""" +"""Staged cold-compile artifacts publish safely beside existing trees and concurrent cleanup.""" from __future__ import annotations @@ -11,26 +11,20 @@ import pytest from filelock import FileLock, Timeout +from sqlbuild._native import PreparedCompileArtifacts from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.commands._helpers.compile.target_writer import ( - publish_static_compile_target, - staged_artifact_files, - write_static_compile_target, -) -from sqlbuild.cli.commands.classes import prepared_compile_artifacts -from sqlbuild.cli.commands.classes.prepared_compile_artifacts import PreparedCompileArtifacts from sqlbuild.cli.commands.exceptions import StagedArtifactsChangedError from sqlbuild.cli.output.models import WrittenTarget from tests.unit.src.sqlbuild.cli.commands.main.compile._test_types import ( - CrossDeviceStagedPublishTestCase, + StagedPublishTestCase, StagingInterleavingTestCase, TamperedStagedPublishTestCase, ) from tests.unit.src.sqlbuild.cli.commands.main.compile.helpers import ( + StagingEveryProject, build_static_target_writer_project, - cross_device_rename, - cross_device_replace, read_target_files, + staged_then_published, write_relative_files, ) @@ -49,14 +43,14 @@ @pytest.mark.parametrize( "test_case", [ - CrossDeviceStagedPublishTestCase( - description="new files join an existing tree on another filesystem", + StagedPublishTestCase( + description="new files join an existing tree and replace its stale files", existing_files={"compiled/models/obsolete.sql": "SELECT 0\n"}, expected_files=_PUBLISHED_FILES, expected_removed=("compiled/models/obsolete.sql",), ), - CrossDeviceStagedPublishTestCase( - description="a cold tree that cannot be moved is copied file by file", + StagedPublishTestCase( + description="a cold tree is published whole", existing_files={}, expected_files=_PUBLISHED_FILES, expected_removed=(), @@ -64,31 +58,20 @@ ], ids=lambda case: case.description, ) -def test_given_cross_device_target_when_publishing_staged_artifacts_then_files_are_copied( - test_case: CrossDeviceStagedPublishTestCase, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, +def test_given_staged_artifacts_when_publishing_then_target_holds_exactly_the_staged_files( + test_case: StagedPublishTestCase, tmp_path: Path ) -> None: - staging: Path = tmp_path / "staging" target: Path = tmp_path / "target" - prepared: WrittenTarget = write_static_compile_target( - target_dir=staging, adapter=DuckDbAdapter(), project=build_static_target_writer_project() - ) - expected_names: frozenset[str] = staged_artifact_files(target_dir=staging) write_relative_files(root=target, files=test_case.existing_files) - monkeypatch.setattr( - os, "replace", cross_device_replace(staging_root=staging, real_replace=os.replace) - ) - monkeypatch.setattr(os, "rename", cross_device_rename) - written: WrittenTarget = publish_static_compile_target( - prepared=prepared, target_dir=target, manifest=None, expected_files=expected_names - ) + written: WrittenTarget | None = staged_then_published(target=target) + assert written is not None assert written.target_dir == target assert read_target_files(target, test_case.expected_files) == test_case.expected_files assert not any((target / relative).exists() for relative in test_case.expected_removed) assert sorted(path.name for path in target.rglob("*.tmp")) == [] + assert sorted(path.name for path in target.glob(".sqlbuild-staging-*")) == [] @pytest.mark.parametrize( @@ -106,20 +89,16 @@ def test_given_cross_device_target_when_publishing_staged_artifacts_then_files_a def test_given_changed_staging_tree_when_publishing_then_nothing_stale_is_deleted( test_case: TamperedStagedPublishTestCase, tmp_path: Path ) -> None: - staging: Path = tmp_path / "staging" target: Path = tmp_path / "target" - prepared: WrittenTarget = write_static_compile_target( - target_dir=staging, adapter=DuckDbAdapter(), project=build_static_target_writer_project() - ) - expected_names: frozenset[str] = staged_artifact_files(target_dir=staging) write_relative_files(root=target, files=test_case.existing_files) - for relative in test_case.removed_staged_files: - (staging / relative).unlink() + + def remove_staged(root: Path) -> None: + for relative in test_case.removed_staged_files: + for staged in root.glob(f".sqlbuild-staging-*/{relative}"): + staged.unlink() with pytest.raises(StagedArtifactsChangedError): - publish_static_compile_target( - prepared=prepared, target_dir=target, manifest=None, expected_files=expected_names - ) + _ = staged_then_published(target=target, before_publish=remove_staged) assert read_target_files(target, test_case.expected_kept_files) == ( test_case.expected_kept_files @@ -149,24 +128,22 @@ def test_given_cleanup_interleaved_with_staging_when_compiling_then_live_staging target: Path = tmp_path / "target" write_relative_files(root=target, files=test_case.existing_files) survived: list[bool] = [] - create_staging: Callable[[Path], None] = prepared_compile_artifacts._create_staging_directory remove_tree: Callable[..., None] = shutil.rmtree - def create_then_clean(staging_dir: Path) -> None: - create_staging(staging_dir) - PreparedCompileArtifacts(enabled=True).remove_abandoned_staging(target_dir=target) - survived.append((staging_dir / ".owner").is_file()) + class CleanedWhileStaging(StagingEveryProject): + def create_staging_directory(self, staging_dir: Path) -> None: + super().create_staging_directory(staging_dir) + PreparedCompileArtifacts(enabled=True).remove_abandoned_staging(target_dir=target) + survived.append((staging_dir / ".owner").is_file()) def remove_while_locked(path: Any, *args: Any, **kwargs: Any) -> None: with pytest.raises(Timeout): FileLock(f"{os.fspath(path)}.lock", timeout=0).acquire() remove_tree(path, *args, **kwargs) - monkeypatch.setattr(prepared_compile_artifacts, "_MIN_PREPARED_ARTIFACT_MODELS", 1) - monkeypatch.setattr(prepared_compile_artifacts, "_create_staging_directory", create_then_clean) monkeypatch.setattr(shutil, "rmtree", remove_while_locked) - with PreparedCompileArtifacts(enabled=True) as artifacts: + with CleanedWhileStaging(enabled=True) as artifacts: artifacts.remove_abandoned_staging(target_dir=target) artifacts.start( project=build_static_target_writer_project(), @@ -196,15 +173,12 @@ def remove_while_locked(path: Any, *args: Any, **kwargs: Any) -> None: ids=lambda case: case.description, ) def test_given_staging_owner_lost_when_publishing_then_compile_fails_without_deleting( - test_case: TamperedStagedPublishTestCase, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, + test_case: TamperedStagedPublishTestCase, tmp_path: Path ) -> None: target: Path = tmp_path / "target" write_relative_files(root=target, files=test_case.existing_files) - monkeypatch.setattr(prepared_compile_artifacts, "_MIN_PREPARED_ARTIFACT_MODELS", 1) - with PreparedCompileArtifacts(enabled=True) as artifacts: + with StagingEveryProject(enabled=True) as artifacts: artifacts.start( project=build_static_target_writer_project(), adapter=DuckDbAdapter(), diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py index c6345f23a3..cd7a18324b 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py @@ -15,7 +15,6 @@ from tests.unit.src.sqlbuild.cli.compile_reuse.main.helpers import compile_in_fresh_process _FULL_COMPILE_MODULES: tuple[str, ...] = ( - "sqlbuild.cli.commands._helpers.compile.output", "sqlbuild.compiler.compile._helpers.native_stages.assembly", ) From 2fea226041c24d38198fb4e5ebe871567fc1ffd7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 06:34:07 +0100 Subject: [PATCH 2/4] chore: update the Python boundary ratchet after the compile command port --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index 1d21fac1af..af057baa57 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 120834 +remaining_lines = 119534 [[stays]] path = "executor" From 720097a6bc63339fe63a9f3d52470e3f82355eae Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 06:53:00 +0100 Subject: [PATCH 3/4] refactor: validate project Python paths and parse header values natively --- crates/sqlbuild-discovery/src/lib.rs | 1 + .../src/python_paths/_helpers/mod.rs | 2 + .../src/python_paths/_helpers/python_files.rs | 44 +++++ .../src/python_paths/_helpers/supported.rs | 53 ++++++ .../src/python_paths/constants.rs | 31 ++++ .../src/python_paths/main/mod.rs | 1 + .../main/unsupported_python_paths.rs | 14 ++ .../src/python_paths/mod.rs | 6 + .../src/python_paths/tests.rs | 4 + .../src/python_paths/tests/test_types.rs | 6 + .../tests/test_unsupported_python_paths.rs | 65 +++++++ .../_helpers/compile_driver/discovery.rs | 7 +- .../_helpers/discovery/header_values.rs | 160 ++++++++++++++++++ .../src/bindings/_helpers/discovery/mod.rs | 2 + .../_helpers/discovery/python_paths.rs | 32 ++++ .../src/bindings/_helpers/discovery/raised.rs | 7 +- .../_helpers/discovery/registration.rs | 5 +- fensu.toml | 5 - src/sqlbuild/_native.pyi | 14 ++ .../_helpers/filesystem/python_paths.py | 116 ------------- .../discovery/_helpers/sql/model_files.py | 126 -------------- .../main/_model_output_column_locations.py | 4 +- src/sqlbuild/lint/_helpers/native.py | 5 +- .../discovery/test_deep_header_values.py | 2 +- .../discovery/_helpers/test_sql_models.py | 8 +- 25 files changed, 453 insertions(+), 267 deletions(-) create mode 100644 crates/sqlbuild-discovery/src/python_paths/_helpers/mod.rs create mode 100644 crates/sqlbuild-discovery/src/python_paths/_helpers/python_files.rs create mode 100644 crates/sqlbuild-discovery/src/python_paths/_helpers/supported.rs create mode 100644 crates/sqlbuild-discovery/src/python_paths/constants.rs create mode 100644 crates/sqlbuild-discovery/src/python_paths/main/mod.rs create mode 100644 crates/sqlbuild-discovery/src/python_paths/main/unsupported_python_paths.rs create mode 100644 crates/sqlbuild-discovery/src/python_paths/mod.rs create mode 100644 crates/sqlbuild-discovery/src/python_paths/tests.rs create mode 100644 crates/sqlbuild-discovery/src/python_paths/tests/test_types.rs create mode 100644 crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/discovery/python_paths.rs delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/filesystem/python_paths.py delete mode 100644 src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py diff --git a/crates/sqlbuild-discovery/src/lib.rs b/crates/sqlbuild-discovery/src/lib.rs index e68026dc37..9226c314a6 100644 --- a/crates/sqlbuild-discovery/src/lib.rs +++ b/crates/sqlbuild-discovery/src/lib.rs @@ -10,6 +10,7 @@ pub mod key_checks; pub mod model_files; pub mod models; pub mod project_validation; +pub mod python_paths; pub mod sql_tests; pub mod tree; pub mod yaml_entries; diff --git a/crates/sqlbuild-discovery/src/python_paths/_helpers/mod.rs b/crates/sqlbuild-discovery/src/python_paths/_helpers/mod.rs new file mode 100644 index 0000000000..49550a2e37 --- /dev/null +++ b/crates/sqlbuild-discovery/src/python_paths/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod python_files; +pub(crate) mod supported; diff --git a/crates/sqlbuild-discovery/src/python_paths/_helpers/python_files.rs b/crates/sqlbuild-discovery/src/python_paths/_helpers/python_files.rs new file mode 100644 index 0000000000..1c700e44e6 --- /dev/null +++ b/crates/sqlbuild-discovery/src/python_paths/_helpers/python_files.rs @@ -0,0 +1,44 @@ +//! Every `.py` file below a project, as relative path parts, skipping ignored directories. + +use std::fs; +use std::path::Path; + +use crate::constants::PYTHON_FILE_SUFFIX; +use crate::python_paths::constants::{HIDDEN_PREFIX, IGNORED_ROOTS, PYTHON_CACHE_DIRECTORY}; + +/// The relative parts of every Python file below `project_dir`, sorted part by part. +pub(crate) fn python_files(project_dir: &Path) -> Vec> { + let mut files: Vec> = Vec::new(); + let mut pending: Vec> = vec![Vec::new()]; + while let Some(parts) = pending.pop() { + let directory = parts + .iter() + .fold(project_dir.to_path_buf(), |path, part| path.join(part)); + let Ok(entries) = fs::read_dir(&directory) else { + continue; + }; + for entry in entries.flatten() { + let name: String = entry.file_name().to_string_lossy().into_owned(); + let followed_is_dir: bool = fs::metadata(entry.path()).is_ok_and(|meta| meta.is_dir()); + let mut child: Vec = parts.clone(); + child.push(name.clone()); + if followed_is_dir { + let is_link: bool = entry.file_type().is_ok_and(|kind| kind.is_symlink()); + if !is_link && walked(&parts, &name) { + pending.push(child); + } + } else if name.ends_with(PYTHON_FILE_SUFFIX) { + files.push(child); + } + } + } + files.sort(); + files +} + +/// Whether the walk enters directory `name` below `parts`. +fn walked(parts: &[String], name: &str) -> bool { + !name.starts_with(HIDDEN_PREFIX) + && name != PYTHON_CACHE_DIRECTORY + && !(parts.is_empty() && IGNORED_ROOTS.contains(&name)) +} diff --git a/crates/sqlbuild-discovery/src/python_paths/_helpers/supported.rs b/crates/sqlbuild-discovery/src/python_paths/_helpers/supported.rs new file mode 100644 index 0000000000..fdcd731dc2 --- /dev/null +++ b/crates/sqlbuild-discovery/src/python_paths/_helpers/supported.rs @@ -0,0 +1,53 @@ +//! Whether one project Python file lives under a documented executable root. + +use crate::constants::{CANONICAL_AUTHORED_ROOTS, SCOPED_DECLARATION_DIRECTORIES}; +use crate::python_paths::constants::{ + DIRECT_PYTHON_ROOTS, GROUPED_PYTHON_HOOK_ROLES, LANGUAGE_PYTHON_ROOTS, SUPPORTED_ROOT_FILES, +}; + +/// Whether the file at relative `parts` is supported project Python. +pub(crate) fn is_supported(parts: &[String]) -> bool { + let Some(first) = parts.first() else { + return false; + }; + if parts.len() == 1 && SUPPORTED_ROOT_FILES.contains(&first.as_str()) { + return true; + } + if DIRECT_PYTHON_ROOTS.contains(&first.as_str()) { + return true; + } + if let [root, language, ..] = parts + && LANGUAGE_PYTHON_ROOTS.contains(&(root.as_str(), language.as_str())) + { + return true; + } + scoped_declaration_path(parts) +} + +/// Whether the file sits in a scoped declaration or grouped hook directory of an authored root. +fn scoped_declaration_path(parts: &[String]) -> bool { + for root in CANONICAL_AUTHORED_ROOTS { + if !starts_with(parts, root) { + continue; + } + let end: usize = parts.len().saturating_sub(1).max(root.len()); + let scoped: &[String] = &parts[root.len()..end]; + for (index, part) in scoped.iter().enumerate() { + if SCOPED_DECLARATION_DIRECTORIES.contains(&part.as_str()) { + return true; + } + for role in GROUPED_PYTHON_HOOK_ROLES { + if starts_with(&scoped[index..], &role) { + return true; + } + } + } + return false; + } + false +} + +/// Whether `parts` begins with every name of `prefix`. +fn starts_with(parts: &[String], prefix: &[&str]) -> bool { + parts.len() >= prefix.len() && parts.iter().zip(prefix).all(|(part, name)| part == name) +} diff --git a/crates/sqlbuild-discovery/src/python_paths/constants.rs b/crates/sqlbuild-discovery/src/python_paths/constants.rs new file mode 100644 index 0000000000..eacdb349f9 --- /dev/null +++ b/crates/sqlbuild-discovery/src/python_paths/constants.rs @@ -0,0 +1,31 @@ +//! The documented roots project Python may live under, and the directories the walk skips. + +/// Top-level directories whose Python files are always supported. +pub(crate) const DIRECT_PYTHON_ROOTS: [&str; 10] = [ + "adapters", + "materializations", + "providers", + "python", + "rules", + "sinks", + "macros", + "enums", + "constants", + "event_exporters", +]; +/// `(root, language)` directory pairs whose Python files are supported. +pub(crate) const LANGUAGE_PYTHON_ROOTS: [(&str, &str); 2] = + [("functions", "python"), ("hooks", "python")]; +/// Top-level directories the walk does not enter. +pub(crate) const IGNORED_ROOTS: [&str; 4] = ["logs", "target", "tests", "venv"]; +/// Root-level Python files that are supported. +pub(crate) const SUPPORTED_ROOT_FILES: [&str; 1] = ["adapter.py"]; +/// Directories the walk never enters. +pub(crate) const PYTHON_CACHE_DIRECTORY: &str = "__pycache__"; +/// Grouped hook roles whose Python files are supported inside an authored tree. +pub(crate) const GROUPED_PYTHON_HOOK_ROLES: [[&str; 3]; 2] = [ + ["_sqlbuild", "hooks", "python"], + ["_sqlbuild", "_hooks", "python"], +]; +/// The prefix of a hidden directory the walk skips. +pub(crate) const HIDDEN_PREFIX: &str = "."; diff --git a/crates/sqlbuild-discovery/src/python_paths/main/mod.rs b/crates/sqlbuild-discovery/src/python_paths/main/mod.rs new file mode 100644 index 0000000000..0767e274b7 --- /dev/null +++ b/crates/sqlbuild-discovery/src/python_paths/main/mod.rs @@ -0,0 +1 @@ +pub mod unsupported_python_paths; diff --git a/crates/sqlbuild-discovery/src/python_paths/main/unsupported_python_paths.rs b/crates/sqlbuild-discovery/src/python_paths/main/unsupported_python_paths.rs new file mode 100644 index 0000000000..3375292810 --- /dev/null +++ b/crates/sqlbuild-discovery/src/python_paths/main/unsupported_python_paths.rs @@ -0,0 +1,14 @@ +//! Project Python files outside every documented executable root. + +use std::path::Path; + +use crate::python_paths::_helpers::python_files::python_files; +use crate::python_paths::_helpers::supported::is_supported; + +/// Each unsupported Python file below `project_dir` as relative parts, sorted part by part. +pub fn unsupported_python_paths(project_dir: &Path) -> Vec> { + python_files(project_dir) + .into_iter() + .filter(|parts| !is_supported(parts)) + .collect() +} diff --git a/crates/sqlbuild-discovery/src/python_paths/mod.rs b/crates/sqlbuild-discovery/src/python_paths/mod.rs new file mode 100644 index 0000000000..37f9081f4f --- /dev/null +++ b/crates/sqlbuild-discovery/src/python_paths/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod main; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-discovery/src/python_paths/tests.rs b/crates/sqlbuild-discovery/src/python_paths/tests.rs new file mode 100644 index 0000000000..66a9fa4139 --- /dev/null +++ b/crates/sqlbuild-discovery/src/python_paths/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_unsupported_python_paths.rs"] +mod unsupported_python_paths; diff --git a/crates/sqlbuild-discovery/src/python_paths/tests/test_types.rs b/crates/sqlbuild-discovery/src/python_paths/tests/test_types.rs new file mode 100644 index 0000000000..65946a359b --- /dev/null +++ b/crates/sqlbuild-discovery/src/python_paths/tests/test_types.rs @@ -0,0 +1,6 @@ +/// One project layout and the Python files outside documented roots. +pub(super) struct PythonPathsTestCase { + pub(super) description: &'static str, + pub(super) files: &'static [&'static str], + pub(super) expected_unsupported: &'static [&'static str], +} diff --git a/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs b/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs new file mode 100644 index 0000000000..64ecd45aa6 --- /dev/null +++ b/crates/sqlbuild-discovery/src/python_paths/tests/test_unsupported_python_paths.rs @@ -0,0 +1,65 @@ +use std::fs; + +use crate::python_paths::main::unsupported_python_paths::unsupported_python_paths; +use crate::python_paths::tests::test_types::PythonPathsTestCase; + +#[test] +fn given_project_python_when_listing_unsupported_paths_then_only_undocumented_roots_remain() { + let test_cases = [ + PythonPathsTestCase { + description: "documented roots", + files: &[ + "adapter.py", + "python/orders.py", + "functions/python/f.py", + "macros/m.py", + "models/_macros/m.py", + "models/a/_sqlbuild/hooks/python/h.py", + "tests/unit/x.py", + ".hidden/x.py", + "target/x.py", + ], + expected_unsupported: &[], + }, + PythonPathsTestCase { + description: "undocumented roots sorted by parts", + files: &[ + "a-b.py", + "a/b.py", + "models/x.py", + "functions/sql/f.py", + "helpers.py", + ], + expected_unsupported: &[ + "a/b.py", + "a-b.py", + "functions/sql/f.py", + "helpers.py", + "models/x.py", + ], + }, + ]; + for test_case in test_cases { + let directory = std::env::temp_dir().join(format!( + "sqlbuild-python-paths-{}-{}", + std::process::id(), + test_case.description.replace(' ', "-") + )); + let _ = fs::remove_dir_all(&directory); + for file in test_case.files { + let path = directory.join(file); + let _ = fs::create_dir_all(path.parent().unwrap_or(&directory)); + let _ = fs::write(&path, ""); + } + let unsupported: Vec = unsupported_python_paths(&directory) + .into_iter() + .map(|parts| parts.join("/")) + .collect(); + let _ = fs::remove_dir_all(&directory); + assert_eq!( + unsupported, test_case.expected_unsupported, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs index 82c8c76ae2..ef760e90cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/discovery.rs @@ -15,6 +15,7 @@ use crate::bindings::_helpers::discovery::declaration_records::{ use crate::bindings::_helpers::discovery::model_records::{ ModelFileSelection, discovered_model_files, }; +use crate::bindings::_helpers::discovery::python_paths::reject_unsupported_python_paths; use crate::bindings::_helpers::discovery::sql_test_records::{ discover_scenario_files, discover_test_files, }; @@ -28,7 +29,6 @@ const LIFECYCLE_MODULE: &str = "sqlbuild.runtime.observability.classes.operation const PROJECT_YAML_MODULE: &str = "sqlbuild.compiler.discovery._helpers.yml.project"; const TARGET_CONNECTIONS_MODULE: &str = "sqlbuild.compiler.discovery._helpers.validation.target_connections"; -const PYTHON_PATHS_MODULE: &str = "sqlbuild.compiler.discovery._helpers.filesystem.python_paths"; const VALIDATION_MODULE: &str = "sqlbuild.compiler.discovery._helpers.validation.discovery"; const DISCOVERY_CONSTANTS_MODULE: &str = "sqlbuild.compiler.discovery.constants"; const EVENT_EXPORTERS_MODULE: &str = @@ -240,10 +240,7 @@ fn assembled_inputs<'py>( ] { call_with(&imported(py, module, name)?, &configured)?; } - call_with( - &imported(py, PYTHON_PATHS_MODULE, "validate_project_python_paths")?, - &directory, - )?; + reject_unsupported_python_paths(project_dir)?; let sql_analysis_override: &Bound<'py, PyAny> = request.sql_analysis_enabled_override.bind(py); let sql_analysis_enabled: Bound<'py, PyAny> = if !sql_analysis_override.is_none() { sql_analysis_override.clone() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs new file mode 100644 index 0000000000..8ada475801 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs @@ -0,0 +1,160 @@ +//! MODEL-style header values with a bounded parse cache, and authored column locations. + +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::sync::PyOnceLock; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::boundary::keyword_request::Request; + +const NATIVE_MODULE: &str = "sqlbuild._native"; +const DISCOVERY_ERRORS_MODULE: &str = "sqlbuild.compiler.discovery.exceptions"; +const CONTRACT_MODELS_MODULE: &str = "sqlbuild.spec.contracts.models"; +const HEADER_CACHE_SIZE: usize = 4096; + +/// `(name, line, column, end line, end column)`. +type LocationRow<'py> = ( + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, + Bound<'py, PyAny>, +); + +/// Parsed headers by text: `(values or None, error or None)`. +static HEADER_CACHE: PyOnceLock> = PyOnceLock::new(); + +fn header_cache(py: Python<'_>) -> Bound<'_, PyDict> { + HEADER_CACHE + .get_or_init(py, || PyDict::new(py).unbind()) + .bind(py) + .clone() +} + +/// Parse every header the cache misses in one native batch. +#[pyfunction] +fn prepare_model_header_tokens(headers: &Bound<'_, PyAny>) -> PyResult<()> { + let py: Python<'_> = headers.py(); + let cache: Bound<'_, PyDict> = header_cache(py); + let missing: Bound<'_, PyDict> = PyDict::new(py); + for header in headers.try_iter()? { + let header: Bound<'_, PyAny> = header?; + if !cache.contains(&header)? { + missing.set_item(header, py.None())?; + } + } + if missing.is_empty() { + return Ok(()); + } + if cache.len() + missing.len() > HEADER_CACHE_SIZE { + cache.clear(); + } + let names: Bound<'_, PyList> = missing.keys(); + let parsed: Bound<'_, PyAny> = py + .import(NATIVE_MODULE)? + .getattr("parse_model_headers")? + .call1((&names,))?; + for (header, row) in names.iter().zip(parsed.try_iter()?) { + let row: Bound<'_, PyAny> = row?; + cache.set_item( + header, + PyTuple::new(py, [row.get_item(0)?, row.get_item(2)?])?, + )?; + } + Ok(()) +} + +/// `parse_header_values(*, header, file_path, statement_name, header_line, error_class=None)`. +#[pyfunction] +#[pyo3(signature = (**request))] +fn parse_header_values<'py>(request: Option<&Bound<'py, PyDict>>) -> PyResult> { + let request = Request::new("parse_header_values", request)?; + let py: Python<'py> = request.py(); + let header: Bound<'py, PyAny> = request.get("header")?; + let cache: Bound<'py, PyDict> = header_cache(py); + if !cache.contains(&header)? { + prepare_model_header_tokens(&PyTuple::new(py, [&header])?.into_any())?; + } + let (values, error): (Bound<'py, PyAny>, Bound<'py, PyAny>) = + header_cache(py).as_any().get_item(&header)?.extract()?; + if !values.is_none() { + let copied: Bound<'py, PyDict> = PyDict::new(py); + copied.update(values.downcast()?.as_mapping())?; + return Ok(copied); + } + let header_line: Bound<'py, PyAny> = request.get("header_line")?; + let (detail, help, line): (String, Option, Option) = py + .import(NATIVE_MODULE)? + .getattr("model_header_failure")? + .call1((error.str()?, &header, &header_line))? + .extract()?; + let file_path: String = request.get("file_path")?.str()?.extract()?; + let location: String = match line { + Some(line) => format!("{file_path}:{line}"), + None => file_path, + }; + let statement: String = request.get("statement_name")?.extract()?; + let error_class: Bound<'py, PyAny> = match request.optional("error_class")? { + Some(class) if !class.is_none() => class, + _ => py + .import(DISCOVERY_ERRORS_MODULE)? + .getattr("ModelSqlParseError")?, + }; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("help", help)?; + let raised: Bound<'py, PyAny> = error_class.call( + (format!( + "{statement}(...) in '{location}' contains invalid SQLBuild header syntax: {detail}" + ),), + Some(&keywords), + )?; + Err(pyo3::PyErr::from_value(raised)) +} + +/// `{name: SourceLocation}` from native `(name, line, column, end line, end column)` rows. +fn source_locations<'py>( + rows: &Bound<'py, PyAny>, + relative_path: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = rows.py(); + let location_class: Bound<'py, PyAny> = py + .import(CONTRACT_MODELS_MODULE)? + .getattr("SourceLocation")?; + let locations: Bound<'py, PyDict> = PyDict::new(py); + for row in rows.try_iter()? { + let (name, line, column, end_line, end_column): LocationRow<'py> = row?.extract()?; + let arguments: Bound<'py, PyDict> = PyDict::new(py); + arguments.set_item("path", relative_path)?; + arguments.set_item("line", line)?; + arguments.set_item("column", column)?; + arguments.set_item("end_line", end_line)?; + arguments.set_item("end_column", end_column)?; + locations.set_item(name, location_class.call((), Some(&arguments))?)?; + } + Ok(locations) +} + +/// `model_output_column_locations(*, contents, relative_path, extract_implicit_alias_columns=True)`. +#[pyfunction] +#[pyo3(signature = (**request))] +fn model_output_column_locations<'py>( + request: Option<&Bound<'py, PyDict>>, +) -> PyResult> { + let request = Request::new("model_output_column_locations", request)?; + let py: Python<'py> = request.py(); + let implicit: bool = match request.optional("extract_implicit_alias_columns")? { + Some(value) => value.is_truthy()?, + None => true, + }; + let rows: Bound<'py, PyAny> = py + .import(NATIVE_MODULE)? + .getattr("model_output_columns")? + .call1((request.get("contents")?, implicit))?; + source_locations(&rows, &request.get("relative_path")?) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(prepare_model_header_tokens, module)?)?; + module.add_function(wrap_pyfunction!(parse_header_values, module)?)?; + module.add_function(wrap_pyfunction!(model_output_column_locations, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs index 444fa2041c..91be3deb71 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/mod.rs @@ -3,6 +3,7 @@ pub(crate) mod declaration_records; pub(crate) mod faults; pub(crate) mod globbing; pub(crate) mod header_declarations; +pub(crate) mod header_values; pub(crate) mod key_checks; pub(crate) mod layout_groups; pub(crate) mod locations; @@ -11,6 +12,7 @@ pub(crate) mod model_records; pub(crate) mod project_facts; pub(crate) mod project_files; pub(crate) mod project_validation; +pub(crate) mod python_paths; pub(crate) mod raised; pub(crate) mod registration; pub(crate) mod snapshot; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_paths.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_paths.rs new file mode 100644 index 0000000000..d5aafff79f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/python_paths.rs @@ -0,0 +1,32 @@ +//! Reject project Python outside documented executable roots. + +use std::path::PathBuf; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_discovery::python_paths::main::unsupported_python_paths::unsupported_python_paths; + +use crate::bindings::_helpers::discovery::raised::discovery_error; + +const PYTHON_PATH_HELP: &str = "Move pipeline Python nodes and the helper modules they import under \ +python/, or move SQLBuild extensions under their documented extension path."; + +/// Raise `ProjectPythonPathError` naming every unsupported Python file below `project_dir`. +pub(crate) fn reject_unsupported_python_paths(project_dir: &Bound<'_, PyAny>) -> PyResult<()> { + let root: PathBuf = project_dir.extract()?; + let unsupported: Vec = unsupported_python_paths(&root) + .into_iter() + .map(|parts| parts.join(std::path::MAIN_SEPARATOR_STR)) + .collect(); + if unsupported.is_empty() { + return Ok(()); + } + Err(discovery_error( + project_dir.py(), + "python_path", + format!( + "Unsupported project Python path(s): {}", + unsupported.join(", ") + ), + Some(PYTHON_PATH_HELP.to_owned()), + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs index 06b454c4da..459dd535d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/raised.rs @@ -18,7 +18,7 @@ const OS_MODULE: &str = "os"; const DECODE_ENCODING: &str = "utf-8"; /// The exception class of each native failure kind. -const FAILURE_CLASSES: [(&str, &str, &str); 12] = [ +const FAILURE_CLASSES: [(&str, &str, &str); 13] = [ ("model_sql", DISCOVERY_ERRORS_MODULE, "ModelSqlParseError"), ( "declaration", @@ -47,6 +47,11 @@ const FAILURE_CLASSES: [(&str, &str, &str); 12] = [ "DiscoveryConflictError", ), ("seed", DISCOVERY_ERRORS_MODULE, "SeedDiscoveryError"), + ( + "python_path", + DISCOVERY_ERRORS_MODULE, + "ProjectPythonPathError", + ), ]; fn raised(result: PyResult>) -> PyErr { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs index 248a1c5234..b25bcd7b5f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/registration.rs @@ -3,8 +3,8 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::discovery::{ - declaration_files, declaration_records, globbing, header_declarations, key_checks, - layout_groups, locations, model_records, project_files, project_validation, raised, + declaration_files, declaration_records, globbing, header_declarations, header_values, + key_checks, layout_groups, locations, model_records, project_files, project_validation, raised, sql_test_records, yaml_entries, yaml_records, }; @@ -23,5 +23,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { globbing::register(module)?; project_validation::register(module)?; yaml_entries::register(module)?; + header_values::register(module)?; Ok(()) } diff --git a/fensu.toml b/fensu.toml index 58169a47a0..431ab76d2c 100644 --- a/fensu.toml +++ b/fensu.toml @@ -257,11 +257,6 @@ path = "src/sqlbuild/adapter/contract/classes/observed_connection.py" symbols = ["ObservedConnection.execute", "ObservedConnection.executemany"] reason = "This proxy preserves DuckDB's positional connection execution protocol." -[[targets.python.rule_exceptions]] -rule = "FFL102" -path = "src/sqlbuild/lint/_helpers/native.py" -reason = "The lint layer reuses the compiler's MODEL() header parser; the compiler exposes no public facade for it." - [[targets.python.rule_exceptions]] rule = "FFR601" path = "src/sqlbuild/executor/run/_helpers/materializations/microbatch.py" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index c7720dfa57..9e98664bf8 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2216,3 +2216,17 @@ def _oracle_text_positions( ) -> tuple[str, list[tuple[int, int, int] | None]]: ... def _oracle_yaml_load(text: str) -> str: ... def _oracle_toml_load(text: str) -> str: ... + +# Discovery: header values and authored locations. +def prepare_model_header_tokens(headers: Sequence[str], /) -> None: ... +def parse_header_values( + *, + header: str, + file_path: object, + statement_name: str, + header_line: int | None, + error_class: type[Exception] | None = None, +) -> dict[str, object]: ... +def model_output_column_locations( + *, contents: str, relative_path: object, extract_implicit_alias_columns: bool = True +) -> dict[str, Any]: ... diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/python_paths.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/python_paths.py deleted file mode 100644 index f3b82499f7..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/python_paths.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Validation for the supported SQLBuild project Python layout.""" - -from __future__ import annotations - -import os -from pathlib import Path - -from sqlbuild.compiler.discovery.constants import ( - CANONICAL_AUTHORED_ROOTS, - LANGUAGE_PYTHON_ROOT_PART_COUNT, - PYTHON_CACHE_DIRECTORY_NAME, -) -from sqlbuild.compiler.discovery.exceptions import ProjectPythonPathError -from sqlbuild.compiler.scopes.constants import ( - DECLARATION_GROUP_DIRECTORY, - GLOBAL_DECLARATION_DIRECTORIES, - INHERITED_DECLARATION_DIRECTORIES, - LOCAL_DECLARATION_DIRECTORIES, -) - -_DIRECT_PYTHON_ROOTS: frozenset[str] = frozenset( - { - "adapters", - "materializations", - "providers", - "python", - "rules", - "sinks", - } -) -_LANGUAGE_PYTHON_ROOTS: frozenset[tuple[str, str]] = frozenset( - {("functions", "python"), ("hooks", "python")} -) -_IGNORED_ROOTS: frozenset[str] = frozenset({"logs", "target", "tests", "venv"}) -_LEGACY_DIAGNOSTIC_ROOTS: frozenset[str] = frozenset({"event_exporters"}) -_SUPPORTED_ROOT_FILES: frozenset[Path] = frozenset({Path("adapter.py")}) -_SCOPED_DECLARATION_DIRECTORIES: frozenset[str] = ( - INHERITED_DECLARATION_DIRECTORIES | LOCAL_DECLARATION_DIRECTORIES -) -_GROUPED_PYTHON_HOOK_ROLES: frozenset[tuple[str, str, str]] = frozenset( - { - (DECLARATION_GROUP_DIRECTORY, "hooks", "python"), - (DECLARATION_GROUP_DIRECTORY, "_hooks", "python"), - } -) - - -def validate_project_python_paths(*, project_dir: Path) -> None: - """Reject project Python outside documented executable roots.""" - - unsupported: tuple[Path, ...] = _unsupported_python_paths(project_dir=project_dir) - if not unsupported: - return - rendered_paths: str = ", ".join(str(path) for path in unsupported) - raise ProjectPythonPathError( - f"Unsupported project Python path(s): {rendered_paths}", - help=( - "Move pipeline Python nodes and the helper modules they import under python/, " - "or move SQLBuild extensions under their documented extension path." - ), - ) - - -def _unsupported_python_paths(*, project_dir: Path) -> tuple[Path, ...]: - unsupported: list[Path] = [] - for file_path in _project_python_files(project_dir=project_dir): - if _is_supported(file_path=file_path, project_dir=project_dir): - continue - unsupported.append(file_path.relative_to(project_dir)) - return tuple(unsupported) - - -def _project_python_files(*, project_dir: Path) -> tuple[Path, ...]: - files: list[Path] = [] - for root, directories, filenames in os.walk(project_dir): - root_path: Path = Path(root) - directories[:] = [ - name - for name in directories - if not name.startswith(".") - and name != PYTHON_CACHE_DIRECTORY_NAME - and not (root_path == project_dir and name in _IGNORED_ROOTS) - ] - files.extend(root_path / name for name in filenames if name.endswith(".py")) - return tuple(sorted(files)) - - -def _is_supported(*, file_path: Path, project_dir: Path) -> bool: - relative_path: Path = file_path.relative_to(project_dir) - parts: tuple[str, ...] = relative_path.parts - if relative_path in _SUPPORTED_ROOT_FILES: - return True - if ( - parts[0] in _DIRECT_PYTHON_ROOTS - or parts[0] in GLOBAL_DECLARATION_DIRECTORIES - or parts[0] in _LEGACY_DIAGNOSTIC_ROOTS - ): - return True - if ( - len(parts) >= LANGUAGE_PYTHON_ROOT_PART_COUNT - and parts[:LANGUAGE_PYTHON_ROOT_PART_COUNT] in _LANGUAGE_PYTHON_ROOTS - ): - return True - return _is_scoped_declaration_path(parts) - - -def _is_scoped_declaration_path(parts: tuple[str, ...]) -> bool: - for root_parts in CANONICAL_AUTHORED_ROOTS: - if parts[: len(root_parts)] != root_parts: - continue - scoped_parts: tuple[str, ...] = parts[len(root_parts) : -1] - return any(part in _SCOPED_DECLARATION_DIRECTORIES for part in scoped_parts) or any( - scoped_parts[index : index + 3] in _GROUPED_PYTHON_HOOK_ROLES - for index in range(len(scoped_parts)) - ) - return False diff --git a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py b/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py deleted file mode 100644 index 2348239936..0000000000 --- a/src/sqlbuild/compiler/discovery/_helpers/sql/model_files.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Native MODEL header parsing and authored-location queries for SQL model files.""" - -from __future__ import annotations - -from collections import OrderedDict -from dataclasses import dataclass -from pathlib import Path - -import sqlbuild._native as _native -from sqlbuild.compiler.discovery.exceptions import ( - DiscoveryError, - ModelHeaderSyntaxError, - ModelSqlParseError, -) -from sqlbuild.spec.contracts.models import SourceLocation - -_MODEL_HEADER_TOKEN_CACHE_SIZE: int = 4096 - - -@dataclass(frozen=True) -class _ModelHeaderParse: - values: dict[str, object] | None - error: str | None - - -class _ModelHeaderParseCache: - def __init__(self) -> None: - self._values: OrderedDict[str, _ModelHeaderParse] = OrderedDict() - - def missing(self, headers: list[str]) -> list[str]: - return list(dict.fromkeys(header for header in headers if header not in self._values)) - - def get(self, header: str) -> _ModelHeaderParse | None: - parsed: _ModelHeaderParse | None = self._values.get(header) - if parsed is not None: - self._values.move_to_end(header) - return parsed - - def put(self, *, header: str, parsed: _ModelHeaderParse) -> None: - self._values[header] = parsed - self._values.move_to_end(header) - if len(self._values) > _MODEL_HEADER_TOKEN_CACHE_SIZE: - self._values.popitem(last=False) - - -_MODEL_HEADER_PARSE_CACHE: _ModelHeaderParseCache = _ModelHeaderParseCache() - - -def header_column_locations( - *, contents: str, header: str, header_start: int, relative_path: Path -) -> dict[str, SourceLocation]: - """Return authored column locations from a parsed SQLBuild header.""" - - locations, error = _native.model_header_column_locations(contents, header, header_start) - if locations is None: - raise ModelHeaderSyntaxError(str(error)) - return _source_locations(rows=locations, relative_path=relative_path) - - -def model_output_column_locations( - *, contents: str, relative_path: Path, extract_implicit_alias_columns: bool = True -) -> dict[str, SourceLocation]: - """Return authored locations for simple SELECT output expressions.""" - - return _source_locations( - rows=_native.model_output_columns(contents, extract_implicit_alias_columns), - relative_path=relative_path, - ) - - -def _source_locations( - *, rows: list[tuple[str, int, int, int, int]], relative_path: Path -) -> dict[str, SourceLocation]: - return { - name: SourceLocation( - path=relative_path, line=line, column=column, end_line=end_line, end_column=end_column - ) - for name, line, column, end_line, end_column in rows - } - - -def parse_header_values( - *, - header: str, - file_path: Path, - statement_name: str, - header_line: int | None, - error_class: type[DiscoveryError] = ModelSqlParseError, -) -> dict[str, object]: - """Parse one header; a nesting error names its file line when `header_line` is known.""" - - parsed: _ModelHeaderParse = _model_header_parse(header) - if parsed.values is not None: - return dict(parsed.values) - error: ModelHeaderSyntaxError = ModelHeaderSyntaxError(str(parsed.error)) - detail, help_text, line = _native.model_header_failure(str(parsed.error), header, header_line) - location: str = str(file_path) if line is None else f"{file_path}:{line}" - raise error_class( - f"{statement_name}(...) in '{location}' contains invalid SQLBuild header syntax: {detail}", - help=help_text, - ) from error - - -def prepare_model_header_tokens(headers: list[str]) -> None: - """Batch cache-missing MODEL header semantic parsing through the native parser.""" - - missing_headers: list[str] = _MODEL_HEADER_PARSE_CACHE.missing(headers) - if not missing_headers: - return - parsed_headers: list[ - tuple[dict[str, object] | None, list[tuple[str, int, int]] | None, str | None] - ] = _native.parse_model_headers(missing_headers) - for header, (values, _offsets, error) in zip(missing_headers, parsed_headers, strict=True): - _MODEL_HEADER_PARSE_CACHE.put( - header=header, parsed=_ModelHeaderParse(values=values, error=error) - ) - - -def _model_header_parse(header: str) -> _ModelHeaderParse: - parsed: _ModelHeaderParse | None = _MODEL_HEADER_PARSE_CACHE.get(header) - if parsed is None: - prepare_model_header_tokens([header]) - parsed = _MODEL_HEADER_PARSE_CACHE.get(header) - if parsed is None: - raise ModelHeaderSyntaxError("Native MODEL header parser did not populate its cache") - return parsed diff --git a/src/sqlbuild/compiler/discovery/main/_model_output_column_locations.py b/src/sqlbuild/compiler/discovery/main/_model_output_column_locations.py index d1a7888304..d07007a8b7 100644 --- a/src/sqlbuild/compiler/discovery/main/_model_output_column_locations.py +++ b/src/sqlbuild/compiler/discovery/main/_model_output_column_locations.py @@ -4,9 +4,7 @@ from pathlib import Path -from sqlbuild.compiler.discovery._helpers.sql.model_files import ( - model_output_column_locations, -) +from sqlbuild._native import model_output_column_locations from sqlbuild.spec.contracts.models import SourceLocation diff --git a/src/sqlbuild/lint/_helpers/native.py b/src/sqlbuild/lint/_helpers/native.py index 6eccef6f92..6ec46d75b9 100644 --- a/src/sqlbuild/lint/_helpers/native.py +++ b/src/sqlbuild/lint/_helpers/native.py @@ -8,13 +8,10 @@ from dataclasses import dataclass from pathlib import Path +from sqlbuild._native import parse_header_values, prepare_model_header_tokens from sqlbuild.compiler.compile.classes.model_description_resolution import ( ModelDescriptionResolution, ) -from sqlbuild.compiler.discovery._helpers.sql.model_files import ( - parse_header_values, # noqa: FFL102 - the compiler owns the canonical native header grammar - prepare_model_header_tokens, -) from sqlbuild.compiler.discovery.exceptions import DiscoveryError, ModelSqlParseError from sqlbuild.lint.constants import ( CARRIAGE_RETURN_LINE_FEED, diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_deep_header_values.py b/tests/integration/src/sqlbuild/compiler/discovery/test_deep_header_values.py index b8c46d16ea..8ba06c193e 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_deep_header_values.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_deep_header_values.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.compiler.discovery._helpers.sql.model_files import parse_header_values +from sqlbuild._native import parse_header_values from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( DeepHeaderValuesTestCase, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py index 460ef2ab2f..ad13db280f 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_sql_models.py @@ -4,11 +4,11 @@ import pytest +import sqlbuild._native as native_module from sqlbuild._native import ( discover_model_files, + model_output_column_locations, ) -from sqlbuild.compiler.discovery._helpers.sql import model_files as model_file_helpers -from sqlbuild.compiler.discovery._helpers.sql.model_files import model_output_column_locations from sqlbuild.compiler.discovery.models import ( DiscoveredSqlModelFile, DiscoveryFileFault, @@ -87,10 +87,10 @@ def test_given_native_pool_construction_error_when_preparing_headers_then_error_ def rejecting_tokenize(_headers: list[str]) -> object: raise ValueError("MODEL header worker pool construction failed") - monkeypatch.setattr(model_file_helpers._native, "parse_model_headers", rejecting_tokenize) + monkeypatch.setattr(native_module, "parse_model_headers", rejecting_tokenize) with pytest.raises(ValueError, match=test_case.expected_message): - model_file_helpers.prepare_model_header_tokens(["pool_error_marker value"]) + native_module.prepare_model_header_tokens(["pool_error_marker value"]) @pytest.mark.parametrize( From a55fe24ba39f19d5ac1802f847a5fc292e65ee2d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 13 Oct 2026 07:16:11 +0100 Subject: [PATCH 4/4] chore: update the Python boundary ratchet --- python-boundary.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-boundary.toml b/python-boundary.toml index af057baa57..4289ba5037 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 119534 +remaining_lines = 118042 [[stays]] path = "executor"