diff --git a/Cargo.lock b/Cargo.lock index 0b20931b5c..4129f56b2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1282,6 +1282,8 @@ dependencies = [ "sqlbuild-cache", "sqlbuild-config", "sqlbuild-core", + "sqlbuild-cursor", + "sqlbuild-model-config", "sqlbuild-rules", "tempfile", ] diff --git a/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs index bcd4228bfe..ba731740dd 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs @@ -6,6 +6,7 @@ use polyglot_sql::{ ReferenceConfidence, SchemaValidationOptions, TransformKind, ValidationResult, ValidationSchema, analyze_query, analyze_query_for_project_projections, }; +#[cfg(test)] use rayon::iter::{IntoParallelIterator, ParallelIterator}; use serde::ser::SerializeTuple; use serde::{Deserialize, Serialize, Serializer}; @@ -22,6 +23,7 @@ const ANALYSIS_WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; const COMPACT_ANALYSIS_BATCH_SIZE: usize = 64; const LARGE_COMPACT_SQL_BYTES: usize = 32 * 1024 * 1024; +#[cfg(test)] #[derive(Debug, Deserialize)] struct AnalysisBatchRequest { requests: Vec, @@ -92,6 +94,7 @@ impl AnalysisRequest { } } +#[cfg(test)] #[derive(Debug, Serialize)] #[serde(untagged)] enum AnalysisResponse { @@ -99,6 +102,7 @@ enum AnalysisResponse { Failure { error: String }, } +#[cfg(test)] #[derive(Debug, Deserialize)] struct ProjectAnalysisBatchRequest { requests: Vec, @@ -139,6 +143,7 @@ struct CompactProjectAnalysisProjectionRequest { resource_names: HashMap, } +#[cfg(test)] #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct ProjectAnalysisRequest { @@ -163,6 +168,7 @@ struct LineageResource { resource_name: String, } +#[cfg(test)] #[derive(Debug, Serialize)] #[serde(untagged)] enum ProjectAnalysisResponse { @@ -279,6 +285,7 @@ fn default_true() -> bool { true } +#[cfg(test)] pub(crate) fn analyze_json(request_json: &str) -> Result { let request: AnalysisBatchRequest = serde_json::from_str(request_json).map_err(|error| error.to_string())?; @@ -298,6 +305,7 @@ pub(crate) fn analyze_json(request_json: &str) -> Result { serde_json::to_string(&responses).map_err(|error| error.to_string()) } +#[cfg(test)] pub(crate) fn analyze_project_json(request_json: &str) -> Result { let request: ProjectAnalysisBatchRequest = serde_json::from_str(request_json).map_err(|error| error.to_string())?; @@ -317,10 +325,12 @@ pub(crate) fn analyze_project_json(request_json: &str) -> Result serde_json::to_string(&responses).map_err(|error| error.to_string()) } +#[cfg(test)] pub(crate) fn analyze_project_compact_json(request_json: &str) -> Result { analyze_project_compact_with_catalog(request_json, None) } +#[cfg(test)] pub(crate) fn analyze_project_compact_with_catalog( request_json: &str, catalog: Option<&crate::semantic_validation::models::ProjectCatalog>, @@ -1077,6 +1087,7 @@ fn project_query_templates( results } +#[cfg(test)] fn analyze_request(request: AnalysisRequest) -> AnalysisResponse { match query_analysis(request, false) { Ok(analysis) => AnalysisResponse::Success { analysis }, @@ -1110,6 +1121,7 @@ fn query_analysis( .map_err(|error| error.to_string()) } +#[cfg(test)] fn analyze_project_request(request: ProjectAnalysisRequest) -> ProjectAnalysisResponse { match analyze_project_result(request) { Ok(analysis) => ProjectAnalysisResponse::Success { analysis }, @@ -1117,6 +1129,7 @@ fn analyze_project_request(request: ProjectAnalysisRequest) -> ProjectAnalysisRe } } +#[cfg(test)] fn analyze_project_result(request: ProjectAnalysisRequest) -> Result { let analysis = query_analysis(request.query, true)?; Ok(project_analysis(ProjectAnalysisInputs { diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/analyze.rs b/crates/sqlbuild-analysis/src/query_analysis/main/analyze.rs deleted file mode 100644 index c1c36ff2f0..0000000000 --- a/crates/sqlbuild-analysis/src/query_analysis/main/analyze.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub fn analyze_json(request_json: &str) -> Result { - crate::query_analysis::engine::analyze_json(request_json) -} diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/analyze_project.rs b/crates/sqlbuild-analysis/src/query_analysis/main/analyze_project.rs deleted file mode 100644 index 50f36ef4ae..0000000000 --- a/crates/sqlbuild-analysis/src/query_analysis/main/analyze_project.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub fn analyze_project_json(request_json: &str) -> Result { - crate::query_analysis::engine::analyze_project_json(request_json) -} diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/analyze_project_compact.rs b/crates/sqlbuild-analysis/src/query_analysis/main/analyze_project_compact.rs deleted file mode 100644 index 5e83be1b56..0000000000 --- a/crates/sqlbuild-analysis/src/query_analysis/main/analyze_project_compact.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub fn analyze_project_compact_json(request_json: &str) -> Result { - crate::query_analysis::engine::analyze_project_compact_json(request_json) -} diff --git a/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs b/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs index f51bc838b0..5d65d9ca3e 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/main/mod.rs @@ -1,6 +1,3 @@ -pub mod analyze; -pub mod analyze_project; -pub mod analyze_project_compact; pub mod annotated_output_types; pub mod output_types; pub mod prepare_project_catalog; diff --git a/crates/sqlbuild-analysis/src/query_analysis/tests/helpers.rs b/crates/sqlbuild-analysis/src/query_analysis/tests/helpers.rs index 360e565241..14068fe6ea 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/tests/helpers.rs @@ -1,4 +1,4 @@ -use crate::query_analysis::main::analyze_project_compact::analyze_project_compact_json; +use crate::query_analysis::engine::analyze_project_compact_json; use serde_json::{Value, json}; pub(crate) fn borrowed_facts_preserve_named_outputs_and_terminal_sources() -> bool { diff --git a/crates/sqlbuild-analysis/src/query_analysis/tests/test_query_analysis.rs b/crates/sqlbuild-analysis/src/query_analysis/tests/test_query_analysis.rs index 528968835e..04a28c6865 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/tests/test_query_analysis.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/tests/test_query_analysis.rs @@ -1,5 +1,4 @@ -use crate::query_analysis::main::analyze::analyze_json; -use crate::query_analysis::main::analyze_project::analyze_project_json; +use crate::query_analysis::engine::{analyze_json, analyze_project_json}; use crate::query_analysis::tests::helpers::{ binding_warnings_preserve_declared_relationship_evidence, borrowed_facts_preserve_named_outputs_and_terminal_sources, diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index 1666867dbf..4a2253fb56 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -19,11 +19,13 @@ path = "src/bin/sqlbuild.rs" blake3.workspace = true getrandom.workspace = true serde.workspace = true -serde_json.workspace = true +serde_json = { workspace = true, features = ["arbitrary_precision"] } sqlbuild-build-features.workspace = true sqlbuild-cache.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true +sqlbuild-cursor.workspace = true +sqlbuild-model-config.workspace = true sqlbuild-rules.workspace = true [target.'cfg(unix)'.dependencies] diff --git a/crates/sqlbuild-cli/src/cost_store/_helpers/artifacts.rs b/crates/sqlbuild-cli/src/cost_store/_helpers/artifacts.rs new file mode 100644 index 0000000000..753c0ac1c0 --- /dev/null +++ b/crates/sqlbuild-cli/src/cost_store/_helpers/artifacts.rs @@ -0,0 +1,75 @@ +//! Run directory confinement and decoding of native cost artifacts. + +use std::fs; +use std::path::{Path, PathBuf}; + +use sqlbuild_core::cost_report::errors::CostArtifactError; +use sqlbuild_core::cost_report::main::check_cost_run_id::check_cost_run_id; +use sqlbuild_core::cost_report::main::read_cost_artifact::read_cost_artifact; +use sqlbuild_core::cost_report::models::CostRun; +use sqlbuild_core::json::models::{JsonInteger, JsonValue}; + +use crate::cost_store::errors::CostStoreError; + +pub(crate) const ARTIFACT_NAME: &str = "run.json"; +pub(crate) const TEMPORARY_NAME: &str = ".run.json.tmp"; + +pub(crate) fn root(project_dir: &Path) -> PathBuf { + project_dir.join("target").join("executions") +} + +pub(crate) fn directory(project_dir: &Path, run_id: &str) -> Result { + check_cost_run_id(run_id)?; + let root = root(project_dir); + let directory = root.join(run_id); + if let (Ok(resolved), Ok(resolved_root)) = (directory.canonicalize(), root.canonicalize()) + && !resolved.starts_with(resolved_root) + { + return Err(CostArtifactError::new( + "cost run path must remain under the project execution directory", + ) + .into()); + } + Ok(directory) +} + +fn entry( + (key, payload): (String, serde_json::Value), +) -> Result<(String, JsonValue), CostArtifactError> { + Ok((key, value(payload)?)) +} + +fn value(value: serde_json::Value) -> Result { + Ok(match value { + serde_json::Value::Null => JsonValue::Null, + serde_json::Value::Bool(value) => JsonValue::Bool(value), + serde_json::Value::String(value) => JsonValue::String(value), + serde_json::Value::Number(value) => { + if let Some(integer) = JsonInteger::parse(&value.to_string()) { + JsonValue::Integer(integer) + } else { + JsonValue::Float(value.as_f64().ok_or_else(|| { + CostArtifactError::new("cost run artifact number is out of range") + })?) + } + } + serde_json::Value::Array(values) => JsonValue::Array( + values + .into_iter() + .map(self::value) + .collect::>()?, + ), + serde_json::Value::Object(values) => { + JsonValue::Object(values.into_iter().map(entry).collect::>()?) + } + }) +} + +pub(crate) fn load(path: &Path) -> Result { + let bytes = fs::read(path)?; + let text = String::from_utf8(bytes) + .map_err(|_| CostArtifactError::new("cost run artifact is not UTF-8 text"))?; + let payload = serde_json::from_str(&text) + .map_err(|_| CostArtifactError::new("cost run artifact is not valid JSON"))?; + Ok(read_cost_artifact(&value(payload)?)?) +} diff --git a/crates/sqlbuild-cli/src/cost_store/_helpers/mod.rs b/crates/sqlbuild-cli/src/cost_store/_helpers/mod.rs new file mode 100644 index 0000000000..6c466b21af --- /dev/null +++ b/crates/sqlbuild-cli/src/cost_store/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod artifacts; diff --git a/crates/sqlbuild-cli/src/cost_store/errors.rs b/crates/sqlbuild-cli/src/cost_store/errors.rs new file mode 100644 index 0000000000..d99ed10763 --- /dev/null +++ b/crates/sqlbuild-cli/src/cost_store/errors.rs @@ -0,0 +1,22 @@ +//! Cost artifact failures, distinguished from filesystem failures at the Python edge. + +use sqlbuild_core::cost_report::errors::CostArtifactError; +use std::io; + +#[derive(Debug)] +pub enum CostStoreError { + Io(io::Error), + Artifact(CostArtifactError), +} + +impl From for CostStoreError { + fn from(error: io::Error) -> Self { + Self::Io(error) + } +} + +impl From for CostStoreError { + fn from(error: CostArtifactError) -> Self { + Self::Artifact(error) + } +} diff --git a/crates/sqlbuild-cli/src/cost_store/main/list_cost_runs.rs b/crates/sqlbuild-cli/src/cost_store/main/list_cost_runs.rs new file mode 100644 index 0000000000..5f73009967 --- /dev/null +++ b/crates/sqlbuild-cli/src/cost_store/main/list_cost_runs.rs @@ -0,0 +1,33 @@ +//! Enumerate readable, confined run artifacts, newest first. + +use crate::cost_store::_helpers::artifacts::{ARTIFACT_NAME, load, root}; +use sqlbuild_core::cost_report::main::order_cost_runs::order_cost_runs; +use sqlbuild_core::cost_report::models::CostRun; +use std::fs; +use std::path::Path; + +/// List readable artifacts; corrupt records, mismatched IDs and escaping symlinks are skipped. +pub fn list_cost_runs(project_dir: &Path) -> Vec { + let root = root(project_dir); + let Ok(entries) = fs::read_dir(&root) else { + return Vec::new(); + }; + let mut runs: Vec = Vec::new(); + for entry in entries.flatten() { + let directory = entry.path(); + let path = directory.join(ARTIFACT_NAME); + let (Ok(resolved), Ok(resolved_root)) = (path.canonicalize(), root.canonicalize()) else { + continue; + }; + if !resolved.starts_with(resolved_root) || !path.is_file() { + continue; + } + let Ok(run) = load(&path) else { + continue; + }; + if directory.file_name().and_then(|name| name.to_str()) == Some(run.run_id.as_str()) { + runs.push(run); + } + } + order_cost_runs(runs) +} diff --git a/crates/sqlbuild-cli/src/cost_store/main/mod.rs b/crates/sqlbuild-cli/src/cost_store/main/mod.rs new file mode 100644 index 0000000000..d5971cccdc --- /dev/null +++ b/crates/sqlbuild-cli/src/cost_store/main/mod.rs @@ -0,0 +1,3 @@ +pub mod list_cost_runs; +pub mod read_cost_run; +pub mod write_cost_run; diff --git a/crates/sqlbuild-cli/src/cost_store/main/read_cost_run.rs b/crates/sqlbuild-cli/src/cost_store/main/read_cost_run.rs new file mode 100644 index 0000000000..4293176462 --- /dev/null +++ b/crates/sqlbuild-cli/src/cost_store/main/read_cost_run.rs @@ -0,0 +1,23 @@ +//! Read one exact run artifact under the project execution directory. + +use crate::cost_store::_helpers::artifacts::{ARTIFACT_NAME, directory, load}; +use crate::cost_store::errors::CostStoreError; +use sqlbuild_core::cost_report::errors::CostArtifactError; +use sqlbuild_core::cost_report::models::CostRun; +use std::path::Path; + +/// Return `None` for a missing artifact; report invalid artifacts and I/O failures distinctly. +pub fn read_cost_run(project_dir: &Path, run_id: &str) -> Result, CostStoreError> { + let path = directory(project_dir, run_id)?.join(ARTIFACT_NAME); + if !path.is_file() { + return Ok(None); + } + let run = load(&path)?; + if run.run_id != run_id { + return Err(CostArtifactError::new( + "cost run artifact run ID does not match its directory", + ) + .into()); + } + Ok(Some(run)) +} diff --git a/crates/sqlbuild-cli/src/cost_store/main/write_cost_run.rs b/crates/sqlbuild-cli/src/cost_store/main/write_cost_run.rs new file mode 100644 index 0000000000..7309fa349f --- /dev/null +++ b/crates/sqlbuild-cli/src/cost_store/main/write_cost_run.rs @@ -0,0 +1,19 @@ +//! Atomically replace one native cost run artifact. + +use crate::cost_store::_helpers::artifacts::{ARTIFACT_NAME, TEMPORARY_NAME, directory}; +use crate::cost_store::errors::CostStoreError; +use sqlbuild_core::cost_report::main::cost_artifact_text::cost_artifact_text; +use sqlbuild_core::cost_report::models::CostRun; +use std::fs; +use std::path::{Path, PathBuf}; + +/// Persist the native record under its confined run ID and return the artifact path. +pub fn write_cost_run(project_dir: &Path, run: &CostRun) -> Result { + let directory = directory(project_dir, &run.run_id)?; + fs::create_dir_all(&directory)?; + let destination = directory.join(ARTIFACT_NAME); + let temporary = directory.join(TEMPORARY_NAME); + fs::write(&temporary, cost_artifact_text(run))?; + fs::rename(&temporary, &destination)?; + Ok(destination) +} diff --git a/crates/sqlbuild-cli/src/cost_store/mod.rs b/crates/sqlbuild-cli/src/cost_store/mod.rs new file mode 100644 index 0000000000..9c1bcc9a43 --- /dev/null +++ b/crates/sqlbuild-cli/src/cost_store/mod.rs @@ -0,0 +1,5 @@ +//! Native project-local cost artifacts shared by the launcher and Python API edge. + +pub(crate) mod _helpers; +pub mod errors; +pub mod main; diff --git a/crates/sqlbuild-cli/src/lib.rs b/crates/sqlbuild-cli/src/lib.rs index fc36fb01fd..538dec3263 100644 --- a/crates/sqlbuild-cli/src/lib.rs +++ b/crates/sqlbuild-cli/src/lib.rs @@ -4,6 +4,7 @@ pub mod argv; pub mod compute_logs; +pub mod cost_store; pub mod diff_report; pub mod dispatch; pub mod launcher; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs index ed076cde4b..1ab59675b6 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -12,8 +12,9 @@ use crate::native_commands::constants::{ }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{ - NativeCommand, NativeInvocation, RulesAction, RulesOptions, SkillsOptions, + CostOptions, NativeCommand, NativeInvocation, RulesAction, RulesOptions, SkillsOptions, }; +use sqlbuild_core::cost_command::constants::DESCENDING_ORDER; /// Whether `sqb review --json` failed to parse, which Python reports as a JSON document. pub(crate) fn review_json_failure(arguments: &[String], outcome: &ArgvOutcome) -> bool { @@ -76,6 +77,24 @@ fn native_command( force: flag(namespace, "skills_force"), })), Some(RULES_COMMAND) => rules_command(namespace), + Some("cost") => Ok(NativeCommand::Cost(CostOptions { + selector: text(namespace, "cost_selector").unwrap_or_else(|| "latest".to_owned()), + limit: match namespace.get("cost_limit") { + None | Some(Value::None) => None, + Some(Value::Int(value)) => { + Some(value.parse().map_err(|_| PythonNeeded::Arguments)?) + } + _ => return Err(PythonNeeded::Arguments), + }, + no_limit: flag(namespace, "cost_no_limit"), + sort: text(namespace, "cost_sort"), + descending: text(namespace, "cost_order").is_none_or(|order| order == DESCENDING_ORDER), + since: text(namespace, "cost_since"), + until: text(namespace, "cost_until"), + json: flag(namespace, "json"), + json_path: text(namespace, "json_output").map(PathBuf::from), + no_color: flag(namespace, "no_color"), + })), Some(_) => Err(PythonNeeded::Command), } } diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 7021116ad0..90297029d9 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -20,6 +20,7 @@ use crate::native_commands::_helpers::skills::{run_skills, skill_freshness_notic use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; use crate::native_commands::constants::{UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND}; use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ CommandOutput, InstalledPackage, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; @@ -95,6 +96,7 @@ fn observed_output( command: &NativeCommand, project: &ProjectLocation, package: &InstalledPackage, + error_color: bool, ) -> Result { if !has_no_runtime_extensions(&project.given) { return Err(PythonNeeded::Project); @@ -103,6 +105,7 @@ fn observed_output( NativeCommand::Nothing => Ok(CommandOutput::default()), NativeCommand::Skills(options) => run_skills(&project.resolved, package, options), NativeCommand::Rules(options) => run_rules(&project.resolved, options), + NativeCommand::Cost(options) => run_cost(&project.given, options, error_color), NativeCommand::Init => Err(PythonNeeded::Command), } } @@ -124,7 +127,8 @@ pub(crate) fn run_command( return Ok(finish_lost_stdout(lost.as_ref(), output.exit_code)); } let started_at: SystemTime = SystemTime::now(); - let output: CommandOutput = observed_output(&invocation.command, &project, &package)?; + let output: CommandOutput = + observed_output(&invocation.command, &project, &package, error_color)?; Ok(finish_observed( invocation, (&project, &package), diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 0d395432ff..867ce8bb82 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -34,6 +34,9 @@ pub(crate) const HOME_VARIABLE: &str = "HOME"; pub(crate) const HOME_VARIABLE: &str = "USERPROFILE"; /// The command name compute logs record for a bare `sqb`. pub(crate) const UNKNOWN_COMMAND: &str = "unknown"; +/// The supported Python ISO grammar, independent of whether a Python process is started. +pub(crate) const COST_ISO_PYTHON_MINOR: u8 = 12; +pub(crate) const RUNNING_BUILD_STATUS: &str = "running"; /// Commands with no handler: observed, then successful. pub(crate) const CLEAN_COMMAND: &str = "clean"; pub(crate) const INIT_COMMAND: &str = "init"; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_cost_report_options.rs b/crates/sqlbuild-cli/src/native_commands/main/_cost_report_options.rs new file mode 100644 index 0000000000..b9e3ad55f5 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_cost_report_options.rs @@ -0,0 +1,136 @@ +//! Native cost request validation and history bounds. + +use std::time::{SystemTime, UNIX_EPOCH}; + +use sqlbuild_core::cost_command::constants::{ + DEFAULT_DETAIL_SORT, DEFAULT_HISTORY_SORT, HISTORY_SELECTOR, ISO_DATE_LENGTH, +}; +use sqlbuild_core::cost_command::errors::CostCommandError; +use sqlbuild_core::cost_command::main::is_zero_cost_duration::is_zero_cost_duration; +use sqlbuild_core::cost_command::main::validate_cost_request::validate_cost_request; +use sqlbuild_core::cost_command::models::CostReportOptions; +use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; +use sqlbuild_cursor::algebra::models::DurationDigits; +use sqlbuild_model_config::model_validation::main::parse_iso_timestamp::parse_iso_timestamp; + +use crate::native_commands::_helpers::terminal::{stdout_supports_color, terminal_columns}; +use crate::native_commands::constants::COST_ISO_PYTHON_MINOR; +use crate::native_commands::models::CostOptions; + +fn bound(value: &str, label: &str, end_of_day: bool) -> Result { + let (instant, aware) = + parse_iso_timestamp(&value.replace('Z', "+00:00"), COST_ISO_PYTHON_MINOR).ok_or_else( + || { + CostCommandError::new( + "C707", + format!("cost {label} must be an ISO date or datetime"), + ) + }, + )?; + if value.chars().count() == ISO_DATE_LENGTH { + return Ok(i128::from(instant) + if end_of_day { 86_400_000_000 } else { 0 }); + } + if !aware { + return Err(CostCommandError::new( + "C708", + format!("cost {label} datetime must include a timezone"), + )); + } + Ok(i128::from(instant)) +} + +fn since(value: &str) -> Result { + let now = match SystemTime::now().duration_since(UNIX_EPOCH) { + Ok(elapsed) => elapsed.as_micros() as i128, + Err(error) => -(error.duration().as_micros() as i128), + }; + if is_zero_cost_duration(value) { + return Ok(now); + } + let Some(DurationDigits { + years, + months, + days, + hours, + minutes, + seconds, + }) = parse_duration(value) + else { + return bound(value, "--since", false); + }; + let used = [&years, &months, &days, &hours, &minutes, &seconds] + .into_iter() + .filter(|value| !value.trim_start_matches('0').is_empty()) + .count(); + if used != 1 + || [&years, &months, &seconds] + .into_iter() + .any(|value| !value.trim_start_matches('0').is_empty()) + { + return Err(CostCommandError::new( + "C707", + "cost --since relative duration must use one of: d, h, m", + )); + } + let seconds = [(days, 86_400_i128), (hours, 3_600), (minutes, 60)] + .into_iter() + .try_fold(0_i128, |total, (digits, scale)| { + let Ok(amount) = digits.parse::() else { + return None; + }; + total.checked_add(amount.checked_mul(scale)?) + }) + .ok_or_else(|| { + CostCommandError::new("C707", "cost --since relative duration is out of range") + })?; + seconds + .checked_mul(1_000_000) + .and_then(|micros| now.checked_sub(micros)) + .ok_or_else(|| { + CostCommandError::new("C707", "cost --since relative duration is out of range") + }) +} + +/// Resolve defaults and bounds once, before report selection or output side effects. +pub(super) fn cost_report_options( + options: &CostOptions, +) -> Result { + let history = options.selector == HISTORY_SELECTOR; + let sort = options + .sort + .clone() + .filter(|sort| !sort.is_empty()) + .unwrap_or_else(|| { + if history { + DEFAULT_HISTORY_SORT + } else { + DEFAULT_DETAIL_SORT + } + .to_owned() + }); + validate_cost_request( + &options.selector, + (options.limit, options.no_limit), + &sort, + options.since.is_some() || options.until.is_some(), + )?; + Ok(CostReportOptions { + selector: options.selector.clone(), + limit: options.limit, + no_limit: options.no_limit, + sort, + descending: options.descending, + since: options.since.as_deref().map(since).transpose()?, + until: options + .until + .as_deref() + .map(|value| bound(value, "--until", true)) + .transpose()?, + until_exclusive: options + .until + .as_deref() + .is_some_and(|value| value.chars().count() == ISO_DATE_LENGTH), + use_color: !options.no_color && stdout_supports_color(), + terminal_width: usize::try_from(terminal_columns().unwrap_or(100)).unwrap_or(100), + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_cost.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_cost.rs new file mode 100644 index 0000000000..4bb97e34bf --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_cost.rs @@ -0,0 +1,90 @@ +//! Persisted reports run natively; warehouse refresh belongs to the adapter edge. + +use std::fs; +use std::path::Path; + +use sqlbuild_core::cli_text::main::format_coded_error::format_coded_error; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::cost_command::constants::{ + HISTORY_SELECTOR, LATEST_SELECTOR, REFRESHABLE_STATUSES, +}; +use sqlbuild_core::cost_command::errors::CostCommandError; +use sqlbuild_core::cost_command::main::render_cost_report::render_cost_report; +use sqlbuild_core::cost_command::main::select_cost_run::select_cost_run; +use sqlbuild_core::cost_command::models::CostReport; +use sqlbuild_core::cost_report::models::CostRun; + +use crate::cost_store::main::list_cost_runs::list_cost_runs; +use crate::native_commands::constants::RUNNING_BUILD_STATUS; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::main::_cost_report_options::cost_report_options; +use crate::native_commands::models::{CommandOutput, CostOptions}; + +fn needs_refresh(runs: &[CostRun], selector: &str) -> Result { + if selector == HISTORY_SELECTOR { + return Ok(false); + } + let selected = if selector == LATEST_SELECTOR { + runs.iter() + .find(|run| run.build_status != RUNNING_BUILD_STATUS) + } else { + let ids = runs + .iter() + .map(|run| run.run_id.clone()) + .collect::>(); + runs.get(select_cost_run(&ids, selector)?) + }; + Ok(selected.is_some_and(|run| REFRESHABLE_STATUSES.contains(&run.cost.status.as_str()))) +} + +fn refused(error: CostCommandError, use_color: bool) -> CommandOutput { + let coded = format_coded_error( + (error.code, &error.message, None), + true, + CliStyle { use_color }, + ); + CommandOutput { + stdout: String::new(), + stderr: format!("{coded}\n\n"), + exit_code: 1, + } +} + +/// Render natively, or select the Python adapter edge before any side effect. +pub(crate) fn run_cost( + project_dir: &Path, + options: &CostOptions, + error_color: bool, +) -> Result { + let runs = list_cost_runs(project_dir); + let request = match cost_report_options(options) { + Ok(request) => request, + Err(error) => return Ok(refused(error, error_color)), + }; + match needs_refresh(&runs, &options.selector) { + Ok(true) => return Err(PythonNeeded::Command), + Ok(false) => {} + Err(error) => return Ok(refused(error, error_color)), + } + let CostReport { json, text } = match render_cost_report(runs, &request) { + Ok(report) => report, + Err(error) => return Ok(refused(error, error_color)), + }; + if let Some(path) = &options.json_path { + let written = path + .parent() + .map_or(Ok(()), fs::create_dir_all) + .and_then(|()| fs::write(path, format!("{json}\n"))); + if let Err(error) = written { + return Ok(CommandOutput { + stdout: String::new(), + stderr: format!("sqb cost: cannot write {}: {error}\n", path.display()), + exit_code: 1, + }); + } + } + Ok(CommandOutput { + stdout: format!("{}\n", if options.json { json } else { text }), + ..CommandOutput::default() + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs index f45e2c475a..034b2c7bcd 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/mod.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -1 +1,3 @@ +pub(super) mod _cost_report_options; +pub(super) mod _run_cost; pub mod run_native_invocation; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 9c6ece8ad5..93db875a33 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -25,6 +25,21 @@ pub(crate) struct RulesOptions { pub(crate) json: bool, } +/// Raw `sqb cost` options; only a pending warehouse refresh needs the Python adapter edge. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct CostOptions { + pub(crate) selector: String, + pub(crate) limit: Option, + pub(crate) no_limit: bool, + pub(crate) sort: Option, + pub(crate) descending: bool, + pub(crate) since: Option, + pub(crate) until: Option, + pub(crate) json: bool, + pub(crate) json_path: Option, + pub(crate) no_color: bool, +} + /// A command that runs without Python. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum NativeCommand { @@ -33,6 +48,7 @@ pub(crate) enum NativeCommand { Init, Skills(SkillsOptions), Rules(RulesOptions), + Cost(CostOptions), } /// One parsed invocation `sqb` serves without Python. diff --git a/crates/sqlbuild-core/src/cost_command/_helpers/mod.rs b/crates/sqlbuild-core/src/cost_command/_helpers/mod.rs new file mode 100644 index 0000000000..6592e64ecb --- /dev/null +++ b/crates/sqlbuild-core/src/cost_command/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod sorting; diff --git a/crates/sqlbuild-core/src/cost_command/_helpers/sorting.rs b/crates/sqlbuild-core/src/cost_command/_helpers/sorting.rs new file mode 100644 index 0000000000..be61991d4b --- /dev/null +++ b/crates/sqlbuild-core/src/cost_command/_helpers/sorting.rs @@ -0,0 +1,101 @@ +//! Stable cost ordering with cached exact decimals and UTC instants. + +use crate::cost_command::constants::{HISTORY_METRIC_SORT_FIELDS, MISSING_METRIC_STATUSES}; +use crate::cost_command::errors::CostCommandError; +use crate::cost_command::models::CostSortMetric; +use crate::cost_report::main::cost_decimal_key::cost_decimal_key; +use crate::cost_report::main::cost_timestamp::cost_timestamp; +use crate::cost_report::models::{CostResource, CostRun}; + +fn invalid() -> CostCommandError { + CostCommandError::new("C000", "Invalid persisted cost metric or timestamp") +} + +pub(crate) fn timestamp(value: &str) -> Result { + cost_timestamp(value).ok_or_else(invalid) +} + +fn decimal(value: &str) -> Result { + cost_decimal_key(value) + .map(CostSortMetric::Decimal) + .ok_or_else(invalid) +} + +fn history_metric( + run: &CostRun, + field: &str, + completed: i128, +) -> Result { + Ok(match field { + "cost" => return decimal(&run.cost.estimated_usd), + "credits" => return decimal(&run.cost.estimated_compute_credits), + "duration" => CostSortMetric::Integer(completed - timestamp(&run.started_at)?), + "models" => CostSortMetric::Integer(run.cost.resources.len() as i128), + "status" => CostSortMetric::Text(run.cost.status.clone()), + _ => CostSortMetric::Integer(completed), + }) +} + +pub(crate) fn history( + runs: Vec, + field: &str, + descending: bool, +) -> Result, CostCommandError> { + let mut keyed = Vec::with_capacity(runs.len()); + for run in runs { + let completed = timestamp(&run.completed_at)?; + let missing = HISTORY_METRIC_SORT_FIELDS.contains(&field) + && MISSING_METRIC_STATUSES.contains(&run.cost.status.as_str()); + let metric = if missing { + None + } else { + Some(history_metric(&run, field, completed)?) + }; + keyed.push((run, metric, completed)); + } + keyed.sort_by( + |(left, left_key, left_time), (right, right_key, right_time)| { + let metric = left_key.cmp(right_key); + left_key.is_none().cmp(&right_key.is_none()).then_with(|| { + (if descending { metric.reverse() } else { metric }) + .then_with(|| right_time.cmp(left_time)) + .then_with(|| left.run_id.cmp(&right.run_id)) + }) + }, + ); + Ok(keyed.into_iter().map(|(run, _, _)| run).collect()) +} + +fn resource_metric( + resource: &CostResource, + field: &str, +) -> Result { + Ok(match field { + "credits" => return decimal(&resource.estimated_compute_credits), + "busy" => return decimal(&resource.attributed_seconds), + "scanned" => CostSortMetric::Integer(i128::from(resource.bytes_scanned)), + "model" => CostSortMetric::Text(resource.resource_name.clone()), + "warehouse" => CostSortMetric::Text(resource.warehouse_name.clone()), + _ => return decimal(&resource.estimated_usd), + }) +} + +pub(crate) fn resources( + resources: Vec, + field: &str, + descending: bool, +) -> Result, CostCommandError> { + let mut keyed = Vec::with_capacity(resources.len()); + for resource in resources { + let metric = resource_metric(&resource, field)?; + keyed.push((resource, metric)); + } + keyed.sort_by(|(left, left_key), (right, right_key)| { + let metric = left_key + .cmp(right_key) + .then_with(|| left.resource_name.cmp(&right.resource_name)) + .then_with(|| left.warehouse_name.cmp(&right.warehouse_name)); + if descending { metric.reverse() } else { metric } + }); + Ok(keyed.into_iter().map(|(resource, _)| resource).collect()) +} diff --git a/crates/sqlbuild-core/src/cost_command/main/cost_history_limit.rs b/crates/sqlbuild-core/src/cost_command/main/_cost_history_limit.rs similarity index 66% rename from crates/sqlbuild-core/src/cost_command/main/cost_history_limit.rs rename to crates/sqlbuild-core/src/cost_command/main/_cost_history_limit.rs index 4fa2b7c7ca..b28840029a 100644 --- a/crates/sqlbuild-core/src/cost_command/main/cost_history_limit.rs +++ b/crates/sqlbuild-core/src/cost_command/main/_cost_history_limit.rs @@ -1,10 +1,10 @@ -//! How many history rows to show. +//! The internal history row limit. use crate::cost_command::constants::DEFAULT_HISTORY_LIMIT; /// None for `--no-limit`, else the requested limit or the default. #[must_use] -pub fn cost_history_limit(limit: Option, no_limit: bool) -> Option { +pub(crate) fn cost_history_limit(limit: Option, no_limit: bool) -> Option { if no_limit { return None; } diff --git a/crates/sqlbuild-core/src/cost_command/main/mod.rs b/crates/sqlbuild-core/src/cost_command/main/mod.rs index 3f5f0cdb27..eb846c1cff 100644 --- a/crates/sqlbuild-core/src/cost_command/main/mod.rs +++ b/crates/sqlbuild-core/src/cost_command/main/mod.rs @@ -1,5 +1,6 @@ +pub(super) mod _cost_history_limit; pub mod accepts_cost_refresh; -pub mod cost_history_limit; pub mod is_zero_cost_duration; +pub mod render_cost_report; pub mod select_cost_run; pub mod validate_cost_request; diff --git a/crates/sqlbuild-core/src/cost_command/main/render_cost_report.rs b/crates/sqlbuild-core/src/cost_command/main/render_cost_report.rs new file mode 100644 index 0000000000..cd071d05fb --- /dev/null +++ b/crates/sqlbuild-core/src/cost_command/main/render_cost_report.rs @@ -0,0 +1,115 @@ +//! Build cost history or a selected run report directly from persisted native records. + +use crate::cost_command::_helpers::sorting; +use crate::cost_command::constants::{HISTORY_SELECTOR, LATEST_SELECTOR}; +use crate::cost_command::errors::CostCommandError; +use crate::cost_command::main::_cost_history_limit::cost_history_limit; +use crate::cost_command::main::select_cost_run::select_cost_run; +use crate::cost_command::main::validate_cost_request::validate_cost_request; +use crate::cost_command::models::{CostReport, CostReportOptions}; +use crate::cost_report::main::cost_history_json::cost_history_json; +use crate::cost_report::main::cost_report_json::cost_report_json; +use crate::cost_report::main::format_cost_breakdown::format_cost_breakdown; +use crate::cost_report::main::format_cost_history::format_cost_history; +use crate::cost_report::main::select_cost_run::select_cost_run as latest_run; +use crate::cost_report::models::{BreakdownOptions, CostRun}; + +fn history( + runs: Vec, + options: &CostReportOptions, +) -> Result { + if options + .since + .zip(options.until) + .is_some_and(|(since, until)| since > until) + { + return Err(CostCommandError::new( + "C711", + "cost --since must not be after --until", + )); + } + let mut matching = Vec::with_capacity(runs.len()); + for run in runs { + let completed = sorting::timestamp(&run.completed_at)?; + if options.since.is_none_or(|since| completed >= since) + && options.until.is_none_or(|until| { + if options.until_exclusive { + completed < until + } else { + completed <= until + } + }) + { + matching.push(run); + } + } + let matching_count = matching.len(); + let mut runs = sorting::history(matching, &options.sort, options.descending)?; + if let Some(limit) = cost_history_limit(options.limit, options.no_limit) { + runs.truncate(usize::try_from(limit).unwrap_or(0)); + } + Ok(CostReport { + json: cost_history_json(&runs, Some(matching_count)), + text: format_cost_history(&runs, matching_count, options.use_color), + }) +} + +fn detail( + mut runs: Vec, + options: &CostReportOptions, +) -> Result { + let mut run = if options.selector == LATEST_SELECTOR { + latest_run(runs, LATEST_SELECTOR).ok_or_else(|| { + CostCommandError::new("C704", "No persisted SQLBuild cost runs were found") + })? + } else { + let ids = runs + .iter() + .map(|run| run.run_id.clone()) + .collect::>(); + let index = select_cost_run(&ids, &options.selector)?; + runs.remove(index) + }; + run.cost.resources = sorting::resources(run.cost.resources, &options.sort, options.descending)?; + let limit = if options.no_limit { + None + } else { + options + .limit + .map(|limit| usize::try_from(limit).unwrap_or(usize::MAX)) + }; + let text = format_cost_breakdown( + &run, + BreakdownOptions { + use_color: options.use_color, + limit, + terminal_width: options.terminal_width, + show_run_id: true, + }, + ); + if let Some(limit) = limit { + run.cost.resources.truncate(limit); + } + Ok(CostReport { + json: cost_report_json(&run), + text, + }) +} + +/// The selected report's JSON and text, with history filtering and detail-only row limits. +pub fn render_cost_report( + runs: Vec, + options: &CostReportOptions, +) -> Result { + validate_cost_request( + &options.selector, + (options.limit, options.no_limit), + &options.sort, + options.since.is_some() || options.until.is_some(), + )?; + if options.selector == HISTORY_SELECTOR { + history(runs, options) + } else { + detail(runs, options) + } +} diff --git a/crates/sqlbuild-core/src/cost_command/mod.rs b/crates/sqlbuild-core/src/cost_command/mod.rs index 93d93f21c4..902c1eb20e 100644 --- a/crates/sqlbuild-core/src/cost_command/mod.rs +++ b/crates/sqlbuild-core/src/cost_command/mod.rs @@ -1,5 +1,6 @@ //! `sqb cost`: request validation, run selection, and delayed-refresh acceptance. +pub(crate) mod _helpers; pub mod constants; pub mod errors; pub mod main; diff --git a/crates/sqlbuild-core/src/cost_command/models.rs b/crates/sqlbuild-core/src/cost_command/models.rs index e6d70de69c..cf30e6317a 100644 --- a/crates/sqlbuild-core/src/cost_command/models.rs +++ b/crates/sqlbuild-core/src/cost_command/models.rs @@ -9,3 +9,33 @@ pub struct CostCoverage { pub query_count: i64, pub limitations: Vec, } + +/// A validated cost report request; history bounds are UTC microseconds. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CostReportOptions { + pub selector: String, + pub limit: Option, + pub no_limit: bool, + pub sort: String, + pub descending: bool, + pub since: Option, + pub until: Option, + pub until_exclusive: bool, + pub use_color: bool, + pub terminal_width: usize, +} + +/// Both output forms of one report, built without Python records. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CostReport { + pub json: String, + pub text: String, +} + +/// A cached ordering key; each request compares only one variant. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) enum CostSortMetric { + Integer(i128), + Text(String), + Decimal(crate::cost_report::models::DecimalSortKey), +} diff --git a/crates/sqlbuild-core/src/cost_command/tests/test_cost_command.rs b/crates/sqlbuild-core/src/cost_command/tests/test_cost_command.rs index 02698797c9..5d09098d1a 100644 --- a/crates/sqlbuild-core/src/cost_command/tests/test_cost_command.rs +++ b/crates/sqlbuild-core/src/cost_command/tests/test_cost_command.rs @@ -1,5 +1,5 @@ +use crate::cost_command::main::_cost_history_limit::cost_history_limit; use crate::cost_command::main::accepts_cost_refresh::accepts_cost_refresh; -use crate::cost_command::main::cost_history_limit::cost_history_limit; use crate::cost_command::main::is_zero_cost_duration::is_zero_cost_duration; use crate::cost_command::main::select_cost_run::select_cost_run; use crate::cost_command::main::validate_cost_request::validate_cost_request; diff --git a/crates/sqlbuild-core/src/cost_report/_helpers/decimal_text.rs b/crates/sqlbuild-core/src/cost_report/_helpers/decimal_text.rs index 27f76969c4..d44e6ccd48 100644 --- a/crates/sqlbuild-core/src/cost_report/_helpers/decimal_text.rs +++ b/crates/sqlbuild-core/src/cost_report/_helpers/decimal_text.rs @@ -1,6 +1,7 @@ //! Fixed-point text of Python `Decimal` values and byte sizes, rounded half to even. use crate::cost_report::constants::{BYTE_SCALE, BYTE_UNITS}; +use crate::cost_report::models::DecimalSortKey; /// The digits and power of ten of a finite decimal text, with its sign. struct DecimalParts { @@ -44,6 +45,24 @@ pub(crate) fn is_decimal(text: &str) -> bool { parts(text).is_some() } +/// Exact finite-decimal ordering without rounding or expanding powers of ten. +pub(crate) fn sort_key(text: &str) -> Option { + let value = parts(text)?; + let digits = value.digits.trim_start_matches('0'); + if digits.is_empty() { + return Some(DecimalSortKey::Zero); + } + let magnitude = ( + i128::from(value.exponent) + digits.len() as i128, + digits.trim_end_matches('0').to_owned(), + ); + Some(if value.negative { + DecimalSortKey::Negative(std::cmp::Reverse(magnitude)) + } else { + DecimalSortKey::Positive(magnitude) + }) +} + /// Round the digit string half to even after removing its last `dropped` digits. fn rounded(digits: &str, dropped: usize) -> String { if dropped == 0 { diff --git a/crates/sqlbuild-core/src/cost_report/errors.rs b/crates/sqlbuild-core/src/cost_report/errors.rs index 8677fb0d25..9085ef9021 100644 --- a/crates/sqlbuild-core/src/cost_report/errors.rs +++ b/crates/sqlbuild-core/src/cost_report/errors.rs @@ -9,7 +9,7 @@ pub struct CostArtifactError { } impl CostArtifactError { - pub(crate) fn new(message: impl Into) -> Self { + pub fn new(message: impl Into) -> Self { Self { message: message.into(), } diff --git a/crates/sqlbuild-core/src/cost_report/main/cost_decimal_key.rs b/crates/sqlbuild-core/src/cost_report/main/cost_decimal_key.rs new file mode 100644 index 0000000000..ea2bc91d84 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/main/cost_decimal_key.rs @@ -0,0 +1,9 @@ +//! Exact numeric ordering for persisted decimal metrics. + +use crate::cost_report::_helpers::decimal_text::sort_key; +use crate::cost_report::models::DecimalSortKey; + +/// Return the numeric key of a finite cost decimal, without floating-point rounding. +pub fn cost_decimal_key(value: &str) -> Option { + sort_key(value) +} diff --git a/crates/sqlbuild-core/src/cost_report/main/cost_timestamp.rs b/crates/sqlbuild-core/src/cost_report/main/cost_timestamp.rs new file mode 100644 index 0000000000..e090caef30 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_report/main/cost_timestamp.rs @@ -0,0 +1,8 @@ +//! The UTC instant of a persisted aware cost timestamp. + +use crate::cost_report::_helpers::instants::utc_micros; + +/// Microseconds since the Unix epoch, or `None` for an invalid or naive timestamp. +pub fn cost_timestamp(value: &str) -> Option { + utc_micros(value) +} diff --git a/crates/sqlbuild-core/src/cost_report/main/mod.rs b/crates/sqlbuild-core/src/cost_report/main/mod.rs index 7e161a6249..9f9be66913 100644 --- a/crates/sqlbuild-core/src/cost_report/main/mod.rs +++ b/crates/sqlbuild-core/src/cost_report/main/mod.rs @@ -1,8 +1,10 @@ pub mod check_cost_run_id; pub mod cost_artifact_text; +pub mod cost_decimal_key; pub mod cost_history_json; pub mod cost_output_payload; pub mod cost_report_json; +pub mod cost_timestamp; pub mod format_cost_breakdown; pub mod format_cost_history; pub mod order_cost_runs; diff --git a/crates/sqlbuild-core/src/cost_report/models.rs b/crates/sqlbuild-core/src/cost_report/models.rs index 65f5deef25..78ecb958c1 100644 --- a/crates/sqlbuild-core/src/cost_report/models.rs +++ b/crates/sqlbuild-core/src/cost_report/models.rs @@ -62,3 +62,11 @@ pub struct BreakdownOptions { pub terminal_width: usize, pub show_run_id: bool, } + +/// Canonical exact-decimal order, without expanding scientific exponents. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum DecimalSortKey { + Negative(std::cmp::Reverse<(i128, String)>), + Zero, + Positive((i128, String)), +} diff --git a/crates/sqlbuild-host/src/models.rs b/crates/sqlbuild-host/src/models.rs index ee79ea246c..42571dc8db 100644 --- a/crates/sqlbuild-host/src/models.rs +++ b/crates/sqlbuild-host/src/models.rs @@ -2,6 +2,8 @@ use std::path::PathBuf; +use sqlbuild_core::json::models::JsonInteger; + use crate::types::ResultEnum; /// Whether a build succeeded. Mirrors `sqlbuild.executor.build.types.BuildStatus`. @@ -454,6 +456,13 @@ pub struct MicrobatchAccountingInterval { pub event_id: Option, } +/// A measured number without coercing integer values to floating point. +#[derive(Clone, Debug, PartialEq)] +pub enum MeasuredValue { + Integer(JsonInteger), + Float(f64), +} + /// One executed audit. Mirrors `sqlbuild.executor.auditing.models.AuditExecutionResult`; `V` is a host value. #[derive(Clone, Debug, PartialEq)] pub struct AuditExecutionResult { @@ -470,7 +479,7 @@ pub struct AuditExecutionResult { pub attached_column_name: Option, pub reused: bool, pub evaluation_mode: AuditEvaluationMode, - pub measured_value: Option, + pub measured_value: Option, pub sample_count: Option, pub sample_unit: Option, pub minimum_samples: Option, diff --git a/crates/sqlbuild-model-config/src/model_validation/main/mod.rs b/crates/sqlbuild-model-config/src/model_validation/main/mod.rs index 2724bc0337..f7dcd6997d 100644 --- a/crates/sqlbuild-model-config/src/model_validation/main/mod.rs +++ b/crates/sqlbuild-model-config/src/model_validation/main/mod.rs @@ -1,4 +1,5 @@ pub mod fixed_duration_seconds; +pub mod parse_iso_timestamp; pub mod retention_override; pub mod table_type_override; pub mod validate_column_migrations; diff --git a/crates/sqlbuild-model-config/src/model_validation/main/parse_iso_timestamp.rs b/crates/sqlbuild-model-config/src/model_validation/main/parse_iso_timestamp.rs new file mode 100644 index 0000000000..b1447c4e3d --- /dev/null +++ b/crates/sqlbuild-model-config/src/model_validation/main/parse_iso_timestamp.rs @@ -0,0 +1,16 @@ +//! The existing CPython-compatible ISO parser for native timestamp consumers. + +use crate::model_validation::_helpers::isoformat::{ + datetime_fromisoformat, epoch_micros, local_micros, +}; + +/// Unix microseconds and timezone awareness, using the selected CPython minor's ISO rules. +pub fn parse_iso_timestamp(text: &str, minor: u8) -> Option<(i64, bool)> { + let Ok(parsed) = datetime_fromisoformat(text, minor) else { + return None; + }; + Some(( + local_micros(&parsed) - epoch_micros() - parsed.offset_micros.unwrap_or(0), + parsed.offset_micros.is_some(), + )) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs index a7db77b4eb..5a0dab6dd3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs @@ -396,19 +396,8 @@ pub(crate) fn schema_validations<'py>( PyTuple::new(py, results) } -/// Whether quoted identifiers bind case-insensitively for `connection` and `dialect`. -#[pyfunction] -#[pyo3(name = "ignores_quoted_case", signature = (*, connection, dialect))] -fn ignores_quoted_case_binding( - connection: &Bound<'_, PyAny>, - dialect: &Bound<'_, PyAny>, -) -> PyResult { - ignores_quoted_case(connection, dialect) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_function(wrap_pyfunction!(schema_validations, module)?)?; - module.add_function(wrap_pyfunction!(ignores_quoted_case_binding, module)?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs index b69dd10240..98a6ce5ace 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs @@ -14,43 +14,11 @@ fn query_fingerprint(py: Python<'_>, sql: &str, dialect: &str) -> PyResult, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::query_analysis::main::analyze::analyze_json(request_json) - }) - .map_err(value_error) -} - -#[pyfunction] -fn analyze_project_queries_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::query_analysis::main::analyze_project::analyze_project_json(request_json) - }) - .map_err(value_error) -} - -#[pyfunction] -fn analyze_project_queries_compact_json(py: Python<'_>, request_json: &str) -> PyResult { - py.compiler_detach(|| { - sqlbuild_analysis::query_analysis::main::analyze_project_compact::analyze_project_compact_json( - request_json, - ) - }) - .map_err(value_error) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::analysis::binding_catalog::register(module)?; crate::bindings::_helpers::analysis::required_columns::register(module)?; crate::bindings::_helpers::references::explicit_references::register(module)?; crate::bindings::_helpers::analysis::binding_locations::register(module)?; module.add_function(wrap_pyfunction!(query_fingerprint, module)?)?; - module.add_function(wrap_pyfunction!(analyze_queries_json, module)?)?; - module.add_function(wrap_pyfunction!(analyze_project_queries_json, module)?)?; - module.add_function(wrap_pyfunction!( - analyze_project_queries_compact_json, - module - )?)?; Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs index dc10a996b0..c21b4bf278 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs @@ -10,6 +10,7 @@ use sqlbuild_core::build_command::main::render_phase_timings::render_phase_timin use sqlbuild_core::build_command::models::PhaseTimings; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_host::models::BuildStatus; use crate::bindings::_helpers::boundary::python_calls::{call_with, keywords}; use crate::bindings::_helpers::build_command::execution::{ @@ -20,6 +21,7 @@ use crate::bindings::_helpers::build_command::support::{ boolean, emit, joined, none, text, utc_now, }; use crate::bindings::_helpers::build_progress::footer::format_build_footer_with; +use crate::bindings::_helpers::build_progress::result_text::build_result_text; use crate::bindings::_helpers::check::check_results::check_result_text; use crate::bindings::_helpers::compile_driver::pipeline_entries::plan_has_executable_work; use crate::bindings::_helpers::compile_outputs::runtime_target::{ @@ -34,7 +36,7 @@ use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::result_output::documents::format_build_execution_json; -use crate::bindings::_helpers::result_output::documents::format_degradable_build_execution_json; +use crate::bindings::_helpers::result_output::documents::format_build_execution_json_with; use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use sqlbuild_core::build_progress::models::PythonNodeText; use sqlbuild_core::check_command::models::{CheckResultText, CheckStatus}; @@ -351,10 +353,9 @@ pub(crate) fn complete_direct_build<'py>( ) -> PyResult { let py: Python<'py> = context.py(); let completed_at: Bound<'py, PyAny> = utc_now(py)?; - let result: &Bound<'py, PyAny> = &outcome.result; + let result = &outcome.result; let plan: Bound<'py, PyAny> = native_field(pipeline, "plan_output")?; - let succeeded: bool = - native_field(result, "status")?.str()?.extract::()? == SUCCESS_STATUS; + let succeeded: bool = result.status == BuildStatus::Success; let python_failed: bool = any_failed(&outcome.python_results, |result| { Ok(python_node_failed( &native_field(result, "status")?.str()?.extract::()?, @@ -390,7 +391,7 @@ pub(crate) fn complete_direct_build<'py>( .map(|check| Ok(check_node_text(&check_result_text(&check?)?))) .collect::>()?; let footer: String = format_build_footer_with( - &result.clone(), + &build_result_text(py, result)?, ( native_field(&preparation.callbacks, "elapsed")?.extract::()?, context.use_color, @@ -399,12 +400,12 @@ pub(crate) fn complete_direct_build<'py>( checks, )?; emit(&context.stream, &format!("\n{footer}\n"))?; - let payload: Bound<'py, PyAny> = format_degradable_build_execution_json( + let payload: Bound<'py, PyAny> = format_build_execution_json_with( py, + result, Some(keywords( py, &[ - ("result", result.clone()), ("plan", plan), ("python_node_results", outcome.python_results.clone()), ("python_check_results", check_results.clone()), @@ -419,17 +420,15 @@ pub(crate) fn complete_direct_build<'py>( write_json(context, payload)?; render_build_cost(&record, &context.stream, context.use_color)?; if context.verbose { - let timings: Bound<'py, PyAny> = native_field(result, "timings")?; - let seconds = - |name: &str| -> PyResult> { native_field(&timings, name)?.extract() }; + let timings = &result.timings; write_timings( context, &PhaseTimings { compile_seconds: native_field(pipeline, "compile_seconds")?.extract()?, planning_seconds: native_field(pipeline, "planning_seconds")?.extract()?, - connection_preparation_seconds: seconds("connection_preparation_seconds")?, - schema_preparation_seconds: seconds("schema_preparation_seconds")?, - execution_seconds: seconds("execution_seconds")?, + connection_preparation_seconds: timings.connection_preparation_seconds, + schema_preparation_seconds: timings.schema_preparation_seconds, + execution_seconds: timings.execution_seconds, cost_collection_seconds: Some(cost_seconds), total_seconds: Some(context.started.elapsed().as_secs_f64()), }, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 389eff2226..f0a0f55d12 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -9,6 +9,7 @@ use sqlbuild_core::build_command::main::render_run_context::render_run_context; use sqlbuild_core::build_command::models::{RunContextFacts, SelectorFileFact}; use sqlbuild_core::cli_text::models::CliStyle; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_host::models::{BuildExecutionResult, BuildStatus, ExecutionStatus}; use crate::bindings::_helpers::boundary::python_calls::{call_with, keywords, within}; use crate::bindings::_helpers::build_command::invocation::BuildContext; @@ -35,6 +36,7 @@ use crate::bindings::_helpers::cursor::overrides::{cursor_integer, cursor_timest use crate::bindings::_helpers::freshness::publication::append_freshness_records; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::planner_resolution::cursor_overrides::type_cursor_overrides; use crate::bindings::_helpers::records::construct::new_record; use crate::bindings::_helpers::records::fields::field as native_field; @@ -44,14 +46,13 @@ use crate::bindings::_helpers::sql_test_command::test_command::timeout_record; use crate::bindings::_helpers::sql_test_command::timeouts::{TimeoutKind, resolve_run_timeout}; const EXECUTION_COMMAND: &str = "sqb build"; -const SUCCESS_STATUS: &str = "success"; const NODE_START: [&str; 2] = ["name", "resource_kind"]; const CONNECTION_TIMING: [&str; 2] = ["connection_count", "elapsed_seconds"]; /// Prepared callbacks, concurrency, cursors and Python lifecycle. pub(crate) struct Preparation<'py> { pub(crate) callbacks: Bound<'py, PyAny>, - pub(crate) concurrency: Bound<'py, PyAny>, + pub(crate) concurrency: i64, connection_progress: Bound<'py, PyAny>, lifecycle: Bound<'py, DirectPythonLifecycle>, cursors: [Bound<'py, PyAny>; 4], @@ -59,7 +60,8 @@ pub(crate) struct Preparation<'py> { /// The build result and its finalized Python node results. pub(crate) struct Outcome<'py> { - pub(crate) result: Bound<'py, PyAny>, + pub(crate) result: BuildExecutionResult>, + pub(crate) executor_result: Bound<'py, PyAny>, pub(crate) python_results: Bound<'py, PyAny>, } @@ -225,7 +227,7 @@ pub(crate) fn prepare_build_execution<'py>( ) .and_then(|value| pyo3::Bound::new(py, value)) .map(pyo3::Bound::into_any)?; - let concurrency: Bound<'py, PyAny> = context.concurrency()?.limit.into_pyobject(py)?.into_any(); + let concurrency = context.concurrency()?.limit; if context.verbose { write_run_context(context, pipeline)?; } else { @@ -237,7 +239,7 @@ pub(crate) fn prepare_build_execution<'py>( ("stream", context.stream.clone()), ("command", text(py, EXECUTION_COMMAND)), ("target", none(py)), - ("concurrency", concurrency.clone()), + ("concurrency", concurrency.into_pyobject(py)?.into_any()), ("use_color", boolean(py, context.use_color)), ], )?), @@ -329,7 +331,10 @@ pub(crate) fn execute_build_plan<'py>( ("run_tests", request("run_tests")?), ("run_audits", request("run_audits")?), ("fail_fast", request("fail_fast")?), - ("max_concurrency", preparation.concurrency.clone()), + ( + "max_concurrency", + preparation.concurrency.into_pyobject(py)?.into_any(), + ), ("loader_is_reload", request("reload_sources")?), ("start_cursor_ts", start_ts.clone()), ("end_cursor_ts", end_ts.clone()), @@ -438,15 +443,23 @@ pub(crate) fn execute_build_plan<'py>( ("initial_state", initial_state), ], )?; + let native_result = BuildExecutionResult::read(&result)?; + let successful_models: Vec = native_result + .model_results + .iter() + .filter(|model| model.status == ExecutionStatus::Success) + .map(|model| model.model_name.clone()) + .collect(); append_freshness_records( - (&plan, &result), + (&plan, &successful_models), &adapter, &connection_config, &native_field(&project, "run_id")?, )?; lifecycle.finalize(py)?; Ok(Outcome { - result, + result: native_result, + executor_result: result, python_results: lifecycle.python_results(py)?, }) } @@ -520,13 +533,13 @@ pub(crate) fn run_post_build_python_checks<'py>( providers: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = context.py(); - let result: &Bound<'py, PyAny> = &outcome.result; - if native_field(result, "status")?.str()?.extract::()? != SUCCESS_STATUS { + let result = &outcome.result; + if result.status != BuildStatus::Success { return tuple(py, &[]); } let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?; let plan: Bound<'py, PyAny> = native_field(pipeline, "plan_output")?; - let load_results: Bound<'py, PyAny> = native_field(result, "load_results")?; + let load_results: Bound<'py, PyAny> = native_field(&outcome.executor_result, "load_results")?; let graph: Bound<'py, PyAny> = python_node_graph(&discovered)?; let by_loader = || -> PyResult> { Ok( @@ -538,13 +551,24 @@ pub(crate) fn run_post_build_python_checks<'py>( for name in by_loader()?.try_iter()? { dependencies.push(name?); } - let mut completed: Vec> = - names_of(&native_field(result, "model_results")?, "model_name")?; - completed.extend(names_of( - &native_field(result, "seed_results")?, - "seed_name", - )?); - completed.extend(names_of(&load_results, "source_name")?); + let completed: Vec> = result + .model_results + .iter() + .map(|model| model.model_name.as_str()) + .chain( + result + .seed_results + .iter() + .map(|seed| seed.seed_name.as_str()), + ) + .chain( + result + .load_results + .iter() + .map(|load| load.source_name.as_str()), + ) + .map(|name| text(py, name)) + .collect(); let checks: Vec> = relevant_check_functions( &discovered, &graph, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs index 894c07caaf..f19280d63c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/footer.rs @@ -3,7 +3,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::build_progress::main::format_build_footer::format_build_footer as footer; -use sqlbuild_core::build_progress::models::PythonNodeText; +use sqlbuild_core::build_progress::models::{BuildResultText, PythonNodeText}; use sqlbuild_core::cli_text::models::CliStyle; use crate::bindings::_helpers::build_progress::result_text::{build_text, python_node_text}; @@ -18,7 +18,7 @@ pub(crate) fn format_build_footer( python_node_results: Option<&Bound<'_, PyAny>>, ) -> PyResult { format_build_footer_with( - result, + &build_text(result)?, (elapsed, use_color), python_node_results, Vec::new(), @@ -27,7 +27,7 @@ pub(crate) fn format_build_footer( /// The build footer, counting `checks` (Python check results) with the Python node results. pub(crate) fn format_build_footer_with( - result: &Bound<'_, PyAny>, + result: &BuildResultText, (elapsed, use_color): (f64, bool), python_node_results: Option<&Bound<'_, PyAny>>, checks: Vec, @@ -39,12 +39,7 @@ pub(crate) fn format_build_footer_with( } } python.extend(checks); - Ok(footer( - &build_text(result)?, - &python, - elapsed, - CliStyle { use_color }, - )) + Ok(footer(result, &python, elapsed, CliStyle { use_color })) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs index 31bbc2b8c5..699a622437 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_progress/result_text.rs @@ -2,28 +2,34 @@ use std::collections::HashMap; +use pyo3::IntoPyObjectExt; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_core::build_progress::models::{ AuditResultText, BuildResultText, ErrorText, HookResultText, LifecycleEventText, LoadRowText, ModelResultText, NodeResultText, PythonNodeText, SkipText, TestResultText, TestStepText, }; use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::main::python_float_repr::python_float_repr; use sqlbuild_core::json::models::{JsonDialect, StdlibJsonOptions}; use sqlbuild_core::progress::main::format_expectation_detail::format_expectation_detail; +use sqlbuild_core::progress::models::ExpectationStep; use sqlbuild_core::test_command::main::parameterized_test_label::parameterized_test_label; use sqlbuild_core::test_command::models::TestParameter; +use sqlbuild_host::models::{ + AuditEvaluationMode, AuditExecutionResult, BuildExecutionResult, BuildStatus, ExecutionStatus, + FunctionExecutionResult, HookExecutionResult, HookPhase, LifeCycleEvent, LifeCycleEventKind, + LoadExecutionResult, MeasuredValue, ModelExecutionResult, SeedExecutionResult, + SqlTestExecutionResult, StepResult, +}; +use sqlbuild_host::types::ResultEnum; -use crate::bindings::_helpers::build_progress::progress_text::expectation_step; +use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::plan_output::display::{cursor_bound_display, inclusive_cursor_end}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::result_output::integration_values::json_value; -const SQL_EVENT: &str = "sql"; -const PRE_HOOKS: &str = "pre_hooks"; -const MEASUREMENT: &str = "measurement"; -const FAILED: &str = "failed"; const DECIMAL_KIND: &str = "decimal"; pub(crate) fn text(value: &Bound<'_, PyAny>, name: &str) -> PyResult { @@ -45,84 +51,80 @@ fn texts(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { .collect() } -fn items<'py>(value: &Bound<'py, PyAny>, name: &str) -> PyResult>> { - native_field(value, name)?.try_iter()?.collect() -} - -fn duration(value: &Bound<'_, PyAny>) -> PyResult> { - native_field(value, "duration_ms")?.extract() +fn error(code: &Option, message: &Option, help: &Option) -> ErrorText { + ErrorText { + code: code.clone(), + message: message.clone(), + help: help.clone(), + } } -fn error(value: &Bound<'_, PyAny>) -> PyResult { - Ok(ErrorText { - code: optional(value, "error_code")?, - message: optional(value, "error_message")?, - help: optional(value, "error_help")?, - }) +fn events(value: &[LifeCycleEvent]) -> Vec { + value + .iter() + .map(|event| match event.kind { + LifeCycleEventKind::Sql => LifecycleEventText::Sql(event.content.clone()), + LifeCycleEventKind::Log => LifecycleEventText::Log(event.content.clone()), + }) + .collect() } -fn skip(value: &Bound<'_, PyAny>) -> PyResult { - Ok(SkipText { - mode: optional(value, "skip_mode")?, - reason: optional(value, "skip_reason")?, - }) +pub(crate) fn audit_text(audit: &Bound<'_, PyAny>) -> PyResult { + Ok(audit_result_text(&AuditExecutionResult::read(audit)?)) } -fn events(value: &Bound<'_, PyAny>) -> PyResult> { - let mut events: Vec = Vec::new(); - for event in items(value, "lifecycle_events")? { - let content: String = native_field(&event, "content")?.extract()?; - events.push(if text(&event, "kind")? == SQL_EVENT { - LifecycleEventText::Sql(content) - } else { - LifecycleEventText::Log(content) - }); +fn audit_result_text(audit: &AuditExecutionResult>) -> AuditResultText { + AuditResultText { + audit_name: audit.audit_name.clone(), + attached_column_name: audit.attached_column_name.clone(), + attached_target_name: audit.attached_target_name.clone(), + run_scope_phase: audit.run_scope_phase.value().to_owned(), + outcome: audit.outcome.value().to_owned(), + row_count: audit.row_count, + reused: audit.reused, + executed_sql: Some(audit.executed_sql.clone()), + measurement: audit.evaluation_mode == AuditEvaluationMode::Measurement, + measured_value: match &audit.measured_value { + Some(MeasuredValue::Integer(value)) => value.as_str().to_owned(), + Some(MeasuredValue::Float(value)) => python_float_repr(*value), + None => "None".to_owned(), + }, + execution_error: audit.execution_error.clone(), + skip_reason: audit.skip_reason.clone(), } - Ok(events) -} - -pub(crate) fn audit_text(audit: &Bound<'_, PyAny>) -> PyResult { - Ok(AuditResultText { - audit_name: text(audit, "audit_name")?, - attached_column_name: optional(audit, "attached_column_name")?, - attached_target_name: optional(audit, "attached_target_name")?, - run_scope_phase: text(audit, "run_scope_phase")?, - outcome: text(audit, "outcome")?, - row_count: native_field(audit, "row_count")?.extract()?, - reused: native_field(audit, "reused")?.is_truthy()?, - executed_sql: optional(audit, "executed_sql")?, - measurement: text(audit, "evaluation_mode")? == MEASUREMENT, - measured_value: text(audit, "measured_value")?, - execution_error: optional(audit, "execution_error")?, - skip_reason: optional(audit, "skip_reason")?, - }) } -fn audits(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { - items(value, name)?.iter().map(audit_text).collect() +fn audits(value: &[AuditExecutionResult>]) -> Vec { + value.iter().map(audit_result_text).collect() } -fn hook_text(hook: &Bound<'_, PyAny>) -> PyResult { - Ok(HookResultText { - pre_hook: text(hook, "phase")? == PRE_HOOKS, - hook_type: text(hook, "hook_type")?, - label: text(hook, "label")?, - status: text(hook, "status")?, - skip: skip(hook)?, - }) +fn hook_text(hook: &HookExecutionResult) -> HookResultText { + HookResultText { + pre_hook: hook.phase == HookPhase::PreHooks, + hook_type: hook.hook_type.clone(), + label: hook.label.clone(), + status: hook.status.value().to_owned(), + skip: SkipText { + mode: hook.skip_mode.map(|mode| mode.value().to_owned()), + reason: hook.skip_reason.clone(), + }, + } } -fn cursor_range(model: &Bound<'_, PyAny>) -> PyResult> { - let start: Bound<'_, PyAny> = native_field(model, "cursor_range_start")?; - let end: Bound<'_, PyAny> = native_field(model, "cursor_range_end")?; - if start.is_none() - || end.is_none() - || native_field(model, "empty_cursor_inputs")?.is_truthy()? - { +fn cursor_range( + py: Python<'_>, + model: &ModelExecutionResult>, +) -> PyResult> { + let (Some(start), Some(end)) = (&model.cursor_range_start, &model.cursor_range_end) else { + return Ok(None); + }; + if !model.empty_cursor_inputs.is_empty() { return Ok(None); } - let cursor_type: Bound<'_, PyAny> = native_field(model, "cursor_type")?; - let cursor_grain: Bound<'_, PyAny> = native_field(model, "cursor_grain")?; + let start = start.into_bound_py_any(py)?; + let end = end.into_bound_py_any(py)?; + let cursor_type = model.cursor_type.as_deref().into_bound_py_any(py)?; + let cursor_grain = model.cursor_grain.as_deref().into_bound_py_any(py)?; Ok(Some(( cursor_bound_display(&start, &cursor_type, &cursor_grain)?, inclusive_cursor_end(&end, &cursor_type, &cursor_grain)?, @@ -130,85 +132,168 @@ fn cursor_range(model: &Bound<'_, PyAny>) -> PyResult> } pub(crate) fn model_text(model: &Bound<'_, PyAny>) -> PyResult { + model_result_text(model.py(), &ModelExecutionResult::read(model)?) +} + +fn model_result_text( + py: Python<'_>, + model: &ModelExecutionResult>, +) -> PyResult { Ok(ModelResultText { - name: text(model, "model_name")?, - status: text(model, "status")?, - failed_phase: optional(model, "failed_phase")?, - staging_relation: optional(model, "staging_relation")?, - duration_ms: duration(model)?, - batch_count: native_field(model, "batch_count")?.extract()?, - batch_size: optional(model, "batch_size")?, - rows_affected: native_field(model, "rows_affected")?.extract()?, - cursor_range: cursor_range(model)?, - empty_cursor_inputs: texts(model, "empty_cursor_inputs")?, - waiting_on_empty_inputs: native_field(model, "waiting_on_empty_inputs")?.is_truthy()?, - recovery_batch_count: native_field(model, "microbatch_recovery_batch_count")?.extract()?, - synthetic_completion_count: native_field(model, "microbatch_synthetic_completion_count")? - .extract()?, - replay_requirement_state: optional(model, "microbatch_replay_requirement_state")?, - audits: audits(model, "audit_results")?, - warnings: texts(model, "warning_messages")?, - events: events(model)?, - hooks: items(model, "hook_results")? - .iter() - .map(hook_text) - .collect::>()?, - skip: skip(model)?, - error: error(model)?, + name: model.model_name.clone(), + status: model.status.value().to_owned(), + failed_phase: model.failed_phase.map(|phase| phase.value().to_owned()), + staging_relation: model.staging_relation.clone(), + duration_ms: model.duration_ms.map(|duration| duration as f64), + batch_count: model.batch_count, + batch_size: model.batch_size.clone(), + rows_affected: model.rows_affected, + cursor_range: cursor_range(py, model)?, + empty_cursor_inputs: model.empty_cursor_inputs.clone(), + waiting_on_empty_inputs: model.waiting_on_empty_inputs, + recovery_batch_count: model.microbatch_recovery_batch_count, + synthetic_completion_count: model.microbatch_synthetic_completion_count, + replay_requirement_state: model.microbatch_replay_requirement_state.clone(), + audits: audits(&model.audit_results), + warnings: model.warning_messages.clone(), + events: events(&model.lifecycle_events), + hooks: model.hook_results.iter().map(hook_text).collect(), + skip: SkipText { + mode: model.skip_mode.map(|mode| mode.value().to_owned()), + reason: model.skip_reason.clone(), + }, + error: error(&model.error_code, &model.error_message, &model.error_help), }) } pub(crate) fn seed_text(seed: &Bound<'_, PyAny>) -> PyResult { - Ok(NodeResultText { - name: text(seed, "seed_name")?, - kind: "seed".to_owned(), - status: text(seed, "status")?, - duration_ms: duration(seed)?, - error: error(seed)?, - warnings: Vec::new(), - }) + Ok(seed_result_text(&SeedExecutionResult::read(seed)?)) +} + +fn seed_result_text(seed: &SeedExecutionResult) -> NodeResultText { + node_result_text( + (&seed.seed_name, "seed"), + (seed.status, seed.duration_ms), + error(&seed.error_code, &seed.error_message, &seed.error_help), + &[], + ) } pub(crate) fn function_text(function: &Bound<'_, PyAny>) -> PyResult { - Ok(NodeResultText { - name: text(function, "function_name")?, - kind: text(function, "function_kind")?, - status: text(function, "status")?, - duration_ms: duration(function)?, - error: error(function)?, - warnings: texts(function, "warning_messages")?, - }) + Ok(function_result_text(&FunctionExecutionResult::read( + function, + )?)) +} + +fn function_result_text(function: &FunctionExecutionResult) -> NodeResultText { + node_result_text( + (&function.function_name, &function.function_kind), + (function.status, function.duration_ms), + error( + &function.error_code, + &function.error_message, + &function.error_help, + ), + &function.warning_messages, + ) +} + +fn node_result_text( + (name, kind): (&str, &str), + (status, duration_ms): (ExecutionStatus, Option), + error: ErrorText, + warnings: &[String], +) -> NodeResultText { + NodeResultText { + name: name.to_owned(), + kind: kind.to_owned(), + status: status.value().to_owned(), + duration_ms: duration_ms.map(|duration| duration as f64), + error, + warnings: warnings.to_vec(), + } } pub(crate) fn load_text(load: &Bound<'_, PyAny>) -> PyResult { - Ok(LoadRowText { - source_name: text(load, "source_name")?, - loader_name: text(load, "loader_name")?, - resource_kind: text(load, "resource_kind")?, - status: text(load, "status")?, - rows_loaded: native_field(load, "rows_loaded")?.extract()?, - duration_ms: duration(load)?, - events: events(load)?, - skip: skip(load)?, - error_message: optional(load, "error_message")?, - warnings: texts(load, "warning_messages")?, - }) + Ok(load_result_text(&LoadExecutionResult::read(load)?)) +} + +fn load_result_text(load: &LoadExecutionResult>) -> LoadRowText { + LoadRowText { + source_name: load.source_name.clone(), + loader_name: load.loader_name.clone(), + resource_kind: load.resource_kind.value().to_owned(), + status: load.status.value().to_owned(), + rows_loaded: load.rows_loaded, + duration_ms: load.duration_ms.map(|duration| duration as f64), + events: events(&load.lifecycle_events), + skip: SkipText { + mode: load.skip_mode.map(|mode| mode.value().to_owned()), + reason: load.skip_reason.clone(), + }, + error_message: load.error_message.clone(), + warnings: load.warning_messages.clone(), + } } pub(crate) fn test_text(test: &Bound<'_, PyAny>) -> PyResult { - let mut steps: Vec = Vec::new(); - for step in items(test, "step_results")? { - steps.push(TestStepText { - model_name: text(&step, "model_name")?, - outcome: text(&step, "outcome")?, - detail: format_expectation_detail(&expectation_step(&step)?), - }); + test_result_text(&SqlTestExecutionResult::read(test)?) +} + +fn test_step_text(step: &StepResult) -> TestStepText { + TestStepText { + model_name: step.model_name.clone(), + outcome: step.outcome.value().to_owned(), + detail: format_expectation_detail(&ExpectationStep { + model_name: step.model_name.clone(), + outcome: step.outcome.value().to_owned(), + actual_row_count: step.actual_row_count, + expected_row_count: step.expected_row_count, + mismatched_row_count: step.mismatched_row_count, + unexpected_row_count: step.unexpected_row_count, + missing_row_count: step.missing_row_count, + unexpected_samples: step + .unexpected_samples + .iter() + .map(|sample| sample.values.clone()) + .collect(), + missing_samples: step + .missing_samples + .iter() + .map(|sample| sample.values.clone()) + .collect(), + column_differences: step + .column_differences + .iter() + .map(|column| { + ( + column.name.clone(), + column.actual.clone(), + column.expected.clone(), + ) + }) + .collect(), + }), } +} + +fn test_result_text(test: &SqlTestExecutionResult>) -> PyResult { Ok(TestResultText { - label: test_label(test, "test_name")?, - outcome: text(test, "outcome")?, - steps, - error: error(test)?, + label: test_label_values( + &test.test_name, + test.source_path + .as_ref() + .map(|path| { + path.to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "/") + }) + .as_deref(), + &test.parameter_schema, + &test.parameter_values, + )?, + outcome: test.outcome.value().to_owned(), + steps: test.step_results.iter().map(test_step_text).collect(), + error: error(&test.error_code, &test.error_message, &test.error_help), }) } @@ -223,32 +308,38 @@ pub(crate) fn python_node_text(node: &Bound<'_, PyAny>) -> PyResult) -> PyResult { + let py = result.py(); + let native = BuildExecutionResult::read(result)?; + build_result_text(py, &native) +} + +pub(crate) fn build_result_text( + py: Python<'_>, + native: &BuildExecutionResult>, +) -> PyResult { Ok(BuildResultText { - failed: text(result, "status")? == FAILED, - warning_count: native_field(result, "warning_count")?.extract()?, - insufficient_count: native_field(result, "insufficient_count")?.extract()?, - models: items(result, "model_results")? - .iter() - .map(model_text) - .collect::>()?, - seeds: items(result, "seed_results")? + failed: native.status == BuildStatus::Failed, + warning_count: native.warning_count, + insufficient_count: native.insufficient_count, + models: native + .model_results .iter() - .map(seed_text) + .map(|model| model_result_text(py, model)) .collect::>()?, - functions: items(result, "function_results")? + seeds: native.seed_results.iter().map(seed_result_text).collect(), + functions: native + .function_results .iter() - .map(function_text) - .collect::>()?, - loads: items(result, "load_results")? - .iter() - .map(load_text) - .collect::>()?, - tests: items(result, "test_results")? + .map(function_result_text) + .collect(), + loads: native.load_results.iter().map(load_result_text).collect(), + tests: native + .test_results .iter() - .map(test_text) + .map(test_result_text) .collect::>()?, - source_audits: audits(result, "source_audit_results")?, - end_audits: audits(result, "end_audit_results")?, + source_audits: audits(&native.source_audit_results), + end_audits: audits(&native.end_audit_results), }) } @@ -256,22 +347,32 @@ pub(crate) fn build_text(result: &Bound<'_, PyAny>) -> PyResult pub(crate) fn test_label(item: &Bound<'_, PyAny>, name_field: &str) -> PyResult { let name: String = native_field(item, name_field)?.extract()?; let source_path: Bound<'_, PyAny> = item.getattr("source_path")?; - let values: Vec> = native_field(item, "parameter_values")? - .try_iter()? - .collect::>()?; + let values = Vec::<(String, Bound<'_, PyAny>)>::read(&native_field(item, "parameter_values")?)?; if source_path.is_none() || values.is_empty() { return Ok(name); } + let path: String = source_path.call_method0("as_posix")?.extract()?; + let declarations = Vec::>::read(&item.getattr("parameter_schema")?)?; + test_label_values(&name, Some(&path), &declarations, &values) +} + +fn test_label_values( + name: &str, + path: Option<&str>, + declarations: &[Bound<'_, PyAny>], + values: &[(String, Bound<'_, PyAny>)], +) -> PyResult { + if path.is_none() || values.is_empty() { + return Ok(name.to_owned()); + } let mut schema: HashMap> = HashMap::new(); - for declaration in item.getattr("parameter_schema")?.try_iter()? { - let declaration: Bound<'_, PyAny> = declaration?; - schema.insert(declaration.getattr("name")?.extract()?, declaration); + for declaration in declarations { + schema.insert(declaration.getattr("name")?.extract()?, declaration.clone()); } let mut parameters: Vec = Vec::new(); - for pair in values { - let (value_name, value): (String, Bound<'_, PyAny>) = pair.extract()?; + for (value_name, value) in values { let declaration: &Bound<'_, PyAny> = schema - .get(&value_name) + .get(value_name) .ok_or_else(|| pyo3::exceptions::PyKeyError::new_err(value_name.clone()))?; let raw: Bound<'_, PyAny> = value.getattr("value")?; let kind: String = value.getattr("kind")?.getattr("value")?.extract()?; @@ -281,15 +382,14 @@ pub(crate) fn test_label(item: &Bound<'_, PyAny>, name_field: &str) -> PyResult< raw }; parameters.push(TestParameter { - name: value_name, + name: value_name.clone(), value_type: native_field(&native_field(declaration, "value_type")?, "value")? .extract()?, nullable: declaration.getattr("nullable")?.is_truthy()?, rendered: compact_json(&payload)?, }); } - let path: String = source_path.call_method0("as_posix")?.extract()?; - Ok(parameterized_test_label(&name, Some(&path), ¶meters)) + Ok(parameterized_test_label(name, path, ¶meters)) } /// A payload as `json.dumps(payload, ensure_ascii=True, separators=(",", ":"))`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs index 3ce0e1ffaf..36d922bc4a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs @@ -13,11 +13,11 @@ use sqlbuild_analysis::target_paths::main::runtime_sql_text::runtime_sql_text; use sqlbuild_analysis::target_paths::main::sql_test_output_path::sql_test_output_path; use sqlbuild_analysis::target_paths::models::SqlTestPathFacts; use sqlbuild_core::text::main::python_isspace::python_isspace; -use sqlbuild_host::entries::models::AdapterHost; +use sqlbuild_host::models::{BuildExecutionResult, LifeCycleEvent, LifeCycleEventKind}; use crate::bindings::_helpers::analysis::sql_test_comparisons::single_comparison_sql; use crate::bindings::_helpers::boundary::python_calls::imported; -use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::records::fields::field as native_field; const RUN_DIR: &str = "run"; @@ -107,19 +107,6 @@ fn write_text(path: &Bound<'_, PyAny>, text: &str) -> PyResult<()> { Ok(()) } -fn sql_statements(result: &Bound<'_, PyAny>) -> PyResult> { - let sql_kind: Bound<'_, PyAny> = - host::entry(result.py(), AdapterHost::LifeCycleEventKind)?.getattr("SQL")?; - let mut statements: Vec = Vec::new(); - for event in native_field(result, "lifecycle_events")?.try_iter()? { - let event: Bound<'_, PyAny> = event?; - if native_field(&event, "kind")?.eq(&sql_kind)? { - statements.push(native_field(&event, "content")?.extract()?); - } - } - Ok(statements) -} - fn entry_paths( entries: &Bound<'_, PyAny>, path_of: impl Fn(&Bound<'_, PyAny>) -> PyResult, @@ -142,7 +129,7 @@ fn relative_posix(entry: &Bound<'_, PyAny>) -> PyResult { pub(crate) fn write_runtime_target( target_dir: &Bound<'_, PyAny>, plan: &Bound<'_, PyAny>, - result: &Bound<'_, PyAny>, + result: &BuildExecutionResult>, ) -> PyResult<()> { let run_dir: Bound<'_, PyAny> = target_dir.div(RUN_DIR)?; let models: HashMap = @@ -158,20 +145,26 @@ pub(crate) fn write_runtime_target( .extract::()?, )) })?; - for (results, name_field, paths) in [ - ("model_results", "model_name", &models), - ("function_results", "function_name", &functions), - ] { - for item in native_field(result, results)?.try_iter()? { - let item: Bound<'_, PyAny> = item?; - let statements: Vec = sql_statements(&item)?; - if statements.is_empty() { - continue; - } - let name: String = native_field(&item, name_field)?.extract()?; - if let Some(path) = paths.get(&name) { - write_text(&run_dir.div(path)?, &runtime_sql_text(&statements))?; - } + let models = result.model_results.iter().map(|model| { + ( + models.get(&model.model_name), + model.lifecycle_events.as_slice(), + ) + }); + let functions = result.function_results.iter().map(|function| { + ( + functions.get(&function.function_name), + function.lifecycle_events.as_slice(), + ) + }); + for (path, events) in models.chain(functions) { + let statements: Vec = events + .iter() + .filter(|event: &&LifeCycleEvent| event.kind == LifeCycleEventKind::Sql) + .map(|event| event.content.clone()) + .collect(); + if let Some(path) = path.filter(|_| !statements.is_empty()) { + write_text(&run_dir.div(path)?, &runtime_sql_text(&statements))?; } } Ok(()) @@ -265,7 +258,11 @@ fn write_runtime_target_binding( plan_output: &Bound<'_, PyAny>, result: &Bound<'_, PyAny>, ) -> PyResult<()> { - write_runtime_target(target_dir, plan_output, result) + write_runtime_target( + target_dir, + plan_output, + &BuildExecutionResult::read(result)?, + ) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs index 3330e9d5df..4622ba1f67 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_views/relation_keys.rs @@ -168,8 +168,6 @@ fn unique_key(values: &Bound<'_, PyDict>) -> PyResult>> { } /// The key row of a seed, source or hand-built model from its config values and schema entry. -#[pyfunction(name = "relation_key_row")] -#[pyo3(signature = (kind, name, values, entry))] pub(crate) fn entry_relation_key_row( kind: &str, name: String, @@ -347,7 +345,6 @@ fn declared_relation(row: RelationKeyRow) -> DeclaredRelation { } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(entry_relation_key_row, module)?)?; module.add_function(wrap_pyfunction!(declared_relation_keys, module)?)?; module.add_function(wrap_pyfunction!(compiled_relation_keys, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs index b4b4cfdfe5..9f9f57ce84 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs @@ -1,320 +1,132 @@ -//! `sqb cost`: persisted run-cost history or one run's breakdown. +//! The permanent Python request/refresh edge of native cost reporting. + +use std::path::PathBuf; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyDict, PyDictMethods, PyInt, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::cost_command::constants::{ DEFAULT_DETAIL_SORT, DEFAULT_HISTORY_SORT, DESCENDING_ORDER, HISTORY_SELECTOR, ISO_DATE_LENGTH, - LATEST_SELECTOR, }; -use sqlbuild_core::cost_command::errors::CostCommandError; -use sqlbuild_core::cost_command::main::cost_history_limit::cost_history_limit; -use sqlbuild_core::cost_command::main::select_cost_run::select_cost_run; +use sqlbuild_core::cost_command::main::render_cost_report::render_cost_report; use sqlbuild_core::cost_command::main::validate_cost_request::validate_cost_request; +use sqlbuild_core::cost_command::models::{CostReport, CostReportOptions}; +use sqlbuild_core::cost_report::main::cost_timestamp::cost_timestamp; use crate::bindings::_helpers::boundary::python_calls::imported; use crate::bindings::_helpers::cost_command::bounds::{parse_bound, parse_since}; use crate::bindings::_helpers::cost_command::refresh::refresh_pending_cost_run; use crate::bindings::_helpers::cost_command::refusal::cost_error; -use crate::bindings::_helpers::cost_command::sorting::{sort_history, sort_resources}; -use crate::bindings::_helpers::cost_report::functions::format_cost_breakdown; -use crate::bindings::_helpers::cost_report::functions::format_cost_history; -use crate::bindings::_helpers::cost_report::functions::format_cost_history_json; -use crate::bindings::_helpers::cost_report::functions::format_cost_json; -use crate::bindings::_helpers::cost_report::functions::list_cost_runs; -use crate::bindings::_helpers::cost_report::functions::resolve_cost_run; +use crate::bindings::_helpers::cost_report::store; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::records::fields::replaced_keywords; +use crate::bindings::_helpers::records::fields::field; -/// The request fields every cost view reads. -struct CostRequest<'py> { - request: Bound<'py, PyAny>, - project_dir: Bound<'py, PyAny>, - selector: String, - limit: Option, - no_limit: bool, - descending: bool, - since: Option, - until: Option, +/// Preserve Python's datetime parsing at the public request edge, then leave Python records behind. +fn instant(value: &Bound<'_, PyAny>) -> PyResult { + let text: String = value.call_method0("isoformat")?.extract()?; + cost_timestamp(&text) + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Invalid UTC cost bound")) } -impl<'py> CostRequest<'py> { - fn read(request: &Bound<'py, PyAny>) -> PyResult { - let py: Python<'py> = request.py(); - let requested: Bound<'py, PyAny> = native_field(request, "project_dir")?; - let order: Option = native_field(request, "order")?.extract()?; - Ok(Self { - request: request.clone(), - project_dir: if requested.is_none() { - imported(py, "pathlib", "Path")?.call_method0("cwd")? - } else { - requested - }, - selector: native_field(request, "selector")?.extract()?, - limit: native_field(request, "limit")?.extract()?, - no_limit: native_field(request, "no_limit")?.is_truthy()?, - descending: order - .filter(|order| !order.is_empty()) - .is_none_or(|order| order == DESCENDING_ORDER), - since: native_field(request, "since")?.extract()?, - until: native_field(request, "until")?.extract()?, - }) - } - - fn sort_field(&self, default: &str) -> PyResult { - let sort: Option = self.request.getattr("sort")?.extract()?; - Ok(sort - .filter(|sort| !sort.is_empty()) - .unwrap_or_else(|| default.to_owned())) - } - - fn use_color(&self) -> PyResult { - if self.request.getattr("no_color")?.is_truthy()? { - return Ok(false); - } - stdout_supports_color(self.request.py()) - } - - /// Write the JSON file when asked, then the JSON or the rendered text to stdout. - fn emit(&self, payload: &str, text: impl FnOnce() -> PyResult) -> PyResult<()> { - let py: Python<'py> = self.request.py(); - let path: Bound<'py, PyAny> = self.request.getattr("json_output_path")?; - if !path.is_none() { - let options: Bound<'py, PyDict> = PyDict::new(py); - options.set_item("parents", true)?; - options.set_item("exist_ok", true)?; - native_field(&path, "parent")?.call_method("mkdir", (), Some(&options))?; - let encoding: Bound<'py, PyDict> = PyDict::new(py); - encoding.set_item("encoding", "utf-8")?; - path.call_method("write_text", (format!("{payload}\n"),), Some(&encoding))?; - } - let rendered: String = if self.request.getattr("json_output")?.is_truthy()? { - payload.to_owned() +fn options(request: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = request.py(); + let selector: String = field(request, "selector")?.extract()?; + let history: bool = selector == HISTORY_SELECTOR; + let sort: Option = field(request, "sort")?.extract()?; + let sort: String = sort.filter(|sort| !sort.is_empty()).unwrap_or_else(|| { + if history { + DEFAULT_HISTORY_SORT } else { - text()? - }; - py.import("sys")? - .getattr("stdout")? - .call_method1("write", (PyString::new(py, &format!("{rendered}\n")),))?; - Ok(()) - } -} - -fn refused(py: Python<'_>, result: Result<(), CostCommandError>) -> PyResult<()> { - result.map_err(|error| cost_error(py, &error)) -} - -fn cost_runs<'py>(request: &CostRequest<'py>) -> PyResult>> { - let py: Python<'py> = request.request.py(); - list_cost_runs( - py, - request - .project_dir - .clone() - .extract::()?, - )? - .try_iter()? - .collect() -} - -/// Whether the record completed inside the `--since`/`--until` window. -fn in_window( - record: &Bound<'_, PyAny>, - since: Option<&Bound<'_, PyAny>>, - (until, until_is_date): (Option<&Bound<'_, PyAny>>, bool), -) -> PyResult { - let completed: Bound<'_, PyAny> = native_field(record, "completed_at")?; - if let Some(since) = since - && !completed.ge(since)? - { - return Ok(false); - } - match until { - Some(until) if until_is_date => completed.lt(until), - Some(until) => completed.le(until), - None => Ok(true), - } -} - -fn run_history(request: &CostRequest<'_>) -> PyResult { - let py: Python<'_> = request.request.py(); - let sort_field: String = request.sort_field(DEFAULT_HISTORY_SORT)?; - refused( - py, - validate_cost_request( - &request.selector, - (request.limit, request.no_limit), - &sort_field, - request.since.is_some() || request.until.is_some(), - ), - )?; - let since: Option> = request - .since - .as_deref() - .map(|value| parse_since(py, value)) - .transpose()?; - let until: Option> = request - .until - .as_deref() - .map(|value| parse_bound(py, value, "--until", true)) - .transpose()?; - if let (Some(since), Some(until)) = (&since, &until) - && since.gt(until)? - { - return Err(cost_error( - py, - &CostCommandError::new("C711", "cost --since must not be after --until"), - )); - } - let until_is_date: bool = request - .until - .as_deref() - .is_some_and(|value| value.chars().count() == ISO_DATE_LENGTH); - let mut matching: Vec> = Vec::new(); - for record in cost_runs(request)? { - if in_window(&record, since.as_ref(), (until.as_ref(), until_is_date))? { - matching.push(record); + DEFAULT_DETAIL_SORT } - } - let matching_count: usize = matching.len(); - let mut records: Vec> = - sort_history(py, matching, &sort_field, request.descending)?; - if let Some(limit) = cost_history_limit(request.limit, request.no_limit) { - records.truncate(usize::try_from(limit).unwrap_or(0)); - } - let records: Bound<'_, PyAny> = PyTuple::new(py, records)?.into_any(); - let count: Bound<'_, PyAny> = PyInt::new(py, matching_count).into_any(); - let payload: String = - format_cost_history_json(&records.clone(), count.clone().extract::>()?) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? - .extract()?; - request.emit(&payload, || { - format_cost_history( - &records, - count.extract::()?, - PyBool::new(py, request.use_color()?) - .to_owned() - .into_any() - .extract::()?, - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? - .extract() - })?; - Ok(0) + .to_owned() + }); + let limit: Option = field(request, "limit")?.extract()?; + let no_limit: bool = field(request, "no_limit")?.is_truthy()?; + let since: Option = field(request, "since")?.extract()?; + let until: Option = field(request, "until")?.extract()?; + validate_cost_request( + &selector, + (limit, no_limit), + &sort, + since.is_some() || until.is_some(), + ) + .map_err(|error| cost_error(py, &error))?; + let order: Option = field(request, "order")?.extract()?; + Ok(CostReportOptions { + selector, + limit, + no_limit, + sort, + descending: order + .filter(|order| !order.is_empty()) + .is_none_or(|order| order == DESCENDING_ORDER), + since: since + .as_deref() + .map(|value| instant(&parse_since(py, value)?)) + .transpose()?, + until: until + .as_deref() + .map(|value| instant(&parse_bound(py, value, "--until", true)?)) + .transpose()?, + until_exclusive: until + .as_deref() + .is_some_and(|value| value.chars().count() == ISO_DATE_LENGTH), + use_color: !field(request, "no_color")?.is_truthy()? && stdout_supports_color(py)?, + terminal_width: py + .import("shutil")? + .call_method1("get_terminal_size", ((100, 24),))? + .getattr("columns")? + .extract()?, + }) } -/// The persisted run the selector names. -fn resolve_run<'py>(request: &CostRequest<'py>) -> PyResult> { - let py: Python<'py> = request.request.py(); - let records: Vec> = cost_runs(request)?; - if request.selector == LATEST_SELECTOR { - let latest: Bound<'py, PyAny> = resolve_cost_run( - py, - request - .project_dir - .clone() - .extract::()?, - &PyString::new(py, &request.selector) - .into_any() - .extract::()?, - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; - if latest.is_none() { - return Err(cost_error( - py, - &CostCommandError::new("C704", "No persisted SQLBuild cost runs were found"), - )); - } - return Ok(latest); +fn emit(request: &Bound<'_, PyAny>, report: CostReport) -> PyResult<()> { + let py: Python<'_> = request.py(); + let path: Bound<'_, PyAny> = field(request, "json_output_path")?; + if !path.is_none() { + let options: Bound<'_, PyDict> = PyDict::new(py); + options.set_item("parents", true)?; + options.set_item("exist_ok", true)?; + field(&path, "parent")?.call_method("mkdir", (), Some(&options))?; + let encoding: Bound<'_, PyDict> = PyDict::new(py); + encoding.set_item("encoding", "utf-8")?; + path.call_method( + "write_text", + (format!("{}\n", report.json),), + Some(&encoding), + )?; } - let run_ids: Vec = records - .iter() - .map(|record| native_field(record, "run_id")?.extract()) - .collect::>()?; - let index: usize = - select_cost_run(&run_ids, &request.selector).map_err(|error| cost_error(py, &error))?; - Ok(records[index].clone()) -} - -/// `dataclasses.replace(record, cost=replace(record.cost, resources=resources))`. -fn with_resources<'py>( - record: &Bound<'py, PyAny>, - resources: &[Bound<'py, PyAny>], -) -> PyResult> { - let py: Python<'py> = record.py(); - let cost_changes: Bound<'py, PyDict> = PyDict::new(py); - cost_changes.set_item("resources", PyTuple::new(py, resources)?)?; - let cost: Bound<'py, PyAny> = replaced_keywords(&native_field(record, "cost")?, &cost_changes)?; - let changes: Bound<'py, PyDict> = PyDict::new(py); - changes.set_item("cost", cost)?; - replaced_keywords(record, &changes) -} - -fn run_detail(request: &CostRequest<'_>) -> PyResult { - let py: Python<'_> = request.request.py(); - let sort_field: String = request.sort_field(DEFAULT_DETAIL_SORT)?; - refused( - py, - validate_cost_request( - &request.selector, - (request.limit, request.no_limit), - &sort_field, - request.since.is_some() || request.until.is_some(), - ), - )?; - let record: Bound<'_, PyAny> = resolve_run(request)?; - let resources: Vec> = native_field(&record, "cost")? - .getattr("resources")? - .try_iter()? - .collect::>()?; - let resources: Vec> = - sort_resources(py, resources, &sort_field, request.descending)?; - let record: Bound<'_, PyAny> = with_resources(&record, &resources)?; - let limit: Option = if request.no_limit { - None + let rendered: String = if field(request, "json_output")?.is_truthy()? { + report.json } else { - request.limit - }; - let json_record: Bound<'_, PyAny> = match limit { - Some(limit) => { - let kept: usize = usize::try_from(limit).unwrap_or(0).min(resources.len()); - with_resources(&record, &resources[..kept])? - } - None => record.clone(), + report.text }; - let payload: String = format_cost_json(&json_record) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? - .extract()?; - request.emit(&payload, || { - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("record", &record)?; - options.set_item("use_color", request.use_color()?)?; - options.set_item("limit", limit)?; - format_cost_breakdown(py, Some(options.clone())) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? - .extract() - })?; - Ok(0) -} - -/// Render persisted cost history or one run breakdown, without the delayed refresh. -fn report(request: &Bound<'_, PyAny>) -> PyResult { - let request: CostRequest<'_> = CostRequest::read(request)?; - if request.selector == HISTORY_SELECTOR { - return run_history(&request); - } - run_detail(&request) + py.import("sys")? + .getattr("stdout")? + .call_method1("write", (format!("{rendered}\n"),))?; + Ok(()) } -/// Refresh a delayed run, then render persisted cost history or one run breakdown. +/// Refresh through the adapter edge, then render entirely from native persisted records. #[pyfunction] pub(crate) fn run_cost_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { refresh_pending_cost_run(py, request); - report(request) + let options: CostReportOptions = options(request)?; + let requested: Bound<'_, PyAny> = field(request, "project_dir")?; + let project_dir: PathBuf = if requested.is_none() { + imported(py, "pathlib", "Path")? + .call_method0("cwd")? + .extract()? + } else { + requested.extract()? + }; + let report: CostReport = render_cost_report(store::list(py, &project_dir)?, &options) + .map_err(|error| cost_error(py, &error))?; + emit(request, report)?; + Ok(0) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(run_cost_command, module)?)?; - Ok(()) + module.add_function(wrap_pyfunction!(run_cost_command, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/mod.rs index d5730144e0..7df0ec70e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/mod.rs @@ -3,4 +3,3 @@ pub(crate) mod command; pub(crate) mod refresh; pub(crate) mod refusal; pub(crate) mod registration; -pub(crate) mod sorting; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/sorting.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/sorting.rs deleted file mode 100644 index a6c9205b50..0000000000 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/sorting.rs +++ /dev/null @@ -1,142 +0,0 @@ -//! Python's stable `sorted` over cost records and resources, keyed by the requested metric. - -use pyo3::exceptions::PyKeyError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyInt, PyList, PyTuple, PyTupleMethods}; -use sqlbuild_core::cost_command::constants::{HISTORY_METRIC_SORT_FIELDS, MISSING_METRIC_STATUSES}; - -use crate::bindings::_helpers::boundary::python_calls::imported; -use crate::bindings::_helpers::records::fields::field as native_field; - -type SortKey = fn(&Bound<'_, PyAny>, &str) -> PyResult>; - -/// `sorted(items, key=lambda item: key(item, field), reverse=reverse)`. -fn sorted_with<'py>( - py: Python<'py>, - items: Vec>, - (key, field): (SortKey, &str), - reverse: bool, -) -> PyResult>> { - let field: String = field.to_owned(); - let function = move |arguments: &Bound<'_, PyTuple>, - _: Option<&Bound<'_, PyDict>>| - -> PyResult> { key(&arguments.get_item(0)?, &field) }; - let options: Bound<'py, PyDict> = PyDict::new(py); - options.set_item("key", PyCFunction::new_closure(py, None, None, function)?)?; - options.set_item("reverse", reverse)?; - imported(py, "builtins", "sorted")? - .call((PyList::new(py, items)?,), Some(&options))? - .try_iter()? - .collect() -} - -fn attribute(item: &Bound<'_, PyAny>, path: &str) -> PyResult> { - let mut value: Bound<'_, PyAny> = item.clone(); - for name in path.split('.') { - value = native_field(&value, name)?; - } - Ok(value.unbind()) -} - -fn history_value(record: &Bound<'_, PyAny>, field: &str) -> PyResult> { - let py: Python<'_> = record.py(); - match field { - "completed" => attribute(record, "completed_at"), - "cost" => attribute(record, "cost.estimated_usd"), - "credits" => attribute(record, "cost.estimated_compute_credits"), - "duration" => Ok(native_field(record, "completed_at")? - .sub(native_field(record, "started_at")?)? - .unbind()), - "models" => Ok(PyInt::new( - py, - native_field(record, "cost")?.getattr("resources")?.len()?, - ) - .into_any() - .unbind()), - "status" => attribute(record, "cost.status.value"), - _ => Err(PyKeyError::new_err(field.to_owned())), - } -} - -fn resource_value(resource: &Bound<'_, PyAny>, field: &str) -> PyResult> { - let path: &str = match field { - "cost" => "estimated_usd", - "credits" => "estimated_compute_credits", - "busy" => "attributed_seconds", - "scanned" => "bytes_scanned", - "model" => "resource_name", - "warehouse" => "warehouse_name", - _ => return Err(PyKeyError::new_err(field.to_owned())), - }; - let py: Python<'_> = resource.py(); - Ok(PyTuple::new( - py, - [ - attribute(resource, path)?, - attribute(resource, "resource_name")?, - attribute(resource, "warehouse_name")?, - ], - )? - .into_any() - .unbind()) -} - -fn by_attribute(item: &Bound<'_, PyAny>, path: &str) -> PyResult> { - attribute(item, path) -} - -/// Newest first with run ids as the tie-break, then stably by the field. -fn sort_partition<'py>( - py: Python<'py>, - records: Vec>, - field: &str, - descending: bool, -) -> PyResult>> { - let by_run_id = sorted_with(py, records, (by_attribute, "run_id"), false)?; - let by_completion = sorted_with(py, by_run_id, (by_attribute, "completed_at"), true)?; - sorted_with(py, by_completion, (history_value, field), descending) -} - -/// History records by the field; metric sorts put runs without metrics last, newest first. -pub(crate) fn sort_history<'py>( - py: Python<'py>, - records: Vec>, - field: &str, - descending: bool, -) -> PyResult>> { - if !HISTORY_METRIC_SORT_FIELDS.contains(&field) { - return sort_partition(py, records, field, descending); - } - let mut available: Vec> = Vec::new(); - let mut missing: Vec> = Vec::new(); - for record in records { - let status: String = native_field(&record, "cost")? - .getattr("status")? - .getattr("value")? - .extract()?; - if MISSING_METRIC_STATUSES.contains(&status.as_str()) { - missing.push(record); - } else { - available.push(record); - } - } - let mut sorted: Vec> = sort_partition(py, available, field, descending)?; - let missing = sorted_with(py, missing, (by_attribute, "run_id"), false)?; - sorted.extend(sorted_with( - py, - missing, - (by_attribute, "completed_at"), - true, - )?); - Ok(sorted) -} - -/// Breakdown resources by the field, then resource and warehouse name. -pub(crate) fn sort_resources<'py>( - py: Python<'py>, - resources: Vec>, - field: &str, - descending: bool, -) -> PyResult>> { - sorted_with(py, resources, (resource_value, field), descending) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs index 2679840a9f..ecc1512232 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs @@ -1,127 +1,34 @@ -//! Project-local cost run artifacts under `target/executions//run.json`. +//! Python error translation for the shared native cost artifact store. -use std::fs; use std::path::{Path, PathBuf}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyResult, Python}; -use sqlbuild_core::cost_report::errors::CostArtifactError; -use sqlbuild_core::cost_report::main::check_cost_run_id::check_cost_run_id; -use sqlbuild_core::cost_report::main::cost_artifact_text::cost_artifact_text; -use sqlbuild_core::cost_report::main::order_cost_runs::order_cost_runs; -use sqlbuild_core::cost_report::main::read_cost_artifact::read_cost_artifact; +use pyo3::prelude::{PyErr, PyResult, Python}; +use sqlbuild_cli::cost_store::errors::CostStoreError; +use sqlbuild_cli::cost_store::main::list_cost_runs::list_cost_runs; +use sqlbuild_cli::cost_store::main::read_cost_run::read_cost_run; +use sqlbuild_cli::cost_store::main::write_cost_run::write_cost_run; use sqlbuild_core::cost_report::models::CostRun; use crate::bindings::_helpers::cost_report::records::artifact_error; -use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::result_output::integration_values::json_value; -const ARTIFACT_NAME: &str = "run.json"; -const TEMPORARY_NAME: &str = ".run.json.tmp"; - -fn invalid(message: &str) -> CostArtifactError { - CostArtifactError { - message: message.to_owned(), - } -} - -/// The directory every run's artifact lives under. -fn executions_root(project_dir: &Path) -> PathBuf { - project_dir.join("target").join("executions") -} - -/// `_run_directory`: the run's directory, which must stay under the executions root. -fn run_directory(project_dir: &Path, run_id: &str) -> Result { - check_cost_run_id(run_id)?; - let root: PathBuf = executions_root(project_dir); - let directory: PathBuf = root.join(run_id); - if let (Ok(resolved), Ok(resolved_root)) = (directory.canonicalize(), root.canonicalize()) - && !resolved.starts_with(resolved_root) - { - return Err(invalid( - "cost run path must remain under the project execution directory", - )); +fn failure(py: Python<'_>, error: CostStoreError) -> PyErr { + match error { + CostStoreError::Io(error) => PyErr::from(error), + CostStoreError::Artifact(error) => artifact_error(py, &error), } - Ok(directory) -} - -/// The run an artifact file describes; I/O failures stay Python `OSError`s. -fn load(py: Python<'_>, path: &Path) -> PyResult> { - let bytes: Vec = fs::read(path).map_err(PyErr::from)?; - let Ok(text) = String::from_utf8(bytes) else { - return Ok(Err(invalid("cost run artifact is not UTF-8 text"))); - }; - let json: Bound<'_, PyModule> = py.import("json")?; - let payload: Bound<'_, PyAny> = match json.call_method1("loads", (text,)) { - Ok(payload) => payload, - Err(error) if error.is_instance(py, &native_field(&json, "JSONDecodeError")?) => { - return Ok(Err(invalid("cost run artifact is not valid JSON"))); - } - Err(error) => return Err(error), - }; - let Some(payload) = json_value(&payload)? else { - return Ok(Err(invalid("cost run artifact must be an object"))); - }; - Ok(read_cost_artifact(&payload)) } -/// Raise the artifact failure Python reported for the same file. -fn checked(py: Python<'_>, result: Result) -> PyResult { - result.map_err(|error| artifact_error(py, &error)) -} - -/// `write_cost_run`: atomically replace the run's artifact and return its path. +/// Atomically persist one run using the same implementation as the native launcher. pub(crate) fn write(py: Python<'_>, project_dir: &Path, run: &CostRun) -> PyResult { - let directory: PathBuf = checked(py, run_directory(project_dir, &run.run_id))?; - fs::create_dir_all(&directory).map_err(PyErr::from)?; - let destination: PathBuf = directory.join(ARTIFACT_NAME); - let temporary: PathBuf = directory.join(TEMPORARY_NAME); - fs::write(&temporary, cost_artifact_text(run)).map_err(PyErr::from)?; - fs::rename(&temporary, &destination).map_err(PyErr::from)?; - Ok(destination) + write_cost_run(project_dir, run).map_err(|error| failure(py, error)) } -/// `read_cost_run`: the run stored under the ID, `None` when it has no artifact. +/// Read one exact artifact, preserving Python's artifact versus I/O error distinction. pub(crate) fn read(py: Python<'_>, project_dir: &Path, run_id: &str) -> PyResult> { - let path: PathBuf = checked(py, run_directory(project_dir, run_id))?.join(ARTIFACT_NAME); - if !path.is_file() { - return Ok(None); - } - let run: CostRun = checked(py, load(py, &path)?)?; - if run.run_id != run_id { - return Err(artifact_error( - py, - &invalid("cost run artifact run ID does not match its directory"), - )); - } - Ok(Some(run)) + read_cost_run(project_dir, run_id).map_err(|error| failure(py, error)) } -fn listed_run(py: Python<'_>, root: &Path, directory: &Path) -> PyResult> { - let path: PathBuf = directory.join(ARTIFACT_NAME); - let (Ok(resolved), Ok(resolved_root)) = (path.canonicalize(), root.canonicalize()) else { - return Ok(None); - }; - if !resolved.starts_with(resolved_root) || !path.is_file() { - return Ok(None); - } - let Ok(Ok(run)) = load(py, &path) else { - return Ok(None); - }; - let name: Option<&str> = directory.file_name().and_then(|name| name.to_str()); - Ok((name == Some(run.run_id.as_str())).then_some(run)) -} - -/// `list_cost_runs`: every readable run, newest first; unreadable artifacts are skipped. -pub(crate) fn list(py: Python<'_>, project_dir: &Path) -> PyResult> { - let root: PathBuf = executions_root(project_dir); - let Ok(entries) = fs::read_dir(&root) else { - return Ok(Vec::new()); - }; - let mut runs: Vec = Vec::new(); - for entry in entries.flatten() { - if let Some(run) = listed_run(py, &root, &entry.path())? { - runs.push(run); - } - } - Ok(order_cost_runs(runs)) +/// The shared store skips unreadable artifacts and orders the rest newest first. +pub(crate) fn list(_py: Python<'_>, project_dir: &Path) -> PyResult> { + Ok(list_cost_runs(project_dir)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs index 1d94516925..4a37ac2c33 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs @@ -156,11 +156,6 @@ impl<'py> CursorSession<'py> { if inner { held.clone() } else { value.clone() } } - /// The `date`, `datetime` or `int` input value `handle` holds. - pub(crate) fn held(&self, handle: usize) -> Bound<'py, PyAny> { - self.input(handle, true) - } - /// The `datetime` holding `local` in the zone and type of input `template`. fn date_time(&self, timestamp: &Timestamp) -> PyResult> { match timestamp.template { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs b/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs index c75f175a30..002b76b844 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cursor/scalars.rs @@ -22,12 +22,12 @@ use sqlbuild_cursor::algebra::main::next_boundary::next_boundary; use sqlbuild_cursor::algebra::main::normalize_timestamp_text::normalize_timestamp_text; use sqlbuild_cursor::algebra::main::parse_integer_text::parse_integer_text; use sqlbuild_cursor::algebra::main::render::render; -use sqlbuild_cursor::algebra::models::{Extreme, Grain, Scalar, ScalarValue, SortKey, ZoneOffset}; +use sqlbuild_cursor::algebra::models::{Extreme, Grain, Scalar, SortKey}; use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::boundary::python_calls::imported; use crate::bindings::_helpers::cursor::conversion::{CursorSession, algebra_error}; -use crate::bindings::_helpers::cursor::values::{DATETIME_MODULE, naive_date_time}; +use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; @@ -35,7 +35,6 @@ use crate::bindings::_helpers::records::ids::RecordId; const DECIMAL_MODULE: &str = "decimal"; const INTEGER_CURSOR: &str = "integer"; -const UTC_OFFSET: i64 = 0; const TIMESTAMP_CURSOR: &str = "timestamp"; /// `CursorAlgebraError(message)` raised from `cause`. @@ -254,34 +253,6 @@ pub(crate) fn compare_values(left: &Bound<'_, PyAny>, right: &Bound<'_, PyAny>) }) } -/// The key as `int` or naive `datetime`; converting from UTC itself keeps the fold. -fn python_key<'py>(py: Python<'py>, key: &SortKey, fold: bool) -> PyResult> { - match key { - SortKey::Integer(number) => Ok(number.into_pyobject(py)?.into_any()), - SortKey::Moment(local) => naive_date_time(py, local, fold), - } -} - -/// The canonical sortable key of one raw cursor bound. -#[pyfunction(name = "cursor_sort_key")] -#[pyo3(signature = (*, raw, cursor_type))] -fn sortable_key<'py>( - raw: &Bound<'py, PyAny>, - cursor_type: &Bound<'py, PyAny>, -) -> PyResult> { - let session: CursorSession<'py> = CursorSession::new(raw.py())?; - let scalar: Scalar = as_scalar(&session, raw, cursor_type)?; - let key: SortKey = session.checked(sort_key(&scalar, &session))?; - let ScalarValue::Timestamp(timestamp) = &scalar.value else { - return python_key(raw.py(), &key, false); - }; - match (timestamp.offset, timestamp.template) { - (ZoneOffset::Naive, Some(template)) => Ok(session.held(template)), - (ZoneOffset::Fixed(UTC_OFFSET), _) => python_key(raw.py(), &key, timestamp.fold), - _ => python_key(raw.py(), &key, false), - } -} - fn extreme_bound<'py>( values: &Bound<'py, PyAny>, cursor_type: &Bound<'py, PyAny>, @@ -403,7 +374,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(sentinel_to_token, module)?)?; module.add_function(wrap_pyfunction!(sentinel_from_token, module)?)?; module.add_function(wrap_pyfunction!(compare_values, module)?)?; - module.add_function(wrap_pyfunction!(sortable_key, module)?)?; module.add_function(wrap_pyfunction!(max_bound, module)?)?; module.add_function(wrap_pyfunction!(min_bound, module)?)?; module.add_function(wrap_pyfunction!(floor_value, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs index bc10e0eff5..9aaa429cd5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs @@ -8,6 +8,7 @@ use sqlbuild_core::build_command::constants::{ SNAPSHOT_FULL_REFRESH_DBT_HELP, TABLE_TYPE_DOWNGRADE_DBT_HELP, }; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_host::models::{BuildExecutionResult, BuildStatus}; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, keywords}; use crate::bindings::_helpers::build_command::policies::enforce_plan_safety_policies_binding; @@ -17,11 +18,13 @@ use crate::bindings::_helpers::build_command::support::{ use crate::bindings::_helpers::build_progress::build_callbacks::{ BuildProgressCallbacks, batch_complete_callback, }; -use crate::bindings::_helpers::build_progress::footer::format_build_footer; +use crate::bindings::_helpers::build_progress::footer::format_build_footer_with; use crate::bindings::_helpers::build_progress::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::build_progress::progress_text::format_execution_header; +use crate::bindings::_helpers::build_progress::result_text::build_result_text; use crate::bindings::_helpers::compile_outputs::runtime_target::write_runtime_target; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::presentation_text::style::PythonCliStyle; use crate::bindings::_helpers::records::construct::{new_record_with, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; @@ -239,22 +242,23 @@ pub(crate) fn execute_dbt_sqlbuild_work<'py>( ("callbacks", hooks), ], )?; + let result = BuildExecutionResult::read(&result)?; write_runtime_target( &joined(&args.value("project_dir")?, &["target"])?, &plan, &result, )?; - let footer = format_build_footer( - &result.clone(), - native_field(&callbacks, "elapsed")?.extract::()?, - color.extract::()?, + let footer = format_build_footer_with( + &build_result_text(py, &result)?, + ( + native_field(&callbacks, "elapsed")?.extract::()?, + color.extract::()?, + ), None, - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; - emit(&output, &format!("\n{}\n", footer.extract::()?))?; - Ok(i64::from(!native_field(&result, "status")?.eq( - host::entry(py, ExecutorHost::BuildStatus)?.getattr("SUCCESS")?, - )?)) + Vec::new(), + )?; + emit(&output, &format!("\n{footer}\n"))?; + Ok(i64::from(result.status != BuildStatus::Success)) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs index 25361a5242..ab91f568ae 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/freshness/publication.rs @@ -25,17 +25,17 @@ type RecordsByLocation<'py> = Vec<( )>; /// The names of the selected models that built successfully. -fn successful_model_names<'py>(result: &Bound<'py, PyAny>) -> PyResult>> { - let py: Python<'py> = result.py(); - let success: Bound<'py, PyAny> = +fn successful_model_names(result: &Bound<'_, PyAny>) -> PyResult> { + let py = result.py(); + let success: Bound<'_, PyAny> = host::entry(py, ExecutorHost::ExecutionStatus)?.getattr("SUCCESS")?; - let mut names: Vec> = Vec::new(); + let mut names: Vec = Vec::new(); for model_result in native_field(result, "model_results")?.try_iter()? { - let model_result: Bound<'py, PyAny> = model_result?; - let status: Bound<'py, PyAny> = native_field(&model_result, "status")?; - let value: Bound<'py, PyAny> = status.getattr_opt("value")?.unwrap_or(status); + let model_result: Bound<'_, PyAny> = model_result?; + let status: Bound<'_, PyAny> = native_field(&model_result, "status")?; + let value: Bound<'_, PyAny> = status.getattr_opt("value")?.unwrap_or(status); if value.eq(&success)? { - names.push(native_field(&model_result, "model_name")?); + names.push(native_field(&model_result, "model_name")?.extract()?); } } Ok(names) @@ -104,7 +104,7 @@ fn contains_location( /// The observed records to publish, keyed by the schema of each affected model. fn eligible_records<'py>( plan: &Bound<'py, PyAny>, - result: &Bound<'py, PyAny>, + successful_models: &[String], run_id: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = plan.py(); @@ -121,8 +121,7 @@ fn eligible_records<'py>( return Ok(Vec::new()); } let selected: Bound<'py, PyFrozenSet> = PyFrozenSet::new(py, entries_by_name.keys())?; - let successful: Bound<'py, PyFrozenSet> = - PyFrozenSet::new(py, successful_model_names(result)?)?; + let successful: Bound<'py, PyFrozenSet> = PyFrozenSet::new(py, successful_models)?; let observed: Bound<'py, PyDict> = PyDict::new(py); for record in native_field(&freshness, "observed_records")?.try_iter()? { let record: Bound<'py, PyAny> = record?; @@ -247,13 +246,13 @@ fn write_records<'py>( /// Append observed source freshness only after the affected selected models succeed. pub(crate) fn append_freshness_records<'py>( - (plan, result): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + (plan, successful_models): (&Bound<'py, PyAny>, &[String]), adapter: &Bound<'py, PyAny>, connection_config: &Bound<'py, PyAny>, run_id: &Bound<'py, PyAny>, ) -> PyResult<()> { let py: Python<'py> = plan.py(); - let grouped: RecordsByLocation<'py> = eligible_records(plan, result, run_id)?; + let grouped: RecordsByLocation<'py> = eligible_records(plan, successful_models, run_id)?; let record_count: usize = grouped.iter().map(|(_, records)| records.len()).sum(); if record_count == 0 { return Ok(()); @@ -310,7 +309,10 @@ fn append_eligible_direct_source_freshness_records<'py>( ), )?; append_freshness_records( - (&arguments.value("plan")?, &arguments.value("result")?), + ( + &arguments.value("plan")?, + &successful_model_names(&arguments.value("result")?)?, + ), &arguments.value("adapter")?, &arguments.value("connection_config")?, &arguments.value("run_id")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs index 124c11f231..835332ee7d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs @@ -1,19 +1,13 @@ //! Ordering edges attached audits and model hooks add, and the keys hooks and SQL refs name. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; -use crate::bindings::_helpers::graph::keys::KeyCache; use crate::bindings::_helpers::records::construct::{enum_from_value, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::RecordId; -use crate::bindings::_helpers::references::explicit_references::{ - GateRow, HookRow, native_execution_gate_edges, -}; - -const PATHLIB_MODULE: &str = "pathlib"; /// The model config keys whose entries are hooks, in lifecycle order. const MODEL_HOOK_KEYS: [&str; 2] = ["pre_hooks", "post_hooks"]; @@ -28,78 +22,6 @@ fn keyword_call<'py>( callable.call((), Some(&keywords)) } -/// Edges from each extra audit read to its model or seed target, or source triggers. -#[pyfunction] -#[pyo3(signature = (*, project))] -fn attached_audit_gate_edges<'py>( - py: Python<'py>, - project: &Bound<'py, PyAny>, -) -> PyResult> { - let (gates, _): (Vec, Vec) = native_execution_gate_edges(project)?; - let cache: KeyCache<'py> = KeyCache::new(py)?; - let edge_type: Bound<'py, PyAny> = record_class(py, RecordId::AttachedAuditGateEdge)?; - let mut edges: Vec> = Vec::with_capacity(gates.len()); - for GateRow { - audit_name, - target, - gated, - read, - } in gates - { - edges.push(keyword_call( - &edge_type, - &[ - ("audit_name", PyString::new(py, &audit_name).into_any()), - ("target", cache.key(&target)?), - ("gated", cache.key(&gated)?), - ("read", cache.key(&read)?), - ], - )?); - } - PyTuple::new(py, edges) -} - -/// Edges from each hook read to the model that runs the hook. -#[pyfunction] -#[pyo3(signature = (*, project))] -fn hook_read_edges<'py>( - py: Python<'py>, - project: &Bound<'py, PyAny>, -) -> PyResult> { - let (_, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; - let cache: KeyCache<'py> = KeyCache::new(py)?; - let edge_type: Bound<'py, PyAny> = record_class(py, RecordId::HookReadEdge)?; - let path_type: Bound<'py, PyAny> = py.import(PATHLIB_MODULE)?.getattr("Path")?; - let mut edges: Vec> = Vec::with_capacity(hooks.len()); - for HookRow { - hook_name, - gated, - read, - hook_type: kind, - path, - } in hooks - { - let hook_path: Bound<'py, PyAny> = match path { - None => py.None().into_bound(py), - Some(path) => path_type.call1((path,))?, - }; - edges.push(keyword_call( - &edge_type, - &[ - ("hook_name", PyString::new(py, &hook_name).into_any()), - ("gated", cache.key(&gated)?), - ("read", cache.key(&read)?), - ( - "hook_type", - enum_from_value(py, RecordId::HookReadType, kind)?, - ), - ("hook_path", hook_path), - ], - )?); - } - PyTuple::new(py, edges) -} - /// The Python hooks attached to one model, in lifecycle order without repeats. #[pyfunction] #[pyo3(signature = (*, model))] @@ -153,8 +75,6 @@ pub(crate) fn sql_ref_key<'py>( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(attached_audit_gate_edges, module)?)?; - module.add_function(wrap_pyfunction!(hook_read_edges, module)?)?; module.add_function(wrap_pyfunction!(model_python_hook_names, module)?)?; module.add_function(wrap_pyfunction!(sql_ref_key, module)?)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs index 514ff42697..f06d515bb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/mod.rs @@ -3,3 +3,4 @@ pub(crate) mod connections; pub(crate) mod host; pub(crate) mod registration; +pub(crate) mod results; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/results.rs new file mode 100644 index 0000000000..9686c85cdf --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/results.rs @@ -0,0 +1,302 @@ +//! Read executor dataclasses once into the host contract. Only opaque payloads retain Python values. + +use std::path::PathBuf; + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::PyInt; +use sqlbuild_core::json::models::JsonInteger; +use sqlbuild_host::models::{ + AttachedAuditTargetKind, AuditAttachmentKind, AuditEvaluationMode, AuditExecutionResult, + AuditOutcome, AuditRunScope, AuditSeverity, BuildExecutionResult, BuildExecutionTimings, + BuildStatus, CancellationOutcome, ExecutionPhase, ExecutionResourceKind, ExecutionStatus, + FunctionExecutionResult, HookExecutionResult, HookPhase, ItemKind, ItemTimeoutReport, + LifeCycleEvent, LifeCycleEventKind, LoadExecutionResult, MeasuredValue, + MicrobatchAccountingInterval, MicrobatchLimitAction, ModelExecutionResult, SeedExecutionResult, + SkipMode, SqlTestColumnDifference, SqlTestDifferenceSample, SqlTestExecutionResult, + SqlTestOutcome, StepResult, +}; +use sqlbuild_host::types::ResultEnum; + +use crate::bindings::_helpers::records::fields::field; + +/// Extraction at the permanent Python executor edge, including nested result records. +pub(crate) trait ReadResult<'py>: Sized { + fn read(value: &Bound<'py, PyAny>) -> PyResult; +} + +macro_rules! scalar { + ($($kind:ty),+ $(,)?) => {$ ( + impl<'py> ReadResult<'py> for $kind { + fn read(value: &Bound<'py, PyAny>) -> PyResult { + value.extract() + } + } + )+}; +} + +scalar!(String, i64, f64, bool, PathBuf); + +impl<'py> ReadResult<'py> for MeasuredValue { + fn read(value: &Bound<'py, PyAny>) -> PyResult { + if value.is_instance_of::() { + let decimal = value.str()?.to_string(); + return JsonInteger::parse(&decimal) + .map(Self::Integer) + .ok_or_else(|| PyValueError::new_err(format!("invalid integer text {decimal}"))); + } + value.extract().map(Self::Float) + } +} + +impl<'py> ReadResult<'py> for Bound<'py, PyAny> { + fn read(value: &Bound<'py, PyAny>) -> PyResult { + Ok(value.clone()) + } +} + +impl<'py, T: ReadResult<'py>> ReadResult<'py> for Option { + fn read(value: &Bound<'py, PyAny>) -> PyResult { + if value.is_none() { + Ok(None) + } else { + T::read(value).map(Some) + } + } +} + +impl<'py, T: ReadResult<'py>> ReadResult<'py> for Vec { + fn read(value: &Bound<'py, PyAny>) -> PyResult { + value.try_iter()?.map(|item| T::read(&item?)).collect() + } +} + +impl<'py, A: ReadResult<'py>, B: ReadResult<'py>> ReadResult<'py> for (A, B) { + fn read(value: &Bound<'py, PyAny>) -> PyResult { + let (first, second): (Bound<'py, PyAny>, Bound<'py, PyAny>) = value.extract()?; + Ok((A::read(&first)?, B::read(&second)?)) + } +} + +macro_rules! enums { + ($($kind:ty),+ $(,)?) => {$ ( + impl<'py> ReadResult<'py> for $kind { + fn read(value: &Bound<'py, PyAny>) -> PyResult { + let text: String = field(value, "value")?.extract()?; + Self::from_value(&text).ok_or_else(|| PyValueError::new_err( + format!("invalid {} value: {text}", stringify!($kind)) + )) + } + } + )+}; +} + +enums!( + BuildStatus, + ExecutionStatus, + ExecutionPhase, + HookPhase, + AuditRunScope, + AttachedAuditTargetKind, + AuditEvaluationMode, + LifeCycleEventKind, + AuditAttachmentKind, + AuditSeverity, + AuditOutcome, + SkipMode, + ExecutionResourceKind, + SqlTestOutcome, + ItemKind, + CancellationOutcome, + MicrobatchLimitAction, +); + +macro_rules! record { + ($kind:ident { $($name:ident),+ $(,)? }) => { + impl<'py> ReadResult<'py> for $kind { + fn read(value: &Bound<'py, PyAny>) -> PyResult { + Ok(Self { $($name: ReadResult::read(&field(value, stringify!($name))?)?,)+ }) + } + } + }; + ($kind:ident { $($name:ident),+ $(,)? }) => { + impl<'py> ReadResult<'py> for $kind> { + fn read(value: &Bound<'py, PyAny>) -> PyResult { + Ok(Self { $($name: ReadResult::read(&field(value, stringify!($name))?)?,)+ }) + } + } + }; +} + +record!(BuildExecutionTimings { + connection_preparation_seconds, + schema_preparation_seconds, + execution_seconds, +}); +record!(LifeCycleEvent { kind, content }); +record!(HookExecutionResult { + phase, + index, + hook_type, + label, + status, + skip_mode, + skip_reason, + error_message, +}); +record!(MicrobatchAccountingInterval { + partition_start, + partition_end, + accounting_status, + fingerprint_status, + model_version_hash, + completion_type, + event_id, +}); +record!(ModelExecutionResult { + model_name, status, failed_phase, staging_relation, promoted_relation, duration_ms, + batch_count, batch_size, rows_affected, cursor_range_start, cursor_range_end, cursor_type, + cursor_grain, empty_cursor_inputs, waiting_on_empty_inputs, future_cursor_safety, + maximum_start_safety, audit_results, warning_messages, lifecycle_events, hook_results, + skip_mode, skip_reason, error_code, error_help, error_message, failed_sql, + microbatch_run_type, microbatch_strategy, microbatch_plan_reason, + microbatch_recovery_batch_count, microbatch_known_gap_count, + microbatch_unaccounted_interval_count, microbatch_synthetic_completion_count, + microbatch_unknown_fingerprint_count, microbatch_contiguous_frontier, + microbatch_unaccounted_partition_policy, microbatch_replay_requirement_id, + microbatch_required_model_version_hash, microbatch_physical_generation_id, + microbatch_concurrent_enabled, microbatch_batch_concurrency, microbatch_global_concurrency, + microbatch_replay_requirement_state, microbatch_accounting_intervals, + microbatch_limit, microbatch_limit_count, microbatch_limit_action, microbatch_limit_warning, +}); +record!(SeedExecutionResult { + seed_name, + status, + duration_ms, + lifecycle_events, + error_code, + error_help, + error_message, + warning_messages, +}); +record!(FunctionExecutionResult { + function_name, + status, + function_kind, + duration_ms, + error_code, + error_help, + error_message, + warning_messages, + lifecycle_events, +}); +record!(SqlTestDifferenceSample { values }); +record!(SqlTestColumnDifference { + name, + actual, + expected +}); +record!(StepResult { + model_name, + outcome, + actual_row_count, + expected_row_count, + mismatched_row_count, + unexpected_row_count, + missing_row_count, + unexpected_samples, + missing_samples, + column_differences, + error_code, + error_help, + error_message, +}); +record!(ItemTimeoutReport { + kind, + name, + limit, + source, + elapsed_seconds, + query_ids, + cancellation, + cancellation_detail, +}); +record!(SqlTestExecutionResult { + test_name, outcome, source_path, block_index, parent_name, case_name, case_index, + case_fingerprint, parameter_schema, parameter_values, step_results, error_code, error_help, + error_message, real_model_count, timeout, +}); +record!(BuildExecutionResult { + status, model_results, seed_results, function_results, load_results, test_results, + source_audit_results, end_audit_results, success_count, failure_count, timeout_count, + skipped_count, warning_count, insufficient_count, audit_result_projection_attempted_count, + audit_result_projection_written_count, audit_result_projection_failed_count, timings, +}); + +fn mapping<'py>(value: &Bound<'py, PyAny>) -> PyResult)>> { + Vec::read(&value.call_method0("items")?) +} + +impl<'py> ReadResult<'py> for AuditExecutionResult> { + fn read(value: &Bound<'py, PyAny>) -> PyResult { + let meta = field(value, "audit_meta")?; + Ok(Self { + audit_name: ReadResult::read(&field(value, "audit_name")?)?, + audit_definition_name: ReadResult::read(&field(value, "audit_definition_name")?)?, + attachment_kind: ReadResult::read(&field(value, "attachment_kind")?)?, + severity: ReadResult::read(&field(value, "severity")?)?, + outcome: ReadResult::read(&field(value, "outcome")?)?, + row_count: ReadResult::read(&field(value, "row_count")?)?, + executed_sql: ReadResult::read(&field(value, "executed_sql")?)?, + run_scope_phase: ReadResult::read(&field(value, "run_scope_phase")?)?, + attached_target_kind: ReadResult::read(&field(value, "attached_target_kind")?)?, + attached_target_name: ReadResult::read(&field(value, "attached_target_name")?)?, + attached_column_name: ReadResult::read(&field(value, "attached_column_name")?)?, + reused: ReadResult::read(&field(value, "reused")?)?, + evaluation_mode: ReadResult::read(&field(value, "evaluation_mode")?)?, + measured_value: ReadResult::read(&field(value, "measured_value")?)?, + sample_count: ReadResult::read(&field(value, "sample_count")?)?, + sample_unit: ReadResult::read(&field(value, "sample_unit")?)?, + minimum_samples: ReadResult::read(&field(value, "minimum_samples")?)?, + thresholds: ReadResult::read(&field(value, "thresholds")?)?, + evidence_rows: field(value, "evidence_rows")? + .try_iter()? + .map(|row| mapping(&row?)) + .collect::>()?, + evidence_truncated: ReadResult::read(&field(value, "evidence_truncated")?)?, + evidence_error: ReadResult::read(&field(value, "evidence_error")?)?, + evidence_sql: ReadResult::read(&field(value, "evidence_sql")?)?, + audit_description: ReadResult::read(&field(value, "audit_description")?)?, + execution_error: ReadResult::read(&field(value, "execution_error")?)?, + audit_meta: if meta.is_none() { + None + } else { + Some(mapping(&meta)?) + }, + skip_reason: ReadResult::read(&field(value, "skip_reason")?)?, + }) + } +} + +impl<'py> ReadResult<'py> for LoadExecutionResult> { + fn read(value: &Bound<'py, PyAny>) -> PyResult { + Ok(Self { + source_name: ReadResult::read(&field(value, "source_name")?)?, + loader_name: ReadResult::read(&field(value, "loader_name")?)?, + status: ReadResult::read(&field(value, "status")?)?, + target: ReadResult::read(&field(value, "target")?)?, + resource_kind: ReadResult::read(&field(value, "resource_kind")?)?, + staging_relation: ReadResult::read(&field(value, "staging_relation")?)?, + rows_loaded: ReadResult::read(&field(value, "rows_loaded")?)?, + duration_ms: ReadResult::read(&field(value, "duration_ms")?)?, + lifecycle_events: ReadResult::read(&field(value, "lifecycle_events")?)?, + skip_mode: ReadResult::read(&field(value, "skip_mode")?)?, + skip_reason: ReadResult::read(&field(value, "skip_reason")?)?, + error_message: ReadResult::read(&field(value, "error_message")?)?, + result_payload: ReadResult::read(&field(value, "result_payload")?)?, + result_metadata: mapping(&field(value, "result_metadata")?)?, + result_materialized: ReadResult::read(&field(value, "result_materialized")?)?, + warning_messages: ReadResult::read(&field(value, "warning_messages")?)?, + }) + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs index f1e6b2d95e..ca7cda04fd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs @@ -129,15 +129,6 @@ pub(crate) fn migration_lines( )) } -#[pyfunction(name = "format_model_migrations", signature = (*, lines, plan, display_options))] -fn format_migrations( - lines: Vec, - plan: &Bound<'_, PyAny>, - display_options: &Bound<'_, PyAny>, -) -> PyResult> { - migration_lines(lines, plan, display_options) -} - fn days(value: &Bound<'_, PyAny>) -> PyResult> { value.extract() } @@ -213,7 +204,6 @@ fn render_old_name(entry: &Bound<'_, PyAny>, style: &Bound<'_, PyAny>) -> PyResu } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(format_migrations, module)?)?; module.add_function(wrap_pyfunction!(format_pre_build, module)?)?; module.add_function(wrap_pyfunction!(render_old_name, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs index db2e066a99..6d93857fcf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs @@ -1,7 +1,7 @@ //! Shared CLI text helpers as `_native` functions; styles are read from a `CliStyle`'s `use_color`. use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyList, PyListMethods, PySlice}; +use pyo3::types::{PyDict, PyList, PyListMethods}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_core::cli_text::main::format_aligned_name_value::format_aligned_name_value as aligned; use sqlbuild_core::cli_text::main::format_completion_line::format_completion_line as completion; @@ -135,21 +135,6 @@ fn append_overflow_line<'py>( Ok(lines) } -/// Return entries visible under the current display cap. -#[pyfunction] -#[pyo3(signature = (*, entries, options))] -fn visible_entries<'py>( - entries: Bound<'py, PyAny>, - options: &Bound<'py, PyAny>, -) -> PyResult> { - let limit: Bound<'py, PyAny> = native_field(options, "max_entries_per_section")?; - if limit.is_none() { - return Ok(entries); - } - let slice: Bound<'py, PySlice> = PySlice::new(entries.py(), 0, limit.extract()?, 1); - entries.get_item(slice) -} - /// A header style that renders `Title (count)` as a styled title and a dim count. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct CountHeaderStyle { @@ -199,6 +184,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(format_aligned_name_value, module)?)?; module.add_function(wrap_pyfunction!(resolve_name_column_width, module)?)?; module.add_function(wrap_pyfunction!(append_overflow_line, module)?)?; - module.add_function(wrap_pyfunction!(visible_entries, module)?)?; module.add_function(wrap_pyfunction!(count_header_style, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/references/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/references/explicit_references.rs index 60a01dbf20..1d672af48a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/references/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/references/explicit_references.rs @@ -431,14 +431,13 @@ pub(crate) struct GateRow { pub(crate) read: ResourceKey, } -/// One hook-read ordering edge: the hook, the gated and read resources, its type and path. +/// One hook-read ordering edge: the hook, the gated and read resources, and its type. pub(crate) struct HookRow { pub(crate) hook_name: String, pub(crate) gated: ResourceKey, pub(crate) read: ResourceKey, /// The `HookReadType` value. pub(crate) hook_type: &'static str, - pub(crate) path: Option, } /// The attached-audit gate edges and hook-read edges of one project, in Python's order. @@ -478,7 +477,6 @@ fn hook_row(edge: HookEdge) -> HookRow { gated: edge.gated, read: edge.read, hook_type, - path: edge.path, } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs index 566e1b4b0f..da21068156 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_values.rs @@ -429,26 +429,6 @@ pub(crate) fn constant_declaration_sql<'py>( Err(PyErr::from_value(compile_error)) } -/// Render one validated constant with the active adapter's typed-value contract. -#[pyfunction] -#[pyo3(signature = (*, declaration, value_renderer, collection_rendering, file_path=None))] -fn render_constant_declaration<'py>( - declaration: &Bound<'py, PyAny>, - value_renderer: &Bound<'py, PyAny>, - collection_rendering: &Bound<'py, PyAny>, - file_path: Option>, -) -> PyResult { - let location: Option = match file_path { - Some(file_path) => Some(file_path.str()?.to_string()), - None => None, - }; - constant_declaration_sql( - declaration, - value_renderer, - (collection_rendering, location.as_deref()), - ) -} - /// The adapter's constant rendering a compile's declaration expansions share. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct ConstantRendering { @@ -511,7 +491,6 @@ fn resolve_declaration_context<'py>( pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(build_native_declaration_contexts, module)?)?; - module.add_function(wrap_pyfunction!(render_constant_declaration, module)?)?; module.add_function(wrap_pyfunction!(render_enum_member_value, module)?)?; module.add_function(wrap_pyfunction!(resolve_declaration_context, module)?)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs index 76a5730fec..448dcb6591 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs @@ -1,16 +1,125 @@ //! Build `BuildExecutionResult` with aggregate counts and overall status from typed results. use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyTuple}; +use pyo3::types::PyDict; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_core::build_aggregate::main::aggregate_build_counts::aggregate_build_counts; use sqlbuild_core::build_aggregate::models::{BuildCounts, BuildResultFacts, ModelResultFacts}; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_host::models::{ + AuditExecutionResult, BuildExecutionResult, FunctionExecutionResult, LoadExecutionResult, + ModelExecutionResult, SeedExecutionResult, SqlTestExecutionResult, +}; +use sqlbuild_host::types::ResultEnum; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::boundary::python_calls::call_with; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::result_output::projection::Projection; + +/// Borrowed executor results with canonical terminals, used by the build JSON projection. +pub(crate) struct RetainedBuildResults<'a, V> { + pub(crate) models: Vec<&'a ModelExecutionResult>, + pub(crate) seeds: Vec<&'a SeedExecutionResult>, + pub(crate) functions: Vec<&'a FunctionExecutionResult>, + pub(crate) loads: Vec<&'a LoadExecutionResult>, + pub(crate) tests: Vec<&'a SqlTestExecutionResult>, + pub(crate) source_audits: Vec<&'a AuditExecutionResult>, + pub(crate) end_audits: Vec<&'a AuditExecutionResult>, +} + +impl<'a, V> RetainedBuildResults<'a, V> { + pub(crate) fn new( + result: &'a BuildExecutionResult, + projection: &Projection<'_>, + ) -> PyResult { + Ok(Self { + models: projection.retained(&result.model_results)?, + seeds: projection.retained(&result.seed_results)?, + functions: projection.retained(&result.function_results)?, + loads: projection.retained(&result.load_results)?, + tests: projection.retained(&result.test_results)?, + source_audits: projection.retained(&result.source_audit_results)?, + end_audits: projection.retained(&result.end_audit_results)?, + }) + } + + pub(crate) fn is_partial(&self, result: &BuildExecutionResult) -> bool { + [ + self.models.len(), + self.seeds.len(), + self.functions.len(), + self.loads.len(), + self.tests.len(), + self.source_audits.len(), + self.end_audits.len(), + ] != [ + result.model_results.len(), + result.seed_results.len(), + result.function_results.len(), + result.load_results.len(), + result.test_results.len(), + result.source_audit_results.len(), + result.end_audit_results.len(), + ] + } + + pub(crate) fn counts(&self) -> BuildCounts { + let mut models: Vec = Vec::new(); + for model in &self.models { + let audit_outcomes: Vec = model + .audit_results + .iter() + .map(|audit| audit.outcome.value().to_owned()) + .collect(); + models.push(ModelResultFacts { + status: model.status.value().to_owned(), + error_code: model.error_code.clone(), + warning_count: model.warning_messages.len(), + audit_outcomes, + }); + } + aggregate_build_counts(&BuildResultFacts { + models, + seed_statuses: self + .seeds + .iter() + .map(|result| result.status.value().to_owned()) + .collect(), + functions: self + .functions + .iter() + .map(|result| { + ( + result.status.value().to_owned(), + result.warning_messages.len(), + ) + }) + .collect(), + load_statuses: self + .loads + .iter() + .map(|result| result.status.value().to_owned()) + .collect(), + test_outcomes: self + .tests + .iter() + .map(|result| result.outcome.value().to_owned()) + .collect(), + source_audit_outcomes: self + .source_audits + .iter() + .map(|result| result.outcome.value().to_owned()) + .collect(), + end_audit_outcomes: self + .end_audits + .iter() + .map(|result| result.outcome.value().to_owned()) + .collect(), + }) + } +} const RESULT_FIELDS: [&str; 7] = [ "model_results", @@ -104,18 +213,6 @@ fn aggregate_build_result<'py>( aggregated_build_result(request.py(), §ions) } -/// `aggregate_build_result` over retained result lists, as tuples. -pub(crate) fn aggregated_from_lists<'py>( - py: Python<'py>, - retained: &[Vec>], -) -> PyResult> { - let sections: Vec> = retained - .iter() - .map(|kept| Ok(PyTuple::new(py, kept)?.into_any())) - .collect::>()?; - aggregated_build_result(py, §ions) -} - pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(aggregate_build_result, module)?)?; Ok(()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs index 9ad30581f7..8286e427a1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs @@ -2,17 +2,19 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList}; +use sqlbuild_host::models::{ + ExecutionStatus, FunctionExecutionResult, LifeCycleEvent, LifeCycleEventKind, + LoadExecutionResult, ModelExecutionResult, SeedExecutionResult, +}; +use sqlbuild_host::types::ResultEnum; use crate::bindings::_helpers::plan_output::safety::{future_cursor_safety, maximum_start_safety}; -use crate::bindings::_helpers::plan_output::values::{any, dict, items, optional_enum_value}; +use crate::bindings::_helpers::plan_output::values::{any, dict, items}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::fields::mapping_items; use crate::bindings::_helpers::result_output::projection::{Projection, rounded}; -const FAILED_STATUS: &str = "failed"; -const SQL_EVENT_KIND: &str = "sql"; - /// A dictionary of the entries whose value is not `None`, in order. pub(crate) fn without_none<'py>( py: Python<'py>, @@ -64,106 +66,138 @@ fn target<'py>( } } -fn last_recorded_sql<'py>(result: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = result.py(); - if !native_field(&native_field(result, "status")?, "value")?.eq(FAILED_STATUS)? { - return Ok(py.None().into_bound(py)); - } - let events: Vec> = items(&native_field(result, "lifecycle_events")?)?; - for event in events.iter().rev() { - if native_field(&native_field(event, "kind")?, "value")?.eq(SQL_EVENT_KIND)? { - return native_field(event, "content"); - } +fn last_recorded_sql(status: ExecutionStatus, events: &[LifeCycleEvent]) -> Option<&str> { + if status != ExecutionStatus::Failed { + return None; } - Ok(py.None().into_bound(py)) + events + .iter() + .rev() + .find(|event| event.kind == LifeCycleEventKind::Sql) + .map(|event| event.content.as_str()) } -/// The run type and accounting of a microbatch model result. -fn microbatch<'py>(result: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = result.py(); - let run_type: Bound<'py, PyAny> = native_field(result, "microbatch_run_type")?; - if run_type.is_none() { - return Ok(run_type); - } - let state: Bound<'py, PyDict> = dict(py, vec![("run_type", run_type)])?; - for (key, field) in [ - ("strategy", "microbatch_strategy"), - ("reason", "microbatch_plan_reason"), +/// Shared run type and bounded accounting summary, without per-interval evidence. +pub(crate) fn microbatch_summary<'py>( + py: Python<'py>, + result: &ModelExecutionResult>, +) -> PyResult> { + let Some(run_type) = &result.microbatch_run_type else { + return Ok(py.None().into_bound(py)); + }; + let state: Bound<'py, PyDict> = dict(py, vec![("run_type", any(py, run_type)?)])?; + for (key, value) in [ + ("strategy", &result.microbatch_strategy), + ("reason", &result.microbatch_plan_reason), ] { - let value: Bound<'py, PyAny> = native_field(result, field)?; - if !value.is_none() { + if let Some(value) = value { state.set_item(key, value)?; } } - for (key, field) in [ - ("batch_count", "batch_count"), - ("batch_size", "batch_size"), - ("recovery_batch_count", "microbatch_recovery_batch_count"), - ("known_gap_count", "microbatch_known_gap_count"), + for (key, value) in [ + ("batch_count", any(py, result.batch_count)?), + ("batch_size", any(py, &result.batch_size)?), + ( + "recovery_batch_count", + any(py, result.microbatch_recovery_batch_count)?, + ), + ( + "known_gap_count", + any(py, result.microbatch_known_gap_count)?, + ), ( "unaccounted_interval_count", - "microbatch_unaccounted_interval_count", + any(py, result.microbatch_unaccounted_interval_count)?, ), ( "synthetic_completion_count", - "microbatch_synthetic_completion_count", + any(py, result.microbatch_synthetic_completion_count)?, ), ( "unknown_fingerprint_count", - "microbatch_unknown_fingerprint_count", + any(py, result.microbatch_unknown_fingerprint_count)?, + ), + ( + "contiguous_frontier", + any(py, &result.microbatch_contiguous_frontier)?, ), - ("contiguous_frontier", "microbatch_contiguous_frontier"), ( "unaccounted_partition_policy", - "microbatch_unaccounted_partition_policy", + any(py, &result.microbatch_unaccounted_partition_policy)?, + ), + ( + "replay_requirement_id", + any(py, &result.microbatch_replay_requirement_id)?, ), - ("replay_requirement_id", "microbatch_replay_requirement_id"), ( "required_model_version_hash", - "microbatch_required_model_version_hash", + any(py, &result.microbatch_required_model_version_hash)?, ), ( "physical_generation_id", - "microbatch_physical_generation_id", + any(py, &result.microbatch_physical_generation_id)?, + ), + ( + "concurrent_enabled", + any(py, result.microbatch_concurrent_enabled)?, + ), + ( + "batch_concurrency", + any(py, result.microbatch_batch_concurrency)?, + ), + ( + "global_concurrency", + any(py, result.microbatch_global_concurrency)?, ), - ("concurrent_enabled", "microbatch_concurrent_enabled"), - ("batch_concurrency", "microbatch_batch_concurrency"), - ("global_concurrency", "microbatch_global_concurrency"), ( "replay_requirement_state", - "microbatch_replay_requirement_state", + any(py, &result.microbatch_replay_requirement_state)?, ), ] { - state.set_item(key, native_field(result, field)?)?; + state.set_item(key, value)?; + } + Ok(state.into_any()) +} + +/// The run type and accounting of a microbatch model result, including interval evidence. +fn microbatch<'py>( + py: Python<'py>, + result: &ModelExecutionResult>, +) -> PyResult> { + let summary: Bound<'py, PyAny> = microbatch_summary(py, result)?; + if summary.is_none() { + return Ok(summary); } + let state: &Bound<'py, PyDict> = summary.downcast()?; let intervals: Bound<'py, PyList> = PyList::empty(py); - for interval in items(&native_field(result, "microbatch_accounting_intervals")?)? { - let fields: Vec<(&str, Bound<'py, PyAny>)> = [ - "partition_start", - "partition_end", - "accounting_status", - "fingerprint_status", - "model_version_hash", - "completion_type", - "event_id", - ] - .into_iter() - .map(|field| Ok((field, native_field(&interval, field)?))) - .collect::>()?; - pyo3::types::PyListMethods::append(&intervals, dict(py, fields)?)?; + for interval in &result.microbatch_accounting_intervals { + pyo3::types::PyListMethods::append( + &intervals, + dict( + py, + vec![ + ("partition_start", any(py, &interval.partition_start)?), + ("partition_end", any(py, &interval.partition_end)?), + ("accounting_status", any(py, &interval.accounting_status)?), + ("fingerprint_status", any(py, &interval.fingerprint_status)?), + ("model_version_hash", any(py, &interval.model_version_hash)?), + ("completion_type", any(py, &interval.completion_type)?), + ("event_id", any(py, &interval.event_id)?), + ], + )?, + )?; } state.set_item("intervals", intervals)?; - let limit: Bound<'py, PyAny> = native_field(result, "microbatch_limit")?; - if !limit.is_none() { + if let Some(limit) = result.microbatch_limit { state.set_item("limit", limit)?; - state.set_item("count", native_field(result, "microbatch_limit_count")?)?; + state.set_item("count", result.microbatch_limit_count)?; state.set_item( "action", - optional_enum_value(&native_field(result, "microbatch_limit_action")?)?, + result.microbatch_limit_action.map(ResultEnum::value), )?; - state.set_item("warning", native_field(result, "microbatch_limit_warning")?)?; + state.set_item("warning", &result.microbatch_limit_warning)?; } - Ok(state.into_any()) + Ok(summary) } fn value<'py>(result: &Bound<'py, PyAny>, field: &str) -> PyResult> { @@ -181,59 +215,67 @@ fn optional_value<'py>(result: &Bound<'py, PyAny>, field: &str) -> PyResult( projection: &Projection<'py>, - result: &Bound<'py, PyAny>, + result: &ModelExecutionResult>, targets: Option<&Bound<'py, PyAny>>, ) -> PyResult> { - let py: Python<'py> = result.py(); - let name: Bound<'py, PyAny> = native_field(result, "model_name")?; + let py: Python<'py> = projection.py; + let name: Bound<'py, PyAny> = any(py, &result.model_name)?; let planned: Bound<'py, PyAny> = target(py, targets, &name)?; - let empty_inputs: Bound<'py, PyAny> = - PyList::new(py, items(&native_field(result, "empty_cursor_inputs")?)?)?.into_any(); + let empty_inputs: Bound<'py, PyAny> = any(py, &result.empty_cursor_inputs)?; without_none( py, vec![ ("kind", any(py, "model")?), ("name", name), - ("status", value(result, "status")?), + ("status", any(py, result.status.value())?), ( "duration_ms", - projection.result_duration(result, &native_field(result, "duration_ms")?)?, + projection.result_duration(result, &any(py, result.duration_ms)?)?, ), ( "target", if planned.is_truthy()? { planned } else { - native_field(result, "promoted_relation")? + any(py, &result.promoted_relation)? }, ), + ("staging_relation", any(py, &result.staging_relation)?), + ( + "failed_phase", + any(py, result.failed_phase.map(ResultEnum::value))?, + ), + ( + "skip_mode", + any(py, result.skip_mode.map(ResultEnum::value))?, + ), + ("skip_reason", any(py, &result.skip_reason)?), + ("error_code", any(py, &result.error_code)?), + ("error_help", any(py, &result.error_help)?), + ("error_message", any(py, &result.error_message)?), + ("failed_sql", any(py, &result.failed_sql)?), ( - "staging_relation", - native_field(result, "staging_relation")?, + "last_recorded_sql", + any( + py, + last_recorded_sql(result.status, &result.lifecycle_events), + )?, ), - ("failed_phase", optional_value(result, "failed_phase")?), - ("skip_mode", optional_value(result, "skip_mode")?), - ("skip_reason", native_field(result, "skip_reason")?), - ("error_code", native_field(result, "error_code")?), - ("error_help", native_field(result, "error_help")?), - ("error_message", native_field(result, "error_message")?), - ("failed_sql", native_field(result, "failed_sql")?), - ("last_recorded_sql", last_recorded_sql(result)?), - ("warnings", native_field(result, "warning_messages")?), + ("warnings", any(py, &result.warning_messages)?), ("empty_cursor_inputs", or_none(empty_inputs)?), ( "waiting_on_empty_inputs", - or_none(native_field(result, "waiting_on_empty_inputs")?)?, + or_none(any(py, result.waiting_on_empty_inputs)?)?, ), ( "future_cursor_safety", - future_cursor_safety(&native_field(result, "future_cursor_safety")?)?, + future_cursor_safety(&any(py, &result.future_cursor_safety)?)?, ), ( "maximum_start_safety", - maximum_start_safety(&native_field(result, "maximum_start_safety")?)?, + maximum_start_safety(&any(py, &result.maximum_start_safety)?)?, ), - ("microbatch", microbatch(result)?), + ("microbatch", microbatch(py, result)?), ], ) } @@ -241,7 +283,7 @@ fn model_asset<'py>( /// Model result assets, with plan targets when a plan is given. pub(crate) fn model_assets<'py>( projection: &Projection<'py>, - results: &[Bound<'py, PyAny>], + results: &[&ModelExecutionResult>], plan: Option<&Bound<'py, PyAny>>, ) -> PyResult>> { let targets: Option> = targets(plan, "model_locations")?; @@ -257,7 +299,7 @@ pub(crate) fn model_assets<'py>( /// Seed result assets, with plan targets and reasons when a plan is given. pub(crate) fn seed_assets<'py>( projection: &Projection<'py>, - results: &[Bound<'py, PyAny>], + results: &[&SeedExecutionResult], plan: Option<&Bound<'py, PyAny>>, ) -> PyResult>> { let targets: Option> = targets(plan, "seed_locations")?; @@ -276,23 +318,23 @@ pub(crate) fn seed_assets<'py>( if !projection.terminal_recorded(result)? { continue; } - let py: Python<'py> = result.py(); - let name: Bound<'py, PyAny> = native_field(result, "seed_name")?; + let py: Python<'py> = projection.py; + let name: Bound<'py, PyAny> = any(py, &result.seed_name)?; assets.push(without_none( py, vec![ ("kind", any(py, "seed")?), ("name", name.clone()), - ("status", value(result, "status")?), + ("status", any(py, result.status.value())?), ( "duration_ms", - projection.result_duration(result, &native_field(result, "duration_ms")?)?, + projection.result_duration(result, &any(py, result.duration_ms)?)?, ), ("target", target(py, targets.as_ref(), &name)?), ("reason", target(py, reasons.as_ref(), &name)?), - ("error_code", native_field(result, "error_code")?), - ("error_help", native_field(result, "error_help")?), - ("error_message", native_field(result, "error_message")?), + ("error_code", any(py, &result.error_code)?), + ("error_help", any(py, &result.error_help)?), + ("error_message", any(py, &result.error_message)?), ], )?); } @@ -302,7 +344,7 @@ pub(crate) fn seed_assets<'py>( /// Function result assets, with plan targets when a plan is given. pub(crate) fn function_assets<'py>( projection: &Projection<'py>, - results: &[Bound<'py, PyAny>], + results: &[&FunctionExecutionResult], plan: Option<&Bound<'py, PyAny>>, ) -> PyResult>> { let targets: Option> = targets(plan, "function_locations")?; @@ -311,23 +353,23 @@ pub(crate) fn function_assets<'py>( if !projection.terminal_recorded(result)? { continue; } - let py: Python<'py> = result.py(); - let name: Bound<'py, PyAny> = native_field(result, "function_name")?; + let py: Python<'py> = projection.py; + let name: Bound<'py, PyAny> = any(py, &result.function_name)?; assets.push(without_none( py, vec![ - ("kind", native_field(result, "function_kind")?), + ("kind", any(py, &result.function_kind)?), ("name", name.clone()), - ("status", value(result, "status")?), + ("status", any(py, result.status.value())?), ( "duration_ms", - projection.result_duration(result, &native_field(result, "duration_ms")?)?, + projection.result_duration(result, &any(py, result.duration_ms)?)?, ), ("target", target(py, targets.as_ref(), &name)?), - ("error_code", native_field(result, "error_code")?), - ("error_help", native_field(result, "error_help")?), - ("error_message", native_field(result, "error_message")?), - ("warnings", native_field(result, "warning_messages")?), + ("error_code", any(py, &result.error_code)?), + ("error_help", any(py, &result.error_help)?), + ("error_message", any(py, &result.error_message)?), + ("warnings", any(py, &result.warning_messages)?), ], )?); } @@ -337,38 +379,35 @@ pub(crate) fn function_assets<'py>( /// Source and loader load result assets. pub(crate) fn load_assets<'py>( projection: &Projection<'py>, - results: &[Bound<'py, PyAny>], + results: &[&LoadExecutionResult>], ) -> PyResult>> { let mut assets: Vec> = Vec::new(); for result in results { if !projection.terminal_recorded(result)? { continue; } - let py: Python<'py> = result.py(); + let py: Python<'py> = projection.py; assets.push(without_none( py, vec![ - ("kind", value(result, "resource_kind")?), - ("name", native_field(result, "source_name")?), - ("status", value(result, "status")?), + ("kind", any(py, result.resource_kind.value())?), + ("name", any(py, &result.source_name)?), + ("status", any(py, result.status.value())?), ( "duration_ms", - projection.result_duration(result, &native_field(result, "duration_ms")?)?, + projection.result_duration(result, &any(py, result.duration_ms)?)?, ), - ("target", native_field(result, "target")?), + ("target", any(py, &result.target)?), + ("staging_relation", any(py, &result.staging_relation)?), + ("loader", any(py, &result.loader_name)?), + ("rows_loaded", any(py, result.rows_loaded)?), ( - "staging_relation", - native_field(result, "staging_relation")?, - ), - ("loader", native_field(result, "loader_name")?), - ("rows_loaded", native_field(result, "rows_loaded")?), - ("skip_mode", optional_value(result, "skip_mode")?), - ("skip_reason", native_field(result, "skip_reason")?), - ("error_message", native_field(result, "error_message")?), - ( - "warnings", - or_none(native_field(result, "warning_messages")?)?, + "skip_mode", + any(py, result.skip_mode.map(ResultEnum::value))?, ), + ("skip_reason", any(py, &result.skip_reason)?), + ("error_message", any(py, &result.error_message)?), + ("warnings", or_none(any(py, &result.warning_messages)?)?), ], )?); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs index 891e19d7fa..ca5b3f660b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs @@ -2,17 +2,19 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use sqlbuild_core::json::models::JsonValue; +use sqlbuild_host::models::{ + AuditEvaluationMode, AuditExecutionResult, AuditOutcome, MeasuredValue, + SqlTestDifferenceSample, SqlTestExecutionResult, SqlTestOutcome, StepResult, +}; +use sqlbuild_host::types::ResultEnum; use crate::bindings::_helpers::plan_output::values::{any, dict, items, text}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::result_output::assets::{or_none, without_none}; -use crate::bindings::_helpers::result_output::projection::{ - Projection, audit_check_id, rounded, sql_test_check_id, -}; +use crate::bindings::_helpers::result_output::projection::{Projection, ResultIdentity, rounded}; +use crate::bindings::_helpers::scopes::scope_report_objects::json_to_python; -const PASS_OUTCOME: &str = "pass"; -const INSUFFICIENT_OUTCOME: &str = "insufficient"; -const MEASUREMENT_MODE: &str = "measurement"; const SUCCESS_STATUS: &str = "success"; const DECIMAL_KIND: &str = "decimal"; const SCALAR_KINDS: [&str; 5] = ["string", "integer", "boolean", "float", "null"]; @@ -31,14 +33,6 @@ fn value<'py>(result: &Bound<'py, PyAny>, field: &str) -> PyResult(result: &Bound<'py, PyAny>, field: &str) -> PyResult> { - let value: Bound<'py, PyAny> = native_field(result, field)?; - if value.is_none() { - return Ok(value); - } - native_field(&value, "value") -} - fn fields<'py>( result: &Bound<'py, PyAny>, names: &[&'static str], @@ -81,131 +75,140 @@ fn sql_value_json<'py>(value: &Bound<'py, PyAny>) -> PyResult> /// The case, schema and parameter values of a parameterized SQL test. fn case_metadata<'py>( - result: &Bound<'py, PyAny>, + py: Python<'py>, + result: &SqlTestExecutionResult>, ) -> PyResult)>> { - let py: Python<'py> = result.py(); - let source_path: Bound<'py, PyAny> = native_field(result, "source_path")?; - if native_field(result, "case_name")?.is_none() || source_path.is_none() { + let Some(source_path) = result + .source_path + .as_ref() + .filter(|_| result.case_name.is_some()) + else { return Ok(Vec::new()); - } + }; let parameter_types: Bound<'py, PyDict> = PyDict::new(py); let mut schema: Vec> = Vec::new(); - for parameter in items(&native_field(result, "parameter_schema")?)? { - let parameter_type: Bound<'py, PyAny> = value(¶meter, "value_type")?; - parameter_types.set_item(native_field(¶meter, "name")?, ¶meter_type)?; + for parameter in &result.parameter_schema { + let parameter_type: Bound<'py, PyAny> = value(parameter, "value_type")?; + parameter_types.set_item(native_field(parameter, "name")?, ¶meter_type)?; schema.push(dict( py, vec![ - ("name", native_field(¶meter, "name")?), + ("name", native_field(parameter, "name")?), ("type", parameter_type), - ("nullable", native_field(¶meter, "nullable")?), + ("nullable", native_field(parameter, "nullable")?), ], )?); } let mut parameters: Vec> = Vec::new(); - for item in items(&native_field(result, "parameter_values")?)? { - let (name, parameter_value): (Bound<'py, PyAny>, Bound<'py, PyAny>) = item.extract()?; - let parameter_type: Bound<'py, PyAny> = parameter_types.as_any().get_item(&name)?; + for (name, parameter_value) in &result.parameter_values { + let parameter_type: Bound<'py, PyAny> = parameter_types.as_any().get_item(name)?; parameters.push(dict( py, vec![ - ("name", name), + ("name", any(py, name)?), ("type", parameter_type), - ("value", sql_value_json(¶meter_value)?), + ("value", sql_value_json(parameter_value)?), ], )?); } - let mut metadata: Vec<(&'static str, Bound<'py, PyAny>)> = - vec![("source_path", source_path.call_method0("as_posix")?)]; - metadata.extend(fields( - result, - &[ - "block_index", - "parent_name", - "case_name", - "case_index", - "case_fingerprint", - ], - )?); + let mut metadata: Vec<(&'static str, Bound<'py, PyAny>)> = vec![ + ( + "source_path", + any( + py, + source_path + .to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "/"), + )?, + ), + ("block_index", any(py, result.block_index)?), + ("parent_name", any(py, &result.parent_name)?), + ("case_name", any(py, &result.case_name)?), + ("case_index", any(py, result.case_index)?), + ("case_fingerprint", any(py, &result.case_fingerprint)?), + ]; metadata.push(("parameter_schema", tuple_of(py, schema)?)); metadata.push(("parameters", tuple_of(py, parameters)?)); Ok(metadata) } -fn difference_samples<'py>(samples: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = samples.py(); +fn difference_samples<'py>( + py: Python<'py>, + samples: &[SqlTestDifferenceSample], +) -> PyResult> { let formatted: Bound<'py, PyList> = PyList::empty(py); - for sample in items(samples)? { + for sample in samples { let pairs: Bound<'py, PyList> = PyList::empty(py); - for pair in items(&native_field(&sample, "values")?)? { - let (name, sample_value): (Bound<'py, PyAny>, Bound<'py, PyAny>) = pair.extract()?; - pairs.append(dict(py, vec![("name", name), ("value", sample_value)])?)?; + for (name, sample_value) in &sample.values { + pairs.append(dict( + py, + vec![("name", any(py, name)?), ("value", any(py, sample_value)?)], + )?)?; } formatted.append(pairs)?; } or_none(formatted.into_any()) } -fn sql_test_step<'py>(step: &Bound<'py, PyAny>) -> PyResult> { - let py: Python<'py> = step.py(); +fn sql_test_step<'py>(py: Python<'py>, step: &StepResult) -> PyResult> { let differences: Bound<'py, PyList> = PyList::empty(py); - for difference in items(&native_field(step, "column_differences")?)? { + for difference in &step.column_differences { differences.append(dict( py, - fields(&difference, &["name", "actual", "expected"])?, + vec![ + ("name", any(py, &difference.name)?), + ("actual", any(py, &difference.actual)?), + ("expected", any(py, &difference.expected)?), + ], )?)?; } let mut entries: Vec<(&str, Bound<'py, PyAny>)> = vec![ - ("model_name", native_field(step, "model_name")?), - ("status", value(step, "outcome")?), + ("model_name", any(py, &step.model_name)?), + ("status", any(py, step.outcome.value())?), + ("actual_row_count", any(py, step.actual_row_count)?), + ("expected_row_count", any(py, step.expected_row_count)?), + ("mismatched_row_count", any(py, step.mismatched_row_count)?), + ("unexpected_row_count", any(py, step.unexpected_row_count)?), + ("missing_row_count", any(py, step.missing_row_count)?), ]; - entries.extend(fields( - step, - &[ - "actual_row_count", - "expected_row_count", - "mismatched_row_count", - "unexpected_row_count", - "missing_row_count", - ], - )?); entries.push(( "unexpected_samples", - difference_samples(&native_field(step, "unexpected_samples")?)?, + difference_samples(py, &step.unexpected_samples)?, )); entries.push(( "missing_samples", - difference_samples(&native_field(step, "missing_samples")?)?, + difference_samples(py, &step.missing_samples)?, )); entries.push(("column_differences", or_none(differences.into_any())?)); - entries.extend(fields( - step, - &["error_code", "error_help", "error_message"], - )?); + entries.extend([ + ("error_code", any(py, &step.error_code)?), + ("error_help", any(py, &step.error_help)?), + ("error_message", any(py, &step.error_message)?), + ]); without_none(py, entries) } /// SQL test checks with their steps and parameterized case metadata. pub(crate) fn sql_test_checks<'py>( projection: &Projection<'py>, - results: &[Bound<'py, PyAny>], + results: &[&SqlTestExecutionResult>], ) -> PyResult>> { let mut checks: Vec> = Vec::new(); for result in results { if !projection.terminal_recorded(result)? { continue; } - let py: Python<'py> = result.py(); - let steps: Vec> = items(&native_field(result, "step_results")?)?; - let outcome: Bound<'py, PyAny> = value(result, "outcome")?; + let py: Python<'py> = projection.py; + let steps = &result.step_results; + let outcome = result.outcome; let mut entries: Vec<(&str, Bound<'py, PyAny>)> = vec![ ("kind", any(py, "sql_test")?), - ("name", native_field(result, "test_name")?), - ("check_id", any(py, sql_test_check_id(result)?)?), + ("name", any(py, &result.test_name)?), + ("check_id", any(py, result.identity()?.map(|(_, id)| id))?), ]; - entries.extend(case_metadata(result)?); - entries.push(("passed", any(py, outcome.eq(PASS_OUTCOME)?)?)); - entries.push(("status", outcome)); + entries.extend(case_metadata(py, result)?); + entries.push(("passed", any(py, outcome == SqlTestOutcome::Pass)?)); + entries.push(("status", any(py, outcome.value())?)); entries.push(( "duration_ms", projection.result_duration(result, &py.None().into_bound(py))?, @@ -213,19 +216,23 @@ pub(crate) fn sql_test_checks<'py>( entries.push(( "asset_name", match steps.first() { - Some(step) => native_field(step, "model_name")?, + Some(step) => any(py, &step.model_name)?, None => py.None().into_bound(py), }, )); - entries.extend(fields( - result, - &["error_code", "error_help", "error_message"], - )?); + entries.extend([ + ("error_code", any(py, &result.error_code)?), + ("error_help", any(py, &result.error_help)?), + ("error_message", any(py, &result.error_message)?), + ]); entries.push(( "steps", tuple_of( py, - steps.iter().map(sql_test_step).collect::>()?, + steps + .iter() + .map(|step| sql_test_step(py, step)) + .collect::>()?, )?, )); checks.push(without_none(py, entries)?); @@ -263,79 +270,103 @@ pub(crate) fn measurement_thresholds<'py>( Ok(rendered.into_any()) } -/// `len(evidence_rows)` and `evidence_truncated` of a measurement audit, else `None`. -pub(crate) fn measurement_evidence<'py>( - result: &Bound<'py, PyAny>, -) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { - let py: Python<'py> = result.py(); - if !value(result, "evaluation_mode")?.eq(MEASUREMENT_MODE)? { - return Ok((py.None().into_bound(py), py.None().into_bound(py))); +/// Keep integer measurements exact across both JSON output contracts. +pub(crate) fn measured_value<'py>( + py: Python<'py>, + value: Option<&MeasuredValue>, +) -> PyResult> { + match value { + Some(MeasuredValue::Integer(value)) => { + Ok(json_to_python(py, JsonValue::Integer(value.clone()), false)?.into_bound(py)) + } + Some(MeasuredValue::Float(value)) => any(py, *value), + None => Ok(py.None().into_bound(py)), } - Ok(( - any(py, native_field(result, "evidence_rows")?.len()?)?, - native_field(result, "evidence_truncated")?, - )) -} - -/// Whether an audit outcome counts as passed. -pub(crate) fn audit_passed(result: &Bound<'_, PyAny>) -> PyResult { - let outcome: Bound<'_, PyAny> = value(result, "outcome")?; - Ok(outcome.eq(PASS_OUTCOME)? || outcome.eq(INSUFFICIENT_OUTCOME)?) } /// Audit checks; `terminal_evidence` keeps results whose model already has a terminal. pub(crate) fn audit_checks<'py>( projection: &Projection<'py>, - results: &Bound<'py, PyAny>, + results: &[&AuditExecutionResult>], terminal_evidence: bool, ) -> PyResult>> { let mut checks: Vec> = Vec::new(); - for result in items(results)? { + for result in results { if !terminal_evidence && !projection.terminal_recorded(&result)? { continue; } - let py: Python<'py> = result.py(); - let (evidence_count, evidence_truncated) = measurement_evidence(&result)?; + let py: Python<'py> = projection.py; + let measurement = result.evaluation_mode == AuditEvaluationMode::Measurement; + let measured = measured_value(py, result.measured_value.as_ref())?; + let meta = match &result.audit_meta { + Some(meta) => dict( + py, + meta.iter() + .map(|(key, value)| (key.as_str(), value.clone())) + .collect(), + )? + .into_any(), + None => py.None().into_bound(py), + }; let mut entries: Vec<(&str, Bound<'py, PyAny>)> = vec![("kind", any(py, "audit")?)]; - entries.push(("name", native_field(&result, "audit_name")?)); - entries.extend(fields( - &result, - &["audit_definition_name", "audit_description", "audit_meta"], - )?); - entries.push(("check_id", any(py, audit_check_id(&result)?)?)); - entries.push(("passed", any(py, audit_passed(&result)?)?)); - entries.push(("status", value(&result, "outcome")?)); - entries.push(("severity", value(&result, "severity")?)); - entries.push(("row_count", native_field(&result, "row_count")?)); - entries.push(("attachment_kind", value(&result, "attachment_kind")?)); + entries.push(("name", any(py, &result.audit_name)?)); + entries.extend([ + ( + "audit_definition_name", + any(py, &result.audit_definition_name)?, + ), + ("audit_description", any(py, &result.audit_description)?), + ("audit_meta", meta), + ]); + entries.push(("check_id", any(py, result.identity()?.map(|(_, id)| id))?)); + entries.push(( + "passed", + any( + py, + matches!( + result.outcome, + AuditOutcome::Pass | AuditOutcome::Insufficient + ), + )?, + )); + entries.push(("status", any(py, result.outcome.value())?)); + entries.push(("severity", any(py, result.severity.value())?)); + entries.push(("row_count", any(py, result.row_count)?)); + entries.push(("attachment_kind", any(py, result.attachment_kind.value())?)); entries.push(( "attached_target_kind", - optional_value(&result, "attached_target_kind")?, + any(py, result.attached_target_kind.map(ResultEnum::value))?, )); - entries.push(("asset_name", native_field(&result, "attached_target_name")?)); + entries.push(("asset_name", any(py, &result.attached_target_name)?)); entries.push(( "attached_column_name", - native_field(&result, "attached_column_name")?, + any(py, &result.attached_column_name)?, )); - entries.push(("run_scope_phase", value(&result, "run_scope_phase")?)); - entries.push(("reused", native_field(&result, "reused")?)); - entries.push(("evaluation_mode", value(&result, "evaluation_mode")?)); - entries.extend(fields( - &result, - &[ - "measured_value", - "sample_count", - "sample_unit", - "minimum_samples", - ], - )?); + entries.push(("run_scope_phase", any(py, result.run_scope_phase.value())?)); + entries.push(("reused", any(py, result.reused)?)); + entries.push(("evaluation_mode", any(py, result.evaluation_mode.value())?)); + entries.extend([ + ("measured_value", measured), + ("sample_count", any(py, result.sample_count)?), + ("sample_unit", any(py, &result.sample_unit)?), + ("minimum_samples", any(py, result.minimum_samples)?), + ]); entries.push(( "thresholds", - measurement_thresholds(&native_field(&result, "thresholds")?)?, + measurement_thresholds(&any(py, &result.thresholds)?)?, + )); + entries.push(( + "evidence_count", + any(py, measurement.then_some(result.evidence_rows.len()))?, + )); + entries.push(( + "evidence_truncated", + any(py, measurement.then_some(result.evidence_truncated))?, )); - entries.push(("evidence_count", evidence_count)); - entries.push(("evidence_truncated", evidence_truncated)); - entries.extend(fields(&result, &["execution_error", "skip_reason"])?); + entries.extend([ + ("execution_error", any(py, &result.execution_error)?), + ("skip_reason", any(py, &result.skip_reason)?), + ]); checks.push(without_none(py, entries)?); } Ok(checks) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs index 7227f41fe8..533ecb2537 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs @@ -4,13 +4,19 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_host::models::{ + AuditExecutionResult, AuditOutcome, BuildExecutionResult, BuildStatus, ExecutionStatus, + FunctionExecutionResult, LoadExecutionResult, ModelExecutionResult, SeedExecutionResult, + SqlTestExecutionResult, SqlTestOutcome, +}; use crate::bindings::_helpers::boundary::python_calls::KeywordArguments; use crate::bindings::_helpers::compile_outputs::json_report::indented_json; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::plan_output::values::{any, dict, items, text}; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::result_output::aggregation::aggregated_from_lists; +use crate::bindings::_helpers::result_output::aggregation::RetainedBuildResults; use crate::bindings::_helpers::result_output::assets::{ clone_asset, function_assets, load_assets, model_assets, python_node_assets, seed_assets, without_none, @@ -24,10 +30,6 @@ const JSON_VERSION: i64 = 1; const FAILED: &str = "failed"; const SUCCESS: &str = "success"; const SKIPPED: &str = "skipped"; -const ERROR_OUTCOME: &str = "error"; -const PASS_OUTCOME: &str = "pass"; -const TIMEOUT_OUTCOME: &str = "timeout"; -const WARN_OUTCOME: &str = "warn"; const BUILD_OPTIONS: (&[&str], &[&str]) = ( &["result", "plan"], &[ @@ -48,6 +50,7 @@ const DEGRADABLE_BUILD_OPTIONS: (&[&str], &[&str]) = ( "cost", ], ); +const TYPED_BUILD_OPTIONS: (&[&str], &[&str]) = (&["plan", "execution_succeeded"], BUILD_OPTIONS.1); const SCENARIO_OPTIONS: (&[&str], &[&str]) = ( &["results"], &["local", "run_namespace", "namespace_source", "duration_ms"], @@ -56,16 +59,6 @@ const SNAPSHOT_OPTIONS: (&[&str], &[&str]) = ( &["results"], &["refresh", "run_namespace", "namespace_source"], ); -const BUILD_RESULT_FIELDS: [&str; 7] = [ - "model_results", - "seed_results", - "function_results", - "load_results", - "test_results", - "source_audit_results", - "end_audit_results", -]; - /// The optional sections of a document after `checks`. #[derive(Default)] struct Extras<'py> { @@ -176,8 +169,10 @@ fn sequence<'py>(py: Python<'py>, value: Option>) -> Bound<'py value.unwrap_or_else(|| PyTuple::empty(py).into_any()) } -fn count(result: &Bound<'_, PyAny>, field: &str) -> PyResult { - native_field(result, field)?.extract() +fn count(value: i64) -> PyResult { + usize::try_from(value).map_err(|_| { + pyo3::exceptions::PyOverflowError::new_err("can't convert negative int to unsigned") + }) } /// Format build command execution results as JSON. @@ -189,21 +184,28 @@ pub(crate) fn format_build_execution_json<'py>( ) -> PyResult { let arguments: KeywordArguments<'py> = KeywordArguments::read(py, "format_build_execution_json", options, BUILD_OPTIONS)?; - build_json(py, &arguments, Vec::new()) + let result = BuildExecutionResult::read(&arguments.value("result")?)?; + build_json(py, &arguments, &result, Vec::new()) } /// The audit-history projection counts, flagged as degraded when any write failed. fn audit_projection<'py>( - result: &Bound<'py, PyAny>, + py: Python<'py>, + result: &BuildExecutionResult>, ) -> PyResult)>> { - let py: Python<'py> = result.py(); let counts: Vec<(&str, Bound<'py, PyAny>)> = [ - ("attempted_count", "audit_result_projection_attempted_count"), - ("written_count", "audit_result_projection_written_count"), - ("failed_count", "audit_result_projection_failed_count"), + ( + "attempted_count", + result.audit_result_projection_attempted_count, + ), + ( + "written_count", + result.audit_result_projection_written_count, + ), + ("failed_count", result.audit_result_projection_failed_count), ] .into_iter() - .map(|(key, field)| Ok((key, native_field(result, field)?))) + .map(|(key, value)| Ok((key, any(py, value)?))) .collect::>()?; let failed: bool = counts[2].1.is_truthy()?; let mut entries: Vec<(&'static str, Bound<'py, PyAny>)> = vec![( @@ -236,9 +238,33 @@ pub(crate) fn format_degradable_build_execution_json<'py>( options, DEGRADABLE_BUILD_OPTIONS, )?; + let result = BuildExecutionResult::read(&arguments.value("result")?)?; + degradable_build_json(py, &arguments, &result) +} + +/// Production build completion already owns the converted executor result. +pub(crate) fn format_build_execution_json_with<'py>( + py: Python<'py>, + result: &BuildExecutionResult>, + options: Option>, +) -> PyResult { + let arguments = KeywordArguments::read( + py, + "format_build_execution_json", + options, + TYPED_BUILD_OPTIONS, + )?; + degradable_build_json(py, &arguments, result) +} + +fn degradable_build_json<'py>( + py: Python<'py>, + arguments: &KeywordArguments<'py>, + result: &BuildExecutionResult>, +) -> PyResult { let execution_succeeded: bool = arguments.flag("execution_succeeded")?; - let rendered: PyResult = audit_projection(&arguments.value("result")?) - .and_then(|trailing| build_json(py, &arguments, trailing)); + let rendered: PyResult = audit_projection(py, result) + .and_then(|trailing| build_json(py, arguments, result, trailing)); let error: pyo3::PyErr = match rendered { Ok(payload) => return Ok(payload), Err(error) => error, @@ -268,9 +294,9 @@ pub(crate) fn format_degradable_build_execution_json<'py>( fn build_json<'py>( py: Python<'py>, arguments: &KeywordArguments<'py>, + result: &BuildExecutionResult>, trailing: Vec<(&'static str, Bound<'py, PyAny>)>, ) -> PyResult { - let result: Bound<'py, PyAny> = arguments.value("result")?; let plan: Bound<'py, PyAny> = arguments.value("plan")?; let command: String = if arguments.passed("command")? { arguments.value("command")?.extract()? @@ -282,17 +308,10 @@ fn build_json<'py>( sequence(py, passed(arguments, "python_node_results")?); let python_check_results: Bound<'py, PyAny> = sequence(py, passed(arguments, "python_check_results")?); - let mut retained: Vec>> = Vec::new(); - let mut partial: bool = false; - for field in BUILD_RESULT_FIELDS { - let all: Bound<'py, PyAny> = native_field(&result, field)?; - let kept: Vec> = projection.terminal_results(&all)?; - partial = partial || kept.len() != all.len()?; - retained.push(kept); - } + let retained = RetainedBuildResults::new(result, &projection)?; let nodes: Vec> = projection.terminal_results(&python_node_results)?; let checks_run: Vec> = projection.terminal_results(&python_check_results)?; - partial = partial + let partial = retained.is_partial(result) || nodes.len() != python_node_results.len()? || checks_run.len() != python_check_results.len()?; let python_fail: usize = count_status(&nodes, "status", FAILED)?; @@ -302,61 +321,68 @@ fn build_json<'py>( let check_warn: usize = count_truthy(&checks_run, "warned")?; let check_pass: usize = count_truthy(&checks_run, "passed")?; let mut python_warnings: usize = 0; - for warned in nodes.iter().chain(&checks_run).chain(&retained[3]) { + for warned in nodes.iter().chain(&checks_run) { python_warnings += native_field(warned, "warning_messages")?.len()?; } - let result_status: String = text(&native_field(&native_field(&result, "status")?, "value")?)?; - let status: String = if result_status == FAILED || python_fail > 0 || check_fail > 0 { - FAILED.to_owned() - } else { - result_status - }; - let mut checks: Vec> = sql_test_checks(&projection, &retained[4])?; - for audits in [&retained[5], &retained[6]] { - checks.extend(audit_checks( - &projection, - PyList::new(py, audits)?.as_any(), - false, - )?); + python_warnings += retained + .loads + .iter() + .map(|result| result.warning_messages.len()) + .sum::(); + let status: String = + if result.status == BuildStatus::Failed || python_fail > 0 || check_fail > 0 { + FAILED.to_owned() + } else { + SUCCESS.to_owned() + }; + let mut checks: Vec> = sql_test_checks(&projection, &retained.tests)?; + for audits in [&retained.source_audits, &retained.end_audits] { + checks.extend(audit_checks(&projection, audits, false)?); } - for model in &retained[0] { + for model in &retained.models { checks.extend(audit_checks( &projection, - &native_field(model, "audit_results")?, + &model.audit_results.iter().collect::>(), projection.terminal_recorded(model)?, )?); } checks.extend(python_checks(&projection, &checks_run)?); - let mut assets: Vec> = model_assets(&projection, &retained[0], Some(&plan))?; - assets.extend(seed_assets(&projection, &retained[1], Some(&plan))?); - assets.extend(function_assets(&projection, &retained[2], Some(&plan))?); - assets.extend(load_assets(&projection, &retained[3])?); + let mut assets: Vec> = + model_assets(&projection, &retained.models, Some(&plan))?; + assets.extend(seed_assets(&projection, &retained.seeds, Some(&plan))?); + assets.extend(function_assets( + &projection, + &retained.functions, + Some(&plan), + )?); + assets.extend(load_assets(&projection, &retained.loads)?); assets.extend(python_node_assets(&projection, &nodes)?); - let counted: Bound<'py, PyAny> = if partial { - aggregated_from_lists(py, &retained)? + let counted: [usize; 5] = if partial { + let counts = retained.counts(); + [ + counts.success, + counts.failure, + counts.timeout, + counts.skipped, + counts.warning, + ] } else { - result.clone() + [ + count(result.success_count)?, + count(result.failure_count)?, + count(result.timeout_count)?, + count(result.skipped_count)?, + count(result.warning_count)?, + ] }; let summary: Bound<'py, PyDict> = summary( py, &[ - ( - "success_count", - count(&counted, "success_count")? + python_success, - ), - ( - "failure_count", - count(&counted, "failure_count")? + python_fail + check_fail, - ), - ("timeout_count", count(&counted, "timeout_count")?), - ( - "skipped_count", - count(&counted, "skipped_count")? + python_skipped, - ), - ( - "warning_count", - count(&counted, "warning_count")? + check_warn + python_warnings, - ), + ("success_count", counted[0] + python_success), + ("failure_count", counted[1] + python_fail + check_fail), + ("timeout_count", counted[2]), + ("skipped_count", counted[3] + python_skipped), + ("warning_count", counted[4] + check_warn + python_warnings), ("python_check_pass_count", check_pass), ("python_check_warn_count", check_warn), ("python_check_fail_count", check_fail), @@ -389,8 +415,12 @@ pub(crate) fn format_seed_execution_json<'py>( plan: Bound<'py, PyAny>, ) -> PyResult { let projection: Projection<'py> = Projection::installed(py)?; - let results: Vec> = projection.terminal_results(&results)?; - let fail_count: usize = count_status(&results, "status", FAILED)?; + let all = Vec::::read(&results)?; + let results = projection.retained(&all)?; + let fail_count = results + .iter() + .filter(|result| result.status == ExecutionStatus::Failed) + .count(); emitted( &projection, Document { @@ -399,7 +429,13 @@ pub(crate) fn format_seed_execution_json<'py>( summary: summary( py, &[ - ("success_count", count_status(&results, "status", SUCCESS)?), + ( + "success_count", + results + .iter() + .filter(|result| result.status == ExecutionStatus::Success) + .count(), + ), ("failure_count", fail_count), ("total_count", results.len()), ], @@ -419,8 +455,12 @@ pub(crate) fn format_load_execution_json<'py>( results: Bound<'py, PyAny>, ) -> PyResult { let projection: Projection<'py> = Projection::installed(py)?; - let results: Vec> = projection.terminal_results(&results)?; - let fail_count: usize = count_status(&results, "status", FAILED)?; + let all = Vec::>::read(&results)?; + let results = projection.retained(&all)?; + let fail_count = results + .iter() + .filter(|result| result.status == ExecutionStatus::Failed) + .count(); emitted( &projection, Document { @@ -429,9 +469,21 @@ pub(crate) fn format_load_execution_json<'py>( summary: summary( py, &[ - ("success_count", count_status(&results, "status", SUCCESS)?), + ( + "success_count", + results + .iter() + .filter(|result| result.status == ExecutionStatus::Success) + .count(), + ), ("failure_count", fail_count), - ("skipped_count", count_status(&results, "status", SKIPPED)?), + ( + "skipped_count", + results + .iter() + .filter(|result| result.status == ExecutionStatus::Skipped) + .count(), + ), ("total_count", results.len()), ], )?, @@ -501,9 +553,16 @@ pub(crate) fn format_test_execution_json<'py>( results: Bound<'py, PyAny>, ) -> PyResult { let projection: Projection<'py> = Projection::installed(py)?; - let results: Vec> = projection.terminal_results(&results)?; - let pass_count: usize = count_status(&results, "outcome", PASS_OUTCOME)?; - let timeout_count: usize = count_status(&results, "outcome", TIMEOUT_OUTCOME)?; + let all = Vec::>::read(&results)?; + let results = projection.retained(&all)?; + let pass_count = results + .iter() + .filter(|result| result.outcome == SqlTestOutcome::Pass) + .count(); + let timeout_count = results + .iter() + .filter(|result| result.outcome == SqlTestOutcome::Timeout) + .count(); let fail_count: usize = results.len() - pass_count; emitted( &projection, @@ -538,8 +597,12 @@ pub(crate) fn format_audit_execution_json<'py>( let configured_concurrency: Bound<'py, PyAny> = configured_concurrency.map_or_else(|| any(py, 1_i64), Ok)?; let projection: Projection<'py> = Projection::installed(py)?; - let results: Vec> = projection.terminal_results(&results)?; - let fail_count: usize = count_status(&results, "outcome", ERROR_OUTCOME)?; + let all = Vec::>::read(&results)?; + let results = projection.retained(&all)?; + let fail_count = results + .iter() + .filter(|result| result.outcome == AuditOutcome::Error) + .count(); let workers: Bound<'py, PyAny> = match worker_count.filter(|count| !count.is_none()) { Some(workers) => workers, None => py @@ -557,18 +620,24 @@ pub(crate) fn format_audit_execution_json<'py>( &[ ( "pass_count", - count_status(&results, "outcome", PASS_OUTCOME)?, + results + .iter() + .filter(|result| result.outcome == AuditOutcome::Pass) + .count(), ), ( "warn_count", - count_status(&results, "outcome", WARN_OUTCOME)?, + results + .iter() + .filter(|result| result.outcome == AuditOutcome::Warn) + .count(), ), ("fail_count", fail_count), ("total_count", results.len()), ], )?, assets: Vec::new(), - checks: audit_checks(&projection, PyList::new(py, &results)?.as_any(), false)?, + checks: audit_checks(&projection, &results, false)?, }, Extras { execution: Some(dict( @@ -645,17 +714,23 @@ pub(crate) fn format_scenario_execution_json<'py>( projection.scenario = Some(text(&native_field(result, "scenario_name")?)?); assets.extend(seed_assets( &projection, - &items(&native_field(result, "seed_results")?)?, + &Vec::::read(&native_field(result, "seed_results")?)? + .iter() + .collect::>(), None, )?); assets.extend(function_assets( &projection, - &items(&native_field(result, "function_results")?)?, + &Vec::::read(&native_field(result, "function_results")?)? + .iter() + .collect::>(), None, )?); assets.extend(model_assets( &projection, - &items(&native_field(result, "model_results")?)?, + &Vec::>::read(&native_field(result, "model_results")?)? + .iter() + .collect::>(), None, )?); projection.scenario = None; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs index 45a7acab30..d35c91e72f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_records.rs @@ -2,71 +2,36 @@ use pyo3::IntoPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::{PyBool, PyFloat, PyInt}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFloat, PyInt}; use sqlbuild_core::integration_results::errors::IntegrationRecordError; use sqlbuild_core::integration_results::main::encode_integration_value::encode_integration_value; use sqlbuild_core::integration_results::main::normalize_integration_record::normalize_integration_record; use sqlbuild_core::json::models::{JsonInteger, JsonValue}; +use sqlbuild_host::models::{ + AuditEvaluationMode, AuditExecutionResult, AuditOutcome, FunctionExecutionResult, + LoadExecutionResult, ModelExecutionResult, SeedExecutionResult, SqlTestExecutionResult, + SqlTestOutcome, +}; +use sqlbuild_host::types::ResultEnum; +use crate::bindings::_helpers::native_host::results::ReadResult; use crate::bindings::_helpers::plan_output::safety::{future_cursor_safety, maximum_start_safety}; -use crate::bindings::_helpers::plan_output::values::items; +use crate::bindings::_helpers::plan_output::values::any; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; +use crate::bindings::_helpers::result_output::assets::microbatch_summary; use crate::bindings::_helpers::result_output::checks::{ - audit_passed, measurement_evidence, measurement_thresholds, python_check_status, + measured_value, measurement_thresholds, python_check_status, }; use crate::bindings::_helpers::result_output::integration_values::{ Checked, bounded_metadata, optional_text, required_json, text, text_or_empty, }; use crate::bindings::_helpers::result_output::projection::{ - ResultKind, audit_check_id, result_kind, sql_test_check_id, + ResultIdentity, ResultKind, result_kind, }; const FAILED_EVENT: &str = "resource_attempt_failed"; const SKIPPED_EVENT: &str = "resource_attempt_skipped"; -const PASS_OUTCOME: &str = "pass"; -const MICROBATCH_FIELDS: [(&str, &str); 19] = [ - ("strategy", "microbatch_strategy"), - ("reason", "microbatch_plan_reason"), - ("batch_count", "batch_count"), - ("batch_size", "batch_size"), - ("recovery_batch_count", "microbatch_recovery_batch_count"), - ("known_gap_count", "microbatch_known_gap_count"), - ( - "unaccounted_interval_count", - "microbatch_unaccounted_interval_count", - ), - ( - "synthetic_completion_count", - "microbatch_synthetic_completion_count", - ), - ( - "unknown_fingerprint_count", - "microbatch_unknown_fingerprint_count", - ), - ("contiguous_frontier", "microbatch_contiguous_frontier"), - ( - "unaccounted_partition_policy", - "microbatch_unaccounted_partition_policy", - ), - ("replay_requirement_id", "microbatch_replay_requirement_id"), - ( - "required_model_version_hash", - "microbatch_required_model_version_hash", - ), - ( - "physical_generation_id", - "microbatch_physical_generation_id", - ), - ("concurrent_enabled", "microbatch_concurrent_enabled"), - ("batch_concurrency", "microbatch_batch_concurrency"), - ("global_concurrency", "microbatch_global_concurrency"), - ( - "replay_requirement_state", - "microbatch_replay_requirement_state", - ), - ("limit", "microbatch_limit"), -]; type Entries = Vec<(String, JsonValue)>; @@ -86,15 +51,6 @@ fn value<'py>(result: &Bound<'py, PyAny>, field: &str) -> PyResult.value` when it is truthy, else `None`. -fn truthy_value<'py>(result: &Bound<'py, PyAny>, field: &str) -> PyResult> { - let value: Bound<'py, PyAny> = native_field(result, field)?; - if !value.is_truthy()? { - return Ok(result.py().None().into_bound(result.py())); - } - native_field(&value, "value") -} - /// `plan.[name].qualified_name` when the plan locates the resource. fn planned_target<'py>( plan: &Bound<'py, PyAny>, @@ -112,27 +68,29 @@ fn planned_target<'py>( native_field(&locations.get_item(name)?, "qualified_name") } -fn model_microbatch(result: &Bound<'_, PyAny>) -> PyResult { - let py: Python<'_> = result.py(); - let run_type: Bound<'_, PyAny> = native_field(result, "microbatch_run_type")?; - if run_type.is_none() { +fn model_microbatch<'py>( + py: Python<'py>, + result: &ModelExecutionResult>, +) -> PyResult { + let rendered: Bound<'py, PyAny> = microbatch_summary(py, result)?; + if rendered.is_none() { return Ok(JsonValue::Object(Vec::new())); } - let values: Bound<'_, pyo3::types::PyDict> = pyo3::types::PyDict::new(py); - pyo3::types::PyDictMethods::set_item(&values, "run_type", run_type)?; - for (key, field) in MICROBATCH_FIELDS { - let item: Bound<'_, PyAny> = native_field(result, field)?; + let values: Bound<'py, PyDict> = PyDict::new(py); + for (key, item) in rendered.downcast::()?.iter() { + let key: String = key.extract()?; if !item.is_none() { - pyo3::types::PyDictMethods::set_item(&values, key, item)?; + values.set_item(key, item)?; } } - let count: Bound<'_, PyAny> = native_field(result, "microbatch_limit_count")?; - if !count.is_none() { - pyo3::types::PyDictMethods::set_item(&values, "count", count)?; + if let Some(limit) = result.microbatch_limit { + values.set_item("limit", limit)?; + } + if let Some(count) = result.microbatch_limit_count { + values.set_item("count", count)?; } - let action: Bound<'_, PyAny> = native_field(result, "microbatch_limit_action")?; - if !action.is_none() { - pyo3::types::PyDictMethods::set_item(&values, "action", native_field(&action, "value")?)?; + if let Some(action) = result.microbatch_limit_action { + values.set_item("action", action.value())?; } bounded_metadata(values.as_any()) } @@ -143,25 +101,27 @@ fn asset<'py>(result: &Bound<'py, PyAny>, plan: &Bound<'py, PyAny>) -> Checked)> = match result_kind(result)? { ResultKind::Model => { - let name: Bound<'py, PyAny> = native_field(result, "model_name")?; + let native: ModelExecutionResult> = + ModelExecutionResult::read(result)?; + let name: Bound<'py, PyAny> = any(py, &native.model_name)?; let planned: Bound<'py, PyAny> = planned_target(plan, "model_locations", &name)?; let mut values: Vec<(&str, Bound<'py, PyAny>)> = vec![ ("kind", "model".into_pyobject(py)?.into_any()), ("name", name), - ("status", value(result, "status")?), + ("status", any(py, native.status.value())?), ( "target", if planned.is_truthy()? { planned } else { - native_field(result, "promoted_relation")? + any(py, &native.promoted_relation)? }, ), + ("staging_relation", any(py, &native.staging_relation)?), ( - "staging_relation", - native_field(result, "staging_relation")?, + "failed_phase", + any(py, native.failed_phase.map(ResultEnum::value))?, ), - ("failed_phase", truthy_value(result, "failed_phase")?), ]; let mut converted: Entries = match entries(std::mem::take(&mut values))? { Ok(converted) => converted, @@ -170,11 +130,11 @@ fn asset<'py>(result: &Bound<'py, PyAny>, plan: &Bound<'py, PyAny>) -> Checked = if evidence.is_none() { @@ -184,39 +144,41 @@ fn asset<'py>(result: &Bound<'py, PyAny>, plan: &Bound<'py, PyAny>) -> Checked { - let name: Bound<'py, PyAny> = native_field(result, "seed_name")?; + let native = SeedExecutionResult::read(result)?; + let name: Bound<'py, PyAny> = any(py, &native.seed_name)?; vec![ ("kind", "seed".into_pyobject(py)?.into_any()), ("target", planned_target(plan, "seed_locations", &name)?), ("name", name), - ("status", value(result, "status")?), + ("status", any(py, native.status.value())?), ] } ResultKind::Function => { - let name: Bound<'py, PyAny> = native_field(result, "function_name")?; + let native = FunctionExecutionResult::read(result)?; + let name: Bound<'py, PyAny> = any(py, &native.function_name)?; vec![ - ("kind", native_field(result, "function_kind")?), + ("kind", any(py, &native.function_kind)?), ("target", planned_target(plan, "function_locations", &name)?), ("name", name), - ("status", value(result, "status")?), + ("status", any(py, native.status.value())?), + ] + } + ResultKind::Load => { + let native: LoadExecutionResult> = LoadExecutionResult::read(result)?; + vec![ + ("kind", any(py, native.resource_kind.value())?), + ("name", any(py, &native.source_name)?), + ("status", any(py, native.status.value())?), + ("target", any(py, &native.target)?), + ("staging_relation", any(py, &native.staging_relation)?), + ("loader", any(py, &native.loader_name)?), + ("rows_loaded", any(py, native.rows_loaded)?), ] } - ResultKind::Load => vec![ - ("kind", value(result, "resource_kind")?), - ("name", native_field(result, "source_name")?), - ("status", value(result, "status")?), - ("target", native_field(result, "target")?), - ( - "staging_relation", - native_field(result, "staging_relation")?, - ), - ("loader", native_field(result, "loader_name")?), - ("rows_loaded", native_field(result, "rows_loaded")?), - ], ResultKind::PythonNode => vec![ ("kind", value(result, "kind")?), ("name", native_field(result, "node_name")?), @@ -233,51 +195,64 @@ fn asset<'py>(result: &Bound<'py, PyAny>, plan: &Bound<'py, PyAny>) -> Checked(result: &Bound<'py, PyAny>) -> Checked { let py: Python<'py> = result.py(); - let check_id: Bound<'py, PyAny> = audit_check_id(result)?.into_pyobject(py)?.into_any(); - let (evidence_count, evidence_truncated) = measurement_evidence(result)?; + let native: AuditExecutionResult> = AuditExecutionResult::read(result)?; + let check_id = any(py, native.identity()?.map(|(_, id)| id))?; + let measurement = native.evaluation_mode == AuditEvaluationMode::Measurement; entries(vec![ ("kind", "audit".into_pyobject(py)?.into_any()), - ("name", native_field(result, "audit_name")?), + ("name", any(py, &native.audit_name)?), ("check_id", check_id.clone()), ("dag_check_id", check_id), ( "passed", - PyBool::new(py, audit_passed(result)?).to_owned().into_any(), + any( + py, + matches!( + native.outcome, + AuditOutcome::Pass | AuditOutcome::Insufficient + ), + )?, ), - ("status", value(result, "outcome")?), - ("severity", value(result, "severity")?), - ("row_count", native_field(result, "row_count")?), - ("attachment_kind", value(result, "attachment_kind")?), - ("asset_name", native_field(result, "attached_target_name")?), + ("status", any(py, native.outcome.value())?), + ("severity", any(py, native.severity.value())?), + ("row_count", any(py, native.row_count)?), + ("attachment_kind", any(py, native.attachment_kind.value())?), + ("asset_name", any(py, &native.attached_target_name)?), ( "attached_column_name", - native_field(result, "attached_column_name")?, + any(py, &native.attached_column_name)?, ), ( "attached_target_name", - native_field(result, "attached_target_name")?, + any(py, &native.attached_target_name)?, ), - ("run_scope_phase", value(result, "run_scope_phase")?), - ("reused", native_field(result, "reused")?), - ("evaluation_mode", value(result, "evaluation_mode")?), - ("measured_value", native_field(result, "measured_value")?), - ("sample_count", native_field(result, "sample_count")?), - ("sample_unit", native_field(result, "sample_unit")?), - ("minimum_samples", native_field(result, "minimum_samples")?), + ("run_scope_phase", any(py, native.run_scope_phase.value())?), + ("reused", any(py, native.reused)?), + ("evaluation_mode", any(py, native.evaluation_mode.value())?), + ( + "measured_value", + measured_value(py, native.measured_value.as_ref())?, + ), + ("sample_count", any(py, native.sample_count)?), + ("sample_unit", any(py, &native.sample_unit)?), + ("minimum_samples", any(py, native.minimum_samples)?), ( "thresholds", - measurement_thresholds(&native_field(result, "thresholds")?)?, + measurement_thresholds(&any(py, &native.thresholds)?)?, ), - ("evidence_count", evidence_count), - ("evidence_truncated", evidence_truncated), ( - "audit_definition_name", - native_field(result, "audit_definition_name")?, + "evidence_count", + any(py, measurement.then_some(native.evidence_rows.len()))?, ), ( - "audit_description", - native_field(result, "audit_description")?, + "evidence_truncated", + any(py, measurement.then_some(native.evidence_truncated))?, ), + ( + "audit_definition_name", + any(py, &native.audit_definition_name)?, + ), + ("audit_description", any(py, &native.audit_description)?), ]) } @@ -290,26 +265,20 @@ fn checks<'py>(result: &Bound<'py, PyAny>) -> Checked> { match result_kind(result)? { ResultKind::Audit => single(audit_check(result)), ResultKind::SqlTest => { - let check_id: Bound<'py, PyAny> = - sql_test_check_id(result)?.into_pyobject(py)?.into_any(); - let steps: Vec> = items(&native_field(result, "step_results")?)?; - let outcome: Bound<'py, PyAny> = value(result, "outcome")?; + let native: SqlTestExecutionResult> = + SqlTestExecutionResult::read(result)?; + let check_id = any(py, native.identity()?.map(|(_, id)| id))?; single(entries(vec![ ("kind", "sql_test".into_pyobject(py)?.into_any()), - ("name", native_field(result, "test_name")?), + ("name", any(py, &native.test_name)?), ("check_id", check_id.clone()), ("dag_check_id", check_id), - ( - "passed", - PyBool::new(py, outcome.eq(PASS_OUTCOME)?) - .to_owned() - .into_any(), - ), - ("status", outcome), + ("passed", any(py, native.outcome == SqlTestOutcome::Pass)?), + ("status", any(py, native.outcome.value())?), ( "asset_name", - match steps.first() { - Some(step) => native_field(step, "model_name")?, + match native.step_results.first() { + Some(step) => any(py, &step.model_name)?, None => py.None().into_bound(py), }, ), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs b/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs index 61ee4d6220..86bd0533a6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/result_output/projection.rs @@ -2,7 +2,14 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult, Python}; use pyo3::types::{PyFloat, PyInt}; +use sqlbuild_core::nested_progress::main::audit_resource_id::audit_resource_id; +use sqlbuild_core::nested_progress::main::sql_test_resource_id::sql_test_resource_id; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_host::models::{ + AuditExecutionResult, FunctionExecutionResult, LoadExecutionResult, ModelExecutionResult, + SeedExecutionResult, SqlTestExecutionResult, +}; +use sqlbuild_host::types::ResultEnum; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::plan_output::values::text; @@ -101,6 +108,86 @@ pub(crate) fn resource_identity(result: &Bound<'_, PyAny>) -> PyResult PyResult>; +} + +impl ResultIdentity for &T { + fn identity(&self) -> PyResult> { + (*self).identity() + } +} + +impl ResultIdentity for Bound<'_, PyAny> { + fn identity(&self) -> PyResult> { + resource_identity(self) + } +} + +macro_rules! named_identity { + ($kind:ident $(<$value:ident>)?, $field:ident, $prefix:literal) => { + impl $(<$value>)? ResultIdentity for $kind $(<$value>)? { + fn identity(&self) -> PyResult> { + Ok(Some(( + self.$field.clone(), + format!("{}:{}", $prefix, self.$field), + ))) + } + } + }; +} + +named_identity!(ModelExecutionResult, model_name, "model"); +named_identity!(SeedExecutionResult, seed_name, "seed"); +named_identity!(LoadExecutionResult, source_name, "source"); + +impl ResultIdentity for FunctionExecutionResult { + fn identity(&self) -> PyResult> { + Ok(Some(( + self.function_name.clone(), + format!("{}:{}", self.function_kind, self.function_name), + ))) + } +} + +impl ResultIdentity for SqlTestExecutionResult { + fn identity(&self) -> PyResult> { + let path = self.source_path.as_ref().map(|path| { + path.to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "/") + }); + let case = path + .as_deref() + .zip(self.block_index) + .zip(self.case_name.as_deref()) + .map(|((path, block), case)| (path, block, case)); + Ok(Some(( + self.test_name.clone(), + sql_test_resource_id(&self.test_name, case), + ))) + } +} + +impl ResultIdentity for AuditExecutionResult { + fn identity(&self) -> PyResult> { + let kind = self + .attached_target_kind + .map_or(self.attachment_kind.value(), ResultEnum::value); + Ok(Some(( + self.audit_name.clone(), + audit_resource_id( + &self.audit_name, + kind, + ( + self.attached_target_name.as_deref(), + self.attached_column_name.as_deref(), + ), + ), + ))) + } +} + /// `round(value)`: the nearest integer, ties to even. pub(crate) fn rounded(py: Python<'_>, value: f64) -> PyResult> { py.import("builtins")?.getattr("round")?.call1((value,)) @@ -116,7 +203,7 @@ pub(crate) fn number(value: &Bound<'_, PyAny>) -> PyResult> { /// Results matched against the installed terminal event index, if any. pub(crate) struct Projection<'py> { - py: Python<'py>, + pub(crate) py: Python<'py>, index: Option>, /// Scenario whose resources are namespaced under `scenario::`. pub(crate) scenario: Option, @@ -173,8 +260,8 @@ impl<'py> Projection<'py> { } /// Whether an executor result has a terminal. - pub(crate) fn terminal_recorded(&self, result: &Bound<'py, PyAny>) -> PyResult { - match resource_identity(result)? { + pub(crate) fn terminal_recorded(&self, result: &T) -> PyResult { + match result.identity()? { Some((name, resource_id)) => self.resource_terminal_recorded(&name, &resource_id), None => Ok(self.index.is_none()), } @@ -195,6 +282,18 @@ impl<'py> Projection<'py> { Ok(kept) } + /// Borrow typed results with recorded terminals; never recreate Python result records. + pub(crate) fn retained<'a, T: ResultIdentity>(&self, results: &'a [T]) -> PyResult> { + results + .iter() + .filter_map(|result| match self.terminal_recorded(result) { + Ok(true) => Some(Ok(result)), + Ok(false) => None, + Err(error) => Some(Err(error)), + }) + .collect() + } + /// The terminal's `duration_ms`, or the fallback without an index. pub(crate) fn resource_duration( &self, @@ -213,12 +312,12 @@ impl<'py> Projection<'py> { } /// The rounded canonical duration of a result, else its own `duration_ms`. - pub(crate) fn result_duration( + pub(crate) fn result_duration( &self, - result: &Bound<'py, PyAny>, + result: &T, fallback: &Bound<'py, PyAny>, ) -> PyResult> { - let Some((name, resource_id)) = resource_identity(result)? else { + let Some((name, resource_id)) = result.identity()? else { return Ok(fallback.clone()); }; match self.resource_duration((&name, &resource_id), number(fallback)?)? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/command.rs index 14d8a121fc..8b1f824636 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/command.rs @@ -4,11 +4,9 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; use std::time::Instant; +use pyo3::IntoPyObject; use pyo3::exceptions::PyException; -use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, -}; -use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; use sqlbuild_analysis::review_command::constants::{ AUDIT_FINDING, BUILT_NODE, FAILED_NODE, FINDING_NODE, MARKDOWN_WRITE_CODE, MODEL_KIND, NO_COMPARISON_MODE, NOT_VALIDATED_NODE, OK_NODE, PASSES_LIMIT, PLANNED_STATUS, SEED_KIND, @@ -702,8 +700,6 @@ fn with_markdown_failure(report: ReviewReport, failure: Option<&String>) -> Revi } /// Run one `sqb review` request. -#[pyfunction] -#[pyo3(signature = (*, request))] pub(crate) fn run_review_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult { let started: Instant = Instant::now(); let machine: bool = native_field(request, "json_output")?.is_truthy()?; @@ -747,7 +743,3 @@ pub(crate) fn run_review_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> } Ok(review_outcome(&report).1) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(run_review_command, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/review_command/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/review_command/registration.rs index f74d4efd27..e1d19a0d5c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/review_command/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/review_command/registration.rs @@ -1,11 +1,10 @@ -//! Register `sqb review` and the review schema name parser the janitor uses. +//! Register the review schema name parser the janitor uses. use pyo3::prelude::{Bound, PyAny, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::review_command::main::parse_review_schema_name::parse_review_schema_name as parse_name; use crate::bindings::_helpers::janitor::janitor_keys::utc_datetime; -use crate::bindings::_helpers::review_command::command; /// The aware UTC creation time a strict review schema name encodes, or `None`. #[pyfunction] @@ -20,6 +19,5 @@ pub(crate) fn parse_review_schema_name<'py>( } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(parse_review_schema_name, module)?)?; - command::register(module) + module.add_function(wrap_pyfunction!(parse_review_schema_name, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs index e9df64c5a8..974d477bc9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs @@ -1,19 +1,16 @@ //! Interpolation sentinels, inline directive findings and fixture-null candidates over lint models. -use std::collections::BTreeSet; use std::path::PathBuf; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString}; +use pyo3::types::{PyDict, PyDictMethods, PyString}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_rules::lint_text::errors::LintTextError; use sqlbuild_rules::lint_text::main::format_sql_bodies::format_sql_bodies; use sqlbuild_rules::lint_text::main::named_declaration_role_kind::named_declaration_role_kind as role_kind; -use sqlbuild_rules::lint_text::main::neutralize_context_interpolation::neutralize_context_interpolation; use sqlbuild_rules::lint_text::main::neutralize_interpolation::neutralize_interpolation; use sqlbuild_rules::lint_text::main::reject_inline_suppressions::reject_inline_suppressions; use sqlbuild_rules::lint_text::main::restore_interpolation::restore_interpolation; -use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_null_candidates; use sqlbuild_rules::lint_text::models::{ FormatFault, FormatFile, FormattedBodies, InterpolationSite, Neutralized, SuppressionFinding, }; @@ -51,13 +48,6 @@ fn neutralize_lint_interpolation( }) } -#[pyfunction] -fn neutralize_lint_context_interpolation( - body: &str, -) -> PyResult<(String, Vec>)> { - compiler_guard(|| Ok(site_tuples(neutralize_context_interpolation(body)))) -} - /// Restored text, or `(False, reason)` when a sentinel is missing or repeated. #[pyfunction] fn restore_lint_interpolation( @@ -72,23 +62,6 @@ fn restore_lint_interpolation( } } -#[pyfunction] -#[pyo3(signature = (project_dir, selected_paths = None))] -fn scan_fixture_typed_null_candidates<'py>( - py: Python<'py>, - project_dir: PathBuf, - selected_paths: Option>, -) -> PyResult<(Bound<'py, PyFrozenSet>, Bound<'py, PyFrozenSet>)> { - let candidates = compiler_guard(|| { - scan_fixture_null_candidates(&project_dir, selected_paths.as_ref()).map_err(value_error) - })?; - let paths: Vec = candidates.paths.into_iter().map(PathBuf::from).collect(); - Ok(( - PyFrozenSet::new(py, paths)?, - PyFrozenSet::new(py, candidates.model_names)?, - )) -} - /// The named declaration kind of a project-relative path, as `DeclarationKind`. #[pyfunction] #[pyo3(signature = (*, relative_path))] @@ -231,15 +204,7 @@ pub(crate) fn format_lint_sql_bodies<'py>( pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(neutralize_lint_interpolation, module)?)?; - module.add_function(wrap_pyfunction!( - neutralize_lint_context_interpolation, - module - )?)?; module.add_function(wrap_pyfunction!(restore_lint_interpolation, module)?)?; - module.add_function(wrap_pyfunction!( - scan_fixture_typed_null_candidates, - module - )?)?; module.add_function(wrap_pyfunction!(format_lint_sql_bodies, module)?)?; module.add_function(wrap_pyfunction!(named_declaration_role_kind, module)?)?; module.add_function(wrap_pyfunction!(reject_lint_suppressions, module)?) diff --git a/fensu.toml b/fensu.toml index 4d1674dafa..8b03b9572a 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1629,6 +1629,14 @@ forbidden_owners = ["src/sqlbuild/adapter/contract/classes/base_adapter.py:BaseA paths = ["src/sqlbuild/adapter/contract/classes/*", "src/sqlbuild/adapters/*/classes/*"] reason = "tests/unit/src/sqlbuild/adapter/contract/classes/strict_adapter/test_strict_adapter.py requires every first-class adapter to define each StrictAdapter method itself instead of inheriting it from BaseAdapter." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/result_output/assets.rs"] +reason = "Resource asset projections intentionally mirror their public JSON schemas: seeds include planning reasons, functions include warnings and their function kind, and loads include loader and skip metadata." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs"] +reason = "Command document projections intentionally mirror the public envelope while preserving command-specific summaries: seeds require a plan and loads report skipped_count." + [[targets.python.rule_exceptions]] rule = "XSB051" path = "src/sqlbuild/executor/migrations/_helpers/old_name_views.py" @@ -1765,7 +1773,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/artifacts.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs=11", + "crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/values.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_views/audit_views.rs=4", @@ -1788,14 +1796,13 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/contracts/spec_resolution.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/contracts/state_literals.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/bounds.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs=12", + "crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/cost_command/refusal.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/cost_command/sorting.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/functions.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=12", - "crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/cost_report/store.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/allocation.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs=1", @@ -1841,7 +1848,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs=26", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs=8", @@ -1909,7 +1916,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/frontier/python_types.rs=29", "crates/sqlbuild-python/src/bindings/_helpers/frontier/stage_capture.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/graph/algorithms.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/graph/keys.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/graph/python_lifecycle.rs=2", @@ -2144,7 +2151,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/result_output/aggregation.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/result_output/checks.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/result_output/documents.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/result_output/event_writer.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/result_output/integration_envelope.rs=5", @@ -2328,7 +2335,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs=1", - "crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/cost_command/command.rs=0", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=7", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 2b8218d67a..d50926603f 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -2027,9 +2027,6 @@ class NativeMacroExpansion: def declaration_read(self, kind: str, name: str) -> None: ... def block_memo(self) -> None: ... -def relation_key_row( - kind: str, name: str, values: dict[str, object], entry: object -) -> RelationKeyRow: ... def declared_relation_keys( *, model_rows: Iterable[RelationKeyRow], @@ -4785,7 +4782,6 @@ def write_check_results( def format_check_json(*, results: tuple[PythonCheckExecutionResult, ...]) -> str: ... def run_test_command(*, request: TestCommandRequest) -> int: ... def run_build_command(*, request: BuildCommandRequest) -> int: ... -def run_review_command(*, request: ReviewCommandRequest) -> int: ... def enforce_plan_safety_policies( *, plan: PlanOutput, diff --git a/tests/integration/src/sqlbuild/cli/entry/_test_types.py b/tests/integration/src/sqlbuild/cli/entry/_test_types.py index 4df4329ad5..ac7764e6ba 100644 --- a/tests/integration/src/sqlbuild/cli/entry/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/entry/_test_types.py @@ -135,3 +135,12 @@ class MachineOutputResetTestCase: failing_args: tuple[str, ...] failing_extra_toml: str expected_error_code: str + + +@dataclass(frozen=True) +class NativeCostTestCase: + """A persisted history request and its independently specified ordering.""" + + description: str + args: tuple[str, ...] + expected_run_ids: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index 33c4425cfd..b510600905 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -5,10 +5,15 @@ import signal import subprocess import sys +from datetime import UTC, datetime, timedelta +from decimal import Decimal from pathlib import Path import sqlbuild from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR +from sqlbuild._native import CostRunRecord, ResourceCost, write_cost_run +from sqlbuild.cost.models import RunCostSummary +from sqlbuild.cost.types import CostStatus type LauncherRun = tuple[int, bytes, bytes] @@ -354,3 +359,51 @@ def write_janitor_project(project_dir: Path, *, extra_toml: str) -> Path: "MODEL (description 'Orders.');\n\nSELECT 1 AS order_id\n", encoding="utf-8" ) return project_dir + + +def write_cost_project(project_dir: Path) -> Path: + """Persist exact, asymmetric metrics through the public API, with ties and missing metrics.""" + + project: Path = write_project(project_dir) + origin: datetime = datetime(2026, 8, 23, tzinfo=UTC) + for index, (name, usd, status) in enumerate( + ( + ("negative", "-3", CostStatus.COMPLETE), + ("small", "2", CostStatus.COMPLETE), + ("large", "10.0000000000000000001", CostStatus.COMPLETE), + ("tie", "2.00", CostStatus.COMPLETE), + ("pending", "100", CostStatus.PENDING), + ) + ): + completed: datetime = origin + timedelta(days=index) + resource: ResourceCost = ResourceCost( + resource_type="model", + resource_name="orders", + warehouse_name="warehouse", + warehouse_size="small", + warehouse_type="STANDARD", + cluster_number=1, + query_count=1, + attributed_seconds=Decimal(index + 1), + bytes_scanned=index + 1, + estimated_compute_credits=Decimal(usd), + estimated_usd=Decimal(usd), + ) + record: CostRunRecord = CostRunRecord( + run_id=f"run-{name}", + adapter_name="duckdb", + target_name="dev", + build_status="success", + started_at=completed - timedelta(seconds=index + 1), + completed_at=completed, + cost=RunCostSummary( + status=status, + usd_per_credit=Decimal(1), + rate_source="default", + resources=(resource,), + estimated_compute_credits=Decimal(usd), + estimated_usd=Decimal(usd), + ), + ) + _ = write_cost_run(project_dir=project, record=record) + return project diff --git a/tests/integration/src/sqlbuild/cli/entry/test_native_cost.py b/tests/integration/src/sqlbuild/cli/entry/test_native_cost.py new file mode 100644 index 0000000000..a7d9a6f364 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/entry/test_native_cost.py @@ -0,0 +1,211 @@ +import json +import sys +from pathlib import Path +from typing import Any + +import pytest + +from tests.integration.src.sqlbuild.cli.entry._test_types import HandoffTestCase, NativeCostTestCase +from tests.integration.src.sqlbuild.cli.entry.helpers import ( + PYTHON_CLI, + LauncherRun, + counting_environment, + python_starts, + run, + write_cost_project, +) + + +@pytest.mark.skipif(sys.platform == "win32", reason="the counting environment is a POSIX script") +@pytest.mark.parametrize( + "test_case", + [ + NativeCostTestCase( + "exact ascending decimals and newest tie", + ("--sort", "cost", "--order", "asc"), + ("run-negative", "run-tie", "run-small", "run-large", "run-pending"), + ), + NativeCostTestCase( + "missing metrics stay last descending", + ("--sort", "credits"), + ("run-large", "run-tie", "run-small", "run-negative", "run-pending"), + ), + NativeCostTestCase( + "duration differs from decimal order", + ("--sort", "duration", "--order", "asc"), + ("run-negative", "run-small", "run-large", "run-tie", "run-pending"), + ), + NativeCostTestCase( + "whole day includes its final instant", + ("--since", "2026-08-24", "--until", "2026-08-25"), + ("run-large", "run-small"), + ), + NativeCostTestCase( + "basic date and clock", + ("--since", "20260824T000000Z", "--until", "20260825T000000Z"), + ("run-large", "run-small"), + ), + NativeCostTestCase( + "week date", + ("--since", "2026-W35-1T00:00:00Z", "--until", "2026-W35-2T00:00:00Z"), + ("run-large", "run-small"), + ), + NativeCostTestCase( + "comma and long subsecond precision", + ("--since", "2026-08-24T00:00:00,0000009Z", "--until", "2026-08-25T00:00:00.0000009Z"), + ("run-large", "run-small"), + ), + NativeCostTestCase( + "nonstandard separator and offset", + ("--since", "2026-08-24X01:00:00+01:00", "--until", "2026-08-25X01:00:00+01:00"), + ("run-large", "run-small"), + ), + NativeCostTestCase( + "limit follows filtering and ordering", + ("--sort", "cost", "--limit", "2"), + ("run-large", "run-tie"), + ), + ], + ids=lambda case: case.description, +) +def test_given_persisted_cost_history_when_launching_then_exact_order_and_output_need_no_python( + test_case: NativeCostTestCase, tmp_path: Path +) -> None: + workspace: Path = write_cost_project(tmp_path / "workspace") + marker: Path = tmp_path / "python-starts.log" + _ = marker.write_text("", encoding="utf-8") + program: Path = counting_environment(tmp_path / "environment", marker) + args: tuple[str, ...] = ("cost", "history", "--json", *test_case.args) + + native: LauncherRun = run(program, args=args, cwd=workspace) + python: LauncherRun = run(PYTHON_CLI, args=args, cwd=workspace) + + assert native == python + assert native[0] == 0 + assert python_starts(marker) == 0 + payload: dict[str, Any] = json.loads(native[1]) + assert tuple(record["run_id"] for record in payload["runs"]) == test_case.expected_run_ids + + +@pytest.mark.skipif(sys.platform == "win32", reason="the counting environment is a POSIX script") +@pytest.mark.parametrize( + "test_case", + [HandoffTestCase("completed persisted cost", ("cost", "run-large"), 0)], + ids=lambda case: case.description, +) +def test_given_completed_cost_when_reporting_then_native_text_json_and_file_match( + test_case: HandoffTestCase, + tmp_path: Path, +) -> None: + workspace: Path = write_cost_project(tmp_path / "workspace") + marker: Path = tmp_path / "python-starts.log" + _ = marker.write_text("", encoding="utf-8") + program: Path = counting_environment(tmp_path / "environment", marker) + + text: LauncherRun = run(program, args=test_case.args, cwd=workspace) + assert text == run(PYTHON_CLI, args=test_case.args, cwd=workspace) + assert text[0] == test_case.expected_exit_code + document: LauncherRun = run( + program, + args=(*test_case.args, "--json", "--json-output", "reports/cost.json"), + cwd=workspace, + ) + assert document == run(PYTHON_CLI, args=(*test_case.args, "--json"), cwd=workspace) + assert document[0] == 0 + assert (workspace / "reports" / "cost.json").read_bytes() == document[1] + assert json.loads(document[1])["build_status"] == "success" + assert python_starts(marker) == 0 + + +@pytest.mark.skipif(sys.platform == "win32", reason="the counting environment is a POSIX script") +@pytest.mark.parametrize( + "test_case", + [ + NativeCostTestCase( + "pending detail refresh", ("cost", "run-pending", "--json"), ("run-pending",) + ) + ], + ids=lambda case: case.description, +) +def test_given_pending_cost_when_selecting_detail_then_existing_adapter_refresh_still_runs( + test_case: NativeCostTestCase, + tmp_path: Path, +) -> None: + workspace: Path = write_cost_project(tmp_path / "workspace") + marker: Path = tmp_path / "python-starts.log" + _ = marker.write_text("", encoding="utf-8") + program: Path = counting_environment(tmp_path / "environment", marker) + + native: LauncherRun = run(program, args=test_case.args, cwd=workspace) + + assert native == run(PYTHON_CLI, args=test_case.args, cwd=workspace) + assert native[0] == 0 + assert (json.loads(native[1])["run_id"],) == test_case.expected_run_ids + assert python_starts(marker) == 1 + + +@pytest.mark.skipif(sys.platform == "win32", reason="the counting environment is a POSIX script") +@pytest.mark.parametrize( + "test_case", + [ + HandoffTestCase("missing run", ("cost", "missing", "--json"), 1), + HandoffTestCase("ambiguous prefix", ("cost", "run-", "--json"), 1), + HandoffTestCase("invalid date", ("cost", "history", "--since", "invalid", "--json"), 1), + HandoffTestCase( + "naive datetime", ("cost", "history", "--since", "2026-08-24T00:00:00", "--json"), 1 + ), + HandoffTestCase( + "reversed bounds", + ("cost", "history", "--since", "2026-08-25", "--until", "2026-08-23", "--json"), + 1, + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_cost_request_when_launching_then_coded_error_matches_without_python( + test_case: HandoffTestCase, + tmp_path: Path, +) -> None: + workspace: Path = write_cost_project(tmp_path / "workspace") + marker: Path = tmp_path / "python-starts.log" + _ = marker.write_text("", encoding="utf-8") + program: Path = counting_environment(tmp_path / "environment", marker) + + native: LauncherRun = run(program, args=test_case.args, cwd=workspace) + + assert native == run(PYTHON_CLI, args=test_case.args, cwd=workspace) + assert native[0] == test_case.expected_exit_code + assert native[1] == b"" + assert native[2].startswith(b"error[") + assert python_starts(marker) == 0 + + +@pytest.mark.skipif(sys.platform == "win32", reason="the counting environment is a POSIX script") +@pytest.mark.parametrize( + "test_case", + [ + HandoffTestCase( + "output parent is a file", + ("cost", "run-small", "--json-output", "reports/cost.json"), + 1, + ) + ], + ids=lambda case: case.description, +) +def test_given_cost_output_failure_when_launching_then_failure_never_retries_in_python( + test_case: HandoffTestCase, + tmp_path: Path, +) -> None: + workspace: Path = write_cost_project(tmp_path / "workspace") + _ = (workspace / "reports").write_text("keep this file", encoding="utf-8") + marker: Path = tmp_path / "python-starts.log" + _ = marker.write_text("", encoding="utf-8") + program: Path = counting_environment(tmp_path / "environment", marker) + + native: LauncherRun = run(program, args=test_case.args, cwd=workspace) + + assert native[0] == test_case.expected_exit_code + assert native[1] == b"" + assert b"cannot write" in native[2] + assert (workspace / "reports").read_text(encoding="utf-8") == "keep this file" + assert python_starts(marker) == 0 diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py index 8abd6652c8..5ab62fd1a4 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/_test_types.py @@ -16,6 +16,13 @@ class BuildFooterTestCase: use_color: bool = False +@dataclass(frozen=True) +class BuildFooterMeasurementTestCase: + description: str + value: int | float | None + expected_text: str + + @dataclass(frozen=True) class ExecutionHeaderTestCase: description: str diff --git a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py index 3926dfa2ff..94cbc8af7a 100644 --- a/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py +++ b/tests/unit/src/sqlbuild/cli/commands/shared/_helpers/test_progress.py @@ -3,6 +3,7 @@ from __future__ import annotations import time +from dataclasses import replace from io import StringIO import pytest @@ -19,9 +20,10 @@ ) from sqlbuild.adapter.contract.models import LifeCycleEvent from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.compiler.auditing.types import AuditOutcome +from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditOutcome from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.cost.models import StatementExecutionTelemetry +from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.executor.build.models import ( BuildExecutionResult, FunctionExecutionResult, @@ -37,6 +39,7 @@ from sqlbuild.executor.testing.types import SqlTestOutcome from sqlbuild.runtime.contracts.types import ExecutionResourceKind from tests.unit.src.sqlbuild.cli.commands.shared._helpers._test_types import ( + BuildFooterMeasurementTestCase, BuildFooterTestCase, BuildProgressActiveSpinnerTestCase, BuildProgressFailureOutputTestCase, @@ -540,6 +543,48 @@ def test_given_failed_resource_result_when_formatting_footer_then_includes_error assert unexpected_fragment not in footer +@pytest.mark.parametrize( + "test_case", + [ + BuildFooterMeasurementTestCase(description="integer text", value=1, expected_text="1"), + BuildFooterMeasurementTestCase( + description="whole float text", value=1.0, expected_text="1.0" + ), + BuildFooterMeasurementTestCase( + description="scientific notation", value=0.000001, expected_text="1e-06" + ), + BuildFooterMeasurementTestCase(description="signed zero", value=-0.0, expected_text="-0.0"), + BuildFooterMeasurementTestCase( + description="missing measurement", value=None, expected_text="None" + ), + ], + ids=lambda case: case.description, +) +def test_given_measurement_warning_when_formatting_footer_then_preserves_numeric_text( + test_case: BuildFooterMeasurementTestCase, +) -> None: + audit: AuditExecutionResult = replace( + build_audit_result(name="order_count", outcome=AuditOutcome.WARN), + evaluation_mode=AuditEvaluationMode.MEASUREMENT, + measured_value=test_case.value, + ) + result: BuildExecutionResult = BuildExecutionResult( + status=BuildStatus.SUCCESS, + model_results=( + ModelExecutionResult( + model_name="orders", + status=ExecutionStatus.SUCCESS, + audit_results=(audit,), + ), + ), + warning_count=1, + ) + + footer: str = format_build_footer(result=result, elapsed=1.25, use_color=False) + + assert f"audit order_count measured {test_case.expected_text}\n" in footer + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py b/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py index 559a92b544..60d0e39ac9 100644 --- a/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py +++ b/tests/unit/src/sqlbuild/cli/output/_helpers/test_execution_result_document.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from dataclasses import replace from decimal import Decimal from pathlib import Path @@ -20,6 +21,7 @@ SqlValue, SqlValueKind, format_audit_execution_json, + format_build_footer, format_parameterized_test_label, ) from sqlbuild.adapter.contract.models import LifeCycleEvent @@ -33,6 +35,8 @@ ) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind from sqlbuild.executor.auditing.models import AuditExecutionResult +from sqlbuild.executor.build.models import BuildExecutionResult +from sqlbuild.executor.build.types import BuildStatus from sqlbuild.executor.run.models import ( MicrobatchAccountingInterval, ModelExecutionResult, @@ -548,3 +552,12 @@ def test_given_parameterized_sql_test_result_when_formatting_json_then_case_meta ) == test_case.expected_text_label ) + footer: str = format_build_footer( + result=BuildExecutionResult( + status=BuildStatus.FAILED, + test_results=(replace(result, outcome=SqlTestOutcome.FAIL),), + ), + elapsed=1.0, + use_color=False, + ) + assert test_case.expected_text_label in footer