diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/bounded_sections.rs b/crates/sqlbuild-core/src/cli_text/_helpers/bounded_sections.rs new file mode 100644 index 0000000000..27fec08493 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/_helpers/bounded_sections.rs @@ -0,0 +1,102 @@ +//! Bounded sections, aligned name/value rows and ANSI stripping, as `sqlbuild.presentation` does. + +use crate::cli_text::constants::MIN_NAME_COLUMN_WIDTH; +use crate::cli_text::models::{CliStyle, DisplayLimit}; + +/// The text without ANSI SGR sequences (`ESC [ digits-and-semicolons m`). +pub(crate) fn strip_ansi(text: &str) -> String { + let mut stripped: String = String::with_capacity(text.len()); + let mut rest: &str = text; + while let Some(start) = rest.find("\x1b[") { + stripped.push_str(&rest[..start]); + let after: &str = &rest[start + 2..]; + let parameters: usize = after + .find(|character: char| !(character.is_ascii_digit() || character == ';')) + .unwrap_or(after.len()); + if after[parameters..].starts_with('m') { + rest = &after[parameters + 1..]; + } else { + stripped.push('\x1b'); + rest = &rest[start + 1..]; + } + } + stripped.push_str(rest); + stripped +} + +/// `count_header_style` over `plan_section`: `Title (count)` as a bold title and dim count. +pub(crate) fn section_header(style: CliStyle, text: &str) -> String { + if let Some(split) = text.rfind(" (") + && text.ends_with(')') + { + return format!( + "{} {}", + style.plan_section(&text[..split]), + style.muted(&text[split + 1..]) + ); + } + style.plan_section(text) +} + +/// How many of `total` entries a section shows. +pub(crate) fn visible_count(total: usize, display: &DisplayLimit) -> usize { + display.max_entries.map_or(total, |limit| total.min(limit)) +} + +/// The dim `... and N more` line when a section hides entries. +pub(crate) fn overflow_line( + (total, visible): (usize, usize), + indent: &str, + display: &DisplayLimit, +) -> Option { + (total > visible).then(|| { + CliStyle { use_color: true }.muted(&format!( + "{indent}... and {} more (use {} to show all)", + total - visible, + display.overflow_flag + )) + }) +} + +/// The widest name, but at least the minimum name column. +pub(crate) fn name_column_width<'a>(names: impl Iterator) -> usize { + names + .map(|name| name.chars().count()) + .max() + .unwrap_or(0) + .max(MIN_NAME_COLUMN_WIDTH) +} + +/// ` name value`, with the value dim unless `dim_value` is off. +pub(crate) fn name_value_line(name: &str, value: &str, width: usize, dim_value: bool) -> String { + let style: CliStyle = CliStyle { use_color: true }; + let rendered: String = if dim_value { + style.muted(value) + } else { + value.to_owned() + }; + let padding: String = " ".repeat(width.saturating_sub(name.chars().count())); + format!(" {}{padding} {rendered}", style.object_name(name)) +} + +/// A comma-separated name list cut to the section limit with a `+N more` note. +pub(crate) fn capped_names( + names: &[String], + display: &DisplayLimit, + name_style: impl Fn(&str) -> String, +) -> String { + let visible: usize = visible_count(names.len(), display); + let base: String = names[..visible] + .iter() + .map(|name| name_style(name)) + .collect::>() + .join(", "); + if visible >= names.len() { + return base; + } + format!( + "{base}, ... (+{} more; use {} to show all)", + names.len() - visible, + display.overflow_flag + ) +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs b/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs index 6bc7e5db96..2fdfbd3544 100644 --- a/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/cli_text/_helpers/mod.rs @@ -1,2 +1,6 @@ +pub(crate) mod bounded_sections; pub(crate) mod diagnostics; pub(crate) mod layout; +pub(crate) mod plan_document; +pub(crate) mod plan_sections; +pub(crate) mod tree_rows; diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/plan_document.rs b/crates/sqlbuild-core/src/cli_text/_helpers/plan_document.rs new file mode 100644 index 0000000000..2925f39627 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/_helpers/plan_document.rs @@ -0,0 +1,84 @@ +//! The full-refresh and incremental bodies of the text plan. + +use crate::cli_text::_helpers::layout::surface_header; +use crate::cli_text::_helpers::plan_sections::{ + full_refresh_count_lines, functions_lines, plan_name_width, provider_usage_lines, pruned_lines, + python_entry_lines, ready_summary, reason_group_lines, remaining_stale_lines, + routine_model_lines, seed_lines, source_freshness_lines, source_load_lines, warning_lines, +}; +use crate::cli_text::_helpers::tree_rows::treeify; +use crate::cli_text::constants::{ + PLAN_READY_TITLE, PRE_SQL_INGRESS_PHASE, PYTHON_INGRESS_TITLE, PYTHON_READ_SIDE_TITLE, + READ_SIDE_PHASE, +}; +use crate::cli_text::models::{CliStyle, ModelPlanText, PlanTextReport}; + +fn header_lines(report: &PlanTextReport, full_refresh: bool) -> Vec { + if !report.include_header { + return Vec::new(); + } + vec![surface_header( + CliStyle { use_color: true }, + PLAN_READY_TITLE, + &ready_summary(report, full_refresh), + )] +} + +/// Every active model is a full refresh: counts by materialization instead of detail. +pub(crate) fn full_refresh_lines( + report: &PlanTextReport, + active: &[&ModelPlanText], +) -> Vec { + let width: usize = plan_name_width(report); + let mut lines: Vec = header_lines(report, true); + lines.extend(provider_usage_lines(report)); + lines.extend(python_entry_lines( + (report, PRE_SQL_INGRESS_PHASE, PYTHON_INGRESS_TITLE), + width, + )); + lines.extend(source_load_lines(report, width)); + lines.extend(functions_lines(report, true, width)); + lines.extend(functions_lines(report, false, width)); + if !lines.is_empty() { + lines.push(String::new()); + } + lines.extend(full_refresh_count_lines(active)); + lines.extend(seed_lines(report)); + lines.extend(python_entry_lines( + (report, READ_SIDE_PHASE, PYTHON_READ_SIDE_TITLE), + width, + )); + lines.extend(warning_lines(report)); + lines.extend(report.pre_build_lines.iter().cloned()); + lines +} + +/// Sections by change reason, drawn as trees. +pub(crate) fn incremental_lines(report: &PlanTextReport, active: &[&ModelPlanText]) -> Vec { + let width: usize = plan_name_width(report); + let mut lines: Vec = header_lines(report, false); + if report.include_freshness_diagnostics { + lines.extend(source_freshness_lines(report)); + } + lines.extend(remaining_stale_lines(report)); + lines.extend(provider_usage_lines(report)); + lines.extend(pruned_lines(report)); + lines.extend(report.pre_build_lines.iter().cloned()); + lines.extend(python_entry_lines( + (report, PRE_SQL_INGRESS_PHASE, PYTHON_INGRESS_TITLE), + width, + )); + lines.extend(source_load_lines(report, width)); + lines.extend(functions_lines(report, true, width)); + lines.extend(reason_group_lines(report, active, width)); + lines.extend(routine_model_lines(report, active, width)); + lines.extend(functions_lines(report, false, width)); + lines.extend(seed_lines(report)); + lines.extend(python_entry_lines( + (report, READ_SIDE_PHASE, PYTHON_READ_SIDE_TITLE), + width, + )); + let mut lines: Vec = treeify(lines); + lines.extend(warning_lines(report)); + lines +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs new file mode 100644 index 0000000000..19b556e8a7 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/_helpers/plan_sections.rs @@ -0,0 +1,854 @@ +//! The sections of the text plan: header, providers, loads, models, functions, seeds, warnings. + +use crate::cli_text::_helpers::bounded_sections::{ + capped_names, name_column_width, name_value_line, overflow_line, section_header, strip_ansi, + visible_count, +}; +use crate::cli_text::_helpers::layout::tree_connector; +use crate::cli_text::constants::{ + BOUNDED_BACKFILL, CHANGED_IDENTITY, COLUMN_ADDED, COLUMN_REMOVED, COLUMN_TYPE_CHANGED, + CONFIG_CHANGED_REASON, CREATE_VIEW_ACTION, CUSTOM_ACTION, DIFF_HEADER_MARKER, + EFFECTIVE_BATCH_SIZE_TOKEN, ERROR_SEVERITY_VALUE, FIRST_RUN_REASON, FORWARD_BACKFILL, + FULL_BACKFILL, FULL_REFRESH_REASON, INFO_SEVERITY_VALUE, NO_CHANGE_REASON, + NORMAL_INCREMENTAL_REASON, QUERY_CHANGED_REASON, REASON_GROUPS, SCHEMA_CHANGED_REASON, + SKIP_ACTION, SOURCE_FRESHNESS_UNKNOWN_WARNING_TITLE, STALE_INPUT_WARNING_TITLE, +}; +use crate::cli_text::models::{ + CliStyle, CursorPlanText, DisplayLimit, FunctionPlanText, ModelPlanText, PlanTextReport, + PlanWarningText, ProviderUsageText, PythonPlanText, SchemaFindingText, SourceFreshnessText, + SourceLoadPlanText, +}; + +const STYLE: CliStyle = CliStyle { use_color: true }; + +/// The rows of one bounded section: blank line, `Title (count)` header, rows, overflow line. +fn section_lines( + title: &str, + entries: &[T], + display: &DisplayLimit, + row: impl Fn(&T) -> Vec, +) -> Vec { + let mut lines: Vec = vec![ + String::new(), + section_header(STYLE, &format!("{title} ({})", entries.len())), + ]; + let visible: usize = visible_count(entries.len(), display); + for entry in &entries[..visible] { + lines.extend(row(entry)); + } + lines.extend(overflow_line((entries.len(), visible), " ", display)); + lines +} + +/// Models whose action is not `skip`. +pub(crate) fn active_models(report: &PlanTextReport) -> Vec<&ModelPlanText> { + report + .models + .iter() + .filter(|model| model.action != SKIP_ACTION) + .collect() +} + +/// Whether the plan renders as a full refresh: every active model is one, or nothing is active. +pub(crate) fn full_refresh_layout(report: &PlanTextReport) -> bool { + let active: Vec<&ModelPlanText> = active_models(report); + (!active.is_empty() + && active + .iter() + .all(|model| model.reason == FULL_REFRESH_REASON)) + || (report.full_refresh && active.is_empty()) +} + +/// `N selected`, with the full-refresh, source-load and Python-node parts. +pub(crate) fn ready_summary(report: &PlanTextReport, full_refresh: bool) -> String { + let mut parts: Vec = Vec::new(); + if full_refresh { + parts.push("full refresh".to_owned()); + } + let selected: usize = report.models.len() + report.seeds.len() + report.functions.len(); + parts.push(format!("{selected} selected")); + let sources: usize = report.source_loads.len(); + if sources > 0 { + let noun: &str = if sources == 1 { "source" } else { "sources" }; + let action: &str = if report.source_loads.iter().any(|load| load.is_reload) { + "reload" + } else { + "load" + }; + parts.push(format!("{sources} {noun} to {action}")); + } + let python: usize = report.python_entries.len(); + if python > 0 { + let noun: &str = if python == 1 { "node" } else { "nodes" }; + parts.push(format!("{python} Python {noun}")); + } + parts.join(", ") +} + +/// The name column every plan section aligns to. +pub(crate) fn plan_name_width(report: &PlanTextReport) -> usize { + name_column_width( + report + .models + .iter() + .map(|model| model.name.as_str()) + .chain( + report + .functions + .iter() + .map(|function| function.name.as_str()), + ) + .chain( + report + .python_entries + .iter() + .map(|entry| entry.name.as_str()), + ), + ) +} + +/// Each provider with its count of selected Python surfaces, or every surface when verbose. +pub(crate) fn provider_usage_lines(report: &PlanTextReport) -> Vec { + if report.provider_usages.is_empty() { + return Vec::new(); + } + let mut providers: Vec<(&str, Vec<&ProviderUsageText>)> = Vec::new(); + for usage in &report.provider_usages { + if let Some((_, usages)) = providers + .iter_mut() + .find(|(name, _)| *name == usage.provider_name) + { + usages.push(usage); + } else { + providers.push((&usage.provider_name, vec![usage])); + } + } + providers.sort_by(|left, right| left.0.cmp(right.0)); + let mut lines: Vec = vec![String::new(), section_header(STYLE, "Providers")]; + if report.display.max_entries.is_some() { + let width: usize = providers + .iter() + .map(|(name, _)| name.chars().count()) + .max() + .unwrap_or(0); + for (name, usages) in &providers { + let surface: &str = if usages.len() == 1 { + "surface" + } else { + "surfaces" + }; + let padding: String = " ".repeat(width - name.chars().count()); + lines.push(format!( + " {}{padding} {}", + STYLE.object_name(name), + STYLE.muted(&format!( + "used by {} selected Python {surface}", + usages.len() + )) + )); + } + return lines; + } + for (name, mut usages) in providers { + lines.push(format!(" {}", STYLE.object_name(name))); + usages.sort_by(|left, right| { + ( + &left.consumer_kind, + &left.consumer_name, + &left.parameter_name, + ) + .cmp(&( + &right.consumer_kind, + &right.consumer_name, + &right.parameter_name, + )) + }); + let visible: usize = visible_count(usages.len(), &report.display); + for usage in &usages[..visible] { + let annotation: String = usage + .annotation_class_name + .as_ref() + .map_or_else(String::new, |class_name| format!(" ({class_name})")); + lines.push(format!( + " {} {}{annotation}", + STYLE.muted(&usage.consumer_kind), + STYLE.object_name(&usage.consumer_name) + )); + } + lines.extend(overflow_line( + (usages.len(), visible), + " ", + &report.display, + )); + } + lines +} + +/// The count of current models a direct plan pruned, listing them when verbose. +pub(crate) fn pruned_lines(report: &PlanTextReport) -> Vec { + let names: &[String] = &report.pruned_models; + if names.is_empty() { + return Vec::new(); + } + let mut lines: Vec = vec![ + String::new(), + STYLE.muted(&format!( + "Skipped current models ({} already up to date)", + names.len() + )), + ]; + if report.display.max_entries.is_some() { + return lines; + } + let width: usize = name_column_width(names.iter().map(String::as_str)); + for name in names { + lines.push(name_value_line(name, "up to date", width, true)); + } + lines +} + +fn identity_diff_lines(entry: &PythonPlanText) -> Vec { + if entry.identity_status != CHANGED_IDENTITY + || (entry.source_diff.is_empty() && entry.dependency_diff.is_empty()) + { + return Vec::new(); + } + let mut lines: Vec = vec![STYLE.label(" python diff:")]; + if !entry.source_diff.is_empty() { + lines.push(STYLE.label(" source diff:")); + lines.extend(entry.source_diff.iter().map(|line| format!(" {line}"))); + } + if !entry.dependency_diff.is_empty() { + lines.push(STYLE.label(" dependency diff:")); + for line in &entry.dependency_diff { + let indented: String = format!(" {line}"); + if strip_ansi(&indented).contains(DIFF_HEADER_MARKER) { + lines.push(STYLE.muted(&indented)); + } else { + lines.push(indented); + } + } + } + lines +} + +/// The Python entries of one run phase under `label`. +pub(crate) fn python_entry_lines( + (report, phase, label): (&PlanTextReport, &str, &str), + width: usize, +) -> Vec { + let entries: Vec<&PythonPlanText> = report + .python_entries + .iter() + .filter(|entry| entry.phase == phase) + .collect(); + if entries.is_empty() { + return Vec::new(); + } + section_lines(label, &entries, &report.display, |entry| { + let mut lines: Vec = vec![name_value_line( + &entry.name, + &format!("{} ({})", entry.kind, entry.identity_status), + width, + true, + )]; + lines.extend(identity_diff_lines(entry)); + lines + }) +} + +fn source_load_label(entry: &SourceLoadPlanText) -> String { + let strategy: String = match (&entry.integration_kind, &entry.write_strategy) { + (Some(kind), _) => format!("external ({kind})"), + (None, Some(strategy)) => strategy.clone(), + (None, None) => "self-managed".to_owned(), + }; + let mut details: Vec = Vec::new(); + if let Some(column) = &entry.cursor_column { + details.push(format!("cursor: {column}")); + } + if !entry.unique_key.is_empty() { + details.push(format!("unique_key: {}", entry.unique_key.join(", "))); + } + if details.is_empty() { + return strategy; + } + format!("{strategy} ({})", details.join("; ")) +} + +/// The loaders, then the sources, to load or reload. +pub(crate) fn source_load_lines(report: &PlanTextReport, width: usize) -> Vec { + let mut lines: Vec = Vec::new(); + for (kind, label) in [("loader", "Loaders"), ("source", "Sources")] { + let entries: Vec<&SourceLoadPlanText> = report + .source_loads + .iter() + .filter(|entry| entry.resource_kind == kind) + .collect(); + if entries.is_empty() { + continue; + } + let action: &str = if entries.iter().any(|entry| entry.is_reload) { + "reload" + } else { + "load" + }; + lines.extend(section_lines( + &format!("{label} to {action}"), + &entries, + &report.display, + |entry| { + vec![name_value_line( + &entry.name, + &source_load_label(entry), + width, + true, + )] + }, + )); + } + lines +} + +fn microbatch_lines(cursor: &CursorPlanText) -> Vec { + let mut lines: Vec = Vec::new(); + if let Some(declared) = &cursor.declared_grain { + let resolved: &str = cursor.effective_grain.as_deref().unwrap_or(declared); + if resolved == declared { + lines.push(format!(" grain: {declared}")); + } else { + lines.push(format!(" grain: {declared} -> {resolved} (effective)")); + } + } + if let Some(declared) = &cursor.declared_batch_size { + let mut text: String = declared.clone(); + if declared == EFFECTIVE_BATCH_SIZE_TOKEN { + let resolved: &str = cursor.effective_batch_size.as_deref().unwrap_or("runtime"); + text = if cursor.effective_grain == cursor.declared_grain { + resolved.to_owned() + } else { + format!("{declared} -> {resolved}") + }; + } + lines.push(format!(" batch size: {text}")); + } + if let (Some(count), Some(size)) = (cursor.planned_batch_count, &cursor.effective_batch_size) { + lines.push(format!(" batches: {count} x {size}")); + } + if cursor.deferred { + lines.push(format!( + " batches: counted at run time, {}", + cursor.runtime_inputs_clause + )); + } + lines.push(format!( + " batch concurrency: {}", + cursor.batch_concurrency + )); + if let Some(policy) = &cursor.unaccounted_partition_policy { + lines.push(format!(" unaccounted partition policy: {policy}")); + } + lines +} + +fn cursor_lines(model: &ModelPlanText, show_range: bool) -> Vec { + let Some(cursor) = &model.cursor else { + return Vec::new(); + }; + let mut lines: Vec = vec![match &cursor.cursor_type { + Some(cursor_type) => format!(" cursor: {} ({cursor_type})", cursor.column), + None => format!(" cursor: {}", cursor.column), + }]; + if cursor.microbatch { + lines.push(" mode: microbatch".to_owned()); + } + if cursor.requested_start.is_some() || cursor.requested_end.is_some() { + lines.push(format!( + " requested: {} -> {}", + cursor + .requested_start + .as_deref() + .unwrap_or("earliest available"), + cursor + .requested_end + .as_deref() + .unwrap_or("latest available") + )); + } + if let Some(window) = &cursor.empty_inputs_window { + let line: String = format!(" window: {window}"); + if cursor.waiting_on_empty_inputs { + lines.push(STYLE.warning(&line)); + } else { + lines.push(line); + } + } + if let (true, Some(range)) = (show_range, &cursor.range) { + lines.push(format!(" range: {range}")); + } + if cursor.microbatch { + lines.extend(microbatch_lines(cursor)); + } + if cursor.runtime_owned { + lines.push(format!( + " bounds: computed at run time, {}", + cursor.runtime_inputs_clause + )); + } else if cursor.resolved { + lines.push(" bounds: planner-resolved".to_owned()); + } + lines +} + +fn schema_change_suffix(model: &ModelPlanText) -> String { + let has = |kind: &str| { + model + .schema_findings + .iter() + .any(|finding| finding.kind == kind) + }; + let mut parts: Vec<&str> = Vec::new(); + if has(COLUMN_ADDED) { + parts.push("add column"); + } + if has(COLUMN_REMOVED) { + parts.push("drop column"); + } + if has(COLUMN_TYPE_CHANGED) { + parts.push("type change"); + } + parts.join(", ") +} + +fn action_text(model: &ModelPlanText) -> String { + if model.action == CREATE_VIEW_ACTION { + return "recreate view".to_owned(); + } + if model.action == CUSTOM_ACTION { + let materialization: &str = model + .custom_materialization_name + .as_deref() + .unwrap_or("custom materialization"); + return format!("run {materialization}"); + } + let suffix: String = schema_change_suffix(model); + let base: String = match (&model.backfill_action, &model.backfill_duration) { + (action, Some(duration)) if action == BOUNDED_BACKFILL => { + format!("rebuild last {duration}") + } + (action, _) if action == FULL_BACKFILL && model.reason != FIRST_RUN_REASON => { + "full rebuild".to_owned() + } + _ => return "continue forward".to_owned(), + }; + if suffix.is_empty() { + return base; + } + format!("{base}, {suffix}") +} + +fn policy_lines(model: &ModelPlanText) -> Vec { + if model.backfill_action == FORWARD_BACKFILL + || !(model.reason == QUERY_CHANGED_REASON || model.reason == SCHEMA_CHANGED_REASON) + { + return Vec::new(); + } + let duration: &str = model.backfill_duration.as_deref().unwrap_or("full"); + let value: String = if model.backfill_action == BOUNDED_BACKFILL { + format!("bounded-{duration}") + } else { + model.backfill_action.clone() + }; + vec![format!(" policy: replay_on_change={value}")] +} + +fn schema_finding_line(finding: &SchemaFindingText) -> String { + let (symbol, label) = match finding.kind.as_str() { + COLUMN_ADDED => ("+", "added"), + COLUMN_REMOVED => ("-", "removed"), + COLUMN_TYPE_CHANGED => ("~", "type changed"), + other => ("?", other), + }; + let expected: &str = finding.expected_type.as_deref().unwrap_or("None"); + let type_info: String = match finding.kind.as_str() { + COLUMN_TYPE_CHANGED => format!( + " {expected} \u{2192} {}", + finding.actual_type.as_deref().unwrap_or("None") + ), + COLUMN_ADDED + if finding + .expected_type + .as_deref() + .is_some_and(|kind| !kind.is_empty()) => + { + format!(" {expected}") + } + _ => String::new(), + }; + let line: String = format!( + " {symbol} {}{type_info} ({label})", + finding.column_name + ); + match finding.kind.as_str() { + COLUMN_ADDED => STYLE.success(&line), + COLUMN_REMOVED => STYLE.error(&line), + COLUMN_TYPE_CHANGED => STYLE.warning(&line), + _ => line, + } +} + +fn schema_diff_lines(model: &ModelPlanText) -> Vec { + if model.schema_findings.is_empty() && model.rename_hints.is_empty() { + return Vec::new(); + } + let mut lines: Vec = vec![STYLE.label(" schema diff:")]; + let mut hinted: Vec = Vec::new(); + for finding in &model.schema_findings { + lines.push(schema_finding_line(finding)); + if finding.kind != COLUMN_ADDED { + continue; + } + for hint in &model.rename_hints { + if hint.added_column.to_lowercase() == finding.column_name.to_lowercase() { + lines.push(format!(" {}", STYLE.warning(&hint.message))); + hinted.push(hint.added_column.to_lowercase()); + } + } + } + for hint in &model.rename_hints { + if !hinted.contains(&hint.added_column.to_lowercase()) { + lines.push(format!( + " {}", + STYLE.warning(&format!("{}: {}", hint.added_column, hint.message)) + )); + } + } + lines +} + +fn diff_lines(label: &str, diff: Option<&Vec>) -> Vec { + let Some(diff) = diff else { + return Vec::new(); + }; + let mut lines: Vec = vec![STYLE.label(label)]; + lines.extend(diff.iter().cloned()); + lines +} + +fn detail_entry_lines(model: &ModelPlanText, reason: &str, width: usize) -> Vec { + if reason == FIRST_RUN_REASON { + let mut lines: Vec = vec![name_value_line( + &model.name, + &model.materialization_label, + width, + true, + )]; + lines.extend(cursor_lines(model, true)); + return lines; + } + let mut lines: Vec = vec![name_value_line( + &model.name, + &STYLE.accent(&action_text(model)), + width, + false, + )]; + lines.extend(cursor_lines(model, reason != FULL_REFRESH_REASON)); + lines.extend(policy_lines(model)); + if !model.changed_functions.is_empty() { + lines.push(format!( + " cause: function {} changed", + model.changed_functions.join(", ") + )); + } + lines.extend(schema_diff_lines(model)); + lines.extend(diff_lines(" config diff:", model.config_diff.as_ref())); + lines.extend(diff_lines(" query diff:", model.query_diff.as_ref())); + lines +} + +/// One section per change reason, in reason order, with each model's detail. +pub(crate) fn reason_group_lines( + report: &PlanTextReport, + active: &[&ModelPlanText], + width: usize, +) -> Vec { + let mut lines: Vec = Vec::new(); + for (reason, label) in REASON_GROUPS { + let entries: Vec<&ModelPlanText> = active + .iter() + .copied() + .filter(|model| model.reason == reason) + .collect(); + if entries.is_empty() { + continue; + } + lines.extend(section_lines(label, &entries, &report.display, |model| { + detail_entry_lines(model, reason, width) + })); + } + lines +} + +/// Unchanged and normally incremental models. +pub(crate) fn routine_model_lines( + report: &PlanTextReport, + active: &[&ModelPlanText], + width: usize, +) -> Vec { + let routine: Vec<&ModelPlanText> = active + .iter() + .copied() + .filter(|model| { + model.reason == NO_CHANGE_REASON || model.reason == NORMAL_INCREMENTAL_REASON + }) + .collect(); + if routine.is_empty() { + return Vec::new(); + } + section_lines("Models", &routine, &report.display, |model| { + let mut lines: Vec = vec![name_value_line( + &model.name, + &model.materialization_label, + width, + true, + )]; + lines.extend(cursor_lines(model, true)); + lines + }) +} + +fn function_lines(function: &FunctionPlanText, show_details: bool, width: usize) -> Vec { + let mut lines: Vec = vec![name_value_line(&function.name, &function.kind, width, true)]; + if !show_details { + return lines; + } + match function.reason.as_str() { + FIRST_RUN_REASON => lines.push(" reason: first run".to_owned()), + FULL_REFRESH_REASON => lines.push(" reason: full refresh".to_owned()), + QUERY_CHANGED_REASON => { + lines.extend(diff_lines(" query diff:", function.query_diff.as_ref())); + } + _ => {} + } + lines +} + +/// The changed functions with their detail, or the unchanged ones. +pub(crate) fn functions_lines(report: &PlanTextReport, changed: bool, width: usize) -> Vec { + let entries: Vec<&FunctionPlanText> = report + .functions + .iter() + .filter(|function| (function.reason != NO_CHANGE_REASON) == changed) + .collect(); + if entries.is_empty() { + return Vec::new(); + } + let title: &str = if changed { + "Changed functions" + } else { + "Functions" + }; + section_lines(title, &entries, &report.display, |function| { + function_lines(function, changed, width) + }) +} + +/// The seeds with their load reason. +pub(crate) fn seed_lines(report: &PlanTextReport) -> Vec { + if report.seeds.is_empty() { + return Vec::new(); + } + section_lines("Seeds", &report.seeds, &report.display, |seed| { + let label: &str = match seed.reason.as_str() { + FIRST_RUN_REASON => "first_run", + CONFIG_CHANGED_REASON => "seed_changed", + NO_CHANGE_REASON => "current", + _ => "", + }; + if label.is_empty() { + return vec![format!(" {}", seed.name)]; + } + vec![format!(" {} ({label})", seed.name)] + }) +} + +/// `Full refresh (N)` with one count row per materialization label, most common first. +pub(crate) fn full_refresh_count_lines(active: &[&ModelPlanText]) -> Vec { + let mut counts: Vec<(&str, usize)> = Vec::new(); + for model in active { + if let Some((_, count)) = counts + .iter_mut() + .find(|(label, _)| *label == model.materialization_label) + { + *count += 1; + } else { + counts.push((&model.materialization_label, 1)); + } + } + counts.sort_by_key(|(_, count)| std::cmp::Reverse(*count)); + let mut lines: Vec = vec![section_header( + STYLE, + &format!("Full refresh ({})", active.len()), + )]; + for (label, count) in counts { + lines.push(format!(" {count:>3} {label}")); + } + lines +} + +fn diagnostic_section_lines( + (entries, title, error): (&[&PlanWarningText], &str, bool), + display: &DisplayLimit, +) -> Vec { + if entries.is_empty() { + return Vec::new(); + } + let line_style = |text: &str| { + if error { + STYLE.error(text) + } else { + STYLE.warning(text) + } + }; + let heading: String = format!("{title} ({})", entries.len()); + let mut lines: Vec = vec![ + String::new(), + if error { + STYLE.error_strong(&heading) + } else { + STYLE.warning_strong(&heading) + }, + ]; + let visible: usize = visible_count(entries.len(), display); + for (index, warning) in entries[..visible].iter().enumerate() { + let connector: String = tree_connector(STYLE, index + 1 == visible); + let message_lines: Vec<&str> = warning.message.split('\n').collect(); + if let Some(model) = &warning.model_name { + lines.push(format!("{connector} {}", STYLE.object_name(model))); + for (line_index, line) in message_lines.iter().enumerate() { + let child: String = tree_connector(STYLE, line_index + 1 == message_lines.len()); + lines.push(format!(" {child} {}", line_style(line))); + } + } else { + lines.push(format!("{connector} {}", line_style(message_lines[0]))); + for line in &message_lines[1..] { + lines.push(format!(" {}", line_style(line))); + } + } + } + lines.extend(overflow_line((entries.len(), visible), " ", display)); + lines +} + +/// The plan's errors, then its warnings, without the diagnostics other sections already show. +pub(crate) fn warning_lines(report: &PlanTextReport) -> Vec { + let include_stale: bool = report + .selection_diagnostics + .unwrap_or(report.include_freshness_diagnostics); + let diagnostics: Vec<&PlanWarningText> = report + .warnings + .iter() + .filter(|warning| { + warning.severity != INFO_SEVERITY_VALUE + && (include_stale || !warning.message.starts_with(STALE_INPUT_WARNING_TITLE)) + && !(report.include_freshness_diagnostics + && warning + .message + .starts_with(SOURCE_FRESHNESS_UNKNOWN_WARNING_TITLE)) + }) + .collect(); + let (errors, warnings): (Vec<&PlanWarningText>, Vec<&PlanWarningText>) = diagnostics + .into_iter() + .partition(|warning| warning.severity == ERROR_SEVERITY_VALUE); + let mut lines: Vec = + diagnostic_section_lines((&errors, "Errors", true), &report.display); + lines.extend(diagnostic_section_lines( + (&warnings, "Warnings", false), + &report.display, + )); + lines +} + +fn freshness_count(label: &str, names: &[String], warn_nonzero: bool) -> String { + let count: String = names.len().to_string(); + let styled: String = if warn_nonzero && !names.is_empty() { + STYLE.warning(&count) + } else if names.is_empty() { + STYLE.muted(&count) + } else { + count + }; + format!(" {} {styled}", STYLE.label(&format!("{label}:"))) +} + +fn freshness_set(label: &str, names: &[String], display: &DisplayLimit, warn: bool) -> String { + let rendered: String = capped_names(names, display, |name| { + if warn { + STYLE.warning(name) + } else { + STYLE.object_name(name) + } + }); + format!(" {} {rendered}", STYLE.label(&format!("{label}:"))) +} + +/// The source freshness a direct plan observed, by outcome. +pub(crate) fn source_freshness_lines(report: &PlanTextReport) -> Vec { + let Some(freshness) = &report.source_freshness else { + return Vec::new(); + }; + let SourceFreshnessText { + observed, + changed, + unchanged, + unknown, + age_warnings, + age_errors, + stale_models, + blocked_models, + } = freshness; + if observed.is_empty() && unknown.is_empty() { + return Vec::new(); + } + let display: &DisplayLimit = &report.display; + let mut lines: Vec = vec![String::new(), section_header(STYLE, "Source freshness")]; + for (label, names, warn) in [ + ("observed", observed, false), + ("changed", changed, true), + ("unchanged", unchanged, false), + ("unknown", unknown, true), + ] { + lines.push(freshness_count(label, names, warn)); + if !names.is_empty() { + lines.push(freshness_set(&format!("{label} set"), names, display, warn)); + } + } + for (label, names) in [ + ("age warnings", age_warnings), + ("age errors", age_errors), + ("source-stale models", stale_models), + ("source-blocked models", blocked_models), + ] { + if !names.is_empty() { + lines.push(freshness_set(label, names, display, true)); + } + } + lines +} + +/// Stale models outside the selection, unless selection diagnostics are off. +pub(crate) fn remaining_stale_lines(report: &PlanTextReport) -> Vec { + if report.selection_diagnostics == Some(false) || report.remaining_stale_models.is_empty() { + return Vec::new(); + } + let names: &[String] = &report.remaining_stale_models; + vec![ + String::new(), + section_header(STYLE, "Remaining stale"), + STYLE.muted(&format!(" models outside selection: {}", names.len())), + STYLE.muted(&format!( + " model set: {}", + capped_names(names, &report.display, |name| STYLE.muted(name)) + )), + ] +} diff --git a/crates/sqlbuild-core/src/cli_text/_helpers/tree_rows.rs b/crates/sqlbuild-core/src/cli_text/_helpers/tree_rows.rs new file mode 100644 index 0000000000..db02ee17d0 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/_helpers/tree_rows.rs @@ -0,0 +1,82 @@ +//! Section entry and detail rows connected into trees. + +use crate::cli_text::_helpers::bounded_sections::strip_ansi; +use crate::cli_text::_helpers::layout::tree_connector; +use crate::cli_text::constants::{ + ENTRY_ROW_PREFIX, LEAF_KEY_SEPARATOR, LEAF_ROW_PREFIX, NESTED_ROW_PREFIX, TREE_EXEMPT_CHARS, +}; +use crate::cli_text::models::{CliStyle, RowKind}; + +fn classify(line: &str) -> RowKind { + let plain: String = strip_ansi(line); + if plain.trim().is_empty() || !plain.starts_with(ENTRY_ROW_PREFIX) { + return RowKind::Other; + } + if plain.starts_with(NESTED_ROW_PREFIX) { + return RowKind::Nested; + } + let (prefix, kind) = if plain.starts_with(LEAF_ROW_PREFIX) { + (LEAF_ROW_PREFIX, RowKind::Leaf) + } else { + (ENTRY_ROW_PREFIX, RowKind::Entry) + }; + if plain[prefix.len()..] + .chars() + .next() + .is_some_and(|first| TREE_EXEMPT_CHARS.contains(&first)) + { + return RowKind::Other; + } + kind +} + +/// Whether the row is the final one of its kind within its block. +fn is_last(kinds: &[RowKind], index: usize, row_kind: RowKind) -> bool { + for kind in &kinds[index + 1..] { + match (row_kind, kind) { + (_, RowKind::Other) | (RowKind::Leaf, RowKind::Entry) => return true, + (RowKind::Entry, RowKind::Entry) | (RowKind::Leaf, RowKind::Leaf) => return false, + _ => {} + } + } + true +} + +/// `_treeify_plan_lines`: entry rows and their detail rows drawn with tree connectors. +pub(crate) fn treeify(lines: Vec) -> Vec { + let style: CliStyle = CliStyle { use_color: true }; + let kinds: Vec = lines.iter().map(|line| classify(line)).collect(); + let mut out: Vec = Vec::with_capacity(lines.len()); + for (index, line) in lines.into_iter().enumerate() { + match kinds[index] { + RowKind::Entry => { + let connector: String = + tree_connector(style, is_last(&kinds, index, RowKind::Entry)); + out.push(format!("{connector} {}", &line[ENTRY_ROW_PREFIX.len()..])); + } + RowKind::Leaf => { + let connector: String = + tree_connector(style, is_last(&kinds, index, RowKind::Leaf)); + let content: String = match line.strip_prefix(LEAF_ROW_PREFIX) { + Some(content) => content.to_owned(), + None => strip_ansi(&line)[LEAF_ROW_PREFIX.len()..].to_owned(), + }; + if let Some((key, value)) = content.split_once(LEAF_KEY_SEPARATOR) { + out.push(format!( + "{LEAF_ROW_PREFIX}{connector} {} {value}", + style.muted(key) + )); + } else if content.ends_with(':') { + out.push(format!( + "{LEAF_ROW_PREFIX}{connector} {}", + style.muted(&content) + )); + } else { + out.push(format!("{LEAF_ROW_PREFIX}{connector} {content}")); + } + } + RowKind::Nested | RowKind::Other => out.push(line), + } + } + out +} diff --git a/crates/sqlbuild-core/src/cli_text/constants.rs b/crates/sqlbuild-core/src/cli_text/constants.rs index 5f26c08244..86d35be7a9 100644 --- a/crates/sqlbuild-core/src/cli_text/constants.rs +++ b/crates/sqlbuild-core/src/cli_text/constants.rs @@ -29,3 +29,51 @@ pub(crate) const OPEN_INPUT: &str = "open input: "; pub(crate) const INVALID_UPSTREAM: &str = "depends on invalid model "; pub(crate) const DEFAULT_ACTION: &str = "declare the input shape, then run `sqb compile --no-cache`"; +/// Names narrower than this still get a column this wide. +pub(crate) const MIN_NAME_COLUMN_WIDTH: usize = 20; +/// Indent of a section's entry rows before tree conversion. +pub(crate) const ENTRY_ROW_PREFIX: &str = " "; +/// Indent of an entry's detail rows before tree conversion. +pub(crate) const LEAF_ROW_PREFIX: &str = " "; +/// Indent of rows nested below a detail row, which keep their text. +pub(crate) const NESTED_ROW_PREFIX: &str = " "; +pub(crate) const LEAF_KEY_SEPARATOR: &str = ": "; +/// A row whose text starts with one of these after its indent is not a tree row. +pub(crate) const TREE_EXEMPT_CHARS: [char; 2] = [' ', '.']; +pub(crate) const STALE_INPUT_WARNING_TITLE: &str = "Stale inputs detected"; +pub(crate) const SOURCE_FRESHNESS_UNKNOWN_WARNING_TITLE: &str = "source freshness unknown"; +pub(crate) const DIFF_HEADER_MARKER: &str = "# "; +pub(crate) const EFFECTIVE_BATCH_SIZE_TOKEN: &str = "effective"; +pub(crate) const REASON_GROUPS: [(&str, &str); 7] = [ + ("full_refresh", "Full refresh"), + ("query_changed", "Query changed"), + ("function_changed", "Function changed"), + ("config_changed", "Config changed"), + ("schema_changed", "Schema changed"), + ("first_run", "First run"), + ("renamed", "Renamed"), +]; +pub(crate) const SKIP_ACTION: &str = "skip"; +pub(crate) const CREATE_VIEW_ACTION: &str = "create_view"; +pub(crate) const CUSTOM_ACTION: &str = "custom"; +pub(crate) const FULL_REFRESH_REASON: &str = "full_refresh"; +pub(crate) const FIRST_RUN_REASON: &str = "first_run"; +pub(crate) const QUERY_CHANGED_REASON: &str = "query_changed"; +pub(crate) const SCHEMA_CHANGED_REASON: &str = "schema_changed"; +pub(crate) const CONFIG_CHANGED_REASON: &str = "config_changed"; +pub(crate) const NO_CHANGE_REASON: &str = "no_change"; +pub(crate) const NORMAL_INCREMENTAL_REASON: &str = "normal_incremental"; +pub(crate) const BOUNDED_BACKFILL: &str = "bounded"; +pub(crate) const FULL_BACKFILL: &str = "full"; +pub(crate) const FORWARD_BACKFILL: &str = "forward"; +pub(crate) const COLUMN_ADDED: &str = "column_added"; +pub(crate) const COLUMN_REMOVED: &str = "column_removed"; +pub(crate) const COLUMN_TYPE_CHANGED: &str = "column_type_changed"; +pub(crate) const ERROR_SEVERITY_VALUE: &str = "error"; +pub(crate) const INFO_SEVERITY_VALUE: &str = "info"; +pub(crate) const CHANGED_IDENTITY: &str = "changed"; +pub(crate) const PRE_SQL_INGRESS_PHASE: &str = "pre_sql_ingress"; +pub(crate) const READ_SIDE_PHASE: &str = "read_side"; +pub(crate) const PYTHON_INGRESS_TITLE: &str = "Python ingress"; +pub(crate) const PYTHON_READ_SIDE_TITLE: &str = "Python read-side"; +pub(crate) const PLAN_READY_TITLE: &str = "Plan ready"; diff --git a/crates/sqlbuild-core/src/cli_text/main/format_plan_completion.rs b/crates/sqlbuild-core/src/cli_text/main/format_plan_completion.rs new file mode 100644 index 0000000000..b386b0bb0c --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_plan_completion.rs @@ -0,0 +1,27 @@ +//! The completion line that closes text plan output. + +use crate::cli_text::_helpers::plan_sections::{full_refresh_layout, ready_summary}; +use crate::cli_text::constants::{ERROR_SEVERITY_VALUE, PHASE_OK_GLYPH}; +use crate::cli_text::models::{CliStyle, PlanTextReport}; + +/// `✓ Plan complete `, warning-styled when the plan has errors. +#[must_use] +pub fn format_plan_completion(report: &PlanTextReport) -> String { + let style: CliStyle = CliStyle { + use_color: report.use_color, + }; + let has_errors: bool = report + .warnings + .iter() + .any(|warning| warning.severity == ERROR_SEVERITY_VALUE); + let (glyph, label) = if has_errors { + ( + style.warning(PHASE_OK_GLYPH), + style.warning_strong("Plan complete with errors"), + ) + } else { + (style.success(PHASE_OK_GLYPH), "Plan complete".to_owned()) + }; + let summary: String = ready_summary(report, full_refresh_layout(report)); + format!("{glyph} {label} {summary}") +} diff --git a/crates/sqlbuild-core/src/cli_text/main/format_plan_text.rs b/crates/sqlbuild-core/src/cli_text/main/format_plan_text.rs new file mode 100644 index 0000000000..822e9a26b4 --- /dev/null +++ b/crates/sqlbuild-core/src/cli_text/main/format_plan_text.rs @@ -0,0 +1,22 @@ +//! The text plan, grouped by reason with inline detail. + +use crate::cli_text::_helpers::bounded_sections::strip_ansi; +use crate::cli_text::_helpers::plan_document::{full_refresh_lines, incremental_lines}; +use crate::cli_text::_helpers::plan_sections::{active_models, full_refresh_layout}; +use crate::cli_text::models::{ModelPlanText, PlanTextReport}; + +/// The plan as `format_plan` printed it, without the completion line. +#[must_use] +pub fn format_plan_text(report: &PlanTextReport) -> String { + let active: Vec<&ModelPlanText> = active_models(report); + let lines: Vec = if full_refresh_layout(report) { + full_refresh_lines(report, &active) + } else { + incremental_lines(report, &active) + }; + let text: String = lines.join("\n"); + if report.use_color { + return text; + } + strip_ansi(&text) +} diff --git a/crates/sqlbuild-core/src/cli_text/main/mod.rs b/crates/sqlbuild-core/src/cli_text/main/mod.rs index 75b612fceb..1d6c262c16 100644 --- a/crates/sqlbuild-core/src/cli_text/main/mod.rs +++ b/crates/sqlbuild-core/src/cli_text/main/mod.rs @@ -1,2 +1,4 @@ pub mod format_compile_text; +pub mod format_plan_completion; +pub mod format_plan_text; pub mod semantic_coverage_notice; diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 3054461df0..97843c1373 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -30,6 +30,26 @@ impl CliStyle { self.apply(DIM, text) } + #[must_use] + pub fn label(self, text: &str) -> String { + self.apply(DIM, text) + } + + #[must_use] + pub fn accent(self, text: &str) -> String { + self.apply(BLUE, text) + } + + #[must_use] + pub fn plan_section(self, text: &str) -> String { + self.apply(BOLD, text) + } + + #[must_use] + pub fn warning_strong(self, text: &str) -> String { + self.apply(&format!("{YELLOW}{BOLD}"), text) + } + #[must_use] pub fn object_name(self, text: &str) -> String { self.apply("", text) @@ -154,3 +174,191 @@ pub struct CompileTextReport { pub manifest: bool, pub lineage: LineageNote, } + +/// How many entries each bounded section of human output shows. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DisplayLimit { + /// `None` shows every entry. + pub max_entries: Option, + /// The flag the overflow line suggests, such as `--verbose`. + pub overflow_flag: String, +} + +/// The cursor detail of one planned model, with the planner's display text already rendered. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CursorPlanText { + pub column: String, + pub cursor_type: Option, + pub microbatch: bool, + pub requested_start: Option, + pub requested_end: Option, + /// The planner's no-input-rows or waiting text, when inputs are empty. + pub empty_inputs_window: Option, + pub waiting_on_empty_inputs: bool, + /// `start → inclusive end`, when the bounds are resolved. + pub range: Option, + pub declared_grain: Option, + pub effective_grain: Option, + pub declared_batch_size: Option, + pub effective_batch_size: Option, + pub planned_batch_count: Option, + /// The runtime resolves the bounds. + pub runtime_owned: bool, + /// Resolution status `resolved` or `deferred`; anything else is neither. + pub resolved: bool, + pub deferred: bool, + /// `once ... is up to date`, naming the runtime-owned cursor inputs. + pub runtime_inputs_clause: String, + /// `str(entry.batch_concurrency)`. + pub batch_concurrency: String, + pub unaccounted_partition_policy: Option, +} + +/// One schema finding of a planned model: `column_added`, `column_removed` or `column_type_changed`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SchemaFindingText { + pub kind: String, + pub column_name: String, + pub expected_type: Option, + pub actual_type: Option, +} + +/// A hint that an added column may be a rename. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RenameHintText { + pub added_column: String, + pub message: String, +} + +/// One planned model as the text plan shows it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ModelPlanText { + pub name: String, + /// `PlanAction` value. + pub action: String, + /// `PlanReason` value. + pub reason: String, + pub materialization_label: String, + pub custom_materialization_name: Option, + /// `BackfillAction` value. + pub backfill_action: String, + pub backfill_duration: Option, + pub cursor: Option, + pub changed_functions: Vec, + pub schema_findings: Vec, + pub rename_hints: Vec, + /// The rendered config diff, when the version config changed. + pub config_diff: Option>, + /// The rendered query diff, when the query changed and the previous query is known. + pub query_diff: Option>, +} + +/// One planned function. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FunctionPlanText { + pub name: String, + pub reason: String, + /// `table function` or ` udf`. + pub kind: String, + /// The rendered query diff, when the query changed and the previous query is known. + pub query_diff: Option>, +} + +/// One planned seed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SeedPlanText { + pub name: String, + pub reason: String, +} + +/// One planned source or loader load. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SourceLoadPlanText { + pub name: String, + /// `ExecutionResourceKind` value; only `loader` and `source` loads are listed. + pub resource_kind: String, + pub is_reload: bool, + pub integration_kind: Option, + pub write_strategy: Option, + pub cursor_column: Option, + pub unique_key: Vec, +} + +/// One planned Python node, with its rendered identity diffs. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PythonPlanText { + pub name: String, + pub kind: String, + /// `pre_sql_ingress` or `read_side`. + pub phase: String, + pub identity_status: String, + pub source_diff: Vec, + pub dependency_diff: Vec, +} + +/// One provider used by a selected Python surface. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ProviderUsageText { + pub provider_name: String, + pub consumer_kind: String, + pub consumer_name: String, + pub parameter_name: String, + pub annotation_class_name: Option, +} + +/// One plan warning. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PlanWarningText { + /// `error`, `warning` or `info`. + pub severity: String, + pub message: String, + pub model_name: Option, +} + +/// The source freshness observed by a direct plan, by outcome. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SourceFreshnessText { + pub observed: Vec, + pub changed: Vec, + pub unchanged: Vec, + pub unknown: Vec, + pub age_warnings: Vec, + pub age_errors: Vec, + pub stale_models: Vec, + pub blocked_models: Vec, +} + +/// Everything the text plan renders. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PlanTextReport { + pub use_color: bool, + pub include_header: bool, + /// The plan was requested with `--full-refresh`. + pub full_refresh: bool, + pub display: DisplayLimit, + pub include_freshness_diagnostics: bool, + /// The plan's explicit selection-diagnostics policy, when it sets one. + pub selection_diagnostics: Option, + pub models: Vec, + pub functions: Vec, + pub seeds: Vec, + pub source_loads: Vec, + pub python_entries: Vec, + /// The plan's provider usages, then each Python entry's. + pub provider_usages: Vec, + pub warnings: Vec, + pub source_freshness: Option, + pub remaining_stale_models: Vec, + pub pruned_models: Vec, + /// Table type, retention and migration lines the planner renders. + pub pre_build_lines: Vec, +} + +/// The structural kind of a rendered plan line. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum RowKind { + Entry, + Leaf, + Nested, + Other, +} diff --git a/crates/sqlbuild-core/src/cli_text/tests/helpers.rs b/crates/sqlbuild-core/src/cli_text/tests/helpers.rs index 1b9a8dbf19..7d5b244a9f 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/helpers.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/helpers.rs @@ -1,4 +1,6 @@ -use crate::cli_text::models::{DiagnosticText, LocationText}; +use crate::cli_text::models::{ + DiagnosticText, DisplayLimit, LocationText, ModelPlanText, PlanTextReport, +}; /// A location in `models/a.sql`, whose text has a CRLF line, a non-ASCII name and a vertical tab. pub(super) fn location(line: i64, column: i64, end: (Option, Option)) -> LocationText { @@ -29,3 +31,41 @@ pub(super) fn diagnostic(severity: &str, code: &str, message: &str) -> Diagnosti occurrences: 1, } } + +/// An empty uncolored plan report that shows every entry. +pub(super) fn plan_report(models: Vec) -> PlanTextReport { + PlanTextReport { + use_color: false, + include_header: true, + full_refresh: false, + display: DisplayLimit { + max_entries: None, + overflow_flag: "--verbose".to_owned(), + }, + include_freshness_diagnostics: false, + selection_diagnostics: None, + models, + functions: Vec::new(), + seeds: Vec::new(), + source_loads: Vec::new(), + python_entries: Vec::new(), + provider_usages: Vec::new(), + warnings: Vec::new(), + source_freshness: None, + remaining_stale_models: Vec::new(), + pruned_models: Vec::new(), + pre_build_lines: Vec::new(), + } +} + +/// A planned table model with a change reason and full backfill. +pub(super) fn table_model(name: &str, reason: &str) -> ModelPlanText { + ModelPlanText { + name: name.to_owned(), + action: "create_table".to_owned(), + reason: reason.to_owned(), + materialization_label: "table".to_owned(), + backfill_action: "full".to_owned(), + ..ModelPlanText::default() + } +} diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs index 2f47337f66..e1b88d9e36 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs @@ -1,12 +1,15 @@ use crate::cli_text::_helpers::diagnostics::diagnostics_text; use crate::cli_text::main::format_compile_text::format_compile_text; +use crate::cli_text::main::format_plan_completion::format_plan_completion; +use crate::cli_text::main::format_plan_text::format_plan_text; use crate::cli_text::main::semantic_coverage_notice::semantic_coverage_notice; use crate::cli_text::models::{ - CliStyle, CompileTextReport, DiagnosticText, LineageNote, ModelLine, RelatedText, + CliStyle, CompileTextReport, DiagnosticText, LineageNote, ModelLine, ModelPlanText, + PlanTextReport, PlanWarningText, RelatedText, }; -use crate::cli_text::tests::helpers::{diagnostic, location}; +use crate::cli_text::tests::helpers::{diagnostic, location, plan_report, table_model}; use crate::cli_text::tests::test_types::{ - CompileTextTestCase, DiagnosticsTextTestCase, NoticeTestCase, + CompileTextTestCase, DiagnosticsTextTestCase, NoticeTestCase, PlanTextTestCase, }; /// Expectations recorded from the retired Python formatter. @@ -122,3 +125,60 @@ fn given_report_when_formatting_then_lists_models_and_summary() { ); } } + +#[test] +fn given_plan_report_when_rendering_then_groups_models_by_reason() { + let test_cases = [ + PlanTextTestCase { + description: "a changed query with its diff, a routine model and a warning", + report: PlanTextReport { + warnings: vec![PlanWarningText { + severity: "warning".to_owned(), + message: "orders lag\nsecond line".to_owned(), + model_name: Some("orders".to_owned()), + }], + ..plan_report(vec![ + ModelPlanText { + query_diff: Some(vec![ + " -SELECT 1".to_owned(), + " +SELECT 2".to_owned(), + ]), + ..table_model("orders", "query_changed") + }, + table_model("customers", "no_change"), + ModelPlanText { + action: "skip".to_owned(), + ..table_model("skipped", "no_change") + }, + ]) + }, + expected_text: "Plan ready 3 selected\n\nQuery changed (1)\n\u{2514}\u{2500}\u{2500} orders full rebuild\n \u{251c}\u{2500}\u{2500} policy replay_on_change=full\n \u{2514}\u{2500}\u{2500} query diff:\n -SELECT 1\n +SELECT 2\n\nModels (1)\n\u{2514}\u{2500}\u{2500} customers table\n\nWarnings (1)\n\u{2514}\u{2500}\u{2500} orders\n \u{251c}\u{2500}\u{2500} orders lag\n \u{2514}\u{2500}\u{2500} second line", + expected_completion: "\u{2713} Plan complete 3 selected", + }, + PlanTextTestCase { + description: "every active model is a full refresh", + report: plan_report(vec![ + table_model("orders", "full_refresh"), + table_model("customers", "full_refresh"), + ModelPlanText { + materialization_label: "view".to_owned(), + ..table_model("stg_orders", "full_refresh") + }, + ]), + expected_text: "Plan ready full refresh, 3 selected\n\nFull refresh (3)\n 2 table\n 1 view", + expected_completion: "\u{2713} Plan complete full refresh, 3 selected", + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + format_plan_text(&test_case.report).as_str(), + format_plan_completion(&test_case.report).as_str() + ), + (test_case.expected_text, test_case.expected_completion), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/cli_text/tests/test_types.rs b/crates/sqlbuild-core/src/cli_text/tests/test_types.rs index 9f4c59aac0..09c3dd0dda 100644 --- a/crates/sqlbuild-core/src/cli_text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/cli_text/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::cli_text::models::{CliStyle, CompileTextReport, DiagnosticText}; +use crate::cli_text::models::{CliStyle, CompileTextReport, DiagnosticText, PlanTextReport}; pub(super) struct DiagnosticsTextTestCase { pub(super) description: &'static str, @@ -18,3 +18,10 @@ pub(super) struct CompileTextTestCase { pub(super) report: CompileTextReport, pub(super) expected_text: &'static str, } + +pub(super) struct PlanTextTestCase { + pub(super) description: &'static str, + pub(super) report: PlanTextReport, + pub(super) expected_text: &'static str, + pub(super) expected_completion: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs index b289578fe4..1e499b85fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/json_report.rs @@ -91,6 +91,19 @@ pub(crate) fn orjson_sorted(value: &Bound<'_, PyAny>) -> PyResult { ) } +/// `json.dumps(value, indent=2)`, raising what `json.dumps` raises. +pub(crate) fn indented_json(value: &Bound<'_, PyAny>) -> PyResult { + let mut conversion: Conversion = Conversion::default(); + let converted: JsonValue = conversion.json_value(value)?; + if let Some(error) = conversion.stdlib_error { + return Err(error); + } + emitted( + &converted, + &JsonDialect::Stdlib(StdlibJsonOptions::new(Some(2))), + ) +} + /// `json.dumps(value, ensure_ascii=True, separators=(",", ":"), sort_keys=True)`. pub(crate) fn compact_sorted_json(value: &Bound<'_, PyAny>) -> PyResult { let mut conversion: Conversion = Conversion::default(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 86f280618a..f8b0832769 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -20,6 +20,7 @@ pub(crate) mod lineage; pub(crate) mod macro_bridge; pub(crate) mod manifest; pub(crate) mod model_config; +pub(crate) mod plan_output; pub(crate) mod project_assembly; pub(crate) mod python_values; pub(crate) mod refactor; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs new file mode 100644 index 0000000000..40a9d2d458 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs @@ -0,0 +1,262 @@ +//! One planned model's requested, resolved and deferred cursor state, for text and JSON plans. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::plan_output::values::{optional_text, text}; + +const BOUND_TYPES_MODULE: &str = "sqlbuild.cursor_algebra.types"; +const CURSOR_MODELS_MODULE: &str = "sqlbuild.cursor_algebra.models"; +const ALIGNED_BOUNDS_MODULE: &str = + "sqlbuild.compiler.planner.main.execution.aligned_timestamp_bounds"; +const BATCH_SIZE_MODULE: &str = + "sqlbuild.compiler.planner.main.execution.effective_microbatch_batch_size"; +const MICROBATCH_MODE: &str = "microbatch"; +const TIMESTAMP_CURSOR: &str = "timestamp"; +const INTEGER_CURSOR: &str = "integer"; +const WATERMARK_STRATEGY: &str = "watermark"; +const SECOND_GRAIN: &str = "second"; +/// `GRAIN_ORDER`, finest first. +const GRAIN_ORDER: [&str; 6] = ["second", "minute", "hour", "day", "month", "year"]; + +/// `CursorBoundsOwner` and `CursorResolutionStatus` of one model, as their values. +pub(crate) const RUNTIME_OWNER: &str = "runtime"; +pub(crate) const PLANNER_OWNER: &str = "planner"; +pub(crate) const NO_INPUT_ROWS: &str = "no_input_rows"; +pub(crate) const RESOLVED: &str = "resolved"; +pub(crate) const DEFERRED: &str = "deferred"; +pub(crate) const UNAVAILABLE: &str = "unavailable"; + +/// `CursorPlanDetails` of one model entry. +pub(crate) struct CursorDetails<'py> { + pub(crate) requested_start: Option, + pub(crate) requested_end: Option, + pub(crate) runtime_owned: bool, + pub(crate) resolution_status: &'static str, + /// Both bounds are scalars, not sentinels. + pub(crate) resolved_bounds: Option>, + pub(crate) declared_grain: Option, + pub(crate) effective_grain: Option, + pub(crate) declared_batch_size: Option, + pub(crate) effective_batch_size: Option, + pub(crate) planned_batch_count: Option, + pub(crate) empty_inputs: Bound<'py, PyAny>, + pub(crate) waiting_on_empty_inputs: bool, +} + +fn grain_rank(grain: &str) -> PyResult { + GRAIN_ORDER + .iter() + .position(|known| *known == grain) + .ok_or_else(|| { + pyo3::exceptions::PyValueError::new_err(format!("'{grain}' is not a valid CursorGrain")) + }) +} + +/// The coarsest timestamp grain declared across a model and its inputs. +fn effective_grain(entry: &Bound<'_, PyAny>) -> PyResult> { + let declared: Option = optional_text(&entry.getattr("cursor_grain")?)?; + if !entry.getattr("cursor_type")?.eq(TIMESTAMP_CURSOR)? { + return Ok(declared); + } + let mut effective: String = declared.unwrap_or_else(|| SECOND_GRAIN.to_owned()); + if entry + .getattr("microbatch_strategy")? + .eq(WATERMARK_STRATEGY)? + { + return Ok(Some(effective)); + } + for relation in entry.getattr("cursor_input_relations")?.try_iter()? { + let relation_grain: String = optional_text(&relation?.getattr("cursor_grain")?)? + .unwrap_or_else(|| SECOND_GRAIN.to_owned()); + if grain_rank(&relation_grain)? > grain_rank(&effective)? { + effective = relation_grain; + } + } + Ok(Some(effective)) +} + +/// How many windows the executor will create over timestamp or integer bounds. +fn count_batches( + bounds: &Bound<'_, PyAny>, + batch_size: &str, + cursor_type: &str, +) -> PyResult> { + let py: Python<'_> = bounds.py(); + let (start, end) = (bounds.getattr("start")?, bounds.getattr("end")?); + if cursor_type == TIMESTAMP_CURSOR { + let duration: Bound<'_, PyAny> = + imported(py, CURSOR_MODELS_MODULE, "Duration")?.call_method1("parse", (batch_size,))?; + if duration.is_none() { + return Ok(None); + } + let temporal = |value: &Bound<'_, PyAny>| -> PyResult { + Ok( + value.is_instance(&imported(py, CURSOR_MODELS_MODULE, "DateValue")?)? + || value.is_instance(&imported(py, CURSOR_MODELS_MODULE, "TimestampValue")?)?, + ) + }; + if !temporal(&start)? || !temporal(&end)? { + return Ok(None); + } + let (mut current, end): (Bound<'_, PyAny>, Bound<'_, PyAny>) = call_with( + &imported(py, ALIGNED_BOUNDS_MODULE, "aligned_timestamp_bounds")?, + &[("start", start), ("end", end)], + )? + .extract()?; + let minimum: Bound<'_, PyAny> = py.import("builtins")?.getattr("min")?; + let mut count: i64 = 0; + while current.lt(&end)? { + current = minimum.call1((duration.call_method1("add_to", (¤t,))?, &end))?; + count += 1; + } + return Ok(Some(count)); + } + if cursor_type != INTEGER_CURSOR { + return Ok(None); + } + let integer: Bound<'_, PyAny> = imported(py, CURSOR_MODELS_MODULE, "IntegerValue")?; + if !start.is_instance(&integer)? || !end.is_instance(&integer)? { + return Ok(None); + } + let (start, end): (i128, i128) = ( + start.getattr("value")?.extract()?, + end.getattr("value")?.extract()?, + ); + let Ok(size) = batch_size.trim().replace('_', "").parse::() else { + return Ok(None); + }; + if size <= 0 || start >= end { + return Ok(Some(0)); + } + Ok(Some( + i64::try_from((end - start + size - 1) / size).unwrap_or(i64::MAX), + )) +} + +/// `build_cursor_plan_details`: the model's cursor details, or `None` without a cursor column. +pub(crate) fn cursor_details<'py>( + entry: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = entry.py(); + if entry.getattr("cursor_column")?.is_none() { + return Ok(None); + } + let requested_start: Option = optional_text(&entry.getattr("start_cursor_override")?)?; + let requested_end: Option = optional_text(&entry.getattr("end_cursor_override")?)?; + let mut any_runtime: bool = false; + for relation in entry.getattr("cursor_input_relations")?.try_iter()? { + any_runtime = any_runtime || relation?.getattr("is_runtime_owned")?.is_truthy()?; + } + let runtime_owned: bool = + any_runtime && !(requested_start.is_some() && requested_end.is_some()); + let microbatch_range: Bound<'py, PyAny> = entry.getattr("microbatch_range")?; + let mut resolved_bounds: Option> = if microbatch_range.is_truthy()? { + Some(microbatch_range) + } else { + let bounds: Bound<'py, PyAny> = entry.getattr("cursor_bounds")?; + (!bounds.is_none()).then_some(bounds) + }; + let sentinel: Bound<'py, PyAny> = imported(py, BOUND_TYPES_MODULE, "BoundSentinel")?; + if let Some(bounds) = &resolved_bounds + && (bounds.getattr("start")?.is_instance(&sentinel)? + || bounds.getattr("end")?.is_instance(&sentinel)?) + { + resolved_bounds = None; + } + let effective_grain: Option = effective_grain(entry)?; + let declared_batch_size: Option = optional_text(&entry.getattr("batch_size")?)?; + let effective_batch_size: Option = match (&declared_batch_size, &effective_grain) { + (Some(batch_size), Some(grain)) => Some(text(&call_with( + &imported( + py, + BATCH_SIZE_MODULE, + "resolve_effective_microbatch_batch_size", + )?, + &[ + ( + "batch_size", + pyo3::types::PyString::new(py, batch_size).into_any(), + ), + ( + "effective_grain", + pyo3::types::PyString::new(py, grain).into_any(), + ), + ], + )?)?), + (batch_size, _) => batch_size.clone(), + }; + let cursor_type: Option = optional_text(&entry.getattr("cursor_type")?)?; + let planned_batch_count: Option = + match (&resolved_bounds, &effective_batch_size, &cursor_type) { + (Some(bounds), Some(batch_size), Some(cursor_type)) + if entry.getattr("incremental_mode")?.eq(MICROBATCH_MODE)? => + { + count_batches(bounds, batch_size, cursor_type)? + } + _ => None, + }; + let empty_inputs: Bound<'py, PyAny> = entry.getattr("empty_cursor_inputs")?; + let resolution_status: &'static str = if empty_inputs.is_truthy()? { + NO_INPUT_ROWS + } else if resolved_bounds.is_some() { + RESOLVED + } else if runtime_owned { + DEFERRED + } else { + UNAVAILABLE + }; + Ok(Some(CursorDetails { + requested_start, + requested_end, + runtime_owned, + resolution_status, + resolved_bounds, + declared_grain: optional_text(&entry.getattr("cursor_grain")?)?, + effective_grain, + declared_batch_size, + effective_batch_size, + planned_batch_count, + empty_inputs, + waiting_on_empty_inputs: entry.getattr("waiting_on_empty_inputs")?.is_truthy()?, + })) +} + +/// `runtime_cursor_inputs_clause`: the runtime-owned cursor inputs this run brings up to date. +pub(crate) fn runtime_inputs_clause(entry: &Bound<'_, PyAny>) -> PyResult { + let mut labels: Vec = Vec::new(); + for relation in entry.getattr("cursor_input_relations")?.try_iter()? { + let relation: Bound<'_, PyAny> = relation?; + if !relation.getattr("is_runtime_owned")?.is_truthy()? { + continue; + } + let input_name: Bound<'_, PyAny> = relation.getattr("input_name")?; + let name: String = if input_name.is_truthy()? { + text(&input_name)? + } else { + text(&relation.getattr("relation")?)? + }; + let kind: Bound<'_, PyAny> = relation.getattr("input_kind")?; + let label: String = if kind.is_none() { + name + } else { + format!("{} {name}", text(&kind.getattr("value")?)?) + }; + if !labels.contains(&label) { + labels.push(label); + } + } + Ok(match labels.as_slice() { + [] => "once its inputs are up to date".to_owned(), + [only] => format!("once {only} is up to date"), + [first @ .., last] => format!("once {} and {last} are up to date", first.join(", ")), + }) +} + +pub(crate) fn owner(details: &CursorDetails<'_>) -> &'static str { + if details.runtime_owned { + RUNTIME_OWNER + } else { + PLANNER_OWNER + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs new file mode 100644 index 0000000000..11ea8bead3 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod cursor_details; +pub(crate) mod plan_json; +pub(crate) mod plan_text; +pub(crate) mod python_identity; +pub(crate) mod registration; +pub(crate) mod safety; +pub(crate) mod values; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs new file mode 100644 index 0000000000..264356778f --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs @@ -0,0 +1,756 @@ +//! The JSON plan document, read from the planner's output by attribute. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::compile_outputs::json_report::indented_json; +use crate::bindings::_helpers::plan_output::cursor_details::{ + CursorDetails, cursor_details, owner, +}; +use crate::bindings::_helpers::plan_output::python_identity::{ + definition_source_text, dependency_source_text, +}; +use crate::bindings::_helpers::plan_output::safety::{future_cursor_safety, maximum_start_safety}; +use crate::bindings::_helpers::plan_output::values::{ + any, dict, enum_value, iso, items, list, mapped, optional_enum_value, relation_name, text, +}; + +const SENTINEL_TOKEN_MODULE: &str = "sqlbuild.cursor_algebra.main.sentinel_to_token"; +const MICROBATCH_MODE: &str = "microbatch"; +const RENAMED_REASON: &str = "renamed"; +const RENAMED_DECISION: &str = "renamed"; +const VIEW_MATERIALIZATION: &str = "view"; +const CHANGED_IDENTITY: &str = "changed"; + +fn token<'py>(bound: Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = bound.py(); + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("sentinel", bound)?; + imported(py, SENTINEL_TOKEN_MODULE, "sentinel_to_token")?.call((), Some(&keywords)) +} + +fn token_bounds<'py>(bounds: &Bound<'py, PyAny>) -> PyResult> { + Ok(dict( + bounds.py(), + vec![ + ("start", token(bounds.getattr("start")?)?), + ("end", token(bounds.getattr("end")?)?), + ], + )? + .into_any()) +} + +/// `_serialize_cursor_details`: requested, resolved and deferred cursor-plan state. +fn cursor_json<'py>(py: Python<'py>, details: &CursorDetails<'py>) -> PyResult> { + let resolved: Bound<'py, PyAny> = match &details.resolved_bounds { + Some(bounds) => token_bounds(bounds)?, + None => py.None().into_bound(py), + }; + Ok(dict( + py, + vec![ + ( + "requested_bounds", + dict( + py, + vec![ + ("start", any(py, details.requested_start.clone())?), + ("end", any(py, details.requested_end.clone())?), + ], + )? + .into_any(), + ), + ("bounds_owner", any(py, owner(details))?), + ("resolution_status", any(py, details.resolution_status)?), + ("resolved_bounds", resolved), + ("declared_grain", any(py, details.declared_grain.clone())?), + ("effective_grain", any(py, details.effective_grain.clone())?), + ( + "declared_batch_size", + any(py, details.declared_batch_size.clone())?, + ), + ( + "effective_batch_size", + any(py, details.effective_batch_size.clone())?, + ), + ("planned_batch_count", any(py, details.planned_batch_count)?), + ("empty_inputs", list(&details.empty_inputs)?), + ( + "waiting_on_empty_inputs", + any(py, details.waiting_on_empty_inputs)?, + ), + ], + )? + .into_any()) +} + +fn identity_status(entry: &Bound<'_, PyAny>) -> PyResult<&'static str> { + let expected: Bound<'_, PyAny> = entry.getattr("fingerprint_version_hash")?; + let built: Bound<'_, PyAny> = entry.getattr("previous_version_hash")?; + Ok(if expected.is_none() { + "unknown" + } else if built.is_none() { + "missing" + } else if built.eq(&expected)? { + "current" + } else { + "stale" + }) +} + +/// The safety evidence of the microbatch range when it has some, else of the cursor bounds. +fn evidence<'py>(entry: &Bound<'py, PyAny>, field: &str) -> PyResult> { + let microbatch_range: Bound<'py, PyAny> = entry.getattr("microbatch_range")?; + if !microbatch_range.is_none() { + let found: Bound<'py, PyAny> = microbatch_range.getattr(field)?; + if !found.is_none() { + return Ok(found); + } + } + let bounds: Bound<'py, PyAny> = entry.getattr("cursor_bounds")?; + if bounds.is_none() { + return Ok(bounds); + } + bounds.getattr(field) +} + +fn microbatch_state<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = entry.py(); + let state: Bound<'py, PyDict> = dict( + py, + vec![ + ("completion_tracking", any(py, "universal")?), + ("batch_concurrency", entry.getattr("batch_concurrency")?), + ( + "unaccounted_partition_policy", + entry.getattr("unaccounted_partition_policy")?, + ), + ("reconciliation", any(py, "runtime")?), + ], + )?; + let limit: Bound<'py, PyAny> = entry.getattr("microbatch_limit")?; + if !limit.is_none() { + state.set_item("limit", limit)?; + state.set_item("count", entry.getattr("microbatch_limit_count")?)?; + state.set_item( + "action", + optional_enum_value(&entry.getattr("microbatch_limit_action")?)?, + )?; + state.set_item("warning", entry.getattr("microbatch_limit_warning")?)?; + } + Ok(state.into_any()) +} + +fn model_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = entry.py(); + let model: Bound<'py, PyDict> = dict( + py, + vec![ + ("name", entry.getattr("name")?), + ( + "relative_path", + entry.getattr("relative_path")?.str()?.into_any(), + ), + ( + "materialization_type", + enum_value(entry, "materialization_type")?, + ), + ("action", enum_value(entry, "action")?), + ("reason", enum_value(entry, "reason")?), + ( + "expected_version_hash", + entry.getattr("fingerprint_version_hash")?, + ), + ( + "built_version_hash", + entry.getattr("previous_version_hash")?, + ), + ( + "built_version_present", + any(py, !entry.getattr("previous_version_hash")?.is_none())?, + ), + ("identity_status", any(py, identity_status(entry)?)?), + ], + )?; + for key in ["incremental_strategy", "incremental_mode"] { + let value: Bound<'py, PyAny> = entry.getattr(key)?; + if !value.is_none() { + model.set_item(key, value)?; + } + } + if entry.getattr("incremental_mode")?.eq(MICROBATCH_MODE)? { + model.set_item("microbatch_state", microbatch_state(entry)?)?; + } + for key in ["cursor_column", "cursor_type"] { + let value: Bound<'py, PyAny> = entry.getattr(key)?; + if !value.is_none() { + model.set_item(key, value)?; + } + } + let bounds: Bound<'py, PyAny> = entry.getattr("cursor_bounds")?; + if !bounds.is_none() { + model.set_item("cursor_bounds", token_bounds(&bounds)?)?; + } + let future: Bound<'py, PyAny> = future_cursor_safety(&evidence(entry, "future_safety")?)?; + if !future.is_none() { + model.set_item("future_cursor_safety", future)?; + } + let maximum: Bound<'py, PyAny> = + maximum_start_safety(&evidence(entry, "maximum_start_safety")?)?; + if !maximum.is_none() { + model.set_item("maximum_start_safety", maximum)?; + } + if let Some(details) = cursor_details(entry)? { + model.set_item("cursor", cursor_json(py, &details)?)?; + } + let backfill: Bound<'py, PyAny> = entry.getattr("backfill")?; + model.set_item( + "backfill", + dict( + py, + vec![ + ("action", enum_value(&backfill, "action")?), + ("duration", backfill.getattr("duration")?), + ], + )?, + )?; + if entry.getattr("reason")?.eq(RENAMED_REASON)? + && entry.getattr("query_changed")?.is_truthy()? + { + model.set_item("query_changed", true)?; + } + let changed_functions: Bound<'py, PyAny> = entry.getattr("changed_functions")?; + if changed_functions.is_truthy()? { + model.set_item("changed_functions", list(&changed_functions)?)?; + } + let qualified: Bound<'py, PyAny> = entry.getattr("destination")?.getattr("qualified_name")?; + if !qualified.is_none() { + model.set_item("qualified_name", qualified)?; + } + let hints: Bound<'py, PyAny> = entry.getattr("column_rename_hints")?; + if hints.is_truthy()? { + model.set_item( + "column_rename_hints", + mapped(&hints, |hint| { + Ok(dict( + py, + vec![ + ("column", hint.getattr("added_column")?), + ("candidates", list(&hint.getattr("candidate_columns")?)?), + ("identical", hint.getattr("identical")?), + ("hint", hint.getattr("message")?), + ], + )? + .into_any()) + })?, + )?; + } + Ok(model.into_any()) +} + +fn with_qualified_name<'py>( + item: Bound<'py, PyDict>, + entry: &Bound<'py, PyAny>, +) -> PyResult> { + let qualified: Bound<'py, PyAny> = entry.getattr("destination")?.getattr("qualified_name")?; + if !qualified.is_none() { + item.set_item("qualified_name", qualified)?; + } + Ok(item.into_any()) +} + +fn seed_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = entry.py(); + with_qualified_name( + dict( + py, + vec![ + ("name", entry.getattr("name")?), + ("reason", enum_value(entry, "reason")?), + ], + )?, + entry, + ) +} + +fn source_load_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = entry.py(); + let load: Bound<'py, PyDict> = dict( + py, + vec![ + ("name", entry.getattr("name")?), + ("loader", entry.getattr("loader")?), + ("kind", enum_value(entry, "resource_kind")?), + ("target", entry.getattr("destination")?), + ("is_reload", entry.getattr("is_reload")?), + ], + )?; + let strategy: Bound<'py, PyAny> = entry.getattr("write_strategy")?; + if !strategy.is_none() { + load.set_item("write_strategy", strategy.getattr("value")?)?; + } + let cursor: Bound<'py, PyAny> = entry.getattr("cursor_column")?; + if !cursor.is_none() { + load.set_item("cursor_column", cursor)?; + } + let unique_key: Bound<'py, PyAny> = entry.getattr("unique_key")?; + if unique_key.is_truthy()? { + load.set_item("unique_key", unique_key)?; + } + Ok(load.into_any()) +} + +fn function_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = entry.py(); + let columns: Bound<'py, PyAny> = entry.getattr("return_columns")?; + with_qualified_name( + dict( + py, + vec![ + ("name", entry.getattr("name")?), + ( + "relative_path", + entry.getattr("relative_path")?.str()?.into_any(), + ), + ("language", enum_value(entry, "language")?), + ( + "return_kind", + any( + py, + if columns.is_truthy()? { + "table" + } else { + "scalar" + }, + )?, + ), + ("returns", entry.getattr("returns")?), + ( + "return_columns", + mapped(&columns, |column| { + Ok(dict( + py, + vec![ + ("name", column.getattr("name")?), + ("type", column.getattr("type")?), + ], + )? + .into_any()) + })?, + ), + ], + )?, + entry, + ) +} + +fn warning_json<'py>(warning: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = warning.py(); + let item: Bound<'py, PyDict> = dict( + py, + vec![ + ("severity", enum_value(warning, "severity")?), + ("message", warning.getattr("message")?), + ], + )?; + for key in ["model_name", "code"] { + let value: Bound<'py, PyAny> = warning.getattr(key)?; + if !value.is_none() { + item.set_item(key, value)?; + } + } + Ok(item.into_any()) +} + +/// `difflib.unified_diff` between two texts, each line without its newline. +fn unified_diff<'py>(py: Python<'py>, previous: &str, current: &str) -> PyResult> { + let lines = |text: &str| -> PyResult> { + PyString::new(py, text).call_method1("splitlines", (true,)) + }; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("fromfile", "previous")?; + keywords.set_item("tofile", "current")?; + let diff: Bound<'py, PyAny> = py + .import("difflib")? + .getattr("unified_diff")? + .call((lines(previous)?, lines(current)?), Some(&keywords))?; + let mut rendered: Vec = Vec::new(); + for line in diff.try_iter()? { + let line: String = line?.extract()?; + rendered.push(line.trim_end_matches('\n').to_owned()); + } + Ok(rendered) +} + +fn identity_diff<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = entry.py(); + let result: Bound<'py, PyDict> = PyDict::new(py); + if !entry.getattr("identity_status")?.eq(CHANGED_IDENTITY)? { + return Ok(result); + } + let pairs = [ + ( + "source_diff", + definition_source_text(&entry.getattr("previous_definition_json")?)?, + definition_source_text(&entry.getattr("current_definition_json")?)?, + ), + ( + "dependency_diff", + dependency_source_text(&entry.getattr("previous_metadata_json")?)?, + dependency_source_text(&entry.getattr("current_metadata_json")?)?, + ), + ]; + for (key, previous, current) in pairs { + if let (Some(previous), Some(current)) = (previous, current) + && previous != current + { + let diff: Vec = unified_diff(py, &previous, ¤t)?; + if !diff.is_empty() { + result.set_item(key, diff)?; + } + } + } + Ok(result) +} + +fn python_entry_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = entry.py(); + let item: Bound<'py, PyDict> = dict( + py, + vec![ + ("name", entry.getattr("name")?), + ("kind", enum_value(entry, "kind")?), + ("phase", enum_value(entry, "phase")?), + ("identity_status", enum_value(entry, "identity_status")?), + ], + )?; + let diff: Bound<'py, PyDict> = identity_diff(entry)?; + if !diff.is_empty() { + item.set_item("identity_diff", diff)?; + } + Ok(item.into_any()) +} + +/// One provider usage: `(provider, kind, name, parameter, annotation class, annotation module)`. +type Usage<'py> = ( + String, + (String, String, String), + Bound<'py, PyAny>, + Bound<'py, PyAny>, +); + +fn providers_json<'py>( + plan: &Bound<'py, PyAny>, + python_entries: &[Bound<'py, PyAny>], +) -> PyResult> { + let py: Python<'py> = plan.py(); + let mut usages: Vec> = Vec::new(); + for usage in items(&plan.getattr("provider_usages")?)? { + usages.push(( + text(&usage.getattr("provider_name")?)?, + ( + text(&usage.getattr("consumer_kind")?)?, + text(&usage.getattr("consumer_name")?)?, + text(&usage.getattr("parameter_name")?)?, + ), + usage.getattr("annotation_class_name")?, + usage.getattr("annotation_module")?, + )); + } + for entry in python_entries { + let kind: String = text(&enum_value(entry, "kind")?)?; + let name: String = text(&entry.getattr("name")?)?; + for usage in items(&entry.getattr("provider_usages")?)? { + usages.push(( + text(&usage.getattr("provider_name")?)?, + ( + kind.clone(), + name.clone(), + text(&usage.getattr("parameter_name")?)?, + ), + usage.getattr("annotation_class_name")?, + usage.getattr("annotation_module")?, + )); + } + } + usages.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1))); + let providers: Bound<'py, PyList> = PyList::empty(py); + let mut current: Option<(String, Bound<'py, PyList>)> = None; + for (provider, (kind, name, parameter), class_name, module) in usages { + if current.as_ref().is_none_or(|(open, _)| *open != provider) { + let used_by: Bound<'py, PyList> = PyList::empty(py); + providers.append(dict( + py, + vec![ + ("name", any(py, provider.as_str())?), + ("used_by", used_by.clone().into_any()), + ], + )?)?; + current = Some((provider, used_by)); + } + let usage: Bound<'py, PyDict> = dict( + py, + vec![ + ("kind", any(py, kind)?), + ("name", any(py, name)?), + ("parameter", any(py, parameter)?), + ], + )?; + if !class_name.is_none() || !module.is_none() { + usage.set_item( + "annotation", + dict(py, vec![("class_name", class_name), ("module", module)])?, + )?; + } + if let Some((_, used_by)) = ¤t { + used_by.append(usage)?; + } + } + Ok(providers.into_any()) +} + +fn old_name_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = entry.py(); + Ok(dict( + py, + vec![ + ("model", entry.getattr("model_name")?), + ("action", enum_value(entry, "action")?), + ("view", relation_name(&entry.getattr("origin")?)?), + ("reads", relation_name(&entry.getattr("destination")?)?), + ("expires_at", iso(&entry.getattr("expires_at")?)?), + ( + "column_aliases", + py.import("builtins")? + .getattr("dict")? + .call1((entry.getattr("column_aliases")?,))?, + ), + ("grants_copied", entry.getattr("grants_copied")?), + ("reason", entry.getattr("reason")?), + ], + )? + .into_any()) +} + +fn migrations_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = plan.py(); + let old_names: Bound<'py, PyDict> = PyDict::new(py); + for entry in items(&plan.getattr("old_name_view_entries")?)? { + old_names.set_item(entry.getattr("model_name")?, entry)?; + } + let materializations: Bound<'py, PyDict> = PyDict::new(py); + for entry in items(&plan.getattr("model_entries")?)? { + materializations.set_item( + entry.getattr("name")?, + entry.getattr("materialization_type")?.str()?, + )?; + } + mapped(&plan.getattr("migration_entries")?, |entry| { + let model: Bound<'py, PyAny> = entry.getattr("model_name")?; + let transfer: Bound<'py, PyAny> = entry.getattr("transfer")?; + let completed_at: Bound<'py, PyAny> = entry.getattr("completed_at")?; + let transfer_value: Bound<'py, PyAny> = if !transfer.is_none() { + transfer.getattr("value")? + } else if !entry.getattr("decision")?.eq(RENAMED_DECISION)? || !completed_at.is_none() { + py.None().into_bound(py) + } else { + let view: bool = materializations + .get_item(&model)? + .is_some_and(|kind| kind.eq(VIEW_MATERIALIZATION).unwrap_or(false)); + any(py, if view { "recreate" } else { "rebuild" })? + }; + let old_name: Bound<'py, PyAny> = match old_names.get_item(&model)? { + Some(old_name) => old_name_json(&old_name)?, + None => py.None().into_bound(py), + }; + Ok(dict( + py, + vec![ + ("kind", any(py, "model_migration")?), + ("model", model.clone()), + ("discovery", enum_value(entry, "discovery")?), + ("decision", enum_value(entry, "decision")?), + ("compatibility", enum_value(entry, "compatibility")?), + ( + "compatibility_findings", + list(&entry.getattr("compatibility_findings")?)?, + ), + ("origin_model", entry.getattr("origin_model")?), + ("origin", relation_name(&entry.getattr("origin")?)?), + ( + "destination", + relation_name(&entry.getattr("destination")?)?, + ), + ("target", entry.getattr("target_name")?), + ("origin_version_hash", entry.getattr("origin_version_hash")?), + ("transfer", transfer_value), + ( + "transfer_fallback", + optional_enum_value(&entry.getattr("transfer_fallback")?)?, + ), + ("storage_transition", entry.getattr("storage_transition")?), + ( + "promotion", + optional_enum_value(&entry.getattr("promotion")?)?, + ), + ("completed_at", iso(&completed_at)?), + ("old_name", old_name), + ], + )? + .into_any()) + }) +} + +fn column_migrations_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = plan.py(); + mapped(&plan.getattr("column_migration_entries")?, |entry| { + Ok(dict( + py, + vec![ + ("kind", any(py, "column_migration")?), + ("model", entry.getattr("model_name")?), + ("relation", relation_name(&entry.getattr("destination")?)?), + ("origin_column", entry.getattr("origin_column")?), + ("destination_column", entry.getattr("destination_column")?), + ("discovery", enum_value(entry, "discovery")?), + ("decision", enum_value(entry, "decision")?), + ("target", entry.getattr("target_name")?), + ("completed_at", iso(&entry.getattr("completed_at")?)?), + ], + )? + .into_any()) + }) +} + +fn retention_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = plan.py(); + mapped(&plan.getattr("retention_entries")?, |entry| { + let request: Bound<'py, PyAny> = entry.getattr("request")?; + Ok(dict( + py, + vec![ + ("scope", enum_value(&request, "scope")?), + ("database", request.getattr("database")?), + ("schema", request.getattr("schema")?), + ("name", request.getattr("name")?), + ("models", entry.getattr("model_names")?), + ("desired_days", request.getattr("desired_days")?), + ("actual_days", entry.getattr("actual_days")?), + ("effective_days", entry.getattr("effective_days")?), + ("source", entry.getattr("source")?), + ("direction", enum_value(entry, "direction")?), + ("phase", enum_value(entry, "phase")?), + ("statements", entry.getattr("statements")?), + ( + "irreversible_warning", + entry.getattr("irreversible_warning")?, + ), + ("decrease_policy", entry.getattr("decrease_policy")?), + ], + )? + .into_any()) + }) +} + +fn table_type_json<'py>(plan: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = plan.py(); + mapped(&plan.getattr("table_type_entries")?, |entry| { + Ok(dict( + py, + vec![ + ("kind", any(py, "table_type_conversion")?), + ("model", entry.getattr("model_name")?), + ("desired_type", entry.getattr("desired_type")?), + ("actual_type", entry.getattr("actual_type")?), + ("source", entry.getattr("source")?), + ("downgrade", entry.getattr("downgrade")?), + ("downgrade_policy", entry.getattr("downgrade_policy")?), + ( + "irreversible_warning", + entry.getattr("irreversible_warning")?, + ), + ], + )? + .into_any()) + }) +} + +/// The plan as one JSON document, `json.dumps(indent=2)`. +#[pyfunction] +#[pyo3(signature = (**options))] +fn format_plan_json(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = KeywordArguments::read( + py, + "format_plan_json", + options, + (&["plan"], &["python_plan_entries"]), + )?; + let plan: Bound<'_, PyAny> = arguments.value("plan")?; + let python_entries: Vec> = if arguments.passed("python_plan_entries")? { + items(&arguments.value("python_plan_entries")?)? + } else { + Vec::new() + }; + let length = |field: &str| -> PyResult { plan.getattr(field)?.len() }; + let python_nodes: Vec> = python_entries + .iter() + .map(python_entry_json) + .collect::>()?; + let result: Bound<'_, PyDict> = dict( + py, + vec![ + ( + "selected_count", + any( + py, + length("model_entries")? + + length("seed_entries")? + + length("function_entries")?, + )?, + ), + ( + "source_load_count", + any(py, length("source_load_entries")?)?, + ), + ("python_node_count", any(py, python_nodes.len())?), + ( + "models", + mapped(&plan.getattr("model_entries")?, model_json)?, + ), + ("seeds", mapped(&plan.getattr("seed_entries")?, seed_json)?), + ( + "source_loads", + mapped(&plan.getattr("source_load_entries")?, source_load_json)?, + ), + ( + "functions", + mapped(&plan.getattr("function_entries")?, function_json)?, + ), + ("python_nodes", PyList::new(py, python_nodes)?.into_any()), + ("providers", providers_json(&plan, &python_entries)?), + ( + "warnings", + mapped(&plan.getattr("warnings")?, warning_json)?, + ), + ("retention", retention_json(&plan)?), + ("table_type_conversions", table_type_json(&plan)?), + ("migrations", migrations_json(&plan)?), + ("column_migrations", column_migrations_json(&plan)?), + ( + "old_names", + mapped(&plan.getattr("old_name_view_entries")?, old_name_json)?, + ), + ], + )?; + let metadata: Bound<'_, PyAny> = plan.getattr("metadata")?; + if metadata.is_truthy()? { + result.set_item("metadata", metadata)?; + } + indented_json(result.as_any()) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_plan_json, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs new file mode 100644 index 0000000000..1feb11dc79 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_text.rs @@ -0,0 +1,531 @@ +//! The text plan: planner objects read by attribute into the native plan report. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_core::cli_text::main::format_plan_completion::format_plan_completion; +use sqlbuild_core::cli_text::main::format_plan_text::format_plan_text; +use sqlbuild_core::cli_text::models::{ + CursorPlanText, DisplayLimit, FunctionPlanText, ModelPlanText, PlanTextReport, PlanWarningText, + ProviderUsageText, PythonPlanText, RenameHintText, SchemaFindingText, SeedPlanText, + SourceFreshnessText, SourceLoadPlanText, +}; + +use crate::bindings::_helpers::compile_driver::python_calls::{ + KeywordArguments, call_with, imported, +}; +use crate::bindings::_helpers::plan_output::cursor_details::{ + CursorDetails, DEFERRED, RESOLVED, cursor_details, runtime_inputs_clause, +}; +use crate::bindings::_helpers::plan_output::python_identity::{ + definition_source_text, dependency_source_text, +}; +use crate::bindings::_helpers::plan_output::values::{items, optional_text, text, texts}; + +const QUERY_DIFF_MODULE: &str = "sqlbuild.compiler.planner.main.changes._query_diff"; +const NO_INPUT_ROWS_MODULE: &str = + "sqlbuild.compiler.planner.main.cursor_window.no_input_rows_display"; +const WAITING_MODULE: &str = + "sqlbuild.compiler.planner.main.cursor_window.waiting_on_empty_inputs_display"; +const BOUND_DISPLAY_MODULE: &str = "sqlbuild.compiler.planner.main.execution.cursor_bound_display"; +const INCLUSIVE_END_MODULE: &str = "sqlbuild.compiler.planner.main.execution.inclusive_cursor_end"; +const MATERIALIZATION_LABEL_MODULE: &str = + "sqlbuild.compiler.planner.main.execution._model_materialization_label"; +const PRE_BUILD_MODULE: &str = "sqlbuild.compiler.planner.main.pre_build.pre_build_work_display"; +const SENTINEL_TOKEN_MODULE: &str = "sqlbuild.cursor_algebra.main.sentinel_to_token"; +const DISPLAY_MODELS_MODULE: &str = "sqlbuild.presentation.models"; +const SKIP_ACTION: &str = "skip"; +const QUERY_CHANGED: &str = "query_changed"; +const CONFIG_CHANGED: &str = "config_changed"; +const MICROBATCH_MODE: &str = "microbatch"; +const CHANGED_IDENTITY: &str = "changed"; +const PLAN_OPTIONS: (&[&str], &[&str]) = ( + &["plan"], + &[ + "full_refresh", + "use_color", + "include_header", + "display_options", + "python_plan_entries", + "include_direct_freshness_diagnostics", + "include_completion", + ], +); + +/// `format_query_diff(previous=..., current=...)`. +pub(crate) fn query_diff( + previous: &Bound<'_, PyAny>, + current: &Bound<'_, PyAny>, +) -> PyResult> { + call_with( + &imported(previous.py(), QUERY_DIFF_MODULE, "format_query_diff")?, + &[("previous", previous.clone()), ("current", current.clone())], + )? + .extract() +} + +fn token<'py>(bound: Bound<'py, PyAny>) -> PyResult> { + call_with( + &imported(bound.py(), SENTINEL_TOKEN_MODULE, "sentinel_to_token")?, + &[("sentinel", bound)], + ) +} + +/// `start → inclusive end` of resolved bounds, in the model's cursor display. +fn cursor_range(entry: &Bound<'_, PyAny>, bounds: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = entry.py(); + let typing: [(&str, Bound<'_, PyAny>); 2] = [ + ("cursor_type", entry.getattr("cursor_type")?), + ("cursor_grain", entry.getattr("cursor_grain")?), + ]; + let start: String = call_with( + &imported(py, BOUND_DISPLAY_MODULE, "cursor_bound_display")?, + &[ + ("value", token(bounds.getattr("start")?)?), + typing[0].clone(), + typing[1].clone(), + ], + )? + .extract()?; + let end: String = call_with( + &imported(py, INCLUSIVE_END_MODULE, "inclusive_cursor_end")?, + &[ + ("end", token(bounds.getattr("end")?)?), + typing[0].clone(), + typing[1].clone(), + ], + )? + .extract()?; + Ok(format!("{start} \u{2192} {end}")) +} + +fn cursor_text(entry: &Bound<'_, PyAny>, details: CursorDetails<'_>) -> PyResult { + let py: Python<'_> = entry.py(); + let empty_inputs_window: Option = if details.empty_inputs.is_truthy()? { + let (module, function) = if details.waiting_on_empty_inputs { + (WAITING_MODULE, "format_waiting_on_empty_inputs") + } else { + (NO_INPUT_ROWS_MODULE, "format_no_input_rows") + }; + Some( + imported(py, module, function)? + .call1((&details.empty_inputs,))? + .extract()?, + ) + } else { + None + }; + let range: Option = details + .resolved_bounds + .as_ref() + .map(|bounds| cursor_range(entry, bounds)) + .transpose()?; + Ok(CursorPlanText { + column: optional_text(&entry.getattr("cursor_column")?)?.unwrap_or_default(), + cursor_type: optional_text(&entry.getattr("cursor_type")?)?, + microbatch: entry.getattr("incremental_mode")?.eq(MICROBATCH_MODE)?, + requested_start: details.requested_start, + requested_end: details.requested_end, + empty_inputs_window, + waiting_on_empty_inputs: details.waiting_on_empty_inputs, + range, + declared_grain: details.declared_grain, + effective_grain: details.effective_grain, + declared_batch_size: details.declared_batch_size, + effective_batch_size: details.effective_batch_size, + planned_batch_count: details.planned_batch_count, + runtime_owned: details.runtime_owned, + resolved: details.resolution_status == RESOLVED, + deferred: details.resolution_status == DEFERRED, + runtime_inputs_clause: runtime_inputs_clause(entry)?, + batch_concurrency: text(&entry.getattr("batch_concurrency")?)?, + unaccounted_partition_policy: optional_text( + &entry.getattr("unaccounted_partition_policy")?, + )?, + }) +} + +/// `json.dumps(json.loads(metadata).get("config", {}), sort_keys=True, indent=2, default=str)`. +fn config_json<'py>(metadata_json: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'_> = metadata_json.py(); + let json: Bound<'_, PyModule> = py.import("json")?; + let payload: Bound<'_, PyAny> = match json.call_method1("loads", (metadata_json,)) { + Ok(payload) => payload, + Err(error) if error.is_instance(py, &json.getattr("JSONDecodeError")?) => { + return Ok(metadata_json.clone()); + } + Err(error) => return Err(error), + }; + let config: Bound<'_, PyAny> = match payload.downcast::() { + Ok(payload) => payload + .get_item("config")? + .unwrap_or_else(|| PyDict::new(py).into_any()), + Err(_) => PyDict::new(py).into_any(), + }; + let keywords: Bound<'_, PyDict> = PyDict::new(py); + keywords.set_item("sort_keys", true)?; + keywords.set_item("indent", 2)?; + keywords.set_item("default", py.import("builtins")?.getattr("str")?)?; + json.getattr("dumps")?.call((config,), Some(&keywords)) +} + +fn config_diff(entry: &Bound<'_, PyAny>, reason: &str) -> PyResult>> { + if !entry.getattr("config_changed")?.is_truthy()? && reason != CONFIG_CHANGED { + return Ok(None); + } + let (previous, current) = ( + entry.getattr("previous_metadata_json")?, + entry.getattr("fingerprint_metadata_json")?, + ); + if previous.is_none() || current.is_none() { + return Ok(None); + } + let (previous, current) = (config_json(&previous)?, config_json(¤t)?); + if previous.eq(¤t)? { + return Ok(None); + } + query_diff(&previous, ¤t).map(Some) +} + +fn model_text(entry: &Bound<'_, PyAny>) -> PyResult { + let py: Python<'_> = entry.py(); + let action: String = text(&entry.getattr("action")?.getattr("value")?)?; + let reason: String = text(&entry.getattr("reason")?.getattr("value")?)?; + let name: String = text(&entry.getattr("name")?)?; + if action == SKIP_ACTION { + return Ok(ModelPlanText { + name, + action, + reason, + ..ModelPlanText::default() + }); + } + let backfill: Bound<'_, PyAny> = entry.getattr("backfill")?; + let mut schema_findings: Vec = Vec::new(); + for finding in items(&entry.getattr("schema_findings")?)? { + schema_findings.push(SchemaFindingText { + kind: text(&finding.getattr("kind")?.getattr("value")?)?, + column_name: text(&finding.getattr("column_name")?)?, + expected_type: optional_text(&finding.getattr("expected_type")?)?, + actual_type: optional_text(&finding.getattr("actual_type")?)?, + }); + } + let mut rename_hints: Vec = Vec::new(); + for hint in items(&entry.getattr("column_rename_hints")?)? { + rename_hints.push(RenameHintText { + added_column: text(&hint.getattr("added_column")?)?, + message: text(&hint.getattr("message")?)?, + }); + } + let previous_query: Bound<'_, PyAny> = entry.getattr("previous_query_sql")?; + let query_diff_lines: Option> = if previous_query.is_none() + || (!entry.getattr("query_changed")?.is_truthy()? && reason != QUERY_CHANGED) + { + None + } else { + Some(query_diff( + &previous_query, + &entry.getattr("fingerprint_query_sql")?, + )?) + }; + Ok(ModelPlanText { + materialization_label: imported( + py, + MATERIALIZATION_LABEL_MODULE, + "model_materialization_label", + )? + .call1((entry,))? + .extract()?, + custom_materialization_name: optional_text(&entry.getattr("custom_materialization_name")?)?, + backfill_action: text(&backfill.getattr("action")?.getattr("value")?)?, + backfill_duration: optional_text(&backfill.getattr("duration")?)?, + cursor: cursor_details(entry)? + .map(|details| cursor_text(entry, details)) + .transpose()?, + changed_functions: texts(&entry.getattr("changed_functions")?)?, + schema_findings, + rename_hints, + config_diff: config_diff(entry, &reason)?, + query_diff: query_diff_lines, + name, + action, + reason, + }) +} + +fn function_text(entry: &Bound<'_, PyAny>) -> PyResult { + let reason: String = text(&entry.getattr("reason")?.getattr("value")?)?; + let previous: Bound<'_, PyAny> = entry.getattr("previous_query_sql")?; + let query_diff_lines: Option> = if reason == QUERY_CHANGED && !previous.is_none() { + Some(query_diff( + &previous, + &entry.getattr("fingerprint_query_sql")?, + )?) + } else { + None + }; + Ok(FunctionPlanText { + name: text(&entry.getattr("name")?)?, + kind: if entry.getattr("return_columns")?.is_truthy()? { + "table function".to_owned() + } else { + format!( + "{} udf", + text(&entry.getattr("language")?.getattr("value")?)? + ) + }, + reason, + query_diff: query_diff_lines, + }) +} + +fn source_load_text(entry: &Bound<'_, PyAny>) -> PyResult { + let strategy: Bound<'_, PyAny> = entry.getattr("write_strategy")?; + Ok(SourceLoadPlanText { + name: text(&entry.getattr("name")?)?, + resource_kind: text(&entry.getattr("resource_kind")?.getattr("value")?)?, + is_reload: entry.getattr("is_reload")?.is_truthy()?, + integration_kind: optional_text(&entry.getattr("integration_kind")?)?, + write_strategy: if strategy.is_none() { + None + } else { + Some(text(&strategy.getattr("value")?)?) + }, + cursor_column: optional_text(&entry.getattr("cursor_column")?)?, + unique_key: texts(&entry.getattr("unique_key")?)?, + }) +} + +/// Both identity diffs of a changed Python entry, each as `format_query_diff` lines. +fn python_text(entry: &Bound<'_, PyAny>) -> PyResult { + let identity_status: String = text(&entry.getattr("identity_status")?.getattr("value")?)?; + let mut diffs: [Vec; 2] = [Vec::new(), Vec::new()]; + if identity_status == CHANGED_IDENTITY { + let sources = [ + ( + definition_source_text(&entry.getattr("previous_definition_json")?)?, + definition_source_text(&entry.getattr("current_definition_json")?)?, + ), + ( + dependency_source_text(&entry.getattr("previous_metadata_json")?)?, + dependency_source_text(&entry.getattr("current_metadata_json")?)?, + ), + ]; + for (diff, sources) in diffs.iter_mut().zip(sources) { + if let (Some(previous), Some(current)) = sources + && previous != current + { + let py: Python<'_> = entry.py(); + *diff = query_diff( + &pyo3::types::PyString::new(py, &previous).into_any(), + &pyo3::types::PyString::new(py, ¤t).into_any(), + )?; + } + } + } + let [source_diff, dependency_diff] = diffs; + Ok(PythonPlanText { + name: text(&entry.getattr("name")?)?, + kind: text(&entry.getattr("kind")?.getattr("value")?)?, + phase: text(&entry.getattr("phase")?.getattr("value")?)?, + identity_status, + source_diff, + dependency_diff, + }) +} + +fn provider_usages( + plan: &Bound<'_, PyAny>, + python_entries: &[Bound<'_, PyAny>], +) -> PyResult> { + let mut usages: Vec = Vec::new(); + for usage in items(&plan.getattr("provider_usages")?)? { + usages.push(ProviderUsageText { + provider_name: text(&usage.getattr("provider_name")?)?, + consumer_kind: text(&usage.getattr("consumer_kind")?)?, + consumer_name: text(&usage.getattr("consumer_name")?)?, + parameter_name: text(&usage.getattr("parameter_name")?)?, + annotation_class_name: optional_text(&usage.getattr("annotation_class_name")?)?, + }); + } + for entry in python_entries { + let (kind, name) = ( + text(&entry.getattr("kind")?.getattr("value")?)?, + text(&entry.getattr("name")?)?, + ); + for usage in items(&entry.getattr("provider_usages")?)? { + usages.push(ProviderUsageText { + provider_name: text(&usage.getattr("provider_name")?)?, + consumer_kind: kind.clone(), + consumer_name: name.clone(), + parameter_name: text(&usage.getattr("parameter_name")?)?, + annotation_class_name: optional_text(&usage.getattr("annotation_class_name")?)?, + }); + } + } + Ok(usages) +} + +/// `str(item)` of a metadata tuple or list, else nothing. +fn metadata_names(value: Option>) -> PyResult> { + match value { + Some(value) if value.is_instance_of::() || value.is_instance_of::() => { + texts(&value) + } + _ => Ok(Vec::new()), + } +} + +/// The metadata `selection_diagnostics.enabled` flag, when it is a bool. +pub(crate) fn selection_diagnostics(metadata: &Bound<'_, PyAny>) -> PyResult> { + let diagnostics: Bound<'_, PyAny> = metadata.call_method1("get", ("selection_diagnostics",))?; + let Ok(diagnostics) = diagnostics.downcast::() else { + return Ok(None); + }; + match diagnostics.get_item("enabled")? { + Some(enabled) if enabled.is_instance_of::() => { + Ok(Some(enabled.is_truthy()?)) + } + _ => Ok(None), + } +} + +fn source_freshness(metadata: &Bound<'_, PyAny>) -> PyResult> { + let freshness: Bound<'_, PyAny> = metadata.call_method1("get", ("direct_source_freshness",))?; + let Ok(freshness) = freshness.downcast::() else { + return Ok(None); + }; + let names = |key: &str| -> PyResult> { metadata_names(freshness.get_item(key)?) }; + Ok(Some(SourceFreshnessText { + observed: names("observed_source_names")?, + changed: names("changed_source_names")?, + unchanged: names("unchanged_source_names")?, + unknown: names("unknown_source_names")?, + age_warnings: names("age_warning_source_names")?, + age_errors: names("age_error_source_names")?, + stale_models: names("stale_model_names")?, + blocked_models: names("blocked_model_names")?, + })) +} + +/// The names of a metadata tuple that are strings. +fn pruned_models(metadata: &Bound<'_, PyAny>) -> PyResult> { + let names: Bound<'_, PyAny> = metadata.call_method1("get", ("direct_pruned_model_names",))?; + if !names.is_instance_of::() { + return Ok(Vec::new()); + } + let mut kept: Vec = Vec::new(); + for name in items(&names)? { + if name.is_instance_of::() { + kept.push(name.extract()?); + } + } + Ok(kept) +} + +fn display_limit(options: &Bound<'_, PyAny>) -> PyResult { + Ok(DisplayLimit { + max_entries: options.getattr("max_entries_per_section")?.extract()?, + overflow_flag: text(&options.getattr("overflow_flag")?)?, + }) +} + +fn report(arguments: &KeywordArguments<'_>) -> PyResult { + let py: Python<'_> = arguments.value("plan")?.py(); + let plan: Bound<'_, PyAny> = arguments.value("plan")?; + let display_options: Bound<'_, PyAny> = { + let given: Bound<'_, PyAny> = arguments.value("display_options")?; + if given.is_none() { + imported(py, DISPLAY_MODELS_MODULE, "DisplayOptions")?.call0()? + } else { + given + } + }; + let python_entries: Vec> = if arguments.passed("python_plan_entries")? { + items(&arguments.value("python_plan_entries")?)? + } else { + Vec::new() + }; + let flag_or = |name: &str, default: bool| -> PyResult { + if arguments.passed(name)? { + arguments.flag(name) + } else { + Ok(default) + } + }; + let metadata: Bound<'_, PyAny> = plan.getattr("metadata")?; + let mut warnings: Vec = Vec::new(); + for warning in items(&plan.getattr("warnings")?)? { + warnings.push(PlanWarningText { + severity: text(&warning.getattr("severity")?.getattr("value")?)?, + message: text(&warning.getattr("message")?)?, + model_name: optional_text(&warning.getattr("model_name")?)?, + }); + } + let mut seeds: Vec = Vec::new(); + for seed in items(&plan.getattr("seed_entries")?)? { + seeds.push(SeedPlanText { + name: text(&seed.getattr("name")?)?, + reason: text(&seed.getattr("reason")?)?, + }); + } + let pre_build: Bound<'_, PyAny> = call_with( + &imported(py, PRE_BUILD_MODULE, "format_pre_build_work")?, + &[ + ("lines", PyList::empty(py).into_any()), + ("plan", plan.clone()), + ("display_options", display_options.clone()), + ], + )?; + Ok(PlanTextReport { + use_color: flag_or("use_color", true)?, + include_header: flag_or("include_header", true)?, + full_refresh: arguments.flag("full_refresh")?, + display: display_limit(&display_options)?, + include_freshness_diagnostics: flag_or("include_direct_freshness_diagnostics", true)?, + selection_diagnostics: selection_diagnostics(&metadata)?, + models: items(&plan.getattr("model_entries")?)? + .iter() + .map(model_text) + .collect::>()?, + functions: items(&plan.getattr("function_entries")?)? + .iter() + .map(function_text) + .collect::>()?, + seeds, + source_loads: items(&plan.getattr("source_load_entries")?)? + .iter() + .map(source_load_text) + .collect::>()?, + provider_usages: provider_usages(&plan, &python_entries)?, + python_entries: python_entries + .iter() + .map(python_text) + .collect::>()?, + warnings, + source_freshness: source_freshness(&metadata)?, + remaining_stale_models: metadata_names(Some( + metadata.call_method1("get", ("direct_remaining_stale_model_names",))?, + ))?, + pruned_models: pruned_models(&metadata)?, + pre_build_lines: pre_build.extract()?, + }) +} + +/// The text plan grouped by reason with inline detail, optionally closed by its completion line. +#[pyfunction] +#[pyo3(signature = (**options))] +fn format_plan(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "format_plan", options, PLAN_OPTIONS)?; + let report: PlanTextReport = report(&arguments)?; + let text: String = format_plan_text(&report); + if !arguments.flag("include_completion")? { + return Ok(text); + } + Ok(format!("{text}\n\n{}", format_plan_completion(&report))) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(format_plan, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs new file mode 100644 index 0000000000..edc5b1481a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/python_identity.rs @@ -0,0 +1,89 @@ +//! The source texts a changed Python node's identity diffs compare. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; + +const DEPENDENCY_HEADER_SEPARATOR: &str = " :: "; +const DEPENDENCY_HEADER_FALLBACK: &str = "dependency"; +const HEADER_FIELDS: [&str; 3] = ["source_path", "module", "qualname"]; + +/// The JSON object a recorded identity holds, or `None` when it is missing or not an object. +fn json_object<'py>(raw_json: &Bound<'py, PyAny>) -> PyResult>> { + let py: Python<'py> = raw_json.py(); + if raw_json.is_none() { + return Ok(None); + } + let json: Bound<'py, PyModule> = py.import("json")?; + let payload: Bound<'py, PyAny> = match json.call_method1("loads", (raw_json,)) { + Ok(payload) => payload, + Err(error) if error.is_instance(py, &json.getattr("JSONDecodeError")?) => return Ok(None), + Err(error) => return Err(error), + }; + let Ok(payload) = payload.downcast_into::() else { + return Ok(None); + }; + Ok(Some(payload)) +} + +/// `payload[key]` when it is a string. +fn string_field(payload: &Bound<'_, PyDict>, key: &str) -> PyResult> { + match payload.get_item(key)? { + Some(value) if value.is_instance_of::() => Ok(Some(value.extract()?)), + _ => Ok(None), + } +} + +/// The recorded definition's `source_text`. +pub(crate) fn definition_source_text(raw_json: &Bound<'_, PyAny>) -> PyResult> { + match json_object(raw_json)? { + Some(payload) => string_field(&payload, "source_text"), + None => Ok(None), + } +} + +/// Each dependency's source under a `# path :: module :: qualname` header, sorted by that header. +pub(crate) fn dependency_source_text(raw_json: &Bound<'_, PyAny>) -> PyResult> { + let Some(payload) = json_object(raw_json)? else { + return Ok(None); + }; + let Some(dependencies) = payload.get_item("dependencies")? else { + return Ok(None); + }; + let Ok(dependencies) = dependencies.downcast::() else { + return Ok(None); + }; + let mut keyed: Vec<([String; 3], Option>)> = Vec::new(); + for dependency in dependencies.iter() { + let Ok(dependency) = dependency.downcast_into::() else { + keyed.push((Default::default(), None)); + continue; + }; + let mut key: [String; 3] = Default::default(); + for (slot, field) in key.iter_mut().zip(HEADER_FIELDS) { + *slot = string_field(&dependency, field)?.unwrap_or_default(); + } + keyed.push((key, Some(dependency))); + } + keyed.sort_by(|left, right| left.0.cmp(&right.0)); + let mut blocks: Vec = Vec::new(); + for (key, dependency) in keyed { + let Some(dependency) = dependency else { + continue; + }; + let Some(source_text) = string_field(&dependency, "source_text")? else { + continue; + }; + let parts: Vec<&str> = key + .iter() + .map(String::as_str) + .filter(|part| !part.is_empty()) + .collect(); + let header: String = if parts.is_empty() { + DEPENDENCY_HEADER_FALLBACK.to_owned() + } else { + parts.join(DEPENDENCY_HEADER_SEPARATOR) + }; + blocks.push(format!("# {header}\n{source_text}")); + } + Ok(Some(blocks.join("\n\n"))) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs new file mode 100644 index 0000000000..ce31fcb06b --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/registration.rs @@ -0,0 +1,11 @@ +//! Register the text and JSON plan output and the cursor safety serializers. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::plan_output::{plan_json, plan_text, safety}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + plan_json::register(module)?; + plan_text::register(module)?; + safety::register(module) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/safety.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/safety.rs new file mode 100644 index 0000000000..0280c81335 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/safety.rs @@ -0,0 +1,178 @@ +//! Structured future-cursor and maximum-start safety evidence for plan and result JSON. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::types::PyDict; +use pyo3::{pyfunction, wrap_pyfunction}; + +use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; +use crate::bindings::_helpers::plan_output::values::{dict, enum_value, items}; + +const RENDER_MODULE: &str = "sqlbuild.cursor_algebra.main.render"; + +/// `render(value=value)`. +fn render<'py>(value: Bound<'py, PyAny>) -> PyResult> { + call_with( + &imported(value.py(), RENDER_MODULE, "render")?, + &[("value", value)], + ) +} + +/// `render(value=value)` when set, else `None`. +fn optional_render<'py>(value: Bound<'py, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(value); + } + render(value) +} + +fn bounds<'py>( + evidence: &Bound<'py, PyAny>, + (start, end): (&str, &str), +) -> PyResult> { + Ok(dict( + evidence.py(), + vec![ + ("start", render(evidence.getattr(start)?)?), + ("end", render(evidence.getattr(end)?)?), + ], + )? + .into_any()) +} + +/// Cap policy, bounds and physical input evidence of future-cursor safety, or `None`. +pub(crate) fn future_cursor_safety<'py>( + evidence: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = evidence.py(); + if evidence.is_none() { + return Ok(evidence.clone()); + } + let relation: Bound<'py, PyAny> = evidence.getattr("determining_relation")?; + let determining: Bound<'py, PyAny> = if relation.is_none() { + relation + } else { + dict( + py, + vec![ + ("relation", relation), + ( + "cursor_column", + evidence.getattr("determining_cursor_column")?, + ), + ], + )? + .into_any() + }; + let mut inputs: Vec> = Vec::new(); + for item in items(&evidence.getattr("inputs")?)? { + inputs.push(dict( + py, + vec![ + ("relation", item.getattr("relation")?), + ("cursor_column", item.getattr("cursor_column")?), + ("minimum", optional_render(item.getattr("minimum")?)?), + ("maximum", render(item.getattr("maximum")?)?), + ], + )?); + } + Ok(dict( + py, + vec![ + ("action", enum_value(evidence, "action")?), + ("max_distance", evidence.getattr("max_distance")?), + ( + "invocation_time", + render(evidence.getattr("invocation_time")?)?, + ), + ( + "discovered_bounds", + bounds(evidence, ("discovered_start", "discovered_end"))?, + ), + ( + "applied_bounds", + bounds(evidence, ("applied_start", "applied_end"))?, + ), + ( + "maximum_allowed_bounds", + bounds(evidence, ("maximum_allowed_start", "maximum_allowed_end"))?, + ), + ( + "future_start_detected", + evidence.getattr("future_start_detected")?, + ), + ( + "future_end_detected", + evidence.getattr("future_end_detected")?, + ), + ("determining_input", determining), + ("inputs", pyo3::types::PyList::new(py, inputs)?.into_any()), + ], + )? + .into_any()) +} + +/// Automatic-start policy and physical eligibility evidence, or `None`. +pub(crate) fn maximum_start_safety<'py>( + evidence: &Bound<'py, PyAny>, +) -> PyResult> { + let py: Python<'py> = evidence.py(); + if evidence.is_none() { + return Ok(evidence.clone()); + } + Ok(dict( + py, + vec![ + ("action", enum_value(evidence, "action")?), + ("max_ahead", evidence.getattr("max_ahead")?), + ( + "invocation_time", + render(evidence.getattr("invocation_time")?)?, + ), + ( + "physical_target_max", + render(evidence.getattr("physical_target_max")?)?, + ), + ( + "highest_eligible_target_max", + optional_render(evidence.getattr("highest_eligible_target_max")?)?, + ), + ( + "effective_start", + render(evidence.getattr("effective_start")?)?, + ), + ( + "maximum_allowed_start", + render(evidence.getattr("maximum_allowed_start")?)?, + ), + ( + "input", + dict( + py, + vec![ + ("relation", evidence.getattr("target_relation")?), + ("cursor_column", evidence.getattr("cursor_column")?), + ], + )? + .into_any(), + ), + ], + )? + .into_any()) +} + +/// Serialize future-cursor safety evidence; `None` stays `None`. +#[pyfunction] +fn serialize_future_cursor_safety<'py>(evidence: Bound<'py, PyAny>) -> PyResult> { + future_cursor_safety(&evidence) +} + +/// Serialize maximum automatic-start safety evidence; `None` stays `None`. +#[pyfunction] +fn serialize_maximum_start_safety<'py>(evidence: Bound<'py, PyAny>) -> PyResult> { + maximum_start_safety(&evidence) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(serialize_future_cursor_safety, module)?)?; + module.add_function(wrap_pyfunction!(serialize_maximum_start_safety, module)?) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs new file mode 100644 index 0000000000..e10c15f827 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/values.rs @@ -0,0 +1,95 @@ +//! Reading planner values by attribute and building the JSON documents' dictionaries. + +use pyo3::IntoPyObjectExt; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyList}; + +/// `str(value)`. +pub(crate) fn text(value: &Bound<'_, PyAny>) -> PyResult { + Ok(value.str()?.to_string()) +} + +/// `str(value)`, or `None`. +pub(crate) fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + text(value).map(Some) +} + +/// `str(item)` of every item. +pub(crate) fn texts(value: &Bound<'_, PyAny>) -> PyResult> { + value.try_iter()?.map(|item| text(&item?)).collect() +} + +/// Every item of an iterable. +pub(crate) fn items<'py>(value: &Bound<'py, PyAny>) -> PyResult>> { + value.try_iter()?.collect() +} + +/// `value..value`, the value of an enum attribute. +pub(crate) fn enum_value<'py>( + value: &Bound<'py, PyAny>, + name: &str, +) -> PyResult> { + value.getattr(name)?.getattr("value") +} + +/// `value.value` when `value` is set, else `None`. +pub(crate) fn optional_enum_value<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(value.clone()); + } + value.getattr("value") +} + +/// A dictionary of the entries, in order. +pub(crate) fn dict<'py>( + py: Python<'py>, + entries: Vec<(&str, Bound<'py, PyAny>)>, +) -> PyResult> { + let built: Bound<'py, PyDict> = PyDict::new(py); + for (key, value) in entries { + built.set_item(key, value)?; + } + Ok(built) +} + +/// Any Rust value as a Python object. +pub(crate) fn any<'py, T: IntoPyObjectExt<'py>>( + py: Python<'py>, + value: T, +) -> PyResult> { + value.into_bound_py_any(py) +} + +/// `list(value)`. +pub(crate) fn list<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + Ok(PyList::new(value.py(), items(value)?)?.into_any()) +} + +/// `[build(item) for item in value]`. +pub(crate) fn mapped<'py>( + value: &Bound<'py, PyAny>, + build: impl Fn(&Bound<'py, PyAny>) -> PyResult>, +) -> PyResult> { + let built: Vec> = items(value)?.iter().map(build).collect::>()?; + Ok(PyList::new(value.py(), built)?.into_any()) +} + +/// `value.qualified_name or value.name`. +pub(crate) fn relation_name<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let qualified: Bound<'py, PyAny> = value.getattr("qualified_name")?; + if qualified.is_truthy()? { + return Ok(qualified); + } + value.getattr("name") +} + +/// `value.isoformat()` when set, else `None`. +pub(crate) fn iso<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(value.clone()); + } + value.call_method0("isoformat") +} diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index bc99bd56b7..bbae1a8936 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -7,8 +7,8 @@ use crate::bindings::_helpers::{ analysis, analysis_session, attachments, boundary, cache, cli, compile_attachment, compile_command, compile_driver, compile_outputs, compiled_project, contracts, dag_artifact, discovery, fingerprints, frontier, graph, lineage, macro_bridge, manifest, model_config, - project_assembly, refactor, render, rules, scopes, semantic_checks, semantic_shapes, - sql_test_glue, sqltext, type_system, + plan_output, project_assembly, refactor, render, rules, scopes, semantic_checks, + semantic_shapes, sql_test_glue, sqltext, type_system, }; use crate::bindings::models; @@ -45,6 +45,7 @@ fn register_project_stages(module: &Bound<'_, PyModule>) -> PyResult<()> { cache::registration::register(module)?; attachments::registration::register(module)?; refactor::registration::register(module)?; + plan_output::registration::register(module)?; Ok(()) } diff --git a/python-boundary.toml b/python-boundary.toml index 4289ba5037..35d97952bf 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -2,7 +2,7 @@ # migration work that moves to Rust. `remaining_lines` must equal the current count of that work: # a PR that moves code lowers it, and adding movable Python fails the check. # Native failures must surface as errors; `handled_native_errors` lists the reviewed exceptions. -remaining_lines = 118042 +remaining_lines = 115330 [[stays]] path = "executor" diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 9e98664bf8..bcb105ed5f 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -119,7 +119,12 @@ from sqlbuild.compiler.pipeline.models import ( PythonRunPlanOutputs, StaticCommandContext, ) -from sqlbuild.compiler.planner.models import DeferralInputs, PlanOutput +from sqlbuild.compiler.planner.models import ( + DeferralInputs, + FutureCursorSafetyEvidence, + MaximumStartSafetyEvidence, + PlanOutput, +) from sqlbuild.compiler.python_nodes.models import ( PythonNodeGraph, PythonNodeIdentity, @@ -156,6 +161,7 @@ from sqlbuild.compiler.sql_test_glue.models import ( from sqlbuild.compiler.sql_test_glue.types import NativeSqlTestAssemblyRow, NativeSqlTestPlanRow from sqlbuild.lint.types import RelationKeyRow from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter +from sqlbuild.presentation.models import DisplayOptions from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.spec.contracts.models import ( @@ -1022,6 +1028,26 @@ class RecordedCompileOutput: @property def stdout(self) -> str | None: ... +def format_plan( + *, + plan: PlanOutput, + full_refresh: bool = False, + use_color: bool = True, + include_header: bool = True, + display_options: DisplayOptions | None = None, + python_plan_entries: tuple[PythonPlanEntry, ...] = (), + include_direct_freshness_diagnostics: bool = True, + include_completion: bool = False, +) -> str: ... +def format_plan_json( + *, plan: PlanOutput, python_plan_entries: tuple[PythonPlanEntry, ...] = () +) -> str: ... +def serialize_future_cursor_safety( + evidence: FutureCursorSafetyEvidence | None, +) -> dict[str, object] | None: ... +def serialize_maximum_start_safety( + evidence: MaximumStartSafetyEvidence | None, +) -> dict[str, object] | None: ... def write_reusable_compile( *, attempt: CompileReuseAttempt, diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/outputs.py b/src/sqlbuild/cli/commands/_helpers/build_execution/outputs.py index 200bc8e58c..bceb25f04e 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/build_execution/outputs.py @@ -6,6 +6,7 @@ import sys from typing import TextIO +from sqlbuild._native import format_plan from sqlbuild.cli.commands._helpers.build_python_nodes.python_node_output import ( python_node_results_failed, ) @@ -22,7 +23,6 @@ degraded_execution_json, ) from sqlbuild.cli.output.main._write_execution_json_output import write_execution_json_output -from sqlbuild.cli.output.main.plan import format_plan from sqlbuild.cli.target_artifacts.main._write_python_check_runtime_target import ( write_python_check_runtime_target, ) diff --git a/src/sqlbuild/cli/commands/_helpers/plan/outputs.py b/src/sqlbuild/cli/commands/_helpers/plan/outputs.py index 95fda411d1..c513c58271 100644 --- a/src/sqlbuild/cli/commands/_helpers/plan/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/plan/outputs.py @@ -2,9 +2,8 @@ from __future__ import annotations +from sqlbuild._native import format_plan, format_plan_json from sqlbuild.cli.commands.models import PlanCommandRequest, PlanInvocation -from sqlbuild.cli.output.main._plan_json import format_plan_json -from sqlbuild.cli.output.main.plan import format_plan from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.presentation.models import DisplayOptions diff --git a/src/sqlbuild/cli/output/_helpers/cursor_plan.py b/src/sqlbuild/cli/output/_helpers/cursor_plan.py deleted file mode 100644 index a18d1a9b6a..0000000000 --- a/src/sqlbuild/cli/output/_helpers/cursor_plan.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Shared cursor-plan projection for text and JSON output.""" - -from __future__ import annotations - -from datetime import datetime - -from sqlbuild.cli.output.models import CursorPlanDetails -from sqlbuild.cli.output.types import CursorBoundsOwner, CursorResolutionStatus -from sqlbuild.compiler.planner.main.execution.aligned_timestamp_bounds import ( - aligned_timestamp_bounds, -) -from sqlbuild.compiler.planner.main.execution.effective_microbatch_batch_size import ( - resolve_effective_microbatch_batch_size, -) -from sqlbuild.compiler.planner.models import ( - CursorBounds, - CursorInputRelation, - Duration, - ModelPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - CursorGrain, - CursorType, - IncrementalMode, - MicrobatchStrategy, -) -from sqlbuild.cursor_algebra.constants import GRAIN_ORDER -from sqlbuild.cursor_algebra.models import DateValue, IntegerValue, TimestampValue -from sqlbuild.cursor_algebra.types import BoundSentinel -from sqlbuild.spec.contracts.constants import EFFECTIVE_BATCH_SIZE_TOKEN - - -def build_cursor_plan_details(*, entry: ModelPlanEntry) -> CursorPlanDetails | None: - """Project one model entry into stable cursor details for operator output.""" - - if entry.cursor_column is None: - return None - runtime_owned: bool = any( - relation.is_runtime_owned for relation in entry.cursor_input_relations - ) and not (entry.start_cursor_override is not None and entry.end_cursor_override is not None) - resolved_bounds: CursorBounds | None = entry.microbatch_range or entry.cursor_bounds - if resolved_bounds is not None and ( - isinstance(resolved_bounds.start, BoundSentinel) - or isinstance(resolved_bounds.end, BoundSentinel) - ): - resolved_bounds = None - effective_grain: str | None = _effective_grain(entry=entry) - effective_batch_size: str | None = _effective_batch_size( - entry=entry, effective_grain=effective_grain - ) - planned_batch_count: int | None = None - if ( - entry.incremental_mode == IncrementalMode.MICROBATCH - and resolved_bounds is not None - and effective_batch_size is not None - and entry.cursor_type is not None - ): - planned_batch_count = _count_batches( - bounds=resolved_bounds, - batch_size=effective_batch_size, - cursor_type=entry.cursor_type, - ) - if entry.empty_cursor_inputs: - resolution_status: CursorResolutionStatus = CursorResolutionStatus.NO_INPUT_ROWS - elif resolved_bounds is not None: - resolution_status = CursorResolutionStatus.RESOLVED - elif runtime_owned: - resolution_status = CursorResolutionStatus.DEFERRED - else: - resolution_status = CursorResolutionStatus.UNAVAILABLE - return CursorPlanDetails( - requested_start=entry.start_cursor_override, - requested_end=entry.end_cursor_override, - bounds_owner=(CursorBoundsOwner.RUNTIME if runtime_owned else CursorBoundsOwner.PLANNER), - resolution_status=resolution_status, - resolved_bounds=resolved_bounds, - declared_grain=entry.cursor_grain, - effective_grain=effective_grain, - declared_batch_size=entry.batch_size, - effective_batch_size=effective_batch_size, - planned_batch_count=planned_batch_count, - empty_inputs=entry.empty_cursor_inputs, - waiting_on_empty_inputs=entry.waiting_on_empty_inputs, - ) - - -def _effective_grain(*, entry: ModelPlanEntry) -> str | None: - """Return the coarsest timestamp grain declared across a model and its inputs.""" - - if entry.cursor_type != CursorType.TIMESTAMP: - return entry.cursor_grain - effective: str = entry.cursor_grain or CursorGrain.SECOND - if entry.microbatch_strategy == MicrobatchStrategy.WATERMARK: - return effective - for relation in entry.cursor_input_relations: - relation_grain: str = relation.cursor_grain or CursorGrain.SECOND - if GRAIN_ORDER[CursorGrain(relation_grain)] > GRAIN_ORDER[CursorGrain(effective)]: - effective = relation_grain - return effective - - -def _effective_batch_size(*, entry: ModelPlanEntry, effective_grain: str | None) -> str | None: - """Return the batch size the current executor path will apply.""" - - batch_size: str | None = entry.batch_size - if batch_size is None or effective_grain is None: - return batch_size - return resolve_effective_microbatch_batch_size( - batch_size=batch_size, - effective_grain=effective_grain, - ) - - -def _count_batches(*, bounds: CursorBounds, batch_size: str, cursor_type: str) -> int | None: - """Count the windows the executor will create without materializing them.""" - - if cursor_type == CursorType.TIMESTAMP: - duration: Duration | None = Duration.parse(batch_size) - if duration is None: - return None - if not isinstance(bounds.start, DateValue | TimestampValue) or not isinstance( - bounds.end, DateValue | TimestampValue - ): - return None - current: datetime - end: datetime - current, end = aligned_timestamp_bounds(start=bounds.start, end=bounds.end) - count: int = 0 - while current < end: - current = min(duration.add_to(current), end) - count += 1 - return count - if cursor_type == CursorType.INTEGER: - if not isinstance(bounds.start, IntegerValue) or not isinstance(bounds.end, IntegerValue): - return None - start: int = bounds.start.value - end: int = bounds.end.value - try: - size: int = int(batch_size) - except ValueError: - return None - if size <= 0 or start >= end: - return 0 - return (end - start + size - 1) // size - return None - - -def runtime_cursor_inputs_clause(*, entry: ModelPlanEntry) -> str: - """Name the cursor inputs this run brings up to date before the model's range is known.""" - - labels: list[str] = [] - relation: CursorInputRelation - for relation in entry.cursor_input_relations: - if not relation.is_runtime_owned: - continue - name: str = relation.input_name or relation.relation - label: str = ( - f"{relation.input_kind.value} {name}" if relation.input_kind is not None else name - ) - if label not in labels: - labels.append(label) - if not labels: - return "once its inputs are up to date" - if len(labels) == 1: - return f"once {labels[0]} is up to date" - return f"once {', '.join(labels[:-1])} and {labels[-1]} are up to date" - - -def append_microbatch_plan_detail( - *, lines: list[str], details: CursorPlanDetails, entry: ModelPlanEntry -) -> list[str]: - """Append grain, batch size, and known-or-deferred batch count.""" - - if details.declared_grain is not None: - grain_text: str = details.declared_grain - resolved_grain: str = details.effective_grain or details.declared_grain - if resolved_grain != details.declared_grain: - grain_text = f"{details.declared_grain} -> {resolved_grain} (effective)" - lines.append(f" grain: {grain_text}") - if details.declared_batch_size is not None: - batch_size_text: str = details.declared_batch_size - if details.declared_batch_size == EFFECTIVE_BATCH_SIZE_TOKEN: - resolved_batch_size: str = details.effective_batch_size or "runtime" - batch_size_text = ( - resolved_batch_size - if details.effective_grain == details.declared_grain - else f"{details.declared_batch_size} -> {resolved_batch_size}" - ) - lines.append(f" batch size: {batch_size_text}") - if details.planned_batch_count is not None and details.effective_batch_size is not None: - lines.append(f" batches: {details.planned_batch_count} x {details.effective_batch_size}") - if details.resolution_status == CursorResolutionStatus.DEFERRED: - lines.append( - f" batches: counted at run time, {runtime_cursor_inputs_clause(entry=entry)}" - ) - lines.append(f" batch concurrency: {entry.batch_concurrency}") - if entry.unaccounted_partition_policy is not None: - lines.append(f" unaccounted partition policy: {entry.unaccounted_partition_policy}") - return lines diff --git a/src/sqlbuild/cli/output/_helpers/execution_result_document.py b/src/sqlbuild/cli/output/_helpers/execution_result_document.py index dfea846d8c..3bf914d28f 100644 --- a/src/sqlbuild/cli/output/_helpers/execution_result_document.py +++ b/src/sqlbuild/cli/output/_helpers/execution_result_document.py @@ -10,11 +10,10 @@ from pathlib import Path from typing import cast +from sqlbuild._native import serialize_future_cursor_safety, serialize_maximum_start_safety from sqlbuild.adapter.contract.models import LifeCycleEvent as StatementLifeCycleEvent from sqlbuild.adapter.contract.types import LifeCycleEventKind -from sqlbuild.cli.output._helpers.future_cursor_safety import serialize_future_cursor_safety from sqlbuild.cli.output._helpers.integration_result import _render_measurement_thresholds -from sqlbuild.cli.output._helpers.maximum_start_safety import serialize_maximum_start_safety from sqlbuild.cli.output.classes.terminal_event_index import ( TerminalEventIndex, current_terminal_event_index, diff --git a/src/sqlbuild/cli/output/_helpers/future_cursor_safety.py b/src/sqlbuild/cli/output/_helpers/future_cursor_safety.py deleted file mode 100644 index 4871624562..0000000000 --- a/src/sqlbuild/cli/output/_helpers/future_cursor_safety.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Structured future-cursor safety output.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.models import FutureCursorSafetyEvidence -from sqlbuild.cursor_algebra.main.render import render - - -def serialize_future_cursor_safety( - evidence: FutureCursorSafetyEvidence | None, -) -> dict[str, object] | None: - """Serialize cap policy, bounds, and physical input evidence.""" - - if evidence is None: - return None - return { - "action": evidence.action.value, - "max_distance": evidence.max_distance, - "invocation_time": render(value=evidence.invocation_time), - "discovered_bounds": { - "start": render(value=evidence.discovered_start), - "end": render(value=evidence.discovered_end), - }, - "applied_bounds": { - "start": render(value=evidence.applied_start), - "end": render(value=evidence.applied_end), - }, - "maximum_allowed_bounds": { - "start": render(value=evidence.maximum_allowed_start), - "end": render(value=evidence.maximum_allowed_end), - }, - "future_start_detected": evidence.future_start_detected, - "future_end_detected": evidence.future_end_detected, - "determining_input": ( - { - "relation": evidence.determining_relation, - "cursor_column": evidence.determining_cursor_column, - } - if evidence.determining_relation is not None - else None - ), - "inputs": [ - { - "relation": item.relation, - "cursor_column": item.cursor_column, - "minimum": render(value=item.minimum) if item.minimum is not None else None, - "maximum": render(value=item.maximum), - } - for item in evidence.inputs - ], - } diff --git a/src/sqlbuild/cli/output/_helpers/integration_result.py b/src/sqlbuild/cli/output/_helpers/integration_result.py index 6b854d4033..9cbd71605d 100644 --- a/src/sqlbuild/cli/output/_helpers/integration_result.py +++ b/src/sqlbuild/cli/output/_helpers/integration_result.py @@ -7,10 +7,9 @@ from dataclasses import replace from typing import Any, cast -from sqlbuild.cli.output._helpers.future_cursor_safety import serialize_future_cursor_safety +from sqlbuild._native import serialize_future_cursor_safety, serialize_maximum_start_safety from sqlbuild.cli.output._helpers.integration_identity import integration_resource_id from sqlbuild.cli.output._helpers.integration_validation import encode_integration_json -from sqlbuild.cli.output._helpers.maximum_start_safety import serialize_maximum_start_safety from sqlbuild.cli.output.constants import ( INTEGRATION_RESOURCE_FAILED_EVENT, INTEGRATION_RESULT_RECORD_KIND, diff --git a/src/sqlbuild/cli/output/_helpers/maximum_start_safety.py b/src/sqlbuild/cli/output/_helpers/maximum_start_safety.py deleted file mode 100644 index 4ea548354a..0000000000 --- a/src/sqlbuild/cli/output/_helpers/maximum_start_safety.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Structured maximum automatic-start safety output.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.models import MaximumStartSafetyEvidence -from sqlbuild.cursor_algebra.main.render import render - - -def serialize_maximum_start_safety( - evidence: MaximumStartSafetyEvidence | None, -) -> dict[str, object] | None: - """Serialize automatic-start policy and physical eligibility evidence.""" - - if evidence is None: - return None - return { - "action": evidence.action.value, - "max_ahead": evidence.max_ahead, - "invocation_time": render(value=evidence.invocation_time), - "physical_target_max": render(value=evidence.physical_target_max), - "highest_eligible_target_max": ( - render(value=evidence.highest_eligible_target_max) - if evidence.highest_eligible_target_max is not None - else None - ), - "effective_start": render(value=evidence.effective_start), - "maximum_allowed_start": render(value=evidence.maximum_allowed_start), - "input": { - "relation": evidence.target_relation, - "cursor_column": evidence.cursor_column, - }, - } diff --git a/src/sqlbuild/cli/output/_helpers/plan_json.py b/src/sqlbuild/cli/output/_helpers/plan_json.py deleted file mode 100644 index 24389270f5..0000000000 --- a/src/sqlbuild/cli/output/_helpers/plan_json.py +++ /dev/null @@ -1,552 +0,0 @@ -"""JSON presentation for plans.""" - -from __future__ import annotations - -import difflib -import json -from typing import cast - -from sqlbuild.cli.output._helpers.cursor_plan import build_cursor_plan_details -from sqlbuild.cli.output._helpers.future_cursor_safety import serialize_future_cursor_safety -from sqlbuild.cli.output._helpers.maximum_start_safety import serialize_maximum_start_safety -from sqlbuild.cli.output.models import CursorPlanDetails -from sqlbuild.compiler.migrations.types import MigrationDecision -from sqlbuild.compiler.pipeline.models import PythonPlanEntry -from sqlbuild.compiler.planner.models import ( - FunctionPlanEntry, - FutureCursorSafetyEvidence, - MaximumStartSafetyEvidence, - ModelMigrationPlanEntry, - ModelPlanEntry, - OldNameViewPlanEntry, - PlanOutput, - PlanProviderUsage, - PlanWarning, - SeedPlanEntry, - SourceLoadPlanEntry, -) -from sqlbuild.compiler.planner.types import IncrementalMode, MaterializationType, PlanReason -from sqlbuild.compiler.python_nodes.types import PythonIdentityStatus -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token - - -def format_plan_json( - *, plan: PlanOutput, python_plan_entries: tuple[PythonPlanEntry, ...] = () -) -> str: - """Serialize a PlanOutput to JSON.""" - - models: list[dict[str, object]] = [_serialize_model_entry(e) for e in plan.model_entries] - seeds: list[dict[str, object]] = [_serialize_seed_entry(e) for e in plan.seed_entries] - functions: list[dict[str, object]] = [ - _serialize_function_entry(e) for e in plan.function_entries - ] - source_loads: list[dict[str, object]] = [ - _serialize_source_load_entry(e) for e in plan.source_load_entries - ] - warnings: list[dict[str, object]] = [_serialize_warning(w) for w in plan.warnings] - retention: list[dict[str, object]] = [ - { - "scope": entry.request.scope.value, - "database": entry.request.database, - "schema": entry.request.schema, - "name": entry.request.name, - "models": entry.model_names, - "desired_days": entry.request.desired_days, - "actual_days": entry.actual_days, - "effective_days": entry.effective_days, - "source": entry.source, - "direction": entry.direction.value, - "phase": entry.phase.value, - "statements": entry.statements, - "irreversible_warning": entry.irreversible_warning, - "decrease_policy": entry.decrease_policy, - } - for entry in plan.retention_entries - ] - table_type_conversions: list[dict[str, object]] = [ - { - "kind": "table_type_conversion", - "model": entry.model_name, - "desired_type": entry.desired_type, - "actual_type": entry.actual_type, - "source": entry.source, - "downgrade": entry.downgrade, - "downgrade_policy": entry.downgrade_policy, - "irreversible_warning": entry.irreversible_warning, - } - for entry in plan.table_type_entries - ] - python_nodes: list[dict[str, object]] = [ - _serialize_python_plan_entry(entry) for entry in python_plan_entries - ] - providers: list[dict[str, object]] = _serialize_provider_usages( - plan=plan, - python_plan_entries=python_plan_entries, - ) - - result: dict[str, object] = { - "selected_count": len(plan.model_entries) - + len(plan.seed_entries) - + len(plan.function_entries), - "source_load_count": len(source_loads), - "python_node_count": len(python_nodes), - "models": models, - "seeds": seeds, - "source_loads": source_loads, - "functions": functions, - "python_nodes": python_nodes, - "providers": providers, - "warnings": warnings, - "retention": retention, - "table_type_conversions": table_type_conversions, - "migrations": _serialize_model_migrations(plan), - "column_migrations": _serialize_column_migrations(plan), - "old_names": [_serialize_old_name(entry) for entry in plan.old_name_view_entries], - } - if plan.metadata: - result["metadata"] = plan.metadata - return json.dumps(result, indent=2) - - -def _serialize_model_entry(entry: ModelPlanEntry) -> dict[str, object]: - """Serialize one ModelPlanEntry for plan JSON output.""" - - model: dict[str, object] = { - "name": entry.name, - "relative_path": str(entry.relative_path), - "materialization_type": entry.materialization_type.value, - "action": entry.action.value, - "reason": entry.reason.value, - "expected_version_hash": entry.fingerprint_version_hash, - "built_version_hash": entry.previous_version_hash, - "built_version_present": entry.previous_version_hash is not None, - "identity_status": _model_identity_status(entry), - } - - if entry.incremental_strategy is not None: - model["incremental_strategy"] = entry.incremental_strategy - if entry.incremental_mode is not None: - model["incremental_mode"] = entry.incremental_mode - if entry.incremental_mode == IncrementalMode.MICROBATCH: - microbatch_state: dict[str, object] = { - "completion_tracking": "universal", - "batch_concurrency": entry.batch_concurrency, - "unaccounted_partition_policy": entry.unaccounted_partition_policy, - "reconciliation": "runtime", - } - if entry.microbatch_limit is not None: - microbatch_state.update( - { - "limit": entry.microbatch_limit, - "count": entry.microbatch_limit_count, - "action": ( - entry.microbatch_limit_action.value - if entry.microbatch_limit_action is not None - else None - ), - "warning": entry.microbatch_limit_warning, - } - ) - model["microbatch_state"] = microbatch_state - if entry.cursor_column is not None: - model["cursor_column"] = entry.cursor_column - if entry.cursor_type is not None: - model["cursor_type"] = entry.cursor_type - if entry.cursor_bounds is not None: - model["cursor_bounds"] = { - "start": sentinel_to_token(sentinel=entry.cursor_bounds.start), - "end": sentinel_to_token(sentinel=entry.cursor_bounds.end), - } - future_safety: FutureCursorSafetyEvidence | None = ( - entry.microbatch_range.future_safety - if entry.microbatch_range is not None and entry.microbatch_range.future_safety is not None - else (entry.cursor_bounds.future_safety if entry.cursor_bounds is not None else None) - ) - serialized_future_safety: dict[str, object] | None = serialize_future_cursor_safety( - future_safety - ) - if serialized_future_safety is not None: - model["future_cursor_safety"] = serialized_future_safety - maximum_start_safety: MaximumStartSafetyEvidence | None = ( - entry.microbatch_range.maximum_start_safety - if entry.microbatch_range is not None - and entry.microbatch_range.maximum_start_safety is not None - else (entry.cursor_bounds.maximum_start_safety if entry.cursor_bounds is not None else None) - ) - serialized_maximum_start: dict[str, object] | None = serialize_maximum_start_safety( - maximum_start_safety - ) - if serialized_maximum_start is not None: - model["maximum_start_safety"] = serialized_maximum_start - cursor_details: CursorPlanDetails | None = build_cursor_plan_details(entry=entry) - if cursor_details is not None: - model["cursor"] = _serialize_cursor_details(details=cursor_details) - - model["backfill"] = { - "action": entry.backfill.action.value, - "duration": entry.backfill.duration, - } - - if entry.reason == PlanReason.RENAMED and entry.query_changed: - model["query_changed"] = True - if entry.changed_functions: - model["changed_functions"] = list(entry.changed_functions) - - if entry.destination.qualified_name is not None: - model["qualified_name"] = entry.destination.qualified_name - - if entry.column_rename_hints: - model["column_rename_hints"] = [ - { - "column": hint.added_column, - "candidates": list(hint.candidate_columns), - "identical": hint.identical, - "hint": hint.message, - } - for hint in entry.column_rename_hints - ] - - return model - - -def _serialize_cursor_details(*, details: CursorPlanDetails) -> dict[str, object]: - """Serialize requested, resolved, and deferred cursor-plan state.""" - - resolved_bounds: dict[str, str] | None = None - if details.resolved_bounds is not None: - resolved_bounds = { - "start": sentinel_to_token(sentinel=details.resolved_bounds.start), - "end": sentinel_to_token(sentinel=details.resolved_bounds.end), - } - return { - "requested_bounds": { - "start": details.requested_start, - "end": details.requested_end, - }, - "bounds_owner": details.bounds_owner.value, - "resolution_status": details.resolution_status.value, - "resolved_bounds": resolved_bounds, - "declared_grain": details.declared_grain, - "effective_grain": details.effective_grain, - "declared_batch_size": details.declared_batch_size, - "effective_batch_size": details.effective_batch_size, - "planned_batch_count": details.planned_batch_count, - "empty_inputs": list(details.empty_inputs), - "waiting_on_empty_inputs": details.waiting_on_empty_inputs, - } - - -def _model_identity_status(entry: ModelPlanEntry) -> str: - """Return a stable JSON status for expected-vs-built model identity.""" - - if entry.fingerprint_version_hash is None: - return "unknown" - if entry.previous_version_hash is None: - return "missing" - if entry.previous_version_hash == entry.fingerprint_version_hash: - return "current" - return "stale" - - -def _serialize_seed_entry(entry: SeedPlanEntry) -> dict[str, object]: - """Serialize one SeedPlanEntry.""" - - seed: dict[str, object] = {"name": entry.name, "reason": entry.reason.value} - if entry.destination.qualified_name is not None: - seed["qualified_name"] = entry.destination.qualified_name - return seed - - -def _serialize_source_load_entry(entry: SourceLoadPlanEntry) -> dict[str, object]: - source_load: dict[str, object] = { - "name": entry.name, - "loader": entry.loader, - "kind": entry.resource_kind.value, - "target": entry.destination, - "is_reload": entry.is_reload, - } - if entry.write_strategy is not None: - source_load["write_strategy"] = entry.write_strategy.value - if entry.cursor_column is not None: - source_load["cursor_column"] = entry.cursor_column - if entry.unique_key: - source_load["unique_key"] = entry.unique_key - return source_load - - -def _serialize_function_entry(entry: FunctionPlanEntry) -> dict[str, object]: - """Serialize one FunctionPlanEntry.""" - - function: dict[str, object] = { - "name": entry.name, - "relative_path": str(entry.relative_path), - "language": entry.language.value, - "return_kind": "table" if entry.return_columns else "scalar", - "returns": entry.returns, - "return_columns": [ - {"name": column.name, "type": column.type} for column in entry.return_columns - ], - } - if entry.destination.qualified_name is not None: - function["qualified_name"] = entry.destination.qualified_name - return function - - -def _serialize_warning(warning: PlanWarning) -> dict[str, object]: - """Serialize one PlanWarning.""" - - result: dict[str, object] = { - "severity": warning.severity.value, - "message": warning.message, - } - if warning.model_name is not None: - result["model_name"] = warning.model_name - if warning.code is not None: - result["code"] = warning.code - return result - - -def _serialize_python_plan_entry(entry: PythonPlanEntry) -> dict[str, object]: - result: dict[str, object] = { - "name": entry.name, - "kind": entry.kind.value, - "phase": entry.phase.value, - "identity_status": entry.identity_status.value, - } - identity_diff: dict[str, object] = _serialize_python_identity_diff(entry) - if identity_diff: - result["identity_diff"] = identity_diff - return result - - -def _serialize_python_identity_diff(entry: PythonPlanEntry) -> dict[str, object]: - if entry.identity_status != PythonIdentityStatus.CHANGED: - return {} - result: dict[str, object] = {} - source_diff: list[str] = _python_source_diff(entry) - if source_diff: - result["source_diff"] = source_diff - dependency_diff: list[str] = _python_dependency_diff(entry) - if dependency_diff: - result["dependency_diff"] = dependency_diff - return result - - -def _python_source_diff(entry: PythonPlanEntry) -> list[str]: - previous: str | None = _python_definition_source_text(entry.previous_definition_json) - current: str | None = _python_definition_source_text(entry.current_definition_json) - if previous is None or current is None or previous == current: - return [] - return _unified_diff(previous=previous, current=current) - - -def _python_dependency_diff(entry: PythonPlanEntry) -> list[str]: - previous: str | None = _python_dependency_source_text(entry.previous_metadata_json) - current: str | None = _python_dependency_source_text(entry.current_metadata_json) - if previous is None or current is None or previous == current: - return [] - return _unified_diff(previous=previous, current=current) - - -def _python_definition_source_text(raw_json: str | None) -> str | None: - payload: dict[str, object] | None = _json_object(raw_json) - if payload is None: - return None - source_text: object = payload.get("source_text") - return source_text if isinstance(source_text, str) else None - - -def _python_dependency_source_text(raw_json: str | None) -> str | None: - payload: dict[str, object] | None = _json_object(raw_json) - if payload is None: - return None - raw_dependencies: object = payload.get("dependencies") - if not isinstance(raw_dependencies, list): - return None - blocks: list[str] = [] - dependency: object - for dependency in sorted(raw_dependencies, key=_python_dependency_sort_key): - if not isinstance(dependency, dict): - continue - dependency_payload: dict[object, object] = cast(dict[object, object], dependency) - source_text: object = dependency_payload.get("source_text") - if not isinstance(source_text, str): - continue - source_path: object = dependency_payload.get("source_path") - module: object = dependency_payload.get("module") - qualname: object = dependency_payload.get("qualname") - header_parts: list[str] = [] - if isinstance(source_path, str) and source_path: - header_parts.append(source_path) - if isinstance(module, str) and module: - header_parts.append(module) - if isinstance(qualname, str) and qualname: - header_parts.append(qualname) - header: str = " :: ".join(header_parts) if header_parts else "dependency" - blocks.append(f"# {header}\n{source_text}") - return "\n\n".join(blocks) - - -def _python_dependency_sort_key(dependency: object) -> tuple[str, str, str]: - if not isinstance(dependency, dict): - return ("", "", "") - dependency_payload: dict[object, object] = cast(dict[object, object], dependency) - source_path: object = dependency_payload.get("source_path") - module: object = dependency_payload.get("module") - qualname: object = dependency_payload.get("qualname") - return ( - source_path if isinstance(source_path, str) else "", - module if isinstance(module, str) else "", - qualname if isinstance(qualname, str) else "", - ) - - -def _json_object(raw_json: str | None) -> dict[str, object] | None: - if raw_json is None: - return None - try: - payload: object = json.loads(raw_json) - except json.JSONDecodeError: - return None - return cast(dict[str, object], payload) if isinstance(payload, dict) else None - - -def _unified_diff(*, previous: str, current: str) -> list[str]: - return [ - line.rstrip("\n") - for line in difflib.unified_diff( - previous.splitlines(keepends=True), - current.splitlines(keepends=True), - fromfile="previous", - tofile="current", - ) - ] - - -def _serialize_provider_usages( - *, plan: PlanOutput, python_plan_entries: tuple[PythonPlanEntry, ...] -) -> list[dict[str, object]]: - usage_by_provider: dict[str, list[PlanProviderUsage]] = {} - usage: PlanProviderUsage - for usage in plan.provider_usages: - usage_by_provider.setdefault(usage.provider_name, []).append(usage) - python_entry: PythonPlanEntry - for python_entry in python_plan_entries: - for provider_usage in python_entry.provider_usages: - usage_by_provider.setdefault(provider_usage.provider_name, []).append( - PlanProviderUsage( - provider_name=provider_usage.provider_name, - consumer_kind=python_entry.kind.value, - consumer_name=python_entry.name, - parameter_name=provider_usage.parameter_name, - annotation_class_name=provider_usage.annotation_class_name, - annotation_module=provider_usage.annotation_module, - ) - ) - providers: list[dict[str, object]] = [] - for provider_name, usages in sorted(usage_by_provider.items()): - serialized_usages: list[dict[str, object]] = [] - for usage in sorted( - usages, - key=lambda item: (item.consumer_kind, item.consumer_name, item.parameter_name), - ): - serialized_usages.append(_serialize_provider_usage(usage)) - providers.append({"name": provider_name, "used_by": serialized_usages}) - return providers - - -def _serialize_provider_usage(usage: PlanProviderUsage) -> dict[str, object]: - payload: dict[str, object] = { - "kind": usage.consumer_kind, - "name": usage.consumer_name, - "parameter": usage.parameter_name, - } - if usage.annotation_class_name is not None or usage.annotation_module is not None: - payload["annotation"] = { - "class_name": usage.annotation_class_name, - "module": usage.annotation_module, - } - return payload - - -def _serialize_column_migrations(plan: PlanOutput) -> list[dict[str, object]]: - return [ - { - "kind": "column_migration", - "model": entry.model_name, - "relation": entry.destination.qualified_name or entry.destination.name, - "origin_column": entry.origin_column, - "destination_column": entry.destination_column, - "discovery": entry.discovery.value, - "decision": entry.decision.value, - "target": entry.target_name, - "completed_at": ( - entry.completed_at.isoformat() if entry.completed_at is not None else None - ), - } - for entry in plan.column_migration_entries - ] - - -def _serialize_old_name(entry: OldNameViewPlanEntry) -> dict[str, object]: - return { - "model": entry.model_name, - "action": entry.action.value, - "view": entry.origin.qualified_name or entry.origin.name, - "reads": entry.destination.qualified_name or entry.destination.name, - "expires_at": entry.expires_at.isoformat() if entry.expires_at is not None else None, - "column_aliases": dict(entry.column_aliases), - "grants_copied": entry.grants_copied, - "reason": entry.reason, - } - - -def _serialize_model_migrations(plan: PlanOutput) -> list[dict[str, object]]: - old_names: dict[str, OldNameViewPlanEntry] = { - entry.model_name: entry for entry in plan.old_name_view_entries - } - materializations: dict[str, str] = { - entry.name: str(entry.materialization_type) for entry in plan.model_entries - } - return [ - { - "kind": "model_migration", - "model": entry.model_name, - "discovery": entry.discovery.value, - "decision": entry.decision.value, - "compatibility": entry.compatibility.value, - "compatibility_findings": list(entry.compatibility_findings), - "origin_model": entry.origin_model, - "origin": entry.origin.qualified_name or entry.origin.name, - "destination": entry.destination.qualified_name or entry.destination.name, - "target": entry.target_name, - "origin_version_hash": entry.origin_version_hash, - "transfer": _migration_transfer( - entry=entry, materialization=materializations.get(entry.model_name) - ), - "transfer_fallback": ( - entry.transfer_fallback.value if entry.transfer_fallback is not None else None - ), - "storage_transition": entry.storage_transition, - "promotion": entry.promotion.value if entry.promotion is not None else None, - "completed_at": ( - entry.completed_at.isoformat() if entry.completed_at is not None else None - ), - "old_name": ( - _serialize_old_name(old_names[entry.model_name]) - if entry.model_name in old_names - else None - ), - } - for entry in plan.migration_entries - ] - - -def _migration_transfer( - *, entry: ModelMigrationPlanEntry, materialization: str | None -) -> str | None: - if entry.transfer is not None: - return entry.transfer.value - if entry.decision != MigrationDecision.RENAMED or entry.completed_at is not None: - return None - return "recreate" if materialization == MaterializationType.VIEW else "rebuild" diff --git a/src/sqlbuild/cli/output/_helpers/plan_text.py b/src/sqlbuild/cli/output/_helpers/plan_text.py deleted file mode 100644 index 329f94b679..0000000000 --- a/src/sqlbuild/cli/output/_helpers/plan_text.py +++ /dev/null @@ -1,1747 +0,0 @@ -"""Plan output formatting grouped by reason with inline detail.""" - -from __future__ import annotations - -import json -import re -from collections import Counter -from collections.abc import Callable, Sequence -from typing import cast - -from sqlbuild.cli.output._helpers.cursor_plan import ( - append_microbatch_plan_detail, - build_cursor_plan_details, - runtime_cursor_inputs_clause, -) -from sqlbuild.cli.output._helpers.selection_diagnostics import direct_selection_diagnostics_enabled -from sqlbuild.cli.output.models import CursorPlanDetails -from sqlbuild.cli.output.types import CursorBoundsOwner, CursorResolutionStatus, PlanRowKind -from sqlbuild.compiler.pipeline.models import PythonPlanEntry -from sqlbuild.compiler.planner.constants import SOURCE_FRESHNESS_UNKNOWN_WARNING_TITLE -from sqlbuild.compiler.planner.main.changes.query_diff import format_query_diff -from sqlbuild.compiler.planner.main.cursor_window.no_input_rows_display import format_no_input_rows -from sqlbuild.compiler.planner.main.cursor_window.waiting_on_empty_inputs_display import ( - format_waiting_on_empty_inputs, -) -from sqlbuild.compiler.planner.main.execution.cursor_bound_display import cursor_bound_display -from sqlbuild.compiler.planner.main.execution.inclusive_cursor_end import inclusive_cursor_end -from sqlbuild.compiler.planner.main.execution.model_materialization_label import ( - model_materialization_label, -) -from sqlbuild.compiler.planner.main.pre_build.pre_build_work_display import ( - format_pre_build_work, -) -from sqlbuild.compiler.planner.models import ( - ColumnRenameHint, - CursorBounds, - FunctionPlanEntry, - ModelPlanEntry, - PlanOutput, - PlanProviderUsage, - PlanWarning, - SchemaFinding, - SourceLoadPlanEntry, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - IncrementalMode, - PlanAction, - PlanReason, - SchemaChangeKind, - WarningSeverity, -) -from sqlbuild.compiler.python_nodes.types import PythonIdentityStatus, PythonRunPhase -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.aligned_name_value import format_aligned_name_value -from sqlbuild.presentation.main.append_overflow_line import append_overflow_line -from sqlbuild.presentation.main.completion_line import format_completion_line -from sqlbuild.presentation.main.count_header import count_header_style -from sqlbuild.presentation.main.resolve_name_column_width import resolve_name_column_width -from sqlbuild.presentation.main.surface_header import format_surface_header -from sqlbuild.presentation.main.tree_connector import tree_connector -from sqlbuild.presentation.main.visible_entries import visible_entries -from sqlbuild.presentation.models import DisplayOptions -from sqlbuild.presentation.types import CompletionState -from sqlbuild.runtime.contracts.types import ExecutionResourceKind - -_DIFF_HEADER_MARKER: str = "# " -_ENTRY_ROW_PREFIX: str = " " -_LEAF_KEY_SEPARATOR: str = ": " -_LEAF_ROW_PREFIX: str = " " -_NESTED_ROW_PREFIX: str = " " -_STALE_INPUT_WARNING_TITLE: str = "Stale inputs detected" -_TREE_EXEMPT_CHARS: frozenset[str] = frozenset({" ", "."}) - -_REASON_GROUP_ORDER: tuple[PlanReason, ...] = ( - PlanReason.FULL_REFRESH, - PlanReason.QUERY_CHANGED, - PlanReason.FUNCTION_CHANGED, - PlanReason.CONFIG_CHANGED, - PlanReason.SCHEMA_CHANGED, - PlanReason.FIRST_RUN, - PlanReason.RENAMED, -) - -_REASON_GROUP_LABELS: dict[PlanReason, str] = { - PlanReason.FULL_REFRESH: "Full refresh", - PlanReason.QUERY_CHANGED: "Query changed", - PlanReason.FUNCTION_CHANGED: "Function changed", - PlanReason.CONFIG_CHANGED: "Config changed", - PlanReason.SCHEMA_CHANGED: "Schema changed", - PlanReason.FIRST_RUN: "First run", - PlanReason.RENAMED: "Renamed", -} - -_ANSI_ESCAPE_PATTERN: re.Pattern[str] = re.compile(r"\033\[[0-9;]*m") -_SCHEMA_CHANGE_SYMBOLS: dict[SchemaChangeKind, str] = { - SchemaChangeKind.COLUMN_ADDED: "+", - SchemaChangeKind.COLUMN_REMOVED: "-", - SchemaChangeKind.COLUMN_TYPE_CHANGED: "~", -} - - -def format_plan( - *, - plan: PlanOutput, - full_refresh: bool = False, - use_color: bool = True, - include_header: bool = True, - display_options: DisplayOptions | None = None, - section_header_style: Callable[[str], str] | None = None, - python_plan_entries: tuple[PythonPlanEntry, ...] = (), - include_direct_freshness_diagnostics: bool = True, -) -> str: - """Format plan output grouped by reason with inline detail.""" - - lines: list[str] = [] - - resolved_display_options: DisplayOptions = display_options or DisplayOptions() - style: CliStyle = CliStyle(use_color=True) - resolved_section_header_style: Callable[[str], str] = count_header_style( - style=style, title_style=section_header_style or style.plan_section - ) - - active_model_entries: tuple[ModelPlanEntry, ...] = tuple( - entry for entry in plan.model_entries if entry.action != PlanAction.SKIP - ) - all_active_models_full_refresh: bool = bool(active_model_entries) and all( - entry.reason == PlanReason.FULL_REFRESH for entry in active_model_entries - ) - if all_active_models_full_refresh or (full_refresh and not active_model_entries): - lines = _format_full_refresh( - lines=lines, - plan=plan, - include_header=include_header, - display_options=resolved_display_options, - section_header_style=resolved_section_header_style, - python_plan_entries=python_plan_entries, - ) - lines = _format_warnings( - lines=lines, - plan=plan, - include_direct_freshness_diagnostics=(include_direct_freshness_diagnostics), - display_options=resolved_display_options, - ) - lines = format_pre_build_work( - lines=lines, plan=plan, display_options=resolved_display_options - ) - result: str = "\n".join(lines) - return result if use_color else _strip_ansi(result) - - active: list[ModelPlanEntry] = [e for e in plan.model_entries if e.action != PlanAction.SKIP] - selected_count: int = _selected_count(plan) - name_column_width: int = _resolve_name_column_width( - plan=plan, python_plan_entries=python_plan_entries - ) - - if include_header: - header: str = _plan_ready_header( - selected_count=selected_count, - source_load_entries=plan.source_load_entries, - python_plan_entries=python_plan_entries, - full_refresh=False, - ) - lines.append(format_surface_header(style=style, title="Plan ready", context=header)) - - if include_direct_freshness_diagnostics: - lines = _format_direct_source_freshness_metadata( - lines=lines, - plan=plan, - section_header_style=resolved_section_header_style, - display_options=resolved_display_options, - ) - lines = _format_direct_remaining_stale_metadata( - lines=lines, - plan=plan, - section_header_style=resolved_section_header_style, - display_options=resolved_display_options, - ) - lines = _format_provider_usages( - lines=lines, - plan=plan, - python_plan_entries=python_plan_entries, - display_options=resolved_display_options, - section_header_style=resolved_section_header_style, - ) - lines = _format_direct_pruned_metadata( - lines=lines, - plan=plan, - display_options=resolved_display_options, - skipped_header_style=style.muted, - ) - lines = format_pre_build_work(lines=lines, plan=plan, display_options=resolved_display_options) - - lines = _format_python_plan_entries( - lines=lines, - entries=_python_plan_entries_for_phase( - entries=python_plan_entries, phase=PythonRunPhase.PRE_SQL_INGRESS - ), - label="Python ingress", - name_column_width=name_column_width, - display_options=resolved_display_options, - section_header_style=resolved_section_header_style, - ) - - _format_source_loads( - lines=lines, - plan=plan, - name_column_width=name_column_width, - display_options=resolved_display_options, - section_header_style=resolved_section_header_style, - ) - - normal: list[ModelPlanEntry] = _collect_normal(active) - groups: dict[PlanReason, list[ModelPlanEntry]] = _group_by_reason(entries=active) - - lines = _format_changed_functions( - lines=lines, - plan=plan, - name_column_width=name_column_width, - display_options=resolved_display_options, - section_header_style=resolved_section_header_style, - ) - - reason: PlanReason - for reason in _REASON_GROUP_ORDER: - entries: list[ModelPlanEntry] | None = groups.get(reason) - if not entries: - continue - label: str = _REASON_GROUP_LABELS[reason] - lines.append("") - lines.append(resolved_section_header_style(f"{label} ({len(entries)})")) - entry: ModelPlanEntry - visible: Sequence[ModelPlanEntry] = visible_entries( - entries=entries, options=resolved_display_options - ) - for entry in visible: - lines = _format_detail_entry( - lines=lines, entry=entry, reason=reason, name_column_width=name_column_width - ) - lines = append_overflow_line( - lines=lines, - total_count=len(entries), - visible_count=len(visible), - indent=" ", - options=resolved_display_options, - ) - - if normal: - lines.append("") - lines = _format_routine_models_section( - lines=lines, - entries=normal, - name_column_width=name_column_width, - display_options=resolved_display_options, - section_header_style=resolved_section_header_style, - ) - - lines = _format_routine_functions( - lines=lines, - plan=plan, - name_column_width=name_column_width, - display_options=resolved_display_options, - section_header_style=resolved_section_header_style, - ) - - lines = _format_seeds( - lines=lines, - plan=plan, - display_options=resolved_display_options, - section_header_style=resolved_section_header_style, - ) - - lines = _format_python_plan_entries( - lines=lines, - entries=_python_plan_entries_for_phase( - entries=python_plan_entries, phase=PythonRunPhase.READ_SIDE - ), - label="Python read-side", - name_column_width=name_column_width, - display_options=resolved_display_options, - section_header_style=resolved_section_header_style, - ) - lines = _treeify_plan_lines(lines) - lines = _format_warnings( - lines=lines, - plan=plan, - include_direct_freshness_diagnostics=include_direct_freshness_diagnostics, - display_options=resolved_display_options, - ) - - output: str = "\n".join(lines) - return output if use_color else _strip_ansi(output) - - -def _format_full_refresh( - *, - lines: list[str], - plan: PlanOutput, - include_header: bool, - display_options: DisplayOptions, - section_header_style: Callable[[str], str], - python_plan_entries: tuple[PythonPlanEntry, ...], -) -> list[str]: - """Format the full refresh variant of plan output.""" - - selected_count: int = _selected_count(plan) - active: list[ModelPlanEntry] = [e for e in plan.model_entries if e.action != PlanAction.SKIP] - name_column_width: int = _resolve_name_column_width( - plan=plan, python_plan_entries=python_plan_entries - ) - - if include_header: - lines.append( - format_surface_header( - style=CliStyle(use_color=True), - title="Plan ready", - context=_plan_ready_header( - selected_count=selected_count, - source_load_entries=plan.source_load_entries, - python_plan_entries=python_plan_entries, - full_refresh=True, - ), - ) - ) - - lines = _format_provider_usages( - lines=lines, - plan=plan, - python_plan_entries=python_plan_entries, - display_options=display_options, - section_header_style=section_header_style, - ) - - lines = _format_python_plan_entries( - lines=lines, - entries=_python_plan_entries_for_phase( - entries=python_plan_entries, phase=PythonRunPhase.PRE_SQL_INGRESS - ), - label="Python ingress", - name_column_width=name_column_width, - display_options=display_options, - section_header_style=section_header_style, - ) - - _format_source_loads( - lines=lines, - plan=plan, - name_column_width=name_column_width, - display_options=display_options, - section_header_style=section_header_style, - ) - - _format_functions( - lines=lines, - plan=plan, - name_column_width=name_column_width, - display_options=display_options, - section_header_style=section_header_style, - ) - if lines: - lines.append("") - - counts: Counter[str] = Counter() - entry: ModelPlanEntry - for entry in active: - label: str = model_materialization_label(entry) - counts[label] += 1 - - lines.append(section_header_style(f"Full refresh ({len(active)})")) - count_label: str - count_value: int - for count_label, count_value in counts.most_common(): - lines.append(f" {count_value:>3} {count_label}") - - lines = _format_seeds( - lines=lines, - plan=plan, - display_options=display_options, - section_header_style=section_header_style, - ) - - lines = _format_python_plan_entries( - lines=lines, - entries=_python_plan_entries_for_phase( - entries=python_plan_entries, phase=PythonRunPhase.READ_SIDE - ), - label="Python read-side", - name_column_width=name_column_width, - display_options=display_options, - section_header_style=section_header_style, - ) - return lines - - -def format_plan_completion( - *, - plan: PlanOutput, - full_refresh: bool, - use_color: bool, - python_plan_entries: tuple[PythonPlanEntry, ...], -) -> str: - """Render the terminal completion line that closes text plan output.""" - - has_errors: bool = any(warning.severity == WarningSeverity.ERROR for warning in plan.warnings) - active_model_entries: tuple[ModelPlanEntry, ...] = tuple( - entry for entry in plan.model_entries if entry.action != PlanAction.SKIP - ) - all_active_models_full_refresh: bool = bool(active_model_entries) and all( - entry.reason == PlanReason.FULL_REFRESH for entry in active_model_entries - ) - return format_completion_line( - style=CliStyle(use_color=use_color), - state=CompletionState.WARN if has_errors else CompletionState.OK, - label="Plan complete with errors" if has_errors else "Plan complete", - summary=_plan_ready_header( - selected_count=_selected_count(plan), - source_load_entries=plan.source_load_entries, - python_plan_entries=python_plan_entries, - full_refresh=all_active_models_full_refresh - or (full_refresh and not active_model_entries), - ), - ) - - -def _selected_count(plan: PlanOutput) -> int: - """Count selected executable resources shown in plan output.""" - - return len(plan.model_entries) + len(plan.seed_entries) + len(plan.function_entries) - - -def _plan_ready_header( - *, - selected_count: int, - source_load_entries: tuple[SourceLoadPlanEntry, ...], - python_plan_entries: tuple[PythonPlanEntry, ...], - full_refresh: bool, -) -> str: - source_count: int = len(source_load_entries) - parts: list[str] = [] - if full_refresh: - parts.append("full refresh") - parts.append(f"{selected_count} selected") - if source_count: - source_noun: str = "source" if source_count == 1 else "sources" - action: str = "reload" if any(e.is_reload for e in source_load_entries) else "load" - source_label: str = f"{source_noun} to {action}" - parts.append(f"{source_count} {source_label}") - python_count: int = len(python_plan_entries) - if python_count: - node_noun: str = "node" if python_count == 1 else "nodes" - parts.append(f"{python_count} Python {node_noun}") - return ", ".join(parts) - - -def _python_plan_entries_for_phase( - *, entries: tuple[PythonPlanEntry, ...], phase: PythonRunPhase -) -> tuple[PythonPlanEntry, ...]: - return tuple(entry for entry in entries if entry.phase == phase) - - -def _format_provider_usages( - *, - lines: list[str], - plan: PlanOutput, - python_plan_entries: tuple[PythonPlanEntry, ...], - display_options: DisplayOptions, - section_header_style: Callable[[str], str], -) -> list[str]: - usages: tuple[PlanProviderUsage, ...] = _all_provider_usages( - plan=plan, - python_plan_entries=python_plan_entries, - ) - if not usages: - return lines - usage_by_provider: dict[str, list[PlanProviderUsage]] = {} - usage: PlanProviderUsage - for usage in usages: - usage_by_provider.setdefault(usage.provider_name, []).append(usage) - lines.append("") - lines.append(section_header_style("Providers")) - verbose: bool = display_options.max_entries_per_section is None - provider_names: list[str] = sorted(usage_by_provider) - style: CliStyle = CliStyle(use_color=True) - if not verbose: - provider_width: int = max(len(name) for name in provider_names) - for provider_name in provider_names: - count: int = len(usage_by_provider[provider_name]) - surface_word: str = "surface" if count == 1 else "surfaces" - padding: str = " " * max(0, provider_width - len(provider_name)) - lines.append( - f" {style.object_name(provider_name)}{padding} " - f"{style.muted(f'used by {count} selected Python {surface_word}')}" - ) - return lines - for provider_name in provider_names: - lines.append(f" {style.object_name(provider_name)}") - provider_usages: list[PlanProviderUsage] = sorted( - usage_by_provider[provider_name], - key=lambda item: (item.consumer_kind, item.consumer_name, item.parameter_name), - ) - visible: Sequence[PlanProviderUsage] = visible_entries( - entries=provider_usages, - options=display_options, - ) - for usage in visible: - annotation: str = ( - f" ({usage.annotation_class_name})" - if usage.annotation_class_name is not None - else "" - ) - lines.append( - f" {style.muted(usage.consumer_kind)} " - f"{style.object_name(usage.consumer_name)}{annotation}" - ) - lines = append_overflow_line( - lines=lines, - total_count=len(provider_usages), - visible_count=len(visible), - indent=" ", - options=display_options, - ) - return lines - - -def _all_provider_usages( - *, plan: PlanOutput, python_plan_entries: tuple[PythonPlanEntry, ...] -) -> tuple[PlanProviderUsage, ...]: - usages: list[PlanProviderUsage] = list(plan.provider_usages) - python_entry: PythonPlanEntry - for python_entry in python_plan_entries: - usages.extend( - PlanProviderUsage( - provider_name=provider_usage.provider_name, - consumer_kind=python_entry.kind.value, - consumer_name=python_entry.name, - parameter_name=provider_usage.parameter_name, - annotation_class_name=provider_usage.annotation_class_name, - annotation_module=provider_usage.annotation_module, - ) - for provider_usage in python_entry.provider_usages - ) - return tuple(usages) - - -def _format_direct_pruned_metadata( - *, - lines: list[str], - plan: PlanOutput, - display_options: DisplayOptions, - skipped_header_style: Callable[[str], str], -) -> list[str]: - raw_names: object = plan.metadata.get("direct_pruned_model_names") - if not isinstance(raw_names, tuple): - return lines - names: tuple[str, ...] = tuple(name for name in raw_names if isinstance(name, str)) - if not names: - return lines - lines.append("") - lines.append(skipped_header_style(f"Skipped current models ({len(names)} already up to date)")) - if display_options.max_entries_per_section is not None: - return lines - visible_names: Sequence[str] = visible_entries(entries=names, options=display_options) - name: str - name_column_width: int = resolve_name_column_width(names=names) - for name in visible_names: - lines.append( - _format_name_value_line( - name=name, value="up to date", name_column_width=name_column_width - ) - ) - lines = append_overflow_line( - lines=lines, - total_count=len(names), - visible_count=len(visible_names), - indent=" ", - options=display_options, - ) - return lines - - -def _format_python_plan_entries( - *, - lines: list[str], - entries: tuple[PythonPlanEntry, ...], - label: str, - name_column_width: int, - display_options: DisplayOptions, - section_header_style: Callable[[str], str], -) -> list[str]: - if not entries: - return lines - lines.append("") - lines.append(section_header_style(f"{label} ({len(entries)})")) - visible: Sequence[PythonPlanEntry] = visible_entries(entries=entries, options=display_options) - entry: PythonPlanEntry - for entry in visible: - lines.append( - _format_name_value_line( - name=entry.name, - value=f"{entry.kind.value} ({entry.identity_status.value})", - name_column_width=name_column_width, - ) - ) - lines = _append_python_identity_diff(lines=lines, entry=entry) - lines = append_overflow_line( - lines=lines, - total_count=len(entries), - visible_count=len(visible), - indent=" ", - options=display_options, - ) - return lines - - -def _append_python_identity_diff(*, lines: list[str], entry: PythonPlanEntry) -> list[str]: - if entry.identity_status != PythonIdentityStatus.CHANGED: - return lines - - source_diff: list[str] = _format_python_source_diff(entry) - dependency_diff: list[str] = _format_python_dependency_diff(entry) - if not source_diff and not dependency_diff: - return lines - - style: CliStyle = CliStyle(use_color=True) - lines.append(style.label(" python diff:")) - if source_diff: - lines.append(style.label(" source diff:")) - lines.extend(source_diff) - if dependency_diff: - lines.append(style.label(" dependency diff:")) - lines.extend(dependency_diff) - return lines - - -def _format_python_source_diff(entry: PythonPlanEntry) -> list[str]: - previous: str | None = _python_definition_source_text(entry.previous_definition_json) - current: str | None = _python_definition_source_text(entry.current_definition_json) - if previous is None or current is None or previous == current: - return [] - return _indent_diff( - lines=format_query_diff(previous=previous, current=current), extra_indent=" " - ) - - -def _format_python_dependency_diff(entry: PythonPlanEntry) -> list[str]: - previous: str | None = _python_dependency_source_text(entry.previous_metadata_json) - current: str | None = _python_dependency_source_text(entry.current_metadata_json) - if previous is None or current is None or previous == current: - return [] - return _dim_python_dependency_headers( - _indent_diff(lines=format_query_diff(previous=previous, current=current), extra_indent=" ") - ) - - -def _dim_python_dependency_headers(lines: list[str]) -> list[str]: - style: CliStyle = CliStyle(use_color=True) - result: list[str] = [] - line: str - for line in lines: - if _DIFF_HEADER_MARKER in _strip_ansi(line): - result.append(style.muted(line)) - else: - result.append(line) - return result - - -def _python_definition_source_text(raw_json: str | None) -> str | None: - payload: dict[str, object] | None = _json_object(raw_json) - if payload is None: - return None - source_text: object = payload.get("source_text") - return source_text if isinstance(source_text, str) else None - - -def _python_dependency_source_text(raw_json: str | None) -> str | None: - payload: dict[str, object] | None = _json_object(raw_json) - if payload is None: - return None - raw_dependencies: object = payload.get("dependencies") - if not isinstance(raw_dependencies, list): - return None - - dependency_blocks: list[str] = [] - dependency: object - for dependency in sorted(raw_dependencies, key=_python_dependency_sort_key): - if not isinstance(dependency, dict): - continue - dependency_payload: dict[object, object] = cast(dict[object, object], dependency) - source_text: object = dependency_payload.get("source_text") - if not isinstance(source_text, str): - continue - source_path: object = dependency_payload.get("source_path") - module: object = dependency_payload.get("module") - qualname: object = dependency_payload.get("qualname") - header_parts: list[str] = [] - if isinstance(source_path, str) and source_path: - header_parts.append(source_path) - if isinstance(module, str) and module: - header_parts.append(module) - if isinstance(qualname, str) and qualname: - header_parts.append(qualname) - header: str = " :: ".join(header_parts) if header_parts else "dependency" - dependency_blocks.append(f"# {header}\n{source_text}") - return "\n\n".join(dependency_blocks) - - -def _python_dependency_sort_key(dependency: object) -> tuple[str, str, str]: - if not isinstance(dependency, dict): - return ("", "", "") - dependency_payload: dict[object, object] = cast(dict[object, object], dependency) - source_path: object = dependency_payload.get("source_path") - module: object = dependency_payload.get("module") - qualname: object = dependency_payload.get("qualname") - return ( - source_path if isinstance(source_path, str) else "", - module if isinstance(module, str) else "", - qualname if isinstance(qualname, str) else "", - ) - - -def _json_object(raw_json: str | None) -> dict[str, object] | None: - if raw_json is None: - return None - try: - payload: object = json.loads(raw_json) - except json.JSONDecodeError: - return None - return cast(dict[str, object], payload) if isinstance(payload, dict) else None - - -def _indent_diff(*, lines: list[str], extra_indent: str) -> list[str]: - return [f"{extra_indent}{line}" for line in lines] - - -def _format_source_loads( - *, - lines: list[str], - plan: PlanOutput, - name_column_width: int, - display_options: DisplayOptions, - section_header_style: Callable[[str], str], -) -> None: - """Append the managed source loaders section.""" - - if not plan.source_load_entries: - return - loader_entries: tuple[SourceLoadPlanEntry, ...] = tuple( - entry - for entry in plan.source_load_entries - if entry.resource_kind == ExecutionResourceKind.LOADER - ) - source_entries: tuple[SourceLoadPlanEntry, ...] = tuple( - entry - for entry in plan.source_load_entries - if entry.resource_kind == ExecutionResourceKind.SOURCE - ) - lines = _format_load_entry_group( - lines=lines, - entries=loader_entries, - label="Loaders", - name_column_width=name_column_width, - display_options=display_options, - section_header_style=section_header_style, - ) - lines = _format_load_entry_group( - lines=lines, - entries=source_entries, - label="Sources", - name_column_width=name_column_width, - display_options=display_options, - section_header_style=section_header_style, - ) - - -def _format_load_entry_group( - *, - lines: list[str], - entries: tuple[SourceLoadPlanEntry, ...], - label: str, - name_column_width: int, - display_options: DisplayOptions, - section_header_style: Callable[[str], str], -) -> list[str]: - if not entries: - return lines - action: str = "reload" if any(entry.is_reload for entry in entries) else "load" - lines.append("") - lines.append(section_header_style(f"{label} to {action} ({len(entries)})")) - visible: Sequence[SourceLoadPlanEntry] = visible_entries( - entries=entries, options=display_options - ) - source_load_entry: SourceLoadPlanEntry - for source_load_entry in visible: - lines.append( - _format_name_value_line( - name=source_load_entry.name, - value=_source_load_label(source_load_entry), - name_column_width=name_column_width, - ) - ) - lines = append_overflow_line( - lines=lines, - total_count=len(entries), - visible_count=len(visible), - indent=" ", - options=display_options, - ) - return lines - - -def _source_load_label(entry: SourceLoadPlanEntry) -> str: - strategy: str = _source_load_strategy_label(entry) - details: list[str] = [] - if entry.cursor_column is not None: - details.append(f"cursor: {entry.cursor_column}") - if entry.unique_key: - key_text: str = ", ".join(entry.unique_key) - details.append(f"unique_key: {key_text}") - if not details: - return strategy - return f"{strategy} ({'; '.join(details)})" - - -def _source_load_strategy_label(entry: SourceLoadPlanEntry) -> str: - if entry.integration_kind is not None: - return f"external ({entry.integration_kind})" - if entry.write_strategy is not None: - return entry.write_strategy.value - return "self-managed" - - -def _collect_normal(entries: list[ModelPlanEntry]) -> list[ModelPlanEntry]: - """Collect entries that belong in the Normal aggregate section.""" - - result: list[ModelPlanEntry] = [] - entry: ModelPlanEntry - for entry in entries: - if entry.reason not in (PlanReason.NO_CHANGE, PlanReason.NORMAL_INCREMENTAL): - continue - result.append(entry) - return result - - -def _group_by_reason(*, entries: list[ModelPlanEntry]) -> dict[PlanReason, list[ModelPlanEntry]]: - """Group entries by reason, excluding normal/no-change entries.""" - - groups: dict[PlanReason, list[ModelPlanEntry]] = {} - entry: ModelPlanEntry - for entry in entries: - if entry.reason in (PlanReason.NO_CHANGE, PlanReason.NORMAL_INCREMENTAL): - continue - groups.setdefault(entry.reason, []).append(entry) - return groups - - -def _format_routine_models_section( - *, - lines: list[str], - entries: list[ModelPlanEntry], - name_column_width: int, - display_options: DisplayOptions, - section_header_style: Callable[[str], str], -) -> list[str]: - """Format routine model work by resource name.""" - - lines.append(section_header_style(f"Models ({len(entries)})")) - visible: Sequence[ModelPlanEntry] = visible_entries(entries=entries, options=display_options) - entry: ModelPlanEntry - for entry in visible: - lines.append( - _format_name_value_line( - name=entry.name, - value=model_materialization_label(entry), - name_column_width=name_column_width, - ) - ) - lines = _append_cursor_detail(lines=lines, entry=entry) - lines = append_overflow_line( - lines=lines, - total_count=len(entries), - visible_count=len(visible), - indent=" ", - options=display_options, - ) - return lines - - -def _format_detail_entry( - *, - lines: list[str], - entry: ModelPlanEntry, - reason: PlanReason, - name_column_width: int, -) -> list[str]: - """Format a per-model entry with action text and detail lines.""" - - if reason == PlanReason.FIRST_RUN: - mat_label: str = model_materialization_label(entry) - lines.append( - _format_name_value_line( - name=entry.name, value=mat_label, name_column_width=name_column_width - ) - ) - lines = _append_cursor_detail(lines=lines, entry=entry) - return lines - - action_text: str = CliStyle(use_color=True).accent(_action_text(entry)) - lines.append( - _format_name_value_line( - name=entry.name, - value=action_text, - name_column_width=name_column_width, - dim_value=False, - ) - ) - lines = _append_cursor_detail( - lines=lines, - entry=entry, - show_range=reason != PlanReason.FULL_REFRESH, - ) - lines = _append_policy_line(lines=lines, entry=entry) - if entry.changed_functions: - lines.append(f" cause: function {', '.join(entry.changed_functions)} changed") - lines = _append_schema_diff(lines=lines, entry=entry) - lines = _append_config_diff(lines=lines, entry=entry) - lines = _append_query_diff(lines=lines, entry=entry) - return lines - - -def _append_cursor_detail( - *, lines: list[str], entry: ModelPlanEntry, show_range: bool = True -) -> list[str]: - """Append cursor column, mode, and range detail lines.""" - - details: CursorPlanDetails | None = build_cursor_plan_details(entry=entry) - if details is None: - return lines - cursor_value: str = entry.cursor_column or "" - if entry.cursor_type is not None: - cursor_value = f"{cursor_value} ({entry.cursor_type})" - lines.append(f" cursor: {cursor_value}") - if entry.incremental_mode == IncrementalMode.MICROBATCH: - lines.append(f" mode: {IncrementalMode.MICROBATCH.value}") - if details.requested_start is not None or details.requested_end is not None: - requested_start: str = details.requested_start or "earliest available" - requested_end: str = details.requested_end or "latest available" - lines.append(f" requested: {requested_start} -> {requested_end}") - if details.empty_inputs and details.waiting_on_empty_inputs: - lines.append( - CliStyle(use_color=True).warning( - f" window: {format_waiting_on_empty_inputs(details.empty_inputs)}" - ) - ) - elif details.empty_inputs: - lines.append(f" window: {format_no_input_rows(details.empty_inputs)}") - if show_range and details.resolved_bounds is not None: - lines.append( - f" range: {_format_cursor_range(bounds=details.resolved_bounds, entry=entry)}" - ) - if entry.incremental_mode == IncrementalMode.MICROBATCH: - lines = append_microbatch_plan_detail(lines=lines, details=details, entry=entry) - if details.bounds_owner == CursorBoundsOwner.RUNTIME: - lines.append( - f" bounds: computed at run time, {runtime_cursor_inputs_clause(entry=entry)}" - ) - elif details.resolution_status == CursorResolutionStatus.RESOLVED: - lines.append(" bounds: planner-resolved") - return lines - - -def _format_cursor_range(*, bounds: CursorBounds, entry: ModelPlanEntry) -> str: - """Render a cursor range with an inclusive end bound.""" - - start: str = cursor_bound_display( - value=sentinel_to_token(sentinel=bounds.start), - cursor_type=entry.cursor_type, - cursor_grain=entry.cursor_grain, - ) - inclusive_end: str = inclusive_cursor_end( - end=sentinel_to_token(sentinel=bounds.end), - cursor_type=entry.cursor_type, - cursor_grain=entry.cursor_grain, - ) - return f"{start} \u2192 {inclusive_end}" - - -def _append_policy_line(*, lines: list[str], entry: ModelPlanEntry) -> list[str]: - """Append the policy line if a backfill policy triggered.""" - - if entry.backfill.action == BackfillAction.FORWARD_ONLY: - return lines - duration: str = entry.backfill.duration or "full" - policy_value: str = _backfill_value(action=entry.backfill.action, duration=duration) - if entry.reason in (PlanReason.QUERY_CHANGED, PlanReason.SCHEMA_CHANGED): - lines.append(f" policy: replay_on_change={policy_value}") - return lines - - -def _append_schema_diff(*, lines: list[str], entry: ModelPlanEntry) -> list[str]: - """Append schema diff lines if findings exist.""" - - if not entry.schema_findings and not entry.column_rename_hints: - return lines - style: CliStyle = CliStyle(use_color=True) - lines.append(style.label(" schema diff:")) - finding_lines: list[str] = _format_schema_findings(entry.schema_findings) - hinted: set[str] = set() - finding: SchemaFinding - line: str - for finding, line in zip(entry.schema_findings, finding_lines, strict=True): - lines.append(line) - if finding.kind != SchemaChangeKind.COLUMN_ADDED: - continue - hint: ColumnRenameHint - for hint in entry.column_rename_hints: - if hint.added_column.lower() == finding.column_name.lower(): - lines.append(f" {style.warning(hint.message)}") - hinted.add(hint.added_column.lower()) - lines.extend( - f" {style.warning(f'{hint.added_column}: {hint.message}')}" - for hint in entry.column_rename_hints - if hint.added_column.lower() not in hinted - ) - return lines - - -def _append_query_diff(*, lines: list[str], entry: ModelPlanEntry) -> list[str]: - """Append query diff lines if previous SQL is available.""" - - if entry.previous_query_sql is None: - return lines - if not entry.query_changed and entry.reason != PlanReason.QUERY_CHANGED: - return lines - style: CliStyle = CliStyle(use_color=True) - lines.append(style.label(" query diff:")) - lines.extend( - format_query_diff(previous=entry.previous_query_sql, current=entry.fingerprint_query_sql) - ) - return lines - - -def _append_config_diff(*, lines: list[str], entry: ModelPlanEntry) -> list[str]: - """Append version-identity config diff lines if metadata changed.""" - - if not entry.config_changed and entry.reason != PlanReason.CONFIG_CHANGED: - return lines - if entry.previous_metadata_json is None or entry.fingerprint_metadata_json is None: - return lines - previous_config: str = _format_config_json(entry.previous_metadata_json) - current_config: str = _format_config_json(entry.fingerprint_metadata_json) - if previous_config == current_config: - return lines - style: CliStyle = CliStyle(use_color=True) - lines.append(style.label(" config diff:")) - lines.extend(format_query_diff(previous=previous_config, current=current_config)) - return lines - - -def _format_config_json(metadata_json: str) -> str: - try: - payload: object = json.loads(metadata_json) - except json.JSONDecodeError: - return metadata_json - config: object = payload.get("config", {}) if isinstance(payload, dict) else {} - return json.dumps(config, sort_keys=True, indent=2, default=str) - - -def _action_text(entry: ModelPlanEntry) -> str: - """Human-readable action text for inline display.""" - - if entry.action == PlanAction.CREATE_VIEW: - return "recreate view" - if entry.action == PlanAction.CUSTOM: - materialization: str = entry.custom_materialization_name or "custom materialization" - return f"run {materialization}" - if entry.backfill.action == BackfillAction.BOUNDED and entry.backfill.duration is not None: - base: str = f"rebuild last {entry.backfill.duration}" - suffix: str = _schema_change_suffix(entry) - return f"{base}, {suffix}" if suffix else base - if entry.backfill.action == BackfillAction.FULL and entry.reason != PlanReason.FIRST_RUN: - suffix = _schema_change_suffix(entry) - return f"full rebuild, {suffix}" if suffix else "full rebuild" - return "continue forward" - - -def _schema_change_suffix(entry: ModelPlanEntry) -> str: - """Short suffix describing schema changes for inline display.""" - - if not entry.schema_findings: - return "" - kinds: set[SchemaChangeKind] = {f.kind for f in entry.schema_findings} - parts: list[str] = [] - if SchemaChangeKind.COLUMN_ADDED in kinds: - parts.append("add column") - if SchemaChangeKind.COLUMN_REMOVED in kinds: - parts.append("drop column") - if SchemaChangeKind.COLUMN_TYPE_CHANGED in kinds: - parts.append("type change") - return ", ".join(parts) - - -def _backfill_value(*, action: BackfillAction, duration: str) -> str: - """Format a backfill action as a policy value string.""" - - if action == BackfillAction.BOUNDED: - return f"bounded-{duration}" - return str(action) - - -def _format_seeds( - *, - lines: list[str], - plan: PlanOutput, - display_options: DisplayOptions, - section_header_style: Callable[[str], str], -) -> list[str]: - """Append the seeds section.""" - - if not plan.seed_entries: - return lines - lines.append("") - lines.append(section_header_style(f"Seeds ({len(plan.seed_entries)})")) - seed_entry: object - visible: Sequence[object] = visible_entries(entries=plan.seed_entries, options=display_options) - for seed_entry in visible: - reason: object | None = getattr(seed_entry, "reason", None) - reason_label: str = _seed_reason_label(reason) - suffix: str = f" ({reason_label})" if reason_label else "" - lines.append(f" {getattr(seed_entry, 'name', str(seed_entry))}{suffix}") - lines = append_overflow_line( - lines=lines, - total_count=len(plan.seed_entries), - visible_count=len(visible), - indent=" ", - options=display_options, - ) - return lines - - -def _seed_reason_label(reason: object | None) -> str: - if reason == PlanReason.FIRST_RUN: - return "first_run" - if reason == PlanReason.CONFIG_CHANGED: - return "seed_changed" - if reason == PlanReason.NO_CHANGE: - return "current" - return "" - - -def _format_functions( - *, - lines: list[str], - plan: PlanOutput, - name_column_width: int, - display_options: DisplayOptions, - section_header_style: Callable[[str], str], -) -> None: - """Append the functions section.""" - - lines = _format_changed_functions( - lines=lines, - plan=plan, - name_column_width=name_column_width, - display_options=display_options, - section_header_style=section_header_style, - ) - lines = _format_routine_functions( - lines=lines, - plan=plan, - name_column_width=name_column_width, - display_options=display_options, - section_header_style=section_header_style, - ) - - -def _format_changed_functions( - *, - lines: list[str], - plan: PlanOutput, - name_column_width: int, - display_options: DisplayOptions, - section_header_style: Callable[[str], str], -) -> list[str]: - """Append changed functions with details.""" - - if not plan.function_entries: - return lines - changed_entries: list[FunctionPlanEntry] = [ - entry for entry in plan.function_entries if entry.reason != PlanReason.NO_CHANGE - ] - if not changed_entries: - return lines - lines.append("") - lines.append(section_header_style(f"Changed functions ({len(changed_entries)})")) - function_entry: FunctionPlanEntry - visible_changed: Sequence[FunctionPlanEntry] = visible_entries( - entries=changed_entries, options=display_options - ) - for function_entry in visible_changed: - lines = _format_function_entry( - lines=lines, - function_entry=function_entry, - show_details=True, - name_column_width=name_column_width, - ) - lines = append_overflow_line( - lines=lines, - total_count=len(changed_entries), - visible_count=len(visible_changed), - indent=" ", - options=display_options, - ) - return lines - - -def _format_routine_functions( - *, - lines: list[str], - plan: PlanOutput, - name_column_width: int, - display_options: DisplayOptions, - section_header_style: Callable[[str], str], -) -> list[str]: - """Append routine functions by resource name.""" - - if not plan.function_entries: - return lines - unchanged_entries: list[FunctionPlanEntry] = [ - entry for entry in plan.function_entries if entry.reason == PlanReason.NO_CHANGE - ] - if not unchanged_entries: - return lines - lines.append("") - lines.append(section_header_style(f"Functions ({len(unchanged_entries)})")) - visible_unchanged: Sequence[FunctionPlanEntry] = visible_entries( - entries=unchanged_entries, options=display_options - ) - for function_entry in visible_unchanged: - lines = _format_function_entry( - lines=lines, - function_entry=function_entry, - show_details=False, - name_column_width=name_column_width, - ) - lines = append_overflow_line( - lines=lines, - total_count=len(unchanged_entries), - visible_count=len(visible_unchanged), - indent=" ", - options=display_options, - ) - return lines - - -def _format_function_entry( - *, - lines: list[str], - function_entry: FunctionPlanEntry, - show_details: bool, - name_column_width: int, -) -> list[str]: - """Append one function line and optional change details.""" - - function_kind: str = ( - "table function" - if function_entry.return_columns - else f"{function_entry.language.value} udf" - ) - lines.append( - _format_name_value_line( - name=function_entry.name, - value=function_kind, - name_column_width=name_column_width, - ) - ) - if not show_details: - return lines - if function_entry.reason == PlanReason.FIRST_RUN: - lines.append(" reason: first run") - elif function_entry.reason == PlanReason.FULL_REFRESH: - lines.append(" reason: full refresh") - elif function_entry.reason == PlanReason.QUERY_CHANGED: - if function_entry.previous_query_sql is not None: - style: CliStyle = CliStyle(use_color=True) - lines.append(style.label(" query diff:")) - lines.extend( - format_query_diff( - previous=function_entry.previous_query_sql, - current=function_entry.fingerprint_query_sql, - ) - ) - return lines - - -def _format_warnings( - *, - lines: list[str], - plan: PlanOutput, - include_direct_freshness_diagnostics: bool, - display_options: DisplayOptions, -) -> list[str]: - """Append the errors and warnings sections.""" - - selection_diagnostics: bool | None = direct_selection_diagnostics_enabled(plan) - include_stale_input_warnings: bool = ( - selection_diagnostics - if selection_diagnostics is not None - else include_direct_freshness_diagnostics - ) - diagnostics: list[PlanWarning] = [ - warning - for warning in plan.warnings - if warning.severity != WarningSeverity.INFO - and ( - include_stale_input_warnings - or not warning.message.startswith(_STALE_INPUT_WARNING_TITLE) - ) - and not ( - include_direct_freshness_diagnostics - and warning.message.startswith(SOURCE_FRESHNESS_UNKNOWN_WARNING_TITLE) - ) - ] - style: CliStyle = CliStyle(use_color=True) - lines = _format_diagnostic_section( - lines=lines, - entries=[entry for entry in diagnostics if entry.severity == WarningSeverity.ERROR], - title="Errors", - heading_style=style.error_strong, - line_style=style.error, - display_options=display_options, - ) - return _format_diagnostic_section( - lines=lines, - entries=[entry for entry in diagnostics if entry.severity != WarningSeverity.ERROR], - title="Warnings", - heading_style=style.warning_strong, - line_style=style.warning, - display_options=display_options, - ) - - -def _format_diagnostic_section( - *, - lines: list[str], - entries: list[PlanWarning], - title: str, - heading_style: Callable[[str], str], - line_style: Callable[[str], str], - display_options: DisplayOptions, -) -> list[str]: - """Append one severity-grouped plan diagnostic section.""" - - if not entries: - return lines - style: CliStyle = CliStyle(use_color=True) - lines.append("") - lines.append(heading_style(f"{title} ({len(entries)})")) - shown: Sequence[PlanWarning] = visible_entries(entries=entries, options=display_options) - warning: PlanWarning - warning_index: int - for warning_index, warning in enumerate(shown): - connector: str = tree_connector(style=style, last=warning_index == len(shown) - 1) - message_lines: list[str] = warning.message.split("\n") - if warning.model_name is not None: - lines.append(f"{connector} {style.object_name(warning.model_name)}") - message_index: int - for message_index, message_line in enumerate(message_lines): - child_connector: str = tree_connector( - style=style, - last=message_index == len(message_lines) - 1, - ) - lines.append(f" {child_connector} {line_style(message_line)}") - else: - lines.append(f"{connector} {line_style(message_lines[0])}") - continuation: str - for continuation in message_lines[1:]: - lines.append(f" {line_style(continuation)}") - return append_overflow_line( - lines=lines, - total_count=len(entries), - visible_count=len(shown), - indent=" ", - options=display_options, - ) - - -def _format_direct_source_freshness_metadata( - *, - lines: list[str], - plan: PlanOutput, - section_header_style: Callable[[str], str], - display_options: DisplayOptions, -) -> list[str]: - raw_metadata: object | None = plan.metadata.get("direct_source_freshness") - if not isinstance(raw_metadata, dict): - return lines - source_freshness_metadata: dict[str, object] = cast(dict[str, object], raw_metadata) - observed_source_names: tuple[str, ...] = _metadata_string_tuple( - source_freshness_metadata.get("observed_source_names") - ) - changed_source_names: tuple[str, ...] = _metadata_string_tuple( - source_freshness_metadata.get("changed_source_names") - ) - unchanged_source_names: tuple[str, ...] = _metadata_string_tuple( - source_freshness_metadata.get("unchanged_source_names") - ) - unknown_source_names: tuple[str, ...] = _metadata_string_tuple( - source_freshness_metadata.get("unknown_source_names") - ) - age_warning_source_names: tuple[str, ...] = _metadata_string_tuple( - source_freshness_metadata.get("age_warning_source_names") - ) - age_error_source_names: tuple[str, ...] = _metadata_string_tuple( - source_freshness_metadata.get("age_error_source_names") - ) - stale_model_names: tuple[str, ...] = _metadata_string_tuple( - source_freshness_metadata.get("stale_model_names") - ) - blocked_model_names: tuple[str, ...] = _metadata_string_tuple( - source_freshness_metadata.get("blocked_model_names") - ) - if not observed_source_names and not unknown_source_names: - return lines - style: CliStyle = CliStyle(use_color=True) - lines.append("") - lines.append(section_header_style("Source freshness")) - lines.append( - _source_freshness_count_line(style=style, label="observed", names=observed_source_names) - ) - if observed_source_names: - lines.append( - _source_freshness_set_line( - style=style, - label="observed set", - names=observed_source_names, - display_options=display_options, - ) - ) - lines.append( - _source_freshness_count_line( - style=style, label="changed", names=changed_source_names, warn_nonzero=True - ) - ) - if changed_source_names: - lines.append( - _source_freshness_set_line( - style=style, - label="changed set", - names=changed_source_names, - display_options=display_options, - warn=True, - ) - ) - lines.append( - _source_freshness_count_line(style=style, label="unchanged", names=unchanged_source_names) - ) - if unchanged_source_names: - lines.append( - _source_freshness_set_line( - style=style, - label="unchanged set", - names=unchanged_source_names, - display_options=display_options, - ) - ) - lines.append( - _source_freshness_count_line( - style=style, label="unknown", names=unknown_source_names, warn_nonzero=True - ) - ) - if unknown_source_names: - lines.append( - _source_freshness_set_line( - style=style, - label="unknown set", - names=unknown_source_names, - display_options=display_options, - warn=True, - ) - ) - if age_warning_source_names: - lines.append( - _source_freshness_set_line( - style=style, - label="age warnings", - names=age_warning_source_names, - display_options=display_options, - warn=True, - ) - ) - if age_error_source_names: - lines.append( - _source_freshness_set_line( - style=style, - label="age errors", - names=age_error_source_names, - display_options=display_options, - warn=True, - ) - ) - if stale_model_names: - lines.append( - _source_freshness_set_line( - style=style, - label="source-stale models", - names=stale_model_names, - display_options=display_options, - warn=True, - ) - ) - if blocked_model_names: - lines.append( - _source_freshness_set_line( - style=style, - label="source-blocked models", - names=blocked_model_names, - display_options=display_options, - warn=True, - ) - ) - return lines - - -def _format_direct_remaining_stale_metadata( - *, - lines: list[str], - plan: PlanOutput, - section_header_style: Callable[[str], str], - display_options: DisplayOptions, -) -> list[str]: - if direct_selection_diagnostics_enabled(plan) is False: - return lines - remaining_stale_model_names: tuple[str, ...] = _metadata_string_tuple( - plan.metadata.get("direct_remaining_stale_model_names") - ) - if not remaining_stale_model_names: - return lines - style: CliStyle = CliStyle(use_color=True) - lines.append("") - lines.append(section_header_style("Remaining stale")) - lines.append(style.muted(f" models outside selection: {len(remaining_stale_model_names)}")) - lines.append( - style.muted( - " model set: " - + _format_capped_name_list( - names=remaining_stale_model_names, - display_options=display_options, - name_style=style.muted, - ) - ) - ) - return lines - - -def _metadata_string_tuple(raw_value: object | None) -> tuple[str, ...]: - return tuple(str(item) for item in raw_value) if isinstance(raw_value, (tuple, list)) else () - - -def _source_freshness_count_line( - *, - style: CliStyle, - label: str, - names: tuple[str, ...], - warn_nonzero: bool = False, -) -> str: - count_text: str = str(len(names)) - styled_count: str - if warn_nonzero and names: - styled_count = style.warning(count_text) - elif not names: - styled_count = style.muted(count_text) - else: - styled_count = count_text - return f" {style.label(label + ':')} {styled_count}" - - -def _source_freshness_set_line( - *, - style: CliStyle, - label: str, - names: tuple[str, ...], - display_options: DisplayOptions, - warn: bool = False, -) -> str: - formatted_names: str = _format_capped_name_list( - names=names, - display_options=display_options, - name_style=style.warning if warn else style.object_name, - ) - return f" {style.label(label + ':')} {formatted_names}" - - -def _format_capped_name_list( - *, - names: tuple[str, ...], - display_options: DisplayOptions, - name_style: Callable[[str], str] | None = None, -) -> str: - """Format a capped comma-separated name list.""" - - limit: int | None = display_options.max_entries_per_section - visible_names: tuple[str, ...] = names if limit is None else names[:limit] - remaining_count: int = len(names) - len(visible_names) - rendered_names: tuple[str, ...] = ( - visible_names if name_style is None else tuple(name_style(name) for name in visible_names) - ) - base: str = ", ".join(rendered_names) - if remaining_count <= 0: - return base - return f"{base}, ... (+{remaining_count} more; use {display_options.overflow_flag} to show all)" - - -def _resolve_name_column_width( - *, plan: PlanOutput, python_plan_entries: tuple[PythonPlanEntry, ...] = () -) -> int: - names: list[str] = [entry.name for entry in plan.model_entries] - names.extend(entry.name for entry in plan.function_entries) - names.extend(entry.name for entry in python_plan_entries) - return resolve_name_column_width(names=names) - - -def _format_name_value_line( - *, name: str, value: str, name_column_width: int, dim_value: bool = True -) -> str: - style: CliStyle = CliStyle(use_color=True) - rendered_value: str = style.muted(value) if dim_value else value - return format_aligned_name_value( - plain_name=name, - styled_name=style.object_name(name), - value=rendered_value, - name_column_width=name_column_width, - ) - - -def _treeify_plan_lines(lines: list[str]) -> list[str]: - """Convert section entry and detail lines into tree-connected rows.""" - - style: CliStyle = CliStyle(use_color=True) - kinds: list[PlanRowKind] = [_classify_plan_row(line) for line in lines] - out: list[str] = [] - index: int - line: str - for index, line in enumerate(lines): - kind: PlanRowKind = kinds[index] - if kind == PlanRowKind.ENTRY: - last: bool = _is_last_tree_row(kinds=kinds, index=index, row_kind=PlanRowKind.ENTRY) - connector: str = tree_connector(style=style, last=last) - out.append(f"{connector} {line[len(_ENTRY_ROW_PREFIX) :]}") - continue - if kind == PlanRowKind.LEAF: - last = _is_last_tree_row(kinds=kinds, index=index, row_kind=PlanRowKind.LEAF) - connector = tree_connector(style=style, last=last) - plain_line: str = _strip_ansi(line) - content: str = ( - line[len(_LEAF_ROW_PREFIX) :] - if line.startswith(_LEAF_ROW_PREFIX) - else plain_line[len(_LEAF_ROW_PREFIX) :] - ) - key, separator, value = content.partition(_LEAF_KEY_SEPARATOR) - if separator: - out.append(f"{_LEAF_ROW_PREFIX}{connector} {style.muted(key)} {value}") - elif content.endswith(":"): - out.append(f"{_LEAF_ROW_PREFIX}{connector} {style.muted(content)}") - else: - out.append(f"{_LEAF_ROW_PREFIX}{connector} {content}") - continue - out.append(line) - return out - - -def _classify_plan_row(line: str) -> PlanRowKind: - """Classify a rendered plan line for tree conversion.""" - - plain: str = _strip_ansi(line) - if not plain.strip() or not plain.startswith(_ENTRY_ROW_PREFIX): - return PlanRowKind.OTHER - if plain.startswith(_NESTED_ROW_PREFIX): - return PlanRowKind.NESTED - if plain.startswith(_LEAF_ROW_PREFIX): - if plain[len(_LEAF_ROW_PREFIX)] in _TREE_EXEMPT_CHARS: - return PlanRowKind.OTHER - return PlanRowKind.LEAF - if plain[len(_ENTRY_ROW_PREFIX)] in _TREE_EXEMPT_CHARS: - return PlanRowKind.OTHER - return PlanRowKind.ENTRY - - -def _is_last_tree_row(*, kinds: list[PlanRowKind], index: int, row_kind: PlanRowKind) -> bool: - """Return whether the row is the final one of its kind within its block.""" - - scan: int = index + 1 - while scan < len(kinds): - kind: PlanRowKind = kinds[scan] - if kind == PlanRowKind.OTHER: - return True - if row_kind == PlanRowKind.ENTRY and kind == PlanRowKind.ENTRY: - return False - if row_kind == PlanRowKind.LEAF: - if kind == PlanRowKind.LEAF: - return False - if kind == PlanRowKind.ENTRY: - return True - scan += 1 - return True - - -def _format_schema_findings(findings: tuple[SchemaFinding, ...]) -> list[str]: - """Format schema findings as indented diff lines.""" - - style: CliStyle = CliStyle(use_color=True) - lines: list[str] = [] - finding: SchemaFinding - for finding in findings: - symbol: str = _SCHEMA_CHANGE_SYMBOLS.get(finding.kind, "?") - type_info: str = "" - if finding.kind == SchemaChangeKind.COLUMN_TYPE_CHANGED: - type_info = f" {finding.expected_type} \u2192 {finding.actual_type}" - elif finding.kind == SchemaChangeKind.COLUMN_ADDED: - type_info = f" {finding.expected_type}" if finding.expected_type else "" - kind_label: str = _schema_kind_label(finding.kind) - line: str = f" {symbol} {finding.column_name}{type_info} ({kind_label})" - if finding.kind == SchemaChangeKind.COLUMN_ADDED: - lines.append(style.success(line)) - elif finding.kind == SchemaChangeKind.COLUMN_REMOVED: - lines.append(style.error(line)) - elif finding.kind == SchemaChangeKind.COLUMN_TYPE_CHANGED: - lines.append(style.warning(line)) - else: - lines.append(line) - return lines - - -def _schema_kind_label(kind: SchemaChangeKind) -> str: - """Human-readable schema change kind.""" - if kind == SchemaChangeKind.COLUMN_ADDED: - return "added" - if kind == SchemaChangeKind.COLUMN_REMOVED: - return "removed" - if kind == SchemaChangeKind.COLUMN_TYPE_CHANGED: - return "type changed" - return str(kind) - - -def _strip_ansi(text: str) -> str: - """Remove ANSI escape sequences from text.""" - return _ANSI_ESCAPE_PATTERN.sub("", text) diff --git a/src/sqlbuild/cli/output/_helpers/selection_diagnostics.py b/src/sqlbuild/cli/output/_helpers/selection_diagnostics.py deleted file mode 100644 index 25ef436970..0000000000 --- a/src/sqlbuild/cli/output/_helpers/selection_diagnostics.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Selection diagnostics output policy.""" - -from __future__ import annotations - -from typing import cast - -from sqlbuild.compiler.planner.models import PlanOutput - - -def direct_selection_diagnostics_enabled(plan: PlanOutput) -> bool | None: - """Return the explicit direct diagnostics policy, if present.""" - - raw_diagnostics: object | None = plan.metadata.get("selection_diagnostics") - if not isinstance(raw_diagnostics, dict): - return None - diagnostics: dict[str, object] = cast(dict[str, object], raw_diagnostics) - enabled: object | None = diagnostics.get("enabled") - return enabled if isinstance(enabled, bool) else None diff --git a/src/sqlbuild/cli/output/main/_plan_json.py b/src/sqlbuild/cli/output/main/_plan_json.py deleted file mode 100644 index c388ac7e30..0000000000 --- a/src/sqlbuild/cli/output/main/_plan_json.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Public JSON plan formatting entrypoint.""" - -from __future__ import annotations - -from sqlbuild.cli.output._helpers.plan_json import format_plan_json as _format_plan_json -from sqlbuild.compiler.pipeline.models import PythonPlanEntry -from sqlbuild.compiler.planner.models import PlanOutput - - -def format_plan_json( - *, plan: PlanOutput, python_plan_entries: tuple[PythonPlanEntry, ...] = () -) -> str: - """Serialize a PlanOutput to JSON.""" - - return _format_plan_json(plan=plan, python_plan_entries=python_plan_entries) diff --git a/src/sqlbuild/cli/output/main/plan.py b/src/sqlbuild/cli/output/main/plan.py deleted file mode 100644 index 340baee71a..0000000000 --- a/src/sqlbuild/cli/output/main/plan.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Public text plan formatting entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.cli.output._helpers.plan_text import format_plan as _format_plan -from sqlbuild.cli.output._helpers.plan_text import format_plan_completion -from sqlbuild.compiler.pipeline.models import PythonPlanEntry -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.presentation.models import DisplayOptions - - -def format_plan( - *, - plan: PlanOutput, - full_refresh: bool = False, - use_color: bool = True, - include_header: bool = True, - display_options: DisplayOptions | None = None, - section_header_style: Callable[[str], str] | None = None, - python_plan_entries: tuple[PythonPlanEntry, ...] = (), - include_direct_freshness_diagnostics: bool = True, - include_completion: bool = False, -) -> str: - """Format plan output grouped by reason with inline detail.""" - - plan_text: str = _format_plan( - plan=plan, - full_refresh=full_refresh, - use_color=use_color, - include_header=include_header, - display_options=display_options, - section_header_style=section_header_style, - python_plan_entries=python_plan_entries, - include_direct_freshness_diagnostics=include_direct_freshness_diagnostics, - ) - if not include_completion: - return plan_text - completion: str = format_plan_completion( - plan=plan, - full_refresh=full_refresh, - use_color=use_color, - python_plan_entries=python_plan_entries, - ) - return f"{plan_text}\n\n{completion}" diff --git a/src/sqlbuild/cli/output/models.py b/src/sqlbuild/cli/output/models.py index 0e0603c16c..12c78955a0 100644 --- a/src/sqlbuild/cli/output/models.py +++ b/src/sqlbuild/cli/output/models.py @@ -48,10 +48,7 @@ MAX_INTEGRATION_RECORD_BYTES, ) from sqlbuild.cli.output.types import ( - CursorBoundsOwner, - CursorResolutionStatus, IntegrationOutputKind, - PlannedCursorBounds, WrittenTargetDiagnostic, ) from sqlbuild.compiler.auditing.types import ( @@ -77,24 +74,6 @@ class TerminalEventClaim: event_sequence: int -@dataclass(frozen=True) -class CursorPlanDetails: - """Operator-facing cursor details shared by plan renderers.""" - - requested_start: str | None - requested_end: str | None - bounds_owner: CursorBoundsOwner - resolution_status: CursorResolutionStatus - resolved_bounds: PlannedCursorBounds | None - declared_grain: str | None - effective_grain: str | None - declared_batch_size: str | None - effective_batch_size: str | None - planned_batch_count: int | None - empty_inputs: tuple[str, ...] = () - waiting_on_empty_inputs: bool = False - - @dataclass(frozen=True) class IntegrationAssetResult: """Bounded asset enrichment consumed by SQLBuild-owned integrations.""" diff --git a/src/sqlbuild/cli/output/types.py b/src/sqlbuild/cli/output/types.py index f84f12bde5..85785c3f17 100644 --- a/src/sqlbuild/cli/output/types.py +++ b/src/sqlbuild/cli/output/types.py @@ -5,37 +5,9 @@ if TYPE_CHECKING: from sqlbuild.compiler.compile.models import CompilerDiagnostic - from sqlbuild.compiler.planner.models import CursorBounds type WrittenTargetDiagnostic = CompilerDiagnostic -type PlannedCursorBounds = CursorBounds - - -class CursorBoundsOwner(StrEnum): - """Component responsible for resolving effective cursor bounds.""" - - PLANNER = "planner" - RUNTIME = "runtime" - - -class CursorResolutionStatus(StrEnum): - """Availability of effective cursor bounds at plan time.""" - - DEFERRED = "deferred" - RESOLVED = "resolved" - UNAVAILABLE = "unavailable" - NO_INPUT_ROWS = "no_input_rows" - - -class PlanRowKind(StrEnum): - """Structural classification of a rendered plan line.""" - - ENTRY = "entry" - LEAF = "leaf" - NESTED = "nested" - OTHER = "other" - class IntegrationOutputKind(StrEnum): """Kind of integration-facing result enrichment.""" diff --git a/src/sqlbuild/compiler/planner/main/changes/query_diff.py b/src/sqlbuild/compiler/planner/main/changes/_query_diff.py similarity index 100% rename from src/sqlbuild/compiler/planner/main/changes/query_diff.py rename to src/sqlbuild/compiler/planner/main/changes/_query_diff.py diff --git a/src/sqlbuild/compiler/planner/main/execution/model_materialization_label.py b/src/sqlbuild/compiler/planner/main/execution/_model_materialization_label.py similarity index 100% rename from src/sqlbuild/compiler/planner/main/execution/model_materialization_label.py rename to src/sqlbuild/compiler/planner/main/execution/_model_materialization_label.py diff --git a/src/sqlbuild/integrations/dbt/_helpers/planning/plan.py b/src/sqlbuild/integrations/dbt/_helpers/planning/plan.py index 3d9dc6d79a..0a5371ae24 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/planning/plan.py +++ b/src/sqlbuild/integrations/dbt/_helpers/planning/plan.py @@ -7,7 +7,7 @@ from collections import defaultdict from collections.abc import Sequence -from sqlbuild.cli.output.main.plan import format_plan +from sqlbuild._native import format_plan from sqlbuild.compiler.planner.models import PlanOutput from sqlbuild.integrations.dbt.models import DbtInteropPlan, DbtInteropSelectionResult, DbtLsNode from sqlbuild.integrations.dbt.types import DbtInteropCommand, DbtInteropSkipReason diff --git a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py index 893d4b43bb..323774a72f 100644 --- a/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py +++ b/tests/integration/src/sqlbuild/compiler/planner/main/test_build_execution_plan.py @@ -9,11 +9,11 @@ from sqlbuild._native import ( compiled_model_record, + format_plan, project_with, ) from sqlbuild.adapter.contract.types import BuiltinAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.output.main.plan import format_plan from sqlbuild.compiler.compile.types import CompiledProjectView from sqlbuild.compiler.planner._helpers.changes import detect as change_detection from sqlbuild.compiler.planner._helpers.planning import identities as identity_planning diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py index 88c8e2872c..988650b137 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.cli.output.main.plan import format_plan +from sqlbuild._native import format_plan from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.compiler.pipeline.models import PythonPlanEntry from sqlbuild.compiler.planner.models import ( diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py index 2e3644d217..a26ec438f1 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py @@ -7,7 +7,7 @@ import pytest -from sqlbuild.cli.output.main._plan_json import format_plan_json +from sqlbuild._native import format_plan_json from sqlbuild.compiler.pipeline.models import PythonPlanEntry from sqlbuild.compiler.planner.models import ( CursorBounds, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py index 6e0dc98a6b..e3ddb6f0ba 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/execution/test_display_plan.py @@ -6,7 +6,7 @@ import pytest -from sqlbuild.cli.output.main.plan import format_plan +from sqlbuild._native import format_plan from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, diff --git a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py index f63ccc5e6b..ea481f74eb 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py +++ b/tests/unit/src/sqlbuild/compiler/planner/main/pre_build/test_pre_build_work_display.py @@ -4,7 +4,7 @@ import pytest -from sqlbuild.cli.output.main.plan import format_plan +from sqlbuild._native import format_plan from sqlbuild.compiler.migrations.types import ( ColumnMigrationDecision, MigrationCompatibility, diff --git a/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py b/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py index 1f08f44954..b088b56696 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/test_resource.py @@ -13,7 +13,7 @@ import pytest from _pytest.capture import CaptureResult -from sqlbuild.cli.output._helpers.maximum_start_safety import serialize_maximum_start_safety +from sqlbuild._native import serialize_maximum_start_safety from sqlbuild.cli.output.models import IntegrationCheckResult, IntegrationResultEnvelope from sqlbuild.compiler.planner.models import MaximumStartSafetyEvidence from sqlbuild.integrations.dagster import (