Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
SHELL := /bin/bash

.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-python-boundary check-release-gate \
.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-python-boundary check-release-gate check-docs-skill \
check-e2e-shards test-e2e-duckdb test-e2e-duckdb-build-core \
test-e2e-duckdb-build-incremental \
test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-inspection \
Expand Down Expand Up @@ -625,7 +625,7 @@ verify-quick:
exit $$status


check-ci: check-e2e-shards check-native-layering check-python-boundary check-release-gate rust-lint
check-ci: check-e2e-shards check-native-layering check-python-boundary check-release-gate check-docs-skill rust-lint
uv run ruff format --check .
uv run ruff check .
uv run ty check src tests
Expand All @@ -645,6 +645,10 @@ check-release-gate:
uv run python -m scripts.check_release_gate


check-docs-skill:
uv run python -m scripts.check_docs_skill


check-pr-metadata:
@test -n "$(PR_TITLE)" || { echo "PR_TITLE is required" >&2; exit 2; }
@test -n "$(PR_BODY_FILE)" || { echo "PR_BODY_FILE is required" >&2; exit 2; }
Expand Down
26 changes: 24 additions & 2 deletions crates/sqlbuild-attachments/src/audits/_helpers/rendering.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

use crate::audits::_helpers::parameters::{RenderStop, render_parameterized_sql};
use crate::audits::_helpers::policies::{resolve_run_scope, resolve_severity};
use crate::audits::constants::{MODEL_RESOURCE, SEED_RESOURCE, SOURCE_RESOURCE};
use crate::audits::models::{ArgumentValue, AuditAttachment, PolicySource};

/// Severity first, then run scope, as Python resolves them after expanding the SQL.
Expand Down Expand Up @@ -39,8 +40,10 @@ pub(crate) fn rendered(
) -> Result<String, String> {
render_parameterized_sql(sql, arguments).map_err(|stop| match stop {
RenderStop::MissingArgument(name) => format!(
"{} is missing argument '{name}' for generic audit '{}'",
attachment.owner_label, attachment.definition_name
"{} is missing argument '{name}' for generic audit '{}'{}",
attachment.owner_label,
attachment.definition_name,
owner_argument_hint(attachment, &name)
),
RenderStop::UnsupportedValue(name) => format!(
"{} generic audit '{}' argument '{name}' uses an unsupported value",
Expand All @@ -49,6 +52,25 @@ pub(crate) fn rendered(
})
}

/// Why `@model` is missing on a seed or source: each owner passes its own kind's name argument.
fn owner_argument_hint(attachment: &AuditAttachment, missing: &str) -> String {
let owner_arguments: [&str; 3] = [MODEL_RESOURCE, SOURCE_RESOURCE, SEED_RESOURCE];
if !owner_arguments.contains(&missing) {
return String::new();
}
let Some((owner_kind, _)) = attachment
.implicit_arguments
.iter()
.find(|(argument, _)| owner_arguments.contains(&argument.as_str()))
else {
return String::new();
};
format!(
"; a {owner_kind} audit receives '{owner_kind}' and 'relation', not '{missing}', so read \
the attached relation with @relation"
)
}

/// `merge_audit_arguments`: an explicit argument may repeat an implicit one only as equal text.
pub(crate) fn merged_arguments(
attachment: &AuditAttachment,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,23 @@ fn given_attached_audits_when_rendering_then_python_rendering_is_returned() {
'accepted_values'",
),
},
AttachedAuditTestCase {
description: "a seed audit reading @model is told to read @relation",
attachment: AuditAttachment {
owner_label: "seeds/waffle_types.yml".to_owned(),
sql_body: "SELECT * FROM __ref(\"@model\") WHERE NOT (price_cents >= 0)".to_owned(),
implicit_arguments: vec![
("seed".to_owned(), "waffle_types".to_owned()),
("relation".to_owned(), "__seed(\"waffle_types\")".to_owned()),
],
..attachment(Vec::new(), None, None)
},
expected_rendering: failed(
"seeds/waffle_types.yml is missing argument 'model' for generic audit \
'accepted_values'; a seed audit receives 'seed' and 'relation', not 'model', \
so read the attached relation with @relation",
),
},
AttachedAuditTestCase {
description: "an opaque value inside a referenced list is Python's value error",
attachment: attachment(
Expand Down
49 changes: 42 additions & 7 deletions crates/sqlbuild-core/src/cli_text/_helpers/tree_rows.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
use crate::cli_text::_helpers::bounded_sections::strip_ansi;
use crate::cli_text::_helpers::layout::tree_connector;
use crate::cli_text::constants::{
ENTRY_ROW_PREFIX, LEAF_KEY_SEPARATOR, LEAF_ROW_PREFIX, NESTED_ROW_PREFIX, TREE_EXEMPT_CHARS,
ENTRY_ROW_PREFIX, LEAF_KEY_SEPARATOR, LEAF_ROW_PREFIX, NESTED_ROW_PREFIX, STYLE_ESCAPE_END,
STYLE_ESCAPE_START, TREE_CONTINUATION_GLYPH, TREE_EXEMPT_CHARS,
};
use crate::cli_text::models::{CliStyle, RowKind};

Expand Down Expand Up @@ -42,19 +43,49 @@ fn is_last(kinds: &[RowKind], index: usize, row_kind: RowKind) -> bool {
true
}

/// A nested row whose leaf indentation becomes a non-last entry's bar, keeping style escapes.
fn continued(line: String, continuation: Option<&str>) -> String {
let Some(continuation) = continuation else {
return line;
};
let mut kept: String = String::with_capacity(line.len());
let mut removed: usize = 0;
let mut characters = line.chars().peekable();
while removed < LEAF_ROW_PREFIX.len() {
match characters.next() {
Some(STYLE_ESCAPE_START) => {
kept.push(STYLE_ESCAPE_START);
for character in characters.by_ref() {
kept.push(character);
if character == STYLE_ESCAPE_END {
break;
}
}
}
Some(' ') => removed += 1,
_ => return line,
}
}
kept.extend(characters);
format!("{continuation}{kept}")
}

/// `_treeify_plan_lines`: entry rows and their detail rows drawn with tree connectors.
pub(crate) fn treeify(lines: Vec<String>) -> Vec<String> {
let style: CliStyle = CliStyle { use_color: true };
let kinds: Vec<RowKind> = lines.iter().map(|line| classify(line)).collect();
let mut out: Vec<String> = Vec::with_capacity(lines.len());
let mut bar: Option<String> = None;
for (index, line) in lines.into_iter().enumerate() {
match kinds[index] {
RowKind::Entry => {
let connector: String =
tree_connector(style, is_last(&kinds, index, RowKind::Entry));
let last: bool = is_last(&kinds, index, RowKind::Entry);
bar = (!last).then(|| style.muted(TREE_CONTINUATION_GLYPH));
let connector: String = tree_connector(style, last);
out.push(format!("{connector} {}", &line[ENTRY_ROW_PREFIX.len()..]));
}
RowKind::Leaf => {
let continuation: &str = bar.as_deref().unwrap_or(LEAF_ROW_PREFIX);
let connector: String =
tree_connector(style, is_last(&kinds, index, RowKind::Leaf));
let content: String = match line.strip_prefix(LEAF_ROW_PREFIX) {
Expand All @@ -63,19 +94,23 @@ pub(crate) fn treeify(lines: Vec<String>) -> Vec<String> {
};
if let Some((key, value)) = content.split_once(LEAF_KEY_SEPARATOR) {
out.push(format!(
"{LEAF_ROW_PREFIX}{connector} {} {value}",
"{continuation}{connector} {} {value}",
style.muted(key)
));
} else if content.ends_with(':') {
out.push(format!(
"{LEAF_ROW_PREFIX}{connector} {}",
"{continuation}{connector} {}",
style.muted(&content)
));
} else {
out.push(format!("{LEAF_ROW_PREFIX}{connector} {content}"));
out.push(format!("{continuation}{connector} {content}"));
}
}
RowKind::Nested | RowKind::Other => out.push(line),
RowKind::Nested => out.push(continued(line, bar.as_deref())),
RowKind::Other => {
bar = None;
out.push(line);
}
}
}
out
Expand Down
5 changes: 5 additions & 0 deletions crates/sqlbuild-core/src/cli_text/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ pub(crate) const PHASE_OK_GLYPH: &str = "\u{2713}";
pub(crate) const PHASE_FAIL_GLYPH: &str = "\u{2717}";
pub(crate) const TREE_BRANCH_GLYPH: &str = "\u{251c}\u{2500}\u{2500}";
pub(crate) const TREE_LAST_GLYPH: &str = "\u{2514}\u{2500}\u{2500}";
/// The bar a non-last entry's detail rows carry under its connector.
pub(crate) const TREE_CONTINUATION_GLYPH: &str = "\u{2502} ";
/// The start of a terminal style escape and the character that ends it.
pub(crate) const STYLE_ESCAPE_START: char = '\u{1b}';
pub(crate) const STYLE_ESCAPE_END: char = 'm';
/// The human report lists at most this many models.
pub(crate) const HUMAN_MODEL_LIMIT: usize = 100;
pub(crate) const MIN_MODEL_NAME_WIDTH: usize = 24;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
//! The completion line that closes text plan output.

use crate::cli_text::_helpers::plan_sections::{full_refresh_layout, ready_summary};
use crate::cli_text::constants::{ERROR_SEVERITY_VALUE, PHASE_OK_GLYPH};
use crate::cli_text::constants::{ERROR_SEVERITY_VALUE, PHASE_FAIL_GLYPH, PHASE_OK_GLYPH};
use crate::cli_text::models::{CliStyle, PlanTextReport};

/// `✓ Plan complete <summary>`, warning-styled when the plan has errors.
/// `✓ Plan complete <summary>`, or `✗ Plan failed` when the plan has errors a build refuses.
#[must_use]
pub fn format_plan_completion(report: &PlanTextReport) -> String {
let style: CliStyle = CliStyle {
Expand All @@ -16,8 +16,8 @@ pub fn format_plan_completion(report: &PlanTextReport) -> String {
.any(|warning| warning.severity == ERROR_SEVERITY_VALUE);
let (glyph, label) = if has_errors {
(
style.warning(PHASE_OK_GLYPH),
style.warning_strong("Plan complete with errors"),
style.error(PHASE_FAIL_GLYPH),
style.error_strong("Plan failed with errors"),
)
} else {
(style.success(PHASE_OK_GLYPH), "Plan complete".to_owned())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

use std::collections::BTreeMap;

use crate::cli_text::_helpers::summary_text::count_noun;

use crate::cli_text::constants::{
DEFAULT_ACTION, INVALID_UPSTREAM, NOTICE_DISPLAY_LIMIT, OPEN_INPUT, OPEN_SOURCE,
};
Expand Down Expand Up @@ -53,9 +55,10 @@ pub fn semantic_coverage_notice(reasons: &[(String, Vec<String>)]) -> Option<Str
.push(model.as_str());
}
}
let count: i64 = i64::try_from(reasons.len()).unwrap_or(i64::MAX);
let mut lines: Vec<String> = vec![format!(
"Semantic checks were partial for {} models:",
reasons.len()
"Semantic checks were partial for {}:",
count_noun(count, "model", None)
)];
for (reason, models) in groups.iter().take(NOTICE_DISPLAY_LIMIT) {
let suffix: String = if models.len() > NOTICE_DISPLAY_LIMIT {
Expand Down
32 changes: 32 additions & 0 deletions crates/sqlbuild-core/src/cli_text/tests/test_cli_text.rs
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,13 @@ fn given_partial_reasons_when_building_notice_then_groups_by_reason() {
"Semantic checks were partial for 2 models:\n zeta: SQL analysis disabled; enable sql_analysis, then run `sqb compile --no-cache`.\n alpha, zeta: reads open source raw_orders; run `sqb contract generate --from <target> --select source:raw_orders --write` and enforce the complete source contract.\n alpha: weird; declare the input shape, then run `sqb compile --no-cache`.",
),
},
NoticeTestCase {
description: "one partial model is counted in the singular",
reasons: vec![("alpha".to_owned(), vec!["SQL analysis disabled".to_owned()])],
expected_notice: Some(
"Semantic checks were partial for 1 model:\n alpha: SQL analysis disabled; enable sql_analysis, then run `sqb compile --no-cache`.",
),
},
NoticeTestCase {
description: "complete checks print no notice",
reasons: Vec::new(),
Expand Down Expand Up @@ -158,6 +165,31 @@ fn given_plan_report_when_rendering_then_groups_models_by_reason() {
expected_text: "Plan ready 3 selected\n\nQuery changed (1)\n\u{2514}\u{2500}\u{2500} orders full rebuild\n \u{251c}\u{2500}\u{2500} policy replay_on_change=full\n \u{2514}\u{2500}\u{2500} query diff:\n -SELECT 1\n +SELECT 2\n\nModels (1)\n\u{2514}\u{2500}\u{2500} customers table\n\nWarnings (1)\n\u{2514}\u{2500}\u{2500} orders\n \u{251c}\u{2500}\u{2500} orders lag\n \u{2514}\u{2500}\u{2500} second line",
expected_completion: "\u{2713} Plan complete 3 selected",
},
PlanTextTestCase {
description: "detail rows under a non-last entry keep the tree's vertical bar",
report: plan_report(vec![
ModelPlanText {
query_diff: Some(vec![" -SELECT 1".to_owned()]),
..table_model("orders", "query_changed")
},
table_model("customers", "query_changed"),
]),
expected_text: "Plan ready 2 selected\n\nQuery changed (2)\n\u{251c}\u{2500}\u{2500} orders full rebuild\n\u{2502} \u{251c}\u{2500}\u{2500} policy replay_on_change=full\n\u{2502} \u{2514}\u{2500}\u{2500} query diff:\n\u{2502} -SELECT 1\n\u{2514}\u{2500}\u{2500} customers full rebuild\n \u{2514}\u{2500}\u{2500} policy replay_on_change=full",
expected_completion: "\u{2713} Plan complete 2 selected",
},
PlanTextTestCase {
description: "an error finding makes the final line state failure",
report: PlanTextReport {
warnings: vec![PlanWarningText {
severity: "error".to_owned(),
message: "would replace analytics.orders".to_owned(),
model_name: None,
}],
..plan_report(vec![table_model("orders", "no_change")])
},
expected_text: "Plan ready 1 selected\n\nModels (1)\n\u{2514}\u{2500}\u{2500} orders table\n\nErrors (1)\n\u{2514}\u{2500}\u{2500} would replace analytics.orders",
expected_completion: "\u{2717} Plan failed with errors 1 selected",
},
PlanTextTestCase {
description: "every active model is a full refresh",
report: plan_report(vec![
Expand Down
44 changes: 41 additions & 3 deletions crates/sqlbuild-core/src/progress/_helpers/resources.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@ use std::collections::VecDeque;

use crate::cli_text::models::CliStyle;
use crate::progress::constants::{
FAILED_SUFFIX, KIND_WIDTH, SCENARIO_RESOURCE_PREFIX, SKIPPED_SUFFIX,
AUDIT_OUTCOME_STATUSES, AUDIT_RESOURCE_KIND, AUDIT_RESOURCE_PREFIX, FAILED_SUFFIX, KIND_WIDTH,
SCENARIO_RESOURCE_PREFIX, SKIPPED_SUFFIX,
};
use crate::progress::models::{ProgressEvent, ProgressProjection};
use crate::terminal_events::models::TerminalRequest;
Expand All @@ -19,6 +20,27 @@ fn display_resource_name(resource_id: Option<&str>, resource_name: &str) -> Stri
format!("{scenario_name}/{resource_name}")
}

/// `name (target.column)` from an audit identifier `audit:<name>:<kind>[:<target>[:<column>]]`.
fn audit_display_name(resource_id: Option<&str>, resource_name: &str) -> String {
let Some(identity) = resource_id.and_then(|id| id.strip_prefix(AUDIT_RESOURCE_PREFIX)) else {
return resource_name.to_owned();
};
let attached: Vec<&str> = identity.split(':').skip(2).collect();
if attached.is_empty() {
return resource_name.to_owned();
}
format!("{resource_name} ({})", attached.join("."))
}

/// The row name of one resource: audits name their target, scenario resources their scenario.
fn resource_display_name(kind: &str, resource_id: Option<&str>, resource_name: &str) -> String {
if kind == AUDIT_RESOURCE_KIND {
audit_display_name(resource_id, resource_name)
} else {
display_resource_name(resource_id, resource_name)
}
}

/// Remember an attempt, keeping the position of its first sighting.
fn store_attempt(
mut projection: ProgressProjection,
Expand Down Expand Up @@ -84,7 +106,8 @@ pub(crate) fn consume_resource_start(
use_color: projection.settings.use_color,
}
.status("START");
let display_name: String = display_resource_name(Some(resource_id), resource_name);
let display_name: String =
resource_display_name(resource_kind, Some(resource_id), resource_name);
lines.push(format!(
" {counter}{resource_kind:<KIND_WIDTH$}{display_name} {status}"
));
Expand Down Expand Up @@ -132,10 +155,24 @@ pub(crate) fn render_resource_terminal(
let Some(resource_kind) = &event.payload.resource_kind else {
return (projection, None);
};
let audit: bool = resource_kind == AUDIT_RESOURCE_KIND;
let recorded: Option<&str> = projection
.audit_outcomes
.get(attempt_id)
.and_then(|outcome| {
AUDIT_OUTCOME_STATUSES
.iter()
.find(|(value, _)| value == outcome)
.map(|(_, status)| *status)
});
let status_text: &str = if event.event_type.ends_with(FAILED_SUFFIX) {
"FAIL"
} else if event.event_type.ends_with(SKIPPED_SUFFIX) {
"SKIP"
} else if let Some(status) = recorded {
status
} else if audit {
"PASS"
} else {
"OK"
};
Expand All @@ -148,7 +185,8 @@ pub(crate) fn render_resource_terminal(
.duration_ms
.map(|duration| format!(" {:.2}s", duration / 1000.0))
.unwrap_or_default();
let display_name: String = display_resource_name(event.resource_id.as_deref(), resource_name);
let display_name: String =
resource_display_name(resource_kind, event.resource_id.as_deref(), resource_name);
let line: String = format!(" {resource_kind:<KIND_WIDTH$}{display_name} {status}{elapsed}");
(projection, Some(line))
}
Expand Down
10 changes: 10 additions & 0 deletions crates/sqlbuild-core/src/progress/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -111,3 +111,13 @@ pub(crate) const JSON_FORMAT: &str = "json";
pub(crate) const CSV_FORMAT: &str = "csv";
/// What replaces the cut tail of a transient status message.
pub(crate) const STATUS_ELLIPSIS: &str = "...";
/// The resource kind of audits and the prefix of their resource identifiers.
pub(crate) const AUDIT_RESOURCE_KIND: &str = "audit";
pub(crate) const AUDIT_RESOURCE_PREFIX: &str = "audit:";
/// Audit outcomes and the status words their rows show.
pub(crate) const AUDIT_OUTCOME_STATUSES: [(&str, &str); 4] = [
("pass", "PASS"),
("warn", "WARN"),
("error", "FAIL"),
("insufficient", "INSUFFICIENT"),
];
Loading
Loading