From 3deb6a2121b6412e614026c8592a0b14443e9a67 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:23:10 +0100 Subject: [PATCH 1/5] refactor: share project scaffolding and rule listing text outside the Python bindings --- .../sqlbuild-core/src/init_command/errors.rs | 11 ++++ .../src/init_command/main/mod.rs | 1 + .../main/scaffold_blank_project.rs | 57 +++++++++++++++++ crates/sqlbuild-core/src/init_command/mod.rs | 1 + .../_helpers/init_command/scaffold.rs | 51 ++------------- .../bindings/_helpers/plan_command/rules.rs | 64 ++++++++----------- .../src/rule_catalogue/main/mod.rs | 1 + .../rule_catalogue/main/render_rule_list.rs | 44 +++++++++++++ .../src/rule_catalogue/models.rs | 11 ++++ fensu.toml | 2 +- 10 files changed, 157 insertions(+), 86 deletions(-) create mode 100644 crates/sqlbuild-core/src/init_command/errors.rs create mode 100644 crates/sqlbuild-core/src/init_command/main/scaffold_blank_project.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/main/render_rule_list.rs diff --git a/crates/sqlbuild-core/src/init_command/errors.rs b/crates/sqlbuild-core/src/init_command/errors.rs new file mode 100644 index 0000000000..752e7680d6 --- /dev/null +++ b/crates/sqlbuild-core/src/init_command/errors.rs @@ -0,0 +1,11 @@ +//! Why a blank project could not be created. + +use std::io; +use std::path::PathBuf; + +/// The I/O failure and the path it happened at, as Python's `OSError` names its `filename`. +#[derive(Debug)] +pub struct ScaffoldError { + pub error: io::Error, + pub path: PathBuf, +} diff --git a/crates/sqlbuild-core/src/init_command/main/mod.rs b/crates/sqlbuild-core/src/init_command/main/mod.rs index b8eae0db00..815db5c0cc 100644 --- a/crates/sqlbuild-core/src/init_command/main/mod.rs +++ b/crates/sqlbuild-core/src/init_command/main/mod.rs @@ -1,3 +1,4 @@ pub mod init_project_name; pub mod init_project_toml; pub mod render_init_summary; +pub mod scaffold_blank_project; diff --git a/crates/sqlbuild-core/src/init_command/main/scaffold_blank_project.rs b/crates/sqlbuild-core/src/init_command/main/scaffold_blank_project.rs new file mode 100644 index 0000000000..eb9c89ff78 --- /dev/null +++ b/crates/sqlbuild-core/src/init_command/main/scaffold_blank_project.rs @@ -0,0 +1,57 @@ +//! Create the blank project tree as Python's `pathlib` calls did. + +use std::fs::{self, OpenOptions}; +use std::io::ErrorKind; +use std::path::Path; + +use crate::constants::PROJECT_CONFIG_FILENAME; +use crate::init_command::constants::{GITKEEP_FILE, SCAFFOLD_DIRECTORIES}; +use crate::init_command::errors::ScaffoldError; + +fn failed(error: std::io::Error, path: &Path) -> ScaffoldError { + ScaffoldError { + error, + path: path.to_path_buf(), + } +} + +/// `Path.mkdir(parents=True, exist_ok=True)`. +fn make_directories(path: &Path) -> Result<(), ScaffoldError> { + match fs::create_dir(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == ErrorKind::NotFound => { + let Some(parent) = path.parent().filter(|parent| *parent != path) else { + return Err(failed(error, path)); + }; + make_directories(parent)?; + match fs::create_dir(path) { + Err(error) if !path.is_dir() => Err(failed(error, path)), + _ => Ok(()), + } + } + Err(_) if path.is_dir() => Ok(()), + Err(error) => Err(failed(error, path)), + } +} + +/// Create the resource directories, the project file, and `.gitkeep` in empty directories. +pub fn scaffold_blank_project(base_dir: &Path, project_toml: &str) -> Result<(), ScaffoldError> { + for directory in SCAFFOLD_DIRECTORIES { + make_directories(&base_dir.join(directory))?; + } + let project_file = base_dir.join(PROJECT_CONFIG_FILENAME); + fs::write(&project_file, project_toml).map_err(|error| failed(error, &project_file))?; + for directory in SCAFFOLD_DIRECTORIES { + let directory = base_dir.join(directory); + let mut entries = fs::read_dir(&directory).map_err(|error| failed(error, &directory))?; + if entries.next().is_none() { + let gitkeep = directory.join(GITKEEP_FILE); + OpenOptions::new() + .create(true) + .append(true) + .open(&gitkeep) + .map_err(|error| failed(error, &gitkeep))?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-core/src/init_command/mod.rs b/crates/sqlbuild-core/src/init_command/mod.rs index d9f45c6513..5aedb44cc0 100644 --- a/crates/sqlbuild-core/src/init_command/mod.rs +++ b/crates/sqlbuild-core/src/init_command/mod.rs @@ -1,6 +1,7 @@ //! `sqb init`: the blank project layout, its configuration, and the created summary. pub mod constants; +pub mod errors; pub mod main; #[cfg(test)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs b/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs index 8e969b0c1c..35d083baeb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs @@ -1,13 +1,12 @@ //! Create the blank project tree, raising the `OSError` Python's `pathlib` would. -use std::fs::{self, OpenOptions}; -use std::io::{self, ErrorKind}; +use std::io; use std::path::Path; use crate::bindings::_helpers::records::fields::field as native_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_FILE; -use sqlbuild_core::init_command::constants::{GITKEEP_FILE, SCAFFOLD_DIRECTORIES}; +use sqlbuild_core::init_command::errors::ScaffoldError; +use sqlbuild_core::init_command::main::scaffold_blank_project::scaffold_blank_project as scaffold; /// `OSError(errno, strerror, filename)`, which Python narrows to its errno subclass. fn os_error(py: Python<'_>, error: &io::Error, path: &Path) -> PyErr { @@ -35,52 +34,12 @@ fn os_error(py: Python<'_>, error: &io::Error, path: &Path) -> PyErr { } } -/// `Path.mkdir(parents=True, exist_ok=True)`. -fn make_directories(path: &Path) -> Result<(), (io::Error, std::path::PathBuf)> { - match fs::create_dir(path) { - Ok(()) => Ok(()), - Err(error) if error.kind() == ErrorKind::NotFound => { - let Some(parent) = path.parent().filter(|parent| *parent != path) else { - return Err((error, path.to_path_buf())); - }; - make_directories(parent)?; - match fs::create_dir(path) { - Err(error) if !path.is_dir() => Err((error, path.to_path_buf())), - _ => Ok(()), - } - } - Err(error) if path.is_dir() => { - drop(error); - Ok(()) - } - Err(error) => Err((error, path.to_path_buf())), - } -} - /// Create the resource directories, the project file, and `.gitkeep` in empty directories. pub(crate) fn scaffold_blank_project( py: Python<'_>, base_dir: &Path, project_toml: &str, ) -> PyResult<()> { - for directory in SCAFFOLD_DIRECTORIES { - make_directories(&base_dir.join(directory)) - .map_err(|(error, path)| os_error(py, &error, &path))?; - } - let project_file = base_dir.join(PROJECT_FILE); - fs::write(&project_file, project_toml).map_err(|error| os_error(py, &error, &project_file))?; - for directory in SCAFFOLD_DIRECTORIES { - let directory = base_dir.join(directory); - let mut entries = - fs::read_dir(&directory).map_err(|error| os_error(py, &error, &directory))?; - if entries.next().is_none() { - let gitkeep = directory.join(GITKEEP_FILE); - OpenOptions::new() - .create(true) - .append(true) - .open(&gitkeep) - .map_err(|error| os_error(py, &error, &gitkeep))?; - } - } - Ok(()) + scaffold(base_dir, project_toml) + .map_err(|ScaffoldError { error, path }| os_error(py, &error, &path)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs index a3be4ae5df..9fcc0ef2bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -3,8 +3,10 @@ use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyfunction, }; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::wrap_pyfunction; +use sqlbuild_rules::rule_catalogue::main::render_rule_list::render_rule_list; +use sqlbuild_rules::rule_catalogue::models::RuleListing; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; @@ -81,47 +83,31 @@ fn run_skills( Ok(if fresh { 0 } else { 1 }) } +fn rule_listing(rule: &Bound<'_, PyAny>) -> PyResult { + let subject: Bound<'_, PyAny> = native_field(rule, "subject")?; + Ok(RuleListing { + code: native_field(rule, "code")?.str()?.extract()?, + family: native_field(rule, "family")?.extract()?, + message: native_field(rule, "message")?.str()?.extract()?, + custom: native_field(rule, "custom")?.is_truthy()?, + subject: if subject.is_none() { + None + } else { + Some(native_field(&subject, "value")?.extract()?) + }, + }) +} + fn list_rules(catalogue: &Bound<'_, PyAny>, json_output: bool) -> PyResult { - let py: Python<'_> = catalogue.py(); - if !json_output { - let mut lines: Vec = Vec::new(); - for rule in catalogue.try_iter()? { - let rule: Bound<'_, PyAny> = rule?; - lines.push(format!( - "{} {}", - native_field(&rule, "code")?.str()?, - native_field(&rule, "message")?.str()? - )); - } - print_line(py, &lines.join("\n"), false)?; - return Ok(0); - } - let rules: Bound<'_, PyList> = PyList::empty(py); + let mut rules: Vec = Vec::new(); for rule in catalogue.try_iter()? { - let rule: Bound<'_, PyAny> = rule?; - let item: Bound<'_, PyDict> = PyDict::new(py); - for field in ["code", "family", "message", "custom"] { - item.set_item(field, native_field(&rule, field)?)?; - } - let subject: Bound<'_, PyAny> = native_field(&rule, "subject")?; - item.set_item( - "subject", - if subject.is_none() { - subject - } else { - native_field(&subject, "value")? - }, - )?; - rules.append(item)?; + rules.push(rule_listing(&rule?)?); } - let payload: Bound<'_, PyDict> = PyDict::new(py); - payload.set_item("rules", rules)?; - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("sort_keys", true)?; - let text: String = imported(py, "json", "dumps")? - .call((payload,), Some(&options))? - .extract()?; - print_line(py, &text, false)?; + print_line( + catalogue.py(), + &render_rule_list(&rules, json_output), + false, + )?; Ok(0) } diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs index 618095dd7e..0c4b11601a 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs @@ -1,5 +1,6 @@ pub mod format_rule_text; pub mod install_rendered_skills; +pub mod render_rule_list; pub mod render_skills_body; pub mod ruleset_fingerprint; pub mod skill_lock_name; diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/render_rule_list.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/render_rule_list.rs new file mode 100644 index 0000000000..358d57d1b1 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/render_rule_list.rs @@ -0,0 +1,44 @@ +//! `sqb rules list` text: one `code message` line per rule, or the sorted JSON document. + +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::rule_catalogue::models::RuleListing; + +fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +fn rule_object(rule: &RuleListing) -> JsonValue { + JsonValue::Object(vec![ + ("code".to_owned(), text(&rule.code)), + ("family".to_owned(), text(&rule.family)), + ("message".to_owned(), text(&rule.message)), + ("custom".to_owned(), JsonValue::Bool(rule.custom)), + ( + "subject".to_owned(), + rule.subject.as_deref().map_or(JsonValue::Null, text), + ), + ]) +} + +/// The listing without its trailing newline: `json.dumps({"rules": ...}, sort_keys=True)` or lines. +#[must_use] +pub fn render_rule_list(rules: &[RuleListing], json_output: bool) -> String { + if !json_output { + return rules + .iter() + .map(|rule| format!("{} {}", rule.code, rule.message)) + .collect::>() + .join("\n"); + } + let payload: JsonValue = JsonValue::Object(vec![( + "rules".to_owned(), + JsonValue::Array(rules.iter().map(rule_object).collect()), + )]); + let options: StdlibJsonOptions = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None) + }; + dumps(&payload, &JsonDialect::Stdlib(options)).unwrap_or_default() +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/models.rs b/crates/sqlbuild-rules/src/rule_catalogue/models.rs index 2ebd66fc1b..a3f5fd695c 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/models.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/models.rs @@ -85,3 +85,14 @@ pub struct SkillTarget { pub path: std::path::PathBuf, pub state: String, } + +/// One catalogue rule as `sqb rules list` prints it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RuleListing { + pub code: String, + pub family: String, + pub message: String, + pub custom: bool, + /// The `RuleSubject` value, or `None` when the rule declares none. + pub subject: Option, +} diff --git a/fensu.toml b/fensu.toml index 154893a523..b09c54fa00 100644 --- a/fensu.toml +++ b/fensu.toml @@ -842,7 +842,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=12", - "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=10", + "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs=1", From cf463ff3524b7cd06b4148546ce0f1c052ce0028 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:23:11 +0100 Subject: [PATCH 2/5] feat: run help, usage errors, init, skills, clean and rules list/show in sqb without Python --- Cargo.lock | 2 + Cargo.toml | 2 +- crates/sqlbuild-cli/Cargo.toml | 8 +- .../src/launcher/_helpers/dispatch.rs | 17 +- crates/sqlbuild-cli/src/lib.rs | 3 +- .../src/native_commands/_helpers/argv_host.rs | 119 +++++++ .../src/native_commands/_helpers/decision.rs | 92 +++++ .../src/native_commands/_helpers/execution.rs | 117 ++++++ .../src/native_commands/_helpers/init.rs | 101 ++++++ .../src/native_commands/_helpers/mod.rs | 10 + .../src/native_commands/_helpers/package.rs | 129 +++++++ .../src/native_commands/_helpers/paths.rs | 56 +++ .../native_commands/_helpers/project_facts.rs | 66 ++++ .../src/native_commands/_helpers/rules.rs | 169 +++++++++ .../src/native_commands/_helpers/skills.rs | 49 +++ .../src/native_commands/_helpers/terminal.rs | 93 +++++ .../src/native_commands/constants.rs | 75 ++++ .../src/native_commands/errors.rs | 16 + .../src/native_commands/main/mod.rs | 1 + .../main/run_native_invocation.rs | 52 +++ .../sqlbuild-cli/src/native_commands/mod.rs | 10 + .../src/native_commands/models.rs | 73 ++++ .../sqlbuild-cli/src/native_commands/tests.rs | 14 + .../src/native_commands/tests/helpers.rs | 28 ++ .../native_commands/tests/test_argv_host.rs | 154 ++++++++ .../native_commands/tests/test_decision.rs | 129 +++++++ .../src/native_commands/tests/test_paths.rs | 77 ++++ .../tests/test_project_facts.rs | 60 ++++ .../src/native_commands/tests/test_rules.rs | 108 ++++++ .../src/native_commands/tests/test_types.rs | 68 ++++ .../src/sqlbuild/cli/entry/_test_types.py | 31 ++ .../src/sqlbuild/cli/entry/helpers.py | 92 +++++ .../cli/entry/test_python_free_commands.py | 333 ++++++++++++++++++ 33 files changed, 2343 insertions(+), 11 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/init.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/mod.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/package.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/constants.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/errors.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/main/mod.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/mod.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/models.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/helpers.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_argv_host.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_decision.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_paths.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_project_facts.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_rules.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_types.rs create mode 100644 tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py diff --git a/Cargo.lock b/Cargo.lock index 3a3986c304..4907f98599 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1241,7 +1241,9 @@ dependencies = [ "serde", "serde_json", "sqlbuild-cache", + "sqlbuild-config", "sqlbuild-core", + "sqlbuild-rules", "tempfile", ] diff --git a/Cargo.toml b/Cargo.toml index 0ffdd46f94..994caecd9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,7 +90,7 @@ ruff_text_size = "=0.0.4" rustc-hash = "2" # Recall's local store: SQLite compiled in, so every build has the same engine version. rusqlite = { version = "0.37", features = ["bundled"] } -rustix = { version = "1.1.4", default-features = false, features = ["fs", "std"] } +rustix = { version = "1.1.4", default-features = false, features = ["fs", "std", "termios"] } saphyr-parser = "0.1" regex = "1" serde = { version = "1", features = ["derive"] } diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index 8daa9c84c3..cdfce87880 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "sqlbuild-cli" version.workspace = true -description = "The native `sqb` executable: reused compiles served without Python, every other command handed to the Python CLI." +description = "The native `sqb` executable: reused compiles and Python-free commands served without Python, every other command handed to the Python CLI." edition.workspace = true license-file = "LICENSE" publish.workspace = true @@ -17,18 +17,18 @@ path = "src/bin/sqlbuild.rs" [dependencies] blake3.workspace = true +ctrlc.workspace = true getrandom.workspace = true serde.workspace = true serde_json.workspace = true sqlbuild-cache.workspace = true +sqlbuild-config.workspace = true sqlbuild-core.workspace = true +sqlbuild-rules.workspace = true [target.'cfg(unix)'.dependencies] rustix.workspace = true -[target.'cfg(windows)'.dependencies] -ctrlc.workspace = true - [dev-dependencies] tempfile.workspace = true diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs index c8c85f8e54..e6f689c242 100644 --- a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs +++ b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs @@ -1,4 +1,4 @@ -//! Decide how one invocation runs: natively, from a launch record, or in the Python CLI. +//! Decide how one invocation runs: from a launch record, natively, or in the Python CLI. use std::ffi::OsString; use std::path::{Path, PathBuf}; @@ -10,6 +10,7 @@ use crate::launcher::_helpers::messages::{print_version, report_missing_python_c use crate::launcher::constants::{ HANDOFF_FAILURE_EXIT_CODE, LAUNCH_RECORD_OPTION, VERSION_OPTIONS, }; +use crate::native_commands::main::run_native_invocation::run_native_invocation; use crate::replay::main::serve_reused_compile::serve_reused_compile; /// The launch record path and key for these arguments, when the directory is known. @@ -43,10 +44,16 @@ pub(crate) fn launch(arguments: Vec, started: (Instant, SystemTime)) - return HANDOFF_FAILURE_EXIT_CODE; }; let mut handed: Vec = Vec::with_capacity(arguments.len() + 1); - if let Some((path, key)) = launch_record(&program, &script, &arguments) { - if let Some(code) = serve_reused_compile(&path, &key, started) { - return exit_code(code); - } + let record: Option<(PathBuf, String)> = launch_record(&program, &script, &arguments); + if let Some((path, key)) = &record + && let Some(code) = serve_reused_compile(path, key, started) + { + return exit_code(code); + } + if let Ok(code) = run_native_invocation(&arguments, &script) { + return code; + } + if let Some((path, _)) = record { let mut option = OsString::from(LAUNCH_RECORD_OPTION); option.push(path.as_os_str()); handed.push(option); diff --git a/crates/sqlbuild-cli/src/lib.rs b/crates/sqlbuild-cli/src/lib.rs index 4bbb72fffb..fc36fb01fd 100644 --- a/crates/sqlbuild-cli/src/lib.rs +++ b/crates/sqlbuild-cli/src/lib.rs @@ -1,4 +1,4 @@ -//! The native `sqb` executable: replays reused compiles and hands other invocations to Python. +//! The native `sqb` executable: replays reused compiles, runs Python-free commands and hands the rest to Python. #![forbid(unsafe_code)] @@ -7,4 +7,5 @@ pub mod compute_logs; pub mod diff_report; pub mod dispatch; pub mod launcher; +pub mod native_commands; pub mod replay; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs new file mode 100644 index 0000000000..9e26816a90 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs @@ -0,0 +1,119 @@ +//! The parse's interpreter side without an interpreter: Python's `int`, `repr` and the version. + +use sqlbuild_core::text::main::decimal_digit_value::decimal_digit_value; +use sqlbuild_core::text::main::python_isspace::python_isspace; +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +use crate::argv::main::cli_namespace_defaults::cli_namespace_defaults; +use crate::argv::models::{NamespaceDefault, TypeFailure, Value}; +use crate::argv::types::ArgvHost; +use crate::native_commands::constants::{ + DIGIT_SEPARATOR, INT_ASCII_SPACES, INT_CONVERTER, NEGATIVE_SIGN, POSITIVE_INTEGER_CONVERTER, + POSITIVE_INTEGER_MESSAGE, POSITIVE_SIGN, PYTHON_INT_MAX_STR_DIGITS, ZERO_TEXT, +}; + +/// The ASCII text CPython parses: other decimal digits become ASCII, other spaces a space. +fn ascii_digits_and_spaces(text: &str) -> Option { + text.chars() + .map(|character| { + if character.is_ascii() { + Some(character) + } else if python_isspace(character) { + Some(' ') + } else { + decimal_digit_value(character).and_then(|digit| char::from_digit(digit, 10)) + } + }) + .collect() +} + +/// The canonical decimal text of Python's `int(text)`, or `None` where it raises `ValueError`. +pub(crate) fn python_int(text: &str) -> Option { + let ascii: String = ascii_digits_and_spaces(text)?; + let trimmed: &str = ascii.trim_matches(|character: char| INT_ASCII_SPACES.contains(character)); + let (negative, body): (bool, &str) = match trimmed.strip_prefix(NEGATIVE_SIGN) { + Some(body) => (true, body), + None => ( + false, + trimmed.strip_prefix(POSITIVE_SIGN).unwrap_or(trimmed), + ), + }; + let mut digits: String = String::with_capacity(body.len()); + let mut after_digit: bool = false; + for character in body.chars() { + if character.is_ascii_digit() { + digits.push(character); + after_digit = true; + } else if character == DIGIT_SEPARATOR && after_digit { + after_digit = false; + } else { + return None; + } + } + if !after_digit || digits.len() > PYTHON_INT_MAX_STR_DIGITS { + return None; + } + let significant: &str = digits.trim_start_matches('0'); + Some(match (significant.is_empty(), negative) { + (true, _) => ZERO_TEXT.to_owned(), + (false, true) => format!("{NEGATIVE_SIGN}{significant}"), + (false, false) => significant.to_owned(), + }) +} + +/// Converts what it can as Python would and records any converter only Python can apply. +#[derive(Debug, Default)] +pub(crate) struct NativeArgvHost { + /// Whether the parse met a converter, such as `--vars`, that needs the interpreter. + pub(crate) needs_python: bool, +} + +impl ArgvHost for NativeArgvHost { + fn convert(&mut self, converter: &str, text: &str) -> Result { + match converter { + INT_CONVERTER => python_int(text).map(Value::Int).ok_or(TypeFailure::Invalid), + POSITIVE_INTEGER_CONVERTER => { + let number: String = python_int(text).ok_or(TypeFailure::Invalid)?; + if number == ZERO_TEXT || number.starts_with(NEGATIVE_SIGN) { + return Err(TypeFailure::ArgumentType( + POSITIVE_INTEGER_MESSAGE.to_owned(), + )); + } + Ok(Value::Int(number)) + } + _ => { + self.needs_python = true; + Ok(Value::None) + } + } + } + + fn repr(&self, text: &str) -> String { + python_str_repr(text) + } + + fn program_version(&mut self) -> String { + env!("CARGO_PKG_VERSION").to_owned() + } +} + +/// The parse's starting namespace: every `CliNamespace` default. +pub(crate) fn namespace_defaults() -> Vec<(String, Value)> { + cli_namespace_defaults() + .iter() + .map(|(name, default)| { + let value: Value = match default { + NamespaceDefault::None => Value::None, + NamespaceDefault::Bool(flag) => Value::Bool(*flag), + NamespaceDefault::Int(number) => Value::Int(number.to_string()), + NamespaceDefault::Text(text) => Value::Str((*text).to_owned()), + NamespaceDefault::EmptyList => Value::List(Vec::new()), + NamespaceDefault::EmptyDict => Value::EmptyDict, + NamespaceDefault::CompileLineageFast | NamespaceDefault::ColumnLineageRich => { + Value::Host(0) + } + }; + ((*name).to_owned(), value) + }) + .collect() +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs new file mode 100644 index 0000000000..30ede63f68 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -0,0 +1,92 @@ +//! Which parsed invocations run without Python, decided from the parsed values alone. + +use std::path::PathBuf; + +use crate::argv::models::{Namespace, Value}; +use crate::dispatch::main::invocation_policy::invocation_policy; +use crate::dispatch::models::{CommandFacts, InvocationPolicy}; +use crate::native_commands::constants::{ + CLEAN_COMMAND, INIT_COMMAND, RULES_COMMAND, RULES_LIST_ACTION, RULES_SHOW_ACTION, + SKILLS_COMMAND, +}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{ + NativeCommand, NativeInvocation, RulesAction, RulesOptions, SkillsOptions, +}; + +fn text(namespace: &Namespace, name: &str) -> Option { + match namespace.get(name) { + Some(Value::Str(text) | Value::Path(text)) => Some(text.clone()), + _ => None, + } +} + +fn flag(namespace: &Namespace, name: &str) -> bool { + namespace.get(name) == Some(&Value::Bool(true)) +} + +/// The strings of a list attribute, or `None` when it holds anything else. +fn texts(namespace: &Namespace, name: &str) -> Option> { + match namespace.get(name) { + None | Some(Value::None) => Some(Vec::new()), + Some(Value::List(items)) => items + .iter() + .map(|item| match item { + Value::Str(text) => Some(text.clone()), + _ => None, + }) + .collect(), + Some(_) => None, + } +} + +fn rules_command(namespace: &Namespace) -> Result { + let action: RulesAction = match text(namespace, "rules_command").as_deref() { + None | Some(RULES_LIST_ACTION) => RulesAction::List, + Some(RULES_SHOW_ACTION) => RulesAction::Show, + Some(_) => return Err(PythonNeeded::Command), + }; + Ok(NativeCommand::Rules(RulesOptions { + action, + selector: text(namespace, "rules_rule_selector"), + json: flag(namespace, "json"), + })) +} + +fn native_command( + command: Option<&str>, + namespace: &Namespace, +) -> Result { + match command { + None | Some(CLEAN_COMMAND) => Ok(NativeCommand::Nothing), + Some(INIT_COMMAND) => Ok(NativeCommand::Init), + Some(SKILLS_COMMAND) => Ok(NativeCommand::Skills(SkillsOptions { + global_install: flag(namespace, "skills_global"), + targets: texts(namespace, "skills_target").ok_or(PythonNeeded::Arguments)?, + force: flag(namespace, "skills_force"), + })), + Some(RULES_COMMAND) => rules_command(namespace), + Some(_) => Err(PythonNeeded::Command), + } +} + +/// The command `namespace` runs, when it runs without Python; `--debug` always uses Python. +pub(crate) fn native_invocation(namespace: &Namespace) -> Result { + if flag(namespace, "debug") { + return Err(PythonNeeded::Command); + } + let command_name: Option = text(namespace, "command"); + let command: NativeCommand = native_command(command_name.as_deref(), namespace)?; + let policy: InvocationPolicy = invocation_policy(&CommandFacts { + command: command_name.clone(), + ..CommandFacts::default() + }); + Ok(NativeInvocation { + command, + command_name, + project_dir: text(namespace, "project_dir").map(PathBuf::from), + target: text(namespace, "target"), + creates_project: policy.creates_project, + maintains_skills: policy.maintains_skills, + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs new file mode 100644 index 0000000000..278a397f81 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -0,0 +1,117 @@ +//! Running a Python-free command: its checks, then its output with the observed services. + +use std::io; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use crate::argv::models::Stream; +use crate::compute_logs::constants::DEFAULT_RETENTION_COUNT; +use crate::compute_logs::main::finish_capture::finish_capture; +use crate::compute_logs::main::open_log_root::open_log_root; +use crate::compute_logs::main::start_capture::start_capture; +use crate::compute_logs::models::{CaptureEnd, CaptureStart, OpenCapture}; +use crate::native_commands::_helpers::init::run_init; +use crate::native_commands::_helpers::package::installed_package; +use crate::native_commands::_helpers::paths::project_location; +use crate::native_commands::_helpers::project_facts::has_no_runtime_extensions; +use crate::native_commands::_helpers::rules::run_rules; +use crate::native_commands::_helpers::skills::{run_skills, skill_freshness_notice}; +use crate::native_commands::_helpers::terminal::{exit_on_interrupt, write_text}; +use crate::native_commands::constants::UNKNOWN_COMMAND; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{ + CommandOutput, InstalledPackage, NativeCommand, NativeInvocation, ProjectLocation, +}; + +/// The capture Python's `dispatch_with_compute_logs` opens; a failure only loses the log. +fn open_capture( + invocation: &NativeInvocation, + project: &ProjectLocation, + started_at: SystemTime, +) -> io::Result { + let retention: i64 = i64::try_from(DEFAULT_RETENTION_COUNT).unwrap_or(i64::MAX); + let root: PathBuf = open_log_root(&project.given, None, Some(retention)) + .map_err(|error| io::Error::other(format!("{error:?}")))?; + start_capture(CaptureStart { + root, + command: invocation + .command_name + .clone() + .unwrap_or_else(|| UNKNOWN_COMMAND.to_owned()), + project_dir: project.resolved.display().to_string(), + target: invocation.target.clone(), + started_at, + }) +} + +/// Print an observed command's output after the skill notice, retaining both in its capture. +fn finish_observed( + invocation: &NativeInvocation, + (project, package): (&ProjectLocation, &InstalledPackage), + started_at: SystemTime, + output: CommandOutput, +) -> i32 { + let notice: String = if invocation.maintains_skills { + skill_freshness_notice(&project.given, package) + } else { + String::new() + }; + let capture: io::Result = open_capture(invocation, project, started_at); + let stderr: String = notice + &output.stderr; + write_text(Stream::Stderr, &stderr); + write_text(Stream::Stdout, &output.stdout); + if let Ok(capture) = capture { + let finished: io::Result<()> = finish_capture( + capture, + &CaptureEnd { + stdout: output.stdout.into_bytes(), + stderr: stderr.into_bytes(), + exit_code: i64::from(output.exit_code), + }, + ); + drop(finished); + } + output.exit_code +} + +/// What an observed command prints, unless it needs Python after all. +fn observed_output( + command: &NativeCommand, + project: &ProjectLocation, + package: &InstalledPackage, +) -> Result { + if !has_no_runtime_extensions(&project.given) { + return Err(PythonNeeded::Project); + } + match command { + NativeCommand::Nothing => Ok(CommandOutput::default()), + NativeCommand::Skills(options) => run_skills(&project.resolved, package, options), + NativeCommand::Rules(options) => run_rules(&project.resolved, options), + NativeCommand::Init => Err(PythonNeeded::Command), + } +} + +/// Run a parsed Python-free command; an error hands it to Python before any side effect. +pub(crate) fn run_command( + invocation: &NativeInvocation, + python_cli: &Path, + error_color: bool, +) -> Result { + exit_on_interrupt(); + let package: InstalledPackage = installed_package(python_cli)?; + let project: ProjectLocation = project_location(invocation.project_dir.as_deref())?; + if invocation.creates_project { + let output: CommandOutput = run_init(&project.resolved, &package, error_color)?; + write_text(Stream::Stderr, &output.stderr); + write_text(Stream::Stdout, &output.stdout); + return Ok(output.exit_code); + } + let started_at: SystemTime = SystemTime::now(); + let output: CommandOutput = observed_output(&invocation.command, &project, &package)?; + Ok(finish_observed( + invocation, + (&project, &package), + started_at, + output, + )) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs new file mode 100644 index 0000000000..dfee4cd22b --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs @@ -0,0 +1,101 @@ +//! `sqb init`: scaffold a blank project and install its skills. + +use std::path::{Path, PathBuf}; + +use sqlbuild_config::workspace::errors::WorkspaceError; +use sqlbuild_config::workspace::main::install_skills::install_skills; +use sqlbuild_config::workspace::models::SkillInstallRequest; +use sqlbuild_core::cli_text::main::format_coded_error::format_coded_error; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::init_command::errors::ScaffoldError; +use sqlbuild_core::init_command::main::init_project_name::init_project_name; +use sqlbuild_core::init_command::main::init_project_toml::init_project_toml; +use sqlbuild_core::init_command::main::render_init_summary::render_init_summary; +use sqlbuild_core::init_command::main::scaffold_blank_project::scaffold_blank_project; + +use crate::dispatch::main::failure_report::failure_report; +use crate::dispatch::models::CommandFailure; +use crate::native_commands::_helpers::skills::home_dir; +use crate::native_commands::_helpers::terminal::stdout_supports_color; +use crate::native_commands::constants::{INIT_FAILURE_PREFIX, USER_ERROR_CLASS}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{CommandOutput, InstalledPackage}; + +/// A failure once the project file exists, which a Python rerun would report differently. +fn failed(stderr: String) -> Result { + Ok(CommandOutput { + stdout: String::new(), + stderr, + exit_code: 1, + }) +} + +/// The project's skills could not be installed: what the Python CLI prints for its error. +fn skills_failure(error: &WorkspaceError, use_color: bool) -> Result { + let WorkspaceError::User { + code, + message, + help, + } = error + else { + return failed(format!("{INIT_FAILURE_PREFIX}{error:?}\n")); + }; + let (_, log_message) = failure_report(CommandFailure::User); + let coded: String = format_coded_error( + (code, message, help.as_deref()), + true, + CliStyle { use_color }, + ); + failed(format!( + "{log_message}\n{USER_ERROR_CLASS}: {message}\n{coded}\n\n" + )) +} + +/// Create the project; a failure before the project file exists is Python's to report. +pub(crate) fn run_init( + base_dir: &Path, + package: &InstalledPackage, + error_color: bool, +) -> Result { + let project_file: PathBuf = base_dir.join(PROJECT_CONFIG_FILENAME); + if project_file.exists() { + return Err(PythonNeeded::Failure); + } + let directory_name: &str = match base_dir.file_name() { + Some(name) => name.to_str().ok_or(PythonNeeded::Arguments)?, + None => "", + }; + let home: PathBuf = home_dir()?; + let project_name: String = init_project_name(directory_name); + if let Err(ScaffoldError { error, path }) = + scaffold_blank_project(base_dir, &init_project_toml(&project_name)) + { + if !project_file.exists() { + return Err(PythonNeeded::Failure); + } + return failed(format!( + "{INIT_FAILURE_PREFIX}{error}: {}\n", + path.display() + )); + } + if let Err(error) = install_skills(&SkillInstallRequest { + project_dir: base_dir.to_path_buf(), + skills_root: package.skills_root.clone(), + home_dir: home, + global_install: false, + requested_targets: Vec::new(), + force: false, + }) { + return skills_failure(&error, error_color); + } + Ok(CommandOutput { + stdout: render_init_summary( + &project_name, + CliStyle { + use_color: stdout_supports_color(), + }, + ), + ..CommandOutput::default() + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/mod.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/mod.rs new file mode 100644 index 0000000000..c01acfd621 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod argv_host; +pub(crate) mod decision; +pub(crate) mod execution; +pub(crate) mod init; +pub(crate) mod package; +pub(crate) mod paths; +pub(crate) mod project_facts; +pub(crate) mod rules; +pub(crate) mod skills; +pub(crate) mod terminal; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/package.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/package.rs new file mode 100644 index 0000000000..ce77e6aa57 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/package.rs @@ -0,0 +1,129 @@ +//! The installed `sqlbuild` package Python would import, found without starting Python. + +use std::path::{Path, PathBuf}; + +use crate::native_commands::constants::{ + PACKAGE_MARKER, PACKAGE_NAME, PATH_FILE_COMMENT, PATH_FILE_EXTENSION, + PATH_FILE_IMPORT_PREFIXES, PYTHON_PATH_VARIABLES, SKILLS_DIRECTORY, +}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::InstalledPackage; + +/// The site directories of the installation whose scripts directory holds `script`. +#[cfg(not(windows))] +fn site_directories(prefix: &Path) -> Vec { + use crate::native_commands::constants::{ + INTERPRETER_DIRECTORY_PREFIX, LIBRARY_DIRECTORY, SITE_DIRECTORIES, + }; + + let Ok(entries) = std::fs::read_dir(prefix.join(LIBRARY_DIRECTORY)) else { + return Vec::new(); + }; + let mut directories: Vec = Vec::new(); + for entry in entries.flatten() { + if !entry + .file_name() + .to_string_lossy() + .starts_with(INTERPRETER_DIRECTORY_PREFIX) + { + continue; + } + for site in SITE_DIRECTORIES { + let directory: PathBuf = entry.path().join(site); + if directory.is_dir() { + directories.push(directory); + } + } + } + directories.sort(); + directories +} + +#[cfg(windows)] +fn site_directories(prefix: &Path) -> Vec { + use crate::native_commands::constants::WINDOWS_SITE_DIRECTORY; + + let directory: PathBuf = WINDOWS_SITE_DIRECTORY + .iter() + .fold(prefix.to_path_buf(), |path, part| path.join(part)); + if directory.is_dir() { + vec![directory] + } else { + Vec::new() + } +} + +fn package_beneath(directory: &Path) -> Option { + let package: PathBuf = directory.join(PACKAGE_NAME); + package.join(PACKAGE_MARKER).is_file().then_some(package) +} + +/// The paths a site directory's `.pth` files add, in the order `site` reads them. +fn path_file_entries(site: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(site) else { + return Vec::new(); + }; + let mut files: Vec = entries + .flatten() + .map(|entry| entry.path()) + .filter(|path| { + path.extension() + .is_some_and(|extension| extension == PATH_FILE_EXTENSION) + }) + .collect(); + files.sort(); + let mut added: Vec = Vec::new(); + for file in files { + let Ok(text) = std::fs::read_to_string(&file) else { + continue; + }; + for line in text.lines() { + let line: &str = line.trim_end(); + if line.is_empty() + || line.starts_with(PATH_FILE_COMMENT) + || PATH_FILE_IMPORT_PREFIXES + .iter() + .any(|prefix| line.starts_with(prefix)) + { + continue; + } + added.push(site.join(line)); + } + } + added +} + +/// The one `sqlbuild` package the installation beside `script` holds, unless that is unclear. +pub(crate) fn installed_package(script: &Path) -> Result { + if PYTHON_PATH_VARIABLES + .iter() + .any(|name| std::env::var_os(name).is_some_and(|value| !value.is_empty())) + { + return Err(PythonNeeded::Installation); + } + let script: PathBuf = std::fs::canonicalize(script).map_err(|_| PythonNeeded::Installation)?; + let prefix: &Path = script + .parent() + .and_then(Path::parent) + .ok_or(PythonNeeded::Installation)?; + let mut found: Vec = Vec::new(); + for site in site_directories(prefix) { + let package: Option = package_beneath(&site).or_else(|| { + path_file_entries(&site) + .iter() + .find_map(|directory| package_beneath(directory)) + }); + if let Some(package) = package { + found.push(std::fs::canonicalize(&package).unwrap_or(package)); + } + } + found.dedup(); + let [package] = found.as_slice() else { + return Err(PythonNeeded::Installation); + }; + Ok(InstalledPackage { + skills_root: SKILLS_DIRECTORY + .iter() + .fold(package.clone(), |path, part| path.join(part)), + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs new file mode 100644 index 0000000000..c50eea5af9 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs @@ -0,0 +1,56 @@ +//! `pathlib.Path` construction and `Path.resolve()` over native paths. + +use std::path::{Component, Path, PathBuf}; + +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::ProjectLocation; + +/// `Path(text)`: repeated separators, `.` parts and trailing separators dropped. +pub(crate) fn pathlib_path(text: &Path) -> PathBuf { + let path: PathBuf = text + .components() + .filter(|component| *component != Component::CurDir) + .collect(); + if path.as_os_str().is_empty() { + return PathBuf::from("."); + } + path +} + +/// `Path.resolve()`: absolute, with every existing prefix's links followed, as `realpath` does. +pub(crate) fn resolved_path(path: &Path, cwd: &Path) -> PathBuf { + let absolute: PathBuf = if path.is_absolute() { + path.to_path_buf() + } else { + cwd.join(path) + }; + if let Ok(real) = std::fs::canonicalize(&absolute) { + return real; + } + let mut resolved: PathBuf = PathBuf::new(); + for component in absolute.components() { + match component { + Component::Prefix(_) | Component::RootDir => resolved.push(component.as_os_str()), + Component::CurDir => {} + Component::ParentDir => { + resolved.pop(); + } + Component::Normal(name) => { + let candidate: PathBuf = resolved.join(name); + resolved = std::fs::canonicalize(&candidate).unwrap_or(candidate); + } + } + } + resolved +} + +/// The project directory as given and as `Path.resolve()` reads it. +pub(crate) fn project_location(given: Option<&Path>) -> Result { + let cwd: PathBuf = std::env::current_dir().map_err(|_| PythonNeeded::Installation)?; + let given: PathBuf = match given { + Some(raw) => pathlib_path(raw), + None => cwd.clone(), + }; + let resolved: PathBuf = resolved_path(&given, &cwd); + Ok(ProjectLocation { given, resolved }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs new file mode 100644 index 0000000000..e94539aa29 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs @@ -0,0 +1,66 @@ +//! Project facts that decide, before any side effect, whether a command needs Python. + +use std::path::Path; + +use sqlbuild_config::project::main::load_project_config::load_project_config; +use sqlbuild_config::project::models::{Built, ConfigPaths}; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; + +use crate::native_commands::constants::{ + CUSTOM_RULES_DIRECTORY, EVENT_EXPORTERS_DIRECTORY, LEGACY_LOCAL_CONFIG_FILENAME, + LEGACY_PROJECT_CONFIG_FILENAME, LIFECYCLE_SINK_PATH, LOCAL_CONFIG_FILENAME, SINKS_DIRECTORY, +}; + +fn object_field<'a>(built: &'a Built, name: &str) -> Option<&'a Built> { + match built { + Built::Object(_, fields) => fields + .iter() + .find(|(field, _)| *field == name) + .map(|(_, value)| value), + _ => None, + } +} + +/// Whether the loaded configuration names a lifecycle sink, which only Python can bind. +fn names_lifecycle_sinks(config: &Built) -> bool { + let named: Option<&Built> = LIFECYCLE_SINK_PATH + .iter() + .try_fold(config, |value, name| object_field(value, name)); + match named { + None => false, + Some(Built::Dict(entries)) => !entries.is_empty(), + Some(_) => true, + } +} + +/// The four configuration files of `project_dir`. +fn config_paths(project_dir: &Path) -> ConfigPaths { + ConfigPaths { + project: project_dir.join(PROJECT_CONFIG_FILENAME), + legacy_project: project_dir.join(LEGACY_PROJECT_CONFIG_FILENAME), + local: project_dir.join(LOCAL_CONFIG_FILENAME), + legacy_local: project_dir.join(LEGACY_LOCAL_CONFIG_FILENAME), + } +} + +/// Whether `discover_runtime_extensions` would import nothing and accept the project file. +pub(crate) fn has_no_runtime_extensions(project_dir: &Path) -> bool { + if project_dir.join(EVENT_EXPORTERS_DIRECTORY).exists() + || project_dir.join(SINKS_DIRECTORY).is_dir() + { + return false; + } + let paths: ConfigPaths = config_paths(project_dir); + if !paths.project.exists() { + return true; + } + match load_project_config(&paths) { + Ok(config) => !names_lifecycle_sinks(&config), + Err(_) => false, + } +} + +/// Whether the project defines custom rules, which Python imports. +pub(crate) fn has_custom_rules(project_dir: &Path) -> bool { + project_dir.join(CUSTOM_RULES_DIRECTORY).is_dir() +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs new file mode 100644 index 0000000000..c956dd94fe --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs @@ -0,0 +1,169 @@ +//! `sqb rules list|show` over the built-in catalogue, when no custom rule or option needs Python. + +use std::collections::BTreeMap; +use std::path::Path; + +use sqlbuild_rules::configuration::main::forbidden_exceptions::forbidden_exceptions; +use sqlbuild_rules::configuration::main::load_rules_config::load_rules_config; +use sqlbuild_rules::models::{RuleMetadata, RulesConfig}; +use sqlbuild_rules::rule_catalogue::main::format_rule_text::format_rule_text; +use sqlbuild_rules::rule_catalogue::main::render_rule_list::render_rule_list; +use sqlbuild_rules::rule_catalogue::main::sorted_catalogue_order::sorted_catalogue_order; +use sqlbuild_rules::rule_catalogue::models::{ + CatalogueRule, OptionKind, OptionSpec, OptionText, OptionValue, RuleListing, RuleText, +}; +use sqlbuild_rules::rules::main::builtin_options::builtin_options; +use sqlbuild_rules::rules::main::catalogue::catalogue; +use sqlbuild_rules::rules::models::{BuiltinOption, BuiltinOptionKind}; + +use crate::native_commands::_helpers::project_facts::has_custom_rules; +use crate::native_commands::constants::{ + EMPTY_TUPLE_REPR, FALSE_REPR, FAMILY_SUFFIX_LENGTH, MODEL_SUBJECT, PROJECT_SUBJECT, + RULE_SELECTOR_REQUIRED, RULES_USAGE_EXIT_CODE, UNKNOWN_RULE_PREFIX, +}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{CommandOutput, RulesAction, RulesOptions}; + +/// `RuleOption.boolean(...)` or `RuleOption.string_list(...)` as the catalogue validates it. +fn option_spec(option: &BuiltinOption) -> OptionSpec { + let (value_type, default, item_type) = match option.kind { + BuiltinOptionKind::Boolean => (OptionKind::Bool, OptionValue::Bool(false), None), + BuiltinOptionKind::StringList => ( + OptionKind::Tuple, + OptionValue::Tuple(Vec::new()), + Some(OptionKind::Str), + ), + }; + OptionSpec { + name: option.name.to_owned(), + description: option.description.to_owned(), + value_type, + default, + choices: Vec::new(), + minimum: None, + maximum: None, + minimum_items: None, + item_type, + } +} + +/// The option as `rules show` prints it: the `repr` of its default, which is also effective. +fn option_text(option: &BuiltinOption) -> OptionText { + let default_repr: &str = match option.kind { + BuiltinOptionKind::Boolean => FALSE_REPR, + BuiltinOptionKind::StringList => EMPTY_TUPLE_REPR, + }; + OptionText { + name: option.name.to_owned(), + description: option.description.to_owned(), + default_repr: default_repr.to_owned(), + effective_repr: default_repr.to_owned(), + } +} + +/// The rule code without its three-digit number, as the built-in `Rule` records it. +fn family(rule: &RuleMetadata) -> String { + rule.code[..rule.code.len().saturating_sub(FAMILY_SUFFIX_LENGTH)].to_owned() +} + +fn listing(rule: &RuleMetadata) -> RuleListing { + RuleListing { + code: rule.code.clone(), + family: family(rule), + message: rule.message.clone(), + custom: false, + subject: Some( + if rule.project_wide { + PROJECT_SUBJECT + } else { + MODEL_SUBJECT + } + .to_owned(), + ), + } +} + +fn rule_text(rule: &RuleMetadata) -> RuleText { + RuleText { + code: rule.code.clone(), + family: family(rule), + slug: rule.slug.clone(), + message: rule.message.clone(), + remediation: rule.remediation.clone(), + enabled_by_default: rule.enabled_by_default, + custom: false, + source: None, + options: builtin_options(&rule.code) + .iter() + .map(option_text) + .collect(), + good_example: rule + .guidance + .as_ref() + .map(|guidance| guidance.good_example.clone()), + } +} + +/// The built-in catalogue in the order `build_rule_catalogue` sorts it. +fn sorted_catalogue() -> Result, PythonNeeded> { + let rules: Vec = catalogue(); + let views: Vec = rules + .iter() + .map(|rule| CatalogueRule { + code: rule.code.clone(), + options: builtin_options(&rule.code) + .iter() + .map(option_spec) + .collect(), + }) + .collect(); + let order: Vec = + sorted_catalogue_order(&views, &BTreeMap::new()).map_err(|_| PythonNeeded::Failure)?; + Ok(order + .into_iter() + .filter_map(|index| rules.get(index).cloned()) + .collect()) +} + +fn printed(stdout: String, exit_code: i32) -> CommandOutput { + CommandOutput { + stdout: format!("{stdout}\n"), + stderr: String::new(), + exit_code, + } +} + +fn show(selector: Option<&str>, rules: &[RuleMetadata], config: &RulesConfig) -> CommandOutput { + let Some(selector) = selector else { + return printed(RULE_SELECTOR_REQUIRED.to_owned(), RULES_USAGE_EXIT_CODE); + }; + match rules.iter().find(|rule| rule.code == selector) { + Some(rule) => printed(format_rule_text(&rule_text(rule), Some(config)), 0), + None => printed( + format!("{UNKNOWN_RULE_PREFIX}{selector}"), + RULES_USAGE_EXIT_CODE, + ), + } +} + +/// What `rules list|show` prints for `project_dir`, unless it needs Python. +pub(crate) fn run_rules( + project_dir: &Path, + options: &RulesOptions, +) -> Result { + if has_custom_rules(project_dir) { + return Err(PythonNeeded::Project); + } + let config: RulesConfig = load_rules_config(project_dir).map_err(|_| PythonNeeded::Failure)?; + if forbidden_exceptions(&config).is_some() || !config.rule_options.is_empty() { + return Err(PythonNeeded::Project); + } + let rules: Vec = sorted_catalogue()?; + Ok(match options.action { + RulesAction::List => { + let listings: Vec = rules.iter().map(listing).collect(); + printed(render_rule_list(&listings, options.json), 0) + } + RulesAction::Show => show(options.selector.as_deref(), &rules, &config), + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs new file mode 100644 index 0000000000..4272d40185 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs @@ -0,0 +1,49 @@ +//! `sqb skills` and the skill freshness check every observed command makes first. + +use std::path::{Path, PathBuf}; + +use sqlbuild_config::workspace::main::install_skills::install_skills; +use sqlbuild_config::workspace::main::maintain_skills::maintain_skills; +use sqlbuild_config::workspace::models::SkillInstallRequest; + +use crate::native_commands::constants::{HOME_VARIABLE, SKILL_FILES_HEADER}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{CommandOutput, InstalledPackage, SkillsOptions}; + +/// `Path.home()` from `HOME`, unless only Python's password lookup could tell. +pub(crate) fn home_dir() -> Result { + std::env::var_os(HOME_VARIABLE) + .filter(|home| !home.is_empty()) + .map(PathBuf::from) + .ok_or(PythonNeeded::Installation) +} + +/// The packaged skill files installed into `project_dir`; Python reports any failure. +pub(crate) fn run_skills( + project_dir: &Path, + package: &InstalledPackage, + options: &SkillsOptions, +) -> Result { + let written: Vec = install_skills(&SkillInstallRequest { + project_dir: project_dir.to_path_buf(), + skills_root: package.skills_root.clone(), + home_dir: home_dir()?, + global_install: options.global_install, + requested_targets: options.targets.clone(), + force: options.force, + }) + .map_err(|_| PythonNeeded::Failure)?; + let mut stdout: String = SKILL_FILES_HEADER.to_owned(); + for path in written { + stdout.push_str(&format!(" {}\n", path.display())); + } + Ok(CommandOutput { + stdout, + ..CommandOutput::default() + }) +} + +/// The stale-skill notice, refreshing generated skills; failures stay silent, as in Python. +pub(crate) fn skill_freshness_notice(project_dir: &Path, package: &InstalledPackage) -> String { + maintain_skills(project_dir, &package.skills_root).unwrap_or_default() +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs new file mode 100644 index 0000000000..3a584e7abd --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs @@ -0,0 +1,93 @@ +//! The process's terminal: its width, colour support, stream writes and Ctrl+C. + +use std::io::{self, IsTerminal, Write}; + +use crate::argv::models::Stream; +use crate::native_commands::_helpers::argv_host::python_int; +use crate::native_commands::constants::{COLUMNS_VARIABLE, FALLBACK_COLUMNS, NO_COLOR_VARIABLE}; +use crate::native_commands::errors::PythonNeeded; + +/// `int(os.environ["COLUMNS"])`, `0` when unset or not an integer; beyond 64 bits is Python's. +fn columns_variable() -> Result { + let Ok(text) = std::env::var(COLUMNS_VARIABLE) else { + return Ok(0); + }; + let Some(number) = python_int(&text) else { + return Ok(0); + }; + number.parse::().map_err(|_| PythonNeeded::Arguments) +} + +/// The width of the terminal on stdout, or the fallback when stdout is not one. +#[cfg(unix)] +fn stdout_columns() -> Result { + match rustix::termios::tcgetwinsize(io::stdout()) { + Ok(size) if size.ws_col > 0 => Ok(i64::from(size.ws_col)), + _ => Ok(FALLBACK_COLUMNS), + } +} + +/// A console's width needs the console API; only a redirected stdout has a known width. +#[cfg(not(unix))] +fn stdout_columns() -> Result { + if io::stdout().is_terminal() { + return Err(PythonNeeded::Arguments); + } + Ok(FALLBACK_COLUMNS) +} + +/// `shutil.get_terminal_size().columns`, unless only Python can tell. +pub(crate) fn terminal_columns() -> Result { + let columns: i64 = columns_variable()?; + if columns > 0 { + return Ok(columns); + } + stdout_columns() +} + +/// `stdout_supports_color`: stdout is a terminal and `NO_COLOR` is unset. +pub(crate) fn stdout_supports_color() -> bool { + std::env::var_os(NO_COLOR_VARIABLE).is_none() && io::stdout().is_terminal() +} + +/// `\n` as a Python text stream writes it on this platform. +fn console_bytes(text: &str) -> Vec { + if cfg!(windows) { + text.replace('\n', "\r\n").into_bytes() + } else { + text.as_bytes().to_vec() + } +} + +/// Write and flush one text; a closed or full stream drops it, as the parse's writes do. +pub(crate) fn write_text(stream: Stream, text: &str) { + if text.is_empty() { + return; + } + let bytes: Vec = console_bytes(text); + let written: io::Result<()> = match stream { + Stream::Stdout => { + let mut output = io::stdout().lock(); + output.write_all(&bytes).and_then(|()| output.flush()) + } + Stream::Stderr => { + let mut output = io::stderr().lock(); + output.write_all(&bytes).and_then(|()| output.flush()) + } + }; + drop(written); +} + +/// End the process with status 130 on Ctrl+C; handing off by `exec` restores the default. +#[cfg(unix)] +pub(crate) fn exit_on_interrupt() { + use crate::native_commands::constants::INTERRUPTED_EXIT_CODE; + + drop(ctrlc::set_handler(|| { + std::process::exit(INTERRUPTED_EXIT_CODE) + })); +} + +/// The Python child waited for alone handles Ctrl+C, so the native command keeps the default. +#[cfg(not(unix))] +pub(crate) fn exit_on_interrupt() {} diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs new file mode 100644 index 0000000000..9c01a84a6b --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -0,0 +1,75 @@ +//! Names and limits of the Python-free command path. + +/// `type=` converters the native parse applies itself; any other needs Python. +pub(crate) const INT_CONVERTER: &str = "int"; +pub(crate) const POSITIVE_INTEGER_CONVERTER: &str = "_positive_integer"; +pub(crate) const POSITIVE_INTEGER_MESSAGE: &str = "value must be a positive integer"; +pub(crate) const NO_COLOR_OPTION: &str = "--no-color"; +pub(crate) const NO_COLOR_VARIABLE: &str = "NO_COLOR"; +pub(crate) const COLUMNS_VARIABLE: &str = "COLUMNS"; +/// `shutil.get_terminal_size`'s fallback width. +pub(crate) const FALLBACK_COLUMNS: i64 = 80; +/// Variables that change where Python imports `sqlbuild` from. +pub(crate) const PYTHON_PATH_VARIABLES: [&str; 2] = ["PYTHONPATH", "PYTHONHOME"]; +pub(crate) const PACKAGE_NAME: &str = "sqlbuild"; +pub(crate) const PACKAGE_MARKER: &str = "__init__.py"; +pub(crate) const PATH_FILE_EXTENSION: &str = "pth"; +/// `.pth` lines that `site` executes instead of adding to the search path. +pub(crate) const PATH_FILE_IMPORT_PREFIXES: [&str; 2] = ["import ", "import\t"]; +pub(crate) const PATH_FILE_COMMENT: char = '#'; +#[cfg(not(windows))] +pub(crate) const LIBRARY_DIRECTORY: &str = "lib"; +#[cfg(not(windows))] +pub(crate) const INTERPRETER_DIRECTORY_PREFIX: &str = "python3"; +#[cfg(not(windows))] +pub(crate) const SITE_DIRECTORIES: [&str; 2] = ["site-packages", "dist-packages"]; +#[cfg(windows)] +pub(crate) const WINDOWS_SITE_DIRECTORY: [&str; 2] = ["Lib", "site-packages"]; +/// The packaged skill beneath the `sqlbuild` package directory. +pub(crate) const SKILLS_DIRECTORY: [&str; 3] = [".agents", "skills", "sqlbuild"]; +pub(crate) const HOME_VARIABLE: &str = "HOME"; +/// The exit status of a command stopped by Ctrl+C. +#[cfg(unix)] +pub(crate) const INTERRUPTED_EXIT_CODE: i32 = 130; +/// The command name compute logs record for a bare `sqb`. +pub(crate) const UNKNOWN_COMMAND: &str = "unknown"; +/// Commands with no handler: observed, then successful. +pub(crate) const CLEAN_COMMAND: &str = "clean"; +pub(crate) const INIT_COMMAND: &str = "init"; +pub(crate) const SKILLS_COMMAND: &str = "skills"; +pub(crate) const RULES_COMMAND: &str = "rules"; +pub(crate) const RULES_LIST_ACTION: &str = "list"; +pub(crate) const RULES_SHOW_ACTION: &str = "show"; +/// Project directories holding Python the command would import. +pub(crate) const EVENT_EXPORTERS_DIRECTORY: &str = "event_exporters"; +pub(crate) const SINKS_DIRECTORY: &str = "sinks"; +pub(crate) const CUSTOM_RULES_DIRECTORY: &str = "rules"; +pub(crate) const SKILL_FILES_HEADER: &str = "Updated SQLBuild skill files:\n"; +pub(crate) const RULE_SELECTOR_REQUIRED: &str = "A rule code or family prefix is required"; +/// `run_rules_command`'s exit status for a missing or unknown rule. +pub(crate) const RULES_USAGE_EXIT_CODE: i32 = 2; +pub(crate) const LEGACY_PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.yml"; +pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; +pub(crate) const LEGACY_LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.yml"; +/// The class Python's last-resort log handler names for a user error. +pub(crate) const USER_ERROR_CLASS: &str = "sqlbuild._native.CliUserError"; +pub(crate) const INIT_FAILURE_PREFIX: &str = "sqb init: "; +/// A built-in rule's family is its code without the trailing three-digit number. +pub(crate) const FAMILY_SUFFIX_LENGTH: usize = 3; +/// `RuleSubject` values. +pub(crate) const MODEL_SUBJECT: &str = "model"; +pub(crate) const PROJECT_SUBJECT: &str = "project"; +/// Python `repr` of the built-in option defaults: `False` and `()`. +pub(crate) const FALSE_REPR: &str = "False"; +pub(crate) const EMPTY_TUPLE_REPR: &str = "()"; +pub(crate) const UNKNOWN_RULE_PREFIX: &str = "Unknown rule: "; +/// The whitespace `int()` strips once other spaces are ASCII spaces. +pub(crate) const INT_ASCII_SPACES: &str = " \t\n\x0b\x0c\r"; +pub(crate) const NEGATIVE_SIGN: char = '-'; +pub(crate) const POSITIVE_SIGN: char = '+'; +pub(crate) const ZERO_TEXT: &str = "0"; +/// `sys.int_info.default_max_str_digits`: longer decimal text makes `int()` raise. +pub(crate) const PYTHON_INT_MAX_STR_DIGITS: usize = 4300; +pub(crate) const DIGIT_SEPARATOR: char = '_'; +/// Where a loaded project configuration names its lifecycle sinks. +pub(crate) const LIFECYCLE_SINK_PATH: [&str; 3] = ["sinks", "lifecycle", "named"]; diff --git a/crates/sqlbuild-cli/src/native_commands/errors.rs b/crates/sqlbuild-cli/src/native_commands/errors.rs new file mode 100644 index 0000000000..78e1542aa8 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/errors.rs @@ -0,0 +1,16 @@ +//! Why `sqb` hands an invocation to Python after all. + +/// What made the invocation need Python, found before any side effect. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PythonNeeded { + /// The arguments are not text, or a converter or terminal fact needs the interpreter. + Arguments, + /// The command, or an option such as `--debug`, runs in Python. + Command, + /// The installed package, home directory or working directory is not known natively. + Installation, + /// The project holds Python the command imports, or files Python reports on. + Project, + /// The command failed before any side effect, and Python reports the failure. + Failure, +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs new file mode 100644 index 0000000000..f45e2c475a --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -0,0 +1 @@ +pub mod run_native_invocation; diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs new file mode 100644 index 0000000000..222d953909 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs @@ -0,0 +1,52 @@ +//! Serve one invocation without Python when its parse and project facts allow it. + +use std::ffi::OsString; +use std::path::Path; + +use crate::argv::main::parse_cli_arguments::parse_cli_arguments; +use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace}; +use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; +use crate::native_commands::_helpers::decision::native_invocation; +use crate::native_commands::_helpers::execution::run_command; +use crate::native_commands::_helpers::terminal::{ + stdout_supports_color, terminal_columns, write_text, +}; +use crate::native_commands::constants::NO_COLOR_OPTION; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::NativeInvocation; + +/// The exit status of an invocation `sqb` served, or why the same arguments go to Python. +pub fn run_native_invocation( + arguments: &[OsString], + python_cli: &Path, +) -> Result { + let arguments: Vec = arguments + .iter() + .map(|argument| argument.to_str().map(str::to_owned)) + .collect::>() + .ok_or(PythonNeeded::Arguments)?; + let use_color: bool = + !arguments.iter().any(|argument| argument == NO_COLOR_OPTION) && stdout_supports_color(); + let settings: ArgvSettings = ArgvSettings { + columns: terminal_columns()?, + use_color, + }; + let mut host: NativeArgvHost = NativeArgvHost::default(); + let outcome: ArgvOutcome = + parse_cli_arguments(&arguments, namespace_defaults(), settings, &mut host); + if host.needs_python { + return Err(PythonNeeded::Arguments); + } + let namespace: Namespace = match outcome.parsed { + Err(code) => { + for (stream, text) in &outcome.writes { + write_text(*stream, text); + } + return Ok(i32::try_from(code).unwrap_or(1)); + } + Ok(namespace) if outcome.writes.is_empty() => namespace, + Ok(_) => return Err(PythonNeeded::Arguments), + }; + let invocation: NativeInvocation = native_invocation(&namespace)?; + run_command(&invocation, python_cli, use_color) +} diff --git a/crates/sqlbuild-cli/src/native_commands/mod.rs b/crates/sqlbuild-cli/src/native_commands/mod.rs new file mode 100644 index 0000000000..d99ca200f2 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/mod.rs @@ -0,0 +1,10 @@ +//! Python-free commands `sqb` runs itself: help, usage errors and the commands needing no Python. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub(crate) mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs new file mode 100644 index 0000000000..f0ce868fa2 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -0,0 +1,73 @@ +//! Python-free commands, the invocation facts they run with and what they print. + +use std::path::PathBuf; + +/// What `sqb skills` installs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct SkillsOptions { + pub(crate) global_install: bool, + pub(crate) targets: Vec, + pub(crate) force: bool, +} + +/// Which `sqb rules` inspection runs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RulesAction { + List, + Show, +} + +/// What `sqb rules list|show` prints. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RulesOptions { + pub(crate) action: RulesAction, + pub(crate) selector: Option, + pub(crate) json: bool, +} + +/// A command that runs without Python. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum NativeCommand { + /// `clean` and a bare `sqb`: observed, with no handler to run. + Nothing, + Init, + Skills(SkillsOptions), + Rules(RulesOptions), +} + +/// One parsed invocation `sqb` serves without Python. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NativeInvocation { + pub(crate) command: NativeCommand, + /// The parsed command name; `None` for a bare `sqb`. + pub(crate) command_name: Option, + /// `--project-dir` as given. + pub(crate) project_dir: Option, + pub(crate) target: Option, + pub(crate) creates_project: bool, + pub(crate) maintains_skills: bool, +} + +/// What a command printed and the status it ends with. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct CommandOutput { + pub(crate) stdout: String, + pub(crate) stderr: String, + pub(crate) exit_code: i32, +} + +/// Where the installed `sqlbuild` package keeps what the commands read. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct InstalledPackage { + /// The packaged `.agents/skills/sqlbuild` directory. + pub(crate) skills_root: PathBuf, +} + +/// The directories one command runs against. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ProjectLocation { + /// `Path(--project-dir)`, or `Path.cwd()`, unresolved. + pub(crate) given: PathBuf, + /// `Path(--project-dir).resolve()`, or `Path.cwd()`. + pub(crate) resolved: PathBuf, +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests.rs b/crates/sqlbuild-cli/src/native_commands/tests.rs new file mode 100644 index 0000000000..9119c141c9 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests.rs @@ -0,0 +1,14 @@ +#[path = "tests/test_argv_host.rs"] +mod argv_host; +#[path = "tests/test_decision.rs"] +mod decision; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_paths.rs"] +mod paths; +#[path = "tests/test_project_facts.rs"] +mod project_facts; +#[path = "tests/test_rules.rs"] +mod rules; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs new file mode 100644 index 0000000000..e54ee1dd6d --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs @@ -0,0 +1,28 @@ +use std::path::Path; + +use crate::argv::main::parse_cli_arguments::parse_cli_arguments; +use crate::argv::models::{ArgvOutcome, ArgvSettings}; +use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; + +/// Parse `argv` onto the `CliNamespace` defaults with the native host. +pub(super) fn parse(argv: &[&str]) -> (ArgvOutcome, NativeArgvHost) { + let argv: Vec = argv.iter().map(|arg| (*arg).to_owned()).collect(); + let settings: ArgvSettings = ArgvSettings { + columns: 100, + use_color: false, + }; + let mut host: NativeArgvHost = NativeArgvHost::default(); + let outcome: ArgvOutcome = + parse_cli_arguments(&argv, namespace_defaults(), settings, &mut host); + (outcome, host) +} + +/// Create `directories` and write `files` beneath `root`. +pub(super) fn write_tree(root: &Path, directories: &[&str], files: &[(&str, &str)]) { + for directory in directories { + std::fs::create_dir_all(root.join(directory)).expect("create directory"); + } + for (relative, contents) in files { + std::fs::write(root.join(relative), contents).expect("write file"); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_argv_host.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_argv_host.rs new file mode 100644 index 0000000000..be011f8c40 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_argv_host.rs @@ -0,0 +1,154 @@ +use crate::argv::models::{TypeFailure, Value}; +use crate::argv::types::ArgvHost; +use crate::native_commands::_helpers::argv_host::NativeArgvHost; +use crate::native_commands::tests::test_types::ConversionTestCase; + +/// Integer expectations recorded from CPython 3.12 `int(text)`. +#[test] +fn given_converter_when_converting_natively_then_python_conversions_match() { + let test_cases = [ + ConversionTestCase { + description: "surrounding ASCII spaces", + converter: "int", + text: " 5 ", + expected_value: Ok(Value::Int("5".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "underscores between digits", + converter: "int", + text: "1_0", + expected_value: Ok(Value::Int("10".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a leading underscore", + converter: "int", + text: "_1", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a trailing underscore", + converter: "int", + text: "1_", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "doubled underscores", + converter: "int", + text: "1__0", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "an Arabic-Indic digit", + converter: "int", + text: "\u{663}", + expected_value: Ok(Value::Int("3".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a non-breaking space", + converter: "int", + text: "\u{a0}5", + expected_value: Ok(Value::Int("5".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "an ASCII separator is not stripped", + converter: "int", + text: "\u{1c}5", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a zero-width space is not a space", + converter: "int", + text: "\u{200b}7", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "negative zero", + converter: "int", + text: "-0", + expected_value: Ok(Value::Int("0".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a sign and leading zeros", + converter: "int", + text: "+007", + expected_value: Ok(Value::Int("7".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "beyond 64 bits", + converter: "int", + text: "-123456789012345678901234567890", + expected_value: Ok(Value::Int("-123456789012345678901234567890".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a decimal point", + converter: "int", + text: "1.5", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "empty text", + converter: "int", + text: "", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a positive integer rejects zero with its message", + converter: "_positive_integer", + text: "0", + expected_value: Err(TypeFailure::ArgumentType( + "value must be a positive integer".to_owned(), + )), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a positive integer rejects negatives", + converter: "_positive_integer", + text: "-3", + expected_value: Err(TypeFailure::ArgumentType( + "value must be a positive integer".to_owned(), + )), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a positive integer that is not an integer is invalid", + converter: "_positive_integer", + text: "many", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "--vars JSON needs Python", + converter: "parse_cli_vars", + text: "{}", + expected_value: Ok(Value::None), + expected_needs_python: true, + }, + ]; + + for test_case in test_cases { + let mut host: NativeArgvHost = NativeArgvHost::default(); + + let value = host.convert(test_case.converter, test_case.text); + + assert_eq!( + (value, host.needs_python), + (test_case.expected_value, test_case.expected_needs_python), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_decision.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_decision.rs new file mode 100644 index 0000000000..02dd4d48be --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_decision.rs @@ -0,0 +1,129 @@ +use crate::argv::models::Namespace; +use crate::native_commands::_helpers::decision::native_invocation; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{NativeCommand, RulesAction, RulesOptions, SkillsOptions}; +use crate::native_commands::tests::helpers::parse; +use crate::native_commands::tests::test_types::{DecisionTestCase, ParseTestCase}; + +#[test] +fn given_parsed_command_when_deciding_then_only_python_free_commands_run_natively() { + let test_cases = [ + DecisionTestCase { + description: "a bare sqb has no handler", + argv: &[], + expected_command: Ok(NativeCommand::Nothing), + expected_creates_project: false, + }, + DecisionTestCase { + description: "clean has no handler", + argv: &["clean"], + expected_command: Ok(NativeCommand::Nothing), + expected_creates_project: false, + }, + DecisionTestCase { + description: "init creates a project", + argv: &["init", "--project-dir", "shop"], + expected_command: Ok(NativeCommand::Init), + expected_creates_project: true, + }, + DecisionTestCase { + description: "skills with targets", + argv: &[ + "skills", "--target", "claude", "--target", "agents", "--force", + ], + expected_command: Ok(NativeCommand::Skills(SkillsOptions { + global_install: false, + targets: vec!["claude".to_owned(), "agents".to_owned()], + force: true, + })), + expected_creates_project: false, + }, + DecisionTestCase { + description: "rules list as JSON", + argv: &["rules", "--json", "list"], + expected_command: Ok(NativeCommand::Rules(RulesOptions { + action: RulesAction::List, + selector: None, + json: true, + })), + expected_creates_project: false, + }, + DecisionTestCase { + description: "rules show", + argv: &["rules", "show", "SQBRMODEL101"], + expected_command: Ok(NativeCommand::Rules(RulesOptions { + action: RulesAction::Show, + selector: Some("SQBRMODEL101".to_owned()), + json: false, + })), + expected_creates_project: false, + }, + DecisionTestCase { + description: "rules run evaluates the project in Python", + argv: &["rules", "run", "SQBRMODEL"], + expected_command: Err(PythonNeeded::Command), + expected_creates_project: false, + }, + DecisionTestCase { + description: "debug output is Python's", + argv: &["clean", "--debug"], + expected_command: Err(PythonNeeded::Command), + expected_creates_project: false, + }, + DecisionTestCase { + description: "compile needs Python", + argv: &["compile"], + expected_command: Err(PythonNeeded::Command), + expected_creates_project: false, + }, + ]; + + for test_case in test_cases { + let (outcome, _) = parse(test_case.argv); + let namespace: Namespace = outcome.parsed.expect(test_case.description); + + let decided = native_invocation(&namespace); + + assert_eq!( + decided + .as_ref() + .map(|invocation| invocation.command.clone()) + .map_err(|reason| *reason), + test_case.expected_command.clone(), + "{}", + test_case.description + ); + assert_eq!( + decided.is_ok_and(|invocation| invocation.creates_project), + test_case.expected_creates_project, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_command_line_when_parsing_natively_then_only_python_converters_need_python() { + let test_cases = [ + ParseTestCase { + description: "--vars JSON is parsed by Python", + argv: &["compile", "--vars", "{}"], + expected_needs_python: true, + }, + ParseTestCase { + description: "an integer option converts natively", + argv: &["build", "--concurrency", "4"], + expected_needs_python: false, + }, + ]; + + for test_case in test_cases { + let (_, host) = parse(test_case.argv); + + assert_eq!( + host.needs_python, test_case.expected_needs_python, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_paths.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_paths.rs new file mode 100644 index 0000000000..6183cb8ae0 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_paths.rs @@ -0,0 +1,77 @@ +use std::path::{Path, PathBuf}; + +use crate::native_commands::_helpers::paths::{pathlib_path, resolved_path}; +use crate::native_commands::tests::test_types::{PathlibTestCase, ResolveTestCase}; + +#[test] +fn given_project_dir_text_when_building_a_path_then_pathlib_normalisation_applies() { + let test_cases = [ + PathlibTestCase { + description: "a leading dot and a trailing slash drop", + text: "./shop/", + expected_path: "shop", + }, + PathlibTestCase { + description: "inner dots and repeated slashes drop", + text: "shop/.//orders", + expected_path: "shop/orders", + }, + PathlibTestCase { + description: "a lone dot stays", + text: ".", + expected_path: ".", + }, + PathlibTestCase { + description: "parent references stay", + text: "../shop", + expected_path: "../shop", + }, + ]; + + for test_case in test_cases { + assert_eq!( + pathlib_path(Path::new(test_case.text)), + PathBuf::from(test_case.expected_path), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_relative_path_when_resolving_then_existing_prefix_is_real_and_rest_is_kept() { + let test_cases = [ + ResolveTestCase { + description: "a missing directory is joined as written", + text: "new/shop", + expected_parts: &["new", "shop"], + }, + ResolveTestCase { + description: "a parent reference after a missing directory removes it", + text: "missing/../new", + expected_parts: &["new"], + }, + ResolveTestCase { + description: "the working directory itself", + text: ".", + expected_parts: &[], + }, + ]; + + for test_case in test_cases { + let root = tempfile::tempdir().expect("temporary directory"); + let real: PathBuf = std::fs::canonicalize(root.path()).expect("real root"); + + let resolved: PathBuf = resolved_path(Path::new(test_case.text), &real); + + assert_eq!( + resolved, + test_case + .expected_parts + .iter() + .fold(real.clone(), |path, part| path.join(part)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_project_facts.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_project_facts.rs new file mode 100644 index 0000000000..c7b317e626 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_project_facts.rs @@ -0,0 +1,60 @@ +use crate::native_commands::_helpers::project_facts::has_no_runtime_extensions; +use crate::native_commands::tests::helpers::write_tree; +use crate::native_commands::tests::test_types::ProjectFactsTestCase; + +const PROJECT: &str = "name = \"shop\"\nadapter = \"duckdb\"\n"; +const NAMED_SINK_PROJECT: &str = "name = \"shop\"\nadapter = \"duckdb\"\n\n[sinks.lifecycle.named.audit]\nmin_severity = \"error\"\n"; + +#[test] +fn given_project_files_when_checking_runtime_extensions_then_python_owned_ones_need_python() { + let test_cases = [ + ProjectFactsTestCase { + description: "no project file", + directories: &[], + files: &[], + expected_native: true, + }, + ProjectFactsTestCase { + description: "a plain project", + directories: &[], + files: &[("sqlbuild_project.toml", PROJECT)], + expected_native: true, + }, + ProjectFactsTestCase { + description: "a sinks directory is imported", + directories: &["sinks"], + files: &[("sqlbuild_project.toml", PROJECT)], + expected_native: false, + }, + ProjectFactsTestCase { + description: "the replaced event exporters directory is rejected in Python", + directories: &["event_exporters"], + files: &[], + expected_native: false, + }, + ProjectFactsTestCase { + description: "a named lifecycle sink must be bound in Python", + directories: &[], + files: &[("sqlbuild_project.toml", NAMED_SINK_PROJECT)], + expected_native: false, + }, + ProjectFactsTestCase { + description: "an invalid project file is reported by Python", + directories: &[], + files: &[("sqlbuild_project.toml", "name = \n")], + expected_native: false, + }, + ]; + + for test_case in test_cases { + let root = tempfile::tempdir().expect("temporary directory"); + write_tree(root.path(), test_case.directories, test_case.files); + + assert_eq!( + has_no_runtime_extensions(root.path()), + test_case.expected_native, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_rules.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_rules.rs new file mode 100644 index 0000000000..4eedd7443e --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_rules.rs @@ -0,0 +1,108 @@ +use crate::native_commands::_helpers::rules::run_rules; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{RulesAction, RulesOptions}; +use crate::native_commands::tests::helpers::write_tree; +use crate::native_commands::tests::test_types::{RulesTestCase, output_start}; + +const PROJECT: &str = "name = \"shop\"\nadapter = \"duckdb\"\n"; +const OPTIONS_PROJECT: &str = "name = \"shop\"\nadapter = \"duckdb\"\n\n[rules.rule_options.SQBRCONTRACT102]\nallow_numeric_indicators = true\n"; + +#[test] +fn given_project_when_inspecting_rules_then_builtin_catalogue_prints_unless_python_owns_it() { + let test_cases = [ + RulesTestCase { + description: "list prints code and message lines", + directories: &[], + project_file: PROJECT, + action: RulesAction::List, + selector: None, + json: false, + expected_output: Ok(( + "SQBRCONTRACT101 models must declare an enforced output contract\n", + 0, + )), + }, + RulesTestCase { + description: "list --json prints the sorted document", + directories: &[], + project_file: PROJECT, + action: RulesAction::List, + selector: None, + json: true, + expected_output: Ok(( + "{\"rules\": [{\"code\": \"SQBRCONTRACT101\", \"custom\": false, \"family\": \"SQBRCONTRACT\", ", + 0, + )), + }, + RulesTestCase { + description: "show prints the rule", + directories: &[], + project_file: PROJECT, + action: RulesAction::Show, + selector: Some("SQBRMODEL101"), + json: false, + expected_output: Ok(( + "SQBRMODEL101: dependency-import-ctes\nFamily: SQBRMODEL\n", + 0, + )), + }, + RulesTestCase { + description: "an unknown rule is a usage failure", + directories: &[], + project_file: PROJECT, + action: RulesAction::Show, + selector: Some("SQBRNOPE101"), + json: false, + expected_output: Ok(("Unknown rule: SQBRNOPE101\n", 2)), + }, + RulesTestCase { + description: "custom rules are imported by Python", + directories: &["rules"], + project_file: PROJECT, + action: RulesAction::List, + selector: None, + json: false, + expected_output: Err(PythonNeeded::Project), + }, + RulesTestCase { + description: "configured options are rendered by Python", + directories: &[], + project_file: OPTIONS_PROJECT, + action: RulesAction::List, + selector: None, + json: false, + expected_output: Err(PythonNeeded::Project), + }, + ]; + + for test_case in test_cases { + let root = tempfile::tempdir().expect("temporary directory"); + write_tree( + root.path(), + test_case.directories, + &[("sqlbuild_project.toml", test_case.project_file)], + ); + + let output = run_rules( + root.path(), + &RulesOptions { + action: test_case.action, + selector: test_case.selector.map(str::to_owned), + json: test_case.json, + }, + ); + + let length: usize = test_case + .expected_output + .map_or(0, |(prefix, _)| prefix.chars().count()); + + assert_eq!( + output.map(|output| output_start(&output, length)), + test_case + .expected_output + .map(|(prefix, code)| (prefix.to_owned(), code)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs new file mode 100644 index 0000000000..528aeaf4e4 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -0,0 +1,68 @@ +use crate::argv::models::{TypeFailure, Value}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{CommandOutput, NativeCommand, RulesAction}; + +/// One converter applied to one command-line string. +pub(super) struct ConversionTestCase { + pub(super) description: &'static str, + pub(super) converter: &'static str, + pub(super) text: &'static str, + pub(super) expected_value: Result, + pub(super) expected_needs_python: bool, +} + +/// One accepted command line, the native command it runs and its invocation services. +pub(super) struct DecisionTestCase { + pub(super) description: &'static str, + pub(super) argv: &'static [&'static str], + pub(super) expected_command: Result, + pub(super) expected_creates_project: bool, +} + +/// One command line and whether its parse needs Python. +pub(super) struct ParseTestCase { + pub(super) description: &'static str, + pub(super) argv: &'static [&'static str], + pub(super) expected_needs_python: bool, +} + +/// One `--project-dir` string and the `pathlib.Path` it names. +pub(super) struct PathlibTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_path: &'static str, +} + +/// A path beneath a real directory and what `Path.resolve()` makes of it. +pub(super) struct ResolveTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_parts: &'static [&'static str], +} + +/// Project directories and files, and whether an observed command can run without Python. +pub(super) struct ProjectFactsTestCase { + pub(super) description: &'static str, + pub(super) directories: &'static [&'static str], + pub(super) files: &'static [(&'static str, &'static str)], + pub(super) expected_native: bool, +} + +/// One `rules` inspection over a project and the start of what it prints. +pub(super) struct RulesTestCase { + pub(super) description: &'static str, + pub(super) directories: &'static [&'static str], + pub(super) project_file: &'static str, + pub(super) action: RulesAction, + pub(super) selector: Option<&'static str>, + pub(super) json: bool, + pub(super) expected_output: Result<(&'static str, i32), PythonNeeded>, +} + +/// The printed start and exit status of a command's output, for comparison. +pub(super) fn output_start(output: &CommandOutput, length: usize) -> (String, i32) { + ( + output.stdout.chars().take(length).collect(), + output.exit_code, + ) +} diff --git a/tests/integration/src/sqlbuild/cli/entry/_test_types.py b/tests/integration/src/sqlbuild/cli/entry/_test_types.py index 2f6143377e..a39ae9398f 100644 --- a/tests/integration/src/sqlbuild/cli/entry/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/entry/_test_types.py @@ -82,3 +82,34 @@ class LaunchedTeardownTestCase: description: str args: tuple[str, ...] expected_hits: int + + +@dataclass(frozen=True) +class PythonFreeTestCase: + """One command line `sqb` serves without Python, in a directory `prepare` sets up.""" + + description: str + args: tuple[str, ...] + prepare: Callable[[Path], None] + expected_exit_code: int + expected_captures: int + + +@dataclass(frozen=True) +class PythonStartTestCase: + """One command line `sqb` hands to Python because of the project `prepare` writes.""" + + description: str + args: tuple[str, ...] + prepare: Callable[[Path], None] + expected_python_starts: int + + +@dataclass(frozen=True) +class InterruptTestCase: + """One Python-free command line interrupted while it waits on the project `prepare` writes.""" + + description: str + args: tuple[str, ...] + prepare: Callable[[Path], None] + expected_exit_code: int diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index e81c78ba3a..09fbd33876 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -2,6 +2,7 @@ import os import re import shutil +import signal import subprocess import sys from pathlib import Path @@ -199,3 +200,94 @@ def compiled_files(project_dir: Path) -> dict[str, bytes]: path.relative_to(compiled).as_posix(): path.read_bytes() for path in sorted(compiled.rglob("*.sql")) } + + +def counting_environment(directory: Path, marker: Path) -> Path: + """Install `sqb` in an environment whose `sqlbuild-python` logs each start; return `sqb`.""" + + program: Path = counting_install(directory / "bin", marker) + site_packages: Path = ( + directory / "lib" / f"python{sys.version_info.major}.{sys.version_info.minor}" + ) / "site-packages" + site_packages.mkdir(parents=True) + _ = (site_packages / "sqlbuild_source.pth").write_text( + str(Path(sqlbuild.__file__).resolve().parent.parent) + "\n", encoding="utf-8" + ) + return program + + +def reset_directory(directory: Path) -> Path: + """An empty `directory`, removing whatever an earlier run left.""" + + shutil.rmtree(directory, ignore_errors=True) + directory.mkdir(parents=True) + return directory + + +def _capture_files(capture: Path) -> dict[str, object]: + metadata: dict[str, object] = capture_metadata(capture) + for volatile in ("invocation_id", "started_at", "completed_at", "capture_date"): + _ = metadata.pop(volatile) + streams: dict[str, bytes] = { + name: (capture / name).read_bytes() + for name in ("stdout.log", "stderr.log", "diagnostics.jsonl") + } + return {"complete": (capture / "complete").exists(), "metadata": metadata, **streams} + + +def compute_log_files(project_dir: Path) -> list[dict[str, object]]: + """Each capture's streams and metadata, without per-run identity and times.""" + + return [ + _capture_files(metadata.parent) + for metadata in sorted((project_dir / "logs").glob("*/*/metadata.json")) + ] + + +def prepare_nothing(directory: Path) -> None: + """Leave the directory empty.""" + + _ = directory + + +def prepare_project(directory: Path) -> None: + """Write the one-model project.""" + + _ = write_project(directory) + + +def prepare_custom_skill(directory: Path) -> None: + """Write a hand-written skill where `sqb init` installs the packaged one.""" + + custom: Path = directory / ".claude" / "skills" / "sqlbuild" / "SKILL.md" + custom.parent.mkdir(parents=True) + _ = custom.write_text("my own skill\n", encoding="utf-8") + + +def prepare_stale_skills(directory: Path) -> None: + """Write the project configured for Claude skills that are not installed yet.""" + + _ = write_project(directory) + _ = (directory / "sqlbuild_project.toml").write_text( + PROJECT_FILES["sqlbuild_project.toml"] + '\n[skills]\ntargets = ["claude"]\n', + encoding="utf-8", + ) + + +def prepare_custom_rules(directory: Path) -> None: + """Write the project with a custom rules directory.""" + + _ = write_project(directory) + (directory / "rules").mkdir() + + +def prepare_named_pipe_project(directory: Path) -> None: + """Make the project file a named pipe that blocks its reader.""" + + os.mkfifo(directory / "sqlbuild_project.toml") + + +def restore_default_interrupt() -> None: + """Give a spawned command default Ctrl-C handling even when its parent ignores SIGINT.""" + + _ = signal.signal(signal.SIGINT, signal.SIG_DFL) diff --git a/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py new file mode 100644 index 0000000000..6a787e7454 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py @@ -0,0 +1,333 @@ +import signal +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.entry._test_types import ( + InterruptTestCase, + PythonFreeTestCase, + PythonStartTestCase, +) +from tests.integration.src.sqlbuild.cli.entry.helpers import ( + PYTHON_CLI, + LauncherRun, + compute_log_files, + counting_environment, + prepare_custom_rules, + prepare_custom_skill, + prepare_named_pipe_project, + prepare_nothing, + prepare_project, + prepare_stale_skills, + python_starts, + reset_directory, + restore_default_interrupt, + run, +) + +INTERRUPT_DELAY_SECONDS: float = 1.0 +INTERRUPT_TIMEOUT_SECONDS: float = 30.0 + + +@pytest.mark.skipif(sys.platform == "win32", reason="the counting environment is a POSIX script") +@pytest.mark.parametrize( + "test_case", + [ + PythonFreeTestCase( + description="top-level help", + args=("--help",), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="short help after options", + args=("--no-color", "-h"), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="command help", + args=("compile", "--help"), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="nested command help", + args=("rules", "show", "--help"), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="version among options", + args=("--no-color", "--version"), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="unknown command", + args=("not-a-command",), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="invalid choice", + args=("skills", "--target", "bogus"), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="invalid integer", + args=("build", "--concurrency", "many"), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="post-parse rule", + args=("build", "--concurrency", "0"), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="missing option value", + args=("--project-dir",), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="unrecognized argument", + args=("rules", "list", "--json"), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="mutually exclusive options", + args=("lineage", "orders", "--json", "--format", "tree"), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="bare sqb", + args=(), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="clean", + args=("clean",), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="clean outside a project", + args=("clean",), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules list", + args=("rules", "list"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules without an action", + args=("rules",), + prepare=prepare_project, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="rules list json", + args=("rules", "--json", "list"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules show", + args=("rules", "show", "SQBRMODEL101"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules show with thresholds", + args=("rules", "show", "SQBRTEST101"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules show unknown", + args=("rules", "show", "SQBRNOPE101"), + prepare=prepare_project, + expected_exit_code=2, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules with stale skills", + args=("rules", "list"), + prepare=prepare_stale_skills, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="skills", + args=("skills",), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="skills for one target", + args=("skills", "--target", "claude"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="init", + args=("init",), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="init a new directory", + args=("init", "--project-dir", "shop/"), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="init beside a custom skill", + args=("init",), + prepare=prepare_custom_skill, + expected_exit_code=1, + expected_captures=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_free_invocation_when_running_sqb_then_python_never_starts_and_output_matches( + test_case: PythonFreeTestCase, tmp_path: Path +) -> None: + marker: Path = tmp_path / "python-starts.log" + _ = marker.write_text("", encoding="utf-8") + program: Path = counting_environment(tmp_path / "environment", marker) + workspace: Path = tmp_path / "workspace" + test_case.prepare(reset_directory(workspace)) + python: LauncherRun = run(PYTHON_CLI, args=test_case.args, cwd=workspace) + python_logs: list[dict[str, object]] = compute_log_files(workspace) + test_case.prepare(reset_directory(workspace)) + + native: LauncherRun = run(program, args=test_case.args, cwd=workspace) + + assert native == python + assert native[0] == test_case.expected_exit_code + assert python_starts(marker) == 0 + assert compute_log_files(workspace) == python_logs + assert len(python_logs) == test_case.expected_captures + + +@pytest.mark.skipif(sys.platform == "win32", reason="the counting environment is a POSIX script") +@pytest.mark.parametrize( + "test_case", + [ + PythonStartTestCase( + description="custom rules", + args=("rules", "list"), + prepare=prepare_custom_rules, + expected_python_starts=1, + ), + PythonStartTestCase( + description="dbt passthrough help", + args=("dbt", "plan", "--help"), + prepare=prepare_nothing, + expected_python_starts=1, + ), + PythonStartTestCase( + description="debug diagnostics", + args=("clean", "--debug"), + prepare=prepare_project, + expected_python_starts=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_invocation_needing_python_when_running_sqb_then_python_runs_it( + test_case: PythonStartTestCase, tmp_path: Path +) -> None: + marker: Path = tmp_path / "python-starts.log" + _ = marker.write_text("", encoding="utf-8") + program: Path = counting_environment(tmp_path / "environment", marker) + workspace: Path = reset_directory(tmp_path / "workspace") + test_case.prepare(workspace) + python: LauncherRun = run(PYTHON_CLI, args=test_case.args, cwd=workspace) + test_case.prepare(reset_directory(workspace)) + + native: LauncherRun = run(program, args=test_case.args, cwd=workspace) + + assert native[:2] == python[:2] + assert python_starts(marker) == test_case.expected_python_starts + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX signals and named pipes") +@pytest.mark.parametrize( + "test_case", + [ + InterruptTestCase( + description="rules list waiting on the project file", + args=("rules", "list"), + prepare=prepare_named_pipe_project, + expected_exit_code=130, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_free_command_when_interrupted_then_sqb_exits_130_without_python( + test_case: InterruptTestCase, tmp_path: Path +) -> None: + marker: Path = tmp_path / "python-starts.log" + _ = marker.write_text("", encoding="utf-8") + program: Path = counting_environment(tmp_path / "environment", marker) + project_dir: Path = reset_directory(tmp_path / "project") + test_case.prepare(project_dir) + process: subprocess.Popen[bytes] = subprocess.Popen( + [str(program), *test_case.args], + cwd=project_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=restore_default_interrupt, + ) + time.sleep(INTERRUPT_DELAY_SECONDS) + assert process.poll() is None, "sqb returned before reading the project file" + + process.send_signal(signal.SIGINT) + _, stderr = process.communicate(timeout=INTERRUPT_TIMEOUT_SECONDS) + + assert process.returncode == test_case.expected_exit_code, stderr + assert python_starts(marker) == 0 + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 986b63d4c8a0a594a755b7a841f73da94311af8b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:28:38 +0100 Subject: [PATCH 3/5] docs: note that Recall does not record commands sqb answers without Python --- .../references/docs/concepts/observability/recall.md | 5 +++-- .../src/content/docs/docs/concepts/observability/recall.mdx | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index 2b92ac0dcb..b481c1fbcf 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -41,8 +41,9 @@ of its history still succeeds, prints a warning followed by its own final status status `abandoned`. Every command is recorded except `sqb recall` itself, commands that create a project (such as -`sqb init`), and raw-query `sqb diff`. A `sqb compile` answered from the launcher's warm cache is not -recorded either. +`sqb init`), and raw-query `sqb diff`. Commands `sqb` answers without starting Python are not +recorded either: a `sqb compile` answered from the launcher's warm cache, and `sqb clean`, `sqb skills`, +`sqb rules list` and `sqb rules show` in a project with no custom rules, rule options or sinks. The store is `logs/recall.sqlite3` in the project directory. Add `logs/` to `.gitignore`; the store can contain sensitive SQL and output. diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 69a1328e64..2511e7fb0f 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -26,8 +26,9 @@ of its history still succeeds, prints a warning followed by its own final status status `abandoned`. Every command is recorded except `sqb recall` itself, commands that create a project (such as -`sqb init`), and raw-query `sqb diff`. A `sqb compile` answered from the launcher's warm cache is not -recorded either. +`sqb init`), and raw-query `sqb diff`. Commands `sqb` answers without starting Python are not +recorded either: a `sqb compile` answered from the launcher's warm cache, and `sqb clean`, `sqb skills`, +`sqb rules list` and `sqb rules show` in a project with no custom rules, rule options or sinks. The store is `logs/recall.sqlite3` in the project directory. Add `logs/` to `.gitignore`; the store can contain sensitive SQL and output. From 2ca8a36757fa64e0a0019528a3b666ce3bda9f31 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:19:08 +0100 Subject: [PATCH 4/5] fix: resolve native project paths and the home directory as Python does on Windows --- .../src/native_commands/_helpers/paths.rs | 33 +++++++++++++++++-- .../src/native_commands/_helpers/skills.rs | 2 +- .../src/native_commands/constants.rs | 4 +++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs index c50eea5af9..302e8a5a66 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs @@ -2,6 +2,8 @@ use std::path::{Component, Path, PathBuf}; +use sqlbuild_rules::project_tree::main::project_real_path::project_real_path; + use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::ProjectLocation; @@ -24,8 +26,8 @@ pub(crate) fn resolved_path(path: &Path, cwd: &Path) -> PathBuf { } else { cwd.join(path) }; - if let Ok(real) = std::fs::canonicalize(&absolute) { - return real; + if absolute.exists() { + return project_real_path(&absolute); } let mut resolved: PathBuf = PathBuf::new(); for component in absolute.components() { @@ -37,7 +39,11 @@ pub(crate) fn resolved_path(path: &Path, cwd: &Path) -> PathBuf { } Component::Normal(name) => { let candidate: PathBuf = resolved.join(name); - resolved = std::fs::canonicalize(&candidate).unwrap_or(candidate); + resolved = if candidate.exists() { + project_real_path(&candidate) + } else { + candidate + }; } } } @@ -52,5 +58,26 @@ pub(crate) fn project_location(given: Option<&Path>) -> Result cwd.clone(), }; let resolved: PathBuf = resolved_path(&given, &cwd); + if !spelled_as_given(&resolved, &cwd.join(&given)) { + return Err(PythonNeeded::Installation); + } Ok(ProjectLocation { given, resolved }) } + +/// Links resolve as in Python on Unix; Windows drive substitutions and mappings stay Python's. +#[cfg(unix)] +fn spelled_as_given(_resolved: &Path, _absolute: &Path) -> bool { + true +} + +/// Whether the resolved path is the absolute path as given, apart from letter case. +#[cfg(not(unix))] +fn spelled_as_given(resolved: &Path, absolute: &Path) -> bool { + let absolute: PathBuf = absolute + .components() + .filter(|component| *component != Component::CurDir) + .collect(); + resolved + .to_string_lossy() + .eq_ignore_ascii_case(&absolute.to_string_lossy()) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs index 4272d40185..ac7eee9a36 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs @@ -10,7 +10,7 @@ use crate::native_commands::constants::{HOME_VARIABLE, SKILL_FILES_HEADER}; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{CommandOutput, InstalledPackage, SkillsOptions}; -/// `Path.home()` from `HOME`, unless only Python's password lookup could tell. +/// `Path.home()` from `HOME` (`USERPROFILE` on Windows), unless only Python could tell. pub(crate) fn home_dir() -> Result { std::env::var_os(HOME_VARIABLE) .filter(|home| !home.is_empty()) diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 9c01a84a6b..fdef90a8cf 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -27,7 +27,11 @@ pub(crate) const SITE_DIRECTORIES: [&str; 2] = ["site-packages", "dist-packages" pub(crate) const WINDOWS_SITE_DIRECTORY: [&str; 2] = ["Lib", "site-packages"]; /// The packaged skill beneath the `sqlbuild` package directory. pub(crate) const SKILLS_DIRECTORY: [&str; 3] = [".agents", "skills", "sqlbuild"]; +/// The variable `Path.home()` reads first; without it `sqb` defers to Python's other lookups. +#[cfg(not(windows))] pub(crate) const HOME_VARIABLE: &str = "HOME"; +#[cfg(windows)] +pub(crate) const HOME_VARIABLE: &str = "USERPROFILE"; /// The exit status of a command stopped by Ctrl+C. #[cfg(unix)] pub(crate) const INTERRUPTED_EXIT_CODE: i32 = 130; From 8ccb01fc426492287a993b1d466440bf78775801 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:19:27 +0100 Subject: [PATCH 5/5] fix: report lost stdout, Ctrl+C, init failures and the version as the Python CLI does --- crates/sqlbuild-cli/Cargo.toml | 4 +- .../src/compute_logs/constants.rs | 5 + .../src/compute_logs/main/abandon_capture.rs | 72 ++++++++++++ .../sqlbuild-cli/src/compute_logs/main/mod.rs | 1 + .../src/launcher/_helpers/messages.rs | 11 +- crates/sqlbuild-cli/src/launcher/constants.rs | 2 + .../src/native_commands/_helpers/argv_host.rs | 3 +- .../src/native_commands/_helpers/execution.rs | 43 ++++--- .../src/native_commands/_helpers/init.rs | 43 ++++--- .../src/native_commands/_helpers/terminal.rs | 108 ++++++++++++++---- .../src/native_commands/constants.rs | 11 +- .../main/run_native_invocation.rs | 20 ++-- .../src/native_commands/models.rs | 11 ++ .../sqlbuild-cli/src/native_commands/tests.rs | 4 + .../src/native_commands/tests/test_init.rs | 56 +++++++++ .../native_commands/tests/test_terminal.rs | 75 ++++++++++++ .../src/native_commands/tests/test_types.rs | 15 +++ .../src/sqlbuild/cli/entry/_test_types.py | 11 ++ .../src/sqlbuild/cli/entry/helpers.py | 25 +++- .../cli/entry/test_python_free_commands.py | 63 +++++++++- 20 files changed, 515 insertions(+), 68 deletions(-) create mode 100644 crates/sqlbuild-cli/src/compute_logs/main/abandon_capture.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_init.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_terminal.rs diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index cdfce87880..791fafe765 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -17,7 +17,6 @@ path = "src/bin/sqlbuild.rs" [dependencies] blake3.workspace = true -ctrlc.workspace = true getrandom.workspace = true serde.workspace = true serde_json.workspace = true @@ -29,6 +28,9 @@ sqlbuild-rules.workspace = true [target.'cfg(unix)'.dependencies] rustix.workspace = true +[target.'cfg(windows)'.dependencies] +ctrlc.workspace = true + [dev-dependencies] tempfile.workspace = true diff --git a/crates/sqlbuild-cli/src/compute_logs/constants.rs b/crates/sqlbuild-cli/src/compute_logs/constants.rs index ba6664c9b8..44dfb4738b 100644 --- a/crates/sqlbuild-cli/src/compute_logs/constants.rs +++ b/crates/sqlbuild-cli/src/compute_logs/constants.rs @@ -33,3 +33,8 @@ pub const DEFAULT_LOG_DIRECTORY: &str = "logs"; pub const MONTHS_PER_YEAR: i64 = 12; pub const MAX_DAY_OF_MONTH: i64 = 31; pub const HOURS_PER_DAY: i64 = 24; +/// The debug record Python's capture logs when its streams could not be finalized. +pub const CAPTURE_FAILURE_LOGGER: &str = "sqlbuild.cli.compute_logs"; +pub const CAPTURE_FAILURE_MESSAGE: &str = "local compute log capture unavailable"; +pub const DIAGNOSTIC_PRODUCER: &str = "sqlbuild"; +pub const DIAGNOSTIC_SCHEMA_VERSION: i64 = 1; diff --git a/crates/sqlbuild-cli/src/compute_logs/main/abandon_capture.rs b/crates/sqlbuild-cli/src/compute_logs/main/abandon_capture.rs new file mode 100644 index 0000000000..1697a7896b --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/main/abandon_capture.rs @@ -0,0 +1,72 @@ +//! Leave a capture incomplete after its final flush failed, as Python's capture cleanup does. + +use std::io::{self, Write}; +use std::time::SystemTime; + +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, OrjsonOptions}; + +use crate::compute_logs::_helpers::timestamps::{epoch_microseconds, utc_timestamp}; +use crate::compute_logs::constants::{ + CAPTURE_FAILURE_LOGGER, CAPTURE_FAILURE_MESSAGE, DIAGNOSTIC_PRODUCER, DIAGNOSTIC_SCHEMA_VERSION, +}; +use crate::compute_logs::models::{CaptureEnd, OpenCapture}; + +fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +/// The debug record `log_compute_capture_failure` writes to the capture's diagnostics. +fn capture_failure_record(invocation_id: &str, occurred_at: SystemTime) -> String { + let record: JsonValue = JsonValue::Object(vec![ + ("fields".to_owned(), JsonValue::Object(Vec::new())), + ("invocation_id".to_owned(), text(invocation_id)), + ("log_stream_id".to_owned(), JsonValue::Null), + ("logger".to_owned(), text(CAPTURE_FAILURE_LOGGER)), + ("message".to_owned(), text(CAPTURE_FAILURE_MESSAGE)), + ( + "occurred_at".to_owned(), + text(&utc_timestamp(epoch_microseconds(occurred_at))), + ), + ("operation_id".to_owned(), JsonValue::Null), + ("producer".to_owned(), text(DIAGNOSTIC_PRODUCER)), + ( + "producer_version".to_owned(), + text(env!("CARGO_PKG_VERSION")), + ), + ("resource_attempt_id".to_owned(), JsonValue::Null), + ("resource_id".to_owned(), JsonValue::Null), + ("run_id".to_owned(), JsonValue::Null), + ( + "schema_version".to_owned(), + JsonValue::Integer(JsonInteger::from(DIAGNOSTIC_SCHEMA_VERSION)), + ), + ("severity".to_owned(), text("debug")), + ("source".to_owned(), text("python_logging")), + ("statement_id".to_owned(), JsonValue::Null), + ]); + let options: OrjsonOptions = OrjsonOptions { + sort_keys: true, + ..OrjsonOptions::default() + }; + dumps(&record, &JsonDialect::Orjson(options)).unwrap_or_default() + "\n" +} + +/// Append the streams the command wrote and the capture failure, without completing it. +pub fn abandon_capture(capture: OpenCapture, end: &CaptureEnd) -> io::Result<()> { + let OpenCapture { + invocation_id, + mut streams, + .. + } = capture; + let [stdout, stderr, diagnostics] = streams.as_mut_slice() else { + return Err(io::Error::other("capture streams are not open")); + }; + stdout.write_all(&end.stdout)?; + stderr.write_all(&end.stderr)?; + diagnostics.write_all(capture_failure_record(&invocation_id, SystemTime::now()).as_bytes())?; + for stream in [stdout, stderr, diagnostics] { + stream.flush()?; + } + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/compute_logs/main/mod.rs b/crates/sqlbuild-cli/src/compute_logs/main/mod.rs index 8e44c3eed2..60606a3a12 100644 --- a/crates/sqlbuild-cli/src/compute_logs/main/mod.rs +++ b/crates/sqlbuild-cli/src/compute_logs/main/mod.rs @@ -1,3 +1,4 @@ +pub mod abandon_capture; pub mod capture_inventory; pub mod capture_is_complete; pub mod check_completion; diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/messages.rs b/crates/sqlbuild-cli/src/launcher/_helpers/messages.rs index 8a91e6387d..6a9994f629 100644 --- a/crates/sqlbuild-cli/src/launcher/_helpers/messages.rs +++ b/crates/sqlbuild-cli/src/launcher/_helpers/messages.rs @@ -3,11 +3,16 @@ use std::io::{self, Write}; use std::path::Path; -use crate::launcher::constants::{PROGRAM_NAME, PYTHON_CLI_SCRIPT}; +use crate::launcher::constants::{PROGRAM_NAME, PROGRAM_VERSION, PYTHON_CLI_SCRIPT}; -/// argparse's `--version` line: `sqb ` on stdout. +/// argparse's `--version` line: `sqb `. +pub(crate) fn version_line() -> String { + format!("{PROGRAM_NAME} {PROGRAM_VERSION}\n") +} + +/// Print the `--version` line on stdout. pub(crate) fn print_version() { - let _ = writeln!(io::stdout(), "{PROGRAM_NAME} {}", env!("CARGO_PKG_VERSION")); + let _ = io::stdout().write_all(version_line().as_bytes()); } /// The Python CLI script is not installed beside this program. diff --git a/crates/sqlbuild-cli/src/launcher/constants.rs b/crates/sqlbuild-cli/src/launcher/constants.rs index 82cad78763..ce9a8716eb 100644 --- a/crates/sqlbuild-cli/src/launcher/constants.rs +++ b/crates/sqlbuild-cli/src/launcher/constants.rs @@ -11,6 +11,8 @@ pub const LAUNCH_RECORD_OPTION: &str = "--sqlbuild-launch-record="; /// The program name argparse reports, fixed by the Python parser. pub const PROGRAM_NAME: &str = "sqb"; pub const VERSION_OPTIONS: [&str; 2] = ["--version", "-V"]; +/// The version `--version` prints, alone or among other options; the wheels share it. +pub const PROGRAM_VERSION: &str = env!("CARGO_PKG_VERSION"); /// Global options naming the project directory, exactly as the Python parser spells them. pub const PROJECT_DIR_OPTIONS: [&str; 2] = ["--project-dir", "--sqb-project-dir"]; /// Prefixes argparse could expand to a project directory option. diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs index 9e26816a90..4a7f8b62de 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs @@ -7,6 +7,7 @@ use sqlbuild_core::text::main::python_str_repr::python_str_repr; use crate::argv::main::cli_namespace_defaults::cli_namespace_defaults; use crate::argv::models::{NamespaceDefault, TypeFailure, Value}; use crate::argv::types::ArgvHost; +use crate::launcher::constants::PROGRAM_VERSION; use crate::native_commands::constants::{ DIGIT_SEPARATOR, INT_ASCII_SPACES, INT_CONVERTER, NEGATIVE_SIGN, POSITIVE_INTEGER_CONVERTER, POSITIVE_INTEGER_MESSAGE, POSITIVE_SIGN, PYTHON_INT_MAX_STR_DIGITS, ZERO_TEXT, @@ -93,7 +94,7 @@ impl ArgvHost for NativeArgvHost { } fn program_version(&mut self) -> String { - env!("CARGO_PKG_VERSION").to_owned() + PROGRAM_VERSION.to_owned() } } diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 278a397f81..7021116ad0 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -6,6 +6,7 @@ use std::time::SystemTime; use crate::argv::models::Stream; use crate::compute_logs::constants::DEFAULT_RETENTION_COUNT; +use crate::compute_logs::main::abandon_capture::abandon_capture; use crate::compute_logs::main::finish_capture::finish_capture; use crate::compute_logs::main::open_log_root::open_log_root; use crate::compute_logs::main::start_capture::start_capture; @@ -16,11 +17,11 @@ use crate::native_commands::_helpers::paths::project_location; use crate::native_commands::_helpers::project_facts::has_no_runtime_extensions; use crate::native_commands::_helpers::rules::run_rules; use crate::native_commands::_helpers::skills::{run_skills, skill_freshness_notice}; -use crate::native_commands::_helpers::terminal::{exit_on_interrupt, write_text}; -use crate::native_commands::constants::UNKNOWN_COMMAND; +use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; +use crate::native_commands::constants::{UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND}; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{ - CommandOutput, InstalledPackage, NativeCommand, NativeInvocation, ProjectLocation, + CommandOutput, InstalledPackage, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; /// The capture Python's `dispatch_with_compute_logs` opens; a failure only loses the log. @@ -44,6 +45,19 @@ fn open_capture( }) } +/// Close the capture the way Python's does once the command's output met its reader. +fn close_capture(capture: OpenCapture, mut end: CaptureEnd, lost: Option<&LostStdout>) { + match lost { + None => drop(finish_capture(capture, &end)), + Some(LostStdout::Raised(_)) => { + end.stdout.clear(); + end.exit_code = i64::from(UNCAUGHT_EXCEPTION_STATUS); + drop(finish_capture(capture, &end)); + } + Some(LostStdout::AtExit(_) | LostStdout::Dropped) => drop(abandon_capture(capture, &end)), + } +} + /// Print an observed command's output after the skill notice, retaining both in its capture. fn finish_observed( invocation: &NativeInvocation, @@ -58,20 +72,22 @@ fn finish_observed( }; let capture: io::Result = open_capture(invocation, project, started_at); let stderr: String = notice + &output.stderr; - write_text(Stream::Stderr, &stderr); - write_text(Stream::Stdout, &output.stdout); + let lost: Option = write_output(&[ + (Stream::Stderr, stderr.as_str()), + (Stream::Stdout, output.stdout.as_str()), + ]); if let Ok(capture) = capture { - let finished: io::Result<()> = finish_capture( + close_capture( capture, - &CaptureEnd { + CaptureEnd { stdout: output.stdout.into_bytes(), stderr: stderr.into_bytes(), exit_code: i64::from(output.exit_code), }, + lost.as_ref(), ); - drop(finished); } - output.exit_code + finish_lost_stdout(lost.as_ref(), output.exit_code) } /// What an observed command prints, unless it needs Python after all. @@ -97,14 +113,15 @@ pub(crate) fn run_command( python_cli: &Path, error_color: bool, ) -> Result { - exit_on_interrupt(); let package: InstalledPackage = installed_package(python_cli)?; let project: ProjectLocation = project_location(invocation.project_dir.as_deref())?; if invocation.creates_project { let output: CommandOutput = run_init(&project.resolved, &package, error_color)?; - write_text(Stream::Stderr, &output.stderr); - write_text(Stream::Stdout, &output.stdout); - return Ok(output.exit_code); + let lost: Option = write_output(&[ + (Stream::Stderr, output.stderr.as_str()), + (Stream::Stdout, output.stdout.as_str()), + ]); + return Ok(finish_lost_stdout(lost.as_ref(), output.exit_code)); } let started_at: SystemTime = SystemTime::now(); let output: CommandOutput = observed_output(&invocation.command, &project, &package)?; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs index dfee4cd22b..ec54a0f30a 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs @@ -17,7 +17,7 @@ use sqlbuild_core::init_command::main::scaffold_blank_project::scaffold_blank_pr use crate::dispatch::main::failure_report::failure_report; use crate::dispatch::models::CommandFailure; use crate::native_commands::_helpers::skills::home_dir; -use crate::native_commands::_helpers::terminal::stdout_supports_color; +use crate::native_commands::_helpers::terminal::{error_description, stdout_supports_color}; use crate::native_commands::constants::{INIT_FAILURE_PREFIX, USER_ERROR_CLASS}; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{CommandOutput, InstalledPackage}; @@ -31,15 +31,31 @@ fn failed(stderr: String) -> Result { }) } -/// The project's skills could not be installed: what the Python CLI prints for its error. -fn skills_failure(error: &WorkspaceError, use_color: bool) -> Result { - let WorkspaceError::User { - code, - message, - help, - } = error - else { - return failed(format!("{INIT_FAILURE_PREFIX}{error:?}\n")); +/// The project's skills could not be installed: what the Python CLI prints for its user error. +fn skills_failure( + base_dir: &Path, + error: &WorkspaceError, + use_color: bool, +) -> Result { + let (code, message, help) = match error { + WorkspaceError::User { + code, + message, + help, + } => (code, message, help), + WorkspaceError::Decode { path, .. } => { + return failed(format!( + "{INIT_FAILURE_PREFIX}cannot install SQLBuild skills: {} is not UTF-8 text\n", + path.display() + )); + } + WorkspaceError::Io(error) => { + return failed(format!( + "{INIT_FAILURE_PREFIX}cannot install SQLBuild skills into {}: {}\n", + base_dir.display(), + error_description(error) + )); + } }; let (_, log_message) = failure_report(CommandFailure::User); let coded: String = format_coded_error( @@ -75,8 +91,9 @@ pub(crate) fn run_init( return Err(PythonNeeded::Failure); } return failed(format!( - "{INIT_FAILURE_PREFIX}{error}: {}\n", - path.display() + "{INIT_FAILURE_PREFIX}cannot create {}: {}\n", + path.display(), + error_description(&error) )); } if let Err(error) = install_skills(&SkillInstallRequest { @@ -87,7 +104,7 @@ pub(crate) fn run_init( requested_targets: Vec::new(), force: false, }) { - return skills_failure(&error, error_color); + return skills_failure(base_dir, &error, error_color); } Ok(CommandOutput { stdout: render_init_summary( diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs index 3a584e7abd..44d55e4570 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs @@ -1,11 +1,15 @@ -//! The process's terminal: its width, colour support, stream writes and Ctrl+C. +//! The process's terminal: its width, colour support and stream writes. use std::io::{self, IsTerminal, Write}; use crate::argv::models::Stream; use crate::native_commands::_helpers::argv_host::python_int; -use crate::native_commands::constants::{COLUMNS_VARIABLE, FALLBACK_COLUMNS, NO_COLOR_VARIABLE}; +use crate::native_commands::constants::{ + BROKEN_PIPE_EXCEPTION, COLUMNS_VARIABLE, EXIT_FLUSH_FAILURE, EXIT_FLUSH_FAILURE_STATUS, + FALLBACK_COLUMNS, NO_COLOR_VARIABLE, PYTHON_STDOUT_BUFFER_SIZE, UNCAUGHT_EXCEPTION_STATUS, +}; use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::LostStdout; /// `int(os.environ["COLUMNS"])`, `0` when unset or not an integer; beyond 64 bits is Python's. fn columns_variable() -> Result { @@ -59,35 +63,93 @@ fn console_bytes(text: &str) -> Vec { } } -/// Write and flush one text; a closed or full stream drops it, as the parse's writes do. -pub(crate) fn write_text(stream: Stream, text: &str) { - if text.is_empty() { - return; - } - let bytes: Vec = console_bytes(text); - let written: io::Result<()> = match stream { +fn write_stream(stream: Stream, bytes: &[u8]) -> io::Result<()> { + match stream { Stream::Stdout => { let mut output = io::stdout().lock(); - output.write_all(&bytes).and_then(|()| output.flush()) + output.write_all(bytes).and_then(|()| output.flush()) } Stream::Stderr => { let mut output = io::stderr().lock(); - output.write_all(&bytes).and_then(|()| output.flush()) + output.write_all(bytes).and_then(|()| output.flush()) } - }; - drop(written); + } } -/// End the process with status 130 on Ctrl+C; handing off by `exec` restores the default. -#[cfg(unix)] -pub(crate) fn exit_on_interrupt() { - use crate::native_commands::constants::INTERRUPTED_EXIT_CODE; +/// The system's description of an I/O failure, without Rust's `(os error N)` suffix. +pub(crate) fn error_description(error: &io::Error) -> String { + let described: String = error.to_string(); + match error.raw_os_error() { + Some(errno) => described + .strip_suffix(&format!(" (os error {errno})")) + .unwrap_or(&described) + .to_owned(), + None => described, + } +} - drop(ctrlc::set_handler(|| { - std::process::exit(INTERRUPTED_EXIT_CODE) - })); +/// The last line of the traceback Python prints for a failed write. +fn exception_line(error: &io::Error) -> String { + if error.kind() == io::ErrorKind::BrokenPipe { + return BROKEN_PIPE_EXCEPTION.to_owned(); + } + match error.raw_os_error() { + Some(errno) => format!("OSError: [Errno {errno}] {}", error_description(error)), + None => format!("OSError: {error}"), + } } -/// The Python child waited for alone handles Ctrl+C, so the native command keeps the default. -#[cfg(not(unix))] -pub(crate) fn exit_on_interrupt() {} +/// CPython's 8 KiB stdout buffer keeps up to half until exit, drops more, and fails a full write. +pub(crate) fn lost_stdout(total: usize, error: &io::Error) -> LostStdout { + if total >= PYTHON_STDOUT_BUFFER_SIZE { + LostStdout::Raised(exception_line(error)) + } else if total <= PYTHON_STDOUT_BUFFER_SIZE / 2 { + LostStdout::AtExit(exception_line(error)) + } else { + LostStdout::Dropped + } +} + +/// Write each text in order and flush it; errors on stderr are ignored, as Python's are. +pub(crate) fn write_output(writes: &[(Stream, &str)]) -> Option { + let encoded: Vec<(Stream, Vec)> = writes + .iter() + .filter(|(_, text)| !text.is_empty()) + .map(|(stream, text)| (*stream, console_bytes(text))) + .collect(); + let total: usize = encoded + .iter() + .filter(|(stream, _)| *stream == Stream::Stdout) + .map(|(_, bytes)| bytes.len()) + .sum(); + let mut lost: Option = None; + for (stream, bytes) in &encoded { + let written: io::Result<()> = match (stream, &lost) { + (Stream::Stdout, Some(_)) => Ok(()), + _ => write_stream(*stream, bytes), + }; + if let (Stream::Stdout, Err(error)) = (stream, written) { + lost = Some(lost_stdout(total, &error)); + } + } + lost +} + +/// Report lost stdout as Python does and return the exit status that results. +pub(crate) fn finish_lost_stdout(lost: Option<&LostStdout>, exit_code: i32) -> i32 { + match lost { + None | Some(LostStdout::Dropped) => exit_code, + Some(LostStdout::AtExit(exception)) => { + let text: String = format!("{EXIT_FLUSH_FAILURE}\n{exception}\n"); + drop(write_stream(Stream::Stderr, &console_bytes(&text))); + EXIT_FLUSH_FAILURE_STATUS + } + Some(LostStdout::Raised(exception)) => { + drop(write_stream( + Stream::Stderr, + &console_bytes(&format!("{exception}\n")), + )); + UNCAUGHT_EXCEPTION_STATUS + } + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index fdef90a8cf..5b1dd8ff84 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -32,9 +32,6 @@ pub(crate) const SKILLS_DIRECTORY: [&str; 3] = [".agents", "skills", "sqlbuild"] pub(crate) const HOME_VARIABLE: &str = "HOME"; #[cfg(windows)] pub(crate) const HOME_VARIABLE: &str = "USERPROFILE"; -/// The exit status of a command stopped by Ctrl+C. -#[cfg(unix)] -pub(crate) const INTERRUPTED_EXIT_CODE: i32 = 130; /// The command name compute logs record for a bare `sqb`. pub(crate) const UNKNOWN_COMMAND: &str = "unknown"; /// Commands with no handler: observed, then successful. @@ -77,3 +74,11 @@ pub(crate) const PYTHON_INT_MAX_STR_DIGITS: usize = 4300; pub(crate) const DIGIT_SEPARATOR: char = '_'; /// Where a loaded project configuration names its lifecycle sinks. pub(crate) const LIFECYCLE_SINK_PATH: [&str; 3] = ["sinks", "lifecycle", "named"]; +/// CPython's default buffer size for a piped stdout. +pub(crate) const PYTHON_STDOUT_BUFFER_SIZE: usize = 8192; +pub(crate) const BROKEN_PIPE_EXCEPTION: &str = "BrokenPipeError: [Errno 32] Broken pipe"; +/// What Python prints when the final flush of a lost stdout fails, and the status it exits with. +pub(crate) const EXIT_FLUSH_FAILURE: &str = + "Exception ignored in: <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>"; +pub(crate) const EXIT_FLUSH_FAILURE_STATUS: i32 = 120; +pub(crate) const UNCAUGHT_EXCEPTION_STATUS: i32 = 1; diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs index 222d953909..1233e99081 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs @@ -4,16 +4,16 @@ use std::ffi::OsString; use std::path::Path; use crate::argv::main::parse_cli_arguments::parse_cli_arguments; -use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace}; +use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace, Stream}; use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; use crate::native_commands::_helpers::decision::native_invocation; use crate::native_commands::_helpers::execution::run_command; use crate::native_commands::_helpers::terminal::{ - stdout_supports_color, terminal_columns, write_text, + finish_lost_stdout, stdout_supports_color, terminal_columns, write_output, }; use crate::native_commands::constants::NO_COLOR_OPTION; use crate::native_commands::errors::PythonNeeded; -use crate::native_commands::models::NativeInvocation; +use crate::native_commands::models::{LostStdout, NativeInvocation}; /// The exit status of an invocation `sqb` served, or why the same arguments go to Python. pub fn run_native_invocation( @@ -39,10 +39,16 @@ pub fn run_native_invocation( } let namespace: Namespace = match outcome.parsed { Err(code) => { - for (stream, text) in &outcome.writes { - write_text(*stream, text); - } - return Ok(i32::try_from(code).unwrap_or(1)); + let writes: Vec<(Stream, &str)> = outcome + .writes + .iter() + .map(|(stream, text)| (*stream, text.as_str())) + .collect(); + let lost: Option = write_output(&writes); + return Ok(finish_lost_stdout( + lost.as_ref(), + i32::try_from(code).unwrap_or(1), + )); } Ok(namespace) if outcome.writes.is_empty() => namespace, Ok(_) => return Err(PythonNeeded::Arguments), diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index f0ce868fa2..9c6ece8ad5 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -71,3 +71,14 @@ pub(crate) struct ProjectLocation { /// `Path(--project-dir).resolve()`, or `Path.cwd()`. pub(crate) resolved: PathBuf, } + +/// What Python's buffered stdout does with output its reader no longer takes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum LostStdout { + /// Kept until exit, whose failed flush Python reports with this exception and status 120. + AtExit(String), + /// Dropped by a failed flush; the command's own status stands. + Dropped, + /// Raised by the write: an uncaught exception, with this last line and status 1. + Raised(String), +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests.rs b/crates/sqlbuild-cli/src/native_commands/tests.rs index 9119c141c9..2f86f371e1 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests.rs @@ -4,11 +4,15 @@ mod argv_host; mod decision; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_init.rs"] +mod init; #[path = "tests/test_paths.rs"] mod paths; #[path = "tests/test_project_facts.rs"] mod project_facts; #[path = "tests/test_rules.rs"] mod rules; +#[path = "tests/test_terminal.rs"] +mod terminal; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_init.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_init.rs new file mode 100644 index 0000000000..29d90d5f7f --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_init.rs @@ -0,0 +1,56 @@ +use std::path::Path; + +use crate::native_commands::_helpers::init::run_init; +use crate::native_commands::models::InstalledPackage; +use crate::native_commands::tests::helpers::write_tree; +use crate::native_commands::tests::test_types::InitFailureTestCase; + +#[cfg(unix)] +#[test] +fn given_unwritable_skill_directory_when_initialising_then_a_readable_error_names_the_path() { + let test_cases = [InitFailureTestCase { + description: "a file where the agents skill directory goes", + blocking_file: ".agents", + expected_stderr_start: "sqb init: cannot install SQLBuild skills into ", + expected_exit_code: 1, + }]; + + for test_case in test_cases { + let root = tempfile::tempdir().expect("temporary directory"); + let skills = root.path().join("packaged"); + write_tree( + root.path(), + &["packaged", "shop"], + &[("packaged/SKILL.md", "# Skill\n")], + ); + let project: &Path = &root.path().join("shop"); + write_tree( + project, + &[], + &[(test_case.blocking_file, "not a directory\n")], + ); + + let output = run_init( + project, + &InstalledPackage { + skills_root: skills, + }, + false, + ) + .expect(test_case.description); + + assert!( + output.stderr.starts_with(test_case.expected_stderr_start) + && output.stderr.contains(&project.display().to_string()) + && output.stderr.ends_with("Not a directory\n"), + "{}: {}", + test_case.description, + output.stderr + ); + assert_eq!( + output.exit_code, test_case.expected_exit_code, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_terminal.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_terminal.rs new file mode 100644 index 0000000000..19be6f26cd --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_terminal.rs @@ -0,0 +1,75 @@ +use std::io; + +use crate::argv::models::Stream; +use crate::launcher::constants::{PROGRAM_NAME, PROGRAM_VERSION}; +use crate::native_commands::_helpers::terminal::lost_stdout; +use crate::native_commands::models::LostStdout; +use crate::native_commands::tests::helpers::parse; +use crate::native_commands::tests::test_types::{LostStdoutTestCase, ParseTestCase}; + +/// Thresholds recorded from CPython 3.12 writing to a pipe without a reader. +#[test] +fn given_lost_stdout_bytes_when_classifying_then_python_buffer_outcome_matches() { + let test_cases = [ + LostStdoutTestCase { + description: "half the buffer stays until the final flush", + total: 4096, + expected_lost: LostStdout::AtExit("BrokenPipeError: [Errno 32] Broken pipe".to_owned()), + }, + LostStdoutTestCase { + description: "more than half is dropped by the failed flush", + total: 4097, + expected_lost: LostStdout::Dropped, + }, + LostStdoutTestCase { + description: "a full buffer fails the write", + total: 8192, + expected_lost: LostStdout::Raised("BrokenPipeError: [Errno 32] Broken pipe".to_owned()), + }, + ]; + + for test_case in test_cases { + let error: io::Error = io::Error::from(io::ErrorKind::BrokenPipe); + + assert_eq!( + lost_stdout(test_case.total, &error), + test_case.expected_lost, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_version_among_options_when_parsing_natively_then_the_lone_version_line_prints() { + let test_cases = [ + ParseTestCase { + description: "after --no-color", + argv: &["--no-color", "--version"], + expected_needs_python: false, + }, + ParseTestCase { + description: "the short option after --debug", + argv: &["--debug", "-V"], + expected_needs_python: false, + }, + ]; + + for test_case in test_cases { + let (outcome, host) = parse(test_case.argv); + + assert_eq!( + (outcome.writes, outcome.parsed, host.needs_python), + ( + vec![( + Stream::Stdout, + format!("{PROGRAM_NAME} {PROGRAM_VERSION}\n") + )], + Err(0), + test_case.expected_needs_python, + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs index 528aeaf4e4..252ef24b95 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -66,3 +66,18 @@ pub(super) fn output_start(output: &CommandOutput, length: usize) -> (String, i3 output.exit_code, ) } + +/// Bytes a reader no longer takes and what Python's stdout buffer makes of them. +pub(super) struct LostStdoutTestCase { + pub(super) description: &'static str, + pub(super) total: usize, + pub(super) expected_lost: crate::native_commands::models::LostStdout, +} + +/// A directory `init` meets once the project file exists, and the start of its error. +pub(super) struct InitFailureTestCase { + pub(super) description: &'static str, + pub(super) blocking_file: &'static str, + pub(super) expected_stderr_start: &'static str, + pub(super) expected_exit_code: i32, +} diff --git a/tests/integration/src/sqlbuild/cli/entry/_test_types.py b/tests/integration/src/sqlbuild/cli/entry/_test_types.py index a39ae9398f..5be5472a2a 100644 --- a/tests/integration/src/sqlbuild/cli/entry/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/entry/_test_types.py @@ -113,3 +113,14 @@ class InterruptTestCase: args: tuple[str, ...] prepare: Callable[[Path], None] expected_exit_code: int + + +@dataclass(frozen=True) +class BrokenPipeTestCase: + """One Python-free command line whose stdout has no reader.""" + + description: str + args: tuple[str, ...] + prepare: Callable[[Path], None] + expected_exit_code: int + expected_stderr_end: bytes diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index 09fbd33876..15825c8079 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -14,6 +14,7 @@ _ELAPSED_SECONDS: re.Pattern[bytes] = re.compile(rb"\(\d+\.\d s\)") _PHASE_SECONDS: re.Pattern[bytes] = re.compile(rb"\(\d+\.\d+s\)") +_DIAGNOSTIC_IDENTITY: re.Pattern[bytes] = re.compile(rb'"(invocation_id|occurred_at)":"[^"]*"') NATIVE_SQB: Path = Path(sys.executable).with_name("sqb") PYTHON_CLI: Path = Path(sys.executable).with_name("sqlbuild-python") PROJECT_FILES: dict[str, str] = { @@ -227,9 +228,9 @@ def reset_directory(directory: Path) -> Path: def _capture_files(capture: Path) -> dict[str, object]: metadata: dict[str, object] = capture_metadata(capture) for volatile in ("invocation_id", "started_at", "completed_at", "capture_date"): - _ = metadata.pop(volatile) + _ = metadata.pop(volatile, None) streams: dict[str, bytes] = { - name: (capture / name).read_bytes() + name: _DIAGNOSTIC_IDENTITY.sub(rb'"\1":""', (capture / name).read_bytes()) for name in ("stdout.log", "stderr.log", "diagnostics.jsonl") } return {"complete": (capture / "complete").exists(), "metadata": metadata, **streams} @@ -291,3 +292,23 @@ def restore_default_interrupt() -> None: """Give a spawned command default Ctrl-C handling even when its parent ignores SIGINT.""" _ = signal.signal(signal.SIGINT, signal.SIG_DFL) + + +def run_without_reader(program: Path, *, args: tuple[str, ...], cwd: Path) -> LauncherRun: + """Run with a stdout pipe whose reader has already closed; return exit code and stderr.""" + + reader, writer = os.pipe() + os.close(reader) + try: + completed: subprocess.CompletedProcess[bytes] = subprocess.run( + [str(program), *args], + cwd=cwd, + stdout=writer, + stderr=subprocess.PIPE, + env={**os.environ, "NO_COLOR": "1"}, + check=False, + timeout=600, + ) + finally: + os.close(writer) + return completed.returncode, b"", completed.stderr diff --git a/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py index 6a787e7454..fe5412e49b 100644 --- a/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py +++ b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py @@ -7,6 +7,7 @@ import pytest from tests.integration.src.sqlbuild.cli.entry._test_types import ( + BrokenPipeTestCase, InterruptTestCase, PythonFreeTestCase, PythonStartTestCase, @@ -26,9 +27,11 @@ reset_directory, restore_default_interrupt, run, + run_without_reader, ) INTERRUPT_DELAY_SECONDS: float = 1.0 +TRACEBACK_LAST_LINE: bytes = b"BrokenPipeError: [Errno 32] Broken pipe\n" INTERRUPT_TIMEOUT_SECONDS: float = 30.0 @@ -299,12 +302,12 @@ def test_given_invocation_needing_python_when_running_sqb_then_python_runs_it( description="rules list waiting on the project file", args=("rules", "list"), prepare=prepare_named_pipe_project, - expected_exit_code=130, + expected_exit_code=-signal.SIGINT, ), ], ids=lambda case: case.description, ) -def test_given_python_free_command_when_interrupted_then_sqb_exits_130_without_python( +def test_given_python_free_command_when_interrupted_then_sqb_dies_by_sigint_without_python( test_case: InterruptTestCase, tmp_path: Path ) -> None: marker: Path = tmp_path / "python-starts.log" @@ -329,5 +332,61 @@ def test_given_python_free_command_when_interrupted_then_sqb_exits_130_without_p assert python_starts(marker) == 0 +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX pipes") +@pytest.mark.parametrize( + "test_case", + [ + BrokenPipeTestCase( + description="help kept until the final flush", + args=("--help",), + prepare=prepare_nothing, + expected_exit_code=120, + expected_stderr_end=TRACEBACK_LAST_LINE, + ), + BrokenPipeTestCase( + description="a short observed report kept until the final flush", + args=("rules", "show", "SQBRMODEL101"), + prepare=prepare_project, + expected_exit_code=120, + expected_stderr_end=TRACEBACK_LAST_LINE, + ), + BrokenPipeTestCase( + description="a report the failed flush drops", + args=("rules", "list"), + prepare=prepare_project, + expected_exit_code=0, + expected_stderr_end=b"", + ), + BrokenPipeTestCase( + description="a report whose write fails", + args=("rules", "--json", "list"), + prepare=prepare_project, + expected_exit_code=1, + expected_stderr_end=TRACEBACK_LAST_LINE, + ), + ], + ids=lambda case: case.description, +) +def test_given_stdout_without_reader_when_running_sqb_then_status_and_logs_match_python( + test_case: BrokenPipeTestCase, tmp_path: Path +) -> None: + marker: Path = tmp_path / "python-starts.log" + _ = marker.write_text("", encoding="utf-8") + program: Path = counting_environment(tmp_path / "environment", marker) + workspace: Path = reset_directory(tmp_path / "workspace") + test_case.prepare(workspace) + python: LauncherRun = run_without_reader(PYTHON_CLI, args=test_case.args, cwd=workspace) + python_logs: list[dict[str, object]] = compute_log_files(workspace) + test_case.prepare(reset_directory(workspace)) + + native: LauncherRun = run_without_reader(program, args=test_case.args, cwd=workspace) + + assert (native[0], python[0]) == (test_case.expected_exit_code,) * 2 + assert native[2].endswith(test_case.expected_stderr_end), native[2] + assert python[2].endswith(test_case.expected_stderr_end), python[2] + assert python_starts(marker) == 0 + assert compute_log_files(workspace) == python_logs + + if __name__ == "__main__": pytest.main([__file__, "-vv"])