From 44cba91fc1c66321d1f3fd8d1373914da9c3a300 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 6 Nov 2026 07:47:09 +0000 Subject: [PATCH 1/2] feat: read a SQLBuild-style project beside sqlquack.toml convert and census read the folder holding the policy file as a project: tables from sources/*.yml in SQLBuild's source format, functions from functions/sql/*.sql (FUNCTION header) and functions/python/*.py (@udf decorator from sqlbuild.functions or sqlquack), and overrides from overrides/. [project.session] and [project.confirm] give session parameters and confirmations; flags apply on top for one run, --schema adds to the project's declarations, and a name declared twice is an error naming both files. The Python API takes ConvertOptions(project=). A refused Python function's help names the line to add under [project.confirm]. Override headers share the project's header parser, which now keeps commas and parentheses inside quoted descriptions. --- Cargo.lock | 43 +++- Cargo.toml | 1 + README.md | 11 + .../src/verify/_helpers/session.rs | 17 +- .../src/verify/tests/helpers.rs | 3 +- crates/sqlquack-policy/src/policy/models.rs | 50 ++++ .../src/bindings/_helpers/convert_json.rs | 6 +- crates/sqlquack-python/src/bindings/models.rs | 2 + .../_helpers/expressions/project_functions.rs | 5 +- .../src/session/_helpers/session_profiles.rs | 17 +- .../main/session_conversion_options.rs | 4 +- crates/sqlquack/Cargo.toml | 1 + .../analysis/_helpers/override_definition.rs | 101 +------- .../src/cli/_helpers/census_command.rs | 30 ++- crates/sqlquack/src/cli/_helpers/evidence.rs | 18 +- .../src/cli/_helpers/function_command.rs | 2 +- .../src/cli/_helpers/index_command.rs | 2 +- crates/sqlquack/src/cli/_helpers/mod.rs | 2 +- crates/sqlquack/src/cli/_helpers/project.rs | 68 +++++ .../sqlquack/src/cli/main/convert_report.rs | 24 +- crates/sqlquack/src/cli/models.rs | 22 +- crates/sqlquack/src/lib.rs | 1 + .../src/project/_helpers/declarations.rs | 122 +++++++++ .../src/project/_helpers/function_files.rs | 121 +++++++++ .../src/project/_helpers/header_parts.rs | 84 ++++++ crates/sqlquack/src/project/_helpers/mod.rs | 8 + .../src/project/_helpers/python_literal.rs | 164 ++++++++++++ .../src/project/_helpers/source_files.rs | 64 +++++ .../src/project/_helpers/yaml_value.rs | 109 ++++++++ crates/sqlquack/src/project/constants.rs | 42 +++ .../src/project/main/function_header.rs | 9 + .../schema.rs => project/main/load_schema.rs} | 4 +- crates/sqlquack/src/project/main/mod.rs | 5 + .../sqlquack/src/project/main/named_types.rs | 33 +++ .../src/project/main/project_folder.rs | 20 ++ .../src/project/main/project_schema.rs | 47 ++++ crates/sqlquack/src/project/mod.rs | 6 + crates/sqlquack/src/project/models.rs | 14 + crates/sqlquack/tests/cli.rs | 2 + crates/sqlquack/tests/cli/helpers.rs | 35 ++- crates/sqlquack/tests/cli/test_project.rs | 241 ++++++++++++++++++ crates/sqlquack/tests/cli/test_types.rs | 26 ++ python/src/sqlquack/__init__.py | 2 + python/src/sqlquack/api/main/convert.py | 1 + python/src/sqlquack/api/main/udf.py | 16 ++ python/src/sqlquack/api/models.py | 3 +- .../src/sqlquack/api/main/_test_types.py | 8 + .../src/sqlquack/api/main/test_convert.py | 40 +++ website/src/content/docs/docs/cli/convert.mdx | 6 +- website/src/content/docs/docs/policy.mdx | 6 +- website/src/content/docs/docs/project.mdx | 146 +++++++++++ website/src/content/docs/docs/python.mdx | 14 +- website/src/content/docs/docs/schema.mdx | 6 +- website/src/sidebar.json | 2 +- 54 files changed, 1660 insertions(+), 176 deletions(-) create mode 100644 crates/sqlquack/src/cli/_helpers/project.rs create mode 100644 crates/sqlquack/src/project/_helpers/declarations.rs create mode 100644 crates/sqlquack/src/project/_helpers/function_files.rs create mode 100644 crates/sqlquack/src/project/_helpers/header_parts.rs create mode 100644 crates/sqlquack/src/project/_helpers/mod.rs create mode 100644 crates/sqlquack/src/project/_helpers/python_literal.rs create mode 100644 crates/sqlquack/src/project/_helpers/source_files.rs create mode 100644 crates/sqlquack/src/project/_helpers/yaml_value.rs create mode 100644 crates/sqlquack/src/project/constants.rs create mode 100644 crates/sqlquack/src/project/main/function_header.rs rename crates/sqlquack/src/{cli/_helpers/schema.rs => project/main/load_schema.rs} (80%) create mode 100644 crates/sqlquack/src/project/main/mod.rs create mode 100644 crates/sqlquack/src/project/main/named_types.rs create mode 100644 crates/sqlquack/src/project/main/project_folder.rs create mode 100644 crates/sqlquack/src/project/main/project_schema.rs create mode 100644 crates/sqlquack/src/project/mod.rs create mode 100644 crates/sqlquack/src/project/models.rs create mode 100644 crates/sqlquack/tests/cli/test_project.rs create mode 100644 python/src/sqlquack/api/main/udf.py create mode 100644 website/src/content/docs/docs/project.mdx diff --git a/Cargo.lock b/Cargo.lock index 149ecdafe..661899d09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -99,6 +99,12 @@ dependencies = [ "object", ] +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "arrow" version = "60.0.0" @@ -989,7 +995,7 @@ dependencies = [ "serde", "serde_json", "stacker", - "thiserror", + "thiserror 1.0.69", "unicode-segmentation", ] @@ -1252,6 +1258,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "saphyr-parser" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7429158804c36e705d9423b7848018e535b3e92d439a763e364ee12670ef473b" +dependencies = [ + "arraydeque", + "thiserror 2.0.21", +] + [[package]] name = "semver" version = "1.0.28" @@ -1347,6 +1363,7 @@ dependencies = [ "clap", "duckdb", "rayon", + "saphyr-parser", "serde", "serde_json", "sha2", @@ -1486,7 +1503,7 @@ dependencies = [ "serde_json", "sqlquack-syntax-derive", "stacker", - "thiserror", + "thiserror 1.0.69", "unicode-segmentation", ] @@ -1616,7 +1633,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl 2.0.21", ] [[package]] @@ -1630,6 +1656,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + [[package]] name = "tiny-keccak" version = "2.0.2" diff --git a/Cargo.toml b/Cargo.toml index b9691a176..671d3f1fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,7 @@ rayon = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" # The Python module; abi3, so one build loads into every CPython from 3.12. +saphyr-parser = "0.1" pyo3 = { version = "0.30.1", features = ["abi3-py312"] } sha2 = "0.10" sqlquack = { path = "crates/sqlquack" } diff --git a/README.md b/README.md index 394ecc542..8dafd7640 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,17 @@ They share the policy flags (`--policy`, `--builtin-policy`, `--strict`, `--utc- on divergences whose converted SQL raises rather than returns another value). Every command with a report takes `--format text|json`; `--json` is short for `--format json`. +The folder holding the policy file is a project, laid out as a SQLBuild project. `convert` and +`census` read the tables in `sources/*.yml` (SQLBuild's source format), the functions in +`functions/sql/*.sql` (a `FUNCTION ( ... );` header, then the body) and `functions/python/*.py` (a +`@udf(...)` decorator imported from `sqlbuild.functions` or `sqlquack`), and the +overrides in `overrides/`, so a SQLBuild project's folders work unchanged. `sqlquack.toml` holds +`[project.session]` (as `--session`) and `[project.confirm]` with `python_runtime`, `utc_session` +and `declared_offsets` (as the confirmation flags), and folder paths under `[project]` only where +they differ from the defaults. Flags apply on top for one run, `--schema` adds to the project's +declarations, and a name declared twice is an error. The Python API takes the folder as +`ConvertOptions(project=...)`. + HASH is refused by default: the converted SQL computes DuckDB's hash, which is consistent within one run but never Snowflake's value. A project that relies on HASH only for joins, grouping, deduplication or change detection can opt in. A policy file is complete on its own, so start from diff --git a/crates/sqlquack-evidence/src/verify/_helpers/session.rs b/crates/sqlquack-evidence/src/verify/_helpers/session.rs index 2e0ee9d5a..86b492e24 100644 --- a/crates/sqlquack-evidence/src/verify/_helpers/session.rs +++ b/crates/sqlquack-evidence/src/verify/_helpers/session.rs @@ -33,7 +33,7 @@ pub(crate) fn session_check( return SessionCheck::Mismatch(format!( "session parameter {key} was recorded as {}, but the profile sets {}", actual.as_deref().unwrap_or("nothing"), - parameter_text(value) + value.text() )); } } @@ -51,25 +51,14 @@ pub(crate) fn session_check( return SessionCheck::Mismatch(format!( "recorded before {key} was pinned, and the account default is {}, not {}", default.map_or("unrecorded", String::as_str), - parameter_text(value) + value.text() )); } } SessionCheck::Matches } -/// A parameter value as `SHOW PARAMETERS` writes it. -pub(crate) fn parameter_text(value: &SessionValue) -> String { - match value { - SessionValue::Boolean(flag) => flag.to_string(), - SessionValue::Integer(number) => number.to_string(), - SessionValue::Text(text) => text.clone(), - } -} - /// Whether a reported value is the profile's, ignoring case and surrounding space. fn same(actual: &str, value: &SessionValue) -> bool { - actual - .trim() - .eq_ignore_ascii_case(parameter_text(value).trim()) + actual.trim().eq_ignore_ascii_case(value.text().trim()) } diff --git a/crates/sqlquack-evidence/src/verify/tests/helpers.rs b/crates/sqlquack-evidence/src/verify/tests/helpers.rs index 286b07145..ab3011e46 100644 --- a/crates/sqlquack-evidence/src/verify/tests/helpers.rs +++ b/crates/sqlquack-evidence/src/verify/tests/helpers.rs @@ -21,7 +21,6 @@ use crate::results::models::{ RecordedFailure, RecordedOutcome, ResultMetadata, SessionDefaults, WarehouseColumn, }; use crate::verify::_helpers::compile_errors::undeclared_compile_error; -use crate::verify::_helpers::session::parameter_text; use crate::verify::main::verify_cases::verify_cases; use crate::verify::models::VerifyInputs; use crate::verify::tests::test_types::{RoundTrip, Tampering, UnchangedSqlForTests}; @@ -330,7 +329,7 @@ fn record_stand_in(pending: &PendingCase, results: &Path) { .session_parameters .iter() .chain(&pending.pinned_parameters) - .map(|(key, value)| (key.clone(), Some(parameter_text(value)))) + .map(|(key, value)| (key.clone(), Some(value.text()))) .collect(), ), parameters_sha256: Some(STAND_IN_PARAMETERS_SHA256.to_owned()), diff --git a/crates/sqlquack-policy/src/policy/models.rs b/crates/sqlquack-policy/src/policy/models.rs index 62dd9ba2d..9b89f3c79 100644 --- a/crates/sqlquack-policy/src/policy/models.rs +++ b/crates/sqlquack-policy/src/policy/models.rs @@ -26,6 +26,45 @@ pub struct Policy { /// Limits on converting one statement; absent, the built-in defaults apply. #[serde(default)] pub conversion_budget: Option, + /// The project around the file: its folders, session and confirmations. + #[serde(default)] + pub project: ProjectPolicy, +} + +/// `[project]`: folders that differ from the defaults, the caller's session and what it confirms. +#[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProjectPolicy { + /// The folder of source YAML files, relative to the file; `sources` when absent. + #[serde(default)] + pub sources: Option, + /// The folder of SQL and Python function files, relative to the file; `functions` when absent. + #[serde(default)] + pub functions: Option, + /// The folder of override files, relative to the file; `overrides` when absent. + #[serde(default)] + pub overrides: Option, + /// Snowflake session parameters of the caller's session, as `--session` gives them. + #[serde(default)] + pub session: BTreeMap, + /// What the caller confirms, as the command-line confirmation flags do. + #[serde(default)] + pub confirm: ProjectConfirm, +} + +/// `[project.confirm]`: assumptions the project confirms for every run. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProjectConfirm { + /// `--python-runtime`. + #[serde(default)] + pub python_runtime: bool, + /// `--utc-session`. + #[serde(default)] + pub utc_session: bool, + /// `--confirm-declared-offsets`. + #[serde(default)] + pub declared_offsets: bool, } /// How long converting one statement may take and how much SQL it may emit, before it is refused. @@ -151,6 +190,17 @@ pub enum SessionValue { Text(String), } +impl SessionValue { + /// The value as text: booleans in lower case, integers in digits, text as written. + pub fn text(&self) -> String { + match self { + Self::Boolean(flag) => flag.to_string(), + Self::Integer(number) => number.to_string(), + Self::Text(text) => text.clone(), + } + } +} + /// The input tags a category's case files must cover between them. #[derive(Clone, Debug, PartialEq, Eq, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/crates/sqlquack-python/src/bindings/_helpers/convert_json.rs b/crates/sqlquack-python/src/bindings/_helpers/convert_json.rs index b265a356f..4fff37574 100644 --- a/crates/sqlquack-python/src/bindings/_helpers/convert_json.rs +++ b/crates/sqlquack-python/src/bindings/_helpers/convert_json.rs @@ -26,7 +26,11 @@ pub(crate) fn convert_json(sql: &str, request: &str, path: &str) -> PyResult policy, + (None, Some(project)) => project.join(POLICY_FILE_NAME), + (None, None) => POLICY_FILE_NAME.into(), + }, builtin_policy: request.builtin_policy, strict: request.strict, utc_session: request.utc_session, diff --git a/crates/sqlquack-python/src/bindings/models.rs b/crates/sqlquack-python/src/bindings/models.rs index b2e2f412a..4dade20aa 100644 --- a/crates/sqlquack-python/src/bindings/models.rs +++ b/crates/sqlquack-python/src/bindings/models.rs @@ -18,6 +18,8 @@ pub struct Rendered { #[derive(Debug, Default, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct ConvertRequest { + /// The project folder, whose policy file is the default `--policy`. + pub project: Option, /// `--schema`. pub schema: Option, /// `--policy`; the command line's default when absent. diff --git a/crates/sqlquack-rules/src/conversion/_helpers/expressions/project_functions.rs b/crates/sqlquack-rules/src/conversion/_helpers/expressions/project_functions.rs index ffc18b64d..a830213a3 100644 --- a/crates/sqlquack-rules/src/conversion/_helpers/expressions/project_functions.rs +++ b/crates/sqlquack-rules/src/conversion/_helpers/expressions/project_functions.rs @@ -3,6 +3,7 @@ use serde_json::Value; use sqlquack_core::classification::models::Construct; use sqlquack_core::provenance::models::ProvenanceLog; +use sqlquack_core::refusals::constants::NEEDS_DIVERGENCE; use sqlquack_core::refusals::main::outside_conditions::outside_conditions; use sqlquack_core::refusals::main::refused_divergence::refused_divergence; use sqlquack_core::refusals::models::Refusal; @@ -68,8 +69,8 @@ fn rely_on_python(name: &str, construct: &Construct, context: &Context<'_>) -> R if !context.options.divergences.contains_key(PROJECT_UDF) { return Err(Refusal { reason: format!( - "the Python function {name}, which runs only where a Python runtime registers it: \ - assumption `{PROJECT_UDF}`, which needs --python-runtime" + "the Python function {name}, which runs only where a Python runtime registers it,\ + {NEEDS_DIVERGENCE}{PROJECT_UDF}`, an assumption the caller has not confirmed" ), ..refused_divergence(construct.clone(), PROJECT_UDF) }); diff --git a/crates/sqlquack-rules/src/session/_helpers/session_profiles.rs b/crates/sqlquack-rules/src/session/_helpers/session_profiles.rs index a2c91e365..018d9ace5 100644 --- a/crates/sqlquack-rules/src/session/_helpers/session_profiles.rs +++ b/crates/sqlquack-rules/src/session/_helpers/session_profiles.rs @@ -46,7 +46,7 @@ pub(crate) fn matching_profile( .snowflake .get(name) .or_else(|| defaults.pinned.get(name)) - .map(value_text) + .map(SessionValue::text) }) }; let mut unmatched: BTreeSet = BTreeSet::new(); @@ -79,22 +79,11 @@ pub(crate) fn matching_profile( )) } -/// A parameter value as text: booleans in lower case, integers in digits, text as written. -pub(crate) fn value_text(value: &SessionValue) -> String { - match value { - SessionValue::Boolean(flag) => flag.to_string(), - SessionValue::Integer(number) => number.to_string(), - SessionValue::Text(text) => text.clone(), - } -} - /// Whether a caller's value is the profile's: booleans ignoring case, others exactly. fn same_value(given: Option<&str>, recorded: &SessionValue) -> bool { match (given, recorded) { - (Some(given), SessionValue::Boolean(_)) => { - given.eq_ignore_ascii_case(&value_text(recorded)) - } - (Some(given), _) => given == value_text(recorded), + (Some(given), SessionValue::Boolean(_)) => given.eq_ignore_ascii_case(&recorded.text()), + (Some(given), _) => given == recorded.text(), (None, _) => false, } } diff --git a/crates/sqlquack-rules/src/session/main/session_conversion_options.rs b/crates/sqlquack-rules/src/session/main/session_conversion_options.rs index b027c7614..5db023492 100644 --- a/crates/sqlquack-rules/src/session/main/session_conversion_options.rs +++ b/crates/sqlquack-rules/src/session/main/session_conversion_options.rs @@ -10,7 +10,7 @@ use sqlquack_policy::policy::constants::{ }; use sqlquack_policy::policy::models::{Policy, SessionValue}; -use crate::session::_helpers::session_profiles::{matching_profile, value_text}; +use crate::session::_helpers::session_profiles::matching_profile; use crate::session::constants::BUILTIN_PROFILE; use crate::session::main::confirm_utc_session::confirm_utc_session; use crate::session::main::conversion_options::conversion_options; @@ -46,7 +46,7 @@ pub fn session_conversion_options( .session_profiles .get(&profile) .and_then(|matched| matched.snowflake.get(QUOTED_IDENTIFIERS_IGNORE_CASE)) - .is_some_and(|value| value_text(value) == value_text(&SessionValue::Boolean(true))); + .is_some_and(|value| value.text() == SessionValue::Boolean(true).text()); let mut options = ConversionOptions { quoted_identifiers_ignore_case: ignore_case, ..conversion_options(policy, strict) diff --git a/crates/sqlquack/Cargo.toml b/crates/sqlquack/Cargo.toml index 6089ff83f..5dd332789 100644 --- a/crates/sqlquack/Cargo.toml +++ b/crates/sqlquack/Cargo.toml @@ -20,6 +20,7 @@ path = "src/bin/sqlquack.rs" clap.workspace = true rayon.workspace = true serde.workspace = true +saphyr-parser.workspace = true serde_json.workspace = true sha2.workspace = true sqlquack-compare.workspace = true diff --git a/crates/sqlquack/src/analysis/_helpers/override_definition.rs b/crates/sqlquack/src/analysis/_helpers/override_definition.rs index dc5d6f7db..6754199c3 100644 --- a/crates/sqlquack/src/analysis/_helpers/override_definition.rs +++ b/crates/sqlquack/src/analysis/_helpers/override_definition.rs @@ -9,8 +9,10 @@ use sqlquack_snowflake::typing::main::cast_target::cast_target; use crate::analysis::constants::VARIANT_HOLDING_TYPE; use crate::analysis::constants::{ AGGREGATE_OR_WINDOW_NAMES, OVERRIDE_ARGUMENTS, OVERRIDE_DIALECT, OVERRIDE_DUCKDB, - OVERRIDE_HEADER, OVERRIDE_REPLACES, OVERRIDE_RETURNS, SHA256_HEX_LENGTH, + OVERRIDE_REPLACES, OVERRIDE_RETURNS, SHA256_HEX_LENGTH, }; +use crate::project::main::function_header::function_header; +use crate::project::main::named_types::named_types; use sqlquack_duckdb::blocks::main::column_type::column_type; use sqlquack_duckdb::blocks::main::quoted_identifier::quoted_identifier; use sqlquack_evidence::replay::main::bind_macro::bind_macro; @@ -62,43 +64,16 @@ pub(crate) fn load_override(name: &str, text: &str) -> Result<(String, UserOverr /// The function `text` replaces, upper case, and the override named `name`, or why it is not one. fn override_definition(name: &str, text: &str) -> Result<(String, UserOverride), String> { let failed = |reason: &str| format!("override {name}: {reason}"); - let rest = text.trim_start(); - let rest = rest - .get(..OVERRIDE_HEADER.len()) - .filter(|head| head.eq_ignore_ascii_case(OVERRIDE_HEADER)) - .map(|_| rest[OVERRIDE_HEADER.len()..].trim_start()) - .ok_or_else(|| failed("does not start with FUNCTION ("))?; - let rest = rest - .strip_prefix('(') - .ok_or_else(|| failed("does not start with FUNCTION ("))?; - let close = closing(rest).ok_or_else(|| failed("its FUNCTION ( header is not closed"))?; - let header = &rest[..close]; - let body = rest[close + 1..] - .trim_start() - .strip_prefix(';') - .ok_or_else(|| failed("its header must end with );"))? - .trim(); - let body = body.trim_end_matches(';').trim_end(); - if body.is_empty() { - return Err(failed("has no body")); - } + let (properties, body) = function_header(text).map_err(|reason| failed(&reason))?; let (mut arguments, mut returns, mut replaces, mut dialect) = (None, None, None, None); - for property in top_level(header) - .iter() - .map(|part| part.trim()) - .filter(|part| !part.is_empty()) - { - let (key, value) = property - .split_once(|character: char| character.is_whitespace() || character == '(') - .map(|(key, _)| (key, property[key.len()..].trim())) - .ok_or_else(|| failed(&format!("the property `{property}` has no value")))?; - match key.to_ascii_lowercase().as_str() { + for (key, value) in properties { + match key.as_str() { OVERRIDE_ARGUMENTS => { - arguments = Some(parameters(value).map_err(|reason| failed(&reason))?) + arguments = Some(parameters(&value).map_err(|reason| failed(&reason))?) } OVERRIDE_RETURNS => { returns = Some( - cast_target(value, &ProvenanceLog::default()) + cast_target(&value, &ProvenanceLog::default()) .map_err(|reason| failed(&reason))? .data_type, ); @@ -128,75 +103,21 @@ fn override_definition(name: &str, text: &str) -> Result<(String, UserOverride), .iter() .map(|byte| format!("{byte:02x}")) .collect(), - body: body.to_owned(), + body, }, )) } /// `(name TYPE, ...)` as parameters. fn parameters(value: &str) -> Result, String> { - let inner = value - .strip_prefix('(') - .and_then(|inner| inner.strip_suffix(')')) - .ok_or_else(|| "arguments must be written `arguments (name TYPE, ...)`".to_owned())?; let mut parameters: Vec = Vec::new(); - for parameter in top_level(inner) - .iter() - .map(|part| part.trim()) - .filter(|part| !part.is_empty()) - { - let (name, type_name) = parameter - .split_once(char::is_whitespace) - .ok_or_else(|| format!("the argument `{parameter}` has no type"))?; - let identifier = name.chars().enumerate().all(|(index, character)| { - character == '_' - || character.is_ascii_alphabetic() - || (index > 0 && character.is_ascii_digit()) - }); - if !identifier { - return Err(format!( - "the argument name `{name}` is not a plain identifier" - )); - } - let target = cast_target(type_name.trim(), &ProvenanceLog::default())?; + for (name, type_name) in named_types(value)? { + let target = cast_target(&type_name, &ProvenanceLog::default())?; parameters.push(ColumnSpec::new(name.to_ascii_uppercase(), target.data_type)); } Ok(parameters) } -/// The byte offset of the parenthesis closing `text`'s open one, or None. -fn closing(text: &str) -> Option { - let mut depth: usize = 1; - for (offset, character) in text.char_indices() { - match character { - '(' => depth += 1, - ')' if depth == 1 => return Some(offset), - ')' => depth -= 1, - _ => {} - } - } - None -} - -/// `text` split at commas outside parentheses. -fn top_level(text: &str) -> Vec<&str> { - let mut parts: Vec<&str> = Vec::new(); - let (mut depth, mut start) = (0_usize, 0_usize); - for (offset, character) in text.char_indices() { - match character { - '(' => depth += 1, - ')' => depth = depth.saturating_sub(1), - ',' if depth == 0 => { - parts.push(&text[start..offset]); - start = offset + 1; - } - _ => {} - } - } - parts.push(&text[start..]); - parts -} - /// The upper-case name and lower-case hash of a `NAME=` verification, or why it is not one. pub(crate) fn verified_override(named: &str) -> Result<(String, String), String> { let (name, sha256) = named diff --git a/crates/sqlquack/src/cli/_helpers/census_command.rs b/crates/sqlquack/src/cli/_helpers/census_command.rs index 7a67e40c8..bed606463 100644 --- a/crates/sqlquack/src/cli/_helpers/census_command.rs +++ b/crates/sqlquack/src/cli/_helpers/census_command.rs @@ -13,7 +13,7 @@ use crate::census::main::census_with_progress::census_with_progress; use crate::census::main::render_census_text::render_census_text; use crate::census::models::CensusOptions; use crate::cli::_helpers::evidence::{chosen_policy, divergence_options}; -use crate::cli::_helpers::schema::load_schema; +use crate::cli::_helpers::project::project_choices; use crate::cli::models::CensusArguments; use sqlquack_rules::catalogue::constants::RULES; @@ -30,29 +30,35 @@ pub(crate) fn census_command( ); return ExitCode::from(2); }; - let schema = match arguments.schema.as_deref().map(load_schema).transpose() { - Ok(schema) => schema.unwrap_or_default(), + let policy = match chosen_policy( + &arguments.conversion.policy, + arguments.conversion.builtin_policy, + ) { + Ok(policy) => policy, Err(error) => { let _ = writeln!(stderr, "sqlquack census: failed: {error}"); return ExitCode::from(2); } }; - let conversion = match chosen_policy( - &arguments.conversion.policy, - arguments.conversion.builtin_policy, + let project = match project_choices( + policy.as_ref(), + &arguments.conversion, + ( + arguments.schema.as_deref(), + &[], + arguments.overrides.as_deref(), + ), ) { - Ok(policy) => divergence_options(policy.as_ref(), &arguments.conversion), + Ok(project) => project, Err(error) => { let _ = writeln!(stderr, "sqlquack census: failed: {error}"); return ExitCode::from(2); } }; + let schema = project.schema; let conversion = match with_overrides( - conversion, - ( - arguments.overrides.as_deref(), - &arguments.verified_overrides, - ), + divergence_options(policy.as_ref(), &project.conversion), + (project.overrides.as_deref(), &arguments.verified_overrides), ) { Ok(conversion) => conversion, Err(error) => { diff --git a/crates/sqlquack/src/cli/_helpers/evidence.rs b/crates/sqlquack/src/cli/_helpers/evidence.rs index 287788dc2..090f66edf 100644 --- a/crates/sqlquack/src/cli/_helpers/evidence.rs +++ b/crates/sqlquack/src/cli/_helpers/evidence.rs @@ -144,20 +144,28 @@ fn divergence_help( } let mut fixes: Vec = Vec::new(); let confirmation = match id { - SESSION_TIMEZONE_UTC => Some(("--utc-session", arguments.utc_session)), + SESSION_TIMEZONE_UTC => Some(("utc_session", "--utc-session", arguments.utc_session)), DECLARED_OFFSET => Some(( + "declared_offsets", "--confirm-declared-offsets", arguments.confirm_declared_offsets, )), - PROJECT_UDF => Some(("--python-runtime", arguments.python_runtime)), + PROJECT_UDF => Some(( + "python_runtime", + "--python-runtime", + arguments.python_runtime, + )), _ => None, }; let confirmable = match confirmation { - Some((flag, false)) => { - fixes.push(format!("confirm its assumption with {flag}")); + Some((key, flag, false)) => { + fixes.push(format!( + "confirm its assumption: add `{key} = true` under [project.confirm] in {file}, or \ + pass {flag} for one run" + )); true } - Some((_, true)) => true, + Some((_, _, true)) => true, None => false, }; let mode = match arguments.strict { diff --git a/crates/sqlquack/src/cli/_helpers/function_command.rs b/crates/sqlquack/src/cli/_helpers/function_command.rs index eac31d2e6..89a8b56c5 100644 --- a/crates/sqlquack/src/cli/_helpers/function_command.rs +++ b/crates/sqlquack/src/cli/_helpers/function_command.rs @@ -21,8 +21,8 @@ use crate::census::main::resolve_references::resolve_references; use crate::cli::_helpers::evidence::caller_choices; use crate::cli::_helpers::evidence::chosen_policy; use crate::cli::_helpers::evidence::with_divergence_help; -use crate::cli::_helpers::schema::load_schema; use crate::cli::models::FunctionArguments; +use crate::project::main::load_schema::load_schema; use sqlquack_rules::session::main::parse_session::parse_session; use sqlquack_rules::session::main::session_conversion_options::session_conversion_options; diff --git a/crates/sqlquack/src/cli/_helpers/index_command.rs b/crates/sqlquack/src/cli/_helpers/index_command.rs index 0ccedba70..f21843b87 100644 --- a/crates/sqlquack/src/cli/_helpers/index_command.rs +++ b/crates/sqlquack/src/cli/_helpers/index_command.rs @@ -13,13 +13,13 @@ use crate::analysis::models::PipelineConverter; use crate::census::main::inspect_corpus::inspect_corpus; use crate::census::models::CensusOptions; use crate::cli::_helpers::evidence::{chosen_policy, divergence_options}; -use crate::cli::_helpers::schema::load_schema; use crate::cli::models::IndexArguments; use crate::index::main::build_index::build_index; use crate::index::main::case_records::case_records; use crate::index::main::corpus_records::corpus_records; use crate::index::main::render_index_text::render_index_text; use crate::index::models::{ConversionRecord, ProvenanceIndex}; +use crate::project::main::load_schema::load_schema; /// The case directory indexed when no path is given. const DEFAULT_CASES: &str = "cases"; diff --git a/crates/sqlquack/src/cli/_helpers/mod.rs b/crates/sqlquack/src/cli/_helpers/mod.rs index 8dd66d17f..b774d6561 100644 --- a/crates/sqlquack/src/cli/_helpers/mod.rs +++ b/crates/sqlquack/src/cli/_helpers/mod.rs @@ -5,6 +5,6 @@ pub(crate) mod evidence; pub(crate) mod function_command; pub(crate) mod index_command; pub(crate) mod pending_command; +pub(crate) mod project; pub(crate) mod review_command; -pub(crate) mod schema; pub(crate) mod verify_command; diff --git a/crates/sqlquack/src/cli/_helpers/project.rs b/crates/sqlquack/src/cli/_helpers/project.rs new file mode 100644 index 000000000..56afcbde6 --- /dev/null +++ b/crates/sqlquack/src/cli/_helpers/project.rs @@ -0,0 +1,68 @@ +//! The project beside the policy file: its schema, session, confirmations and overrides, under the caller's flags. + +use std::collections::BTreeSet; +use std::path::Path; + +use sqlquack_policy::policy::models::Policy; +use sqlquack_semantics::types::models::Schema; + +use crate::cli::models::{PolicyArguments, ProjectChoices}; +use crate::project::constants::OVERRIDES_FOLDER; +use crate::project::main::load_schema::load_schema; +use crate::project::main::project_folder::project_folder; +use crate::project::main::project_schema::project_schema; + +/// The project choices for `policy`, read from `conversion.policy`, under the caller's flags. +pub(crate) fn project_choices( + policy: Option<&Policy>, + conversion: &PolicyArguments, + (schema, session, overrides): (Option<&Path>, &[String], Option<&Path>), +) -> Result { + let file = schema.map(schema_file).transpose()?; + let mut conversion = conversion.clone(); + let Some(policy) = policy else { + return Ok(ProjectChoices { + schema: file.map(|(_, schema)| schema).unwrap_or_default(), + session: session.to_vec(), + overrides: overrides.map(Path::to_path_buf), + conversion, + }); + }; + let root = conversion + .policy + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or(Path::new(".")) + .to_path_buf(); + let project = &policy.project; + conversion.python_runtime |= project.confirm.python_runtime; + conversion.utc_session |= project.confirm.utc_session; + conversion.confirm_declared_offsets |= project.confirm.declared_offsets; + let given: BTreeSet = session + .iter() + .filter_map(|setting| setting.split_once('=')) + .map(|(name, _)| name.trim().to_ascii_uppercase()) + .collect(); + let session: Vec = project + .session + .iter() + .filter(|(name, _)| !given.contains(&name.to_ascii_uppercase())) + .map(|(name, value)| format!("{name}={}", value.text())) + .chain(session.iter().cloned()) + .collect(); + let overrides = match overrides { + Some(path) => Some(path.to_path_buf()), + None => project_folder(&root, project.overrides.as_deref(), OVERRIDES_FOLDER)?, + }; + Ok(ProjectChoices { + schema: project_schema(&root, project, file)?, + session, + overrides, + conversion, + }) +} + +/// The schema file at `path`, kept with its path to name it in a clash. +fn schema_file(path: &Path) -> Result<(&Path, Schema), String> { + load_schema(path).map(|schema| (path, schema)) +} diff --git a/crates/sqlquack/src/cli/main/convert_report.rs b/crates/sqlquack/src/cli/main/convert_report.rs index 374271984..f9c8b1478 100644 --- a/crates/sqlquack/src/cli/main/convert_report.rs +++ b/crates/sqlquack/src/cli/main/convert_report.rs @@ -6,23 +6,33 @@ use sqlquack_rules::catalogue::main::builtin_catalogue::builtin_catalogue; use crate::analysis::main::analyze::analyze; use crate::cli::_helpers::convert_command::{analysis_report, chosen_options, refused_session}; use crate::cli::_helpers::evidence::chosen_policy; -use crate::cli::_helpers::schema::load_schema; +use crate::cli::_helpers::project::project_choices; use crate::cli::models::{ConvertArguments, ConvertReport}; use sqlquack_rules::session::main::parse_session::parse_session; use sqlquack_rules::session::main::session_conversion_options::session_conversion_options; /// The report `convert` prints for `sql` under `arguments`, or why it cannot be made (exit 2). pub fn convert_report(arguments: &ConvertArguments, sql: &str) -> Result { - let schema = arguments - .schema - .as_deref() - .map(load_schema) - .transpose()? - .unwrap_or_default(); let policy = chosen_policy( &arguments.conversion.policy, arguments.conversion.builtin_policy, )?; + let project = project_choices( + policy.as_ref(), + &arguments.conversion, + ( + arguments.schema.as_deref(), + &arguments.session, + arguments.overrides.as_deref(), + ), + )?; + let schema = project.schema; + let arguments = &ConvertArguments { + session: project.session, + overrides: project.overrides, + conversion: project.conversion, + ..arguments.clone() + }; let parameters = parse_session(&arguments.session)?; let session = ( arguments.conversion.strict, diff --git a/crates/sqlquack/src/cli/models.rs b/crates/sqlquack/src/cli/models.rs index c727b54f9..d7d068c32 100644 --- a/crates/sqlquack/src/cli/models.rs +++ b/crates/sqlquack/src/cli/models.rs @@ -13,6 +13,7 @@ use sqlquack_evidence::constants::{ use sqlquack_evidence::coverage::constants::{DOMAINS_DIR, HAZARDS_FILE}; use sqlquack_policy::policy::constants::POLICY_FILE_NAME; use sqlquack_policy::policy::models::DivergenceFailure; +use sqlquack_semantics::types::models::Schema; use crate::analysis::models::PythonSignature; use crate::census::constants::DEFAULT_TEXT_LIMIT; @@ -87,7 +88,7 @@ pub struct CheckArguments { } /// Arguments of `sqlquack convert`. -#[derive(Debug, Args)] +#[derive(Clone, Debug, Args)] pub struct ConvertArguments { /// The SQL file; standard input when absent or `-`. pub path: Option, @@ -268,7 +269,7 @@ pub struct IndexArguments { } /// Arguments of `sqlquack census`. -#[derive(Debug, Args)] +#[derive(Clone, Debug, Args)] pub struct CensusArguments { /// SQL files, or directories searched for `.sql` files: one query or compiled model per file. #[arg(required = true)] @@ -309,7 +310,7 @@ pub struct CensusArguments { } /// The policy conversion relies on, and what the caller confirms about its sessions. -#[derive(Debug, Args)] +#[derive(Clone, Debug, Args)] pub struct PolicyArguments { /// The policy file whose known divergences conversion may rely on. #[arg(long, default_value = POLICY_FILE_NAME)] @@ -345,7 +346,7 @@ pub enum OutputFormat { } /// The report format: `--format json`, or its short form `--json`. -#[derive(Debug, Args)] +#[derive(Clone, Debug, Args)] pub struct OutputArguments { /// Print the report as text or as JSON. #[arg(long, value_enum, default_value_t = OutputFormat::Text)] @@ -442,3 +443,16 @@ pub(crate) struct Comparison { /// Whether the rows compare in order. pub(crate) ordered: bool, } + +/// What a command reads from the project once the caller's flags are applied. +#[derive(Debug)] +pub struct ProjectChoices { + /// The schema of `--schema` merged with the project's sources and functions. + pub schema: Schema, + /// `[project.session]`, each parameter a `--session` flag gives replaced by the flag's value. + pub session: Vec, + /// `--overrides`, or the project's overrides folder. + pub overrides: Option, + /// The caller's flags with `[project.confirm]` added: a flag confirms on top of the file. + pub conversion: PolicyArguments, +} diff --git a/crates/sqlquack/src/lib.rs b/crates/sqlquack/src/lib.rs index 220b654f1..ce2f98828 100644 --- a/crates/sqlquack/src/lib.rs +++ b/crates/sqlquack/src/lib.rs @@ -7,5 +7,6 @@ pub mod census; pub mod cli; pub mod index; pub(crate) mod planning; +pub mod project; pub mod review; pub mod statements; diff --git a/crates/sqlquack/src/project/_helpers/declarations.rs b/crates/sqlquack/src/project/_helpers/declarations.rs new file mode 100644 index 000000000..945b28c4d --- /dev/null +++ b/crates/sqlquack/src/project/_helpers/declarations.rs @@ -0,0 +1,122 @@ +//! The tables and functions a project's folders declare, and the files declaring them. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use serde_json::{Map, Value}; +use sqlquack_policy::policy::models::ProjectPolicy; +use walkdir::WalkDir; + +use crate::project::_helpers::function_files::{python_function, sql_function}; +use crate::project::_helpers::source_files::source_relations; +use crate::project::constants::{ + FUNCTION_KIND, FUNCTIONS_FOLDER, PYTHON_EXTENSION, PYTHON_FUNCTIONS_FOLDER, SOURCE_EXTENSIONS, + SOURCES_FOLDER, SQL_EXTENSION, SQL_FUNCTIONS_FOLDER, TABLE_KIND, +}; +use crate::project::main::project_folder::project_folder; +use crate::project::models::Declaration; + +/// Declarations as the schema file writes them, by name, and where each came from. +pub(crate) type Declared = (Map, Vec); + +/// The tables of the project's source files. +pub(crate) fn source_tables(root: &Path, project: &ProjectPolicy) -> Result { + let (mut tables, mut declarations) = (Map::new(), Vec::new()); + let Some(folder) = project_folder(root, project.sources.as_deref(), SOURCES_FOLDER)? else { + return Ok((tables, declarations)); + }; + for path in files(&folder, SOURCE_EXTENSIONS) { + for (name, columns) in source_relations(&path)? { + declarations.push(declaration(TABLE_KIND, &name, &path)); + tables.insert(name, columns); + } + } + Ok((tables, declarations)) +} + +/// The functions of the project's SQL and Python function files, named after their files. +pub(crate) fn project_functions(root: &Path, project: &ProjectPolicy) -> Result { + let Some(folder) = project_folder(root, project.functions.as_deref(), FUNCTIONS_FOLDER)? else { + return Ok((Map::new(), Vec::new())); + }; + let (mut functions, mut declarations) = function_files( + &folder.join(SQL_FUNCTIONS_FOLDER), + SQL_EXTENSION, + sql_function, + )?; + let (python, python_declarations) = function_files( + &folder.join(PYTHON_FUNCTIONS_FOLDER), + PYTHON_EXTENSION, + python_function, + )?; + functions.extend(python); + declarations.extend(python_declarations); + Ok((functions, declarations)) +} + +/// The functions of the `extension` files in `folder`, each read by `read`. +fn function_files( + folder: &Path, + extension: &str, + read: fn(&str) -> Result, +) -> Result { + let (mut functions, mut declarations) = (Map::new(), Vec::new()); + for path in files(folder, &[extension]) { + let name = path + .file_stem() + .and_then(|stem| stem.to_str()) + .unwrap_or_default() + .to_ascii_uppercase(); + let text = std::fs::read_to_string(&path) + .map_err(|error| format!("cannot read {}: {error}", path.display()))?; + let declared = read(&text).map_err(|reason| format!("{}: {reason}", path.display()))?; + declarations.push(declaration(FUNCTION_KIND, &name, &path)); + functions.insert(name, declared); + } + Ok((functions, declarations)) +} + +/// `name`, of `kind`, declared in `path`. +pub(crate) fn declaration(kind: &'static str, name: &str, path: &Path) -> Declaration { + Declaration { + kind, + name: name.to_owned(), + path: path.to_path_buf(), + } +} + +/// An error naming both files when two declare one kind and name, ignoring case. +pub(crate) fn no_clash(declarations: &[Declaration]) -> Result<(), String> { + let mut seen: BTreeMap<(&str, String), &Declaration> = BTreeMap::new(); + for declared in declarations { + let key = (declared.kind, declared.name.to_ascii_uppercase()); + if let Some(earlier) = seen.insert(key, declared) { + return Err(format!( + "the {} {} is declared twice, in {} and in {}; keep one", + declared.kind, + declared.name, + earlier.path.display(), + declared.path.display() + )); + } + } + Ok(()) +} + +/// The files under `folder` with one of `extensions`, in path order. +fn files(folder: &Path, extensions: &[&str]) -> Vec { + let mut files: Vec = WalkDir::new(folder) + .into_iter() + .filter_map(Result::ok) + .map(walkdir::DirEntry::into_path) + .filter(|file| { + file.is_file() + && file + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extensions.contains(&extension)) + }) + .collect(); + files.sort(); + files +} diff --git a/crates/sqlquack/src/project/_helpers/function_files.rs b/crates/sqlquack/src/project/_helpers/function_files.rs new file mode 100644 index 000000000..d9e016548 --- /dev/null +++ b/crates/sqlquack/src/project/_helpers/function_files.rs @@ -0,0 +1,121 @@ +//! Function signatures from SQLBuild's function files: a SQL `FUNCTION (...)` header or a Python `@udf(...)`. + +use serde_json::{Map, Value, json}; + +use crate::analysis::constants::{OVERRIDE_ARGUMENTS, OVERRIDE_RETURNS}; +use crate::project::_helpers::python_literal::python_keywords; +use crate::project::constants::{ + SQL_FUNCTION_KEYS, TABLE_RETURNS, UDF_DECORATOR, UDF_IMPORTS, UDF_KEYS, +}; +use crate::project::main::function_header::function_header; +use crate::project::main::named_types::named_types; + +/// The schema declaration of the SQL function file `text`. +pub(crate) fn sql_function(text: &str) -> Result { + let (properties, _body) = function_header(text)?; + let mut function: Map = Map::new(); + function.insert("language".to_owned(), json!("sql")); + for (key, value) in properties { + match key.as_str() { + OVERRIDE_ARGUMENTS => { + function.insert(key, pairs(named_types(&value)?)); + } + OVERRIDE_RETURNS => { + let returns = match value.get(..TABLE_RETURNS.len()) { + Some(head) if head.eq_ignore_ascii_case(TABLE_RETURNS) => { + let columns = value[TABLE_RETURNS.len()..].trim(); + json!({ "table": pairs(named_types(columns)?) }) + } + _ => json!(value), + }; + function.insert(key, returns); + } + known if SQL_FUNCTION_KEYS.contains(&known) => {} + other => return Err(format!("the property `{other}` is not understood")), + } + } + if !function.contains_key(OVERRIDE_RETURNS) { + return Err("needs `returns `".to_owned()); + } + Ok(Value::Object(function)) +} + +/// The schema declaration of the Python function file `text`; errors name the decorator's line. +pub(crate) fn python_function(text: &str) -> Result { + let imported = text.lines().any(|line| { + let words: Vec<&str> = line.split_whitespace().collect(); + matches!(words.as_slice(), ["from", module, "import", names @ ..] + if UDF_IMPORTS.contains(module) + && names.iter().any(|name| name.trim_end_matches(',') == "udf")) + }); + let Some((line, start)) = text + .lines() + .scan(0_usize, |offset, line| { + let start = *offset; + *offset += line.len() + 1; + Some((line, start)) + }) + .enumerate() + .find(|(_, (line, _))| line.trim_start().starts_with(UDF_DECORATOR)) + .map(|(index, (line, start))| (index + 1, start + line.find('(').unwrap_or_default())) + else { + return Err(format!( + "has no {UDF_DECORATOR}...) decorator; mark the function as SQLBuild does" + )); + }; + if !imported { + return Err(format!( + "line {line}: import udf from {} first", + UDF_IMPORTS.join(" or ") + )); + } + let keywords = + python_keywords(&text[start..]).map_err(|reason| format!("line {line}: {reason}"))?; + let mut function: Map = Map::new(); + function.insert("language".to_owned(), json!("python")); + for (key, value) in keywords { + match (key.as_str(), value) { + (OVERRIDE_ARGUMENTS, Value::Object(arguments)) => { + let mut named: Vec<(String, String)> = Vec::new(); + for (name, type_name) in arguments { + let Value::String(type_name) = type_name else { + return Err(format!( + "line {line}: the argument {name}'s type is not text" + )); + }; + named.push((name, type_name)); + } + function.insert(key, pairs(named)); + } + (OVERRIDE_RETURNS, Value::String(type_name)) => { + function.insert(key, json!(type_name)); + } + (OVERRIDE_ARGUMENTS | OVERRIDE_RETURNS, _) => { + return Err(format!( + "line {line}: `{key}` must be a dict of names to types (arguments) or a type \ + (returns)" + )); + } + (known, _) if UDF_KEYS.contains(&known) => {} + (other, _) => { + return Err(format!( + "line {line}: the keyword `{other}` is not understood" + )); + } + } + } + if !function.contains_key(OVERRIDE_RETURNS) { + return Err(format!("line {line}: needs returns=\"\"")); + } + Ok(Value::Object(function)) +} + +/// Name and type pairs as the schema file writes columns. +fn pairs(named: Vec<(String, String)>) -> Value { + Value::Array( + named + .into_iter() + .map(|(name, type_name)| json!([name, type_name])) + .collect(), + ) +} diff --git a/crates/sqlquack/src/project/_helpers/header_parts.rs b/crates/sqlquack/src/project/_helpers/header_parts.rs new file mode 100644 index 000000000..36fd8eef1 --- /dev/null +++ b/crates/sqlquack/src/project/_helpers/header_parts.rs @@ -0,0 +1,84 @@ +//! Splitting SQLBuild's `FUNCTION ( ... );` header from its body, and into its properties. + +use crate::analysis::constants::OVERRIDE_HEADER; + +/// The text inside `FUNCTION ( ... );` and the body after it, or why `text` does not start with one. +pub(crate) fn header_and_body(text: &str) -> Result<(&str, String), String> { + let rest = text.trim_start(); + let rest = rest + .get(..OVERRIDE_HEADER.len()) + .filter(|head| head.eq_ignore_ascii_case(OVERRIDE_HEADER)) + .map(|_| rest[OVERRIDE_HEADER.len()..].trim_start()) + .and_then(|rest| rest.strip_prefix('(')) + .ok_or_else(|| "does not start with FUNCTION (".to_owned())?; + let close = closing(rest).ok_or_else(|| "its FUNCTION ( header is not closed".to_owned())?; + let body = rest[close + 1..] + .trim_start() + .strip_prefix(';') + .ok_or_else(|| "its header must end with );".to_owned())? + .trim() + .trim_end_matches(';') + .trim_end(); + if body.is_empty() { + return Err("has no body".to_owned()); + } + Ok((&rest[..close], body.to_owned())) +} + +/// The header's `key value` properties, keys lower-cased. +pub(crate) fn properties(header: &str) -> Result, String> { + let mut properties: Vec<(String, String)> = Vec::new(); + for property in top_level(header) + .iter() + .map(|part| part.trim()) + .filter(|part| !part.is_empty()) + { + let (key, value) = property + .split_once(|character: char| character.is_whitespace() || character == '(') + .map(|(key, _)| (key, property[key.len()..].trim())) + .ok_or_else(|| format!("the property `{property}` has no value"))?; + properties.push((key.to_ascii_lowercase(), value.to_owned())); + } + Ok(properties) +} + +/// The byte offset of the parenthesis closing `text`'s open one, outside quotes, or None. +fn closing(text: &str) -> Option { + let mut depth: usize = 1; + let mut quote: Option = None; + for (offset, character) in text.char_indices() { + match (quote, character) { + (Some(open), _) if character == open => quote = None, + (Some(_), _) => {} + (None, '\'' | '"') => quote = Some(character), + (None, '(') => depth += 1, + (None, ')') if depth == 1 => return Some(offset), + (None, ')') => depth -= 1, + (None, _) => {} + } + } + None +} + +/// `text` split at commas outside parentheses and quotes. +pub(crate) fn top_level(text: &str) -> Vec<&str> { + let mut parts: Vec<&str> = Vec::new(); + let (mut depth, mut start) = (0_usize, 0_usize); + let mut quote: Option = None; + for (offset, character) in text.char_indices() { + match (quote, character) { + (Some(open), _) if character == open => quote = None, + (Some(_), _) => {} + (None, '\'' | '"') => quote = Some(character), + (None, '(') => depth += 1, + (None, ')') => depth = depth.saturating_sub(1), + (None, ',') if depth == 0 => { + parts.push(&text[start..offset]); + start = offset + 1; + } + (None, _) => {} + } + } + parts.push(&text[start..]); + parts +} diff --git a/crates/sqlquack/src/project/_helpers/mod.rs b/crates/sqlquack/src/project/_helpers/mod.rs new file mode 100644 index 000000000..956fdd19d --- /dev/null +++ b/crates/sqlquack/src/project/_helpers/mod.rs @@ -0,0 +1,8 @@ +//! Readers of a project's files. + +pub(crate) mod declarations; +pub(crate) mod function_files; +pub(crate) mod header_parts; +pub(crate) mod python_literal; +pub(crate) mod source_files; +pub(crate) mod yaml_value; diff --git a/crates/sqlquack/src/project/_helpers/python_literal.rs b/crates/sqlquack/src/project/_helpers/python_literal.rs new file mode 100644 index 000000000..d0a275931 --- /dev/null +++ b/crates/sqlquack/src/project/_helpers/python_literal.rs @@ -0,0 +1,164 @@ +//! A Python call's keyword arguments as JSON: the strings, numbers, booleans, None, lists and dicts a `@udf(...)` holds. + +use serde_json::{Map, Value}; + +use crate::project::_helpers::yaml_value::number; + +/// The call arguments `(keyword=literal, ...)` at the start of `text`, as an object. +pub(crate) fn python_keywords(text: &str) -> Result, String> { + Reader { + characters: text.chars().collect(), + index: 0, + } + .keywords() +} + +struct Reader { + characters: Vec, + index: usize, +} + +impl Reader { + /// The next character after spaces, line breaks and comments, without taking it. + fn peek(&mut self) -> Option { + loop { + match self.characters.get(self.index) { + Some(character) if character.is_whitespace() => self.index += 1, + Some('#') => { + while !matches!(self.characters.get(self.index), None | Some('\n')) { + self.index += 1; + } + } + other => return other.copied(), + } + } + } + + fn take(&mut self, expected: char) -> Result<(), String> { + match self.peek() { + Some(found) if found == expected => { + self.index += 1; + Ok(()) + } + Some(found) => Err(format!("expected `{expected}`, found `{found}`")), + None => Err(format!("expected `{expected}`, found the end")), + } + } + + /// `(name=value, ...)`. + fn keywords(&mut self) -> Result, String> { + self.take('(')?; + let mut keywords: Map = Map::new(); + while self.peek() != Some(')') { + let name = self.word(); + if name.is_empty() { + return Err("expected a keyword argument".to_owned()); + } + self.take('=')?; + let value = self.value()?; + keywords.insert(name, value); + if self.peek() != Some(')') { + self.take(',')?; + } + } + self.index += 1; + Ok(keywords) + } + + fn value(&mut self) -> Result { + match self.peek() { + Some(quote @ ('\'' | '"')) => self.string(quote).map(Value::String), + Some(open @ ('[' | '(')) => { + self.index += 1; + let close = if open == '[' { ']' } else { ')' }; + let mut items: Vec = Vec::new(); + while self.peek() != Some(close) { + items.push(self.value()?); + if self.peek() != Some(close) { + self.take(',')?; + } + } + self.index += 1; + Ok(Value::Array(items)) + } + Some('{') => { + self.index += 1; + let mut entries: Map = Map::new(); + while self.peek() != Some('}') { + let key = match self.value()? { + Value::String(key) => key, + other => return Err(format!("a dict key is not text: {other}")), + }; + self.take(':')?; + entries.insert(key, self.value()?); + if self.peek() != Some('}') { + self.take(',')?; + } + } + self.index += 1; + Ok(Value::Object(entries)) + } + Some(_) => match self.word().as_str() { + "True" => Ok(Value::Bool(true)), + "False" => Ok(Value::Bool(false)), + "None" => Ok(Value::Null), + word => number(word) + .map(Value::Number) + .ok_or_else(|| format!("`{word}` is not a literal")), + }, + None => Err("expected a value, found the end".to_owned()), + } + } + + /// A name or number: letters, digits, `_`, `.`, `+` and `-`. + fn word(&mut self) -> String { + self.peek(); + let start = self.index; + while self.characters.get(self.index).is_some_and(|character| { + character.is_alphanumeric() || matches!(character, '_' | '.' | '+' | '-') + }) { + self.index += 1; + } + self.characters[start..self.index].iter().collect() + } + + /// A quoted string, single or triple, with backslash escapes; adjacent strings join. + fn string(&mut self, quote: char) -> Result { + let triple = self.characters[self.index..].starts_with(&[quote, quote, quote]); + let width = if triple { 3 } else { 1 }; + self.index += width; + let mut text = String::new(); + loop { + match self.characters.get(self.index).copied() { + None => return Err("a string is not closed".to_owned()), + Some('\\') => { + let escaped = self.characters.get(self.index + 1).copied(); + text.push(match escaped { + Some('n') => '\n', + Some('t') => '\t', + Some(other) => other, + None => return Err("a string is not closed".to_owned()), + }); + self.index += 2; + } + Some(character) + if character == quote + && (!triple + || self.characters[self.index..] + .starts_with(&[quote, quote, quote])) => + { + self.index += width; + break; + } + Some(character) => { + text.push(character); + self.index += 1; + } + } + } + match self.peek() { + Some(next @ ('\'' | '"')) => Ok(text + &self.string(next)?), + _ => Ok(text), + } + } +} diff --git a/crates/sqlquack/src/project/_helpers/source_files.rs b/crates/sqlquack/src/project/_helpers/source_files.rs new file mode 100644 index 000000000..824c1306f --- /dev/null +++ b/crates/sqlquack/src/project/_helpers/source_files.rs @@ -0,0 +1,64 @@ +//! Tables declared in SQLBuild's source YAML files: a name and columns of a name and a type each. + +use std::path::Path; + +use serde::Deserialize; +use serde_json::{Value, json}; + +use crate::project::_helpers::yaml_value::yaml_value; + +/// One source file; keys SQLQuack does not read, such as contracts and audits, are ignored. +#[derive(Deserialize)] +struct SourceFile { + #[serde(default)] + sources: Vec, +} + +#[derive(Deserialize)] +struct SourceEntry { + name: String, + #[serde(default)] + columns: Vec, + /// An inline query rather than a table; its columns may be partial, so it is not declared. + #[serde(default)] + expression: Option, +} + +#[derive(Deserialize)] +struct SourceColumn { + name: String, + #[serde(rename = "type", default)] + type_name: Option, +} + +/// Each table `path` declares, upper-cased as Snowflake folds SQLBuild's unquoted names. +pub(crate) fn source_relations(path: &Path) -> Result, String> { + let failed = |reason: String| format!("{}: {reason}", path.display()); + let text = std::fs::read_to_string(path).map_err(|error| failed(error.to_string()))?; + let value = yaml_value(&text).map_err(failed)?; + if value.is_null() { + return Ok(Vec::new()); + } + let file: SourceFile = serde_json::from_value(value) + .map_err(|error| failed(format!("is not a SQLBuild source file: {error}")))?; + Ok(file.sources.into_iter().filter_map(table).collect()) +} + +/// A source as a table and its `[column, type]` pairs; None for an inline query or an untyped column. +fn table(source: SourceEntry) -> Option<(String, Value)> { + if source.expression.is_some() || source.columns.is_empty() { + return None; + } + let columns: Option> = source.columns.into_iter().map(typed_column).collect(); + Some((source.name.to_ascii_uppercase(), Value::Array(columns?))) +} + +/// A column as an upper-case name and its type text, or None without a text type. +fn typed_column(column: SourceColumn) -> Option { + match column.type_name { + Some(Value::String(type_name)) => { + Some(json!([column.name.to_ascii_uppercase(), type_name])) + } + _ => None, + } +} diff --git a/crates/sqlquack/src/project/_helpers/yaml_value.rs b/crates/sqlquack/src/project/_helpers/yaml_value.rs new file mode 100644 index 000000000..8241baa51 --- /dev/null +++ b/crates/sqlquack/src/project/_helpers/yaml_value.rs @@ -0,0 +1,109 @@ +//! A YAML document as a JSON value, for reading SQLBuild's source files. + +use std::collections::HashMap; + +use saphyr_parser::{Event, Parser, ScalarStyle, StrInput}; +use serde_json::{Map, Number, Value}; + +/// The first document of `text` as a JSON value: plain scalars typed as YAML 1.2's core schema does. +pub(crate) fn yaml_value(text: &str) -> Result { + let mut reader = Reader { + parser: Parser::new_from_str(text), + anchors: HashMap::new(), + }; + loop { + match reader.next()? { + Event::StreamStart | Event::DocumentStart(_) | Event::Nothing => {} + Event::StreamEnd => return Ok(Value::Null), + event => return reader.node(event), + } + } +} + +/// The parser and the anchored values seen so far. +struct Reader<'input> { + parser: Parser<'input, StrInput<'input>>, + anchors: HashMap, +} + +impl<'input> Reader<'input> { + fn next(&mut self) -> Result, String> { + match self.parser.next_event() { + Some(Ok((event, _))) => Ok(event), + Some(Err(error)) => Err(format!( + "line {} column {}: {}", + error.marker().line(), + error.marker().col() + 1, + error.info() + )), + None => Err("the YAML ends unexpectedly".to_owned()), + } + } + + /// The value `event` starts. + fn node(&mut self, event: Event<'input>) -> Result { + let (value, anchor) = match event { + Event::Alias(anchor) => { + return self + .anchors + .get(&anchor) + .cloned() + .ok_or_else(|| "an alias names no anchor".to_owned()); + } + Event::Scalar(text, style, anchor, _) => (scalar(&text, style), anchor), + Event::SequenceStart(anchor, _) => { + let mut items: Vec = Vec::new(); + loop { + match self.next()? { + Event::SequenceEnd => break, + event => items.push(self.node(event)?), + } + } + (Value::Array(items), anchor) + } + Event::MappingStart(anchor, _) => { + let mut entries: Map = Map::new(); + loop { + let key = match self.next()? { + Event::MappingEnd => break, + event => self.node(event)?, + }; + let key = match key { + Value::String(text) => text, + other => other.to_string(), + }; + let event = self.next()?; + entries.insert(key, self.node(event)?); + } + (Value::Object(entries), anchor) + } + _ => return Err("the YAML holds an unexpected event".to_owned()), + }; + if anchor != 0 { + self.anchors.insert(anchor, value.clone()); + } + Ok(value) + } +} + +/// A scalar: quoted and block text as written, plain text typed as null, a boolean or a number. +fn scalar(text: &str, style: ScalarStyle) -> Value { + if style != ScalarStyle::Plain { + return Value::String(text.to_owned()); + } + match text { + "" | "~" | "null" | "Null" | "NULL" => Value::Null, + "true" | "True" | "TRUE" => Value::Bool(true), + "false" | "False" | "FALSE" => Value::Bool(false), + _ => number(text).map_or_else(|| Value::String(text.to_owned()), Value::Number), + } +} + +/// `text` as an integer or a finite float, or None when it is neither. +pub(crate) fn number(text: &str) -> Option { + match (text.parse::(), text.parse::()) { + (Ok(integer), _) => Some(Number::from(integer)), + (Err(_not_integer), Ok(float)) => Number::from_f64(float), + (Err(_not_integer), Err(_not_float)) => None, + } +} diff --git a/crates/sqlquack/src/project/constants.rs b/crates/sqlquack/src/project/constants.rs new file mode 100644 index 000000000..287137d7e --- /dev/null +++ b/crates/sqlquack/src/project/constants.rs @@ -0,0 +1,42 @@ +//! Project folder names, file kinds and the SQLBuild header and decorator keys SQLQuack reads. + +/// The project folder of source YAML files, beside the policy file. +pub const SOURCES_FOLDER: &str = "sources"; +/// The project folder of function files, beside the policy file. +pub const FUNCTIONS_FOLDER: &str = "functions"; +/// The project folder of user overrides, beside the policy file. +pub const OVERRIDES_FOLDER: &str = "overrides"; +/// The functions subfolder of SQL function files. +pub const SQL_FUNCTIONS_FOLDER: &str = "sql"; +/// The functions subfolder of Python function files. +pub const PYTHON_FUNCTIONS_FOLDER: &str = "python"; +/// The extensions of source files. +pub const SOURCE_EXTENSIONS: &[&str] = &["yml", "yaml"]; +/// The extension of Python function files. +pub const PYTHON_EXTENSION: &str = "py"; +/// SQL function header properties SQLBuild reads that a signature does not need. +pub const SQL_FUNCTION_KEYS: &[&str] = &["description", "tags", "database", "schema"]; +/// The word that starts a table function's `returns table (...)`. +pub const TABLE_RETURNS: &str = "table"; +/// How a Python function file's decorator starts. +pub const UDF_DECORATOR: &str = "@udf("; +/// The modules a Python function file may import `udf` from: SQLBuild's and SQLQuack's. +pub const UDF_IMPORTS: &[&str] = &["sqlbuild.functions", "sqlbuild", "sqlquack"]; +/// The keywords SQLBuild's `@udf(...)` accepts. +pub const UDF_KEYS: &[&str] = &[ + "description", + "arguments", + "returns", + "runtime_version", + "entry_point", + "packages", + "database", + "schema", + "tags", +]; +/// The extension of SQL function files. +pub const SQL_EXTENSION: &str = "sql"; +/// How a clash names a declared table. +pub const TABLE_KIND: &str = "table"; +/// How a clash names a declared function. +pub const FUNCTION_KIND: &str = "function"; diff --git a/crates/sqlquack/src/project/main/function_header.rs b/crates/sqlquack/src/project/main/function_header.rs new file mode 100644 index 000000000..cccf5ff9e --- /dev/null +++ b/crates/sqlquack/src/project/main/function_header.rs @@ -0,0 +1,9 @@ +//! `function_header`: SQLBuild's `FUNCTION ( key value, ... );` header split into properties and the body. + +use crate::project::_helpers::header_parts::{header_and_body, properties}; + +/// The header's properties as lower-case keys and their values, and the body, or why it is not one. +pub fn function_header(text: &str) -> Result<(Vec<(String, String)>, String), String> { + let (header, body) = header_and_body(text)?; + Ok((properties(header)?, body)) +} diff --git a/crates/sqlquack/src/cli/_helpers/schema.rs b/crates/sqlquack/src/project/main/load_schema.rs similarity index 80% rename from crates/sqlquack/src/cli/_helpers/schema.rs rename to crates/sqlquack/src/project/main/load_schema.rs index 7f733c98e..ecabaa9a8 100644 --- a/crates/sqlquack/src/cli/_helpers/schema.rs +++ b/crates/sqlquack/src/project/main/load_schema.rs @@ -1,4 +1,4 @@ -//! Reading a schema file: typed relations, and the signatures of project functions. +//! `load_schema`: a schema file's typed relations and project function signatures. use std::path::Path; @@ -7,7 +7,7 @@ use sqlquack_semantics::types::main::parse_schema::parse_schema; use sqlquack_semantics::types::models::Schema; /// The schema in `path`; a relation or function using a type SQLQuack does not know is left out. -pub(crate) fn load_schema(path: &Path) -> Result { +pub fn load_schema(path: &Path) -> Result { let text = std::fs::read_to_string(path) .map_err(|error| format!("cannot read {}: {error}", path.display()))?; parse_schema(&text).map_err(|error| match error { diff --git a/crates/sqlquack/src/project/main/mod.rs b/crates/sqlquack/src/project/main/mod.rs new file mode 100644 index 000000000..8ef65d30f --- /dev/null +++ b/crates/sqlquack/src/project/main/mod.rs @@ -0,0 +1,5 @@ +pub mod function_header; +pub mod load_schema; +pub mod named_types; +pub mod project_folder; +pub mod project_schema; diff --git a/crates/sqlquack/src/project/main/named_types.rs b/crates/sqlquack/src/project/main/named_types.rs new file mode 100644 index 000000000..c5cd86baf --- /dev/null +++ b/crates/sqlquack/src/project/main/named_types.rs @@ -0,0 +1,33 @@ +//! `named_types`: a header's `(name TYPE, ...)` list as name and type text pairs. + +use crate::project::_helpers::header_parts::top_level; + +/// `(name TYPE, ...)` as name and type text pairs, the name a plain identifier. +pub fn named_types(value: &str) -> Result, String> { + let inner = value + .strip_prefix('(') + .and_then(|inner| inner.strip_suffix(')')) + .ok_or_else(|| "arguments must be written `arguments (name TYPE, ...)`".to_owned())?; + let mut named: Vec<(String, String)> = Vec::new(); + for part in top_level(inner) + .iter() + .map(|part| part.trim()) + .filter(|part| !part.is_empty()) + { + let (name, type_name) = part + .split_once(char::is_whitespace) + .ok_or_else(|| format!("the argument `{part}` has no type"))?; + let identifier = name.chars().enumerate().all(|(index, character)| { + character == '_' + || character.is_ascii_alphabetic() + || (index > 0 && character.is_ascii_digit()) + }); + if !identifier { + return Err(format!( + "the argument name `{name}` is not a plain identifier" + )); + } + named.push((name.to_owned(), type_name.trim().to_owned())); + } + Ok(named) +} diff --git a/crates/sqlquack/src/project/main/project_folder.rs b/crates/sqlquack/src/project/main/project_folder.rs new file mode 100644 index 000000000..58e422867 --- /dev/null +++ b/crates/sqlquack/src/project/main/project_folder.rs @@ -0,0 +1,20 @@ +//! `project_folder`: a project folder beside the policy file, when it exists. + +use std::path::{Path, PathBuf}; + +/// `configured` or `default` under `root`; None when the default is absent, an error when the configured one is. +pub fn project_folder( + root: &Path, + configured: Option<&str>, + default: &str, +) -> Result, String> { + let folder = root.join(configured.unwrap_or(default)); + match (folder.is_dir(), configured) { + (true, _) => Ok(Some(folder)), + (false, None) => Ok(None), + (false, Some(_)) => Err(format!( + "[project] names the folder {}, which does not exist", + folder.display() + )), + } +} diff --git a/crates/sqlquack/src/project/main/project_schema.rs b/crates/sqlquack/src/project/main/project_schema.rs new file mode 100644 index 000000000..ea21d4d51 --- /dev/null +++ b/crates/sqlquack/src/project/main/project_schema.rs @@ -0,0 +1,47 @@ +//! `project_schema`: the tables and functions a project's folders declare, with a schema file's. + +use std::path::Path; + +use serde_json::json; +use sqlquack_policy::policy::models::ProjectPolicy; +use sqlquack_semantics::types::errors::SchemaError; +use sqlquack_semantics::types::main::parse_schema::parse_schema; +use sqlquack_semantics::types::models::Schema; + +use crate::project::_helpers::declarations::{ + declaration, no_clash, project_functions, source_tables, +}; +use crate::project::constants::{FUNCTION_KIND, TABLE_KIND}; + +/// The schema of the project at `root` merged with `file`'s; one name declared twice is an error. +pub fn project_schema( + root: &Path, + project: &ProjectPolicy, + file: Option<(&Path, Schema)>, +) -> Result { + let (tables, mut declarations) = source_tables(root, project)?; + let (functions, function_declarations) = project_functions(root, project)?; + declarations.extend(function_declarations); + let declared = json!({ "relations": tables, "functions": functions }).to_string(); + let mut schema = parse_schema(&declared).map_err(|error| match error { + SchemaError::Shape(reason) | SchemaError::Declaration(reason) => { + format!("the project at {}: {reason}", root.display()) + } + })?; + if let Some((path, file)) = file { + declarations.extend( + file.relations + .keys() + .map(|name| declaration(TABLE_KIND, name, path)), + ); + declarations.extend( + file.functions + .keys() + .map(|name| declaration(FUNCTION_KIND, name, path)), + ); + schema.relations.extend(file.relations); + schema.functions.extend(file.functions); + } + no_clash(&declarations)?; + Ok(schema) +} diff --git a/crates/sqlquack/src/project/mod.rs b/crates/sqlquack/src/project/mod.rs new file mode 100644 index 000000000..38e08dd79 --- /dev/null +++ b/crates/sqlquack/src/project/mod.rs @@ -0,0 +1,6 @@ +//! A project folder laid out as SQLBuild lays one out: sources, functions and overrides beside the policy file. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlquack/src/project/models.rs b/crates/sqlquack/src/project/models.rs new file mode 100644 index 000000000..cc72d4d86 --- /dev/null +++ b/crates/sqlquack/src/project/models.rs @@ -0,0 +1,14 @@ +//! What reading a project's files keeps track of. + +use std::path::PathBuf; + +/// One table or function a file declares, kept to name both files of a clash. +#[derive(Debug)] +pub struct Declaration { + /// `table` or `function`. + pub kind: &'static str, + /// The name as declared. + pub name: String, + /// The file declaring it. + pub path: PathBuf, +} diff --git a/crates/sqlquack/tests/cli.rs b/crates/sqlquack/tests/cli.rs index 4d05fd8e7..e346b6cf7 100644 --- a/crates/sqlquack/tests/cli.rs +++ b/crates/sqlquack/tests/cli.rs @@ -18,6 +18,8 @@ mod test_evidence_commands; mod test_explain; #[path = "cli/test_function.rs"] mod test_function; +#[path = "cli/test_project.rs"] +mod test_project; #[path = "cli/test_review.rs"] mod test_review; #[path = "cli/test_types.rs"] diff --git a/crates/sqlquack/tests/cli/helpers.rs b/crates/sqlquack/tests/cli/helpers.rs index 56dbcb2a7..ccbafd0c5 100644 --- a/crates/sqlquack/tests/cli/helpers.rs +++ b/crates/sqlquack/tests/cli/helpers.rs @@ -9,8 +9,8 @@ use walkdir::WalkDir; use crate::test_types::{ CommandOutcome, CompareTestCase, ConvertOutcome, CoverageOutcome, DivergenceOutcome, - ErrorCheckOutcome, FunctionMacroOutcome, FunctionStdinOutcome, ReviewOutcome, - ReviewedRepository, + ErrorCheckOutcome, FunctionMacroOutcome, FunctionStdinOutcome, ProjectOutcome, ProjectTestCase, + ReviewOutcome, ReviewedRepository, }; fn repository_root() -> PathBuf { @@ -1094,3 +1094,34 @@ pub(crate) fn schema_failure(text: &str) -> (Option, String) { String::from_utf8_lossy(&output.stderr).into_owned(), ) } + +/// `sqlquack convert --json` run inside a project folder built from `test_case`. +pub(crate) fn project_convert(test_case: &ProjectTestCase) -> ProjectOutcome { + let directory = tempfile::tempdir().expect("temporary directory"); + let policy = + std::fs::read_to_string(repository_root().join("sqlquack.toml")).expect("policy read"); + std::fs::write( + directory.path().join("sqlquack.toml"), + format!("{policy}\n{}", test_case.project), + ) + .expect("policy written"); + for file in test_case.files { + let path = directory.path().join(file.path); + std::fs::create_dir_all(path.parent().expect("a parent directory")) + .expect("directory created"); + std::fs::write(path, file.text).expect("file written"); + } + std::fs::write(directory.path().join("query.sql"), test_case.sql).expect("query written"); + let output = Command::new(env!("CARGO_BIN_EXE_sqlquack")) + .args(["convert", "--json"]) + .args(test_case.flags) + .arg("query.sql") + .current_dir(directory.path()) + .output() + .expect("sqlquack runs"); + ProjectOutcome { + exit_code: output.status.code(), + stdout: String::from_utf8_lossy(&output.stdout).into_owned(), + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + } +} diff --git a/crates/sqlquack/tests/cli/test_project.rs b/crates/sqlquack/tests/cli/test_project.rs new file mode 100644 index 000000000..6c3ea5657 --- /dev/null +++ b/crates/sqlquack/tests/cli/test_project.rs @@ -0,0 +1,241 @@ +use crate::helpers::project_convert; +use crate::test_types::{ProjectFile, ProjectTestCase}; + +/// A SQLBuild source file, with keys SQLQuack does not read. +const SOURCES: ProjectFile = ProjectFile { + path: "sources/raw.yml", + text: "sources:\n - name: raw__orders\n description: Orders, as loaded\n contract: enforced\n columns:\n - name: id\n type: INTEGER\n nullable: false\n - name: status\n type: VARCHAR\n audits: [not_null]\n - name: raw__inline\n expression: SELECT 1 AS id\n columns:\n - name: id\n type: INTEGER\n", +}; +/// A SQLBuild SQL function file, its description holding a comma and parentheses. +const SQL_FUNCTION: ProjectFile = ProjectFile { + path: "functions/sql/is_done.sql", + text: "FUNCTION (\n description \"Whether a status counts as done (shipped, or delivered)\",\n arguments (order_status VARCHAR),\n returns BOOLEAN,\n);\n\norder_status IN ('shipped', 'delivered')\n", +}; +/// A SQLBuild Python function file, unchanged. +const PYTHON_FUNCTION: ProjectFile = ProjectFile { + path: "functions/python/shout.py", + text: "from sqlbuild.functions import udf\n\n\n@udf(\n arguments={\"text\": \"VARCHAR\"},\n returns=\"VARCHAR\",\n runtime_version=\"3.11\",\n description=\"\"\"Upper case, with a comma, here.\"\"\",\n)\ndef main(text: str | None) -> str | None:\n return None if text is None else text.upper()\n", +}; + +#[test] +fn given_a_sqlbuild_project_when_converting_then_its_folders_declare_tables_and_functions() { + let test_cases = [ + ProjectTestCase { + description: "a source table types the query without --schema", + files: &[SOURCES], + project: "", + flags: &[], + sql: "SELECT id + 1 AS n FROM raw__orders", + expected_exit: Some(0), + expected_output: "\"converted\": true", + }, + ProjectTestCase { + description: "an inline-expression source is not a table, so a query of it refuses", + files: &[SOURCES], + project: "", + flags: &[], + sql: "SELECT id + 1 AS n FROM raw__inline", + expected_exit: Some(1), + expected_output: "\"converted\": false", + }, + ProjectTestCase { + description: "a SQL function file declares a function the query may call", + files: &[SOURCES, SQL_FUNCTION], + project: "", + flags: &[], + sql: "SELECT is_done(status) AS d FROM raw__orders", + expected_exit: Some(0), + expected_output: "\"converted\": true", + }, + ProjectTestCase { + description: "a Python function file is listed once the project confirms its runtime", + files: &[SOURCES, PYTHON_FUNCTION], + project: "[project.confirm]\npython_runtime = true\n", + flags: &[], + sql: "SELECT shout(status) AS s FROM raw__orders", + expected_exit: Some(0), + expected_output: "\"name\": \"SHOUT\"", + }, + ProjectTestCase { + description: "a Python function file may import udf from sqlquack instead", + files: &[ + SOURCES, + ProjectFile { + path: "functions/python/shout.py", + text: "from sqlquack import udf\n\n@udf(arguments={'text': 'VARCHAR'}, returns='VARCHAR')\ndef main(text):\n return text\n", + }, + ], + project: "[project.confirm]\npython_runtime = true\n", + flags: &[], + sql: "SELECT shout(status) AS s FROM raw__orders", + expected_exit: Some(0), + expected_output: "\"name\": \"SHOUT\"", + }, + ProjectTestCase { + description: "folders named in [project] replace the defaults", + files: &[ProjectFile { + path: "tables/raw.yml", + text: SOURCES.text, + }], + project: "[project]\nsources = \"tables\"\n", + flags: &[], + sql: "SELECT id + 1 AS n FROM raw__orders", + expected_exit: Some(0), + expected_output: "\"converted\": true", + }, + ]; + for test_case in test_cases { + let outcome = project_convert(&test_case); + assert_eq!( + outcome.exit_code, test_case.expected_exit, + "{}: {}", + test_case.description, outcome.stderr + ); + assert!( + outcome.stdout.contains(test_case.expected_output), + "{}: {}", + test_case.description, + outcome.stdout + ); + } +} + +#[test] +fn given_project_settings_when_flags_are_given_then_the_flags_win_for_the_run() { + let test_cases = [ + ProjectTestCase { + description: "the project's session applies without flags", + files: &[SOURCES], + project: "[project.session]\nTIMEZONE = \"America/New_York\"\n", + flags: &[], + sql: "SELECT id AS n FROM raw__orders", + expected_exit: Some(0), + expected_output: "TIMEZONE = America/New_York is not UTC", + }, + ProjectTestCase { + description: "a --session flag replaces the project's value of that parameter", + files: &[SOURCES], + project: "[project.session]\nTIMEZONE = \"America/New_York\"\n", + flags: &["--session", "TIMEZONE=UTC"], + sql: "SELECT id AS n FROM raw__orders", + expected_exit: Some(0), + expected_output: "\"notes\": []", + }, + ProjectTestCase { + description: "a --python-runtime flag confirms what the project does not", + files: &[SOURCES, PYTHON_FUNCTION], + project: "", + flags: &["--python-runtime"], + sql: "SELECT shout(status) AS s FROM raw__orders", + expected_exit: Some(0), + expected_output: "\"name\": \"SHOUT\"", + }, + ProjectTestCase { + description: "--overrides replaces the project's overrides folder", + files: &[ + SOURCES, + ProjectFile { + path: "overrides/broken.sql", + text: "not an override", + }, + ProjectFile { + path: "elsewhere/plain.sql", + text: "FUNCTION (\n arguments (x VARCHAR),\n returns VARCHAR,\n replaces UPPER,\n dialect duckdb\n);\nupper(x)\n", + }, + ], + project: "", + flags: &["--overrides", "elsewhere"], + sql: "SELECT UPPER(status) AS s FROM raw__orders", + expected_exit: Some(0), + expected_output: "\"converted\": true", + }, + ]; + for test_case in test_cases { + let outcome = project_convert(&test_case); + assert_eq!( + outcome.exit_code, test_case.expected_exit, + "{}: {}", + test_case.description, outcome.stderr + ); + assert!( + outcome.stdout.contains(test_case.expected_output), + "{}: {}", + test_case.description, + outcome.stdout + ); + } +} + +#[test] +fn given_a_project_when_a_declaration_clashes_or_is_unconfirmed_then_the_error_says_what_to_change() +{ + let test_cases = [ + ProjectTestCase { + description: "a table in both a source file and --schema is an error naming both", + files: &[ + SOURCES, + ProjectFile { + path: "schema.json", + text: "{\"RAW__ORDERS\": [[\"id\", \"INTEGER\"]]}", + }, + ], + project: "", + flags: &["--schema", "schema.json"], + sql: "SELECT id AS n FROM raw__orders", + expected_exit: Some(2), + expected_output: "the table RAW__ORDERS is declared twice, in ./sources/raw.yml and in schema.json", + }, + ProjectTestCase { + description: "a SQL and a Python function of one name is an error", + files: &[ + SQL_FUNCTION, + ProjectFile { + path: "functions/python/is_done.py", + text: PYTHON_FUNCTION.text, + }, + ], + project: "", + flags: &[], + sql: "SELECT 1 AS n", + expected_exit: Some(2), + expected_output: "the function IS_DONE is declared twice", + }, + ProjectTestCase { + description: "a Python function without the confirmation names the line to add", + files: &[SOURCES, PYTHON_FUNCTION], + project: "", + flags: &[], + sql: "SELECT shout(status) AS s FROM raw__orders", + expected_exit: Some(1), + expected_output: "add `python_runtime = true` under [project.confirm] in sqlquack.toml, or pass --python-runtime for one run", + }, + ProjectTestCase { + description: "a Python function file not marked as SQLBuild marks one is an error", + files: &[ProjectFile { + path: "functions/python/plain.py", + text: "def main(text):\n return text\n", + }], + project: "", + flags: &[], + sql: "SELECT 1 AS n", + expected_exit: Some(2), + expected_output: "plain.py: has no @udf(...) decorator", + }, + ]; + for test_case in test_cases { + let outcome = project_convert(&test_case); + assert_eq!( + outcome.exit_code, test_case.expected_exit, + "{}: {}", + test_case.description, outcome.stderr + ); + assert!( + outcome.stdout.contains(test_case.expected_output) + || outcome.stderr.contains(test_case.expected_output), + "{}: {}{}", + test_case.description, + outcome.stdout, + outcome.stderr + ); + } +} diff --git a/crates/sqlquack/tests/cli/test_types.rs b/crates/sqlquack/tests/cli/test_types.rs index 191b9a7bf..3724dfa6d 100644 --- a/crates/sqlquack/tests/cli/test_types.rs +++ b/crates/sqlquack/tests/cli/test_types.rs @@ -293,3 +293,29 @@ pub(crate) struct SchemaErrorTestCase { pub(crate) schema: &'static str, pub(crate) expected_error: &'static str, } + +/// A file of a project, by its path under the project folder. +#[derive(Clone, Copy)] +pub(crate) struct ProjectFile { + pub(crate) path: &'static str, + pub(crate) text: &'static str, +} + +/// A project's files and `[project]` settings, the convert flags, and what the report must hold. +pub(crate) struct ProjectTestCase { + pub(crate) description: &'static str, + pub(crate) files: &'static [ProjectFile], + /// TOML appended to the repository's policy file. + pub(crate) project: &'static str, + pub(crate) flags: &'static [&'static str], + pub(crate) sql: &'static str, + pub(crate) expected_exit: Option, + pub(crate) expected_output: &'static str, +} + +/// How `sqlquack convert --json` ended in a project folder. +pub(crate) struct ProjectOutcome { + pub(crate) exit_code: Option, + pub(crate) stdout: String, + pub(crate) stderr: String, +} diff --git a/python/src/sqlquack/__init__.py b/python/src/sqlquack/__init__.py index ca28fbea4..715af5789 100644 --- a/python/src/sqlquack/__init__.py +++ b/python/src/sqlquack/__init__.py @@ -6,6 +6,7 @@ from sqlquack.api.exceptions import SqlQuackError from sqlquack.api.main.check import check from sqlquack.api.main.convert import convert +from sqlquack.api.main.udf import udf from sqlquack.api.models import CheckResult, Conversion, ConvertOptions, Diagnostic, Span __all__ = [ @@ -18,4 +19,5 @@ "__version__", "check", "convert", + "udf", ] diff --git a/python/src/sqlquack/api/main/convert.py b/python/src/sqlquack/api/main/convert.py index ee400b4e7..c980b0b9d 100644 --- a/python/src/sqlquack/api/main/convert.py +++ b/python/src/sqlquack/api/main/convert.py @@ -17,6 +17,7 @@ def convert( """Convert `sql` or refuse it as `sqlquack convert` does; exit-2 failures raise.""" chosen: ConvertOptions = options or ConvertOptions() request: dict[str, Any] = { + "project": path_argument(chosen.project), "schema": path_argument(chosen.schema), "policy": path_argument(chosen.policy), "builtin_policy": chosen.builtin_policy, diff --git a/python/src/sqlquack/api/main/udf.py b/python/src/sqlquack/api/main/udf.py new file mode 100644 index 000000000..febe5eb46 --- /dev/null +++ b/python/src/sqlquack/api/main/udf.py @@ -0,0 +1,16 @@ +"""`udf`: marks a project's Python function, as SQLBuild's `sqlbuild.functions.udf` does.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + + +def udf(**kwargs: object) -> Callable[[Callable[..., Any]], Callable[..., Any]]: + """Mark a Python function as a project function; its keywords are read statically.""" + del kwargs + + def _decorate(function: Callable[..., Any]) -> Callable[..., Any]: + return function + + return _decorate diff --git a/python/src/sqlquack/api/models.py b/python/src/sqlquack/api/models.py index ef9d09ee8..d82d961ff 100644 --- a/python/src/sqlquack/api/models.py +++ b/python/src/sqlquack/api/models.py @@ -63,8 +63,9 @@ class Conversion: @dataclass(frozen=True) class ConvertOptions: - """The `sqlquack convert` flags, one field each; the defaults are the flags' own.""" + """A project folder and one field per `sqlquack convert` flag, with the flags' defaults.""" + project: str | os.PathLike[str] | None = None schema: str | os.PathLike[str] | None = None policy: str | os.PathLike[str] | None = None builtin_policy: bool = False diff --git a/tests/unit/python/src/sqlquack/api/main/_test_types.py b/tests/unit/python/src/sqlquack/api/main/_test_types.py index b8cc4d383..6cbc93d3e 100644 --- a/tests/unit/python/src/sqlquack/api/main/_test_types.py +++ b/tests/unit/python/src/sqlquack/api/main/_test_types.py @@ -57,3 +57,11 @@ class DeepChainTestCase: sql: str expected_converted: bool expected_codes: tuple[str, ...] + + +@dataclass(frozen=True) +class ProjectTestCase: + description: str + session: dict[str, str] + arguments: tuple[str, ...] + expected_notes: int diff --git a/tests/unit/python/src/sqlquack/api/main/test_convert.py b/tests/unit/python/src/sqlquack/api/main/test_convert.py index 44c8a5df9..12d7bfaa6 100644 --- a/tests/unit/python/src/sqlquack/api/main/test_convert.py +++ b/tests/unit/python/src/sqlquack/api/main/test_convert.py @@ -12,6 +12,7 @@ CommandLineTestCase, DeepChainTestCase, FailureTestCase, + ProjectTestCase, TypedDiagnosticTestCase, ) from tests.unit.python.src.sqlquack.api.main.helpers import command_report, schema_file @@ -173,3 +174,42 @@ def test_given_a_long_flat_chain_when_converting_then_the_interpreter_survives( ) assert conversion.converted is test_case.expected_converted assert tuple(found.code for found in conversion.diagnostics) == test_case.expected_codes + + +@pytest.mark.parametrize( + "test_case", + ( + ProjectTestCase( + description="the project's sources type the query and its session applies", + session={}, + arguments=(), + expected_notes=1, + ), + ProjectTestCase( + description="a session option replaces the project's value for the call", + session={"TIMEZONE": "UTC"}, + arguments=("--session", "TIMEZONE=UTC"), + expected_notes=0, + ), + ), + ids=lambda case: case.description, +) +def test_given_a_project_folder_when_converting_then_its_sources_and_settings_apply( + test_case: ProjectTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = tmp_path / "project" + (project / "sources").mkdir(parents=True) + (project / "sqlquack.toml").write_text( + Path(POLICY).read_text() + '\n[project.session]\nTIMEZONE = "America/New_York"\n' + ) + (project / "sources" / "raw.yml").write_text( + "sources:\n - name: orders\n columns:\n - name: order_id\n" + " type: NUMBER(38,0)\n" + ) + monkeypatch.chdir(tmp_path) + sql: str = "SELECT order_id + 1 AS next_id FROM orders" + options: ConvertOptions = ConvertOptions(project=project, session=test_case.session) + conversion: Conversion = sqlquack.convert(sql=sql, options=options) + assert conversion.converted is True + assert conversion.report == command_report(sql, test_case.arguments, project) + assert len(conversion.notes) == test_case.expected_notes diff --git a/website/src/content/docs/docs/cli/convert.mdx b/website/src/content/docs/docs/cli/convert.mdx index 2da414bc7..c39f859f6 100644 --- a/website/src/content/docs/docs/cli/convert.mdx +++ b/website/src/content/docs/docs/cli/convert.mdx @@ -52,9 +52,9 @@ SQL that does not parse, or that Snowflake rejects, is refused with the same cod | Option | Meaning | |--------|---------| -| `--schema ` | The relations and functions the SQL reads. See [Schema files](/docs/schema/). | -| `--session NAME=VALUE` | A Snowflake session parameter of your session. Repeatable. See [Sessions](/docs/policy/#sessions). | -| `--overrides ` | User overrides: a `FUNCTION()` file, or a directory of them, each with `replaces` and `dialect duckdb`. | +| `--schema ` | The relations and functions the SQL reads, added to the [project](/docs/project/)'s. See [Schema files](/docs/schema/). | +| `--session NAME=VALUE` | A Snowflake session parameter of your session, replacing the project's. Repeatable. See [Sessions](/docs/policy/#sessions). | +| `--overrides ` | User overrides: a `FUNCTION()` file, or a directory of them, each with `replaces` and `dialect duckdb`. Replaces the project's `overrides/` folder. | | `--verified-override NAME=SHA256` | An override verified for the body with that hash, as `convert --json` reports. | | `--format text\|json`, `--json` | Print the report as text or JSON. | diff --git a/website/src/content/docs/docs/policy.mdx b/website/src/content/docs/docs/policy.mdx index f6b118f40..6a8579228 100644 --- a/website/src/content/docs/docs/policy.mdx +++ b/website/src/content/docs/docs/policy.mdx @@ -55,6 +55,9 @@ sqlquack convert: done, converted under utc-v1 | `--utc-session` | Confirm both sessions run in UTC, so strict mode relies on `session_timezone_utc`. | | `--confirm-declared-offsets` | Confirm the `TIMESTAMP_TZ` offsets the schema declares. | | `--python-runtime` | Confirm a Python runtime registers the project's Python functions. | + +A project confirms the same three for every run under `[project.confirm]` in the policy file; see +[Projects](/docs/project/#settings). | `--loud-divergences-only` | Rely only on divergences whose converted SQL raises rather than returns another value. | ## Sessions @@ -63,7 +66,8 @@ Converted SQL is proven only where both sessions run in UTC: Snowflake's `TIMEZO `TimeZone` must be `'UTC'`. Statements whose result depends on the session time zone carry the `session_timezone_utc` assumption, and strict mode refuses them unless you pass `--utc-session`. -`--session NAME=VALUE` passes your Snowflake session parameters, one flag each: +`--session NAME=VALUE` passes your Snowflake session parameters, one flag each, or +`[project.session]` in the policy file for every run (see [Projects](/docs/project/#settings)): - Conversion runs under the recorded session profile they match: `utc-v1`, or `utc-qiic-v1` for `QUOTED_IDENTIFIERS_IGNORE_CASE = true`. A parameter left unset takes Snowflake's documented diff --git a/website/src/content/docs/docs/project.mdx b/website/src/content/docs/docs/project.mdx new file mode 100644 index 000000000..29fef42ae --- /dev/null +++ b/website/src/content/docs/docs/project.mdx @@ -0,0 +1,146 @@ +--- +title: "Projects" +description: "Keep tables, functions, overrides and session settings in a project folder, laid out as SQLBuild lays them out." +--- + +A project folder holds `sqlquack.toml` and the folders `convert` and `census` read without flags. +A SQLBuild project's `sources/` and `functions/` folders work unchanged. + +```text +sqlquack.toml +sources/raw.yml +functions/sql/is_shipped.sql +functions/python/label.py +overrides/ +``` + +```console +$ sqlquack convert shipped.sql +sqlquack convert: converting… +SELECT "ORDERS"."ORDER_ID" AS "ORDER_ID", CAST("IS_SHIPPED"("ORDERS"."STATUS") AS BOOLEAN) AS "SHIPPED" FROM "ORDERS" AS "ORDERS" +sqlquack convert: done, converted under utc-v1 +``` + +The project is the folder holding the policy file: `sqlquack.toml` in the current directory, or +the file `--policy` names. A folder that does not exist is skipped. + +## Declaring tables + +Each `.yml` or `.yaml` file under `sources/` lists tables in SQLBuild's source format. SQLQuack +reads each source's `name` and its columns' `name` and `type`, and ignores other keys such as +`description`, `nullable`, `contract` and `audits`. + +```yaml +sources: + - name: orders + columns: + - name: order_id + type: NUMBER(38,0) + - name: status + type: VARCHAR +``` + +Names are unquoted, so `orders` declares the table `ORDERS`, as Snowflake folds it. A source with +an `expression` is an inline query, not a table, and is not declared. A source with a column +without a type is left out, as a [schema file](/docs/schema/) leaves out an unknown type. + +## Declaring functions + +A function is named after its file, upper-cased: `functions/sql/is_shipped.sql` declares +`IS_SHIPPED`. + +A SQL function file starts with SQLBuild's `FUNCTION ( ... );` header, then the body: + +```sql +FUNCTION ( + description "Whether an order has shipped", + arguments (order_status VARCHAR), + returns BOOLEAN, +); + +order_status = 'shipped' +``` + +`returns table (name TYPE, ...)` declares a table function. `description`, `tags`, `database` and +`schema` are accepted and ignored. + +A Python function file is marked with SQLBuild's `@udf(...)` decorator, imported from +`sqlbuild.functions` or `sqlquack`: + +```python +from sqlbuild.functions import udf + + +@udf(arguments={"status": "VARCHAR"}, returns="VARCHAR", runtime_version="3.11") +def main(status: str | None) -> str | None: + return None if status is None else status.title() +``` + +SQLQuack reads the decorator's `arguments` and `returns` from the file without running it. Calls +convert only once the project confirms a Python runtime, as [`--python-runtime`](/docs/policy/#flags) +does: + +```console +$ sqlquack convert labels.sql +error[SQ0008]: the conversion needs a known divergence the policy refuses: `fn:LABEL` + --> labels.sql:1:18 + | +1 | SELECT order_id, label(status) AS label FROM orders + | ^^^^^^^^^^^^^ the Python function LABEL, which runs only where a Python runtime registers it, needs known divergence `project_udf`, an assumption the caller has not confirmed + | + = help: to rely on `project_udf`: confirm its assumption: add `python_runtime = true` under [project.confirm] in sqlquack.toml, or pass --python-runtime for one run +``` + +## Overriding built-in functions + +Each file under `overrides/` replaces a Snowflake built-in with DuckDB SQL. Its header names what +it replaces with `replaces ` and `dialect duckdb`; see +[`convert`](/docs/cli/convert/). + +## Settings + +`sqlquack.toml` holds the project's settings beside its policy: + +```toml +[project] +sources = "tables" + +[project.session] +TIMEZONE = "UTC" + +[project.confirm] +python_runtime = true +``` + +| Key | Default | Meaning | +|-----|---------|---------| +| `[project] sources` | `sources` | The folder of source files. | +| `[project] functions` | `functions` | The folder holding `sql/` and `python/` function files. | +| `[project] overrides` | `overrides` | The folder of override files. | +| `[project.session]` | none | Snowflake session parameters, as [`--session`](/docs/policy/#sessions) gives them. | +| `[project.confirm] python_runtime` | `false` | As `--python-runtime`. | +| `[project.confirm] utc_session` | `false` | As `--utc-session`. | +| `[project.confirm] declared_offsets` | `false` | As `--confirm-declared-offsets`. | + +A folder named in `[project]` must exist. Write a folder key only when it differs from the default. + +## Flags and the project + +Flags apply on top of the project for one run: + +| Flag | With a project | +|------|----------------| +| `--session NAME=VALUE` | Replaces the project's value of `NAME`; other project parameters still apply. | +| `--overrides ` | Replaces the project's overrides folder. | +| `--schema ` | Adds the file's relations and functions to the project's. | +| `--utc-session`, `--confirm-declared-offsets`, `--python-runtime` | Confirm on top of `[project.confirm]`. | +| `--builtin-policy` | Reads no policy file, so no project. | + +A table or function declared twice, in two project files or in a project file and `--schema`, is +an error naming both, and `convert` exits 2: + +```console +$ sqlquack convert shipped.sql --schema schema.json +sqlquack convert: converting… +sqlquack convert: failed: the table ORDERS is declared twice, in ./sources/raw.yml and in schema.json; keep one +``` diff --git a/website/src/content/docs/docs/python.mdx b/website/src/content/docs/docs/python.mdx index 3af634699..35948db14 100644 --- a/website/src/content/docs/docs/python.mdx +++ b/website/src/content/docs/docs/python.mdx @@ -73,14 +73,18 @@ SQL in rendered diagnostics. It returns a `CheckResult`: | `rendered` | `str` | The refusals as the command line prints them. | | `report` | `Mapping[str, Any]` | Exactly `sqlquack convert --json`'s report. | -`ConvertOptions` has one field per [`convert`](/docs/cli/convert/) flag, with the same defaults: -`schema`, `policy`, `builtin_policy`, `session`, `strict`, `utc_session`, +`ConvertOptions` has `project` and one field per [`convert`](/docs/cli/convert/) flag, with the +same defaults: `schema`, `policy`, `builtin_policy`, `session`, `strict`, `utc_session`, `confirm_declared_offsets`, `python_runtime`, `loud_divergences_only`, `overrides` and `verified_overrides`. `schema`, `policy` and `overrides` take a path (`str` or `os.PathLike`), not the parsed contents. `session` and `verified_overrides` take a mapping, such as -`session={"TIMEZONE": "UTC"}`, not a list of `NAME=VALUE` strings. With `policy` and -`builtin_policy` both unset, `convert` reads `sqlquack.toml` from the current directory and raises -`SqlQuackError` when there is none. +`session={"TIMEZONE": "UTC"}`, not a list of `NAME=VALUE` strings. + +`project` takes a [project](/docs/project/) folder: its `sqlquack.toml` is the policy unless +`policy` names another, and its sources, functions, overrides, session and confirmations apply with +the other options on top, as flags do. With `project`, `policy` and `builtin_policy` all unset, +`convert` reads `sqlquack.toml` from the current directory and raises `SqlQuackError` when there is +none. ## Diagnostics diff --git a/website/src/content/docs/docs/schema.mdx b/website/src/content/docs/docs/schema.mdx index 271823e2e..1c968dc6c 100644 --- a/website/src/content/docs/docs/schema.mdx +++ b/website/src/content/docs/docs/schema.mdx @@ -4,8 +4,10 @@ description: "Declare the relations and functions your SQL reads, so conversion --- Conversion needs to know what each relation and column name refers to and what Snowflake type it -has. A schema file declares them as JSON. Pass it with `--schema` to `convert`, `function`, -`census` and `index`. +has. A [project](/docs/project/) declares them in `sources/` and `functions/` files; a schema file +declares them as JSON, the format programs such as SQLBuild write. Pass it with `--schema` to +`convert`, `function`, `census` and `index`. With a project, the schema file adds to the project's +declarations, and a name declared in both is an error. ```json { diff --git a/website/src/sidebar.json b/website/src/sidebar.json index c9a55c970..7e2cc511e 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -1,6 +1,6 @@ [ { "label": "Getting Started", "items": ["docs", "docs/quickstart"] }, - { "label": "Concepts", "items": ["docs/schema", "docs/policy", "docs/coverage"] }, + { "label": "Concepts", "items": ["docs/project", "docs/schema", "docs/policy", "docs/coverage"] }, { "label": "Diagnostics", "items": [ From b401cba0f46a96f912f824b29cd08e3bc5cc7c23 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 6 Nov 2026 07:58:04 +0000 Subject: [PATCH 2/2] feat: warn when a source with an untyped column is left out The table is left out whole, so SELECT * never misses a column; convert and census name the file, the source and the column on standard error, and convert's report lists the same message under warnings. --- .../src/cli/_helpers/census_command.rs | 3 ++ .../src/cli/_helpers/convert_command.rs | 5 +++ crates/sqlquack/src/cli/_helpers/project.rs | 5 ++- .../sqlquack/src/cli/main/convert_report.rs | 13 ++++++-- crates/sqlquack/src/cli/models.rs | 4 +++ .../src/project/_helpers/declarations.rs | 17 ++++++---- .../src/project/_helpers/source_files.rs | 32 ++++++++++++++++--- .../src/project/main/project_schema.rs | 8 ++--- crates/sqlquack/src/project/models.rs | 11 +++++++ crates/sqlquack/tests/cli/test_project.rs | 12 +++++++ website/src/content/docs/docs/project.mdx | 14 ++++++-- 11 files changed, 104 insertions(+), 20 deletions(-) diff --git a/crates/sqlquack/src/cli/_helpers/census_command.rs b/crates/sqlquack/src/cli/_helpers/census_command.rs index bed606463..44c2a53ed 100644 --- a/crates/sqlquack/src/cli/_helpers/census_command.rs +++ b/crates/sqlquack/src/cli/_helpers/census_command.rs @@ -55,6 +55,9 @@ pub(crate) fn census_command( return ExitCode::from(2); } }; + for warning in &project.warnings { + let _ = writeln!(stderr, "sqlquack census: warning: {warning}"); + } let schema = project.schema; let conversion = match with_overrides( divergence_options(policy.as_ref(), &project.conversion), diff --git a/crates/sqlquack/src/cli/_helpers/convert_command.rs b/crates/sqlquack/src/cli/_helpers/convert_command.rs index 853e50caf..c27907f20 100644 --- a/crates/sqlquack/src/cli/_helpers/convert_command.rs +++ b/crates/sqlquack/src/cli/_helpers/convert_command.rs @@ -71,6 +71,9 @@ pub(crate) fn convert_command( if stdout.write_all(rendered.as_bytes()).is_err() { return ExitCode::FAILURE; } + for warning in &report.warnings { + let _ = writeln!(stderr, "sqlquack convert: warning: {warning}"); + } for note in &report.notes { let _ = writeln!(stderr, "sqlquack convert: note: {note}"); } @@ -228,6 +231,7 @@ pub(crate) fn refused_session(reason: String) -> ConvertReport { assumptions: Vec::new(), divergence_failures: BTreeMap::new(), notes: Vec::new(), + warnings: Vec::new(), python_functions: Vec::new(), macros: Vec::new(), overrides: Vec::new(), @@ -288,6 +292,7 @@ pub(crate) fn analysis_report( converted, profile, notes, + warnings: Vec::new(), sql: match converted { true => analysis .conversions diff --git a/crates/sqlquack/src/cli/_helpers/project.rs b/crates/sqlquack/src/cli/_helpers/project.rs index 56afcbde6..2a2acec39 100644 --- a/crates/sqlquack/src/cli/_helpers/project.rs +++ b/crates/sqlquack/src/cli/_helpers/project.rs @@ -26,6 +26,7 @@ pub(crate) fn project_choices( session: session.to_vec(), overrides: overrides.map(Path::to_path_buf), conversion, + warnings: Vec::new(), }); }; let root = conversion @@ -54,8 +55,10 @@ pub(crate) fn project_choices( Some(path) => Some(path.to_path_buf()), None => project_folder(&root, project.overrides.as_deref(), OVERRIDES_FOLDER)?, }; + let (schema, warnings) = project_schema(&root, project, file)?; Ok(ProjectChoices { - schema: project_schema(&root, project, file)?, + schema, + warnings, session, overrides, conversion, diff --git a/crates/sqlquack/src/cli/main/convert_report.rs b/crates/sqlquack/src/cli/main/convert_report.rs index f9c8b1478..0db17b060 100644 --- a/crates/sqlquack/src/cli/main/convert_report.rs +++ b/crates/sqlquack/src/cli/main/convert_report.rs @@ -27,6 +27,7 @@ pub fn convert_report(arguments: &ConvertArguments, sql: &str) -> Result Result matched, - Err(reason) => return Ok(refused_session(reason)), + Err(reason) => { + return Ok(ConvertReport { + warnings, + ..refused_session(reason) + }); + } }; let options = chosen_options(arguments, policy.as_ref(), options)?; let catalogue = builtin_catalogue(); @@ -54,10 +60,11 @@ pub fn convert_report(arguments: &ConvertArguments, sql: &str) -> Result, /// What the session parameters changed about conversion. pub notes: Vec, + /// What the project leaves out, such as a source with an untyped column. + pub warnings: Vec, /// The Python functions the SQL calls, which the caller registers before running it. pub python_functions: Vec, /// The macros of the user overrides the SQL calls, which the caller runs before it. @@ -455,4 +457,6 @@ pub struct ProjectChoices { pub overrides: Option, /// The caller's flags with `[project.confirm]` added: a flag confirms on top of the file. pub conversion: PolicyArguments, + /// What the project leaves out, such as a source with an untyped column. + pub warnings: Vec, } diff --git a/crates/sqlquack/src/project/_helpers/declarations.rs b/crates/sqlquack/src/project/_helpers/declarations.rs index 945b28c4d..4a85d9daa 100644 --- a/crates/sqlquack/src/project/_helpers/declarations.rs +++ b/crates/sqlquack/src/project/_helpers/declarations.rs @@ -19,19 +19,24 @@ use crate::project::models::Declaration; /// Declarations as the schema file writes them, by name, and where each came from. pub(crate) type Declared = (Map, Vec); -/// The tables of the project's source files. -pub(crate) fn source_tables(root: &Path, project: &ProjectPolicy) -> Result { - let (mut tables, mut declarations) = (Map::new(), Vec::new()); +/// The tables of the project's source files, and a warning for each source left out. +pub(crate) fn source_tables( + root: &Path, + project: &ProjectPolicy, +) -> Result<(Declared, Vec), String> { + let (mut tables, mut declarations, mut warnings) = (Map::new(), Vec::new(), Vec::new()); let Some(folder) = project_folder(root, project.sources.as_deref(), SOURCES_FOLDER)? else { - return Ok((tables, declarations)); + return Ok(((tables, declarations), warnings)); }; for path in files(&folder, SOURCE_EXTENSIONS) { - for (name, columns) in source_relations(&path)? { + let found = source_relations(&path)?; + warnings.extend(found.warnings); + for (name, columns) in found.tables { declarations.push(declaration(TABLE_KIND, &name, &path)); tables.insert(name, columns); } } - Ok((tables, declarations)) + Ok(((tables, declarations), warnings)) } /// The functions of the project's SQL and Python function files, named after their files. diff --git a/crates/sqlquack/src/project/_helpers/source_files.rs b/crates/sqlquack/src/project/_helpers/source_files.rs index 824c1306f..db57aed6c 100644 --- a/crates/sqlquack/src/project/_helpers/source_files.rs +++ b/crates/sqlquack/src/project/_helpers/source_files.rs @@ -6,6 +6,7 @@ use serde::Deserialize; use serde_json::{Value, json}; use crate::project::_helpers::yaml_value::yaml_value; +use crate::project::models::SourceTables; /// One source file; keys SQLQuack does not read, such as contracts and audits, are ignored. #[derive(Deserialize)] @@ -31,17 +32,40 @@ struct SourceColumn { type_name: Option, } -/// Each table `path` declares, upper-cased as Snowflake folds SQLBuild's unquoted names. -pub(crate) fn source_relations(path: &Path) -> Result, String> { +/// Each table `path` declares, upper-cased as Snowflake folds SQLBuild's unquoted names, and warnings. +pub(crate) fn source_relations(path: &Path) -> Result { let failed = |reason: String| format!("{}: {reason}", path.display()); let text = std::fs::read_to_string(path).map_err(|error| failed(error.to_string()))?; let value = yaml_value(&text).map_err(failed)?; if value.is_null() { - return Ok(Vec::new()); + return Ok(SourceTables::default()); } let file: SourceFile = serde_json::from_value(value) .map_err(|error| failed(format!("is not a SQLBuild source file: {error}")))?; - Ok(file.sources.into_iter().filter_map(table).collect()) + let mut found = SourceTables::default(); + for source in file.sources { + match untyped_column(&source) { + Some(column) => found.warnings.push(format!( + "{}: the source {} is left out, because its column {column} has no type", + path.display(), + source.name + )), + None => found.tables.extend(table(source)), + } + } + Ok(found) +} + +/// The first column of a table source without a text type. +fn untyped_column(source: &SourceEntry) -> Option<&str> { + match source.expression { + Some(_) => None, + None => source + .columns + .iter() + .find(|column| !matches!(column.type_name, Some(Value::String(_)))) + .map(|column| column.name.as_str()), + } } /// A source as a table and its `[column, type]` pairs; None for an inline query or an untyped column. diff --git a/crates/sqlquack/src/project/main/project_schema.rs b/crates/sqlquack/src/project/main/project_schema.rs index ea21d4d51..1f95f2620 100644 --- a/crates/sqlquack/src/project/main/project_schema.rs +++ b/crates/sqlquack/src/project/main/project_schema.rs @@ -13,13 +13,13 @@ use crate::project::_helpers::declarations::{ }; use crate::project::constants::{FUNCTION_KIND, TABLE_KIND}; -/// The schema of the project at `root` merged with `file`'s; one name declared twice is an error. +/// The project's schema merged with `file`'s, and warnings; a name declared twice is an error. pub fn project_schema( root: &Path, project: &ProjectPolicy, file: Option<(&Path, Schema)>, -) -> Result { - let (tables, mut declarations) = source_tables(root, project)?; +) -> Result<(Schema, Vec), String> { + let ((tables, mut declarations), warnings) = source_tables(root, project)?; let (functions, function_declarations) = project_functions(root, project)?; declarations.extend(function_declarations); let declared = json!({ "relations": tables, "functions": functions }).to_string(); @@ -43,5 +43,5 @@ pub fn project_schema( schema.functions.extend(file.functions); } no_clash(&declarations)?; - Ok(schema) + Ok((schema, warnings)) } diff --git a/crates/sqlquack/src/project/models.rs b/crates/sqlquack/src/project/models.rs index cc72d4d86..a43ac911d 100644 --- a/crates/sqlquack/src/project/models.rs +++ b/crates/sqlquack/src/project/models.rs @@ -2,6 +2,8 @@ use std::path::PathBuf; +use serde_json::Value; + /// One table or function a file declares, kept to name both files of a clash. #[derive(Debug)] pub struct Declaration { @@ -12,3 +14,12 @@ pub struct Declaration { /// The file declaring it. pub path: PathBuf, } + +/// The tables one source file declares, and a warning for each source it leaves out. +#[derive(Debug, Default)] +pub struct SourceTables { + /// Upper-case table names and their `[column, type]` pairs. + pub tables: Vec<(String, Value)>, + /// One message per source left out, naming the file, the source and the column. + pub warnings: Vec, +} diff --git a/crates/sqlquack/tests/cli/test_project.rs b/crates/sqlquack/tests/cli/test_project.rs index 6c3ea5657..ddfea6c16 100644 --- a/crates/sqlquack/tests/cli/test_project.rs +++ b/crates/sqlquack/tests/cli/test_project.rs @@ -209,6 +209,18 @@ fn given_a_project_when_a_declaration_clashes_or_is_unconfirmed_then_the_error_s expected_exit: Some(1), expected_output: "add `python_runtime = true` under [project.confirm] in sqlquack.toml, or pass --python-runtime for one run", }, + ProjectTestCase { + description: "a source with an untyped column is left out with a warning naming it", + files: &[ProjectFile { + path: "sources/raw.yml", + text: "sources:\n - name: raw__customers\n columns:\n - name: id\n type: INTEGER\n - name: email\n", + }], + project: "", + flags: &[], + sql: "SELECT id AS n FROM raw__customers", + expected_exit: Some(1), + expected_output: "sqlquack convert: warning: ./sources/raw.yml: the source raw__customers is left out, because its column email has no type", + }, ProjectTestCase { description: "a Python function file not marked as SQLBuild marks one is an error", files: &[ProjectFile { diff --git a/website/src/content/docs/docs/project.mdx b/website/src/content/docs/docs/project.mdx index 29fef42ae..9977022b0 100644 --- a/website/src/content/docs/docs/project.mdx +++ b/website/src/content/docs/docs/project.mdx @@ -41,8 +41,18 @@ sources: ``` Names are unquoted, so `orders` declares the table `ORDERS`, as Snowflake folds it. A source with -an `expression` is an inline query, not a table, and is not declared. A source with a column -without a type is left out, as a [schema file](/docs/schema/) leaves out an unknown type. +an `expression` is an inline query, not a table, and is not declared. + +A source with a column without a `type` is left out whole, so `SELECT *` never misses a column. +`convert` and `census` warn, naming the file, the source and the column: + +```console +$ sqlquack convert customers.sql +sqlquack convert: converting… +sqlquack convert: warning: ./sources/raw.yml: the source customers is left out, because its column email has no type +``` + +The report's `warnings` lists the same messages. ## Declaring functions