diff --git a/Cargo.lock b/Cargo.lock index 3cc452ec33..09550ca684 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -756,6 +756,9 @@ dependencies = [ [[package]] name = "sqlbuild-model-config" version = "0.153.0" +dependencies = [ + "sqlbuild-core", +] [[package]] name = "sqlbuild-python" diff --git a/crates/sqlbuild-analysis/src/assembly/project/_helpers/templates.rs b/crates/sqlbuild-analysis/src/assembly/project/_helpers/templates.rs index 4a5b17d1ab..f97b281c3a 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/_helpers/templates.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/_helpers/templates.rs @@ -9,7 +9,8 @@ use sqlbuild_model_config::templates::models::{ use sqlbuild_model_config::templates::types::TemplateHost; use crate::assembly::project::constants::{ - PYTHON_FALSE, PYTHON_NONE, PYTHON_TRUE, TEMPLATE_DEFERRAL, TEMPLATE_OPEN_TOKEN, + PROJECT_VAR_FALSE, PROJECT_VAR_TRUE, PYTHON_FALSE, PYTHON_NONE, PYTHON_TRUE, TEMPLATE_DEFERRAL, + TEMPLATE_LABEL, TEMPLATE_OPEN_TOKEN, }; use crate::assembly::project::models::{InputRead, Variable}; use crate::assembly::project::types::Fact; @@ -43,7 +44,7 @@ impl TemplateHost for ScalarHost<'_> { { None => Ok(None), Some((_, Variable::Scalar(value))) => Ok(Some(value.clone())), - Some((_, Variable::Unsupported)) => Err(TemplateFailure::Unsupported), + Some((_, Variable::Unsupported)) => Err(TemplateFailure::Host), } } @@ -58,7 +59,7 @@ impl TemplateHost for ScalarHost<'_> { .iter() .find(|(variable, _)| variable == name) { - None => Err(TemplateFailure::Unsupported), + None => Err(TemplateFailure::Host), Some((_, value)) => Ok(value.clone().map(Scalar::Text)), } } @@ -87,8 +88,21 @@ impl TemplateHost for ScalarHost<'_> { Scalar::Null } - fn scalar(&self, value: &Self::Value) -> Option { - Some(value.clone()) + fn scalar(&self, value: &Self::Value) -> Result { + Ok(value.clone()) + } + + fn render(&self, value: &Self::Value, _label: &str) -> Result { + Ok(match value { + Scalar::Null => String::new(), + Scalar::Bool(true) => PROJECT_VAR_TRUE.to_owned(), + Scalar::Bool(false) => PROJECT_VAR_FALSE.to_owned(), + Scalar::Text(text) => text.clone(), + }) + } + + fn label(&self) -> &str { + TEMPLATE_LABEL } } diff --git a/crates/sqlbuild-analysis/src/assembly/project/constants.rs b/crates/sqlbuild-analysis/src/assembly/project/constants.rs index 2fb32b6c70..dc6f8f874f 100644 --- a/crates/sqlbuild-analysis/src/assembly/project/constants.rs +++ b/crates/sqlbuild-analysis/src/assembly/project/constants.rs @@ -28,6 +28,11 @@ pub(crate) const DESTINATION_TABLE_CONTEXT: &str = "destination.table"; pub(crate) const DESTINATION_QUALIFIED_CONTEXT: &str = "destination.qualified"; pub(crate) const AUDIT_TARGET_DEFERRAL: &str = "audit_target_kind"; pub(crate) const TEMPLATE_DEFERRAL: &str = "target_template"; +/// `render_project_var_text` of booleans inside a larger template string. +pub(crate) const PROJECT_VAR_TRUE: &str = "true"; +pub(crate) const PROJECT_VAR_FALSE: &str = "false"; +/// The label template errors would name; assembly defers every template error. +pub(crate) const TEMPLATE_LABEL: &str = "project assembly template"; pub(crate) const PRESERVED_NAMESPACE_DEFERRAL: &str = "preserved_namespace"; pub(crate) const SYNTAX_DEFERRAL: &str = "syntax_error"; pub(crate) const PLACEHOLDER_DEFERRAL: &str = "placeholder_word"; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/mod.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/mod.rs index 0b35188f4d..db9c168b3d 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/mod.rs @@ -1 +1,2 @@ +pub(crate) mod polyglot; pub mod sql_tests; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/function_types.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/function_types.rs new file mode 100644 index 0000000000..330f9618fa --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/function_types.rs @@ -0,0 +1,30 @@ +//! Function argument and return types parsed with the adapter's SQL analysis dialect. + +use polyglot_sql::Dialect; + +use crate::compiler::_helpers::polyglot::parse_options::guarded_parse_options; +use crate::compiler::constants::FUNCTION_TYPE_DIALECTS; + +/// The error for a type the adapter's dialect cannot parse; adapters without a dialect pass. +pub(crate) fn function_type_error( + type_sql: &str, + adapter_name: &str, + context: &str, +) -> Result, String> { + let Some((_, dialect_name, dialect)) = FUNCTION_TYPE_DIALECTS + .iter() + .find(|(adapter, _, _)| *adapter == adapter_name) + else { + return Ok(None); + }; + let options = guarded_parse_options().map_err(|error| error.to_string())?; + Ok(Dialect::get(*dialect) + .parse_data_type_with_options(type_sql, &options) + .err() + .map(|error| { + format!( + "{context} type '{type_sql}' is not valid for adapter '{adapter_name}' SQL \ + analysis dialect '{dialect_name}': {error}" + ) + })) +} diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/mod.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/mod.rs new file mode 100644 index 0000000000..bfe54fc9de --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod function_types; +pub(crate) mod parse_options; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/parse_options.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/parse_options.rs new file mode 100644 index 0000000000..773f7acad5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/polyglot/parse_options.rs @@ -0,0 +1,15 @@ +//! The Polyglot parse options SQLBuild's wheel proxy applies to every parse. + +use polyglot_sql::{ComplexityGuardOptions, ParseOptions}; + +use crate::lineage::constants::MAX_FUNCTION_CALL_DEPTH; + +/// The proxy's complexity guard, decoded as the wheel decodes it. +pub(crate) fn guarded_parse_options() -> Result { + let guard: ComplexityGuardOptions = serde_json::from_value(serde_json::json!({ + "maxFunctionCallDepth": MAX_FUNCTION_CALL_DEPTH, + }))?; + Ok(ParseOptions { + complexity_guard: Some(guard), + }) +} diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/check_independence.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/check_independence.rs new file mode 100644 index 0000000000..32e895995e --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/check_independence.rs @@ -0,0 +1,265 @@ +//! Expected results and assertions must not define or read each other, even through helpers. + +use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; + +use polyglot_sql::{Dialect, DialectType, Expression, ExpressionWalk}; +use serde_json::Value; +use sqlbuild_sqltext::sql_scan::main::dialect_non_code_end::dialect_non_code_end; +use sqlbuild_sqltext::sql_scan::models::{LexicalSyntax, Unclosed}; + +use crate::compiler::_helpers::polyglot::parse_options::guarded_parse_options; +use crate::compiler::constants::{ALIAS_FIELD, CTES_FIELD, NAME_FIELD}; + +const DIRECT_DEPENDENCY_PATH_LENGTH: usize = 2; +const TABLE_KIND: &str = "table"; +const IDENTIFIER_QUOTES: &[u8] = b"\"`"; + +/// The check CTEs of one file and the labels its messages name. +pub(crate) struct IndependenceCheck<'a> { + pub(crate) ctes: &'a [(String, String)], + pub(crate) expected_prefix: &'a str, + pub(crate) assertion_prefix: &'a str, + pub(crate) file_label: &'a str, + pub(crate) context_label: &'a str, +} + +/// The first independence error of `check`, if any. +pub(crate) fn independence_error( + check: &IndependenceCheck<'_>, + syntax: &LexicalSyntax, +) -> Option { + let mut names_by_key: HashMap = HashMap::new(); + for (name, _) in check.ctes { + names_by_key.insert(casefold(name), name); + } + let keys_with_prefix = |prefix: &str| -> BTreeSet { + names_by_key + .iter() + .filter(|(_, name)| name.starts_with(prefix)) + .map(|(key, _)| key.clone()) + .collect() + }; + let expected: BTreeSet = keys_with_prefix(check.expected_prefix); + let assertions: BTreeSet = keys_with_prefix(check.assertion_prefix); + for (name, body) in check.ctes { + let key: String = casefold(name); + let (prohibited_prefix, prohibited_label) = if expected.contains(&key) { + (check.assertion_prefix, "assertion") + } else if assertions.contains(&key) { + (check.expected_prefix, "expected result") + } else { + continue; + }; + if !casefold(body).contains(&casefold(prohibited_prefix)) { + continue; + } + if let Some(nested) = defined_cte_names(body) + .into_iter() + .find(|nested| nested.starts_with(prohibited_prefix)) + { + return Some(format!( + "{} '{}' check CTE '{name}' must not define {prohibited_label} CTE '{nested}'; \ + expected results and assertions must be independent", + check.context_label, check.file_label + )); + } + } + if expected.is_empty() || assertions.is_empty() { + return None; + } + let mut dependencies: HashMap> = HashMap::new(); + for (name, body) in check.ctes { + let references = match known_cte_references(body, &names_by_key, syntax) { + Ok(references) => references, + Err(message) => return Some(message), + }; + dependencies.insert(casefold(name), references); + } + for (origins, prohibited) in [(&expected, &assertions), (&assertions, &expected)] { + for origin in origins { + let Some(path) = dependency_path(origin, prohibited, &dependencies) else { + continue; + }; + let rendered: Vec<&str> = path.iter().map(|key| names_by_key[key]).collect(); + let through: String = if rendered.len() > DIRECT_DEPENDENCY_PATH_LENGTH { + format!( + " through {}", + rendered[1..rendered.len() - 1] + .iter() + .map(|name| format!("'{name}'")) + .collect::>() + .join(" -> ") + ) + } else { + String::new() + }; + return Some(format!( + "{} '{}' check CTE '{}' must not depend on '{}'{through}; expected results and \ + assertions must be independent", + check.context_label, + check.file_label, + rendered[0], + rendered[rendered.len() - 1] + )); + } + } + None +} + +/// `str.casefold`, through Rust's Unicode lower-case mapping. +fn casefold(text: &str) -> String { + text.to_lowercase() +} + +/// `parse_one(sql, dialect="generic")` under SQLBuild's complexity guard, or `None` on error. +fn parse_one(sql: &str) -> Option { + let Ok(options) = guarded_parse_options() else { + return None; + }; + let Ok(mut statements) = Dialect::get(DialectType::Generic).parse_with_options(sql, &options) + else { + return None; + }; + (statements.len() == 1).then(|| statements.remove(0)) +} + +/// Every CTE alias defined anywhere in `sql`, in the order `to_dict()` lists them. +fn defined_cte_names(sql: &str) -> Vec { + let Some(parsed) = parse_one(sql) else { + return Vec::new(); + }; + let Ok(value) = serde_json::to_value(&parsed) else { + return Vec::new(); + }; + collect_cte_names(&value, Vec::new()) +} + +/// `names` followed by each new CTE alias under `value`, depth first. +fn collect_cte_names(value: &Value, mut names: Vec) -> Vec { + match value { + Value::Object(fields) => { + if let Some(Value::Array(ctes)) = fields.get(CTES_FIELD) { + for cte in ctes { + if let Some(Value::String(name)) = cte + .get(ALIAS_FIELD) + .filter(|alias| alias.is_object()) + .and_then(|alias| alias.get(NAME_FIELD)) + && !names.contains(name) + { + names.push(name.clone()); + } + } + } + fields + .values() + .fold(names, |names, child| collect_cte_names(child, names)) + } + Value::Array(items) => items + .iter() + .fold(names, |names, item| collect_cte_names(item, names)), + _ => names, + } +} + +/// The keys of the file's CTEs that `sql` reads, in reading order. +fn known_cte_references( + sql: &str, + names_by_key: &HashMap, + syntax: &LexicalSyntax, +) -> Result, String> { + let folded: String = casefold(sql); + if !names_by_key.keys().any(|key| folded.contains(key.as_str())) { + return Ok(Vec::new()); + } + let Some(parsed) = parse_one(sql) else { + return identifier_references(sql, names_by_key, syntax); + }; + let mut references: Vec = Vec::new(); + for table in parsed + .dfs() + .skip(1) + .filter(|node| node.variant_name() == TABLE_KIND) + { + references = push_known(references, casefold(table.get_name()), names_by_key); + } + Ok(references) +} + +/// The identifier scan used when Polyglot cannot parse a body. +fn identifier_references( + sql: &str, + names_by_key: &HashMap, + syntax: &LexicalSyntax, +) -> Result, String> { + let bytes: &[u8] = sql.as_bytes(); + let mut references: Vec = Vec::new(); + let mut index: usize = 0; + while let Some(character) = sql[index..].chars().next() { + match dialect_non_code_end(bytes, index, syntax) { + Err(construct) => return Err(unclosed_message(construct)), + Ok(Some(end)) if IDENTIFIER_QUOTES.contains(&bytes[index]) => { + let quote: &str = &sql[index..=index]; + let name: String = sql[index + 1..end - 1].replace("e.repeat(2), quote); + references = push_known(references, casefold(&name), names_by_key); + index = end; + } + Ok(Some(end)) => index = end, + Ok(None) if character.is_alphabetic() || character == '_' => { + let end: usize = sql[index..] + .char_indices() + .skip(1) + .find(|(_, next)| !(next.is_alphanumeric() || *next == '_')) + .map_or(sql.len(), |(offset, _)| index + offset); + references = push_known(references, casefold(&sql[index..end]), names_by_key); + index = end; + } + Ok(None) => index += character.len_utf8(), + } + } + Ok(references) +} + +/// `references` with `key` appended when it names a file CTE not yet read. +fn push_known( + mut references: Vec, + key: String, + names_by_key: &HashMap, +) -> Vec { + if names_by_key.contains_key(&key) && !references.contains(&key) { + references.push(key); + } + references +} + +fn unclosed_message(construct: Unclosed) -> String { + let construct: &str = match construct { + Unclosed::BlockComment => "block comment", + Unclosed::Quote => "quoted string", + Unclosed::Parenthesis => "parenthesis", + }; + format!("SQL test contains an unclosed {construct}") +} + +/// The shortest dependency path from `origin` to a prohibited CTE, breadth first. +fn dependency_path( + origin: &str, + prohibited: &BTreeSet, + dependencies: &HashMap>, +) -> Option> { + let mut pending: VecDeque> = VecDeque::from([vec![origin.to_owned()]]); + let mut visited: HashSet = HashSet::from([origin.to_owned()]); + while let Some(path) = pending.pop_front() { + let last: &String = path.last()?; + for dependency in dependencies.get(last).into_iter().flatten() { + let mut candidate: Vec = path.clone(); + candidate.push(dependency.clone()); + if prohibited.contains(dependency) { + return Some(candidate); + } + if visited.insert(dependency.clone()) { + pending.push_back(candidate); + } + } + } + None +} diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mock_reads.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mock_reads.rs index f381916286..5e9793b5e8 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mock_reads.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mock_reads.rs @@ -253,9 +253,7 @@ fn references(sql: &str, syntax: &LexicalSyntax) -> Deferrable> { ReferenceExtraction::Extracted(scan) => { Ok(scan.references.into_iter().map(scanned).collect()) } - ReferenceExtraction::Failed(_) | ReferenceExtraction::Deferred => { - Err(SqlTestAssemblyDeferral::ReferenceScan) - } + ReferenceExtraction::Failed(_) => Err(SqlTestAssemblyDeferral::ReferenceScan), } } diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs index 2661b0ec0a..7207851e88 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/mod.rs @@ -1,6 +1,7 @@ pub mod assembly; pub mod authored_macro_calls; pub mod case_identity; +pub(crate) mod check_independence; pub mod cte_rename; pub mod cte_slices; pub mod cte_sql; diff --git a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs index fd8c6488b1..9a2d1fcf0c 100644 --- a/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs +++ b/crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/scenario_extraction.rs @@ -1,12 +1,14 @@ -//! Python's `extract_sql_scenario_ctes`: the exact top-level scan, then classification and errors. - -use std::collections::{HashMap, HashSet}; +//! `extract_sql_scenario_ctes`: the top-level scan, the independence check, then classification. use serde::Serialize; use sqlbuild_scopes::relationship_names::main::top_level_ctes::scan_top_level_ctes; use sqlbuild_scopes::relationship_names::models::{RelationshipSource, TopLevelCtes}; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; +use crate::compiler::_helpers::sql_tests::check_independence::{ + IndependenceCheck, independence_error, +}; + type Cte = (String, String); const SOURCE_PREFIX: &str = "__source__"; @@ -16,6 +18,7 @@ const DBT_REF_PREFIX: &str = "__dbt_ref__"; const EXPECTED_PREFIX: &str = "__expected__"; const ASSERT_PREFIX: &str = "__assert__"; const MACRO_PREFIX: &str = "__macro__"; +const SCENARIO_CONTEXT: &str = "SQL scenario"; #[derive(Debug, Default, Serialize)] #[serde(rename_all = "camelCase")] @@ -31,96 +34,52 @@ struct ClassifiedScenario { assertion_names: Vec, } -/// One scenario's error, or its classification after any Polyglot independence check. +/// One scenario's error, or its classification. #[derive(Debug, Default, Serialize)] #[serde(rename_all = "camelCase")] struct ScenarioOutcome { - #[serde(skip_serializing_if = "Option::is_none")] - independence: Option>, #[serde(skip_serializing_if = "Option::is_none")] error: Option, #[serde(skip_serializing_if = "Option::is_none")] scenario: Option, } +/// The scenario's classification or error, as JSON; independence errors come first. pub(crate) fn extract_scenario_json( sql: &str, file: &str, syntax: &LexicalSyntax, -) -> Result, String> { +) -> Result { let outcome: ScenarioOutcome = match scan_top_level_ctes(sql, file, RelationshipSource::Scenario, syntax) { - TopLevelCtes::Deferred => return Ok(None), TopLevelCtes::Failed(message) => ScenarioOutcome { error: Some(message), ..ScenarioOutcome::default() }, TopLevelCtes::Scanned(ctes) => { - let independence: Option> = - python_checks_independence(&ctes).then(|| ctes.clone()); - match classify(ctes, file) { + let independence: Option = independence_error( + &IndependenceCheck { + ctes: &ctes, + expected_prefix: EXPECTED_PREFIX, + assertion_prefix: ASSERT_PREFIX, + file_label: file, + context_label: SCENARIO_CONTEXT, + }, + syntax, + ); + match independence.map_or_else(|| classify(ctes, file), Err) { Ok(scenario) => ScenarioOutcome { - independence, scenario: Some(scenario), ..ScenarioOutcome::default() }, Err(message) => ScenarioOutcome { - independence, error: Some(message), ..ScenarioOutcome::default() }, } } }; - serde_json::to_string(&outcome) - .map(Some) - .map_err(|error| error.to_string()) -} - -/// Whether Python's independence check parses a body; when it does not, the check passes. -fn python_checks_independence(ctes: &[Cte]) -> bool { - let mut names_by_key: HashMap = HashMap::new(); - for (name, _) in ctes { - names_by_key.insert(name.to_ascii_lowercase(), name); - } - let keys_with_prefix = |prefix: &str| -> HashSet<&String> { - names_by_key - .iter() - .filter(|(_, name)| name.starts_with(prefix)) - .map(|(key, _)| key) - .collect() - }; - let expected: HashSet<&String> = keys_with_prefix(EXPECTED_PREFIX); - let assertions: HashSet<&String> = keys_with_prefix(ASSERT_PREFIX); - for (name, body) in ctes { - let key: String = name.to_ascii_lowercase(); - let prohibited: &str = if expected.contains(&key) { - ASSERT_PREFIX - } else if assertions.contains(&key) { - EXPECTED_PREFIX - } else { - continue; - }; - if folded_body_may_contain(body, prohibited) { - return true; - } - } - if expected.is_empty() || assertions.is_empty() { - return false; - } - for (_, body) in ctes { - for key in names_by_key.keys() { - if folded_body_may_contain(body, key) { - return true; - } - } - } - false -} - -/// Whether the body's Python `casefold` may contain the text; non-ASCII may fold onto ASCII. -fn folded_body_may_contain(body: &str, lower_ascii: &str) -> bool { - !body.is_ascii() || body.to_ascii_lowercase().contains(lower_ascii) + serde_json::to_string(&outcome).map_err(|error| error.to_string()) } fn classify(ctes: Vec, file: &str) -> Result { diff --git a/crates/sqlbuild-analysis/src/compiler/constants.rs b/crates/sqlbuild-analysis/src/compiler/constants.rs new file mode 100644 index 0000000000..179f2ae1b7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/constants.rs @@ -0,0 +1,15 @@ +//! Fixed names and tables of native compiler checks. + +use polyglot_sql::DialectType; + +/// The adapters whose function types are checked, with the dialect name and parser of each. +pub(crate) const FUNCTION_TYPE_DIALECTS: [(&str, &str, DialectType); 4] = [ + ("duckdb", "duckdb", DialectType::DuckDB), + ("bigquery", "bigquery", DialectType::BigQuery), + ("snowflake", "snowflake", DialectType::Snowflake), + ("databricks", "databricks", DialectType::Databricks), +]; +/// `to_dict()` fields the CTE alias walk reads. +pub(crate) const CTES_FIELD: &str = "ctes"; +pub(crate) const ALIAS_FIELD: &str = "alias"; +pub(crate) const NAME_FIELD: &str = "name"; diff --git a/crates/sqlbuild-analysis/src/compiler/main/function_type_validation.rs b/crates/sqlbuild-analysis/src/compiler/main/function_type_validation.rs new file mode 100644 index 0000000000..1673cafcd2 --- /dev/null +++ b/crates/sqlbuild-analysis/src/compiler/main/function_type_validation.rs @@ -0,0 +1,12 @@ +//! Function argument and return types checked against the adapter's SQL analysis dialect. + +use crate::compiler::_helpers; + +/// The error for a type the adapter's dialect cannot parse, or `Err` for a broken parse guard. +pub fn function_type_error( + type_sql: &str, + adapter_name: &str, + context: &str, +) -> Result, String> { + _helpers::polyglot::function_types::function_type_error(type_sql, adapter_name, context) +} diff --git a/crates/sqlbuild-analysis/src/compiler/main/mod.rs b/crates/sqlbuild-analysis/src/compiler/main/mod.rs index 94b6925b7e..bb4c16b14f 100644 --- a/crates/sqlbuild-analysis/src/compiler/main/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/main/mod.rs @@ -1,3 +1,4 @@ +pub mod function_type_validation; pub mod sql_scenario_extraction; pub mod sql_test_assembly; pub mod sql_test_chain_resolution; diff --git a/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_extraction.rs b/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_extraction.rs index ccf4589ca4..9c2d99e7af 100644 --- a/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_extraction.rs +++ b/crates/sqlbuild-analysis/src/compiler/main/sql_scenario_extraction.rs @@ -2,11 +2,11 @@ use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::compiler::_helpers; -/// Python's scenario extraction outcome as JSON, or `None` where Python must extract the scenario. +/// The scenario's classification or error as JSON; `Err` holds the JSON encoder's message. pub fn extract_scenario_json( sql: &str, file_label: &str, syntax: &LexicalSyntax, -) -> Result, String> { +) -> Result { _helpers::sql_tests::scenario_extraction::extract_scenario_json(sql, file_label, syntax) } diff --git a/crates/sqlbuild-analysis/src/compiler/mod.rs b/crates/sqlbuild-analysis/src/compiler/mod.rs index 6059132c3b..b9965bee44 100644 --- a/crates/sqlbuild-analysis/src/compiler/mod.rs +++ b/crates/sqlbuild-analysis/src/compiler/mod.rs @@ -1,4 +1,5 @@ pub mod _helpers; +pub(crate) mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_scenario_extraction.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_scenario_extraction.rs index 39606b5f89..363581a468 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_sql_scenario_extraction.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_sql_scenario_extraction.rs @@ -4,7 +4,7 @@ use crate::compiler::main::sql_scenario_extraction::extract_scenario_json; use crate::compiler::tests::test_types::ScenarioExtractionTestCase; #[test] -fn given_scenarios_when_extracting_natively_then_python_outcomes_or_deferrals_are_returned() { +fn given_scenarios_when_extracting_natively_then_outcomes_are_returned() { let syntax = LexicalSyntax { line_comment_prefixes: vec!["--".to_owned()], ..LexicalSyntax::default() @@ -14,85 +14,79 @@ fn given_scenarios_when_extracting_natively_then_python_outcomes_or_deferrals_ar description: "fixtures, helpers and expected results without cross references", sql: "WITH helper AS (SELECT 1 AS id), __source__raw_orders AS (SELECT 2 AS id), \ __expected__orders AS (SELECT 1 AS id) SELECT 1", - expected_json: Some( - r#"{"scenario":{"authored":[["helper","SELECT 1 AS id"],["__source__raw_orders","SELECT 2 AS id"]],"expected":[["__expected__orders","SELECT 1 AS id"]],"assertions":[],"sourceFixtures":["raw_orders"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":["orders"],"assertionNames":[]}}"#, - ), + expected_json: r#"{"scenario":{"authored":[["helper","SELECT 1 AS id"],["__source__raw_orders","SELECT 2 AS id"]],"expected":[["__expected__orders","SELECT 1 AS id"]],"assertions":[],"sourceFixtures":["raw_orders"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":["orders"],"assertionNames":[]}}"#, }, ScenarioExtractionTestCase { - description: "expected results and assertions reading a CTE leave independence to Python", + description: "expected results and assertions reading a CTE pass the independence check", sql: "WITH helper AS (SELECT 1 AS id), __source__raw AS (SELECT id FROM helper), \ __expected__orders AS (SELECT 1 AS id), __assert__positive AS (SELECT 1) SELECT 1", - expected_json: Some( - r#"{"independence":[["helper","SELECT 1 AS id"],["__source__raw","SELECT id FROM helper"],["__expected__orders","SELECT 1 AS id"],["__assert__positive","SELECT 1"]],"scenario":{"authored":[["helper","SELECT 1 AS id"],["__source__raw","SELECT id FROM helper"]],"expected":[["__expected__orders","SELECT 1 AS id"]],"assertions":[["__assert__positive","SELECT 1"]],"sourceFixtures":["raw"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":["orders"],"assertionNames":["positive"]}}"#, - ), + expected_json: r#"{"scenario":{"authored":[["helper","SELECT 1 AS id"],["__source__raw","SELECT id FROM helper"]],"expected":[["__expected__orders","SELECT 1 AS id"]],"assertions":[["__assert__positive","SELECT 1"]],"sourceFixtures":["raw"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":["orders"],"assertionNames":["positive"]}}"#, }, ScenarioExtractionTestCase { - description: "an expected result naming the assertion prefix leaves independence to Python", + description: "an expected result naming the assertion prefix passes the independence check", sql: "WITH __source__raw AS (SELECT 1), __expected__orders AS (SELECT '__ASSERT__' AS x) \ SELECT 1", - expected_json: Some( - r#"{"independence":[["__source__raw","SELECT 1"],["__expected__orders","SELECT '__ASSERT__' AS x"]],"scenario":{"authored":[["__source__raw","SELECT 1"]],"expected":[["__expected__orders","SELECT '__ASSERT__' AS x"]],"assertions":[],"sourceFixtures":["raw"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":["orders"],"assertionNames":[]}}"#, - ), + expected_json: r#"{"scenario":{"authored":[["__source__raw","SELECT 1"]],"expected":[["__expected__orders","SELECT '__ASSERT__' AS x"]],"assertions":[],"sourceFixtures":["raw"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":["orders"],"assertionNames":[]}}"#, }, ScenarioExtractionTestCase { description: "a macro mock raises Python's error", sql: "WITH __source__raw AS (SELECT 1), __macro__tidy AS (SELECT 'x'), \ __expected__orders AS (SELECT 1 AS id) SELECT 1", - expected_json: Some( - r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' does not support macro mock CTE '__macro__tidy'. Scenarios run real project macros; use SQL unit tests for macro mocks."}"#, - ), + expected_json: r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' does not support macro mock CTE '__macro__tidy'. Scenarios run real project macros; use SQL unit tests for macro mocks."}"#, }, ScenarioExtractionTestCase { description: "a bare fixture prefix raises Python's error before later CTEs", sql: "WITH __seed__ AS (SELECT 1), __macro__tidy AS (SELECT 'x') SELECT 1", - expected_json: Some( - r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' must use __seed__ to identify a target"}"#, - ), + expected_json: r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' must use __seed__ to identify a target"}"#, }, ScenarioExtractionTestCase { description: "a scenario without fixtures raises Python's error", sql: "WITH __expected__orders AS (SELECT 1 AS id) SELECT 1", - expected_json: Some( - r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' must define at least one __source__*, __ref__*, __seed__*, or __dbt_ref__* fixture CTE"}"#, - ), + expected_json: r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' must define at least one __source__*, __ref__*, __seed__*, or __dbt_ref__* fixture CTE"}"#, }, ScenarioExtractionTestCase { description: "a scenario without checks raises Python's error", sql: "WITH __dbt_ref__shop__orders AS (SELECT 1 AS id)", - expected_json: Some( - r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' must define at least one __expected__ or __assert__ CTE"}"#, - ), + expected_json: r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' must define at least one __expected__ or __assert__ CTE"}"#, }, ScenarioExtractionTestCase { description: "a quoted CTE name is Python's scanner error", sql: "WITH \"__source__raw\" AS (SELECT 1), __expected__orders AS (SELECT 1 AS id)", - expected_json: Some( - r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' expected a CTE name"}"#, - ), + expected_json: r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' expected a CTE name"}"#, }, ScenarioExtractionTestCase { description: "an unclosed body is Python's scanner error naming the scenario context", sql: "WITH __source__raw AS (SELECT 1", - expected_json: Some(r#"{"error":"SQL scenario contains an unclosed parenthesis"}"#), + expected_json: r#"{"error":"SQL scenario contains an unclosed parenthesis"}"#, }, ScenarioExtractionTestCase { description: "a materialization hint is Python's scanner error", sql: "WITH __source__raw AS NOT MATERIALIZED (SELECT 1), __expected__orders AS (SELECT 1)", - expected_json: Some( - r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' CTE '__source__raw' must not use AS NOT MATERIALIZED; materialization hints are not supported in SQL scenario CTEs"}"#, - ), + expected_json: r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' CTE '__source__raw' must not use AS NOT MATERIALIZED; materialization hints are not supported in SQL scenario CTEs"}"#, }, ScenarioExtractionTestCase { - description: "a keyword Python would match by case mapping defers", + description: "a keyword matched by case mapping reads as the keyword", sql: "w\u{131}th __source__raw AS (SELECT 1), __expected__orders AS (SELECT 1 AS id)", - expected_json: None, + expected_json: r#"{"scenario":{"authored":[["__source__raw","SELECT 1"]],"expected":[["__expected__orders","SELECT 1 AS id"]],"assertions":[],"sourceFixtures":["raw"],"refFixtures":[],"seedFixtures":[],"dbtRefFixtures":[],"expectedModels":["orders"],"assertionNames":[]}}"#, + }, + ScenarioExtractionTestCase { + description: "an expected result reading an assertion through a helper fails", + sql: "WITH __source__raw AS (SELECT 1), helper AS (SELECT * FROM __assert__positive), \ + __expected__orders AS (SELECT * FROM helper), __assert__positive AS (SELECT 1) SELECT 1", + expected_json: r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' check CTE '__expected__orders' must not depend on '__assert__positive' through 'helper'; expected results and assertions must be independent"}"#, + }, + ScenarioExtractionTestCase { + description: "an assertion defining a nested expected result fails", + sql: "WITH __source__raw AS (SELECT 1), __expected__orders AS (SELECT 1), \ + __assert__positive AS (WITH __expected__x AS (SELECT 1) SELECT * FROM __expected__x) SELECT 1", + expected_json: r#"{"error":"SQL scenario 'tests/scenarios/orders.sql' check CTE '__assert__positive' must not define expected result CTE '__expected__x'; expected results and assertions must be independent"}"#, }, ]; for test_case in test_cases { assert_eq!( extract_scenario_json(test_case.sql, "tests/scenarios/orders.sql", &syntax), - Ok(test_case.expected_json.map(str::to_owned)), + Ok(test_case.expected_json.to_owned()), "{}", test_case.description ); diff --git a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs index 3d59ff16d4..c348ab5536 100644 --- a/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/compiler/tests/test_types.rs @@ -136,7 +136,7 @@ pub(crate) struct HelperReferenceTestCase { pub(crate) struct ScenarioExtractionTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, - pub(crate) expected_json: Option<&'static str>, + pub(crate) expected_json: &'static str, } pub(crate) struct UnresolvedReaderReferenceTestCase { diff --git a/crates/sqlbuild-attachments/src/audits/_helpers/parameters.rs b/crates/sqlbuild-attachments/src/audits/_helpers/parameters.rs index 586f51d388..ba7f5ea542 100644 --- a/crates/sqlbuild-attachments/src/audits/_helpers/parameters.rs +++ b/crates/sqlbuild-attachments/src/audits/_helpers/parameters.rs @@ -1,5 +1,7 @@ //! Python's raw and quoted SQL argument patterns and argument value rendering. +use sqlbuild_core::text::main::is_python_space::is_python_space; + use crate::audits::models::ArgumentValue; /// One `@name` or `@'name'` occurrence, in byte offsets. @@ -10,8 +12,8 @@ pub(crate) struct ParameterMatch<'sql> { pub(crate) quoted: bool, } -/// Every parameter in start order, or None where Python's Unicode whitespace could decide. -pub(crate) fn parameter_matches(sql: &str) -> Option>> { +/// Every parameter in start order. +pub(crate) fn parameter_matches(sql: &str) -> Vec> { let bytes: &[u8] = sql.as_bytes(); let mut matches: Vec> = Vec::new(); let mut quoted_end: usize = 0; @@ -47,7 +49,7 @@ pub(crate) fn parameter_matches(sql: &str) -> Option>> { let Some(end) = identifier_end(bytes, index + 1) else { continue; }; - if !opens_call(bytes, end)? { + if !opens_call(sql, end) { matches.push(ParameterMatch { name: &sql[index + 1..end], start: index, @@ -57,13 +59,13 @@ pub(crate) fn parameter_matches(sql: &str) -> Option>> { raw_end = end; } } - Some(matches) + matches } /// Python's `render_sql_argument_value` for one value; `None` where it holds an opaque value. pub(crate) fn render_value(value: &ArgumentValue, quoted: bool) -> Option { Some(match value { - ArgumentValue::List(items) => items + ArgumentValue::List(items) | ArgumentValue::Tuple(items) => items .iter() .map(|item| render_value(item, quoted)) .collect::>>()? @@ -94,46 +96,28 @@ fn identifier_end(bytes: &[u8], start: usize) -> Option { Some(end) } -/// Python's `(?!\s*\()` after a name; non-ASCII whitespace candidates defer. -fn opens_call(bytes: &[u8], mut index: usize) -> Option { - while let Some(byte) = bytes.get(index) { - if !byte.is_ascii() { - return None; - } - if !matches!( - byte, - b' ' | b'\t' | b'\n' | b'\r' | 0x0b | 0x0c | 0x1c..=0x1f - ) { - return Some(*byte == b'('); - } - index += 1; - } - Some(false) +/// Python's `(?=\s*\()` after a name, with `\s` matching what `str.isspace` accepts. +fn opens_call(sql: &str, end: usize) -> bool { + sql[end..] + .chars() + .find(|character| !is_python_space(*character)) + == Some('(') } -/// Why rendering stops: Python's missing-argument or value error, or input Python must judge. +/// Why rendering stops: a missing argument or a value that cannot be rendered. #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum RenderStop { MissingArgument(String), - /// The argument whose value Python cannot render. + /// The argument whose value cannot be rendered. UnsupportedValue(String), - Deferred, } -/// Python's `render_parameterized_sql`; unused arguments under `reject_unused` defer. +/// `render_parameterized_sql` without `reject_unused`. pub(crate) fn render_parameterized_sql( sql: &str, arguments: &[(String, ArgumentValue)], - reject_unused: bool, ) -> Result { - let matches = parameter_matches(sql).ok_or(RenderStop::Deferred)?; - if reject_unused { - for (name, _) in arguments { - if !matches.iter().any(|item| item.name == name.as_str()) { - return Err(RenderStop::Deferred); - } - } - } + let matches = parameter_matches(sql); let mut rendered: String = String::with_capacity(sql.len()); let mut previous_end: usize = 0; for item in &matches { diff --git a/crates/sqlbuild-attachments/src/audits/_helpers/rendering.rs b/crates/sqlbuild-attachments/src/audits/_helpers/rendering.rs index cc794952a2..5e9481fe81 100644 --- a/crates/sqlbuild-attachments/src/audits/_helpers/rendering.rs +++ b/crates/sqlbuild-attachments/src/audits/_helpers/rendering.rs @@ -31,89 +31,44 @@ pub(crate) fn policies( Ok((severity, run_scope_source)) } -/// One SQL text rendered, Python's missing-argument error, or None to defer. +/// One SQL text rendered, or the missing-argument or unsupported-value error. pub(crate) fn rendered( attachment: &AuditAttachment, sql: &str, arguments: &[(String, ArgumentValue)], -) -> Option> { - match render_parameterized_sql(sql, arguments, false) { - Ok(rendered) => Some(Ok(rendered)), - Err(RenderStop::MissingArgument(name)) => Some(Err(format!( +) -> Result { + render_parameterized_sql(sql, arguments).map_err(|stop| match stop { + RenderStop::MissingArgument(name) => format!( "{} is missing argument '{name}' for generic audit '{}'", attachment.owner_label, attachment.definition_name - ))), - Err(RenderStop::UnsupportedValue(name)) => Some(Err(format!( + ), + RenderStop::UnsupportedValue(name) => format!( "{} generic audit '{}' argument '{name}' uses an unsupported value", attachment.owner_label, attachment.definition_name - ))), - Err(RenderStop::Deferred) => None, - } + ), + }) } -/// Python's `merge_audit_arguments`; None where only Python's `!=` can compare the values. +/// `merge_audit_arguments`: an explicit argument may repeat an implicit one only as equal text. pub(crate) fn merged_arguments( attachment: &AuditAttachment, -) -> Option, String>> { - let mut merged: Vec<(String, ArgumentValue)> = attachment.implicit_arguments.clone(); +) -> Result, String> { + let mut merged: Vec<(String, ArgumentValue)> = attachment + .implicit_arguments + .iter() + .map(|(name, value)| (name.clone(), ArgumentValue::Text(value.clone()))) + .collect(); for (name, value) in &attachment.explicit_arguments { - match merged.iter_mut().find(|(existing, _)| existing == name) { - Some((_, existing)) => { - if python_unequal(existing, value)? { - return Some(Err(format!( - "{} audit '{}' must not override implicit {name} from attached context", - attachment.owner_label, attachment.definition_name - ))); - } + match merged.iter().find(|(existing, _)| existing == name) { + Some((_, existing)) if existing != value => { + return Err(format!( + "{} audit '{}' must not override implicit {name} from attached context", + attachment.owner_label, attachment.definition_name + )); } + Some(_) => {} None => merged.push((name.clone(), value.clone())), } } - Some(Ok(merged)) -} - -/// Python's `left != right`, or None for nested opaque values and numbers differing in text. -fn python_unequal(left: &ArgumentValue, right: &ArgumentValue) -> Option { - if contains_opaque(left) || contains_opaque(right) { - return None; - } - if left == right { - return Some(false); - } - match (left, right) { - (ArgumentValue::List(left), ArgumentValue::List(right)) => { - if left.len() != right.len() { - return Some(true); - } - let mut undecided: bool = false; - for (left, right) in left.iter().zip(right) { - match python_unequal(left, right) { - Some(true) => return Some(true), - Some(false) => {} - None => undecided = true, - } - } - if undecided { None } else { Some(false) } - } - ( - ArgumentValue::Number(_) | ArgumentValue::Boolean(_), - ArgumentValue::Number(_) | ArgumentValue::Boolean(_), - ) if !matches!( - (left, right), - (ArgumentValue::Boolean(_), ArgumentValue::Boolean(_)) - ) => - { - None - } - _ => Some(true), - } -} - -/// Whether `value` holds an opaque value at any depth, which only Python can compare. -fn contains_opaque(value: &ArgumentValue) -> bool { - match value { - ArgumentValue::Opaque => true, - ArgumentValue::List(items) => items.iter().any(contains_opaque), - _ => false, - } + Ok(merged) } diff --git a/crates/sqlbuild-attachments/src/audits/main/render_attached_audit.rs b/crates/sqlbuild-attachments/src/audits/main/render_attached_audit.rs index 832440a26a..e26aaed35c 100644 --- a/crates/sqlbuild-attachments/src/audits/main/render_attached_audit.rs +++ b/crates/sqlbuild-attachments/src/audits/main/render_attached_audit.rs @@ -1,36 +1,27 @@ -//! Python's attached generic audit rendering: argument merge, SQL, severity and run scope. +//! Attached generic audit rendering: argument merge, SQL, severity and run scope. use crate::audits::_helpers::rendering::{merged_arguments, policies, rendered}; use crate::audits::models::{ArgumentValue, AuditAttachment, AuditRendering, RenderedAudit}; -/// Render one attachment as Python does, with its errors, or None where Python must decide. +/// Render one attachment, or the error rendering raises. #[must_use] -pub fn render_attached_audit(attachment: &AuditAttachment) -> Option { - if attachment.measurement { - if attachment.instance_severity.is_some() || !attachment.has_thresholds { - return None; - } - } else if attachment.has_thresholds || attachment.has_minimum_samples { - return None; - } - let arguments: Vec<(String, ArgumentValue)> = match merged_arguments(attachment)? { - Ok(arguments) => arguments, - Err(message) => return Some(AuditRendering::Failed(message)), - }; - let sql_body: String = match rendered(attachment, &attachment.sql_body, &arguments)? { - Ok(sql) => sql, - Err(message) => return Some(AuditRendering::Failed(message)), +pub fn render_attached_audit(attachment: &AuditAttachment) -> AuditRendering { + let rendering = || -> Result { + let arguments: Vec<(String, ArgumentValue)> = merged_arguments(attachment)?; + let sql_body: String = rendered(attachment, &attachment.sql_body, &arguments)?; + let evidence_sql: Option = attachment + .evidence_sql + .as_deref() + .map(|evidence| rendered(attachment, evidence, &arguments)) + .transpose()?; + Ok(RenderedAudit { + sql_body, + evidence_sql, + policies: policies(attachment), + }) }; - let evidence_sql: Option = match &attachment.evidence_sql { - Some(evidence) => match rendered(attachment, evidence, &arguments)? { - Ok(sql) => Some(sql), - Err(message) => return Some(AuditRendering::Failed(message)), - }, - None => None, - }; - Some(AuditRendering::Rendered(RenderedAudit { - sql_body, - evidence_sql, - policies: policies(attachment), - })) + match rendering() { + Ok(rendered) => AuditRendering::Rendered(rendered), + Err(message) => AuditRendering::Failed(message), + } } diff --git a/crates/sqlbuild-attachments/src/audits/models.rs b/crates/sqlbuild-attachments/src/audits/models.rs index cb99f154c1..9b5a709349 100644 --- a/crates/sqlbuild-attachments/src/audits/models.rs +++ b/crates/sqlbuild-attachments/src/audits/models.rs @@ -8,7 +8,9 @@ pub enum ArgumentValue { Number(String), Text(String), List(Vec), - /// A value Python cannot render, such as a mapping; rendering it is Python's error. + /// A tuple renders like a list. + Tuple(Vec), + /// A value that cannot be rendered, such as a mapping; rendering it is an error. Opaque, } @@ -28,11 +30,10 @@ pub struct AuditAttachment { pub definition_name: String, pub sql_body: String, pub evidence_sql: Option, - pub implicit_arguments: Vec<(String, ArgumentValue)>, + /// Arguments from the attached resource, such as `model` and `column`; always text. + pub implicit_arguments: Vec<(String, String)>, pub explicit_arguments: Vec<(String, ArgumentValue)>, - pub measurement: bool, pub has_thresholds: bool, - pub has_minimum_samples: bool, pub threshold_error: bool, pub instance_severity: Option, pub default_severity: Option, diff --git a/crates/sqlbuild-attachments/src/audits/tests/helpers.rs b/crates/sqlbuild-attachments/src/audits/tests/helpers.rs index cff143ce9e..c41c742aa8 100644 --- a/crates/sqlbuild-attachments/src/audits/tests/helpers.rs +++ b/crates/sqlbuild-attachments/src/audits/tests/helpers.rs @@ -23,8 +23,8 @@ pub(super) fn order_arguments() -> Vec<(String, ArgumentValue)> { ] } -pub(super) fn rendered_sql(sql: &str, reject_unused: bool) -> Result { - render_parameterized_sql(sql, &order_arguments(), reject_unused) +pub(super) fn rendered_sql(sql: &str) -> Result { + render_parameterized_sql(sql, &order_arguments()) } /// A violations audit over `@column` with the given explicit arguments and policies. @@ -38,14 +38,9 @@ pub(super) fn attachment( definition_name: "accepted_values".to_owned(), sql_body: "SELECT * FROM t WHERE @column NOT IN (@'values')".to_owned(), evidence_sql: Some("SELECT @column FROM t".to_owned()), - implicit_arguments: vec![( - "column".to_owned(), - ArgumentValue::Text("status".to_owned()), - )], + implicit_arguments: vec![("column".to_owned(), "status".to_owned())], explicit_arguments: explicit, - measurement: false, has_thresholds: false, - has_minimum_samples: false, threshold_error: false, instance_severity: severity.map(str::to_owned), default_severity: None, @@ -58,14 +53,14 @@ pub(super) fn attachment( pub(super) fn rendered( sql_body: &str, policies: Result<(&'static str, PolicySource), &str>, -) -> Option { - Some(AuditRendering::Rendered(RenderedAudit { +) -> AuditRendering { + AuditRendering::Rendered(RenderedAudit { sql_body: sql_body.to_owned(), evidence_sql: Some("SELECT status FROM t".to_owned()), policies: policies.map_err(str::to_owned), - })) + }) } -pub(super) fn failed(message: &str) -> Option { - Some(AuditRendering::Failed(message.to_owned())) +pub(super) fn failed(message: &str) -> AuditRendering { + AuditRendering::Failed(message.to_owned()) } diff --git a/crates/sqlbuild-attachments/src/audits/tests/test_render_attached_audit.rs b/crates/sqlbuild-attachments/src/audits/tests/test_render_attached_audit.rs index 6792966c6e..9b571785ff 100644 --- a/crates/sqlbuild-attachments/src/audits/tests/test_render_attached_audit.rs +++ b/crates/sqlbuild-attachments/src/audits/tests/test_render_attached_audit.rs @@ -42,37 +42,43 @@ fn given_attached_audits_when_rendering_then_python_rendering_is_returned() { ), }, AttachedAuditTestCase { - description: "an override Python's equality must judge defers", - attachment: AuditAttachment { - implicit_arguments: vec![( - "values".to_owned(), - ArgumentValue::Number("1".to_owned()), - )], - ..attachment( - vec![("values".to_owned(), ArgumentValue::Number("1.0".to_owned()))], - None, - None, - ) - }, - expected_rendering: None, + description: "an explicit argument repeating the implicit text is accepted", + attachment: attachment( + vec![ + ( + "column".to_owned(), + ArgumentValue::Text("status".to_owned()), + ), + ( + "values".to_owned(), + ArgumentValue::Tuple(vec![ + ArgumentValue::Text("a".to_owned()), + ArgumentValue::Number("inf".to_owned()), + ]), + ), + ], + None, + None, + ), + expected_rendering: rendered( + "SELECT * FROM t WHERE status NOT IN ('a', inf)", + Ok(("error", PolicySource::Fallback)), + ), }, AttachedAuditTestCase { - description: "an override holding a nested opaque value defers to Python's equality", - attachment: AuditAttachment { - implicit_arguments: vec![( - "values".to_owned(), - ArgumentValue::List(vec![ArgumentValue::Opaque]), + description: "an explicit argument repeating the implicit name with a list is an override", + attachment: attachment( + vec![( + "column".to_owned(), + ArgumentValue::List(vec![ArgumentValue::Text("status".to_owned())]), )], - ..attachment( - vec![( - "values".to_owned(), - ArgumentValue::List(vec![ArgumentValue::Opaque]), - )], - None, - None, - ) - }, - expected_rendering: None, + None, + None, + ), + expected_rendering: failed( + "models/schema.yml audit 'accepted_values' must not override \ + implicit column from attached context", + ), }, AttachedAuditTestCase { description: "a missing argument is Python's error before the policies", diff --git a/crates/sqlbuild-attachments/src/audits/tests/test_render_parameterized_sql.rs b/crates/sqlbuild-attachments/src/audits/tests/test_render_parameterized_sql.rs index 7b19079955..b3e1bc6736 100644 --- a/crates/sqlbuild-attachments/src/audits/tests/test_render_parameterized_sql.rs +++ b/crates/sqlbuild-attachments/src/audits/tests/test_render_parameterized_sql.rs @@ -8,7 +8,6 @@ fn given_parameterized_sql_when_rendering_then_python_text_is_returned() { ParameterizedSqlTestCase { description: "raw and quoted parameters: lists joined, only text quoted, quotes doubled", sql: "WHERE @column IN (@'values') AND x = @values", - reject_unused: false, expected_sql: Ok( "WHERE status IN ('it''s', 2, NULL, TRUE) AND x = it's, 2, NULL, TRUE", ), @@ -16,32 +15,23 @@ fn given_parameterized_sql_when_rendering_then_python_text_is_returned() { ParameterizedSqlTestCase { description: "macro calls, doubled @ and runtime placeholders are left alone", sql: "@column (x) @@column @@@column @column(y)", - reject_unused: false, expected_sql: Ok("@column (x) @@column @@@column @column(y)"), }, ParameterizedSqlTestCase { description: "the first missing argument stops rendering with its name", sql: "WHERE @column = @missing AND @'other'", - reject_unused: false, expected_sql: Err(RenderStop::MissingArgument("missing".to_owned())), }, ParameterizedSqlTestCase { - description: "an unused argument defers when unused arguments are rejected", - sql: "WHERE @column", - reject_unused: true, - expected_sql: Err(RenderStop::Deferred), - }, - ParameterizedSqlTestCase { - description: "a non-ASCII character after a name defers to Python's whitespace", - sql: "WHERE @column\u{a0}(", - reject_unused: false, - expected_sql: Err(RenderStop::Deferred), + description: "Unicode whitespace before a parenthesis makes a macro call", + sql: "WHERE @column\u{a0}( AND @column\u{e9}", + expected_sql: Ok("WHERE @column\u{a0}( AND status\u{e9}"), }, ]; for test_case in test_cases { assert_eq!( - rendered_sql(test_case.sql, test_case.reject_unused), + rendered_sql(test_case.sql), test_case.expected_sql.map(str::to_owned), "{}", test_case.description diff --git a/crates/sqlbuild-attachments/src/audits/tests/test_types.rs b/crates/sqlbuild-attachments/src/audits/tests/test_types.rs index b9e2aee9a9..f93c7734a4 100644 --- a/crates/sqlbuild-attachments/src/audits/tests/test_types.rs +++ b/crates/sqlbuild-attachments/src/audits/tests/test_types.rs @@ -4,13 +4,11 @@ use crate::audits::models::{AuditAttachment, AuditRendering}; pub(super) struct ParameterizedSqlTestCase { pub(super) description: &'static str, pub(super) sql: &'static str, - pub(super) reject_unused: bool, pub(super) expected_sql: Result<&'static str, RenderStop>, } pub(super) struct AttachedAuditTestCase { pub(super) description: &'static str, pub(super) attachment: AuditAttachment, - /// None where Python must decide. - pub(super) expected_rendering: Option, + pub(super) expected_rendering: AuditRendering, } diff --git a/crates/sqlbuild-attachments/src/ceremonial_select/_helpers/boundaries.rs b/crates/sqlbuild-attachments/src/ceremonial_select/_helpers/boundaries.rs index 2820bd2271..b945d8e926 100644 --- a/crates/sqlbuild-attachments/src/ceremonial_select/_helpers/boundaries.rs +++ b/crates/sqlbuild-attachments/src/ceremonial_select/_helpers/boundaries.rs @@ -1,4 +1,6 @@ -//! Python's code boundary pattern and `WITH` keyword check. +//! The code boundary pattern and `WITH` keyword check. + +use crate::ceremonial_select::constants::WITH_KEYWORD; /// Python's boundary pattern: parentheses, `/*`, quotes, `$` and the line comment prefixes. pub(crate) fn next_boundary(bytes: &[u8], start: usize, prefixes: &[String]) -> Option { @@ -17,19 +19,15 @@ fn is_boundary(rest: &[u8], prefixes: &[String]) -> bool { false } -/// Whether `WITH` (any case) starts at `first` as a whole word; None for non-ASCII text. -pub(crate) fn starts_with_keyword(sql: &str, first: usize) -> Option { +/// Whether `sql[first:first + 4].upper() == "WITH"` and no identifier character follows. +pub(crate) fn starts_with_keyword(sql: &str, first: usize) -> bool { let mut characters = sql[first..].chars(); - let keyword: String = characters.by_ref().take(4).collect(); - if !keyword.is_ascii() { - return None; - } - if !keyword.eq_ignore_ascii_case("WITH") { - return Some(false); - } - match characters.next() { - None => Some(true), - Some(next) if !next.is_ascii() => None, - Some(next) => Some(!(next.is_ascii_alphanumeric() || next == '_')), - } + let keyword: String = characters + .by_ref() + .take(WITH_KEYWORD.chars().count()) + .collect(); + keyword.to_uppercase() == WITH_KEYWORD + && !characters + .next() + .is_some_and(|next| next.is_alphanumeric() || next == '_') } diff --git a/crates/sqlbuild-attachments/src/ceremonial_select/constants.rs b/crates/sqlbuild-attachments/src/ceremonial_select/constants.rs new file mode 100644 index 0000000000..9848d4b39a --- /dev/null +++ b/crates/sqlbuild-attachments/src/ceremonial_select/constants.rs @@ -0,0 +1,4 @@ +//! Keywords the ceremonial `SELECT 1` scan matches. + +/// The keyword an omitted-`SELECT` body starts with, upper-cased. +pub(crate) const WITH_KEYWORD: &str = "WITH"; diff --git a/crates/sqlbuild-attachments/src/ceremonial_select/main/omitted_select_offset.rs b/crates/sqlbuild-attachments/src/ceremonial_select/main/omitted_select_offset.rs index f4402ea257..bd6206ed04 100644 --- a/crates/sqlbuild-attachments/src/ceremonial_select/main/omitted_select_offset.rs +++ b/crates/sqlbuild-attachments/src/ceremonial_select/main/omitted_select_offset.rs @@ -14,12 +14,6 @@ use crate::ceremonial_select::models::OmittedSelect; pub fn omitted_select_offset(sql: &str, syntax: &LexicalSyntax) -> OmittedSelect { let bytes: &[u8] = sql.as_bytes(); let prefixes: &[String] = &syntax.line_comment_prefixes; - if prefixes - .iter() - .any(|prefix| prefix.is_empty() || !prefix.is_ascii()) - { - return OmittedSelect::Deferred; - } let tail: &[u8] = &bytes[..trimmed_end(bytes, 0, bytes.len())]; if !tail.ends_with(b")") && !may_end_with_comment(tail, prefixes) { return OmittedSelect::Absent; @@ -55,9 +49,9 @@ pub fn omitted_select_offset(sql: &str, syntax: &LexicalSyntax) -> OmittedSelect if Some(last) != last_close { return OmittedSelect::Absent; } - match starts_with_keyword(sql, first) { - Some(true) => OmittedSelect::At(sql[..=last].chars().count()), - Some(false) => OmittedSelect::Absent, - None => OmittedSelect::Deferred, + if starts_with_keyword(sql, first) { + OmittedSelect::At(sql[..=last].chars().count()) + } else { + OmittedSelect::Absent } } diff --git a/crates/sqlbuild-attachments/src/ceremonial_select/mod.rs b/crates/sqlbuild-attachments/src/ceremonial_select/mod.rs index e6e5a52eb4..1e64f3d116 100644 --- a/crates/sqlbuild-attachments/src/ceremonial_select/mod.rs +++ b/crates/sqlbuild-attachments/src/ceremonial_select/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod _helpers; +pub(crate) mod constants; pub mod main; pub mod models; diff --git a/crates/sqlbuild-attachments/src/ceremonial_select/models.rs b/crates/sqlbuild-attachments/src/ceremonial_select/models.rs index 55d9e386e8..95fa3a8e1c 100644 --- a/crates/sqlbuild-attachments/src/ceremonial_select/models.rs +++ b/crates/sqlbuild-attachments/src/ceremonial_select/models.rs @@ -1,11 +1,10 @@ //! Where a SQL test or scenario body omits its trailing ceremonial `SELECT 1`. -/// Python's answer, or a deferral where only Python's Unicode rules can decide. +/// Where to insert `SELECT 1`, if anywhere. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum OmittedSelect { /// Insert `SELECT 1` at this code-point offset. At(usize), /// The body already ends correctly or is not a bare `WITH` statement. Absent, - Deferred, } diff --git a/crates/sqlbuild-attachments/src/ceremonial_select/tests/test_omitted_select_offset.rs b/crates/sqlbuild-attachments/src/ceremonial_select/tests/test_omitted_select_offset.rs index e46b61ebc2..018d5aa5bd 100644 --- a/crates/sqlbuild-attachments/src/ceremonial_select/tests/test_omitted_select_offset.rs +++ b/crates/sqlbuild-attachments/src/ceremonial_select/tests/test_omitted_select_offset.rs @@ -33,9 +33,14 @@ fn given_test_bodies_when_locating_omitted_select_then_python_offset_is_returned expected_offset: OmittedSelect::Absent, }, OmittedSelectTestCase { - description: "non-ASCII keyword text defers to Python's case mapping", + description: "a dotless i upper-cases to WITH", sql: "w\u{131}th a AS (SELECT 1)", - expected_offset: OmittedSelect::Deferred, + expected_offset: OmittedSelect::At(20), + }, + OmittedSelectTestCase { + description: "a non-ASCII letter after WITH continues the word", + sql: "WITH\u{e9} a AS (SELECT 1)", + expected_offset: OmittedSelect::Absent, }, ]; diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/scan.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/scan.rs index 15d1f92248..dc8c00c570 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/scan.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/_helpers/scan.rs @@ -1,6 +1,8 @@ -//! Python's cursor intrinsic scan over quotes, comments, identifier boundaries and calls. +//! The cursor intrinsic scan over quotes, comments, identifier boundaries and calls. +use sqlbuild_core::text::main::is_python_alnum::is_python_alnum; use sqlbuild_core::text::main::is_python_space::is_python_space; +use sqlbuild_core::text::models::PythonText; use sqlbuild_sqltext::sql_scan::models::Unclosed; use crate::sql_lexing::main::python_non_code_end::python_non_code_end; @@ -9,7 +11,7 @@ use crate::sql_lexing::models::NonCode; const INTRINSIC_NAMES: [&str; 2] = ["__cursor_start", "__cursor_end"]; -/// What Python's scan finds at one offset. +/// What the scan finds at one offset. pub(crate) enum ScanStep { Next(usize), Unclosed(Unclosed), @@ -18,23 +20,21 @@ pub(crate) enum ScanStep { name: &'static str, end: usize, }, - /// Python decides by its Unicode identifier rules. - Deferred, } -/// How the text after an intrinsic name reads as Python's empty call. +/// How the text after an intrinsic name reads as an empty call. pub(crate) enum CallScan { /// `name()`, with the offset of the closing parenthesis. Called(usize), NotCalled, Arguments, - UnclosedParenthesis, - /// Quotes, comments or nested parentheses inside the call, which only Python matches. - Deferred, + /// The call reaches the end of the SQL inside this construct. + Unclosed(Unclosed), } -/// Python's scan at `index`. -pub(crate) fn step(bytes: &[u8], index: usize) -> ScanStep { +/// The scan at `index`, a character boundary of `sql`. +pub(crate) fn step(python: PythonText, sql: &str, index: usize) -> ScanStep { + let bytes: &[u8] = sql.as_bytes(); match python_non_code_end(bytes, index) { NonCode::End(end) => return ScanStep::Next(end), NonCode::Raises => return ScanStep::Unclosed(unclosed_construct(bytes, index)), @@ -43,21 +43,19 @@ pub(crate) fn step(bytes: &[u8], index: usize) -> ScanStep { for name in INTRINSIC_NAMES { if bytes[index..].starts_with(name.as_bytes()) { let end: usize = index + name.len(); - let before: Option = index.checked_sub(1).map(|previous| bytes[previous]); - return match ( - continues_identifier(before), - continues_identifier(bytes.get(end).copied()), - ) { - (Some(false), Some(false)) => ScanStep::Intrinsic { name, end }, - (Some(_), Some(_)) => ScanStep::Next(end), - _ => ScanStep::Deferred, + let bounded: bool = !continues_identifier(python, sql[..index].chars().next_back()) + && !continues_identifier(python, sql[end..].chars().next()); + return if bounded { + ScanStep::Intrinsic { name, end } + } else { + ScanStep::Next(end) }; } } - ScanStep::Next(index + 1) + ScanStep::Next(index + sql[index..].chars().next().map_or(1, char::len_utf8)) } -/// Python's whitespace skip, `(` check, parenthesis match and argument check after a name. +/// The whitespace skip, `(` check, parenthesis match and argument check after a name. pub(crate) fn call_scan(sql: &str, name_end: usize) -> CallScan { let mut open: usize = name_end; while let Some(character) = sql[open..].chars().next() { @@ -70,32 +68,39 @@ pub(crate) fn call_scan(sql: &str, name_end: usize) -> CallScan { if bytes.get(open) != Some(&b'(') { return CallScan::NotCalled; } + let mut depth: usize = 1; let mut index: usize = open + 1; - while let Some(byte) = bytes.get(index) { - match byte { + while index < bytes.len() { + match python_non_code_end(bytes, index) { + NonCode::End(end) => { + index = end; + continue; + } + NonCode::Raises => return CallScan::Unclosed(unclosed_construct(bytes, index)), + NonCode::Code => {} + } + match bytes[index] { + b'(' => depth += 1, b')' => { - return if sql[open + 1..index].chars().all(is_python_space) { - CallScan::Called(index) - } else { - CallScan::Arguments - }; + depth -= 1; + if depth == 0 { + return if sql[open + 1..index].chars().all(is_python_space) { + CallScan::Called(index) + } else { + CallScan::Arguments + }; + } } - b'-' if bytes.get(index + 1) == Some(&b'-') => return CallScan::Deferred, - b'/' if bytes.get(index + 1) == Some(&b'*') => return CallScan::Deferred, - b'#' | b'\'' | b'"' | b'`' | b'$' | b'(' => return CallScan::Deferred, - _ => index += 1, + _ => {} } + index += 1; } - CallScan::UnclosedParenthesis + CallScan::Unclosed(Unclosed::Parenthesis) } -/// Python's `isalnum() or _`; non-ASCII neighbours defer to Python's Unicode rules. -fn continues_identifier(byte: Option) -> Option { - match byte { - None => Some(false), - Some(byte) if !byte.is_ascii() => None, - Some(byte) => Some(byte.is_ascii_alphanumeric() || byte == b'_'), - } +/// Python's `isalnum() or _` for a neighbouring character; the SQL edge continues nothing. +fn continues_identifier(python: PythonText, character: Option) -> bool { + character.is_some_and(|character| character == '_' || is_python_alnum(python, character)) } /// Python's `any(name in sql for name in _INTRINSIC_NAMES)`. diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/intrinsic_free.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/intrinsic_free.rs index 57c70b8242..9cf1d47b55 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/intrinsic_free.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/intrinsic_free.rs @@ -1,15 +1,18 @@ -//! Python's `reject_cursor_intrinsics`, with the error it raises. +//! `reject_cursor_intrinsics`, with the error it raises. -use crate::cursor_intrinsics::_helpers::scan::{ - CallScan, ScanStep, call_scan, contains_intrinsic_name, step, -}; +use sqlbuild_core::text::models::PythonText; + +use crate::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; use crate::cursor_intrinsics::models::IntrinsicCheck; -use crate::sql_lexing::main::unclosed_message::unclosed_message; -use sqlbuild_sqltext::sql_scan::models::Unclosed; -/// Python's acceptance of `sql`, its error for `context`, or `Deferred` where only Python decides. +/// Accept `sql` free of cursor intrinsics and reserved markers, or return its error for `context`. #[must_use] -pub fn intrinsic_free(sql: &str, reserved_markers: &[String], context: &str) -> IntrinsicCheck { +pub fn intrinsic_free( + python: PythonText, + sql: &str, + reserved_markers: &[String], + context: &str, +) -> IntrinsicCheck { if reserved_markers .iter() .any(|marker| sql.contains(marker.as_str())) @@ -18,54 +21,12 @@ pub fn intrinsic_free(sql: &str, reserved_markers: &[String], context: &str) -> "{context} contains a reserved internal cursor marker" )); } - if !contains_intrinsic_name(sql) { - return IntrinsicCheck::Free; - } - let bytes: &[u8] = sql.as_bytes(); - let mut found: bool = false; - let mut index: usize = 0; - while index < bytes.len() { - let (name, end): (&str, usize) = match step(bytes, index) { - ScanStep::Next(next) => { - index = next; - continue; - } - ScanStep::Unclosed(construct) => { - return IntrinsicCheck::Rejected(unclosed_message(context, construct)); - } - ScanStep::Deferred => return IntrinsicCheck::Deferred, - ScanStep::Intrinsic { name, end } => (name, end), - }; - match call_scan(sql, end) { - CallScan::Called(close) => { - found = true; - index = close + 1; - } - CallScan::NotCalled => { - return IntrinsicCheck::Rejected(format!( - "{context} intrinsic {name} must be called with ()" - )); - } - CallScan::Arguments => { - return IntrinsicCheck::Rejected(format!( - "{context} intrinsic {name} does not accept arguments" - )); - } - CallScan::UnclosedParenthesis => { - return IntrinsicCheck::Rejected(unclosed_message( - &format!("{context} cursor intrinsic"), - Unclosed::Parenthesis, - )); - } - CallScan::Deferred => return IntrinsicCheck::Deferred, - } - } - if found { - IntrinsicCheck::Rejected(format!( + match replace_intrinsics(python, sql, context, str::to_owned) { + Err(message) => IntrinsicCheck::Rejected(message), + Ok((_, true)) => IntrinsicCheck::Rejected(format!( "{context} uses cursor intrinsics, which are only supported in cursor-based \ incremental model query SQL" - )) - } else { - IntrinsicCheck::Free + )), + Ok((_, false)) => IntrinsicCheck::Free, } } diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/mod.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/mod.rs index 78a625860d..ac124fa97a 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/mod.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/mod.rs @@ -1 +1,3 @@ pub mod intrinsic_free; +pub mod replace_intrinsics; +pub mod validated_model_intrinsics; diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics.rs new file mode 100644 index 0000000000..cb6ea5dacb --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/replace_intrinsics.rs @@ -0,0 +1,58 @@ +//! `_transform_cursor_intrinsics`: replace every intrinsic call, or raise its error. + +use sqlbuild_core::text::models::PythonText; + +use crate::cursor_intrinsics::_helpers::scan::{ + CallScan, ScanStep, call_scan, contains_intrinsic_name, step, +}; +use crate::sql_lexing::main::unclosed_message::unclosed_message; + +/// `sql` with each cursor intrinsic call replaced and whether any was, or the malformed-call error. +pub fn replace_intrinsics( + python: PythonText, + sql: &str, + context: &str, + replacement: impl Fn(&str) -> String, +) -> Result<(String, bool), String> { + if !contains_intrinsic_name(sql) { + return Ok((sql.to_owned(), false)); + } + let mut replaced: String = String::with_capacity(sql.len()); + let mut last: usize = 0; + let mut found: bool = false; + let mut index: usize = 0; + while index < sql.len() { + let (name, end): (&str, usize) = match step(python, sql, index) { + ScanStep::Next(next) => { + index = next; + continue; + } + ScanStep::Unclosed(construct) => return Err(unclosed_message(context, construct)), + ScanStep::Intrinsic { name, end } => (name, end), + }; + let close: usize = match call_scan(sql, end) { + CallScan::Called(close) => close, + CallScan::NotCalled => { + return Err(format!("{context} intrinsic {name} must be called with ()")); + } + CallScan::Arguments => { + return Err(format!( + "{context} intrinsic {name} does not accept arguments" + )); + } + CallScan::Unclosed(construct) => { + return Err(unclosed_message( + &format!("{context} cursor intrinsic"), + construct, + )); + } + }; + replaced.push_str(&sql[last..index]); + replaced.push_str(&replacement(name)); + last = close + 1; + index = close + 1; + found = true; + } + replaced.push_str(&sql[last..]); + Ok((replaced, found)) +} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/main/validated_model_intrinsics.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/validated_model_intrinsics.rs new file mode 100644 index 0000000000..99fed9a7c1 --- /dev/null +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/main/validated_model_intrinsics.rs @@ -0,0 +1,43 @@ +//! `get_validated_model_cursor_intrinsics`: check and canonicalize one model query's intrinsics. + +use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_core::text::models::PythonText; + +use crate::cursor_intrinsics::main::replace_intrinsics::replace_intrinsics; +use crate::cursor_intrinsics::models::IntrinsicModel; + +/// The query with every intrinsic call written `name()`, or why the model may not use them. +pub fn validated_model_intrinsics( + python: PythonText, + sql: &str, + reserved_markers: &[String], + model: &IntrinsicModel<'_>, +) -> Result { + let context: String = format!("Model '{}'", model.name); + if reserved_markers + .iter() + .any(|marker| sql.contains(marker.as_str())) + { + return Err(format!( + "{context} contains a reserved internal cursor marker" + )); + } + let (canonical, found) = replace_intrinsics(python, sql, &context, |name| format!("{name}()"))?; + if !found { + return Ok(canonical); + } + if !model.incremental { + return Err(format!( + "{context} uses cursor intrinsics but is not a built-in incremental model" + )); + } + if model + .cursor + .is_none_or(|cursor| python_strip(cursor).is_empty()) + { + return Err(format!( + "{context} uses cursor intrinsics but does not declare a cursor" + )); + } + Ok(canonical) +} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/models.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/models.rs index 4844810821..7474639f96 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/models.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/models.rs @@ -1,9 +1,16 @@ //! Outcome of the native cursor intrinsic check. -/// Whether Python accepts SQL as intrinsic-free, rejects it with this error, or must decide. +/// Whether SQL is free of cursor intrinsics, or the error rejecting it. #[derive(Clone, Debug, PartialEq, Eq)] pub enum IntrinsicCheck { Free, Rejected(String), - Deferred, +} + +/// The model whose query is checked: name, built-in incremental flag and string `cursor`. +#[derive(Clone, Copy, Debug)] +pub struct IntrinsicModel<'model> { + pub name: &'model str, + pub incremental: bool, + pub cursor: Option<&'model str>, } diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/helpers.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/helpers.rs index 8bf22ff204..80e257ebef 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/helpers.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/helpers.rs @@ -1,5 +1,12 @@ +use sqlbuild_core::text::main::python_text::python_text; +use sqlbuild_core::text::models::PythonText; + use crate::cursor_intrinsics::models::IntrinsicCheck; pub(super) fn rejected(message: &str) -> IntrinsicCheck { IntrinsicCheck::Rejected(message.to_owned()) } + +pub(super) fn python() -> PythonText { + python_text((3, 12), "15.0.0").expect("Python 3.12 is supported") +} diff --git a/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_intrinsic_free.rs b/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_intrinsic_free.rs index f95d1c9bd7..7256d952c7 100644 --- a/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_intrinsic_free.rs +++ b/crates/sqlbuild-attachments/src/cursor_intrinsics/tests/test_intrinsic_free.rs @@ -1,6 +1,6 @@ use crate::cursor_intrinsics::main::intrinsic_free::intrinsic_free; use crate::cursor_intrinsics::models::IntrinsicCheck; -use crate::cursor_intrinsics::tests::helpers::rejected; +use crate::cursor_intrinsics::tests::helpers::{python, rejected}; use crate::cursor_intrinsics::tests::test_types::IntrinsicCheckTestCase; #[test] @@ -60,20 +60,42 @@ fn given_sql_when_checking_cursor_intrinsics_then_python_acceptance_or_error_is_ expected_check: rejected("Audit 'fresh' contains a reserved internal cursor marker"), }, IntrinsicCheckTestCase { - description: "quotes inside a call defer to Python's parenthesis matching", + description: "a quoted parenthesis inside a call is an argument", sql: "WHERE ts >= __cursor_start(')')", - expected_check: IntrinsicCheck::Deferred, + expected_check: rejected( + "Audit 'fresh' intrinsic __cursor_start does not accept arguments", + ), + }, + IntrinsicCheckTestCase { + description: "a comment-only call is an argument", + sql: "WHERE ts >= __cursor_start(/* now */)", + expected_check: rejected( + "Audit 'fresh' intrinsic __cursor_start does not accept arguments", + ), + }, + IntrinsicCheckTestCase { + description: "an unclosed quote inside a call names the cursor intrinsic context", + sql: "WHERE ts >= __cursor_start('open", + expected_check: rejected( + "Audit 'fresh' cursor intrinsic contains an unclosed quoted string", + ), }, IntrinsicCheckTestCase { - description: "a non-ASCII neighbour defers to Python's Unicode identifier rules", - sql: "SELECT é__cursor_end", - expected_check: IntrinsicCheck::Deferred, + description: "a non-ASCII letter neighbour continues the identifier", + sql: "SELECT é__cursor_end, __cursor_endé", + expected_check: IntrinsicCheck::Free, + }, + IntrinsicCheckTestCase { + description: "a non-ASCII symbol neighbour is a boundary", + sql: "SELECT ·__cursor_end", + expected_check: rejected("Audit 'fresh' intrinsic __cursor_end must be called with ()"), }, ]; for test_case in test_cases { assert_eq!( intrinsic_free( + python(), test_case.sql, &["__reserved_marker__".to_owned()], "Audit 'fresh'" diff --git a/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs b/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs index cf4cef94a7..4c81042156 100644 --- a/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs +++ b/crates/sqlbuild-core/src/text/_helpers/alnum_tables.rs @@ -1,4 +1,4 @@ -//! The generated `str.isalnum()` tables, one per Unicode version of a supported CPython. +//! The generated `str.isalnum()`, `str.isalpha()` and `str.isdecimal()` tables, per Unicode version. /// `(unicodedata.unidata_version, ranges)` for every table native text helpers can select. pub(crate) const ALNUM_TABLES: [(&str, &[(u32, u32)]); 3] = [ @@ -6,3 +6,17 @@ pub(crate) const ALNUM_TABLES: [(&str, &[(u32, u32)]); 3] = [ ("15.1.0", include!("../alnum_ranges/unicode_15_1_0.in")), ("16.0.0", include!("../alnum_ranges/unicode_16_0_0.in")), ]; + +/// `(unicodedata.unidata_version, ranges)` of `str.isalpha()`, in the order of `ALNUM_TABLES`. +pub(crate) const ALPHA_TABLES: [(&str, &[(u32, u32)]); 3] = [ + ("15.0.0", include!("../alpha_ranges/unicode_15_0_0.in")), + ("15.1.0", include!("../alpha_ranges/unicode_15_1_0.in")), + ("16.0.0", include!("../alpha_ranges/unicode_16_0_0.in")), +]; + +/// `(unicodedata.unidata_version, ranges)` of `str.isdecimal()` (`\d`), in the same order. +pub(crate) const DECIMAL_TABLES: [(&str, &[(u32, u32)]); 3] = [ + ("15.0.0", include!("../decimal_ranges/unicode_15_0_0.in")), + ("15.1.0", include!("../decimal_ranges/unicode_15_1_0.in")), + ("16.0.0", include!("../decimal_ranges/unicode_16_0_0.in")), +]; diff --git a/crates/sqlbuild-core/src/text/_helpers/code_point_ranges.rs b/crates/sqlbuild-core/src/text/_helpers/code_point_ranges.rs new file mode 100644 index 0000000000..2a0a763184 --- /dev/null +++ b/crates/sqlbuild-core/src/text/_helpers/code_point_ranges.rs @@ -0,0 +1,10 @@ +//! Membership in the generated, ordered code point range tables. + +/// Whether one character falls inside the inclusive, ordered code point `ranges`. +pub(crate) fn in_ranges(ranges: &[(u32, u32)], character: char) -> bool { + let code_point: u32 = u32::from(character); + let index: usize = ranges.partition_point(|(_, end)| *end < code_point); + ranges + .get(index) + .is_some_and(|(start, _)| *start <= code_point) +} diff --git a/crates/sqlbuild-core/src/text/_helpers/mod.rs b/crates/sqlbuild-core/src/text/_helpers/mod.rs index 0a2f6ee237..ede18696f6 100644 --- a/crates/sqlbuild-core/src/text/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/text/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod alnum_tables; pub(crate) mod cleandoc; +pub(crate) mod code_point_ranges; pub(crate) mod sequence_matcher; diff --git a/crates/sqlbuild-core/src/text/alpha_ranges/unicode_15_0_0.in b/crates/sqlbuild-core/src/text/alpha_ranges/unicode_15_0_0.in new file mode 100644 index 0000000000..303ce0bae7 --- /dev/null +++ b/crates/sqlbuild-core/src/text/alpha_ranges/unicode_15_0_0.in @@ -0,0 +1,663 @@ +// Generated by `scripts/generate_python_char_table.py`; do not edit. +// Inclusive code point ranges where `str.isalpha()` is true in Unicode 15.0.0, in order. +&[ + (0x0041, 0x005A), + (0x0061, 0x007A), + (0x00AA, 0x00AA), + (0x00B5, 0x00B5), + (0x00BA, 0x00BA), + (0x00C0, 0x00D6), + (0x00D8, 0x00F6), + (0x00F8, 0x02C1), + (0x02C6, 0x02D1), + (0x02E0, 0x02E4), + (0x02EC, 0x02EC), + (0x02EE, 0x02EE), + (0x0370, 0x0374), + (0x0376, 0x0377), + (0x037A, 0x037D), + (0x037F, 0x037F), + (0x0386, 0x0386), + (0x0388, 0x038A), + (0x038C, 0x038C), + (0x038E, 0x03A1), + (0x03A3, 0x03F5), + (0x03F7, 0x0481), + (0x048A, 0x052F), + (0x0531, 0x0556), + (0x0559, 0x0559), + (0x0560, 0x0588), + (0x05D0, 0x05EA), + (0x05EF, 0x05F2), + (0x0620, 0x064A), + (0x066E, 0x066F), + (0x0671, 0x06D3), + (0x06D5, 0x06D5), + (0x06E5, 0x06E6), + (0x06EE, 0x06EF), + (0x06FA, 0x06FC), + (0x06FF, 0x06FF), + (0x0710, 0x0710), + (0x0712, 0x072F), + (0x074D, 0x07A5), + (0x07B1, 0x07B1), + (0x07CA, 0x07EA), + (0x07F4, 0x07F5), + (0x07FA, 0x07FA), + (0x0800, 0x0815), + (0x081A, 0x081A), + (0x0824, 0x0824), + (0x0828, 0x0828), + (0x0840, 0x0858), + (0x0860, 0x086A), + (0x0870, 0x0887), + (0x0889, 0x088E), + (0x08A0, 0x08C9), + (0x0904, 0x0939), + (0x093D, 0x093D), + (0x0950, 0x0950), + (0x0958, 0x0961), + (0x0971, 0x0980), + (0x0985, 0x098C), + (0x098F, 0x0990), + (0x0993, 0x09A8), + (0x09AA, 0x09B0), + (0x09B2, 0x09B2), + (0x09B6, 0x09B9), + (0x09BD, 0x09BD), + (0x09CE, 0x09CE), + (0x09DC, 0x09DD), + (0x09DF, 0x09E1), + (0x09F0, 0x09F1), + (0x09FC, 0x09FC), + (0x0A05, 0x0A0A), + (0x0A0F, 0x0A10), + (0x0A13, 0x0A28), + (0x0A2A, 0x0A30), + (0x0A32, 0x0A33), + (0x0A35, 0x0A36), + (0x0A38, 0x0A39), + (0x0A59, 0x0A5C), + (0x0A5E, 0x0A5E), + (0x0A72, 0x0A74), + (0x0A85, 0x0A8D), + (0x0A8F, 0x0A91), + (0x0A93, 0x0AA8), + (0x0AAA, 0x0AB0), + (0x0AB2, 0x0AB3), + (0x0AB5, 0x0AB9), + (0x0ABD, 0x0ABD), + (0x0AD0, 0x0AD0), + (0x0AE0, 0x0AE1), + (0x0AF9, 0x0AF9), + (0x0B05, 0x0B0C), + (0x0B0F, 0x0B10), + (0x0B13, 0x0B28), + (0x0B2A, 0x0B30), + (0x0B32, 0x0B33), + (0x0B35, 0x0B39), + (0x0B3D, 0x0B3D), + (0x0B5C, 0x0B5D), + (0x0B5F, 0x0B61), + (0x0B71, 0x0B71), + (0x0B83, 0x0B83), + (0x0B85, 0x0B8A), + (0x0B8E, 0x0B90), + (0x0B92, 0x0B95), + (0x0B99, 0x0B9A), + (0x0B9C, 0x0B9C), + (0x0B9E, 0x0B9F), + (0x0BA3, 0x0BA4), + (0x0BA8, 0x0BAA), + (0x0BAE, 0x0BB9), + (0x0BD0, 0x0BD0), + (0x0C05, 0x0C0C), + (0x0C0E, 0x0C10), + (0x0C12, 0x0C28), + (0x0C2A, 0x0C39), + (0x0C3D, 0x0C3D), + (0x0C58, 0x0C5A), + (0x0C5D, 0x0C5D), + (0x0C60, 0x0C61), + (0x0C80, 0x0C80), + (0x0C85, 0x0C8C), + (0x0C8E, 0x0C90), + (0x0C92, 0x0CA8), + (0x0CAA, 0x0CB3), + (0x0CB5, 0x0CB9), + (0x0CBD, 0x0CBD), + (0x0CDD, 0x0CDE), + (0x0CE0, 0x0CE1), + (0x0CF1, 0x0CF2), + (0x0D04, 0x0D0C), + (0x0D0E, 0x0D10), + (0x0D12, 0x0D3A), + (0x0D3D, 0x0D3D), + (0x0D4E, 0x0D4E), + (0x0D54, 0x0D56), + (0x0D5F, 0x0D61), + (0x0D7A, 0x0D7F), + (0x0D85, 0x0D96), + (0x0D9A, 0x0DB1), + (0x0DB3, 0x0DBB), + (0x0DBD, 0x0DBD), + (0x0DC0, 0x0DC6), + (0x0E01, 0x0E30), + (0x0E32, 0x0E33), + (0x0E40, 0x0E46), + (0x0E81, 0x0E82), + (0x0E84, 0x0E84), + (0x0E86, 0x0E8A), + (0x0E8C, 0x0EA3), + (0x0EA5, 0x0EA5), + (0x0EA7, 0x0EB0), + (0x0EB2, 0x0EB3), + (0x0EBD, 0x0EBD), + (0x0EC0, 0x0EC4), + (0x0EC6, 0x0EC6), + (0x0EDC, 0x0EDF), + (0x0F00, 0x0F00), + (0x0F40, 0x0F47), + (0x0F49, 0x0F6C), + (0x0F88, 0x0F8C), + (0x1000, 0x102A), + (0x103F, 0x103F), + (0x1050, 0x1055), + (0x105A, 0x105D), + (0x1061, 0x1061), + (0x1065, 0x1066), + (0x106E, 0x1070), + (0x1075, 0x1081), + (0x108E, 0x108E), + (0x10A0, 0x10C5), + (0x10C7, 0x10C7), + (0x10CD, 0x10CD), + (0x10D0, 0x10FA), + (0x10FC, 0x1248), + (0x124A, 0x124D), + (0x1250, 0x1256), + (0x1258, 0x1258), + (0x125A, 0x125D), + (0x1260, 0x1288), + (0x128A, 0x128D), + (0x1290, 0x12B0), + (0x12B2, 0x12B5), + (0x12B8, 0x12BE), + (0x12C0, 0x12C0), + (0x12C2, 0x12C5), + (0x12C8, 0x12D6), + (0x12D8, 0x1310), + (0x1312, 0x1315), + (0x1318, 0x135A), + (0x1380, 0x138F), + (0x13A0, 0x13F5), + (0x13F8, 0x13FD), + (0x1401, 0x166C), + (0x166F, 0x167F), + (0x1681, 0x169A), + (0x16A0, 0x16EA), + (0x16F1, 0x16F8), + (0x1700, 0x1711), + (0x171F, 0x1731), + (0x1740, 0x1751), + (0x1760, 0x176C), + (0x176E, 0x1770), + (0x1780, 0x17B3), + (0x17D7, 0x17D7), + (0x17DC, 0x17DC), + (0x1820, 0x1878), + (0x1880, 0x1884), + (0x1887, 0x18A8), + (0x18AA, 0x18AA), + (0x18B0, 0x18F5), + (0x1900, 0x191E), + (0x1950, 0x196D), + (0x1970, 0x1974), + (0x1980, 0x19AB), + (0x19B0, 0x19C9), + (0x1A00, 0x1A16), + (0x1A20, 0x1A54), + (0x1AA7, 0x1AA7), + (0x1B05, 0x1B33), + (0x1B45, 0x1B4C), + (0x1B83, 0x1BA0), + (0x1BAE, 0x1BAF), + (0x1BBA, 0x1BE5), + (0x1C00, 0x1C23), + (0x1C4D, 0x1C4F), + (0x1C5A, 0x1C7D), + (0x1C80, 0x1C88), + (0x1C90, 0x1CBA), + (0x1CBD, 0x1CBF), + (0x1CE9, 0x1CEC), + (0x1CEE, 0x1CF3), + (0x1CF5, 0x1CF6), + (0x1CFA, 0x1CFA), + (0x1D00, 0x1DBF), + (0x1E00, 0x1F15), + (0x1F18, 0x1F1D), + (0x1F20, 0x1F45), + (0x1F48, 0x1F4D), + (0x1F50, 0x1F57), + (0x1F59, 0x1F59), + (0x1F5B, 0x1F5B), + (0x1F5D, 0x1F5D), + (0x1F5F, 0x1F7D), + (0x1F80, 0x1FB4), + (0x1FB6, 0x1FBC), + (0x1FBE, 0x1FBE), + (0x1FC2, 0x1FC4), + (0x1FC6, 0x1FCC), + (0x1FD0, 0x1FD3), + (0x1FD6, 0x1FDB), + (0x1FE0, 0x1FEC), + (0x1FF2, 0x1FF4), + (0x1FF6, 0x1FFC), + (0x2071, 0x2071), + (0x207F, 0x207F), + (0x2090, 0x209C), + (0x2102, 0x2102), + (0x2107, 0x2107), + (0x210A, 0x2113), + (0x2115, 0x2115), + (0x2119, 0x211D), + (0x2124, 0x2124), + (0x2126, 0x2126), + (0x2128, 0x2128), + (0x212A, 0x212D), + (0x212F, 0x2139), + (0x213C, 0x213F), + (0x2145, 0x2149), + (0x214E, 0x214E), + (0x2183, 0x2184), + (0x2C00, 0x2CE4), + (0x2CEB, 0x2CEE), + (0x2CF2, 0x2CF3), + (0x2D00, 0x2D25), + (0x2D27, 0x2D27), + (0x2D2D, 0x2D2D), + (0x2D30, 0x2D67), + (0x2D6F, 0x2D6F), + (0x2D80, 0x2D96), + (0x2DA0, 0x2DA6), + (0x2DA8, 0x2DAE), + (0x2DB0, 0x2DB6), + (0x2DB8, 0x2DBE), + (0x2DC0, 0x2DC6), + (0x2DC8, 0x2DCE), + (0x2DD0, 0x2DD6), + (0x2DD8, 0x2DDE), + (0x2E2F, 0x2E2F), + (0x3005, 0x3006), + (0x3031, 0x3035), + (0x303B, 0x303C), + (0x3041, 0x3096), + (0x309D, 0x309F), + (0x30A1, 0x30FA), + (0x30FC, 0x30FF), + (0x3105, 0x312F), + (0x3131, 0x318E), + (0x31A0, 0x31BF), + (0x31F0, 0x31FF), + (0x3400, 0x4DBF), + (0x4E00, 0xA48C), + (0xA4D0, 0xA4FD), + (0xA500, 0xA60C), + (0xA610, 0xA61F), + (0xA62A, 0xA62B), + (0xA640, 0xA66E), + (0xA67F, 0xA69D), + (0xA6A0, 0xA6E5), + (0xA717, 0xA71F), + (0xA722, 0xA788), + (0xA78B, 0xA7CA), + (0xA7D0, 0xA7D1), + (0xA7D3, 0xA7D3), + (0xA7D5, 0xA7D9), + (0xA7F2, 0xA801), + (0xA803, 0xA805), + (0xA807, 0xA80A), + (0xA80C, 0xA822), + (0xA840, 0xA873), + (0xA882, 0xA8B3), + (0xA8F2, 0xA8F7), + (0xA8FB, 0xA8FB), + (0xA8FD, 0xA8FE), + (0xA90A, 0xA925), + (0xA930, 0xA946), + (0xA960, 0xA97C), + (0xA984, 0xA9B2), + (0xA9CF, 0xA9CF), + (0xA9E0, 0xA9E4), + (0xA9E6, 0xA9EF), + (0xA9FA, 0xA9FE), + (0xAA00, 0xAA28), + (0xAA40, 0xAA42), + (0xAA44, 0xAA4B), + (0xAA60, 0xAA76), + (0xAA7A, 0xAA7A), + (0xAA7E, 0xAAAF), + (0xAAB1, 0xAAB1), + (0xAAB5, 0xAAB6), + (0xAAB9, 0xAABD), + (0xAAC0, 0xAAC0), + (0xAAC2, 0xAAC2), + (0xAADB, 0xAADD), + (0xAAE0, 0xAAEA), + (0xAAF2, 0xAAF4), + (0xAB01, 0xAB06), + (0xAB09, 0xAB0E), + (0xAB11, 0xAB16), + (0xAB20, 0xAB26), + (0xAB28, 0xAB2E), + (0xAB30, 0xAB5A), + (0xAB5C, 0xAB69), + (0xAB70, 0xABE2), + (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), + (0xD7CB, 0xD7FB), + (0xF900, 0xFA6D), + (0xFA70, 0xFAD9), + (0xFB00, 0xFB06), + (0xFB13, 0xFB17), + (0xFB1D, 0xFB1D), + (0xFB1F, 0xFB28), + (0xFB2A, 0xFB36), + (0xFB38, 0xFB3C), + (0xFB3E, 0xFB3E), + (0xFB40, 0xFB41), + (0xFB43, 0xFB44), + (0xFB46, 0xFBB1), + (0xFBD3, 0xFD3D), + (0xFD50, 0xFD8F), + (0xFD92, 0xFDC7), + (0xFDF0, 0xFDFB), + (0xFE70, 0xFE74), + (0xFE76, 0xFEFC), + (0xFF21, 0xFF3A), + (0xFF41, 0xFF5A), + (0xFF66, 0xFFBE), + (0xFFC2, 0xFFC7), + (0xFFCA, 0xFFCF), + (0xFFD2, 0xFFD7), + (0xFFDA, 0xFFDC), + (0x10000, 0x1000B), + (0x1000D, 0x10026), + (0x10028, 0x1003A), + (0x1003C, 0x1003D), + (0x1003F, 0x1004D), + (0x10050, 0x1005D), + (0x10080, 0x100FA), + (0x10280, 0x1029C), + (0x102A0, 0x102D0), + (0x10300, 0x1031F), + (0x1032D, 0x10340), + (0x10342, 0x10349), + (0x10350, 0x10375), + (0x10380, 0x1039D), + (0x103A0, 0x103C3), + (0x103C8, 0x103CF), + (0x10400, 0x1049D), + (0x104B0, 0x104D3), + (0x104D8, 0x104FB), + (0x10500, 0x10527), + (0x10530, 0x10563), + (0x10570, 0x1057A), + (0x1057C, 0x1058A), + (0x1058C, 0x10592), + (0x10594, 0x10595), + (0x10597, 0x105A1), + (0x105A3, 0x105B1), + (0x105B3, 0x105B9), + (0x105BB, 0x105BC), + (0x10600, 0x10736), + (0x10740, 0x10755), + (0x10760, 0x10767), + (0x10780, 0x10785), + (0x10787, 0x107B0), + (0x107B2, 0x107BA), + (0x10800, 0x10805), + (0x10808, 0x10808), + (0x1080A, 0x10835), + (0x10837, 0x10838), + (0x1083C, 0x1083C), + (0x1083F, 0x10855), + (0x10860, 0x10876), + (0x10880, 0x1089E), + (0x108E0, 0x108F2), + (0x108F4, 0x108F5), + (0x10900, 0x10915), + (0x10920, 0x10939), + (0x10980, 0x109B7), + (0x109BE, 0x109BF), + (0x10A00, 0x10A00), + (0x10A10, 0x10A13), + (0x10A15, 0x10A17), + (0x10A19, 0x10A35), + (0x10A60, 0x10A7C), + (0x10A80, 0x10A9C), + (0x10AC0, 0x10AC7), + (0x10AC9, 0x10AE4), + (0x10B00, 0x10B35), + (0x10B40, 0x10B55), + (0x10B60, 0x10B72), + (0x10B80, 0x10B91), + (0x10C00, 0x10C48), + (0x10C80, 0x10CB2), + (0x10CC0, 0x10CF2), + (0x10D00, 0x10D23), + (0x10E80, 0x10EA9), + (0x10EB0, 0x10EB1), + (0x10F00, 0x10F1C), + (0x10F27, 0x10F27), + (0x10F30, 0x10F45), + (0x10F70, 0x10F81), + (0x10FB0, 0x10FC4), + (0x10FE0, 0x10FF6), + (0x11003, 0x11037), + (0x11071, 0x11072), + (0x11075, 0x11075), + (0x11083, 0x110AF), + (0x110D0, 0x110E8), + (0x11103, 0x11126), + (0x11144, 0x11144), + (0x11147, 0x11147), + (0x11150, 0x11172), + (0x11176, 0x11176), + (0x11183, 0x111B2), + (0x111C1, 0x111C4), + (0x111DA, 0x111DA), + (0x111DC, 0x111DC), + (0x11200, 0x11211), + (0x11213, 0x1122B), + (0x1123F, 0x11240), + (0x11280, 0x11286), + (0x11288, 0x11288), + (0x1128A, 0x1128D), + (0x1128F, 0x1129D), + (0x1129F, 0x112A8), + (0x112B0, 0x112DE), + (0x11305, 0x1130C), + (0x1130F, 0x11310), + (0x11313, 0x11328), + (0x1132A, 0x11330), + (0x11332, 0x11333), + (0x11335, 0x11339), + (0x1133D, 0x1133D), + (0x11350, 0x11350), + (0x1135D, 0x11361), + (0x11400, 0x11434), + (0x11447, 0x1144A), + (0x1145F, 0x11461), + (0x11480, 0x114AF), + (0x114C4, 0x114C5), + (0x114C7, 0x114C7), + (0x11580, 0x115AE), + (0x115D8, 0x115DB), + (0x11600, 0x1162F), + (0x11644, 0x11644), + (0x11680, 0x116AA), + (0x116B8, 0x116B8), + (0x11700, 0x1171A), + (0x11740, 0x11746), + (0x11800, 0x1182B), + (0x118A0, 0x118DF), + (0x118FF, 0x11906), + (0x11909, 0x11909), + (0x1190C, 0x11913), + (0x11915, 0x11916), + (0x11918, 0x1192F), + (0x1193F, 0x1193F), + (0x11941, 0x11941), + (0x119A0, 0x119A7), + (0x119AA, 0x119D0), + (0x119E1, 0x119E1), + (0x119E3, 0x119E3), + (0x11A00, 0x11A00), + (0x11A0B, 0x11A32), + (0x11A3A, 0x11A3A), + (0x11A50, 0x11A50), + (0x11A5C, 0x11A89), + (0x11A9D, 0x11A9D), + (0x11AB0, 0x11AF8), + (0x11C00, 0x11C08), + (0x11C0A, 0x11C2E), + (0x11C40, 0x11C40), + (0x11C72, 0x11C8F), + (0x11D00, 0x11D06), + (0x11D08, 0x11D09), + (0x11D0B, 0x11D30), + (0x11D46, 0x11D46), + (0x11D60, 0x11D65), + (0x11D67, 0x11D68), + (0x11D6A, 0x11D89), + (0x11D98, 0x11D98), + (0x11EE0, 0x11EF2), + (0x11F02, 0x11F02), + (0x11F04, 0x11F10), + (0x11F12, 0x11F33), + (0x11FB0, 0x11FB0), + (0x12000, 0x12399), + (0x12480, 0x12543), + (0x12F90, 0x12FF0), + (0x13000, 0x1342F), + (0x13441, 0x13446), + (0x14400, 0x14646), + (0x16800, 0x16A38), + (0x16A40, 0x16A5E), + (0x16A70, 0x16ABE), + (0x16AD0, 0x16AED), + (0x16B00, 0x16B2F), + (0x16B40, 0x16B43), + (0x16B63, 0x16B77), + (0x16B7D, 0x16B8F), + (0x16E40, 0x16E7F), + (0x16F00, 0x16F4A), + (0x16F50, 0x16F50), + (0x16F93, 0x16F9F), + (0x16FE0, 0x16FE1), + (0x16FE3, 0x16FE3), + (0x17000, 0x187F7), + (0x18800, 0x18CD5), + (0x18D00, 0x18D08), + (0x1AFF0, 0x1AFF3), + (0x1AFF5, 0x1AFFB), + (0x1AFFD, 0x1AFFE), + (0x1B000, 0x1B122), + (0x1B132, 0x1B132), + (0x1B150, 0x1B152), + (0x1B155, 0x1B155), + (0x1B164, 0x1B167), + (0x1B170, 0x1B2FB), + (0x1BC00, 0x1BC6A), + (0x1BC70, 0x1BC7C), + (0x1BC80, 0x1BC88), + (0x1BC90, 0x1BC99), + (0x1D400, 0x1D454), + (0x1D456, 0x1D49C), + (0x1D49E, 0x1D49F), + (0x1D4A2, 0x1D4A2), + (0x1D4A5, 0x1D4A6), + (0x1D4A9, 0x1D4AC), + (0x1D4AE, 0x1D4B9), + (0x1D4BB, 0x1D4BB), + (0x1D4BD, 0x1D4C3), + (0x1D4C5, 0x1D505), + (0x1D507, 0x1D50A), + (0x1D50D, 0x1D514), + (0x1D516, 0x1D51C), + (0x1D51E, 0x1D539), + (0x1D53B, 0x1D53E), + (0x1D540, 0x1D544), + (0x1D546, 0x1D546), + (0x1D54A, 0x1D550), + (0x1D552, 0x1D6A5), + (0x1D6A8, 0x1D6C0), + (0x1D6C2, 0x1D6DA), + (0x1D6DC, 0x1D6FA), + (0x1D6FC, 0x1D714), + (0x1D716, 0x1D734), + (0x1D736, 0x1D74E), + (0x1D750, 0x1D76E), + (0x1D770, 0x1D788), + (0x1D78A, 0x1D7A8), + (0x1D7AA, 0x1D7C2), + (0x1D7C4, 0x1D7CB), + (0x1DF00, 0x1DF1E), + (0x1DF25, 0x1DF2A), + (0x1E030, 0x1E06D), + (0x1E100, 0x1E12C), + (0x1E137, 0x1E13D), + (0x1E14E, 0x1E14E), + (0x1E290, 0x1E2AD), + (0x1E2C0, 0x1E2EB), + (0x1E4D0, 0x1E4EB), + (0x1E7E0, 0x1E7E6), + (0x1E7E8, 0x1E7EB), + (0x1E7ED, 0x1E7EE), + (0x1E7F0, 0x1E7FE), + (0x1E800, 0x1E8C4), + (0x1E900, 0x1E943), + (0x1E94B, 0x1E94B), + (0x1EE00, 0x1EE03), + (0x1EE05, 0x1EE1F), + (0x1EE21, 0x1EE22), + (0x1EE24, 0x1EE24), + (0x1EE27, 0x1EE27), + (0x1EE29, 0x1EE32), + (0x1EE34, 0x1EE37), + (0x1EE39, 0x1EE39), + (0x1EE3B, 0x1EE3B), + (0x1EE42, 0x1EE42), + (0x1EE47, 0x1EE47), + (0x1EE49, 0x1EE49), + (0x1EE4B, 0x1EE4B), + (0x1EE4D, 0x1EE4F), + (0x1EE51, 0x1EE52), + (0x1EE54, 0x1EE54), + (0x1EE57, 0x1EE57), + (0x1EE59, 0x1EE59), + (0x1EE5B, 0x1EE5B), + (0x1EE5D, 0x1EE5D), + (0x1EE5F, 0x1EE5F), + (0x1EE61, 0x1EE62), + (0x1EE64, 0x1EE64), + (0x1EE67, 0x1EE6A), + (0x1EE6C, 0x1EE72), + (0x1EE74, 0x1EE77), + (0x1EE79, 0x1EE7C), + (0x1EE7E, 0x1EE7E), + (0x1EE80, 0x1EE89), + (0x1EE8B, 0x1EE9B), + (0x1EEA1, 0x1EEA3), + (0x1EEA5, 0x1EEA9), + (0x1EEAB, 0x1EEBB), + (0x20000, 0x2A6DF), + (0x2A700, 0x2B739), + (0x2B740, 0x2B81D), + (0x2B820, 0x2CEA1), + (0x2CEB0, 0x2EBE0), + (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), + (0x31350, 0x323AF), +] diff --git a/crates/sqlbuild-core/src/text/alpha_ranges/unicode_15_1_0.in b/crates/sqlbuild-core/src/text/alpha_ranges/unicode_15_1_0.in new file mode 100644 index 0000000000..ceaf88df48 --- /dev/null +++ b/crates/sqlbuild-core/src/text/alpha_ranges/unicode_15_1_0.in @@ -0,0 +1,664 @@ +// Generated by `scripts/generate_python_char_table.py`; do not edit. +// Inclusive code point ranges where `str.isalpha()` is true in Unicode 15.1.0, in order. +&[ + (0x0041, 0x005A), + (0x0061, 0x007A), + (0x00AA, 0x00AA), + (0x00B5, 0x00B5), + (0x00BA, 0x00BA), + (0x00C0, 0x00D6), + (0x00D8, 0x00F6), + (0x00F8, 0x02C1), + (0x02C6, 0x02D1), + (0x02E0, 0x02E4), + (0x02EC, 0x02EC), + (0x02EE, 0x02EE), + (0x0370, 0x0374), + (0x0376, 0x0377), + (0x037A, 0x037D), + (0x037F, 0x037F), + (0x0386, 0x0386), + (0x0388, 0x038A), + (0x038C, 0x038C), + (0x038E, 0x03A1), + (0x03A3, 0x03F5), + (0x03F7, 0x0481), + (0x048A, 0x052F), + (0x0531, 0x0556), + (0x0559, 0x0559), + (0x0560, 0x0588), + (0x05D0, 0x05EA), + (0x05EF, 0x05F2), + (0x0620, 0x064A), + (0x066E, 0x066F), + (0x0671, 0x06D3), + (0x06D5, 0x06D5), + (0x06E5, 0x06E6), + (0x06EE, 0x06EF), + (0x06FA, 0x06FC), + (0x06FF, 0x06FF), + (0x0710, 0x0710), + (0x0712, 0x072F), + (0x074D, 0x07A5), + (0x07B1, 0x07B1), + (0x07CA, 0x07EA), + (0x07F4, 0x07F5), + (0x07FA, 0x07FA), + (0x0800, 0x0815), + (0x081A, 0x081A), + (0x0824, 0x0824), + (0x0828, 0x0828), + (0x0840, 0x0858), + (0x0860, 0x086A), + (0x0870, 0x0887), + (0x0889, 0x088E), + (0x08A0, 0x08C9), + (0x0904, 0x0939), + (0x093D, 0x093D), + (0x0950, 0x0950), + (0x0958, 0x0961), + (0x0971, 0x0980), + (0x0985, 0x098C), + (0x098F, 0x0990), + (0x0993, 0x09A8), + (0x09AA, 0x09B0), + (0x09B2, 0x09B2), + (0x09B6, 0x09B9), + (0x09BD, 0x09BD), + (0x09CE, 0x09CE), + (0x09DC, 0x09DD), + (0x09DF, 0x09E1), + (0x09F0, 0x09F1), + (0x09FC, 0x09FC), + (0x0A05, 0x0A0A), + (0x0A0F, 0x0A10), + (0x0A13, 0x0A28), + (0x0A2A, 0x0A30), + (0x0A32, 0x0A33), + (0x0A35, 0x0A36), + (0x0A38, 0x0A39), + (0x0A59, 0x0A5C), + (0x0A5E, 0x0A5E), + (0x0A72, 0x0A74), + (0x0A85, 0x0A8D), + (0x0A8F, 0x0A91), + (0x0A93, 0x0AA8), + (0x0AAA, 0x0AB0), + (0x0AB2, 0x0AB3), + (0x0AB5, 0x0AB9), + (0x0ABD, 0x0ABD), + (0x0AD0, 0x0AD0), + (0x0AE0, 0x0AE1), + (0x0AF9, 0x0AF9), + (0x0B05, 0x0B0C), + (0x0B0F, 0x0B10), + (0x0B13, 0x0B28), + (0x0B2A, 0x0B30), + (0x0B32, 0x0B33), + (0x0B35, 0x0B39), + (0x0B3D, 0x0B3D), + (0x0B5C, 0x0B5D), + (0x0B5F, 0x0B61), + (0x0B71, 0x0B71), + (0x0B83, 0x0B83), + (0x0B85, 0x0B8A), + (0x0B8E, 0x0B90), + (0x0B92, 0x0B95), + (0x0B99, 0x0B9A), + (0x0B9C, 0x0B9C), + (0x0B9E, 0x0B9F), + (0x0BA3, 0x0BA4), + (0x0BA8, 0x0BAA), + (0x0BAE, 0x0BB9), + (0x0BD0, 0x0BD0), + (0x0C05, 0x0C0C), + (0x0C0E, 0x0C10), + (0x0C12, 0x0C28), + (0x0C2A, 0x0C39), + (0x0C3D, 0x0C3D), + (0x0C58, 0x0C5A), + (0x0C5D, 0x0C5D), + (0x0C60, 0x0C61), + (0x0C80, 0x0C80), + (0x0C85, 0x0C8C), + (0x0C8E, 0x0C90), + (0x0C92, 0x0CA8), + (0x0CAA, 0x0CB3), + (0x0CB5, 0x0CB9), + (0x0CBD, 0x0CBD), + (0x0CDD, 0x0CDE), + (0x0CE0, 0x0CE1), + (0x0CF1, 0x0CF2), + (0x0D04, 0x0D0C), + (0x0D0E, 0x0D10), + (0x0D12, 0x0D3A), + (0x0D3D, 0x0D3D), + (0x0D4E, 0x0D4E), + (0x0D54, 0x0D56), + (0x0D5F, 0x0D61), + (0x0D7A, 0x0D7F), + (0x0D85, 0x0D96), + (0x0D9A, 0x0DB1), + (0x0DB3, 0x0DBB), + (0x0DBD, 0x0DBD), + (0x0DC0, 0x0DC6), + (0x0E01, 0x0E30), + (0x0E32, 0x0E33), + (0x0E40, 0x0E46), + (0x0E81, 0x0E82), + (0x0E84, 0x0E84), + (0x0E86, 0x0E8A), + (0x0E8C, 0x0EA3), + (0x0EA5, 0x0EA5), + (0x0EA7, 0x0EB0), + (0x0EB2, 0x0EB3), + (0x0EBD, 0x0EBD), + (0x0EC0, 0x0EC4), + (0x0EC6, 0x0EC6), + (0x0EDC, 0x0EDF), + (0x0F00, 0x0F00), + (0x0F40, 0x0F47), + (0x0F49, 0x0F6C), + (0x0F88, 0x0F8C), + (0x1000, 0x102A), + (0x103F, 0x103F), + (0x1050, 0x1055), + (0x105A, 0x105D), + (0x1061, 0x1061), + (0x1065, 0x1066), + (0x106E, 0x1070), + (0x1075, 0x1081), + (0x108E, 0x108E), + (0x10A0, 0x10C5), + (0x10C7, 0x10C7), + (0x10CD, 0x10CD), + (0x10D0, 0x10FA), + (0x10FC, 0x1248), + (0x124A, 0x124D), + (0x1250, 0x1256), + (0x1258, 0x1258), + (0x125A, 0x125D), + (0x1260, 0x1288), + (0x128A, 0x128D), + (0x1290, 0x12B0), + (0x12B2, 0x12B5), + (0x12B8, 0x12BE), + (0x12C0, 0x12C0), + (0x12C2, 0x12C5), + (0x12C8, 0x12D6), + (0x12D8, 0x1310), + (0x1312, 0x1315), + (0x1318, 0x135A), + (0x1380, 0x138F), + (0x13A0, 0x13F5), + (0x13F8, 0x13FD), + (0x1401, 0x166C), + (0x166F, 0x167F), + (0x1681, 0x169A), + (0x16A0, 0x16EA), + (0x16F1, 0x16F8), + (0x1700, 0x1711), + (0x171F, 0x1731), + (0x1740, 0x1751), + (0x1760, 0x176C), + (0x176E, 0x1770), + (0x1780, 0x17B3), + (0x17D7, 0x17D7), + (0x17DC, 0x17DC), + (0x1820, 0x1878), + (0x1880, 0x1884), + (0x1887, 0x18A8), + (0x18AA, 0x18AA), + (0x18B0, 0x18F5), + (0x1900, 0x191E), + (0x1950, 0x196D), + (0x1970, 0x1974), + (0x1980, 0x19AB), + (0x19B0, 0x19C9), + (0x1A00, 0x1A16), + (0x1A20, 0x1A54), + (0x1AA7, 0x1AA7), + (0x1B05, 0x1B33), + (0x1B45, 0x1B4C), + (0x1B83, 0x1BA0), + (0x1BAE, 0x1BAF), + (0x1BBA, 0x1BE5), + (0x1C00, 0x1C23), + (0x1C4D, 0x1C4F), + (0x1C5A, 0x1C7D), + (0x1C80, 0x1C88), + (0x1C90, 0x1CBA), + (0x1CBD, 0x1CBF), + (0x1CE9, 0x1CEC), + (0x1CEE, 0x1CF3), + (0x1CF5, 0x1CF6), + (0x1CFA, 0x1CFA), + (0x1D00, 0x1DBF), + (0x1E00, 0x1F15), + (0x1F18, 0x1F1D), + (0x1F20, 0x1F45), + (0x1F48, 0x1F4D), + (0x1F50, 0x1F57), + (0x1F59, 0x1F59), + (0x1F5B, 0x1F5B), + (0x1F5D, 0x1F5D), + (0x1F5F, 0x1F7D), + (0x1F80, 0x1FB4), + (0x1FB6, 0x1FBC), + (0x1FBE, 0x1FBE), + (0x1FC2, 0x1FC4), + (0x1FC6, 0x1FCC), + (0x1FD0, 0x1FD3), + (0x1FD6, 0x1FDB), + (0x1FE0, 0x1FEC), + (0x1FF2, 0x1FF4), + (0x1FF6, 0x1FFC), + (0x2071, 0x2071), + (0x207F, 0x207F), + (0x2090, 0x209C), + (0x2102, 0x2102), + (0x2107, 0x2107), + (0x210A, 0x2113), + (0x2115, 0x2115), + (0x2119, 0x211D), + (0x2124, 0x2124), + (0x2126, 0x2126), + (0x2128, 0x2128), + (0x212A, 0x212D), + (0x212F, 0x2139), + (0x213C, 0x213F), + (0x2145, 0x2149), + (0x214E, 0x214E), + (0x2183, 0x2184), + (0x2C00, 0x2CE4), + (0x2CEB, 0x2CEE), + (0x2CF2, 0x2CF3), + (0x2D00, 0x2D25), + (0x2D27, 0x2D27), + (0x2D2D, 0x2D2D), + (0x2D30, 0x2D67), + (0x2D6F, 0x2D6F), + (0x2D80, 0x2D96), + (0x2DA0, 0x2DA6), + (0x2DA8, 0x2DAE), + (0x2DB0, 0x2DB6), + (0x2DB8, 0x2DBE), + (0x2DC0, 0x2DC6), + (0x2DC8, 0x2DCE), + (0x2DD0, 0x2DD6), + (0x2DD8, 0x2DDE), + (0x2E2F, 0x2E2F), + (0x3005, 0x3006), + (0x3031, 0x3035), + (0x303B, 0x303C), + (0x3041, 0x3096), + (0x309D, 0x309F), + (0x30A1, 0x30FA), + (0x30FC, 0x30FF), + (0x3105, 0x312F), + (0x3131, 0x318E), + (0x31A0, 0x31BF), + (0x31F0, 0x31FF), + (0x3400, 0x4DBF), + (0x4E00, 0xA48C), + (0xA4D0, 0xA4FD), + (0xA500, 0xA60C), + (0xA610, 0xA61F), + (0xA62A, 0xA62B), + (0xA640, 0xA66E), + (0xA67F, 0xA69D), + (0xA6A0, 0xA6E5), + (0xA717, 0xA71F), + (0xA722, 0xA788), + (0xA78B, 0xA7CA), + (0xA7D0, 0xA7D1), + (0xA7D3, 0xA7D3), + (0xA7D5, 0xA7D9), + (0xA7F2, 0xA801), + (0xA803, 0xA805), + (0xA807, 0xA80A), + (0xA80C, 0xA822), + (0xA840, 0xA873), + (0xA882, 0xA8B3), + (0xA8F2, 0xA8F7), + (0xA8FB, 0xA8FB), + (0xA8FD, 0xA8FE), + (0xA90A, 0xA925), + (0xA930, 0xA946), + (0xA960, 0xA97C), + (0xA984, 0xA9B2), + (0xA9CF, 0xA9CF), + (0xA9E0, 0xA9E4), + (0xA9E6, 0xA9EF), + (0xA9FA, 0xA9FE), + (0xAA00, 0xAA28), + (0xAA40, 0xAA42), + (0xAA44, 0xAA4B), + (0xAA60, 0xAA76), + (0xAA7A, 0xAA7A), + (0xAA7E, 0xAAAF), + (0xAAB1, 0xAAB1), + (0xAAB5, 0xAAB6), + (0xAAB9, 0xAABD), + (0xAAC0, 0xAAC0), + (0xAAC2, 0xAAC2), + (0xAADB, 0xAADD), + (0xAAE0, 0xAAEA), + (0xAAF2, 0xAAF4), + (0xAB01, 0xAB06), + (0xAB09, 0xAB0E), + (0xAB11, 0xAB16), + (0xAB20, 0xAB26), + (0xAB28, 0xAB2E), + (0xAB30, 0xAB5A), + (0xAB5C, 0xAB69), + (0xAB70, 0xABE2), + (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), + (0xD7CB, 0xD7FB), + (0xF900, 0xFA6D), + (0xFA70, 0xFAD9), + (0xFB00, 0xFB06), + (0xFB13, 0xFB17), + (0xFB1D, 0xFB1D), + (0xFB1F, 0xFB28), + (0xFB2A, 0xFB36), + (0xFB38, 0xFB3C), + (0xFB3E, 0xFB3E), + (0xFB40, 0xFB41), + (0xFB43, 0xFB44), + (0xFB46, 0xFBB1), + (0xFBD3, 0xFD3D), + (0xFD50, 0xFD8F), + (0xFD92, 0xFDC7), + (0xFDF0, 0xFDFB), + (0xFE70, 0xFE74), + (0xFE76, 0xFEFC), + (0xFF21, 0xFF3A), + (0xFF41, 0xFF5A), + (0xFF66, 0xFFBE), + (0xFFC2, 0xFFC7), + (0xFFCA, 0xFFCF), + (0xFFD2, 0xFFD7), + (0xFFDA, 0xFFDC), + (0x10000, 0x1000B), + (0x1000D, 0x10026), + (0x10028, 0x1003A), + (0x1003C, 0x1003D), + (0x1003F, 0x1004D), + (0x10050, 0x1005D), + (0x10080, 0x100FA), + (0x10280, 0x1029C), + (0x102A0, 0x102D0), + (0x10300, 0x1031F), + (0x1032D, 0x10340), + (0x10342, 0x10349), + (0x10350, 0x10375), + (0x10380, 0x1039D), + (0x103A0, 0x103C3), + (0x103C8, 0x103CF), + (0x10400, 0x1049D), + (0x104B0, 0x104D3), + (0x104D8, 0x104FB), + (0x10500, 0x10527), + (0x10530, 0x10563), + (0x10570, 0x1057A), + (0x1057C, 0x1058A), + (0x1058C, 0x10592), + (0x10594, 0x10595), + (0x10597, 0x105A1), + (0x105A3, 0x105B1), + (0x105B3, 0x105B9), + (0x105BB, 0x105BC), + (0x10600, 0x10736), + (0x10740, 0x10755), + (0x10760, 0x10767), + (0x10780, 0x10785), + (0x10787, 0x107B0), + (0x107B2, 0x107BA), + (0x10800, 0x10805), + (0x10808, 0x10808), + (0x1080A, 0x10835), + (0x10837, 0x10838), + (0x1083C, 0x1083C), + (0x1083F, 0x10855), + (0x10860, 0x10876), + (0x10880, 0x1089E), + (0x108E0, 0x108F2), + (0x108F4, 0x108F5), + (0x10900, 0x10915), + (0x10920, 0x10939), + (0x10980, 0x109B7), + (0x109BE, 0x109BF), + (0x10A00, 0x10A00), + (0x10A10, 0x10A13), + (0x10A15, 0x10A17), + (0x10A19, 0x10A35), + (0x10A60, 0x10A7C), + (0x10A80, 0x10A9C), + (0x10AC0, 0x10AC7), + (0x10AC9, 0x10AE4), + (0x10B00, 0x10B35), + (0x10B40, 0x10B55), + (0x10B60, 0x10B72), + (0x10B80, 0x10B91), + (0x10C00, 0x10C48), + (0x10C80, 0x10CB2), + (0x10CC0, 0x10CF2), + (0x10D00, 0x10D23), + (0x10E80, 0x10EA9), + (0x10EB0, 0x10EB1), + (0x10F00, 0x10F1C), + (0x10F27, 0x10F27), + (0x10F30, 0x10F45), + (0x10F70, 0x10F81), + (0x10FB0, 0x10FC4), + (0x10FE0, 0x10FF6), + (0x11003, 0x11037), + (0x11071, 0x11072), + (0x11075, 0x11075), + (0x11083, 0x110AF), + (0x110D0, 0x110E8), + (0x11103, 0x11126), + (0x11144, 0x11144), + (0x11147, 0x11147), + (0x11150, 0x11172), + (0x11176, 0x11176), + (0x11183, 0x111B2), + (0x111C1, 0x111C4), + (0x111DA, 0x111DA), + (0x111DC, 0x111DC), + (0x11200, 0x11211), + (0x11213, 0x1122B), + (0x1123F, 0x11240), + (0x11280, 0x11286), + (0x11288, 0x11288), + (0x1128A, 0x1128D), + (0x1128F, 0x1129D), + (0x1129F, 0x112A8), + (0x112B0, 0x112DE), + (0x11305, 0x1130C), + (0x1130F, 0x11310), + (0x11313, 0x11328), + (0x1132A, 0x11330), + (0x11332, 0x11333), + (0x11335, 0x11339), + (0x1133D, 0x1133D), + (0x11350, 0x11350), + (0x1135D, 0x11361), + (0x11400, 0x11434), + (0x11447, 0x1144A), + (0x1145F, 0x11461), + (0x11480, 0x114AF), + (0x114C4, 0x114C5), + (0x114C7, 0x114C7), + (0x11580, 0x115AE), + (0x115D8, 0x115DB), + (0x11600, 0x1162F), + (0x11644, 0x11644), + (0x11680, 0x116AA), + (0x116B8, 0x116B8), + (0x11700, 0x1171A), + (0x11740, 0x11746), + (0x11800, 0x1182B), + (0x118A0, 0x118DF), + (0x118FF, 0x11906), + (0x11909, 0x11909), + (0x1190C, 0x11913), + (0x11915, 0x11916), + (0x11918, 0x1192F), + (0x1193F, 0x1193F), + (0x11941, 0x11941), + (0x119A0, 0x119A7), + (0x119AA, 0x119D0), + (0x119E1, 0x119E1), + (0x119E3, 0x119E3), + (0x11A00, 0x11A00), + (0x11A0B, 0x11A32), + (0x11A3A, 0x11A3A), + (0x11A50, 0x11A50), + (0x11A5C, 0x11A89), + (0x11A9D, 0x11A9D), + (0x11AB0, 0x11AF8), + (0x11C00, 0x11C08), + (0x11C0A, 0x11C2E), + (0x11C40, 0x11C40), + (0x11C72, 0x11C8F), + (0x11D00, 0x11D06), + (0x11D08, 0x11D09), + (0x11D0B, 0x11D30), + (0x11D46, 0x11D46), + (0x11D60, 0x11D65), + (0x11D67, 0x11D68), + (0x11D6A, 0x11D89), + (0x11D98, 0x11D98), + (0x11EE0, 0x11EF2), + (0x11F02, 0x11F02), + (0x11F04, 0x11F10), + (0x11F12, 0x11F33), + (0x11FB0, 0x11FB0), + (0x12000, 0x12399), + (0x12480, 0x12543), + (0x12F90, 0x12FF0), + (0x13000, 0x1342F), + (0x13441, 0x13446), + (0x14400, 0x14646), + (0x16800, 0x16A38), + (0x16A40, 0x16A5E), + (0x16A70, 0x16ABE), + (0x16AD0, 0x16AED), + (0x16B00, 0x16B2F), + (0x16B40, 0x16B43), + (0x16B63, 0x16B77), + (0x16B7D, 0x16B8F), + (0x16E40, 0x16E7F), + (0x16F00, 0x16F4A), + (0x16F50, 0x16F50), + (0x16F93, 0x16F9F), + (0x16FE0, 0x16FE1), + (0x16FE3, 0x16FE3), + (0x17000, 0x187F7), + (0x18800, 0x18CD5), + (0x18D00, 0x18D08), + (0x1AFF0, 0x1AFF3), + (0x1AFF5, 0x1AFFB), + (0x1AFFD, 0x1AFFE), + (0x1B000, 0x1B122), + (0x1B132, 0x1B132), + (0x1B150, 0x1B152), + (0x1B155, 0x1B155), + (0x1B164, 0x1B167), + (0x1B170, 0x1B2FB), + (0x1BC00, 0x1BC6A), + (0x1BC70, 0x1BC7C), + (0x1BC80, 0x1BC88), + (0x1BC90, 0x1BC99), + (0x1D400, 0x1D454), + (0x1D456, 0x1D49C), + (0x1D49E, 0x1D49F), + (0x1D4A2, 0x1D4A2), + (0x1D4A5, 0x1D4A6), + (0x1D4A9, 0x1D4AC), + (0x1D4AE, 0x1D4B9), + (0x1D4BB, 0x1D4BB), + (0x1D4BD, 0x1D4C3), + (0x1D4C5, 0x1D505), + (0x1D507, 0x1D50A), + (0x1D50D, 0x1D514), + (0x1D516, 0x1D51C), + (0x1D51E, 0x1D539), + (0x1D53B, 0x1D53E), + (0x1D540, 0x1D544), + (0x1D546, 0x1D546), + (0x1D54A, 0x1D550), + (0x1D552, 0x1D6A5), + (0x1D6A8, 0x1D6C0), + (0x1D6C2, 0x1D6DA), + (0x1D6DC, 0x1D6FA), + (0x1D6FC, 0x1D714), + (0x1D716, 0x1D734), + (0x1D736, 0x1D74E), + (0x1D750, 0x1D76E), + (0x1D770, 0x1D788), + (0x1D78A, 0x1D7A8), + (0x1D7AA, 0x1D7C2), + (0x1D7C4, 0x1D7CB), + (0x1DF00, 0x1DF1E), + (0x1DF25, 0x1DF2A), + (0x1E030, 0x1E06D), + (0x1E100, 0x1E12C), + (0x1E137, 0x1E13D), + (0x1E14E, 0x1E14E), + (0x1E290, 0x1E2AD), + (0x1E2C0, 0x1E2EB), + (0x1E4D0, 0x1E4EB), + (0x1E7E0, 0x1E7E6), + (0x1E7E8, 0x1E7EB), + (0x1E7ED, 0x1E7EE), + (0x1E7F0, 0x1E7FE), + (0x1E800, 0x1E8C4), + (0x1E900, 0x1E943), + (0x1E94B, 0x1E94B), + (0x1EE00, 0x1EE03), + (0x1EE05, 0x1EE1F), + (0x1EE21, 0x1EE22), + (0x1EE24, 0x1EE24), + (0x1EE27, 0x1EE27), + (0x1EE29, 0x1EE32), + (0x1EE34, 0x1EE37), + (0x1EE39, 0x1EE39), + (0x1EE3B, 0x1EE3B), + (0x1EE42, 0x1EE42), + (0x1EE47, 0x1EE47), + (0x1EE49, 0x1EE49), + (0x1EE4B, 0x1EE4B), + (0x1EE4D, 0x1EE4F), + (0x1EE51, 0x1EE52), + (0x1EE54, 0x1EE54), + (0x1EE57, 0x1EE57), + (0x1EE59, 0x1EE59), + (0x1EE5B, 0x1EE5B), + (0x1EE5D, 0x1EE5D), + (0x1EE5F, 0x1EE5F), + (0x1EE61, 0x1EE62), + (0x1EE64, 0x1EE64), + (0x1EE67, 0x1EE6A), + (0x1EE6C, 0x1EE72), + (0x1EE74, 0x1EE77), + (0x1EE79, 0x1EE7C), + (0x1EE7E, 0x1EE7E), + (0x1EE80, 0x1EE89), + (0x1EE8B, 0x1EE9B), + (0x1EEA1, 0x1EEA3), + (0x1EEA5, 0x1EEA9), + (0x1EEAB, 0x1EEBB), + (0x20000, 0x2A6DF), + (0x2A700, 0x2B739), + (0x2B740, 0x2B81D), + (0x2B820, 0x2CEA1), + (0x2CEB0, 0x2EBE0), + (0x2EBF0, 0x2EE5D), + (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), + (0x31350, 0x323AF), +] diff --git a/crates/sqlbuild-core/src/text/alpha_ranges/unicode_16_0_0.in b/crates/sqlbuild-core/src/text/alpha_ranges/unicode_16_0_0.in new file mode 100644 index 0000000000..d2080cb95d --- /dev/null +++ b/crates/sqlbuild-core/src/text/alpha_ranges/unicode_16_0_0.in @@ -0,0 +1,681 @@ +// Generated by `scripts/generate_python_char_table.py`; do not edit. +// Inclusive code point ranges where `str.isalpha()` is true in Unicode 16.0.0, in order. +&[ + (0x0041, 0x005A), + (0x0061, 0x007A), + (0x00AA, 0x00AA), + (0x00B5, 0x00B5), + (0x00BA, 0x00BA), + (0x00C0, 0x00D6), + (0x00D8, 0x00F6), + (0x00F8, 0x02C1), + (0x02C6, 0x02D1), + (0x02E0, 0x02E4), + (0x02EC, 0x02EC), + (0x02EE, 0x02EE), + (0x0370, 0x0374), + (0x0376, 0x0377), + (0x037A, 0x037D), + (0x037F, 0x037F), + (0x0386, 0x0386), + (0x0388, 0x038A), + (0x038C, 0x038C), + (0x038E, 0x03A1), + (0x03A3, 0x03F5), + (0x03F7, 0x0481), + (0x048A, 0x052F), + (0x0531, 0x0556), + (0x0559, 0x0559), + (0x0560, 0x0588), + (0x05D0, 0x05EA), + (0x05EF, 0x05F2), + (0x0620, 0x064A), + (0x066E, 0x066F), + (0x0671, 0x06D3), + (0x06D5, 0x06D5), + (0x06E5, 0x06E6), + (0x06EE, 0x06EF), + (0x06FA, 0x06FC), + (0x06FF, 0x06FF), + (0x0710, 0x0710), + (0x0712, 0x072F), + (0x074D, 0x07A5), + (0x07B1, 0x07B1), + (0x07CA, 0x07EA), + (0x07F4, 0x07F5), + (0x07FA, 0x07FA), + (0x0800, 0x0815), + (0x081A, 0x081A), + (0x0824, 0x0824), + (0x0828, 0x0828), + (0x0840, 0x0858), + (0x0860, 0x086A), + (0x0870, 0x0887), + (0x0889, 0x088E), + (0x08A0, 0x08C9), + (0x0904, 0x0939), + (0x093D, 0x093D), + (0x0950, 0x0950), + (0x0958, 0x0961), + (0x0971, 0x0980), + (0x0985, 0x098C), + (0x098F, 0x0990), + (0x0993, 0x09A8), + (0x09AA, 0x09B0), + (0x09B2, 0x09B2), + (0x09B6, 0x09B9), + (0x09BD, 0x09BD), + (0x09CE, 0x09CE), + (0x09DC, 0x09DD), + (0x09DF, 0x09E1), + (0x09F0, 0x09F1), + (0x09FC, 0x09FC), + (0x0A05, 0x0A0A), + (0x0A0F, 0x0A10), + (0x0A13, 0x0A28), + (0x0A2A, 0x0A30), + (0x0A32, 0x0A33), + (0x0A35, 0x0A36), + (0x0A38, 0x0A39), + (0x0A59, 0x0A5C), + (0x0A5E, 0x0A5E), + (0x0A72, 0x0A74), + (0x0A85, 0x0A8D), + (0x0A8F, 0x0A91), + (0x0A93, 0x0AA8), + (0x0AAA, 0x0AB0), + (0x0AB2, 0x0AB3), + (0x0AB5, 0x0AB9), + (0x0ABD, 0x0ABD), + (0x0AD0, 0x0AD0), + (0x0AE0, 0x0AE1), + (0x0AF9, 0x0AF9), + (0x0B05, 0x0B0C), + (0x0B0F, 0x0B10), + (0x0B13, 0x0B28), + (0x0B2A, 0x0B30), + (0x0B32, 0x0B33), + (0x0B35, 0x0B39), + (0x0B3D, 0x0B3D), + (0x0B5C, 0x0B5D), + (0x0B5F, 0x0B61), + (0x0B71, 0x0B71), + (0x0B83, 0x0B83), + (0x0B85, 0x0B8A), + (0x0B8E, 0x0B90), + (0x0B92, 0x0B95), + (0x0B99, 0x0B9A), + (0x0B9C, 0x0B9C), + (0x0B9E, 0x0B9F), + (0x0BA3, 0x0BA4), + (0x0BA8, 0x0BAA), + (0x0BAE, 0x0BB9), + (0x0BD0, 0x0BD0), + (0x0C05, 0x0C0C), + (0x0C0E, 0x0C10), + (0x0C12, 0x0C28), + (0x0C2A, 0x0C39), + (0x0C3D, 0x0C3D), + (0x0C58, 0x0C5A), + (0x0C5D, 0x0C5D), + (0x0C60, 0x0C61), + (0x0C80, 0x0C80), + (0x0C85, 0x0C8C), + (0x0C8E, 0x0C90), + (0x0C92, 0x0CA8), + (0x0CAA, 0x0CB3), + (0x0CB5, 0x0CB9), + (0x0CBD, 0x0CBD), + (0x0CDD, 0x0CDE), + (0x0CE0, 0x0CE1), + (0x0CF1, 0x0CF2), + (0x0D04, 0x0D0C), + (0x0D0E, 0x0D10), + (0x0D12, 0x0D3A), + (0x0D3D, 0x0D3D), + (0x0D4E, 0x0D4E), + (0x0D54, 0x0D56), + (0x0D5F, 0x0D61), + (0x0D7A, 0x0D7F), + (0x0D85, 0x0D96), + (0x0D9A, 0x0DB1), + (0x0DB3, 0x0DBB), + (0x0DBD, 0x0DBD), + (0x0DC0, 0x0DC6), + (0x0E01, 0x0E30), + (0x0E32, 0x0E33), + (0x0E40, 0x0E46), + (0x0E81, 0x0E82), + (0x0E84, 0x0E84), + (0x0E86, 0x0E8A), + (0x0E8C, 0x0EA3), + (0x0EA5, 0x0EA5), + (0x0EA7, 0x0EB0), + (0x0EB2, 0x0EB3), + (0x0EBD, 0x0EBD), + (0x0EC0, 0x0EC4), + (0x0EC6, 0x0EC6), + (0x0EDC, 0x0EDF), + (0x0F00, 0x0F00), + (0x0F40, 0x0F47), + (0x0F49, 0x0F6C), + (0x0F88, 0x0F8C), + (0x1000, 0x102A), + (0x103F, 0x103F), + (0x1050, 0x1055), + (0x105A, 0x105D), + (0x1061, 0x1061), + (0x1065, 0x1066), + (0x106E, 0x1070), + (0x1075, 0x1081), + (0x108E, 0x108E), + (0x10A0, 0x10C5), + (0x10C7, 0x10C7), + (0x10CD, 0x10CD), + (0x10D0, 0x10FA), + (0x10FC, 0x1248), + (0x124A, 0x124D), + (0x1250, 0x1256), + (0x1258, 0x1258), + (0x125A, 0x125D), + (0x1260, 0x1288), + (0x128A, 0x128D), + (0x1290, 0x12B0), + (0x12B2, 0x12B5), + (0x12B8, 0x12BE), + (0x12C0, 0x12C0), + (0x12C2, 0x12C5), + (0x12C8, 0x12D6), + (0x12D8, 0x1310), + (0x1312, 0x1315), + (0x1318, 0x135A), + (0x1380, 0x138F), + (0x13A0, 0x13F5), + (0x13F8, 0x13FD), + (0x1401, 0x166C), + (0x166F, 0x167F), + (0x1681, 0x169A), + (0x16A0, 0x16EA), + (0x16F1, 0x16F8), + (0x1700, 0x1711), + (0x171F, 0x1731), + (0x1740, 0x1751), + (0x1760, 0x176C), + (0x176E, 0x1770), + (0x1780, 0x17B3), + (0x17D7, 0x17D7), + (0x17DC, 0x17DC), + (0x1820, 0x1878), + (0x1880, 0x1884), + (0x1887, 0x18A8), + (0x18AA, 0x18AA), + (0x18B0, 0x18F5), + (0x1900, 0x191E), + (0x1950, 0x196D), + (0x1970, 0x1974), + (0x1980, 0x19AB), + (0x19B0, 0x19C9), + (0x1A00, 0x1A16), + (0x1A20, 0x1A54), + (0x1AA7, 0x1AA7), + (0x1B05, 0x1B33), + (0x1B45, 0x1B4C), + (0x1B83, 0x1BA0), + (0x1BAE, 0x1BAF), + (0x1BBA, 0x1BE5), + (0x1C00, 0x1C23), + (0x1C4D, 0x1C4F), + (0x1C5A, 0x1C7D), + (0x1C80, 0x1C8A), + (0x1C90, 0x1CBA), + (0x1CBD, 0x1CBF), + (0x1CE9, 0x1CEC), + (0x1CEE, 0x1CF3), + (0x1CF5, 0x1CF6), + (0x1CFA, 0x1CFA), + (0x1D00, 0x1DBF), + (0x1E00, 0x1F15), + (0x1F18, 0x1F1D), + (0x1F20, 0x1F45), + (0x1F48, 0x1F4D), + (0x1F50, 0x1F57), + (0x1F59, 0x1F59), + (0x1F5B, 0x1F5B), + (0x1F5D, 0x1F5D), + (0x1F5F, 0x1F7D), + (0x1F80, 0x1FB4), + (0x1FB6, 0x1FBC), + (0x1FBE, 0x1FBE), + (0x1FC2, 0x1FC4), + (0x1FC6, 0x1FCC), + (0x1FD0, 0x1FD3), + (0x1FD6, 0x1FDB), + (0x1FE0, 0x1FEC), + (0x1FF2, 0x1FF4), + (0x1FF6, 0x1FFC), + (0x2071, 0x2071), + (0x207F, 0x207F), + (0x2090, 0x209C), + (0x2102, 0x2102), + (0x2107, 0x2107), + (0x210A, 0x2113), + (0x2115, 0x2115), + (0x2119, 0x211D), + (0x2124, 0x2124), + (0x2126, 0x2126), + (0x2128, 0x2128), + (0x212A, 0x212D), + (0x212F, 0x2139), + (0x213C, 0x213F), + (0x2145, 0x2149), + (0x214E, 0x214E), + (0x2183, 0x2184), + (0x2C00, 0x2CE4), + (0x2CEB, 0x2CEE), + (0x2CF2, 0x2CF3), + (0x2D00, 0x2D25), + (0x2D27, 0x2D27), + (0x2D2D, 0x2D2D), + (0x2D30, 0x2D67), + (0x2D6F, 0x2D6F), + (0x2D80, 0x2D96), + (0x2DA0, 0x2DA6), + (0x2DA8, 0x2DAE), + (0x2DB0, 0x2DB6), + (0x2DB8, 0x2DBE), + (0x2DC0, 0x2DC6), + (0x2DC8, 0x2DCE), + (0x2DD0, 0x2DD6), + (0x2DD8, 0x2DDE), + (0x2E2F, 0x2E2F), + (0x3005, 0x3006), + (0x3031, 0x3035), + (0x303B, 0x303C), + (0x3041, 0x3096), + (0x309D, 0x309F), + (0x30A1, 0x30FA), + (0x30FC, 0x30FF), + (0x3105, 0x312F), + (0x3131, 0x318E), + (0x31A0, 0x31BF), + (0x31F0, 0x31FF), + (0x3400, 0x4DBF), + (0x4E00, 0xA48C), + (0xA4D0, 0xA4FD), + (0xA500, 0xA60C), + (0xA610, 0xA61F), + (0xA62A, 0xA62B), + (0xA640, 0xA66E), + (0xA67F, 0xA69D), + (0xA6A0, 0xA6E5), + (0xA717, 0xA71F), + (0xA722, 0xA788), + (0xA78B, 0xA7CD), + (0xA7D0, 0xA7D1), + (0xA7D3, 0xA7D3), + (0xA7D5, 0xA7DC), + (0xA7F2, 0xA801), + (0xA803, 0xA805), + (0xA807, 0xA80A), + (0xA80C, 0xA822), + (0xA840, 0xA873), + (0xA882, 0xA8B3), + (0xA8F2, 0xA8F7), + (0xA8FB, 0xA8FB), + (0xA8FD, 0xA8FE), + (0xA90A, 0xA925), + (0xA930, 0xA946), + (0xA960, 0xA97C), + (0xA984, 0xA9B2), + (0xA9CF, 0xA9CF), + (0xA9E0, 0xA9E4), + (0xA9E6, 0xA9EF), + (0xA9FA, 0xA9FE), + (0xAA00, 0xAA28), + (0xAA40, 0xAA42), + (0xAA44, 0xAA4B), + (0xAA60, 0xAA76), + (0xAA7A, 0xAA7A), + (0xAA7E, 0xAAAF), + (0xAAB1, 0xAAB1), + (0xAAB5, 0xAAB6), + (0xAAB9, 0xAABD), + (0xAAC0, 0xAAC0), + (0xAAC2, 0xAAC2), + (0xAADB, 0xAADD), + (0xAAE0, 0xAAEA), + (0xAAF2, 0xAAF4), + (0xAB01, 0xAB06), + (0xAB09, 0xAB0E), + (0xAB11, 0xAB16), + (0xAB20, 0xAB26), + (0xAB28, 0xAB2E), + (0xAB30, 0xAB5A), + (0xAB5C, 0xAB69), + (0xAB70, 0xABE2), + (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), + (0xD7CB, 0xD7FB), + (0xF900, 0xFA6D), + (0xFA70, 0xFAD9), + (0xFB00, 0xFB06), + (0xFB13, 0xFB17), + (0xFB1D, 0xFB1D), + (0xFB1F, 0xFB28), + (0xFB2A, 0xFB36), + (0xFB38, 0xFB3C), + (0xFB3E, 0xFB3E), + (0xFB40, 0xFB41), + (0xFB43, 0xFB44), + (0xFB46, 0xFBB1), + (0xFBD3, 0xFD3D), + (0xFD50, 0xFD8F), + (0xFD92, 0xFDC7), + (0xFDF0, 0xFDFB), + (0xFE70, 0xFE74), + (0xFE76, 0xFEFC), + (0xFF21, 0xFF3A), + (0xFF41, 0xFF5A), + (0xFF66, 0xFFBE), + (0xFFC2, 0xFFC7), + (0xFFCA, 0xFFCF), + (0xFFD2, 0xFFD7), + (0xFFDA, 0xFFDC), + (0x10000, 0x1000B), + (0x1000D, 0x10026), + (0x10028, 0x1003A), + (0x1003C, 0x1003D), + (0x1003F, 0x1004D), + (0x10050, 0x1005D), + (0x10080, 0x100FA), + (0x10280, 0x1029C), + (0x102A0, 0x102D0), + (0x10300, 0x1031F), + (0x1032D, 0x10340), + (0x10342, 0x10349), + (0x10350, 0x10375), + (0x10380, 0x1039D), + (0x103A0, 0x103C3), + (0x103C8, 0x103CF), + (0x10400, 0x1049D), + (0x104B0, 0x104D3), + (0x104D8, 0x104FB), + (0x10500, 0x10527), + (0x10530, 0x10563), + (0x10570, 0x1057A), + (0x1057C, 0x1058A), + (0x1058C, 0x10592), + (0x10594, 0x10595), + (0x10597, 0x105A1), + (0x105A3, 0x105B1), + (0x105B3, 0x105B9), + (0x105BB, 0x105BC), + (0x105C0, 0x105F3), + (0x10600, 0x10736), + (0x10740, 0x10755), + (0x10760, 0x10767), + (0x10780, 0x10785), + (0x10787, 0x107B0), + (0x107B2, 0x107BA), + (0x10800, 0x10805), + (0x10808, 0x10808), + (0x1080A, 0x10835), + (0x10837, 0x10838), + (0x1083C, 0x1083C), + (0x1083F, 0x10855), + (0x10860, 0x10876), + (0x10880, 0x1089E), + (0x108E0, 0x108F2), + (0x108F4, 0x108F5), + (0x10900, 0x10915), + (0x10920, 0x10939), + (0x10980, 0x109B7), + (0x109BE, 0x109BF), + (0x10A00, 0x10A00), + (0x10A10, 0x10A13), + (0x10A15, 0x10A17), + (0x10A19, 0x10A35), + (0x10A60, 0x10A7C), + (0x10A80, 0x10A9C), + (0x10AC0, 0x10AC7), + (0x10AC9, 0x10AE4), + (0x10B00, 0x10B35), + (0x10B40, 0x10B55), + (0x10B60, 0x10B72), + (0x10B80, 0x10B91), + (0x10C00, 0x10C48), + (0x10C80, 0x10CB2), + (0x10CC0, 0x10CF2), + (0x10D00, 0x10D23), + (0x10D4A, 0x10D65), + (0x10D6F, 0x10D85), + (0x10E80, 0x10EA9), + (0x10EB0, 0x10EB1), + (0x10EC2, 0x10EC4), + (0x10F00, 0x10F1C), + (0x10F27, 0x10F27), + (0x10F30, 0x10F45), + (0x10F70, 0x10F81), + (0x10FB0, 0x10FC4), + (0x10FE0, 0x10FF6), + (0x11003, 0x11037), + (0x11071, 0x11072), + (0x11075, 0x11075), + (0x11083, 0x110AF), + (0x110D0, 0x110E8), + (0x11103, 0x11126), + (0x11144, 0x11144), + (0x11147, 0x11147), + (0x11150, 0x11172), + (0x11176, 0x11176), + (0x11183, 0x111B2), + (0x111C1, 0x111C4), + (0x111DA, 0x111DA), + (0x111DC, 0x111DC), + (0x11200, 0x11211), + (0x11213, 0x1122B), + (0x1123F, 0x11240), + (0x11280, 0x11286), + (0x11288, 0x11288), + (0x1128A, 0x1128D), + (0x1128F, 0x1129D), + (0x1129F, 0x112A8), + (0x112B0, 0x112DE), + (0x11305, 0x1130C), + (0x1130F, 0x11310), + (0x11313, 0x11328), + (0x1132A, 0x11330), + (0x11332, 0x11333), + (0x11335, 0x11339), + (0x1133D, 0x1133D), + (0x11350, 0x11350), + (0x1135D, 0x11361), + (0x11380, 0x11389), + (0x1138B, 0x1138B), + (0x1138E, 0x1138E), + (0x11390, 0x113B5), + (0x113B7, 0x113B7), + (0x113D1, 0x113D1), + (0x113D3, 0x113D3), + (0x11400, 0x11434), + (0x11447, 0x1144A), + (0x1145F, 0x11461), + (0x11480, 0x114AF), + (0x114C4, 0x114C5), + (0x114C7, 0x114C7), + (0x11580, 0x115AE), + (0x115D8, 0x115DB), + (0x11600, 0x1162F), + (0x11644, 0x11644), + (0x11680, 0x116AA), + (0x116B8, 0x116B8), + (0x11700, 0x1171A), + (0x11740, 0x11746), + (0x11800, 0x1182B), + (0x118A0, 0x118DF), + (0x118FF, 0x11906), + (0x11909, 0x11909), + (0x1190C, 0x11913), + (0x11915, 0x11916), + (0x11918, 0x1192F), + (0x1193F, 0x1193F), + (0x11941, 0x11941), + (0x119A0, 0x119A7), + (0x119AA, 0x119D0), + (0x119E1, 0x119E1), + (0x119E3, 0x119E3), + (0x11A00, 0x11A00), + (0x11A0B, 0x11A32), + (0x11A3A, 0x11A3A), + (0x11A50, 0x11A50), + (0x11A5C, 0x11A89), + (0x11A9D, 0x11A9D), + (0x11AB0, 0x11AF8), + (0x11BC0, 0x11BE0), + (0x11C00, 0x11C08), + (0x11C0A, 0x11C2E), + (0x11C40, 0x11C40), + (0x11C72, 0x11C8F), + (0x11D00, 0x11D06), + (0x11D08, 0x11D09), + (0x11D0B, 0x11D30), + (0x11D46, 0x11D46), + (0x11D60, 0x11D65), + (0x11D67, 0x11D68), + (0x11D6A, 0x11D89), + (0x11D98, 0x11D98), + (0x11EE0, 0x11EF2), + (0x11F02, 0x11F02), + (0x11F04, 0x11F10), + (0x11F12, 0x11F33), + (0x11FB0, 0x11FB0), + (0x12000, 0x12399), + (0x12480, 0x12543), + (0x12F90, 0x12FF0), + (0x13000, 0x1342F), + (0x13441, 0x13446), + (0x13460, 0x143FA), + (0x14400, 0x14646), + (0x16100, 0x1611D), + (0x16800, 0x16A38), + (0x16A40, 0x16A5E), + (0x16A70, 0x16ABE), + (0x16AD0, 0x16AED), + (0x16B00, 0x16B2F), + (0x16B40, 0x16B43), + (0x16B63, 0x16B77), + (0x16B7D, 0x16B8F), + (0x16D40, 0x16D6C), + (0x16E40, 0x16E7F), + (0x16F00, 0x16F4A), + (0x16F50, 0x16F50), + (0x16F93, 0x16F9F), + (0x16FE0, 0x16FE1), + (0x16FE3, 0x16FE3), + (0x17000, 0x187F7), + (0x18800, 0x18CD5), + (0x18CFF, 0x18D08), + (0x1AFF0, 0x1AFF3), + (0x1AFF5, 0x1AFFB), + (0x1AFFD, 0x1AFFE), + (0x1B000, 0x1B122), + (0x1B132, 0x1B132), + (0x1B150, 0x1B152), + (0x1B155, 0x1B155), + (0x1B164, 0x1B167), + (0x1B170, 0x1B2FB), + (0x1BC00, 0x1BC6A), + (0x1BC70, 0x1BC7C), + (0x1BC80, 0x1BC88), + (0x1BC90, 0x1BC99), + (0x1D400, 0x1D454), + (0x1D456, 0x1D49C), + (0x1D49E, 0x1D49F), + (0x1D4A2, 0x1D4A2), + (0x1D4A5, 0x1D4A6), + (0x1D4A9, 0x1D4AC), + (0x1D4AE, 0x1D4B9), + (0x1D4BB, 0x1D4BB), + (0x1D4BD, 0x1D4C3), + (0x1D4C5, 0x1D505), + (0x1D507, 0x1D50A), + (0x1D50D, 0x1D514), + (0x1D516, 0x1D51C), + (0x1D51E, 0x1D539), + (0x1D53B, 0x1D53E), + (0x1D540, 0x1D544), + (0x1D546, 0x1D546), + (0x1D54A, 0x1D550), + (0x1D552, 0x1D6A5), + (0x1D6A8, 0x1D6C0), + (0x1D6C2, 0x1D6DA), + (0x1D6DC, 0x1D6FA), + (0x1D6FC, 0x1D714), + (0x1D716, 0x1D734), + (0x1D736, 0x1D74E), + (0x1D750, 0x1D76E), + (0x1D770, 0x1D788), + (0x1D78A, 0x1D7A8), + (0x1D7AA, 0x1D7C2), + (0x1D7C4, 0x1D7CB), + (0x1DF00, 0x1DF1E), + (0x1DF25, 0x1DF2A), + (0x1E030, 0x1E06D), + (0x1E100, 0x1E12C), + (0x1E137, 0x1E13D), + (0x1E14E, 0x1E14E), + (0x1E290, 0x1E2AD), + (0x1E2C0, 0x1E2EB), + (0x1E4D0, 0x1E4EB), + (0x1E5D0, 0x1E5ED), + (0x1E5F0, 0x1E5F0), + (0x1E7E0, 0x1E7E6), + (0x1E7E8, 0x1E7EB), + (0x1E7ED, 0x1E7EE), + (0x1E7F0, 0x1E7FE), + (0x1E800, 0x1E8C4), + (0x1E900, 0x1E943), + (0x1E94B, 0x1E94B), + (0x1EE00, 0x1EE03), + (0x1EE05, 0x1EE1F), + (0x1EE21, 0x1EE22), + (0x1EE24, 0x1EE24), + (0x1EE27, 0x1EE27), + (0x1EE29, 0x1EE32), + (0x1EE34, 0x1EE37), + (0x1EE39, 0x1EE39), + (0x1EE3B, 0x1EE3B), + (0x1EE42, 0x1EE42), + (0x1EE47, 0x1EE47), + (0x1EE49, 0x1EE49), + (0x1EE4B, 0x1EE4B), + (0x1EE4D, 0x1EE4F), + (0x1EE51, 0x1EE52), + (0x1EE54, 0x1EE54), + (0x1EE57, 0x1EE57), + (0x1EE59, 0x1EE59), + (0x1EE5B, 0x1EE5B), + (0x1EE5D, 0x1EE5D), + (0x1EE5F, 0x1EE5F), + (0x1EE61, 0x1EE62), + (0x1EE64, 0x1EE64), + (0x1EE67, 0x1EE6A), + (0x1EE6C, 0x1EE72), + (0x1EE74, 0x1EE77), + (0x1EE79, 0x1EE7C), + (0x1EE7E, 0x1EE7E), + (0x1EE80, 0x1EE89), + (0x1EE8B, 0x1EE9B), + (0x1EEA1, 0x1EEA3), + (0x1EEA5, 0x1EEA9), + (0x1EEAB, 0x1EEBB), + (0x20000, 0x2A6DF), + (0x2A700, 0x2B739), + (0x2B740, 0x2B81D), + (0x2B820, 0x2CEA1), + (0x2CEB0, 0x2EBE0), + (0x2EBF0, 0x2EE5D), + (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), + (0x31350, 0x323AF), +] diff --git a/crates/sqlbuild-core/src/text/decimal_ranges/unicode_15_0_0.in b/crates/sqlbuild-core/src/text/decimal_ranges/unicode_15_0_0.in new file mode 100644 index 0000000000..237a5d10d5 --- /dev/null +++ b/crates/sqlbuild-core/src/text/decimal_ranges/unicode_15_0_0.in @@ -0,0 +1,68 @@ +// Generated by `scripts/generate_python_char_table.py`; do not edit. +// Inclusive code point ranges where `str.isdecimal()` is true in Unicode 15.0.0, in order. +&[ + (0x0030, 0x0039), + (0x0660, 0x0669), + (0x06F0, 0x06F9), + (0x07C0, 0x07C9), + (0x0966, 0x096F), + (0x09E6, 0x09EF), + (0x0A66, 0x0A6F), + (0x0AE6, 0x0AEF), + (0x0B66, 0x0B6F), + (0x0BE6, 0x0BEF), + (0x0C66, 0x0C6F), + (0x0CE6, 0x0CEF), + (0x0D66, 0x0D6F), + (0x0DE6, 0x0DEF), + (0x0E50, 0x0E59), + (0x0ED0, 0x0ED9), + (0x0F20, 0x0F29), + (0x1040, 0x1049), + (0x1090, 0x1099), + (0x17E0, 0x17E9), + (0x1810, 0x1819), + (0x1946, 0x194F), + (0x19D0, 0x19D9), + (0x1A80, 0x1A89), + (0x1A90, 0x1A99), + (0x1B50, 0x1B59), + (0x1BB0, 0x1BB9), + (0x1C40, 0x1C49), + (0x1C50, 0x1C59), + (0xA620, 0xA629), + (0xA8D0, 0xA8D9), + (0xA900, 0xA909), + (0xA9D0, 0xA9D9), + (0xA9F0, 0xA9F9), + (0xAA50, 0xAA59), + (0xABF0, 0xABF9), + (0xFF10, 0xFF19), + (0x104A0, 0x104A9), + (0x10D30, 0x10D39), + (0x11066, 0x1106F), + (0x110F0, 0x110F9), + (0x11136, 0x1113F), + (0x111D0, 0x111D9), + (0x112F0, 0x112F9), + (0x11450, 0x11459), + (0x114D0, 0x114D9), + (0x11650, 0x11659), + (0x116C0, 0x116C9), + (0x11730, 0x11739), + (0x118E0, 0x118E9), + (0x11950, 0x11959), + (0x11C50, 0x11C59), + (0x11D50, 0x11D59), + (0x11DA0, 0x11DA9), + (0x11F50, 0x11F59), + (0x16A60, 0x16A69), + (0x16AC0, 0x16AC9), + (0x16B50, 0x16B59), + (0x1D7CE, 0x1D7FF), + (0x1E140, 0x1E149), + (0x1E2F0, 0x1E2F9), + (0x1E4F0, 0x1E4F9), + (0x1E950, 0x1E959), + (0x1FBF0, 0x1FBF9), +] diff --git a/crates/sqlbuild-core/src/text/decimal_ranges/unicode_15_1_0.in b/crates/sqlbuild-core/src/text/decimal_ranges/unicode_15_1_0.in new file mode 100644 index 0000000000..2715a72404 --- /dev/null +++ b/crates/sqlbuild-core/src/text/decimal_ranges/unicode_15_1_0.in @@ -0,0 +1,68 @@ +// Generated by `scripts/generate_python_char_table.py`; do not edit. +// Inclusive code point ranges where `str.isdecimal()` is true in Unicode 15.1.0, in order. +&[ + (0x0030, 0x0039), + (0x0660, 0x0669), + (0x06F0, 0x06F9), + (0x07C0, 0x07C9), + (0x0966, 0x096F), + (0x09E6, 0x09EF), + (0x0A66, 0x0A6F), + (0x0AE6, 0x0AEF), + (0x0B66, 0x0B6F), + (0x0BE6, 0x0BEF), + (0x0C66, 0x0C6F), + (0x0CE6, 0x0CEF), + (0x0D66, 0x0D6F), + (0x0DE6, 0x0DEF), + (0x0E50, 0x0E59), + (0x0ED0, 0x0ED9), + (0x0F20, 0x0F29), + (0x1040, 0x1049), + (0x1090, 0x1099), + (0x17E0, 0x17E9), + (0x1810, 0x1819), + (0x1946, 0x194F), + (0x19D0, 0x19D9), + (0x1A80, 0x1A89), + (0x1A90, 0x1A99), + (0x1B50, 0x1B59), + (0x1BB0, 0x1BB9), + (0x1C40, 0x1C49), + (0x1C50, 0x1C59), + (0xA620, 0xA629), + (0xA8D0, 0xA8D9), + (0xA900, 0xA909), + (0xA9D0, 0xA9D9), + (0xA9F0, 0xA9F9), + (0xAA50, 0xAA59), + (0xABF0, 0xABF9), + (0xFF10, 0xFF19), + (0x104A0, 0x104A9), + (0x10D30, 0x10D39), + (0x11066, 0x1106F), + (0x110F0, 0x110F9), + (0x11136, 0x1113F), + (0x111D0, 0x111D9), + (0x112F0, 0x112F9), + (0x11450, 0x11459), + (0x114D0, 0x114D9), + (0x11650, 0x11659), + (0x116C0, 0x116C9), + (0x11730, 0x11739), + (0x118E0, 0x118E9), + (0x11950, 0x11959), + (0x11C50, 0x11C59), + (0x11D50, 0x11D59), + (0x11DA0, 0x11DA9), + (0x11F50, 0x11F59), + (0x16A60, 0x16A69), + (0x16AC0, 0x16AC9), + (0x16B50, 0x16B59), + (0x1D7CE, 0x1D7FF), + (0x1E140, 0x1E149), + (0x1E2F0, 0x1E2F9), + (0x1E4F0, 0x1E4F9), + (0x1E950, 0x1E959), + (0x1FBF0, 0x1FBF9), +] diff --git a/crates/sqlbuild-core/src/text/decimal_ranges/unicode_16_0_0.in b/crates/sqlbuild-core/src/text/decimal_ranges/unicode_16_0_0.in new file mode 100644 index 0000000000..b6964c3b9c --- /dev/null +++ b/crates/sqlbuild-core/src/text/decimal_ranges/unicode_16_0_0.in @@ -0,0 +1,75 @@ +// Generated by `scripts/generate_python_char_table.py`; do not edit. +// Inclusive code point ranges where `str.isdecimal()` is true in Unicode 16.0.0, in order. +&[ + (0x0030, 0x0039), + (0x0660, 0x0669), + (0x06F0, 0x06F9), + (0x07C0, 0x07C9), + (0x0966, 0x096F), + (0x09E6, 0x09EF), + (0x0A66, 0x0A6F), + (0x0AE6, 0x0AEF), + (0x0B66, 0x0B6F), + (0x0BE6, 0x0BEF), + (0x0C66, 0x0C6F), + (0x0CE6, 0x0CEF), + (0x0D66, 0x0D6F), + (0x0DE6, 0x0DEF), + (0x0E50, 0x0E59), + (0x0ED0, 0x0ED9), + (0x0F20, 0x0F29), + (0x1040, 0x1049), + (0x1090, 0x1099), + (0x17E0, 0x17E9), + (0x1810, 0x1819), + (0x1946, 0x194F), + (0x19D0, 0x19D9), + (0x1A80, 0x1A89), + (0x1A90, 0x1A99), + (0x1B50, 0x1B59), + (0x1BB0, 0x1BB9), + (0x1C40, 0x1C49), + (0x1C50, 0x1C59), + (0xA620, 0xA629), + (0xA8D0, 0xA8D9), + (0xA900, 0xA909), + (0xA9D0, 0xA9D9), + (0xA9F0, 0xA9F9), + (0xAA50, 0xAA59), + (0xABF0, 0xABF9), + (0xFF10, 0xFF19), + (0x104A0, 0x104A9), + (0x10D30, 0x10D39), + (0x10D40, 0x10D49), + (0x11066, 0x1106F), + (0x110F0, 0x110F9), + (0x11136, 0x1113F), + (0x111D0, 0x111D9), + (0x112F0, 0x112F9), + (0x11450, 0x11459), + (0x114D0, 0x114D9), + (0x11650, 0x11659), + (0x116C0, 0x116C9), + (0x116D0, 0x116E3), + (0x11730, 0x11739), + (0x118E0, 0x118E9), + (0x11950, 0x11959), + (0x11BF0, 0x11BF9), + (0x11C50, 0x11C59), + (0x11D50, 0x11D59), + (0x11DA0, 0x11DA9), + (0x11F50, 0x11F59), + (0x16130, 0x16139), + (0x16A60, 0x16A69), + (0x16AC0, 0x16AC9), + (0x16B50, 0x16B59), + (0x16D70, 0x16D79), + (0x1CCF0, 0x1CCF9), + (0x1D7CE, 0x1D7FF), + (0x1E140, 0x1E149), + (0x1E2F0, 0x1E2F9), + (0x1E4F0, 0x1E4F9), + (0x1E5F1, 0x1E5FA), + (0x1E950, 0x1E959), + (0x1FBF0, 0x1FBF9), +] diff --git a/crates/sqlbuild-core/src/text/main/is_python_alnum.rs b/crates/sqlbuild-core/src/text/main/is_python_alnum.rs index f8575f0ba2..d7024a577d 100644 --- a/crates/sqlbuild-core/src/text/main/is_python_alnum.rs +++ b/crates/sqlbuild-core/src/text/main/is_python_alnum.rs @@ -1,5 +1,6 @@ //! Python's `str.isalnum()` for one character, from a table generated by Python. +use crate::text::_helpers::code_point_ranges::in_ranges; use crate::text::models::PythonText; /// Whether Python's `str.isalnum()` is true for one character. @@ -7,12 +8,5 @@ pub fn is_python_alnum(python: PythonText, character: char) -> bool { if character.is_ascii() { return character.is_ascii_alphanumeric(); } - let code_point: u32 = u32::from(character); - let index: usize = python - .alnum_ranges - .partition_point(|(_, end)| *end < code_point); - python - .alnum_ranges - .get(index) - .is_some_and(|(start, _)| *start <= code_point) + in_ranges(python.alnum_ranges, character) } diff --git a/crates/sqlbuild-core/src/text/main/is_python_alpha.rs b/crates/sqlbuild-core/src/text/main/is_python_alpha.rs new file mode 100644 index 0000000000..aca761d0ed --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/is_python_alpha.rs @@ -0,0 +1,12 @@ +//! Python's `str.isalpha()` for one character, from a table generated by Python. + +use crate::text::_helpers::code_point_ranges::in_ranges; +use crate::text::models::PythonText; + +/// Whether Python's `str.isalpha()` is true for one character. +pub fn is_python_alpha(python: PythonText, character: char) -> bool { + if character.is_ascii() { + return character.is_ascii_alphabetic(); + } + in_ranges(python.alpha_ranges, character) +} diff --git a/crates/sqlbuild-core/src/text/main/is_python_decimal.rs b/crates/sqlbuild-core/src/text/main/is_python_decimal.rs new file mode 100644 index 0000000000..c98d7658b3 --- /dev/null +++ b/crates/sqlbuild-core/src/text/main/is_python_decimal.rs @@ -0,0 +1,12 @@ +//! Python's `str.isdecimal()`, the regex `\d` of `str` patterns, from a generated table. + +use crate::text::_helpers::code_point_ranges::in_ranges; +use crate::text::models::PythonText; + +/// Whether Python's `str.isdecimal()` is true for one character. +pub fn is_python_decimal(python: PythonText, character: char) -> bool { + if character.is_ascii() { + return character.is_ascii_digit(); + } + in_ranges(python.decimal_ranges, character) +} diff --git a/crates/sqlbuild-core/src/text/main/mod.rs b/crates/sqlbuild-core/src/text/main/mod.rs index a0a0883793..2b20809e36 100644 --- a/crates/sqlbuild-core/src/text/main/mod.rs +++ b/crates/sqlbuild-core/src/text/main/mod.rs @@ -1,6 +1,8 @@ pub mod close_matches; pub mod decode_python_text; pub mod is_python_alnum; +pub mod is_python_alpha; +pub mod is_python_decimal; pub mod is_python_space; pub mod is_python_word; pub mod python_cleandoc; diff --git a/crates/sqlbuild-core/src/text/main/python_text.rs b/crates/sqlbuild-core/src/text/main/python_text.rs index 23f6a8efee..f8d2bcfcdf 100644 --- a/crates/sqlbuild-core/src/text/main/python_text.rs +++ b/crates/sqlbuild-core/src/text/main/python_text.rs @@ -1,6 +1,6 @@ //! Select the Python string semantics of one CPython release. -use crate::text::_helpers::alnum_tables::ALNUM_TABLES; +use crate::text::_helpers::alnum_tables::{ALNUM_TABLES, ALPHA_TABLES, DECIMAL_TABLES}; use crate::text::models::{CleandocMargin, PythonText}; /// The semantics of one Python and its `unicodedata.unidata_version`, or `None` if unknown. @@ -14,8 +14,19 @@ pub fn python_text(python_version: (u8, u8), unicode_version: &str) -> Option LineIndex<'text> { pub struct PythonText { /// Inclusive code point ranges where `str.isalnum()` is true, in order. pub(crate) alnum_ranges: &'static [(u32, u32)], + /// Inclusive code point ranges where `str.isalpha()` is true, in order. + pub(crate) alpha_ranges: &'static [(u32, u32)], + /// Inclusive code point ranges where `str.isdecimal()` (regex `\d`) is true, in order. + pub(crate) decimal_ranges: &'static [(u32, u32)], pub(crate) cleandoc_margin: CleandocMargin, + /// The CPython minor version, for library behaviour that changed between 3.x releases. + pub(crate) minor_version: u8, +} + +impl PythonText { + /// The CPython minor version these semantics follow. + pub fn minor_version(&self) -> u8 { + self.minor_version + } } /// The characters `inspect.cleandoc` strips from line starts. diff --git a/crates/sqlbuild-core/src/text/tests/test_python_compat.rs b/crates/sqlbuild-core/src/text/tests/test_python_compat.rs index e78053d265..e09e2ec546 100644 --- a/crates/sqlbuild-core/src/text/tests/test_python_compat.rs +++ b/crates/sqlbuild-core/src/text/tests/test_python_compat.rs @@ -1,5 +1,7 @@ use crate::text::main::close_matches::close_matches; use crate::text::main::is_python_alnum::is_python_alnum; +use crate::text::main::is_python_alpha::is_python_alpha; +use crate::text::main::is_python_decimal::is_python_decimal; use crate::text::main::is_python_space::is_python_space; use crate::text::main::is_python_word::is_python_word; use crate::text::main::python_cleandoc::python_cleandoc; @@ -93,6 +95,8 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { CharacterClassTestCase { description: "ASCII letter", character: 'a', + expected_decimal: false, + expected_alpha: true, expected_alnum: true, expected_word: true, expected_space: false, @@ -100,6 +104,8 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { CharacterClassTestCase { description: "underscore is a word character only", character: '_', + expected_decimal: false, + expected_alpha: false, expected_alnum: false, expected_word: true, expected_space: false, @@ -107,6 +113,8 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { CharacterClassTestCase { description: "Arabic-Indic digit", character: '\u{663}', + expected_decimal: true, + expected_alpha: false, expected_alnum: true, expected_word: true, expected_space: false, @@ -114,6 +122,8 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { CharacterClassTestCase { description: "letter number", character: '\u{2167}', + expected_decimal: false, + expected_alpha: false, expected_alnum: true, expected_word: true, expected_space: false, @@ -121,6 +131,8 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { CharacterClassTestCase { description: "combining mark that Rust calls alphabetic", character: '\u{345}', + expected_decimal: false, + expected_alpha: false, expected_alnum: false, expected_word: false, expected_space: false, @@ -128,6 +140,8 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { CharacterClassTestCase { description: "circled letter symbol", character: '\u{24b6}', + expected_decimal: false, + expected_alpha: false, expected_alnum: false, expected_word: false, expected_space: false, @@ -135,13 +149,26 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { CharacterClassTestCase { description: "information separator is whitespace", character: '\u{1c}', + expected_decimal: false, + expected_alpha: false, expected_alnum: false, expected_word: false, expected_space: true, }, + CharacterClassTestCase { + description: "Latin letter with an accent is alphabetic", + character: '\u{e9}', + expected_decimal: false, + expected_alpha: true, + expected_alnum: true, + expected_word: true, + expected_space: false, + }, CharacterClassTestCase { description: "ideographic space", character: '\u{3000}', + expected_decimal: false, + expected_alpha: false, expected_alnum: false, expected_word: false, expected_space: true, @@ -151,11 +178,15 @@ fn given_characters_when_classifying_then_python_str_methods_agree() { for test_case in test_cases { assert_eq!( ( + is_python_decimal(python, test_case.character), + is_python_alpha(python, test_case.character), is_python_alnum(python, test_case.character), is_python_word(python, test_case.character), is_python_space(test_case.character) ), ( + test_case.expected_decimal, + test_case.expected_alpha, test_case.expected_alnum, test_case.expected_word, test_case.expected_space diff --git a/crates/sqlbuild-core/src/text/tests/test_types.rs b/crates/sqlbuild-core/src/text/tests/test_types.rs index 18e6fbd116..4b5d761cf5 100644 --- a/crates/sqlbuild-core/src/text/tests/test_types.rs +++ b/crates/sqlbuild-core/src/text/tests/test_types.rs @@ -30,6 +30,8 @@ pub(super) struct CloseMatchesTestCase { pub(super) struct CharacterClassTestCase { pub(super) description: &'static str, pub(super) character: char, + pub(super) expected_decimal: bool, + pub(super) expected_alpha: bool, pub(super) expected_alnum: bool, pub(super) expected_word: bool, pub(super) expected_space: bool, diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/python_values.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/python_values.rs index f59b447d76..477d4aadf5 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/python_values.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/python_values.rs @@ -2,9 +2,18 @@ use crate::declaration_files::_helpers::checks::stops::ParseStop; use crate::models::{DiscoveryFailure, FailureKind}; +use sqlbuild_core::text::main::is_python_decimal::is_python_decimal; use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_core::text::models::PythonText; use sqlbuild_sqltext::compiler::models::AuthoredValue; +/// The Python whose `\d` decides bare-word numbers, and the file a rejected word is reported in. +#[derive(Clone, Copy, Debug)] +pub(crate) struct WordRules<'file> { + pub(crate) python: PythonText, + pub(crate) file_path: &'file str, +} + /// The Python type a header value projects to. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum PythonType { @@ -19,12 +28,15 @@ pub(crate) enum PythonType { Other, } -/// The projected type; a bare word with non-ASCII text may be a Unicode number, so it defers. -pub(crate) fn python_type(value: &AuthoredValue) -> Result { +/// The projected type; a bare number written with non-ASCII digits is rejected. +pub(crate) fn python_type( + value: &AuthoredValue, + words: WordRules, +) -> Result { Ok(match value { AuthoredValue::Null => PythonType::None, AuthoredValue::Boolean(_) => PythonType::Bool, - AuthoredValue::BareWord(word) => word_type(word)?, + AuthoredValue::BareWord(word) => word_type(word, words)?, AuthoredValue::String(_) => PythonType::Str, AuthoredValue::List(_) => PythonType::List, AuthoredValue::Map(_) => PythonType::Dict, @@ -37,18 +49,46 @@ pub(crate) fn python_type(value: &AuthoredValue) -> Result Result<(), ParseStop> { + match value { + AuthoredValue::BareWord(word) => word_type(word, words).map(|_| ()), + AuthoredValue::List(items) | AuthoredValue::Set(items) | AuthoredValue::Tuple(items) => { + items + .iter() + .try_for_each(|item| check_bare_numbers(item, words)) + } + AuthoredValue::Map(entries) + | AuthoredValue::TypedConstant(entries) + | AuthoredValue::NamedSqlHook(_, entries) + | AuthoredValue::PythonHook(_, entries) => entries + .iter() + .try_for_each(|(_, item)| check_bare_numbers(item, words)), + AuthoredValue::Null + | AuthoredValue::Boolean(_) + | AuthoredValue::String(_) + | AuthoredValue::InlineSqlHook(_) => Ok(()), + } +} + /// The text of a value that projects to `str`, or `None` for any other type. -pub(crate) fn python_str(value: &AuthoredValue) -> Result, ParseStop> { +pub(crate) fn python_str<'value>( + value: &'value AuthoredValue, + words: WordRules, +) -> Result, ParseStop> { Ok(match value { AuthoredValue::String(text) => Some(text), - AuthoredValue::BareWord(word) if word_type(word)? == PythonType::Str => Some(word), + AuthoredValue::BareWord(word) if word_type(word, words)? == PythonType::Str => Some(word), _ => None, }) } /// `isinstance(value, str) and value.strip()`. -pub(crate) fn non_empty_str(value: &AuthoredValue) -> Result, ParseStop> { - Ok(python_str(value)?.filter(|text| !python_strip(text).is_empty())) +pub(crate) fn non_empty_str<'value>( + value: &'value AuthoredValue, + words: WordRules, +) -> Result, ParseStop> { + Ok(python_str(value, words)?.filter(|text| !python_strip(text).is_empty())) } /// Python's `dict.get(key)`. @@ -89,9 +129,22 @@ pub(crate) fn failure(kind: FailureKind, message: String) -> ParseStop { } /// `_parse_word_value`: `true`, `false` and `null` are parsed as values; numbers become numbers. -fn word_type(word: &str) -> Result { +fn word_type(word: &str, words: WordRules) -> Result { if !word.is_ascii() { - return Err(ParseStop::Deferred); + if is_unicode_number(word, words.python) { + return Err(ParseStop::Failed(DiscoveryFailure { + kind: FailureKind::Declaration, + message: format!( + "{} has the bare number '{word}', written with non-ASCII digits", + words.file_path + ), + help: Some(format!( + "Quote it to keep it as text (\"{word}\"), or write the number with ASCII \ + digits 0-9" + )), + })); + } + return Ok(PythonType::Str); } Ok(match word { "true" | "false" => PythonType::Bool, @@ -120,3 +173,18 @@ fn is_float(word: &str) -> bool { let digits = |part: &str| part.bytes().all(|byte| byte.is_ascii_digit()); digits(whole) && digits(fraction) && !(whole.is_empty() && fraction.is_empty()) } + +/// Whether Python's `^[+-]?\d+$` or float pattern, with Unicode `\d`, matches a non-ASCII word. +fn is_unicode_number(word: &str, python: PythonText) -> bool { + let digits = |part: &str| { + part.chars() + .all(|character| is_python_decimal(python, character)) + }; + let unsigned = unsigned(word); + if !unsigned.is_empty() && digits(unsigned) { + return true; + } + unsigned.split_once('.').is_some_and(|(whole, fraction)| { + digits(whole) && digits(fraction) && !(whole.is_empty() && fraction.is_empty()) + }) +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/stops.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/stops.rs index 3ce19f339b..b08389b284 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/stops.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/checks/stops.rs @@ -2,11 +2,10 @@ use crate::models::DiscoveryFailure; -/// The first failure Python raises for the file, or a file only the Python parser reproduces. +/// The first failure Python raises for the file. #[derive(Clone, Debug, PartialEq)] pub(crate) enum ParseStop { Failed(DiscoveryFailure), - Deferred, } impl From for ParseStop { diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collection_reads.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collection_reads.rs index 0c6eceefd3..b41b9b9674 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collection_reads.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collection_reads.rs @@ -37,14 +37,14 @@ pub(crate) fn read_collection( CollectionKind::Enums => DeclarationCollection::Enums( declaration_files(source, DeclarationKind::Enum, isolate_kind).map(|files| { read_files(root, tree, files, |file_path, _, contents| { - parse_enum_file(file_path, contents) + parse_enum_file(file_path, contents, options.python) }) }), ), CollectionKind::Constants => DeclarationCollection::Constants( declaration_files(source, DeclarationKind::Constant, isolate_kind).map(|files| { read_files(root, tree, files, |file_path, _, contents| { - parse_constant_file(file_path, contents) + parse_constant_file(file_path, contents, options.python) }) }), ), @@ -58,7 +58,7 @@ pub(crate) fn read_collection( CollectionKind::ModelSchemas => DeclarationCollection::ModelSchemas( named_declaration_files(source, &[NamedKind::Schema], false).map(|files| { read_files(root, tree, files, |file_path, _, contents| { - parse_schema_file(file_path, contents) + parse_schema_file(file_path, contents, options.python) }) }), ), diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collections.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collections.rs index 7d51f06e03..c21c648495 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collections.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/listing/collections.rs @@ -1,6 +1,6 @@ //! The files of each collection in Python's discovery order, read and parsed in parallel. -use crate::_helpers::reading::read_authored_text; +use crate::_helpers::reading::{read_authored_text, undecodable_path_failure}; use crate::_helpers::scoped_paths::is_in_scoped_declaration_tree; use crate::declaration_files::_helpers::checks::stops::ParseStop; use crate::declaration_files::_helpers::listing::collection_reads::CollectionSource; @@ -258,14 +258,13 @@ fn read_one( relative_path: &str, parse: &impl Fn(&str, &str, String) -> Result, ) -> FileOutcome { - if tree.is_undecodable(relative_path) { - return FileOutcome::Deferred; + if let Some(failure) = undecodable_path_failure(root, tree, relative_path) { + return FileOutcome::Failed(failure); } match read_authored_text(&tree.absolute(relative_path)) { Ok(contents) => match parse(&root.display_path(relative_path), relative_path, contents) { Ok(parsed) => FileOutcome::Parsed(parsed), Err(ParseStop::Failed(failure)) => FileOutcome::Failed(failure), - Err(ParseStop::Deferred) => FileOutcome::Deferred, }, Err(failure) => FileOutcome::Unreadable(failure), } diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_files.rs index ad089ae598..e6225e3603 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_files.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/audit_files.rs @@ -2,7 +2,7 @@ use crate::_helpers::statement_headers::{StatementHeader, parse_statement_header}; use crate::declaration_files::_helpers::checks::python_values::{ - PythonType, failure, get, non_empty_str, python_str, python_type, + PythonType, WordRules, failure, get, non_empty_str, python_str, python_type, }; use crate::declaration_files::_helpers::checks::stops::ParseStop; use crate::declaration_files::_helpers::parsing::audit_text::{AuditText, chars, text}; @@ -11,6 +11,7 @@ use crate::models::FailureKind; use sqlbuild_core::text::main::is_python_space::is_python_space; use sqlbuild_core::text::main::python_cleandoc::python_cleandoc; use sqlbuild_core::text::main::python_strip::python_strip; +use sqlbuild_core::text::models::PythonText; use sqlbuild_sqltext::compiler::models::AuthoredValue; const AUDIT_KEYWORD: &str = "AUDIT"; @@ -113,7 +114,8 @@ impl AuditParse<'_> { .filter(|character| **character == '\n') .count(); let header_values: Vec<(String, AuthoredValue)> = self.header(&header, header_line)?; - let evaluation_mode: &'static str = evaluation_mode(&header_values, file_path)?; + let evaluation_mode: &'static str = + evaluation_mode(&header_values, file_path, self.options.python)?; let body: &[char] = &block_chars[body_start..]; let mut measure_sql: Option = None; let mut evidence_sql: Option = None; @@ -143,7 +145,14 @@ impl AuditParse<'_> { sql_body }; let name: Option = match get(&header_values, "name") { - Some(value) => python_str(value)?.map(str::to_owned), + Some(value) => python_str( + value, + WordRules { + python: self.options.python, + file_path, + }, + )? + .map(str::to_owned), None => None, }; Ok(AuditBlock { @@ -177,7 +186,14 @@ impl AuditParse<'_> { )?; for key in STRING_KEYS { if let Some(value) = get(&values, key) - && non_empty_str(value)?.is_none() + && non_empty_str( + value, + WordRules { + python: self.options.python, + file_path, + }, + )? + .is_none() { return Err(audit_failure(format!( "AUDIT() {key} in '{file_path}' must be a non-empty string" @@ -186,7 +202,13 @@ impl AuditParse<'_> { } for key in BOOLEAN_KEYS { if let Some(value) = get(&values, key) - && python_type(value)? != PythonType::Bool + && python_type( + value, + WordRules { + python: self.options.python, + file_path, + }, + )? != PythonType::Bool { return Err(audit_failure(format!( "AUDIT() {key} in '{file_path}' must be a boolean" @@ -343,9 +365,10 @@ fn keyword_at_slice_start( fn evaluation_mode( header_values: &[(String, AuthoredValue)], file_path: &str, + python: PythonText, ) -> Result<&'static str, ParseStop> { let mode: &str = match get(header_values, "evaluation") { - Some(value) => python_str(value)?.unwrap_or_default(), + Some(value) => python_str(value, WordRules { python, file_path })?.unwrap_or_default(), None => VIOLATIONS, }; let mode: &'static str = match mode { diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs index d6e61a5d28..4a940f315c 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/constant_files.rs @@ -1,13 +1,14 @@ //! Python's `parse_constant_declaration_file` up to value normalisation, which Python performs. use crate::declaration_files::_helpers::checks::python_values::{ - failure, get, python_str, unknown_keys, + WordRules, check_bare_numbers, failure, get, python_str, unknown_keys, }; use crate::declaration_files::_helpers::checks::stops::ParseStop; use crate::declaration_files::_helpers::parsing::declaration_headers::declaration_headers; use crate::declaration_files::_helpers::parsing::enum_files::declaration_name; use crate::declaration_files::models::{ConstantDeclaration, ConstantFile}; use crate::models::FailureKind; +use sqlbuild_core::text::models::PythonText; use sqlbuild_sqltext::compiler::models::AuthoredValue; const CONSTANT_KEYS: [&str; 4] = ["name", "value", "type", "render_as"]; @@ -17,18 +18,18 @@ const COLLECTION_RENDERINGS: [&str; 2] = ["value_list", "array"]; pub(crate) fn parse_constant_file( file_path: &str, contents: String, + python: PythonText, ) -> Result { let headers = declaration_headers(&contents, file_path, "CONSTANT")?; let mut declarations: Vec = Vec::with_capacity(headers.len()); let mut failure: Option<_> = None; for header in headers { - match parse_constant(&header.values, file_path) { + match parse_constant(&header.values, file_path, python) { Ok(declaration) => declarations.push(declaration), Err(ParseStop::Failed(stopped)) => { failure = Some(stopped); break; } - Err(ParseStop::Deferred) => return Err(ParseStop::Deferred), } } Ok(ConstantFile { @@ -45,6 +46,7 @@ fn declaration(message: String) -> ParseStop { fn parse_constant( values: &[(String, AuthoredValue)], file_path: &str, + python: PythonText, ) -> Result { let unknown: Vec = unknown_keys(values, &CONSTANT_KEYS); if !unknown.is_empty() { @@ -53,7 +55,7 @@ fn parse_constant( unknown.join(", ") ))); } - let name: String = declaration_name(get(values, "name"), file_path, "constant")?; + let name: String = declaration_name(get(values, "name"), file_path, "constant", python)?; let mut value: Option<&AuthoredValue> = get(values, "value"); let mut explicit_type: Option<&AuthoredValue> = get(values, "type"); let mut render_as: Option<&AuthoredValue> = get(values, "render_as"); @@ -79,12 +81,18 @@ fn parse_constant( "{file_path} constant '{name}' is missing required value" ))); }; - let explicit_type: Option = - constant_option(explicit_type, file_path, &format!("constant '{name}' type"))?; + check_bare_numbers(value, WordRules { python, file_path })?; + let explicit_type: Option = constant_option( + explicit_type, + file_path, + &format!("constant '{name}' type"), + python, + )?; let render_as: Option = constant_option( render_as, file_path, &format!("constant '{name}' render_as"), + python, )?; if let Some(rendering) = &render_as && !COLLECTION_RENDERINGS.contains(&rendering.as_str()) @@ -106,11 +114,12 @@ fn constant_option( raw_value: Option<&AuthoredValue>, file_path: &str, label: &str, + python: PythonText, ) -> Result, ParseStop> { let Some(raw_value) = raw_value else { return Ok(None); }; - match python_str(raw_value)? { + match python_str(raw_value, WordRules { python, file_path })? { Some(text) if !text.is_empty() => Ok(Some(text.to_owned())), _ => Err(declaration(format!( "{file_path} {label} must be an identifier" diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/enum_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/enum_files.rs index f19e8e8e2a..6364dc4be7 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/enum_files.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/enum_files.rs @@ -2,22 +2,27 @@ use crate::declaration_files::_helpers::checks::identities::validate_public_identity; use crate::declaration_files::_helpers::checks::python_values::{ - PythonType, failure, get, is_identifier, python_str, python_type, unknown_keys, + PythonType, WordRules, failure, get, is_identifier, python_str, python_type, unknown_keys, }; use crate::declaration_files::_helpers::checks::stops::ParseStop; use crate::declaration_files::_helpers::parsing::declaration_headers::declaration_headers; use crate::declaration_files::models::{EnumDeclaration, EnumFile}; use crate::models::FailureKind; +use sqlbuild_core::text::models::PythonText; use sqlbuild_sqltext::compiler::models::AuthoredValue; const ENUM_KEYS: [&str; 2] = ["name", "members"]; const VARCHAR_TYPE: &str = "VARCHAR"; const INTEGER_TYPE: &str = "INTEGER"; -pub(crate) fn parse_enum_file(file_path: &str, contents: String) -> Result { +pub(crate) fn parse_enum_file( + file_path: &str, + contents: String, + python: PythonText, +) -> Result { let declarations: Vec = declaration_headers(&contents, file_path, "ENUM")? .into_iter() - .map(|header| parse_enum(&header.values, file_path)) + .map(|header| parse_enum(&header.values, file_path, python)) .collect::>()?; Ok(EnumFile { contents, @@ -32,6 +37,7 @@ fn declaration(message: String) -> ParseStop { fn parse_enum( values: &[(String, AuthoredValue)], file_path: &str, + python: PythonText, ) -> Result { let unknown: Vec = unknown_keys(values, &ENUM_KEYS); if !unknown.is_empty() { @@ -40,10 +46,10 @@ fn parse_enum( unknown.join(", ") ))); } - let name: String = declaration_name(get(values, "name"), file_path, "enum")?; + let name: String = declaration_name(get(values, "name"), file_path, "enum", python)?; let members: Vec<(String, AuthoredValue)> = match get(values, "members") { - Some(AuthoredValue::List(items)) => shorthand_members(items, file_path, &name)?, - Some(AuthoredValue::Map(items)) => explicit_members(items, file_path, &name)?, + Some(AuthoredValue::List(items)) => shorthand_members(items, file_path, &name, python)?, + Some(AuthoredValue::Map(items)) => explicit_members(items, file_path, &name, python)?, _ => { return Err(declaration(format!( "{file_path} enum '{name}' members must use [...] or (...)" @@ -65,7 +71,7 @@ fn parse_enum( } let mut types: Vec = Vec::with_capacity(members.len()); for (_, value) in &members { - types.push(python_type(value)?); + types.push(python_type(value, WordRules { python, file_path })?); } if types.iter().any(|value_type| *value_type != types[0]) { return Err(declaration(format!( @@ -88,9 +94,10 @@ pub(crate) fn declaration_name( raw_name: Option<&AuthoredValue>, file_path: &str, kind: &str, + python: PythonText, ) -> Result { let name: Option<&str> = match raw_name { - Some(value) => python_str(value)?, + Some(value) => python_str(value, WordRules { python, file_path })?, None => None, }; let Some(name) = name.filter(|name| is_identifier(name)) else { @@ -106,10 +113,13 @@ fn shorthand_members( items: &[AuthoredValue], file_path: &str, enum_name: &str, + python: PythonText, ) -> Result, ParseStop> { let mut members: Vec<(String, AuthoredValue)> = Vec::with_capacity(items.len()); for item in items { - let Some(member) = python_str(item)?.filter(|member| is_identifier(member)) else { + let Some(member) = python_str(item, WordRules { python, file_path })? + .filter(|member| is_identifier(member)) + else { return Err(declaration(format!( "{file_path} enum '{enum_name}' shorthand members must be identifiers" ))); @@ -128,6 +138,7 @@ fn explicit_members( items: &[(String, AuthoredValue)], file_path: &str, enum_name: &str, + python: PythonText, ) -> Result, ParseStop> { let mut members: Vec<(String, AuthoredValue)> = Vec::with_capacity(items.len()); for (member, value) in items { @@ -136,7 +147,10 @@ fn explicit_members( "{file_path} enum '{enum_name}' member name must be a SQL identifier" ))); } - if !matches!(python_type(value)?, PythonType::Str | PythonType::Int) { + if !matches!( + python_type(value, WordRules { python, file_path })?, + PythonType::Str | PythonType::Int + ) { return Err(declaration(format!( "{file_path} enum '{enum_name}' member '{member}' value must be a string or \ integer" diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/hook_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/hook_files.rs index 05b1c6dbc0..0a989bceb4 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/hook_files.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/hook_files.rs @@ -2,7 +2,7 @@ use crate::_helpers::statement_headers::{StatementHeader, parse_statement_header}; use crate::declaration_files::_helpers::checks::python_values::{ - failure, get, non_empty_str, python_str, + WordRules, failure, get, non_empty_str, python_str, }; use crate::declaration_files::_helpers::checks::stops::ParseStop; use crate::declaration_files::models::{DeclarationFileOptions, HookFile}; @@ -41,12 +41,27 @@ pub(crate) fn parse_hook_file( )?; let description: Option = match get(&header_values, DESCRIPTION_KEY) { Some(value) => { - if non_empty_str(value)?.is_none() { + if non_empty_str( + value, + WordRules { + python: options.python, + file_path, + }, + )? + .is_none() + { return Err(hook_failure(format!( "HOOK() description in '{file_path}' must be a non-empty string" ))); } - python_str(value)?.map(str::to_owned) + python_str( + value, + WordRules { + python: options.python, + file_path, + }, + )? + .map(str::to_owned) } None => None, }; diff --git a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/schema_files.rs b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/schema_files.rs index 8e1054486f..94feecee54 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/schema_files.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/_helpers/parsing/schema_files.rs @@ -2,13 +2,14 @@ use crate::_helpers::locations::header_column_locations; use crate::declaration_files::_helpers::checks::python_values::{ - PythonType, failure, get, is_identifier, non_empty_str, python_type, unknown_keys, + PythonType, WordRules, failure, get, is_identifier, non_empty_str, python_type, unknown_keys, }; use crate::declaration_files::_helpers::checks::stops::ParseStop; use crate::declaration_files::_helpers::parsing::declaration_headers::declaration_headers; use crate::declaration_files::_helpers::parsing::enum_files::declaration_name; use crate::declaration_files::models::{SchemaDeclaration, SchemaFile}; use crate::models::FailureKind; +use sqlbuild_core::text::models::PythonText; use sqlbuild_sqltext::compiler::models::AuthoredValue; const SCHEMA_KEYS: [&str; 4] = ["name", "description", "extends", "columns"]; @@ -16,12 +17,13 @@ const SCHEMA_KEYS: [&str; 4] = ["name", "description", "extends", "columns"]; pub(crate) fn parse_schema_file( file_path: &str, contents: String, + python: PythonText, ) -> Result { let headers = declaration_headers(&contents, file_path, "SCHEMA")?; let mut declarations: Vec = Vec::with_capacity(headers.len()); let mut failure: Option<_> = None; for header in headers { - match parse_schema(&header.values, file_path) { + match parse_schema(&header.values, file_path, python) { Ok((name, description, extends)) => declarations.push(SchemaDeclaration { name, description, @@ -37,7 +39,6 @@ pub(crate) fn parse_schema_file( failure = Some(stopped); break; } - Err(ParseStop::Deferred) => return Err(ParseStop::Deferred), } } Ok(SchemaFile { @@ -56,6 +57,7 @@ type SchemaFacts = (String, Option, Option); fn parse_schema( values: &[(String, AuthoredValue)], file_path: &str, + python: PythonText, ) -> Result { let unknown: Vec = unknown_keys(values, &SCHEMA_KEYS); if !unknown.is_empty() { @@ -64,16 +66,18 @@ fn parse_schema( unknown.join(", ") ))); } - let name: String = declaration_name(get(values, "name"), file_path, "schema")?; + let name: String = declaration_name(get(values, "name"), file_path, "schema", python)?; let description: Option = optional_string( get(values, "description"), file_path, &format!("schema '{name}' description"), + python, )?; let extends: Option = optional_string( get(values, "extends"), file_path, &format!("schema '{name}' extends"), + python, )?; if let Some(parent) = &extends && !is_identifier(parent) @@ -90,14 +94,15 @@ fn optional_string( raw_value: Option<&AuthoredValue>, file_path: &str, label: &str, + python: PythonText, ) -> Result, ParseStop> { let Some(raw_value) = raw_value else { return Ok(None); }; - if python_type(raw_value)? == PythonType::None { + if python_type(raw_value, WordRules { python, file_path })? == PythonType::None { return Ok(None); } - match non_empty_str(raw_value)? { + match non_empty_str(raw_value, WordRules { python, file_path })? { Some(text) => Ok(Some(text.to_owned())), None => Err(declaration(format!( "{file_path} {label} must be a non-empty string" diff --git a/crates/sqlbuild-discovery/src/declaration_files/main/bare_number_failure.rs b/crates/sqlbuild-discovery/src/declaration_files/main/bare_number_failure.rs new file mode 100644 index 0000000000..f1370b62e2 --- /dev/null +++ b/crates/sqlbuild-discovery/src/declaration_files/main/bare_number_failure.rs @@ -0,0 +1,18 @@ +//! Bare-word numbers in an authored header value, checked as declaration files check them. + +use sqlbuild_core::text::models::PythonText; +use sqlbuild_sqltext::compiler::models::AuthoredValue; + +use crate::declaration_files::_helpers::checks::python_values::{WordRules, check_bare_numbers}; +use crate::declaration_files::_helpers::checks::stops::ParseStop; +use crate::models::DiscoveryFailure; + +/// The failure for the first bare word in `value` that is not a valid number, if any. +pub(crate) fn bare_number_failure( + value: &AuthoredValue, + python: PythonText, + file_path: &str, +) -> Result<(), DiscoveryFailure> { + check_bare_numbers(value, WordRules { python, file_path }) + .map_err(|ParseStop::Failed(failure)| failure) +} diff --git a/crates/sqlbuild-discovery/src/declaration_files/main/mod.rs b/crates/sqlbuild-discovery/src/declaration_files/main/mod.rs index f16e07d5e1..1868d60b93 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/main/mod.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/main/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod bare_number_failure; pub mod discover_declaration_collection; pub mod parse_declaration_text; pub mod retained_collection; diff --git a/crates/sqlbuild-discovery/src/declaration_files/main/parse_declaration_text.rs b/crates/sqlbuild-discovery/src/declaration_files/main/parse_declaration_text.rs index 749733c062..efac1f59ec 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/main/parse_declaration_text.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/main/parse_declaration_text.rs @@ -31,14 +31,12 @@ fn parse_text( ) -> Option> { let parsed: Result = match kind { CollectionKind::Enums => { - parse_enum_file(file_path, contents).map(ParsedDeclarationText::Enum) - } - CollectionKind::Constants => { - parse_constant_file(file_path, contents).map(ParsedDeclarationText::Constant) - } - CollectionKind::ModelSchemas => { - parse_schema_file(file_path, contents).map(ParsedDeclarationText::ModelSchema) + parse_enum_file(file_path, contents, options.python).map(ParsedDeclarationText::Enum) } + CollectionKind::Constants => parse_constant_file(file_path, contents, options.python) + .map(ParsedDeclarationText::Constant), + CollectionKind::ModelSchemas => parse_schema_file(file_path, contents, options.python) + .map(ParsedDeclarationText::ModelSchema), CollectionKind::SqlFunctions => parse_function_file(file_path, contents, options) .map(ParsedDeclarationText::SqlFunction), CollectionKind::SqlHooks => parse_hook_file(file_path, hook_name, contents, options) @@ -51,6 +49,5 @@ fn parse_text( Some(match parsed { Ok(parsed) => FileOutcome::Parsed(parsed), Err(ParseStop::Failed(failure)) => FileOutcome::Failed(failure), - Err(ParseStop::Deferred) => FileOutcome::Deferred, }) } diff --git a/crates/sqlbuild-discovery/src/declaration_files/tests/test_parse_declaration_text.rs b/crates/sqlbuild-discovery/src/declaration_files/tests/test_parse_declaration_text.rs index 5f38e8839b..1d47a86848 100644 --- a/crates/sqlbuild-discovery/src/declaration_files/tests/test_parse_declaration_text.rs +++ b/crates/sqlbuild-discovery/src/declaration_files/tests/test_parse_declaration_text.rs @@ -44,10 +44,28 @@ fn given_declaration_files_when_parsing_then_values_and_failures_match_python() ], }, DeclarationTextTestCase { - description: "a non-ASCII bare word may be a Unicode number", + description: "a bare number in non-ASCII digits is rejected with help", kind: CollectionKind::Enums, contents: "ENUM (name status, members (A \u{661}));", - expected_fragments: &["Some(Deferred)"], + expected_fragments: &[ + "/project/declarations/orders.sql has the bare number '\u{661}', written with non-ASCII digits", + "Quote it to keep it as text", + ], + }, + DeclarationTextTestCase { + description: "a constant bare number in non-ASCII digits is rejected, nested too", + kind: CollectionKind::Constants, + contents: "CONSTANT (name limits, value [1, \u{663}]);", + expected_fragments: &[ + "/project/declarations/orders.sql has the bare number '\u{663}', written with non-ASCII digits", + "Quote it to keep it as text", + ], + }, + DeclarationTextTestCase { + description: "a non-ASCII bare word that is not a number is text", + kind: CollectionKind::Enums, + contents: "ENUM (name status, members (A münchen, B \u{2167}));", + expected_fragments: &["Parsed", "münchen"], }, DeclarationTextTestCase { description: "a camel-case name suggests snake case", diff --git a/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs b/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs index abfed5e535..c3736d2238 100644 --- a/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs +++ b/crates/sqlbuild-discovery/src/model_files/_helpers/parsing.rs @@ -3,6 +3,7 @@ use crate::_helpers::header_keys::{UnsupportedKeys, unsupported_keys_failure}; use crate::_helpers::header_syntax::{FailedHeader, header_syntax_failure}; use crate::_helpers::locations::header_column_locations; +use crate::declaration_files::main::bare_number_failure::bare_number_failure; use crate::model_files::_helpers::output_columns::output_column_locations; use crate::model_files::models::{DiscoveredModelFile, ModelFileOptions}; use crate::models::{DiscoveryFailure, FailureKind}; @@ -73,6 +74,14 @@ pub(crate) fn parse_model_file( python: options.python, })); } + for (_, value) in &header_values { + bare_number_failure(value, options.python, file_path).map_err(|failure| { + DiscoveryFailure { + kind: FailureKind::ModelSql, + ..failure + } + })?; + } let query_sql: String = python_strip(&contents[sql_start..]).to_owned(); if query_sql.is_empty() { return Err(failure(format!( diff --git a/crates/sqlbuild-discovery/src/models.rs b/crates/sqlbuild-discovery/src/models.rs index 3f8be1e5de..b38c79864f 100644 --- a/crates/sqlbuild-discovery/src/models.rs +++ b/crates/sqlbuild-discovery/src/models.rs @@ -84,8 +84,6 @@ pub enum FileOutcome { Failed(DiscoveryFailure), /// The file could not be read or decoded as Python's `read_text` would. Unreadable(ReadFailure), - /// Native parsing cannot reproduce Python for this file; the Python parser reads it. - Deferred, } /// One discovered file, by `/`-separated path relative to the project directory. diff --git a/crates/sqlbuild-model-config/Cargo.toml b/crates/sqlbuild-model-config/Cargo.toml index a4cf9e2917..15f3596bb0 100644 --- a/crates/sqlbuild-model-config/Cargo.toml +++ b/crates/sqlbuild-model-config/Cargo.toml @@ -9,3 +9,6 @@ rust-version.workspace = true [lints] workspace = true + +[dependencies] +sqlbuild-core.workspace = true diff --git a/crates/sqlbuild-model-config/src/config_presence/_helpers/macro_calls.rs b/crates/sqlbuild-model-config/src/config_presence/_helpers/macro_calls.rs index 2d8517fa20..808bf6ae64 100644 --- a/crates/sqlbuild-model-config/src/config_presence/_helpers/macro_calls.rs +++ b/crates/sqlbuild-model-config/src/config_presence/_helpers/macro_calls.rs @@ -1,18 +1,17 @@ //! Python's `@[A-Za-z_][A-Za-z0-9_]*\s*\(` search over one string. -use crate::config_presence::models::Presence; +use sqlbuild_core::text::main::is_python_space::is_python_space; -/// Search `text` for a macro call opening, deferring where Python's `\s` decides. -pub(crate) fn macro_call(text: &str) -> Presence { +/// Whether `text` holds a macro call opening; `\s` is Python's Unicode whitespace class. +pub(crate) fn macro_call(text: &str) -> bool { let bytes = text.as_bytes(); - let mut deferred = false; - for (at, _) in text.match_indices('@') { + text.match_indices('@').any(|(at, _)| { let mut index = at + 1; if !bytes .get(index) .is_some_and(|byte| byte.is_ascii_alphabetic() || *byte == b'_') { - continue; + return false; } while bytes .get(index) @@ -20,23 +19,11 @@ pub(crate) fn macro_call(text: &str) -> Presence { { index += 1; } - while bytes.get(index).copied().is_some_and(is_python_ascii_space) { - index += 1; - } - match bytes.get(index) { - Some(b'(') => return Presence::Present, - Some(byte) if !byte.is_ascii() => deferred = true, - _ => {} - } - } - if deferred { - Presence::Deferred - } else { - Presence::Absent - } + opens_call(&text[index..]) + }) } -/// ASCII characters Python's `\s` matches in a `str` pattern. -fn is_python_ascii_space(byte: u8) -> bool { - matches!(byte, b'\t'..=b'\r' | 0x1c..=0x1f | b' ') +/// Whether the first non-whitespace character of `rest` is `(`. +fn opens_call(rest: &str) -> bool { + rest.trim_start_matches(is_python_space).starts_with('(') } diff --git a/crates/sqlbuild-model-config/src/config_presence/_helpers/scan.rs b/crates/sqlbuild-model-config/src/config_presence/_helpers/scan.rs index 34669a21a2..892fd6fea8 100644 --- a/crates/sqlbuild-model-config/src/config_presence/_helpers/scan.rs +++ b/crates/sqlbuild-model-config/src/config_presence/_helpers/scan.rs @@ -1,36 +1,18 @@ //! The recursive config scan shared by every presence check. -use crate::config_presence::models::Presence; use crate::types::{AuthoredNode, NodeKind}; /// Scan mapping values, list and tuple items and strings like Python's recursive config scans. -pub(crate) fn scan( - node: &N, - text_presence: &impl Fn(&str) -> Presence, -) -> Presence { +pub(crate) fn scan(node: &N, text_presence: &impl Fn(&str) -> bool) -> bool { match node.kind() { - NodeKind::Str => node.with_text(text_presence).unwrap_or(Presence::Deferred), - NodeKind::Map => any_present(node.entries().iter().map(|(_, value)| value), text_presence), - NodeKind::List | NodeKind::Tuple => any_present(node.items().iter(), text_presence), - _ => Presence::Absent, - } -} - -fn any_present<'node, N: AuthoredNode + 'node>( - nodes: impl Iterator, - text_presence: &impl Fn(&str) -> Presence, -) -> Presence { - let mut deferred = false; - for node in nodes { - match scan(node, text_presence) { - Presence::Present => return Presence::Present, - Presence::Deferred => deferred = true, - Presence::Absent => {} + NodeKind::Str => node.with_text(text_presence).unwrap_or(false), + NodeKind::Map => node + .entries() + .iter() + .any(|(_, value)| scan(value, text_presence)), + NodeKind::List | NodeKind::Tuple => { + node.items().iter().any(|item| scan(item, text_presence)) } - } - if deferred { - Presence::Deferred - } else { - Presence::Absent + _ => false, } } diff --git a/crates/sqlbuild-model-config/src/config_presence/_helpers/template_tokens.rs b/crates/sqlbuild-model-config/src/config_presence/_helpers/template_tokens.rs index dcf2a1d7f5..6691ba666b 100644 --- a/crates/sqlbuild-model-config/src/config_presence/_helpers/template_tokens.rs +++ b/crates/sqlbuild-model-config/src/config_presence/_helpers/template_tokens.rs @@ -1,13 +1,8 @@ //! Python's `TEMPLATE_OPEN_TOKEN in value` over one string. use crate::config_presence::constants::TEMPLATE_OPEN_TOKEN; -use crate::config_presence::models::Presence; /// Return whether `text` holds a template opening. -pub(crate) fn template_token(text: &str) -> Presence { - if text.contains(TEMPLATE_OPEN_TOKEN) { - Presence::Present - } else { - Presence::Absent - } +pub(crate) fn template_token(text: &str) -> bool { + text.contains(TEMPLATE_OPEN_TOKEN) } diff --git a/crates/sqlbuild-model-config/src/config_presence/main/contains_macro_call.rs b/crates/sqlbuild-model-config/src/config_presence/main/contains_macro_call.rs deleted file mode 100644 index ba6326a391..0000000000 --- a/crates/sqlbuild-model-config/src/config_presence/main/contains_macro_call.rs +++ /dev/null @@ -1,11 +0,0 @@ -//! Recursive macro call presence over authored config values. - -use crate::config_presence::_helpers::macro_calls::macro_call; -use crate::config_presence::_helpers::scan::scan; -use crate::config_presence::models::Presence; -use crate::types::AuthoredNode; - -/// Return whether `MACRO_CALL_PATTERN` matches any nested string, or defer to Python. -pub fn contains_macro_call(node: &N) -> Presence { - scan(node, ¯o_call) -} diff --git a/crates/sqlbuild-model-config/src/config_presence/main/contains_template.rs b/crates/sqlbuild-model-config/src/config_presence/main/contains_template.rs index bacc702131..c4f3a3832d 100644 --- a/crates/sqlbuild-model-config/src/config_presence/main/contains_template.rs +++ b/crates/sqlbuild-model-config/src/config_presence/main/contains_template.rs @@ -2,10 +2,9 @@ use crate::config_presence::_helpers::scan::scan; use crate::config_presence::_helpers::template_tokens::template_token; -use crate::config_presence::models::Presence; use crate::types::AuthoredNode; -/// Return whether `contains_template_data` finds `${` in any nested string, or defer to Python. -pub fn contains_template(node: &N) -> Presence { +/// Return whether `contains_template_data` finds `${` in any nested string. +pub fn contains_template(node: &N) -> bool { scan(node, &template_token) } diff --git a/crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs b/crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs index 2490eb32ae..d3ec7c465a 100644 --- a/crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs +++ b/crates/sqlbuild-model-config/src/config_presence/main/first_macro_path.rs @@ -1,7 +1,7 @@ //! The first config field, in Python's walk order, whose string holds a macro call. use crate::config_presence::_helpers::macro_calls::macro_call; -use crate::config_presence::models::{MacroPath, Presence}; +use crate::config_presence::models::MacroPath; use crate::types::{AuthoredNode, NodeKind}; /// Walk config values as `validate_no_macros_in_config_value` does, skipping the `skipped` keys. @@ -13,11 +13,9 @@ pub fn first_macro_path(values: &N, skipped: &[&str]) -> MacroP fn walk(node: &N, path: &[String]) -> MacroPath { match node.kind() { - NodeKind::Str => match node.with_text(macro_call) { - Some(Presence::Present) => MacroPath::Found(path.to_vec()), - Some(Presence::Absent) => MacroPath::Absent, - Some(Presence::Deferred) | None => MacroPath::Deferred, - }, + NodeKind::Str if node.with_text(macro_call).unwrap_or(false) => { + MacroPath::Found(path.to_vec()) + } NodeKind::Map => walk_entries(&node.entries(), path), NodeKind::List | NodeKind::Tuple => { for item in node.items() { @@ -38,7 +36,7 @@ fn walk_entries(entries: &[(N, N)], path: &[String]) -> MacroPa continue; } let Some(name) = key.text() else { - return MacroPath::Deferred; + continue; }; let mut child: Vec = path.to_vec(); child.push(name); diff --git a/crates/sqlbuild-model-config/src/config_presence/main/mod.rs b/crates/sqlbuild-model-config/src/config_presence/main/mod.rs index 03e068523b..9cfc885af6 100644 --- a/crates/sqlbuild-model-config/src/config_presence/main/mod.rs +++ b/crates/sqlbuild-model-config/src/config_presence/main/mod.rs @@ -1,3 +1,2 @@ -pub mod contains_macro_call; pub mod contains_template; pub mod first_macro_path; diff --git a/crates/sqlbuild-model-config/src/config_presence/models.rs b/crates/sqlbuild-model-config/src/config_presence/models.rs index 38b270b275..6eba20ef0f 100644 --- a/crates/sqlbuild-model-config/src/config_presence/models.rs +++ b/crates/sqlbuild-model-config/src/config_presence/models.rs @@ -1,13 +1,4 @@ -//! Whether authored config contains a construct, or that only Python can tell. - -/// The answer Python's recursive config scan gives, or a deferral to that scan. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Presence { - Present, - Absent, - /// A string needs Python's Unicode `\s` class to decide. - Deferred, -} +//! Where a config walk finds a macro call. /// Where Python's config walk first finds a macro call. #[derive(Clone, Debug, PartialEq, Eq)] @@ -15,6 +6,4 @@ pub enum MacroPath { /// The mapping keys leading to the first string holding a macro call. Found(Vec), Absent, - /// A string before any match needs Python's Unicode rules to decide. - Deferred, } diff --git a/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs b/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs index 5bf585a298..40c033c0d8 100644 --- a/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs +++ b/crates/sqlbuild-model-config/src/config_presence/tests/test_config_presence.rs @@ -1,11 +1,11 @@ -use crate::config_presence::main::contains_macro_call::contains_macro_call; +use crate::config_presence::_helpers::macro_calls::macro_call; +use crate::config_presence::_helpers::scan::scan; use crate::config_presence::main::contains_template::contains_template; -use crate::config_presence::models::Presence; use crate::config_presence::tests::test_types::PresenceTestCase; use crate::tests::test_types::Value; #[test] -fn given_config_values_when_scanning_then_presence_matches_python_or_defers() { +fn given_config_values_when_scanning_then_presence_matches_python() { let test_cases = [ PresenceTestCase { description: "plain scalars contain nothing", @@ -15,8 +15,8 @@ fn given_config_values_when_scanning_then_presence_matches_python_or_defers() { Value::Null, Value::Float, ]), - expected_template: Presence::Absent, - expected_macro_call: Presence::Absent, + expected_template: false, + expected_macro_call: false, }, PresenceTestCase { description: "nested mapping values are scanned, keys are not", @@ -27,38 +27,44 @@ fn given_config_values_when_scanning_then_presence_matches_python_or_defers() { Value::Tuple(vec![Value::Str("@cents (amount)")]), ), ]), - expected_template: Presence::Absent, - expected_macro_call: Presence::Present, + expected_template: false, + expected_macro_call: true, }, PresenceTestCase { description: "a template token anywhere in a string", value: Value::Str("prefix_${target.schema}"), - expected_template: Presence::Present, - expected_macro_call: Presence::Absent, + expected_template: true, + expected_macro_call: false, }, PresenceTestCase { description: "an at-name without a call or with an invalid start is no macro", value: Value::Str("mail@example.com @1x( @ (x)"), - expected_template: Presence::Absent, - expected_macro_call: Presence::Absent, + expected_template: false, + expected_macro_call: false, }, PresenceTestCase { description: "Python separators count as whitespace before the call", value: Value::Str("@m\u{1f}\t("), - expected_template: Presence::Absent, - expected_macro_call: Presence::Present, + expected_template: false, + expected_macro_call: true, }, PresenceTestCase { - description: "non-ASCII after a macro name defers to Python's whitespace class", - value: Value::Str("@m\u{a0}("), - expected_template: Presence::Absent, - expected_macro_call: Presence::Deferred, + description: "Unicode whitespace after a macro name counts as Python's \\s", + value: Value::Str("@m\u{a0}\u{3000}("), + expected_template: false, + expected_macro_call: true, }, PresenceTestCase { - description: "a later call outweighs an earlier deferral", + description: "other non-ASCII after a macro name is no call", + value: Value::Str("@m\u{e9}( @n\u{200b}("), + expected_template: false, + expected_macro_call: false, + }, + PresenceTestCase { + description: "a later call is found after Unicode whitespace that is no call", value: Value::List(vec![Value::Str("@m\u{2003}("), Value::Str("@n(")]), - expected_template: Presence::Absent, - expected_macro_call: Presence::Present, + expected_template: false, + expected_macro_call: true, }, ]; @@ -66,7 +72,7 @@ fn given_config_values_when_scanning_then_presence_matches_python_or_defers() { assert_eq!( ( contains_template(&test_case.value), - contains_macro_call(&test_case.value) + scan(&test_case.value, ¯o_call) ), (test_case.expected_template, test_case.expected_macro_call), "{}", diff --git a/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs b/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs index 917d2a3746..cb04f91f0a 100644 --- a/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs +++ b/crates/sqlbuild-model-config/src/config_presence/tests/test_types.rs @@ -1,10 +1,9 @@ -use crate::config_presence::models::Presence; use crate::tests::test_types::Value; /// One authored config value and what each recursive scan finds in it. pub(super) struct PresenceTestCase { pub(super) description: &'static str, pub(super) value: Value, - pub(super) expected_template: Presence, - pub(super) expected_macro_call: Presence, + pub(super) expected_template: bool, + pub(super) expected_macro_call: bool, } diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/audits.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/audits.rs index 5e7915de96..39e58b373e 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/_helpers/audits.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/audits.rs @@ -1,18 +1,20 @@ //! Audit lists and instances, as `parse_audit_instances` reads them with `null_as_empty`. +use crate::errors::ConfigError; use crate::header_metadata::_helpers::identity::check_identity; use crate::header_metadata::_helpers::text::{ Site, entry, non_blank_text, optional_bool, optional_count, optional_text, }; +use crate::header_metadata::_helpers::thresholds::thresholds; use crate::header_metadata::constants::{AUDIT_OPTION_KEYS, AUDIT_SEVERITIES}; -use crate::header_metadata::models::{HeaderMetadataStop, ParsedAudit}; +use crate::header_metadata::models::ParsedAudit; use crate::types::{AuthoredNode, NodeKind}; /// Parse an authored audit list; Python's `None` is an empty list. pub(crate) fn audit_list( node: &N, site: Site<'_>, -) -> Result>, HeaderMetadataStop> { +) -> Result>, ConfigError> { match node.kind() { NodeKind::Null => Ok(Vec::new()), NodeKind::List => node @@ -27,10 +29,10 @@ pub(crate) fn audit_list( fn audit_instance( node: &N, site: Site<'_>, -) -> Result, HeaderMetadataStop> { +) -> Result, ConfigError> { let definition_kind = format!("{} audit definition", site.label); if node.kind() == NodeKind::Str { - let Some(text) = non_blank_text(node)? else { + let Some(text) = non_blank_text(node) else { return Err(site.error("audits must not contain empty names")); }; check_identity(&text, &definition_kind, site.path)?; @@ -43,7 +45,7 @@ fn audit_instance( if node.kind() != NodeKind::Map { return Err(site.error("audits must be strings or single-key mappings")); } - let Some(definition) = non_blank_text(definition_name)? else { + let Some(definition) = non_blank_text(definition_name) else { return Err(site.error("audit names must be non-empty strings")); }; check_identity(&definition, &definition_kind, site.path)?; @@ -59,15 +61,16 @@ fn configured_audit( definition: &str, options: &[(N, N)], site: Site<'_>, -) -> Result, HeaderMetadataStop> { +) -> Result, ConfigError> { let option_label = format!("{} audit '{definition}'", site.label); let option_site = Site { path: site.path, label: &option_label, + column: site.column, }; let name = optional_text(entry(options, "name"), option_site, "name")?; if let Some(name) = &name { - let text = name.text().ok_or(HeaderMetadataStop::Unsupported)?; + let text = name.text().unwrap_or_default(); check_identity(&text, &format!("{} audit instance", site.label), site.path)?; } let description = optional_text(entry(options, "description"), option_site, "description")?; @@ -82,18 +85,16 @@ fn configured_audit( } let run_scope = optional_text(entry(options, "run_scope"), option_site, "run_scope")?; let always_run = optional_bool(entry(options, "always_run"), option_site, "always_run")?; - if entry(options, "thresholds").is_some_and(|value| value.kind() != NodeKind::Null) { - return Err(HeaderMetadataStop::Unsupported); - } + let thresholds = thresholds(entry(options, "thresholds"), option_site)?; let minimum_samples = optional_count( entry(options, "minimum_samples"), option_site, - "minimum_samples", + (definition, "minimum_samples"), )?; let evidence_limit = optional_count( entry(options, "evidence_limit"), option_site, - "evidence_limit", + (definition, "evidence_limit"), )?; let arguments: Vec<(N, N)> = options .iter() @@ -108,6 +109,7 @@ fn configured_audit( severity, run_scope, always_run, + thresholds, minimum_samples, evidence_limit, }) @@ -122,6 +124,7 @@ fn bare_audit(definition_name: N) -> ParsedAudit { severity: None, run_scope: None, always_run: None, + thresholds: None, minimum_samples: None, evidence_limit: None, } diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/columns.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/columns.rs index df77dcdcb0..e7219b4df3 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/_helpers/columns.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/columns.rs @@ -10,17 +10,18 @@ use crate::header_metadata::_helpers::text::{ use crate::header_metadata::constants::{ MODEL_COLUMN_KEYS, MODEL_LABEL, NOT_NULL_AUDIT_NAME, NULLABLE_NOT_NULL_CODE, }; -use crate::header_metadata::models::{HeaderMetadataStop, ParsedColumn}; +use crate::header_metadata::models::ParsedColumn; use crate::types::{AuthoredNode, NodeKind}; /// Parse an authored columns mapping; Python's `None` declares no columns. pub(crate) fn column_mapping( node: &N, path: &str, -) -> Result>, HeaderMetadataStop> { +) -> Result>, ConfigError> { let site = Site { path, label: MODEL_LABEL, + column: None, }; match node.kind() { NodeKind::Null => return Ok(Vec::new()), @@ -30,13 +31,10 @@ pub(crate) fn column_mapping( let mut seen: HashSet = HashSet::new(); let mut columns: Vec> = Vec::new(); for (name, metadata) in node.entries() { - let Some(text) = non_blank_text(&name)? else { + let Some(text) = non_blank_text(&name) else { return Err(site.error("column names must be non-empty strings")); }; - if !text.is_ascii() { - return Err(HeaderMetadataStop::Unsupported); - } - if !seen.insert(text.to_ascii_lowercase()) { + if !seen.insert(text.to_lowercase()) { return Err(site.error(&format!( "has duplicate column '{text}' (column names are case-insensitive)" ))); @@ -51,17 +49,18 @@ fn column( text: &str, metadata: &N, path: &str, -) -> Result, HeaderMetadataStop> { +) -> Result, ConfigError> { let label = format!("{MODEL_LABEL} column '{text}'"); let site = Site { path, label: &label, + column: Some(text), }; if metadata.kind() != NodeKind::Map { return Err(site.error("metadata must be a mapping")); } let entries = metadata.entries(); - let unknown = unknown_keys(&entries)?; + let unknown = unknown_keys(&entries); if !unknown.is_empty() { return Err(site.error(&format!( "has unknown metadata keys: {}", @@ -81,13 +80,11 @@ fn column( .iter() .any(|audit| audit.definition_name.is_text(NOT_NULL_AUDIT_NAME)) { - return Err(HeaderMetadataStop::Error( - ConfigError::compile(format!( - "{path} column '{text}' cannot set nullable = true and audit not_null" - )) - .with_code(NULLABLE_NOT_NULL_CODE) - .with_help("remove the not_null audit or set nullable = false"), - )); + return Err(ConfigError::compile(format!( + "{path} column '{text}' cannot set nullable = true and audit not_null" + )) + .with_code(NULLABLE_NOT_NULL_CODE) + .with_help("remove the not_null audit or set nullable = false")); } Ok(ParsedColumn { name: name.clone(), @@ -99,18 +96,18 @@ fn column( }) } -/// Return the sorted metadata keys outside `MODEL_COLUMN_KEYS`, deferring keys Python cannot sort. -fn unknown_keys(entries: &[(N, N)]) -> Result, HeaderMetadataStop> { - let mut unknown: Vec = Vec::new(); - for (key, _) in entries { - if MODEL_COLUMN_KEYS.iter().any(|name| key.is_text(name)) { - continue; - } - if key.kind() != NodeKind::Str { - return Err(HeaderMetadataStop::Unsupported); - } - unknown.push(key.text().ok_or(HeaderMetadataStop::Unsupported)?); - } +/// Return the sorted metadata keys outside `MODEL_COLUMN_KEYS`; header keys are always strings. +fn unknown_keys(entries: &[(N, N)]) -> Vec { + let mut unknown: Vec = entries + .iter() + .filter(|(key, _)| !is_model_column_key(key)) + .map(|(key, _)| key.python_str()) + .collect(); unknown.sort_unstable(); - Ok(unknown) + unknown +} + +/// Whether `key` is one of `MODEL_COLUMN_KEYS`. +fn is_model_column_key(key: &N) -> bool { + MODEL_COLUMN_KEYS.iter().any(|name| key.is_text(name)) } diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/identity.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/identity.rs index c046a733ab..b32e40efc8 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/_helpers/identity.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/identity.rs @@ -1,26 +1,21 @@ -//! `validate_resource_identity` for ASCII names, with its suggested snake_case spelling. +//! `validate_resource_identity`, with its suggested snake_case spelling. use crate::errors::{ConfigError, ErrorClass}; -use crate::header_metadata::_helpers::text::ascii_strip; +use sqlbuild_core::text::main::python_strip::python_strip; + use crate::header_metadata::constants::{FALLBACK_IDENTITY, IDENTITY_HELP}; -use crate::header_metadata::models::HeaderMetadataStop; -/// Raise `ResourceIdentityError` unless `name` is canonical snake_case; defer non-ASCII names. -pub(crate) fn check_identity(name: &str, kind: &str, path: &str) -> Result<(), HeaderMetadataStop> { +/// Raise `ResourceIdentityError` unless `name` is canonical snake_case. +pub(crate) fn check_identity(name: &str, kind: &str, path: &str) -> Result<(), ConfigError> { if is_snake_case(name) { return Ok(()); } - if !name.is_ascii() { - return Err(HeaderMetadataStop::Unsupported); - } let suggestion = suggested_identity(name); - Err(HeaderMetadataStop::Error( - ConfigError::compile(format!( - "Invalid {kind} identity '{name}' in {path}; use snake_case '{suggestion}'" - )) - .with_class(ErrorClass::ResourceIdentity) - .with_help(IDENTITY_HELP), + Err(ConfigError::compile(format!( + "Invalid {kind} identity '{name}' in {path}; use snake_case '{suggestion}'" )) + .with_class(ErrorClass::ResourceIdentity) + .with_help(IDENTITY_HELP)) } /// Return whether `name` fully matches `^[a-z](?:[a-z0-9_]*[a-z0-9])?$`. @@ -38,9 +33,9 @@ fn is_snake_case(name: &str) -> bool { } } -/// Return `suggest_resource_identity` for an ASCII public identity. +/// Return `suggest_resource_identity` for a public identity; its patterns are ASCII-only. fn suggested_identity(name: &str) -> String { - let characters: Vec = ascii_strip(name).chars().collect(); + let characters: Vec = python_strip(name).chars().collect(); let acronyms = split_where(&characters, |index| { characters[index - 1].is_ascii_uppercase() && characters[index].is_ascii_uppercase() diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/mod.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/mod.rs index 3e2c0f3b41..2f9c85d8b4 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/_helpers/mod.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod audits; pub(crate) mod columns; pub(crate) mod identity; pub(crate) mod text; +pub(crate) mod thresholds; diff --git a/crates/sqlbuild-model-config/src/header_metadata/_helpers/text.rs b/crates/sqlbuild-model-config/src/header_metadata/_helpers/text.rs index 14989b1747..b2bc03343e 100644 --- a/crates/sqlbuild-model-config/src/header_metadata/_helpers/text.rs +++ b/crates/sqlbuild-model-config/src/header_metadata/_helpers/text.rs @@ -1,7 +1,9 @@ -//! Text checks that agree with Python's `str` methods or defer, and the errors they raise. +//! Text checks that agree with Python's `str` methods, and the errors they raise. + +use crate::model_validation::constants::SETTING_SNIPPET_INDENT; +use sqlbuild_core::text::main::python_strip::python_strip; use crate::errors::ConfigError; -use crate::header_metadata::models::HeaderMetadataStop; use crate::types::{AuthoredNode, NodeKind}; /// Where one parse reports its errors: the file path and the label the messages name. @@ -9,36 +11,20 @@ use crate::types::{AuthoredNode, NodeKind}; pub(crate) struct Site<'a> { pub(crate) path: &'a str, pub(crate) label: &'a str, + /// The column whose metadata is read, or `None` at the model level. + pub(crate) column: Option<&'a str>, } impl Site<'_> { /// The `CompileInputError` `