From 350ab2590f2970359951d94f565330445307c023 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:19:42 +0100 Subject: [PATCH 1/2] feat(native): normalise and compare types natively behind the preview engine --- crates/README.md | 2 +- .../src/type_system/_helpers/dialects.rs | 38 +++ .../src/type_system/_helpers/fallback.rs | 161 ++++++++++++ .../src/type_system/_helpers/mod.rs | 4 + .../src/type_system/_helpers/parsed.rs | 239 ++++++++++++++++++ .../src/type_system/_helpers/python_text.rs | 100 ++++++++ .../src/type_system/constants.rs | 39 +++ .../src/type_system/main/mod.rs | 1 + .../src/type_system/main/normalize_type.rs | 26 ++ .../sqlbuild-analysis/src/type_system/mod.rs | 8 + .../src/type_system/models.rs | 63 +++++ .../src/type_system/tests.rs | 8 + .../src/type_system/tests/helpers.rs | 15 ++ .../type_system/tests/test_normalize_type.rs | 107 ++++++++ .../src/type_system/tests/test_python_text.rs | 102 ++++++++ .../src/type_system/tests/test_types.rs | 21 ++ .../src/bindings/_helpers/type_system/mod.rs | 1 + .../_helpers/type_system/normalization.rs | 43 ++++ .../_helpers/type_system/registration.rs | 5 +- src/sqlbuild/_native.pyi | 3 + .../_helpers/type_normalization.py | 17 +- src/sqlbuild/adapter/type_system/constants.py | 4 + .../main/_native_normalize_type.py | 34 ++- .../cli/commands/main/compile/_test_types.py | 10 + .../cli/commands/main/compile/helpers.py | 30 +++ .../main/compile/test_native_type_system.py | 96 +++++++ .../sqlbuild/adapter/type_system/__init__.py | 0 .../adapter/type_system/_test_types.py | 26 ++ .../sqlbuild/adapter/type_system/helpers.py | 235 +++++++++++++++++ .../type_system/test_native_type_parity.py | 142 +++++++++++ 30 files changed, 1574 insertions(+), 6 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/type_system/_helpers/dialects.rs create mode 100644 crates/sqlbuild-analysis/src/type_system/_helpers/fallback.rs create mode 100644 crates/sqlbuild-analysis/src/type_system/_helpers/mod.rs create mode 100644 crates/sqlbuild-analysis/src/type_system/_helpers/parsed.rs create mode 100644 crates/sqlbuild-analysis/src/type_system/_helpers/python_text.rs create mode 100644 crates/sqlbuild-analysis/src/type_system/constants.rs create mode 100644 crates/sqlbuild-analysis/src/type_system/main/mod.rs create mode 100644 crates/sqlbuild-analysis/src/type_system/main/normalize_type.rs create mode 100644 crates/sqlbuild-analysis/src/type_system/models.rs create mode 100644 crates/sqlbuild-analysis/src/type_system/tests.rs create mode 100644 crates/sqlbuild-analysis/src/type_system/tests/helpers.rs create mode 100644 crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs create mode 100644 crates/sqlbuild-analysis/src/type_system/tests/test_python_text.rs create mode 100644 crates/sqlbuild-analysis/src/type_system/tests/test_types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs create mode 100644 src/sqlbuild/adapter/type_system/constants.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_type_system.py create mode 100644 tests/integration/src/sqlbuild/adapter/type_system/__init__.py create mode 100644 tests/integration/src/sqlbuild/adapter/type_system/_test_types.py create mode 100644 tests/integration/src/sqlbuild/adapter/type_system/helpers.py create mode 100644 tests/integration/src/sqlbuild/adapter/type_system/test_native_type_parity.py diff --git a/crates/README.md b/crates/README.md index 6cb8ef83ac..05398296bd 100644 --- a/crates/README.md +++ b/crates/README.md @@ -15,7 +15,7 @@ that can be tested and benchmarked with `cargo test` alone. | `sqlbuild-model-config` | Model configuration: MODEL header columns and audits, `${...}` template expansion with its environment and context reads, and the template and macro presence scans over authored config values, all read through a trait over the caller's values. Anything it cannot reproduce exactly defers to Python, which also raises every model config error. | | `sqlbuild-render` | Native rendering: the macro call scanner (a byte-for-byte port of Python's), splicing of rendered calls with code-point spans, and the in-compile memo of recorded macro calls and their replayable events, which can carry results across compiles through the shared native store. Anything it cannot reproduce exactly defers to Python. | | `sqlbuild-attachments` | Compile attachments for tests, audits, sources, functions, scenarios and seeds: attached generic audit argument merge, raw and quoted argument rendering, and severity and run scope resolution. Anything it cannot reproduce exactly defers to Python, which also raises every attachment error. | -| `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, and SQL-test extraction, planning and rendering. | +| `sqlbuild-analysis` | SQL analysis over polyglot: SQL tokens, query analysis, the binding catalog, semantic validation and usage, column references, type normalization, and SQL-test extraction, planning and rendering. Type normalization defers to Python for anything it cannot reproduce exactly. | | `sqlbuild-rules` | Built-in rules and the rules engine, the custom-rule host, SQL lint, quality checks and formatting, rules configuration and the request models. It also owns the build identity script. | | `sqlbuild-python` | The only PyO3 crate: the `_native` module, its Python classes and functions, conversions from Python objects, and the process allocator. | diff --git a/crates/sqlbuild-analysis/src/type_system/_helpers/dialects.rs b/crates/sqlbuild-analysis/src/type_system/_helpers/dialects.rs new file mode 100644 index 0000000000..6cabfabfeb --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/_helpers/dialects.rs @@ -0,0 +1,38 @@ +//! Dialect names: Polyglot's resolution and Python's `TypeDialect` coercion. + +use polyglot_sql::{Dialect, DialectType}; + +use crate::type_system::models::TypeDialect; + +/// The Polyglot dialect Python's wheel resolves `name` to, when this build includes it. +pub(crate) fn polyglot_dialect(name: &str) -> Option { + let Ok(dialect_type) = name.parse::() else { + return None; + }; + matches!( + dialect_type, + DialectType::Generic + | DialectType::PostgreSQL + | DialectType::BigQuery + | DialectType::Snowflake + | DialectType::DuckDB + | DialectType::TSQL + | DialectType::Databricks + ) + .then(|| Dialect::get(dialect_type)) +} + +/// Python's `_coerce_type_dialect`: an exact `TypeDialect` value, otherwise None. +pub(crate) fn coerce_type_dialect(name: &str) -> Option { + Some(match name { + "generic" => TypeDialect::Generic, + "bigquery" => TypeDialect::BigQuery, + "snowflake" => TypeDialect::Snowflake, + "duckdb" => TypeDialect::DuckDb, + "motherduck" => TypeDialect::MotherDuck, + "databricks" => TypeDialect::Databricks, + "postgres" => TypeDialect::Postgres, + "tsql" => TypeDialect::Tsql, + _ => return None, + }) +} diff --git a/crates/sqlbuild-analysis/src/type_system/_helpers/fallback.rs b/crates/sqlbuild-analysis/src/type_system/_helpers/fallback.rs new file mode 100644 index 0000000000..53d2ee1d28 --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/_helpers/fallback.rs @@ -0,0 +1,161 @@ +//! Python's `_normalize_with_fallback`: type text normalized without a parser. + +use crate::type_system::_helpers::python_text::{ + python_strip, remove_python_space, split_type_and_params, +}; +use crate::type_system::constants::{ + BIGNUMERIC_TYPE_NAME, BOOLEAN_TYPE_NAMES, DATE_TYPE_NAME, DATETIME_TYPE_NAME, + DECIMAL_TYPE_NAMES, DEFAULT_TEXT_LENGTH, FLOAT_TYPE_NAMES, INTEGER_PRECISION, INTEGER_SCALE, + INTEGER_TYPE_NAMES, NORMALIZED_LTZ_INPUT_TYPE_NAME, NORMALIZED_NTZ_INPUT_TYPE_NAMES, + NORMALIZED_TZ_INPUT_TYPE_NAME, STRING_TYPE_NAMES, TEXT_TYPE_NAME, TIMESTAMP_TYPE_TOKEN, + UNBOUNDED_TEXT_TYPE_NAMES, +}; +use crate::type_system::models::{NormalizedType, TypeDialect, TypeFamily}; + +/// Normalize ASCII type text as Python's fallback does; None for integers beyond `i64`. +pub(crate) fn normalize_with_fallback( + type_sql: &str, + dialect: Option, +) -> Option { + let normalized: String = remove_python_space(python_strip(&type_sql.to_ascii_uppercase())); + let (base_type, params) = split_type_and_params(&normalized)?; + let first: Option = params.first().copied(); + let second: Option = params.get(1).copied(); + let snowflake: bool = dialect == Some(TypeDialect::Snowflake); + let bigquery: bool = dialect == Some(TypeDialect::BigQuery); + + if INTEGER_TYPE_NAMES.contains(&base_type) { + if snowflake { + return Some(snowflake_integer()); + } + let name: &str = if bigquery { "INT64" } else { base_type }; + return Some(simple(name, TypeFamily::Integer)); + } + if DECIMAL_TYPE_NAMES.contains(&base_type) { + let (mut precision, mut scale) = (first, second); + if snowflake && precision.is_none() { + precision = Some(INTEGER_PRECISION); + scale = Some(INTEGER_SCALE); + } + let base_name: &str = if bigquery { + if base_type == BIGNUMERIC_TYPE_NAME { + "BIGNUMERIC" + } else { + "NUMERIC" + } + } else if snowflake { + "DECIMAL" + } else { + base_type + }; + let normalized_name: String = match (precision, scale) { + (Some(precision), Some(scale)) => format!("{base_name}({precision},{scale})"), + (Some(precision), None) => format!("{base_name}({precision})"), + _ => base_name.to_owned(), + }; + return Some(NormalizedType { + normalized_name, + family: TypeFamily::Decimal, + precision, + scale, + length: None, + }); + } + if FLOAT_TYPE_NAMES.contains(&base_type) { + let name: &str = if bigquery { "FLOAT64" } else { base_type }; + return Some(simple(name, TypeFamily::Float)); + } + if STRING_TYPE_NAMES.contains(&base_type) { + let mut length: Option = first; + let base_name: &str = if bigquery { + "STRING" + } else if snowflake && base_type == TEXT_TYPE_NAME { + "VARCHAR" + } else { + base_type + }; + let mut normalized_name: String = match length { + Some(length) => format!("{base_name}({length})"), + None => base_name.to_owned(), + }; + if snowflake && UNBOUNDED_TEXT_TYPE_NAMES.contains(&base_type) { + let bounded: i64 = python_or_default(length); + length = Some(bounded); + normalized_name = format!("VARCHAR({bounded})"); + } + return Some(NormalizedType { + normalized_name, + family: TypeFamily::String, + precision: None, + scale: None, + length, + }); + } + if BOOLEAN_TYPE_NAMES.contains(&base_type) { + let name: &str = if bigquery { "BOOL" } else { base_type }; + return Some(simple(name, TypeFamily::Boolean)); + } + if base_type.contains(TIMESTAMP_TYPE_TOKEN) { + return Some(simple( + ×tamp_normalized_name(base_type, dialect), + TypeFamily::Timestamp, + )); + } + if base_type == DATE_TYPE_NAME { + return Some(simple(base_type, TypeFamily::Date)); + } + if base_type == DATETIME_TYPE_NAME { + return Some(simple(base_type, TypeFamily::Datetime)); + } + Some(simple(&normalized, TypeFamily::Other)) +} + +/// Python's `_timestamp_normalized_name`: Snowflake spells its timestamp aliases one way. +pub(crate) fn timestamp_normalized_name( + normalized_name: &str, + dialect: Option, +) -> String { + if dialect != Some(TypeDialect::Snowflake) { + return normalized_name.to_owned(); + } + let compact: String = normalized_name.replace('_', ""); + if NORMALIZED_NTZ_INPUT_TYPE_NAMES.contains(&compact.as_str()) { + return "TIMESTAMP_NTZ".to_owned(); + } + if compact == NORMALIZED_LTZ_INPUT_TYPE_NAME { + return "TIMESTAMP_LTZ".to_owned(); + } + if compact == NORMALIZED_TZ_INPUT_TYPE_NAME { + return "TIMESTAMP_TZ".to_owned(); + } + normalized_name.to_owned() +} + +/// Snowflake's integers: `DECIMAL(38,0)`. +pub(crate) fn snowflake_integer() -> NormalizedType { + NormalizedType { + normalized_name: format!("DECIMAL({INTEGER_PRECISION},{INTEGER_SCALE})"), + family: TypeFamily::Decimal, + precision: Some(INTEGER_PRECISION), + scale: Some(INTEGER_SCALE), + length: None, + } +} + +/// Python's `length or DEFAULT_TEXT_LENGTH`: a missing or zero length takes the default. +pub(crate) fn python_or_default(length: Option) -> i64 { + match length { + Some(length) if length != 0 => length, + _ => DEFAULT_TEXT_LENGTH, + } +} + +pub(crate) fn simple(name: &str, family: TypeFamily) -> NormalizedType { + NormalizedType { + normalized_name: name.to_owned(), + family, + precision: None, + scale: None, + length: None, + } +} diff --git a/crates/sqlbuild-analysis/src/type_system/_helpers/mod.rs b/crates/sqlbuild-analysis/src/type_system/_helpers/mod.rs new file mode 100644 index 0000000000..51643d4b4a --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod dialects; +pub(crate) mod fallback; +pub(crate) mod parsed; +pub(crate) mod python_text; diff --git a/crates/sqlbuild-analysis/src/type_system/_helpers/parsed.rs b/crates/sqlbuild-analysis/src/type_system/_helpers/parsed.rs new file mode 100644 index 0000000000..ff4b9e99f7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/_helpers/parsed.rs @@ -0,0 +1,239 @@ +//! Python's `_normalize_with_polyglot`: the parsed type's SQL and arguments. + +use polyglot_sql::ast_json::{expression_from_value, expressions_from_value}; +use polyglot_sql::{ComplexityGuardOptions, DataType, Dialect, Expression, ParseOptions}; +use serde_json::{Map, Value}; + +use crate::type_system::_helpers::fallback::{ + normalize_with_fallback, python_or_default, simple, snowflake_integer, + timestamp_normalized_name, +}; +use crate::type_system::_helpers::python_text::split_type_and_params; +use crate::type_system::constants::{ + BIGNUMERIC_TYPE_NAME, BOOLEAN_TYPE_NAMES, CUSTOM_NORMALIZATION_TYPE_NAMES, DATE_TYPE_NAME, + DATETIME_TYPE_NAME, DECIMAL_TYPE_NAMES, FLOAT_TYPE_NAMES, FLOAT_WIRE_TYPE_NAME, + INTEGER_PARSE_TYPE_NAMES, INTEGER_PRECISION, INTEGER_SCALE, INTEGER_TYPE_NAMES, + MAX_FUNCTION_CALL_DEPTH, POLYGLOT_CUSTOM_TYPE_NAME, POLYGLOT_TYPE_NAME_ALIASES, + STRING_TYPE_NAMES, TIMESTAMP_TYPE_NAMES, UNBOUNDED_TEXT_TYPE_NAMES, +}; +use crate::type_system::models::{NormalizedType, TypeDialect, TypeFamily}; + +/// Parse one type with SQLBuild's Polyglot guard; the error text is what Python logs. +pub(crate) fn parse_type(type_sql: &str, dialect: &Dialect) -> Result { + dialect + .parse_data_type_with_options( + type_sql, + &ParseOptions { + complexity_guard: Some(ComplexityGuardOptions { + max_function_call_depth: Some(MAX_FUNCTION_CALL_DEPTH), + ..Default::default() + }), + }, + ) + .map_err(|error| error.to_string()) +} + +/// Python's `_normalized_from_parsed_type`, or None where Python raises or holds an object. +pub(crate) fn normalize_parsed( + parsed: &DataType, + polyglot: &Dialect, + dialect: Option, +) -> Option { + let expression: Expression = Expression::DataType(parsed.clone()); + let Ok(generated) = polyglot.generate(&expression) else { + return None; + }; + if !generated.is_ascii() { + return None; + } + let normalized_name: String = generated.to_ascii_uppercase().replace(' ', ""); + let args: Map = expression_args(&expression)?; + let dtype_name: String = polyglot_type_name(&args)?; + let params: Vec = polyglot_type_params(&args)?; + let snowflake: bool = dialect == Some(TypeDialect::Snowflake); + let bigquery: bool = dialect == Some(TypeDialect::BigQuery); + let dtype: &str = dtype_name.as_str(); + + if bigquery && INTEGER_PARSE_TYPE_NAMES.contains(&dtype) { + return Some(simple("INT64", TypeFamily::Integer)); + } + if bigquery && dtype == BIGNUMERIC_TYPE_NAME { + return Some(simple("BIGNUMERIC", TypeFamily::Decimal)); + } + if bigquery && dtype == FLOAT_WIRE_TYPE_NAME { + return Some(simple("FLOAT64", TypeFamily::Float)); + } + if dtype == POLYGLOT_CUSTOM_TYPE_NAME { + let raw_name: String = match args.get("name") { + Some(value) => python_str(value)?, + None => normalized_name.clone(), + } + .to_ascii_uppercase() + .replace(' ', ""); + if snowflake && raw_name.starts_with("NUMBER") { + return snowflake_number(&raw_name); + } + return normalize_with_fallback(&raw_name, dialect); + } + if INTEGER_TYPE_NAMES.contains(&dtype) { + if snowflake { + return Some(snowflake_integer()); + } + return Some(simple(&normalized_name, TypeFamily::Integer)); + } + if DECIMAL_TYPE_NAMES.contains(&dtype) { + let mut precision: Option = params.first().copied(); + let mut scale: Option = params.get(1).copied(); + let mut name: String = normalized_name; + if snowflake && precision.is_none() { + precision = Some(INTEGER_PRECISION); + scale = Some(INTEGER_SCALE); + name = format!("DECIMAL({INTEGER_PRECISION},{INTEGER_SCALE})"); + } + return Some(NormalizedType { + normalized_name: name, + family: TypeFamily::Decimal, + precision, + scale, + length: None, + }); + } + if FLOAT_TYPE_NAMES.contains(&dtype) { + return Some(simple(&normalized_name, TypeFamily::Float)); + } + if STRING_TYPE_NAMES.contains(&dtype) { + let mut length: Option = params.first().copied(); + let mut name: String = normalized_name; + let base_name: &str = name.split('(').next().unwrap_or_default(); + if snowflake && UNBOUNDED_TEXT_TYPE_NAMES.contains(&base_name) { + let bounded: i64 = python_or_default(length); + length = Some(bounded); + name = format!("VARCHAR({bounded})"); + } + return Some(NormalizedType { + normalized_name: name, + family: TypeFamily::String, + precision: None, + scale: None, + length, + }); + } + if BOOLEAN_TYPE_NAMES.contains(&dtype) { + return Some(simple(&normalized_name, TypeFamily::Boolean)); + } + if TIMESTAMP_TYPE_NAMES.contains(&dtype) { + return Some(simple( + ×tamp_normalized_name(&normalized_name, dialect), + TypeFamily::Timestamp, + )); + } + if dtype == DATE_TYPE_NAME { + return Some(simple(&normalized_name, TypeFamily::Date)); + } + if dtype == DATETIME_TYPE_NAME { + return Some(simple(&normalized_name, TypeFamily::Datetime)); + } + Some(simple(&normalized_name, TypeFamily::Other)) +} + +/// Snowflake's `NUMBER(p,s)` spelled as a custom type: Python's `DECIMAL` rewrite. +fn snowflake_number(raw_name: &str) -> Option { + let mut decimal_name: String = raw_name.replacen("NUMBER", "DECIMAL", 1); + let (_, params) = split_type_and_params(&decimal_name)?; + let mut precision: Option = params.first().copied(); + let mut scale: Option = params.get(1).copied(); + if precision.is_none() { + precision = Some(INTEGER_PRECISION); + scale = Some(INTEGER_SCALE); + decimal_name = format!("DECIMAL({INTEGER_PRECISION},{INTEGER_SCALE})"); + } + Some(NormalizedType { + normalized_name: decimal_name, + family: TypeFamily::Decimal, + precision, + scale, + length: None, + }) +} + +/// The wheel's `Expression.args`: the serde payload of the expression's variant. +fn expression_args(expression: &Expression) -> Option> { + let Ok(value) = serde_json::to_value(expression) else { + return None; + }; + match value { + Value::Object(map) => match map.into_iter().next() { + Some((_, Value::Object(payload))) => Some(payload), + _ => Some(Map::new()), + }, + _ => Some(Map::new()), + } +} + +/// Python's `_polyglot_type_name`. +fn polyglot_type_name(args: &Map) -> Option { + let data_type: String = match args.get("data_type") { + Some(value) => python_str(value)?, + None => String::new(), + } + .to_ascii_uppercase(); + if let Some((_, alias)) = POLYGLOT_TYPE_NAME_ALIASES + .iter() + .find(|(name, _)| *name == data_type) + { + return Some((*alias).to_owned()); + } + if data_type == POLYGLOT_CUSTOM_TYPE_NAME { + let name: String = match args.get("name") { + Some(value) => python_str(value)?, + None => String::new(), + } + .to_ascii_uppercase() + .replace(' ', ""); + if CUSTOM_NORMALIZATION_TYPE_NAMES.contains(&name.as_str()) { + return Some(name); + } + } + Some(data_type) +} + +/// Python's `_polyglot_type_params`; text and integers beyond `i64` defer. +fn polyglot_type_params(args: &Map) -> Option> { + let mut params: Vec = Vec::new(); + for key in ["precision", "scale", "length"] { + let Some(value) = args.get(key) else { + continue; + }; + match value { + Value::Null | Value::Array(_) | Value::Object(_) => {} + Value::Bool(flag) => params.push(i64::from(*flag)), + Value::Number(number) => params.push(number.as_i64()?), + Value::String(_) if is_python_expression(value) => {} + Value::String(_) => return None, + } + } + Some(params) +} + +/// Python's `str(value)` for the text values Polyglot returns; None for anything else. +fn python_str(value: &Value) -> Option { + if is_python_expression(value) { + return None; + } + match value { + Value::String(text) if text.is_ascii() => Some(text.clone()), + Value::Null => Some("None".to_owned()), + _ => None, + } +} + +/// Whether the wheel hands Python this value as an expression object rather than data. +fn is_python_expression(value: &Value) -> bool { + if let Ok(_expression) = expression_from_value(value.clone()) { + return true; + } + let Ok(_expressions) = expressions_from_value(value.clone()) else { + return false; + }; + true +} diff --git a/crates/sqlbuild-analysis/src/type_system/_helpers/python_text.rs b/crates/sqlbuild-analysis/src/type_system/_helpers/python_text.rs new file mode 100644 index 0000000000..a1dda52dbf --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/_helpers/python_text.rs @@ -0,0 +1,100 @@ +//! Python string and integer semantics the type normalization relies on, for ASCII text. + +use crate::type_system::constants::PYTHON_ASCII_WHITESPACE; + +/// The result of Python's `int(text)` on one parameter. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PythonInt { + Value(i64), + Invalid, + /// A valid integer outside `i64`; Python keeps it, so the caller defers. + TooLarge, +} + +pub(crate) fn is_python_space(byte: u8) -> bool { + PYTHON_ASCII_WHITESPACE.contains(&byte) +} + +/// Python's `str.strip()`. +pub(crate) fn python_strip(text: &str) -> &str { + text.trim_matches(|character: char| character.is_ascii() && is_python_space(character as u8)) +} + +/// Python's `re.sub(r"\s+", "", text)`. +pub(crate) fn remove_python_space(text: &str) -> String { + text.chars() + .filter(|character| !(character.is_ascii() && is_python_space(*character as u8))) + .collect() +} + +/// Python's base-10 `int(text)`: whitespace, a sign, and single underscores between digits. +pub(crate) fn python_int(text: &str) -> PythonInt { + let stripped: &str = python_strip(text); + let (negative, digits) = match stripped.as_bytes().first() { + Some(b'-') => (true, &stripped[1..]), + Some(b'+') => (false, &stripped[1..]), + _ => (false, stripped), + }; + let bytes: &[u8] = digits.as_bytes(); + let valid: bool = !bytes.is_empty() + && bytes.first().is_some_and(u8::is_ascii_digit) + && bytes.last().is_some_and(u8::is_ascii_digit) + && bytes + .iter() + .all(|byte| byte.is_ascii_digit() || *byte == b'_') + && !digits.contains("__"); + if !valid { + return PythonInt::Invalid; + } + let mut value: i64 = 0; + for byte in bytes.iter().filter(|byte| byte.is_ascii_digit()) { + let digit: i64 = i64::from(byte - b'0'); + let next: Option = value.checked_mul(10).and_then(|scaled| { + if negative { + scaled.checked_sub(digit) + } else { + scaled.checked_add(digit) + } + }); + let Some(next) = next else { + return PythonInt::TooLarge; + }; + value = next; + } + PythonInt::Value(value) +} + +/// Python's `_split_type_and_params`; None for a parameter beyond `i64`. +pub(crate) fn split_type_and_params(type_sql: &str) -> Option<(&str, Vec)> { + let candidate: &str = type_sql.strip_suffix('\n').unwrap_or(type_sql); + let name_end: usize = candidate + .bytes() + .position(|byte| !(byte.is_ascii_uppercase() || byte.is_ascii_digit() || byte == b'_')) + .unwrap_or(candidate.len()); + if name_end == 0 { + return Some((type_sql, Vec::new())); + } + let (name, rest) = candidate.split_at(name_end); + if rest.is_empty() { + return Some((name, Vec::new())); + } + let Some(raw_params) = rest + .strip_prefix('(') + .and_then(|inner| inner.strip_suffix(')')) + .filter(|inner| !inner.contains(')')) + else { + return Some((type_sql, Vec::new())); + }; + let mut params: Vec = Vec::new(); + if raw_params.is_empty() { + return Some((name, params)); + } + for raw_part in raw_params.split(',') { + match python_int(raw_part) { + PythonInt::Value(value) => params.push(value), + PythonInt::Invalid => {} + PythonInt::TooLarge => return None, + } + } + Some((name, params)) +} diff --git a/crates/sqlbuild-analysis/src/type_system/constants.rs b/crates/sqlbuild-analysis/src/type_system/constants.rs new file mode 100644 index 0000000000..51852fa0b6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/constants.rs @@ -0,0 +1,39 @@ +//! Type names and defaults that decide a normalized type's family and spelling. + +pub(crate) const BOOLEAN_TYPE_NAMES: &[&str] = &["BOOL", "BOOLEAN"]; +pub(crate) const DATE_TYPE_NAME: &str = "DATE"; +pub(crate) const DATETIME_TYPE_NAME: &str = "DATETIME"; +pub(crate) const DECIMAL_TYPE_NAMES: &[&str] = + &["BIGDECIMAL", "BIGNUMERIC", "DECIMAL", "NUMBER", "NUMERIC"]; +pub(crate) const FLOAT_TYPE_NAMES: &[&str] = &["DOUBLE", "FLOAT", "FLOAT64", "REAL"]; +pub(crate) const INTEGER_TYPE_NAMES: &[&str] = &[ + "BIGINT", "INT", "INT64", "INTEGER", "LONG", "SMALLINT", "TINYINT", +]; +pub(crate) const POLYGLOT_CUSTOM_TYPE_NAME: &str = "CUSTOM"; +pub(crate) const STRING_TYPE_NAMES: &[&str] = &["CHAR", "CHARACTER", "STRING", "TEXT", "VARCHAR"]; +pub(crate) const TIMESTAMP_TYPE_NAMES: &[&str] = + &["TIMESTAMP", "TIMESTAMPLTZ", "TIMESTAMPNTZ", "TIMESTAMPTZ"]; +pub(crate) const TIMESTAMP_TYPE_TOKEN: &str = "TIMESTAMP"; +pub(crate) const BIGNUMERIC_TYPE_NAME: &str = "BIGNUMERIC"; +pub(crate) const CUSTOM_NORMALIZATION_TYPE_NAMES: &[&str] = &["BIGNUMERIC", "FLOAT64", "INT64"]; +pub(crate) const FLOAT_WIRE_TYPE_NAME: &str = "FLOAT64"; +pub(crate) const INTEGER_PARSE_TYPE_NAMES: &[&str] = &["BIGINT", "INT"]; +pub(crate) const DEFAULT_TEXT_LENGTH: i64 = 16_777_216; +pub(crate) const INTEGER_PRECISION: i64 = 38; +pub(crate) const INTEGER_SCALE: i64 = 0; +pub(crate) const NORMALIZED_LTZ_INPUT_TYPE_NAME: &str = "TIMESTAMPLTZ"; +pub(crate) const NORMALIZED_NTZ_INPUT_TYPE_NAMES: &[&str] = &["TIMESTAMP", "TIMESTAMPNTZ"]; +pub(crate) const NORMALIZED_TZ_INPUT_TYPE_NAME: &str = "TIMESTAMPTZ"; +pub(crate) const TEXT_TYPE_NAME: &str = "TEXT"; +pub(crate) const UNBOUNDED_TEXT_TYPE_NAMES: &[&str] = &["STRING", "TEXT", "VARCHAR"]; +/// Polyglot `data_type` tags whose Python spelling drops the word separator. +pub(crate) const POLYGLOT_TYPE_NAME_ALIASES: &[(&str, &str)] = &[ + ("BIG_INT", "BIGINT"), + ("SMALL_INT", "SMALLINT"), + ("TINY_INT", "TINYINT"), + ("VAR_CHAR", "VARCHAR"), +]; +/// The function call depth limit SQLBuild passes to every Polyglot parse. +pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 512; +/// Characters Python's `str.strip()` and the `\s` regex class treat as ASCII whitespace. +pub(crate) const PYTHON_ASCII_WHITESPACE: &[u8] = b" \t\n\r\x0b\x0c\x1c\x1d\x1e\x1f"; diff --git a/crates/sqlbuild-analysis/src/type_system/main/mod.rs b/crates/sqlbuild-analysis/src/type_system/main/mod.rs new file mode 100644 index 0000000000..81be6727b9 --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/main/mod.rs @@ -0,0 +1 @@ +pub mod normalize_type; diff --git a/crates/sqlbuild-analysis/src/type_system/main/normalize_type.rs b/crates/sqlbuild-analysis/src/type_system/main/normalize_type.rs new file mode 100644 index 0000000000..506ba08ad4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/main/normalize_type.rs @@ -0,0 +1,26 @@ +//! Python's `normalize_type`: Polyglot's parsed type first, then the text fallback. + +use crate::type_system::_helpers::dialects::{coerce_type_dialect, polyglot_dialect}; +use crate::type_system::_helpers::fallback::normalize_with_fallback; +use crate::type_system::_helpers::parsed::{normalize_parsed, parse_type}; +use crate::type_system::models::{TypeDialect, TypeNormalization}; + +/// Normalize one type as Python does for the dialect name it hands Polyglot, or None to defer. +#[must_use] +pub fn normalize_type(type_sql: &str, dialect: &str) -> Option { + if !type_sql.is_ascii() || !dialect.is_ascii() { + return None; + } + let polyglot = polyglot_dialect(dialect)?; + let type_dialect: Option = coerce_type_dialect(dialect); + match parse_type(type_sql, &polyglot) { + Ok(parsed) => Some(TypeNormalization { + normalized: normalize_parsed(&parsed, &polyglot, type_dialect)?, + parse_error: None, + }), + Err(error) => Some(TypeNormalization { + normalized: normalize_with_fallback(type_sql, type_dialect)?, + parse_error: Some(error), + }), + } +} diff --git a/crates/sqlbuild-analysis/src/type_system/mod.rs b/crates/sqlbuild-analysis/src/type_system/mod.rs index b053fa0aea..89ff092ae4 100644 --- a/crates/sqlbuild-analysis/src/type_system/mod.rs +++ b/crates/sqlbuild-analysis/src/type_system/mod.rs @@ -1 +1,9 @@ //! Type normalization and type equality shared by every analysis stage. + +pub(crate) mod _helpers; +mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-analysis/src/type_system/models.rs b/crates/sqlbuild-analysis/src/type_system/models.rs new file mode 100644 index 0000000000..0250342b03 --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/models.rs @@ -0,0 +1,63 @@ +//! Normalized type shapes, matching Python's `NormalizedType` and `TypeFamily`. + +/// The semantic family of one normalized type. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TypeFamily { + Integer, + Decimal, + Float, + String, + Boolean, + Timestamp, + Date, + Datetime, + Other, +} + +impl TypeFamily { + /// The value of Python's `TypeFamily` member. + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::Integer => "integer", + Self::Decimal => "decimal", + Self::Float => "float", + Self::String => "string", + Self::Boolean => "boolean", + Self::Timestamp => "timestamp", + Self::Date => "date", + Self::Datetime => "datetime", + Self::Other => "other", + } + } +} + +/// The dialects Python's type normalization distinguishes (`TypeDialect`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TypeDialect { + Generic, + BigQuery, + Snowflake, + DuckDb, + MotherDuck, + Databricks, + Postgres, + Tsql, +} + +/// One type string's comparison shape. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NormalizedType { + pub normalized_name: String, + pub family: TypeFamily, + pub precision: Option, + pub scale: Option, + pub length: Option, +} + +/// A normalized type and the Polyglot parse error Python logs before its text fallback. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TypeNormalization { + pub normalized: NormalizedType, + pub parse_error: Option, +} diff --git a/crates/sqlbuild-analysis/src/type_system/tests.rs b/crates/sqlbuild-analysis/src/type_system/tests.rs new file mode 100644 index 0000000000..e27bb932fd --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_normalize_type.rs"] +mod normalize_type; +#[path = "tests/test_python_text.rs"] +mod python_text; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-analysis/src/type_system/tests/helpers.rs b/crates/sqlbuild-analysis/src/type_system/tests/helpers.rs new file mode 100644 index 0000000000..300ca546a8 --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/tests/helpers.rs @@ -0,0 +1,15 @@ +use crate::type_system::models::TypeNormalization; + +pub(super) fn normalization_text(normalization: &TypeNormalization) -> String { + let optional = |value: Option| value.map_or("-".to_owned(), |value| value.to_string()); + let normalized = &normalization.normalized; + format!( + "{} | {} | {} | {} | {} | {}", + normalized.normalized_name, + normalized.family.as_str(), + optional(normalized.precision), + optional(normalized.scale), + optional(normalized.length), + normalization.parse_error.as_deref().unwrap_or("-"), + ) +} diff --git a/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs b/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs new file mode 100644 index 0000000000..4aa51ed32a --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs @@ -0,0 +1,107 @@ +use crate::type_system::main::normalize_type::normalize_type; +use crate::type_system::tests::helpers::normalization_text; +use crate::type_system::tests::test_types::NormalizeTypeTestCase; + +#[test] +fn given_type_strings_when_normalizing_then_python_normalization_is_returned() { + let test_cases = [ + NormalizeTypeTestCase { + description: "integer aliases keep their spelling outside Snowflake and BigQuery", + type_sql: "integer", + dialect: "duckdb", + expected_normalization: Some("INT | integer | - | - | - | -"), + }, + NormalizeTypeTestCase { + description: "BigQuery integers are INT64", + type_sql: "INT", + dialect: "bigquery", + expected_normalization: Some("INT64 | integer | - | - | - | -"), + }, + NormalizeTypeTestCase { + description: "Snowflake integers are DECIMAL(38,0)", + type_sql: "BIGINT", + dialect: "snowflake", + expected_normalization: Some("DECIMAL(38,0) | decimal | 38 | 0 | - | -"), + }, + NormalizeTypeTestCase { + description: "Snowflake NUMBER is a custom type rewritten to DECIMAL", + type_sql: "NUMBER(12, 2)", + dialect: "snowflake", + expected_normalization: Some("DECIMAL(12,2) | decimal | 12 | 2 | - | -"), + }, + NormalizeTypeTestCase { + description: "Snowflake NUMBER without precision takes the integer default", + type_sql: "NUMBER", + dialect: "snowflake", + expected_normalization: Some("DECIMAL(38,0) | decimal | 38 | 0 | - | -"), + }, + NormalizeTypeTestCase { + description: "Snowflake unbounded text takes the default length", + type_sql: "TEXT", + dialect: "snowflake", + expected_normalization: Some("VARCHAR(16777216) | string | - | - | 16777216 | -"), + }, + NormalizeTypeTestCase { + description: "Snowflake timestamp aliases", + type_sql: "TIMESTAMP_NTZ", + dialect: "snowflake", + expected_normalization: Some("TIMESTAMP_NTZ | timestamp | - | - | - | -"), + }, + NormalizeTypeTestCase { + description: "decimal precision and scale", + type_sql: "decimal(10, 2)", + dialect: "duckdb", + expected_normalization: Some("DECIMAL(10,2) | decimal | 10 | 2 | - | -"), + }, + NormalizeTypeTestCase { + description: "nested types are other", + type_sql: "ARRAY>", + dialect: "bigquery", + expected_normalization: Some("ARRAY> | other | - | - | - | -"), + }, + NormalizeTypeTestCase { + description: "a parse failure falls back to the text and reports the error", + type_sql: "order status", + dialect: "generic", + expected_normalization: Some( + "ORDERSTATUS | other | - | - | - | Parse error at line 1, column 13: Unexpected token after data type: status", + ), + }, + NormalizeTypeTestCase { + description: "a dialect this build does not include defers", + type_sql: "INT", + dialect: "mysql", + expected_normalization: None, + }, + NormalizeTypeTestCase { + description: "a dialect Polyglot does not know defers so Python raises", + type_sql: "INT", + dialect: "motherduck", + expected_normalization: None, + }, + NormalizeTypeTestCase { + description: "non-ASCII text defers", + type_sql: "caf\u{e9}", + dialect: "generic", + expected_normalization: None, + }, + NormalizeTypeTestCase { + description: "a parameter beyond i64 defers", + type_sql: "order status(99999999999999999999)", + dialect: "generic", + expected_normalization: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + normalize_type(test_case.type_sql, test_case.dialect) + .as_ref() + .map(normalization_text) + .as_deref(), + test_case.expected_normalization, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/type_system/tests/test_python_text.rs b/crates/sqlbuild-analysis/src/type_system/tests/test_python_text.rs new file mode 100644 index 0000000000..aec6520811 --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/tests/test_python_text.rs @@ -0,0 +1,102 @@ +use crate::type_system::_helpers::python_text::{PythonInt, python_int, split_type_and_params}; +use crate::type_system::tests::test_types::{PythonIntTestCase, SplitTypeTestCase}; + +#[test] +fn given_parameter_text_when_parsing_as_python_int_then_python_result_is_returned() { + let test_cases = [ + PythonIntTestCase { + description: "padded digits", + text: " \t42\x1c", + expected_value: PythonInt::Value(42), + }, + PythonIntTestCase { + description: "signs", + text: "-7", + expected_value: PythonInt::Value(-7), + }, + PythonIntTestCase { + description: "single underscores between digits", + text: "+1_000", + expected_value: PythonInt::Value(1000), + }, + PythonIntTestCase { + description: "double underscores", + text: "1__0", + expected_value: PythonInt::Invalid, + }, + PythonIntTestCase { + description: "leading underscore", + text: "_1", + expected_value: PythonInt::Invalid, + }, + PythonIntTestCase { + description: "words", + text: "MAX", + expected_value: PythonInt::Invalid, + }, + PythonIntTestCase { + description: "the smallest i64", + text: "-9223372036854775808", + expected_value: PythonInt::Value(i64::MIN), + }, + PythonIntTestCase { + description: "beyond i64", + text: "9223372036854775808", + expected_value: PythonInt::TooLarge, + }, + ]; + + for test_case in test_cases { + assert_eq!( + python_int(test_case.text), + test_case.expected_value, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_type_text_when_splitting_then_python_name_and_parameters_are_returned() { + let test_cases = [ + SplitTypeTestCase { + description: "name only", + type_sql: "VARCHAR", + expected_split: Some(("VARCHAR", vec![])), + }, + SplitTypeTestCase { + description: "parameters skip what int() rejects", + type_sql: "DECIMAL(10,X,2)", + expected_split: Some(("DECIMAL", vec![10, 2])), + }, + SplitTypeTestCase { + description: "a final newline is allowed by the regex end anchor", + type_sql: "NUMBER(5)\n", + expected_split: Some(("NUMBER", vec![5])), + }, + SplitTypeTestCase { + description: "anything else keeps the whole text", + type_sql: "ARRAY", + expected_split: Some(("ARRAY", vec![])), + }, + SplitTypeTestCase { + description: "nested parentheses keep the whole text", + type_sql: "A(1)(2)", + expected_split: Some(("A(1)(2)", vec![])), + }, + SplitTypeTestCase { + description: "a parameter beyond i64 defers", + type_sql: "A(99999999999999999999)", + expected_split: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + split_type_and_params(test_case.type_sql), + test_case.expected_split, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/type_system/tests/test_types.rs b/crates/sqlbuild-analysis/src/type_system/tests/test_types.rs new file mode 100644 index 0000000000..6213929267 --- /dev/null +++ b/crates/sqlbuild-analysis/src/type_system/tests/test_types.rs @@ -0,0 +1,21 @@ +use crate::type_system::_helpers::python_text::PythonInt; + +pub(super) struct NormalizeTypeTestCase { + pub(super) description: &'static str, + pub(super) type_sql: &'static str, + pub(super) dialect: &'static str, + /// `name | family | precision | scale | length | parse error`, or None to defer. + pub(super) expected_normalization: Option<&'static str>, +} + +pub(super) struct PythonIntTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_value: PythonInt, +} + +pub(super) struct SplitTypeTestCase { + pub(super) description: &'static str, + pub(super) type_sql: &'static str, + pub(super) expected_split: Option<(&'static str, Vec)>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs index 7444bd0374..bcc09d3157 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/mod.rs @@ -1 +1,2 @@ +pub(crate) mod normalization; pub(crate) mod registration; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs new file mode 100644 index 0000000000..c4c2d802e5 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs @@ -0,0 +1,43 @@ +//! Native type normalization for the preview type system. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::type_system::main::normalize_type::normalize_type; +use sqlbuild_analysis::type_system::models::{NormalizedType, TypeNormalization}; + +use crate::bindings::_helpers::boundary::panics::compiler_guard; + +/// `(name, family, precision, scale, length)` of one normalized type. +type NormalizedRow = (String, &'static str, Option, Option, Option); + +/// Normalize one type: `(normalized row, Polyglot parse error)`, or None to defer to Python. +#[pyfunction] +#[pyo3(name = "normalize_type")] +fn normalize_type_binding( + type_sql: &str, + dialect: &str, +) -> PyResult)>> { + compiler_guard(|| { + Ok(normalize_type(type_sql, dialect).map( + |TypeNormalization { + normalized, + parse_error, + }| (normalized_row(normalized), parse_error), + )) + }) +} + +fn normalized_row(normalized: NormalizedType) -> NormalizedRow { + ( + normalized.normalized_name, + normalized.family.as_str(), + normalized.precision, + normalized.scale, + normalized.length, + ) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(normalize_type_binding, module)?)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs index cd60143ae3..b1afbad36f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/registration.rs @@ -2,6 +2,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; -pub(crate) fn register(_module: &Bound<'_, PyModule>) -> PyResult<()> { +use crate::bindings::_helpers::type_system::normalization; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + normalization::register(module)?; Ok(()) } diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index cc0cc43c2f..8a6f7abbf6 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -262,6 +262,9 @@ def fingerprint_project_files(root: str, excluded_files: list[str]) -> str: ... def digest_files(paths: list[str]) -> list[str | None]: ... # Native analysis: type system. +def normalize_type( + type_sql: str, dialect: str +) -> tuple[tuple[str, str, int | None, int | None, int | None], str | None] | None: ... # Native analysis: model analysis session. diff --git a/src/sqlbuild/adapter/type_system/_helpers/type_normalization.py b/src/sqlbuild/adapter/type_system/_helpers/type_normalization.py index 95fd782cf1..b169051872 100644 --- a/src/sqlbuild/adapter/type_system/_helpers/type_normalization.py +++ b/src/sqlbuild/adapter/type_system/_helpers/type_normalization.py @@ -21,6 +21,11 @@ ) from sqlbuild.adapter.contract.models import NormalizedType from sqlbuild.adapter.contract.types import TypeDialect, TypeFamily +from sqlbuild.adapter.type_system.constants import ( + TYPE_NORMALIZATION_LOGGER_NAME, + TYPE_PARSE_FALLBACK_MESSAGE, +) +from sqlbuild.adapter.type_system.main._native_normalize_type import normalize_native_type from sqlbuild.adapters.bigquery.constants import ( BIGNUMERIC_TYPE_NAME, CUSTOM_NORMALIZATION_TYPE_NAMES, @@ -37,16 +42,24 @@ TEXT_TYPE_NAME, UNBOUNDED_TEXT_TYPE_NAMES, ) +from sqlbuild.compiler.frontier.main.native_stage_enabled import native_stage_enabled +from sqlbuild.compiler.frontier.types import NativeStage from sqlbuild.compiler.sql_analysis.main.import_polyglot import import_polyglot from sqlbuild.diagnostics.main.log_debug_event import log_debug_event -_DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.adapter") +_DEBUG_LOGGER: logging.Logger = logging.getLogger(TYPE_NORMALIZATION_LOGGER_NAME) @lru_cache(maxsize=1024) def normalize_type(*, type_sql: str, dialect: TypeDialect | str | None) -> NormalizedType: """Normalize one warehouse type string into a semantic comparison shape.""" + if native_stage_enabled(NativeStage.TYPE_SYSTEM): + native_normalized: NormalizedType | None = normalize_native_type( + type_sql=type_sql, dialect=dialect + ) + if native_normalized is not None: + return native_normalized polyglot_normalized: NormalizedType | None = _normalize_with_polyglot( type_sql=type_sql, dialect=dialect, @@ -83,7 +96,7 @@ def _normalize_with_polyglot( except polyglot_module.PolyglotError as error: log_debug_event( logger=_DEBUG_LOGGER, - message="type normalization polyglot parse failed; falling back", + message=TYPE_PARSE_FALLBACK_MESSAGE, type_sql=type_sql, dialect=str(dialect), sqlbuild_error=str(error), diff --git a/src/sqlbuild/adapter/type_system/constants.py b/src/sqlbuild/adapter/type_system/constants.py new file mode 100644 index 0000000000..345fe35380 --- /dev/null +++ b/src/sqlbuild/adapter/type_system/constants.py @@ -0,0 +1,4 @@ +"""Logging names shared by the Python and native type normalization.""" + +TYPE_NORMALIZATION_LOGGER_NAME: str = "sqlbuild.adapter" +TYPE_PARSE_FALLBACK_MESSAGE: str = "type normalization polyglot parse failed; falling back" diff --git a/src/sqlbuild/adapter/type_system/main/_native_normalize_type.py b/src/sqlbuild/adapter/type_system/main/_native_normalize_type.py index 37efa7eb36..6d9a9ceb50 100644 --- a/src/sqlbuild/adapter/type_system/main/_native_normalize_type.py +++ b/src/sqlbuild/adapter/type_system/main/_native_normalize_type.py @@ -2,8 +2,16 @@ from __future__ import annotations +import logging + +import sqlbuild._native as _native from sqlbuild.adapter.contract.models import NormalizedType -from sqlbuild.adapter.contract.types import TypeDialect +from sqlbuild.adapter.contract.types import TypeDialect, TypeFamily +from sqlbuild.adapter.type_system.constants import ( + TYPE_NORMALIZATION_LOGGER_NAME, + TYPE_PARSE_FALLBACK_MESSAGE, +) +from sqlbuild.diagnostics.main.log_debug_event import log_debug_event def normalize_native_type( @@ -11,4 +19,26 @@ def normalize_native_type( ) -> NormalizedType | None: """Return the native normalization, or None where Python must normalize the type.""" - return None + if not isinstance(type_sql, str) or not isinstance(dialect, str | None): + return None + native: tuple[tuple[str, str, int | None, int | None, int | None], str | None] | None = ( + _native.normalize_type(type_sql, str(dialect or "generic")) + ) + if native is None: + return None + (normalized_name, family, precision, scale, length), parse_error = native + if parse_error is not None: + log_debug_event( + logger=logging.getLogger(TYPE_NORMALIZATION_LOGGER_NAME), + message=TYPE_PARSE_FALLBACK_MESSAGE, + type_sql=type_sql, + dialect=str(dialect), + sqlbuild_error=parse_error, + ) + return NormalizedType( + normalized_name=normalized_name, + family=TypeFamily(family), + precision=precision, + scale=scale, + length=length, + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 14a36598f9..666ca70806 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -812,6 +812,16 @@ class CompilerEngineParityTestCase: expected_exit_codes: tuple[int, int] +@dataclass(frozen=True) +class NativeTypeSystemTestCase: + """Python and preview compiles of one project whose contracts compare type spellings.""" + + description: str + project_files: dict[str, str] + expected_exit_code: int + expected_report_text: str + + @dataclass(frozen=True) class NativeAnalysisSeamTestCase: """Every engine compiling a copy of one project while the analysis stage seams are recorded.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 665e118250..309e13770b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -27,6 +27,8 @@ import duckdb import pytest +import sqlbuild._native as native_module +import sqlbuild.adapter.type_system._helpers.type_normalization as type_normalization import sqlbuild.cli.commands._helpers.compile.target_writer as target_writer import sqlbuild.cli.commands.main.project._compile as compile_command_module import sqlbuild.cli.compile_render_reuse._helpers.load_notice as render_load_notice @@ -4802,3 +4804,31 @@ def engine_in_process_compile( seams: dict[str, list[object]] = record_analysis_seams(monkeypatch=patch) run: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) return run, seams + + +def type_system_engine_compile( + *, + project_dir: Path, + engine: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> tuple[CompileReuseRun, list[bool]]: + """Compile in this process under `engine`; return the run and whether each native type + normalization answered.""" + + answered: list[bool] = [] + normalize: Callable[[str, str], object] = native_module.normalize_type + + def recorded(type_sql: str, dialect: str) -> object: + result: object = normalize(type_sql, dialect) + answered.append(result is not None) + return result + + type_normalization.normalize_type.cache_clear() + with monkeypatch.context() as patch: + patch.setattr(native_module, "normalize_type", recorded) + for name, value in {COMPILER_ENGINE_ENV_VAR: engine, REUSE_DISABLE_ENV_VAR: "1"}.items(): + patch.setenv(name, value) + run: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) + type_normalization.normalize_type.cache_clear() + return run, answered diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_type_system.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_type_system.py new file mode 100644 index 0000000000..d58d453159 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_native_type_system.py @@ -0,0 +1,96 @@ +"""Contracts compare type spellings through the native type system under the preview engine.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + NativeTypeSystemTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + CompileReuseRun, + report_without_engine, + type_system_engine_compile, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project + +_PROJECT_TOML: str = ( + 'name = "orders"\nadapter = "duckdb"\n\n[connection]\ndatabase = "warehouse.duckdb"\n' +) +_ORDERS_MODEL: str = """MODEL ( + description "Orders with contracted column types.", + materialized table, + contract enforced, + columns ( + order_id (type INTEGER), + amount (type numeric( 10 , 2 )), + note (type TEXT), + ), +); + +SELECT CAST(1 AS INT) AS order_id, CAST(2.5 AS DECIMAL(10, 2)) AS amount, CAST('gift' AS VARCHAR) AS note +""" +_ORDER_TOTALS_MODEL: str = """MODEL ( + description "Order totals whose contract disagrees with the query.", + materialized table, + contract enforced, + columns ( + order_id (type BIGINT), + total (type DECIMAL(12, 2)), + ), +); + +SELECT order_id, amount AS total FROM __ref("orders") +""" + + +@pytest.mark.parametrize( + "test_case", + [ + NativeTypeSystemTestCase( + description="matching spellings pass and a changed precision is reported", + project_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/orders.sql": _ORDERS_MODEL, + "models/order_totals.sql": _ORDER_TOTALS_MODEL, + }, + expected_exit_code=1, + expected_report_text="inferred as numeric(10,2) but declared type is DECIMAL(12,2)", + ), + ], + ids=lambda case: case.description, +) +def test_given_contract_types_when_compiling_with_preview_then_native_types_match_python( + test_case: NativeTypeSystemTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + outcomes: dict[str, tuple[CompileReuseRun, list[bool]]] = { + engine: type_system_engine_compile( + project_dir=prepare_inline_project( + tmp_path=tmp_path, project_name=engine, repo_files=test_case.project_files + ), + engine=engine, + monkeypatch=monkeypatch, + capsys=capsys, + ) + for engine in ("python", "native-preview") + } + python_run, python_answers = outcomes["python"] + preview_run, preview_answers = outcomes["native-preview"] + + assert (python_run.returncode, preview_run.returncode) == ( + test_case.expected_exit_code, + test_case.expected_exit_code, + ) + assert test_case.expected_report_text in python_run.report + assert report_without_engine(preview_run) == report_without_engine(python_run) + assert preview_run.compiled == python_run.compiled + assert (python_answers, any(preview_answers)) == ([], True) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/adapter/type_system/__init__.py b/tests/integration/src/sqlbuild/adapter/type_system/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/adapter/type_system/_test_types.py b/tests/integration/src/sqlbuild/adapter/type_system/_test_types.py new file mode 100644 index 0000000000..ae90c6d36d --- /dev/null +++ b/tests/integration/src/sqlbuild/adapter/type_system/_test_types.py @@ -0,0 +1,26 @@ +"""Test case types for native type normalization parity.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class GeneratedTypeParityTestCase: + """Seeded generated type strings normalized under every dialect.""" + + description: str + seed: int + count: int + expected_minimum_native: int + expected_minimum_parse_errors: int + expected_minimum_deferred: int + + +@dataclass(frozen=True) +class TypeParityTestCase: + """One type string normalized natively and by Python under every dialect.""" + + description: str + type_sql: str + expected_native_dialects: frozenset[str | None] diff --git a/tests/integration/src/sqlbuild/adapter/type_system/helpers.py b/tests/integration/src/sqlbuild/adapter/type_system/helpers.py new file mode 100644 index 0000000000..1fbe015ea9 --- /dev/null +++ b/tests/integration/src/sqlbuild/adapter/type_system/helpers.py @@ -0,0 +1,235 @@ +"""Generated type strings and Python-versus-native normalization outcomes.""" + +from __future__ import annotations + +import logging +import random +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from itertools import product + +import pytest + +from sqlbuild.adapter.contract.models import NormalizedType +from sqlbuild.adapter.type_system._helpers import type_normalization +from sqlbuild.adapter.type_system.constants import TYPE_NORMALIZATION_LOGGER_NAME +from sqlbuild.adapter.type_system.main._native_normalize_type import normalize_native_type +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.types import CompilerEngine + +DIALECTS: tuple[str | None, ...] = ( + None, + "generic", + "bigquery", + "snowflake", + "duckdb", + "motherduck", + "databricks", + "postgres", + "tsql", + "postgresql", + "DuckDB", + "mysql", +) +_BASE_TYPES: tuple[str, ...] = ( + "INT", + "INTEGER", + "BIGINT", + "SMALLINT", + "TINYINT", + "INT64", + "LONG", + "int", + "Integer", + "DECIMAL", + "NUMERIC", + "NUMBER", + "BIGNUMERIC", + "BIGDECIMAL", + "DEC", + "FLOAT", + "FLOAT64", + "DOUBLE", + "REAL", + "DOUBLE PRECISION", + "VARCHAR", + "CHAR", + "CHARACTER", + "CHARACTER VARYING", + "STRING", + "TEXT", + "NVARCHAR", + "BOOL", + "BOOLEAN", + "TIMESTAMP", + "TIMESTAMP_NTZ", + "TIMESTAMP_LTZ", + "TIMESTAMP_TZ", + "TIMESTAMPTZ", + "TIMESTAMPNTZ", + "TIMESTAMP WITH TIME ZONE", + "TIMESTAMP WITHOUT TIME ZONE", + "DATE", + "DATETIME", + "TIME", + "INTERVAL", + "JSON", + "VARIANT", + "BYTES", + "BLOB", + "UUID", + "GEOGRAPHY", + "HUGEINT", + "MONEY", + '"quoted"', + '"Mixed Case"', + "order_status", + "foo bar", + "1abc", + "INT;", + "", +) +_PARAMETERS: tuple[str, ...] = ( + "", + "(10)", + "(10,2)", + "(10, 2)", + "( 38 , 0 )", + "(0)", + "(-1)", + "(1_000)", + "(MAX)", + "(abc)", + "()", + "(99999999999999999999)", + "(10,2,3)", + "(+5)", +) +_PADDING: tuple[str, ...] = ("", " ", "\t", "\n", " \n") +_MAX_NESTING: int = 3 +_NESTED_TEMPLATES: tuple[str, ...] = ( + "ARRAY<{0}>", + "{0}[]", + "STRUCT", + "STRUCT(a {0})", + "MAP({0}, {1})", +) +_TEMPLATES_BY_DEPTH: tuple[tuple[str, ...], ...] = ( + *((("{scalar}",) * 15 + _NESTED_TEMPLATES,) * _MAX_NESTING), + ("{scalar}",), +) +_CASINGS: tuple[Callable[[str], str], ...] = (str,) * 6 + (str.lower,) +_SPACINGS: tuple[str, ...] = (" ", " ") + + +@dataclass(frozen=True) +class TypeOutcome: + """A normalized type, Python's error, or None where native defers, and the errors logged.""" + + normalized: NormalizedType | str | None + logged_errors: tuple[tuple[str, object, object, object], ...] + + +@dataclass(frozen=True) +class TypeParity: + """One type string and dialect, normalized by Python and natively.""" + + type_sql: str + dialect: str | None + python: TypeOutcome + native: TypeOutcome + + +def generated_type(*, rng: random.Random, depth: int = 0) -> str: + """Return a type string: scalar, parameterized, nested, quoted, padded or invalid.""" + + children: list[str] = [ + generated_type(rng=rng, depth=depth + 1) for _ in range(2 * (depth < _MAX_NESTING)) + ] + template: str = rng.choice(_TEMPLATES_BY_DEPTH[min(depth, _MAX_NESTING)]) + return template.format(*children, scalar=_generated_scalar(rng=rng)) + + +def type_parities(*, type_strings: list[str], monkeypatch: pytest.MonkeyPatch) -> list[TypeParity]: + """Normalize every type string under every dialect with Python and natively.""" + + parities: list[TypeParity] = [] + for type_sql, dialect in product(type_strings, DIALECTS): + parities.append(type_parity(type_sql=type_sql, dialect=dialect, monkeypatch=monkeypatch)) + return parities + + +def type_parity( + *, type_sql: str, dialect: str | None, monkeypatch: pytest.MonkeyPatch +) -> TypeParity: + """Normalize one type with the Python engine and with the native type system.""" + + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, CompilerEngine.PYTHON.value) + with _logged_errors() as python_errors: + python: NormalizedType | str = _python_normalized(type_sql=type_sql, dialect=dialect) + with _logged_errors() as native_errors: + native: NormalizedType | None = normalize_native_type(type_sql=type_sql, dialect=dialect) + return TypeParity( + type_sql=type_sql, + dialect=dialect, + python=TypeOutcome(normalized=python, logged_errors=tuple(python_errors)), + native=TypeOutcome(normalized=native, logged_errors=tuple(native_errors)), + ) + + +def is_native(parity: TypeParity) -> bool: + """Return whether the native type system answered.""" + + return parity.native.normalized is not None + + +def logged_parse_error(parity: TypeParity) -> bool: + """Return whether Python logged a Polyglot parse failure for this type.""" + + return bool(parity.python.logged_errors) + + +def _generated_scalar(*, rng: random.Random) -> str: + type_sql: str = rng.choice(_CASINGS)(rng.choice(_BASE_TYPES) + rng.choice(_PARAMETERS)) + return ( + rng.choice(_PADDING) + type_sql.replace(" ", rng.choice(_SPACINGS)) + rng.choice(_PADDING) + ) + + +def _python_normalized(*, type_sql: str, dialect: str | None) -> NormalizedType | str: + try: + return type_normalization.normalize_type.__wrapped__(type_sql=type_sql, dialect=dialect) + except ValueError as error: + return f"{type(error).__name__}: {error}" + + +@contextmanager +def _logged_errors() -> Iterator[list[tuple[str, object, object, object]]]: + records: list[tuple[str, object, object, object]] = [] + handler: _RecordingHandler = _RecordingHandler(records=records) + logger: logging.Logger = logging.getLogger(TYPE_NORMALIZATION_LOGGER_NAME) + previous_level: int = logger.level + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + try: + yield records + finally: + logger.removeHandler(handler) + logger.setLevel(previous_level) + + +class _RecordingHandler(logging.Handler): + def __init__(self, *, records: list[tuple[str, object, object, object]]) -> None: + super().__init__(level=logging.DEBUG) + self._records: list[tuple[str, object, object, object]] = records + + def emit(self, record: logging.LogRecord) -> None: + self._records.append( + ( + record.getMessage(), + getattr(record, "type_sql", None), + getattr(record, "dialect", None), + getattr(record, "sqlbuild_error", None), + ) + ) diff --git a/tests/integration/src/sqlbuild/adapter/type_system/test_native_type_parity.py b/tests/integration/src/sqlbuild/adapter/type_system/test_native_type_parity.py new file mode 100644 index 0000000000..c1d8da5436 --- /dev/null +++ b/tests/integration/src/sqlbuild/adapter/type_system/test_native_type_parity.py @@ -0,0 +1,142 @@ +"""Native type normalization matches Python for every type it answers, or defers.""" + +from __future__ import annotations + +import random + +import pytest + +from tests.integration.src.sqlbuild.adapter.type_system._test_types import ( + GeneratedTypeParityTestCase, + TypeParityTestCase, +) +from tests.integration.src.sqlbuild.adapter.type_system.helpers import ( + TypeParity, + generated_type, + is_native, + logged_parse_error, + type_parities, +) +from tests.integration.src.sqlbuild.compiler.helpers import mismatches + +_NATIVE_DIALECTS: frozenset[str | None] = frozenset( + {None, "generic", "bigquery", "snowflake", "duckdb", "databricks", "postgres", "tsql"} + | {"postgresql", "DuckDB"} +) + + +@pytest.mark.parametrize( + "test_case", + [ + GeneratedTypeParityTestCase( + description="seeded scalar, parameterized, nested, padded and invalid types", + seed=20261008, + count=1500, + expected_minimum_native=12000, + expected_minimum_parse_errors=5000, + expected_minimum_deferred=3000, + ), + ], + ids=lambda case: case.description, +) +def test_given_generated_types_when_normalizing_natively_then_python_normalization_matches( + test_case: GeneratedTypeParityTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + rng: random.Random = random.Random(test_case.seed) + type_strings: list[str] = [generated_type(rng=rng) for _ in range(test_case.count)] + parities: list[TypeParity] = type_parities(type_strings=type_strings, monkeypatch=monkeypatch) + + answered: list[TypeParity] = list(filter(is_native, parities)) + assert ( + mismatches( + inputs=[(parity.type_sql, parity.dialect) for parity in answered], + expected=[parity.python for parity in answered], + actual=[parity.native for parity in answered], + ), + len(answered) >= test_case.expected_minimum_native, + sum(map(logged_parse_error, answered)) >= test_case.expected_minimum_parse_errors, + len(parities) - len(answered) >= test_case.expected_minimum_deferred, + ) == ([], True, True, True), test_case.description + + +@pytest.mark.parametrize( + "test_case", + [ + TypeParityTestCase( + description="integer aliases", + type_sql="INTEGER", + expected_native_dialects=_NATIVE_DIALECTS, + ), + TypeParityTestCase( + description="snowflake number precision and scale defaults", + type_sql="NUMBER", + expected_native_dialects=_NATIVE_DIALECTS, + ), + TypeParityTestCase( + description="spaced decimal parameters", + type_sql=" decimal( 10 , 2 ) ", + expected_native_dialects=_NATIVE_DIALECTS, + ), + TypeParityTestCase( + description="unbounded text", + type_sql="TEXT", + expected_native_dialects=_NATIVE_DIALECTS, + ), + TypeParityTestCase( + description="nested array of structs", + type_sql="ARRAY>>", + expected_native_dialects=_NATIVE_DIALECTS, + ), + TypeParityTestCase( + description="map of varchar to decimal", + type_sql="MAP(VARCHAR, DECIMAL(18, 4))", + expected_native_dialects=_NATIVE_DIALECTS, + ), + TypeParityTestCase( + description="quoted custom type", + type_sql='"Order Status"', + expected_native_dialects=_NATIVE_DIALECTS, + ), + TypeParityTestCase( + description="timestamp aliases", + type_sql="timestamp_ntz(9)", + expected_native_dialects=_NATIVE_DIALECTS, + ), + TypeParityTestCase( + description="invalid type falls back after a logged parse error", + type_sql="order status", + expected_native_dialects=_NATIVE_DIALECTS, + ), + TypeParityTestCase( + description="parameters Python keeps as large integers defer", + type_sql="VARCHAR(99999999999999999999)", + expected_native_dialects=frozenset(), + ), + TypeParityTestCase( + description="non-ASCII text defers", + type_sql="caf\u00e9", + expected_native_dialects=frozenset(), + ), + ], + ids=lambda case: case.description, +) +def test_given_type_when_normalizing_under_each_dialect_then_native_answers_match_python( + test_case: TypeParityTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + parities: list[TypeParity] = type_parities( + type_strings=[test_case.type_sql], monkeypatch=monkeypatch + ) + + answered: list[TypeParity] = list(filter(is_native, parities)) + assert ( + mismatches( + inputs=[parity.dialect for parity in answered], + expected=[parity.python for parity in answered], + actual=[parity.native for parity in answered], + ), + frozenset(parity.dialect for parity in answered), + ) == ([], test_case.expected_native_dialects), test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 42e09749570b5567b90f67de8507fb6ed746ce76 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:45:57 +0100 Subject: [PATCH 2/2] fix(native): defer types with deep array suffixes to Python --- .../src/type_system/_helpers/parsed.rs | 4 +- .../src/type_system/constants.rs | 2 + .../src/type_system/main/normalize_type.rs | 8 +++- .../type_system/tests/test_normalize_type.rs | 38 ++++++++++++++++++- .../src/type_system/tests/test_types.rs | 6 +++ .../adapter/type_system/_test_types.py | 9 +++++ .../type_system/test_native_type_parity.py | 31 +++++++++++++++ 7 files changed, 93 insertions(+), 5 deletions(-) diff --git a/crates/sqlbuild-analysis/src/type_system/_helpers/parsed.rs b/crates/sqlbuild-analysis/src/type_system/_helpers/parsed.rs index ff4b9e99f7..dfc944b99f 100644 --- a/crates/sqlbuild-analysis/src/type_system/_helpers/parsed.rs +++ b/crates/sqlbuild-analysis/src/type_system/_helpers/parsed.rs @@ -35,11 +35,11 @@ pub(crate) fn parse_type(type_sql: &str, dialect: &Dialect) -> Result, ) -> Option { - let expression: Expression = Expression::DataType(parsed.clone()); + let expression: Expression = Expression::DataType(parsed); let Ok(generated) = polyglot.generate(&expression) else { return None; }; diff --git a/crates/sqlbuild-analysis/src/type_system/constants.rs b/crates/sqlbuild-analysis/src/type_system/constants.rs index 51852fa0b6..9e05c81f24 100644 --- a/crates/sqlbuild-analysis/src/type_system/constants.rs +++ b/crates/sqlbuild-analysis/src/type_system/constants.rs @@ -33,6 +33,8 @@ pub(crate) const POLYGLOT_TYPE_NAME_ALIASES: &[(&str, &str)] = &[ ("TINY_INT", "TINYINT"), ("VAR_CHAR", "VARCHAR"), ]; +/// The most `[` a type may hold before Python decides it; Polyglot's guards do not bound `[]`. +pub(crate) const MAX_TYPE_BRACKETS: usize = 32; /// The function call depth limit SQLBuild passes to every Polyglot parse. pub(crate) const MAX_FUNCTION_CALL_DEPTH: usize = 512; /// Characters Python's `str.strip()` and the `\s` regex class treat as ASCII whitespace. diff --git a/crates/sqlbuild-analysis/src/type_system/main/normalize_type.rs b/crates/sqlbuild-analysis/src/type_system/main/normalize_type.rs index 506ba08ad4..466d0d2186 100644 --- a/crates/sqlbuild-analysis/src/type_system/main/normalize_type.rs +++ b/crates/sqlbuild-analysis/src/type_system/main/normalize_type.rs @@ -3,19 +3,23 @@ use crate::type_system::_helpers::dialects::{coerce_type_dialect, polyglot_dialect}; use crate::type_system::_helpers::fallback::normalize_with_fallback; use crate::type_system::_helpers::parsed::{normalize_parsed, parse_type}; +use crate::type_system::constants::MAX_TYPE_BRACKETS; use crate::type_system::models::{TypeDialect, TypeNormalization}; /// Normalize one type as Python does for the dialect name it hands Polyglot, or None to defer. #[must_use] pub fn normalize_type(type_sql: &str, dialect: &str) -> Option { - if !type_sql.is_ascii() || !dialect.is_ascii() { + if !type_sql.is_ascii() + || !dialect.is_ascii() + || type_sql.bytes().filter(|byte| *byte == b'[').count() > MAX_TYPE_BRACKETS + { return None; } let polyglot = polyglot_dialect(dialect)?; let type_dialect: Option = coerce_type_dialect(dialect); match parse_type(type_sql, &polyglot) { Ok(parsed) => Some(TypeNormalization { - normalized: normalize_parsed(&parsed, &polyglot, type_dialect)?, + normalized: normalize_parsed(parsed, &polyglot, type_dialect)?, parse_error: None, }), Err(error) => Some(TypeNormalization { diff --git a/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs b/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs index 4aa51ed32a..63ea20f15e 100644 --- a/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs +++ b/crates/sqlbuild-analysis/src/type_system/tests/test_normalize_type.rs @@ -1,6 +1,6 @@ use crate::type_system::main::normalize_type::normalize_type; use crate::type_system::tests::helpers::normalization_text; -use crate::type_system::tests::test_types::NormalizeTypeTestCase; +use crate::type_system::tests::test_types::{BracketDepthTestCase, NormalizeTypeTestCase}; #[test] fn given_type_strings_when_normalizing_then_python_normalization_is_returned() { @@ -105,3 +105,39 @@ fn given_type_strings_when_normalizing_then_python_normalization_is_returned() { ); } } + +#[test] +fn given_array_suffixes_when_normalizing_on_a_small_stack_then_deep_types_defer() { + let test_cases = [ + BracketDepthTestCase { + description: "at the bracket cap, parsed on a 1 MiB debug-build stack", + depth: 32, + expected_native: true, + }, + BracketDepthTestCase { + description: "one past the bracket cap", + depth: 33, + expected_native: false, + }, + BracketDepthTestCase { + description: "far past the bracket cap", + depth: 100_000, + expected_native: false, + }, + ]; + + for test_case in test_cases { + let type_sql: String = format!("INT{}", "[]".repeat(test_case.depth)); + let answered: bool = std::thread::Builder::new() + .stack_size(1024 * 1024) + .spawn(move || normalize_type(&type_sql, "generic").is_some()) + .expect("the test thread starts") + .join() + .expect("normalization does not overflow the stack"); + assert_eq!( + answered, test_case.expected_native, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/type_system/tests/test_types.rs b/crates/sqlbuild-analysis/src/type_system/tests/test_types.rs index 6213929267..6044e68168 100644 --- a/crates/sqlbuild-analysis/src/type_system/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/type_system/tests/test_types.rs @@ -19,3 +19,9 @@ pub(super) struct SplitTypeTestCase { pub(super) type_sql: &'static str, pub(super) expected_split: Option<(&'static str, Vec)>, } + +pub(super) struct BracketDepthTestCase { + pub(super) description: &'static str, + pub(super) depth: usize, + pub(super) expected_native: bool, +} diff --git a/tests/integration/src/sqlbuild/adapter/type_system/_test_types.py b/tests/integration/src/sqlbuild/adapter/type_system/_test_types.py index ae90c6d36d..762e82405f 100644 --- a/tests/integration/src/sqlbuild/adapter/type_system/_test_types.py +++ b/tests/integration/src/sqlbuild/adapter/type_system/_test_types.py @@ -24,3 +24,12 @@ class TypeParityTestCase: description: str type_sql: str expected_native_dialects: frozenset[str | None] + + +@dataclass(frozen=True) +class DeepTypeTestCase: + """A type deeper than the wheel can parse, which only native sees.""" + + description: str + type_sql: str + expected_native: None diff --git a/tests/integration/src/sqlbuild/adapter/type_system/test_native_type_parity.py b/tests/integration/src/sqlbuild/adapter/type_system/test_native_type_parity.py index c1d8da5436..eb481b27a3 100644 --- a/tests/integration/src/sqlbuild/adapter/type_system/test_native_type_parity.py +++ b/tests/integration/src/sqlbuild/adapter/type_system/test_native_type_parity.py @@ -6,7 +6,9 @@ import pytest +from sqlbuild.adapter.type_system.main._native_normalize_type import normalize_native_type from tests.integration.src.sqlbuild.adapter.type_system._test_types import ( + DeepTypeTestCase, GeneratedTypeParityTestCase, TypeParityTestCase, ) @@ -112,6 +114,16 @@ def test_given_generated_types_when_normalizing_natively_then_python_normalizati type_sql="VARCHAR(99999999999999999999)", expected_native_dialects=frozenset(), ), + TypeParityTestCase( + description="array suffixes at the bracket cap", + type_sql="INT" + "[]" * 32, + expected_native_dialects=_NATIVE_DIALECTS, + ), + TypeParityTestCase( + description="array suffixes past the bracket cap defer", + type_sql="INT" + "[]" * 33, + expected_native_dialects=frozenset(), + ), TypeParityTestCase( description="non-ASCII text defers", type_sql="caf\u00e9", @@ -138,5 +150,24 @@ def test_given_type_when_normalizing_under_each_dialect_then_native_answers_matc ) == ([], test_case.expected_native_dialects), test_case.description +@pytest.mark.parametrize( + "test_case", + [ + DeepTypeTestCase( + description="100,000 array suffixes, past where the wheel itself overflows", + type_sql="INT" + "[]" * 100_000, + expected_native=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_type_past_the_wheel_depth_when_normalizing_natively_then_it_defers( + test_case: DeepTypeTestCase, +) -> None: + assert normalize_native_type(type_sql=test_case.type_sql, dialect="generic") is ( + test_case.expected_native + ) + + if __name__ == "__main__": pytest.main([__file__, "-vv"])