diff --git a/Cargo.lock b/Cargo.lock index 2b6a5aceb6..93e9016bfc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1319,7 +1319,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2174,7 +2174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cde48f36530cbd91b65f050101e6c056d6eb743cea2a8f0f19beef69c6742ff8" dependencies = [ "scopeguard", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3199,7 +3199,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3637,7 +3637,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4091,7 +4091,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4143,6 +4143,7 @@ dependencies = [ "sqlbuild-sqltext", "sqlclean", "sqlquack-syntax", + "sqlquack-syntax-bigquery", "sqlquack-syntax-postgres", ] @@ -4499,7 +4500,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=4d0c6bc56da0e44eb164601a2af673ae7998257d#4d0c6bc56da0e44eb164601a2af673ae7998257d" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=98eefae657b57566a091e39f7cde992bee45eb6f#98eefae657b57566a091e39f7cde992bee45eb6f" dependencies = [ "serde", "serde_json", @@ -4509,10 +4510,18 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "sqlquack-syntax-bigquery" +version = "0.2.0" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=98eefae657b57566a091e39f7cde992bee45eb6f#98eefae657b57566a091e39f7cde992bee45eb6f" +dependencies = [ + "sqlquack-syntax", +] + [[package]] name = "sqlquack-syntax-derive" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=4d0c6bc56da0e44eb164601a2af673ae7998257d#4d0c6bc56da0e44eb164601a2af673ae7998257d" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=98eefae657b57566a091e39f7cde992bee45eb6f#98eefae657b57566a091e39f7cde992bee45eb6f" dependencies = [ "proc-macro2", "quote", @@ -4522,7 +4531,7 @@ dependencies = [ [[package]] name = "sqlquack-syntax-postgres" version = "0.2.0" -source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=4d0c6bc56da0e44eb164601a2af673ae7998257d#4d0c6bc56da0e44eb164601a2af673ae7998257d" +source = "git+https://github.com/sqlstack-io/sqlquack.git?rev=98eefae657b57566a091e39f7cde992bee45eb6f#98eefae657b57566a091e39f7cde992bee45eb6f" dependencies = [ "sqlquack-syntax", ] @@ -4618,7 +4627,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4740,7 +4749,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5054,7 +5063,7 @@ checksum = "f2f6fb2847f6742cd76af783a2a2c49e9375d0a111c7bef6f71cd9e738c72d6e" dependencies = [ "memoffset", "tempfile", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -5387,7 +5396,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f03cafe024..9a7f79dcfd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -176,9 +176,10 @@ sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } sqlbuild-state = { path = "crates/sqlbuild-state" } # SQL lint Rules, quality Rules and the formatter. sqlclean = { git = "https://github.com/sqlstack-io/sqlclean.git", rev = "cf79656d247277a30bc81ef39b6dee420e721221" } -# The lossless SQL tokenizer behind query fingerprints. -sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "4d0c6bc56da0e44eb164601a2af673ae7998257d" } -sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "4d0c6bc56da0e44eb164601a2af673ae7998257d" } +# SQLQuack's lossless tokenizer behind query fingerprints and its parsers for warehouse types. +sqlquack-syntax = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "98eefae657b57566a091e39f7cde992bee45eb6f" } +sqlquack-syntax-bigquery = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "98eefae657b57566a091e39f7cde992bee45eb6f" } +sqlquack-syntax-postgres = { git = "https://github.com/sqlstack-io/sqlquack.git", rev = "98eefae657b57566a091e39f7cde992bee45eb6f" } stacker = "0.1" syn = { version = "2", features = ["full", "visit", "extra-traits"] } subtle = "2" diff --git a/crates/sqlbuild-analysis/Cargo.toml b/crates/sqlbuild-analysis/Cargo.toml index 80a5ae2313..9a7d4b5a89 100644 --- a/crates/sqlbuild-analysis/Cargo.toml +++ b/crates/sqlbuild-analysis/Cargo.toml @@ -27,6 +27,7 @@ sqlbuild-scopes.workspace = true sqlbuild-sqltext.workspace = true sqlclean.workspace = true sqlquack-syntax.workspace = true +sqlquack-syntax-bigquery.workspace = true sqlquack-syntax-postgres.workspace = true [lints] diff --git a/crates/sqlbuild-analysis/src/lib.rs b/crates/sqlbuild-analysis/src/lib.rs index 5ad16bf8b6..cf2e089071 100644 --- a/crates/sqlbuild-analysis/src/lib.rs +++ b/crates/sqlbuild-analysis/src/lib.rs @@ -51,4 +51,5 @@ pub mod sql_tokens; pub mod sql_tree; pub mod target_paths; pub mod type_system; +pub mod warehouse_types; pub mod worker_stack; diff --git a/crates/sqlbuild-analysis/src/warehouse_types/constants.rs b/crates/sqlbuild-analysis/src/warehouse_types/constants.rs new file mode 100644 index 0000000000..3a91139238 --- /dev/null +++ b/crates/sqlbuild-analysis/src/warehouse_types/constants.rs @@ -0,0 +1,12 @@ +//! Constants of warehouse type parsing. + +/// The tree key of a cast. +pub(crate) const CAST_KEY: &str = "cast"; +/// The tree key of a cast's target type. +pub(crate) const CAST_TARGET_KEY: &str = "to"; +/// The dialect whose type names go through PostgreSQL's own type grammar. +pub(crate) const POSTGRES_DIALECT: &str = "postgres"; +/// The dialect whose type names are parsed as a Snowflake cast's target. +pub(crate) const SNOWFLAKE_DIALECT: &str = "snowflake"; +/// The dialect whose type names are parsed as a BigQuery cast's target. +pub(crate) const BIGQUERY_DIALECT: &str = "bigquery"; diff --git a/crates/sqlbuild-analysis/src/warehouse_types/main/mod.rs b/crates/sqlbuild-analysis/src/warehouse_types/main/mod.rs new file mode 100644 index 0000000000..fdbebafce0 --- /dev/null +++ b/crates/sqlbuild-analysis/src/warehouse_types/main/mod.rs @@ -0,0 +1 @@ +pub mod parse_warehouse_type; diff --git a/crates/sqlbuild-analysis/src/warehouse_types/main/parse_warehouse_type.rs b/crates/sqlbuild-analysis/src/warehouse_types/main/parse_warehouse_type.rs new file mode 100644 index 0000000000..dc6682be9a --- /dev/null +++ b/crates/sqlbuild-analysis/src/warehouse_types/main/parse_warehouse_type.rs @@ -0,0 +1,59 @@ +//! `parse_warehouse_type`: a warehouse type name as SQLQuack's parser for its dialect reads it. + +use serde_json::Value; +use sqlquack_syntax::tokens::models::Dialect; +use sqlquack_syntax::{Expression, ParseOptions, parse_strict}; + +use crate::warehouse_types::constants::{ + BIGQUERY_DIALECT, CAST_KEY, CAST_TARGET_KEY, POSTGRES_DIALECT, SNOWFLAKE_DIALECT, +}; + +/// The type tree of `spelling` in `dialect`, None without a parser, or why the parser rejects it. +pub fn parse_warehouse_type(spelling: &str, dialect: &str) -> Result, String> { + let cast: String = format!("SELECT CAST(NULL AS {spelling})"); + let statements: Vec = match dialect { + POSTGRES_DIALECT => { + let data_type = sqlquack_syntax_postgres::parse_type(spelling) + .map_err(|error| format!("{error:?}"))?; + return serde_json::to_value(data_type) + .map(Some) + .map_err(|error| error.to_string()); + } + SNOWFLAKE_DIALECT => parse_strict(&cast, Dialect::Snowflake, &ParseOptions::default()) + .map_err(|error| format!("{error:?}"))?, + BIGQUERY_DIALECT => { + sqlquack_syntax_bigquery::parse_strict(&cast) + .map_err(|error| format!("{error:?}"))? + .statements + } + _ => return Ok(None), + }; + let tree: Value = serde_json::to_value(&statements).map_err(|error| error.to_string())?; + match (statements.len(), cast_targets(&tree).as_slice()) { + (1, [target]) => Ok(Some((*target).clone())), + _ => Err(format!("`{spelling}` is not a single type name")), + } +} + +/// Every cast's target type in `tree`, in order. +fn cast_targets(tree: &Value) -> Vec<&Value> { + let mut targets: Vec<&Value> = Vec::new(); + let mut pending: Vec<&Value> = vec![tree]; + while let Some(node) = pending.pop() { + match node { + Value::Object(fields) => { + for (key, value) in fields.iter().rev() { + pending.push(value); + if key == CAST_KEY + && let Some(target) = value.get(CAST_TARGET_KEY) + { + targets.push(target); + } + } + } + Value::Array(items) => pending.extend(items.iter().rev()), + _ => {} + } + } + targets +} diff --git a/crates/sqlbuild-analysis/src/warehouse_types/mod.rs b/crates/sqlbuild-analysis/src/warehouse_types/mod.rs new file mode 100644 index 0000000000..c0ebb26c0f --- /dev/null +++ b/crates/sqlbuild-analysis/src/warehouse_types/mod.rs @@ -0,0 +1,4 @@ +//! Warehouse column type names as SQLQuack's parser for each warehouse dialect reads them. + +pub(crate) mod constants; +pub mod main; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs index 98a6ce5ace..b719e29b6b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/queries.rs @@ -14,11 +14,24 @@ fn query_fingerprint(py: Python<'_>, sql: &str, dialect: &str) -> PyResult, spelling: &str, dialect: &str) -> PyResult> { + py.compiler_detach(|| { + sqlbuild_analysis::warehouse_types::main::parse_warehouse_type::parse_warehouse_type( + spelling, dialect, + ) + .map(|tree| tree.as_ref().map(serde_json::Value::to_string)) + }) + .map_err(value_error) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { crate::bindings::_helpers::analysis::binding_catalog::register(module)?; crate::bindings::_helpers::analysis::required_columns::register(module)?; crate::bindings::_helpers::references::explicit_references::register(module)?; crate::bindings::_helpers::analysis::binding_locations::register(module)?; module.add_function(wrap_pyfunction!(query_fingerprint, module)?)?; + module.add_function(wrap_pyfunction!(parse_warehouse_type, module)?)?; Ok(()) } diff --git a/fensu.toml b/fensu.toml index bee7f18a71..bf5365df88 100644 --- a/fensu.toml +++ b/fensu.toml @@ -2932,3 +2932,7 @@ reason = "The override keeps queue.Queue.put's positional protocol, which every rule = "FFS201" path = "src/sqlbuild/orchestration_declarations/models/sensor_context.py" reason = "Only SensorContext lives here: invocation-local ctx.cursor assignment proposes progress, committed solely by native fenced completion. The package regression guards this declaration-only scope; class-symbol exceptions are unsupported." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-analysis/src/warehouse_types/main/parse_warehouse_type.rs", "crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dict_walk.rs", "crates/sqlbuild-analysis/src/relation_names/_helpers/walk.rs"] +reason = "Each is a short stack walk over a JSON syntax tree with its own stop and collect rule: cast_targets collects every cast's target type, branch_selects stops at each set-operation branch, and walked lists every object. A shared visitor would need a callback for each rule and save no logic." diff --git a/python-boundary.toml b/python-boundary.toml index 7b959ef22a..0aa3f9f4ee 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -567,11 +567,6 @@ path = "compiler/auditing/main/get_audit_factory_definition.py" rule = 3 note = "Reads the declaration attached to a project audit factory." -[[stays]] -path = "compiler/sql_analysis/main/import_polyglot.py" -rule = 4 -note = "Imports the Polyglot Python package for the Python API." - [[stays]] path = "compiler/sql_analysis/main/import_polyglot_sql.py" rule = 4 diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 2ecb23ece5..d9dfe2d850 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3458,6 +3458,7 @@ class PreparedCompileArtifacts: def format_sql_json(request_json: str) -> str: ... def query_fingerprint(sql: str, dialect: str) -> str: ... +def parse_warehouse_type(spelling: str, dialect: str) -> str | None: ... def lint_sql_json(request_json: str) -> str: ... def prepare_lint_sql( request: LintPreparationRequest, diff --git a/src/sqlbuild/compiler/sql_analysis/main/import_polyglot.py b/src/sqlbuild/compiler/sql_analysis/main/import_polyglot.py deleted file mode 100644 index d38e3925d8..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/import_polyglot.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public required Polyglot import entry.""" - -from typing import Any - -from sqlbuild.compiler.sql_analysis._helpers.polyglot import ( - import_polyglot as _import_polyglot, -) - - -def import_polyglot() -> Any: - """Return SQLBuild's required Polyglot SQL module.""" - - return _import_polyglot() diff --git a/src/sqlbuild/executor/scenario/_helpers/capture/columns.py b/src/sqlbuild/executor/scenario/_helpers/capture/columns.py index 6614e7cf63..add2b8fe3f 100644 --- a/src/sqlbuild/executor/scenario/_helpers/capture/columns.py +++ b/src/sqlbuild/executor/scenario/_helpers/capture/columns.py @@ -2,50 +2,63 @@ from __future__ import annotations +import json import logging +import re from dataclasses import dataclass from typing import Any +from sqlbuild._native import parse_warehouse_type from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ColumnInfo from sqlbuild.adapter.contract.types import TypeDialect -from sqlbuild.compiler.sql_analysis.main.import_polyglot import import_polyglot from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.scenario.constants import ( BIGQUERY_BIGNUMERIC_TYPE, - BIGQUERY_BYTES_TYPE, BIGQUERY_TIMESTAMP_TYPE, BIGQUERY_VARCHAR_TYPES, BIGQUERY_WIDEN_TO_DOUBLE_TYPES, + DATABRICKS_ARRAY_TYPE, + DATABRICKS_INTERVAL_PREFIX, + DATABRICKS_MAP_ARGUMENT_COUNT, + DATABRICKS_MAP_TYPE, + DATABRICKS_NESTED_TYPES, + DATABRICKS_STRUCT_FIELD_PATTERN, + DATABRICKS_STRUCT_TYPE, DATABRICKS_TIMESTAMP_NTZ_TYPE, DATABRICKS_TIMESTAMP_TYPE, + DATABRICKS_VARIANT_TYPE, DATABRICKS_VOID_TYPE, DUCKDB_BLOB_TYPE, DUCKDB_HUGEINT_TYPE, DUCKDB_INTEGER_TYPE, DUCKDB_LIST_TYPE, + DUCKDB_SPATIAL_TYPES, DUCKDB_UHUGEINT_TYPE, FALLBACK_BINARY_TYPES, FALLBACK_BOOLEAN_TYPES, FALLBACK_DATE_TYPE, FALLBACK_DECIMAL_TYPES, + FALLBACK_DOUBLE_TYPES, FALLBACK_FLOAT_TYPES, FALLBACK_JSON_TYPES, - FALLBACK_POLYGLOT_BASE_TYPES, - FALLBACK_SIGNED_INTEGER_TYPES, + FALLBACK_SIGNED_INTEGER_LOCAL_TYPES, FALLBACK_STRUCTURED_TYPES, FALLBACK_TIME_TYPES, FALLBACK_TIMESTAMP_TOKEN, FALLBACK_TIMESTAMP_TYPES, FALLBACK_UNSIGNED_INTEGER_TYPES, FALLBACK_VARCHAR_TYPES, + GENERIC_LOCAL_BASE_TYPES, GENERIC_WIDEN_TO_DOUBLE_TYPES, LOCAL_TYPE_BIGINT, LOCAL_TYPE_BOOLEAN, LOCAL_TYPE_DATE, LOCAL_TYPE_DECIMAL, + LOCAL_TYPE_DIALECTS, LOCAL_TYPE_DOUBLE, + LOCAL_TYPE_FLOAT, LOCAL_TYPE_INT, LOCAL_TYPE_INT128, LOCAL_TYPE_JSON, @@ -55,16 +68,10 @@ LOCAL_TYPE_TIME, LOCAL_TYPE_TIMESTAMP, LOCAL_TYPE_TIMESTAMPTZ, + LOCAL_TYPE_TIMETZ, LOCAL_TYPE_UINT128, LOCAL_TYPE_VARCHAR, - MALFORMED_RENDERED_TYPE_TOKEN, - POLYGLOT_ARRAY_TYPE, - POLYGLOT_COLLECTION_TYPES, - POLYGLOT_CUSTOM_TYPE, - POLYGLOT_DECIMAL_TYPES, - POLYGLOT_OBJECT_TYPE, - POLYGLOT_SPATIAL_AND_MONEY_TYPES, - POLYGLOT_VARCHAR_TYPES, + LOCAL_TYPE_VARIANT, POSTGRES_BIGSERIAL_TYPE, POSTGRES_PARAMETERIZED_DECIMAL_TYPES, POSTGRES_PARAMETERIZED_TEXT_TYPES, @@ -79,6 +86,8 @@ SNOWFLAKE_TIMEZONE_TIMESTAMP_TYPES, SNOWFLAKE_VECTOR_TYPE, SNOWFLAKE_WIDEN_TO_DOUBLE_TYPES, + STRUCT_FIELD_NAME_PATTERN, + TSQL_LOCAL_TYPES, TYPE_ARGUMENT_CLOSE_ANGLE, TYPE_ARGUMENT_CLOSE_PAREN, TYPE_ARGUMENT_CLOSE_TOKENS, @@ -86,7 +95,16 @@ TYPE_ARGUMENT_OPEN_TOKENS, TYPE_ARGUMENT_SEPARATOR, TYPE_PATTERN_WILDCARD, - UNTYPED_ARRAY_TYPE, + TYPE_TREE_COLLECTIONS, + TYPE_TREE_DECIMAL, + TYPE_TREE_FLOAT, + TYPE_TREE_KIND, + TYPE_TREE_LOCAL_TYPES, + TYPE_TREE_MAP, + TYPE_TREE_STRUCT, + TYPE_TREE_TEXT_KINDS, + TYPE_TREE_TIME, + TYPE_TREE_TIMESTAMP, ) from sqlbuild.executor.scenario.models import ScenarioSnapshotColumn @@ -194,12 +212,12 @@ def local_type_for_warehouse_type( if dialect_type is not None: return dialect_type - polyglot_local_type: str | None = _local_type_with_polyglot( + parsed_local_type: str | None = _local_type_with_sqlquack( warehouse_type=warehouse_type, sql_analysis_dialect=sql_analysis_dialect, ) - if polyglot_local_type is not None: - return polyglot_local_type + if parsed_local_type is not None: + return parsed_local_type return _fallback_local_type_for_warehouse_type(warehouse_type) @@ -277,10 +295,12 @@ def _dialect_pre_local_type(*, warehouse_type: str, sql_analysis_dialect: str | if dialect == TypeDialect.BIGQUERY: return _bigquery_pre_local_type(base=base, args=args) if dialect == TypeDialect.DATABRICKS: - return _databricks_pre_local_type(base=base, args=args) - if dialect == TypeDialect.DUCKDB: + return _databricks_local_type(warehouse_type) + if dialect in LOCAL_TYPE_DIALECTS: return _duckdb_pre_local_type(base=base, args=args) - if dialect is None and base not in FALLBACK_POLYGLOT_BASE_TYPES: + if dialect == TypeDialect.TSQL: + return TSQL_LOCAL_TYPES.get(base) + if dialect is None and base not in GENERIC_LOCAL_BASE_TYPES: return LOCAL_TYPE_VARCHAR return None @@ -317,22 +337,72 @@ def _bigquery_pre_local_type(*, base: str, args: tuple[str, ...]) -> str | None: return None -def _databricks_pre_local_type(*, base: str, args: tuple[str, ...]) -> str | None: +def _databricks_local_type(type_text: str) -> str | None: + """Map a Databricks type, nesting included; None leaves a scalar to the fallback.""" + + pattern: _TypePattern = _split_type_text(type_text.strip()) + base: str = pattern.base.upper() + if base in DATABRICKS_NESTED_TYPES: + return _databricks_nested_local_type(type_text=type_text.strip(), pattern=pattern) if base == DATABRICKS_TIMESTAMP_TYPE: return LOCAL_TYPE_TIMESTAMPTZ if base == DATABRICKS_TIMESTAMP_NTZ_TYPE: return LOCAL_TYPE_TIMESTAMP if base == DATABRICKS_VOID_TYPE: return LOCAL_TYPE_VARCHAR + if base.startswith(DATABRICKS_INTERVAL_PREFIX): + return " ".join(base.split()) + if base == DATABRICKS_VARIANT_TYPE: + return LOCAL_TYPE_VARIANT return None +def _databricks_nested_local_type(*, type_text: str, pattern: _TypePattern) -> str: + base: str = pattern.base.upper() + args: tuple[str, ...] = pattern.args + if base == DATABRICKS_ARRAY_TYPE and len(args) == 1: + return _databricks_element_local_type(element=args[0], type_text=type_text) + "[]" + if base == DATABRICKS_MAP_TYPE and len(args) == DATABRICKS_MAP_ARGUMENT_COUNT: + key: str = _databricks_element_local_type(element=args[0], type_text=type_text) + value: str = _databricks_element_local_type(element=args[1], type_text=type_text) + return f"MAP({key}, {value})" + if base == DATABRICKS_STRUCT_TYPE and args: + fields: list[str] = [] + for field in args: + match: re.Match[str] | None = DATABRICKS_STRUCT_FIELD_PATTERN.fullmatch(field) + if match is None: + raise _unmapped_databricks_type(element=field, type_text=type_text) + name: str = match.group("name").strip("`") + field_type: str = _databricks_element_local_type( + element=match.group("type"), type_text=type_text + ) + fields.append(f"{_struct_field_name(name)} {field_type}") + return f"STRUCT({', '.join(fields)})" + raise _unmapped_databricks_type(element=type_text, type_text=type_text) + + +def _databricks_element_local_type(*, element: str, type_text: str) -> str: + local_type: str | None = _databricks_local_type(element) or _known_local_type(element) + if local_type is None: + raise _unmapped_databricks_type(element=element, type_text=type_text) + return local_type + + +def _unmapped_databricks_type(*, element: str, type_text: str) -> ExecutorInputError: + return ExecutorInputError( + f"Databricks type '{type_text}' has no local DuckDB type: SQLBuild cannot map " + f"'{element.strip()}' inside it", + code=SCENARIO_LOCAL_TYPE_INVALID, + help="Add a scenario local type override for this warehouse type.", + ) + + def _duckdb_pre_local_type(*, base: str, args: tuple[str, ...]) -> str | None: if base == DUCKDB_HUGEINT_TYPE: return LOCAL_TYPE_INT128 if base == DUCKDB_UHUGEINT_TYPE: return LOCAL_TYPE_UINT128 - if base == DUCKDB_BLOB_TYPE: + if base == DUCKDB_BLOB_TYPE or base in DUCKDB_SPATIAL_TYPES: return LOCAL_TYPE_VARCHAR if base == DUCKDB_LIST_TYPE and args: inner_type: str = _fallback_local_type_for_warehouse_type(args[0]) @@ -342,68 +412,37 @@ def _duckdb_pre_local_type(*, base: str, args: tuple[str, ...]) -> str | None: return None -def _local_type_with_polyglot( +def _local_type_with_sqlquack( *, warehouse_type: str, sql_analysis_dialect: str | None ) -> str | None: - polyglot_module: Any = import_polyglot() + """Map a type through SQLQuack's parser for its dialect; DuckDB spellings are already local.""" + dialect: TypeDialect | None = _coerce_type_dialect(sql_analysis_dialect) + if dialect in LOCAL_TYPE_DIALECTS: + return warehouse_type.strip() try: - data_type: Any = polyglot_module.parse_data_type( - warehouse_type, - dialect=sql_analysis_dialect or "generic", - ) - except polyglot_module.PolyglotError as error: + tree_json: str | None = parse_warehouse_type(warehouse_type, sql_analysis_dialect or "") + except ValueError as error: log_debug_event( logger=_DEBUG_LOGGER, - message="scenario snapshot type polyglot conversion failed; falling back", + message="scenario snapshot type parse failed; falling back", warehouse_type=warehouse_type, sql_analysis_dialect=sql_analysis_dialect, sqlbuild_error=str(error), ) return None - - type_name: str = _polyglot_type_name(data_type) + if tree_json is None: + return None base_type: str = _parse_type_pattern(warehouse_type).base - if type_name in POLYGLOT_DECIMAL_TYPES: - rendered_bigdecimal_type: str = data_type.sql(dialect="duckdb").strip() - if MALFORMED_RENDERED_TYPE_TOKEN not in rendered_bigdecimal_type: - return rendered_bigdecimal_type - return _decimal_local_type_from_polyglot(data_type) or LOCAL_TYPE_VARCHAR - if type_name in POLYGLOT_VARCHAR_TYPES: - return LOCAL_TYPE_VARCHAR - if type_name == POLYGLOT_OBJECT_TYPE: - return LOCAL_TYPE_JSON - if type_name == POLYGLOT_ARRAY_TYPE and not _polyglot_element_type(data_type): - return LOCAL_TYPE_JSON - if type_name in POLYGLOT_COLLECTION_TYPES and _polyglot_element_type(data_type): - return data_type.sql(dialect="duckdb").strip() - if type_name in POLYGLOT_SPATIAL_AND_MONEY_TYPES: - return LOCAL_TYPE_VARCHAR - dialect: TypeDialect | None = _coerce_type_dialect(sql_analysis_dialect) if dialect == TypeDialect.POSTGRES and base_type in POSTGRES_VARCHAR_TYPES: return LOCAL_TYPE_VARCHAR if dialect == TypeDialect.POSTGRES and base_type in POSTGRES_SERIAL_TYPES: return _postgres_serial_local_type(base_type) - if ( - dialect == TypeDialect.BIGQUERY - and base_type == BIGQUERY_BYTES_TYPE - and data_type.expressions - ): - return LOCAL_TYPE_VARCHAR if dialect == TypeDialect.SNOWFLAKE and base_type == SNOWFLAKE_VECTOR_TYPE: return LOCAL_TYPE_JSON - - local_type: str = data_type.sql(dialect="duckdb").strip() - if ( - not local_type - or local_type == UNTYPED_ARRAY_TYPE - or MALFORMED_RENDERED_TYPE_TOKEN in local_type - ): + local_type: str | None = _local_type_from_tree(json.loads(tree_json)) + if local_type is None: return _fallback_local_type_for_warehouse_type(warehouse_type) - if local_type.startswith("BLOB"): - return "VARCHAR" - if local_type.startswith("TIMESTAMPTZ("): - return "TIMESTAMPTZ" if local_type == LOCAL_TYPE_REAL and _should_widen_real_to_double( warehouse_type=warehouse_type, sql_analysis_dialect=sql_analysis_dialect, @@ -412,18 +451,70 @@ def _local_type_with_polyglot( return local_type +def _local_type_from_tree(tree: dict[str, Any]) -> str | None: + """The DuckDB type of a SQLQuack type tree, or None when it has no fixed local type.""" + + kind: str = str(tree.get(TYPE_TREE_KIND, "")) + if kind in TYPE_TREE_LOCAL_TYPES: + return TYPE_TREE_LOCAL_TYPES[kind] + if kind in TYPE_TREE_TEXT_KINDS: + length: Any = tree.get("length") + return LOCAL_TYPE_TEXT if length is None else f"{LOCAL_TYPE_TEXT}({length})" + if kind == TYPE_TREE_FLOAT: + return LOCAL_TYPE_REAL if tree.get("real_spelling") else LOCAL_TYPE_FLOAT + if kind == TYPE_TREE_DECIMAL: + arguments: list[str] = [ + str(value) for value in (tree.get("precision"), tree.get("scale")) if value is not None + ] + return LOCAL_TYPE_DECIMAL + (f"({', '.join(arguments)})" if arguments else "") + if kind == TYPE_TREE_TIME: + return LOCAL_TYPE_TIMETZ if tree.get("timezone") else LOCAL_TYPE_TIME + if kind == TYPE_TREE_TIMESTAMP: + return LOCAL_TYPE_TIMESTAMPTZ if tree.get("timezone") else LOCAL_TYPE_TIMESTAMP + if kind in TYPE_TREE_COLLECTIONS: + element: str | None = _local_type_from_tree(tree.get("element_type") or {}) + return None if element is None else f"{element}[]" + if kind == TYPE_TREE_STRUCT: + fields: list[str] = [] + for field in tree.get("fields") or []: + field_type: str | None = _local_type_from_tree(field.get("data_type") or {}) + if field_type is None: + return None + fields.append(f"{_struct_field_name(str(field.get('name', '')))} {field_type}") + return f"STRUCT({', '.join(fields)})" if fields else None + if kind == TYPE_TREE_MAP: + key: str | None = _local_type_from_tree(tree.get("key_type") or {}) + value: str | None = _local_type_from_tree(tree.get("value_type") or {}) + return None if key is None or value is None else f"MAP({key}, {value})" + return None + + +def _struct_field_name(name: str) -> str: + if STRUCT_FIELD_NAME_PATTERN.fullmatch(name): + return name + return '"' + name.replace('"', '""') + '"' + + def _fallback_local_type_for_warehouse_type(warehouse_type: str) -> str: """Return a conservative local type when SQL analysis cannot classify a type.""" + return _known_local_type(warehouse_type) or LOCAL_TYPE_VARCHAR + + +def _known_local_type(warehouse_type: str) -> str | None: + """The local type of a type name SQLBuild knows by its base name alone, else None.""" + normalized_type: str = warehouse_type.strip().upper() - base_type: str = normalized_type.split("(", 1)[0].strip() + base_type: str = _parse_type_pattern(normalized_type).base if base_type in FALLBACK_BOOLEAN_TYPES: return LOCAL_TYPE_BOOLEAN - if base_type in FALLBACK_SIGNED_INTEGER_TYPES: - return LOCAL_TYPE_BIGINT + if base_type in FALLBACK_SIGNED_INTEGER_LOCAL_TYPES: + return FALLBACK_SIGNED_INTEGER_LOCAL_TYPES[base_type] if base_type in FALLBACK_UNSIGNED_INTEGER_TYPES: return LOCAL_TYPE_BIGINT if base_type in FALLBACK_FLOAT_TYPES: + return LOCAL_TYPE_FLOAT + if base_type in FALLBACK_DOUBLE_TYPES: return LOCAL_TYPE_DOUBLE if base_type in FALLBACK_DECIMAL_TYPES: return _decimal_local_type(normalized_type) @@ -441,7 +532,7 @@ def _fallback_local_type_for_warehouse_type(warehouse_type: str) -> str: return LOCAL_TYPE_VARCHAR if base_type in FALLBACK_STRUCTURED_TYPES: return LOCAL_TYPE_JSON - return LOCAL_TYPE_VARCHAR + return None def _postgres_serial_local_type(base_type: str) -> str: @@ -453,7 +544,10 @@ def _postgres_serial_local_type(base_type: str) -> str: def _parse_type_pattern(type_text: str) -> _TypePattern: - normalized_type: str = type_text.strip().upper() + return _split_type_text(type_text.strip().upper()) + + +def _split_type_text(normalized_type: str) -> _TypePattern: if not normalized_type: return _TypePattern(base="") open_index: int = _first_type_arg_open_index(normalized_type) @@ -538,37 +632,6 @@ def _decimal_local_type(normalized_type: str) -> str: return f"DECIMAL({precision_scale})" -def _decimal_local_type_from_polyglot(data_type: Any) -> str | None: - precision_scale: str | None = _polyglot_precision_scale(data_type) - if precision_scale is None: - return LOCAL_TYPE_DECIMAL - return f"DECIMAL({precision_scale})" - - -def _polyglot_precision_scale(data_type: Any) -> str | None: - args: dict[str, Any] = dict(getattr(data_type, "args", {}) or {}) - values: list[str] = [ - str(value) for value in (args.get("precision"), args.get("scale")) if value is not None - ] - if not values: - return None - return ", ".join(values) - - -def _polyglot_type_name(data_type: Any) -> str: - args: dict[str, Any] = dict(getattr(data_type, "args", {}) or {}) - data_type_name: str = str(args.get("data_type", "")).upper() - if data_type_name == POLYGLOT_CUSTOM_TYPE: - return str(args.get("name", "")).upper().replace(" ", "") - return data_type_name - - -def _polyglot_element_type(data_type: Any) -> dict[str, Any] | None: - args: dict[str, Any] = dict(getattr(data_type, "args", {}) or {}) - element_type: Any = args.get("element_type") - return element_type if isinstance(element_type, dict) else None - - def _should_widen_real_to_double(*, warehouse_type: str, sql_analysis_dialect: str | None) -> bool: normalized_type: str = warehouse_type.strip().upper() base_type: str = normalized_type.split("(", 1)[0].strip() diff --git a/src/sqlbuild/executor/scenario/constants.py b/src/sqlbuild/executor/scenario/constants.py index 1f87454379..416287896c 100644 --- a/src/sqlbuild/executor/scenario/constants.py +++ b/src/sqlbuild/executor/scenario/constants.py @@ -1,5 +1,9 @@ """Scenario execution and local replay constants.""" +import re + +from sqlbuild.adapter.contract.types import TypeDialect + SCENARIO_EXEC_FIXTURE_FAILED: str = "X501" SCENARIO_EXEC_SEED_FAILED: str = "X502" SCENARIO_EXEC_MODEL_FAILED: str = "X503" @@ -35,6 +39,9 @@ LOCAL_TYPE_REAL: str = "REAL" LOCAL_TYPE_SMALLINT: str = "SMALLINT" LOCAL_TYPE_TEXT: str = "TEXT" +LOCAL_TYPE_FLOAT: str = "FLOAT" +LOCAL_TYPE_TIMETZ: str = "TIMETZ" +LOCAL_TYPE_VARIANT: str = "VARIANT" LOCAL_TYPE_TIME: str = "TIME" LOCAL_TYPE_TIMESTAMP: str = "TIMESTAMP" LOCAL_TYPE_TIMESTAMPTZ: str = "TIMESTAMPTZ" @@ -82,19 +89,29 @@ SNOWFLAKE_VECTOR_TYPE: str = "VECTOR" BIGQUERY_BIGNUMERIC_TYPE: str = "BIGNUMERIC" BIGQUERY_VARCHAR_TYPES: frozenset[str] = frozenset({"RANGE", "BYTES"}) -BIGQUERY_BYTES_TYPE: str = "BYTES" BIGQUERY_TIMESTAMP_TYPE: str = "TIMESTAMP" BIGQUERY_WIDEN_TO_DOUBLE_TYPES: frozenset[str] = frozenset({"FLOAT64"}) DATABRICKS_TIMESTAMP_TYPE: str = "TIMESTAMP" DATABRICKS_TIMESTAMP_NTZ_TYPE: str = "TIMESTAMP_NTZ" DATABRICKS_VOID_TYPE: str = "VOID" +DATABRICKS_INTERVAL_PREFIX: str = "INTERVAL" +DATABRICKS_VARIANT_TYPE: str = "VARIANT" +DATABRICKS_ARRAY_TYPE: str = "ARRAY" +DATABRICKS_MAP_TYPE: str = "MAP" +DATABRICKS_STRUCT_TYPE: str = "STRUCT" +DATABRICKS_NESTED_TYPES: frozenset[str] = frozenset({"ARRAY", "MAP", "STRUCT"}) +DATABRICKS_MAP_ARGUMENT_COUNT: int = 2 +DATABRICKS_STRUCT_FIELD_PATTERN: re.Pattern[str] = re.compile( + r"\s*(?P`[^`]+`|[A-Za-z_][A-Za-z0-9_]*)\s*(?::\s*|\s+)(?P\S.*?)\s*" +) DUCKDB_HUGEINT_TYPE: str = "HUGEINT" DUCKDB_UHUGEINT_TYPE: str = "UHUGEINT" DUCKDB_BLOB_TYPE: str = "BLOB" +DUCKDB_SPATIAL_TYPES: frozenset[str] = frozenset({"GEOMETRY", "GEOGRAPHY"}) DUCKDB_LIST_TYPE: str = "LIST" DUCKDB_INTEGER_TYPE: str = "INTEGER" -FALLBACK_POLYGLOT_BASE_TYPES: frozenset[str] = frozenset( +GENERIC_LOCAL_BASE_TYPES: frozenset[str] = frozenset( { "BOOL", "BOOLEAN", @@ -118,24 +135,72 @@ "JSON", } ) -POLYGLOT_DECIMAL_TYPES: frozenset[str] = frozenset({"BIGDECIMAL", "DECIMAL"}) -POLYGLOT_VARCHAR_TYPES: frozenset[str] = frozenset({"NULL", "RANGE", "XML"}) -POLYGLOT_OBJECT_TYPE: str = "OBJECT" -POLYGLOT_ARRAY_TYPE: str = "ARRAY" -POLYGLOT_COLLECTION_TYPES: frozenset[str] = frozenset({"ARRAY", "LIST"}) -POLYGLOT_SPATIAL_AND_MONEY_TYPES: frozenset[str] = frozenset({"GEOGRAPHY", "GEOMETRY", "MONEY"}) -POLYGLOT_CUSTOM_TYPE: str = "CUSTOM" +TYPE_TREE_KIND: str = "data_type" +TYPE_TREE_LOCAL_TYPES: dict[str, str] = { + "boolean": "BOOLEAN", + "tiny_int": "TINYINT", + "small_int": "SMALLINT", + "int": "INT", + "big_int": "BIGINT", + "int128": "HUGEINT", + "uint8": "UTINYINT", + "uint16": "USMALLINT", + "uint32": "UINTEGER", + "uint64": "UBIGINT", + "uint128": "UHUGEINT", + "double": "DOUBLE", + "date": "DATE", + "interval": "INTERVAL", + "json": "JSON", + "json_b": "JSON", + "uuid": "UUID", + "binary": "VARCHAR", + "var_binary": "VARCHAR", + "blob": "VARCHAR", + "geography": "VARCHAR", + "geometry": "VARCHAR", + "object": "JSON", +} +TYPE_TREE_TEXT_KINDS: frozenset[str] = frozenset({"char", "var_char", "string", "text"}) +TYPE_TREE_FLOAT: str = "float" +TYPE_TREE_DECIMAL: str = "decimal" +TYPE_TREE_TIME: str = "time" +TYPE_TREE_TIMESTAMP: str = "timestamp" +TYPE_TREE_COLLECTIONS: frozenset[str] = frozenset({"array", "list"}) +TYPE_TREE_STRUCT: str = "struct" +TYPE_TREE_MAP: str = "map" +STRUCT_FIELD_NAME_PATTERN: re.Pattern[str] = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +LOCAL_TYPE_DIALECTS: frozenset[TypeDialect] = frozenset( + {TypeDialect.DUCKDB, TypeDialect.MOTHERDUCK} +) FALLBACK_BOOLEAN_TYPES: frozenset[str] = frozenset({"BOOL", "BOOLEAN"}) -FALLBACK_SIGNED_INTEGER_TYPES: frozenset[str] = frozenset( - {"TINYINT", "SMALLINT", "INT", "INTEGER", "BIGINT", "HUGEINT"} -) +FALLBACK_SIGNED_INTEGER_LOCAL_TYPES: dict[str, str] = { + "TINYINT": "TINYINT", + "SMALLINT": "SMALLINT", + "INT": "INT", + "INTEGER": "INT", + "BIGINT": "BIGINT", + "HUGEINT": "HUGEINT", +} FALLBACK_UNSIGNED_INTEGER_TYPES: frozenset[str] = frozenset( {"UTINYINT", "USMALLINT", "UINTEGER", "UBIGINT", "UHUGEINT"} ) -FALLBACK_FLOAT_TYPES: frozenset[str] = frozenset( - {"FLOAT", "FLOAT4", "REAL", "DOUBLE", "DOUBLE PRECISION", "FLOAT8"} +FALLBACK_FLOAT_TYPES: frozenset[str] = frozenset({"FLOAT", "FLOAT4", "REAL"}) +FALLBACK_DOUBLE_TYPES: frozenset[str] = frozenset( + {"DOUBLE", "DOUBLE PRECISION", "FLOAT8", "FLOAT64"} ) +TSQL_LOCAL_TYPES: dict[str, str] = { + "BIT": "BOOLEAN", + "DATETIME2": "TIMESTAMP", + "SMALLDATETIME": "TIMESTAMP", + "DATETIMEOFFSET": "TIMESTAMPTZ", + "UNIQUEIDENTIFIER": "UUID", + "FLOAT": "DOUBLE", + "REAL": "FLOAT", + "MONEY": "DECIMAL(19,4)", + "SMALLMONEY": "DECIMAL(10,4)", +} FALLBACK_DECIMAL_TYPES: frozenset[str] = frozenset({"DECIMAL", "DEC", "NUMERIC", "NUMBER"}) FALLBACK_DATE_TYPE: str = "DATE" FALLBACK_TIME_TYPES: frozenset[str] = frozenset({"TIME", "TIME WITH TIME ZONE", "TIMETZ"}) @@ -147,7 +212,7 @@ FALLBACK_JSON_TYPES: frozenset[str] = frozenset({"JSON", "JSONB"}) FALLBACK_BINARY_TYPES: frozenset[str] = frozenset({"BLOB", "BYTEA", "BINARY", "VARBINARY"}) FALLBACK_STRUCTURED_TYPES: frozenset[str] = frozenset( - {"ARRAY", "OBJECT", "VARIANT", "JSON", "JSONB"} + {"ARRAY", "OBJECT", "VARIANT", "JSON", "JSONB", "MAP", "STRUCT"} ) SNOWFLAKE_WIDEN_TO_DOUBLE_TYPES: frozenset[str] = SNOWFLAKE_DOUBLE_TYPES GENERIC_WIDEN_TO_DOUBLE_TYPES: frozenset[str] = frozenset({"DOUBLE", "DOUBLE PRECISION", "FLOAT8"}) @@ -159,8 +224,6 @@ TYPE_ARGUMENT_CLOSE_TOKENS: frozenset[str] = frozenset({")", ">"}) TYPE_ARGUMENT_SEPARATOR: str = "," TYPE_PATTERN_WILDCARD: str = "*" -MALFORMED_RENDERED_TYPE_TOKEN: str = ")(" -UNTYPED_ARRAY_TYPE: str = "[]" JSONL_BLANK_LINE: str = "" LOCAL_SCENARIO_DIALECT: str = "duckdb" diff --git a/tests/integration/src/sqlbuild/executor/scenario/test_capture_types.py b/tests/integration/src/sqlbuild/executor/scenario/test_capture_types.py index fd7dd93469..f60ab5ff05 100644 --- a/tests/integration/src/sqlbuild/executor/scenario/test_capture_types.py +++ b/tests/integration/src/sqlbuild/executor/scenario/test_capture_types.py @@ -32,7 +32,7 @@ ScenarioSnapshotCaptureTypesIntegrationTestCase( description="typed duckdb capture jsonl reloads with manifest schema", expected_local_types={ - "decimal_scale_col": "DECIMAL(12, 2)", + "decimal_scale_col": "DECIMAL(12,2)", "date_col": "DATE", "timestamp_col": "TIMESTAMP", "json_col": "JSON", diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py index bd4f84044e..fb5658e8cb 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/_test_types.py @@ -145,6 +145,14 @@ class ScenarioSnapshotLocalTypeMappingTestCase: local_type_overrides: dict[str, str] | None = None +@dataclass(frozen=True) +class ScenarioSnapshotLocalTypeRefusalTestCase: + description: str + sql_analysis_dialect: str | None + warehouse_type: str + expected_message: str + + @dataclass(frozen=True) class ExecuteScenarioSnapshotCaptureStepsTestCase: description: str diff --git a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_capture_columns.py b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_capture_columns.py index 7f363d0be5..f39bcb92b7 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_capture_columns.py +++ b/tests/unit/src/sqlbuild/executor/scenario/_helpers/test_capture_columns.py @@ -3,9 +3,11 @@ import duckdb import pytest +from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.scenario._helpers.capture.columns import local_type_for_warehouse_type from tests.unit.src.sqlbuild.executor.scenario._helpers._test_types import ( ScenarioSnapshotLocalTypeMappingTestCase, + ScenarioSnapshotLocalTypeRefusalTestCase, ) @@ -157,10 +159,10 @@ expected_local_type="JSON", ), ScenarioSnapshotLocalTypeMappingTestCase( - description="snowflake variant remains variant", + description="snowflake variant is held as json as sqlquack converts it", sql_analysis_dialect="snowflake", warehouse_type="VARIANT", - expected_local_type="VARIANT", + expected_local_type="JSON", ), ScenarioSnapshotLocalTypeMappingTestCase( description="snowflake geography falls back to varchar", @@ -187,10 +189,10 @@ expected_local_type="BIGINT", ), ScenarioSnapshotLocalTypeMappingTestCase( - description="bigquery integer maps to int", + description="bigquery integer is int64 and maps to bigint", sql_analysis_dialect="bigquery", warehouse_type="INTEGER", - expected_local_type="INT", + expected_local_type="BIGINT", ), ScenarioSnapshotLocalTypeMappingTestCase( description="bigquery boolean maps to boolean", @@ -502,7 +504,7 @@ description="databricks map maps to duckdb map type", sql_analysis_dialect="databricks", warehouse_type="MAP", - expected_local_type="MAP(TEXT, INT)", + expected_local_type="MAP(VARCHAR, INT)", ), ScenarioSnapshotLocalTypeMappingTestCase( description="databricks tinyint maps to tinyint", @@ -514,19 +516,19 @@ description="databricks decimal precision and scale are preserved", sql_analysis_dialect="databricks", warehouse_type="DECIMAL(10,2)", - expected_local_type="DECIMAL(10, 2)", + expected_local_type="DECIMAL(10,2)", ), ScenarioSnapshotLocalTypeMappingTestCase( - description="databricks string maps to text", + description="databricks string maps to varchar", sql_analysis_dialect="databricks", warehouse_type="STRING", - expected_local_type="TEXT", + expected_local_type="VARCHAR", ), ScenarioSnapshotLocalTypeMappingTestCase( - description="databricks varchar length maps to text length", + description="databricks varchar length maps to varchar", sql_analysis_dialect="databricks", warehouse_type="VARCHAR(20)", - expected_local_type="TEXT(20)", + expected_local_type="VARCHAR", ), ScenarioSnapshotLocalTypeMappingTestCase( description="databricks timestamp maps to timestamptz", @@ -568,7 +570,73 @@ description="databricks nested map of arrays maps to duckdb map", sql_analysis_dialect="databricks", warehouse_type="MAP>", - expected_local_type="MAP(TEXT, INT[])", + expected_local_type="MAP(VARCHAR, INT[])", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="databricks struct nests lists, maps and quoted field names", + sql_analysis_dialect="databricks", + warehouse_type="STRUCT>, `b c`: TINYINT>", + expected_local_type='STRUCT(a MAP(VARCHAR, BIGINT)[], "b c" TINYINT)', + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="sql server datetime2 is a timestamp", + sql_analysis_dialect="tsql", + warehouse_type="datetime2", + expected_local_type="TIMESTAMP", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="sql server smalldatetime is a timestamp", + sql_analysis_dialect="tsql", + warehouse_type="smalldatetime", + expected_local_type="TIMESTAMP", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="sql server datetimeoffset is a timestamp with time zone", + sql_analysis_dialect="tsql", + warehouse_type="datetimeoffset", + expected_local_type="TIMESTAMPTZ", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="sql server uniqueidentifier is a uuid", + sql_analysis_dialect="tsql", + warehouse_type="uniqueidentifier", + expected_local_type="UUID", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="sql server bit is a boolean", + sql_analysis_dialect="tsql", + warehouse_type="bit", + expected_local_type="BOOLEAN", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="sql server float is eight bytes", + sql_analysis_dialect="tsql", + warehouse_type="float", + expected_local_type="DOUBLE", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="sql server real is four bytes", + sql_analysis_dialect="tsql", + warehouse_type="real", + expected_local_type="FLOAT", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="sql server money is an exact four-place decimal", + sql_analysis_dialect="tsql", + warehouse_type="money", + expected_local_type="DECIMAL(19,4)", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="databricks float keeps four bytes, nested too", + sql_analysis_dialect="databricks", + warehouse_type="ARRAY", + expected_local_type="FLOAT[]", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="generic float keeps four bytes", + sql_analysis_dialect="generic", + warehouse_type="FLOAT", + expected_local_type="FLOAT", ), ScenarioSnapshotLocalTypeMappingTestCase( description="databricks interval year is preserved", @@ -619,16 +687,16 @@ expected_local_type="VARCHAR", ), ScenarioSnapshotLocalTypeMappingTestCase( - description="duckdb struct is preserved with normalized inner type", + description="duckdb struct passes through unchanged", sql_analysis_dialect="duckdb", warehouse_type="STRUCT(a INTEGER)", - expected_local_type="STRUCT(a INT)", + expected_local_type="STRUCT(a INTEGER)", ), ScenarioSnapshotLocalTypeMappingTestCase( - description="duckdb union is preserved with normalized inner types", + description="duckdb union passes through unchanged", sql_analysis_dialect="duckdb", warehouse_type="UNION(num INTEGER, str VARCHAR)", - expected_local_type="UNION(num INT, str TEXT)", + expected_local_type="UNION(num INTEGER, str VARCHAR)", ), ScenarioSnapshotLocalTypeMappingTestCase( description="duckdb list angle syntax maps to bracket list syntax", @@ -643,10 +711,10 @@ expected_local_type="ENUM('a', 'b')", ), ScenarioSnapshotLocalTypeMappingTestCase( - description="duckdb time with timezone maps to timetz", + description="duckdb time with time zone passes through unchanged", sql_analysis_dialect="duckdb", warehouse_type="TIME WITH TIME ZONE", - expected_local_type="TIMETZ", + expected_local_type="TIME WITH TIME ZONE", ), ScenarioSnapshotLocalTypeMappingTestCase( description="unknown type falls back to varchar", @@ -855,10 +923,10 @@ def test_given_warehouse_type_when_mapping_to_local_type_then_returns_expected_d expected_local_type="JSON", ), ScenarioSnapshotLocalTypeMappingTestCase( - description="snowflake variant remains variant", + description="snowflake variant is held as json as sqlquack converts it", sql_analysis_dialect="snowflake", warehouse_type="VARIANT", - expected_local_type="VARIANT", + expected_local_type="JSON", ), ScenarioSnapshotLocalTypeMappingTestCase( description="snowflake geography falls back to varchar", @@ -885,10 +953,10 @@ def test_given_warehouse_type_when_mapping_to_local_type_then_returns_expected_d expected_local_type="BIGINT", ), ScenarioSnapshotLocalTypeMappingTestCase( - description="bigquery integer maps to int", + description="bigquery integer is int64 and maps to bigint", sql_analysis_dialect="bigquery", warehouse_type="INTEGER", - expected_local_type="INT", + expected_local_type="BIGINT", ), ScenarioSnapshotLocalTypeMappingTestCase( description="bigquery boolean maps to boolean", @@ -1200,7 +1268,7 @@ def test_given_warehouse_type_when_mapping_to_local_type_then_returns_expected_d description="databricks map maps to duckdb map type", sql_analysis_dialect="databricks", warehouse_type="MAP", - expected_local_type="MAP(TEXT, INT)", + expected_local_type="MAP(VARCHAR, INT)", ), ScenarioSnapshotLocalTypeMappingTestCase( description="databricks tinyint maps to tinyint", @@ -1212,19 +1280,19 @@ def test_given_warehouse_type_when_mapping_to_local_type_then_returns_expected_d description="databricks decimal precision and scale are preserved", sql_analysis_dialect="databricks", warehouse_type="DECIMAL(10,2)", - expected_local_type="DECIMAL(10, 2)", + expected_local_type="DECIMAL(10,2)", ), ScenarioSnapshotLocalTypeMappingTestCase( - description="databricks string maps to text", + description="databricks string maps to varchar", sql_analysis_dialect="databricks", warehouse_type="STRING", - expected_local_type="TEXT", + expected_local_type="VARCHAR", ), ScenarioSnapshotLocalTypeMappingTestCase( - description="databricks varchar length maps to text length", + description="databricks varchar length maps to varchar", sql_analysis_dialect="databricks", warehouse_type="VARCHAR(20)", - expected_local_type="TEXT(20)", + expected_local_type="VARCHAR", ), ScenarioSnapshotLocalTypeMappingTestCase( description="databricks timestamp maps to timestamptz", @@ -1266,7 +1334,73 @@ def test_given_warehouse_type_when_mapping_to_local_type_then_returns_expected_d description="databricks nested map of arrays maps to duckdb map", sql_analysis_dialect="databricks", warehouse_type="MAP>", - expected_local_type="MAP(TEXT, INT[])", + expected_local_type="MAP(VARCHAR, INT[])", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="databricks struct nests lists, maps and quoted field names", + sql_analysis_dialect="databricks", + warehouse_type="STRUCT>, `b c`: TINYINT>", + expected_local_type='STRUCT(a MAP(VARCHAR, BIGINT)[], "b c" TINYINT)', + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="sql server datetime2 is a timestamp", + sql_analysis_dialect="tsql", + warehouse_type="datetime2", + expected_local_type="TIMESTAMP", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="sql server smalldatetime is a timestamp", + sql_analysis_dialect="tsql", + warehouse_type="smalldatetime", + expected_local_type="TIMESTAMP", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="sql server datetimeoffset is a timestamp with time zone", + sql_analysis_dialect="tsql", + warehouse_type="datetimeoffset", + expected_local_type="TIMESTAMPTZ", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="sql server uniqueidentifier is a uuid", + sql_analysis_dialect="tsql", + warehouse_type="uniqueidentifier", + expected_local_type="UUID", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="sql server bit is a boolean", + sql_analysis_dialect="tsql", + warehouse_type="bit", + expected_local_type="BOOLEAN", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="sql server float is eight bytes", + sql_analysis_dialect="tsql", + warehouse_type="float", + expected_local_type="DOUBLE", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="sql server real is four bytes", + sql_analysis_dialect="tsql", + warehouse_type="real", + expected_local_type="FLOAT", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="sql server money is an exact four-place decimal", + sql_analysis_dialect="tsql", + warehouse_type="money", + expected_local_type="DECIMAL(19,4)", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="databricks float keeps four bytes, nested too", + sql_analysis_dialect="databricks", + warehouse_type="ARRAY", + expected_local_type="FLOAT[]", + ), + ScenarioSnapshotLocalTypeMappingTestCase( + description="generic float keeps four bytes", + sql_analysis_dialect="generic", + warehouse_type="FLOAT", + expected_local_type="FLOAT", ), ScenarioSnapshotLocalTypeMappingTestCase( description="databricks interval year is preserved", @@ -1317,16 +1451,16 @@ def test_given_warehouse_type_when_mapping_to_local_type_then_returns_expected_d expected_local_type="VARCHAR", ), ScenarioSnapshotLocalTypeMappingTestCase( - description="duckdb struct is preserved with normalized inner type", + description="duckdb struct passes through unchanged", sql_analysis_dialect="duckdb", warehouse_type="STRUCT(a INTEGER)", - expected_local_type="STRUCT(a INT)", + expected_local_type="STRUCT(a INTEGER)", ), ScenarioSnapshotLocalTypeMappingTestCase( - description="duckdb union is preserved with normalized inner types", + description="duckdb union passes through unchanged", sql_analysis_dialect="duckdb", warehouse_type="UNION(num INTEGER, str VARCHAR)", - expected_local_type="UNION(num INT, str TEXT)", + expected_local_type="UNION(num INTEGER, str VARCHAR)", ), ScenarioSnapshotLocalTypeMappingTestCase( description="duckdb list angle syntax maps to bracket list syntax", @@ -1341,10 +1475,10 @@ def test_given_warehouse_type_when_mapping_to_local_type_then_returns_expected_d expected_local_type="ENUM('a', 'b')", ), ScenarioSnapshotLocalTypeMappingTestCase( - description="duckdb time with timezone maps to timetz", + description="duckdb time with time zone passes through unchanged", sql_analysis_dialect="duckdb", warehouse_type="TIME WITH TIME ZONE", - expected_local_type="TIMETZ", + expected_local_type="TIME WITH TIME ZONE", ), ScenarioSnapshotLocalTypeMappingTestCase( description="unknown type falls back to varchar", @@ -1405,3 +1539,33 @@ def test_given_mapped_local_type_when_creating_duckdb_table_then_type_is_accepte assert local_type == test_case.expected_local_type connection.execute(f'CREATE TABLE local_type_probe ("value" {local_type})') + + +@pytest.mark.parametrize( + "test_case", + [ + ScenarioSnapshotLocalTypeRefusalTestCase( + description="databricks array of an unmapped element type is refused", + sql_analysis_dialect="databricks", + warehouse_type="ARRAY", + expected_message="SQLBuild cannot map 'GEOGRAPHY' inside it", + ), + ScenarioSnapshotLocalTypeRefusalTestCase( + description="databricks map missing its value type is refused", + sql_analysis_dialect="databricks", + warehouse_type="MAP", + expected_message="SQLBuild cannot map 'MAP' inside it", + ), + ], + ids=lambda case: case.description, +) +def test_given_unmapped_nested_type_when_mapping_to_local_type_then_it_is_refused( + test_case: ScenarioSnapshotLocalTypeRefusalTestCase, +) -> None: + with pytest.raises(ExecutorInputError) as raised: + local_type_for_warehouse_type( + warehouse_type=test_case.warehouse_type, + sql_analysis_dialect=test_case.sql_analysis_dialect, + ) + + assert test_case.expected_message in raised.value.message diff --git a/tests/unit/src/sqlbuild/executor/scenario/main/test_capture.py b/tests/unit/src/sqlbuild/executor/scenario/main/test_capture.py index da2be93081..2e450b4c68 100644 --- a/tests/unit/src/sqlbuild/executor/scenario/main/test_capture.py +++ b/tests/unit/src/sqlbuild/executor/scenario/main/test_capture.py @@ -137,7 +137,7 @@ def render_qualified_name( ScenarioSnapshotColumn( name="customer_id", warehouse_type="DECIMAL(38,0)", - local_type="DECIMAL(38, 0)", + local_type="DECIMAL(38,0)", ), ), ), @@ -151,7 +151,7 @@ def render_qualified_name( ScenarioSnapshotColumn( name="country_code", warehouse_type="VARCHAR", - local_type="TEXT", + local_type="VARCHAR", ), ), ), @@ -170,7 +170,7 @@ def render_qualified_name( ScenarioSnapshotColumn( name="amount", warehouse_type="DECIMAL(10,2)", - local_type="DECIMAL(10, 2)", + local_type="DECIMAL(10,2)", ), ), ),