diff --git a/Cargo.lock b/Cargo.lock index 2b797f0b6d..f237f257a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -658,7 +658,7 @@ dependencies = [ [[package]] name = "polyglot-sql" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=ca9e7b6e405fb6f5484472bb08780268789c5e04#ca9e7b6e405fb6f5484472bb08780268789c5e04" +source = "git+https://github.com/chio-labs/polyglot.git?rev=b18dc647e797e480d2faadf3af407077f8e063df#b18dc647e797e480d2faadf3af407077f8e063df" dependencies = [ "polyglot-sql-ast-derive", "polyglot-sql-function-catalogs", @@ -673,7 +673,7 @@ dependencies = [ [[package]] name = "polyglot-sql-ast-derive" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=ca9e7b6e405fb6f5484472bb08780268789c5e04#ca9e7b6e405fb6f5484472bb08780268789c5e04" +source = "git+https://github.com/chio-labs/polyglot.git?rev=b18dc647e797e480d2faadf3af407077f8e063df#b18dc647e797e480d2faadf3af407077f8e063df" dependencies = [ "proc-macro2", "quote", @@ -683,7 +683,7 @@ dependencies = [ [[package]] name = "polyglot-sql-function-catalogs" version = "0.13.9" -source = "git+https://github.com/chio-labs/polyglot.git?rev=ca9e7b6e405fb6f5484472bb08780268789c5e04#ca9e7b6e405fb6f5484472bb08780268789c5e04" +source = "git+https://github.com/chio-labs/polyglot.git?rev=b18dc647e797e480d2faadf3af407077f8e063df#b18dc647e797e480d2faadf3af407077f8e063df" [[package]] name = "portable-atomic" diff --git a/Cargo.toml b/Cargo.toml index 78a797517c..e96db210db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,14 +64,14 @@ junction = { version = "2.1.0", default-features = false } mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] } proc-macro2 = { version = "1", features = ["span-locations"] } pyo3 = { version = "0.26", features = ["abi3-py312"] } -polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "ca9e7b6e405fb6f5484472bb08780268789c5e04", default-features = false, features = [ +polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "b18dc647e797e480d2faadf3af407077f8e063df", default-features = false, features = [ "ast-tools", "builder", "all-dialects", "semantic", "stacker", ] } -polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "ca9e7b6e405fb6f5484472bb08780268789c5e04", default-features = false, features = [ +polyglot-sql-function-catalogs = { git = "https://github.com/chio-labs/polyglot.git", rev = "b18dc647e797e480d2faadf3af407077f8e063df", default-features = false, features = [ "dialect-duckdb", ] } blake3 = { version = "1", features = ["rayon"] } diff --git a/Makefile b/Makefile index aadd9d9307..efba5d41b3 100644 --- a/Makefile +++ b/Makefile @@ -86,6 +86,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_expression_sources.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_lifecycle_commands.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_manifest_artifact_gating.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_materialization_type_change.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_config_build.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_plan_command_surface.py \ @@ -126,7 +127,8 @@ E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_model_backed_cursor_build.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_replay_on_change_full_build.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_seed_watermark_build.py \ - tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py + tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_validity_columns.py E2E_DUCKDB_CLI_DATA_PATHS := \ diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs index 45f4427c36..2d69ab47a5 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/enrichment.rs @@ -170,12 +170,22 @@ fn projected( return Ok(Projected::Declined); }; let shape: Option<&str> = analysis.get("shape").and_then(Value::as_str); - if !matches!(shape, Some(SELECT_SHAPE | SET_OPERATION_SHAPE)) || !eligible(projections) { + if !matches!(shape, Some(SELECT_SHAPE | SET_OPERATION_SHAPE)) + || !(has_star_projection(analysis) || eligible(projections)) + { return Ok(Projected::Declined); } native_projection(input, analysis, cleaned, projections).map(Projected::Native) } +/// Whether the analysis expanded a star; the legacy analysis cannot, so it never answers one. +fn has_star_projection(analysis: &Value) -> bool { + analysis + .get("starProjections") + .and_then(Value::as_array) + .is_some_and(|stars| !stars.is_empty()) +} + /// The compact facts as Python's columns and lineage. fn native_projection( input: &EnrichmentInput<'_>, @@ -248,10 +258,7 @@ fn native_projection( sources, }); } - let has_star: bool = analysis - .get("starProjections") - .and_then(Value::as_array) - .is_some_and(|stars| !stars.is_empty()); + let has_star: bool = has_star_projection(analysis); if has_star && input.model.references.len() == 1 { let order: HashMap<&str, usize> = declared_order(input); let fallback: usize = order.len(); diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs index 28908a0405..cef7394585 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs @@ -265,6 +265,17 @@ pub(crate) fn session_lines(models: &[ModelSpec]) -> (Vec>, Vec Vec> { + let mut request: SessionRequest = orders_request(model_requests(models)); + request.rich_type_inference = rich_type_inference; + for (name, ..) in models { + request.column_types.push(((*name).to_owned(), Vec::new())); + } + let (steps, _, _) = completed(started(request)); + steps.iter().map(step_lines).collect() +} + /// Run `models` and describe the facts the finished session kept for each, or `none`. pub(crate) fn fact_lines(models: &[ModelSpec]) -> Vec { let (_, _, finished) = completed(started(orders_request(model_requests(models)))); diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_session.rs index 03e6f7d0fc..77aab0583f 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_analysis_session.rs @@ -6,11 +6,12 @@ use crate::assembly::analysis_session::models::PivotOutcome; use crate::assembly::analysis_session::tests::helpers::{ catalog, expected_facts, fact_lines, failed, finished_session, kept_model_analyses, legacy_lines, lines, model_requests, orders_request, pivot_request, proven, recovered_facts, - session_lines, + session_lines, session_steps, }; use crate::assembly::analysis_session::tests::test_types::{ CteRecoveryTestCase, KeptAnalysisTestCase, LegacyAnalysisTestCase, ModelSpec, PivotTestCase, - SessionFactsTestCase, SessionFailureTestCase, SessionTestCase, UnscheduledTestCase, + SessionFactsTestCase, SessionFailureTestCase, SessionTestCase, StarExpansionTestCase, + UnscheduledTestCase, }; /// A model whose missing column stays untyped over a known input, so completion re-analyses it. @@ -21,6 +22,9 @@ const UNTYPED_ORDERS: ModelSpec = ( &[], ); +/// A producer whose output types stay unknown without rich inference. +const STAR_PRODUCER: ModelSpec = ("stg_orders", "SELECT 1 AS order_id, 10 AS amount", &[], &[]); + #[test] fn given_models_when_running_the_session_then_publishes_as_python_does() { let test_cases = [ @@ -174,6 +178,68 @@ fn given_models_when_running_the_session_then_publishes_as_python_does() { } } +#[test] +fn given_a_star_beside_a_cast_constant_when_running_the_session_then_expands_the_star() { + let test_cases = [ + StarExpansionTestCase { + description: "fast inference keeps the starred columns before the cast constant", + rich_type_inference: false, + models: &[ + STAR_PRODUCER, + ( + "orders_flagged", + "SELECT *, CAST(1 AS INTEGER) AS flag FROM __ref(\"stg_orders\")", + &[], + &["stg_orders"], + ), + ], + expected_steps: &[&[ + "publish stg_orders order_id:UNKNOWN amount:UNKNOWN", + "publish orders_flagged order_id:UNKNOWN amount:UNKNOWN flag:INT", + ]], + }, + StarExpansionTestCase { + description: "fast inference expands a star over a derived table", + rich_type_inference: false, + models: &[ + STAR_PRODUCER, + ( + "orders_flagged", + "SELECT *, CAST(1 AS INTEGER) AS flag \ + FROM (SELECT order_id, amount FROM __ref(\"stg_orders\")) AS edited", + &[], + &["stg_orders"], + ), + ], + expected_steps: &[&[ + "publish stg_orders order_id:UNKNOWN amount:UNKNOWN", + "publish orders_flagged order_id:UNKNOWN amount:UNKNOWN flag:INT", + ]], + }, + StarExpansionTestCase { + description: "rich inference keeps the starred columns beside the cast constant", + rich_type_inference: true, + models: &[ + STAR_PRODUCER, + ( + "orders_flagged", + "SELECT *, CAST(1 AS INTEGER) AS flag FROM __ref(\"stg_orders\")", + &[], + &["stg_orders"], + ), + ], + expected_steps: &[&[ + "publish stg_orders order_id:INT amount:INT", + "publish orders_flagged order_id:INT amount:INT flag:INT", + ]], + }, + ]; + for test_case in test_cases { + let steps = session_steps(test_case.models, test_case.rich_type_inference); + assert_eq!(steps, test_case.expected_steps, "{}", test_case.description); + } +} + #[test] fn given_finished_sessions_when_reading_model_facts_then_keeps_only_native_successes() { let test_cases = [ diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs index c74ecd0b9e..6a3236ccd9 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/test_types.rs @@ -17,6 +17,15 @@ pub(crate) struct SessionTestCase { pub(crate) expected_outcomes: &'static [&'static [&'static str]], } +/// A session over the orders request, run with or without rich type inference. +pub(crate) struct StarExpansionTestCase { + pub(crate) description: &'static str, + pub(crate) rich_type_inference: bool, + pub(crate) models: &'static [ModelSpec], + /// `step_lines` of each step, in order. + pub(crate) expected_steps: &'static [&'static [&'static str]], +} + pub(crate) struct SessionFactsTestCase { pub(crate) description: &'static str, pub(crate) models: &'static [ModelSpec], diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs index 25feefc33a..f64dbe36ab 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/normalization.rs @@ -9,6 +9,8 @@ use std::sync::LazyLock; const DBT_REF: &str = "dbt_ref"; const UDF: &str = "udf"; +/// Snowflake's second line comment prefix; only Snowflake SQL reaches the variant scan. +const SNOWFLAKE_LINE_COMMENT: &str = "//"; type Edit = (usize, usize, MappedSql); static REFERENCES: LazyLock> = LazyLock::new(|| { Regex::new(r#"__(ref|seed|source|dbt_ref)\((?:"[^"]+"\s*,\s*)?"([^"]+)"\)"#) @@ -201,6 +203,13 @@ fn skip_non_code(sql: &str, index: usize) -> Result, String> { .map_or(sql.len(), |end| index + 4 + end), )); } + if sql[index..].starts_with(SNOWFLAKE_LINE_COMMENT) { + return Ok(Some( + sql[index..] + .find('\n') + .map_or(sql.len(), |offset| index + offset + 1), + )); + } non_code_end( sql.as_bytes(), index, diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs index 3cdbaceca7..112492fef7 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_normalize_batch.rs @@ -30,6 +30,19 @@ fn given_batch_when_normalizing_on_pool_or_in_turn_then_matches_single_calls_in_ ], expected_failures: &[0, 2], }, + NormalizationBatchTestCase { + description: "double-slash comments hide quotes and brackets", + dialect: "snowflake", + requests: &[ + ("SELECT id // it's the key\nFROM orders", &[]), + ( + "// don't read \"items[0\nSELECT payload:item[0] AS item FROM inventory", + &[], + ), + ("SELECT id FROM orders // trailing 'note", &[]), + ], + expected_failures: &[], + }, ]; let pool = rayon::ThreadPoolBuilder::new() .num_threads(3) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs index 52e3f8e2ef..45f40afa35 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs @@ -48,6 +48,7 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::fields::mapping_items; use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::_helpers::semantic_shapes::shapes::snapshot_validity_columns; use crate::bindings::_helpers::type_system::equality::types_equal as native_types_equal; const CHANGES_LOGGER: &str = "sqlbuild.planner.changes"; @@ -531,6 +532,11 @@ fn model_schema_findings<'py>( if warehouse.is_none() { return Ok(Vec::new()); } + let validity: Vec = snapshot_validity_columns(model)?; + let mut warehouse: Vec<(String, String)> = column_pairs(&warehouse)?; + for column in &validity { + warehouse.retain(|(name, _)| !column.eq_ignore_ascii_case(name)); + } let schema_entry: Bound<'py, PyAny> = native_field(model, "schema_entry")?; let mut yml: Vec<(String, String)> = Vec::new(); let mut type_enforcement: bool = false; @@ -559,7 +565,7 @@ fn model_schema_findings<'py>( SchemaColumns { yml, inferred, - warehouse: column_pairs(&warehouse)?, + warehouse, }, (type_enforcement, inferred_schema_complete), (dynamic, dialect), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs index 2973fa30de..e5dfd2cf62 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs @@ -107,27 +107,36 @@ pub(crate) fn published_shape<'py>( inputs: &Bound<'py, PyDict>, profile: &Bound<'py, PyAny>, ) -> PyResult> { - let py: Python<'py> = model.py(); let shape: Bound<'py, PyDict> = binding_shape(model, columns, inputs, profile)?; - let values: Bound<'py, PyAny> = native_field(&native_field(model, "config")?, "values")?; - let snapshot: Bound<'py, PyAny> = enum_member(py, RecordId::MaterializationType, "SNAPSHOT")?; - if mapping_get(&values, "materialized")?.eq(snapshot)? { - for (key, default) in [ - ("valid_from_column", VALID_FROM_DEFAULT), - ("valid_to_column", VALID_TO_DEFAULT), - ] { - let name: Bound<'py, PyAny> = mapping_get(&values, key)?; - let name: Bound<'py, PyAny> = if name.is_truthy()? { - name - } else { - PyString::new(py, default).into_any() - }; - shape.set_item(name.str()?, UNKNOWN)?; - } + for name in snapshot_validity_columns(model)? { + shape.set_item(name, UNKNOWN)?; } Ok(shape) } +/// A snapshot model's validity column names, which the snapshot adds to its query's columns. +pub(crate) fn snapshot_validity_columns(model: &Bound<'_, PyAny>) -> PyResult> { + let py: Python<'_> = model.py(); + let values: Bound<'_, PyAny> = native_field(&native_field(model, "config")?, "values")?; + let snapshot: Bound<'_, PyAny> = enum_member(py, RecordId::MaterializationType, "SNAPSHOT")?; + if !mapping_get(&values, "materialized")?.eq(snapshot)? { + return Ok(Vec::new()); + } + let mut names: Vec = Vec::with_capacity(2); + for (key, default) in [ + ("valid_from_column", VALID_FROM_DEFAULT), + ("valid_to_column", VALID_TO_DEFAULT), + ] { + let name: Bound<'_, PyAny> = mapping_get(&values, key)?; + names.push(if name.is_truthy()? { + name.str()?.extract()? + } else { + default.to_owned() + }); + } + Ok(names) +} + /// Inferred columns, re-quoted from authored SQL where the dialect keeps quoted case. fn binding_shape<'py>( model: &Bound<'py, PyAny>, diff --git a/src/sqlbuild/adapter/relations/constants.py b/src/sqlbuild/adapter/relations/constants.py index 8fa46168db..0c144b1164 100644 --- a/src/sqlbuild/adapter/relations/constants.py +++ b/src/sqlbuild/adapter/relations/constants.py @@ -5,3 +5,4 @@ METADATA_NAME_FILTER_LIMIT: int = 250 RELATION_COLUMNS_LOOKUP: str = "columns" RELATION_EXISTS_LOOKUP: str = "exists" +RELATION_TYPE_LOOKUP: str = "type" diff --git a/src/sqlbuild/adapter/relations/main/cached_relation_type.py b/src/sqlbuild/adapter/relations/main/cached_relation_type.py new file mode 100644 index 0000000000..2d39339f59 --- /dev/null +++ b/src/sqlbuild/adapter/relations/main/cached_relation_type.py @@ -0,0 +1,56 @@ +"""Read one relation's live type through the open relation metadata cache.""" + +from __future__ import annotations + +from functools import partial +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup +from sqlbuild.adapter.contract.types import RelationType +from sqlbuild.adapter.relations._helpers.relation_metadata_context import ( + read_through_relation_metadata_cache, +) +from sqlbuild.adapter.relations.constants import RELATION_TYPE_LOOKUP +from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type + + +def cached_relation_type( + *, adapter: BaseAdapter, connection: Any, database: str | None, schema: str | None, name: str +) -> RelationType | None: + """The warehouse type of the relation at this location, or `None` when none exists.""" + + return read_through_relation_metadata_cache( + adapter=adapter, + kind=RELATION_TYPE_LOOKUP, + database=database, + schema=schema, + name=name, + read=partial( + _relation_type, + adapter=adapter, + connection=connection, + database=database, + schema=schema, + name=name, + ), + ) + + +def _relation_type( + *, adapter: BaseAdapter, connection: Any, database: str | None, schema: str | None, name: str +) -> RelationType | None: + relations: tuple[RelationInfo, ...] = adapter.list_relations( + connection=connection, + database=database, + schemas=None if schema is None else (schema,), + names=(name,), + ) + wanted: tuple[str | None, str | None, str] = RelationLookup.key(schema=schema, name=name) + for relation in relations: + found: tuple[str | None, str | None, str] = RelationLookup.key( + schema=None if schema is None else relation.schema, name=relation.name + ) + if found == wanted: + return normalize_relation_type(relation.relation_type) + return None diff --git a/src/sqlbuild/executor/run/_helpers/execution/promotion.py b/src/sqlbuild/executor/run/_helpers/execution/promotion.py index 63b72639b1..edf070a67b 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/promotion.py +++ b/src/sqlbuild/executor/run/_helpers/execution/promotion.py @@ -7,8 +7,9 @@ from sqlbuild._native import OPERATION_STRATEGIES from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.types import PromotionStrategy +from sqlbuild.adapter.contract.types import PromotionStrategy, RelationType from sqlbuild.adapter.relations.main.cached_relation_exists import cached_relation_exists +from sqlbuild.adapter.relations.main.cached_relation_type import cached_relation_type from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.run.models import BoundViewGuard from sqlbuild.runtime.observability.classes.operation_lifecycle import ( @@ -100,6 +101,52 @@ def promote_relation_to_destination( else: raise ExecutorInputError(f"Unsupported promotion strategy: {strategy}") except Exception as error: - lifecycle.failed(error=error, attributes=terminal_attributes) - raise + if ( + isinstance(error, ExecutorInputError) + or not existing + or not _dropped_replaced_view( + adapter=adapter, + connection=connection, + destination=(destination_database, destination_schema, destination_name), + destination_relation=destination_relation, + statement_recorder=statement_recorder, + ) + ): + lifecycle.failed(error=error, attributes=terminal_attributes) + raise + try: + adapter.rename( + connection=connection, + origin=origin_relation, + destination=destination_relation, + statement_recorder=statement_recorder, + ) + except Exception as rename_error: + lifecycle.failed(error=rename_error, attributes=terminal_attributes) + raise lifecycle.completed(metadata={"changed_count": 1}, attributes=terminal_attributes) + + +def _dropped_replaced_view( + *, + adapter: BaseAdapter, + connection: Any, + destination: tuple[str | None, str | None, str], + destination_relation: str, + statement_recorder: StatementRecorder, +) -> bool: + """Drop the live destination when it is a view a table cannot replace; whether it was one.""" + + database, schema, name = destination + existing: RelationType | None = cached_relation_type( + adapter=adapter, connection=connection, database=database, schema=schema, name=name + ) + if existing != RelationType.VIEW: + return False + adapter.drop_view( + connection=connection, + destination=destination_relation, + if_exists=True, + statement_recorder=statement_recorder, + ) + return True diff --git a/src/sqlbuild/executor/run/_helpers/materializations/view.py b/src/sqlbuild/executor/run/_helpers/materializations/view.py index 4d3d1453b6..eca08c28fb 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/view.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/view.py @@ -14,6 +14,8 @@ ) from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.types import RelationType +from sqlbuild.adapter.relations.main.cached_relation_type import cached_relation_type from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( resolve_relation_location_qualified_name, ) @@ -111,10 +113,11 @@ def execute_view_entry( statement_recorder=statement_recorder, ) with diagnostics_context(sqlbuild_phase="materialize", sqlbuild_action_name="create_view"): - adapter.create_view_as( + _create_view_replacing_table( + adapter=adapter, connection=connection, - destination=target_qualified, - sql=entry.resolved_sql, + entry=entry, + target_qualified=target_qualified, statement_recorder=statement_recorder, ) except Exception as exc: @@ -222,3 +225,50 @@ def execute_view_entry( lifecycle_events=statement_recorder.snapshot(), hook_results=tuple(hook_results), ) + + +def _create_view_replacing_table( + *, + adapter: BaseAdapter, + connection: Any, + entry: ModelPlanEntry, + target_qualified: str, + statement_recorder: StatementRecorder, +) -> None: + """Create the view; a table the model built before it became a view is dropped first.""" + + try: + adapter.create_view_as( + connection=connection, + destination=target_qualified, + sql=entry.resolved_sql, + statement_recorder=statement_recorder, + ) + except Exception as error: + if not _holds_table(adapter=adapter, connection=connection, entry=entry): + raise error + adapter.drop( + connection=connection, + destination=target_qualified, + if_exists=True, + statement_recorder=statement_recorder, + ) + adapter.create_view_as( + connection=connection, + destination=target_qualified, + sql=entry.resolved_sql, + statement_recorder=statement_recorder, + ) + + +def _holds_table(*, adapter: BaseAdapter, connection: Any, entry: ModelPlanEntry) -> bool: + """Whether the view's destination holds a relation other than a view.""" + + existing: RelationType | None = cached_relation_type( + adapter=adapter, + connection=connection, + database=entry.destination.database, + schema=entry.destination.schema, + name=entry.destination.name, + ) + return existing is not None and existing != RelationType.VIEW diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 7816b2f698..4067389ef9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1354,3 +1354,21 @@ class StandaloneAuditRowE2ETestCase: status_values: tuple[str, ...] expected_exit_code: int expected_rows: tuple[str, ...] + + +@dataclass(frozen=True) +class SnapshotValidityColumnsTestCase: + """A snapshot's validity column config and the column names its built relation holds.""" + + description: str + validity_config: str + expected_columns: tuple[str, ...] + + +@dataclass(frozen=True) +class MaterializationTypeChangeTestCase: + """A built model's materialization change and the relation type each build leaves.""" + + description: str + materializations: tuple[str, ...] + expected_table_types: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 98556b8bf9..6a99667565 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1142,3 +1142,12 @@ def seed_audit_files(status_values: tuple[str, ...]) -> dict[str, str]: " severity: warn\n" ), } + + +def order_totals_model_sql(*, materialized: str) -> str: + """One order totals model with the given materialization.""" + + return ( + f"MODEL (description 'Order totals.', materialized {materialized});\n\n" + "SELECT 1 AS order_id, 250 AS total_cents\n" + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_materialization_type_change.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_materialization_type_change.py new file mode 100644 index 0000000000..8c30847451 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_materialization_type_change.py @@ -0,0 +1,81 @@ +"""A built model switching between table and view replaces the existing relation.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + MaterializationTypeChangeTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import order_totals_model_sql +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + query_duckdb, + run_sqb, +) + +_PROJECT_TOML: str = ( + 'name = "orders"\nadapter = "duckdb"\n[connection]\ndatabase = "orders.duckdb"\n' +) +_MODEL_PATH: str = "models/order_totals.sql" + + +@pytest.mark.parametrize( + "test_case", + [ + MaterializationTypeChangeTestCase( + description="table then view then table", + materializations=("table", "view", "table"), + expected_table_types=("BASE TABLE", "VIEW", "BASE TABLE"), + ), + MaterializationTypeChangeTestCase( + description="view then table then view", + materializations=("view", "table", "view"), + expected_table_types=("VIEW", "BASE TABLE", "VIEW"), + ), + ], + ids=lambda case: case.description, +) +def test_given_built_model_when_materialization_changes_then_build_replaces_relation( + test_case: MaterializationTypeChangeTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_project", + repo_files={"sqlbuild_project.toml": _PROJECT_TOML}, + ) + model_path: Path = project_dir / _MODEL_PATH + model_path.parent.mkdir(parents=True, exist_ok=True) + outcomes: list[tuple[int, str, list[tuple[Any, ...]]]] = [] + for materialized in test_case.materializations: + _ = model_path.write_text( + order_totals_model_sql(materialized=materialized), encoding="utf-8" + ) + built: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "build") + ) + table_types: list[tuple[Any, ...]] = query_duckdb( + db_path=project_dir / "orders.duckdb", + sql="SELECT table_type FROM information_schema.tables WHERE table_name = 'order_totals'", + ) + outcomes.append((built.returncode, built.stdout + built.stderr, table_types)) + rows: list[tuple[Any, ...]] = query_duckdb( + db_path=project_dir / "orders.duckdb", sql="SELECT order_id, total_cents FROM order_totals" + ) + + assert [outcome[0] for outcome in outcomes] == [0] * len(test_case.materializations), ( + f"{test_case.description}: {[outcome[1] for outcome in outcomes]}" + ) + assert tuple(outcome[2][0][0] for outcome in outcomes) == test_case.expected_table_types, ( + test_case.description + ) + assert rows == [(1, 250)], test_case.description + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_validity_columns.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_validity_columns.py new file mode 100644 index 0000000000..7bba63162b --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_validity_columns.py @@ -0,0 +1,93 @@ +"""A rebuilt snapshot does not report its own validity columns as schema changes.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + SnapshotValidityColumnsTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + query_duckdb, + run_sqb, +) + +_PROJECT_TOML: str = ( + 'name = "customers"\nadapter = "duckdb"\n[connection]\ndatabase = "customers.duckdb"\n' +) +_CUSTOMERS_SQL: str = ( + "MODEL (description 'Customers.', materialized table);\n\n" + "SELECT 1 AS customer_id, 'basic' AS plan, " + "CAST('2026-01-01 00:00:00' AS TIMESTAMP) AS updated_at\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + SnapshotValidityColumnsTestCase( + description="default validity columns", + validity_config="", + expected_columns=("customer_id", "plan", "updated_at", "valid_from", "valid_to"), + ), + SnapshotValidityColumnsTestCase( + description="renamed validity columns", + validity_config=" valid_from_column effective_from,\n valid_to_column effective_to,\n", + expected_columns=( + "customer_id", + "plan", + "updated_at", + "effective_from", + "effective_to", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_built_snapshot_when_building_again_then_validity_columns_are_not_schema_changes( + test_case: SnapshotValidityColumnsTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="customers_project", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/customers.sql": _CUSTOMERS_SQL, + "models/customer_history.sql": ( + "MODEL (\n description 'Customer history.',\n materialized snapshot,\n" + " unique_key [customer_id],\n snapshot_strategy timestamp,\n" + f" updated_at updated_at,\n{test_case.validity_config});\n\n" + 'SELECT customer_id, plan, updated_at FROM __ref("customers")\n' + ), + }, + ) + + first: subprocess.CompletedProcess[str] = run_sqb(project_dir=project_dir, command=("build",)) + second: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("--no-color", "build") + ) + built_columns: list[tuple[Any, ...]] = query_duckdb( + db_path=project_dir / "customers.duckdb", + sql=( + "SELECT column_name FROM information_schema.columns " + "WHERE table_name = 'customer_history' ORDER BY ordinal_position" + ), + ) + + assert first.returncode == 0, f"{test_case.description}: {first.stdout}{first.stderr}" + assert second.returncode == 0, f"{test_case.description}: {second.stdout}{second.stderr}" + assert "Schema changed" not in second.stdout, f"{test_case.description}: {second.stdout}" + assert "(removed)" not in second.stdout, f"{test_case.description}: {second.stdout}" + assert tuple(row[0] for row in built_columns) == test_case.expected_columns, ( + test_case.description + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index a07b1346fa..c669e5b8a6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -538,6 +538,17 @@ class SetOperationLifecycleTestCase: expected_indented_set_operation_lines: tuple[str, ...] +@dataclass(frozen=True) +class FromQuerySetOperationTestCase: + description: str + adapter: str + models: tuple[SetOperationModel, ...] + expected_column_counts: tuple[tuple[str, int], ...] + lineage_target: str + expected_lineage: tuple[tuple[str, str, str], ...] + expected_rule_summary: str + + @dataclass(frozen=True) class SetOperationArityMismatchTestCase: description: str @@ -1168,3 +1179,22 @@ class DeltaCompileOptOutTestCase: env: dict[str, str] expected_delta_line: bool = False expected_images: tuple[str, ...] = () + + +@dataclass(frozen=True) +class StarBesideCastConstantTestCase: + """A model selecting a star beside a cast constant and the columns it must compile and build.""" + + description: str + query_sql: str + expected_columns: tuple[str, ...] + + +@dataclass(frozen=True) +class DialectLineCommentTestCase: + """A dialect's own line comment holding quotes and reference text, and the columns it keeps.""" + + description: str + project_toml: str + query_sql: str + expected_column_count: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index da20018bd0..b532e5a73e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -2879,15 +2879,21 @@ class SetOperationLifecycleResult(NamedTuple): def write_set_operation_project( - *, project_dir: Path, models: tuple[SetOperationModel, ...] + *, project_dir: Path, models: tuple[SetOperationModel, ...], adapter: str = "duckdb" ) -> Path: - """Write a DuckDB project with one view per model and return its database path.""" + """Write a project with one view per model and return its DuckDB database path. + + Other adapters compile offline against a default database and never connect. + """ database: Path = project_dir / "orders.duckdb" + target: str = { + True: f'[connection]\ndatabase = "{database}"\n', + False: '[defaults]\ndatabase = "example"\n', + }[adapter == "duckdb"] files: dict[str, str] = { "sqlbuild_project.toml": ( - f'name = "orders"\nadapter = "duckdb"\n[connection]\ndatabase = "{database}"\n' - '[rules]\nselect = ["SQBRSQL020"]\n' + f'name = "orders"\nadapter = "{adapter}"\n{target}[rules]\nselect = ["SQBRSQL020"]\n' ), } for model in models: @@ -2938,9 +2944,32 @@ def run_set_operation_lifecycle( f"SELECT * FROM analytics.{model.name}" ) relation_shapes[model.name] = (len(relation.columns), len(relation.fetchall())) - rules: subprocess.CompletedProcess[str] = run_sqb( - project_dir=project_dir, command=("rules", "run", "SQBRSQL020") + rules: subprocess.CompletedProcess[str] = run_set_arity_rule(project_dir=project_dir) + formatted: subprocess.CompletedProcess[str] + indented_set_operation_lines: tuple[str, ...] + formatted, indented_set_operation_lines = format_set_operation_models(project_dir=project_dir) + return SetOperationLifecycleResult( + compiled=compiled, + build=build, + relation_shapes=relation_shapes, + rules=rules, + format=formatted, + indented_set_operation_lines=indented_set_operation_lines, + recompiled=compile_set_operation_project(project_dir=project_dir), ) + + +def run_set_arity_rule(*, project_dir: Path) -> subprocess.CompletedProcess[str]: + """Run the set-operation arity Rule over every model.""" + + return run_sqb(project_dir=project_dir, command=("rules", "run", "SQBRSQL020")) + + +def format_set_operation_models( + *, project_dir: Path +) -> tuple[subprocess.CompletedProcess[str], tuple[str, ...]]: + """Format every model and return the result and any set-operation lines left indented.""" + models_dir: Path = project_dir / "models" formatted: subprocess.CompletedProcess[str] = run_sqb( project_dir=project_dir, command=("format", str(models_dir)) @@ -2948,15 +2977,23 @@ def run_set_operation_lifecycle( indented_set_operation_lines: list[str] = [] for path in sorted(models_dir.glob("*.sql")): indented_set_operation_lines.extend(_indented_set_operation_lines(path)) - return SetOperationLifecycleResult( - compiled=compiled, - build=build, - relation_shapes=relation_shapes, - rules=rules, - format=formatted, - indented_set_operation_lines=tuple(indented_set_operation_lines), - recompiled=compile_set_operation_project(project_dir=project_dir), + return formatted, tuple(indented_set_operation_lines) + + +def column_lineage_edges( + *, project_dir: Path, target: str +) -> tuple[subprocess.CompletedProcess[str], tuple[tuple[str, str, str], ...]]: + """Return the lineage result and sorted `(model, column, transform)` upstream edges.""" + + result: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("lineage", target, "--json") ) + payload: dict[str, Any] = json.loads(result.stdout or '{"trace": []}') + edges: list[tuple[str, str, str]] = [ + (edge["source"]["resource_name"], edge["source"]["column_name"], edge["transform"]) + for edge in payload["trace"] + ] + return result, tuple(sorted(edges)) class UnionFixtureCompileMeasurement(NamedTuple): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index c2c786313e..8e7d1e126a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -53,28 +53,28 @@ DenseCompileGuardTestCase( "dense_models_1000_all_rules", 1000, - 7.5, + 6.0, 800 * _MIB, "de23b14d87e954a65c2da91918fce062d158419edd63cc8cf3dc938b3e47cd44", ), DenseCompileGuardTestCase( "dense_models_3000_all_rules", 3000, - 16.5, + 12.0, 3 * _GIB // 2, "196fca3e87989eb3e89e443b7ca93bb094198538d5bde32b9d6fcca58ffc8eed", ), DenseCompileGuardTestCase( "dense_models_5000_all_rules", 5000, - 26.0, + 18.0, 9 * _GIB // 4, "3477a472fa76d6236ebd0c9758d79c18f67f1bf55e9bbec05b002ba3744cfd22", ), DenseCompileGuardTestCase( "dense_models_10000_all_rules", 10000, - 51.0, + 35.0, 15 * _GIB // 4, "f2a574af6e8068a0634fa3c078b168dff9a475ada5d71502482ad2926c90e1c3", ), @@ -154,8 +154,8 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ "dense_models_3000_warm_and_one_edit", 3000, 1521, - 8.0, - 8.5, + 6.0, + 6.0, 3 * _GIB // 2, "196fca3e87989eb3e89e443b7ca93bb094198538d5bde32b9d6fcca58ffc8eed", "1afe754dd8c66bc36e38a40c66e9183ecbaf365fd8d87a784069cd615c3ce64a", @@ -165,8 +165,8 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ "dense_models_5000_warm_and_one_edit", 5000, 2521, - 13.0, - 13.5, + 9.0, + 9.5, 9 * _GIB // 4, "3477a472fa76d6236ebd0c9758d79c18f67f1bf55e9bbec05b002ba3744cfd22", "dfbada4c59a3ab8ec7733784890d74c7b64e9b0adcb03364baaabef5bdba5574", @@ -240,9 +240,9 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches model_count=3000, edited_model_index=1521, full_pairs=4, - expected_max_full_seconds=15.0, - expected_max_delta_seconds=4.5, - expected_max_delta_cpu_ratio=0.375, + expected_max_full_seconds=7.0, + expected_max_delta_seconds=2.0, + expected_max_delta_cpu_ratio=0.30, expected_max_fallback_cpu_ratio=1.05, expected_max_rss_bytes=3 * _GIB // 2, ), @@ -251,9 +251,9 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches model_count=10000, edited_model_index=5021, full_pairs=3, - expected_max_full_seconds=60.0, - expected_max_delta_seconds=9.0, - expected_max_delta_cpu_ratio=0.30, + expected_max_full_seconds=28.0, + expected_max_delta_seconds=8.0, + expected_max_delta_cpu_ratio=0.275, expected_max_fallback_cpu_ratio=1.05, expected_max_rss_bytes=15 * _GIB // 4, ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_line_comments.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_line_comments.py new file mode 100644 index 0000000000..8b71f18374 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dialect_line_comments.py @@ -0,0 +1,88 @@ +"""A dialect's own line comments may hold quotes and reference text through the real CLI.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + DialectLineCommentTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_SNOWFLAKE_TOML: str = ( + 'name = "orders"\nadapter = "snowflake"\n\n[connection]\naccount = "example"\n' + 'user = "analyst"\npassword = "unused"\ndatabase = "ANALYTICS"\nwarehouse = "COMPUTE"\n' + 'schema = "ORDERS"\n\n[defaults]\ndatabase = "ANALYTICS"\n' +) +_BIGQUERY_TOML: str = ( + 'name = "orders"\nadapter = "bigquery"\n\n[connection]\nproject = "example-project"\n' + 'schema = "orders"\n\n[defaults]\ndatabase = "example-project"\n' +) +_BASE_ORDERS_SQL: str = "MODEL (description 'Orders.', materialized table);\n\nSELECT 1 AS order_id, 2 AS amount_cents\n" + + +@pytest.mark.parametrize( + "test_case", + [ + DialectLineCommentTestCase( + description="snowflake double-slash comments with quotes, brackets and references", + project_toml=_SNOWFLAKE_TOML, + query_sql=( + '// it\'s the order key, "quoted" and items[0]\n' + 'SELECT order_id, amount_cents // don\'t read __ref("missing_orders")\n' + 'FROM __ref("base_orders")' + ), + expected_column_count=2, + ), + DialectLineCommentTestCase( + description="bigquery hash comments with quotes and references", + project_toml=_BIGQUERY_TOML, + query_sql=( + '# it\'s the order key and "quoted"\n' + 'SELECT order_id, amount_cents # don\'t read __ref("missing_orders")\n' + 'FROM __ref("base_orders")' + ), + expected_column_count=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_dialect_line_comment_with_quotes_when_compiling_then_model_is_analysed( + test_case: DialectLineCommentTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_project", + repo_files={ + "sqlbuild_project.toml": test_case.project_toml, + "models/base_orders.sql": _BASE_ORDERS_SQL, + "models/commented_orders.sql": ( + "MODEL (description 'Commented orders.', materialized table);\n\n" + f"{test_case.query_sql}\n" + ), + }, + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("compile", "--no-cache", "--json") + ) + payload: dict[str, Any] = json.loads(compiled.stdout) + column_counts: dict[str, int] = { + model["name"]: model["column_count"] for model in payload["resources"]["models"] + } + + assert compiled.returncode == 0, f"{test_case.description}: {compiled.stdout}{compiled.stderr}" + assert payload["diagnostics"] == [], test_case.description + assert column_counts["commented_orders"] == test_case.expected_column_count, ( + test_case.description + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py index 601275d93e..aeb16ae9de 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py @@ -50,14 +50,14 @@ macro_count=500, test_count=2_945, audit_count=5_056, - expected_cold_max_wall_seconds=10.5, - expected_warm_max_wall_seconds=6.0, - expected_edit_max_wall_seconds=6.0, + expected_cold_max_wall_seconds=9.0, + expected_warm_max_wall_seconds=4.5, + expected_edit_max_wall_seconds=4.5, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO, expected_max_cache_write_wall_overhead_ratio=_MAX_CACHE_WRITE_WALL_OVERHEAD_RATIO, - expected_max_rss_bytes=2 * _GIB, + expected_max_rss_bytes=1152 * _MIB, expected_max_cache_bytes=96 * _MIB, expected_cold_fingerprint=( "1914b788abcb452422eaf6c3a2c240ff390bab9530217285a5ca93334db7b6c3" @@ -84,14 +84,14 @@ macro_count=834, test_count=4_908, audit_count=8_427, - expected_cold_max_wall_seconds=18.0, - expected_warm_max_wall_seconds=10.5, - expected_edit_max_wall_seconds=10.5, + expected_cold_max_wall_seconds=14.5, + expected_warm_max_wall_seconds=7.0, + expected_edit_max_wall_seconds=7.0, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO, expected_max_cache_write_wall_overhead_ratio=_MAX_CACHE_WRITE_WALL_OVERHEAD_RATIO, - expected_max_rss_bytes=2 * _GIB, + expected_max_rss_bytes=1664 * _MIB, expected_max_cache_bytes=160 * _MIB, expected_cold_fingerprint=( "2f1e5dfd29025c25e0dffc6e74f4c2be020a5ef133ecd081f44e0c55c24858b4" @@ -118,15 +118,15 @@ macro_count=1_667, test_count=9_816, audit_count=16_855, - expected_cold_max_wall_seconds=34.5, - expected_warm_max_wall_seconds=19.5, - expected_edit_max_wall_seconds=19.5, + expected_cold_max_wall_seconds=28.0, + expected_warm_max_wall_seconds=14.0, + expected_edit_max_wall_seconds=13.5, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_cache_write_cpu_overhead_ratio=_MAX_LARGE_CACHE_WRITE_CPU_OVERHEAD_RATIO, expected_max_cache_write_wall_overhead_ratio=_MAX_CACHE_WRITE_WALL_OVERHEAD_RATIO, # Temporary memory exception (2026-10-10); remove after conversion optimisation. - expected_max_rss_bytes=3 * _GIB, + expected_max_rss_bytes=3008 * _MIB, expected_max_cache_bytes=320 * _MIB, expected_cold_fingerprint=( "ecdd7f2cf961fab739aa806b6d90283916cffad2007e1e2f3ba90a466db6cb75" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py index 0431b89012..c1bd3fc98c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py @@ -19,7 +19,8 @@ ) _LOGGER: logging.Logger = logging.getLogger(__name__) -_GIB: int = 1024 * 1024 * 1024 +_MIB: int = 1024 * 1024 +_GIB: int = 1024 * _MIB @pytest.mark.performance @@ -36,8 +37,8 @@ macro_count=37, test_count=2_945, audit_count=5_056, - expected_max_wall_seconds=9.5, - expected_max_rss_bytes=int(1.5 * _GIB), + expected_max_wall_seconds=8.0, + expected_max_rss_bytes=1088 * _MIB, expected_semantic_fingerprint=( "d935821df54043fabc3a6911c1926578178c9b744f2a589cbef99a10d5207543" ), @@ -51,8 +52,8 @@ macro_count=61, test_count=4_908, audit_count=8_427, - expected_max_wall_seconds=16.0, - expected_max_rss_bytes=int(1.75 * _GIB), + expected_max_wall_seconds=12.5, + expected_max_rss_bytes=1664 * _MIB, expected_semantic_fingerprint=( "874cbfd473846c6a335ec2f79cb6b7b0b77e8e8d58652815fad1e9d59d423c46" ), @@ -66,7 +67,7 @@ macro_count=123, test_count=9_816, audit_count=16_855, - expected_max_wall_seconds=31.0, + expected_max_wall_seconds=25.5, expected_max_rss_bytes=int(2.5 * _GIB), expected_semantic_fingerprint=( "5ccdabeaabfdfba1ecbae80259dd3c6eb801d683df140b7fb7a249b9555308c6" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_set_operation_after_subquery_operand.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_set_operation_after_subquery_operand.py index 05c010250e..1908970a79 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_set_operation_after_subquery_operand.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_set_operation_after_subquery_operand.py @@ -2,11 +2,13 @@ from __future__ import annotations +import subprocess from pathlib import Path import pytest from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + FromQuerySetOperationTestCase, SetOperationArityMismatchTestCase, SetOperationLifecycleTestCase, SetOperationModel, @@ -14,7 +16,10 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( SetOperationCompileResult, SetOperationLifecycleResult, + column_lineage_edges, compile_set_operation_project, + format_set_operation_models, + run_set_arity_rule, run_set_operation_lifecycle, write_set_operation_project, ) @@ -25,6 +30,43 @@ "), limits AS (SELECT 10 AS max_amount)\n" ) _ALL_ORDERS: str = "SELECT o.order_id, o.amount FROM orders AS o" +_ORDERS_QUERY: str = 'SELECT o.order_id, o.amount, o.status FROM __ref("orders") AS o' +_RETURNS_QUERY: str = 'SELECT r.order_id, r.amount FROM __ref("returns") AS r' +_FROM_QUERY_MODELS: tuple[SetOperationModel, ...] = ( + SetOperationModel( + description="three-column orders", + name="orders", + query_sql="SELECT 1 AS order_id, 20 AS amount, 'open' AS status", + ), + SetOperationModel( + description="two-column returns", + name="returns", + query_sql="SELECT 2 AS order_id, 5 AS amount", + ), + SetOperationModel( + description="narrowed FROM query before UNION ALL", + name="union_all_orders", + query_sql=f"SELECT order_id, amount FROM ({_ORDERS_QUERY})\nUNION ALL\n{_RETURNS_QUERY}", + ), + SetOperationModel( + description="narrowed FROM query before EXCEPT", + name="except_orders", + query_sql=f"SELECT order_id, amount FROM ({_ORDERS_QUERY})\nEXCEPT\n{_RETURNS_QUERY}", + ), + SetOperationModel( + description="narrowed FROM query before INTERSECT", + name="intersect_orders", + query_sql=f"SELECT order_id, amount FROM ({_ORDERS_QUERY})\nINTERSECT\n{_RETURNS_QUERY}", + ), + SetOperationModel( + description="star over a FROM query before UNION ALL", + name="starred_orders", + query_sql=( + 'SELECT * FROM (SELECT o.order_id, o.amount FROM __ref("orders") AS o)\n' + f"UNION ALL\n{_RETURNS_QUERY}" + ), + ), +) @pytest.mark.parametrize( @@ -128,6 +170,16 @@ ), expected_row_count=4, ), + SetOperationModel( + description="narrowed FROM query before UNION ALL", + name="from_query_union_all", + query_sql=( + f"{_INPUTS}SELECT order_id, amount\n" + "FROM (SELECT o.order_id, o.amount, 'open' AS status FROM orders AS o)\n" + f"UNION ALL\n{_ALL_ORDERS}" + ), + expected_row_count=4, + ), SetOperationModel( description="parenthesized branches around a scalar subquery", name="parenthesized_branches_union_all", @@ -141,7 +193,7 @@ ), expected_column_count=2, expected_diagnostics=(), - expected_rule_summary="10 models evaluated, 0 findings", + expected_rule_summary="11 models evaluated, 0 findings", expected_indented_set_operation_lines=(), ) ], @@ -175,6 +227,63 @@ def test_given_set_operation_after_subquery_operand_when_compiling_building_and_ assert result.recompiled.column_counts == expected_column_counts +@pytest.mark.parametrize( + "test_case", + [ + FromQuerySetOperationTestCase( + description=f"{adapter} FROM query before a set operation", + adapter=adapter, + models=_FROM_QUERY_MODELS, + expected_column_counts=( + ("union_all_orders", 2), + ("except_orders", 2), + ("intersect_orders", 2), + ), + lineage_target="starred_orders.order_id", + expected_lineage=(("orders", "order_id", "direct"), ("returns", "order_id", "direct")), + expected_rule_summary="6 models evaluated, 0 findings", + ) + for adapter in ("duckdb", "snowflake", "bigquery", "postgres") + ], + ids=lambda case: case.description, +) +def test_given_set_operation_after_parenthesized_from_query_when_compiling_then_outer_queries_combine( + test_case: FromQuerySetOperationTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = tmp_path / "orders_project" + write_set_operation_project( + project_dir=project_dir, models=test_case.models, adapter=test_case.adapter + ) + + compiled: SetOperationCompileResult = compile_set_operation_project(project_dir=project_dir) + lineage: subprocess.CompletedProcess[str] + lineage_edges: tuple[tuple[str, str, str], ...] + lineage, lineage_edges = column_lineage_edges( + project_dir=project_dir, target=test_case.lineage_target + ) + rules: subprocess.CompletedProcess[str] = run_set_arity_rule(project_dir=project_dir) + formatted: subprocess.CompletedProcess[str] + indented_set_operation_lines: tuple[str, ...] + formatted, indented_set_operation_lines = format_set_operation_models(project_dir=project_dir) + recompiled: SetOperationCompileResult = compile_set_operation_project(project_dir=project_dir) + + expected_column_counts: dict[str, int] = dict(test_case.expected_column_counts) + assert compiled.exit_code == 0, compiled.output + assert compiled.diagnostics == () + assert { + name: compiled.column_counts[name] for name in expected_column_counts + } == expected_column_counts + assert lineage.returncode == 0, lineage.stdout + lineage.stderr + assert lineage_edges == test_case.expected_lineage + assert rules.returncode == 0, rules.stdout + rules.stderr + assert test_case.expected_rule_summary in rules.stdout + rules.stderr + assert formatted.returncode == 0, formatted.stdout + formatted.stderr + assert indented_set_operation_lines == () + assert recompiled.exit_code == 0, recompiled.output + assert recompiled.diagnostics == () + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_star_beside_cast_constant.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_star_beside_cast_constant.py new file mode 100644 index 0000000000..311dbaece0 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_star_beside_cast_constant.py @@ -0,0 +1,92 @@ +"""A star projected beside a cast constant keeps every starred column through the real CLI.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + StarBesideCastConstantTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + query_duckdb, + run_sqb, +) + +_PROJECT_TOML: str = ( + 'name = "orders"\nadapter = "duckdb"\n[connection]\ndatabase = "orders.duckdb"\n' +) +_BASE_ORDERS_SQL: str = ( + "MODEL (description 'Orders.', materialized table);\n\n" + "SELECT 1 AS order_id, 10 AS amount_cents\n" +) +_FLAGGED_ORDERS_HEADER: str = ( + "MODEL (description 'Flagged orders.', materialized table, unique_key order_id);\n\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + StarBesideCastConstantTestCase( + description="star over a derived table beside a cast constant", + query_sql=( + "SELECT *, CAST(1 AS INTEGER) AS is_flagged\n" + 'FROM (SELECT order_id, amount_cents FROM __ref("base_orders")) AS edited' + ), + expected_columns=("order_id", "amount_cents", "is_flagged"), + ), + StarBesideCastConstantTestCase( + description="star over a reference beside a cast constant", + query_sql='SELECT *, CAST(1 AS INTEGER) AS is_flagged FROM __ref("base_orders")', + expected_columns=("order_id", "amount_cents", "is_flagged"), + ), + ], + ids=lambda case: case.description, +) +def test_given_star_beside_cast_constant_when_compiling_and_building_then_starred_columns_are_kept( + test_case: StarBesideCastConstantTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_project", + repo_files={ + "sqlbuild_project.toml": _PROJECT_TOML, + "models/base_orders.sql": _BASE_ORDERS_SQL, + "models/flagged_orders.sql": f"{_FLAGGED_ORDERS_HEADER}{test_case.query_sql}\n", + }, + ) + + compiled: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project_dir, command=("compile", "--no-cache", "--json") + ) + payload: dict[str, Any] = json.loads(compiled.stdout) + column_counts: dict[str, int] = { + model["name"]: model["column_count"] for model in payload["resources"]["models"] + } + built: subprocess.CompletedProcess[str] = run_sqb(project_dir=project_dir, command=("build",)) + built_columns: list[tuple[Any, ...]] = query_duckdb( + db_path=project_dir / "orders.duckdb", + sql=( + "SELECT column_name FROM information_schema.columns " + "WHERE table_name = 'flagged_orders' ORDER BY ordinal_position" + ), + ) + + assert compiled.returncode == 0, f"{test_case.description}: {compiled.stdout}{compiled.stderr}" + assert payload["diagnostics"] == [], test_case.description + assert column_counts["flagged_orders"] == len(test_case.expected_columns), test_case.description + assert built.returncode == 0, f"{test_case.description}: {built.stdout}{built.stderr}" + assert tuple(row[0] for row in built_columns) == test_case.expected_columns, ( + test_case.description + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py index cfbe7d947a..af776b394f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py @@ -31,10 +31,10 @@ VariedCompileCacheTestCase( "models_1000_varied_shared_graph", 1000, - 24.0, - 10.0, - 13.0, - 2 * 1024**3, + 9.0, + 2.5, + 2.5, + 832 * 1024**2, 950, 900, 40,