diff --git a/Makefile b/Makefile index 610731828c..c0b1aaf9d9 100644 --- a/Makefile +++ b/Makefile @@ -174,6 +174,7 @@ E2E_DUCKDB_CLI_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/seed \ tests/e2e/src/sqlbuild/cli/commands/main/skills \ tests/e2e/src/sqlbuild/cli/commands/main/snowflake \ + tests/e2e/src/sqlbuild/cli/commands/main/sqlite \ tests/e2e/src/sqlbuild/cli/commands/main/sqlserver \ tests/e2e/src/sqlbuild/cli/commands/main/test diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs index d8474dd099..6c70b9781a 100644 --- a/crates/sqlbuild-config/src/command_connection/constants.rs +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -9,6 +9,9 @@ pub const CONNECTION_SOURCE_KEY: &str = "source"; pub const DBT_PROFILE_CONNECTION_SOURCE: &str = "dbt_profile"; pub const DUCKDB_ADAPTER: &str = "duckdb"; pub const DUCKDB_MEMORY_DATABASE: &str = ":memory:"; +/// Adapters whose connection names a local database file, and the key that names it. +pub const LOCAL_DATABASE_PATH_KEYS: [(&str, &str); 2] = + [(DUCKDB_ADAPTER, "database"), ("sqlite", "path")]; pub const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; /// Connection keys a dbt profile connection routes on rather than connects with. pub const DBT_PROFILE_ROUTING_KEYS: [&str; 5] = diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index 630457ce08..a7e9135216 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -258,7 +258,7 @@ pub const OPERATION_STRATEGIES: [&str; 10] = [ "sql", "sync_all_columns", ]; -pub const OPERATION_ADAPTERS: [&str; 8] = [ +pub const OPERATION_ADAPTERS: [&str; 9] = [ "bigquery", "custom", "databricks", @@ -266,6 +266,7 @@ pub const OPERATION_ADAPTERS: [&str; 8] = [ "motherduck", "postgres", "snowflake", + "sqlite", "sqlserver", ]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs index 3d69658261..008b378149 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs @@ -6,8 +6,8 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyString}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_config::command_connection::constants::{ - CONNECTION_DATABASE_KEY, CONNECTION_SOURCE_KEY, DBT_PROFILE_CONNECTION_SOURCE, - DBT_PROFILE_ROUTING_KEYS, DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE, + CONNECTION_SOURCE_KEY, DBT_PROFILE_CONNECTION_SOURCE, DBT_PROFILE_ROUTING_KEYS, + DUCKDB_MEMORY_DATABASE, LOCAL_DATABASE_PATH_KEYS, }; use sqlbuild_config::command_connection::main::missing_connection_error::missing_connection_error; use sqlbuild_config::command_connection::main::snowflake_key_warning::snowflake_key_warning; @@ -105,16 +105,19 @@ pub(crate) fn with_dbt_profile<'py>( Ok(merged) } -/// Make a relative DuckDB database path relative to the project directory. -pub(crate) fn anchor_duckdb_database( +/// Make a relative local database file path (DuckDB, SQLite) relative to the project directory. +pub(crate) fn anchor_local_database_path( config: &Bound<'_, PyDict>, (project_dir, adapter_name): (&Bound<'_, PyAny>, &str), ) -> PyResult<()> { let py: Python<'_> = config.py(); - if adapter_name != DUCKDB_ADAPTER { + let Some((_, path_key)) = LOCAL_DATABASE_PATH_KEYS + .iter() + .find(|(adapter, _)| *adapter == adapter_name) + else { return Ok(()); - } - let Some(database) = config.get_item(CONNECTION_DATABASE_KEY)? else { + }; + let Some(database) = config.get_item(*path_key)? else { return Ok(()); }; if !database.is_instance_of::() { @@ -128,7 +131,7 @@ pub(crate) fn anchor_duckdb_database( return Ok(()); } let anchored: Bound<'_, PyAny> = path(py, project_dir)?.div(&database)?.str()?.into_any(); - config.set_item(CONNECTION_DATABASE_KEY, anchored) + config.set_item(*path_key, anchored) } /// Resolve dbt profiles, relative DuckDB paths, and the command group's warehouse. @@ -187,7 +190,7 @@ fn connection_config<'py>( if let Some(warning) = snowflake_key_warning(&adapter, &keys) { warn(py, &warning)?; } - anchor_duckdb_database(&config, (project_dir, &adapter))?; + anchor_local_database_path(&config, (project_dir, &adapter))?; apply_warehouse( config.as_any(), (adapter_name, project_dir), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs index 5c91cc2559..9a68ae4c3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs @@ -6,7 +6,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::build_command::support::{boolean, keyword_relay, none}; use crate::bindings::_helpers::command_runtime::connection::{ - anchor_duckdb_database, with_dbt_profile, + anchor_local_database_path, with_dbt_profile, }; use crate::bindings::_helpers::command_runtime::warehouses::apply_command_warehouse; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config; @@ -54,7 +54,7 @@ pub(crate) fn connection_config<'py>( (&project_dir, &adapter_name), (&discovered, &py.None().into_bound(py)), )?; - anchor_duckdb_database( + anchor_local_database_path( &config, (&project_dir, &adapter_name.str()?.extract::()?), )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs index 5a059259dc..d035277161 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -631,6 +631,7 @@ const R_MODEL_PLAN_ENTRY: RecordSpec = RecordSpec { FieldSpec::new("on_schema_change", FieldDefault::None), FieldSpec::new("type_enforcement", FieldDefault::False), FieldSpec::new("declared_columns", FieldDefault::EmptyTuple), + FieldSpec::new("inferred_columns", FieldDefault::EmptyTuple), FieldSpec::new("contract_enforced", FieldDefault::False), FieldSpec::new("contract_columns", FieldDefault::EmptyTuple), FieldSpec::new("contract_dynamic_columns", FieldDefault::EmptyTuple), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs index 2294eea2c1..8c085ec34e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/values.rs @@ -52,6 +52,7 @@ pub(super) fn cursor_wrap( }; let start = literal("start")?; let end = literal("end")?; + let operand = cursor_operand(adapter, &column, cursor_type)?; let alias = inject_alias && adapter .call_method0("requires_derived_table_aliases")? @@ -59,7 +60,7 @@ pub(super) fn cursor_wrap( Ok(cursor_subquery( &relation, &CursorFilter { - column: &column, + column: &operand, start: &start, end: &end, lower_inclusive: filtering.lower_bound_inclusive, @@ -69,6 +70,27 @@ pub(super) fn cursor_wrap( )) } +/// The adapter's expression for a cursor column compared with bounds, or the column itself. +fn cursor_operand( + adapter: &Bound<'_, PyAny>, + column: &str, + cursor_type: &Bound<'_, PyAny>, +) -> PyResult { + let py = adapter.py(); + if !adapter.hasattr("render_cursor_operand")? { + return Ok(column.to_owned()); + } + method( + adapter, + "render_cursor_operand", + &[ + ("expression", string(py, column)), + ("cursor_type", cursor_type.clone()), + ], + )? + .extract() +} + /// The cursor filtering of a keyword-argument resolution request. pub(super) fn filtering_of<'py>( arguments: &KeywordArguments<'py>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs index 85509850c1..c028a82e37 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/model_entries.rs @@ -122,6 +122,25 @@ fn schema_columns<'py>( PyTuple::new(py, columns) } +/// The model's typed inferred output columns, which adapters without CTAS types declare. +fn inferred_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = model.py(); + let class: Bound<'py, PyAny> = host::entry(py, AdapterHost::ColumnInfo)?; + let mut columns: Vec> = Vec::new(); + for (name, column_type, _) in model_reads::inferred_rows(model)?.unwrap_or_default() { + if let Some(column_type) = column_type { + columns.push(column_info( + &class, + ( + PyString::new(py, &name).into_any(), + PyString::new(py, &column_type).into_any(), + ), + )?); + } + } + PyTuple::new(py, columns) +} + /// A string or list-of-strings config value as a tuple of strings. fn string_list<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult> { let py: Python<'py> = values.py(); @@ -471,6 +490,7 @@ fn contract_fields<'py>( ) -> PyResult<()> { let py: Python<'py> = model.py(); entry.set_item("declared_columns", schema_columns(model, true)?)?; + entry.set_item("inferred_columns", inferred_columns(model)?)?; entry.set_item( "contract_enforced", mapping_get(values, "contract")?.eq(ENFORCED_CONTRACT)?, diff --git a/fensu.toml b/fensu.toml index 1f7570c574..e0ab3886b7 100644 --- a/fensu.toml +++ b/fensu.toml @@ -155,6 +155,11 @@ rule = "FFR309" path = "src/sqlbuild/adapters/snowflake/__init__.py" reason = "The Snowflake backend intentionally owns behavior through its adapter class rather than function entrypoints." +[[targets.python.rule_exceptions]] +rule = "FFR309" +path = "src/sqlbuild/adapters/sqlite/__init__.py" +reason = "The SQLite backend intentionally owns behavior through its adapter class rather than function entrypoints." + [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/adapters/sqlserver/__init__.py" @@ -215,6 +220,11 @@ rule = "FFR601" path = "src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py" reason = "The Snowflake adapter intentionally keeps overrideable backend behavior on one class." +[[targets.python.rule_exceptions]] +rule = "FFR601" +path = "src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py" +reason = "The SQLite adapter intentionally keeps overrideable backend behavior on one class." + [[targets.python.rule_exceptions]] rule = "FFR601" path = "src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py" diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index 8f0d15cb00..336a3a65dc 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -31,6 +31,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [DuckDB](concepts/adapters/duckdb.md) (`concepts/adapters/duckdb`) - DuckDB adapter configuration for SQLBuild. - [MotherDuck](concepts/adapters/motherduck.md) (`concepts/adapters/motherduck`) - MotherDuck adapter configuration for SQLBuild. - [PostgreSQL](concepts/adapters/postgres.md) (`concepts/adapters/postgres`) - PostgreSQL adapter configuration for SQLBuild. +- [SQLite](concepts/adapters/sqlite.md) (`concepts/adapters/sqlite`) - SQLite adapter configuration, capabilities and limits for SQLBuild. - [BigQuery](concepts/adapters/bigquery.md) (`concepts/adapters/bigquery`) - BigQuery adapter configuration for SQLBuild. - [Databricks](concepts/adapters/databricks.md) (`concepts/adapters/databricks`) - Databricks adapter configuration for SQLBuild. - [SQL Server](concepts/adapters/sqlserver.md) (`concepts/adapters/sqlserver`) - Microsoft SQL Server adapter configuration for SQLBuild. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md index ab4ac91e34..e56fc97d91 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters.md @@ -24,6 +24,7 @@ SQLBuild uses adapters to connect to different database engines. Snowflake is th | [BigQuery](adapters/bigquery.md) | Beta | `sqlbuild[bigquery]` | | [Databricks](adapters/databricks.md) | Beta | `sqlbuild[databricks]` | | [SQL Server](adapters/sqlserver.md) | Beta | `sqlbuild[sqlserver]` | +| [SQLite](adapters/sqlite.md) | Beta | included by default | | ClickHouse | Planned | | | Redshift | Planned | | | Trino | Planned | | @@ -108,7 +109,7 @@ A few `StrictAdapter` methods are optional and have a documented default. `creat - SQLBuild discovers all `.py` files under `adapters/` recursively (excluding `__init__.py` and files starting with `_`) - Each file is scanned for classes that define a string `adapter_name` and subclass `StrictAdapter` (or any of its subclasses like `BaseAdapter` or a built-in adapter) - Adapter names must be unique across all adapter files - duplicates raise an error -- Custom adapter names cannot shadow built-in names (`duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlserver`) +- Custom adapter names cannot shadow built-in names (`duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlite`, `sqlserver`) ### Adapter class hierarchy diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md new file mode 100644 index 0000000000..3489c31f9a --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md @@ -0,0 +1,130 @@ + + +# SQLite + +> SQLite adapter configuration, capabilities and limits for SQLBuild. + +Online: https://sqlbuild.com/docs/concepts/adapters/sqlite/ + +## Contents + +- Connection config +- Schemas +- Concurrency +- Types +- Capabilities +- Scenarios +- Limits + +SQLite uses the `sqlite3` module that ships with Python. No extra installation is needed. +SQLBuild requires SQLite 3.44 or newer; Python builds installed by `uv` bundle a recent version. + +## Connection config + +```toml +adapter = "sqlite" +default_target = "dev" + +[connections.local] +path = "warehouse.sqlite" + +[targets.dev] +connection = "local" +schema = "analytics" +``` + +| Field | Description | +|-------|-------------| +| `path` | Path to the database file, relative to the project directory. Use `:memory:` for an in-memory database. The file and its parent directory are created when missing. | +| `busy_timeout` | Seconds a writer waits for another process to release the file before failing. Default `5`. | +| `read_only` | Open the file read-only. Default `false`. | + +On connect SQLBuild turns on write-ahead logging (`journal_mode=WAL`), sets the busy timeout, +enables foreign keys and sets `legacy_alter_table=ON` so that renames keep views bound by name. +WAL needs a local filesystem; do not put the file on a network share. + +## Schemas + +A SQLite connection is one file. SQLBuild stores each schema's relations in that file under +`__`; the `main` schema keeps bare names. For example, model `orders` in schema +`analytics` is the table `analytics__orders`. Schemas are recorded in `_sqlbuild_schemas`, and +every schema relation SQLBuild creates is recorded with its schema and name in +`_sqlbuild_relations`. Listings read that record rather than splitting names, so a table SQLBuild +did not create belongs to `main` even when its name starts with a schema prefix. + +Two relations that would share one table name, such as `orders` in schema `staging` and +`staging__orders` in `main`, fail compilation with error `A706` before anything is written. + +- Targets and models must not set a `database`. +- Schema names cannot contain `__` or end with `_`. +- Attached databases are not used, because a SQLite view cannot reference another attached file. + +## Concurrency + +Readers run alongside one writer. Within one `sqb` process, writers to the same file queue for +each other, so a long model build does not make others time out. A writer in another process waits +up to `busy_timeout` and then fails with error `A702`, naming the file. Every write that spans +statements, such as a promotion, a merge or a seed load, runs in one `BEGIN IMMEDIATE` transaction +and is all or nothing. + +## Types + +SQLite stores values by type affinity, so SQLBuild keeps column types explicitly: + +- Tables are created with explicit column definitions. A column copied from another relation keeps + its declared type; a computed column gets the type SQLBuild infers at compile time. +- Timestamps and dates are ISO-8601 text. `CAST(x AS TIMESTAMP)` and `CAST(x AS DATE)` do not + produce timestamps in SQLite (they keep only the leading number), so write timestamps as text + literals or use SQLite date functions. +- Cursor windows, watermarks and window deletes compare timestamp cursors as UTC text in the + form `YYYY-MM-DD HH:MM:SS[.ffffff]`, so values written with a `T` separator, fractional seconds + or a UTC offset fall in the right window. A value with an offset belongs to the window of its UTC + time; DuckDB's `TIMESTAMP` instead drops the offset. Comparisons you write yourself against + `__cursor_start()` and `__cursor_end()` compare the stored text as is. +- Booleans are stored as `0` and `1`. +- JSON, arrays and objects are stored as JSON text. +- `DECIMAL(p, s)` means exact canonical text. A column whose `DECIMAL(p, s)` or `NUMERIC(p, s)` + type SQLBuild enforces, through a contract or type enforcement, is stored as fixed-scale text + (for example `12.50`), declared `TEXT DECIMAL(p,s)` and reported as `DECIMAL(p,s)`, whether the + table is created, rebuilt, retyped or gains the column. Every other decimal, including a + numeric source column passed through, an inferred one and a bare `DECIMAL`, is a SQLite number + reported as `NUMERIC`; + numbers are binary floating point above 15 significant digits. Text decimals compare as text, so + cast them, for example `CAST(amount AS REAL) > 10`, before comparing with numbers. + +## Capabilities + +| Feature | Support | +|---------|---------| +| Tables, views, seeds, unit tests, audits | Supported | +| Incremental `append`, `merge`, `delete_insert`, cursor windows | Supported, each in one transaction | +| Microbatch | Sequential batches; `batch_concurrency` greater than 1 is refused | +| Snapshots (timestamp, check, historical) | Supported | +| Schema changes | Add, drop and rename columns in place; type changes rebuild the table and its indexes in one transaction | +| `sqb diff`, `sqb clone` | Supported; clones are physical copies | +| Scenarios, `sqb scenario capture`, `--local` replay | Supported; see below | +| Statement time limits | Connection interrupt | + +## Scenarios + +Scenario fixtures and models are tables in the same file, named +`____sqb_____`, and are dropped when the scenario finishes unless you +pass `--retain`. Concurrent scenarios queue for the file's single writer. + +`sqb scenario test --local` transpiles SQLite SQL to DuckDB. Date functions with modifiers, such +as `datetime(ordered_at, 'start of day')`, have no DuckDB translation and fail locally; use +`date(...)` or `strftime(...)` in models you want to replay locally. + +## Limits + +| Limit | Error | +|-------|-------| +| SQL functions and table functions: SQLite has no `CREATE FUNCTION` | `A705` | +| Grants: SQLite has no privileges; grant settings are ignored | none | +| A `database` on a target or model | `A703` | +| Unknown connection keys | `A703` | +| Ambiguous schema names (`__`, trailing `_`) | `A704` | +| Two relations stored under one table name | `A706`, at compile time | +| SQLite older than 3.44 | `A701` | +| Another process holds the write lock beyond `busy_timeout` | `A702` | +| Concurrent microbatch writes: one file has one writer | refused at plan time | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 6a01ea8476..9be589d4d1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -91,7 +91,7 @@ materialized = "view" | Field | Description | |-------|-------------| | `name` | Project name. Used in fingerprint tracking and manifest generation. | -| `adapter` | Database adapter: `duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, or `sqlserver`. See [Adapters](adapters.md). | +| `adapter` | Database adapter: `duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlite`, or `sqlserver`. See [Adapters](adapters.md). | ### Optional fields @@ -418,6 +418,7 @@ Every built-in adapter enforces the limit: | BigQuery | Job timeout (`job_timeout_ms`) on each query job | `jobs.cancel` (asynchronous) | about 24 days | | Databricks | `STATEMENT_TIMEOUT` session setting, capped at 48 hours | Cursor cancel (asynchronous) | about 24 days | | SQL Server | none | Driver cancel | about 24 days | +| SQLite | none | Connection interrupt | about 24 days | BigQuery and Databricks stop cancelled work asynchronously, so SQLBuild reports their cancellation as requested but unconfirmed. SQL Server has no per-statement limit; the driver's query timeout is @@ -454,6 +455,7 @@ finished. Where a warehouse reports queueing separately, queued time is excluded | BigQuery | `jobs.cancel` at the limit; no job timeout, because a job timeout counts from job creation | Excluded: the clock stops while a job is pending | | Databricks | `STATEMENT_TIMEOUT` session setting and cursor cancel | May count | | SQL Server | Driver cancel at the limit | Lock waits count | +| SQLite | Connection interrupt at the limit | Waits for the write lock count | When a model contains unit tests or nested work with its own limit, the earliest limit applies and cancels whatever is running under it. After a timeout interrupted a statement, SQLBuild checks the diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index a95aa3d592..dca9789df7 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -15705,6 +15705,7 @@ class ModelPlanEntry(NativeRecord): on_schema_change: OnSchemaChange | None type_enforcement: bool declared_columns: tuple[ColumnInfo, ...] + inferred_columns: tuple[ColumnInfo, ...] contract_enforced: bool contract_columns: tuple[ColumnInfo, ...] contract_dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] @@ -15788,6 +15789,7 @@ class ModelPlanEntry(NativeRecord): on_schema_change: OnSchemaChange | None = ..., type_enforcement: bool = ..., declared_columns: tuple[ColumnInfo, ...] = ..., + inferred_columns: tuple[ColumnInfo, ...] = ..., contract_enforced: bool = ..., contract_columns: tuple[ColumnInfo, ...] = ..., contract_dynamic_columns: tuple[SchemaDynamicColumnFamily, ...] = ..., diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index a089443596..fafb71c79f 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -1555,7 +1555,11 @@ def _inspect_row_diff_coverage( ) projections: str = "COUNT(*)" if cursor_column is not None: - projections += f", MIN({cursor_column}), MAX({cursor_column})" + operand: str = self.render_cursor_operand( + expression=cursor_column, + cursor_type=None if start_cursor is None else start_cursor.kind, + ) + projections += f", MIN({operand}), MAX({operand})" query: str = f"SELECT {projections} FROM {relation}" if cursor_filter: query += f" WHERE {cursor_filter}" @@ -1691,7 +1695,12 @@ def _render_row_diff_key_hash( value_sql: str = f"CAST({alias}.{key} AS VARCHAR)" key_parts.append(f"CONCAT(LENGTH({value_sql}), ':', {value_sql})") payload_parts: str = ", ".join((f"'{seed}'", *key_parts)) - return f"MD5(CONCAT_WS('|', {payload_parts}))" + return self._render_row_diff_digest(f"CONCAT_WS('|', {payload_parts})") + + def _render_row_diff_digest(self, payload_sql: str) -> str: + """Render the hash function applied to one canonical key payload.""" + + return f"MD5({payload_sql})" def sample_unequal_rows( self, @@ -2537,5 +2546,31 @@ def dimensions(node: SqlValue) -> tuple[int, ...] | None: dimensions(value) +def _render_explicit_source_relation_cast_subquery( + *, + source_relation: str, + cast_projections: tuple[str, ...], + cast_column_names: tuple[str, ...], + warehouse_column_names: tuple[str, ...], + all_columns_cast: bool, + render_identifier: Callable[[str], str], +) -> str: + """Render source casts by listing pass-through columns, for engines without star exclusion.""" + + cast_clause: str = ", ".join(cast_projections) + if all_columns_cast: + return f"(SELECT {cast_clause} FROM {source_relation})" + projection_names: set[str] = set(cast_column_names) + passthrough_clause: str = ", ".join( + render_identifier(column) + for column in warehouse_column_names + if column not in projection_names + ) + projection_clause: str = ( + f"{passthrough_clause}, {cast_clause}" if passthrough_clause else cast_clause + ) + return f"(SELECT {projection_clause} FROM {source_relation})" + + def _quote_sql_string(value: str) -> str: return "'" + value.replace("'", "''") + "'" diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index ac1bf07a68..c1164d99c3 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -116,6 +116,12 @@ def drop_sql_test_output(self, *, connection: Any, relation: str) -> None: del connection, relation + def render_cursor_operand(self, *, expression: str, cursor_type: str | None) -> str: + """Optional: the cursor-column expression compared with cursor bounds and aggregated.""" + + del cursor_type + return expression + @abstractmethod def lists_relations_authoritatively(self, *, database: str | None, schema: str) -> bool: """Return whether ``list_relations`` reports every relation in this scope.""" diff --git a/src/sqlbuild/adapter/contract/types.py b/src/sqlbuild/adapter/contract/types.py index b1c018e69e..0049d7c775 100644 --- a/src/sqlbuild/adapter/contract/types.py +++ b/src/sqlbuild/adapter/contract/types.py @@ -24,6 +24,7 @@ class BuiltinAdapter(StrEnum): DATABRICKS = "databricks" POSTGRES = "postgres" SQLSERVER = "sqlserver" + SQLITE = "sqlite" class RelationReadStatus(StrEnum): diff --git a/src/sqlbuild/adapter/discovery/constants.py b/src/sqlbuild/adapter/discovery/constants.py index 4c24c446ef..9a816bfedb 100644 --- a/src/sqlbuild/adapter/discovery/constants.py +++ b/src/sqlbuild/adapter/discovery/constants.py @@ -16,6 +16,7 @@ "DatabricksAdapter", ), "postgres": ("sqlbuild.adapters.postgres.classes.postgres_adapter", "PostgresAdapter"), + "sqlite": ("sqlbuild.adapters.sqlite.classes.sqlite_adapter", "SqliteAdapter"), "sqlserver": ( "sqlbuild.adapters.sqlserver.classes.sqlserver_adapter", "SqlServerAdapter", diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 0d7ab58d6e..2ccd006066 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -24,6 +24,7 @@ _encode_typed_json, _quote_sql_string, _render_ansi_typed_scalar, + _render_explicit_source_relation_cast_subquery, _render_typed_value_list, _validate_rectangular_typed_array, ) @@ -2022,20 +2023,14 @@ def _render_source_relation_cast_subquery_with_columns( ) -> str: """Render Postgres source casts without relying on unsupported star exclusion.""" - cast_clause: str = ", ".join(cast_projections) - if all_columns_cast: - return f"(SELECT {cast_clause} FROM {source_relation})" - projection_names: set[str] = set(cast_column_names) - passthrough_columns: tuple[str, ...] = tuple( - column for column in warehouse_column_names if column not in projection_names - ) - passthrough_clause: str = ", ".join( - self.render_identifier(name) for name in passthrough_columns + return _render_explicit_source_relation_cast_subquery( + source_relation=source_relation, + cast_projections=cast_projections, + cast_column_names=cast_column_names, + warehouse_column_names=warehouse_column_names, + all_columns_cast=all_columns_cast, + render_identifier=self.render_identifier, ) - projection_clause: str = cast_clause - if passthrough_clause: - projection_clause = f"{passthrough_clause}, {cast_clause}" - return f"(SELECT {projection_clause} FROM {source_relation})" def requires_derived_table_aliases(self) -> bool: """Postgres does not require aliases for derived table factors.""" diff --git a/src/sqlbuild/adapters/sqlite/__init__.py b/src/sqlbuild/adapters/sqlite/__init__.py new file mode 100644 index 0000000000..2de12aa9bb --- /dev/null +++ b/src/sqlbuild/adapters/sqlite/__init__.py @@ -0,0 +1 @@ +"""SQLite adapter.""" diff --git a/src/sqlbuild/adapters/sqlite/_helpers/__init__.py b/src/sqlbuild/adapters/sqlite/_helpers/__init__.py new file mode 100644 index 0000000000..a47e133b33 --- /dev/null +++ b/src/sqlbuild/adapters/sqlite/_helpers/__init__.py @@ -0,0 +1 @@ +"""SQLite adapter helpers.""" diff --git a/src/sqlbuild/adapters/sqlite/_helpers/catalog.py b/src/sqlbuild/adapters/sqlite/_helpers/catalog.py new file mode 100644 index 0000000000..39c41b383d --- /dev/null +++ b/src/sqlbuild/adapters/sqlite/_helpers/catalog.py @@ -0,0 +1,195 @@ +"""SQLite catalog reads: one file holds every schema as `schema__name` relations.""" + +from __future__ import annotations + +import re +from typing import Any + +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapters.sqlite.constants import ( + BASE_TABLE_RELATION_TYPE, + DECIMAL_TEXT_TYPE_PREFIX, + DECIMAL_TYPE_NAMES, + IDENTIFIER_QUOTE, + INTERNAL_TABLE_NAMES, + INVALID_SCHEMA_NAME_CODE, + MAIN_SCHEMA, + MAIN_SCHEMA_PREFIX, + NUMBER_DECIMAL_REPORTED_TYPE, + NUMBER_DECIMAL_TYPE_PATTERN, + RELATION_REGISTRY_TABLE, + SCHEMA_NAME_SEPARATOR, + SCHEMA_REGISTRY_TABLE, + SQLITE_INTERNAL_PREFIX, + SQLITE_VIEW_TYPE, + TYPEOF_TYPE_NAMES, + UNTYPED_COLUMN_TYPE, + VIEW_BODY_PATTERN, + VIEW_RELATION_TYPE, +) + +_NUMBER_DECIMAL_TYPE: re.Pattern[str] = re.compile(NUMBER_DECIMAL_TYPE_PATTERN, re.IGNORECASE) +_VIEW_BODY: re.Pattern[str] = re.compile(VIEW_BODY_PATTERN, re.IGNORECASE | re.DOTALL) + + +def validate_schema_name(schema: str) -> None: + """Reject schema names that would make `schema__name` relations ambiguous.""" + + if SCHEMA_NAME_SEPARATOR in schema or schema.endswith("_") or not schema: + raise AdapterUserError( + message=( + f"SQLite schema '{schema}' cannot contain '{SCHEMA_NAME_SEPARATOR}' or end with " + "'_', because SQLite stores schema relations as '__'" + ), + code=INVALID_SCHEMA_NAME_CODE, + ) + + +def physical_name(*, schema: str | None, name: str) -> str: + """Return the table name one logical relation is stored under.""" + + if schema is None or schema.lower() == MAIN_SCHEMA: + return name + validate_schema_name(schema) + return f"{schema}{SCHEMA_NAME_SEPARATOR}{name}" + + +def unquote_relation(relation: str) -> str: + """Return the physical table name from a rendered relation reference.""" + + text: str = relation.strip() + if text.lower().startswith(MAIN_SCHEMA_PREFIX): + text = text[len(MAIN_SCHEMA_PREFIX) :] + quoted_length: int = 2 + if len(text) >= quoted_length and text[0] == text[-1] == IDENTIFIER_QUOTE: + return text[1:-1].replace('""', '"') + return text + + +def registered_schemas(connection: Any) -> frozenset[str]: + """Return schemas SQLBuild created, which decide how `a__b` names split.""" + + exists: object = connection.execute_with( + sql="SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ?", + parameters=(SCHEMA_REGISTRY_TABLE,), + ).fetchone() + if exists is None: + return frozenset() + rows: list[tuple[Any, ...]] = connection.execute( + f"SELECT schema_name FROM {SCHEMA_REGISTRY_TABLE}" + ).fetchall() + return frozenset(str(row[0]).lower() for row in rows) + + +def registry_exists(connection: Any) -> bool: + """Return whether the relation registry table exists in this file.""" + + row: object = connection.execute_with( + sql="SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = ?", + parameters=(RELATION_REGISTRY_TABLE,), + ).fetchone() + return row is not None + + +def registry_locations(connection: Any) -> dict[str, tuple[str, str]]: + """Return the registered (schema, name) of each schema relation by lowered table name.""" + + if not registry_exists(connection): + return {} + rows: list[tuple[Any, ...]] = connection.execute( + f"SELECT physical_name, schema_name, relation_name FROM {RELATION_REGISTRY_TABLE}" + ).fetchall() + return {str(row[0]).lower(): (str(row[1]), str(row[2])) for row in rows} + + +def logical_location(*, table_name: str, locations: dict[str, tuple[str, str]]) -> tuple[str, str]: + """Return a table's registered (schema, name); unregistered tables belong to `main`.""" + + return locations.get(table_name.lower(), (MAIN_SCHEMA, table_name)) + + +def catalog_relations(connection: Any) -> tuple[tuple[str, str, str, str], ...]: + """Return (schema, name, physical name, relation type) for every user table and view.""" + + locations: dict[str, tuple[str, str]] = registry_locations(connection) + rows: list[tuple[Any, ...]] = connection.execute( + "SELECT name, type FROM sqlite_schema WHERE type IN ('table', 'view') ORDER BY name" + ).fetchall() + relations: list[tuple[str, str, str, str]] = [] + row: tuple[Any, ...] + for row in rows: + table_name: str = str(row[0]) + if table_name.lower().startswith(SQLITE_INTERNAL_PREFIX) or ( + table_name in INTERNAL_TABLE_NAMES + ): + continue + schema: str + name: str + schema, name = logical_location(table_name=table_name, locations=locations) + relation_type: str = ( + VIEW_RELATION_TYPE if row[1] == SQLITE_VIEW_TYPE else BASE_TABLE_RELATION_TYPE + ) + relations.append((schema, name, table_name, relation_type)) + return tuple(relations) + + +def relation_columns(*, connection: Any, table_name: str) -> tuple[ColumnInfo, ...]: + """Return declared column types, sampling stored values for untyped columns.""" + + rows: list[tuple[Any, ...]] = connection.execute( + f"PRAGMA table_xinfo({quote_identifier(table_name)})" + ).fetchall() + visible: list[tuple[str, str]] = [ + (str(row[1]), str(row[2] or "")) for row in rows if not row[6] + ] + untyped: tuple[str, ...] = tuple(name for name, declared in visible if not declared) + sampled: dict[str, str] = _sample_value_types( + connection=connection, table_name=table_name, columns=untyped + ) + return tuple( + ColumnInfo(name=name, type=reported_type(declared) if declared else sampled[name]) + for name, declared in visible + ) + + +def reported_type(declared: str) -> str: + """Report `DECIMAL(p,s)` only for canonical decimal text; other decimals are numbers.""" + + upper: str = declared.upper() + if upper.startswith(DECIMAL_TEXT_TYPE_PREFIX) and upper[ + len(DECIMAL_TEXT_TYPE_PREFIX) : + ].startswith(DECIMAL_TYPE_NAMES): + return declared[len(DECIMAL_TEXT_TYPE_PREFIX) :] + if _NUMBER_DECIMAL_TYPE.match(declared) is not None: + return NUMBER_DECIMAL_REPORTED_TYPE + return declared + + +def view_body(definition: str) -> str: + """Return the SELECT of a stored CREATE VIEW statement.""" + + match: re.Match[str] | None = _VIEW_BODY.match(definition) + return definition if match is None else match.group(1).strip() + + +def quote_identifier(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + +def _sample_value_types( + *, connection: Any, table_name: str, columns: tuple[str, ...] +) -> dict[str, str]: + if not columns: + return {} + relation: str = quote_identifier(table_name) + projections: str = ", ".join( + f"(SELECT typeof({quote_identifier(column)}) FROM {relation} " + f"WHERE {quote_identifier(column)} IS NOT NULL LIMIT 1)" + for column in columns + ) + row: tuple[Any, ...] = connection.execute(f"SELECT {projections}").fetchone() + return { + column: TYPEOF_TYPE_NAMES.get(str(value), UNTYPED_COLUMN_TYPE) + for column, value in zip(columns, row, strict=True) + } diff --git a/src/sqlbuild/adapters/sqlite/_helpers/relations.py b/src/sqlbuild/adapters/sqlite/_helpers/relations.py new file mode 100644 index 0000000000..ca3e9caf79 --- /dev/null +++ b/src/sqlbuild/adapters/sqlite/_helpers/relations.py @@ -0,0 +1,109 @@ +"""Relation registry upkeep and view renames, applied as the adapter executes DDL.""" + +from __future__ import annotations + +import re +from collections.abc import Callable +from typing import Any + +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapters.sqlite._helpers.catalog import registry_exists, unquote_relation, view_body +from sqlbuild.adapters.sqlite.constants import MAIN_SCHEMA, RELATION_REGISTRY_TABLE + +_NAME: str = r"((?:main\.)?(?:\"(?:[^\"]|\"\")+\"|[^\s(;\"]+))" +_CREATE: re.Pattern[str] = re.compile( + r"^\s*CREATE\s+(?:UNIQUE\s+)?(TEMP\s+|TEMPORARY\s+)?(?:TABLE|VIEW)\s+(?:IF\s+NOT\s+EXISTS\s+)?" + + _NAME, + re.IGNORECASE, +) +_DROP: re.Pattern[str] = re.compile( + r"^\s*DROP\s+(?:TABLE|VIEW)\s+(?:IF\s+EXISTS\s+)?" + _NAME, re.IGNORECASE +) +_RENAME_TABLE: re.Pattern[str] = re.compile( + r"^\s*ALTER\s+TABLE\s+" + _NAME + r"\s+RENAME\s+TO\s+" + _NAME + r"\s*;?\s*$", + re.IGNORECASE, +) +_RENAME_VIEW: re.Pattern[str] = re.compile( + r"^\s*ALTER\s+VIEW\s+" + _NAME + r"\s+RENAME\s+TO\s+" + _NAME + r"\s*;?\s*$", + re.IGNORECASE, +) +_TEMP_PREFIX: str = "temp." + + +def split_rename_view(sql: str) -> tuple[str, str] | None: + """Return (origin, destination) of an `ALTER VIEW ... RENAME TO ...` statement.""" + + match: re.Match[str] | None = _RENAME_VIEW.match(sql) + if match is None: + return None + return match.group(1), match.group(2) + + +def rename_view(*, connection: Any, origin: str, destination: str) -> Any: + """Recreate a view under its new name, since SQLite cannot alter a view.""" + + row: tuple[Any, ...] | None = connection.execute_with( + sql="SELECT sql FROM sqlite_schema WHERE type = 'view' AND lower(name) = lower(?)", + parameters=(unquote_relation(origin),), + ).fetchone() + if row is None: + raise AdapterUserError(message=f"SQLite view '{origin}' does not exist") + connection.execute(f"DROP VIEW {origin}") + return connection.execute(f"CREATE VIEW {destination} AS {view_body(str(row[0]))}") + + +def relation_changes(sql: str) -> tuple[tuple[str, ...], tuple[str, ...]]: + """Return the physical names one statement removes and creates.""" + + rename: re.Match[str] | None = _RENAME_TABLE.match(sql) or _RENAME_VIEW.match(sql) + if rename is not None: + return (unquote_relation(rename.group(1)),), (unquote_relation(rename.group(2)),) + create: re.Match[str] | None = _CREATE.match(sql) + if create is not None and create.group(1) is None and not _is_temp(create.group(2)): + return (), (unquote_relation(create.group(2)),) + drop: re.Match[str] | None = _DROP.match(sql) + if drop is not None and not _is_temp(drop.group(1)): + return (unquote_relation(drop.group(1)),), () + return (), () + + +def record_relation_changes( + *, + connection: Any, + removed: tuple[str, ...], + added: tuple[str, ...], + locate: Callable[[str], tuple[str, str] | None], +) -> None: + """Keep the registry's (schema, name) of every schema relation SQLBuild created.""" + + rows: list[tuple[str, str, str]] = [] + physical: str + for physical in added: + location: tuple[str, str] | None = locate(physical) + if location is not None and location[0].lower() != MAIN_SCHEMA: + rows.append((physical, location[0], location[1])) + if rows: + connection.execute( + f"CREATE TABLE IF NOT EXISTS {RELATION_REGISTRY_TABLE} (physical_name TEXT PRIMARY " + "KEY COLLATE NOCASE, schema_name TEXT NOT NULL, relation_name TEXT NOT NULL)" + ) + elif not registry_exists(connection): + return + for physical in (*removed, *added): + connection.execute_with( + sql=f"DELETE FROM {RELATION_REGISTRY_TABLE} WHERE physical_name = ?", + parameters=(physical,), + ) + row: tuple[str, str, str] + for row in rows: + connection.execute_with( + sql=( + f"INSERT INTO {RELATION_REGISTRY_TABLE} (physical_name, schema_name, " + "relation_name) VALUES (?, ?, ?)" + ), + parameters=row, + ) + + +def _is_temp(name: str) -> bool: + return name.lower().startswith(_TEMP_PREFIX) diff --git a/src/sqlbuild/adapters/sqlite/_helpers/statements.py b/src/sqlbuild/adapters/sqlite/_helpers/statements.py new file mode 100644 index 0000000000..eae85654d4 --- /dev/null +++ b/src/sqlbuild/adapters/sqlite/_helpers/statements.py @@ -0,0 +1,17 @@ +"""SQL statement classification for the SQLite write gate.""" + +from __future__ import annotations + + +def leading_keyword(sql: str) -> str: + """Return a statement's first keyword, skipping leading comments.""" + + text: str = sql.lstrip() + while text.startswith(("--", "/*")): + text = ( + text.partition("\n")[2].lstrip() + if text.startswith("--") + else text.partition("*/")[2].lstrip() + ) + word: str = text.split(None, 1)[0] if text else "" + return word.rstrip(";(").upper() diff --git a/src/sqlbuild/adapters/sqlite/_helpers/tables.py b/src/sqlbuild/adapters/sqlite/_helpers/tables.py new file mode 100644 index 0000000000..7ef84946ed --- /dev/null +++ b/src/sqlbuild/adapters/sqlite/_helpers/tables.py @@ -0,0 +1,89 @@ +"""Table creation that keeps declared column types, which SQLite's CTAS discards.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from typing import Any +from uuid import uuid4 + +from sqlbuild.adapters.sqlite._helpers.catalog import quote_identifier +from sqlbuild.adapters.sqlite.constants import DECIMAL_TEXT_TYPE_PREFIX, EXACT_DECIMAL_TYPE_PATTERN + +_CREATE_TABLE_AS: re.Pattern[str] = re.compile( + r"^\s*CREATE\s+TABLE\s+(?!IF\s)(?!TEMP\s)(?!TEMPORARY\s)" + r"(\"(?:[^\"]|\"\")+\"|[^\s(]+)\s+AS\s+(.+)$", + re.IGNORECASE | re.DOTALL, +) +_DECIMAL_TYPE: re.Pattern[str] = re.compile(EXACT_DECIMAL_TYPE_PATTERN, re.IGNORECASE) + + +def split_create_table_as(sql: str) -> tuple[str, str] | None: + """Return (destination, select) when one statement is a plain CREATE TABLE AS.""" + + match: re.Match[str] | None = _CREATE_TABLE_AS.match(sql) + if match is None: + return None + return match.group(1), match.group(2) + + +def create_table_typed( + *, + connection: Any, + destination: str, + sql: str, + column_types: Mapping[str, str], + inferred_types: Mapping[str, str], +) -> Any: + """Create `destination` with passed-through, inferred or enforced types, atomically.""" + + probe: str = f"_sqlbuild_probe_{uuid4().hex}" + owns_transaction: bool = not connection.in_transaction() + connection.execute("BEGIN IMMEDIATE" if owns_transaction else f"SAVEPOINT {probe}") + try: + connection.execute(f"CREATE TEMP VIEW {probe} AS {sql}") + rows: list[tuple[Any, ...]] = connection.execute( + f"PRAGMA temp.table_xinfo({probe})" + ).fetchall() + connection.execute(f"DROP VIEW temp.{probe}") + column_sql: str = ", ".join( + _column_definition( + name=str(row[1]), + declared=str(row[2] or "") or inferred_types.get(str(row[1]).lower(), ""), + column_types=column_types, + ) + for row in rows + ) + connection.execute(f"CREATE TABLE {destination} ({column_sql})") + cursor: Any = connection.execute(f"INSERT INTO {destination} SELECT * FROM ({sql})") + except BaseException: + if connection.in_transaction(): + connection.execute("ROLLBACK" if owns_transaction else f"ROLLBACK TO {probe}") + if not owns_transaction: + connection.execute(f"RELEASE {probe}") + raise + connection.execute("COMMIT" if owns_transaction else f"RELEASE {probe}") + return cursor + + +def storage_type(declared: str) -> str: + """Return the type a column is declared with; `DECIMAL(p,s)` is stored as canonical text.""" + + if _DECIMAL_TYPE.match(declared) is None: + return declared + return f"{DECIMAL_TEXT_TYPE_PREFIX}{declared.strip()}" + + +def decimal_scale(target_type: str) -> int | None: + """Return a `DECIMAL(p,s)` type's scale, or None for any other type, including bare DECIMAL.""" + + match: re.Match[str] | None = _DECIMAL_TYPE.match(target_type) + if match is None: + return None + return int(match.group(2) or 0) + + +def _column_definition(*, name: str, declared: str, column_types: Mapping[str, str]) -> str: + override: str | None = column_types.get(name.lower()) + column_type: str = storage_type(override) if override is not None else declared + return f"{quote_identifier(name)} {column_type}".rstrip() diff --git a/src/sqlbuild/adapters/sqlite/_helpers/timestamps.py b/src/sqlbuild/adapters/sqlite/_helpers/timestamps.py new file mode 100644 index 0000000000..1e246012e1 --- /dev/null +++ b/src/sqlbuild/adapters/sqlite/_helpers/timestamps.py @@ -0,0 +1,27 @@ +"""SQLite timestamp text: the `YYYY-MM-DD HH:MM:SS` form SQLite's date functions produce.""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime + +from sqlbuild.adapters.sqlite.constants import DATE_TIME_PATTERN, TIMESTAMP_TEXT_FORMAT + +_DATE_TIME: re.Pattern[str] = re.compile(DATE_TIME_PATTERN) + + +def canonical_timestamp_text(value: str) -> str: + """Rewrite an ISO-8601 date-time so text comparison orders it like SQLite timestamps.""" + + if _DATE_TIME.match(value) is None: + return value + try: + parsed: datetime = datetime.fromisoformat(value) + except ValueError: + return value + if parsed.tzinfo is not None: + parsed = parsed.astimezone(UTC).replace(tzinfo=None) + text: str = parsed.strftime(TIMESTAMP_TEXT_FORMAT) + if parsed.microsecond: + text += f".{parsed.microsecond:06d}".rstrip("0") + return text diff --git a/src/sqlbuild/adapters/sqlite/classes/__init__.py b/src/sqlbuild/adapters/sqlite/classes/__init__.py new file mode 100644 index 0000000000..7441cf6e7c --- /dev/null +++ b/src/sqlbuild/adapters/sqlite/classes/__init__.py @@ -0,0 +1 @@ +"""SQLite adapter classes.""" diff --git a/src/sqlbuild/adapters/sqlite/classes/logical_relations.py b/src/sqlbuild/adapters/sqlite/classes/logical_relations.py new file mode 100644 index 0000000000..6201383d6d --- /dev/null +++ b/src/sqlbuild/adapters/sqlite/classes/logical_relations.py @@ -0,0 +1,43 @@ +"""The logical relation behind each SQLite table name this adapter rendered.""" + +from __future__ import annotations + +import threading + +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapters.sqlite.constants import RELATION_NAME_COLLISION_CODE + + +class LogicalRelations: + """Map table names to the (schema, name) they were rendered for, refusing collisions.""" + + def __init__(self) -> None: + self._by_physical: dict[str, tuple[str, str]] = {} + self._guard: threading.Lock = threading.Lock() + + def remember(self, *, schema: str, name: str, physical: str) -> None: + """Record one rendering; a second relation stored under the same table name is refused.""" + + with self._guard: + existing: tuple[str, str] = self._by_physical.setdefault( + physical.lower(), (schema, name) + ) + if (existing[0].lower(), existing[1].lower()) == (schema.lower(), name.lower()): + return + raise AdapterUserError( + message=( + f"SQLite relations '{existing[0]}.{existing[1]}' and '{schema}.{name}' would both " + f"be stored as table '{physical}'" + ), + code=RELATION_NAME_COLLISION_CODE, + help=( + "SQLite stores schema relations as '__' in one file. Rename one of " + "the relations, or move it to a schema whose name it does not start with." + ), + ) + + def lookup(self, physical: str) -> tuple[str, str] | None: + """Return the rendered (schema, name) of one physical table name, if known.""" + + with self._guard: + return self._by_physical.get(physical.lower()) diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py new file mode 100644 index 0000000000..330cb97958 --- /dev/null +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -0,0 +1,2724 @@ +"""SQLite adapter implementation.""" + +from __future__ import annotations + +import csv +import hashlib +import json +import logging +import re +import sqlite3 +from collections.abc import Mapping +from contextvars import ContextVar, Token +from datetime import date, datetime +from decimal import Decimal +from pathlib import Path +from types import MappingProxyType +from typing import Any, ClassVar +from urllib.parse import quote + +from sqlbuild._native import ( + MICROBATCH_END_SENTINEL, + MICROBATCH_START_SENTINEL, + SeedCsvSettings, + SourceFreshnessRecord, + SqlValue, + render_insert_source_freshness_records_sql, +) +from sqlbuild.adapter.contract.classes.base_adapter import ( + BaseAdapter, + _encode_typed_json, + _quote_sql_string, + _render_ansi_typed_scalar, + _render_explicit_source_relation_cast_subquery, + _render_typed_value_list, +) +from sqlbuild.adapter.contract.classes.historical_check_snapshot_sql import ( + HistoricalCheckSnapshotSql, +) +from sqlbuild.adapter.contract.classes.historical_snapshot_sql import ( + render_is_distinct_from, +) +from sqlbuild.adapter.contract.classes.historical_timestamp_snapshot_sql import ( + HistoricalTimestampSnapshotSql, +) +from sqlbuild.adapter.contract.classes.microbatch import MicrobatchMixin +from sqlbuild.adapter.contract.classes.snapshot_sql import SnapshotSql +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.classes.unkeyed_diff import UnkeyedDiffMixin +from sqlbuild.adapter.contract.constants import ( + CANCELLABLE_DURATION_LIMIT_SECONDS, + DIFF_LEFT_SIDE, + DIFF_RIGHT_SIDE, +) +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.main.normalize_seed_csv_value import normalize_seed_csv_value +from sqlbuild.adapter.contract.main.run_relation_read_probe import run_relation_read_probe +from sqlbuild.adapter.contract.main.same_view_definition import same_view_definition +from sqlbuild.adapter.contract.models import ( + ColumnInfo, + CursorValue, + ExpressionInferenceProfile, + FunctionDefinition, + FunctionInfo, + MigrationStagePlan, + QueryResult, + RelationGrant, + RelationInfo, + RelationReadProbe, + RelationRowCountEstimate, + RowDiffColumnResult, + RowDiffCoverage, + RowDiffPreparedRelations, + RowDiffResult, + RowDiffSampleCell, + RowDiffSampleRow, + RowDiffSampling, + RowDiffTolerance, + RowDiffTolerances, + SchemaDiffResult, + SnapshotChangeTarget, + SnapshotSqlDialect, + TableFreshnessMetadata, + TableFreshnessRequest, +) +from sqlbuild.adapter.contract.types import ( + BuiltinAdapter, + CursorKind, + FrameworkType, + HistoricalSnapshotCloseStyle, + HistoricalSnapshotInsertStyle, + LoaderLogicalType, + MigrationTransfer, + PromotionStrategy, + RelationReadStatus, + SnapshotLatestVersionStyle, + SnapshotUpdateStyle, + StatementSizeLimit, + TablePromotionMode, +) +from sqlbuild.adapter.relations.main.get_columns_for_relations import ( + get_columns_for_relations_bulk, +) +from sqlbuild.adapter.type_system.main.normalize_numeric_family import normalize_numeric_family +from sqlbuild.adapter.type_system.main.types_equal import types_equal +from sqlbuild.adapters.sqlite._helpers.catalog import ( + catalog_relations, + logical_location, + physical_name, + quote_identifier, + registered_schemas, + registry_locations, + relation_columns, + unquote_relation, + validate_schema_name, + view_body, +) +from sqlbuild.adapters.sqlite._helpers.relations import ( + record_relation_changes, + relation_changes, + rename_view, + split_rename_view, +) +from sqlbuild.adapters.sqlite._helpers.tables import ( + create_table_typed, + decimal_scale, + split_create_table_as, + storage_type, +) +from sqlbuild.adapters.sqlite._helpers.timestamps import canonical_timestamp_text +from sqlbuild.adapters.sqlite.classes.logical_relations import LogicalRelations +from sqlbuild.adapters.sqlite.classes.sqlite_connection import SqliteConnection +from sqlbuild.adapters.sqlite.constants import ( + BOOLEAN_TYPE_MARKER, + CONNECTION_KEYS, + CURSOR_TEXT_FUNCTION_NAME, + DEFAULT_BUSY_TIMEOUT_SECONDS, + HASH_FUNCTION_NAME, + INSERT_KEYWORD, + INVALID_CONNECTION_CODE, + MAIN_SCHEMA, + MEMORY_DATABASE_PATH, + MINIMUM_SQLITE_VERSION, + MISSING_RELATION_ERROR_FRAGMENT, + SCHEMA_REGISTRY_TABLE, + SQLITE_TABLE_TYPE, + UNSUPPORTED_FEATURE_CODE, + VERSION_TOO_OLD_CODE, +) +from sqlbuild.compiler.compile.types import FunctionLanguage +from sqlbuild.diagnostics.main.log_sql import log_sql +from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS + +_ENFORCED_COLUMN_TYPES: ContextVar[Mapping[str, str]] = ContextVar( + "sqlite_enforced_column_types", default=MappingProxyType({}) +) +_INFERRED_COLUMN_TYPES: ContextVar[Mapping[str, str]] = ContextVar( + "sqlite_inferred_column_types", default=MappingProxyType({}) +) +_TRUE_SEED_VALUES: frozenset[str] = frozenset({"true", "t", "yes", "y", "1"}) +_FALSE_SEED_VALUES: frozenset[str] = frozenset({"false", "f", "no", "n", "0"}) + + +class SqliteAdapter(MicrobatchMixin, UnkeyedDiffMixin, BaseAdapter): + """SQLite adapter: one database file, SQLBuild schemas stored as `schema__name` tables.""" + + adapter_name: ClassVar[str] = BuiltinAdapter.SQLITE.value + execution_duration_limit_seconds: ClassVar[int | None] = CANCELLABLE_DURATION_LIMIT_SECONDS + sql_analysis_dialect_name: ClassVar[str | None] = "sqlite" + max_identifier_length: ClassVar[int] = 1024 + relation_grants_supported: ClassVar[bool] = False + allows_implicit_managed_write_schema: ClassVar[bool] = True + _snapshot_sql_dialect: ClassVar[SnapshotSqlDialect] = SnapshotSqlDialect( + timestamp_type="TEXT", + distinct_condition=render_is_distinct_from, + update_style=SnapshotUpdateStyle.UPDATE_FROM, + latest_version=SnapshotLatestVersionStyle.DERIVED_TABLE, + historical_close=HistoricalSnapshotCloseStyle.CORRELATED, + historical_insert=HistoricalSnapshotInsertStyle.WITH_INSERT, + ) + + def __init__(self) -> None: + self._logical_relations: LogicalRelations = LogicalRelations() + + def connect(self, config: dict[str, Any]) -> SqliteConnection: + if sqlite3.sqlite_version_info < MINIMUM_SQLITE_VERSION: + minimum: str = ".".join(str(part) for part in MINIMUM_SQLITE_VERSION) + raise AdapterUserError( + message=( + f"SQLite adapter requires SQLite {minimum} or newer; this Python links " + f"SQLite {sqlite3.sqlite_version}" + ), + code=VERSION_TOO_OLD_CODE, + help="Use a Python build that bundles a newer SQLite, such as one installed by uv.", + ) + unknown: tuple[str, ...] = tuple(sorted(set(config) - CONNECTION_KEYS)) + if unknown: + raise AdapterUserError( + message=f"SQLite connection does not accept: {', '.join(unknown)}", + code=INVALID_CONNECTION_CODE, + help="SQLite connections accept 'path', 'busy_timeout' and 'read_only'.", + ) + path: str = str(config.get("path", MEMORY_DATABASE_PATH)) + busy_timeout: float = float(config.get("busy_timeout", DEFAULT_BUSY_TIMEOUT_SECONDS)) + read_only: bool = bool(config.get("read_only", False)) + if path != MEMORY_DATABASE_PATH and not read_only: + Path(path).parent.mkdir(parents=True, exist_ok=True) + target: str = ( + f"file:{quote(Path(path).as_posix(), safe='/:')}?mode=ro" if read_only else path + ) + raw_connection: sqlite3.Connection = sqlite3.connect( + target, + timeout=busy_timeout, + isolation_level=None, + check_same_thread=False, + uri=read_only, + ) + raw_connection.create_function(HASH_FUNCTION_NAME, 1, _sqb_hash, deterministic=True) + raw_connection.create_function( + CURSOR_TEXT_FUNCTION_NAME, 1, _cursor_text, deterministic=True + ) + connection: SqliteConnection = SqliteConnection( + raw_connection=raw_connection, path=path, busy_timeout_seconds=busy_timeout + ) + if path != MEMORY_DATABASE_PATH and not read_only: + connection.execute("PRAGMA journal_mode=WAL") + connection.execute(f"PRAGMA busy_timeout={int(busy_timeout * 1000)}") + connection.execute("PRAGMA foreign_keys=ON") + connection.execute("PRAGMA legacy_alter_table=ON") + return connection + + def _execute(self, *, connection: SqliteConnection, sql: str) -> Any: + log_sql(logger=logging.getLogger("sqlbuild.adapter.sqlite"), sql=sql) + removed: tuple[str, ...] + added: tuple[str, ...] + removed, added = relation_changes(sql) + if not removed and not added: + return self._execute_statement(connection=connection, sql=sql) + owns_transaction: bool = not connection.in_transaction() + if owns_transaction: + connection.execute("BEGIN IMMEDIATE") + try: + cursor: Any = self._execute_statement(connection=connection, sql=sql) + record_relation_changes( + connection=connection, + removed=removed, + added=added, + locate=self._logical_relations.lookup, + ) + except BaseException: + if owns_transaction and connection.in_transaction(): + connection.execute("ROLLBACK") + raise + if owns_transaction: + connection.execute("COMMIT") + return cursor + + def _execute_statement(self, *, connection: SqliteConnection, sql: str) -> Any: + view_rename: tuple[str, str] | None = split_rename_view(sql) + if view_rename is not None: + return rename_view( + connection=connection, origin=view_rename[0], destination=view_rename[1] + ) + create_table_as: tuple[str, str] | None = split_create_table_as(sql) + if create_table_as is None: + return connection.execute(sql) + return create_table_typed( + connection=connection, + destination=create_table_as[0], + sql=create_table_as[1], + column_types=_ENFORCED_COLUMN_TYPES.get(), + inferred_types=_INFERRED_COLUMN_TYPES.get(), + ) + + def close(self, connection: SqliteConnection) -> None: + connection.close() + + def begin(self, connection: Any) -> None: + """Take the write lock up front so a read lock is never upgraded mid-transaction.""" + + self.execute(connection=connection, sql="BEGIN IMMEDIATE") + + def rollback(self, connection: Any) -> None: + """Roll back only an open transaction; SQLite ends it itself on interrupts.""" + + if connection.in_transaction(): + self.execute(connection=connection, sql="ROLLBACK") + + def interrupt_connection(self, connection: Any) -> bool: + connection.interrupt() + return True + + def lists_relations_authoritatively(self, *, database: str | None, schema: str) -> bool: + del database, schema + return True + + def supports_zero_copy_clone(self) -> bool: + return False + + def supports_durable_clone(self) -> bool: + return False + + def supports_relation_age_metadata(self) -> bool: + """Return False because SQLite records no relation timestamps.""" + + return False + + def supports_table_freshness_metadata(self) -> bool: + return False + + def supports_concurrent_microbatch_dml(self) -> bool: + """Return False: a SQLite file has one writer, so concurrent batches only serialize.""" + + return False + + def probe_relation_read(self, *, connection: Any, relation: str) -> RelationReadProbe: + return run_relation_read_probe( + execute=self.execute, + connection=connection, + relation=relation, + classify_not_found=self._classify_relation_not_found, + ) + + @staticmethod + def _classify_relation_not_found(error: BaseException) -> RelationReadStatus | None: + if isinstance(error, sqlite3.OperationalError) and str(error).startswith( + MISSING_RELATION_ERROR_FRAGMENT + ): + return RelationReadStatus.MISSING + return None + + def get_table_freshness_metadata( + self, + *, + connection: Any, + database: str | None, + schema: str | None, + name: str, + ) -> TableFreshnessMetadata: + raise AdapterUserError( + message=f"adapter '{self.adapter_name}' does not support table freshness metadata" + ) + + def get_tables_freshness_metadata( + self, + *, + connection: Any, + requests: tuple[TableFreshnessRequest, ...], + ) -> dict[TableFreshnessRequest, TableFreshnessMetadata]: + raise AdapterUserError( + message=f"adapter '{self.adapter_name}' does not support table freshness metadata" + ) + + def supports_python_functions(self) -> bool: + return False + + def persists_python_functions(self) -> bool: + return False + + def python_functions_inherit_default_namespace(self) -> bool: + return False + + def supports_unqualified_function_fingerprints(self) -> bool: + return True + + def supports_table_functions(self) -> bool: + return False + + def max_statement_size(self) -> StatementSizeLimit | None: + """SQLITE_MAX_SQL_LENGTH defaults to 1,000,000,000 bytes: https://sqlite.org/limits.html.""" + + return (1_000_000_000, "bytes") + + def maximum_identifier_length(self) -> int: + return self.max_identifier_length + + def query_column_names(self, *, connection: Any, sql: str) -> tuple[str, ...]: + cursor: Any = self.execute( + connection=connection, sql=f"SELECT * FROM ({sql}) AS __describe_source LIMIT 0" + ) + description: Any | None = getattr(cursor, "description", None) + if description is None: + return () + return tuple(str(column[0]) for column in description) + + def render_max_cursor_at_or_before( + self, + *, + relation: str, + cursor_column: str, + maximum_allowed: str, + cursor_type: str | None, + is_date: bool, + ) -> str: + quoted_cursor: str = self.render_cursor_operand( + expression=self.render_identifier(cursor_column), cursor_type=cursor_type + ) + maximum_literal: str = ( + _quote_sql_string(maximum_allowed) + if is_date + else self.render_cursor_bound_literal(value=maximum_allowed, cursor_type=cursor_type) + ) + return ( + f"SELECT MAX({quoted_cursor}) FROM {relation} " + f"WHERE {quoted_cursor} <= {maximum_literal}" + ) + + def build_cursor_filter( + self, + *, + cursor_column: str | None, + start_cursor: CursorValue | None, + end_cursor: CursorValue | None, + ) -> str: + if cursor_column is None or start_cursor is None: + return "" + operand: str = self.render_cursor_operand( + expression=cursor_column, cursor_type=start_cursor.kind + ) + clauses: list[str] = [f"{operand} >= {self._cursor_value_literal(start_cursor)}"] + if end_cursor is not None: + clauses.append(f"{operand} < {self._cursor_value_literal(end_cursor)}") + return " AND ".join(clauses) + + def render_cursor_operand(self, *, expression: str, cursor_type: str | None) -> str: + """Compare timestamp cursors in canonical UTC text, whatever form the column stores.""" + + if str(cursor_type).lower() == CursorKind.INTEGER: + return expression + return f"{CURSOR_TEXT_FUNCTION_NAME}({expression})" + + def schema_exists( + self, + *, + connection: Any, + database: str | None, + schema: str, + ) -> bool: + del database + if schema.lower() == MAIN_SCHEMA: + return True + return schema.lower() in registered_schemas(connection) + + def query(self, *, connection: Any, sql: str, limit: int | None) -> QueryResult: + cursor: Any = self.execute(connection=connection, sql=sql) + description: Any | None = getattr(cursor, "description", None) + if description is None: + return QueryResult() + columns: tuple[str, ...] = tuple(str(column[0]) for column in description) + rows: tuple[tuple[object, ...], ...] + truncated: bool = False + if limit is None: + rows = tuple(tuple(row) for row in cursor.fetchall()) + else: + fetched_rows: list[tuple[object, ...]] = [ + tuple(row) for row in cursor.fetchmany(limit + 1) + ] + truncated = len(fetched_rows) > limit + rows = tuple(fetched_rows[:limit]) + return QueryResult(columns=columns, rows=rows, truncated=truncated) + + def relation_exists( + self, + *, + connection: Any, + database: str | None, + schema: str | None, + name: str, + ) -> bool: + del database + table_name: str = physical_name(schema=schema, name=name) + row: tuple[Any, ...] | None = connection.execute_with( + sql="SELECT name FROM sqlite_schema " + "WHERE type IN ('table', 'view') AND lower(name) = lower(?)", + parameters=(table_name,), + ).fetchone() + if row is None: + return False + located: tuple[str, str] = logical_location( + table_name=str(row[0]), locations=registry_locations(connection) + ) + return (located[0].lower(), located[1].lower()) == ( + (schema or MAIN_SCHEMA).lower(), + name.lower(), + ) + + def list_relations( + self, + *, + connection: Any, + database: str | None, + schemas: tuple[str, ...] | None, + names: tuple[str, ...] | None = None, + ) -> tuple[RelationInfo, ...]: + del database + schema_filter: frozenset[str] | None = ( + None if schemas is None else frozenset(schema.lower() for schema in schemas) + ) + name_filter: frozenset[str] | None = ( + None if names is None else frozenset(name.lower() for name in names) + ) + return tuple( + RelationInfo(database=None, schema=schema, name=name, relation_type=relation_type) + for schema, name, _, relation_type in catalog_relations(connection) + if (schema_filter is None or schema.lower() in schema_filter) + and (name_filter is None or name.lower() in name_filter) + ) + + def with_relation_age_metadata( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> tuple[RelationInfo, ...]: + return relations + + def list_functions( + self, + *, + connection: Any, + database: str | None, + schemas: tuple[str, ...] | None, + names: tuple[str, ...] | None = None, + ) -> tuple[FunctionInfo, ...]: + """Return no functions: SQLite has no persistent function catalog.""" + + del connection, database, schemas, names + return () + + def get_columns( + self, + *, + connection: Any, + database: str | None, + schema: str | None, + name: str, + ) -> tuple[ColumnInfo, ...]: + del database + return relation_columns( + connection=connection, table_name=physical_name(schema=schema, name=name) + ) + + def get_all_columns( + self, + *, + connection: Any, + database: str | None, + schemas: tuple[str, ...] | None, + names: tuple[str, ...] | None = None, + ) -> dict[str, tuple[ColumnInfo, ...]]: + del database + schema_filter: frozenset[str] | None = ( + None if schemas is None else frozenset(schema.lower() for schema in schemas) + ) + name_filter: frozenset[str] | None = ( + None if names is None else frozenset(name.lower() for name in names) + ) + return { + name: relation_columns(connection=connection, table_name=table_name) + for schema, name, table_name, _ in catalog_relations(connection) + if (schema_filter is None or schema.lower() in schema_filter) + and (name_filter is None or name.lower() in name_filter) + } + + def get_columns_for_relations( + self, + *, + connection: Any, + relations: tuple[RelationInfo, ...], + ) -> dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]]: + return get_columns_for_relations_bulk( + adapter=self, connection=connection, relations=relations + ) + + def render_create_schema( + self, + *, + database: str | None, + schema: str, + ) -> tuple[str, ...]: + """Record the schema in the registry that decides how `schema__name` tables split.""" + + del database + if schema.lower() == MAIN_SCHEMA: + return () + validate_schema_name(schema) + return ( + f"CREATE TABLE IF NOT EXISTS {SCHEMA_REGISTRY_TABLE} " + "(schema_name TEXT PRIMARY KEY, created_at TEXT NOT NULL)", + f"INSERT OR IGNORE INTO {SCHEMA_REGISTRY_TABLE} (schema_name, created_at) " + f"VALUES ({_quote_sql_string(schema)}, {self.render_current_timestamp()})", + ) + + def ensure_schema( + self, + *, + connection: Any, + database: str | None, + schema: str | None, + statement_recorder: StatementRecorder, + ) -> None: + if schema is None or self.schema_exists( + connection=connection, database=database, schema=schema + ): + return + statements: tuple[str, ...] = self.render_create_schema(database=database, schema=schema) + statement_recorder.record_many(statements) + with self.transaction(connection): + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def render_create_view_as(self, *, destination: str, sql: str) -> tuple[str, ...]: + return (f"DROP VIEW IF EXISTS {destination}", f"CREATE VIEW {destination} AS {sql}") + + def render_table_function_call(self, *, target: str, call_suffix_sql: str) -> str: + return f"{target}{call_suffix_sql}" + + def render_udf_call(self, *, target: str, call_suffix_sql: str) -> str: + return f"{target}{call_suffix_sql}" + + def render_create_function( + self, + *, + destination: str, + arguments: tuple[Any, ...], + returns: str, + body_sql: str, + return_columns: tuple[Any, ...] = (), + language: FunctionLanguage = FunctionLanguage.SQL, + runtime_version: str | None = None, + entry_point: str | None = None, + packages: tuple[str, ...] = (), + ) -> tuple[str, ...]: + del arguments, returns, body_sql, return_columns, runtime_version, entry_point, packages + raise AdapterUserError( + message=( + f"SQLite cannot create function '{destination}': SQLite has no CREATE FUNCTION, " + f"so {language.value.upper()} functions and table functions are not supported" + ), + code=UNSUPPORTED_FEATURE_CODE, + help="Inline the expression in the model, or use a macro.", + ) + + def render_append( + self, *, destination: str, sql: str, columns: tuple[str, ...] | None = None + ) -> tuple[str, ...]: + column_sql: str = "" + if columns is not None: + column_sql = ( + " (" + ", ".join(self.render_identifier(column) for column in columns) + ")" + ) + return (f"INSERT INTO {destination}{column_sql} {sql}",) + + def render_delete_insert( + self, + *, + destination: str, + sql: str, + unique_key: tuple[str, ...], + columns: tuple[str, ...] | None = None, + ) -> tuple[str, ...]: + staged: str = self._with_replaced_relation_name( + relation=destination, name=f"{self._relation_name(destination)}__delete_insert" + ) + key_list: str = ", ".join(self.render_identifier(key) for key in unique_key) + create_staged: tuple[str, ...] = self.render_create_table_as(destination=staged, sql=sql) + delete_sql: str = ( + f"DELETE FROM {destination} WHERE ({key_list}) IN (SELECT {key_list} FROM {staged})" + ) + insert_stmts: tuple[str, ...] = self.render_append( + destination=destination, + sql=f"SELECT * FROM {staged}", + columns=columns, + ) + drop_staged: tuple[str, ...] = self.render_drop(destination=staged, if_exists=True) + return (*create_staged, delete_sql, *insert_stmts, *drop_staged) + + def render_delete_insert_cursor( + self, + *, + destination: str, + sql: str, + cursor_column: str, + cursor_start: str, + cursor_end: str, + columns: tuple[str, ...] | None = None, + cursor_type: str | None = None, + ) -> tuple[str, ...]: + quoted_cursor_column: str = self.render_cursor_operand( + expression=self.render_identifier(cursor_column), cursor_type=cursor_type + ) + start_literal: str = self.render_cursor_bound_literal( + value=cursor_start, cursor_type=cursor_type + ) + end_literal: str = self.render_cursor_bound_literal( + value=cursor_end, cursor_type=cursor_type + ) + delete_sql: str = ( + f"DELETE FROM {destination} WHERE {quoted_cursor_column} >= {start_literal} " + f"AND {quoted_cursor_column} < {end_literal}" + ) + insert_stmts: tuple[str, ...] = self.render_append( + destination=destination, sql=sql, columns=columns + ) + return (delete_sql, *insert_stmts) + + def render_drop(self, *, destination: str, if_exists: bool = True) -> tuple[str, ...]: + exists_clause: str = " IF EXISTS" if if_exists else "" + return (f"DROP TABLE{exists_clause} {destination}",) + + def render_drop_view(self, *, destination: str, if_exists: bool = True) -> tuple[str, ...]: + exists_clause: str = " IF EXISTS" if if_exists else "" + return (f"DROP VIEW{exists_clause} {destination}",) + + def render_swap(self, *, left: str, right: str) -> tuple[str, ...]: + staging: str = self._with_replaced_relation_name( + relation=left, name=f"{self._relation_name(left)}__swap_staging" + ) + return ( + *self.render_rename(origin=left, destination=staging), + *self.render_rename(origin=right, destination=left), + *self.render_rename(origin=staging, destination=right), + ) + + def render_clone( + self, + *, + origin: str, + destination: str, + hard_copy: bool = False, + origin_is_transient: bool = False, + ) -> tuple[str, ...]: + del hard_copy, origin_is_transient + return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") + + def render_durable_clone( + self, *, origin: str, destination: str, origin_is_transient: bool = False + ) -> tuple[str, ...]: + del origin_is_transient + return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") + + def render_migration_stage( + self, + *, + origin: str, + stage: str, + origin_is_transient: bool = False, + stage_is_transient: bool | None = None, + ) -> MigrationStagePlan: + del origin_is_transient, stage_is_transient + return MigrationStagePlan( + transfer=MigrationTransfer.COPY, + statements=(f"CREATE TABLE {stage} AS SELECT * FROM {origin}",), + ) + + def capture_dependent_view_rebinds( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + """Return nothing: SQLite views bind by name, so a renamed table needs no rebind.""" + + del connection, database, schema, name + return () + + def views_bind_to_relation_identity(self) -> bool: + return False + + def list_dependent_view_names( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> tuple[str, ...]: + del database + target: str = physical_name(schema=schema, name=name) + reference: re.Pattern[str] = re.compile( + r"(? tuple[RelationGrant, ...]: + """Return no grants: SQLite has no privilege system.""" + + del connection, database, schema, name, relation_type + return () + + def render_relation_grants( + self, *, grants: tuple[RelationGrant, ...], destination: str, columns: tuple[str, ...] + ) -> tuple[str, ...]: + del grants, destination, columns + return () + + def render_replace_view_keeping_grants( + self, *, destination: str, sql: str + ) -> tuple[str, ...] | None: + return self.render_create_view_as(destination=destination, sql=sql) + + def read_view_definition( + self, *, connection: Any, database: str | None, schema: str, name: str + ) -> str | None: + del database + row: tuple[Any, ...] | None = connection.execute_with( + sql="SELECT sql FROM sqlite_schema WHERE type = 'view' AND lower(name) = lower(?)", + parameters=(physical_name(schema=schema, name=name),), + ).fetchone() + return None if row is None else view_body(str(row[0])) + + def view_definition_matches( + self, *, connection: Any, database: str | None, schema: str, name: str, sql: str + ) -> bool: + definition: str | None = self.read_view_definition( + connection=connection, database=database, schema=schema, name=name + ) + if definition is None: + return False + return same_view_definition(definition=definition, sql=sql) + + def render_relation_revokes( + self, *, grants: tuple[RelationGrant, ...], destination: str + ) -> tuple[str, ...]: + del grants, destination + return () + + def rename_view( + self, + *, + connection: Any, + origin: str, + destination: str, + statement_recorder: StatementRecorder, + ) -> None: + """Rename a view; SQLite recreates it from its stored definition in one transaction.""" + + statements: tuple[str, ...] = self.render_rename_view( + origin=origin, destination=destination + ) + statement_recorder.record_many(statements) + statement: str + for statement in statements: + self.execute(connection=connection, sql=statement) + + def supports_transactional_ddl(self) -> bool: + return True + + def render_seed_select_before_cursor( + self, + *, + origin: str, + cursor_column: str, + cursor_end_exclusive: str, + cursor_type: str | None, + ) -> str: + operand: str = self.render_cursor_operand( + expression=self.render_identifier(cursor_column), cursor_type=cursor_type + ) + end_literal: str = self.render_cursor_bound_literal( + value=cursor_end_exclusive, cursor_type=cursor_type + ) + return f"SELECT * FROM {origin} WHERE {operand} < {end_literal}" + + def relation_names_match(self, *, left: str, right: str) -> bool: + return self._relation_names_match_impl(left=left, right=right) + + def render_replace_table_from_relation( + self, *, destination: str, origin: str + ) -> tuple[str, ...]: + return self.render_create_table_as(destination=destination, sql=f"SELECT * FROM {origin}") + + def render_add_columns( + self, *, destination: str, columns: tuple[ColumnInfo, ...] + ) -> tuple[str, ...]: + return tuple( + f"ALTER TABLE {destination} ADD COLUMN {self.render_identifier(col.name)} " + f"{storage_type(col.type)}" + for col in columns + ) + + def render_drop_columns( + self, *, destination: str, column_names: tuple[str, ...] + ) -> tuple[str, ...]: + return tuple( + f"ALTER TABLE {destination} DROP COLUMN {self.render_identifier(col_name)}" + for col_name in column_names + ) + + def render_rename_column( + self, *, destination: str, old_name: str, new_name: str + ) -> tuple[str, ...]: + from sqlbuild.adapter.contract.main.render_rename_column_sql import ( + render_alter_rename_column_sql, + ) + + return ( + render_alter_rename_column_sql( + destination=destination, + old_identifier=self.render_exact_identifier(old_name), + new_identifier=self.render_identifier(new_name), + ), + ) + + def render_alter_column_types( + self, *, destination: str, columns: tuple[ColumnInfo, ...] + ) -> tuple[str, ...]: + del columns + raise AdapterUserError( + message=( + f"SQLite cannot alter column types of '{destination}' in place; " + "alter_column_types rebuilds the table instead" + ), + code=UNSUPPORTED_FEATURE_CODE, + ) + + def render_merge( + self, + *, + destination: str, + sql: str, + unique_key: tuple[str, ...], + source_columns: tuple[str, ...] = (), + exclude_columns: tuple[str, ...] = (), + ) -> tuple[str, ...]: + immutable_columns: frozenset[str] = frozenset( + column.lower() for column in (*unique_key, *exclude_columns) + ) + key_match_sql: str = " AND ".join( + f"__target.{self.render_identifier(k)} = __source.{self.render_identifier(k)}" + for k in unique_key + ) + update_set: str = ", ".join( + f"{self.render_identifier(col)} = __source.{self.render_identifier(col)}" + for col in source_columns + if col.lower() not in immutable_columns + ) + col_list: str = ", ".join(self.render_identifier(col) for col in source_columns) + statements: list[str] = [] + if update_set: + statements.append( + f"UPDATE {destination} AS __target SET {update_set} " + f"FROM ({sql}) AS __source WHERE {key_match_sql}" + ) + statements.append( + f"INSERT INTO {destination} ({col_list}) " + f"SELECT {col_list} FROM ({sql}) AS __source " + f"WHERE NOT EXISTS (SELECT 1 FROM {destination} AS __target WHERE {key_match_sql})" + ) + return tuple(statements) + + def render_current_timestamp(self) -> str: + """Render UTC now as ISO-8601 text with milliseconds, the adapter's timestamp form.""" + + return "strftime('%Y-%m-%d %H:%M:%f', 'now')" + + def create_table_as( + self, + *, + connection: Any, + destination: str, + sql: str, + config: dict[str, Any] | None = None, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_create_table_as(destination=destination, sql=sql) + statement_recorder.record_many(statements) + enforced: Token[Mapping[str, str]] = _ENFORCED_COLUMN_TYPES.set( + _lowered((config or {}).get("column_types")) + ) + inferred: Token[Mapping[str, str]] = _INFERRED_COLUMN_TYPES.set( + _lowered((config or {}).get("inferred_column_types")) + ) + try: + with self.transaction(connection): + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + finally: + _INFERRED_COLUMN_TYPES.reset(inferred) + _ENFORCED_COLUMN_TYPES.reset(enforced) + + def create_view_as( + self, + *, + connection: Any, + destination: str, + sql: str, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_create_view_as(destination=destination, sql=sql) + statement_recorder.record_many(statements) + with self.transaction(connection): + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def create_function( + self, + *, + connection: Any, + definition: FunctionDefinition, + statement_recorder: StatementRecorder, + ) -> None: + del connection, statement_recorder + self.render_create_function( + destination=definition.destination, + arguments=definition.arguments, + returns=definition.returns, + body_sql=definition.body_sql, + return_columns=definition.return_columns, + language=definition.language, + ) + + def drop( + self, + *, + connection: Any, + destination: str, + if_exists: bool = True, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_drop(destination=destination, if_exists=if_exists) + statement_recorder.record_many(statements) + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def drop_view( + self, + *, + connection: Any, + destination: str, + if_exists: bool = True, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_drop_view( + destination=destination, if_exists=if_exists + ) + statement_recorder.record_many(statements) + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def rename( + self, + *, + connection: Any, + origin: str, + destination: str, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_rename(origin=origin, destination=destination) + statement_recorder.record_many(statements) + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def swap( + self, + *, + connection: Any, + left: str, + right: str, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_swap(left=left, right=right) + statement_recorder.record_many(statements) + with self.transaction(connection): + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def clone( + self, + *, + connection: Any, + origin: str, + destination: str, + hard_copy: bool = False, + origin_is_transient: bool = False, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_clone( + origin=origin, + destination=destination, + hard_copy=hard_copy, + origin_is_transient=origin_is_transient, + ) + statement_recorder.record_many(statements) + with self.transaction(connection): + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def durable_clone( + self, + *, + connection: Any, + origin: str, + destination: str, + origin_is_transient: bool = False, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_durable_clone( + origin=origin, destination=destination, origin_is_transient=origin_is_transient + ) + statement_recorder.record_many(statements) + with self.transaction(connection): + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def replace_table_from_relation( + self, + *, + connection: Any, + destination: str, + origin: str, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_replace_table_from_relation( + destination=destination, + origin=origin, + ) + statement_recorder.record_many(statements) + with self.transaction(connection): + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def move_or_copy_relation( + self, + *, + connection: Any, + origin: str, + destination: str, + remove_origin: bool, + allow_copy_fallback: bool, + statement_recorder: StatementRecorder, + ) -> None: + """Rename within the file; schemas are name prefixes, so every move is a rename.""" + + del allow_copy_fallback + if remove_origin: + self.rename( + connection=connection, + origin=origin, + destination=destination, + statement_recorder=statement_recorder, + ) + return + self.replace_table_from_relation( + connection=connection, + destination=destination, + origin=origin, + statement_recorder=statement_recorder, + ) + + def append( + self, + *, + connection: Any, + destination: str, + sql: str, + columns: tuple[str, ...] | None = None, + statement_recorder: StatementRecorder, + ) -> int | None: + statements: tuple[str, ...] = self.render_append( + destination=destination, sql=sql, columns=columns + ) + statement_recorder.record_many(statements) + affected: int | None = None + stmt: str + for stmt in statements: + cursor: Any = self.execute(connection=connection, sql=stmt) + affected = self.affected_row_count(cursor=cursor) + return affected + + def delete_insert( + self, + *, + connection: Any, + destination: str, + sql: str, + unique_key: str | tuple[str, ...], + columns: tuple[str, ...] | None = None, + statement_recorder: StatementRecorder, + ) -> int | None: + keys: tuple[str, ...] = (unique_key,) if isinstance(unique_key, str) else unique_key + statements: tuple[str, ...] = self.render_delete_insert( + destination=destination, sql=sql, unique_key=keys, columns=columns + ) + statement_recorder.record_many(statements) + affected: int | None = None + with self.transaction(connection): + stmt: str + for stmt in statements: + cursor: Any = self.execute(connection=connection, sql=stmt) + if stmt.startswith(INSERT_KEYWORD): + affected = self.affected_row_count(cursor=cursor) + return affected + + def delete_insert_cursor( + self, + *, + connection: Any, + destination: str, + sql: str, + cursor_column: str, + cursor_start: str, + cursor_end: str, + columns: tuple[str, ...] | None = None, + statement_recorder: StatementRecorder, + cursor_type: str | None = None, + ) -> int | None: + statements: tuple[str, ...] = self.render_delete_insert_cursor( + destination=destination, + sql=sql, + cursor_column=cursor_column, + cursor_start=cursor_start, + cursor_end=cursor_end, + columns=columns, + cursor_type=cursor_type, + ) + statement_recorder.record_many(statements) + affected: int | None = None + with self.transaction(connection): + stmt: str + for stmt in statements: + cursor: Any = self.execute(connection=connection, sql=stmt) + affected = self.affected_row_count(cursor=cursor) + return affected + + def merge( + self, + *, + connection: Any, + destination: str, + sql: str, + unique_key: str | tuple[str, ...], + statement_recorder: StatementRecorder, + exclude_columns: tuple[str, ...] = (), + ) -> int | None: + keys: tuple[str, ...] = (unique_key,) if isinstance(unique_key, str) else unique_key + source_columns: tuple[str, ...] = self.query_column_names(connection=connection, sql=sql) + statements: tuple[str, ...] = self.render_merge( + destination=destination, + sql=sql, + unique_key=keys, + source_columns=source_columns, + exclude_columns=exclude_columns, + ) + statement_recorder.record_many(statements) + affected: int | None = 0 + with self.transaction(connection): + stmt: str + for stmt in statements: + cursor: Any = self.execute(connection=connection, sql=stmt) + count: int | None = self.affected_row_count(cursor=cursor) + affected = None if affected is None or count is None else affected + count + return affected + + def load_seed( + self, + *, + connection: Any, + destination: str, + file_path: Path, + columns: tuple[ColumnInfo, ...], + csv_settings: SeedCsvSettings = DEFAULT_SEED_CSV_SETTINGS, + replace: bool = True, + infer_types: bool = False, + statement_recorder: StatementRecorder, + ) -> None: + del infer_types + column_names: tuple[str, ...] = tuple(col.name for col in columns) + column_defs: str = ", ".join( + f"{self.render_identifier(col.name)} {col.type}" for col in columns + ) + create_sql: str = f"CREATE TABLE {destination} ({column_defs})" + insert_sql: str = ( + f"INSERT INTO {destination} " + f"({', '.join(self.render_identifier(name) for name in column_names)}) " + f"VALUES ({', '.join(['?'] * len(column_names))})" + ) + boolean_columns: frozenset[str] = frozenset( + col.name for col in columns if BOOLEAN_TYPE_MARKER in col.type.upper() + ) + rows: list[tuple[object, ...]] = [] + with file_path.open( + "r", encoding=csv_settings.encoding or "utf-8", newline="" + ) as seed_file: + reader: csv.DictReader[str] = csv.DictReader( + seed_file, + delimiter=csv_settings.delimiter or ",", + quotechar=csv_settings.quotechar or '"', + escapechar=csv_settings.escapechar, + doublequote=True if csv_settings.doublequote is None else csv_settings.doublequote, + skipinitialspace=( + False + if csv_settings.skipinitialspace is None + else csv_settings.skipinitialspace + ), + ) + for row in reader: + rows.append( + tuple( + _seed_value( + value=normalize_seed_csv_value( + value=row.get(col), column_name=col, csv_settings=csv_settings + ), + boolean=col in boolean_columns, + ) + for col in column_names + ) + ) + with self.transaction(connection): + if replace: + self.drop( + connection=connection, + destination=destination, + if_exists=True, + statement_recorder=statement_recorder, + ) + statement_recorder.record(create_sql) + self.execute(connection=connection, sql=create_sql) + if rows: + statement_recorder.record(insert_sql) + connection.executemany(sql=insert_sql, rows=rows) + + def add_columns( + self, + *, + connection: Any, + destination: str, + columns: tuple[ColumnInfo, ...], + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_add_columns( + destination=destination, columns=columns + ) + statement_recorder.record_many(statements) + with self.transaction(connection): + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def drop_columns( + self, + *, + connection: Any, + destination: str, + column_names: tuple[str, ...], + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_drop_columns( + destination=destination, column_names=column_names + ) + statement_recorder.record_many(statements) + with self.transaction(connection): + for stmt in statements: + self.execute(connection=connection, sql=stmt) + + def rename_column( + self, + *, + connection: Any, + destination: str, + old_name: str, + new_name: str, + statement_recorder: StatementRecorder, + ) -> None: + statements: tuple[str, ...] = self.render_rename_column( + destination=destination, old_name=old_name, new_name=new_name + ) + statement_recorder.record_many(statements) + statement: str + for statement in statements: + self.execute(connection=connection, sql=statement) + + def column_rename_unavailable_reason(self, *, connection: Any, destination: str) -> str | None: + del connection, destination + return None + + def alter_column_types( + self, + *, + connection: Any, + destination: str, + columns: tuple[ColumnInfo, ...], + statement_recorder: StatementRecorder, + ) -> None: + """Rebuild the table with the new types in one transaction, keeping its indexes.""" + + table_name: str = unquote_relation(destination) + rebuilt: str = quote_identifier(f"{table_name}__sqlbuild_retype") + new_types: dict[str, str] = {col.name.lower(): col.type for col in columns} + current: tuple[ColumnInfo, ...] = relation_columns( + connection=connection, table_name=table_name + ) + projections: str = ", ".join( + self.render_source_expression_cast( + expression=self.render_identifier(col.name), + target_type=new_types[col.name.lower()], + alias=self.render_identifier(col.name), + ) + if col.name.lower() in new_types + else self.render_identifier(col.name) + for col in current + ) + index_rows: list[tuple[Any, ...]] = connection.execute_with( + sql="SELECT sql FROM sqlite_schema WHERE type = 'index' AND lower(tbl_name) = lower(?) " + "AND sql IS NOT NULL ORDER BY name", + parameters=(table_name,), + ).fetchall() + statements: tuple[str, ...] = ( + f"DROP TABLE IF EXISTS {rebuilt}", + f"CREATE TABLE {rebuilt} AS SELECT {projections} FROM {destination}", + f"DROP TABLE {destination}", + f"ALTER TABLE {rebuilt} RENAME TO {quote_identifier(table_name)}", + *(str(row[0]) for row in index_rows), + ) + statement_recorder.record_many(statements) + token: Token[Mapping[str, str]] = _ENFORCED_COLUMN_TYPES.set(new_types) + try: + with self.transaction(connection): + stmt: str + for stmt in statements: + self.execute(connection=connection, sql=stmt) + finally: + _ENFORCED_COLUMN_TYPES.reset(token) + + def inspect_row_diff_coverage( + self, + *, + connection: Any, + relation: str, + cursor_column: str | None = None, + start_cursor: Any | None = None, + end_cursor: Any | None = None, + ) -> RowDiffCoverage: + return self._inspect_row_diff_coverage( + connection=connection, + relation=relation, + cursor_column=cursor_column, + start_cursor=start_cursor, + end_cursor=end_cursor, + ) + + def estimate_relation_row_count( + self, + *, + connection: Any, + database: str | None, + schema: str | None, + name: str, + ) -> RelationRowCountEstimate: + """Count table rows: a local file has no metadata count, and counting reads only pages.""" + + del database + table_name: str = physical_name(schema=schema, name=name) + row: tuple[Any, ...] | None = connection.execute_with( + sql="SELECT type FROM sqlite_schema WHERE lower(name) = lower(?)", + parameters=(table_name,), + ).fetchone() + if row is None: + return RelationRowCountEstimate(row_count=None, detail="relation not found") + if row[0] != SQLITE_TABLE_TYPE: + return RelationRowCountEstimate(row_count=None, detail="views have no row count") + count: tuple[Any, ...] = connection.execute( + f"SELECT COUNT(*) FROM {quote_identifier(table_name)}" + ).fetchone() + return RelationRowCountEstimate(row_count=int(count[0])) + + def diff_schema( + self, + *, + connection: Any, + left: str, + right: str, + ) -> SchemaDiffResult: + left_columns: tuple[ColumnInfo, ...] = self.describe_relation( + connection=connection, relation=left + ) + right_columns: tuple[ColumnInfo, ...] = self.describe_relation( + connection=connection, relation=right + ) + left_map: dict[str, str] = {col.name: col.type for col in left_columns} + right_map: dict[str, str] = {col.name: col.type for col in right_columns} + added: list[ColumnInfo] = [] + removed: list[ColumnInfo] = [] + type_changed: list[tuple[ColumnInfo, ColumnInfo]] = [] + for col_name, col_type in right_map.items(): + if col_name not in left_map: + added.append(ColumnInfo(name=col_name, type=col_type)) + elif not types_equal( + left=left_map[col_name], right=col_type, dialect=self.sql_analysis_dialect() + ): + type_changed.append( + ( + ColumnInfo(name=col_name, type=left_map[col_name]), + ColumnInfo(name=col_name, type=col_type), + ) + ) + for col_name, col_type in left_map.items(): + if col_name not in right_map: + removed.append(ColumnInfo(name=col_name, type=col_type)) + return SchemaDiffResult( + added_columns=tuple(added), + removed_columns=tuple(removed), + type_changed_columns=tuple(type_changed), + left_column_count=len(left_columns), + right_column_count=len(right_columns), + ) + + def diff_rows( + self, + *, + connection: Any, + left: str, + right: str, + unique_key: str | tuple[str, ...], + excluded_columns: tuple[str, ...] = (), + tolerances: RowDiffTolerances | None = None, + cursor_column: str | None = None, + start_cursor: Any | None = None, + end_cursor: Any | None = None, + ) -> RowDiffResult: + keys: tuple[str, ...] = (unique_key,) if isinstance(unique_key, str) else unique_key + sampling: RowDiffSampling | None = tolerances.sampling if tolerances is not None else None + left_columns: tuple[ColumnInfo, ...] = self.describe_relation( + connection=connection, relation=left + ) + compare_columns: tuple[str, ...] = tuple( + col.name + for col in left_columns + if col.name not in keys and col.name not in excluded_columns + ) + left_columns_by_name: dict[str, ColumnInfo] = {col.name: col for col in left_columns} + cursor_filter: str = self.build_cursor_filter( + cursor_column=cursor_column, + start_cursor=start_cursor, + end_cursor=end_cursor, + ) + left_cte: str = f"SELECT * FROM {left}" + right_cte: str = f"SELECT * FROM {right}" + if cursor_filter: + left_cte += f" WHERE {cursor_filter}" + right_cte += f" WHERE {cursor_filter}" + self.validate_row_diff_keys( + connection=connection, relation_sql=left_cte, relation_label="left", keys=keys + ) + self.validate_row_diff_keys( + connection=connection, relation_sql=right_cte, relation_label="right", keys=keys + ) + prepared_relations: RowDiffPreparedRelations = self._build_row_diff_relation_ctes( + connection=connection, + left_sql=left_cte, + right_sql=right_cte, + keys=keys, + sampling=sampling, + inspect_population=True, + ) + join_condition: str = " AND ".join(f"__left.{k} = __right.{k}" for k in keys) + column_equal_expressions: dict[str, str] = { + col: self.build_row_diff_equal_expression( + column=col, + column_info=left_columns_by_name[col], + tolerances=tolerances, + ) + for col in compare_columns + } + column_tolerances: dict[str, RowDiffTolerance | None] = { + col: self.resolve_row_diff_tolerance( + column=col, + column_type=left_columns_by_name[col].type, + tolerances=tolerances, + ) + for col in compare_columns + } + equal_condition: str = "TRUE" + if compare_columns: + equal_condition = " AND ".join(column_equal_expressions.values()) + column_count_sql_parts: list[str] = [ + f"COUNT(CASE WHEN __left.{keys[0]} IS NOT NULL " + f"AND __right.{keys[0]} IS NOT NULL " + f"AND NOT ({column_equal_expressions[col]}) THEN 1 END) AS __{col}_mismatch_count" + for col in compare_columns + ] + column_count_sql: str = ( + ", " + ", ".join(column_count_sql_parts) if column_count_sql_parts else "" + ) + diff_sql: str = ( + f"WITH {prepared_relations.cte_sql} " + f"SELECT " + f"COUNT(CASE WHEN __left.{keys[0]} IS NOT NULL THEN 1 END) AS left_count, " + f"COUNT(CASE WHEN __right.{keys[0]} IS NOT NULL THEN 1 END) AS right_count, " + f"COUNT(*) AS joined, " + f"COUNT(CASE WHEN __left.{keys[0]} IS NOT NULL " + f"AND __right.{keys[0]} IS NOT NULL AND ({equal_condition}) THEN 1 END) AS equal, " + f"COUNT(CASE WHEN __left.{keys[0]} IS NOT NULL " + f"AND __right.{keys[0]} IS NOT NULL AND NOT ({equal_condition}) " + f"THEN 1 END) AS unequal, " + f"COUNT(CASE WHEN __right.{keys[0]} IS NULL THEN 1 END) AS left_only, " + f"COUNT(CASE WHEN __left.{keys[0]} IS NULL THEN 1 END) AS right_only" + f"{column_count_sql} FROM __left FULL OUTER JOIN __right ON {join_condition}" + ) + row: tuple[Any, ...] = self.execute(connection=connection, sql=diff_sql).fetchone() + column_results: tuple[RowDiffColumnResult, ...] = tuple( + RowDiffColumnResult( + name=col, + mismatched_count=int(row[index]), + tolerance=column_tolerances[col], + ) + for index, col in enumerate(compare_columns, start=7) + ) + return RowDiffResult( + left_count=int(row[0]), + right_count=int(row[1]), + joined_count=int(row[2]), + equal_count=int(row[3]), + unequal_count=int(row[4]), + left_only_count=int(row[5]), + right_only_count=int(row[6]), + column_results=column_results, + population_count=prepared_relations.population_count or int(row[2]), + compared_count=int(row[2]), + sampling=sampling, + ) + + def sample_unequal_rows( + self, + *, + connection: Any, + left: str, + right: str, + unique_key: str | tuple[str, ...], + excluded_columns: tuple[str, ...] = (), + tolerances: RowDiffTolerances | None = None, + cursor_column: str | None = None, + start_cursor: Any | None = None, + end_cursor: Any | None = None, + limit: int = 20, + ) -> tuple[RowDiffSampleRow, ...]: + sampling: RowDiffSampling | None = tolerances.sampling if tolerances is not None else None + keys: tuple[str, ...] = (unique_key,) if isinstance(unique_key, str) else unique_key + left_columns: tuple[ColumnInfo, ...] = self.describe_relation( + connection=connection, relation=left + ) + compare_columns: tuple[str, ...] = tuple( + col.name + for col in left_columns + if col.name not in keys and col.name not in excluded_columns + ) + left_columns_by_name: dict[str, ColumnInfo] = {col.name: col for col in left_columns} + cursor_filter: str = self.build_cursor_filter( + cursor_column=cursor_column, + start_cursor=start_cursor, + end_cursor=end_cursor, + ) + left_cte: str = f"SELECT * FROM {left}" + right_cte: str = f"SELECT * FROM {right}" + if cursor_filter: + left_cte += f" WHERE {cursor_filter}" + right_cte += f" WHERE {cursor_filter}" + self.validate_row_diff_keys( + connection=connection, relation_sql=left_cte, relation_label="left", keys=keys + ) + self.validate_row_diff_keys( + connection=connection, relation_sql=right_cte, relation_label="right", keys=keys + ) + prepared_relations: RowDiffPreparedRelations = self._build_row_diff_relation_ctes( + connection=connection, + left_sql=left_cte, + right_sql=right_cte, + keys=keys, + sampling=sampling, + inspect_population=False, + ) + column_equal_expressions: dict[str, str] = { + col: self.build_row_diff_equal_expression( + column=col, + column_info=left_columns_by_name[col], + tolerances=tolerances, + ) + for col in compare_columns + } + unequal_condition: str = ( + " OR ".join(f"NOT ({expr})" for expr in column_equal_expressions.values()) + if compare_columns + else "FALSE" + ) + join_condition: str = " AND ".join(f"__left.{k} = __right.{k}" for k in keys) + key_select_sql: str = ", ".join( + f"COALESCE(__left.{k}, __right.{k}) AS __key_{k}" for k in keys + ) + compare_select_sql: str = ", ".join( + f"__left.{col} AS __left_{col}, __right.{col} AS __right_{col}" + for col in compare_columns + ) + if compare_select_sql: + compare_select_sql = ", " + compare_select_sql + sample_sql: str = ( + f"WITH {prepared_relations.cte_sql} " + f"SELECT {key_select_sql}{compare_select_sql} " + f"FROM __left FULL OUTER JOIN __right ON {join_condition} " + f"WHERE __left.{keys[0]} IS NOT NULL AND __right.{keys[0]} IS NOT NULL " + f"AND ({unequal_condition}) " + f"ORDER BY {', '.join(f'__key_{k}' for k in keys)} LIMIT {limit}" + ) + rows: list[tuple[Any, ...]] = self.execute(connection=connection, sql=sample_sql).fetchall() + samples: list[RowDiffSampleRow] = [] + for row in rows: + key_values: tuple[tuple[str, object], ...] = tuple( + (k, row[i]) for i, k in enumerate(keys) + ) + changed_cells: list[RowDiffSampleCell] = [] + for col_index, col in enumerate(compare_columns): + left_i: int = len(keys) + (col_index * 2) + left_val: object = row[left_i] + right_val: object = row[left_i + 1] + if left_val != right_val: + changed_cells.append( + RowDiffSampleCell(name=col, left_value=left_val, right_value=right_val) + ) + samples.append( + RowDiffSampleRow(key_values=key_values, changed_cells=tuple(changed_cells)) + ) + return tuple(samples) + + def sample_side_only_rows( + self, + *, + connection: Any, + left: str, + right: str, + unique_key: str | tuple[str, ...], + side: str, + cursor_column: str | None = None, + start_cursor: Any | None = None, + end_cursor: Any | None = None, + limit: int = 20, + sampling: RowDiffSampling | None = None, + ) -> tuple[tuple[tuple[str, object], ...], ...]: + keys: tuple[str, ...] = (unique_key,) if isinstance(unique_key, str) else unique_key + cursor_filter: str = self.build_cursor_filter( + cursor_column=cursor_column, + start_cursor=start_cursor, + end_cursor=end_cursor, + ) + left_cte: str = f"SELECT * FROM {left}" + right_cte: str = f"SELECT * FROM {right}" + if cursor_filter: + left_cte += f" WHERE {cursor_filter}" + right_cte += f" WHERE {cursor_filter}" + self.validate_row_diff_keys( + connection=connection, relation_sql=left_cte, relation_label="left", keys=keys + ) + self.validate_row_diff_keys( + connection=connection, relation_sql=right_cte, relation_label="right", keys=keys + ) + prepared_relations: RowDiffPreparedRelations = self._build_row_diff_relation_ctes( + connection=connection, + left_sql=left_cte, + right_sql=right_cte, + keys=keys, + sampling=sampling, + inspect_population=False, + ) + join_condition: str = " AND ".join(f"__left.{k} = __right.{k}" for k in keys) + key_select_sql: str = ", ".join( + f"COALESCE(__left.{k}, __right.{k}) AS __key_{k}" for k in keys + ) + if side == DIFF_LEFT_SIDE: + side_condition: str = f"__left.{keys[0]} IS NOT NULL AND __right.{keys[0]} IS NULL" + elif side == DIFF_RIGHT_SIDE: + side_condition = f"__right.{keys[0]} IS NOT NULL AND __left.{keys[0]} IS NULL" + else: + raise AdapterUserError(message="sample_side_only_rows side must be 'left' or 'right'") + sample_sql: str = ( + f"WITH {prepared_relations.cte_sql} " + f"SELECT {key_select_sql} " + f"FROM __left FULL OUTER JOIN __right ON {join_condition} " + f"WHERE {side_condition} " + f"ORDER BY {', '.join(f'__key_{k}' for k in keys)} LIMIT {limit}" + ) + rows: list[tuple[Any, ...]] = self.execute(connection=connection, sql=sample_sql).fetchall() + samples: list[tuple[tuple[str, object], ...]] = [] + row: tuple[Any, ...] + for row in rows: + samples.append(tuple(zip(keys, row, strict=False))) + return tuple(samples) + + def validate_row_diff_keys( + self, + *, + connection: Any, + relation_sql: str, + relation_label: str, + keys: tuple[str, ...], + ) -> None: + if not keys: + raise AdapterUserError(message="row diff requires at least one unique_key column") + null_condition: str = " OR ".join(f"{key} IS NULL" for key in keys) + null_row: tuple[Any, ...] = self.execute( + connection=connection, + sql=f"SELECT COUNT(*) FROM ({relation_sql}) AS __key_check WHERE {null_condition}", + ).fetchone() + if int(null_row[0]) > 0: + raise AdapterUserError( + message=f"row diff {relation_label} relation contains null unique_key values" + ) + key_list: str = ", ".join(keys) + duplicate_row: tuple[Any, ...] = self.execute( + connection=connection, + sql=( + f"SELECT COUNT(*) FROM (SELECT {key_list} FROM ({relation_sql}) AS __key_check " + f"GROUP BY {key_list} HAVING COUNT(*) > 1) AS __duplicates" + ), + ).fetchone() + if int(duplicate_row[0]) > 0: + raise AdapterUserError( + message=f"row diff {relation_label} relation contains duplicate unique_key values" + ) + + def build_row_diff_equal_expression( + self, + *, + column: str, + column_info: ColumnInfo, + tolerances: RowDiffTolerances | None, + ) -> str: + """Compare with tolerances using SQLite's multi-argument max(), which has no GREATEST.""" + + tolerance: RowDiffTolerance | None = self.resolve_row_diff_tolerance( + column=column, + column_type=column_info.type, + tolerances=tolerances, + ) + left_expression: str = f"__left.{column}" + right_expression: str = f"__right.{column}" + if tolerance is None: + return f"{left_expression} IS NOT DISTINCT FROM {right_expression}" + threshold_parts: list[str] = [] + if tolerance.absolute is not None: + threshold_parts.append(self.format_row_diff_decimal_sql(tolerance.absolute)) + if tolerance.relative is not None: + threshold_parts.append( + f"{self.format_row_diff_decimal_sql(tolerance.relative)} * " + f"max(ABS({left_expression}), ABS({right_expression}))" + ) + threshold_sql: str = threshold_parts[0] + if len(threshold_parts) > 1: + threshold_sql = f"max({', '.join(threshold_parts)})" + return ( + f"(({left_expression} IS NULL AND {right_expression} IS NULL) OR " + f"({left_expression} IS NOT NULL AND {right_expression} IS NOT NULL AND " + f"ABS({left_expression} - {right_expression}) <= {threshold_sql}))" + ) + + def resolve_row_diff_tolerance( + self, + *, + column: str, + column_type: str, + tolerances: RowDiffTolerances | None, + ) -> RowDiffTolerance | None: + if tolerances is None: + return None + column_tolerance: RowDiffTolerance | None = tolerances.by_column.get(column) + if column_tolerance is not None: + if self.normalize_row_diff_numeric_type(column_type) is None: + raise AdapterUserError( + message=f"row diff tolerance for non-numeric column '{column}' is invalid" + ) + self.validate_row_diff_tolerance(column=column, tolerance=column_tolerance) + return column_tolerance + normalized_type: str | None = self.normalize_row_diff_numeric_type(column_type) + if normalized_type is None: + return None + type_tolerance: RowDiffTolerance | None = tolerances.by_type.get(normalized_type) + if type_tolerance is not None: + self.validate_row_diff_tolerance(column=column, tolerance=type_tolerance) + return type_tolerance + + def validate_row_diff_tolerance(self, *, column: str, tolerance: RowDiffTolerance) -> None: + if tolerance.absolute is None and tolerance.relative is None: + raise AdapterUserError( + message=f"row diff tolerance for column '{column}' must define absolute or relative" + ) + + def normalize_row_diff_numeric_type(self, column_type: str) -> str | None: + return normalize_numeric_family(type_sql=column_type, dialect=self.sql_analysis_dialect()) + + def format_row_diff_decimal_sql(self, value: Decimal) -> str: + return format(value, "f") + + def _render_row_diff_digest(self, payload_sql: str) -> str: + """Hash through the deterministic function every connection registers.""" + + return f"{HASH_FUNCTION_NAME}({payload_sql})" + + def render_create_initial_snapshot_destination( + self, + *, + table_type: str, + destination: str, + origin: str, + snapshot_strategy: str | None, + updated_at_column: str | None, + observed_at_column: str | None, + valid_from_column: str, + valid_to_column: str, + initial_valid_from: str | None, + ) -> tuple[str, ...]: + return self.render_create_table_as( + destination=destination, + sql=SnapshotSql(dialect=self._snapshot_sql_dialect).initial_select_sql( + origin=origin, + snapshot_strategy=snapshot_strategy, + updated_at_column=updated_at_column, + observed_at_column=observed_at_column, + valid_from_column=valid_from_column, + valid_to_column=valid_to_column, + initial_valid_from=initial_valid_from, + current_timestamp=self.render_current_timestamp(), + ), + ) + + def render_apply_timestamp_snapshot_changes( + self, + *, + destination: str, + origin: str, + unique_key: tuple[str, ...], + updated_at_column: str, + observed_at_column: str | None, + valid_from_column: str, + valid_to_column: str, + initial_valid_from: str | None, + output_columns: tuple[str, ...], + invalidate_hard_deletes: bool, + ) -> tuple[str, ...]: + return SnapshotSql(dialect=self._snapshot_sql_dialect).timestamp_changes_sql( + target=SnapshotChangeTarget( + destination=destination, + origin=origin, + unique_key=unique_key, + valid_from_column=valid_from_column, + valid_to_column=valid_to_column, + output_columns=output_columns, + ), + updated_at_column=updated_at_column, + observed_at_column=observed_at_column, + initial_valid_from=initial_valid_from, + invalidate_hard_deletes=invalidate_hard_deletes, + current_timestamp=self.render_current_timestamp(), + ) + + def render_apply_check_snapshot_changes( + self, + *, + target: SnapshotChangeTarget, + check_columns: tuple[str, ...], + updated_at_column: str | None, + observed_at_column: str | None, + initial_valid_from: str | None, + invalidate_hard_deletes: bool, + ) -> tuple[str, ...]: + return SnapshotSql(dialect=self._snapshot_sql_dialect).check_changes_sql( + target=target, + check_columns=check_columns, + updated_at_column=updated_at_column, + observed_at_column=observed_at_column, + initial_valid_from=initial_valid_from, + invalidate_hard_deletes=invalidate_hard_deletes, + current_timestamp=self.render_current_timestamp(), + ) + + def render_create_initial_historical_timestamp_snapshot_destination( + self, + *, + table_type: str, + destination: str, + origin: str, + unique_key: tuple[str, ...], + updated_at_column: str, + observed_at_column: str, + valid_from_column: str, + valid_to_column: str, + output_columns: tuple[str, ...], + invalidate_hard_deletes: bool, + ) -> tuple[str, ...]: + historical_sql: str = HistoricalTimestampSnapshotSql( + dialect=self._snapshot_sql_dialect + ).initial_select_sql( + origin=origin, + unique_key=unique_key, + updated_at_column=updated_at_column, + observed_at_column=observed_at_column, + valid_from_column=valid_from_column, + valid_to_column=valid_to_column, + output_columns=output_columns, + invalidate_hard_deletes=invalidate_hard_deletes, + ) + return self.render_create_table_as(destination=destination, sql=historical_sql) + + def render_create_initial_historical_timestamp_changes_destination( + self, + *, + table_type: str, + destination: str, + origin: str, + unique_key: tuple[str, ...], + updated_at_column: str, + valid_from_column: str, + valid_to_column: str, + output_columns: tuple[str, ...], + ) -> tuple[str, ...]: + historical_sql: str = HistoricalTimestampSnapshotSql.changes_initial_select_sql( + origin=origin, + unique_key=unique_key, + updated_at_column=updated_at_column, + valid_from_column=valid_from_column, + valid_to_column=valid_to_column, + output_columns=output_columns, + ) + return self.render_create_table_as(destination=destination, sql=historical_sql) + + def render_apply_historical_timestamp_snapshot_changes( + self, + *, + destination: str, + origin: str, + unique_key: tuple[str, ...], + updated_at_column: str, + observed_at_column: str, + valid_from_column: str, + valid_to_column: str, + output_columns: tuple[str, ...], + invalidate_hard_deletes: bool, + ) -> tuple[str, ...]: + return HistoricalTimestampSnapshotSql(dialect=self._snapshot_sql_dialect).apply_sql( + destination=destination, + origin=origin, + unique_key=unique_key, + updated_at_column=updated_at_column, + observed_at_column=observed_at_column, + valid_from_column=valid_from_column, + valid_to_column=valid_to_column, + output_columns=output_columns, + invalidate_hard_deletes=invalidate_hard_deletes, + ) + + def render_apply_historical_timestamp_changes( + self, + *, + destination: str, + origin: str, + unique_key: tuple[str, ...], + updated_at_column: str, + valid_from_column: str, + valid_to_column: str, + output_columns: tuple[str, ...], + ) -> tuple[str, ...]: + return HistoricalTimestampSnapshotSql(dialect=self._snapshot_sql_dialect).changes_apply_sql( + destination=destination, + origin=origin, + unique_key=unique_key, + updated_at_column=updated_at_column, + valid_from_column=valid_from_column, + valid_to_column=valid_to_column, + output_columns=output_columns, + ) + + def render_create_initial_historical_check_snapshot_destination( + self, + *, + table_type: str, + destination: str, + origin: str, + unique_key: tuple[str, ...], + check_columns: tuple[str, ...], + observed_at_column: str, + valid_from_column: str, + valid_to_column: str, + output_columns: tuple[str, ...], + invalidate_hard_deletes: bool, + ) -> tuple[str, ...]: + historical_sql: str = HistoricalCheckSnapshotSql( + dialect=self._snapshot_sql_dialect + ).initial_select_sql( + origin=origin, + unique_key=unique_key, + check_columns=check_columns, + observed_at_column=observed_at_column, + valid_from_column=valid_from_column, + valid_to_column=valid_to_column, + output_columns=output_columns, + invalidate_hard_deletes=invalidate_hard_deletes, + ) + return self.render_create_table_as(destination=destination, sql=historical_sql) + + def render_apply_historical_check_snapshot_changes( + self, + *, + destination: str, + origin: str, + unique_key: tuple[str, ...], + check_columns: tuple[str, ...], + observed_at_column: str, + valid_from_column: str, + valid_to_column: str, + output_columns: tuple[str, ...], + invalidate_hard_deletes: bool, + ) -> tuple[str, ...]: + return HistoricalCheckSnapshotSql(dialect=self._snapshot_sql_dialect).apply_sql( + destination=destination, + origin=origin, + unique_key=unique_key, + check_columns=check_columns, + observed_at_column=observed_at_column, + valid_from_column=valid_from_column, + valid_to_column=valid_to_column, + output_columns=output_columns, + invalidate_hard_deletes=invalidate_hard_deletes, + ) + + def star_exclude_keyword(self) -> str: + """Return EXCLUDE; SQLite has no star exclusion, so casts list columns explicitly.""" + + return "EXCLUDE" + + def render_qualified_name( + self, + *, + database: str | None, + schema: str | None, + name: str, + ) -> str | None: + """Render `schema__name`, with `main` unprefixed; one file has no databases.""" + + if database is not None: + raise AdapterUserError( + message=( + f"SQLite relation '{name}' cannot name database '{database}': a SQLite " + "connection is one file, so targets and models set only a schema" + ), + code=INVALID_CONNECTION_CODE, + ) + if schema is None: + return None + physical: str = physical_name(schema=schema, name=name) + self._logical_relations.remember(schema=schema, name=name, physical=physical) + return physical + + def render_identifier(self, name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + def render_exact_identifier(self, name: str) -> str: + return self.render_identifier(name) + + def render_framework_type(self, type_name: FrameworkType) -> str: + match type_name: + case FrameworkType.STRING: + return "TEXT" + case FrameworkType.INTEGER: + return "INTEGER" + case FrameworkType.TIMESTAMP: + return "TIMESTAMP" + + def render_loader_logical_type(self, type_name: LoaderLogicalType) -> str: + match type_name: + case LoaderLogicalType.BOOLEAN: + return "BOOLEAN" + case LoaderLogicalType.INTEGER: + return "INTEGER" + case LoaderLogicalType.FLOAT: + return "REAL" + case LoaderLogicalType.STRING: + return "TEXT" + case LoaderLogicalType.TIMESTAMP: + return "TIMESTAMP" + case LoaderLogicalType.DATE: + return "DATE" + case LoaderLogicalType.JSON: + return "TEXT" + + def render_typed_scalar(self, *, value: SqlValue) -> str: + return _render_ansi_typed_scalar(value=value) + + def render_typed_value_list(self, *, value: SqlValue) -> str: + return _render_typed_value_list(value=value, render_scalar=self.render_typed_scalar) + + def render_typed_array(self, *, value: SqlValue) -> str: + """Render arrays as JSON text, SQLite's only collection representation.""" + + return _quote_sql_string(_encode_typed_json(value)) + + def render_typed_object(self, *, value: SqlValue) -> str: + return _quote_sql_string(_encode_typed_json(value)) + + def render_loader_value_literal( + self, *, value: object, logical_type: LoaderLogicalType | None + ) -> str: + if value is None: + return "NULL" + if logical_type == LoaderLogicalType.JSON or isinstance(value, dict | list): + return _quote_sql_string(json.dumps(value, sort_keys=True)) + if isinstance(value, bool): + return "1" if value else "0" + if isinstance(value, int | float | Decimal): + return str(value) + if isinstance(value, datetime | date): + return _quote_sql_string(value.isoformat()) + return _quote_sql_string(str(value)) + + def render_loader_rows_select( + self, + *, + rows: tuple[dict[str, object], ...], + column_names: tuple[str, ...], + column_sql_types: dict[str, str], + inferred_types: dict[str, LoaderLogicalType], + ) -> str: + """Render rows as one multi-row VALUES; SQLite names VALUES columns column1, column2.""" + + if not rows: + projections: str = ", ".join( + f"CAST(NULL AS {column_sql_types.get(column_name, 'TEXT')}) AS " + f"{self.render_identifier(column_name)}" + for column_name in column_names + ) + return f"SELECT {projections} WHERE 1 = 0" + value_rows: list[str] = [] + row: dict[str, object] + for row in rows: + literals: str = ", ".join( + self.render_loader_value_literal( + value=row.get(column_name), logical_type=inferred_types.get(column_name) + ) + for column_name in column_names + ) + value_rows.append(f"({literals})") + values_sql: str = ", ".join(value_rows) + select_sql: str = ", ".join( + ( + f"column{index} AS {self.render_identifier(column_name)}" + if column_name not in column_sql_types + else f"CAST(column{index} AS {column_sql_types[column_name]}) " + f"AS {self.render_identifier(column_name)}" + ) + for index, column_name in enumerate(column_names, start=1) + ) + return f"SELECT {select_sql} FROM (VALUES {values_sql}) AS __loader_rows" + + def render_source_expression_cast( + self, *, expression: str, target_type: str, alias: str + ) -> str: + """Cast, rendering exact decimals as canonical fixed-scale text.""" + + scale: int | None = decimal_scale(target_type) + if scale is None: + return f"CAST({expression} AS {target_type}) AS {alias}" + return ( + f"CASE WHEN {expression} IS NULL THEN NULL " + f"ELSE printf('%.{scale}f', {expression}) END AS {alias}" + ) + + def render_source_expression_relation(self, *, expression: str) -> str: + stripped_expression: str = expression.strip().removesuffix(";").strip() + if stripped_expression.startswith("("): + return stripped_expression + if stripped_expression.lower().startswith(("select", "with", "values")): + return f"({stripped_expression})" + return stripped_expression + + def render_source_freshness_max_query( + self, *, column: str, source_relation: str, source_is_subquery: bool, where_sql: str + ) -> str: + relation: str = ( + f"{source_relation} AS __source_freshness" if source_is_subquery else source_relation + ) + return ( + f"SELECT MAX({self.render_identifier(column)}) AS data_version " + f"FROM {relation}{where_sql}" + ) + + def render_source_expression_cast_subquery( + self, *, source_relation: str, projections: tuple[str, ...] + ) -> str: + projection_clause: str = ", ".join(projections) + return f"(SELECT {projection_clause} FROM {source_relation} AS __source_expression)" + + def render_source_relation_cast_subquery( + self, + *, + source_relation: str, + cast_projections: tuple[str, ...], + cast_column_names: tuple[str, ...], + all_columns_cast: bool, + ) -> str: + cast_clause: str = ", ".join(cast_projections) + if all_columns_cast: + return f"(SELECT {cast_clause} FROM {source_relation})" + raise AdapterUserError( + message=( + "SQLite has no SELECT * EXCLUDE; partial source casts need the warehouse columns" + ), + code=UNSUPPORTED_FEATURE_CODE, + ) + + def _render_source_relation_cast_subquery_with_columns( + self, + *, + source_relation: str, + cast_projections: tuple[str, ...], + cast_column_names: tuple[str, ...], + warehouse_column_names: tuple[str, ...], + all_columns_cast: bool, + ) -> str: + return _render_explicit_source_relation_cast_subquery( + source_relation=source_relation, + cast_projections=cast_projections, + cast_column_names=cast_column_names, + warehouse_column_names=warehouse_column_names, + all_columns_cast=all_columns_cast, + render_identifier=self.render_identifier, + ) + + def requires_derived_table_aliases(self) -> bool: + return False + + def render_set_difference_operator(self) -> str: + return "EXCEPT" + + def default_schema(self) -> str: + return MAIN_SCHEMA + + def default_database(self) -> str | None: + return None + + def describe_relation(self, *, connection: Any, relation: str) -> tuple[ColumnInfo, ...]: + return relation_columns(connection=connection, table_name=unquote_relation(relation)) + + def render_create_table_as(self, *, destination: str, sql: str) -> tuple[str, ...]: + return (f"DROP TABLE IF EXISTS {destination}", f"CREATE TABLE {destination} AS {sql}") + + def render_rename(self, *, origin: str, destination: str) -> tuple[str, ...]: + return (f"ALTER TABLE {origin} RENAME TO {destination}",) + + def render_rename_view(self, *, origin: str, destination: str) -> tuple[str, ...]: + """Render ALTER VIEW, which the adapter runs by recreating the view under the new name.""" + + return (f"ALTER VIEW {origin} RENAME TO {destination}",) + + def render_create_fingerprint_table_sql( + self, + *, + database: str | None, + schema: str, + ) -> str: + from sqlbuild._native import build_create_table_sql + + return build_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + + def render_create_fingerprint_index_sqls( + self, + *, + database: str | None, + schema: str, + ) -> tuple[str, ...]: + from sqlbuild._native import FINGERPRINT_TABLE_NAME + + return self._index_sqls( + database=database, + schema=schema, + table=FINGERPRINT_TABLE_NAME, + indexes=(("latest_idx", "node_type, node_name, ts DESC, run_id DESC"),), + ) + + def render_create_microbatch_state_index_sqls( + self, *, database: str | None, schema: str + ) -> tuple[str, ...]: + """Return no indexes: the state table serves only concurrency, which SQLite refuses.""" + + del database, schema + return () + + def render_read_latest_fingerprints_sql( + self, + *, + database: str | None, + schema: str, + ) -> str: + from sqlbuild._native import build_read_latest_sql + + return build_read_latest_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + ) + + def render_prune_fingerprint_history_sql( + self, + *, + database: str | None, + schema: str, + retain_versions: int, + ) -> str: + from sqlbuild._native import FINGERPRINT_TABLE_NAME + + return self._prune_history_sql( + table=self.render_qualified_name( + database=database, schema=schema, name=FINGERPRINT_TABLE_NAME + ), + partition="node_type, node_name", + order="ts DESC, run_id DESC", + retain_versions=retain_versions, + ) + + def render_create_source_freshness_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild._native import build_source_freshness_create_table_sql + + return build_source_freshness_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + + def render_create_source_freshness_index_sqls( + self, + *, + database: str | None, + schema: str, + ) -> tuple[str, ...]: + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME + + return self._index_sqls( + database=database, + schema=schema, + table=SOURCE_FRESHNESS_TABLE_NAME, + indexes=( + ( + "latest_idx", + "source_name, target_database, target_schema, target_name, " + "observed_at DESC, run_id DESC", + ), + ), + ) + + def render_insert_source_freshness_records_sql( + self, + *, + database: str | None, + schema: str, + records: tuple[SourceFreshnessRecord, ...], + ) -> str: + return render_insert_source_freshness_records_sql( + database=database, + schema=schema, + records=records, + render_qualified_name=self.render_qualified_name, + ) + + def render_create_node_result_table_sql( + self, + *, + database: str | None, + schema: str, + ) -> str: + from sqlbuild.executor.node_results.main.create_table_sql import ( + build_node_results_create_table_sql, + ) + + return build_node_results_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + + def render_create_node_result_index_sqls( + self, + *, + database: str | None, + schema: str, + ) -> tuple[str, ...]: + from sqlbuild.executor.node_results.constants import NODE_RESULTS_TABLE_NAME + + return self._index_sqls( + database=database, + schema=schema, + table=NODE_RESULTS_TABLE_NAME, + indexes=( + ( + "latest_idx", + "node_type, node_name, target_database, target_schema, target_name, " + "status, ts DESC, run_id DESC", + ), + ( + "run_id_idx", + "run_id, node_type, node_name, target_database, target_schema, target_name", + ), + ), + ) + + def render_create_audit_result_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.executor.audit_results.main.create_table_sql import ( + build_audit_results_create_table_sql, + ) + + return build_audit_results_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + + def render_create_audit_result_index_sqls( + self, *, database: str | None, schema: str + ) -> tuple[str, ...]: + from sqlbuild.executor.audit_results.constants import AUDIT_RESULTS_TABLE_NAME + + return self._index_sqls( + database=database, + schema=schema, + table=AUDIT_RESULTS_TABLE_NAME, + indexes=( + ( + "latest_idx", + "audit_name, binding_key, execution_fingerprint, run_scope_phase, " + "occurred_at DESC, result_id DESC", + ), + ("run_id_idx", "run_id, invocation_id, result_id"), + ), + ) + + def render_create_janitor_event_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild.executor.janitor_events.main.create_table_sql import ( + build_janitor_events_create_table_sql, + ) + + return build_janitor_events_create_table_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + + def render_create_migration_state_table_sql(self, *, database: str | None, schema: str) -> str: + from sqlbuild._native import render_migration_state_create_sql + + return render_migration_state_create_sql( + kind="model", + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + + def render_create_old_name_view_state_table_sql( + self, *, database: str | None, schema: str + ) -> str: + from sqlbuild._native import render_migration_state_create_sql + + return render_migration_state_create_sql( + kind="old_name", + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + + def render_create_column_migration_state_table_sql( + self, *, database: str | None, schema: str + ) -> str: + from sqlbuild._native import render_migration_state_create_sql + + return render_migration_state_create_sql( + kind="column", + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + render_framework_type=self.render_framework_type, + ) + + def render_read_latest_source_freshness_sql( + self, + *, + database: str | None, + schema: str, + ) -> str: + from sqlbuild._native import build_source_freshness_read_latest_sql + + return build_source_freshness_read_latest_sql( + database=database, + schema=schema, + render_qualified_name=self.render_qualified_name, + ) + + def render_prune_source_freshness_history_sql( + self, + *, + database: str | None, + schema: str, + retain_versions: int, + ) -> str: + from sqlbuild._native import SOURCE_FRESHNESS_TABLE_NAME + + return self._prune_history_sql( + table=self.render_qualified_name( + database=database, schema=schema, name=SOURCE_FRESHNESS_TABLE_NAME + ), + partition="source_name, target_database, target_schema, target_name", + order="observed_at DESC, run_id DESC", + retain_versions=retain_versions, + ) + + def sql_analysis_dialect(self) -> str | None: + return self.sql_analysis_dialect_name + + def expression_inference_profile(self) -> ExpressionInferenceProfile: + return ExpressionInferenceProfile(sql_analysis_dialect=self.sql_analysis_dialect()) + + def render_cursor_bound_literal(self, *, value: str, cursor_type: str | None) -> str: + """Render bounds as literals; timestamps in the text form SQLite stores and compares.""" + + if str(cursor_type).lower() == CursorKind.INTEGER: + return value + if value in {MICROBATCH_START_SENTINEL, MICROBATCH_END_SENTINEL}: + return f"{CURSOR_TEXT_FUNCTION_NAME}({_quote_sql_string(value)})" + return _quote_sql_string(canonical_timestamp_text(value)) + + def default_table_promotion_mode(self) -> TablePromotionMode: + return TablePromotionMode.STAGED + + def default_promotion_strategy(self) -> PromotionStrategy: + return PromotionStrategy.ATOMIC_SWAP + + def _cursor_value_literal(self, cursor: CursorValue) -> str: + value: datetime | int = cursor.value + text: str = value.isoformat() if isinstance(value, datetime) else str(value) + return self.render_cursor_bound_literal(value=text, cursor_type=cursor.kind) + + def _index_sqls( + self, + *, + database: str | None, + schema: str, + table: str, + indexes: tuple[tuple[str, str], ...], + ) -> tuple[str, ...]: + """Name indexes after their table, since SQLite index names span the whole file.""" + + table_name: str | None = self.render_qualified_name( + database=database, schema=schema, name=table + ) + if table_name is None: + return () + return tuple( + f"CREATE INDEX IF NOT EXISTS {quote_identifier(f'{table_name}_{suffix}')} " + f"ON {table_name} ({columns})" + for suffix, columns in indexes + ) + + def _prune_history_sql( + self, *, table: str | None, partition: str, order: str, retain_versions: int + ) -> str: + if table is None: + return "" + return ( + f"DELETE FROM {table} WHERE rowid IN (" + "SELECT rowid FROM (" + f"SELECT rowid, ROW_NUMBER() OVER (PARTITION BY {partition} ORDER BY {order}) " + f"AS __sqlbuild_history_rank FROM {table}" + ") AS __sqlbuild_ranked " + f"WHERE __sqlbuild_history_rank > {retain_versions})" + ) + + +def _lowered(types: object) -> Mapping[str, str]: + if not isinstance(types, Mapping): + return MappingProxyType({}) + return MappingProxyType({str(name).lower(): str(kind) for name, kind in types.items()}) + + +def _cursor_text(value: str | int | float | None) -> str | int | float | None: + return canonical_timestamp_text(value) if isinstance(value, str) else value + + +def _sqb_hash(value: object) -> str | None: + if value is None: + return None + return hashlib.md5(str(value).encode(), usedforsecurity=False).hexdigest() + + +def _seed_value(*, value: str | None, boolean: bool) -> object: + if value is None or not boolean: + return value + lowered: str = value.strip().lower() + if lowered in _TRUE_SEED_VALUES: + return 1 + if lowered in _FALSE_SEED_VALUES: + return 0 + return value diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_connection.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_connection.py new file mode 100644 index 0000000000..7a1c833f18 --- /dev/null +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_connection.py @@ -0,0 +1,103 @@ +"""SQLite connection wrapper with one in-process writer per database file.""" + +from __future__ import annotations + +import sqlite3 +from collections.abc import Callable, Iterable, Sequence + +from sqlbuild.adapter.contract.classes.observed_cursor import ObservedCursor +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapters.sqlite._helpers.statements import leading_keyword +from sqlbuild.adapters.sqlite.classes.write_gate import WriteGate +from sqlbuild.adapters.sqlite.classes.write_gates import WriteGates +from sqlbuild.adapters.sqlite.constants import ( + DATABASE_LOCKED_CODE, + LOCKED_ERROR_FRAGMENTS, + MEMORY_DATABASE_PATH, + READ_STATEMENT_KEYWORDS, +) + +_WRITE_GATES: WriteGates = WriteGates() + + +class SqliteConnection: + """Serialize this process's writers per file so only other processes wait on busy_timeout.""" + + def __init__( + self, *, raw_connection: sqlite3.Connection, path: str, busy_timeout_seconds: float + ) -> None: + self.raw_connection: sqlite3.Connection = raw_connection + self.path: str = path + self.busy_timeout_seconds: float = busy_timeout_seconds + self._gate: WriteGate = _WRITE_GATES.for_file( + path if path != MEMORY_DATABASE_PATH else f"{path}{id(raw_connection)}" + ) + self._holds_gate: bool = False + + def execute(self, sql: str) -> ObservedCursor: + return self.execute_with(sql=sql, parameters=()) + + def execute_with(self, *, sql: str, parameters: Sequence[object]) -> ObservedCursor: + return self._run(sql=sql, run=lambda cursor: cursor.execute(sql, parameters)) + + def executemany(self, *, sql: str, rows: Iterable[Sequence[object]]) -> ObservedCursor: + return self._run(sql=sql, run=lambda cursor: cursor.executemany(sql, rows)) + + def _run(self, *, sql: str, run: Callable[[ObservedCursor], object]) -> ObservedCursor: + self._enter(sql) + cursor: ObservedCursor = self.cursor() + try: + run(cursor) + except sqlite3.Error as error: + coded: AdapterUserError | None = self._locked_error(error) + if coded is None: + raise + raise coded from error + finally: + self._leave() + return cursor + + def cursor(self) -> ObservedCursor: + return ObservedCursor(raw_cursor=self.raw_connection.cursor(), adapter="sqlite") + + def in_transaction(self) -> bool: + return self.raw_connection.in_transaction + + def interrupt(self) -> None: + self.raw_connection.interrupt() + + def close(self) -> None: + try: + self.raw_connection.close() + finally: + self._release() + + def _enter(self, sql: str) -> None: + if self._holds_gate or leading_keyword(sql) in READ_STATEMENT_KEYWORDS: + return + self._holds_gate = self._gate.acquire() + + def _leave(self) -> None: + if self._holds_gate and not self.raw_connection.in_transaction: + self._release() + + def _release(self) -> None: + if self._holds_gate: + self._holds_gate = False + self._gate.release() + + def _locked_error(self, error: sqlite3.Error) -> AdapterUserError | None: + message: str = str(error) + if any(fragment in message.lower() for fragment in LOCKED_ERROR_FRAGMENTS): + return AdapterUserError( + message=( + f"SQLite database '{self.path}' is locked by another writer " + f"(waited {self.busy_timeout_seconds:g}s)" + ), + code=DATABASE_LOCKED_CODE, + help=( + "Another process is writing to this file. Retry when it finishes, or raise " + "busy_timeout in the connection settings." + ), + ) + return None diff --git a/src/sqlbuild/adapters/sqlite/classes/write_gate.py b/src/sqlbuild/adapters/sqlite/classes/write_gate.py new file mode 100644 index 0000000000..a08a4fa496 --- /dev/null +++ b/src/sqlbuild/adapters/sqlite/classes/write_gate.py @@ -0,0 +1,32 @@ +"""One SQLite writer per database file per process.""" + +from __future__ import annotations + +import threading + +from sqlbuild.adapters.sqlite.constants import WRITE_GATE_POLL_SECONDS +from sqlbuild.runtime.execution_limits.main.enforce_execution_deadline import ( + enforce_execution_deadline, +) + + +class WriteGate: + """Queue a process's writers; a thread already writing through another connection passes.""" + + def __init__(self) -> None: + self._lock: threading.Lock = threading.Lock() + self._owner: int | None = None + + def acquire(self) -> bool: + """Wait for the gate, honouring deadlines; False when this thread already holds it.""" + + if self._owner == threading.get_ident(): + return False + while not self._lock.acquire(timeout=WRITE_GATE_POLL_SECONDS): + enforce_execution_deadline() + self._owner = threading.get_ident() + return True + + def release(self) -> None: + self._owner = None + self._lock.release() diff --git a/src/sqlbuild/adapters/sqlite/classes/write_gates.py b/src/sqlbuild/adapters/sqlite/classes/write_gates.py new file mode 100644 index 0000000000..4294d65214 --- /dev/null +++ b/src/sqlbuild/adapters/sqlite/classes/write_gates.py @@ -0,0 +1,19 @@ +"""Write gates shared by every connection to the same SQLite file.""" + +from __future__ import annotations + +import threading + +from sqlbuild.adapters.sqlite.classes.write_gate import WriteGate + + +class WriteGates: + """Hand out one gate per database file so all of a process's connections share it.""" + + def __init__(self) -> None: + self._gates: dict[str, WriteGate] = {} + self._guard: threading.Lock = threading.Lock() + + def for_file(self, key: str) -> WriteGate: + with self._guard: + return self._gates.setdefault(key, WriteGate()) diff --git a/src/sqlbuild/adapters/sqlite/constants.py b/src/sqlbuild/adapters/sqlite/constants.py new file mode 100644 index 0000000000..a542773fa3 --- /dev/null +++ b/src/sqlbuild/adapters/sqlite/constants.py @@ -0,0 +1,52 @@ +"""SQLite adapter constants.""" + +MINIMUM_SQLITE_VERSION: tuple[int, int, int] = (3, 44, 0) +DEFAULT_BUSY_TIMEOUT_SECONDS: float = 5.0 +MEMORY_DATABASE_PATH: str = ":memory:" +MAIN_SCHEMA: str = "main" +SCHEMA_NAME_SEPARATOR: str = "__" +SCHEMA_REGISTRY_TABLE: str = "_sqlbuild_schemas" +RELATION_REGISTRY_TABLE: str = "_sqlbuild_relations" +INTERNAL_TABLE_NAMES: frozenset[str] = frozenset({SCHEMA_REGISTRY_TABLE, RELATION_REGISTRY_TABLE}) +SQLITE_INTERNAL_PREFIX: str = "sqlite_" +CONNECTION_KEYS: frozenset[str] = frozenset({"path", "busy_timeout", "read_only"}) +HASH_FUNCTION_NAME: str = "sqb_hash" +CURSOR_TEXT_FUNCTION_NAME: str = "sqb_cursor_text" +LOCKED_ERROR_FRAGMENTS: tuple[str, ...] = ("database is locked", "database table is locked") +MISSING_RELATION_ERROR_FRAGMENT: str = "no such table" +READ_STATEMENT_KEYWORDS: frozenset[str] = frozenset({"SELECT", "PRAGMA", "EXPLAIN", "VALUES"}) +TRANSACTION_END_KEYWORDS: frozenset[str] = frozenset({"COMMIT", "END", "ROLLBACK"}) +WRITE_GATE_POLL_SECONDS: float = 0.1 +TYPEOF_TYPE_NAMES: dict[str, str] = { + "integer": "INTEGER", + "real": "REAL", + "text": "TEXT", + "blob": "BLOB", +} +UNTYPED_COLUMN_TYPE: str = "ANY" +DECIMAL_TEXT_TYPE_PREFIX: str = "TEXT " +DECIMAL_TYPE_NAMES: tuple[str, ...] = ("DECIMAL", "NUMERIC") +EXACT_DECIMAL_TYPE_PATTERN: str = r"^\s*(?:DECIMAL|NUMERIC)\s*\(\s*(\d+)\s*(?:,\s*(\d+)\s*)?\)\s*$" +NUMBER_DECIMAL_TYPE_PATTERN: str = r"^\s*(?:DECIMAL|NUMERIC)\b" +NUMBER_DECIMAL_REPORTED_TYPE: str = "NUMERIC" +BOOLEAN_TYPE_MARKER: str = "BOOL" +INSERT_KEYWORD: str = "INSERT" +SQLITE_VIEW_TYPE: str = "view" +SQLITE_TABLE_TYPE: str = "table" +IDENTIFIER_QUOTE: str = '"' +MAIN_SCHEMA_PREFIX: str = "main." +BASE_TABLE_RELATION_TYPE: str = "BASE TABLE" +VIEW_RELATION_TYPE: str = "VIEW" +VIEW_BODY_PATTERN: str = ( + r"^\s*CREATE\s+(?:TEMP\s+|TEMPORARY\s+)?VIEW\s+(?:IF\s+NOT\s+EXISTS\s+)?" + r"(?:\"(?:[^\"]|\"\")+\"|\[[^\]]+\]|`[^`]+`|[^\s(]+)\s*(?:\([^)]*\)\s*)?AS\s+(.*)$" +) + +VERSION_TOO_OLD_CODE: str = "A701" +DATABASE_LOCKED_CODE: str = "A702" +INVALID_CONNECTION_CODE: str = "A703" +INVALID_SCHEMA_NAME_CODE: str = "A704" +UNSUPPORTED_FEATURE_CODE: str = "A705" +RELATION_NAME_COLLISION_CODE: str = "A706" +TIMESTAMP_TEXT_FORMAT: str = "%Y-%m-%d %H:%M:%S" +DATE_TIME_PATTERN: str = r"^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}" diff --git a/src/sqlbuild/executor/run/_helpers/execution/inferred_types.py b/src/sqlbuild/executor/run/_helpers/execution/inferred_types.py new file mode 100644 index 0000000000..8c7232ef6f --- /dev/null +++ b/src/sqlbuild/executor/run/_helpers/execution/inferred_types.py @@ -0,0 +1,11 @@ +"""Compiled column types handed to adapters whose CTAS cannot keep expression types.""" + +from __future__ import annotations + +from sqlbuild._native import ModelPlanEntry + + +def inferred_column_types(entry: ModelPlanEntry) -> dict[str, str]: + """Return the model's inferred output column types by name.""" + + return {column.name: column.type for column in entry.inferred_columns} diff --git a/src/sqlbuild/executor/run/_helpers/execution/staging.py b/src/sqlbuild/executor/run/_helpers/execution/staging.py index 8b34bf380e..bab6eeabba 100644 --- a/src/sqlbuild/executor/run/_helpers/execution/staging.py +++ b/src/sqlbuild/executor/run/_helpers/execution/staging.py @@ -7,6 +7,7 @@ from sqlbuild._native import Fingerprint from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.executor.run._helpers.execution.inferred_types import inferred_column_types from sqlbuild.executor.run.models import ModelMaterializationContext from sqlbuild.runtime.observability.classes.operation_lifecycle import ( OperationAttributes, @@ -47,7 +48,10 @@ def create_staging_relation( connection=connection, destination=staging_qualified, sql=resolved_sql, - config={"table_type": context.entry.table_type}, + config={ + "table_type": context.entry.table_type, + "inferred_column_types": inferred_column_types(context.entry), + }, statement_recorder=statement_recorder, ) lifecycle.completed(metadata={"changed_count": 1}) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py index e0c222de4f..5e2ffcfc01 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/incremental.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/incremental.py @@ -37,6 +37,7 @@ run_post_hook_phase, run_pre_hook_phase, ) +from sqlbuild.executor.run._helpers.execution.inferred_types import inferred_column_types from sqlbuild.executor.run._helpers.execution.results import ( build_failed_result, build_skipped_result, @@ -510,6 +511,7 @@ def _prepare_delta_relation( connection=connection, destination=delta_qualified, sql=preparation.resolved_sql, + config={"inferred_column_types": inferred_column_types(context.entry)}, statement_recorder=statement_recorder, ) lifecycle.completed(metadata={"changed_count": 1}) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py index 27f888d74a..6a0b2a2c1f 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/microbatch.py @@ -78,6 +78,7 @@ run_post_hook_phase, run_pre_hook_phase, ) +from sqlbuild.executor.run._helpers.execution.inferred_types import inferred_column_types from sqlbuild.executor.run._helpers.execution.results import ( build_failed_result, build_skipped_result, @@ -2371,7 +2372,10 @@ def _physical_cursor_envelope( cursor_column: str | None = context.entry.cursor_column if cursor_column is None: return None - quoted_cursor: str = context.adapter.render_identifier(cursor_column) + quoted_cursor: str = context.adapter.render_cursor_operand( + expression=context.adapter.render_identifier(cursor_column), + cursor_type=context.entry.cursor_type, + ) try: cursor: Any = context.adapter.execute( connection=context.connection, @@ -2482,7 +2486,10 @@ def _count_unaccounted_intervals( targets: MicrobatchTargets, intervals: tuple[MicrobatchInterval, ...], ) -> dict[tuple[str, str], int] | ModelExecutionResult: - cursor_column: str = context.adapter.render_identifier(context.entry.cursor_column or "") + cursor_column: str = context.adapter.render_cursor_operand( + expression=context.adapter.render_identifier(context.entry.cursor_column or ""), + cursor_type=context.entry.cursor_type, + ) counts: dict[tuple[str, str], int] = {} chunk_start: int for chunk_start in range(0, len(intervals), _UNACCOUNTED_COUNT_CHUNK_SIZE): @@ -2805,6 +2812,7 @@ def _stage_microbatch_delta( connection=context.connection, destination=targets.delta_qualified, sql=batch_sql, + config={"inferred_column_types": inferred_column_types(context.entry)}, statement_recorder=state.statement_recorder, ) lifecycle.completed(metadata={"changed_count": 1}) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py index 0dcf20cd6d..88e075a15d 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py @@ -44,6 +44,7 @@ run_post_hook_phase, run_pre_hook_phase, ) +from sqlbuild.executor.run._helpers.execution.inferred_types import inferred_column_types from sqlbuild.executor.run._helpers.execution.results import ( build_failed_result, build_skipped_result, @@ -191,6 +192,7 @@ def execute_snapshot_entry( # noqa: PLR0915 connection=connection, destination=delta_qualified, sql=entry.resolved_sql, + config={"inferred_column_types": inferred_column_types(entry)}, statement_recorder=statement_recorder, ) lifecycle.completed(metadata={"changed_count": 1}) diff --git a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py index 5879375918..cedbd9543f 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py +++ b/src/sqlbuild/executor/run/_helpers/validation/cursor_bounds.py @@ -541,7 +541,9 @@ def _query_watermark_values( adapter=adapter, connection=connection, relation=relation, - cursor_column=cursor_column, + cursor_column=adapter.render_cursor_operand( + expression=cursor_column, cursor_type=cursor_type + ), read_minimum=read_minimum, query_index=query_index, total=total, @@ -706,7 +708,8 @@ def _query_target_max( name=target_name, ): return None - sql: str = f"SELECT MAX({cursor_column}) FROM {target_relation}" + operand: str = adapter.render_cursor_operand(expression=cursor_column, cursor_type=cursor_type) + sql: str = f"SELECT MAX({operand}) FROM {target_relation}" cursor: Any = adapter.execute(connection=connection, sql=sql) row: Any = cursor.fetchone() if row is None or row[0] is None: diff --git a/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py b/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py index e8863dcb82..41eb23fe1a 100644 --- a/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py +++ b/src/sqlbuild/executor/run/_helpers/validation/type_enforcement.py @@ -74,7 +74,7 @@ def enforce_types_staged( connection=connection, destination=enforced_qualified, sql=f"SELECT {projection_sql} FROM {staging_qualified}", - config={"table_type": table_type}, + config={"table_type": table_type, "column_types": declared_map}, statement_recorder=statement_recorder, ) adapter.drop( diff --git a/src/sqlbuild/executor/run/main/_execute.py b/src/sqlbuild/executor/run/main/_execute.py index 8562c2145c..3ba6b40c38 100644 --- a/src/sqlbuild/executor/run/main/_execute.py +++ b/src/sqlbuild/executor/run/main/_execute.py @@ -26,6 +26,7 @@ run_post_hook_phase, run_pre_hook_phase, ) +from sqlbuild.executor.run._helpers.execution.inferred_types import inferred_column_types from sqlbuild.executor.run._helpers.execution.promotion import promote_relation_to_destination from sqlbuild.executor.run._helpers.execution.results import ( build_failed_result, @@ -421,7 +422,10 @@ def _immediate_lifecycle( connection=connection, destination=target_qualified, sql=state.resolved_sql, - config={"table_type": entry.table_type}, + config={ + "table_type": entry.table_type, + "inferred_column_types": inferred_column_types(entry), + }, statement_recorder=statement_recorder, ) except Exception as exc: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/__init__.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py new file mode 100644 index 0000000000..7646b54426 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py @@ -0,0 +1,87 @@ +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class SqliteOracleE2ETestCase: + description: str + compared_models: tuple[tuple[str, str], ...] + unexpected_rebuild_fragments: tuple[str, ...] + expected_return_code: int = 0 + + +@dataclass(frozen=True) +class SqliteCommandE2ETestCase: + description: str + command: tuple[str, ...] + expected_fragments: tuple[str, ...] + expected_return_code: int = 0 + + +@dataclass(frozen=True) +class SqliteFileLayoutE2ETestCase: + description: str + expected_tables: tuple[str, ...] + absent_tables: tuple[str, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class SqliteProjectE2ETestCase: + description: str + files: dict[str, str] + changed_files: dict[str, str] + query: str + expected_rows: tuple[tuple[object, ...], ...] + absent_tables: tuple[str, ...] = field(default_factory=tuple) + expected_return_code: int = 0 + + +@dataclass(frozen=True) +class SqliteFailureE2ETestCase: + description: str + files: dict[str, str] + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class SqliteLockE2ETestCase: + description: str + busy_timeout: float + hold_seconds: float + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class SqliteScenarioE2ETestCase: + description: str + command: tuple[str, ...] + expected_outcomes: dict[str, tuple[str, str | None]] + expected_tables: tuple[str, ...] + expected_return_code: int = 1 + + +@dataclass(frozen=True) +class SqliteDecimalContractE2ETestCase: + description: str + source_files: dict[str, str] + expected_payment_rows: tuple[tuple[object, ...], ...] + expected_agreeing_rows: int + expected_evolved_rows: tuple[tuple[object, ...], ...] + expected_fee_declaration: str + + +@dataclass(frozen=True) +class SqliteCollisionE2ETestCase: + description: str + first_files: dict[str, str] + second_files: dict[str, str] + expected_fragments: tuple[str, ...] + expected_tables: tuple[tuple[object, ...], ...] + expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class SqliteCursorWindowE2ETestCase: + description: str + events_csv: str + commands: tuple[tuple[str, ...], ...] + expected_order_ids: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/helpers.py new file mode 100644 index 0000000000..2f2aa98e35 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/helpers.py @@ -0,0 +1,724 @@ +"""Helpers for SQLite CLI e2e tests.""" + +from __future__ import annotations + +import re +import sqlite3 +import subprocess +from collections.abc import Callable +from pathlib import Path +from types import MappingProxyType +from typing import Any + +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import query_duckdb, run_sqb + +SQLITE_FILE: str = "warehouse.sqlite" +DUCKDB_FILE: str = "warehouse.duckdb" +SCHEMA: str = "analytics" + +_CUSTOMERS_CSV: str = ( + "customer_id,customer_name,is_active,signed_up_at\n" + "1,Ada,true,2026-01-01 08:00:00\n" + "2,Grace,false,2026-01-02 09:30:00\n" + "3,Linus,true,2026-01-03 10:15:00\n" +) +_ORDER_ROWS: tuple[str, ...] = ( + "1,10,1,1250,2026-01-05 10:00:00,placed", + "2,11,1,725,2026-01-06 11:00:00,shipped", + "3,12,2,3000,2026-01-06 12:00:00,placed", + "4,13,3,500,2026-01-07 13:00:00,placed", +) +LATE_ORDER_ROWS: tuple[str, ...] = ( + "5,12,2,3100,2026-01-06 12:00:00,shipped", + "6,14,3,900,2026-01-08 09:00:00,placed", +) + +_SEEDS_YML: str = """seeds: + - name: customers + description: Customer list. + columns: + - name: customer_id + type: INTEGER + - name: customer_name + type: VARCHAR + - name: is_active + type: BOOLEAN + - name: signed_up_at + type: VARCHAR + - name: orders + description: Order events; a later row for an order replaces earlier ones. + columns: + - name: event_seq + type: INTEGER + - name: order_id + type: INTEGER + - name: customer_id + type: INTEGER + - name: amount_cents + type: INTEGER + - name: ordered_at + type: VARCHAR + - name: status + type: VARCHAR +""" + +_MODELS: dict[str, str] = { + "models/staging/stg_orders.sql": """MODEL ( + materialized view, + description "Latest event per order.", + columns ( + order_id (audits [not_null, unique]), + ), +); + +SELECT order_id, customer_id, amount_cents, ordered_at, status +FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY order_id ORDER BY event_seq DESC) AS event_rank + FROM __seed("orders") +) AS ranked +WHERE event_rank = 1 +""", + "models/marts/customer_totals.sql": """MODEL ( + materialized table, + description "Order totals per customer.", + columns ( + customer_id (audits [not_null, unique]), + ), +); + +SELECT c.customer_id, c.customer_name, COUNT(o.order_id) AS order_count, + COALESCE(SUM(o.amount_cents), 0) AS total_cents +FROM __seed("customers") AS c +LEFT JOIN __ref("stg_orders") AS o ON o.customer_id = c.customer_id +GROUP BY c.customer_id, c.customer_name +""", + "models/marts/orders_appended.sql": """MODEL ( + materialized incremental, + incremental_strategy append, + cursor order_id, + cursor_type integer, + description "Append-only order ids.", +); + +SELECT order_id, customer_id FROM __ref("stg_orders") +""", + "models/marts/orders_merged.sql": """MODEL ( + materialized incremental, + incremental_strategy merge, + unique_key [order_id], + description "Current order status, merged by key.", +); + +SELECT order_id, status, amount_cents FROM __ref("stg_orders") +""", + "models/marts/orders_replaced.sql": """MODEL ( + materialized incremental, + incremental_strategy delete_insert, + unique_key [order_id], + description "Current order status, replaced by key.", +); + +SELECT order_id, status, amount_cents FROM __ref("stg_orders") +""", +} + +_UNIT_TEST: str = """TEST(); + +WITH +__seed__customers AS ( + SELECT 1 AS customer_id, 'Ada' AS customer_name, 1 AS is_active, '2026-01-01' AS signed_up_at +), +__ref__stg_orders AS ( + SELECT 10 AS order_id, 1 AS customer_id, 250 AS amount_cents, + '2026-01-05' AS ordered_at, 'placed' AS status +), +__expected__customer_totals AS ( + SELECT 1 AS customer_id, 'Ada' AS customer_name, 1 AS order_count, 250 AS total_cents +) +SELECT 1 +""" + +COMPARED_MODELS: tuple[tuple[str, str], ...] = ( + ("stg_orders", "order_id"), + ("customer_totals", "customer_id"), + ("orders_appended", "order_id"), + ("orders_merged", "order_id"), + ("orders_replaced", "order_id"), +) + + +def build_orders_project_files(*, adapter: str) -> dict[str, str]: + """Build a synthetic orders project for one adapter, written in portable SQL.""" + + connection: str = _CONNECTIONS[adapter] + return { + "sqlbuild_project.toml": ( + 'name = "orders_shop"\n' + f'adapter = "{adapter}"\n' + 'default_target = "dev"\n\n' + f"[connection]\n{connection}\n" + "[targets.dev]\n" + f'schema = "{SCHEMA}"\n\n' + "[targets.dev.clone]\n" + "allow_as_clone_origin = true\n\n" + "[targets.prod]\n" + 'schema = "prod"\n\n' + "[targets.prod.clone]\n" + "allow_as_clone_destination = true\n\n" + "[defaults]\n" + 'materialized = "table"\n' + ), + "seeds/customers.csv": _CUSTOMERS_CSV, + "seeds/orders.csv": "event_seq,order_id,customer_id,amount_cents,ordered_at,status\n" + + "\n".join(_ORDER_ROWS) + + "\n", + "seeds/seeds.yml": _SEEDS_YML, + "tests/unit/test_customer_totals.sql": _UNIT_TEST, + **_MODELS, + } + + +def append_late_orders(*, project_dir: Path) -> None: + orders_path: Path = project_dir / "seeds" / "orders.csv" + with orders_path.open("a", encoding="utf-8") as orders_file: + orders_file.write("\n".join(LATE_ORDER_ROWS) + "\n") + + +def sqlite_rows(*, db_path: Path, sql: str) -> tuple[tuple[object, ...], ...]: + connection: sqlite3.Connection = sqlite3.connect(db_path) + try: + return tuple(tuple(row) for row in connection.execute(sql).fetchall()) + finally: + connection.close() + + +def model_rows( + *, adapter: str, project_dir: Path, model: str, order_by: str +) -> tuple[tuple[object, ...], ...]: + """Read one model's rows in a stable order from either engine.""" + + return _ROW_READERS[adapter](project_dir, model, order_by) + + +def _sqlite_model_rows( + project_dir: Path, model: str, order_by: str +) -> tuple[tuple[object, ...], ...]: + return sqlite_rows( + db_path=project_dir / SQLITE_FILE, + sql=f'SELECT * FROM "{SCHEMA}__{model}" ORDER BY {order_by}', + ) + + +def _duckdb_model_rows( + project_dir: Path, model: str, order_by: str +) -> tuple[tuple[object, ...], ...]: + return tuple( + tuple(row) + for row in query_duckdb( + db_path=project_dir / DUCKDB_FILE, + sql=f"SELECT * FROM {SCHEMA}.{model} ORDER BY {order_by}", + ) + ) + + +_CONNECTIONS: MappingProxyType[str, str] = MappingProxyType( + {"sqlite": f'path = "{SQLITE_FILE}"\n', "duckdb": f'database = "{DUCKDB_FILE}"\n'} +) +_ROW_READERS: MappingProxyType[str, Callable[[Path, str, str], tuple[tuple[object, ...], ...]]] = ( + MappingProxyType({"sqlite": _sqlite_model_rows, "duckdb": _duckdb_model_rows}) +) + + +_EVENTS_HEADER: str = "order_id,amount_cents,ordered_at\n" +FIRST_EVENTS_CSV: str = _EVENTS_HEADER + ( + "1,100,2026-01-01 10:00:00\n2,200,2026-01-01 11:00:00\n3,300,2026-01-02 09:00:00\n" +) +CHANGED_EVENTS_CSV: str = _EVENTS_HEADER + ( + "2,250,2026-01-02 12:00:00\n3,300,2026-01-02 09:00:00\n4,400,2026-01-03 08:00:00\n" +) + +_SQLITE_TOML: str = ( + 'name = "events_shop"\n' + 'adapter = "sqlite"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + f'path = "{SQLITE_FILE}"\n\n' + "[targets.dev]\n" + f'schema = "{SCHEMA}"\n' +) + +STRATEGY_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": _SQLITE_TOML, + "seeds/order_events.csv": FIRST_EVENTS_CSV, + "seeds/seeds.yml": """seeds: + - name: order_events + description: Order events. + columns: + - name: order_id + type: INTEGER + - name: amount_cents + type: INTEGER + - name: ordered_at + type: TIMESTAMP +""", + "models/daily_revenue.sql": """MODEL ( + description "Daily revenue in sequential microbatches.", + materialized incremental, + incremental_strategy delete_insert, + incremental_mode microbatch, + microbatch_strategy watermark, + cursor_watermark_mode all, + cursor order_day, + cursor_type timestamp, + cursor_grain day, + cursor_inputs (order_events (column ordered_at, roles [filter, watermark]),), + batch_size 1d, +); + +SELECT datetime(ordered_at, 'start of day') AS order_day, COUNT(*) AS order_count, + SUM(amount_cents) AS revenue_cents +FROM __seed("order_events") +WHERE ordered_at >= __cursor_start() AND ordered_at < __cursor_end() +GROUP BY 1 +""", + "models/orders_window.sql": """MODEL ( + description "Orders replaced by daily cursor window.", + materialized incremental, + incremental_strategy delete_insert, + unique_key order_id, + cursor ordered_at, + cursor_type timestamp, + cursor_grain day, + cursor_start "2026-01-01", +); + +SELECT order_id, amount_cents, ordered_at FROM __seed("order_events") +""", + "models/current_orders.sql": """MODEL (description "Current order state.", materialized table); + +SELECT order_id, amount_cents, ordered_at AS updated_at FROM __seed("order_events") +""", + "models/order_history.sql": """MODEL ( + description "Order history by update time.", + materialized snapshot, + unique_key [order_id], + snapshot_strategy timestamp, + updated_at updated_at, + invalidate_hard_deletes true, +); + +SELECT order_id, amount_cents, updated_at FROM __ref("current_orders") +""", + "models/order_amount_history.sql": """MODEL ( + description "Order amount history by checked columns.", + materialized snapshot, + unique_key [order_id], + snapshot_strategy check, + check_columns [amount_cents], +); + +SELECT order_id, amount_cents FROM __ref("current_orders") +""", +} + +CONTRACT_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": _SQLITE_TOML, + "models/order_amounts.sql": """MODEL ( + description "Order amounts with an exact decimal contract.", + contract enforced, + columns ( + order_id (type INTEGER, nullable false), + amount (type DECIMAL(10,2)), + ), +); + +SELECT 1 AS order_id, 12.5 AS amount +UNION ALL SELECT 2, 7 +""", +} + +FUNCTION_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": _SQLITE_TOML, + "functions/sql/is_large.sql": """FUNCTION ( + description "Whether an amount is large.", + arguments (amount INTEGER), + returns BOOLEAN, +); + +amount > 10 +""", + "models/large_orders.sql": """MODEL (description "Large orders.", materialized table); + +SELECT 1 AS order_id +""", +} + +CONCURRENT_MICROBATCH_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": _SQLITE_TOML + "\n[settings]\nmicrobatch_concurrency = true\n", + "seeds/order_events.csv": FIRST_EVENTS_CSV, + "seeds/seeds.yml": STRATEGY_PROJECT_FILES["seeds/seeds.yml"], + "models/daily_revenue.sql": STRATEGY_PROJECT_FILES["models/daily_revenue.sql"].replace( + " batch_size 1d,\n", " batch_size 1d,\n batch_concurrency 4,\n" + ), +} + +LOCK_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": _SQLITE_TOML.replace( + f'path = "{SQLITE_FILE}"\n', f'path = "{SQLITE_FILE}"\nbusy_timeout = 0.5\n' + ), + "models/order_ids.sql": """MODEL (description "Order ids.", materialized table); + +SELECT 1 AS order_id +""", +} + + +SCENARIO_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": _SQLITE_TOML, + "sources/raw.yml": """sources: + - name: raw_orders + description: Raw orders. + schema: main + table: raw_orders + columns: + - name: id + type: INTEGER + - name: customer_id + type: INTEGER + - name: amount + type: INTEGER + - name: ordered_at + type: TIMESTAMP +""", + "models/orders.sql": """MODEL (description 'Orders.', materialized table); + +SELECT id AS order_id, customer_id, amount, ordered_at FROM __source("raw_orders") +""", + "models/customer_totals.sql": """MODEL (description 'Totals per customer.', materialized table); + +SELECT customer_id, SUM(amount) AS total_amount, COUNT(*) AS order_count +FROM __ref("orders") +GROUP BY customer_id +""", + "models/daily_orders.sql": """MODEL ( + description 'Orders by day, incremental.', + materialized incremental, + incremental_strategy delete_insert, + unique_key order_day, + cursor order_day, + cursor_type timestamp, + cursor_grain day, + cursor_start "2026-01-01", +); + +SELECT date(ordered_at) AS order_day, COUNT(*) AS order_count +FROM __ref("orders") +GROUP BY 1 +""", + "tests/scenarios/customer_totals_pass.sql": """SCENARIO (description "Customer totals."); + +WITH +__source__raw_orders AS ( + SELECT 1 AS id, 10 AS customer_id, 10 AS amount, '2026-01-01 10:00:00' AS ordered_at + UNION ALL + SELECT 2, 10, 5, '2026-01-02 10:00:00' + UNION ALL + SELECT 3, 20, 7, '2026-01-02 11:00:00' +), +__expected__customer_totals AS ( + SELECT 10 AS customer_id, 15 AS total_amount, 2 AS order_count + UNION ALL + SELECT 20, 7, 1 +), +__expected__daily_orders AS ( + SELECT '2026-01-01' AS order_day, 1 AS order_count + UNION ALL + SELECT '2026-01-02', 2 +) +SELECT 1 +""", + "tests/scenarios/customer_totals_fail.sql": """SCENARIO (description 'Wrong totals.'); + +WITH +__source__raw_orders AS ( + SELECT 1 AS id, 10 AS customer_id, 10 AS amount, '2026-01-01 10:00:00' AS ordered_at +), +__expected__customer_totals AS ( + SELECT 10 AS customer_id, 11 AS total_amount, 1 AS order_count +) +SELECT 1 +""", + "tests/scenarios/nested/orders_assert_pass.sql": """SCENARIO (description 'No negatives.'); + +WITH +__source__raw_orders AS ( + SELECT 1 AS id, 10 AS customer_id, 10 AS amount, '2026-01-01 10:00:00' AS ordered_at +), +__assert__no_negative_orders AS ( + SELECT * FROM __ref("orders") WHERE amount < 0 +) +SELECT 1 +""", + "tests/scenarios/orders_assert_fail.sql": """SCENARIO (description 'Every order is large.'); + +WITH +__source__raw_orders AS ( + SELECT 1 AS id, 10 AS customer_id, 3 AS amount, '2026-01-01 10:00:00' AS ordered_at +), +__assert__small_orders AS ( + SELECT * FROM __ref("orders") WHERE amount < 5 +) +SELECT 1 +""", +} + +SCENARIO_OUTCOMES: dict[str, tuple[str, str | None]] = { + "customer_totals_pass": ("success", None), + "customer_totals_fail": ("failed", "X506"), + "orders_assert_pass": ("success", None), + "orders_assert_fail": ("failed", "X508"), +} + + +def scenario_outcomes(payload: dict[str, Any]) -> dict[str, tuple[str, str | None]]: + return { + str(scenario["name"]): (str(scenario["status"]), scenario.get("error_code")) + for scenario in payload["scenarios"] + } + + +def scenario_table_shapes(*, db_path: Path) -> tuple[str, ...]: + """Table names with each scenario run identifier replaced, sorted.""" + + return tuple( + sorted( + re.sub(r"__sqb_[0-9a-f]+__", "__sqb___", name) + for name in table_names(db_path=db_path) + ) + ) + + +def table_names(*, db_path: Path) -> tuple[str, ...]: + return tuple( + str(row[0]) + for row in sqlite_rows( + db_path=db_path, sql="SELECT name FROM sqlite_schema WHERE type = 'table'" + ) + ) + + +def hold_sqlite_write_lock(*, db_path: str, ready_path: str, hold_seconds: float) -> None: + """Child process: hold the file's write lock while a CLI writer tries to build.""" + + import time + + connection: sqlite3.Connection = sqlite3.connect(db_path, isolation_level=None) + connection.execute("PRAGMA journal_mode=WAL") + connection.execute("BEGIN IMMEDIATE") + connection.execute("CREATE TABLE IF NOT EXISTS lock_holder (held INTEGER)") + Path(ready_path).write_text("ready", encoding="utf-8") + time.sleep(hold_seconds) + connection.execute("COMMIT") + connection.close() + + +def execute_sqlite(*, db_path: Path, sql: str) -> None: + connection: sqlite3.Connection = sqlite3.connect(db_path) + try: + connection.execute(sql) + connection.commit() + finally: + connection.close() + + +def execute_sqlite_script(*, db_path: Path, sql: str) -> None: + connection: sqlite3.Connection = sqlite3.connect(db_path) + try: + connection.executescript(sql) + finally: + connection.close() + + +RAW_PAYMENTS_SQL: str = ( + "CREATE TABLE raw_payments (payment_id INTEGER, amount DECIMAL(10,2));" + "INSERT INTO raw_payments VALUES (1, 1.5), (2, 2);" +) +CHANGED_RAW_PAYMENTS_SQL: str = ( + "UPDATE raw_payments SET amount = 2.25 WHERE payment_id = 1;" + "INSERT INTO raw_payments VALUES (3, 4);" +) +_PAYMENT_SOURCES: str = """sources: + - name: raw_payments + description: Raw payments. + schema: main + table: raw_payments + columns: + - name: payment_id + type: INTEGER + - name: amount + type: DECIMAL(10,2) +""" +UNDECLARED_PAYMENT_SOURCES: str = """sources: + - name: raw_payments + description: Raw payments read with their warehouse types. + schema: main + table: raw_payments +""" +_PAYMENTS_MODEL: str = """MODEL ( + description "Payments merged by id under an exact decimal contract.", + materialized incremental, + incremental_strategy merge, + unique_key payment_id, + on_schema_change append_new_columns, + contract enforced, + columns ( + payment_id (type INTEGER, nullable false), + amount (type DECIMAL(10,2)), + ), +); + +SELECT payment_id, amount FROM __source("raw_payments") +""" +DECIMAL_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": _SQLITE_TOML, + "sources/raw.yml": _PAYMENT_SOURCES, + "models/payments.sql": _PAYMENTS_MODEL, + "models/payment_ledger.sql": """MODEL ( + description "Payments computed from numbers under the same contract.", + materialized table, + contract enforced, + columns ( + payment_id (type INTEGER, nullable false), + amount (type DECIMAL(10,2)), + ), +); + +SELECT payment_id, amount * 1.0 AS amount FROM __source("raw_payments") +""", +} +EVOLVED_DECIMAL_PROJECT_FILES: dict[str, str] = { + "models/payments.sql": _PAYMENTS_MODEL.replace( + " amount (type DECIMAL(10,2)),\n", + " amount (type DECIMAL(10,2)),\n fee (type DECIMAL(10,2)),\n", + ).replace("SELECT payment_id, amount FROM", "SELECT payment_id, amount, 0.1 AS fee FROM"), +} +PAYMENT_ROWS_SQL: str = ( + "SELECT payment_id, amount, typeof(amount) FROM analytics__payments ORDER BY payment_id" +) +LEDGER_AGREEMENT_SQL: str = ( + "SELECT COUNT(*) FROM analytics__payments AS p JOIN analytics__payment_ledger AS l " + "USING (payment_id) WHERE p.amount = l.amount" +) +EVOLVED_PAYMENT_ROWS_SQL: str = ( + "SELECT payment_id, amount, fee, typeof(fee) FROM analytics__payments ORDER BY payment_id" +) +FEE_DECLARATION_SQL: str = ( + "SELECT type FROM pragma_table_info('analytics__payments') WHERE name = 'fee'" +) + + +_TIMESTAMP_EVENTS_HEADER: str = "order_id,amount_cents,ordered_at\n" +ISO_EVENTS_CSV: str = _TIMESTAMP_EVENTS_HEADER + ( + "1,100,2026-01-01T10:00:00\n" + "2,200,2026-01-01 10:30:00.250\n" + "4,400,2026-01-01 11:59:59.999\n" + "5,500,2026-01-01T11:00:00Z\n" + "6,600,2026-01-01T09:59:59.999\n" +) +OFFSET_EVENTS_CSV: str = ISO_EVENTS_CSV + "3,300,2026-01-01T12:15:00+02:00\n" + + +def build_cursor_window_files(*, adapter: str, events_csv: str) -> dict[str, str]: + """Build an hourly microbatch project over timestamps written in several ISO forms.""" + + return { + "sqlbuild_project.toml": ( + 'name = "hourly_shop"\n' + f'adapter = "{adapter}"\n' + 'default_target = "dev"\n\n' + f"[connection]\n{_CONNECTIONS[adapter]}\n" + "[targets.dev]\n" + f'schema = "{SCHEMA}"\n' + ), + "seeds/order_events.csv": events_csv, + "seeds/seeds.yml": STRATEGY_PROJECT_FILES["seeds/seeds.yml"], + "models/hourly_orders.sql": """MODEL ( + description "Orders in hourly microbatches.", + materialized incremental, + incremental_strategy delete_insert, + incremental_mode microbatch, + microbatch_strategy watermark, + cursor_watermark_mode all, + cursor ordered_at, + cursor_type timestamp, + cursor_grain hour, + cursor_inputs (order_events (column ordered_at, roles [filter, watermark]),), + batch_size 1h, + cursor_start "2026-01-01 08:00:00", +); + +SELECT order_id, amount_cents, ordered_at FROM __seed("order_events") +""", + } + + +def model_order_ids(*, adapter: str, project_dir: Path) -> tuple[tuple[object, ...], ...]: + """Read the hourly model's order ids from either engine.""" + + return _ORDER_ID_READERS[adapter](project_dir) + + +_ORDER_ID_READERS: MappingProxyType[str, Callable[[Path], tuple[tuple[object, ...], ...]]] = ( + MappingProxyType( + { + "sqlite": lambda project_dir: sqlite_rows( + db_path=project_dir / SQLITE_FILE, + sql=f"SELECT order_id FROM {SCHEMA}__hourly_orders ORDER BY 1", + ), + "duckdb": lambda project_dir: tuple( + tuple(row) + for row in query_duckdb( + db_path=project_dir / DUCKDB_FILE, + sql=f"SELECT order_id FROM {SCHEMA}.hourly_orders ORDER BY 1", + ) + ), + } + ) +) + + +_SCHEMA_PER_MODEL_TOML: str = ( + 'name = "staging_shop"\n' + 'adapter = "sqlite"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + f'path = "{SQLITE_FILE}"\n\n' + "[defaults]\n" + 'schema = "main"\n\n' + "[targets.dev]\n" + 'schema = "preserve"\n' +) +STAGING_ORDERS_FILES: dict[str, str] = { + "sqlbuild_project.toml": _SCHEMA_PER_MODEL_TOML, + "models/orders.sql": """MODEL (description "Staged.", materialized table, schema "staging"); + +SELECT 1 AS order_id +""", +} +MAIN_STAGING_ORDERS_FILES: dict[str, str] = { + "sqlbuild_project.toml": _SCHEMA_PER_MODEL_TOML, + "models/staging__orders.sql": """MODEL (description "Orders kept in main.", materialized table); + +SELECT 2 AS order_id +""", +} + + +def run_sqb_commands( + *, project_dir: Path, commands: tuple[tuple[str, ...], ...] +) -> tuple[subprocess.CompletedProcess[str], ...]: + """Run sqb commands in order in one project.""" + + return tuple( + run_sqb(command=("--no-color", *command), project_dir=project_dir) for command in commands + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_cursor_windows.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_cursor_windows.py new file mode 100644 index 0000000000..2c298a0215 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_cursor_windows.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.sqlite._test_types import ( + SqliteCursorWindowE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.sqlite.helpers import ( + ISO_EVENTS_CSV, + OFFSET_EVENTS_CSV, + build_cursor_window_files, + model_order_ids, + run_sqb_commands, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project + +_ONE_WINDOW: tuple[str, ...] = ( + "build", + "--start-cursor-ts", + "2026-01-01T10:00:00", + "--end-cursor-ts", + "2026-01-01T10:00:00", +) +_REBUILD_WINDOWS: tuple[str, ...] = ( + "build", + "--start-cursor-ts", + "2026-01-01T10:00:00", + "--end-cursor-ts", + "2026-01-01T11:00:00", +) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteCursorWindowE2ETestCase( + description="one hourly window selects T-separated and fractional values like DuckDB", + events_csv=ISO_EVENTS_CSV, + commands=(_ONE_WINDOW,), + expected_order_ids=((1,), (2,)), + ), + SqliteCursorWindowE2ETestCase( + description="rebuilt windows replace their rows instead of duplicating them", + events_csv=ISO_EVENTS_CSV, + commands=(("build",), _REBUILD_WINDOWS), + expected_order_ids=((1,), (2,), (4,), (5,), (6,)), + ), + ], + ids=lambda case: case.description, +) +def test_given_iso_timestamps_when_building_hourly_windows_then_sqlite_matches_duckdb( + tmp_path: Path, test_case: SqliteCursorWindowE2ETestCase +) -> None: + sqlite_dir: Path = prepare_inline_project( + tmp_path=tmp_path / "sqlite", + project_name="hourly_shop", + repo_files=build_cursor_window_files(adapter="sqlite", events_csv=test_case.events_csv), + ) + duckdb_dir: Path = prepare_inline_project( + tmp_path=tmp_path / "duckdb", + project_name="hourly_shop", + repo_files=build_cursor_window_files(adapter="duckdb", events_csv=test_case.events_csv), + ) + + results: tuple[subprocess.CompletedProcess[str], ...] = ( + *run_sqb_commands(project_dir=sqlite_dir, commands=test_case.commands), + *run_sqb_commands(project_dir=duckdb_dir, commands=test_case.commands), + ) + + assert all(result.returncode == 0 for result in results), "".join( + result.stdout + result.stderr for result in results + ) + assert model_order_ids(adapter="sqlite", project_dir=sqlite_dir) == ( + test_case.expected_order_ids + ) + assert model_order_ids(adapter="duckdb", project_dir=duckdb_dir) == ( + test_case.expected_order_ids + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteCursorWindowE2ETestCase( + description="a value with a UTC offset falls in the window of its UTC time", + events_csv=OFFSET_EVENTS_CSV, + commands=(_ONE_WINDOW,), + expected_order_ids=((1,), (2,), (3,)), + ), + SqliteCursorWindowE2ETestCase( + description="rebuilding the offset value's UTC window keeps one copy of it", + events_csv=OFFSET_EVENTS_CSV, + commands=(("build",), _ONE_WINDOW), + expected_order_ids=((1,), (2,), (3,), (4,), (5,), (6,)), + ), + ], + ids=lambda case: case.description, +) +def test_given_offset_timestamps_when_building_hourly_windows_then_sqlite_uses_utc( + tmp_path: Path, test_case: SqliteCursorWindowE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="hourly_shop", + repo_files=build_cursor_window_files(adapter="sqlite", events_csv=test_case.events_csv), + ) + + results: tuple[subprocess.CompletedProcess[str], ...] = run_sqb_commands( + project_dir=project_dir, commands=test_case.commands + ) + + assert all(result.returncode == 0 for result in results), "".join( + result.stdout + result.stderr for result in results + ) + assert model_order_ids(adapter="sqlite", project_dir=project_dir) == ( + test_case.expected_order_ids + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_decimal_contracts.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_decimal_contracts.py new file mode 100644 index 0000000000..c11008d650 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_decimal_contracts.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.sqlite._test_types import ( + SqliteDecimalContractE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.sqlite.helpers import ( + CHANGED_RAW_PAYMENTS_SQL, + DECIMAL_PROJECT_FILES, + EVOLVED_DECIMAL_PROJECT_FILES, + EVOLVED_PAYMENT_ROWS_SQL, + FEE_DECLARATION_SQL, + LEDGER_AGREEMENT_SQL, + PAYMENT_ROWS_SQL, + RAW_PAYMENTS_SQL, + SQLITE_FILE, + UNDECLARED_PAYMENT_SOURCES, + execute_sqlite_script, + sqlite_rows, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_EXPECTED_PAYMENT_ROWS: tuple[tuple[object, ...], ...] = ( + (1, "1.50", "text"), + (2, "2.00", "text"), +) +_EXPECTED_EVOLVED_ROWS: tuple[tuple[object, ...], ...] = ( + (1, "2.25", "0.10", "text"), + (2, "2.00", "0.10", "text"), + (3, "4.00", "0.10", "text"), +) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteDecimalContractE2ETestCase( + description="a source declaring DECIMAL columns feeds canonical contract text", + source_files={}, + expected_payment_rows=_EXPECTED_PAYMENT_ROWS, + expected_agreeing_rows=2, + expected_evolved_rows=_EXPECTED_EVOLVED_ROWS, + expected_fee_declaration="TEXT DECIMAL(10,2)", + ), + SqliteDecimalContractE2ETestCase( + description="a numeric DECIMAL(10,2) warehouse column is enforced to contract text", + source_files={"sources/raw.yml": UNDECLARED_PAYMENT_SOURCES}, + expected_payment_rows=_EXPECTED_PAYMENT_ROWS, + expected_agreeing_rows=2, + expected_evolved_rows=_EXPECTED_EVOLVED_ROWS, + expected_fee_declaration="TEXT DECIMAL(10,2)", + ), + ], + ids=lambda case: case.description, +) +def test_given_numeric_source_when_building_contract_models_then_sqlite_stores_decimal_text( + tmp_path: Path, test_case: SqliteDecimalContractE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="payments_shop", + repo_files={**DECIMAL_PROJECT_FILES, **test_case.source_files}, + ) + db_path: Path = project_dir / SQLITE_FILE + execute_sqlite_script(db_path=db_path, sql=RAW_PAYMENTS_SQL) + + first: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + payment_rows: tuple[tuple[object, ...], ...] = sqlite_rows( + db_path=db_path, sql=PAYMENT_ROWS_SQL + ) + agreeing: tuple[tuple[object, ...], ...] = sqlite_rows( + db_path=db_path, sql=LEDGER_AGREEMENT_SQL + ) + execute_sqlite_script(db_path=db_path, sql=CHANGED_RAW_PAYMENTS_SQL) + prepare_inline_project( + tmp_path=tmp_path, project_name="payments_shop", repo_files=EVOLVED_DECIMAL_PROJECT_FILES + ) + second: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--select", "payments"), project_dir=project_dir + ) + + assert (first.returncode, second.returncode) == (0, 0), ( + first.stdout + first.stderr + second.stdout + second.stderr + ) + assert payment_rows == test_case.expected_payment_rows + assert agreeing == ((test_case.expected_agreeing_rows,),) + assert sqlite_rows(db_path=db_path, sql=EVOLVED_PAYMENT_ROWS_SQL) == ( + test_case.expected_evolved_rows + ) + assert sqlite_rows(db_path=db_path, sql=FEE_DECLARATION_SQL) == ( + (test_case.expected_fee_declaration,), + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_relation_names.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_relation_names.py new file mode 100644 index 0000000000..960808861a --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_relation_names.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.sqlite._test_types import ( + SqliteCollisionE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.sqlite.helpers import ( + MAIN_STAGING_ORDERS_FILES, + SQLITE_FILE, + STAGING_ORDERS_FILES, + sqlite_rows, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_COLLISION_FRAGMENTS: tuple[str, ...] = ( + "error[A706]", + "'staging.orders' and 'main.staging__orders'", +) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteCollisionE2ETestCase( + description="adding main staging__orders beside staging.orders is refused unbuilt", + first_files=STAGING_ORDERS_FILES, + second_files=MAIN_STAGING_ORDERS_FILES, + expected_fragments=_COLLISION_FRAGMENTS, + expected_tables=(("staging__orders",),), + expected_rows=((1,),), + ), + SqliteCollisionE2ETestCase( + description="adding staging.orders beside main staging__orders is refused unbuilt", + first_files=MAIN_STAGING_ORDERS_FILES, + second_files=STAGING_ORDERS_FILES, + expected_fragments=_COLLISION_FRAGMENTS, + expected_tables=(("staging__orders",),), + expected_rows=((2,),), + ), + ], + ids=lambda case: case.description, +) +def test_given_colliding_relation_names_when_building_then_sqlite_refuses_before_writing( + tmp_path: Path, test_case: SqliteCollisionE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="staging_shop", repo_files=test_case.first_files + ) + first: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + prepare_inline_project( + tmp_path=tmp_path, project_name="staging_shop", repo_files=test_case.second_files + ) + + second: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert first.returncode == 0, first.stdout + first.stderr + assert second.returncode != 0, second.stdout + second.stderr + output: str = second.stdout + second.stderr + assert all(fragment in output for fragment in test_case.expected_fragments), output + assert ( + sqlite_rows( + db_path=project_dir / SQLITE_FILE, + sql="SELECT name FROM sqlite_schema WHERE type = 'table' AND name LIKE 'staging%orders'", + ) + == test_case.expected_tables + ) + assert ( + sqlite_rows(db_path=project_dir / SQLITE_FILE, sql="SELECT * FROM staging__orders") + == test_case.expected_rows + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_scenarios.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_scenarios.py new file mode 100644 index 0000000000..28abb775e3 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_scenarios.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.sqlite._test_types import ( + SqliteScenarioE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.sqlite.helpers import ( + SCENARIO_OUTCOMES, + SCENARIO_PROJECT_FILES, + SQLITE_FILE, + scenario_outcomes, + scenario_table_shapes, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteScenarioE2ETestCase( + description="fixtures, expected rows and assertions run in the SQLite file", + command=("scenario", "test"), + expected_outcomes=SCENARIO_OUTCOMES, + expected_tables=("_sqlbuild_relations", "_sqlbuild_schemas"), + ), + SqliteScenarioE2ETestCase( + description="concurrent scenarios in one namespace share the SQLite writer", + command=("scenario", "test", "--concurrency", "4", "--scenario-namespace", "ci_one"), + expected_outcomes=SCENARIO_OUTCOMES, + expected_tables=("_sqlbuild_relations", "_sqlbuild_schemas"), + ), + SqliteScenarioE2ETestCase( + description="snapshots captured from SQLite replay locally in DuckDB", + command=("scenario", "test", "--local", "--sync-snapshots"), + expected_outcomes=SCENARIO_OUTCOMES, + expected_tables=("_sqlbuild_relations", "_sqlbuild_schemas"), + ), + SqliteScenarioE2ETestCase( + description="retained scenarios keep their SQLite fixture and model tables", + command=("scenario", "test", "customer_totals_pass", "--retain"), + expected_outcomes={"customer_totals_pass": ("success", None)}, + expected_tables=( + "_sqlbuild_relations", + "_sqlbuild_schemas", + "analytics____sqb___model__customer_totals", + "analytics____sqb___model__daily_orders", + "analytics____sqb___model__orders", + "analytics____sqb___source__raw_orders", + ), + expected_return_code=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_orders_scenarios_when_testing_then_sqlite_reports_each_outcome( + tmp_path: Path, test_case: SqliteScenarioE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="scenario_shop", repo_files=SCENARIO_PROJECT_FILES + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *test_case.command, "--json"), project_dir=project_dir + ) + + assert result.returncode == test_case.expected_return_code, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert scenario_outcomes(payload) == test_case.expected_outcomes + assert scenario_table_shapes(db_path=project_dir / SQLITE_FILE) == test_case.expected_tables diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_sqlite.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_sqlite.py new file mode 100644 index 0000000000..26bdae45dc --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_sqlite.py @@ -0,0 +1,351 @@ +from __future__ import annotations + +import multiprocessing +import subprocess +import time +from multiprocessing.context import SpawnContext +from multiprocessing.process import BaseProcess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.sqlite._test_types import ( + SqliteCommandE2ETestCase, + SqliteFailureE2ETestCase, + SqliteLockE2ETestCase, + SqliteOracleE2ETestCase, + SqliteProjectE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.sqlite.helpers import ( + CHANGED_EVENTS_CSV, + COMPARED_MODELS, + CONCURRENT_MICROBATCH_PROJECT_FILES, + CONTRACT_PROJECT_FILES, + FUNCTION_PROJECT_FILES, + LOCK_PROJECT_FILES, + SQLITE_FILE, + STRATEGY_PROJECT_FILES, + append_late_orders, + build_orders_project_files, + execute_sqlite, + hold_sqlite_write_lock, + model_rows, + sqlite_rows, + table_names, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteOracleE2ETestCase( + description="seed, view, table, append, merge and delete_insert match DuckDB", + compared_models=COMPARED_MODELS, + unexpected_rebuild_fragments=("Schema changed", "type changed"), + ), + ], + ids=lambda case: case.description, +) +def test_given_orders_project_when_building_twice_then_sqlite_matches_duckdb( + tmp_path: Path, test_case: SqliteOracleE2ETestCase +) -> None: + outputs: dict[str, list[str]] = {"sqlite": [], "duckdb": []} + rows: dict[str, list[tuple[tuple[object, ...], ...]]] = {"sqlite": [], "duckdb": []} + adapter: str + for adapter in ("sqlite", "duckdb"): + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name=f"orders_{adapter}", + repo_files=build_orders_project_files(adapter=adapter), + ) + first: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + append_late_orders(project_dir=project_dir) + second: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + outputs[adapter].extend((first.stdout + first.stderr, second.stdout + second.stderr)) + assert (first.returncode, second.returncode) == (test_case.expected_return_code,) * 2, ( + outputs[adapter] + ) + rows[adapter].extend( + model_rows(adapter=adapter, project_dir=project_dir, model=model, order_by=key) + for model, key in test_case.compared_models + ) + + assert rows["sqlite"] == rows["duckdb"] + for fragment in test_case.unexpected_rebuild_fragments: + assert fragment not in outputs["sqlite"][1] + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteCommandE2ETestCase( + description="test runs unit tests against the file", + command=("--no-color", "test"), + expected_fragments=("test_customer_totals", "PASS=1"), + ), + SqliteCommandE2ETestCase( + description="seed reloads seeds", + command=("--no-color", "seed"), + expected_fragments=("Completed successfully", "PASS=2"), + ), + SqliteCommandE2ETestCase( + description="query reads schema relations by their stored name", + command=( + "--no-color", + "query", + "--format", + "csv", + "SELECT customer_id, total_cents FROM analytics__customer_totals ORDER BY 1", + ), + expected_fragments=("customer_id,total_cents", "1,1975", "2,3000", "3,500"), + ), + SqliteCommandE2ETestCase( + description="debug connects and runs a query", + command=("--no-color", "debug"), + expected_fragments=("adapter: sqlite [OK found]", "connection test: [OK connected]"), + ), + SqliteCommandE2ETestCase( + description="clone copies dev into prod within the file", + command=("--no-color", "clone", "--from", "dev", "--to", "prod"), + expected_fragments=( + "analytics__orders_merged -> prod__orders_merged", + "Completed successfully", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_built_project_when_running_command_then_sqlite_succeeds( + tmp_path: Path, test_case: SqliteCommandE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_shop", + repo_files=build_orders_project_files(adapter="sqlite"), + ) + build: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert build.returncode == 0, build.stdout + build.stderr + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_return_code, output + for fragment in test_case.expected_fragments: + assert fragment in output + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteCommandE2ETestCase( + description="diff between a clone and an edited dev reports the changed row", + command=("--no-color", "diff", "prod:dev", "--select", "orders_merged"), + expected_fragments=("status", "mismatches=1", "order_id=12 | placed -> lost"), + expected_return_code=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_cloned_target_when_diffing_then_sqlite_reports_changed_rows( + tmp_path: Path, test_case: SqliteCommandE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_shop", + repo_files=build_orders_project_files(adapter="sqlite"), + ) + setup: tuple[subprocess.CompletedProcess[str], ...] = ( + run_sqb(command=("--no-color", "build"), project_dir=project_dir), + run_sqb( + command=("--no-color", "clone", "--from", "dev", "--to", "prod"), + project_dir=project_dir, + ), + ) + assert [result.returncode for result in setup] == [0, 0], [r.stdout for r in setup] + execute_sqlite( + db_path=project_dir / SQLITE_FILE, + sql="UPDATE analytics__orders_merged SET status = 'lost' WHERE order_id = 12", + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_return_code, output + for fragment in test_case.expected_fragments: + assert fragment in output + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteProjectE2ETestCase( + description="sequential microbatches rewrite each day without state tables", + files=STRATEGY_PROJECT_FILES, + changed_files={"seeds/order_events.csv": CHANGED_EVENTS_CSV}, + query="SELECT * FROM analytics__daily_revenue ORDER BY 1", + expected_rows=(("2026-01-02 00:00:00", 2, 550), ("2026-01-03 00:00:00", 1, 400)), + absent_tables=("analytics___sqlbuild_microbatches", "_sqlbuild_microbatches"), + ), + SqliteProjectE2ETestCase( + description="cursor delete_insert replaces only the cursor window", + files=STRATEGY_PROJECT_FILES, + changed_files={"seeds/order_events.csv": CHANGED_EVENTS_CSV}, + query="SELECT * FROM analytics__orders_window ORDER BY 1, 3", + expected_rows=( + (1, 100, "2026-01-01 10:00:00"), + (2, 200, "2026-01-01 11:00:00"), + (2, 250, "2026-01-02 12:00:00"), + (3, 300, "2026-01-02 09:00:00"), + (4, 400, "2026-01-03 08:00:00"), + ), + ), + SqliteProjectE2ETestCase( + description="timestamp snapshot closes updated and hard-deleted versions", + files=STRATEGY_PROJECT_FILES, + changed_files={"seeds/order_events.csv": CHANGED_EVENTS_CSV}, + query=( + "SELECT order_id, amount_cents, valid_from, valid_to IS NULL " + "FROM analytics__order_history ORDER BY 1, 3" + ), + expected_rows=( + (1, 100, "2026-01-01 10:00:00", 0), + (2, 200, "2026-01-01 11:00:00", 0), + (2, 250, "2026-01-02 12:00:00", 1), + (3, 300, "2026-01-02 09:00:00", 1), + (4, 400, "2026-01-03 08:00:00", 1), + ), + ), + SqliteProjectE2ETestCase( + description="check snapshot versions a changed amount", + files=STRATEGY_PROJECT_FILES, + changed_files={"seeds/order_events.csv": CHANGED_EVENTS_CSV}, + query=( + "SELECT order_id, amount_cents, valid_to IS NULL " + "FROM analytics__order_amount_history ORDER BY 1, 2" + ), + expected_rows=((1, 100, 1), (2, 200, 0), (2, 250, 1), (3, 300, 1), (4, 400, 1)), + ), + SqliteProjectE2ETestCase( + description="an enforced decimal contract stores canonical text", + files=CONTRACT_PROJECT_FILES, + changed_files={}, + query="SELECT order_id, amount, typeof(amount) FROM analytics__order_amounts ORDER BY 1", + expected_rows=((1, "12.50", "text"), (2, "7.00", "text")), + ), + ], + ids=lambda case: case.description, +) +def test_given_strategy_project_when_building_twice_then_sqlite_writes_expected_rows( + tmp_path: Path, test_case: SqliteProjectE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="events_shop", repo_files=test_case.files + ) + first: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + prepare_inline_project( + tmp_path=tmp_path, project_name="events_shop", repo_files=test_case.changed_files + ) + second: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert (first.returncode, second.returncode) == (test_case.expected_return_code,) * 2, ( + first.stdout + first.stderr + second.stdout + second.stderr + ) + assert ( + sqlite_rows(db_path=project_dir / SQLITE_FILE, sql=test_case.query) + == test_case.expected_rows + ) + assert set(test_case.absent_tables).isdisjoint(table_names(db_path=project_dir / SQLITE_FILE)) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteFailureE2ETestCase( + description="SQL functions fail with a coded error naming the limit", + files=FUNCTION_PROJECT_FILES, + expected_fragments=("error[A705]", "SQLite has no CREATE FUNCTION"), + ), + SqliteFailureE2ETestCase( + description="concurrent microbatch DML is refused at plan time", + files=CONCURRENT_MICROBATCH_PROJECT_FILES, + expected_fragments=("does not support concurrent microbatch", "batch_concurrency = 1"), + ), + ], + ids=lambda case: case.description, +) +def test_given_unsupported_feature_when_building_then_sqlite_fails_with_code( + tmp_path: Path, test_case: SqliteFailureE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="function_shop", repo_files=test_case.files + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + output: str = result.stdout + result.stderr + assert result.returncode != 0, output + for fragment in test_case.expected_fragments: + assert fragment in output + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteLockE2ETestCase( + description="a build blocked by another process's writer fails with A702", + busy_timeout=0.5, + hold_seconds=20.0, + expected_fragments=("A702", "is locked by another writer"), + ), + ], + ids=lambda case: case.description, +) +def test_given_locked_file_when_building_then_sqlite_fails_with_locked_code( + tmp_path: Path, test_case: SqliteLockE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="lock_shop", repo_files=LOCK_PROJECT_FILES + ) + context: SpawnContext = multiprocessing.get_context("spawn") + ready_path: Path = tmp_path / "ready" + holder: BaseProcess = context.Process( + target=hold_sqlite_write_lock, + kwargs={ + "db_path": str(project_dir / SQLITE_FILE), + "ready_path": str(ready_path), + "hold_seconds": test_case.hold_seconds, + }, + ) + holder.start() + try: + deadline: float = time.monotonic() + 30 + while not ready_path.exists() and time.monotonic() < deadline: + time.sleep(0.05) + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + finally: + holder.terminate() + holder.join(timeout=30) + + output: str = result.stdout + result.stderr + assert result.returncode != 0, output + for fragment in test_case.expected_fragments: + assert fragment in output diff --git a/tests/goldens/compiler/failure/config-unknown-adapter.json b/tests/goldens/compiler/failure/config-unknown-adapter.json index cb53fda78f..c834d3788e 100644 --- a/tests/goldens/compiler/failure/config-unknown-adapter.json +++ b/tests/goldens/compiler/failure/config-unknown-adapter.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "C601", - "message": "unknown adapter 'warehouse9000'. Available built-in adapters: bigquery, databricks, duckdb, motherduck, postgres, snowflake, sqlserver. Project-local adapters are discovered from adapters/**/*.py." + "message": "unknown adapter 'warehouse9000'. Available built-in adapters: bigquery, databricks, duckdb, motherduck, postgres, snowflake, sqlite, sqlserver. Project-local adapters are discovered from adapters/**/*.py." } ] } diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/__init__.py b/tests/integration/src/sqlbuild/adapters/sqlite/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py b/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py new file mode 100644 index 0000000000..54b25a3560 --- /dev/null +++ b/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py @@ -0,0 +1,203 @@ +from dataclasses import dataclass + +from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo + + +@dataclass(frozen=True) +class SqliteQualifiedNameTestCase: + description: str + schema: str | None + name: str + expected_name: str | None + + +@dataclass(frozen=True) +class SqliteRejectedNameTestCase: + description: str + database: str | None + schema: str + expected_code: str + + +@dataclass(frozen=True) +class SqliteConnectErrorTestCase: + description: str + config: dict[str, object] + expected_code: str + + +@dataclass(frozen=True) +class SqliteNamespaceListingTestCase: + description: str + schemas: tuple[str, ...] | None + expected_relations: tuple[RelationInfo, ...] + + +@dataclass(frozen=True) +class SqliteTypedTableTestCase: + description: str + select_sql: str + config: dict[str, object] + expected_columns: tuple[ColumnInfo, ...] + expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class SqliteStrategyTestCase: + description: str + strategy: str + source_sql: str + select_columns: str + expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class SqliteAtomicityTestCase: + description: str + strategy: str + expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class SqliteSwapTestCase: + description: str + expected_view_rows: tuple[tuple[object, ...], ...] + expected_dependent_views: tuple[str, ...] + + +@dataclass(frozen=True) +class SqliteSchemaEvolutionTestCase: + description: str + operation: str + expected_columns: tuple[ColumnInfo, ...] + expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class SqliteSeedTestCase: + description: str + csv_contents: str + columns: tuple[ColumnInfo, ...] + expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class SqliteStateTableTestCase: + description: str + schema: str + expected_tables: tuple[str, ...] + expected_index_count: int + + +@dataclass(frozen=True) +class SqliteDiffTestCase: + description: str + right_sql: str + expected_counts: tuple[int, int, int, int, int] + expected_changed_columns: tuple[str, ...] + + +@dataclass(frozen=True) +class SqliteViewRenameTestCase: + description: str + expected_definition: str + expected_names: tuple[str, ...] + + +@dataclass(frozen=True) +class SqliteLockTestCase: + description: str + busy_timeout: float + hold_seconds: float + expected_code: str + expected_reader_rows: int + max_wait_seconds: float + + +@dataclass(frozen=True) +class SqliteInProcessWritersTestCase: + description: str + busy_timeout: float + hold_seconds: float + expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class SqliteKilledWriterTestCase: + description: str + expected_rows: tuple[tuple[object, ...], ...] + expected_tables: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class SqliteVersionTestCase: + description: str + linked_version: tuple[int, int, int] + expected_code: str + + +@dataclass(frozen=True) +class SqlitePragmaTestCase: + description: str + pragmas: tuple[str, ...] + expected_values: tuple[object, ...] + + +@dataclass(frozen=True) +class SqliteCastTestCase: + description: str + target_type: str + expected_sql: str + + +@dataclass(frozen=True) +class SqliteCollisionTestCase: + description: str + first: tuple[str, str] + second: tuple[str, str] + expected_code: str + + +@dataclass(frozen=True) +class SqliteJanitorViewRenameTestCase: + description: str + expected_relations: tuple[RelationInfo, ...] + expected_registry_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class SqliteDecimalDiffTestCase: + description: str + left_sql: str + right_sql: str + expected_counts: tuple[int, int, int, int] + expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class SqliteInterruptTestCase: + description: str + interrupt_after_seconds: float + expected_error_fragment: str + expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class SqliteMergeCountTestCase: + description: str + source_sql: str + expected_affected: int + + +@dataclass(frozen=True) +class SqliteReadOnlyPathTestCase: + description: str + file_name: str + expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class SqliteRetypeIndexTestCase: + description: str + expected_indexes: tuple[tuple[object, ...], ...] + expected_columns: tuple[ColumnInfo, ...] diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/conftest.py b/tests/integration/src/sqlbuild/adapters/sqlite/conftest.py new file mode 100644 index 0000000000..b066e65558 --- /dev/null +++ b/tests/integration/src/sqlbuild/adapters/sqlite/conftest.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.adapters.sqlite.classes.sqlite_adapter import SqliteAdapter + + +@pytest.fixture +def adapter() -> SqliteAdapter: + return SqliteAdapter() + + +@pytest.fixture +def database_path(tmp_path: Path) -> Path: + return tmp_path / "shop.sqlite" + + +@pytest.fixture +def connection(adapter: SqliteAdapter, database_path: Path) -> Iterator[Any]: + conn: Any = adapter.connect({"path": str(database_path)}) + try: + yield conn + finally: + adapter.close(conn) diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/helpers.py b/tests/integration/src/sqlbuild/adapters/sqlite/helpers.py new file mode 100644 index 0000000000..48b708b73f --- /dev/null +++ b/tests/integration/src/sqlbuild/adapters/sqlite/helpers.py @@ -0,0 +1,257 @@ +"""Shared helpers for SQLite adapter integration tests.""" + +from __future__ import annotations + +from collections.abc import Callable +from types import MappingProxyType +from typing import Any + +from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder +from sqlbuild.adapter.contract.models import ColumnInfo, RowDiffSampleRow +from sqlbuild.adapters.sqlite.classes.sqlite_adapter import SqliteAdapter +from sqlbuild.executor.run._helpers.validation.type_enforcement import enforce_types_staged + + +def run_sql(*, adapter: SqliteAdapter, connection: Any, statements: tuple[str, ...]) -> None: + for statement in statements: + adapter.execute(connection=connection, sql=statement) + + +def fetch_rows( + *, adapter: SqliteAdapter, connection: Any, sql: str +) -> tuple[tuple[object, ...], ...]: + cursor: Any = adapter.execute(connection=connection, sql=sql) + return tuple(tuple(row) for row in cursor.fetchall()) + + +def render_relations( + *, adapter: SqliteAdapter, relations: tuple[tuple[str, str], ...] +) -> tuple[str | None, ...]: + """Render relations as the planner does, which records the schema each table belongs to.""" + + return tuple( + adapter.render_qualified_name(database=None, schema=schema, name=name) + for schema, name in relations + ) + + +def stage_contract_relation( + *, adapter: SqliteAdapter, connection: Any, name: str, sql: str +) -> None: + """Create a relation and enforce a DECIMAL(10,2) payment contract on it, as staging does.""" + + adapter.create_table_as( + connection=connection, destination=name, sql=sql, statement_recorder=recorder() + ) + enforce_types_staged( + adapter=adapter, + connection=connection, + staging_qualified=name, + staging_database=None, + staging_schema="main", + staging_table=name, + declared_columns=( + ColumnInfo("payment_id", "INTEGER"), + ColumnInfo("amount", "DECIMAL(10,2)"), + ), + table_type="table", + statement_recorder=recorder(), + ) + + +def recorder() -> StatementRecorder: + return StatementRecorder() + + +def create_orders(*, adapter: SqliteAdapter, connection: Any, relation: str) -> None: + run_sql( + adapter=adapter, + connection=connection, + statements=( + f"CREATE TABLE {relation} (order_id INTEGER, customer_id INTEGER, " + "amount DECIMAL(10,2), ordered_at TIMESTAMP, status VARCHAR(16))", + f"INSERT INTO {relation} VALUES " + "(1, 10, 12.5, '2026-01-01 10:00:00', 'placed'), " + "(2, 10, 7.25, '2026-01-02 11:00:00', 'shipped'), " + "(3, 20, 30, '2026-01-03 12:00:00', 'placed')", + ), + ) + + +def hold_write_lock(*, database_path: str, ready_path: str, hold_seconds: float) -> None: + """Child process: take the write lock, signal readiness, hold it, then commit.""" + + import time + from pathlib import Path + + adapter: SqliteAdapter = SqliteAdapter() + connection: Any = adapter.connect({"path": database_path}) + adapter.begin(connection) + adapter.execute(connection=connection, sql="INSERT INTO orders VALUES (100)") + Path(ready_path).write_text("ready", encoding="utf-8") + time.sleep(hold_seconds) + adapter.commit(connection) + adapter.close(connection) + + +def replace_orders_then_hang(*, database_path: str, ready_path: str) -> None: + """Child process: rewrite orders inside an open transaction, signal, then wait to be killed.""" + + import time + from pathlib import Path + + adapter: SqliteAdapter = SqliteAdapter() + connection: Any = adapter.connect({"path": database_path}) + adapter.begin(connection) + adapter.execute(connection=connection, sql="DELETE FROM orders") + adapter.execute(connection=connection, sql="INSERT INTO orders VALUES (999)") + adapter.execute(connection=connection, sql="CREATE TABLE orders_rebuilt (order_id INTEGER)") + Path(ready_path).write_text("ready", encoding="utf-8") + time.sleep(60) + + +def attempt_write(*, database_path: str, busy_timeout: float, result_path: str) -> None: + """Child process: try one insert and record the outcome code and how long it waited.""" + + import json + import time + from pathlib import Path + + from sqlbuild.adapter.contract.exceptions import AdapterUserError + + adapter: SqliteAdapter = SqliteAdapter() + connection: Any = adapter.connect({"path": database_path, "busy_timeout": busy_timeout}) + started: float = time.monotonic() + code: str = "ok" + message: str = "" + try: + adapter.execute(connection=connection, sql="INSERT INTO orders VALUES (200)") + except AdapterUserError as error: + code = error.code + message = error.message + waited: float = time.monotonic() - started + adapter.close(connection) + Path(result_path).write_text( + json.dumps({"code": code, "message": message, "waited": waited}), encoding="utf-8" + ) + + +def pragma_values( + *, adapter: SqliteAdapter, connection: Any, pragmas: tuple[str, ...] +) -> tuple[object, ...]: + return tuple( + fetch_rows(adapter=adapter, connection=connection, sql=f"PRAGMA {name}")[0][0] + for name in pragmas + ) + + +def changed_column_names(samples: tuple[RowDiffSampleRow, ...]) -> tuple[str, ...]: + names: list[str] = [] + sample: RowDiffSampleRow + for sample in samples: + names.extend(cell.name for cell in sample.changed_cells) + return tuple(names) + + +def write_incrementally( + *, adapter: SqliteAdapter, connection: Any, strategy: str, sql: str +) -> None: + _WRITERS[strategy](adapter, connection, sql) + + +def evolve_orders(*, adapter: SqliteAdapter, connection: Any, operation: str) -> None: + _EVOLUTIONS[operation](adapter, connection) + + +def _append(adapter: SqliteAdapter, connection: Any, sql: str) -> None: + adapter.append( + connection=connection, destination="orders", sql=sql, statement_recorder=recorder() + ) + + +def _merge(adapter: SqliteAdapter, connection: Any, sql: str) -> None: + adapter.merge( + connection=connection, + destination="orders", + sql=sql, + unique_key="order_id", + statement_recorder=recorder(), + ) + + +def _delete_insert(adapter: SqliteAdapter, connection: Any, sql: str) -> None: + adapter.delete_insert( + connection=connection, + destination="orders", + sql=sql, + unique_key=("order_id",), + statement_recorder=recorder(), + ) + + +def _cursor_window(adapter: SqliteAdapter, connection: Any, sql: str) -> None: + adapter.delete_insert_cursor( + connection=connection, + destination="orders", + sql=sql, + cursor_column="ordered_at", + cursor_start="2026-01-02 00:00:00", + cursor_end="2026-01-03 00:00:00", + cursor_type="timestamp", + statement_recorder=recorder(), + ) + + +def _add_column(adapter: SqliteAdapter, connection: Any) -> None: + adapter.add_columns( + connection=connection, + destination="orders", + columns=(ColumnInfo("channel", "TEXT"),), + statement_recorder=recorder(), + ) + + +def _drop_column(adapter: SqliteAdapter, connection: Any) -> None: + adapter.drop_columns( + connection=connection, + destination="orders", + column_names=("status",), + statement_recorder=recorder(), + ) + + +def _rename_column(adapter: SqliteAdapter, connection: Any) -> None: + adapter.rename_column( + connection=connection, + destination="orders", + old_name="status", + new_name="order_status", + statement_recorder=recorder(), + ) + + +def _retype_column(adapter: SqliteAdapter, connection: Any) -> None: + adapter.alter_column_types( + connection=connection, + destination="orders", + columns=(ColumnInfo("order_id", "TEXT"),), + statement_recorder=recorder(), + ) + + +_WRITERS: MappingProxyType[str, Callable[[SqliteAdapter, Any, str], None]] = MappingProxyType( + { + "append": _append, + "merge": _merge, + "delete_insert": _delete_insert, + "cursor": _cursor_window, + } +) +_EVOLUTIONS: MappingProxyType[str, Callable[[SqliteAdapter, Any], None]] = MappingProxyType( + { + "add": _add_column, + "drop": _drop_column, + "rename": _rename_column, + "retype": _retype_column, + } +) diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py b/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py new file mode 100644 index 0000000000..c33db3a4db --- /dev/null +++ b/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py @@ -0,0 +1,751 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.models import ( + ColumnInfo, + RelationInfo, + RowDiffResult, + RowDiffSampleRow, +) +from sqlbuild.adapters.sqlite.classes.sqlite_adapter import SqliteAdapter +from tests.integration.src.sqlbuild.adapters.sqlite._test_types import ( + SqliteAtomicityTestCase, + SqliteCastTestCase, + SqliteConnectErrorTestCase, + SqliteDiffTestCase, + SqliteNamespaceListingTestCase, + SqlitePragmaTestCase, + SqliteQualifiedNameTestCase, + SqliteRejectedNameTestCase, + SqliteSchemaEvolutionTestCase, + SqliteSeedTestCase, + SqliteStateTableTestCase, + SqliteStrategyTestCase, + SqliteSwapTestCase, + SqliteTypedTableTestCase, + SqliteVersionTestCase, + SqliteViewRenameTestCase, +) +from tests.integration.src.sqlbuild.adapters.sqlite.helpers import ( + changed_column_names, + create_orders, + evolve_orders, + fetch_rows, + pragma_values, + recorder, + render_relations, + run_sql, + write_incrementally, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteQualifiedNameTestCase( + description="main schema relations keep their bare name", + schema="main", + name="orders", + expected_name="orders", + ), + SqliteQualifiedNameTestCase( + description="other schemas prefix the relation with schema and two underscores", + schema="analytics", + name="orders", + expected_name="analytics__orders", + ), + SqliteQualifiedNameTestCase( + description="no schema leaves the relation unqualified", + schema=None, + name="orders", + expected_name=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_schema_when_rendering_qualified_name_then_sqlite_maps_to_one_file( + test_case: SqliteQualifiedNameTestCase, adapter: SqliteAdapter +) -> None: + rendered: str | None = adapter.render_qualified_name( + database=None, schema=test_case.schema, name=test_case.name + ) + + assert rendered == test_case.expected_name + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteRejectedNameTestCase( + description="a database is rejected because one connection is one file", + database="warehouse", + schema="analytics", + expected_code="A703", + ), + SqliteRejectedNameTestCase( + description="a schema containing two underscores is ambiguous", + database=None, + schema="sales__eu", + expected_code="A704", + ), + SqliteRejectedNameTestCase( + description="a schema ending in an underscore is ambiguous", + database=None, + schema="sales_", + expected_code="A704", + ), + ], + ids=lambda case: case.description, +) +def test_given_unsupported_namespace_when_rendering_name_then_sqlite_raises_coded_error( + test_case: SqliteRejectedNameTestCase, adapter: SqliteAdapter +) -> None: + with pytest.raises(AdapterUserError) as raised: + adapter.render_qualified_name( + database=test_case.database, schema=test_case.schema, name="orders" + ) + + assert raised.value.code == test_case.expected_code + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteConnectErrorTestCase( + description="unknown connection keys are rejected", + config={"path": "shop.sqlite", "database": "shop"}, + expected_code="A703", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_config_when_connecting_then_sqlite_raises_coded_error( + test_case: SqliteConnectErrorTestCase, adapter: SqliteAdapter +) -> None: + with pytest.raises(AdapterUserError) as raised: + adapter.connect(test_case.config) + + assert raised.value.code == test_case.expected_code + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteVersionTestCase( + description="SQLite older than 3.44 is rejected with a coded error", + linked_version=(3, 43, 2), + expected_code="A701", + ), + ], + ids=lambda case: case.description, +) +def test_given_old_sqlite_when_connecting_then_sqlite_raises_version_error( + test_case: SqliteVersionTestCase, adapter: SqliteAdapter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr("sqlite3.sqlite_version_info", test_case.linked_version) + + with pytest.raises(AdapterUserError) as raised: + adapter.connect({"path": ":memory:"}) + + assert raised.value.code == test_case.expected_code + + +@pytest.mark.parametrize( + "test_case", + [ + SqlitePragmaTestCase( + description="file connections use WAL, a busy timeout and legacy renames", + pragmas=("journal_mode", "busy_timeout", "foreign_keys", "legacy_alter_table"), + expected_values=("wal", 5000, 1, 1), + ), + ], + ids=lambda case: case.description, +) +def test_given_file_connection_when_connecting_then_sqlite_enables_wal_and_pragmas( + test_case: SqlitePragmaTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + values: tuple[object, ...] = pragma_values( + adapter=adapter, connection=connection, pragmas=test_case.pragmas + ) + + assert values == test_case.expected_values + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteNamespaceListingTestCase( + description="lists every schema; tables SQLBuild did not create stay in main", + schemas=None, + expected_relations=( + RelationInfo(None, "main", "daily__totals", "BASE TABLE"), + RelationInfo(None, "main", "analytics__legacy_orders", "BASE TABLE"), + RelationInfo(None, "analytics", "orders", "BASE TABLE"), + RelationInfo(None, "analytics", "orders_view", "VIEW"), + ), + ), + SqliteNamespaceListingTestCase( + description="filters to one schema", + schemas=("analytics",), + expected_relations=( + RelationInfo(None, "analytics", "orders", "BASE TABLE"), + RelationInfo(None, "analytics", "orders_view", "VIEW"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_schemas_when_listing_relations_then_sqlite_reports_logical_names( + test_case: SqliteNamespaceListingTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + adapter.ensure_schema( + connection=connection, database=None, schema="analytics", statement_recorder=recorder() + ) + render_relations( + adapter=adapter, relations=(("analytics", "orders"), ("analytics", "orders_view")) + ) + create_orders(adapter=adapter, connection=connection, relation="analytics__orders") + run_sql( + adapter=adapter, + connection=connection, + statements=( + "CREATE VIEW analytics__orders_view AS SELECT * FROM analytics__orders", + "CREATE TABLE daily__totals (total INTEGER)", + "CREATE TABLE analytics__legacy_orders (order_id INTEGER)", + ), + ) + + relations: tuple[RelationInfo, ...] = adapter.list_relations( + connection=connection, database=None, schemas=test_case.schemas + ) + + assert sorted(relations, key=lambda relation: relation.identity) == sorted( + test_case.expected_relations, key=lambda relation: relation.identity + ) + assert adapter.schema_exists(connection=connection, database=None, schema="analytics") + assert not adapter.schema_exists(connection=connection, database=None, schema="daily") + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteTypedTableTestCase( + description="passes types through and reports unenforced decimals as numbers", + select_sql=( + "SELECT order_id, amount, ordered_at, amount * 2 AS doubled, 1 AS one FROM orders" + ), + config={"inferred_column_types": {"doubled": "DECIMAL(12, 2)"}}, + expected_columns=( + ColumnInfo("order_id", "INTEGER"), + ColumnInfo("amount", "NUMERIC"), + ColumnInfo("ordered_at", "TIMESTAMP"), + ColumnInfo("doubled", "NUMERIC"), + ColumnInfo("one", "INTEGER"), + ), + expected_rows=( + (1, 12.5, "2026-01-01 10:00:00", 25.0, 1), + (2, 7.25, "2026-01-02 11:00:00", 14.5, 1), + (3, 30, "2026-01-03 12:00:00", 60, 1), + ), + ), + SqliteTypedTableTestCase( + description="enforced decimals are stored as canonical fixed-scale text", + select_sql=( + "SELECT CAST(order_id AS BIGINT) AS order_id, " + "CASE WHEN amount IS NULL THEN NULL ELSE printf('%.2f', amount) END AS amount " + "FROM orders" + ), + config={"column_types": {"order_id": "BIGINT", "amount": "DECIMAL(10,2)"}}, + expected_columns=( + ColumnInfo("order_id", "BIGINT"), + ColumnInfo("amount", "DECIMAL(10,2)"), + ), + expected_rows=((1, "12.50"), (2, "7.25"), (3, "30.00")), + ), + ], + ids=lambda case: case.description, +) +def test_given_query_when_creating_table_then_sqlite_keeps_column_types( + test_case: SqliteTypedTableTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + create_orders(adapter=adapter, connection=connection, relation="orders") + + adapter.create_table_as( + connection=connection, + destination="order_copy", + sql=test_case.select_sql, + config=test_case.config, + statement_recorder=recorder(), + ) + + assert ( + adapter.describe_relation(connection=connection, relation='"order_copy"') + == test_case.expected_columns + ) + assert ( + fetch_rows(adapter=adapter, connection=connection, sql="SELECT * FROM order_copy") + == test_case.expected_rows + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteCastTestCase( + description="decimal targets render canonical fixed-scale text", + target_type="DECIMAL(10,2)", + expected_sql=( + 'CASE WHEN "amount" IS NULL THEN NULL ' + 'ELSE printf(\'%.2f\', "amount") END AS "amount"' + ), + ), + SqliteCastTestCase( + description="other targets render a plain cast", + target_type="INTEGER", + expected_sql='CAST("amount" AS INTEGER) AS "amount"', + ), + ], + ids=lambda case: case.description, +) +def test_given_contract_type_when_casting_then_sqlite_renders_storage_form( + test_case: SqliteCastTestCase, adapter: SqliteAdapter +) -> None: + rendered: str = adapter.render_source_expression_cast( + expression='"amount"', target_type=test_case.target_type, alias='"amount"' + ) + + assert rendered == test_case.expected_sql + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteStrategyTestCase( + description="append inserts every source row", + strategy="append", + source_sql="SELECT 4, 30, 1, '2026-01-04 09:00:00', 'placed'", + select_columns="order_id", + expected_rows=((1,), (2,), (3,), (4,)), + ), + SqliteStrategyTestCase( + description="merge updates matched keys and inserts new ones", + strategy="merge", + select_columns="order_id, status", + source_sql=( + "SELECT 2 AS order_id, 10 AS customer_id, 9 AS amount, " + "'2026-01-02 11:00:00' AS ordered_at, 'returned' AS status " + "UNION ALL SELECT 5, 30, 4, '2026-01-05 09:00:00', 'placed'" + ), + expected_rows=((1, "placed"), (2, "returned"), (3, "placed"), (5, "placed")), + ), + SqliteStrategyTestCase( + description="delete_insert replaces rows that share a key", + strategy="delete_insert", + select_columns="order_id, status", + source_sql=( + "SELECT 3 AS order_id, 20 AS customer_id, 31 AS amount, " + "'2026-01-03 12:00:00' AS ordered_at, 'shipped' AS status" + ), + expected_rows=((1, "placed"), (2, "shipped"), (3, "shipped")), + ), + SqliteStrategyTestCase( + description="cursor delete_insert replaces exactly the window", + strategy="cursor", + select_columns="order_id, status", + source_sql=( + "SELECT 9 AS order_id, 10 AS customer_id, 1 AS amount, " + "'2026-01-02 12:00:00' AS ordered_at, 'placed' AS status" + ), + expected_rows=((1, "placed"), (3, "placed"), (9, "placed")), + ), + ], + ids=lambda case: case.description, +) +def test_given_strategy_when_writing_incrementally_then_sqlite_produces_expected_rows( + test_case: SqliteStrategyTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + create_orders(adapter=adapter, connection=connection, relation="orders") + + write_incrementally( + adapter=adapter, + connection=connection, + strategy=test_case.strategy, + sql=test_case.source_sql, + ) + + assert ( + fetch_rows( + adapter=adapter, + connection=connection, + sql=f"SELECT {test_case.select_columns} FROM orders ORDER BY order_id", + ) + == test_case.expected_rows + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteAtomicityTestCase( + description="a failing delete_insert leaves the target unchanged", + strategy="delete_insert", + expected_rows=((1,), (2,), (3,)), + ), + SqliteAtomicityTestCase( + description="a failing cursor window leaves the target unchanged", + strategy="cursor", + expected_rows=((1,), (2,), (3,)), + ), + SqliteAtomicityTestCase( + description="a failing merge leaves the target unchanged", + strategy="merge", + expected_rows=((1,), (2,), (3,)), + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_statement_when_writing_then_sqlite_rolls_back_the_transaction( + test_case: SqliteAtomicityTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + create_orders(adapter=adapter, connection=connection, relation="orders") + failing_sql: str = ( + "SELECT 2 AS order_id, 10 AS customer_id, 1 AS amount, " + "'2026-01-02 11:00:00' AS ordered_at, 'x' AS status, 'extra' AS unexpected" + ) + + with pytest.raises(Exception, match="column|values"): + write_incrementally( + adapter=adapter, connection=connection, strategy=test_case.strategy, sql=failing_sql + ) + + assert not connection.in_transaction() + assert ( + fetch_rows( + adapter=adapter, connection=connection, sql="SELECT order_id FROM orders ORDER BY 1" + ) + == test_case.expected_rows + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteSwapTestCase( + description="swap keeps dependent views bound by name to the promoted table", + expected_view_rows=((99,),), + expected_dependent_views=("analytics.order_ids",), + ), + ], + ids=lambda case: case.description, +) +def test_given_dependent_view_when_swapping_then_sqlite_view_reads_new_table( + test_case: SqliteSwapTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + adapter.ensure_schema( + connection=connection, database=None, schema="analytics", statement_recorder=recorder() + ) + render_relations( + adapter=adapter, + relations=( + ("analytics", "orders"), + ("analytics", "orders__staging"), + ("analytics", "order_ids"), + ), + ) + run_sql( + adapter=adapter, + connection=connection, + statements=( + "CREATE TABLE analytics__orders (order_id INTEGER)", + "INSERT INTO analytics__orders VALUES (1)", + "CREATE TABLE analytics__orders__staging (order_id INTEGER)", + "INSERT INTO analytics__orders__staging VALUES (99)", + "CREATE VIEW analytics__order_ids AS SELECT order_id FROM analytics__orders", + ), + ) + + adapter.swap( + connection=connection, + left="analytics__orders", + right="analytics__orders__staging", + statement_recorder=recorder(), + ) + + assert ( + fetch_rows(adapter=adapter, connection=connection, sql="SELECT * FROM analytics__order_ids") + == test_case.expected_view_rows + ) + assert ( + adapter.list_dependent_view_names( + connection=connection, database=None, schema="analytics", name="orders" + ) + == test_case.expected_dependent_views + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteSchemaEvolutionTestCase( + description="adds a nullable column", + operation="add", + expected_columns=( + ColumnInfo("order_id", "INTEGER"), + ColumnInfo("status", "VARCHAR(16)"), + ColumnInfo("channel", "TEXT"), + ), + expected_rows=((1, "placed", None),), + ), + SqliteSchemaEvolutionTestCase( + description="drops a column a view still references", + operation="drop", + expected_columns=(ColumnInfo("order_id", "INTEGER"),), + expected_rows=((1,),), + ), + SqliteSchemaEvolutionTestCase( + description="renames a column in place", + operation="rename", + expected_columns=( + ColumnInfo("order_id", "INTEGER"), + ColumnInfo("order_status", "VARCHAR(16)"), + ), + expected_rows=((1, "placed"),), + ), + SqliteSchemaEvolutionTestCase( + description="retypes by rebuilding the table with its column order", + operation="retype", + expected_columns=(ColumnInfo("order_id", "TEXT"), ColumnInfo("status", "VARCHAR(16)")), + expected_rows=(("1", "placed"),), + ), + ], + ids=lambda case: case.description, +) +def test_given_schema_change_when_evolving_table_then_sqlite_keeps_data( + test_case: SqliteSchemaEvolutionTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + run_sql( + adapter=adapter, + connection=connection, + statements=( + "CREATE TABLE orders (order_id INTEGER, status VARCHAR(16))", + "INSERT INTO orders VALUES (1, 'placed')", + "CREATE VIEW order_status AS SELECT order_id, status FROM orders", + ), + ) + + evolve_orders(adapter=adapter, connection=connection, operation=test_case.operation) + + assert ( + adapter.get_columns(connection=connection, database=None, schema="main", name="orders") + == test_case.expected_columns + ) + assert ( + fetch_rows(adapter=adapter, connection=connection, sql="SELECT * FROM orders") + == test_case.expected_rows + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteSeedTestCase( + description="loads typed rows, booleans as integers and empty cells as null", + csv_contents=( + "customer_id,customer_name,is_active,signed_up_at\n" + "1,Ada,true,2026-01-01 08:00:00\n" + "2,,false,\n" + ), + columns=( + ColumnInfo("customer_id", "INTEGER"), + ColumnInfo("customer_name", "VARCHAR"), + ColumnInfo("is_active", "BOOLEAN"), + ColumnInfo("signed_up_at", "TIMESTAMP"), + ), + expected_rows=((1, "Ada", 1, "2026-01-01 08:00:00"), (2, None, 0, None)), + ), + ], + ids=lambda case: case.description, +) +def test_given_seed_csv_when_loading_twice_then_sqlite_replaces_rows( + test_case: SqliteSeedTestCase, adapter: SqliteAdapter, connection: Any, tmp_path: Path +) -> None: + seed_path: Path = tmp_path / "customers.csv" + seed_path.write_text(test_case.csv_contents, encoding="utf-8") + + for _ in range(2): + adapter.load_seed( + connection=connection, + destination="customers", + file_path=seed_path, + columns=test_case.columns, + statement_recorder=recorder(), + ) + + assert ( + fetch_rows(adapter=adapter, connection=connection, sql="SELECT * FROM customers") + == test_case.expected_rows + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteStateTableTestCase( + description="state tables and file-unique indexes coexist in two schemas", + schema="analytics", + expected_tables=( + "_sqlbuild_audit_results", + "_sqlbuild_fingerprints", + "_sqlbuild_node_results", + "_sqlbuild_source_freshness", + "analytics___sqlbuild_audit_results", + "analytics___sqlbuild_fingerprints", + "analytics___sqlbuild_node_results", + "analytics___sqlbuild_source_freshness", + ), + expected_index_count=12, + ), + ], + ids=lambda case: case.description, +) +def test_given_two_schemas_when_creating_state_tables_then_sqlite_names_indexes_uniquely( + test_case: SqliteStateTableTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + schema: str + for schema in ("main", test_case.schema): + adapter.ensure_schema( + connection=connection, database=None, schema=schema, statement_recorder=recorder() + ) + run_sql( + adapter=adapter, + connection=connection, + statements=( + adapter.render_create_fingerprint_table_sql(database=None, schema=schema), + *adapter.render_create_fingerprint_index_sqls(database=None, schema=schema), + adapter.render_create_source_freshness_table_sql(database=None, schema=schema), + *adapter.render_create_source_freshness_index_sqls(database=None, schema=schema), + adapter.render_create_node_result_table_sql(database=None, schema=schema), + *adapter.render_create_node_result_index_sqls(database=None, schema=schema), + adapter.render_create_audit_result_table_sql(database=None, schema=schema), + *adapter.render_create_audit_result_index_sqls(database=None, schema=schema), + adapter.render_prune_fingerprint_history_sql( + database=None, schema=schema, retain_versions=3 + ), + ), + ) + + tables: tuple[tuple[object, ...], ...] = fetch_rows( + adapter=adapter, + connection=connection, + sql=( + "SELECT name FROM sqlite_schema WHERE type = 'table' " + "AND name LIKE '%sqlbuild%' AND name NOT IN ('_sqlbuild_schemas', '_sqlbuild_relations') " + "ORDER BY name" + ), + ) + indexes: tuple[tuple[object, ...], ...] = fetch_rows( + adapter=adapter, + connection=connection, + sql="SELECT COUNT(*) FROM sqlite_schema WHERE type = 'index' AND sql IS NOT NULL", + ) + assert tuple(row[0] for row in tables) == test_case.expected_tables + assert indexes == ((test_case.expected_index_count,),) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteDiffTestCase( + description="counts equal, changed and side-only rows with full outer join", + right_sql=( + "SELECT order_id, customer_id, amount, ordered_at, " + "CASE WHEN order_id = 2 THEN 'returned' ELSE status END AS status FROM orders " + "WHERE order_id <> 3 UNION ALL SELECT 4, 40, 1, '2026-01-04 00:00:00', 'placed'" + ), + expected_counts=(3, 3, 1, 1, 1), + expected_changed_columns=("status",), + ), + ], + ids=lambda case: case.description, +) +def test_given_two_relations_when_diffing_then_sqlite_reports_row_differences( + test_case: SqliteDiffTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + create_orders(adapter=adapter, connection=connection, relation="orders") + adapter.create_table_as( + connection=connection, + destination="orders_next", + sql=test_case.right_sql, + statement_recorder=recorder(), + ) + + result: RowDiffResult = adapter.diff_rows( + connection=connection, left="orders", right="orders_next", unique_key="order_id" + ) + samples: tuple[RowDiffSampleRow, ...] = adapter.sample_unequal_rows( + connection=connection, left="orders", right="orders_next", unique_key="order_id" + ) + unkeyed: RowDiffResult = adapter.diff_unkeyed_rows( + connection=connection, left="orders", right="orders_next" + ) + + assert ( + result.left_count, + result.right_count, + result.equal_count, + result.unequal_count, + result.left_only_count, + ) == test_case.expected_counts + assert changed_column_names(samples) == test_case.expected_changed_columns + assert unkeyed.equal_count == 1 + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteViewRenameTestCase( + description="recreates the view under its new name from its stored body", + expected_definition="SELECT order_id FROM orders", + expected_names=("orders", "orders_archive"), + ), + ], + ids=lambda case: case.description, +) +def test_given_view_when_renaming_then_sqlite_recreates_it( + test_case: SqliteViewRenameTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + create_orders(adapter=adapter, connection=connection, relation="orders") + adapter.create_view_as( + connection=connection, + destination="orders_view", + sql="SELECT order_id FROM orders", + statement_recorder=recorder(), + ) + + adapter.rename_view( + connection=connection, + origin="orders_view", + destination="orders_archive", + statement_recorder=recorder(), + ) + + assert ( + adapter.read_view_definition( + connection=connection, database=None, schema="main", name="orders_archive" + ) + == test_case.expected_definition + ) + assert ( + tuple( + relation.name + for relation in adapter.list_relations( + connection=connection, database=None, schemas=None + ) + ) + == test_case.expected_names + ) diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/test_locking.py b/tests/integration/src/sqlbuild/adapters/sqlite/test_locking.py new file mode 100644 index 0000000000..31e43e0b82 --- /dev/null +++ b/tests/integration/src/sqlbuild/adapters/sqlite/test_locking.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import json +import multiprocessing +import threading +import time +from multiprocessing.context import SpawnContext +from multiprocessing.process import BaseProcess +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.adapters.sqlite.classes.sqlite_adapter import SqliteAdapter +from tests.integration.src.sqlbuild.adapters.sqlite._test_types import ( + SqliteInProcessWritersTestCase, + SqliteKilledWriterTestCase, + SqliteLockTestCase, +) +from tests.integration.src.sqlbuild.adapters.sqlite.helpers import ( + attempt_write, + fetch_rows, + hold_write_lock, + replace_orders_then_hang, + run_sql, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteLockTestCase( + description="second writer fails with a coded error after busy_timeout; reader reads", + busy_timeout=0.5, + hold_seconds=4.0, + expected_code="A702", + expected_reader_rows=1, + max_wait_seconds=3.0, + ), + ], + ids=lambda case: case.description, +) +def test_given_held_write_lock_when_second_process_writes_then_it_times_out_with_code( + test_case: SqliteLockTestCase, adapter: SqliteAdapter, tmp_path: Path +) -> None: + database_path: Path = tmp_path / "shop.sqlite" + setup: Any = adapter.connect({"path": str(database_path)}) + run_sql( + adapter=adapter, + connection=setup, + statements=("CREATE TABLE orders (order_id INTEGER)", "INSERT INTO orders VALUES (1)"), + ) + adapter.close(setup) + context: SpawnContext = multiprocessing.get_context("spawn") + ready_path: Path = tmp_path / "ready" + result_path: Path = tmp_path / "result.json" + holder: BaseProcess = context.Process( + target=hold_write_lock, + kwargs={ + "database_path": str(database_path), + "ready_path": str(ready_path), + "hold_seconds": test_case.hold_seconds, + }, + ) + holder.start() + try: + deadline: float = time.monotonic() + 30 + while not ready_path.exists() and time.monotonic() < deadline: + time.sleep(0.05) + writer: BaseProcess = context.Process( + target=attempt_write, + kwargs={ + "database_path": str(database_path), + "busy_timeout": test_case.busy_timeout, + "result_path": str(result_path), + }, + ) + writer.start() + reader: Any = adapter.connect({"path": str(database_path)}) + reader_rows: tuple[tuple[object, ...], ...] = fetch_rows( + adapter=adapter, connection=reader, sql="SELECT COUNT(*) FROM orders" + ) + writer.join(timeout=30) + outcome: dict[str, Any] = json.loads(result_path.read_text(encoding="utf-8")) + finally: + holder.join(timeout=30) + committed: tuple[tuple[object, ...], ...] = fetch_rows( + adapter=adapter, connection=reader, sql="SELECT COUNT(*) FROM orders" + ) + adapter.close(reader) + + assert outcome["code"] == test_case.expected_code + assert str(database_path) in outcome["message"] + assert test_case.busy_timeout <= outcome["waited"] < test_case.max_wait_seconds + assert reader_rows == ((test_case.expected_reader_rows,),) + assert committed == ((test_case.expected_reader_rows + 1,),) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteInProcessWritersTestCase( + description="threads of one process queue for the writer instead of timing out", + busy_timeout=0.2, + hold_seconds=1.0, + expected_rows=((1,), (2,)), + ), + ], + ids=lambda case: case.description, +) +def test_given_long_write_in_one_thread_when_another_thread_writes_then_it_waits( + test_case: SqliteInProcessWritersTestCase, adapter: SqliteAdapter, tmp_path: Path +) -> None: + config: dict[str, object] = { + "path": str(tmp_path / "shop.sqlite"), + "busy_timeout": test_case.busy_timeout, + } + first: Any = adapter.connect(config) + second: Any = adapter.connect(config) + run_sql( + adapter=adapter, connection=first, statements=("CREATE TABLE orders (order_id INTEGER)",) + ) + started: threading.Event = threading.Event() + errors: list[BaseException] = [] + + def hold() -> None: + with adapter.transaction(first): + adapter.execute(connection=first, sql="INSERT INTO orders VALUES (1)") + started.set() + time.sleep(test_case.hold_seconds) + + def write() -> None: + started.wait(timeout=10) + try: + adapter.execute(connection=second, sql="INSERT INTO orders VALUES (2)") + except BaseException as error: + errors.append(error) + + threads: tuple[threading.Thread, ...] = ( + threading.Thread(target=hold), + threading.Thread(target=write), + ) + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=30) + rows: tuple[tuple[object, ...], ...] = fetch_rows( + adapter=adapter, connection=first, sql="SELECT order_id FROM orders ORDER BY 1" + ) + adapter.close(first) + adapter.close(second) + + assert errors == [] + assert rows == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteKilledWriterTestCase( + description="a writer killed mid-transaction leaves committed data and no lock behind", + expected_rows=((1,), (2,), (3,)), + expected_tables=(("orders",),), + ), + ], + ids=lambda case: case.description, +) +def test_given_writer_killed_mid_transaction_when_reopening_then_target_is_unchanged( + test_case: SqliteKilledWriterTestCase, adapter: SqliteAdapter, tmp_path: Path +) -> None: + database_path: Path = tmp_path / "shop.sqlite" + setup: Any = adapter.connect({"path": str(database_path)}) + run_sql( + adapter=adapter, + connection=setup, + statements=( + "CREATE TABLE orders (order_id INTEGER)", + "INSERT INTO orders VALUES (1), (2), (3)", + ), + ) + adapter.close(setup) + context: SpawnContext = multiprocessing.get_context("spawn") + ready_path: Path = tmp_path / "ready" + writer: BaseProcess = context.Process( + target=replace_orders_then_hang, + kwargs={"database_path": str(database_path), "ready_path": str(ready_path)}, + ) + writer.start() + try: + deadline: float = time.monotonic() + 30 + while not ready_path.exists() and time.monotonic() < deadline: + time.sleep(0.05) + assert ready_path.exists() + finally: + writer.kill() + writer.join(timeout=30) + reopened: Any = adapter.connect({"path": str(database_path), "busy_timeout": 0.5}) + rows: tuple[tuple[object, ...], ...] = fetch_rows( + adapter=adapter, connection=reopened, sql="SELECT order_id FROM orders ORDER BY 1" + ) + tables: tuple[tuple[object, ...], ...] = fetch_rows( + adapter=adapter, + connection=reopened, + sql="SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'orders%'", + ) + adapter.execute(connection=reopened, sql="INSERT INTO orders VALUES (4)") + adapter.close(reopened) + + assert rows == test_case.expected_rows + assert tables == test_case.expected_tables diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/test_relations.py b/tests/integration/src/sqlbuild/adapters/sqlite/test_relations.py new file mode 100644 index 0000000000..0a833d78f0 --- /dev/null +++ b/tests/integration/src/sqlbuild/adapters/sqlite/test_relations.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from sqlbuild.adapter.contract.exceptions import AdapterUserError +from sqlbuild.adapter.contract.models import RelationInfo, RowDiffResult +from sqlbuild.adapters.sqlite.classes.sqlite_adapter import SqliteAdapter +from tests.integration.src.sqlbuild.adapters.sqlite._test_types import ( + SqliteCollisionTestCase, + SqliteDecimalDiffTestCase, + SqliteJanitorViewRenameTestCase, +) +from tests.integration.src.sqlbuild.adapters.sqlite.helpers import ( + fetch_rows, + recorder, + render_relations, + run_sql, + stage_contract_relation, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteCollisionTestCase( + description="a main table named like a schema relation is refused", + first=("staging", "orders"), + second=("main", "staging__orders"), + expected_code="A706", + ), + SqliteCollisionTestCase( + description="a schema relation named like a main table is refused", + first=("main", "staging__orders"), + second=("staging", "orders"), + expected_code="A706", + ), + ], + ids=lambda case: case.description, +) +def test_given_relations_sharing_a_table_name_when_rendering_then_sqlite_refuses( + test_case: SqliteCollisionTestCase, adapter: SqliteAdapter +) -> None: + render_relations(adapter=adapter, relations=(test_case.first,)) + + with pytest.raises(AdapterUserError) as raised: + render_relations(adapter=adapter, relations=(test_case.second,)) + + assert raised.value.code == test_case.expected_code + assert "staging__orders" in raised.value.message + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteJanitorViewRenameTestCase( + description="the janitor's ALTER VIEW statement recreates the view in its schema", + expected_relations=( + RelationInfo(None, "analytics", "order_ids__archived", "VIEW"), + RelationInfo(None, "analytics", "orders", "BASE TABLE"), + ), + expected_registry_rows=( + ("analytics__order_ids__archived", "analytics", "order_ids__archived"), + ("analytics__orders", "analytics", "orders"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_schema_view_when_running_rename_view_statements_then_sqlite_recreates_it( + test_case: SqliteJanitorViewRenameTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + adapter.ensure_schema( + connection=connection, database=None, schema="analytics", statement_recorder=recorder() + ) + orders, order_ids, archived = render_relations( + adapter=adapter, + relations=( + ("analytics", "orders"), + ("analytics", "order_ids"), + ("analytics", "order_ids__archived"), + ), + ) + run_sql( + adapter=adapter, + connection=connection, + statements=( + f"CREATE TABLE {orders} (order_id INTEGER)", + f"CREATE VIEW {order_ids} AS SELECT order_id FROM {orders}", + ), + ) + + run_sql( + adapter=adapter, + connection=connection, + statements=adapter.render_rename_view(origin=str(order_ids), destination=str(archived)), + ) + + assert adapter.list_relations(connection=connection, database=None, schemas=("analytics",)) == ( + test_case.expected_relations + ) + assert ( + fetch_rows( + adapter=adapter, + connection=connection, + sql="SELECT * FROM _sqlbuild_relations ORDER BY physical_name", + ) + == test_case.expected_registry_rows + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteDecimalDiffTestCase( + description="contract decimals enforced from numeric and text inputs diff as equal", + left_sql="SELECT 1 AS payment_id, 1.5 AS amount UNION ALL SELECT 2, 2", + right_sql="SELECT 1 AS payment_id, '1.50' AS amount UNION ALL SELECT 2, '2.0'", + expected_counts=(2, 2, 2, 0), + expected_rows=((1, "1.50", "1.50"), (2, "2.00", "2.00")), + ), + ], + ids=lambda case: case.description, +) +def test_given_identical_decimal_contracts_when_diffing_then_sqlite_reports_no_changes( + test_case: SqliteDecimalDiffTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + run_sql( + adapter=adapter, + connection=connection, + statements=( + "CREATE TABLE numeric_payments (payment_id INTEGER, amount DECIMAL(10,2))", + f"INSERT INTO numeric_payments {test_case.left_sql}", + ), + ) + stage_contract_relation( + adapter=adapter, + connection=connection, + name="payments_left", + sql="SELECT payment_id, amount FROM numeric_payments", + ) + stage_contract_relation( + adapter=adapter, connection=connection, name="payments_right", sql=test_case.right_sql + ) + + result: RowDiffResult = adapter.diff_rows( + connection=connection, + left="payments_left", + right="payments_right", + unique_key="payment_id", + ) + + assert ( + result.left_count, + result.right_count, + result.equal_count, + result.unequal_count, + ) == test_case.expected_counts + assert ( + fetch_rows( + adapter=adapter, + connection=connection, + sql=( + "SELECT l.payment_id, l.amount, r.amount FROM payments_left AS l " + "JOIN payments_right AS r USING (payment_id) ORDER BY 1" + ), + ) + == test_case.expected_rows + ) diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/test_transactions.py b/tests/integration/src/sqlbuild/adapters/sqlite/test_transactions.py new file mode 100644 index 0000000000..a495678e16 --- /dev/null +++ b/tests/integration/src/sqlbuild/adapters/sqlite/test_transactions.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import sqlite3 +import threading +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapters.sqlite.classes.sqlite_adapter import SqliteAdapter +from tests.integration.src.sqlbuild.adapters.sqlite._test_types import ( + SqliteInterruptTestCase, + SqliteMergeCountTestCase, + SqliteReadOnlyPathTestCase, + SqliteRetypeIndexTestCase, +) +from tests.integration.src.sqlbuild.adapters.sqlite.helpers import ( + create_orders, + fetch_rows, + recorder, + run_sql, +) + +_LARGE_INSERT_SQL: str = ( + "INSERT INTO orders (order_id) WITH RECURSIVE ids(n) AS " + "(SELECT 10 UNION ALL SELECT n + 1 FROM ids WHERE n < 200000000) SELECT n FROM ids" +) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteInterruptTestCase( + description="an interrupted INSERT inside a transaction re-raises the interrupt", + interrupt_after_seconds=0.3, + expected_error_fragment="interrupted", + expected_rows=((1,), (2,), (3,)), + ), + ], + ids=lambda case: case.description, +) +def test_given_interrupt_during_transaction_when_rolling_back_then_sqlite_keeps_original_error( + test_case: SqliteInterruptTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + create_orders(adapter=adapter, connection=connection, relation="orders") + timer: threading.Timer = threading.Timer( + test_case.interrupt_after_seconds, adapter.interrupt_connection, args=(connection,) + ) + + timer.start() + with pytest.raises(sqlite3.OperationalError) as raised, adapter.transaction(connection): + adapter.execute(connection=connection, sql="DELETE FROM orders WHERE order_id = 1") + adapter.execute(connection=connection, sql=_LARGE_INSERT_SQL) + timer.join() + + assert test_case.expected_error_fragment in str(raised.value) + assert not connection.in_transaction() + assert ( + fetch_rows( + adapter=adapter, connection=connection, sql="SELECT order_id FROM orders ORDER BY 1" + ) + == test_case.expected_rows + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteMergeCountTestCase( + description="merge reports updated and inserted rows together", + source_sql=( + "SELECT 1 AS order_id, 11 AS customer_id, 1 AS amount, " + "'2026-01-05 00:00:00' AS ordered_at, 'shipped' AS status " + "UNION ALL SELECT 4, 40, 2, '2026-01-05 00:00:00', 'placed'" + ), + expected_affected=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_matched_and_new_keys_when_merging_then_sqlite_counts_both( + test_case: SqliteMergeCountTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + create_orders(adapter=adapter, connection=connection, relation="orders") + + affected: int | None = adapter.merge( + connection=connection, + destination="orders", + sql=test_case.source_sql, + unique_key="order_id", + statement_recorder=recorder(), + ) + + assert affected == test_case.expected_affected + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteReadOnlyPathTestCase( + description="read-only paths with URI characters open the named file", + file_name="orders ?#%.sqlite", + expected_rows=((1,),), + ), + ], + ids=lambda case: case.description, +) +def test_given_uri_characters_in_path_when_opening_read_only_then_sqlite_reads_the_file( + test_case: SqliteReadOnlyPathTestCase, adapter: SqliteAdapter, tmp_path: Path +) -> None: + path: str = str(tmp_path / test_case.file_name) + writer: Any = adapter.connect({"path": path}) + run_sql( + adapter=adapter, + connection=writer, + statements=("CREATE TABLE orders (order_id INTEGER)", "INSERT INTO orders VALUES (1)"), + ) + adapter.close(writer) + + reader: Any = adapter.connect({"path": path, "read_only": True}) + rows: tuple[tuple[object, ...], ...] = fetch_rows( + adapter=adapter, connection=reader, sql="SELECT order_id FROM orders" + ) + adapter.close(reader) + + assert rows == test_case.expected_rows + assert all(entry.name.startswith(test_case.file_name) for entry in tmp_path.iterdir()) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteRetypeIndexTestCase( + description="a table rebuilt for new column types keeps its indexes", + expected_indexes=( + ("orders_customer_idx", "CREATE INDEX orders_customer_idx ON orders (customer_id)"), + ( + "orders_status_idx", + "CREATE UNIQUE INDEX orders_status_idx ON orders (order_id, status)", + ), + ), + expected_columns=( + ColumnInfo("order_id", "INTEGER"), + ColumnInfo("customer_id", "TEXT"), + ColumnInfo("amount", "NUMERIC"), + ColumnInfo("ordered_at", "TIMESTAMP"), + ColumnInfo("status", "VARCHAR(16)"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_indexed_table_when_altering_column_types_then_sqlite_recreates_indexes( + test_case: SqliteRetypeIndexTestCase, adapter: SqliteAdapter, connection: Any +) -> None: + create_orders(adapter=adapter, connection=connection, relation="orders") + run_sql( + adapter=adapter, + connection=connection, + statements=( + "CREATE INDEX orders_customer_idx ON orders (customer_id)", + "CREATE UNIQUE INDEX orders_status_idx ON orders (order_id, status)", + ), + ) + + adapter.alter_column_types( + connection=connection, + destination="orders", + columns=(ColumnInfo("customer_id", "TEXT"),), + statement_recorder=recorder(), + ) + + assert ( + fetch_rows( + adapter=adapter, + connection=connection, + sql=( + "SELECT name, sql FROM sqlite_schema WHERE type = 'index' " + "AND tbl_name = 'orders' ORDER BY name" + ), + ) + == test_case.expected_indexes + ) + assert ( + adapter.describe_relation(connection=connection, relation="orders") + == test_case.expected_columns + ) diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py index e12691760d..9d0d0d1ae9 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/helpers.py @@ -146,6 +146,10 @@ def relation_exists( def requires_derived_table_aliases(self) -> bool: return False + def render_cursor_operand(self, *, expression: str, cursor_type: str | None) -> str: + del cursor_type + return expression + class RecordingCursorAdapter(FakeCursorAdapter): def __init__(self, *, target_relation_exists: bool) -> None: diff --git a/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py b/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py index de21a651c1..b434d33fb2 100644 --- a/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py +++ b/tests/unit/src/sqlbuild/executor/run/_helpers/test_lifecycle_progress.py @@ -34,6 +34,7 @@ def test_given_staging_creation_when_adapter_blocks_then_start_is_already_dispat adapter: Mock = Mock() context: Mock = Mock(adapter=adapter, connection=object()) context.entry.table_type = "transient" + context.entry.inferred_columns = () adapter.drop.side_effect = lambda **_: barrier_events.append(events[-1]) with dispatcher_scope(dispatcher): diff --git a/website/src/content/docs/docs/concepts/adapters.mdx b/website/src/content/docs/docs/concepts/adapters.mdx index d65f2b3a77..a05d0ed3a0 100644 --- a/website/src/content/docs/docs/concepts/adapters.mdx +++ b/website/src/content/docs/docs/concepts/adapters.mdx @@ -17,6 +17,7 @@ SQLBuild uses adapters to connect to different database engines. Snowflake is th | [BigQuery](/docs/concepts/adapters/bigquery) | Beta | `sqlbuild[bigquery]` | | [Databricks](/docs/concepts/adapters/databricks) | Beta | `sqlbuild[databricks]` | | [SQL Server](/docs/concepts/adapters/sqlserver) | Beta | `sqlbuild[sqlserver]` | +| [SQLite](/docs/concepts/adapters/sqlite) | Beta | included by default | | ClickHouse | Planned | | | Redshift | Planned | | | Trino | Planned | | @@ -101,7 +102,7 @@ A few `StrictAdapter` methods are optional and have a documented default. `creat - SQLBuild discovers all `.py` files under `adapters/` recursively (excluding `__init__.py` and files starting with `_`) - Each file is scanned for classes that define a string `adapter_name` and subclass `StrictAdapter` (or any of its subclasses like `BaseAdapter` or a built-in adapter) - Adapter names must be unique across all adapter files - duplicates raise an error -- Custom adapter names cannot shadow built-in names (`duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlserver`) +- Custom adapter names cannot shadow built-in names (`duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlite`, `sqlserver`) ### Adapter class hierarchy diff --git a/website/src/content/docs/docs/concepts/adapters/sqlite.mdx b/website/src/content/docs/docs/concepts/adapters/sqlite.mdx new file mode 100644 index 0000000000..5999258407 --- /dev/null +++ b/website/src/content/docs/docs/concepts/adapters/sqlite.mdx @@ -0,0 +1,117 @@ +--- +title: "SQLite" +description: "SQLite adapter configuration, capabilities and limits for SQLBuild." +--- + +SQLite uses the `sqlite3` module that ships with Python. No extra installation is needed. +SQLBuild requires SQLite 3.44 or newer; Python builds installed by `uv` bundle a recent version. + +## Connection config + +```toml +adapter = "sqlite" +default_target = "dev" + +[connections.local] +path = "warehouse.sqlite" + +[targets.dev] +connection = "local" +schema = "analytics" +``` + +| Field | Description | +|-------|-------------| +| `path` | Path to the database file, relative to the project directory. Use `:memory:` for an in-memory database. The file and its parent directory are created when missing. | +| `busy_timeout` | Seconds a writer waits for another process to release the file before failing. Default `5`. | +| `read_only` | Open the file read-only. Default `false`. | + +On connect SQLBuild turns on write-ahead logging (`journal_mode=WAL`), sets the busy timeout, +enables foreign keys and sets `legacy_alter_table=ON` so that renames keep views bound by name. +WAL needs a local filesystem; do not put the file on a network share. + +## Schemas + +A SQLite connection is one file. SQLBuild stores each schema's relations in that file under +`__`; the `main` schema keeps bare names. For example, model `orders` in schema +`analytics` is the table `analytics__orders`. Schemas are recorded in `_sqlbuild_schemas`, and +every schema relation SQLBuild creates is recorded with its schema and name in +`_sqlbuild_relations`. Listings read that record rather than splitting names, so a table SQLBuild +did not create belongs to `main` even when its name starts with a schema prefix. + +Two relations that would share one table name, such as `orders` in schema `staging` and +`staging__orders` in `main`, fail compilation with error `A706` before anything is written. + +- Targets and models must not set a `database`. +- Schema names cannot contain `__` or end with `_`. +- Attached databases are not used, because a SQLite view cannot reference another attached file. + +## Concurrency + +Readers run alongside one writer. Within one `sqb` process, writers to the same file queue for +each other, so a long model build does not make others time out. A writer in another process waits +up to `busy_timeout` and then fails with error `A702`, naming the file. Every write that spans +statements, such as a promotion, a merge or a seed load, runs in one `BEGIN IMMEDIATE` transaction +and is all or nothing. + +## Types + +SQLite stores values by type affinity, so SQLBuild keeps column types explicitly: + +- Tables are created with explicit column definitions. A column copied from another relation keeps + its declared type; a computed column gets the type SQLBuild infers at compile time. +- Timestamps and dates are ISO-8601 text. `CAST(x AS TIMESTAMP)` and `CAST(x AS DATE)` do not + produce timestamps in SQLite (they keep only the leading number), so write timestamps as text + literals or use SQLite date functions. +- Cursor windows, watermarks and window deletes compare timestamp cursors as UTC text in the + form `YYYY-MM-DD HH:MM:SS[.ffffff]`, so values written with a `T` separator, fractional seconds + or a UTC offset fall in the right window. A value with an offset belongs to the window of its UTC + time; DuckDB's `TIMESTAMP` instead drops the offset. Comparisons you write yourself against + `__cursor_start()` and `__cursor_end()` compare the stored text as is. +- Booleans are stored as `0` and `1`. +- JSON, arrays and objects are stored as JSON text. +- `DECIMAL(p, s)` means exact canonical text. A column whose `DECIMAL(p, s)` or `NUMERIC(p, s)` + type SQLBuild enforces, through a contract or type enforcement, is stored as fixed-scale text + (for example `12.50`), declared `TEXT DECIMAL(p,s)` and reported as `DECIMAL(p,s)`, whether the + table is created, rebuilt, retyped or gains the column. Every other decimal, including a + numeric source column passed through, an inferred one and a bare `DECIMAL`, is a SQLite number + reported as `NUMERIC`; + numbers are binary floating point above 15 significant digits. Text decimals compare as text, so + cast them, for example `CAST(amount AS REAL) > 10`, before comparing with numbers. + +## Capabilities + +| Feature | Support | +|---------|---------| +| Tables, views, seeds, unit tests, audits | Supported | +| Incremental `append`, `merge`, `delete_insert`, cursor windows | Supported, each in one transaction | +| Microbatch | Sequential batches; `batch_concurrency` greater than 1 is refused | +| Snapshots (timestamp, check, historical) | Supported | +| Schema changes | Add, drop and rename columns in place; type changes rebuild the table and its indexes in one transaction | +| `sqb diff`, `sqb clone` | Supported; clones are physical copies | +| Scenarios, `sqb scenario capture`, `--local` replay | Supported; see below | +| Statement time limits | Connection interrupt | + +## Scenarios + +Scenario fixtures and models are tables in the same file, named +`____sqb_____`, and are dropped when the scenario finishes unless you +pass `--retain`. Concurrent scenarios queue for the file's single writer. + +`sqb scenario test --local` transpiles SQLite SQL to DuckDB. Date functions with modifiers, such +as `datetime(ordered_at, 'start of day')`, have no DuckDB translation and fail locally; use +`date(...)` or `strftime(...)` in models you want to replay locally. + +## Limits + +| Limit | Error | +|-------|-------| +| SQL functions and table functions: SQLite has no `CREATE FUNCTION` | `A705` | +| Grants: SQLite has no privileges; grant settings are ignored | none | +| A `database` on a target or model | `A703` | +| Unknown connection keys | `A703` | +| Ambiguous schema names (`__`, trailing `_`) | `A704` | +| Two relations stored under one table name | `A706`, at compile time | +| SQLite older than 3.44 | `A701` | +| Another process holds the write lock beyond `busy_timeout` | `A702` | +| Concurrent microbatch writes: one file has one writer | refused at plan time | diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 2f8cd9fe14..afaa92fa35 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -68,7 +68,7 @@ materialized = "view" | Field | Description | |-------|-------------| | `name` | Project name. Used in fingerprint tracking and manifest generation. | -| `adapter` | Database adapter: `duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, or `sqlserver`. See [Adapters](/docs/concepts/adapters). | +| `adapter` | Database adapter: `duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlite`, or `sqlserver`. See [Adapters](/docs/concepts/adapters). | ### Optional fields @@ -395,6 +395,7 @@ Every built-in adapter enforces the limit: | BigQuery | Job timeout (`job_timeout_ms`) on each query job | `jobs.cancel` (asynchronous) | about 24 days | | Databricks | `STATEMENT_TIMEOUT` session setting, capped at 48 hours | Cursor cancel (asynchronous) | about 24 days | | SQL Server | none | Driver cancel | about 24 days | +| SQLite | none | Connection interrupt | about 24 days | BigQuery and Databricks stop cancelled work asynchronously, so SQLBuild reports their cancellation as requested but unconfirmed. SQL Server has no per-statement limit; the driver's query timeout is @@ -431,6 +432,7 @@ finished. Where a warehouse reports queueing separately, queued time is excluded | BigQuery | `jobs.cancel` at the limit; no job timeout, because a job timeout counts from job creation | Excluded: the clock stops while a job is pending | | Databricks | `STATEMENT_TIMEOUT` session setting and cursor cancel | May count | | SQL Server | Driver cancel at the limit | Lock waits count | +| SQLite | Connection interrupt at the limit | Waits for the write lock count | When a model contains unit tests or nested work with its own limit, the earliest limit applies and cancels whatever is running under it. After a timeout interrupted a statement, SQLBuild checks the diff --git a/website/src/sidebar.json b/website/src/sidebar.json index 880f2e64cc..5f1dc92e98 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -44,6 +44,7 @@ "docs/concepts/adapters/duckdb", "docs/concepts/adapters/motherduck", "docs/concepts/adapters/postgres", + "docs/concepts/adapters/sqlite", "docs/concepts/adapters/bigquery", "docs/concepts/adapters/databricks", "docs/concepts/adapters/sqlserver"