Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ E2E_DUCKDB_CLI_PATHS := \
tests/e2e/src/sqlbuild/cli/commands/main/seed \
tests/e2e/src/sqlbuild/cli/commands/main/skills \
tests/e2e/src/sqlbuild/cli/commands/main/snowflake \
tests/e2e/src/sqlbuild/cli/commands/main/sqlite \
tests/e2e/src/sqlbuild/cli/commands/main/sqlserver \
tests/e2e/src/sqlbuild/cli/commands/main/test

Expand Down
3 changes: 3 additions & 0 deletions crates/sqlbuild-config/src/command_connection/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ pub const CONNECTION_SOURCE_KEY: &str = "source";
pub const DBT_PROFILE_CONNECTION_SOURCE: &str = "dbt_profile";
pub const DUCKDB_ADAPTER: &str = "duckdb";
pub const DUCKDB_MEMORY_DATABASE: &str = ":memory:";
/// Adapters whose connection names a local database file, and the key that names it.
pub const LOCAL_DATABASE_PATH_KEYS: [(&str, &str); 2] =
[(DUCKDB_ADAPTER, "database"), ("sqlite", "path")];
pub const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml";
/// Connection keys a dbt profile connection routes on rather than connects with.
pub const DBT_PROFILE_ROUTING_KEYS: [&str; 5] =
Expand Down
3 changes: 2 additions & 1 deletion crates/sqlbuild-core/src/lifecycle_events/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -258,14 +258,15 @@ pub const OPERATION_STRATEGIES: [&str; 10] = [
"sql",
"sync_all_columns",
];
pub const OPERATION_ADAPTERS: [&str; 8] = [
pub const OPERATION_ADAPTERS: [&str; 9] = [
"bigquery",
"custom",
"databricks",
"duckdb",
"motherduck",
"postgres",
"snowflake",
"sqlite",
"sqlserver",
];

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ use pyo3::prelude::{
use pyo3::types::{PyDict, PyDictMethods, PyString};
use pyo3::{pyfunction, wrap_pyfunction};
use sqlbuild_config::command_connection::constants::{
CONNECTION_DATABASE_KEY, CONNECTION_SOURCE_KEY, DBT_PROFILE_CONNECTION_SOURCE,
DBT_PROFILE_ROUTING_KEYS, DUCKDB_ADAPTER, DUCKDB_MEMORY_DATABASE,
CONNECTION_SOURCE_KEY, DBT_PROFILE_CONNECTION_SOURCE, DBT_PROFILE_ROUTING_KEYS,
DUCKDB_MEMORY_DATABASE, LOCAL_DATABASE_PATH_KEYS,
};
use sqlbuild_config::command_connection::main::missing_connection_error::missing_connection_error;
use sqlbuild_config::command_connection::main::snowflake_key_warning::snowflake_key_warning;
Expand Down Expand Up @@ -105,16 +105,19 @@ pub(crate) fn with_dbt_profile<'py>(
Ok(merged)
}

/// Make a relative DuckDB database path relative to the project directory.
pub(crate) fn anchor_duckdb_database(
/// Make a relative local database file path (DuckDB, SQLite) relative to the project directory.
pub(crate) fn anchor_local_database_path(
config: &Bound<'_, PyDict>,
(project_dir, adapter_name): (&Bound<'_, PyAny>, &str),
) -> PyResult<()> {
let py: Python<'_> = config.py();
if adapter_name != DUCKDB_ADAPTER {
let Some((_, path_key)) = LOCAL_DATABASE_PATH_KEYS
.iter()
.find(|(adapter, _)| *adapter == adapter_name)
else {
return Ok(());
}
let Some(database) = config.get_item(CONNECTION_DATABASE_KEY)? else {
};
let Some(database) = config.get_item(*path_key)? else {
return Ok(());
};
if !database.is_instance_of::<PyString>() {
Expand All @@ -128,7 +131,7 @@ pub(crate) fn anchor_duckdb_database(
return Ok(());
}
let anchored: Bound<'_, PyAny> = path(py, project_dir)?.div(&database)?.str()?.into_any();
config.set_item(CONNECTION_DATABASE_KEY, anchored)
config.set_item(*path_key, anchored)
}

/// Resolve dbt profiles, relative DuckDB paths, and the command group's warehouse.
Expand Down Expand Up @@ -187,7 +190,7 @@ fn connection_config<'py>(
if let Some(warning) = snowflake_key_warning(&adapter, &keys) {
warn(py, &warning)?;
}
anchor_duckdb_database(&config, (project_dir, &adapter))?;
anchor_local_database_path(&config, (project_dir, &adapter))?;
apply_warehouse(
config.as_any(),
(adapter_name, project_dir),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use pyo3::{pyfunction, wrap_pyfunction};

use crate::bindings::_helpers::build_command::support::{boolean, keyword_relay, none};
use crate::bindings::_helpers::command_runtime::connection::{
anchor_duckdb_database, with_dbt_profile,
anchor_local_database_path, with_dbt_profile,
};
use crate::bindings::_helpers::command_runtime::warehouses::apply_command_warehouse;
use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_connection_config;
Expand Down Expand Up @@ -54,7 +54,7 @@ pub(crate) fn connection_config<'py>(
(&project_dir, &adapter_name),
(&discovered, &py.None().into_bound(py)),
)?;
anchor_duckdb_database(
anchor_local_database_path(
&config,
(&project_dir, &adapter_name.str()?.extract::<String>()?),
)?;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -631,6 +631,7 @@ const R_MODEL_PLAN_ENTRY: RecordSpec = RecordSpec {
FieldSpec::new("on_schema_change", FieldDefault::None),
FieldSpec::new("type_enforcement", FieldDefault::False),
FieldSpec::new("declared_columns", FieldDefault::EmptyTuple),
FieldSpec::new("inferred_columns", FieldDefault::EmptyTuple),
FieldSpec::new("contract_enforced", FieldDefault::False),
FieldSpec::new("contract_columns", FieldDefault::EmptyTuple),
FieldSpec::new("contract_dynamic_columns", FieldDefault::EmptyTuple),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,15 @@ pub(super) fn cursor_wrap(
};
let start = literal("start")?;
let end = literal("end")?;
let operand = cursor_operand(adapter, &column, cursor_type)?;
let alias = inject_alias
&& adapter
.call_method0("requires_derived_table_aliases")?
.is_truthy()?;
Ok(cursor_subquery(
&relation,
&CursorFilter {
column: &column,
column: &operand,
start: &start,
end: &end,
lower_inclusive: filtering.lower_bound_inclusive,
Expand All @@ -69,6 +70,27 @@ pub(super) fn cursor_wrap(
))
}

/// The adapter's expression for a cursor column compared with bounds, or the column itself.
fn cursor_operand(
adapter: &Bound<'_, PyAny>,
column: &str,
cursor_type: &Bound<'_, PyAny>,
) -> PyResult<String> {
let py = adapter.py();
if !adapter.hasattr("render_cursor_operand")? {
return Ok(column.to_owned());
}
method(
adapter,
"render_cursor_operand",
&[
("expression", string(py, column)),
("cursor_type", cursor_type.clone()),
],
)?
.extract()
}

/// The cursor filtering of a keyword-argument resolution request.
pub(super) fn filtering_of<'py>(
arguments: &KeywordArguments<'py>,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,25 @@ fn schema_columns<'py>(
PyTuple::new(py, columns)
}

/// The model's typed inferred output columns, which adapters without CTAS types declare.
fn inferred_columns<'py>(model: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyTuple>> {
let py: Python<'py> = model.py();
let class: Bound<'py, PyAny> = host::entry(py, AdapterHost::ColumnInfo)?;
let mut columns: Vec<Bound<'py, PyAny>> = Vec::new();
for (name, column_type, _) in model_reads::inferred_rows(model)?.unwrap_or_default() {
if let Some(column_type) = column_type {
columns.push(column_info(
&class,
(
PyString::new(py, &name).into_any(),
PyString::new(py, &column_type).into_any(),
),
)?);
}
}
PyTuple::new(py, columns)
}

/// A string or list-of-strings config value as a tuple of strings.
fn string_list<'py>(values: &Bound<'py, PyAny>, key: &str) -> PyResult<Bound<'py, PyTuple>> {
let py: Python<'py> = values.py();
Expand Down Expand Up @@ -471,6 +490,7 @@ fn contract_fields<'py>(
) -> PyResult<()> {
let py: Python<'py> = model.py();
entry.set_item("declared_columns", schema_columns(model, true)?)?;
entry.set_item("inferred_columns", inferred_columns(model)?)?;
entry.set_item(
"contract_enforced",
mapping_get(values, "contract")?.eq(ENFORCED_CONTRACT)?,
Expand Down
10 changes: 10 additions & 0 deletions fensu.toml
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,11 @@ rule = "FFR309"
path = "src/sqlbuild/adapters/snowflake/__init__.py"
reason = "The Snowflake backend intentionally owns behavior through its adapter class rather than function entrypoints."

[[targets.python.rule_exceptions]]
rule = "FFR309"
path = "src/sqlbuild/adapters/sqlite/__init__.py"
reason = "The SQLite backend intentionally owns behavior through its adapter class rather than function entrypoints."

[[targets.python.rule_exceptions]]
rule = "FFR309"
path = "src/sqlbuild/adapters/sqlserver/__init__.py"
Expand Down Expand Up @@ -215,6 +220,11 @@ rule = "FFR601"
path = "src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py"
reason = "The Snowflake adapter intentionally keeps overrideable backend behavior on one class."

[[targets.python.rule_exceptions]]
rule = "FFR601"
path = "src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py"
reason = "The SQLite adapter intentionally keeps overrideable backend behavior on one class."

[[targets.python.rule_exceptions]]
rule = "FFR601"
path = "src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in
- [DuckDB](concepts/adapters/duckdb.md) (`concepts/adapters/duckdb`) - DuckDB adapter configuration for SQLBuild.
- [MotherDuck](concepts/adapters/motherduck.md) (`concepts/adapters/motherduck`) - MotherDuck adapter configuration for SQLBuild.
- [PostgreSQL](concepts/adapters/postgres.md) (`concepts/adapters/postgres`) - PostgreSQL adapter configuration for SQLBuild.
- [SQLite](concepts/adapters/sqlite.md) (`concepts/adapters/sqlite`) - SQLite adapter configuration, capabilities and limits for SQLBuild.
- [BigQuery](concepts/adapters/bigquery.md) (`concepts/adapters/bigquery`) - BigQuery adapter configuration for SQLBuild.
- [Databricks](concepts/adapters/databricks.md) (`concepts/adapters/databricks`) - Databricks adapter configuration for SQLBuild.
- [SQL Server](concepts/adapters/sqlserver.md) (`concepts/adapters/sqlserver`) - Microsoft SQL Server adapter configuration for SQLBuild.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ SQLBuild uses adapters to connect to different database engines. Snowflake is th
| [BigQuery](adapters/bigquery.md) | Beta | `sqlbuild[bigquery]` |
| [Databricks](adapters/databricks.md) | Beta | `sqlbuild[databricks]` |
| [SQL Server](adapters/sqlserver.md) | Beta | `sqlbuild[sqlserver]` |
| [SQLite](adapters/sqlite.md) | Beta | included by default |
| ClickHouse | Planned | |
| Redshift | Planned | |
| Trino | Planned | |
Expand Down Expand Up @@ -108,7 +109,7 @@ A few `StrictAdapter` methods are optional and have a documented default. `creat
- SQLBuild discovers all `.py` files under `adapters/` recursively (excluding `__init__.py` and files starting with `_`)
- Each file is scanned for classes that define a string `adapter_name` and subclass `StrictAdapter` (or any of its subclasses like `BaseAdapter` or a built-in adapter)
- Adapter names must be unique across all adapter files - duplicates raise an error
- Custom adapter names cannot shadow built-in names (`duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlserver`)
- Custom adapter names cannot shadow built-in names (`duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlite`, `sqlserver`)

### Adapter class hierarchy

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
<!-- generated-by: sqlbuild skills -->

# SQLite

> SQLite adapter configuration, capabilities and limits for SQLBuild.

Online: https://sqlbuild.com/docs/concepts/adapters/sqlite/

## Contents

- Connection config
- Schemas
- Concurrency
- Types
- Capabilities
- Scenarios
- Limits

SQLite uses the `sqlite3` module that ships with Python. No extra installation is needed.
SQLBuild requires SQLite 3.44 or newer; Python builds installed by `uv` bundle a recent version.

## Connection config

```toml
adapter = "sqlite"
default_target = "dev"

[connections.local]
path = "warehouse.sqlite"

[targets.dev]
connection = "local"
schema = "analytics"
```

| Field | Description |
|-------|-------------|
| `path` | Path to the database file, relative to the project directory. Use `:memory:` for an in-memory database. The file and its parent directory are created when missing. |
| `busy_timeout` | Seconds a writer waits for another process to release the file before failing. Default `5`. |
| `read_only` | Open the file read-only. Default `false`. |

On connect SQLBuild turns on write-ahead logging (`journal_mode=WAL`), sets the busy timeout,
enables foreign keys and sets `legacy_alter_table=ON` so that renames keep views bound by name.
WAL needs a local filesystem; do not put the file on a network share.

## Schemas

A SQLite connection is one file. SQLBuild stores each schema's relations in that file under
`<schema>__<name>`; the `main` schema keeps bare names. For example, model `orders` in schema
`analytics` is the table `analytics__orders`. Schemas are recorded in `_sqlbuild_schemas`, and
every schema relation SQLBuild creates is recorded with its schema and name in
`_sqlbuild_relations`. Listings read that record rather than splitting names, so a table SQLBuild
did not create belongs to `main` even when its name starts with a schema prefix.

Two relations that would share one table name, such as `orders` in schema `staging` and
`staging__orders` in `main`, fail compilation with error `A706` before anything is written.

- Targets and models must not set a `database`.
- Schema names cannot contain `__` or end with `_`.
- Attached databases are not used, because a SQLite view cannot reference another attached file.

## Concurrency

Readers run alongside one writer. Within one `sqb` process, writers to the same file queue for
each other, so a long model build does not make others time out. A writer in another process waits
up to `busy_timeout` and then fails with error `A702`, naming the file. Every write that spans
statements, such as a promotion, a merge or a seed load, runs in one `BEGIN IMMEDIATE` transaction
and is all or nothing.

## Types

SQLite stores values by type affinity, so SQLBuild keeps column types explicitly:

- Tables are created with explicit column definitions. A column copied from another relation keeps
its declared type; a computed column gets the type SQLBuild infers at compile time.
- Timestamps and dates are ISO-8601 text. `CAST(x AS TIMESTAMP)` and `CAST(x AS DATE)` do not
produce timestamps in SQLite (they keep only the leading number), so write timestamps as text
literals or use SQLite date functions.
- Cursor windows, watermarks and window deletes compare timestamp cursors as UTC text in the
form `YYYY-MM-DD HH:MM:SS[.ffffff]`, so values written with a `T` separator, fractional seconds
or a UTC offset fall in the right window. A value with an offset belongs to the window of its UTC
time; DuckDB's `TIMESTAMP` instead drops the offset. Comparisons you write yourself against
`__cursor_start()` and `__cursor_end()` compare the stored text as is.
- Booleans are stored as `0` and `1`.
- JSON, arrays and objects are stored as JSON text.
- `DECIMAL(p, s)` means exact canonical text. A column whose `DECIMAL(p, s)` or `NUMERIC(p, s)`
type SQLBuild enforces, through a contract or type enforcement, is stored as fixed-scale text
(for example `12.50`), declared `TEXT DECIMAL(p,s)` and reported as `DECIMAL(p,s)`, whether the
table is created, rebuilt, retyped or gains the column. Every other decimal, including a
numeric source column passed through, an inferred one and a bare `DECIMAL`, is a SQLite number
reported as `NUMERIC`;
numbers are binary floating point above 15 significant digits. Text decimals compare as text, so
cast them, for example `CAST(amount AS REAL) > 10`, before comparing with numbers.

## Capabilities

| Feature | Support |
|---------|---------|
| Tables, views, seeds, unit tests, audits | Supported |
| Incremental `append`, `merge`, `delete_insert`, cursor windows | Supported, each in one transaction |
| Microbatch | Sequential batches; `batch_concurrency` greater than 1 is refused |
| Snapshots (timestamp, check, historical) | Supported |
| Schema changes | Add, drop and rename columns in place; type changes rebuild the table and its indexes in one transaction |
| `sqb diff`, `sqb clone` | Supported; clones are physical copies |
| Scenarios, `sqb scenario capture`, `--local` replay | Supported; see below |
| Statement time limits | Connection interrupt |

## Scenarios

Scenario fixtures and models are tables in the same file, named
`<schema>____sqb_<run>__<kind>__<name>`, and are dropped when the scenario finishes unless you
pass `--retain`. Concurrent scenarios queue for the file's single writer.

`sqb scenario test --local` transpiles SQLite SQL to DuckDB. Date functions with modifiers, such
as `datetime(ordered_at, 'start of day')`, have no DuckDB translation and fail locally; use
`date(...)` or `strftime(...)` in models you want to replay locally.

## Limits

| Limit | Error |
|-------|-------|
| SQL functions and table functions: SQLite has no `CREATE FUNCTION` | `A705` |
| Grants: SQLite has no privileges; grant settings are ignored | none |
| A `database` on a target or model | `A703` |
| Unknown connection keys | `A703` |
| Ambiguous schema names (`__`, trailing `_`) | `A704` |
| Two relations stored under one table name | `A706`, at compile time |
| SQLite older than 3.44 | `A701` |
| Another process holds the write lock beyond `busy_timeout` | `A702` |
| Concurrent microbatch writes: one file has one writer | refused at plan time |
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ materialized = "view"
| Field | Description |
|-------|-------------|
| `name` | Project name. Used in fingerprint tracking and manifest generation. |
| `adapter` | Database adapter: `duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, or `sqlserver`. See [Adapters](adapters.md). |
| `adapter` | Database adapter: `duckdb`, `motherduck`, `snowflake`, `bigquery`, `databricks`, `postgres`, `sqlite`, or `sqlserver`. See [Adapters](adapters.md). |

### Optional fields

Expand Down Expand Up @@ -418,6 +418,7 @@ Every built-in adapter enforces the limit:
| BigQuery | Job timeout (`job_timeout_ms`) on each query job | `jobs.cancel` (asynchronous) | about 24 days |
| Databricks | `STATEMENT_TIMEOUT` session setting, capped at 48 hours | Cursor cancel (asynchronous) | about 24 days |
| SQL Server | none | Driver cancel | about 24 days |
| SQLite | none | Connection interrupt | about 24 days |

BigQuery and Databricks stop cancelled work asynchronously, so SQLBuild reports their cancellation
as requested but unconfirmed. SQL Server has no per-statement limit; the driver's query timeout is
Expand Down Expand Up @@ -454,6 +455,7 @@ finished. Where a warehouse reports queueing separately, queued time is excluded
| BigQuery | `jobs.cancel` at the limit; no job timeout, because a job timeout counts from job creation | Excluded: the clock stops while a job is pending |
| Databricks | `STATEMENT_TIMEOUT` session setting and cursor cancel | May count |
| SQL Server | Driver cancel at the limit | Lock waits count |
| SQLite | Connection interrupt at the limit | Waits for the write lock count |

When a model contains unit tests or nested work with its own limit, the earliest limit applies and
cancels whatever is running under it. After a timeout interrupted a statement, SQLBuild checks the
Expand Down
Loading
Loading