diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md index 3489c31f9a..701f486d34 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/sqlite.md @@ -75,7 +75,9 @@ SQLite stores values by type affinity, so SQLBuild keeps column types explicitly its declared type; a computed column gets the type SQLBuild infers at compile time. - Timestamps and dates are ISO-8601 text. `CAST(x AS TIMESTAMP)` and `CAST(x AS DATE)` do not produce timestamps in SQLite (they keep only the leading number), so write timestamps as text - literals or use SQLite date functions. + literals or use SQLite date functions. SQLBuild never applies such a cast itself: source, + contract and type-change casts to a date or time type keep the stored text, and source loaders + write Python `datetime` values as UTC text in the form `YYYY-MM-DD HH:MM:SS[.ffffff]`. - Cursor windows, watermarks and window deletes compare timestamp cursors as UTC text in the form `YYYY-MM-DD HH:MM:SS[.ffffff]`, so values written with a `T` separator, fractional seconds or a UTC offset fall in the right window. A value with an offset belongs to the window of its UTC diff --git a/src/sqlbuild/adapters/sqlite/_helpers/tables.py b/src/sqlbuild/adapters/sqlite/_helpers/tables.py index 7ef84946ed..ac251812e1 100644 --- a/src/sqlbuild/adapters/sqlite/_helpers/tables.py +++ b/src/sqlbuild/adapters/sqlite/_helpers/tables.py @@ -15,6 +15,7 @@ r"(\"(?:[^\"]|\"\")+\"|[^\s(]+)\s+AS\s+(.+)$", re.IGNORECASE | re.DOTALL, ) +_UNTYPED_VIEW_COLUMN_TYPE: str = "BLOB" _DECIMAL_TYPE: re.Pattern[str] = re.compile(EXACT_DECIMAL_TYPE_PATTERN, re.IGNORECASE) @@ -49,7 +50,7 @@ def create_table_typed( column_sql: str = ", ".join( _column_definition( name=str(row[1]), - declared=str(row[2] or "") or inferred_types.get(str(row[1]).lower(), ""), + declared=_probed_type(row[2]) or inferred_types.get(str(row[1]).lower(), ""), column_types=column_types, ) for row in rows @@ -83,6 +84,13 @@ def decimal_scale(target_type: str) -> int | None: return int(match.group(2) or 0) +def _probed_type(declared: object) -> str: + """A view reports an untyped source column as BLOB; keep such columns untyped.""" + + text: str = str(declared or "") + return "" if text.upper() == _UNTYPED_VIEW_COLUMN_TYPE else text + + def _column_definition(*, name: str, declared: str, column_types: Mapping[str, str]) -> str: override: str | None = column_types.get(name.lower()) column_type: str = storage_type(override) if override is not None else declared diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index 21b1f790f6..92a52a6696 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -143,6 +143,7 @@ MISSING_RELATION_ERROR_FRAGMENT, SCHEMA_REGISTRY_TABLE, SQLITE_TABLE_TYPE, + TEMPORAL_TYPE_PATTERN, UNSUPPORTED_FEATURE_CODE, VERSION_TOO_OLD_CODE, ) @@ -156,6 +157,7 @@ _INFERRED_COLUMN_TYPES: ContextVar[Mapping[str, str]] = ContextVar( "sqlite_inferred_column_types", default=MappingProxyType({}) ) +_TEMPORAL_TYPE: re.Pattern[str] = re.compile(TEMPORAL_TYPE_PATTERN, re.IGNORECASE) _TRUE_SEED_VALUES: frozenset[str] = frozenset({"true", "t", "yes", "y", "1"}) _FALSE_SEED_VALUES: frozenset[str] = frozenset({"false", "f", "no", "n", "0"}) @@ -2237,7 +2239,9 @@ def render_loader_value_literal( return "1" if value else "0" if isinstance(value, int | float | Decimal): return str(value) - if isinstance(value, datetime | date): + if isinstance(value, datetime): + return _quote_sql_string(canonical_timestamp_text(value.isoformat())) + if isinstance(value, date): return _quote_sql_string(value.isoformat()) return _quote_sql_string(str(value)) @@ -2249,13 +2253,12 @@ def render_loader_rows_select( column_sql_types: dict[str, str], inferred_types: dict[str, LoaderLogicalType], ) -> str: - """Render rows as one multi-row VALUES; SQLite names VALUES columns column1, column2.""" + """Render uncast rows as one VALUES; staging declares the types and affinity converts.""" + del column_sql_types if not rows: projections: str = ", ".join( - f"CAST(NULL AS {column_sql_types.get(column_name, 'TEXT')}) AS " - f"{self.render_identifier(column_name)}" - for column_name in column_names + f"NULL AS {self.render_identifier(column_name)}" for column_name in column_names ) return f"SELECT {projections} WHERE 1 = 0" value_rows: list[str] = [] @@ -2270,12 +2273,7 @@ def render_loader_rows_select( value_rows.append(f"({literals})") values_sql: str = ", ".join(value_rows) select_sql: str = ", ".join( - ( - f"column{index} AS {self.render_identifier(column_name)}" - if column_name not in column_sql_types - else f"CAST(column{index} AS {column_sql_types[column_name]}) " - f"AS {self.render_identifier(column_name)}" - ) + f"column{index} AS {self.render_identifier(column_name)}" for index, column_name in enumerate(column_names, start=1) ) return f"SELECT {select_sql} FROM (VALUES {values_sql}) AS __loader_rows" @@ -2283,8 +2281,10 @@ def render_loader_rows_select( def render_source_expression_cast( self, *, expression: str, target_type: str, alias: str ) -> str: - """Cast, rendering exact decimals as canonical fixed-scale text.""" + """Cast, keeping date and time text and rendering exact decimals as fixed-scale text.""" + if _TEMPORAL_TYPE.match(target_type) is not None: + return f"{expression} AS {alias}" scale: int | None = decimal_scale(target_type) if scale is None: return f"CAST({expression} AS {target_type}) AS {alias}" diff --git a/src/sqlbuild/adapters/sqlite/constants.py b/src/sqlbuild/adapters/sqlite/constants.py index a542773fa3..dcebc49096 100644 --- a/src/sqlbuild/adapters/sqlite/constants.py +++ b/src/sqlbuild/adapters/sqlite/constants.py @@ -30,6 +30,7 @@ NUMBER_DECIMAL_TYPE_PATTERN: str = r"^\s*(?:DECIMAL|NUMERIC)\b" NUMBER_DECIMAL_REPORTED_TYPE: str = "NUMERIC" BOOLEAN_TYPE_MARKER: str = "BOOL" +TEMPORAL_TYPE_PATTERN: str = r"^\s*(?:DATE|TIME)" INSERT_KEYWORD: str = "INSERT" SQLITE_VIEW_TYPE: str = "view" SQLITE_TABLE_TYPE: str = "table" diff --git a/src/sqlbuild/executor/load/_helpers/rows.py b/src/sqlbuild/executor/load/_helpers/rows.py index 83f1bcab53..54009795c9 100644 --- a/src/sqlbuild/executor/load/_helpers/rows.py +++ b/src/sqlbuild/executor/load/_helpers/rows.py @@ -136,29 +136,45 @@ def build_rows_sql( ) if not resolved_column_names: raise ExecutorInputError("Source loader returned no rows and source declares no columns") - declared_types: dict[str, str] = _declared_types(columns=columns) resolved_inferred_types: dict[str, LoaderLogicalType] = inferred_types or _infer_column_types( rows=rows ) - column_sql_types: dict[str, str] = { + return adapter.render_loader_rows_select( + rows=rows, + column_names=resolved_column_names, + column_sql_types=loader_column_sql_types( + adapter=adapter, + columns=columns, + column_names=resolved_column_names, + inferred_types=resolved_inferred_types, + ), + inferred_types=resolved_inferred_types, + ) + + +def loader_column_sql_types( + *, + adapter: BaseAdapter, + columns: tuple[SourceColumnEntry, ...], + column_names: tuple[str, ...], + inferred_types: dict[str, LoaderLogicalType], +) -> dict[str, str]: + """Return each loader column's declared or inferred warehouse type, when known.""" + + declared_types: dict[str, str] = _declared_types(columns=columns) + return { column_name: sql_type - for column_name in resolved_column_names + for column_name in column_names if ( sql_type := _column_sql_type( adapter=adapter, column_name=column_name, declared_types=declared_types, - inferred_types=resolved_inferred_types, + inferred_types=inferred_types, ) ) is not None } - return adapter.render_loader_rows_select( - rows=rows, - column_names=resolved_column_names, - column_sql_types=column_sql_types, - inferred_types=resolved_inferred_types, - ) def _declared_types(*, columns: tuple[SourceColumnEntry, ...]) -> dict[str, str]: diff --git a/src/sqlbuild/executor/load/_helpers/staging.py b/src/sqlbuild/executor/load/_helpers/staging.py index 351b1a2db2..759a5af123 100644 --- a/src/sqlbuild/executor/load/_helpers/staging.py +++ b/src/sqlbuild/executor/load/_helpers/staging.py @@ -14,6 +14,7 @@ from sqlbuild.executor.load._helpers.rows import ( build_rows_sql, iter_loader_row_batches, + loader_column_sql_types, update_loader_rows_schema, ) from sqlbuild.executor.load.models import LoaderRowsSchema @@ -83,6 +84,12 @@ def write_loader_rows_to_staging( connection=connection, destination=staging, sql=sql, + config=_staging_config( + adapter=adapter, + source_entry=source_entry, + column_names=column_names, + inferred_types=inferred_types, + ), statement_recorder=statement_recorder, ) staging_created = True @@ -99,11 +106,36 @@ def write_loader_rows_to_staging( connection=connection, destination=staging, sql=sql, + config=_staging_config( + adapter=adapter, + source_entry=source_entry, + column_names=column_names, + inferred_types=inferred_types, + ), statement_recorder=statement_recorder, ) return rows_loaded +def _staging_config( + *, + adapter: BaseAdapter, + source_entry: SourceEntry, + column_names: tuple[str, ...], + inferred_types: dict[str, LoaderLogicalType], +) -> dict[str, Any]: + """Hand loader column types to adapters whose CTAS cannot keep them.""" + + return { + "inferred_column_types": loader_column_sql_types( + adapter=adapter, + columns=source_entry.columns, + column_names=column_names, + inferred_types=inferred_types, + ) + } + + def resolve_loader_relations( *, adapter: BaseAdapter, source_entry: SourceEntry, destination_name: str ) -> tuple[str, str]: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py index 7646b54426..02908a71fe 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/_test_types.py @@ -85,3 +85,21 @@ class SqliteCursorWindowE2ETestCase: events_csv: str commands: tuple[tuple[str, ...], ...] expected_order_ids: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class SqliteWaffleShopE2ETestCase: + description: str + commands: tuple[tuple[str, ...], ...] + expected_fragments: tuple[str, ...] + expected_row_counts: dict[str, int] + expected_return_code: int = 0 + + +@dataclass(frozen=True) +class SqliteLoaderE2ETestCase: + description: str + builds: int + relations: tuple[str, ...] + expected_rows: tuple[tuple[object, ...], ...] + expected_declared_types: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/helpers.py index 2f2aa98e35..17b2df60aa 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/helpers.py @@ -10,7 +10,11 @@ from types import MappingProxyType from typing import Any -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import query_duckdb, run_sqb +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_waffle_shop, + query_duckdb, + run_sqb, +) SQLITE_FILE: str = "warehouse.sqlite" DUCKDB_FILE: str = "warehouse.duckdb" @@ -722,3 +726,188 @@ def run_sqb_commands( return tuple( run_sqb(command=("--no-color", *command), project_dir=project_dir) for command in commands ) + + +WAFFLE_SQLITE_FILE: str = "waffle_shop.sqlite" +_WAFFLE_SQLITE_TOML: str = ( + 'name = "waffle_shop"\n' + 'adapter = "sqlite"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + f'path = "{WAFFLE_SQLITE_FILE}"\n\n' + "[settings]\n" + 'default_audit_severity = "warn"\n\n' + "[defaults]\n" + 'materialized = "table"\n\n' + "[targets.dev]\n" + f'schema = "{SCHEMA}"\n\n' + "[path_defaults.staging]\n" + 'materialized = "view"\n' +) +_WAFFLE_SQLITE_UNSUPPORTED_FILES: tuple[str, ...] = ( + "functions/sql/customer_orders.sql", + "functions/sql/is_completed_order.sql", + "functions/python/is_completed_order_py.py", + "tests/unit/test_customer_orders_table_fn.sql", + "tests/unit/test_is_completed_order_udf.sql", + "models/marts/daily_order_partitioned.sql", +) +_WAFFLE_SQLITE_REWRITES: tuple[tuple[str, str, str], ...] = ( + ( + "models/marts/fact_orders.sql", + '__udf("is_completed_order")(o.status) AS is_completed_order,', + "o.status = 'completed' AS is_completed_order,", + ), + ( + "models/marts/fact_orders.sql", + '__udf("is_completed_order_py")(o.status) AS is_completed_order_py,', + "o.status = 'completed' AS is_completed_order_py,", + ), + ( + "models/marts/_macros/time.py", + " return f\"DATE_TRUNC('{grain}', {expression})\"", + ' if ctx.adapter_name == "sqlite" and grain == "day":\n' + " return f\"datetime({expression}, 'start of day')\"\n" + ' if ctx.adapter_name == "sqlite":\n' + " return f\"datetime({expression}, 'start of day', " + "strftime('+%H hours', {expression}))\"\n" + " return f\"DATE_TRUNC('{grain}', {expression})\"", + ), + ("models/marts/daily_revenue.sql", "CAST(o.ordered_at AS DATE)", "date(o.ordered_at)"), +) +_TIMESTAMP_LITERAL_CAST: re.Pattern[str] = re.compile(r"CAST\(('[^']*') AS TIMESTAMP\)") +WAFFLE_ROW_COUNT_RELATIONS: tuple[str, ...] = ( + "raw_customers", + "raw_orders", + "waffle_types", + "stg_customers", + "stg_orders", + "stg_payments", + "fact_orders", + "dim_customers", + "daily_revenue", + "scenario_order_prices", + "customer_status_snapshot", + "order_status_index", + "hourly_order_activity", + "daily_activity_rollup", + "hourly_activity_with_daily_context", +) + + +def prepare_sqlite_waffle_shop(*, tmp_path: Path) -> Path: + """Copy the waffle shop and adapt it to SQLite's documented platform limits. + + SQLite has no CREATE FUNCTION, so SQL and Python functions and their unit tests are removed and + the completed-order check is inlined. SQLite has no DATE_TRUNC, so the truncation macro uses + SQLite date functions. `CAST(x AS DATE)` and `CAST(x AS TIMESTAMP)` keep only the leading number + in SQLite, so the daily revenue model uses `date(...)`, the payments source writes timestamps + as text, and the partition-tracked model, whose materialization deletes by `CAST(... AS DATE)`, + is removed. + """ + + project_dir: Path = prepare_waffle_shop(tmp_path) + (project_dir / "sqlbuild_project.toml").write_text(_WAFFLE_SQLITE_TOML, encoding="utf-8") + relative_path: str + for relative_path in _WAFFLE_SQLITE_UNSUPPORTED_FILES: + (project_dir / relative_path).unlink() + old: str + new: str + for relative_path, old, new in _WAFFLE_SQLITE_REWRITES: + _replace_in_file(path=project_dir / relative_path, old=old, new=new) + sources_path: Path = project_dir / "sources" / "raw.yml" + sources_path.write_text( + _TIMESTAMP_LITERAL_CAST.sub(r"\1", sources_path.read_text(encoding="utf-8")), + encoding="utf-8", + ) + return project_dir + + +def _replace_in_file(*, path: Path, old: str, new: str) -> None: + contents: str = path.read_text(encoding="utf-8") + assert old in contents, f"{path} no longer contains {old!r}" + path.write_text(contents.replace(old, new), encoding="utf-8") + + +def waffle_row_counts(*, adapter: str, project_dir: Path) -> dict[str, int]: + """Count the rows of every compared waffle shop relation in either engine.""" + + return { + relation: _WAFFLE_ROW_COUNTERS[adapter](project_dir, relation) + for relation in WAFFLE_ROW_COUNT_RELATIONS + } + + +def _sqlite_waffle_row_count(project_dir: Path, relation: str) -> int: + rows: tuple[tuple[object, ...], ...] = sqlite_rows( + db_path=project_dir / WAFFLE_SQLITE_FILE, + sql=f'SELECT COUNT(*) FROM "{SCHEMA}__{relation}"', + ) + return int(str(rows[0][0])) + + +def _duckdb_waffle_row_count(project_dir: Path, relation: str) -> int: + rows: list[tuple[Any, ...]] = query_duckdb( + db_path=project_dir / "waffle_shop.duckdb", sql=f"SELECT COUNT(*) FROM main.{relation}" + ) + return int(rows[0][0]) + + +_WAFFLE_ROW_COUNTERS: MappingProxyType[str, Callable[[Path, str], int]] = MappingProxyType( + {"sqlite": _sqlite_waffle_row_count, "duckdb": _duckdb_waffle_row_count} +) + + +LOADER_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": _SQLITE_TOML, + "sources/raw.yml": """sources: + - name: raw_events + description: Order events from a loader. + managed: true + write_strategy: delete_insert + cursor_column: happened_at + columns: + - name: event_id + type: INTEGER + - name: happened_at + type: TIMESTAMP + - name: event_day + type: DATE + - name: logged_at + type: TIMESTAMP +""", + "python/loaders/events.py": '''from datetime import date, datetime, timedelta, timezone + +from sqlbuild.loaders import loader + + +@loader +def raw_events(ctx: object) -> list[dict[str, object]]: + """Order events with naive, zoned and date values.""" + _ = ctx + return [ + { + "event_id": 1, + "happened_at": datetime(2026, 4, 1, 9, 15), + "event_day": date(2026, 4, 1), + "logged_at": datetime(2026, 4, 1, 11, 30, tzinfo=timezone(timedelta(hours=2))), + }, + { + "event_id": 2, + "happened_at": datetime(2026, 4, 2, 8, 45, 30, 250000), + "event_day": date(2026, 4, 2), + "logged_at": None, + }, + ] +''', + "models/events.sql": """MODEL (description "Events from the source.", materialized table); + +SELECT event_id, happened_at, event_day, logged_at FROM __source("raw_events") +""", +} +LOADED_EVENT_ROWS_SQL: str = ( + "SELECT event_id, happened_at, event_day, logged_at FROM analytics__{relation} ORDER BY 1" +) +LOADED_EVENT_TYPES_SQL: str = ( + "SELECT name, type FROM pragma_table_info('analytics__raw_events') ORDER BY cid" +) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_waffle_shop.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_waffle_shop.py new file mode 100644 index 0000000000..6d918b1c6c --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_waffle_shop.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.sqlite._test_types import ( + SqliteLoaderE2ETestCase, + SqliteWaffleShopE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.sqlite.helpers import ( + LOADED_EVENT_ROWS_SQL, + LOADED_EVENT_TYPES_SQL, + LOADER_PROJECT_FILES, + SQLITE_FILE, + prepare_sqlite_waffle_shop, + run_sqb_commands, + sqlite_rows, + waffle_row_counts, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + prepare_waffle_shop, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteWaffleShopE2ETestCase( + description="waffle shop full build succeeds on sqlite", + commands=(("build",), ("test",), ("build",)), + expected_fragments=("Completed successfully", "FAIL=0", "Completed successfully"), + expected_row_counts={ + "raw_customers": 5, + "raw_orders": 10, + "waffle_types": 6, + "stg_customers": 5, + "stg_orders": 10, + "stg_payments": 8, + "fact_orders": 10, + "dim_customers": 5, + "daily_revenue": 3, + "scenario_order_prices": 10, + "customer_status_snapshot": 5, + "order_status_index": 10, + "hourly_order_activity": 8, + "daily_activity_rollup": 3, + "hourly_activity_with_daily_context": 0, + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_waffle_shop_when_building_testing_and_rebuilding_on_sqlite_then_rows_match_duckdb( + tmp_path: Path, test_case: SqliteWaffleShopE2ETestCase +) -> None: + project_dirs: dict[str, Path] = { + "sqlite": prepare_sqlite_waffle_shop(tmp_path=tmp_path / "sqlite"), + "duckdb": prepare_waffle_shop(tmp_path / "duckdb"), + } + row_counts: dict[str, dict[str, int]] = {} + adapter: str + project_dir: Path + for adapter, project_dir in project_dirs.items(): + results: tuple[subprocess.CompletedProcess[str], ...] = run_sqb_commands( + project_dir=project_dir, commands=test_case.commands + ) + result: subprocess.CompletedProcess[str] + fragment: str + for result, fragment in zip(results, test_case.expected_fragments, strict=True): + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_return_code, (adapter, output) + assert fragment in result.stdout, (adapter, output) + row_counts[adapter] = waffle_row_counts(adapter=adapter, project_dir=project_dir) + + assert row_counts["sqlite"] == row_counts["duckdb"] + assert row_counts["sqlite"] == test_case.expected_row_counts + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteLoaderE2ETestCase( + description="loaded timestamps and dates stay ISO text under their declared types", + builds=2, + relations=("raw_events", "events"), + expected_rows=( + (1, "2026-04-01 09:15:00", "2026-04-01", "2026-04-01 09:30:00"), + (2, "2026-04-02 08:45:30.25", "2026-04-02", None), + ), + expected_declared_types=( + ("event_id", "INTEGER"), + ("happened_at", "TIMESTAMP"), + ("event_day", "DATE"), + ("logged_at", "TIMESTAMP"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_loader_with_timestamps_when_building_on_sqlite_then_values_are_kept( + tmp_path: Path, test_case: SqliteLoaderE2ETestCase +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="events_shop", repo_files=LOADER_PROJECT_FILES + ) + + results: tuple[subprocess.CompletedProcess[str], ...] = run_sqb_commands( + project_dir=project_dir, commands=(("build",),) * test_case.builds + ) + + assert [result.returncode for result in results] == [0] * test_case.builds, [ + result.stdout + result.stderr for result in results + ] + db_path: Path = project_dir / SQLITE_FILE + relation: str + for relation in test_case.relations: + assert ( + sqlite_rows(db_path=db_path, sql=LOADED_EVENT_ROWS_SQL.format(relation=relation)) + == test_case.expected_rows + ) + assert sqlite_rows(db_path=db_path, sql=LOADED_EVENT_TYPES_SQL) == ( + test_case.expected_declared_types + ) diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py b/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py index 54b25a3560..d1d89ecfc7 100644 --- a/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py @@ -40,6 +40,7 @@ class SqliteTypedTableTestCase: config: dict[str, object] expected_columns: tuple[ColumnInfo, ...] expected_rows: tuple[tuple[object, ...], ...] + setup_statements: tuple[str, ...] = () @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py b/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py index c33db3a4db..389f56d9ed 100644 --- a/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/sqlite/test_client.py @@ -267,6 +267,17 @@ def test_given_schemas_when_listing_relations_then_sqlite_reports_logical_names( ), expected_rows=((1, "12.50"), (2, "7.25"), (3, "30.00")), ), + SqliteTypedTableTestCase( + description="untyped columns stay untyped instead of becoming BLOB", + setup_statements=( + "CREATE TABLE order_notes (order_id INTEGER, note)", + "INSERT INTO order_notes VALUES (1, 'gift')", + ), + select_sql="SELECT order_id, note FROM order_notes", + config={}, + expected_columns=(ColumnInfo("order_id", "INTEGER"), ColumnInfo("note", "TEXT")), + expected_rows=((1, "gift"),), + ), ], ids=lambda case: case.description, ) @@ -274,6 +285,7 @@ def test_given_query_when_creating_table_then_sqlite_keeps_column_types( test_case: SqliteTypedTableTestCase, adapter: SqliteAdapter, connection: Any ) -> None: create_orders(adapter=adapter, connection=connection, relation="orders") + run_sql(adapter=adapter, connection=connection, statements=test_case.setup_statements) adapter.create_table_as( connection=connection, @@ -304,6 +316,11 @@ def test_given_query_when_creating_table_then_sqlite_keeps_column_types( 'ELSE printf(\'%.2f\', "amount") END AS "amount"' ), ), + SqliteCastTestCase( + description="date and time targets keep their ISO text unchanged", + target_type="TIMESTAMP", + expected_sql='"amount" AS "amount"', + ), SqliteCastTestCase( description="other targets render a plain cast", target_type="INTEGER", diff --git a/website/src/content/docs/docs/concepts/adapters/sqlite.mdx b/website/src/content/docs/docs/concepts/adapters/sqlite.mdx index 5999258407..a1930765ee 100644 --- a/website/src/content/docs/docs/concepts/adapters/sqlite.mdx +++ b/website/src/content/docs/docs/concepts/adapters/sqlite.mdx @@ -62,7 +62,9 @@ SQLite stores values by type affinity, so SQLBuild keeps column types explicitly its declared type; a computed column gets the type SQLBuild infers at compile time. - Timestamps and dates are ISO-8601 text. `CAST(x AS TIMESTAMP)` and `CAST(x AS DATE)` do not produce timestamps in SQLite (they keep only the leading number), so write timestamps as text - literals or use SQLite date functions. + literals or use SQLite date functions. SQLBuild never applies such a cast itself: source, + contract and type-change casts to a date or time type keep the stored text, and source loaders + write Python `datetime` values as UTC text in the form `YYYY-MM-DD HH:MM:SS[.ffffff]`. - Cursor windows, watermarks and window deletes compare timestamp cursors as UTC text in the form `YYYY-MM-DD HH:MM:SS[.ffffff]`, so values written with a `T` separator, fractional seconds or a UTC offset fall in the right window. A value with an offset belongs to the window of its UTC