From 1941dc147c0d2cb408897d434594304c2838d524 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 21 Oct 2026 18:26:42 +0100 Subject: [PATCH] fix(ingestr): pin the DuckDB ADBC driver to the installed duckdb version DuckDB 2.0.0 was released and ingestr installs the newest DuckDB ADBC driver when none is installed, so files it wrote moved to a storage format the locked duckdb package cannot read. SQLBuild now installs the driver matching its duckdb package before running ingestr, unless ADBC_DRIVER_PATH is set, and the ingestr extra includes dbc. --- .../src/lifecycle_events/constants.rs | 3 +- .../sqlbuild-core/src/progress/constants.rs | 6 +- pyproject.toml | 6 +- .../references/docs/integrations/ingestr.md | 5 + .../integrations/ingestr/_helpers/runner.py | 73 ++++++++++++- .../integrations/ingestr/_test_types.py | 15 +++ .../integrations/ingestr/test_runner.py | 103 +++++++++++++++++- uv.lock | 28 +++-- .../docs/docs/integrations/ingestr.mdx | 5 + 9 files changed, 228 insertions(+), 16 deletions(-) diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index 70e6a04f84..f370a52fc6 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -183,7 +183,8 @@ pub const OPERATION_KINDS: [&str; 10] = [ "subprocess", "warehouse", ]; -pub const OPERATION_NAMES: [&str; 55] = [ +pub const OPERATION_NAMES: [&str; 56] = [ + "adbc_driver_install", "audit_evaluation", "clone_execution", "clone_finalization", diff --git a/crates/sqlbuild-core/src/progress/constants.rs b/crates/sqlbuild-core/src/progress/constants.rs index 2fbc23967f..f5fa3267e7 100644 --- a/crates/sqlbuild-core/src/progress/constants.rs +++ b/crates/sqlbuild-core/src/progress/constants.rs @@ -1,7 +1,7 @@ //! Lifecycle event types, operation labels and layout of CLI progress lines. /// Operations with visible progress lines, and the label each renders with. -pub(crate) const VISIBLE_OPERATION_LABELS: [(&str, &str); 31] = [ +pub(crate) const VISIBLE_OPERATION_LABELS: [(&str, &str); 32] = [ ("clone_finalization", "Clone finalization"), ("discovery_declaration_parse", "Declaration parsing"), ("discovery_filesystem_walk", "Filesystem discovery"), @@ -14,6 +14,7 @@ pub(crate) const VISIBLE_OPERATION_LABELS: [(&str, &str); 31] = [ ("dbt_command", "dbt command"), ("external_source_load", "loader callable"), ("ingestr_command", "ingestr subprocess"), + ("adbc_driver_install", "DuckDB driver install"), ("managed_source_load", "loader callable"), ("python_asset", "Python asset attempt"), ("python_check", "Python check attempt"), @@ -38,7 +39,8 @@ pub(crate) const VISIBLE_OPERATION_LABELS: [(&str, &str); 31] = [ ("relation_promotion", "Promote relation"), ]; /// Operations a live transient owner shows on a terminal when no resource attempt owns them. -pub(crate) const TTY_TRANSIENT_OWNER_OPERATIONS: [&str; 18] = [ +pub(crate) const TTY_TRANSIENT_OWNER_OPERATIONS: [&str; 19] = [ + "adbc_driver_install", "dbt_command", "discovery_declaration_parse", "discovery_filesystem_walk", diff --git a/pyproject.toml b/pyproject.toml index 2a20ff6c19..54fcaf4048 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,8 @@ clickhouse = [ "clickhouse-connect>=0.8", ] ingestr = [ - "ingestr>=1.0.0", + "dbc>=0.3.1", + "ingestr>=1.1.73", ] dlt = [ "dlt[duckdb,filesystem,sql_database]>=1.28.0,<2", @@ -72,7 +73,8 @@ dev = [ "dbt-snowflake==1.12.0b1", "dbt-sqlserver>=1.9.0,<1.11", "dlt[duckdb,filesystem,sql_database]>=1.28.0,<2", - "ingestr>=1.0.0", + "dbc>=0.3.1", + "ingestr>=1.1.73", "jsonschema>=4.23.0", "maturin>=1.9,<2", "pre-commit>=4.3.0", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md index b573813aed..58cd89e2b4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/ingestr.md @@ -31,6 +31,11 @@ uv pip install 'sqlbuild[ingestr]' This installs `ingestr` alongside SQLBuild. The `ingestr` CLI must be available on `PATH`. +ingestr reads and writes DuckDB through an ADBC driver. Before the first ingestr run, SQLBuild +installs the DuckDB driver that matches its own `duckdb` package (with `dbc`, included in the extra) +into `/etc/adbc/drivers`, so the files ingestr writes stay readable by SQLBuild. Set +`ADBC_DRIVER_PATH` to manage the driver yourself. + ## How it works 1. Declare an `ingestr` block on a source in `sources/*.yml` diff --git a/src/sqlbuild/integrations/ingestr/_helpers/runner.py b/src/sqlbuild/integrations/ingestr/_helpers/runner.py index 1f33bc29fa..730bc6c822 100644 --- a/src/sqlbuild/integrations/ingestr/_helpers/runner.py +++ b/src/sqlbuild/integrations/ingestr/_helpers/runner.py @@ -5,6 +5,8 @@ import shutil import subprocess import sys +import tomllib +from importlib import metadata from os import environ from pathlib import Path @@ -14,6 +16,7 @@ from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle _DUCKDB_DRIVER_BOOTSTRAP_ERROR: str = "ADBC driver still not available after installation" +_DUCKDB_MANIFEST: str = "duckdb.toml" def run_ingestr_command( @@ -26,6 +29,7 @@ def run_ingestr_command( "This source uses ingestr, but the ingestr CLI is not available. " "Install it with: pip install 'sqlbuild[ingestr]'" ) + ensure_duckdb_driver() command_display: str = format_ingestr_command(command) try: completed: subprocess.CompletedProcess[str] = _run_observed_ingestr_subprocess( @@ -81,7 +85,74 @@ def _ingestr_subprocess_env() -> dict[str, str]: env: dict[str, str] = dict(environ) if env.get("ADBC_DRIVER_PATH"): return env - driver_path: Path = Path(sys.prefix) / "etc" / "adbc" / "drivers" + driver_path: Path = _managed_driver_path() if driver_path.exists(): env["ADBC_DRIVER_PATH"] = str(driver_path) return env + + +def _managed_driver_path() -> Path: + return Path(sys.prefix) / "etc" / "adbc" / "drivers" + + +def ensure_duckdb_driver() -> None: + """Install the DuckDB ADBC driver matching the duckdb package unless the user set one.""" + + if environ.get("ADBC_DRIVER_PATH"): + return + try: + duckdb_version: str = metadata.version("duckdb") + except metadata.PackageNotFoundError: + return + driver_path: Path = _managed_driver_path() + if _installed_driver_version(driver_path / _DUCKDB_MANIFEST) == duckdb_version: + return + dbc: str | None = _dbc_executable() + if dbc is None: + raise IngestrIntegrationError( + "ingestr needs the dbc driver installer to pin its DuckDB driver to duckdb " + f"{duckdb_version}. Install it with: pip install 'sqlbuild[ingestr]'" + ) + driver_path.mkdir(parents=True, exist_ok=True) + with OperationLifecycle( + operation_kind="subprocess", + operation_name="adbc_driver_install", + auto_fail_base_exceptions=False, + ) as lifecycle: + completed: subprocess.CompletedProcess[str] = subprocess.run( + (dbc, "install", "--quiet", f"duckdb={duckdb_version}"), + capture_output=True, + check=False, + env={**environ, "ADBC_DRIVER_PATH": str(driver_path)}, + text=True, + ) + if completed.returncode == 0: + lifecycle.completed(exit_code=0) + else: + lifecycle.failed( + error_code=f"exit_{completed.returncode}", + exit_code=completed.returncode, + signal_number=-completed.returncode if completed.returncode < 0 else None, + ) + if completed.returncode != 0: + raise IngestrIntegrationError( + f"could not install the DuckDB ADBC driver for duckdb {duckdb_version}: " + f"{(completed.stderr or completed.stdout).strip()}" + ) + + +def _installed_driver_version(manifest: Path) -> str | None: + try: + with manifest.open("rb") as handle: + version: object = tomllib.load(handle).get("version") + except (OSError, tomllib.TOMLDecodeError): + return None + return version if isinstance(version, str) else None + + +def _dbc_executable() -> str | None: + for name in ("dbc", "dbc.exe"): + beside_python: Path = Path(sys.executable).parent / name + if beside_python.exists(): + return str(beside_python) + return shutil.which("dbc") diff --git a/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py b/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py index ac2c01efab..1747f36fe3 100644 --- a/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py +++ b/tests/unit/src/sqlbuild/integrations/ingestr/_test_types.py @@ -38,3 +38,18 @@ class IngestrRunnerTestCase: returncode: int expected_signal_number: int expected_event_types: tuple[str, ...] = () + + +@dataclass(frozen=True) +class IngestrDriverTestCase: + description: str + driver_files: tuple[tuple[str, str], ...] + environment: dict[str, str] + expected_installs: tuple[tuple[str, ...], ...] + + +@dataclass(frozen=True) +class IngestrDriverFailureTestCase: + description: str + stderr: str + expected_message: str diff --git a/tests/unit/src/sqlbuild/integrations/ingestr/test_runner.py b/tests/unit/src/sqlbuild/integrations/ingestr/test_runner.py index da3ad84d1b..2e8fea69b1 100644 --- a/tests/unit/src/sqlbuild/integrations/ingestr/test_runner.py +++ b/tests/unit/src/sqlbuild/integrations/ingestr/test_runner.py @@ -3,10 +3,11 @@ from __future__ import annotations import subprocess +from pathlib import Path import pytest -from sqlbuild.integrations.ingestr._helpers.runner import run_ingestr_command +from sqlbuild.integrations.ingestr._helpers.runner import ensure_duckdb_driver, run_ingestr_command from sqlbuild.integrations.ingestr.exceptions import IngestrIntegrationError from sqlbuild.observability import ( EventDispatcher, @@ -14,7 +15,14 @@ dispatcher_scope, invocation_scope, ) -from tests.unit.src.sqlbuild.integrations.ingestr._test_types import IngestrRunnerTestCase +from tests.unit.src.sqlbuild.integrations.ingestr._test_types import ( + IngestrDriverFailureTestCase, + IngestrDriverTestCase, + IngestrRunnerTestCase, +) + +_RUNNER: str = "sqlbuild.integrations.ingestr._helpers.runner" +_INSTALL: tuple[str, ...] = ("dbc", "install", "--quiet", "duckdb=1.5.2") @pytest.mark.parametrize( @@ -39,6 +47,9 @@ def test_given_signalled_ingestr_when_run_then_safe_terminal_excludes_child_stre "sqlbuild.integrations.ingestr._helpers.runner.shutil.which", lambda executable: executable, ) + monkeypatch.setattr( + "sqlbuild.integrations.ingestr._helpers.runner.ensure_duckdb_driver", lambda: None + ) monkeypatch.setattr( "sqlbuild.integrations.ingestr._helpers.runner._run_ingestr_subprocess", lambda **kwargs: subprocess.CompletedProcess( @@ -92,6 +103,9 @@ def test_given_unobserved_ingestr_interruption_when_run_then_only_start_is_publi "sqlbuild.integrations.ingestr._helpers.runner.shutil.which", lambda executable: executable, ) + monkeypatch.setattr( + "sqlbuild.integrations.ingestr._helpers.runner.ensure_duckdb_driver", lambda: None + ) monkeypatch.setattr( "sqlbuild.integrations.ingestr._helpers.runner._run_ingestr_subprocess", lambda **kwargs: (_ for _ in ()).throw(KeyboardInterrupt), @@ -105,3 +119,88 @@ def test_given_unobserved_ingestr_interruption_when_run_then_only_start_is_publi _ = run_ingestr_command(("ingestr", "run")) assert tuple(event.event_type for event in events) == test_case.expected_event_types + + +@pytest.mark.parametrize( + "test_case", + ( + IngestrDriverTestCase( + description="missing driver installs the duckdb package version", + driver_files=(), + environment={}, + expected_installs=(_INSTALL,), + ), + IngestrDriverTestCase( + description="driver of another version is replaced", + driver_files=(("duckdb.toml", "version = '2.0.0'\n"),), + environment={}, + expected_installs=(_INSTALL,), + ), + IngestrDriverTestCase( + description="matching driver is kept", + driver_files=(("duckdb.toml", "version = '1.5.2'\n"),), + environment={}, + expected_installs=(), + ), + IngestrDriverTestCase( + description="user driver path is left alone", + driver_files=(), + environment={"ADBC_DRIVER_PATH": "/custom/adbc"}, + expected_installs=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_duckdb_driver_state_when_ensuring_then_installs_only_a_mismatch( + test_case: IngestrDriverTestCase, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + driver_path: Path = tmp_path / "etc" / "adbc" / "drivers" + driver_path.mkdir(parents=True) + _ = [(driver_path / name).write_text(text) for name, text in test_case.driver_files] + calls: list[tuple[tuple[str, ...], dict[str, str]]] = [] + monkeypatch.setattr(f"{_RUNNER}.sys.prefix", str(tmp_path)) + monkeypatch.setattr(f"{_RUNNER}.environ", dict(test_case.environment)) + monkeypatch.setattr(f"{_RUNNER}.metadata.version", lambda name: "1.5.2") + monkeypatch.setattr(f"{_RUNNER}._dbc_executable", lambda: "dbc") + monkeypatch.setattr( + f"{_RUNNER}.subprocess.run", + lambda args, **kwargs: ( + calls.append((args, kwargs["env"])) or subprocess.CompletedProcess(args, 0, "", "") + ), + ) + + ensure_duckdb_driver() + + assert tuple(args for args, _ in calls) == test_case.expected_installs + assert all(env["ADBC_DRIVER_PATH"] == str(driver_path) for _, env in calls) + + +@pytest.mark.parametrize( + "test_case", + ( + IngestrDriverFailureTestCase( + description="failed install names the duckdb version", + stderr="registry unreachable", + expected_message="duckdb 1.5.2: registry unreachable", + ), + ), + ids=lambda case: case.description, +) +def test_given_failing_driver_install_when_ensuring_then_names_the_duckdb_version( + test_case: IngestrDriverFailureTestCase, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr(f"{_RUNNER}.sys.prefix", str(tmp_path)) + monkeypatch.setattr(f"{_RUNNER}.environ", {}) + monkeypatch.setattr(f"{_RUNNER}.metadata.version", lambda name: "1.5.2") + monkeypatch.setattr(f"{_RUNNER}._dbc_executable", lambda: "dbc") + monkeypatch.setattr( + f"{_RUNNER}.subprocess.run", + lambda args, **kwargs: subprocess.CompletedProcess(args, 1, "", test_case.stderr), + ) + + with pytest.raises(IngestrIntegrationError, match=test_case.expected_message): + ensure_duckdb_driver() diff --git a/uv.lock b/uv.lock index de25d86219..4d039fa507 100644 --- a/uv.lock +++ b/uv.lock @@ -916,6 +916,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/7a/797ed371bde520223e49dbbe0f8762fa4cc724d0e21fc6af944bb8f82150/db_dtypes-1.7.0-py3-none-any.whl", hash = "sha256:c80a1d9bc7dda3cc63638a3f2cf4ec27af50b809ec2d92b8a94cb9d8438cdc76", size = 17740, upload-time = "2026-06-03T14:51:15.287Z" }, ] +[[package]] +name = "dbc" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/10/c595fd891a8a5415378d40fd4d428b72c7266dd5cbf8c04ea5f16279f323/dbc-0.3.1-py3-none-macosx_12_0_arm64.whl", hash = "sha256:9b55f649516055c121d80ebf189ec47451a4f30a3e66577dfb89d4c92c104524", size = 3994959, upload-time = "2026-10-01T16:00:28.743Z" }, + { url = "https://files.pythonhosted.org/packages/ee/38/abf425442f73343e0f27fd32451fdabedd403317d6fc3ec6fe423742cc1f/dbc-0.3.1-py3-none-macosx_12_0_x86_64.whl", hash = "sha256:4d6ac7ed02572a55ce8312ef510367155cbb0fee529512684a460895dfb991f5", size = 4429256, upload-time = "2026-10-01T16:00:30.927Z" }, + { url = "https://files.pythonhosted.org/packages/f1/59/f07789d4db5a8e3f1eb3afd8524d099e048ccb75a4f854ece6ef0cdb402f/dbc-0.3.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:bbb1f137c75d4ac18b1e29f662ddbb06d636cb455a9b76a6fcc9bf4dcf2a8b3a", size = 4288844, upload-time = "2026-10-01T16:00:33.016Z" }, + { url = "https://files.pythonhosted.org/packages/ee/80/55ad27b5b6f21c0af04acb611d9a48da07fc0bb0a287f03515ee4b34aaf3/dbc-0.3.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79a880d10ccb2c4816262ccd4cf11aca0db259bc36a34eced34b533a404ba029", size = 3865144, upload-time = "2026-10-01T16:00:34.848Z" }, + { url = "https://files.pythonhosted.org/packages/27/b9/904c2249a1bfd5a6f34011f33db01b113af0ca7685b1a1218a20b5e8838a/dbc-0.3.1-py3-none-win_amd64.whl", hash = "sha256:5ae67a0d0c9e028a24ed3cd802d85c184f73794e90405c59984471d0e3a3bd1b", size = 4415639, upload-time = "2026-10-01T16:00:36.529Z" }, +] + [[package]] name = "dbt-adapters" version = "1.24.2" @@ -2049,14 +2061,10 @@ wheels = [ [[package]] name = "ingestr" -version = "1.0.0" +version = "1.1.73" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/e8/25b6bddfb758ce416a4ef28d8d42cf80b665216ad6dad1c814ed8e079730/ingestr-1.0.0-py2.py3-none-macosx_10_15_x86_64.whl", hash = "sha256:f32c9d70090c4eaa0f452b6c10aedb2efc146b8ea148e2baa349c6fa626fce5d", size = 55923802, upload-time = "2026-05-22T07:32:01.917Z" }, - { url = "https://files.pythonhosted.org/packages/9f/bc/c92aa774b74c2e6e65c115fcb7920e24c31751c7a4854b458b7917bd240a/ingestr-1.0.0-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:c9ff2e5092f30685338e2a27cf0b384f516db9f1bf0edba580faac294abfd453", size = 51329572, upload-time = "2026-05-22T07:32:05.683Z" }, - { url = "https://files.pythonhosted.org/packages/f7/c1/ee8b02a7266f1f0ab60cf4e55b8ac74d19bfe5905399bd4b323590dd9247/ingestr-1.0.0-py2.py3-none-manylinux2014_aarch64.whl", hash = "sha256:693f98ed6632fa1321ab4184aea25974bbf722478102627564674c3311b39ab3", size = 49705302, upload-time = "2026-05-22T07:32:08.958Z" }, - { url = "https://files.pythonhosted.org/packages/34/0b/9cb4623a5903bf177f0411ead9a50fd8867c24380f574a8567c8ceb7e294/ingestr-1.0.0-py2.py3-none-manylinux2014_x86_64.whl", hash = "sha256:3910167cf18577635b39c695e257e2bcd74c2d4dd956e28ebc2381ca79cbb8cc", size = 55611218, upload-time = "2026-05-22T07:32:11.922Z" }, - { url = "https://files.pythonhosted.org/packages/e3/7a/f0561d903fc96f5824a8e46864dd14f3203f6a3b7b59099f3c731c484a0f/ingestr-1.0.0-py2.py3-none-win_amd64.whl", hash = "sha256:db9460062b3b0466b412023282aa8a7349c44cbf4f7d654c820551fd19af236c", size = 57230133, upload-time = "2026-05-22T07:32:14.842Z" }, + { url = "https://files.pythonhosted.org/packages/03/c4/ec864b94260069470c94f0e6e0a84a10f7838bcc0a21e8ceb7f5196f81d7/ingestr-1.1.73-py3-none-any.whl", hash = "sha256:dad15bc1b0c0c1eaef746f885c89a4986c66a64941660a0e3971f106ee221db8", size = 18387, upload-time = "2026-10-21T16:03:12.817Z" }, ] [[package]] @@ -4275,6 +4283,7 @@ dlt = [ { name = "dlt", extra = ["duckdb", "filesystem", "sql-database"] }, ] ingestr = [ + { name = "dbc" }, { name = "ingestr" }, ] postgres = [ @@ -4293,6 +4302,7 @@ sqlserver = [ [package.dev-dependencies] dev = [ { name = "commitizen" }, + { name = "dbc" }, { name = "dbt-bigquery" }, { name = "dbt-core" }, { name = "dbt-databricks" }, @@ -4320,11 +4330,12 @@ requires-dist = [ { name = "dagster", marker = "extra == 'dagster'", specifier = ">=1.13.0" }, { name = "dagster-webserver", marker = "extra == 'dagster'", specifier = ">=1.13.0" }, { name = "databricks-sql-connector", extras = ["pyarrow"], marker = "extra == 'databricks'", specifier = ">=4.1.2" }, + { name = "dbc", marker = "extra == 'ingestr'", specifier = ">=0.3.1" }, { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], marker = "extra == 'dlt'", specifier = ">=1.28.0,<2" }, { name = "duckdb", specifier = ">=1.3.2" }, { name = "filelock", specifier = ">=3.20.3" }, { name = "google-cloud-bigquery", marker = "extra == 'bigquery'", specifier = ">=3.37.0" }, - { name = "ingestr", marker = "extra == 'ingestr'", specifier = ">=1.0.0" }, + { name = "ingestr", marker = "extra == 'ingestr'", specifier = ">=1.1.73" }, { name = "jinja2", marker = "extra == 'dbt'", specifier = ">=3.1" }, { name = "orjson", specifier = ">=3.11.7" }, { name = "polyglot-sql-chio", specifier = "==0.13.9" }, @@ -4342,6 +4353,7 @@ provides-extras = ["dbt", "snowflake", "bigquery", "databricks", "dagster", "pos [package.metadata.requires-dev] dev = [ { name = "commitizen", specifier = ">=4.9.1" }, + { name = "dbc", specifier = ">=0.3.1" }, { name = "dbt-bigquery", specifier = ">=1.9.0,<1.11" }, { name = "dbt-core", specifier = ">=1.11.6,<1.12" }, { name = "dbt-databricks", specifier = ">=1.9.0,<1.11" }, @@ -4351,7 +4363,7 @@ dev = [ { name = "dbt-sqlserver", specifier = ">=1.9.0,<1.11" }, { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], specifier = ">=1.28.0,<2" }, { name = "fensu", specifier = ">=0.27.0" }, - { name = "ingestr", specifier = ">=1.0.0" }, + { name = "ingestr", specifier = ">=1.1.73" }, { name = "jsonschema", specifier = ">=4.23.0" }, { name = "maturin", specifier = ">=1.9,<2" }, { name = "pre-commit", specifier = ">=4.3.0" }, diff --git a/website/src/content/docs/docs/integrations/ingestr.mdx b/website/src/content/docs/docs/integrations/ingestr.mdx index 47f0b7e9d0..d57f6ecb41 100644 --- a/website/src/content/docs/docs/integrations/ingestr.mdx +++ b/website/src/content/docs/docs/integrations/ingestr.mdx @@ -17,6 +17,11 @@ uv pip install 'sqlbuild[ingestr]' This installs `ingestr` alongside SQLBuild. The `ingestr` CLI must be available on `PATH`. +ingestr reads and writes DuckDB through an ADBC driver. Before the first ingestr run, SQLBuild +installs the DuckDB driver that matches its own `duckdb` package (with `dbc`, included in the extra) +into `/etc/adbc/drivers`, so the files ingestr writes stay readable by SQLBuild. Set +`ADBC_DRIVER_PATH` to manage the driver yourself. + ## How it works 1. Declare an `ingestr` block on a source in `sources/*.yml`