diff --git a/.github/scripts/assert_sdist_sources.sh b/.github/scripts/assert_sdist_sources.sh new file mode 100755 index 00000000..d501c0d0 --- /dev/null +++ b/.github/scripts/assert_sdist_sources.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Asserts every sdist in DIR carries unixODBC's pinned source release, unchanged: the corresponding +# source of the driver manager the Linux and macOS wheels bundle (version and SHA-256 from +# unixodbc_source.sh). +# Usage: assert_sdist_sources.sh DIR +set -euo pipefail + +pins="$(dirname "$0")/unixodbc_source.sh" +version="$(sed -n 's/^UNIXODBC_VERSION=//p' "$pins")" +sha256="$(sed -n 's/^UNIXODBC_SHA256=//p' "$pins")" +name="unixODBC-${version}.tar.gz" + +shopt -s nullglob +sdists=("$1"/*.tar.gz) +if [ "${#sdists[@]}" -eq 0 ]; then + echo "no sdists in $1" >&2 + exit 1 +fi +for sdist in "${sdists[@]}"; do + member="$(tar -tzf "$sdist" | grep -E "^[^/]+/${name//./\\.}\$" || true)" + if [ -z "$member" ]; then + echo "$sdist: no ${name} at its root" >&2 + exit 1 + fi + found="$(tar -xzOf "$sdist" "$member" | sha256sum | cut -d' ' -f1)" + if [ "$found" != "$sha256" ]; then + echo "$sdist: ${name} has SHA-256 ${found}, expected ${sha256}" >&2 + exit 1 + fi + echo "ok: $(basename "$sdist") carries ${name} (SHA-256 ${sha256})" +done diff --git a/.github/scripts/assert_wheel_notices.sh b/.github/scripts/assert_wheel_notices.sh new file mode 100755 index 00000000..575664e1 --- /dev/null +++ b/.github/scripts/assert_wheel_notices.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Asserts every wheel in DIR carries unixODBC's notice and licence text, and that each Linux and +# macOS wheel bundles the unixODBC driver manager as one separate shared library (Windows wheels +# use odbc32.dll and bundle none). +# Usage: assert_wheel_notices.sh DIR +set -euo pipefail + +python="$(command -v python3 || command -v python)" +shopt -s nullglob +wheels=("$1"/*.whl) +if [ "${#wheels[@]}" -eq 0 ]; then + echo "no wheels in $1" >&2 + exit 1 +fi +for wheel in "${wheels[@]}"; do + names="$("$python" -c 'import sys, zipfile; print("\n".join(zipfile.ZipFile(sys.argv[1]).namelist()))' "$wheel")" + for notice in unixODBC-NOTICE.md unixODBC-COPYING.txt; do + if ! grep -Eq "\.dist-info/licenses/licenses/${notice}\$" <<< "$names"; then + echo "$wheel: no ${notice} in its .dist-info/licenses" >&2 + exit 1 + fi + done + bundled="$(grep -Ec '\.(libs|dylibs)/libodbc[^/]*\.(so[.0-9]*|dylib)$' <<< "$names" || true)" + case "$wheel" in + *win_amd64.whl | *win32.whl | *win_arm64.whl) expected=0 ;; + *) expected=1 ;; + esac + if [ "$bundled" -ne "$expected" ]; then + echo "$wheel: bundles ${bundled} unixODBC libraries, expected ${expected}" >&2 + exit 1 + fi + echo "ok: $(basename "$wheel") (bundled libodbc: ${bundled})" +done diff --git a/.github/scripts/build_unixodbc.sh b/.github/scripts/build_unixodbc.sh new file mode 100755 index 00000000..2bca75db --- /dev/null +++ b/.github/scripts/build_unixodbc.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Builds the unixODBC driver manager the Linux and macOS release wheels bundle, from its pinned source +# release, so the wheels link it dynamically and carry it as a separate, replaceable shared library. +# Usage: build_unixodbc.sh PREFIX SYSCONFDIR [LIBDIR] [DRIVER_PATH] +# SYSCONFDIR is where the bundled driver manager reads odbcinst.ini and odbc.ini when ODBCSYSINI +# is unset: /etc on Linux, as distribution driver packages register there; Homebrew's etc on macOS. +# DRIVER_PATH is a colon-separated list of folders searched for a driver registered by file name +# alone, as Debian and Ubuntu packages register theirs (Driver=psqlodbcw.so). +# The source release and its SHA-256 are pinned in unixodbc_source.sh. +set -euo pipefail + +prefix="$1" +sysconfdir="$2" +libdir="${3:-$prefix/lib}" +driver_path="${4:-}" + +work="$(mktemp -d)" +tarball="$(bash "$(dirname "$0")/unixodbc_source.sh" "$work")" +tar -xzf "$work/$tarball" -C "$work" +cd "$work/${tarball%.tar.gz}" +# The included libltdl is built into the libraries, so they need no system libltdl; no readline, as +# only the libraries ship. +./configure --prefix="$prefix" --libdir="$libdir" --sysconfdir="$sysconfdir" \ + ${driver_path:+--with-odbc-driver-path="$driver_path"} \ + --with-included-ltdl --enable-readline=no --enable-editline=no --enable-static=no > "$work/configure.log" || + { tail -50 "$work/configure.log"; exit 1; } +make -j"$(getconf _NPROCESSORS_ONLN)" > "$work/make.log" 2>&1 || { tail -50 "$work/make.log"; exit 1; } +make install > /dev/null +cd / +rm -rf "$work" +echo "${tarball%.tar.gz} installed in ${libdir}, reading ${sysconfdir}, finding drivers by name in ${driver_path:-the library search path}" diff --git a/.github/scripts/install_odbc_drivers.ps1 b/.github/scripts/install_odbc_drivers.ps1 new file mode 100644 index 00000000..0598cdce --- /dev/null +++ b/.github/scripts/install_odbc_drivers.ps1 @@ -0,0 +1,45 @@ +# Installs the 64-bit ODBC drivers the Windows checks read through, each a pinned installer +# checked against its SHA-256, and lists the drivers Windows then has: +# PostgreSQL Unicode(x64) psqlODBC 18.00.0006 +# ODBC Driver 18 for SQL Server Microsoft's msodbcsql18 18.7.1.1 (installing it accepts Microsoft's licence) +# MySQL ODBC 26.10 Unicode Driver MySQL Connector/ODBC 26.10.0 (its SHA-256 taken from the file +# matching MySQL's published MD5, 34541f6d21bb6a4b2f56b3f8f1afcf4b) +# Usage: pwsh .github/scripts/install_odbc_drivers.ps1 +$ErrorActionPreference = "Stop" + +$drivers = @( + @{ + Name = "psqlODBC" + Uri = "https://ftp.postgresql.org/pub/odbc/releases/REL-18_00_0006/psqlodbc_x64.msi" + Sha256 = "8962543113df83410b74f113db1ef80717b65a1dc7869e18c24bd852dd65535f" + Arguments = @() + }, + @{ + Name = "msodbcsql18" + Uri = "https://go.microsoft.com/fwlink/?linkid=2378279" + Sha256 = "21ef69e4b942f18aced55fa7c3a8d7263004e113a96690bac7509645e1b1a310" + Arguments = @("IACCEPTMSODBCSQLLICENSETERMS=YES") + }, + @{ + Name = "MySQL Connector/ODBC" + Uri = "https://cdn.mysql.com/Downloads/Connector-ODBC/26.10/mysql-connector-odbc-26.10.0-winx64.msi" + Sha256 = "ac58d7f1b4d7c1f37630a78c83c659f04c127754bfa299957b4effa32bd83cae" + Arguments = @() + } +) + +foreach ($driver in $drivers) { + $msi = Join-Path $env:RUNNER_TEMP ("odbc-" + [guid]::NewGuid() + ".msi") + Invoke-WebRequest -Uri $driver.Uri -OutFile $msi + $hash = (Get-FileHash -Algorithm SHA256 -Path $msi).Hash.ToLowerInvariant() + if ($hash -ne $driver.Sha256) { + throw "$($driver.Name): the installer's SHA-256 is $hash, not the pinned $($driver.Sha256); check the new release and update the pin" + } + $process = Start-Process msiexec.exe -ArgumentList (@("/i", $msi, "/qn", "/norestart") + $driver.Arguments) -Wait -PassThru + if ($process.ExitCode -ne 0) { + throw "$($driver.Name): msiexec exited $($process.ExitCode)" + } + Remove-Item $msi +} + +Get-OdbcDriver -Platform 64-bit | Where-Object { $_.Name -match "PostgreSQL|SQL Server|MySQL" } | Format-Table Name, Platform diff --git a/.github/scripts/install_odbc_drivers.sh b/.github/scripts/install_odbc_drivers.sh new file mode 100755 index 00000000..9ebe1786 --- /dev/null +++ b/.github/scripts/install_odbc_drivers.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Installs unixODBC and the ODBC drivers the container tests read through, on Debian or Ubuntu x86_64, +# and registers each driver under the name the tests connect with: +# PostgreSQL Unicode psqlODBC, from the distribution +# ODBC Driver 18 for SQL Server Microsoft's msodbcsql18, the same release as on Windows (installing it accepts Microsoft's licence) +# MySQL ODBC Unicode Driver MySQL Connector/ODBC, MySQL's package for this distribution release +# Usage: install_odbc_drivers.sh [--with-dev] (--with-dev adds the unixODBC headers to build against) +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive + +MYSQL_ODBC_VERSION=26.10.0 +MSODBCSQL_VERSION=18.7.1.1-1 +MYSQL_ODBC_NAME="MySQL ODBC Unicode Driver" + +sudo_() { if [ "$(id -u)" = 0 ]; then "$@"; else sudo "$@"; fi; } + +packages=(unixodbc odbc-postgresql curl ca-certificates) +if [ "${1:-}" = "--with-dev" ]; then + packages+=(unixodbc-dev) +fi +sudo_ apt-get update -qq +sudo_ apt-get install -y -qq "${packages[@]}" + +# Microsoft's repository for this distribution release, then msodbcsql18. +. /etc/os-release +if ! dpkg -s msodbcsql18 > /dev/null 2>&1; then + curl -fsSL -o /tmp/packages-microsoft-prod.deb "https://packages.microsoft.com/config/${ID}/${VERSION_ID}/packages-microsoft-prod.deb" + sudo_ dpkg -i /tmp/packages-microsoft-prod.deb > /dev/null + rm -f /tmp/packages-microsoft-prod.deb + sudo_ apt-get update -qq + sudo_ env ACCEPT_EULA=Y apt-get install -y -qq "msodbcsql18=${MSODBCSQL_VERSION}" +fi + +# MySQL Connector/ODBC: MySQL's package built for this distribution release, which links the system +# OpenSSL. Its generic tarball carries a newer private libssl.so.3, which can't load in a process +# where another driver already loaded the system one (OPENSSL_3.2.0 not found on Ubuntu 24.04). +# The SHA-256s are of the packages as cdn.mysql.com served them over TLS. The Unicode driver is +# registered under a name that doesn't change with the version. +case "${ID}${VERSION_ID}" in + ubuntu24.04) mysql_sha256=b99d26f320ca640d24d8a82bdaea33e41f1ce9bb16cbea5b44ee3b7547bc5673 ;; + ubuntu22.04) mysql_sha256=2e89b5ca8149ba77c24e0828597c85d160ff66fe57aa81873c742a152a747c65 ;; + debian13) mysql_sha256=20b506e3b44271fb3b0276333ae6595b9cecf6c2bdf888a8d21ff421c54bbdc9 ;; + debian12) mysql_sha256=119d0a5413a97e155a886df03cc01921a3294ff0cb02569c1d8e11642f4c0b7d ;; + *) echo "no pinned MySQL Connector/ODBC package for ${ID} ${VERSION_ID}" >&2; exit 1 ;; +esac +if ! dpkg -s mysql-connector-odbc > /dev/null 2>&1; then + package="mysql-connector-odbc_${MYSQL_ODBC_VERSION}-1${ID}${VERSION_ID}_amd64.deb" + work="$(mktemp -d)" + curl -fsSL -o "$work/$package" "https://cdn.mysql.com/Downloads/Connector-ODBC/${MYSQL_ODBC_VERSION%.*}/$package" || + curl -fsSL -o "$work/$package" "https://downloads.mysql.com/archives/get/p/10/file/$package" + echo "$mysql_sha256 $work/$package" | sha256sum -c --quiet - + sudo_ apt-get install -y -qq "$work/$package" + rm -rf "$work" +fi +driver="/usr/lib/x86_64-linux-gnu/odbc/libmyodbc${MYSQL_ODBC_VERSION%%.*}w.so" +test -f "$driver" +printf '[%s]\nDescription=MySQL Connector/ODBC %s\nDriver=%s\nUsageCount=1\n' "$MYSQL_ODBC_NAME" "$MYSQL_ODBC_VERSION" "$driver" > /tmp/mysql-odbc.ini +sudo_ odbcinst -i -d -r -f /tmp/mysql-odbc.ini > /dev/null +rm -f /tmp/mysql-odbc.ini + +odbcinst -q -d diff --git a/.github/scripts/smoke_odbc_wheel.py b/.github/scripts/smoke_odbc_wheel.py new file mode 100644 index 00000000..c52347a5 --- /dev/null +++ b/.github/scripts/smoke_odbc_wheel.py @@ -0,0 +1,109 @@ +"""Smoke-test the driver manager an installed wheel's `sqlload` reaches ODBC drivers through. + +Usage: python smoke_odbc_wheel.py SQLLOAD... [--default-ini PATH] [--driver-folder DIR] + +Each SQLLOAD is an installed `sqlload` command (the sqlload wheel's and the sqlload-cli wheel's). +On Linux and macOS it must run where no unixODBC is installed, so the wheel's bundled driver +manager is the only one; it must find a driver registered in an odbcinst.ini named by ODBCSYSINI +and, with --default-ini, one registered in the default odbcinst.ini (/etc on Linux, Homebrew's etc +on macOS). The drivers don't exist, so the driver manager reports the library it tried to load, +which shows the registration it read. With --driver-folder (Linux), a plain shared library placed in +DIR and registered by file name alone, as Debian and Ubuntu driver packages register theirs +(Driver=psqlodbcw.so), must load: it then fails as no driver (IM004), not as a missing file. On Windows, the driver manager built in (odbc32.dll) reports +an unregistered driver. +""" + +import argparse +import glob +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +PROJECT = """\ +[connections.erp] +kind = "odbc" +connection_string = "Driver={{{driver}}};" + +[connections.local] +kind = "duckdb" +database = "out.duckdb" + +[targets.dev] +connection = "local" +loader_schema = "raw" + +[sources.erp] +kind = "odbc" +connection = "erp" +description = "Smoke test." +include = ["*"] +""" + +#: Where a unixODBC library would be if one were installed. +SYSTEM_LIBODBC = ["/usr/lib*/libodbc.so*", "/usr/lib/*/libodbc.so*", "/usr/local/lib/libodbc*", "/opt/homebrew/lib/libodbc*", "/opt/homebrew/opt/unixodbc"] + + +def plan(sqlload, folder, driver, environment): + (folder / "sqlload.toml").write_text(PROJECT.format(driver=driver), encoding="utf-8") + result = subprocess.run([sqlload, "--no-color", "plan"], cwd=folder, capture_output=True, text=True, encoding="utf-8", errors="replace", env=environment, check=False) + return result.returncode, result.stdout + result.stderr + + +def check(condition, what, output=""): + if not condition: + sys.exit(f"{what}\n{output}") + print(f"ok: {what}", flush=True) + + +def registered(ini, name, library): + previous = ini.read_text(encoding="utf-8") if ini.exists() else "" + ini.parent.mkdir(parents=True, exist_ok=True) + ini.write_text(f"{previous}\n[{name}]\nDriver={library}\n", encoding="utf-8") + return previous + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("sqlload", nargs="+") + parser.add_argument("--default-ini", type=Path) + parser.add_argument("--driver-folder", type=Path) + arguments = parser.parse_args() + if os.name != "nt": + found = [path for pattern in SYSTEM_LIBODBC for path in glob.glob(pattern)] + check(not found, "no unixODBC is installed, so only the wheel's driver manager can load", str(found)) + for sqlload in arguments.sqlload: + # Absolute, as each plan runs in its own folder and POSIX resolves a relative path from there. + sqlload = os.path.abspath(shutil.which(sqlload) or sqlload) + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + if os.name == "nt": + code, output = plan(sqlload, folder, "SQLLoad Smoke Missing", dict(os.environ)) + check(code == 1 and "Data source name not found" in output, f"{sqlload}: Windows's driver manager reports an unregistered driver", output) + continue + ini = folder / "ini" / "odbcinst.ini" + registered(ini, "SQLLoad Smoke", "/nonexistent/libsmoke-odbcsysini.so") + code, output = plan(sqlload, folder, "SQLLoad Smoke", {**os.environ, "ODBCSYSINI": str(ini.parent)}) + check(code == 1 and "Can't open lib '/nonexistent/libsmoke-odbcsysini.so'" in output, f"{sqlload}: the bundled driver manager finds a driver registered under ODBCSYSINI", output) + if arguments.driver_folder: + arguments.driver_folder.mkdir(parents=True, exist_ok=True) + plain = sorted(glob.glob("/usr/lib/*-linux-gnu/libz.so.1") + glob.glob("/lib/*-linux-gnu/libz.so.1")) + check(bool(plain), "a plain shared library to stand in for a driver") + shutil.copy(plain[0], arguments.driver_folder / "libsqlload-smoke.so") + registered(ini, "SQLLoad Smoke Bare", "libsqlload-smoke.so") + code, output = plan(sqlload, folder, "SQLLoad Smoke Bare", {**os.environ, "ODBCSYSINI": str(ini.parent)}) + check(code == 1 and "IM004" in output and "file not found" not in output, f"{sqlload}: the bundled driver manager finds a driver registered by file name in {arguments.driver_folder}", output) + if arguments.default_ini: + previous = registered(arguments.default_ini, "SQLLoad Smoke Default", "/nonexistent/libsmoke-default.so") + try: + environment = {name: value for name, value in os.environ.items() if name not in ("ODBCSYSINI", "ODBCINSTINI")} + code, output = plan(sqlload, folder, "SQLLoad Smoke Default", environment) + finally: + arguments.default_ini.write_text(previous, encoding="utf-8") + check(code == 1 and "Can't open lib '/nonexistent/libsmoke-default.so'" in output, f"{sqlload}: the bundled driver manager finds a driver registered in {arguments.default_ini}", output) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/unixodbc_source.sh b/.github/scripts/unixodbc_source.sh new file mode 100755 index 00000000..314a8c38 --- /dev/null +++ b/.github/scripts/unixodbc_source.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Downloads unixODBC's pinned source release, checks its SHA-256 and copies it into each FOLDER. +# The Linux and macOS wheels bundle the driver manager built from it (build_unixodbc.sh), and both +# sdists carry it as the library's corresponding source (LGPL-2.1 section 4). +# Usage: unixodbc_source.sh FOLDER... Prints the tarball's name. +# Keep the version in step with the NOTICE files under packages/*/licenses/. +set -euo pipefail + +UNIXODBC_VERSION=2.3.14 +UNIXODBC_SHA256=4e2814de3e01fc30b0b9f75e83bb5aba91ab0384ee951286504bb70205524771 +name="unixODBC-${UNIXODBC_VERSION}.tar.gz" + +work="$(mktemp -d)" +curl -fsSL -o "$work/$name" "https://www.unixodbc.org/$name" || + curl -fsSL -o "$work/$name" "https://github.com/lurcher/unixODBC/releases/download/v${UNIXODBC_VERSION}/$name" +if command -v sha256sum > /dev/null; then + echo "$UNIXODBC_SHA256 $work/$name" | sha256sum -c --quiet - >&2 +else + echo "$UNIXODBC_SHA256 $work/$name" | shasum -a 256 -c --quiet - >&2 +fi +for folder in "$@"; do + cp "$work/$name" "$folder/$name" +done +rm -rf "$work" +echo "$name" diff --git a/.github/scripts/windows_e2e.py b/.github/scripts/windows_e2e.py index a722ab7d..52cb9ca0 100644 --- a/.github/scripts/windows_e2e.py +++ b/.github/scripts/windows_e2e.py @@ -1,18 +1,23 @@ """End-to-end checks of a Windows sqlload executable: drive paths, a UNC share and an ODBC driver. -Usage: python windows_e2e.py SQLLOAD_EXE WORK_DIR SHARE [--odbc-driver NAME] [--cdc] +Usage: python windows_e2e.py SQLLOAD_EXE WORK_DIR SHARE [--odbc-driver NAME] [--cursor-drivers] [--cdc] WORK_DIR is a local folder (C:\\...) whose `share` folder is shared as SHARE (\\\\localhost\\name), so what the share holds can also be read locally. With --odbc-driver, a table in the local Postgres (the runner's preinstalled server, user postgres) is loaded through that ODBC driver, then loaded again -incrementally. With --cdc, a postgres_cdc source snapshots a table and then streams its changes, which +incrementally. With --cursor-drivers, a millisecond timestamp cursor and a date cursor resume through +Microsoft's ODBC Driver 18 (SQL Server LocalDB instance `sqlload`, datetime2 and legacy datetime), MySQL +Connector/ODBC (a MySQL server on 127.0.0.1:3307, user root without a password) and psqlODBC, and the +next load must read exactly the rows after each boundary; needs the pyodbc package. With --cdc, a postgres_cdc source snapshots a table and then streams its changes, which on Windows go through the loopback TLS relay (that server needs wal_level = logical). Needs the duckdb Python package to read the results back. """ import argparse +import json import os import pathlib +import re import subprocess import sys @@ -27,7 +32,7 @@ def sqlload(exe, *args, config=None): command += ["--config", str(config)] command += list(args) print("$", subprocess.list2cmdline(command), flush=True) - result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", errors="replace") + result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", errors="replace", check=False) print(result.stdout + result.stderr, flush=True) if result.returncode != 0: sys.exit(f"sqlload exited {result.returncode}") @@ -141,6 +146,134 @@ def odbc_checks(exe, work, driver): ) +# The cursor scenario of crates/sqlload/tests/loader/test_odbc_drivers.rs, through Windows drivers. +CURSOR_TABLES = ( + "SELECT (SELECT string_agg(id::VARCHAR, ',' ORDER BY id) || ' to ' || strftime(max(happened_at), '%Y-%m-%d %H:%M:%S.%g') FROM raw.erp__events)" + " || ' | ' || (SELECT string_agg(id::VARCHAR, ',' ORDER BY id) || ' to ' || max(opened)::VARCHAR FROM raw.erp__days)" +) +FIRST = "1,2 to 2026-01-31 23:59:59.997 | 1,2 to 2026-01-31" +SECOND = "1,2,3,4 to 2026-02-01 00:00:00.000 | 1,2,3,4 to 2026-02-01" + + +def cursor_cases(): + import pyodbc + + mysql = next(name for name in pyodbc.drivers() if re.fullmatch(r"MySQL ODBC [\d.]+ Unicode Driver", name)) + local_db = "Driver={ODBC Driver 18 for SQL Server};Server=(localdb)\\sqlload;Trusted_Connection=yes;TrustServerCertificate=yes;" + rows = [ + "INSERT INTO {events} VALUES (1, 'a', '2026-01-31 10:00:00'), (2, 'b', '2026-01-31 23:59:59.997')", + "INSERT INTO {days} VALUES (1, '2026-01-30'), (2, '2026-01-31')", + ] + change = [ + "INSERT INTO {events} VALUES (0, 'early', '2026-01-31 23:59:59.996'), (3, 'next', '2026-01-31 23:59:59.998'), (4, 'later', '2026-02-01 00:00:00')", + "INSERT INTO {days} VALUES (3, '2026-01-31'), (4, '2026-02-01')", + ] + return [ + { + "description": "SQL Server LocalDB through ODBC Driver 18, datetime2(3)", + "admin": (local_db + "Database=master;", ["CREATE DATABASE cursors_datetime2"]), + "connection": local_db + "Database=cursors_datetime2;", + "tables": ("dbo.events", "dbo.days"), + "setup": ["CREATE TABLE {events} (id int PRIMARY KEY, name nvarchar(40), happened_at datetime2(3) NOT NULL)", "CREATE TABLE {days} (id int PRIMARY KEY, opened date NOT NULL)", *rows], + "change": change, + "expected": (FIRST, SECOND), + }, + { + "description": "SQL Server LocalDB through ODBC Driver 18, legacy datetime (.123 is stored as .12333)", + "admin": (local_db + "Database=master;", ["CREATE DATABASE cursors_datetime"]), + "connection": local_db + "Database=cursors_datetime;", + "tables": ("dbo.events", "dbo.days"), + "setup": [ + "CREATE TABLE {events} (id int PRIMARY KEY, name nvarchar(40), happened_at datetime NOT NULL)", + "CREATE TABLE {days} (id int PRIMARY KEY, opened date NOT NULL)", + "INSERT INTO {events} VALUES (1, 'a', '2026-01-31 10:00:00'), (2, 'b', '2026-01-31 23:59:59.123')", + rows[1], + ], + "change": [ + "INSERT INTO {events} VALUES (0, 'early', '2026-01-31 23:59:59.120'), (3, 'next', '2026-01-31 23:59:59.127'), (4, 'later', '2026-02-01 00:00:00')", + change[1], + ], + "expected": ("1,2 to 2026-01-31 23:59:59.123 | 1,2 to 2026-01-31", SECOND), + }, + { + "description": f"MySQL through {mysql}, DATETIME(3), in an ANSI_QUOTES session", + "admin": (f"Driver={{{mysql}}};Server=127.0.0.1;Port=3307;Uid=root;Pwd=;", ["CREATE DATABASE cursors"]), + "connection": f"Driver={{{mysql}}};Server=127.0.0.1;Port=3307;Database=cursors;Uid=root;Pwd=;INITSTMT=SET SESSION sql_mode=CONCAT(@@SESSION.sql_mode,',ANSI_QUOTES');", + "tables": ("events", "days"), + "setup": ["CREATE TABLE {events} (id int PRIMARY KEY, name varchar(40), happened_at DATETIME(3) NOT NULL)", "CREATE TABLE {days} (id int PRIMARY KEY, opened DATE NOT NULL)", *rows], + "change": change, + "expected": (FIRST, SECOND), + }, + { + "description": "Postgres through psqlODBC, timestamp(3)", + "admin": None, + "connection": "Driver={PostgreSQL Unicode(x64)};Server=127.0.0.1;Port=5432;Database=postgres;Uid=postgres;Pwd=root;", + "tables": ("cursors.events", "cursors.days"), + "setup": ["DROP SCHEMA IF EXISTS cursors CASCADE", "CREATE SCHEMA cursors", "CREATE TABLE {events} (id int PRIMARY KEY, name varchar(40), happened_at timestamp(3) NOT NULL)", "CREATE TABLE {days} (id int PRIMARY KEY, opened date NOT NULL)", *rows], + "change": change, + "expected": (FIRST, SECOND), + }, + ] + + +def odbc_run(connection_string, statements, tables=("", "")): + import pyodbc + + with pyodbc.connect(connection_string, autocommit=True) as connection: + for statement in statements: + connection.execute(statement.format(events=tables[0], days=tables[1])) + + +def cursor_checks(exe, work): + for index, case in enumerate(cursor_cases()): + if case["admin"]: + odbc_run(*case["admin"]) + odbc_run(case["connection"], case["setup"], case["tables"]) + database = work / f"cursors-{index}.duckdb" + project = work / f"cursors-{index}" / "sqlload.toml" + project.parent.mkdir(parents=True, exist_ok=True) + events, days = case["tables"] + # A JSON string is a TOML basic string, so the connection string's backslash and quotes stay as they are. + project.write_text( + f""" +[connections.erp] +kind = "odbc" +connection_string = {json.dumps(case["connection"])} + +[connections.local] +kind = "duckdb" +database = '{database}' + +[targets.dev] +connection = "local" +loader_schema = "raw" + +[sources.erp] +kind = "odbc" +connection = "erp" +description = "Events and days of a generic ERP." +include = ["{events}", "{days}"] + +[sources.erp.tables."{events}"] +description = "Events, read after the newest happened_at." +cursor_column = "happened_at" +write_strategy = "append" + +[sources.erp.tables."{days}"] +description = "Days, the last one read again." +cursor_column = "opened" +write_strategy = "delete_insert" +""", + encoding="utf-8", + ) + first, second = case["expected"] + sqlload(exe, "load", "--yes", config=project) + expect(query(database, CURSOR_TABLES), [(first,)], f"{case['description']}, first load") + odbc_run(case["connection"], case["change"], case["tables"]) + sqlload(exe, "load", "--yes", config=project) + expect(query(database, CURSOR_TABLES), [(second,)], f"{case['description']}, the next load reads exactly the rows after each boundary") + + def cdc_checks(exe, work): """A postgres_cdc load streams through the loopback relay: the server offers no TLS, so under the default sslmode (prefer) the replication session is relayed in plain text.""" psql( @@ -205,11 +338,14 @@ def main(): parser.add_argument("work", type=pathlib.Path) parser.add_argument("share") parser.add_argument("--odbc-driver") + parser.add_argument("--cursor-drivers", action="store_true") parser.add_argument("--cdc", action="store_true") arguments = parser.parse_args() files_checks(arguments.exe, arguments.work, arguments.share) if arguments.odbc_driver: odbc_checks(arguments.exe, arguments.work, arguments.odbc_driver) + if arguments.cursor_drivers: + cursor_checks(arguments.exe, arguments.work) if arguments.cdc: cdc_checks(arguments.exe, arguments.work) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8385bb65..b35a07f8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,6 +14,12 @@ on: - .github/workflows/publish.yml - .github/scripts/assert_*_wheel_tags.sh - .github/scripts/interrupt_python_load.py + - .github/scripts/build_unixodbc.sh + - .github/scripts/unixodbc_source.sh + - .github/scripts/assert_sdist_sources.sh + - .github/scripts/assert_wheel_notices.sh + - .github/scripts/smoke_odbc_wheel.py + - packages/*/licenses/** - packages/*/pyproject.toml - Cargo.toml - Cargo.lock @@ -135,19 +141,33 @@ jobs: - name: Build for macOS 11 and newer if: runner.os == 'macOS' run: echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> "$GITHUB_ENV" + # unixODBC built from its pinned source, which the Linux and macOS wheels bundle as a separate + # shared library (licenses/ in each package; Linux builds it in the manylinux container below). + # The runner's Homebrew copy goes first, so the wheels link this one and the smoke test runs + # with no unixODBC installed. It reads Homebrew's etc, where Homebrew's drivers register. + - name: Build unixODBC for macOS + if: runner.os == 'macOS' + run: | + if brew list --versions unixodbc > /dev/null 2>&1; then brew uninstall --ignore-dependencies unixodbc; fi + bash .github/scripts/build_unixodbc.sh "$RUNNER_TEMP/unixodbc" "$(brew --prefix)/etc" '' "$(brew --prefix)/lib" + echo "RUSTFLAGS=-L native=$RUNNER_TEMP/unixodbc/lib" >> "$GITHUB_ENV" # sqlload-cli ships one standalone Rust executable (OpenSSL built in) and links no libpython, # so maturin tags it py3-none-. The tag is asserted before release. - name: Build native CLI wheel uses: PyO3/maturin-action@v1 with: command: build - args: --release --out ../../dist + # The wheels build ODBC (sdists don't, so building one needs no driver manager); repair copies + # unixODBC into the Linux and macOS wheels; Windows uses odbc32.dll. + args: --release --out ../../dist --features odbc ${{ runner.os != 'Windows' && '--auditwheel=repair' || '' }} target: ${{ matrix.platform.target }} manylinux: "2_28" working-directory: packages/sqlload-cli # The vendored OpenSSL build runs Perl's Configure, which needs modules the - # manylinux image leaves out. - before-script-linux: dnf install -y perl-IPC-Cmd perl-Time-Piece + # manylinux image leaves out. unixODBC reads /etc, where distribution drivers register. + before-script-linux: | + dnf install -y perl-IPC-Cmd perl-Time-Piece + bash "$GITHUB_WORKSPACE/.github/scripts/build_unixodbc.sh" /usr /etc /usr/lib64 "/usr/lib/$(uname -m)-linux-gnu/odbc:/usr/lib/$(uname -m)-linux-gnu:/usr/lib64/odbc:/usr/lib64" - name: Assert native CLI wheel is interpreter-agnostic shell: bash run: bash .github/scripts/assert_cli_wheel_tags.sh dist @@ -157,14 +177,21 @@ jobs: uses: PyO3/maturin-action@v1 with: command: build - args: --release --out ../../dist + # The wheels build ODBC (sdists don't, so building one needs no driver manager); repair copies + # unixODBC into the Linux and macOS wheels; Windows uses odbc32.dll. + args: --release --out ../../dist --features odbc ${{ runner.os != 'Windows' && '--auditwheel=repair' || '' }} target: ${{ matrix.platform.target }} manylinux: "2_28" working-directory: packages/sqlload - before-script-linux: dnf install -y perl-IPC-Cmd perl-Time-Piece + before-script-linux: | + dnf install -y perl-IPC-Cmd perl-Time-Piece + bash "$GITHUB_WORKSPACE/.github/scripts/build_unixodbc.sh" /usr /etc /usr/lib64 "/usr/lib/$(uname -m)-linux-gnu/odbc:/usr/lib/$(uname -m)-linux-gnu:/usr/lib64/odbc:/usr/lib64" - name: Assert Python extension wheel targets the stable ABI shell: bash run: bash .github/scripts/assert_python_wheel_tags.sh dist + - name: Assert every wheel carries unixODBC's notice, and Linux and macOS wheels bundle it + shell: bash + run: bash .github/scripts/assert_wheel_notices.sh dist - name: Smoke-test the wheels shell: bash run: | @@ -180,6 +207,23 @@ jobs: if [[ "${RUNNER_OS}" == "Windows" ]]; then cli=smoke-cli/Scripts; fi "${cli}/python" -m pip install --quiet dist/sqlload_cli-*.whl "${cli}/sqlload" --version + # ODBC through each wheel's driver manager: on Linux in a container with no unixODBC (the + # runner image has one), on macOS here (Homebrew's was removed above), on Windows odbc32.dll. + case "${RUNNER_OS}" in + Linux) + docker run --rm -v "$PWD:/w" -w /w python:3.12-slim bash -c ' + python -m venv /v && /v/bin/pip install --quiet dist/sqlload-*.whl && + python -m venv /c && /c/bin/pip install --quiet dist/sqlload_cli-*.whl && + /v/bin/python .github/scripts/smoke_odbc_wheel.py /v/bin/sqlload /c/bin/sqlload --default-ini /etc/odbcinst.ini --driver-folder "/usr/lib/$(uname -m)-linux-gnu/odbc"' + ;; + macOS) + rm -rf "$RUNNER_TEMP/unixodbc" + "${bin}/python" .github/scripts/smoke_odbc_wheel.py "${bin}/sqlload" "${cli}/sqlload" --default-ini "$(brew --prefix)/etc/odbcinst.ini" + ;; + Windows) + "${bin}/python" .github/scripts/smoke_odbc_wheel.py "${bin}/sqlload" "${cli}/sqlload" + ;; + esac - uses: actions/upload-artifact@v5 with: name: wheel-${{ matrix.platform.name }} @@ -196,6 +240,10 @@ jobs: - uses: dtolnay/rust-toolchain@master with: toolchain: 1.95.0 + # unixODBC's source, SHA-checked, into each package: the sdists carry it as the corresponding + # source of the driver manager the Linux and macOS wheels bundle (LGPL-2.1 section 4). + - name: Fetch unixODBC's source for the sdists + run: bash .github/scripts/unixodbc_source.sh packages/sqlload packages/sqlload-cli - name: Build Python extension sdist uses: PyO3/maturin-action@v1 with: @@ -208,6 +256,8 @@ jobs: command: sdist args: --out ../../dist working-directory: packages/sqlload-cli + - name: Assert every sdist carries unixODBC's source, unchanged + run: bash .github/scripts/assert_sdist_sources.sh dist - uses: actions/upload-artifact@v5 with: name: sdist @@ -242,6 +292,14 @@ jobs: shell: bash run: bash .github/scripts/assert_python_wheel_tags.sh dist + - name: Assert every wheel carries unixODBC's notice, and Linux and macOS wheels bundle it + shell: bash + run: bash .github/scripts/assert_wheel_notices.sh dist + + - name: Assert every sdist carries unixODBC's source, unchanged + shell: bash + run: bash .github/scripts/assert_sdist_sources.sh dist + - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 84f34ae7..a0fae884 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -117,8 +117,12 @@ jobs: docker pull clickhouse/clickhouse-server:25.8 docker pull clickhouse/clickhouse-server:26.10 docker pull clickhouse/clickhouse-server:25.3 - - name: Install unixODBC and the PostgreSQL ODBC driver - run: sudo apt-get update -qq && sudo apt-get install -y -qq unixodbc-dev odbc-postgresql + docker pull mysql:8.4 + docker pull mcr.microsoft.com/mssql/server:2022-latest + # unixODBC with psqlODBC, Microsoft's ODBC Driver 18 for SQL Server and MySQL Connector/ODBC, + # which the ODBC source's driver tests read through. + - name: Install unixODBC and the ODBC drivers + run: .github/scripts/install_odbc_drivers.sh --with-dev # Compiles first, so the test step's timeout measures the tests alone. - name: Build container tests run: cargo test -p sqlload --features container-tests,odbc --test exporter --test loader --no-run --quiet @@ -185,13 +189,36 @@ jobs: Add-Content -Path "$env:PGDATA\postgresql.conf" -Value "wal_level = logical" Set-Service -Name postgresql-x64-17 -StartupType Manual Start-Service -Name postgresql-x64-17 - - name: Install psqlODBC + # psqlODBC, Microsoft's ODBC Driver 18 for SQL Server and MySQL Connector/ODBC. + - name: Install the ODBC drivers + shell: pwsh + run: .github/scripts/install_odbc_drivers.ps1 + # The image's SQL Server LocalDB and MySQL 8.4, for the ODBC drivers' cursor checks. MySQL gets a + # data folder and port of its own, its root user without a password, and runs as a service: the + # runner ends the processes a step starts when the step ends. + - name: Start SQL Server LocalDB and MySQL shell: pwsh run: | - $msi = "$env:RUNNER_TEMP\psqlodbc_x64.msi" - Invoke-WebRequest -Uri https://ftp.postgresql.org/pub/odbc/releases/REL-18_00_0006/psqlodbc_x64.msi -OutFile $msi - Start-Process msiexec.exe -ArgumentList "/i", $msi, "/qn", "/norestart" -Wait - Get-OdbcDriver -Name "PostgreSQL*" -Platform 64-bit | Format-Table Name, Platform + $localdb = (Get-Command SqlLocalDB.exe -ErrorAction SilentlyContinue).Source + if (-not $localdb) { $localdb = (Get-ChildItem "C:\Program Files\Microsoft SQL Server" -Recurse -Filter SqlLocalDB.exe | Select-Object -First 1).FullName } + & $localdb create sqlload -s + if ($LASTEXITCODE -ne 0) { throw "SqlLocalDB exited $LASTEXITCODE" } + $mysqld = (Get-ChildItem "C:\Program Files\MySQL" -Recurse -Filter mysqld.exe | Select-Object -First 1).FullName + $ini = "C:\mysql-sqlload.ini" + Set-Content -Path $ini -Value "[mysqld]`ndatadir=C:/mysql-data`nport=3307`nmysqlx=OFF`n" + & $mysqld --defaults-file=$ini --initialize-insecure --console + if ($LASTEXITCODE -ne 0) { throw "mysqld --initialize-insecure exited $LASTEXITCODE" } + & $mysqld --install sqlload-mysql --defaults-file=$ini + if ($LASTEXITCODE -ne 0) { throw "mysqld --install exited $LASTEXITCODE" } + Start-Service -Name sqlload-mysql + foreach ($attempt in 1..60) { + if ((Test-NetConnection 127.0.0.1 -Port 3307 -WarningAction SilentlyContinue).TcpTestSucceeded) { break } + Start-Sleep -Seconds 2 + } + if (-not (Test-NetConnection 127.0.0.1 -Port 3307 -WarningAction SilentlyContinue).TcpTestSucceeded) { + Get-ChildItem C:\mysql-data -Filter *.err | Get-Content + throw "MySQL never listened on 3307" + } - name: Share a folder as \\localhost\sqlload-e2e shell: pwsh run: | @@ -200,8 +227,8 @@ jobs: - name: End-to-end checks shell: pwsh run: | - python -m pip install --quiet duckdb - python .github/scripts/windows_e2e.py target\debug\sqlload.exe $env:E2E_DIR \\localhost\sqlload-e2e --odbc-driver "PostgreSQL Unicode(x64)" --cdc + python -m pip install --quiet duckdb pyodbc + python .github/scripts/windows_e2e.py target\debug\sqlload.exe $env:E2E_DIR \\localhost\sqlload-e2e --odbc-driver "PostgreSQL Unicode(x64)" --cursor-drivers --cdc # Postgres to DuckDB, 1M narrow rows, sqlload alone with a release build: a wall-time figure # in the job summary, not a gate (shared runners are too noisy to gate on time), so Verify diff --git a/.gitignore b/.gitignore index 90f7720e..2f05057f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,8 @@ target/ # Python packaging dist/ +# unixODBC's source, fetched into each package for its release sdist +packages/*/unixODBC-*.tar.gz .venv/ __pycache__/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b10cecac..014e1255 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ make test-python # builds the sqlload extension into .venv and runs its tes `make check-ci` is what CI runs, and must be clean, including fensu with 0 faults. -The unit and container tests link the ODBC source against unixODBC's `libodbc.so`, and the +The unit and container tests, and `make test-python` (the wheel includes ODBC), link the ODBC source against unixODBC's `libodbc.so`, and the container tests also need the PostgreSQL ODBC driver: `unixodbc-dev odbc-postgresql` on Debian and Ubuntu, `unixodbc psqlodbc` with Homebrew. diff --git a/Makefile b/Makefile index aa2768c3..72e3dccf 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ test-clickhouse-older: test-python: uv venv --quiet --allow-existing --python 3.12 .venv uv pip install --quiet --python .venv/bin/python 'maturin>=1.9,<2' pytest pytest-xdist pyarrow polars pandas orjson duckdb - cd packages/sqlload && VIRTUAL_ENV=$(CURDIR)/.venv $(CURDIR)/.venv/bin/maturin develop --uv --quiet + cd packages/sqlload && VIRTUAL_ENV=$(CURDIR)/.venv $(CURDIR)/.venv/bin/maturin develop --uv --quiet --features odbc .venv/bin/python -m pytest packages/sqlload/tests -n auto --dist loadfile test: test-unit test-containers diff --git a/README.md b/README.md index 4538537a..de3ae539 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ versions and how to run them yourself. | HTTP APIs and Python | ✓ | | | ODBC databases¹ | ✓ | | -¹ In builds with the `odbc` feature. +¹ In the published wheels from 0.38.0 (not released yet); before that, in builds with the `odbc` feature. ² From 0.34.0, through a separate DuckDB 1.5 process: install `sqlload[motherduck]`. diff --git a/crates/sqlload-python/Cargo.toml b/crates/sqlload-python/Cargo.toml index 05c08f44..1841aeda 100644 --- a/crates/sqlload-python/Cargo.toml +++ b/crates/sqlload-python/Cargo.toml @@ -18,6 +18,8 @@ doctest = false [features] # TLS with OpenSSL built in, as in the sqlload-cli wheels. ssl-vendored = ["sqlload/ssl-vendored"] +# ODBC sources, as in the sqlload-cli wheels. +odbc = ["sqlload/odbc"] [dependencies] arrow-array.workspace = true diff --git a/crates/sqlload-testkit/src/lib.rs b/crates/sqlload-testkit/src/lib.rs index 9feae2d7..9f0c0d2c 100644 --- a/crates/sqlload-testkit/src/lib.rs +++ b/crates/sqlload-testkit/src/lib.rs @@ -1,4 +1,4 @@ -//! Test doubles for sqlload: a fake Snowflake service, Redpanda and Postgres containers, configurations and an exporter harness. +//! Test doubles for sqlload: a fake Snowflake service, Redpanda, Postgres, MySQL and SQL Server containers, configurations and an exporter harness. #![forbid(unsafe_code)] @@ -9,6 +9,7 @@ pub mod fake_databricks; pub mod fake_sheets; pub mod fake_snowflake; pub mod harness; +pub mod odbc_servers; pub mod parity; pub mod postgres; pub mod records; diff --git a/crates/sqlload-testkit/src/odbc_servers/_helpers/containers.rs b/crates/sqlload-testkit/src/odbc_servers/_helpers/containers.rs new file mode 100644 index 00000000..656430f9 --- /dev/null +++ b/crates/sqlload-testkit/src/odbc_servers/_helpers/containers.rs @@ -0,0 +1,76 @@ +//! The database containers the ODBC drivers are tested against, each with its driver's connection string. + +use std::time::Duration; + +use testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers::runners::AsyncRunner; +use testcontainers::{GenericImage, ImageExt}; + +use crate::odbc_servers::constants::{ + DATABASE, MYSQL_IMAGE, MYSQL_PORT, MYSQL_READY, MYSQL_TAG, PASSWORD, SQL_SERVER_IMAGE, + SQL_SERVER_PASSWORD, SQL_SERVER_PORT, SQL_SERVER_READY, SQL_SERVER_TAG, USER, +}; +use crate::odbc_servers::models::OdbcServer; +use crate::postgres::main::connect::connect; +use crate::postgres::main::start_postgres::start_postgres; + +/// A MySQL server with database `app` and user `sqlload`, on a free host port, and a connection string for MySQL Connector/ODBC that sets the session's SQL mode to read ANSI double quotes, as SQLLoad writes them. +pub(crate) async fn start_mysql() -> Result { + let container = GenericImage::new(MYSQL_IMAGE, MYSQL_TAG) + .with_exposed_port(MYSQL_PORT.tcp()) + .with_wait_for(WaitFor::message_on_stderr(MYSQL_READY)) + .with_env_var("MYSQL_ROOT_PASSWORD", PASSWORD) + .with_env_var("MYSQL_DATABASE", DATABASE) + .with_env_var("MYSQL_USER", USER) + .with_env_var("MYSQL_PASSWORD", PASSWORD) + .with_startup_timeout(Duration::from_secs(180)) + .start() + .await + .map_err(|error| format!("cannot start MySQL: {error}"))?; + let port = container + .get_host_port_ipv4(MYSQL_PORT.tcp()) + .await + .map_err(|error| error.to_string())?; + Ok(OdbcServer { + connection_string: format!( + "Driver={{MySQL ODBC Unicode Driver}};Server=127.0.0.1;Port={port};Database={DATABASE};Uid={USER};Pwd={PASSWORD};INITSTMT=SET SESSION sql_mode=CONCAT(@@SESSION.sql_mode,',ANSI_QUOTES');" + ), + container, + }) +} + +/// A SQL Server (Developer edition, licence accepted for testing) on a free host port, and a connection string for Microsoft's ODBC Driver 18 that signs in as `sa` to `master` and trusts the server's own certificate. +pub(crate) async fn start_sql_server() -> Result { + let container = GenericImage::new(SQL_SERVER_IMAGE, SQL_SERVER_TAG) + .with_exposed_port(SQL_SERVER_PORT.tcp()) + .with_wait_for(WaitFor::message_on_stdout(SQL_SERVER_READY)) + .with_env_var("ACCEPT_EULA", "Y") + .with_env_var("MSSQL_SA_PASSWORD", SQL_SERVER_PASSWORD) + .with_startup_timeout(Duration::from_secs(180)) + .start() + .await + .map_err(|error| format!("cannot start SQL Server: {error}"))?; + let port = container + .get_host_port_ipv4(SQL_SERVER_PORT.tcp()) + .await + .map_err(|error| error.to_string())?; + Ok(OdbcServer { + connection_string: format!( + "Driver={{ODBC Driver 18 for SQL Server}};Server=127.0.0.1,{port};Database=master;Uid=sa;Pwd={SQL_SERVER_PASSWORD};TrustServerCertificate=yes;" + ), + container, + }) +} + +/// A Postgres server with logical replication, once it accepts connections, and a connection string for psqlODBC. +pub(crate) async fn start_postgres_server() -> Result { + let postgres = start_postgres().await?; + connect(&postgres).await?; + Ok(OdbcServer { + connection_string: format!( + "Driver={{PostgreSQL Unicode}};Server=127.0.0.1;Port={};Database={DATABASE};Uid={USER};Pwd={PASSWORD};", + postgres.port + ), + container: postgres.container, + }) +} diff --git a/crates/sqlload-testkit/src/odbc_servers/_helpers/mod.rs b/crates/sqlload-testkit/src/odbc_servers/_helpers/mod.rs new file mode 100644 index 00000000..5c680bf2 --- /dev/null +++ b/crates/sqlload-testkit/src/odbc_servers/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod containers; diff --git a/crates/sqlload-testkit/src/odbc_servers/constants.rs b/crates/sqlload-testkit/src/odbc_servers/constants.rs new file mode 100644 index 00000000..edce1e00 --- /dev/null +++ b/crates/sqlload-testkit/src/odbc_servers/constants.rs @@ -0,0 +1,15 @@ +pub const MYSQL_IMAGE: &str = "mysql"; +pub const MYSQL_TAG: &str = "8.4"; +pub const MYSQL_PORT: u16 = 3306; +/// The line MySQL logs once the real server, not the image's setup server, listens on its port. +pub const MYSQL_READY: &str = "port: 3306 MySQL Community Server"; +pub const SQL_SERVER_IMAGE: &str = "mcr.microsoft.com/mssql/server"; +pub const SQL_SERVER_TAG: &str = "2022-latest"; +pub const SQL_SERVER_PORT: u16 = 1433; +/// The line SQL Server logs once its databases are recovered and `sa` can sign in, after it says it is ready for connections. +pub const SQL_SERVER_READY: &str = "Recovery is complete"; +/// The SQL Server administrator's password, which must meet its complexity rules. +pub const SQL_SERVER_PASSWORD: &str = "Sqlload-Test-1"; +pub const USER: &str = "sqlload"; +pub const PASSWORD: &str = "sqlload"; +pub const DATABASE: &str = "app"; diff --git a/crates/sqlload-testkit/src/odbc_servers/main/mod.rs b/crates/sqlload-testkit/src/odbc_servers/main/mod.rs new file mode 100644 index 00000000..4683325d --- /dev/null +++ b/crates/sqlload-testkit/src/odbc_servers/main/mod.rs @@ -0,0 +1 @@ +pub mod start_odbc_server; diff --git a/crates/sqlload-testkit/src/odbc_servers/main/start_odbc_server.rs b/crates/sqlload-testkit/src/odbc_servers/main/start_odbc_server.rs new file mode 100644 index 00000000..a0ed6ecd --- /dev/null +++ b/crates/sqlload-testkit/src/odbc_servers/main/start_odbc_server.rs @@ -0,0 +1,15 @@ +//! Start a database an ODBC driver test reads. + +use crate::odbc_servers::_helpers::containers::{ + start_mysql, start_postgres_server, start_sql_server, +}; +use crate::odbc_servers::models::{OdbcDatabase, OdbcServer}; + +/// A server of `database` on a free host port, and the connection string its ODBC driver reaches it by. +pub async fn start_odbc_server(database: OdbcDatabase) -> Result { + match database { + OdbcDatabase::SqlServer => start_sql_server().await, + OdbcDatabase::MySql => start_mysql().await, + OdbcDatabase::Postgres => start_postgres_server().await, + } +} diff --git a/crates/sqlload-testkit/src/odbc_servers/mod.rs b/crates/sqlload-testkit/src/odbc_servers/mod.rs new file mode 100644 index 00000000..d05c7854 --- /dev/null +++ b/crates/sqlload-testkit/src/odbc_servers/mod.rs @@ -0,0 +1,6 @@ +//! MySQL, SQL Server and Postgres containers, which the ODBC source tests read through their ODBC drivers. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlload-testkit/src/odbc_servers/models.rs b/crates/sqlload-testkit/src/odbc_servers/models.rs new file mode 100644 index 00000000..6781b8ee --- /dev/null +++ b/crates/sqlload-testkit/src/odbc_servers/models.rs @@ -0,0 +1,19 @@ +use testcontainers::{ContainerAsync, GenericImage}; + +/// A running database server and the ODBC connection string that reaches it; the container stops when this is dropped. +#[derive(Debug)] +pub struct OdbcServer { + pub connection_string: String, + pub container: ContainerAsync, +} + +/// A database the ODBC driver tests read through its own driver. +#[derive(Debug, Clone, Copy)] +pub enum OdbcDatabase { + /// Microsoft's ODBC Driver 18 for SQL Server. + SqlServer, + /// MySQL Connector/ODBC. + MySql, + /// psqlODBC. + Postgres, +} diff --git a/crates/sqlload/src/adapters/contract/constants.rs b/crates/sqlload/src/adapters/contract/constants.rs index 4295c406..87d42422 100644 --- a/crates/sqlload/src/adapters/contract/constants.rs +++ b/crates/sqlload/src/adapters/contract/constants.rs @@ -10,6 +10,9 @@ pub const SECONDS_PER_DAY: u64 = 86_400; /// How cursor timestamps are written in literals (microseconds, no offset). pub const CURSOR_TIMESTAMP_FORMAT: &str = "%Y-%m-%d %H:%M:%S%.6f"; +/// The years an ODBC date or timestamp parameter holds. +pub const BOUND_YEARS: std::ops::RangeInclusive = 1..=9999; + /// Timestamps without an offset accepted on the command line. pub const NAIVE_TIMESTAMP_FORMATS: &[&str] = &["%Y-%m-%dT%H:%M:%S%.f", "%Y-%m-%d %H:%M:%S%.f"]; diff --git a/crates/sqlload/src/adapters/contract/main/bound_parameter.rs b/crates/sqlload/src/adapters/contract/main/bound_parameter.rs new file mode 100644 index 00000000..4a2cf9b5 --- /dev/null +++ b/crates/sqlload/src/adapters/contract/main/bound_parameter.rs @@ -0,0 +1,29 @@ +//! Cursor bounds a source binds to placeholders. + +use chrono::{Datelike, NaiveDate, NaiveDateTime}; + +use crate::adapters::contract::constants::{BOUND_YEARS, NAIVE_TIMESTAMP_FORMATS}; +use crate::adapters::contract::models::BoundParameter; +use crate::column_types::models::ColumnType; + +/// `value`, a normalised cursor bound, as the parameter a cursor of `column_type` compares with: text for a text cursor, a timestamp for a wall-clock timestamp cursor, a date for a date cursor; `None` for any other cursor, or a value that is not of its type or falls outside the years a bound parameter holds, which then stays a literal. +pub fn bound_parameter(value: &str, column_type: &ColumnType) -> Option { + match column_type { + ColumnType::Text | ColumnType::Varchar(_) => Some(BoundParameter::Text(value.to_owned())), + ColumnType::TimestampNtz => NAIVE_TIMESTAMP_FORMATS + .iter() + .find_map( + |format| match NaiveDateTime::parse_from_str(value, format) { + Ok(time) => Some(time), + Err(_) => None, + }, + ) + .filter(|time| BOUND_YEARS.contains(&time.year())) + .map(BoundParameter::Timestamp), + ColumnType::Date => match NaiveDate::parse_from_str(value, "%Y-%m-%d") { + Ok(date) if BOUND_YEARS.contains(&date.year()) => Some(BoundParameter::Date(date)), + Ok(_) | Err(_) => None, + }, + _ => None, + } +} diff --git a/crates/sqlload/src/adapters/contract/main/cursor_text.rs b/crates/sqlload/src/adapters/contract/main/cursor_text.rs index 67bdba39..4b23619b 100644 --- a/crates/sqlload/src/adapters/contract/main/cursor_text.rs +++ b/crates/sqlload/src/adapters/contract/main/cursor_text.rs @@ -5,8 +5,14 @@ use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime}; use crate::adapters::contract::constants::{CURSOR_TIMESTAMP_FORMAT, NAIVE_TIMESTAMP_FORMATS}; use crate::column_types::models::ColumnType; -/// `text` as the cursor literal every system reads alike: zoned instants in UTC with `+00`, wall-clock times without an offset. +/// `text` as the cursor literal every system reads alike: zoned instants in UTC with `+00`, wall-clock times without an offset; for a timestamp or date cursor, an ODBC `{ts '...'}` or `{d '...'}` literal, as a query against an ODBC source writes it, reads as the value inside (any other cursor keeps it as given). pub fn cursor_text(text: &str, column_type: &ColumnType) -> Result { + let text = match column_type { + ColumnType::TimestampTz | ColumnType::TimestampNtz | ColumnType::Date => { + odbc_value(text).unwrap_or(text) + } + _ => text, + }; let zoned = DateTime::parse_from_rfc3339(text); let naive = NAIVE_TIMESTAMP_FORMATS .iter() @@ -67,3 +73,13 @@ pub fn cursor_text(text: &str, column_type: &ColumnType) -> Result Ok(text.to_owned()), } } + +/// The quoted value of an ODBC `{ts '...'}` or `{d '...'}` escape literal, in any letter case and spacing. +fn odbc_value(text: &str) -> Option<&str> { + let inner = text.trim().strip_prefix('{')?.strip_suffix('}')?.trim(); + let (keyword, quoted) = inner.split_at(inner.find('\'')?); + let keyword = keyword.trim(); + (keyword.eq_ignore_ascii_case("ts") || keyword.eq_ignore_ascii_case("d")) + .then(|| quoted.strip_prefix('\'')?.strip_suffix('\'')) + .flatten() +} diff --git a/crates/sqlload/src/adapters/contract/main/mod.rs b/crates/sqlload/src/adapters/contract/main/mod.rs index 718c6044..731b97ee 100644 --- a/crates/sqlload/src/adapters/contract/main/mod.rs +++ b/crates/sqlload/src/adapters/contract/main/mod.rs @@ -1,6 +1,7 @@ pub(in crate::adapters) mod _batch_direct_problem; pub(in crate::adapters) mod _files_direct_problem; pub(in crate::adapters) mod _footer_rows; +pub mod bound_parameter; pub mod cursor_bound; pub mod cursor_from_now; pub mod cursor_text; diff --git a/crates/sqlload/src/adapters/contract/models.rs b/crates/sqlload/src/adapters/contract/models.rs index a1bc7033..b1698f92 100644 --- a/crates/sqlload/src/adapters/contract/models.rs +++ b/crates/sqlload/src/adapters/contract/models.rs @@ -121,6 +121,17 @@ pub enum CursorValue { Plain, } +/// A cursor bound passed to a `?` placeholder rather than written into the filter, typed as its cursor column compares it, so the database's literal rules can't change it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BoundParameter { + /// A text cursor's bound. + Text(String), + /// A wall-clock timestamp cursor's bound. + Timestamp(chrono::NaiveDateTime), + /// A date cursor's bound. + Date(chrono::NaiveDate), +} + /// A query for a cursor's maximum and how to read its result. #[derive(Debug, Clone, PartialEq, Eq)] pub struct CursorRead { diff --git a/crates/sqlload/src/adapters/contract/tests.rs b/crates/sqlload/src/adapters/contract/tests.rs index ef4bdf53..501c19d6 100644 --- a/crates/sqlload/src/adapters/contract/tests.rs +++ b/crates/sqlload/src/adapters/contract/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_bound_parameter.rs"] +mod bound_parameter; #[path = "tests/test_build_table.rs"] mod build_table; #[path = "tests/test_check_query.rs"] diff --git a/crates/sqlload/src/adapters/contract/tests/helpers.rs b/crates/sqlload/src/adapters/contract/tests/helpers.rs index 17eab38c..5f79e322 100644 --- a/crates/sqlload/src/adapters/contract/tests/helpers.rs +++ b/crates/sqlload/src/adapters/contract/tests/helpers.rs @@ -123,3 +123,21 @@ pub(crate) fn writes(statement: &str, (target, name): (&str, &str)) -> bool { || statement.starts_with(&format!("{verb} IF EXISTS {target}")) }) } + +/// `text` (`YYYY-MM-DD HH:MM:SS[.f]`) as a bound timestamp. +pub(crate) fn timestamp_parameter( + text: &str, +) -> Result { + chrono::NaiveDateTime::parse_from_str(text, "%Y-%m-%d %H:%M:%S%.f") + .map(crate::adapters::contract::models::BoundParameter::Timestamp) + .map_err(|error| error.to_string()) +} + +/// `text` (`YYYY-MM-DD`) as a bound date. +pub(crate) fn date_parameter( + text: &str, +) -> Result { + chrono::NaiveDate::parse_from_str(text, "%Y-%m-%d") + .map(crate::adapters::contract::models::BoundParameter::Date) + .map_err(|error| error.to_string()) +} diff --git a/crates/sqlload/src/adapters/contract/tests/test_bound_parameter.rs b/crates/sqlload/src/adapters/contract/tests/test_bound_parameter.rs new file mode 100644 index 00000000..50b78f50 --- /dev/null +++ b/crates/sqlload/src/adapters/contract/tests/test_bound_parameter.rs @@ -0,0 +1,81 @@ +use crate::adapters::contract::main::bound_parameter::bound_parameter; +use crate::adapters::contract::models::BoundParameter; +use crate::adapters::contract::tests::helpers::{date_parameter, timestamp_parameter}; +use crate::adapters::contract::tests::test_types::BoundParameterTestCase; +use crate::column_types::models::ColumnType; + +#[test] +fn given_a_cursor_bound_when_binding_it_for_an_odbc_source_then_text_dates_and_wall_clock_times_keep_their_type_and_every_digit() +-> Result<(), String> { + let test_cases = [ + BoundParameterTestCase { + description: "the millisecond maximum a cursor resumes from keeps its milliseconds exactly", + value: "2026-01-31 23:59:59.997000", + column_type: ColumnType::TimestampNtz, + expected_parameter: Some(timestamp_parameter("2026-01-31 23:59:59.997")?), + }, + BoundParameterTestCase { + description: "a microsecond bound keeps all six digits", + value: "2026-01-31 23:59:59.000001", + column_type: ColumnType::TimestampNtz, + expected_parameter: Some(timestamp_parameter("2026-01-31 23:59:59.000001")?), + }, + BoundParameterTestCase { + description: "a bound written with a T and no fraction reads the same", + value: "2026-02-01T08:30:00", + column_type: ColumnType::TimestampNtz, + expected_parameter: Some(timestamp_parameter("2026-02-01 08:30:00")?), + }, + BoundParameterTestCase { + description: "a date is a date", + value: "2026-02-01", + column_type: ColumnType::Date, + expected_parameter: Some(date_parameter("2026-02-01")?), + }, + BoundParameterTestCase { + description: "a text bound is text, whatever it holds", + value: "o'brien\\", + column_type: ColumnType::Varchar(40), + expected_parameter: Some(BoundParameter::Text("o'brien\\".to_owned())), + }, + BoundParameterTestCase { + description: "a zoned timestamp is not bound, since an ODBC timestamp carries no offset", + value: "2026-01-31 23:59:59.997000+00", + column_type: ColumnType::TimestampTz, + expected_parameter: None, + }, + BoundParameterTestCase { + description: "a value that is not a timestamp is not bound as one", + value: "2026-01-31'} OR 1=1 --", + column_type: ColumnType::TimestampNtz, + expected_parameter: None, + }, + BoundParameterTestCase { + description: "a date cursor given a timestamp is not bound", + value: "2026-02-01 00:00:00", + column_type: ColumnType::Date, + expected_parameter: None, + }, + BoundParameterTestCase { + description: "a year an ODBC date can't hold is not bound", + value: "+10000-01-01", + column_type: ColumnType::Date, + expected_parameter: None, + }, + BoundParameterTestCase { + description: "an integer cursor's bound is a number, not a parameter", + value: "42", + column_type: ColumnType::Int, + expected_parameter: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + bound_parameter(test_case.value, &test_case.column_type), + test_case.expected_parameter, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlload/src/adapters/contract/tests/test_cursor_bound.rs b/crates/sqlload/src/adapters/contract/tests/test_cursor_bound.rs index 65196d08..83c3e705 100644 --- a/crates/sqlload/src/adapters/contract/tests/test_cursor_bound.rs +++ b/crates/sqlload/src/adapters/contract/tests/test_cursor_bound.rs @@ -23,6 +23,13 @@ fn given_a_cursor_maximum_when_making_the_lower_bound_then_lookback_and_zone_are lookback: HOUR, expected_bound: Ok("2026-11-04 09:02:03.000000"), }, + CursorBoundTestCase { + description: "a wall-clock millisecond maximum, as an ODBC source's cursor resumes from, keeps its milliseconds", + raw: "1769903999997000", + value: CursorValue::EpochMicros { zoned: false }, + lookback: None, + expected_bound: Ok("2026-01-31 23:59:59.997000"), + }, CursorBoundTestCase { description: "a date moves back by whole days, rounding the lookback up", raw: "2026-11-04", diff --git a/crates/sqlload/src/adapters/contract/tests/test_cursor_text.rs b/crates/sqlload/src/adapters/contract/tests/test_cursor_text.rs index d95452d4..52527cfc 100644 --- a/crates/sqlload/src/adapters/contract/tests/test_cursor_text.rs +++ b/crates/sqlload/src/adapters/contract/tests/test_cursor_text.rs @@ -29,6 +29,42 @@ fn given_a_cursor_flag_when_normalising_then_every_system_reads_the_same_bound() column_type: ColumnType::TimestampNtz, expected_text: Err("has a UTC offset"), }, + CursorTextTestCase { + description: "a wall-clock cursor reads an ODBC {ts} literal with milliseconds", + text: "{ts '2026-01-31 23:59:59.997'}", + column_type: ColumnType::TimestampNtz, + expected_text: Ok("2026-01-31 23:59:59.997000"), + }, + CursorTextTestCase { + description: "an ODBC literal reads in any letter case and spacing", + text: " { TS'2026-01-31 23:59:59' } ", + column_type: ColumnType::TimestampNtz, + expected_text: Ok("2026-01-31 23:59:59.000000"), + }, + CursorTextTestCase { + description: "a date cursor reads an ODBC {d} literal", + text: "{d '2026-02-01'}", + column_type: ColumnType::Date, + expected_text: Ok("2026-02-01"), + }, + CursorTextTestCase { + description: "a zoned cursor reads an ODBC {ts} literal as UTC", + text: "{ts '2026-01-31 23:59:59.5'}", + column_type: ColumnType::TimestampTz, + expected_text: Ok("2026-01-31 23:59:59.500000+00"), + }, + CursorTextTestCase { + description: "an ODBC literal missing its closing quote is refused", + text: "{ts '2026-01-31 23:59:59}", + column_type: ColumnType::TimestampNtz, + expected_text: Err("is not an RFC 3339 timestamp or a date"), + }, + CursorTextTestCase { + description: "a text cursor keeps a value that looks like an ODBC literal as it is", + text: "{d 'abc'}", + column_type: ColumnType::Text, + expected_text: Ok("{d 'abc'}"), + }, CursorTextTestCase { description: "a date cursor needs a date", text: "2026-11-04T10:00:00Z", diff --git a/crates/sqlload/src/adapters/contract/tests/test_types.rs b/crates/sqlload/src/adapters/contract/tests/test_types.rs index 02e0d501..833c4edc 100644 --- a/crates/sqlload/src/adapters/contract/tests/test_types.rs +++ b/crates/sqlload/src/adapters/contract/tests/test_types.rs @@ -1,7 +1,9 @@ use crate::column_types::models::Warehouse; use std::time::Duration; -use crate::adapters::contract::models::{CheckQuery, CursorValue, NowShift, WriteMode}; +use crate::adapters::contract::models::{ + BoundParameter, CheckQuery, CursorValue, NowShift, WriteMode, +}; use crate::column_types::models::{ColumnType, LiveColumnType, TypeFit}; pub(crate) struct WriteStatementsTestCase { @@ -148,6 +150,13 @@ pub(crate) struct CursorFromNowTestCase { pub(crate) expected_text: Result<&'static str, &'static str>, } +pub(crate) struct BoundParameterTestCase { + pub(crate) description: &'static str, + pub(crate) value: &'static str, + pub(crate) column_type: ColumnType, + pub(crate) expected_parameter: Option, +} + pub(crate) struct MaxCursorTestCase { pub(crate) description: &'static str, pub(crate) warehouse: Warehouse, diff --git a/crates/sqlload/src/loader/_helpers/publish.rs b/crates/sqlload/src/loader/_helpers/publish.rs index e9f4b529..382f5555 100644 --- a/crates/sqlload/src/loader/_helpers/publish.rs +++ b/crates/sqlload/src/loader/_helpers/publish.rs @@ -2,6 +2,7 @@ use crate::adapters::contract::constants::COMMIT_STATEMENT; use crate::adapters::contract::constants::SWAP_SUFFIX; use crate::adapters::contract::main::cursor_text::cursor_text; use crate::adapters::contract::main::staging_id::staging_id; +use crate::adapters::contract::models::BoundParameter; use crate::adapters::contract::models::TableName; use crate::adapters::contract::models::TableShape; use crate::adapters::contract::models::WriteMode; @@ -68,7 +69,7 @@ use crate::loader::_helpers::sources::files_source_parts; use crate::loader::_helpers::sources::nanosecond_columns; use crate::loader::_helpers::sources::read_source; use crate::loader::_helpers::sources::unload_source; -use crate::loader::_helpers::table_plan::source_filter; +use crate::loader::_helpers::table_plan::{binds_cursor, source_filter}; use crate::loader::_helpers::table_plan::{replan_columns, schema_statements}; use crate::loader::constants::EXCLUSIVE_BOUND; use crate::loader::constants::INCLUSIVE_BOUND; @@ -985,8 +986,10 @@ async fn feed( snapshot(reader, SNAPSHOT_BEGIN).await?; for index in 0..windows.count { let window = &window_at(windows, index).map_err(LoaderError::Invalid)?; + let (filter, parameters) = window_filter(table, window); let part = ReadPart { - filter: window_filter(table, window), + filter, + parameters, batch_bytes, ..ReadPart::default() }; @@ -1135,33 +1138,38 @@ async fn stage(publisher: &mut Publisher, batch: RecordBatch) -> Result<(), Load publisher.stage(batch, None).await } -/// The source filter reading one cursor window, in the source's dialect; its bounds are literals, since `split_windows` cuts only number and time cursors, never a text one. -fn window_filter(table: &TablePlan, window: &CursorWindow) -> Option { - let cursor = table.spec.cursor.as_ref()?; - let column = table - .read_columns - .iter() - .find(|column| column.name.eq_ignore_ascii_case(&cursor.column))?; +/// The source filter reading one cursor window, in the source's dialect, and the values bound to its placeholders: an ODBC source's date and timestamp bounds (`split_windows` cuts only number and time cursors, never a text one). +fn window_filter( + table: &TablePlan, + window: &CursorWindow, +) -> (Option, Vec) { + let mut parameters: Vec = Vec::new(); + let Some(column) = table.spec.cursor.as_ref().and_then(|cursor| { + table + .read_columns + .iter() + .find(|column| column.name.eq_ignore_ascii_case(&cursor.column)) + }) else { + return (None, parameters); + }; let operator = match window.inclusive { true => INCLUSIVE_BOUND, false => EXCLUSIVE_BOUND, }; - source_filter( + let source = table.spec.warehouse_source.as_ref(); + let filter = source_filter( &column.expression, ( Some(&column.output), - table - .spec - .warehouse_source - .as_ref() - .and_then(|source| source_dialect(source.system)), + source.and_then(|source| source_dialect(source.system)), ), ( window.from.as_deref().map(|from| (operator, from)), window.to.as_deref(), ), - None, - ) + binds_cursor(&table.spec).then_some(&mut parameters), + ); + (filter, parameters) } /// The replaced files a commit deletes: those of the partitions it wrote, and, at the last commit, every other file the load replaces (everything for `replace`, the window's partitions for `delete_insert`); never one already deleted. diff --git a/crates/sqlload/src/loader/_helpers/sources.rs b/crates/sqlload/src/loader/_helpers/sources.rs index 171791f9..44336f3b 100644 --- a/crates/sqlload/src/loader/_helpers/sources.rs +++ b/crates/sqlload/src/loader/_helpers/sources.rs @@ -1,3 +1,4 @@ +use crate::adapters::contract::models::BoundParameter; use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter; use arrow_array::RecordBatch; use std::collections::BTreeMap; @@ -1186,6 +1187,12 @@ pub(crate) async fn plan_files( pub(crate) struct ReadPart { /// The Postgres filter; `None` reads the plan's. pub(crate) filter: Option, + /// The values bound to `filter`'s `?` placeholders, in order. + #[cfg_attr( + not(feature = "odbc"), + expect(dead_code, reason = "only ODBC sources bind parameters") + )] + pub(crate) parameters: Vec, /// `ORDER BY` this quoted column. pub(crate) order_by: Option, /// The files to read; `None` reads every planned file. @@ -1264,7 +1271,7 @@ async fn read_each( #[cfg(feature = "odbc")] (SourceReader::Odbc(reader), _) => { let (filter, parameters) = match part.filter { - Some(filter) => (Some(filter), &[][..]), + Some(filter) => (Some(filter), &part.parameters[..]), None => (table.source_filter.clone(), &table.source_parameters[..]), }; let request = crate::odbc_source::models::OdbcReadRequest { diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index 0e497c7b..6c804e5c 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -1,6 +1,8 @@ +use crate::adapters::contract::main::bound_parameter::bound_parameter; use crate::adapters::contract::main::cursor_bound::cursor_bound; use crate::adapters::contract::main::cursor_from_now::cursor_from_now; use crate::adapters::contract::main::cursor_text::cursor_text; +use crate::adapters::contract::models::BoundParameter; use crate::adapters::contract::models::CursorQuery; use crate::adapters::contract::models::NowShift; use crate::adapters::contract::models::WriteMode; @@ -970,8 +972,8 @@ pub(crate) struct WindowRequest<'a> { /// The source filter, the write mode and a note for `plan`. pub(crate) struct Window { pub(crate) filter: Option, - /// The values bound to `filter`'s `?` placeholders, in order: an ODBC source's text cursor bounds. - pub(crate) parameters: Vec, + /// The values bound to `filter`'s `?` placeholders, in order: an ODBC source's cursor bounds. + pub(crate) parameters: Vec, /// The cursor range read: `from <= cursor < to`, each open when `None`. pub(crate) bounds: (Option, Option), pub(crate) mode: WriteMode, @@ -1034,22 +1036,23 @@ pub(crate) fn source_cursor( )) } -/// `column >= 'from' AND column < 'to'`, from the bounds that are set, as literals of the source's `dialect` (Postgres's when `None`): a numeric cursor's plain-number bounds as numbers (`render_cursor_number`), as BigQuery compares a number with no string; a text cursor's bounds as `?` placeholders when `parameters` collects their values, in order, to bind; any other bound as the dialect's string literal, so a value holding a quote or a backslash stays one string. +/// `column >= 'from' AND column < 'to'`, from the bounds that are set, as literals of the source's `dialect` (Postgres's when `None`): a numeric cursor's plain-number bounds as numbers (`render_cursor_number`), as BigQuery compares a number with no string; a text, date or wall-clock timestamp cursor's bounds as `?` placeholders when `parameters` collects them, in order, to bind as their type (an ODBC source's: no literal reads alike everywhere, since some databases read a backslash in a string as an escape, OpenEdge compares no string with a timestamp, and SQL Server's `{ts}` escape is a `datetime` that rounds milliseconds); any other bound as the dialect's string literal, so a value holding a quote or a backslash stays one string. pub(crate) fn source_filter( column: &str, (cursor_type, dialect): (Option<&ColumnType>, Option), (lower, upper): (Option<(&str, &str)>, Option<&str>), - mut parameters: Option<&mut Vec>, + mut parameters: Option<&mut Vec>, ) -> Option { let adapter = sql_adapter(dialect.unwrap_or(Warehouse::Postgres)); - let text = matches!(cursor_type, Some(ColumnType::Text | ColumnType::Varchar(_))); let mut literal = |value: &str| match ( cursor_type.filter(|cursor_type| numeric_cursor(cursor_type) && is_number(value)), - parameters.as_mut().filter(|_| text), + parameters + .as_mut() + .zip(cursor_type.and_then(|cursor_type| bound_parameter(value, cursor_type))), ) { (Some(cursor_type), _) => adapter.render_cursor_number(value, cursor_type), - (None, Some(parameters)) => { - parameters.push(value.to_owned()); + (None, Some((parameters, parameter))) => { + parameters.push(parameter); "?".to_owned() } (None, None) => adapter.render_loader_value_literal(value), @@ -1064,6 +1067,21 @@ pub(crate) fn source_filter( (!bounds.is_empty()).then(|| bounds.join(" AND ")) } +/// Whether `spec`'s cursor bounds bind as parameters of the cursor's type: on a source that binds them (ODBC), unless the cursor column is pinned to a type other than text, which may not be the source column's own (a `varchar` read as a timestamp), so its bounds stay literals the database converts; text binds as text whatever the column, as a literal would convert, without a database reading its backslashes. +pub(crate) fn binds_cursor(spec: &TableSpec) -> bool { + let pinned_other = spec.cursor.as_ref().is_some_and(|cursor| { + spec.pinned.iter().any(|(name, column_type)| { + name.eq_ignore_ascii_case(&cursor.column) + && !matches!(column_type, ColumnType::Text | ColumnType::Varchar(_)) + }) + }); + !pinned_other + && spec + .warehouse_source + .as_ref() + .is_some_and(|source| source_binds_text(source.system)) +} + /// Whether a cursor of `column_type` compares as a number. fn numeric_cursor(column_type: &ColumnType) -> bool { matches!( @@ -1276,7 +1294,7 @@ pub(crate) async fn cursor_window( (None, None) => (None, note), }; let source = request.spec.warehouse_source.as_ref(); - let mut parameters: Vec = Vec::new(); + let mut parameters: Vec = Vec::new(); let filter = source_filter( request.cursor_expression.unwrap_or_default(), ( @@ -1284,9 +1302,7 @@ pub(crate) async fn cursor_window( source.and_then(|source| source_dialect(source.system)), ), (from.as_deref().map(|from| (operator, from)), to.as_deref()), - source - .is_some_and(|source| source_binds_text(source.system)) - .then_some(&mut parameters), + binds_cursor(spec).then_some(&mut parameters), ); Ok(Ok(Window { filter, diff --git a/crates/sqlload/src/loader/models.rs b/crates/sqlload/src/loader/models.rs index 46e28390..7e1475c9 100644 --- a/crates/sqlload/src/loader/models.rs +++ b/crates/sqlload/src/loader/models.rs @@ -1,3 +1,4 @@ +use crate::adapters::contract::models::BoundParameter; use crate::adapters::contract::models::ChangeApply; use std::collections::BTreeMap; use std::time::Duration; @@ -282,8 +283,8 @@ pub struct TablePlan { pub mode: WriteMode, /// The source filter for incremental reads, already rendered in the source's dialect. pub source_filter: Option, - /// The values bound to `source_filter`'s `?` placeholders, in order: an ODBC source's text cursor bounds. - pub source_parameters: Vec, + /// The values bound to `source_filter`'s `?` placeholders, in order: an ODBC source's cursor bounds. + pub source_parameters: Vec, /// How the cursor window was chosen, for `plan`. pub cursor_note: Option, /// The cursor range read, for sources that filter it themselves (Python tables). diff --git a/crates/sqlload/src/loader/tests/test_source_filter.rs b/crates/sqlload/src/loader/tests/test_source_filter.rs index 0047b415..d652dbf4 100644 --- a/crates/sqlload/src/loader/tests/test_source_filter.rs +++ b/crates/sqlload/src/loader/tests/test_source_filter.rs @@ -10,7 +10,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the description: "a numeric cursor's bounds are bare numbers, which BigQuery compares with an INT64 column", cursor_type: ColumnType::Int, dialect: None, - binds: false, + odbc: false, lower: Some((">", "41")), upper: Some("100.5"), expected_filter: Some("`id` > 41 AND `id` < 100.5"), @@ -20,7 +20,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the description: "a text cursor holding digits stays a string", cursor_type: ColumnType::Text, dialect: None, - binds: false, + odbc: false, lower: Some((">=", "41")), upper: None, expected_filter: Some("`id` >= '41'"), @@ -33,7 +33,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the scale: 9, }, dialect: Some(Warehouse::BigQuery), - binds: false, + odbc: false, lower: Some((">", "1234567890123.000000001")), upper: Some("-0.5"), expected_filter: Some( @@ -48,7 +48,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the scale: 9, }, dialect: Some(Warehouse::Snowflake), - binds: false, + odbc: false, lower: Some((">", "12.5")), upper: None, expected_filter: Some("`id` > 12.5"), @@ -58,7 +58,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the description: "a BigQuery text bound escapes quotes and backslashes with a backslash, so a value cannot end the string", cursor_type: ColumnType::Text, dialect: Some(Warehouse::BigQuery), - binds: false, + odbc: false, lower: Some((">", r"zz\' UNION ALL SELECT 1 --")), upper: Some(r"o'brien\"), expected_filter: Some(r"`id` > 'zz\\\' UNION ALL SELECT 1 --' AND `id` < 'o\'brien\\'"), @@ -68,7 +68,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the description: "a Snowflake text maximum ending in a backslash doubles it, so it cannot escape the closing quote", cursor_type: ColumnType::Text, dialect: Some(Warehouse::Snowflake), - binds: false, + odbc: false, lower: Some((">", r"o'brien\")), upper: None, expected_filter: Some(r"`id` > 'o''brien\\'"), @@ -78,7 +78,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the description: "a Snowflake text bound holding a backslash before a quote stays one string, so the rest is not SQL", cursor_type: ColumnType::Text, dialect: Some(Warehouse::Snowflake), - binds: false, + odbc: false, lower: Some((">=", r"zz\' OR 1=1 --")), upper: Some(r"zz\'"), expected_filter: Some(r"`id` >= 'zz\\'' OR 1=1 --' AND `id` < 'zz\\'''"), @@ -88,7 +88,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the description: "a Postgres text bound doubles its quotes and keeps backslashes, as standard strings read them", cursor_type: ColumnType::Text, dialect: None, - binds: false, + odbc: false, lower: Some((">", r"o'brien\")), upper: None, expected_filter: Some(r"`id` > 'o''brien\'"), @@ -98,27 +98,90 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the description: "an ODBC text bound holding a quote and a backslash is a bound parameter, so no database's literal rules can misread it", cursor_type: ColumnType::Varchar(40), dialect: None, - binds: true, + odbc: true, lower: Some((">", r"zz\' OR 1=1 --")), upper: Some(r"o'brien\"), expected_filter: Some("`id` > ? AND `id` < ?"), - expected_parameters: &[r"zz\' OR 1=1 --", r"o'brien\"], + expected_parameters: &[r#"Text("zz\\' OR 1=1 --")"#, r#"Text("o'brien\\")"#], }, SourceFilterTestCase { description: "an ODBC numeric bound stays a bare number, not a parameter", cursor_type: ColumnType::Int, dialect: None, - binds: true, + odbc: true, lower: Some((">=", "41")), upper: None, expected_filter: Some("`id` >= 41"), expected_parameters: &[], }, + SourceFilterTestCase { + description: "an ODBC timestamp cursor resumes after its millisecond maximum through timestamp parameters, which every driver compares as timestamps (OpenEdge compares no string with one)", + cursor_type: ColumnType::TimestampNtz, + dialect: None, + odbc: true, + lower: Some((">", "2026-01-31 23:59:59.997000")), + upper: Some("2026-02-01 00:00:00.000000"), + expected_filter: Some("`id` > ? AND `id` < ?"), + expected_parameters: &[ + "Timestamp(2026-01-31T23:59:59.997)", + "Timestamp(2026-02-01T00:00:00)", + ], + }, + SourceFilterTestCase { + description: "an ODBC timestamp bound keeps microseconds rather than rounding them away", + cursor_type: ColumnType::TimestampNtz, + dialect: None, + odbc: true, + lower: Some((">=", "2026-01-31 23:59:59.123456")), + upper: None, + expected_filter: Some("`id` >= ?"), + expected_parameters: &["Timestamp(2026-01-31T23:59:59.123456)"], + }, + SourceFilterTestCase { + description: "an ODBC date cursor's bound is a date parameter", + cursor_type: ColumnType::Date, + dialect: None, + odbc: true, + lower: Some((">=", "2026-02-01")), + upper: None, + expected_filter: Some("`id` >= ?"), + expected_parameters: &["Date(2026-02-01)"], + }, + SourceFilterTestCase { + description: "a timestamp bound from any other source stays a string literal", + cursor_type: ColumnType::TimestampNtz, + dialect: None, + odbc: false, + lower: Some((">", "2026-01-31 23:59:59.997000")), + upper: None, + expected_filter: Some("`id` > '2026-01-31 23:59:59.997000'"), + expected_parameters: &[], + }, + SourceFilterTestCase { + description: "an ODBC zoned timestamp bound stays a string literal, since an ODBC timestamp has no offset", + cursor_type: ColumnType::TimestampTz, + dialect: None, + odbc: true, + lower: Some((">", "2026-01-31 23:59:59.997000+00")), + upper: None, + expected_filter: Some("`id` > '2026-01-31 23:59:59.997000+00'"), + expected_parameters: &[], + }, + SourceFilterTestCase { + description: "an ODBC timestamp bound that is not a timestamp stays one quoted string, never a timestamp parameter", + cursor_type: ColumnType::TimestampNtz, + dialect: None, + odbc: true, + lower: Some((">", "2026-01-31'} OR 1=1 --")), + upper: None, + expected_filter: Some("`id` > '2026-01-31''} OR 1=1 --'"), + expected_parameters: &[], + }, SourceFilterTestCase { description: "a numeric cursor's bound that isn't a plain number is quoted, never written bare", cursor_type: ColumnType::Float, dialect: None, - binds: false, + odbc: false, lower: Some((">=", "1 OR 1=1")), upper: Some("NaN"), expected_filter: Some("`id` >= '1 OR 1=1' AND `id` < 'NaN'"), @@ -128,7 +191,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the description: "no bounds, no filter", cursor_type: ColumnType::Int, dialect: None, - binds: false, + odbc: false, lower: None, upper: None, expected_filter: None, @@ -141,10 +204,16 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the "`id`", (Some(&test_case.cursor_type), test_case.dialect), (test_case.lower, test_case.upper), - test_case.binds.then_some(&mut parameters), + test_case.odbc.then_some(&mut parameters), ); assert_eq!( - (filter.as_deref(), parameters), + ( + filter.as_deref(), + parameters + .iter() + .map(|parameter| format!("{parameter:?}")) + .collect::>() + ), ( test_case.expected_filter, test_case diff --git a/crates/sqlload/src/loader/tests/test_types.rs b/crates/sqlload/src/loader/tests/test_types.rs index 1d5c6c9b..b98f1bb2 100644 --- a/crates/sqlload/src/loader/tests/test_types.rs +++ b/crates/sqlload/src/loader/tests/test_types.rs @@ -91,12 +91,12 @@ pub(crate) struct SourceFilterTestCase { pub(crate) description: &'static str, pub(crate) cursor_type: ColumnType, pub(crate) dialect: Option, - /// Whether text bounds are bound as parameters, as an ODBC source's are. - pub(crate) binds: bool, + /// Whether the source is an ODBC source, whose text, date and timestamp bounds are bound parameters. + pub(crate) odbc: bool, pub(crate) lower: Option<(&'static str, &'static str)>, pub(crate) upper: Option<&'static str>, pub(crate) expected_filter: Option<&'static str>, - /// The values bound to the filter's placeholders, in order. + /// The values bound to the filter's placeholders, in order, as Debug prints them. pub(crate) expected_parameters: &'static [&'static str], } diff --git a/crates/sqlload/src/odbc_source/_helpers/batches.rs b/crates/sqlload/src/odbc_source/_helpers/batches.rs index 64ff42b9..16291696 100644 --- a/crates/sqlload/src/odbc_source/_helpers/batches.rs +++ b/crates/sqlload/src/odbc_source/_helpers/batches.rs @@ -4,19 +4,19 @@ use std::sync::Arc; use arrow_odbc::odbc_api::Cursor; use arrow_odbc::{ColumnFailure, OdbcReaderBuilder, arrow_schema_from}; -use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit}; use crate::odbc_source::constants::TOO_LARGE_VALUE; use crate::odbc_source::errors::OdbcSourceError; use crate::odbc_source::models::OdbcReader; -/// The reader of `cursor`'s rows in batches of `batch_rows`, decimals as text, each buffer capped by the reader's limits; a column the driver gives no length for is refused, naming the setting that bounds it. +/// The reader of `cursor`'s rows in batches of `batch_rows`, as `fetch_schema` fetches them, each buffer capped by the reader's limits; a column the driver gives no length for is refused, naming the setting that bounds it. pub(crate) fn batch_reader( reader: &OdbcReader, mut cursor: C, (dbms, batch_rows): (String, usize), ) -> Result, OdbcSourceError> { - let schema = decimals_as_text(&arrow_schema_from(&mut cursor, Some(&dbms), false)?); + let schema = fetch_schema(&arrow_schema_from(&mut cursor, Some(&dbms), false)?); let mut builder = OdbcReaderBuilder::new(); builder .with_schema(schema) @@ -51,13 +51,17 @@ pub(crate) fn fetch_error(text: &str) -> OdbcSourceError { } } -/// `schema` with every decimal field fetched as text instead. -pub(crate) fn decimals_as_text(schema: &Schema) -> SchemaRef { +/// `schema` with every decimal field fetched as text, and every timestamp coarser than microseconds fetched in microseconds: a driver may describe fewer fractional digits than its values hold (MySQL Connector/ODBC describes every `DATETIME(n)` with none), and the reader would drop the rest. +pub(crate) fn fetch_schema(schema: &Schema) -> SchemaRef { let fields: Vec = schema .fields() .iter() .map(|field| match field.data_type() { DataType::Decimal128(..) => field.as_ref().clone().with_data_type(DataType::Utf8), + DataType::Timestamp(TimeUnit::Second | TimeUnit::Millisecond, zone) => field + .as_ref() + .clone() + .with_data_type(DataType::Timestamp(TimeUnit::Microsecond, zone.clone())), _ => field.as_ref().clone(), }) .collect(); diff --git a/crates/sqlload/src/odbc_source/_helpers/catalog.rs b/crates/sqlload/src/odbc_source/_helpers/catalog.rs index 4ad49bcf..502a3aef 100644 --- a/crates/sqlload/src/odbc_source/_helpers/catalog.rs +++ b/crates/sqlload/src/odbc_source/_helpers/catalog.rs @@ -74,7 +74,7 @@ pub(crate) fn source_table( } } -/// The tables and views whose `schema.table` (or `table`, for a driver without schemas) matches `include` and not `exclude`, with their columns and keys. +/// The tables and views whose `schema.table` (or `table`, for a driver without schemas) matches `include` and not `exclude`, with their columns and keys. A driver that refuses a schema pattern and reports no schema name length (MySQL Connector/ODBC, by default) is asked again with none; one reporting 0 for "no maximum" still lists every schema. pub(crate) fn catalog_tables( connection: &Connection<'_>, (include, exclude): (&[String], &[String]), @@ -82,7 +82,16 @@ pub(crate) fn catalog_tables( let include = table_matcher(include)?; let exclude = table_matcher(exclude)?; let mut names: Vec<(String, String)> = Vec::new(); - for row in connection.tables("", ANY_NAME, ANY_NAME, TABLE_TYPES)? { + let listed = match connection.tables("", ANY_NAME, ANY_NAME, TABLE_TYPES) { + Ok(listed) => listed, + Err(refused) => match connection.max_schema_name_len() { + Ok(0) => connection + .tables("", "", ANY_NAME, TABLE_TYPES) + .map_err(|_| refused)?, + _ => return Err(refused.into()), + }, + }; + for row in listed { let row = row?; let (schema, table) = (catalog_text(&row.schema)?, catalog_text(&row.table)?); let qualified = match schema.is_empty() { diff --git a/crates/sqlload/src/odbc_source/_helpers/connect.rs b/crates/sqlload/src/odbc_source/_helpers/connect.rs index fb6d7485..4dd9c30e 100644 --- a/crates/sqlload/src/odbc_source/_helpers/connect.rs +++ b/crates/sqlload/src/odbc_source/_helpers/connect.rs @@ -1,12 +1,12 @@ //! Connections through the process's one ODBC environment. -use arrow_odbc::odbc_api::{Connection, ConnectionOptions, environment}; +use arrow_odbc::odbc_api::{Connection, ConnectionOptions, Cursor, environment}; -use crate::odbc_source::constants::{IDENTIFIER_QUOTE, NO_QUOTE}; +use crate::odbc_source::constants::{ANSI_QUOTES_PROBE, IDENTIFIER_QUOTE, NO_QUOTE}; use crate::odbc_source::errors::OdbcSourceError; use crate::odbc_source::models::OdbcReader; -/// A connection with the reader's connection string, refused when its driver quotes identifiers other than as ANSI SQL does, since SQLLoad writes `"name"`. +/// A connection with the reader's connection string, refused when its database doesn't read `"name"` as a name, since SQLLoad writes ANSI double quotes. A driver that reports another quote is asked once whether the database reads them anyway: MySQL Connector/ODBC reports a backtick even when the session's SQL mode includes `ANSI_QUOTES`. pub(crate) fn connect(reader: &OdbcReader) -> Result, OdbcSourceError> { let connection = environment()?.connect_with_connection_string( reader.connection_string.expose(), @@ -14,8 +14,17 @@ pub(crate) fn connect(reader: &OdbcReader) -> Result, OdbcSo )?; match connection.identifier_quote_char()? { None | Some(IDENTIFIER_QUOTE | NO_QUOTE) => Ok(connection), + Some(_) if reads_ansi_quotes(&connection) => Ok(connection), Some(quote) => Err(OdbcSourceError::Invalid(format!( - "the driver quotes identifiers with {quote:?}; SQLLoad's ODBC source writes ANSI double quotes, so switch the driver to them (for example ANSI_QUOTES)" + "the driver quotes identifiers with {quote:?}; SQLLoad's ODBC source writes ANSI double quotes, so switch the database to them (for MySQL, add INITSTMT=SET SESSION sql_mode=CONCAT(@@SESSION.sql_mode,',ANSI_QUOTES') to the connection string)" ))), } } + +/// Whether the database reads `"name"` as a column name: the probe returns its row only then (a string literal never equals 1). A failed probe counts as no. +fn reads_ansi_quotes(connection: &Connection<'_>) -> bool { + match connection.execute(ANSI_QUOTES_PROBE, (), None) { + Ok(Some(mut cursor)) => matches!(cursor.next_row(), Ok(Some(_))), + Ok(None) | Err(_) => false, + } +} diff --git a/crates/sqlload/src/odbc_source/_helpers/mod.rs b/crates/sqlload/src/odbc_source/_helpers/mod.rs index 8f1c09da..8c1fd09e 100644 --- a/crates/sqlload/src/odbc_source/_helpers/mod.rs +++ b/crates/sqlload/src/odbc_source/_helpers/mod.rs @@ -2,4 +2,5 @@ pub(crate) mod batches; pub(crate) mod catalog; pub(crate) mod connect; pub(crate) mod convert; +pub(crate) mod parameters; pub(crate) mod query; diff --git a/crates/sqlload/src/odbc_source/_helpers/parameters.rs b/crates/sqlload/src/odbc_source/_helpers/parameters.rs new file mode 100644 index 00000000..342edba1 --- /dev/null +++ b/crates/sqlload/src/odbc_source/_helpers/parameters.rs @@ -0,0 +1,89 @@ +//! Cursor bounds as the ODBC parameters a read binds. + +use arrow_odbc::odbc_api::handles::AsStatementRef; +use arrow_odbc::odbc_api::parameter::{InputParameter, WithDataType}; +use arrow_odbc::odbc_api::sys::{Date, Timestamp}; +use arrow_odbc::odbc_api::{DataType, IntoParameter, Prepared}; +use chrono::{Datelike, NaiveDateTime, Timelike}; + +use crate::adapters::contract::models::BoundParameter; +use crate::odbc_source::constants::{ + LEGACY_DATETIME_DIGITS, SQL_SERVER_DBMS, TIMESTAMP_PARAMETER_PRECISION, TIMESTAMP_TEXT_DIGITS, +}; + +/// `parameters` as `prepared` binds them, in order: text as `VARCHAR`; a date as `DATE`; a timestamp as `TIMESTAMP` with seven fractional digits, which SQL Server compares as `datetime2(7)` (three would make it a `datetime`, rounding milliseconds), except against a SQL Server column the driver describes with one to three fractional digits, a legacy `datetime`: that compares a `datetime2` parameter with the column's exact 1/300 s ticks, which the driver rounded to milliseconds when the bound was read, so the bound goes as text, which SQL Server converts to the column's own type; when the driver can't describe the parameter, as text with seven digits, exact for `datetime2` and refused by a `datetime`. +pub(crate) fn bound_parameters( + prepared: &mut Prepared, + dbms: &str, + parameters: &[BoundParameter], +) -> Vec> { + parameters + .iter() + .zip(1u16..) + .map(|(parameter, number)| match parameter { + BoundParameter::Timestamp(time) if dbms == SQL_SERVER_DBMS => { + match prepared.describe_param(number) { + Ok(described) => match described.data_type { + DataType::Timestamp { precision } + if LEGACY_DATETIME_DIGITS.contains(&precision) => + { + Box::new( + timestamp_text(time, precision.unsigned_abs()).into_parameter(), + ) + } + _ => odbc_parameter(parameter), + }, + Err(_) => { + Box::new(timestamp_text(time, TIMESTAMP_TEXT_DIGITS).into_parameter()) + } + } + } + _ => odbc_parameter(parameter), + }) + .collect() +} + +/// `time` as ISO 8601 `YYYY-MM-DDTHH:MM:SS`, which SQL Server reads alike under every `SET LANGUAGE` and `SET DATEFORMAT` (with a space it reads the month and day by the session's language), with `digits` fractional digits, or with six when it has more than `digits` (which the database then refuses rather than misreads). +pub(crate) fn timestamp_text(time: &NaiveDateTime, digits: u16) -> String { + let nanoseconds = time.nanosecond(); + let unit = 10u32.pow(9u32.saturating_sub(u32::from(digits))); + match (digits, nanoseconds % unit) { + (0, 0) => time.format("%Y-%m-%dT%H:%M:%S").to_string(), + (_, 0) => format!( + "{}.{}", + time.format("%Y-%m-%dT%H:%M:%S"), + &format!("{nanoseconds:09}")[..usize::from(digits)] + ), + _ => time.format("%Y-%m-%dT%H:%M:%S%.6f").to_string(), + } +} + +/// `parameter` bound as its own type. +fn odbc_parameter(parameter: &BoundParameter) -> Box { + let small = |value: u32| u16::try_from(value).unwrap_or(u16::MAX); + match parameter { + BoundParameter::Text(text) => Box::new(text.clone().into_parameter()), + BoundParameter::Timestamp(time) => Box::new(WithDataType::new( + Timestamp { + year: i16::try_from(time.year()).unwrap_or(i16::MAX), + month: small(time.month()), + day: small(time.day()), + hour: small(time.hour()), + minute: small(time.minute()), + second: small(time.second()), + fraction: time.nanosecond(), + }, + DataType::Timestamp { + precision: TIMESTAMP_PARAMETER_PRECISION, + }, + )), + BoundParameter::Date(date) => Box::new(WithDataType::new( + Date { + year: i16::try_from(date.year()).unwrap_or(i16::MAX), + month: small(date.month()), + day: small(date.day()), + }, + DataType::Date, + )), + } +} diff --git a/crates/sqlload/src/odbc_source/constants.rs b/crates/sqlload/src/odbc_source/constants.rs index a8a6cdec..6aefd164 100644 --- a/crates/sqlload/src/odbc_source/constants.rs +++ b/crates/sqlload/src/odbc_source/constants.rs @@ -10,6 +10,10 @@ pub const ANY_NAME: &str = "%"; /// The identifier quote SQLLoad writes, ANSI SQL's. pub const IDENTIFIER_QUOTE: char = '"'; +/// A query returning one row only when the database reads `"name"` as a name rather than a string. +pub const ANSI_QUOTES_PROBE: &str = + "SELECT 1 FROM (SELECT 1 AS sqlload_probe) sqlload_query WHERE \"sqlload_probe\" = 1"; + /// The driver's answer when it does not quote identifiers. pub const NO_QUOTE: char = ' '; @@ -21,3 +25,15 @@ pub const BATCHES_AHEAD: usize = 2; /// What the driver manager's error says when a value is longer than its buffer. pub const TOO_LARGE_VALUE: &str = "too large to be written into the allocated buffer"; + +/// The fractional digits a timestamp cursor bound is described with: seven, so SQL Server compares it as `datetime2(7)`, exactly, rather than as a `datetime`. +pub const TIMESTAMP_PARAMETER_PRECISION: i16 = 7; + +/// The name Microsoft's SQL Server drivers report for the database. +pub const SQL_SERVER_DBMS: &str = "Microsoft SQL Server"; + +/// The fractional digits SQL Server's driver describes a legacy `datetime` parameter with. +pub const LEGACY_DATETIME_DIGITS: std::ops::RangeInclusive = 1..=3; + +/// The fractional digits of a SQL Server timestamp bound sent as text when the driver can't describe its parameter: `datetime2`'s seven, which a legacy `datetime` refuses rather than rounds. +pub const TIMESTAMP_TEXT_DIGITS: u16 = 7; diff --git a/crates/sqlload/src/odbc_source/main/read_rows.rs b/crates/sqlload/src/odbc_source/main/read_rows.rs index a7260749..5b1b3c2f 100644 --- a/crates/sqlload/src/odbc_source/main/read_rows.rs +++ b/crates/sqlload/src/odbc_source/main/read_rows.rs @@ -1,14 +1,16 @@ //! Read a table or query through ODBC as Arrow batches. use arrow_array::RecordBatch; -use arrow_odbc::odbc_api::IntoParameter; use tokio::sync::mpsc::Sender; +use crate::adapters::contract::models::BoundParameter; use crate::odbc_source::_helpers::batches::{batch_reader, fetch_error}; use crate::odbc_source::_helpers::connect::connect; use crate::odbc_source::_helpers::convert::{converted, output_schema}; +use crate::odbc_source::_helpers::parameters::bound_parameters; use crate::odbc_source::_helpers::query::select_query; use crate::odbc_source::constants::BATCHES_AHEAD; + use crate::odbc_source::errors::OdbcSourceError; use crate::odbc_source::models::{OdbcReadRequest, OdbcReader}; @@ -40,20 +42,18 @@ pub async fn read_rows( Ok(rows) } -/// Fetch `query`'s rows, with `parameters` bound to its placeholders as text, into `sender` until they end, the receiver goes away, or a fetch fails (sent on as the error). Decimal columns are fetched as text, which `converted` parses without rounding, since drivers round values to the scale they describe (an unbounded `numeric` as `NUMERIC(28,6)`). +/// Fetch `query`'s rows, with `parameters` bound to its placeholders as `bound_parameters` binds them, into `sender` until they end, the receiver goes away, or a fetch fails (sent on as the error). Decimal columns are fetched as text, which `converted` parses without rounding, since drivers round values to the scale they describe (an unbounded `numeric` as `NUMERIC(28,6)`). fn fetch( reader: &OdbcReader, - (query, parameters): (&str, &[String]), + (query, parameters): (&str, &[BoundParameter]), batch_rows: usize, sender: &Sender>, ) -> Result<(), OdbcSourceError> { let connection = connect(reader)?; let dbms: String = connection.database_management_system_name()?; - let parameters: Vec<_> = parameters - .iter() - .map(|value| value.as_str().into_parameter()) - .collect(); - let Some(cursor) = connection.execute(query, parameters.as_slice(), None)? else { + let mut prepared = connection.prepare(query)?; + let parameters = bound_parameters(&mut prepared, &dbms, parameters); + let Some(cursor) = prepared.execute(parameters.as_slice())? else { return Err(OdbcSourceError::Invalid( "the read returned no result set".to_owned(), )); diff --git a/crates/sqlload/src/odbc_source/models.rs b/crates/sqlload/src/odbc_source/models.rs index 9b6461b5..af1b086c 100644 --- a/crates/sqlload/src/odbc_source/models.rs +++ b/crates/sqlload/src/odbc_source/models.rs @@ -1,3 +1,4 @@ +use crate::adapters::contract::models::BoundParameter; use crate::auth::models::Secret; use crate::postgres_source::models::{ReadColumn, SourceTable}; @@ -24,7 +25,7 @@ pub struct OdbcReadRequest<'a> { pub columns: &'a [ReadColumn], /// A condition on the source's columns, already rendered. pub filter: Option<&'a str>, - /// The values bound to `filter`'s `?` placeholders, in order, as text. - pub parameters: &'a [String], + /// The values bound to `filter`'s `?` placeholders, in order, each as its type. + pub parameters: &'a [BoundParameter], pub batch_rows: usize, } diff --git a/crates/sqlload/src/odbc_source/tests.rs b/crates/sqlload/src/odbc_source/tests.rs index 5c072639..d238ac02 100644 --- a/crates/sqlload/src/odbc_source/tests.rs +++ b/crates/sqlload/src/odbc_source/tests.rs @@ -8,3 +8,5 @@ mod odbc_reader; mod select_query; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_timestamp_text.rs"] +mod timestamp_text; diff --git a/crates/sqlload/src/odbc_source/tests/test_timestamp_text.rs b/crates/sqlload/src/odbc_source/tests/test_timestamp_text.rs new file mode 100644 index 00000000..35f87014 --- /dev/null +++ b/crates/sqlload/src/odbc_source/tests/test_timestamp_text.rs @@ -0,0 +1,56 @@ +use crate::odbc_source::_helpers::parameters::timestamp_text; +use crate::odbc_source::tests::test_types::TimestampTextTestCase; + +#[test] +fn given_a_timestamp_bound_for_a_legacy_sql_server_datetime_when_writing_it_as_text_then_it_has_the_column_s_digits_unless_that_would_cut_it() +-> Result<(), String> { + let test_cases = [ + TimestampTextTestCase { + description: "a millisecond bound keeps its three digits, which datetime converts to the tick it read", + time: "2026-01-31 23:59:59.123", + digits: 3, + expected_text: "2026-01-31T23:59:59.123", + }, + TimestampTextTestCase { + description: "a whole second still has three digits", + time: "2026-02-01 00:00:00", + digits: 3, + expected_text: "2026-02-01T00:00:00.000", + }, + TimestampTextTestCase { + description: "a leading zero in the fraction is kept", + time: "2026-01-31 23:59:59.007", + digits: 3, + expected_text: "2026-01-31T23:59:59.007", + }, + TimestampTextTestCase { + description: "a microsecond bound keeps six digits rather than being cut, so the database refuses it", + time: "2026-01-31 23:59:59.123456", + digits: 3, + expected_text: "2026-01-31T23:59:59.123456", + }, + TimestampTextTestCase { + description: "seven digits, as a bound the driver can't describe goes", + time: "2026-01-31 23:59:59.123456", + digits: 7, + expected_text: "2026-01-31T23:59:59.1234560", + }, + TimestampTextTestCase { + description: "no digits for a whole second", + time: "2026-01-31 23:59:00", + digits: 0, + expected_text: "2026-01-31T23:59:00", + }, + ]; + for test_case in test_cases { + let time = chrono::NaiveDateTime::parse_from_str(test_case.time, "%Y-%m-%d %H:%M:%S%.f") + .map_err(|error| error.to_string())?; + assert_eq!( + timestamp_text(&time, test_case.digits), + test_case.expected_text, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlload/src/odbc_source/tests/test_types.rs b/crates/sqlload/src/odbc_source/tests/test_types.rs index 45b30525..e87cd927 100644 --- a/crates/sqlload/src/odbc_source/tests/test_types.rs +++ b/crates/sqlload/src/odbc_source/tests/test_types.rs @@ -23,3 +23,10 @@ pub(crate) struct OdbcReaderTestCase { pub(crate) password: Option<&'static str>, pub(crate) expected_connection_string: &'static str, } + +pub(crate) struct TimestampTextTestCase { + pub(crate) description: &'static str, + pub(crate) time: &'static str, + pub(crate) digits: u16, + pub(crate) expected_text: &'static str, +} diff --git a/crates/sqlload/tests/loader.rs b/crates/sqlload/tests/loader.rs index 00275ecb..7bc53f48 100644 --- a/crates/sqlload/tests/loader.rs +++ b/crates/sqlload/tests/loader.rs @@ -44,6 +44,9 @@ mod test_loader_edge_cases; #[path = "loader/test_mixed_cursors.rs"] mod test_mixed_cursors; #[cfg(feature = "odbc")] +#[path = "loader/test_odbc_drivers.rs"] +mod test_odbc_drivers; +#[cfg(feature = "odbc")] #[path = "loader/test_odbc_to_duckdb.rs"] mod test_odbc_to_duckdb; #[path = "loader/test_postgres_cdc.rs"] diff --git a/crates/sqlload/tests/loader/odbc/helpers.rs b/crates/sqlload/tests/loader/odbc/helpers.rs index 9be37c6a..1aa2508b 100644 --- a/crates/sqlload/tests/loader/odbc/helpers.rs +++ b/crates/sqlload/tests/loader/odbc/helpers.rs @@ -66,9 +66,67 @@ cursor_column = "label" write_strategy = "append" [sources.erp.tables."erp.codes"] -description = "Codes; the first maximum holds a backslash before a quote." +description = "Codes; the first maximum holds a backslash before a quote. Pinned as text, which still binds." cursor_column = "label" write_strategy = "append" +columns = {{ label = "text" }} +"#, + database = database.display(), + ) +} + +/// Run each of `statements` through ODBC, one at a time. +pub(crate) fn odbc_execute(connection_string: &str, statements: &[&str]) -> Result<(), String> { + let connection = arrow_odbc::odbc_api::environment() + .map_err(|error| error.to_string())? + .connect_with_connection_string( + connection_string, + arrow_odbc::odbc_api::ConnectionOptions::default(), + ) + .map_err(|error| format!("cannot connect: {error}"))?; + statements.iter().try_for_each(|statement| { + connection + .execute(statement, (), None) + .map(|_| ()) + .map_err(|error| format!("{statement}: {error}")) + }) +} + +/// A project reading an events table appended by its millisecond `happened_at` cursor and a days table replaced from its last `opened` day, through `connection_string`, into DuckDB. +pub(crate) fn driver_project( + connection_string: &str, + (events, days): (&str, &str), + database: &std::path::Path, +) -> String { + format!( + r#" +[connections.erp] +kind = "odbc" +connection_string = "{connection_string}" + +[connections.local] +kind = "duckdb" +database = '{database}' + +[targets.dev] +connection = "local" +loader_schema = "raw" + +[sources.erp] +kind = "odbc" +connection = "erp" +description = "Events and days of a generic ERP." +include = ["{events}", "{days}"] + +[sources.erp.tables."{events}"] +description = "Events, read after the newest happened_at." +cursor_column = "happened_at" +write_strategy = "append" + +[sources.erp.tables."{days}"] +description = "Days, the last one read again." +cursor_column = "opened" +write_strategy = "delete_insert" "#, database = database.display(), ) diff --git a/crates/sqlload/tests/loader/test_odbc_drivers.rs b/crates/sqlload/tests/loader/test_odbc_drivers.rs new file mode 100644 index 00000000..073cfa2b --- /dev/null +++ b/crates/sqlload/tests/loader/test_odbc_drivers.rs @@ -0,0 +1,140 @@ +use sqlload_testkit::odbc_servers::main::start_odbc_server::start_odbc_server; +use sqlload_testkit::odbc_servers::models::OdbcDatabase; + +use crate::helpers::{duckdb_text, run_async, run_sqlload}; +use crate::odbc_helpers::{driver_project, odbc_execute}; +use crate::test_types::OdbcDriverTestCase; + +const TABLES: &str = "SELECT (SELECT string_agg(id::VARCHAR, ',' ORDER BY id) || ' to ' || strftime(max(happened_at), '%Y-%m-%d %H:%M:%S.%g') FROM raw.erp__events) || ' | ' || (SELECT string_agg(id::VARCHAR, ',' ORDER BY id) || ' to ' || max(opened)::VARCHAR FROM raw.erp__days)"; + +const FIRST: &str = "1,2 to 2026-01-31 23:59:59.997 | 1,2 to 2026-01-31"; + +const SECOND: &str = "1,2,3,4 to 2026-02-01 00:00:00.000 | 1,2,3,4 to 2026-02-01"; + +const LEGACY_FIRST: &str = "1,2 to 2026-01-05 23:59:59.123 | 1,2 to 2026-01-31"; + +#[test] +fn given_each_odbc_driver_when_a_millisecond_timestamp_cursor_and_a_date_cursor_resume_then_the_next_load_reads_exactly_the_rows_after_each_boundary() +-> Result<(), String> { + let test_cases = [ + OdbcDriverTestCase { + description: "SQL Server through Microsoft's ODBC Driver 18: millisecond and date cursors on datetime2(3) and date", + connection_options: "", + database: OdbcDatabase::SqlServer, + tables: ("erp.events", "erp.days"), + setup: &[ + "CREATE SCHEMA erp", + "CREATE TABLE erp.events (id int PRIMARY KEY, name nvarchar(40), happened_at datetime2(3) NOT NULL)", + "CREATE TABLE erp.days (id int PRIMARY KEY, opened date NOT NULL)", + "INSERT INTO erp.events VALUES (1, 'a', '2026-01-31 10:00:00'), (2, 'b', '2026-01-31 23:59:59.997')", + "INSERT INTO erp.days VALUES (1, '2026-01-30'), (2, '2026-01-31')", + ], + change: &[ + "INSERT INTO erp.events VALUES (0, 'early', '2026-01-31 23:59:59.996'), (3, 'next', '2026-01-31 23:59:59.998'), (4, 'later', '2026-02-01 00:00:00')", + "INSERT INTO erp.days VALUES (3, '2026-01-31'), (4, '2026-02-01')", + ], + expected_tables: (FIRST, SECOND), + }, + OdbcDriverTestCase { + description: "SQL Server's legacy datetime, whose 1/300 s ticks the driver reads rounded to milliseconds (.123 is .12333): a datetime2 bound would read the .123 row again, so it is bound as text in the column's own type, in ISO 8601 form, which a british session (day before month) reads alike: 2026-01-05 with a space would be 1 May", + connection_options: "Language=British;", + database: OdbcDatabase::SqlServer, + tables: ("erp.events", "erp.days"), + setup: &[ + "CREATE SCHEMA erp", + "CREATE TABLE erp.events (id int PRIMARY KEY, name nvarchar(40), happened_at datetime NOT NULL)", + "CREATE TABLE erp.days (id int PRIMARY KEY, opened date NOT NULL)", + "INSERT INTO erp.events VALUES (1, 'a', '2026-01-05 10:00:00'), (2, 'b', '2026-01-05 23:59:59.123')", + "INSERT INTO erp.days VALUES (1, '2026-01-30'), (2, '2026-01-31')", + ], + change: &[ + "INSERT INTO erp.events VALUES (0, 'early', '2026-01-05 23:59:59.120'), (3, 'next', '2026-01-05 23:59:59.127'), (4, 'later', '2026-02-01 00:00:00')", + "INSERT INTO erp.days VALUES (3, '2026-01-31'), (4, '2026-02-01')", + ], + expected_tables: (LEGACY_FIRST, SECOND), + }, + OdbcDriverTestCase { + description: "MySQL through Connector/ODBC, a driver without schemas, in an ANSI_QUOTES session: millisecond and date cursors on DATETIME(3) and DATE", + connection_options: "", + database: OdbcDatabase::MySql, + tables: ("events", "days"), + setup: &[ + "CREATE TABLE events (id int PRIMARY KEY, name varchar(40), happened_at DATETIME(3) NOT NULL)", + "CREATE TABLE days (id int PRIMARY KEY, opened DATE NOT NULL)", + "INSERT INTO events VALUES (1, 'a', '2026-01-31 10:00:00'), (2, 'b', '2026-01-31 23:59:59.997')", + "INSERT INTO days VALUES (1, '2026-01-30'), (2, '2026-01-31')", + ], + change: &[ + "INSERT INTO events VALUES (0, 'early', '2026-01-31 23:59:59.996'), (3, 'next', '2026-01-31 23:59:59.998'), (4, 'later', '2026-02-01 00:00:00')", + "INSERT INTO days VALUES (3, '2026-01-31'), (4, '2026-02-01')", + ], + expected_tables: (FIRST, SECOND), + }, + OdbcDriverTestCase { + description: "Postgres through psqlODBC: millisecond and date cursors on timestamp(3) and date", + connection_options: "", + database: OdbcDatabase::Postgres, + tables: ("erp.events", "erp.days"), + setup: &[ + "CREATE SCHEMA erp", + "CREATE TABLE erp.events (id int PRIMARY KEY, name varchar(40), happened_at timestamp(3) NOT NULL)", + "CREATE TABLE erp.days (id int PRIMARY KEY, opened date NOT NULL)", + "INSERT INTO erp.events VALUES (1, 'a', '2026-01-31 10:00:00'), (2, 'b', '2026-01-31 23:59:59.997')", + "INSERT INTO erp.days VALUES (1, '2026-01-30'), (2, '2026-01-31')", + ], + change: &[ + "INSERT INTO erp.events VALUES (0, 'early', '2026-01-31 23:59:59.996'), (3, 'next', '2026-01-31 23:59:59.998'), (4, 'later', '2026-02-01 00:00:00')", + "INSERT INTO erp.days VALUES (3, '2026-01-31'), (4, '2026-02-01')", + ], + expected_tables: (FIRST, SECOND), + }, + ]; + for test_case in test_cases { + run_async(async { + let server = start_odbc_server(test_case.database).await?; + odbc_execute(&server.connection_string, test_case.setup)?; + let folder = tempfile::tempdir().map_err(|error| error.to_string())?; + let database = folder.path().join("out.duckdb"); + let config = folder.path().join("sqlload.toml"); + std::fs::write( + &config, + driver_project( + &format!( + "{}{}", + server.connection_string, test_case.connection_options + ), + test_case.tables, + &database, + ), + ) + .map_err(|error| error.to_string())?; + let (succeeded, output) = run_sqlload(&config, &["load", "--yes"], ("", ""))?; + assert!( + succeeded && output.contains("2 loaded"), + "{}, first load:\n{output}", + test_case.description + ); + assert_eq!( + duckdb_text(&database, TABLES)?, + test_case.expected_tables.0, + "{}, first load", + test_case.description + ); + odbc_execute(&server.connection_string, test_case.change)?; + let (succeeded, output) = run_sqlload(&config, &["load", "--yes"], ("", ""))?; + assert!( + succeeded && output.contains("2 loaded"), + "{}, second load:\n{output}", + test_case.description + ); + assert_eq!( + duckdb_text(&database, TABLES)?, + test_case.expected_tables.1, + "{}, second load: the event at .996 is before the maximum and the day 2026-01-31 is read again", + test_case.description + ); + Ok(()) + })?; + } + Ok(()) +} diff --git a/crates/sqlload/tests/loader/test_types.rs b/crates/sqlload/tests/loader/test_types.rs index 8f662aeb..feb39e55 100644 --- a/crates/sqlload/tests/loader/test_types.rs +++ b/crates/sqlload/tests/loader/test_types.rs @@ -967,6 +967,23 @@ pub(crate) struct SftpSnowflakePlanTestCase { pub(crate) expected_output: &'static str, } +/// One database's run of the ODBC cursor scenario. +#[cfg(feature = "odbc")] +pub(crate) struct OdbcDriverTestCase { + pub(crate) description: &'static str, + pub(crate) database: sqlload_testkit::odbc_servers::models::OdbcDatabase, + /// Appended to the connection string SQLLoad reads through. + pub(crate) connection_options: &'static str, + /// The events and days tables as the source's `include` names them. + pub(crate) tables: (&'static str, &'static str), + /// Statements creating both tables and their first rows. + pub(crate) setup: &'static [&'static str], + /// Statements adding rows around each cursor's maximum before the second load. + pub(crate) change: &'static [&'static str], + /// Each table's ids and newest cursor value after the first load, then after the second. + pub(crate) expected_tables: (&'static str, &'static str), +} + pub(crate) struct SheetsUserTestCase { pub(crate) description: &'static str, /// The connection's `refresh_token`. diff --git a/fensu.toml b/fensu.toml index 57e9ef3a..4bf2d6ed 100644 --- a/fensu.toml +++ b/fensu.toml @@ -41,6 +41,11 @@ rule = "FPRSL105" path = "crates/sqlload-testkit/src/parity/main/run_cases.rs" reason = "Testkit entry used by sqlload's integration tests, which live outside the analysed crate tree." +[[targets.rust.rule_exceptions]] +rule = "FPRSL105" +path = "crates/sqlload-testkit/src/odbc_servers/main/start_odbc_server.rs" +reason = "Testkit entry used by sqlload's integration tests, which live outside the analysed crate tree." + [[targets.rust.rule_exceptions]] rule = "FPRSL105" path = "crates/sqlload-testkit/src/postgres/main/start_postgres_tls.rs" @@ -146,16 +151,6 @@ rule = "FPRSL105" path = "crates/sqlload-testkit/src/harness/main/read_metrics.rs" reason = "Testkit entry used by sqlload's integration tests, which live outside the analysed crate tree." -[[targets.rust.rule_exceptions]] -rule = "FPRSL105" -path = "crates/sqlload-testkit/src/postgres/main/connect.rs" -reason = "Testkit entry used by sqlload's integration tests, which live outside the analysed crate tree." - -[[targets.rust.rule_exceptions]] -rule = "FPRSL105" -path = "crates/sqlload-testkit/src/postgres/main/start_postgres.rs" -reason = "Testkit entry used by sqlload's integration tests, which live outside the analysed crate tree." - [[targets.rust.rule_exceptions]] rule = "FPRSL105" path = "crates/sqlload-testkit/src/seaweedfs/main/create_bucket.rs" diff --git a/packages/sqlload-cli/licenses/unixODBC-COPYING.txt b/packages/sqlload-cli/licenses/unixODBC-COPYING.txt new file mode 100644 index 00000000..2c012a8c --- /dev/null +++ b/packages/sqlload-cli/licenses/unixODBC-COPYING.txt @@ -0,0 +1,458 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS diff --git a/packages/sqlload-cli/licenses/unixODBC-NOTICE.md b/packages/sqlload-cli/licenses/unixODBC-NOTICE.md new file mode 100644 index 00000000..cf99693a --- /dev/null +++ b/packages/sqlload-cli/licenses/unixODBC-NOTICE.md @@ -0,0 +1,22 @@ +# unixODBC + +The Linux and macOS wheels of this package include the unixODBC driver manager library +(`libodbc`), version 2.3.14, unmodified. Windows wheels include no unixODBC; they use the +driver manager built into Windows (`odbc32.dll`). + +- Licence: GNU Lesser General Public License, version 2.1; its text is in `unixODBC-COPYING.txt` + next to this file. +- Corresponding source: `unixODBC-2.3.14.tar.gz`, SHA-256 + 4e2814de3e01fc30b0b9f75e83bb5aba91ab0384ee951286504bb70205524771, is included in this package's + source distribution (sdist) of the same version on PyPI, at its top level. It is unixODBC's own + release, also at https://www.unixodbc.org/unixODBC-2.3.14.tar.gz and + https://github.com/lurcher/unixODBC/releases/tag/v2.3.14. +- Built by `.github/scripts/build_unixodbc.sh` in https://github.com/sqlstack-io/sqlload, with + `--with-included-ltdl`, the configuration folder set to `/etc` on Linux and Homebrew's `etc` on + macOS, and drivers registered by file name alone looked for in + `/usr/lib/-linux-gnu/odbc`, `/usr/lib/-linux-gnu`, `/usr/lib64/odbc` and `/usr/lib64` + on Linux and Homebrew's `lib` on macOS. + +SQLLoad links the library dynamically. It is installed as a separate shared library next to the +package (`*.libs/libodbc-*.so.2` on Linux, `*.dylibs/libodbc-*.2.dylib` on macOS); you may +replace that file with another build of unixODBC 2.3 with the same interface. diff --git a/packages/sqlload-cli/pyproject.toml b/packages/sqlload-cli/pyproject.toml index bed8f952..13106460 100644 --- a/packages/sqlload-cli/pyproject.toml +++ b/packages/sqlload-cli/pyproject.toml @@ -9,6 +9,7 @@ description = "Native executable for sqlload." readme = "README.md" requires-python = ">=3.10" license = "Apache-2.0" +license-files = ["licenses/*"] classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", @@ -25,7 +26,12 @@ Repository = "https://github.com/sqlstack-io/sqlload" bindings = "bin" manifest-path = "../../crates/sqlload/Cargo.toml" # Kafka TLS with OpenSSL built in; SASL PLAIN and SCRAM need nothing else. GSSAPI is left out. +# The published wheels also build `odbc` (publish.yml's --features odbc), bundling unixODBC on Linux +# and macOS (licenses/); a build from this sdist leaves it out, so it needs no driver manager. features = ["ssl-vendored"] +# unixODBC's source, which the release sdist carries as the corresponding source of the library the +# Linux and macOS wheels bundle; publish.yml fetches it (unixodbc_source.sh) and checks it is there. +include = [{ path = "unixODBC-*.tar.gz", format = "sdist" }] [tool.uv] cache-keys = [ diff --git a/packages/sqlload/README.md b/packages/sqlload/README.md index 4538537a..de3ae539 100644 --- a/packages/sqlload/README.md +++ b/packages/sqlload/README.md @@ -92,7 +92,7 @@ versions and how to run them yourself. | HTTP APIs and Python | ✓ | | | ODBC databases¹ | ✓ | | -¹ In builds with the `odbc` feature. +¹ In the published wheels from 0.38.0 (not released yet); before that, in builds with the `odbc` feature. ² From 0.34.0, through a separate DuckDB 1.5 process: install `sqlload[motherduck]`. diff --git a/packages/sqlload/licenses/unixODBC-COPYING.txt b/packages/sqlload/licenses/unixODBC-COPYING.txt new file mode 100644 index 00000000..2c012a8c --- /dev/null +++ b/packages/sqlload/licenses/unixODBC-COPYING.txt @@ -0,0 +1,458 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS diff --git a/packages/sqlload/licenses/unixODBC-NOTICE.md b/packages/sqlload/licenses/unixODBC-NOTICE.md new file mode 100644 index 00000000..cf99693a --- /dev/null +++ b/packages/sqlload/licenses/unixODBC-NOTICE.md @@ -0,0 +1,22 @@ +# unixODBC + +The Linux and macOS wheels of this package include the unixODBC driver manager library +(`libodbc`), version 2.3.14, unmodified. Windows wheels include no unixODBC; they use the +driver manager built into Windows (`odbc32.dll`). + +- Licence: GNU Lesser General Public License, version 2.1; its text is in `unixODBC-COPYING.txt` + next to this file. +- Corresponding source: `unixODBC-2.3.14.tar.gz`, SHA-256 + 4e2814de3e01fc30b0b9f75e83bb5aba91ab0384ee951286504bb70205524771, is included in this package's + source distribution (sdist) of the same version on PyPI, at its top level. It is unixODBC's own + release, also at https://www.unixodbc.org/unixODBC-2.3.14.tar.gz and + https://github.com/lurcher/unixODBC/releases/tag/v2.3.14. +- Built by `.github/scripts/build_unixodbc.sh` in https://github.com/sqlstack-io/sqlload, with + `--with-included-ltdl`, the configuration folder set to `/etc` on Linux and Homebrew's `etc` on + macOS, and drivers registered by file name alone looked for in + `/usr/lib/-linux-gnu/odbc`, `/usr/lib/-linux-gnu`, `/usr/lib64/odbc` and `/usr/lib64` + on Linux and Homebrew's `lib` on macOS. + +SQLLoad links the library dynamically. It is installed as a separate shared library next to the +package (`*.libs/libodbc-*.so.2` on Linux, `*.dylibs/libodbc-*.2.dylib` on macOS); you may +replace that file with another build of unixODBC 2.3 with the same interface. diff --git a/packages/sqlload/pyproject.toml b/packages/sqlload/pyproject.toml index a2d48359..5bacb312 100644 --- a/packages/sqlload/pyproject.toml +++ b/packages/sqlload/pyproject.toml @@ -9,7 +9,7 @@ description = "SQL data loading tooling." readme = "README.md" requires-python = ">=3.10" license = "Apache-2.0" -license-files = ["LICENSE"] +license-files = ["LICENSE", "licenses/*"] classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", @@ -39,7 +39,12 @@ manifest-path = "../../crates/sqlload-python/Cargo.toml" module-name = "sqlload._native" python-source = "python" # Kafka TLS with OpenSSL built in, as in sqlload-cli; SASL PLAIN and SCRAM need nothing else. +# The published wheels also build `odbc` (publish.yml's --features odbc), bundling unixODBC on Linux +# and macOS (licenses/); a build from this sdist leaves it out, so it needs no driver manager. features = ["ssl-vendored"] +# unixODBC's source, which the release sdist carries as the corresponding source of the library the +# Linux and macOS wheels bundle; publish.yml fetches it (unixodbc_source.sh) and checks it is there. +include = [{ path = "unixODBC-*.tar.gz", format = "sdist" }] [tool.uv] cache-keys = [ diff --git a/website/scripts/check-content.mjs b/website/scripts/check-content.mjs index 0e7e79b3..21fdf0cd 100644 --- a/website/scripts/check-content.mjs +++ b/website/scripts/check-content.mjs @@ -22,7 +22,9 @@ for (const page of pages) { const slug = relative(docs, page).replace(/\.mdx$/, '').replace(/\/index$/, ''); if (!listed.has(slug)) problems.push(`${relative(website, page)}: not in src/sidebar.json`); } -for (const path of [...pages, join(website, 'public/index.html')]) { +// The ODBC source page names the drivers SQLLoad is tested with, which reach these systems through ODBC. +const odbcPage = join(docs, 'docs/sources/odbc.mdx'); +for (const path of [...pages.filter((page) => page !== odbcPage), join(website, 'public/index.html')]) { readFileSync(path, 'utf8') .split('\n') .forEach((line, index) => { diff --git a/website/src/content/docs/docs/index.mdx b/website/src/content/docs/docs/index.mdx index 48b24b18..84cf6ff1 100644 --- a/website/src/content/docs/docs/index.mdx +++ b/website/src/content/docs/docs/index.mdx @@ -22,7 +22,7 @@ SQLLoad loads data into your warehouse, and shows you how before it changes anyt | HTTP APIs and Python | ✓ | | | ODBC databases¹ | ✓ | | -¹ In builds with the `odbc` feature. +¹ In the published wheels from 0.38.0 (not released yet); before that, in builds with the `odbc` feature. ² From 0.34.0, through a separate DuckDB 1.5 process: install `sqlload[motherduck]`. diff --git a/website/src/content/docs/docs/sources/odbc.mdx b/website/src/content/docs/docs/sources/odbc.mdx index c08ab95e..f7ff77c3 100644 --- a/website/src/content/docs/docs/sources/odbc.mdx +++ b/website/src/content/docs/docs/sources/odbc.mdx @@ -7,9 +7,10 @@ From sqlload 0.27.0, an `odbc` source loads tables, views and query results from any database that has an ODBC driver. SQLLoad reads the rows as Arrow through the driver and loads them into the source's target. -ODBC support is built only with the `odbc` feature. The released packages don't include it yet; -build SQLLoad with `cargo install sqlload --features odbc`. Without it, `plan` refuses an `odbc` -source. +From 0.38.0 (not released yet), the `sqlload` and `sqlload-cli` wheels include ODBC on every +platform. Before that, and in a build from source (`cargo install`, or pip building an sdist where +no wheel fits), ODBC is built only with the `odbc` feature (`cargo install sqlload --features odbc`); +without it, `plan` refuses an `odbc` source. ```toml [connections.erp] @@ -70,31 +71,57 @@ Loads (2) sqlload load | `dsn` | A data source name from `odbc.ini`, instead of `connection_string`. It and `user` can't contain `;`, `{` or `}`. | | `user`, `password` | With `dsn`, when the DSN doesn't hold them. | -Set `connection_string` or `dsn`, not both. SQLLoad uses unixODBC on Linux and macOS, and the -driver manager built into Windows. The driver and the driver manager must be installed on the -machine SQLLoad runs on; on Debian and Ubuntu, `unixodbc-dev` (which `cargo install` links -against) plus the driver's package, such as `odbc-postgresql`. +Set `connection_string` or `dsn`, not both. Install the database's ODBC driver on the machine +SQLLoad runs on; SQLLoad reaches it through a driver manager: + +- **Windows:** the driver manager built into Windows; register drivers and DSNs as usual (ODBC Data + Sources). +- **Linux and macOS wheels** (from 0.38.0, not released yet): the unixODBC 2.3.14 driver manager, + bundled as a separate shared library, so no system unixODBC is needed. It reads drivers and + system DSNs from `odbcinst.ini` and `odbc.ini` in `/etc` on Linux, where distribution driver + packages register, and in Homebrew's `etc` on macOS (`/opt/homebrew/etc` on Apple silicon, + `/usr/local/etc` on Intel), or in the folder `ODBCSYSINI` names; user DSNs from `~/.odbc.ini`. + A driver registered by file name alone, as Debian and Ubuntu packages register theirs + (`Driver=psqlodbcw.so`), is looked for in `/usr/lib/-linux-gnu/odbc` and + `/usr/lib/-linux-gnu` (then `/usr/lib64/odbc` and `/usr/lib64`) on Linux, and Homebrew's + `lib` on macOS. unixODBC is LGPL-2.1: its notice and licence are in each wheel's + `.dist-info/licenses`, and its source is in the package's sdist on PyPI. +- **`cargo install --features odbc`:** the system's unixODBC, which it links against: `unixodbc-dev` + on Debian and Ubuntu, `unixodbc` with Homebrew. ## Drivers -SQLLoad writes names in ANSI double quotes (`"sales"."items"`). A driver that quotes them another -way is refused when the source is read; switch such a database to ANSI quoting, for example -with an `ANSI_QUOTES` SQL mode. +SQLLoad writes names in ANSI double quotes (`"sales"."items"`). A database that doesn't read them +as names is refused when the source is read; switch it to ANSI quoting. For MySQL, add +`INITSTMT=SET SESSION sql_mode=CONCAT(@@SESSION.sql_mode,',ANSI_QUOTES');` to the connection string. | Driver | Tested | |--------|--------| -| psqlODBC (`PostgreSQL Unicode`) | Yes, in containers. Add `BoolsAsChar=0` to the connection string so `boolean` columns load as `bool` rather than text. | +| psqlODBC (`PostgreSQL Unicode`) | Yes, against Postgres 17 in containers on Linux and on Windows. Add `BoolsAsChar=0` to the connection string so `boolean` columns load as `bool` rather than text. | +| Microsoft ODBC Driver 18 for SQL Server | Yes, against SQL Server 2022 in containers on Linux and against SQL Server LocalDB on Windows, `datetime2` and legacy `datetime` cursors included. | +| MySQL Connector/ODBC (`MySQL ODBC Unicode Driver`) | Yes, against MySQL 8.4 in containers on Linux and MySQL 8.4 on Windows, in an `ANSI_QUOTES` session (above). On Linux, install MySQL's package for your distribution rather than the generic glibc tarball: the tarball's own OpenSSL fails to load once another driver, such as SQL Server's or psqlODBC, has loaded the system one in the same run. | +| OpenEdge (Progress) | Tested for timestamp handling (parsing `{ts}` values and binding timestamp bounds), not against a live server. | | Others | Not yet. A driver needs to describe prepared statements and list tables; run `sqlload plan` to see the types it reports. | -A cursor filter compares the cursor column with a quoted literal, such as -`"updated_at" >= '2026-03-01 10:00:05.123456'`, which the database converts to the column's type. -Postgres does; check that yours does too before relying on a cursor. From 0.30.0 (not released -yet), a numeric cursor compares with a bare number, such as `"id" > 41`. From 0.30.0 (not released -yet), a text cursor's bounds are bound parameters, such as `"code" > ?`, so the database's own -string-literal rules can't change them. Before that, on a database that reads a backslash in a -string as an escape, a text cursor maximum holding a backslash made -every later incremental load of the table fail, and other escapes could skip rows or read them -again. +From 0.38.0 (not released yet), a cursor filter compares the cursor column with bound parameters, +such as `"updated_at" > ?`, each passed as the cursor's own type: a timestamp without a time zone as +an ODBC `TIMESTAMP`, with every fractional digit; a date as a `DATE`; text as text. No database's +literal rules can change a bound, and a millisecond maximum such as `23:59:59.997` resumes exactly. +SQL Server's legacy `datetime` is the one exception: it stores 1/300 s ticks, which its driver reads +rounded to milliseconds, so its bounds go as ISO 8601 text (`2026-01-31T23:59:59.997`) in the +column's own precision, which SQL Server converts to the column's type alike under every login +language. A cursor column pinned to a type (`columns = { ... }`) compares with a quoted literal, as +the pinned type may not be the column's own, such as a `varchar` read as a timestamp. A numeric +cursor compares with a bare number, such as `"id" > 41`. +Any other cursor, a timestamp with a time zone for one, still compares with a quoted literal, +which the database converts to the column's type; check that yours does before relying on it. +For a timestamp or date cursor, `--start-cursor`, `--end-cursor` and `cursor_start` also take ODBC +escape values, such as +`{ts '2026-01-31 23:59:59.997'}` or `{d '2026-01-31'}`. + +From 0.38.0 (not released yet), a timestamp column is read in at least microseconds whatever precision +the driver describes: MySQL Connector/ODBC describes every `DATETIME(n)` with none, and its fractional +seconds were dropped before. ## Choosing tables diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx index 12783e6b..f73e8330 100644 --- a/website/src/content/docs/docs/supported-systems.mdx +++ b/website/src/content/docs/docs/supported-systems.mdx @@ -10,7 +10,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes | Postgres | Tables, by `include` patterns | Binary `COPY` into Arrow, over TLS when the connection's `sslmode` asks | | Postgres changes (`postgres_cdc`) | Inserts, updates and deletes of tables, by `include` patterns | A snapshot, then a logical replication slot (`pgoutput`), in whole transactions. See [Postgres changes](/docs/sources/postgres-cdc/). | | Snowflake (from 0.26.0) | Tables and views, by `include` patterns, and query results | Parquet unloaded with `COPY INTO` to the user stage or an S3 stage, then read by SQLLoad, or (from 0.27.0) by DuckDB itself from an S3 stage. See [Snowflake](/docs/sources/snowflake/). | -| Databases with an ODBC driver (from 0.27.0) | Tables and views, by `include` patterns, and query results | The driver, as Arrow, in builds with the `odbc` feature. See [ODBC](/docs/sources/odbc/). | +| Databases with an ODBC driver (from 0.27.0) | Tables and views, by `include` patterns, and query results | The driver, as Arrow; in the published wheels from 0.38.0 (not released yet), before that in builds with the `odbc` feature. See [ODBC](/docs/sources/odbc/). | | BigQuery (from 0.30.0) | Tables and views, by `include` patterns, and query results | The Storage Read API, as Arrow in parallel streams; views and queries through a query job. See [BigQuery](/docs/sources/bigquery/). | | Files in S3 | Parquet, CSV, JSON, NDJSON; gzip or zstd for text formats | The destination itself where it can load them exactly, otherwise SQLLoad's reader | | Files in a local folder | As for S3 | As for S3 |