feat(odbc): ODBC in the release wheels, typed cursor bounds, live driver tests on Linux and Windows

.github/scripts/assert_sdist_sources.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Asserts every sdist in DIR carries unixODBC's pinned source release, unchanged: the corresponding
+# source of the driver manager the Linux and macOS wheels bundle (version and SHA-256 from
+# unixodbc_source.sh).
+# Usage: assert_sdist_sources.sh DIR
+set -euo pipefail
+
+pins="$(dirname "$0")/unixodbc_source.sh"
+version="$(sed -n 's/^UNIXODBC_VERSION=//p' "$pins")"
+sha256="$(sed -n 's/^UNIXODBC_SHA256=//p' "$pins")"
+name="unixODBC-${version}.tar.gz"
+
+shopt -s nullglob
+sdists=("$1"/*.tar.gz)
+if [ "${#sdists[@]}" -eq 0 ]; then
+	echo "no sdists in $1" >&2
+	exit 1
+fi
+for sdist in "${sdists[@]}"; do
+	member="$(tar -tzf "$sdist" | grep -E "^[^/]+/${name//./\\.}\$" || true)"
+	if [ -z "$member" ]; then
+		echo "$sdist: no ${name} at its root" >&2
+		exit 1
+	fi
+	found="$(tar -xzOf "$sdist" "$member" | sha256sum | cut -d' ' -f1)"
+	if [ "$found" != "$sha256" ]; then
+		echo "$sdist: ${name} has SHA-256 ${found}, expected ${sha256}" >&2
+		exit 1
+	fi
+	echo "ok: $(basename "$sdist") carries ${name} (SHA-256 ${sha256})"
+done

.github/scripts/assert_wheel_notices.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Asserts every wheel in DIR carries unixODBC's notice and licence text, and that each Linux and
+# macOS wheel bundles the unixODBC driver manager as one separate shared library (Windows wheels
+# use odbc32.dll and bundle none).
+# Usage: assert_wheel_notices.sh DIR
+set -euo pipefail
+
+python="$(command -v python3 || command -v python)"
+shopt -s nullglob
+wheels=("$1"/*.whl)
+if [ "${#wheels[@]}" -eq 0 ]; then
+	echo "no wheels in $1" >&2
+	exit 1
+fi
+for wheel in "${wheels[@]}"; do
+	names="$("$python" -c 'import sys, zipfile; print("\n".join(zipfile.ZipFile(sys.argv[1]).namelist()))' "$wheel")"
+	for notice in unixODBC-NOTICE.md unixODBC-COPYING.txt; do
+		if ! grep -Eq "\.dist-info/licenses/licenses/${notice}\$" <<< "$names"; then
+			echo "$wheel: no ${notice} in its .dist-info/licenses" >&2
+			exit 1
+		fi
+	done
+	bundled="$(grep -Ec '\.(libs|dylibs)/libodbc[^/]*\.(so[.0-9]*|dylib)$' <<< "$names" || true)"
+	case "$wheel" in
+		*win_amd64.whl | *win32.whl | *win_arm64.whl) expected=0 ;;
+		*) expected=1 ;;
+	esac
+	if [ "$bundled" -ne "$expected" ]; then
+		echo "$wheel: bundles ${bundled} unixODBC libraries, expected ${expected}" >&2
+		exit 1
+	fi
+	echo "ok: $(basename "$wheel") (bundled libodbc: ${bundled})"
+done

.github/scripts/build_unixodbc.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Builds the unixODBC driver manager the Linux and macOS release wheels bundle, from its pinned source
+# release, so the wheels link it dynamically and carry it as a separate, replaceable shared library.
+# Usage: build_unixodbc.sh PREFIX SYSCONFDIR [LIBDIR] [DRIVER_PATH]
+#   SYSCONFDIR is where the bundled driver manager reads odbcinst.ini and odbc.ini when ODBCSYSINI
+#   is unset: /etc on Linux, as distribution driver packages register there; Homebrew's etc on macOS.
+#   DRIVER_PATH is a colon-separated list of folders searched for a driver registered by file name
+#   alone, as Debian and Ubuntu packages register theirs (Driver=psqlodbcw.so).
+# The source release and its SHA-256 are pinned in unixodbc_source.sh.
+set -euo pipefail
+
+prefix="$1"
+sysconfdir="$2"
+libdir="${3:-$prefix/lib}"
+driver_path="${4:-}"
+
+work="$(mktemp -d)"
+tarball="$(bash "$(dirname "$0")/unixodbc_source.sh" "$work")"
+tar -xzf "$work/$tarball" -C "$work"
+cd "$work/${tarball%.tar.gz}"
+# The included libltdl is built into the libraries, so they need no system libltdl; no readline, as
+# only the libraries ship.
+./configure --prefix="$prefix" --libdir="$libdir" --sysconfdir="$sysconfdir" \
+	${driver_path:+--with-odbc-driver-path="$driver_path"} \
+	--with-included-ltdl --enable-readline=no --enable-editline=no --enable-static=no > "$work/configure.log" ||
+	{ tail -50 "$work/configure.log"; exit 1; }
+make -j"$(getconf _NPROCESSORS_ONLN)" > "$work/make.log" 2>&1 || { tail -50 "$work/make.log"; exit 1; }
+make install > /dev/null
+cd /
+rm -rf "$work"
+echo "${tarball%.tar.gz} installed in ${libdir}, reading ${sysconfdir}, finding drivers by name in ${driver_path:-the library search path}"

.github/scripts/install_odbc_drivers.ps1

@@ -0,0 +1,45 @@
+# Installs the 64-bit ODBC drivers the Windows checks read through, each a pinned installer
+# checked against its SHA-256, and lists the drivers Windows then has:
+#   PostgreSQL Unicode(x64)        psqlODBC 18.00.0006
+#   ODBC Driver 18 for SQL Server  Microsoft's msodbcsql18 18.7.1.1 (installing it accepts Microsoft's licence)
+#   MySQL ODBC 26.10 Unicode Driver MySQL Connector/ODBC 26.10.0 (its SHA-256 taken from the file
+#                                  matching MySQL's published MD5, 34541f6d21bb6a4b2f56b3f8f1afcf4b)
+# Usage: pwsh .github/scripts/install_odbc_drivers.ps1
+$ErrorActionPreference = "Stop"
+
+$drivers = @(
+    @{
+        Name = "psqlODBC"
+        Uri = "https://ftp.postgresql.org/pub/odbc/releases/REL-18_00_0006/psqlodbc_x64.msi"
+        Sha256 = "8962543113df83410b74f113db1ef80717b65a1dc7869e18c24bd852dd65535f"
+        Arguments = @()
+    },
+    @{
+        Name = "msodbcsql18"
+        Uri = "https://go.microsoft.com/fwlink/?linkid=2378279"
+        Sha256 = "21ef69e4b942f18aced55fa7c3a8d7263004e113a96690bac7509645e1b1a310"
+        Arguments = @("IACCEPTMSODBCSQLLICENSETERMS=YES")
+    },
+    @{
+        Name = "MySQL Connector/ODBC"
+        Uri = "https://cdn.mysql.com/Downloads/Connector-ODBC/26.10/mysql-connector-odbc-26.10.0-winx64.msi"
+        Sha256 = "ac58d7f1b4d7c1f37630a78c83c659f04c127754bfa299957b4effa32bd83cae"
+        Arguments = @()
+    }
+)
+
+foreach ($driver in $drivers) {
+    $msi = Join-Path $env:RUNNER_TEMP ("odbc-" + [guid]::NewGuid() + ".msi")
+    Invoke-WebRequest -Uri $driver.Uri -OutFile $msi
+    $hash = (Get-FileHash -Algorithm SHA256 -Path $msi).Hash.ToLowerInvariant()
+    if ($hash -ne $driver.Sha256) {
+        throw "$($driver.Name): the installer's SHA-256 is $hash, not the pinned $($driver.Sha256); check the new release and update the pin"
+    }
+    $process = Start-Process msiexec.exe -ArgumentList (@("/i", $msi, "/qn", "/norestart") + $driver.Arguments) -Wait -PassThru
+    if ($process.ExitCode -ne 0) {
+        throw "$($driver.Name): msiexec exited $($process.ExitCode)"
+    }
+    Remove-Item $msi
+}
+
+Get-OdbcDriver -Platform 64-bit | Where-Object { $_.Name -match "PostgreSQL|SQL Server|MySQL" } | Format-Table Name, Platform

.github/scripts/install_odbc_drivers.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# Installs unixODBC and the ODBC drivers the container tests read through, on Debian or Ubuntu x86_64,
+# and registers each driver under the name the tests connect with:
+#   PostgreSQL Unicode            psqlODBC, from the distribution
+#   ODBC Driver 18 for SQL Server Microsoft's msodbcsql18, the same release as on Windows (installing it accepts Microsoft's licence)
+#   MySQL ODBC Unicode Driver     MySQL Connector/ODBC, MySQL's package for this distribution release
+# Usage: install_odbc_drivers.sh [--with-dev]   (--with-dev adds the unixODBC headers to build against)
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+
+MYSQL_ODBC_VERSION=26.10.0
+MSODBCSQL_VERSION=18.7.1.1-1
+MYSQL_ODBC_NAME="MySQL ODBC Unicode Driver"
+
+sudo_() { if [ "$(id -u)" = 0 ]; then "$@"; else sudo "$@"; fi; }
+
+packages=(unixodbc odbc-postgresql curl ca-certificates)
+if [ "${1:-}" = "--with-dev" ]; then
+	packages+=(unixodbc-dev)
+fi
+sudo_ apt-get update -qq
+sudo_ apt-get install -y -qq "${packages[@]}"
+
+# Microsoft's repository for this distribution release, then msodbcsql18.
+. /etc/os-release
+if ! dpkg -s msodbcsql18 > /dev/null 2>&1; then
+	curl -fsSL -o /tmp/packages-microsoft-prod.deb "https://packages.microsoft.com/config/${ID}/${VERSION_ID}/packages-microsoft-prod.deb"
+	sudo_ dpkg -i /tmp/packages-microsoft-prod.deb > /dev/null
+	rm -f /tmp/packages-microsoft-prod.deb
+	sudo_ apt-get update -qq
+	sudo_ env ACCEPT_EULA=Y apt-get install -y -qq "msodbcsql18=${MSODBCSQL_VERSION}"
+fi
+
+# MySQL Connector/ODBC: MySQL's package built for this distribution release, which links the system
+# OpenSSL. Its generic tarball carries a newer private libssl.so.3, which can't load in a process
+# where another driver already loaded the system one (OPENSSL_3.2.0 not found on Ubuntu 24.04).
+# The SHA-256s are of the packages as cdn.mysql.com served them over TLS. The Unicode driver is
+# registered under a name that doesn't change with the version.
+case "${ID}${VERSION_ID}" in
+	ubuntu24.04) mysql_sha256=b99d26f320ca640d24d8a82bdaea33e41f1ce9bb16cbea5b44ee3b7547bc5673 ;;
+	ubuntu22.04) mysql_sha256=2e89b5ca8149ba77c24e0828597c85d160ff66fe57aa81873c742a152a747c65 ;;
+	debian13) mysql_sha256=20b506e3b44271fb3b0276333ae6595b9cecf6c2bdf888a8d21ff421c54bbdc9 ;;
+	debian12) mysql_sha256=119d0a5413a97e155a886df03cc01921a3294ff0cb02569c1d8e11642f4c0b7d ;;
+	*) echo "no pinned MySQL Connector/ODBC package for ${ID} ${VERSION_ID}" >&2; exit 1 ;;
+esac
+if ! dpkg -s mysql-connector-odbc > /dev/null 2>&1; then
+	package="mysql-connector-odbc_${MYSQL_ODBC_VERSION}-1${ID}${VERSION_ID}_amd64.deb"
+	work="$(mktemp -d)"
+	curl -fsSL -o "$work/$package" "https://cdn.mysql.com/Downloads/Connector-ODBC/${MYSQL_ODBC_VERSION%.*}/$package" ||
+		curl -fsSL -o "$work/$package" "https://downloads.mysql.com/archives/get/p/10/file/$package"
+	echo "$mysql_sha256  $work/$package" | sha256sum -c --quiet -
+	sudo_ apt-get install -y -qq "$work/$package"
+	rm -rf "$work"
+fi
+driver="/usr/lib/x86_64-linux-gnu/odbc/libmyodbc${MYSQL_ODBC_VERSION%%.*}w.so"
+test -f "$driver"
+printf '[%s]\nDescription=MySQL Connector/ODBC %s\nDriver=%s\nUsageCount=1\n' "$MYSQL_ODBC_NAME" "$MYSQL_ODBC_VERSION" "$driver" > /tmp/mysql-odbc.ini
+sudo_ odbcinst -i -d -r -f /tmp/mysql-odbc.ini > /dev/null
+rm -f /tmp/mysql-odbc.ini
+
+odbcinst -q -d

.github/scripts/smoke_odbc_wheel.py

@@ -0,0 +1,108 @@
+"""Smoke-test the driver manager an installed wheel's `sqlload` reaches ODBC drivers through.
+
+Usage: python smoke_odbc_wheel.py SQLLOAD... [--default-ini PATH] [--driver-folder DIR]
+
+Each SQLLOAD is an installed `sqlload` command (the sqlload wheel's and the sqlload-cli wheel's).
+On Linux and macOS it must run where no unixODBC is installed, so the wheel's bundled driver
+manager is the only one; it must find a driver registered in an odbcinst.ini named by ODBCSYSINI
+and, with --default-ini, one registered in the default odbcinst.ini (/etc on Linux, Homebrew's etc
+on macOS). The drivers don't exist, so the driver manager reports the library it tried to load,
+which shows the registration it read. With --driver-folder (Linux), a plain shared library placed in
+DIR and registered by file name alone, as Debian and Ubuntu driver packages register theirs
+(Driver=psqlodbcw.so), must load: it then fails as no driver (IM004), not as a missing file. On Windows, the driver manager built in (odbc32.dll) reports
+an unregistered driver.
+"""
+
+import argparse
+import glob
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+PROJECT = """\
+[connections.erp]
+kind = "odbc"
+connection_string = "Driver={{{driver}}};"
+
+[connections.local]
+kind = "duckdb"
+database = "out.duckdb"
+
+[targets.dev]
+connection = "local"
+loader_schema = "raw"
+
+[sources.erp]
+kind = "odbc"
+connection = "erp"
+description = "Smoke test."
+include = ["*"]
+"""
+
+#: Where a unixODBC library would be if one were installed.
+SYSTEM_LIBODBC = ["/usr/lib*/libodbc.so*", "/usr/lib/*/libodbc.so*", "/usr/local/lib/libodbc*", "/opt/homebrew/lib/libodbc*", "/opt/homebrew/opt/unixodbc"]
+
+
+def plan(sqlload, folder, driver, environment):
+    (folder / "sqlload.toml").write_text(PROJECT.format(driver=driver), encoding="utf-8")
+    result = subprocess.run([sqlload, "--no-color", "plan"], cwd=folder, capture_output=True, text=True, encoding="utf-8", errors="replace", env=environment, check=False)
+    return result.returncode, result.stdout + result.stderr
+
+
+def check(condition, what, output=""):
+    if not condition:
+        sys.exit(f"{what}\n{output}")
+    print(f"ok: {what}", flush=True)
+
+
+def registered(ini, name, library):
+    previous = ini.read_text(encoding="utf-8") if ini.exists() else ""
+    ini.parent.mkdir(parents=True, exist_ok=True)
+    ini.write_text(f"{previous}\n[{name}]\nDriver={library}\n", encoding="utf-8")
+    return previous
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("sqlload", nargs="+")
+    parser.add_argument("--default-ini", type=Path)
+    parser.add_argument("--driver-folder", type=Path)
+    arguments = parser.parse_args()
+    if os.name != "nt":
+        found = [path for pattern in SYSTEM_LIBODBC for path in glob.glob(pattern)]
+        check(not found, "no unixODBC is installed, so only the wheel's driver manager can load", str(found))
+    for sqlload in arguments.sqlload:
+        sqlload = shutil.which(sqlload) or sqlload
+        with tempfile.TemporaryDirectory() as name:
+            folder = Path(name)
+            if os.name == "nt":
+                code, output = plan(sqlload, folder, "SQLLoad Smoke Missing", dict(os.environ))
+                check(code == 1 and "Data source name not found" in output, f"{sqlload}: Windows's driver manager reports an unregistered driver", output)
+                continue
+            ini = folder / "ini" / "odbcinst.ini"
+            registered(ini, "SQLLoad Smoke", "/nonexistent/libsmoke-odbcsysini.so")
+            code, output = plan(sqlload, folder, "SQLLoad Smoke", {**os.environ, "ODBCSYSINI": str(ini.parent)})
+            check(code == 1 and "Can't open lib '/nonexistent/libsmoke-odbcsysini.so'" in output, f"{sqlload}: the bundled driver manager finds a driver registered under ODBCSYSINI", output)
+            if arguments.driver_folder:
+                arguments.driver_folder.mkdir(parents=True, exist_ok=True)
+                plain = sorted(glob.glob("/usr/lib/*-linux-gnu/libz.so.1") + glob.glob("/lib/*-linux-gnu/libz.so.1"))
+                check(bool(plain), "a plain shared library to stand in for a driver")
+                shutil.copy(plain[0], arguments.driver_folder / "libsqlload-smoke.so")
+                registered(ini, "SQLLoad Smoke Bare", "libsqlload-smoke.so")
+                code, output = plan(sqlload, folder, "SQLLoad Smoke Bare", {**os.environ, "ODBCSYSINI": str(ini.parent)})
+                check(code == 1 and "IM004" in output and "file not found" not in output, f"{sqlload}: the bundled driver manager finds a driver registered by file name in {arguments.driver_folder}", output)
+            if arguments.default_ini:
+                previous = registered(arguments.default_ini, "SQLLoad Smoke Default", "/nonexistent/libsmoke-default.so")
+                try:
+                    environment = {name: value for name, value in os.environ.items() if name not in ("ODBCSYSINI", "ODBCINSTINI")}
+                    code, output = plan(sqlload, folder, "SQLLoad Smoke Default", environment)
+                finally:
+                    arguments.default_ini.write_text(previous, encoding="utf-8")
+                check(code == 1 and "Can't open lib '/nonexistent/libsmoke-default.so'" in output, f"{sqlload}: the bundled driver manager finds a driver registered in {arguments.default_ini}", output)
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/unixodbc_source.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Downloads unixODBC's pinned source release, checks its SHA-256 and copies it into each FOLDER.
+# The Linux and macOS wheels bundle the driver manager built from it (build_unixodbc.sh), and both
+# sdists carry it as the library's corresponding source (LGPL-2.1 section 4).
+# Usage: unixodbc_source.sh FOLDER...      Prints the tarball's name.
+# Keep the version in step with the NOTICE files under packages/*/licenses/.
+set -euo pipefail
+
+UNIXODBC_VERSION=2.3.14
+UNIXODBC_SHA256=4e2814de3e01fc30b0b9f75e83bb5aba91ab0384ee951286504bb70205524771
+name="unixODBC-${UNIXODBC_VERSION}.tar.gz"
+
+work="$(mktemp -d)"
+curl -fsSL -o "$work/$name" "https://www.unixodbc.org/$name" ||
+	curl -fsSL -o "$work/$name" "https://github.com/lurcher/unixODBC/releases/download/v${UNIXODBC_VERSION}/$name"
+if command -v sha256sum > /dev/null; then
+	echo "$UNIXODBC_SHA256  $work/$name" | sha256sum -c --quiet - >&2
+else
+	echo "$UNIXODBC_SHA256  $work/$name" | shasum -a 256 -c --quiet - >&2
+fi
+for folder in "$@"; do
+	cp "$work/$name" "$folder/$name"
+done
+rm -rf "$work"
+echo "$name"