From 2ba63c6af8d0d288e6c34b55b3bcb2e7d2944b9c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 11:56:51 +0000 Subject: [PATCH 01/25] feat: Windows and macOS Intel wheels and CLI Build sqlload and sqlload-cli wheels for Windows x86_64 (MSVC) and macOS x86_64 (macos-15-intel), and test Windows in CI. - librdkafka builds with CMake on Windows, where its configure script and make are missing. - Local paths: drive paths (C:\data), UNC shares (\\server\share), \\?\ paths and file:///C:/ and file://server/share URLs are local folders on Windows; Unix paths are unchanged. - A postgres_cdc replication session's relay on Windows is a 127.0.0.1 port, since the replication client cannot reach a socket file there. It relays only the connection whose startup message carries a random per-session secret, drops every other connection before a byte reaches the server, gives each 5 seconds, and never forwards the secret. - Verify gains a Windows job: unit tests, then the executable end to end over drive paths, a UNC share, psqlODBC against the runner's Postgres, and a postgres_cdc load streamed through the relay. - Publish builds and smoke-tests every wheel on pull requests that change packaging, and publishes only when dispatched. --- .github/scripts/windows_e2e.py | 208 ++++++++++++ .github/workflows/publish.yml | 51 ++- .github/workflows/verify.yml | 59 +++- Cargo.lock | 10 + crates/sqlload/Cargo.toml | 5 + .../src/adapters/files/main/table_path.rs | 9 +- .../sqlload/src/cdc/_helpers/snapshot_log.rs | 8 +- crates/sqlload/src/cdc/_helpers/stream.rs | 15 +- crates/sqlload/src/copy/_helpers/endpoints.rs | 3 +- .../_helpers/loopback_relay.rs | 135 ++++++++ .../src/postgres_source/_helpers/mod.rs | 2 + .../sqlload/src/postgres_source/constants.rs | 12 + .../src/postgres_source/main/relay_tls.rs | 45 ++- crates/sqlload/src/postgres_source/models.rs | 10 + crates/sqlload/src/postgres_source/tests.rs | 2 + .../src/postgres_source/tests/helpers.rs | 46 +++ .../tests/test_loopback_relay.rs | 304 ++++++++++++++++++ .../src/postgres_source/tests/test_types.rs | 7 + .../src/store_paths/_helpers/local_root.rs | 55 ++++ .../src/store_paths/_helpers/locations.rs | 22 +- .../sqlload/src/store_paths/_helpers/mod.rs | 1 + .../src/store_paths/main/file_url_path.rs | 10 + .../src/store_paths/main/local_separators.rs | 10 + crates/sqlload/src/store_paths/main/mod.rs | 2 + .../src/store_paths/main/resolve_location.rs | 25 +- crates/sqlload/src/store_paths/tests.rs | 2 + .../src/store_paths/tests/test_local_root.rs | 110 +++++++ .../tests/test_resolve_location.rs | 83 +++++ .../sqlload/tests/loader/test_postgres_tls.rs | 1 + website/src/content/docs/docs/cli/copy.mdx | 2 + .../src/content/docs/docs/configuration.mdx | 2 +- website/src/content/docs/docs/install.mdx | 3 +- .../content/docs/docs/sources/postgres.mdx | 24 +- .../content/docs/docs/supported-systems.mdx | 3 +- 34 files changed, 1239 insertions(+), 47 deletions(-) create mode 100644 .github/scripts/windows_e2e.py create mode 100644 crates/sqlload/src/postgres_source/_helpers/loopback_relay.rs create mode 100644 crates/sqlload/src/postgres_source/tests/test_loopback_relay.rs create mode 100644 crates/sqlload/src/store_paths/_helpers/local_root.rs create mode 100644 crates/sqlload/src/store_paths/main/file_url_path.rs create mode 100644 crates/sqlload/src/store_paths/main/local_separators.rs create mode 100644 crates/sqlload/src/store_paths/tests/test_local_root.rs diff --git a/.github/scripts/windows_e2e.py b/.github/scripts/windows_e2e.py new file mode 100644 index 00000000..4a555ce1 --- /dev/null +++ b/.github/scripts/windows_e2e.py @@ -0,0 +1,208 @@ +"""End-to-end checks of a Windows sqlload executable: drive paths, a UNC share and an ODBC driver. + +Usage: python windows_e2e.py SQLLOAD_EXE WORK_DIR SHARE [--odbc-driver NAME] [--cdc] + +WORK_DIR is a local folder (C:\\...) whose `share` folder is shared as SHARE (\\\\localhost\\name), +so what the share holds can also be read locally. With --odbc-driver, a table in the local Postgres (the runner's +preinstalled server, user postgres) is loaded through that ODBC driver, then loaded again +incrementally. With --cdc, a postgres_cdc source snapshots a table and then streams its changes, which +on Windows go through the loopback TLS relay (that server needs wal_level = logical). Needs the +duckdb Python package to read the results back. +""" + +import argparse +import os +import pathlib +import subprocess +import sys + +import duckdb + +ROWS = [(1, "first", "2026-01-01 10:00:00"), (2, "second", "2026-01-02 11:30:00"), (3, "third", "2026-01-03 12:45:30")] + + +def sqlload(exe, *args, config=None): + command = [exe, "--no-color"] + if config is not None: + command += ["--config", str(config)] + command += list(args) + print("$", subprocess.list2cmdline(command), flush=True) + result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", errors="replace") + print(result.stdout + result.stderr, flush=True) + if result.returncode != 0: + sys.exit(f"sqlload exited {result.returncode}") + return result.stdout + result.stderr + + +def query(database, sql): + with duckdb.connect(str(database), read_only=True) as connection: + return connection.execute(sql).fetchall() + + +def expect(found, expected, what): + if found != expected: + sys.exit(f"{what}: found {found!r}, expected {expected!r}") + print(f"ok: {what}", flush=True) + + +def files_checks(exe, work, share): + source = work / "in" / "orders" + source.mkdir(parents=True, exist_ok=True) + lines = ["id,name,updated_at"] + [f"{i},{name},{at}" for i, name, at in ROWS] + (source / "orders.csv").write_text("\n".join(lines) + "\n", encoding="utf-8") + database = work / "shop.duckdb" + + # A drive path with backslashes to a DuckDB file on a drive path. + sqlload(exe, "copy", str(source), f"duckdb://{database}#raw.orders", "--format", "csv") + expect(query(database, "SELECT count(*), max(name) FROM raw.orders"), [(3, "third")], "drive folder to DuckDB") + + # The same folder into Parquet files on the UNC share. + lake = f"{share}\\lake\\orders" + sqlload(exe, "copy", str(source), lake, "--format", "csv") + written = sorted(path.name for path in (work / "share" / "lake" / "orders").rglob("*.parquet")) + if not written: + sys.exit(f"no Parquet files under {lake}") + print(f"ok: UNC share holds {written}", flush=True) + + # Read back through a file:// URL naming the share, and through a \\?\UNC\ path. + server_share = share.lstrip("\\").replace("\\", "/") + sqlload(exe, "copy", f"file://{server_share}/lake/orders", f"duckdb://{database}#raw.from_share", "--format", "parquet") + expect(query(database, "SELECT count(*), sum(id) FROM raw.from_share"), [(3, 6)], "file:// share URL to DuckDB") + verbatim = "\\\\?\\UNC\\" + share.lstrip("\\") + "\\lake\\orders" + sqlload(exe, "copy", verbatim, f"duckdb://{database}#raw.from_verbatim", "--format", "parquet") + expect(query(database, "SELECT count(*) FROM raw.from_verbatim"), [(3,)], "\\\\?\\UNC\\ path to DuckDB") + + +def psql(sql): + psql_exe = pathlib.Path(os.environ["PGBIN"]) / "psql.exe" + subprocess.run( + [str(psql_exe), "-v", "ON_ERROR_STOP=1", "-h", "127.0.0.1", "-U", "postgres", "-d", "postgres", "-c", sql], + check=True, + env={**os.environ, "PGPASSWORD": "root"}, + ) + + +def odbc_checks(exe, work, driver): + psql( + "DROP SCHEMA IF EXISTS erp CASCADE; CREATE SCHEMA erp;" + " CREATE TABLE erp.items (id bigint PRIMARY KEY, name text, price numeric(10,2), updated_at timestamp NOT NULL);" + " INSERT INTO erp.items SELECT g, 'item ' || g, g * 1.25, TIMESTAMP '2026-01-01' + g * INTERVAL '1 minute' FROM generate_series(1, 500) g;" + ) + database = work / "odbc.duckdb" + project = work / "odbc" / "sqlload.toml" + project.parent.mkdir(parents=True, exist_ok=True) + # TOML literal strings, so the Windows path keeps its backslashes. + project.write_text( + f""" +[connections.erp] +kind = "odbc" +connection_string = "Driver={{{driver}}};Server=127.0.0.1;Port=5432;Database=postgres;Uid=postgres;Pwd=root;" + +[connections.local] +kind = "duckdb" +database = '{database}' + +[targets.dev] +connection = "local" +loader_schema = "raw" + +[sources.erp] +kind = "odbc" +connection = "erp" +description = "Items of a generic ERP." +include = ["erp.items"] + +[sources.erp.tables."erp.items"] +description = "Items, changed rows found by updated_at." +cursor_column = "updated_at" +unique_key = ["id"] +write_strategy = "merge" +""", + encoding="utf-8", + ) + sqlload(exe, "load", config=project) + expect(query(database, "SELECT count(*), sum(price) FROM raw.erp__items"), [(500, 156562.50)], "ODBC first load") + psql( + "UPDATE erp.items SET name = 'changed', updated_at = TIMESTAMP '2027-01-01 00:00:00.25' WHERE id = 7;" + " INSERT INTO erp.items VALUES (501, 'new', 1, TIMESTAMP '2027-01-01 00:00:00.5');" + ) + sqlload(exe, "load", config=project) + expect( + query(database, "SELECT count(*), (SELECT name FROM raw.erp__items WHERE id = 7), max(updated_at)::VARCHAR FROM raw.erp__items"), + [(501, "changed", "2027-01-01 00:00:00.5")], + "ODBC incremental merge on a fractional-second cursor", + ) + + +def cdc_checks(exe, work): + """A postgres_cdc load streams through the loopback relay: the server offers no TLS, so under the default sslmode (prefer) the replication session is relayed in plain text.""" + psql( + "DROP SCHEMA IF EXISTS shop CASCADE; CREATE SCHEMA shop;" + " CREATE TABLE shop.orders (id bigint PRIMARY KEY, status text, total numeric(10,2));" + " INSERT INTO shop.orders SELECT g, 'new', g FROM generate_series(1, 100) g;" + ) + database = work / "cdc.duckdb" + project = work / "cdc" / "sqlload.toml" + project.parent.mkdir(parents=True, exist_ok=True) + project.write_text( + f""" +[connections.pg] +kind = "postgres" +host = "127.0.0.1" +port = 5432 +user = "postgres" +password = "${{ENV:PGPASSWORD}}" +database = "postgres" + +[connections.local] +kind = "duckdb" +database = '{database}' + +[targets.dev] +connection = "local" +loader_schema = "raw" + +[sources.shop] +kind = "postgres_cdc" +connection = "pg" +include = ["shop.*"] +slot = "sqlload_windows_e2e" +publication = "sqlload_windows_e2e" + +[sources.shop.tables."shop.orders"] +description = "Orders, captured." +""", + encoding="utf-8", + ) + os.environ["PGPASSWORD"] = "root" + sqlload(exe, "load", config=project) + expect(query(database, "SELECT count(*) FROM raw.shop__orders"), [(100,)], "CDC snapshot") + psql("UPDATE shop.orders SET status = 'paid' WHERE id <= 10; DELETE FROM shop.orders WHERE id = 100; INSERT INTO shop.orders VALUES (101, 'new', 5);") + sqlload(exe, "load", config=project) + expect( + query( + database, + "SELECT count(*) FILTER (WHERE status = 'paid'), count(*) FILTER (WHERE id = 101) FROM raw.shop__orders", + ), + [(10, 1)], + "CDC changes streamed through the loopback relay", + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("exe") + parser.add_argument("work", type=pathlib.Path) + parser.add_argument("share") + parser.add_argument("--odbc-driver") + parser.add_argument("--cdc", action="store_true") + arguments = parser.parse_args() + files_checks(arguments.exe, arguments.work, arguments.share) + if arguments.odbc_driver: + odbc_checks(arguments.exe, arguments.work, arguments.odbc_driver) + if arguments.cdc: + cdc_checks(arguments.exe, arguments.work) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6ed94151..dea80311 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,6 +7,15 @@ on: description: "Release tag to publish, for example v0.1.0" required: true type: string + # A pull request changing how wheels are built builds and smoke-tests them all, and publishes + # nothing. + pull_request: + paths: + - .github/workflows/publish.yml + - .github/scripts/assert_*_wheel_tags.sh + - packages/*/pyproject.toml + - crates/sqlload/Cargo.toml + - crates/sqlload-python/Cargo.toml permissions: contents: read @@ -22,6 +31,10 @@ jobs: - name: Resolve release tag id: release-tag run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + echo "A pull request: building its head, publishing nothing." + exit 0 + fi tag="${{ inputs.tag }}" if [[ ! "$tag" =~ ^v0\.[0-9]+\.[0-9]+$ ]]; then @@ -41,6 +54,7 @@ jobs: with: ref: ${{ needs.release-tag.outputs.tag }} - name: Match package versions to release tag + if: github.event_name == 'workflow_dispatch' env: RELEASE_VERSION: ${{ needs.release-tag.outputs.version }} run: | @@ -84,6 +98,13 @@ jobs: - name: macos-arm64 runner: macos-latest target: aarch64 + # GitHub's last Intel macOS image, available until August 2027. + - name: macos-x86_64 + runner: macos-15-intel + target: x86_64 + - name: windows-x86_64 + runner: windows-latest + target: x86_64 steps: - uses: actions/checkout@v5 with: @@ -94,6 +115,17 @@ jobs: - uses: dtolnay/rust-toolchain@master with: toolchain: 1.95.0 + # Vendored OpenSSL's Configure needs a native Windows Perl; Git Bash's lacks modules it loads. + # Without nasm it builds with no-asm. + - name: Use Strawberry Perl for vendored OpenSSL + if: runner.os == 'Windows' + shell: pwsh + run: '"OPENSSL_SRC_PERL=C:\Strawberry\perl\bin\perl.exe" >> $env:GITHUB_ENV' + # The arm64 wheel's floor already; the Intel one would otherwise get Rust's (10.12), older than + # the C++ library DuckDB's bundled build needs. + - name: Build for macOS 11 and newer + if: runner.os == 'macOS' + run: echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> "$GITHUB_ENV" # sqlload-cli ships one standalone Rust executable (OpenSSL built in) and links no libpython, # so maturin tags it py3-none-. The tag is asserted before release. - name: Build native CLI wheel @@ -124,13 +156,20 @@ jobs: - name: Assert Python extension wheel targets the stable ABI shell: bash run: bash .github/scripts/assert_python_wheel_tags.sh dist - - name: Smoke-test the Python extension wheel + - name: Smoke-test the wheels shell: bash run: | - python -m venv /tmp/smoke - /tmp/smoke/bin/pip install --quiet dist/sqlload-*.whl - /tmp/smoke/bin/sqlload --help - /tmp/smoke/bin/python -c "import sqlload, sqlload._native" + python -m venv smoke + bin=smoke/bin + if [[ "${RUNNER_OS}" == "Windows" ]]; then bin=smoke/Scripts; fi + "${bin}/python" -m pip install --quiet dist/sqlload-*.whl + "${bin}/sqlload" --help + "${bin}/python" -c "import sqlload, sqlload._native" + python -m venv smoke-cli + cli=smoke-cli/bin + if [[ "${RUNNER_OS}" == "Windows" ]]; then cli=smoke-cli/Scripts; fi + "${cli}/python" -m pip install --quiet dist/sqlload_cli-*.whl + "${cli}/sqlload" --version - uses: actions/upload-artifact@v5 with: name: wheel-${{ matrix.platform.name }} @@ -166,6 +205,7 @@ jobs: pypi: name: Publish to PyPI + if: github.event_name == 'workflow_dispatch' needs: - release-tag - wheels @@ -200,6 +240,7 @@ jobs: crates: name: Publish to crates.io + if: github.event_name == 'workflow_dispatch' needs: - release-tag - wheels diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index bc06eae3..4e1d80c2 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -146,6 +146,60 @@ jobs: df -h docker system df + # The unit tests on Windows, then the executable end to end: drive paths, a UNC share, the + # psqlODBC driver against the runner's own Postgres, and a postgres_cdc load streaming through + # the loopback relay Windows uses instead of a socket file. + windows: + name: Windows + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: windows-2025 + # A cold cache compiles DuckDB from source with MSVC (about 40 minutes); warm runs take a few. + timeout-minutes: 90 + env: + E2E_DIR: C:\sqlload-e2e + steps: + - uses: actions/checkout@v5 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: 1.95.0 + - uses: Swatinem/rust-cache@v2 + with: + shared-key: windows + - name: Remove incomplete DuckDB sources + shell: bash + run: | + for dir in target/duckdb-sources/*/; do + [ -d "$dir" ] && [ ! -e "$dir.complete" ] && echo "removing incomplete $dir" && rm -rf "$dir" + done; true + - name: Unit tests + run: cargo test -p sqlload --lib --features odbc --quiet + - name: Build the executable + run: cargo build -p sqlload --features odbc --bin sqlload --quiet + - name: Start Postgres with logical replication + shell: pwsh + run: | + Add-Content -Path "$env:PGDATA\postgresql.conf" -Value "wal_level = logical" + Set-Service -Name postgresql-x64-17 -StartupType Manual + Start-Service -Name postgresql-x64-17 + - name: Install psqlODBC + shell: pwsh + run: | + $msi = "$env:RUNNER_TEMP\psqlodbc_x64.msi" + Invoke-WebRequest -Uri https://ftp.postgresql.org/pub/odbc/releases/REL-18_00_0006/psqlodbc_x64.msi -OutFile $msi + Start-Process msiexec.exe -ArgumentList "/i", $msi, "/qn", "/norestart" -Wait + Get-OdbcDriver -Name "PostgreSQL*" -Platform 64-bit | Format-Table Name, Platform + - name: Share a folder as \\localhost\sqlload-e2e + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path "$env:E2E_DIR\share" | Out-Null + New-SmbShare -Name sqlload-e2e -Path "$env:E2E_DIR\share" -FullAccess Everyone | Out-Null + - name: End-to-end checks + shell: pwsh + run: | + python -m pip install --quiet duckdb + python .github/scripts/windows_e2e.py target\debug\sqlload.exe $env:E2E_DIR \\localhost\sqlload-e2e --odbc-driver "PostgreSQL Unicode(x64)" --cdc + # Postgres to DuckDB, 1M narrow rows, sqlload alone with a release build: a wall-time figure # in the job summary, not a gate (shared runners are too noisy to gate on time), so Verify # does not wait on it. @@ -256,7 +310,7 @@ jobs: verify: name: Verify if: always() - needs: [changes, checks, containers, perf] + needs: [changes, checks, containers, perf, windows] runs-on: ubuntu-24.04 steps: - name: Confirm every required job passed or was skipped for a docs-only change @@ -265,12 +319,13 @@ jobs: CHECKS_RESULT: ${{ needs.checks.result }} CONTAINERS_RESULT: ${{ needs.containers.result }} PERF_RESULT: ${{ needs.perf.result }} + WINDOWS_RESULT: ${{ needs.windows.result }} run: | if [[ "${CHANGES_RESULT}" != "success" ]]; then echo "Scope classification: ${CHANGES_RESULT}" >&2 exit 1 fi - for result in "${CHECKS_RESULT}" "${CONTAINERS_RESULT}" "${PERF_RESULT}"; do + for result in "${CHECKS_RESULT}" "${CONTAINERS_RESULT}" "${PERF_RESULT}" "${WINDOWS_RESULT}"; do case "${result}" in success|skipped) ;; *) diff --git a/Cargo.lock b/Cargo.lock index 87608b56..fff20f99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -979,6 +979,15 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "cmov" version = "0.5.4" @@ -4558,6 +4567,7 @@ version = "4.10.0+2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e234cf318915c1059d4921ef7f75616b5219b10b46e9f3a511a15eb4b56a3f77" dependencies = [ + "cmake", "libc", "libz-sys", "num_enum", diff --git a/crates/sqlload/Cargo.toml b/crates/sqlload/Cargo.toml index 062dcbdf..9234ff38 100644 --- a/crates/sqlload/Cargo.toml +++ b/crates/sqlload/Cargo.toml @@ -100,6 +100,11 @@ zstd.workspace = true [target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] rustix.workspace = true +# librdkafka's default build runs its mklove `configure` script and `make`, which MSVC has +# neither of; on Windows it is built with CMake instead. +[target.'cfg(windows)'.dependencies] +rdkafka = { workspace = true, features = ["cmake-build"] } + [dev-dependencies] bytes.workspace = true arrow-array = { workspace = true, features = ["chrono-tz"] } diff --git a/crates/sqlload/src/adapters/files/main/table_path.rs b/crates/sqlload/src/adapters/files/main/table_path.rs index c761ab3d..d5a77935 100644 --- a/crates/sqlload/src/adapters/files/main/table_path.rs +++ b/crates/sqlload/src/adapters/files/main/table_path.rs @@ -3,12 +3,13 @@ use crate::adapters::files::_helpers::layout::template::partition_folder; use crate::adapters::files::constants::{CURSOR_PLACEHOLDER, CURSOR_PLACEHOLDER_START}; use crate::adapters::files::models::{TablePath, TablePathRequest}; +use crate::store_paths::main::local_separators::local_separators; use crate::store_paths::main::render_template::render_template; use crate::store_paths::main::resolve_location::resolve_location; /// The table's folder and partition folder: placeholders filled, the cursor allowed only in the last folder. pub fn table_path(request: &TablePathRequest<'_>) -> Result { - let template = request.template; + let template = &*local_separators(request.template); if template.contains("://") { return Err(format!( "path {template:?} must be relative to connection {:?}, not a URL", @@ -42,10 +43,8 @@ pub fn table_path(request: &TablePathRequest<'_>) -> Result { )); } }; - let root = match template.starts_with('/') { - true => "/", - false => "", - }; + // `/`, or `//` before a Windows share; a drive (`C:`) is the first folder. + let root = &template[..template.len() - template.trim_start_matches('/').len()]; let rendered = render_template(&format!("{root}{}", folder_segments.join("/")), &values) .map_err(|error| error.to_string())?; let folder = resolve_location(Some((request.connection, request.store)), &rendered) diff --git a/crates/sqlload/src/cdc/_helpers/snapshot_log.rs b/crates/sqlload/src/cdc/_helpers/snapshot_log.rs index 92f917f5..d944e2cd 100644 --- a/crates/sqlload/src/cdc/_helpers/snapshot_log.rs +++ b/crates/sqlload/src/cdc/_helpers/snapshot_log.rs @@ -15,6 +15,7 @@ use crate::file_store::main::read_file::read_file; use crate::file_store::main::write_file::write_file; use crate::loader::errors::LoaderError; use crate::schema_plan::constants::CDC_LSN_COLUMN; +use crate::store_paths::main::local_separators::local_separators; use crate::store_paths::models::Location; use crate::time_text::main::now_utc_text::now_utc_text; @@ -205,10 +206,9 @@ impl SnapshotLog<'_> { /// The records folder of `source`'s slot `slot` in a files target whose path template is `template`: the template's folders up to the first one naming anything but `{source}`, with `{source}` filled (all but the last when every one is fixed, the table's own folder then), then `_sqlload/cdc///`. Outside every table's folder, and the same whichever tables the source selects; a table's own `destination_path` does not move it. pub(crate) fn snapshot_folder(template: &str, source: &str, slot: &str) -> String { - let root = match template.starts_with('/') { - true => "/", - false => "", - }; + let template = &*local_separators(template); + // `/`, or `//` before a Windows share; a drive (`C:`) is the first folder. + let root = &template[..template.len() - template.trim_start_matches('/').len()]; let segments: Vec = template .split('/') .filter(|part| !part.is_empty()) diff --git a/crates/sqlload/src/cdc/_helpers/stream.rs b/crates/sqlload/src/cdc/_helpers/stream.rs index 2c73ed05..50fc1d6a 100644 --- a/crates/sqlload/src/cdc/_helpers/stream.rs +++ b/crates/sqlload/src/cdc/_helpers/stream.rs @@ -44,6 +44,7 @@ use crate::loader::models::{ CdcSpec, CommitRule, LoadRun, PlanOptions, ProjectPlan, Publisher, TableLoaded, TableOutcome, TablePlan, WholeReason, }; +use crate::postgres_source::constants::APPLICATION_NAME; use crate::postgres_source::constants::DEFAULT_BATCH_BYTES; use crate::postgres_source::errors::PostgresError; use crate::postgres_source::main::connect_postgres::connect_postgres; @@ -145,19 +146,27 @@ impl CdcSource<'_> { async fn replication_config(&self) -> Result { let address = self.address()?; let relay = relay_tls(&address).await?; + let (host, port, application_name) = match &relay { + Some(relay) => ( + relay.host.as_str(), + relay.port, + relay.application_name.expose(), + ), + None => (address.host, address.port, APPLICATION_NAME), + }; Ok(ReplicationConfig::new( - relay.as_deref().unwrap_or(address.host), + host, address.user, address.password.unwrap_or_default(), address.database, self.cdc.slot.clone(), self.cdc.publication.clone(), ) - .with_port(address.port) + .with_port(port) .with_tls(TlsConfig::disabled()) .with_status_interval(STATUS_INTERVAL) .with_options(REPLICATION_OPTIONS) - .with_application_name("sqlload")) + .with_application_name(application_name)) } /// The source's snapshot records: beside its first table in a warehouse, in the records folder of its path template in a files target (see [`snapshot_folder`]). diff --git a/crates/sqlload/src/copy/_helpers/endpoints.rs b/crates/sqlload/src/copy/_helpers/endpoints.rs index 5f4fb3f3..6a3c0e8e 100644 --- a/crates/sqlload/src/copy/_helpers/endpoints.rs +++ b/crates/sqlload/src/copy/_helpers/endpoints.rs @@ -4,6 +4,7 @@ use toml::{Table, Value}; use crate::copy::constants::{POSTGRES_SETTINGS, REDACTED, S3_SETTINGS}; use crate::copy::models::{Endpoint, Role}; +use crate::store_paths::main::file_url_path::file_url_path; /// The endpoint `text` names at `role`: `connection:locator` for a connection in `connections` (or a bare name), a URI, or a local folder; every message shows `text` with its password redacted. pub(crate) fn endpoint(text: &str, role: Role, connections: &Table) -> Result { @@ -123,7 +124,7 @@ fn uri_endpoint(shown: &str, scheme: &str, rest: &str, role: Role) -> Result s3(shown, body, role), - "file" => Ok(folder(body, shown)), + "file" => Ok(folder(&file_url_path(body), shown)), "snowflake" => Err(format!( "the {} {shown}: a Snowflake connection needs auth and a load_method, which a URI does not carry; declare it under [connections] in sqlload.toml and name it, as warehouse:SCHEMA.TABLE", role.label() diff --git a/crates/sqlload/src/postgres_source/_helpers/loopback_relay.rs b/crates/sqlload/src/postgres_source/_helpers/loopback_relay.rs new file mode 100644 index 00000000..e4b2e073 --- /dev/null +++ b/crates/sqlload/src/postgres_source/_helpers/loopback_relay.rs @@ -0,0 +1,135 @@ +//! A relay on a loopback port, where the replication client cannot reach a private socket (Windows). Any local user can connect to the port, so the relay forwards the session only to the connection whose startup message names this relay's secret as its application name, and nothing reaches the server before that check passes. + +use std::net::Ipv4Addr; + +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::task::JoinSet; + +use crate::auth::models::Secret; +use crate::postgres_source::constants::{ + APPLICATION_NAME, PROTOCOL_VERSION, RELAY_ACCEPT_TIMEOUT, RELAY_STARTUP_TIMEOUT, + STARTUP_MESSAGE_LIMIT, +}; +use crate::postgres_source::models::RelayEndpoint; + +/// A port on 127.0.0.1 that pipes `server` to the first connection presenting the returned application name; it closes once that connection is admitted, or after [`RELAY_ACCEPT_TIMEOUT`] without one. Other connections are dropped, each after [`RELAY_STARTUP_TIMEOUT`] at most, without a byte of theirs reaching the server. +pub(crate) async fn loopback_relay(server: S) -> Result +where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + let mut secret = [0u8; 32]; + getrandom::fill(&mut secret).map_err(|error| error.to_string())?; + let expected = Secret(format!("{APPLICATION_NAME}-{}", hex::encode(secret))); + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .map_err(|error| error.to_string())?; + let port = listener + .local_addr() + .map_err(|error| error.to_string())? + .port(); + tokio::spawn(serve(listener, server, expected.clone())); + Ok(RelayEndpoint { + host: Ipv4Addr::LOCALHOST.to_string(), + port, + application_name: expected, + }) +} + +/// Admit connections to `listener` side by side, so a silent one holds up none of the others, and pipe `server` to the first admitted. +async fn serve(listener: TcpListener, mut server: S, expected: Secret) +where + S: AsyncRead + AsyncWrite + Unpin, +{ + let deadline = tokio::time::Instant::now() + RELAY_ACCEPT_TIMEOUT; + let mut pending = JoinSet::new(); + let (mut local, startup) = loop { + tokio::select! { + accepted = listener.accept() => { + if let Ok((stream, _)) = accepted { + pending.spawn(admit(stream, expected.clone())); + } + } + Some(Ok(Some(admitted))) = pending.join_next() => break admitted, + () = tokio::time::sleep_until(deadline) => return, + } + }; + drop(listener); + drop(pending); + if server.write_all(&startup).await.is_ok() { + let _ = tokio::io::copy_bidirectional(&mut local, &mut server).await; + } +} + +/// `stream` and the startup message to forward for it, when it sends one naming `expected` within [`RELAY_STARTUP_TIMEOUT`]. +async fn admit(mut stream: TcpStream, expected: Secret) -> Option<(TcpStream, Vec)> { + let message = tokio::time::timeout(RELAY_STARTUP_TIMEOUT, read_startup(&mut stream)) + .await + .ok()??; + let forwarded = admitted_startup(&message, expected.expose())?; + let _ = stream.set_nodelay(true); + Some((stream, forwarded)) +} + +/// One startup message: its length, and as many bytes as that gives, up to [`STARTUP_MESSAGE_LIMIT`]. +async fn read_startup(stream: &mut R) -> Option> { + let mut length = [0u8; 4]; + stream.read_exact(&mut length).await.ok()?; + let declared = usize::try_from(u32::from_be_bytes(length)).ok()?; + if !(8..=STARTUP_MESSAGE_LIMIT).contains(&declared) { + return None; + } + let mut message = vec![0; declared]; + message[..4].copy_from_slice(&length); + stream.read_exact(&mut message[4..]).await.ok()?; + Some(message) +} + +/// `message`, a whole protocol 3.0 startup message, with its one `application_name` replaced by [`APPLICATION_NAME`]; `None` unless that name is `expected`. +pub(crate) fn admitted_startup(message: &[u8], expected: &str) -> Option> { + let declared = usize::try_from(u32::from_be_bytes(message.get(..4)?.try_into().ok()?)).ok()?; + if declared != message.len() || message.get(4..8)? != PROTOCOL_VERSION { + return None; + } + // Pairs of NUL-terminated names and values, then one more NUL. + let fields: Vec<&[u8]> = message[8..] + .strip_suffix(b"\0\0")? + .split(|byte| *byte == 0) + .collect(); + if !fields.len().is_multiple_of(2) || fields.iter().step_by(2).any(|name| name.is_empty()) { + return None; + } + let named: Vec = (0..fields.len()) + .step_by(2) + .filter(|index| fields[*index] == b"application_name") + .collect(); + let [name] = named.as_slice() else { + return None; + }; + if !same_secret(fields[name + 1], expected.as_bytes()) { + return None; + } + let mut forwarded = vec![0; 4]; + forwarded.extend_from_slice(&PROTOCOL_VERSION); + for (index, field) in fields.iter().enumerate() { + match index == name + 1 { + true => forwarded.extend_from_slice(APPLICATION_NAME.as_bytes()), + false => forwarded.extend_from_slice(field), + } + forwarded.push(0); + } + forwarded.push(0); + let length = u32::try_from(forwarded.len()).ok()?; + forwarded[..4].copy_from_slice(&length.to_be_bytes()); + Some(forwarded) +} + +/// Whether `given` is `expected`, comparing every byte whatever the first difference, so the time taken tells nothing of the secret. +fn same_secret(given: &[u8], expected: &[u8]) -> bool { + given.len() == expected.len() + && given + .iter() + .zip(expected) + .fold(0u8, |difference, (left, right)| difference | (left ^ right)) + == 0 +} diff --git a/crates/sqlload/src/postgres_source/_helpers/mod.rs b/crates/sqlload/src/postgres_source/_helpers/mod.rs index d329a9dd..ca8074c8 100644 --- a/crates/sqlload/src/postgres_source/_helpers/mod.rs +++ b/crates/sqlload/src/postgres_source/_helpers/mod.rs @@ -2,6 +2,8 @@ pub(crate) mod builders; pub(crate) mod catalog; pub(crate) mod copy_decoder; pub(crate) mod copy_stream; +#[cfg(any(windows, test))] +pub(crate) mod loopback_relay; pub(crate) mod rustls_connect; pub(crate) mod server_reason; pub(crate) mod split_read; diff --git a/crates/sqlload/src/postgres_source/constants.rs b/crates/sqlload/src/postgres_source/constants.rs index d61a157f..9c6e1cea 100644 --- a/crates/sqlload/src/postgres_source/constants.rs +++ b/crates/sqlload/src/postgres_source/constants.rs @@ -126,6 +126,18 @@ pub const SSL_REQUEST: [u8; 8] = [0, 0, 0, 8, 0x04, 0xd2, 0x16, 0x2f]; /// How long a TLS relay waits for its replication client to connect. pub const RELAY_ACCEPT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); +/// How long a loopback relay gives each connection to send its whole startup message, so a silent or slow one is dropped without holding up the replication client. +pub const RELAY_STARTUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +/// The longest startup message a loopback relay reads: Postgres's own limit (`MAX_STARTUP_PACKET_LENGTH`). +pub const STARTUP_MESSAGE_LIMIT: usize = 10_000; + +/// Protocol 3.0, as the replication client's startup message gives it. +pub const PROTOCOL_VERSION: [u8; 4] = [0, 3, 0, 0]; + +/// The application name SQLLoad's sessions show the server. +pub const APPLICATION_NAME: &str = "sqlload"; + /// Signature algorithm OIDs (DER contents) whose certificates bind with SHA-256: sha256WithRSAEncryption, sha1WithRSAEncryption, md5WithRSAEncryption, ecdsa-with-SHA256, ecdsa-with-SHA1. pub const SHA256_SIGNATURES: &[&[u8]] = &[ &[0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x0b], diff --git a/crates/sqlload/src/postgres_source/main/relay_tls.rs b/crates/sqlload/src/postgres_source/main/relay_tls.rs index b6722fb3..6d8b10ec 100644 --- a/crates/sqlload/src/postgres_source/main/relay_tls.rs +++ b/crates/sqlload/src/postgres_source/main/relay_tls.rs @@ -1,19 +1,28 @@ //! TLS for a client that speaks only plain text: the replication client. +#[cfg(unix)] use std::os::unix::fs::PermissionsExt; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; -use tokio::net::{TcpStream, UnixListener}; +use tokio::net::TcpStream; +#[cfg(unix)] +use tokio::net::UnixListener; use tokio_rustls::TlsConnector; +#[cfg(windows)] +use crate::postgres_source::_helpers::loopback_relay::loopback_relay; use crate::postgres_source::_helpers::tls::client_config; -use crate::postgres_source::constants::{RELAY_ACCEPT_TIMEOUT, SSL_REQUEST}; +use crate::postgres_source::constants::SSL_REQUEST; +#[cfg(unix)] +use crate::postgres_source::constants::{APPLICATION_NAME, RELAY_ACCEPT_TIMEOUT}; use crate::postgres_source::errors::PostgresError; -use crate::postgres_source::models::{PostgresAddress, SslMode}; +use crate::postgres_source::models::{PostgresAddress, RelayEndpoint, SslMode}; use crate::ssh_tunnel::main::open_channel::open_channel; -/// When `address` may use TLS or goes through a bastion: a private socket folder whose `.s.PGSQL.` socket carries the next session opened on it to the server, encrypted, or in plain text when the server refuses TLS under `prefer` (one connection either way, so the server logs no abandoned one). `None` to connect directly: `sslmode = "disable"`, a Unix socket host, or a failed handshake under `prefer`, which libpq also retries in plain text. Through a bastion it never connects directly: that retry opens a fresh channel. -pub async fn relay_tls(address: &PostgresAddress<'_>) -> Result, PostgresError> { +/// When `address` may use TLS or goes through a bastion: where the replication client connects (a private socket folder whose `.s.PGSQL.` socket, or on Windows a loopback port, carries the next session opened on it to the server, encrypted, or in plain text when the server refuses TLS under `prefer` (one connection either way, so the server logs no abandoned one), and the application name it must send. `None` to connect directly: `sslmode = "disable"`, a Unix socket host, or a failed handshake under `prefer`, which libpq also retries in plain text. Through a bastion it never connects directly: that retry opens a fresh channel. +pub async fn relay_tls( + address: &PostgresAddress<'_>, +) -> Result, PostgresError> { let failed = |reason: String| PostgresError::Tls { address: address.shown(), reason, @@ -26,7 +35,7 @@ pub async fn relay_tls(address: &PostgresAddress<'_>) -> Result, } } let server = open_channel(tunnel, address.host, address.port).await?; - return Ok(Some(relay(server, address.port).map_err(failed)?)); + return Ok(Some(relay(server, address.port).await.map_err(failed)?)); } if address.tls.mode == SslMode::Disable || address.host.starts_with('/') { return Ok(None); @@ -44,7 +53,7 @@ pub async fn relay_tls(address: &PostgresAddress<'_>) -> Result, async fn negotiate( mut server: S, address: &PostgresAddress<'_>, -) -> Result, PostgresError> +) -> Result, PostgresError> where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, { @@ -64,7 +73,7 @@ where != b'S' { return match address.tls.mode { - SslMode::Prefer => Ok(Some(relay(server, address.port).map_err(failed)?)), + SslMode::Prefer => Ok(Some(relay(server, address.port).await.map_err(failed)?)), _ => Err(failed("the server does not offer TLS".to_owned())), }; } @@ -75,11 +84,12 @@ where Err(_) if address.tls.mode == SslMode::Prefer => return Ok(None), Err(error) => return Err(failed(format!("TLS handshake failed: {error}"))), }; - Ok(Some(relay(session, address.port).map_err(failed)?)) + Ok(Some(relay(session, address.port).await.map_err(failed)?)) } /// A folder only this user can enter (some systems ignore a socket's own permissions) whose `.s.PGSQL.` socket pipes the first session opened on it through `server`; the folder and its socket go once it is accepted, or after [`RELAY_ACCEPT_TIMEOUT`] without one. -fn relay(mut server: S, port: u16) -> Result +#[cfg(unix)] +async fn relay(mut server: S, port: u16) -> Result where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, { @@ -99,5 +109,18 @@ where let _ = tokio::io::copy_bidirectional(&mut local, &mut server).await; } }); - Ok(host) + Ok(RelayEndpoint { + host, + port, + application_name: crate::auth::models::Secret(APPLICATION_NAME.to_owned()), + }) +} + +/// pgwire-replication reaches no socket file on Windows, only TCP, so the session goes through a loopback port that admits only this relay's secret (see [`loopback_relay`]). +#[cfg(windows)] +async fn relay(server: S, _port: u16) -> Result +where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + loopback_relay(server).await } diff --git a/crates/sqlload/src/postgres_source/models.rs b/crates/sqlload/src/postgres_source/models.rs index 950b3013..c1e8ccb0 100644 --- a/crates/sqlload/src/postgres_source/models.rs +++ b/crates/sqlload/src/postgres_source/models.rs @@ -192,3 +192,13 @@ pub struct CopyRequest<'a> { /// More connections to the same database, idle and outside any transaction, to read a large table in parallel; empty reads in one pass. pub helpers: &'a [tokio_postgres::Client], } + +/// Where the replication client reaches a relayed session, and the application name its startup message must carry there. +#[derive(Debug)] +pub struct RelayEndpoint { + /// The relay's socket folder (Unix) or `127.0.0.1` (Windows). + pub host: String, + pub port: u16, + /// `sqlload` on Unix. On Windows `sqlload-`, which the relay checks and replaces with `sqlload` before the server sees it. + pub application_name: crate::auth::models::Secret, +} diff --git a/crates/sqlload/src/postgres_source/tests.rs b/crates/sqlload/src/postgres_source/tests.rs index 934df596..03a616d3 100644 --- a/crates/sqlload/src/postgres_source/tests.rs +++ b/crates/sqlload/src/postgres_source/tests.rs @@ -4,6 +4,8 @@ mod copy_decoder; mod decode_copy; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_loopback_relay.rs"] +mod loopback_relay; #[path = "tests/test_read_column.rs"] mod read_column; #[path = "tests/test_read_numeric.rs"] diff --git a/crates/sqlload/src/postgres_source/tests/helpers.rs b/crates/sqlload/src/postgres_source/tests/helpers.rs index 4cd42e8b..dbe21b06 100644 --- a/crates/sqlload/src/postgres_source/tests/helpers.rs +++ b/crates/sqlload/src/postgres_source/tests/helpers.rs @@ -96,3 +96,49 @@ pub(crate) fn source_column(pg_type: PgType) -> SourceColumn { source_spelling: None, } } + +/// A startup message of protocol `version` with `params`, framed as libpq and pgwire-replication frame it. +pub(crate) fn startup_message(version: [u8; 4], params: &[(&str, &str)]) -> Vec { + let mut body = version.to_vec(); + for (name, value) in params { + body.extend_from_slice(name.as_bytes()); + body.push(0); + body.extend_from_slice(value.as_bytes()); + body.push(0); + } + body.push(0); + let length = u32::try_from(body.len() + 4).unwrap_or(u32::MAX); + let mut message = length.to_be_bytes().to_vec(); + message.extend_from_slice(&body); + message +} + +/// The parameters of a well-formed protocol 3.0 startup message, in order; an error naming what is wrong otherwise. +pub(crate) fn startup_params(message: &[u8]) -> Result, String> { + let declared = message + .get(..4) + .and_then(|bytes| <[u8; 4]>::try_from(bytes).ok()) + .map(u32::from_be_bytes) + .ok_or("no length")?; + if usize::try_from(declared).ok() != Some(message.len()) { + return Err(format!("length {declared} for {} bytes", message.len())); + } + if message.get(4..8) != Some(&[0, 3, 0, 0][..]) { + return Err("not protocol 3.0".to_owned()); + } + let text = String::from_utf8(message[8..].to_vec()).map_err(|error| error.to_string())?; + let mut parts: Vec<&str> = text.split('\0').collect(); + // The pairs' terminators and the final NUL leave two empty parts at the end. + if parts.len() < 2 || parts.pop() != Some("") || parts.pop() != Some("") { + return Err("not NUL-terminated".to_owned()); + } + Ok(parts + .chunks(2) + .map(|pair| { + ( + pair[0].to_owned(), + pair.get(1).copied().unwrap_or_default().to_owned(), + ) + }) + .collect()) +} diff --git a/crates/sqlload/src/postgres_source/tests/test_loopback_relay.rs b/crates/sqlload/src/postgres_source/tests/test_loopback_relay.rs new file mode 100644 index 00000000..8a2645c2 --- /dev/null +++ b/crates/sqlload/src/postgres_source/tests/test_loopback_relay.rs @@ -0,0 +1,304 @@ +use std::time::{Duration, Instant}; + +use tokio::io::{AsyncReadExt, AsyncWriteExt, DuplexStream}; +use tokio::net::TcpStream; + +use crate::postgres_source::_helpers::loopback_relay::{admitted_startup, loopback_relay}; +use crate::postgres_source::constants::{PROTOCOL_VERSION, RELAY_STARTUP_TIMEOUT}; +use crate::postgres_source::models::RelayEndpoint; +use crate::postgres_source::tests::helpers::{startup_message, startup_params}; +use crate::postgres_source::tests::test_types::AdmittedStartupTestCase; + +const SECRET: &str = "sqlload-6f1c0d2a9b8e7f6a5b4c3d2e1f0a9b8c7d6e5f4a3b2c1d0e9f8a7b6c5d4e3f2a"; + +fn block_on(body: impl std::future::Future>) -> Result<(), String> { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .map_err(|error| error.to_string())? + .block_on(body) +} + +fn owned(params: &[(&str, &str)]) -> Vec<(String, String)> { + params + .iter() + .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) + .collect() +} + +/// The replication client's startup message, naming `application_name`. +fn client_startup(application_name: &str) -> Vec { + startup_message( + PROTOCOL_VERSION, + &[ + ("user", "replicator"), + ("database", "shop"), + ("replication", "database"), + ("application_name", application_name), + ], + ) +} + +/// The whole first message the server side of the relay received. +async fn read_forwarded(upstream: &mut DuplexStream) -> Result, String> { + let mut length = [0u8; 4]; + upstream + .read_exact(&mut length) + .await + .map_err(|error| error.to_string())?; + let declared = + usize::try_from(u32::from_be_bytes(length)).map_err(|error| error.to_string())?; + let mut message = length.to_vec(); + message.resize(declared.max(4), 0); + upstream + .read_exact(&mut message[4..]) + .await + .map_err(|error| error.to_string())?; + Ok(message) +} + +fn contains(haystack: &[u8], needle: &str) -> bool { + haystack + .windows(needle.len()) + .any(|window| window == needle.as_bytes()) +} + +async fn connect(endpoint: &RelayEndpoint) -> Result { + TcpStream::connect((endpoint.host.as_str(), endpoint.port)) + .await + .map_err(|error| error.to_string()) +} + +/// Whether the relay closes `stream` without answering, within `wait`. +async fn closed_unanswered(stream: &mut TcpStream, wait: Duration) -> Result { + let mut byte = [0u8; 1]; + match tokio::time::timeout(wait, stream.read(&mut byte)).await { + Ok(Ok(0) | Err(_)) => Ok(true), + Ok(Ok(_)) => Ok(false), + Err(_) => Err("the relay kept the connection open".to_owned()), + } +} + +#[test] +fn given_startup_messages_when_the_relay_checks_them_then_only_the_secret_is_admitted_and_never_forwarded() +-> Result<(), String> { + let wrong_last = format!("{}0", &SECRET[..SECRET.len() - 1]); + let mut short_length = client_startup(SECRET); + short_length[3] -= 1; + let mut no_terminator = client_startup(SECRET); + no_terminator.pop(); + no_terminator[3] -= 1; + let test_cases = [ + AdmittedStartupTestCase { + description: "the secret, among the other parameters", + message: client_startup(SECRET), + expected_forwarded: Some(owned(&[ + ("user", "replicator"), + ("database", "shop"), + ("replication", "database"), + ("application_name", "sqlload"), + ])), + }, + AdmittedStartupTestCase { + description: "the secret first, before an empty value", + message: startup_message( + PROTOCOL_VERSION, + &[("application_name", SECRET), ("options", ""), ("user", "u")], + ), + expected_forwarded: Some(owned(&[ + ("application_name", "sqlload"), + ("options", ""), + ("user", "u"), + ])), + }, + AdmittedStartupTestCase { + description: "a secret wrong in its last character", + message: client_startup(&wrong_last), + expected_forwarded: None, + }, + AdmittedStartupTestCase { + description: "a prefix of the secret", + message: client_startup(&SECRET[..SECRET.len() - 1]), + expected_forwarded: None, + }, + AdmittedStartupTestCase { + description: "the secret with more after it", + message: client_startup(&format!("{SECRET}0")), + expected_forwarded: None, + }, + AdmittedStartupTestCase { + description: "the plain application name", + message: client_startup("sqlload"), + expected_forwarded: None, + }, + AdmittedStartupTestCase { + description: "no application name", + message: startup_message(PROTOCOL_VERSION, &[("user", "u"), ("database", "d")]), + expected_forwarded: None, + }, + AdmittedStartupTestCase { + description: "the secret, then a second application name", + message: startup_message( + PROTOCOL_VERSION, + &[("application_name", SECRET), ("application_name", "other")], + ), + expected_forwarded: None, + }, + AdmittedStartupTestCase { + description: "an SSLRequest", + message: vec![0, 0, 0, 8, 0x04, 0xd2, 0x16, 0x2f], + expected_forwarded: None, + }, + AdmittedStartupTestCase { + description: "protocol 2.0 with the secret", + message: startup_message([0, 2, 0, 0], &[("application_name", SECRET)]), + expected_forwarded: None, + }, + AdmittedStartupTestCase { + description: "a length one short of the message", + message: short_length, + expected_forwarded: None, + }, + AdmittedStartupTestCase { + description: "no final NUL", + message: no_terminator, + expected_forwarded: None, + }, + ]; + for test_case in test_cases { + let forwarded = admitted_startup(&test_case.message, SECRET); + let found = forwarded.as_deref().map(startup_params).transpose()?; + if found != test_case.expected_forwarded { + return Err(format!( + "{}: forwarded {found:?}, expected {:?}", + test_case.description, test_case.expected_forwarded + )); + } + if forwarded.is_some_and(|bytes| contains(&bytes, &SECRET["sqlload-".len()..])) { + return Err(format!( + "{}: the secret was forwarded", + test_case.description + )); + } + } + Ok(()) +} + +#[test] +fn given_a_silent_connection_first_when_the_replication_client_connects_then_it_is_relayed_without_waiting() +-> Result<(), String> { + block_on(async { + let (server, mut upstream) = tokio::io::duplex(64 * 1024); + let endpoint = loopback_relay(server).await?; + let _silent = connect(&endpoint).await?; + let started = Instant::now(); + let mut client = connect(&endpoint).await?; + client + .write_all(&client_startup(endpoint.application_name.expose())) + .await + .map_err(|error| error.to_string())?; + let forwarded = tokio::time::timeout(RELAY_STARTUP_TIMEOUT, read_forwarded(&mut upstream)) + .await + .map_err(|_| "the client was held up behind the silent connection".to_owned())??; + if started.elapsed() >= RELAY_STARTUP_TIMEOUT { + return Err(format!("relayed after {:?}", started.elapsed())); + } + let name = startup_params(&forwarded)? + .into_iter() + .find(|(name, _)| name == "application_name"); + if name != Some(("application_name".to_owned(), "sqlload".to_owned())) { + return Err(format!("forwarded application name {name:?}")); + } + // The session is piped both ways once admitted. + upstream + .write_all(b"R") + .await + .map_err(|error| error.to_string())?; + let mut answer = [0u8; 1]; + client + .read_exact(&mut answer) + .await + .map_err(|error| error.to_string())?; + client + .write_all(b"p") + .await + .map_err(|error| error.to_string())?; + upstream + .read_exact(&mut answer) + .await + .map_err(|error| error.to_string())?; + match &answer { + b"p" => Ok(()), + other => Err(format!("piped {other:?}")), + } + }) +} + +#[test] +fn given_connections_without_the_secret_when_they_connect_then_they_are_dropped_and_the_server_receives_nothing() +-> Result<(), String> { + block_on(async { + let (server, mut upstream) = tokio::io::duplex(64 * 1024); + let endpoint = loopback_relay(server).await?; + let secret = endpoint.application_name.expose().to_owned(); + if format!("{endpoint:?}").contains(&secret) { + return Err("the relay's Debug shows its secret".to_owned()); + } + let wrong = format!("sqlload-{}", "0".repeat(64)); + let attempts: [(&str, Vec); 4] = [ + ("a wrong secret", client_startup(&wrong)), + ("the plain application name", client_startup("sqlload")), + ("an oversize length", vec![0, 1, 0, 0, 0, 3, 0, 0]), + ("an SSLRequest", vec![0, 0, 0, 8, 0x04, 0xd2, 0x16, 0x2f]), + ]; + for (description, bytes) in attempts { + let mut stream = connect(&endpoint).await?; + stream + .write_all(&bytes) + .await + .map_err(|error| error.to_string())?; + if !closed_unanswered(&mut stream, RELAY_STARTUP_TIMEOUT * 2) + .await + .map_err(|error| format!("{description}: {error}"))? + { + return Err(format!("{description}: the relay answered")); + } + } + let mut byte = [0u8; 1]; + if tokio::time::timeout(Duration::from_millis(300), upstream.read(&mut byte)) + .await + .is_ok() + { + return Err("the server received bytes from a refused connection".to_owned()); + } + let mut client = connect(&endpoint).await?; + client + .write_all(&client_startup(&secret)) + .await + .map_err(|error| error.to_string())?; + let forwarded = read_forwarded(&mut upstream).await?; + if contains(&forwarded, &secret["sqlload-".len()..]) { + return Err("the secret reached the server".to_owned()); + } + let expected = owned(&[ + ("user", "replicator"), + ("database", "shop"), + ("replication", "database"), + ("application_name", "sqlload"), + ]); + let found = startup_params(&forwarded)?; + if found != expected { + return Err(format!("forwarded {found:?}")); + } + // Admitted once: the port is closed to every later connection. + match tokio::time::timeout(Duration::from_secs(10), connect(&endpoint)).await { + Ok(Err(_)) => Ok(()), + Ok(Ok(mut late)) => match closed_unanswered(&mut late, Duration::from_secs(2)).await { + Ok(true) => Ok(()), + _ => Err("a connection after the admitted one was kept".to_owned()), + }, + Err(_) => Err("connecting after admission hung".to_owned()), + } + }) +} diff --git a/crates/sqlload/src/postgres_source/tests/test_types.rs b/crates/sqlload/src/postgres_source/tests/test_types.rs index 4443f9d2..87899d09 100644 --- a/crates/sqlload/src/postgres_source/tests/test_types.rs +++ b/crates/sqlload/src/postgres_source/tests/test_types.rs @@ -37,3 +37,10 @@ pub(crate) struct ReadColumnTestCase { /// The select expression, or the error text. pub(crate) expected_result: Result<&'static str, &'static str>, } + +pub(crate) struct AdmittedStartupTestCase { + pub(crate) description: &'static str, + pub(crate) message: Vec, + /// The parameters forwarded to the server, in order; `None` when the connection is refused. + pub(crate) expected_forwarded: Option>, +} diff --git a/crates/sqlload/src/store_paths/_helpers/local_root.rs b/crates/sqlload/src/store_paths/_helpers/local_root.rs new file mode 100644 index 00000000..02fc4e96 --- /dev/null +++ b/crates/sqlload/src/store_paths/_helpers/local_root.rs @@ -0,0 +1,55 @@ +use std::borrow::Cow; + +/// `path` with Windows separators read as `/` and the `\\?\` (`\\?\UNC\`) prefix dropped, when `windows`; unchanged otherwise. +pub(crate) fn windows_separators(path: &str, windows: bool) -> Cow<'_, str> { + if !windows || !path.contains('\\') { + return Cow::Borrowed(path); + } + let path = path.replace('\\', "/"); + Cow::Owned(match path.strip_prefix("//?/") { + Some(rest) => match rest.get(..4) { + Some(unc) if unc.eq_ignore_ascii_case("UNC/") => format!("//{}", &rest[4..]), + _ => rest.to_owned(), + }, + None => path, + }) +} + +/// The root and the rest of `path`, `/`-separated (see [`windows_separators`]), when it is absolute on this machine: `/`, or when `windows` also a drive (`C:/`, `/C:/` as a `file:///C:/` URL gives it) or a share (`//server/share/`). `None` for a relative path. +pub(crate) fn local_root(path: &str, windows: bool) -> Option<(String, &str)> { + if windows { + if let Some(share) = path.strip_prefix("//") { + let mut parts = share.splitn(3, '/'); + let (server, name) = (parts.next()?, parts.next()?); + return (!server.is_empty() && !name.is_empty()).then(|| { + ( + format!("//{server}/{name}/"), + parts.next().unwrap_or_default(), + ) + }); + } + let drive = path.strip_prefix('/').unwrap_or(path); + let bytes = drive.as_bytes(); + if bytes.len() >= 3 + && bytes[0].is_ascii_alphabetic() + && bytes[1] == b':' + && bytes[2] == b'/' + { + return Some(( + format!("{}:/", char::from(bytes[0]).to_ascii_uppercase()), + &drive[3..], + )); + } + } + path.strip_prefix('/').map(|rest| ("/".to_owned(), rest)) +} + +/// The local path a `file://` URL names, given what follows `file://`: on Windows `file:///C:/folder` is `C:/folder` and `file://server/share/folder` is the share `//server/share/folder`. +pub(crate) fn windows_file_url_path(rest: &str, windows: bool) -> Cow<'_, str> { + match (windows, rest.strip_prefix('/')) { + (false, _) => Cow::Borrowed(rest), + (true, Some(path)) if path.as_bytes().get(1) == Some(&b':') => Cow::Borrowed(path), + (true, Some(_)) => Cow::Borrowed(rest), + (true, None) => Cow::Owned(format!("//{rest}")), + } +} diff --git a/crates/sqlload/src/store_paths/_helpers/locations.rs b/crates/sqlload/src/store_paths/_helpers/locations.rs index c5cf45d6..f7af877e 100644 --- a/crates/sqlload/src/store_paths/_helpers/locations.rs +++ b/crates/sqlload/src/store_paths/_helpers/locations.rs @@ -1,8 +1,13 @@ +use std::borrow::Cow; + use crate::store_paths::_helpers::folder::folder; +use crate::store_paths::_helpers::local_root::{ + local_root, windows_file_url_path, windows_separators, +}; use crate::store_paths::errors::PathError; use crate::store_paths::models::{Location, StoreKind}; -/// The location in a URL or absolute path, `None` for a relative path. +/// The location in a URL or absolute path (on Windows also `C:\folder`, `\\server\share\folder` and `file://server/share/folder`), `None` for a relative path. pub(crate) fn url_location(path: &str) -> Result, PathError> { if let Some(rest) = path.strip_prefix("s3://") { let (bucket, key) = rest.split_once('/').unwrap_or((rest, "")); @@ -22,9 +27,14 @@ pub(crate) fn url_location(path: &str) -> Result, PathError> { prefix: folder(&[key])?, })); } - let absolute = path.strip_prefix("file://").unwrap_or(path); - if absolute.starts_with('/') { - let prefix = folder(&[absolute])?; + let windows = cfg!(windows); + let local = match path.strip_prefix("file://") { + Some(rest) => windows_file_url_path(rest, windows), + None => Cow::Borrowed(path), + }; + let local = windows_separators(&local, windows); + if let Some((root, rest)) = local_root(&local, windows) { + let prefix = folder(&[rest])?; if prefix.is_empty() { return Err(PathError::Empty { path: path.to_owned(), @@ -32,9 +42,7 @@ pub(crate) fn url_location(path: &str) -> Result, PathError> { }); } return Ok(Some(Location { - store: StoreKind::Filesystem { - root: "/".to_owned(), - }, + store: StoreKind::Filesystem { root }, prefix, })); } diff --git a/crates/sqlload/src/store_paths/_helpers/mod.rs b/crates/sqlload/src/store_paths/_helpers/mod.rs index c48429aa..78a0a2ce 100644 --- a/crates/sqlload/src/store_paths/_helpers/mod.rs +++ b/crates/sqlload/src/store_paths/_helpers/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod escapes; pub(crate) mod folder; +pub(crate) mod local_root; pub(crate) mod locations; pub(crate) mod segments; diff --git a/crates/sqlload/src/store_paths/main/file_url_path.rs b/crates/sqlload/src/store_paths/main/file_url_path.rs new file mode 100644 index 00000000..b5984fdf --- /dev/null +++ b/crates/sqlload/src/store_paths/main/file_url_path.rs @@ -0,0 +1,10 @@ +//! The local path a `file://` URL names. + +use std::borrow::Cow; + +use crate::store_paths::_helpers::local_root::windows_file_url_path; + +/// The local path a `file://` URL names, given what follows `file://`: the same on Unix; on Windows `file:///C:/folder` is `C:/folder` and `file://server/share/folder` is the share `//server/share/folder`. +pub fn file_url_path(rest: &str) -> Cow<'_, str> { + windows_file_url_path(rest, cfg!(windows)) +} diff --git a/crates/sqlload/src/store_paths/main/local_separators.rs b/crates/sqlload/src/store_paths/main/local_separators.rs new file mode 100644 index 00000000..5efdcd32 --- /dev/null +++ b/crates/sqlload/src/store_paths/main/local_separators.rs @@ -0,0 +1,10 @@ +//! Read a local path's Windows separators as `/`. + +use std::borrow::Cow; + +use crate::store_paths::_helpers::local_root::windows_separators; + +/// `path` with `\` read as `/` and the `\\?\` prefix dropped on Windows, so its folders split on `/` as everywhere else; unchanged on other systems. +pub fn local_separators(path: &str) -> Cow<'_, str> { + windows_separators(path, cfg!(windows)) +} diff --git a/crates/sqlload/src/store_paths/main/mod.rs b/crates/sqlload/src/store_paths/main/mod.rs index f5f96a2e..82df1fed 100644 --- a/crates/sqlload/src/store_paths/main/mod.rs +++ b/crates/sqlload/src/store_paths/main/mod.rs @@ -1,5 +1,7 @@ pub mod escape_partition_value; +pub mod file_url_path; pub mod hive_values; +pub mod local_separators; pub mod render_template; pub mod resolve_location; pub mod unescape_partition_value; diff --git a/crates/sqlload/src/store_paths/main/resolve_location.rs b/crates/sqlload/src/store_paths/main/resolve_location.rs index 3cf38b18..e7e4ccdb 100644 --- a/crates/sqlload/src/store_paths/main/resolve_location.rs +++ b/crates/sqlload/src/store_paths/main/resolve_location.rs @@ -1,6 +1,7 @@ //! Work out which store and folder a path means. use crate::store_paths::_helpers::folder::folder; +use crate::store_paths::_helpers::local_root::{local_root, windows_separators}; use crate::store_paths::_helpers::locations::{describe, same_store, url_location}; use crate::store_paths::errors::PathError; use crate::store_paths::models::{Location, StoreKind}; @@ -40,8 +41,22 @@ pub fn resolve_location( }) } (Some((name, StoreKind::Filesystem { root })), Some(location)) => { - let root_prefix = folder(&[root])?; - match location.prefix.strip_prefix(&root_prefix) { + let windows = cfg!(windows); + let separated = windows_separators(root, windows); + // A relative root is matched on its folders alone, whatever the path's root. + let (same_root, root_folders) = match (local_root(&separated, windows), &location.store) + { + (Some((root, folders)), StoreKind::Filesystem { root: found }) => { + (root.eq_ignore_ascii_case(found), folders) + } + _ => (true, separated.as_ref()), + }; + let root_prefix = folder(&[root_folders])?; + match location + .prefix + .strip_prefix(&root_prefix) + .filter(|_| same_root) + { Some(relative) => Ok(Location { store: StoreKind::Filesystem { root: root.clone() }, prefix: relative.to_owned(), @@ -49,7 +64,7 @@ pub fn resolve_location( None => Err(PathError::ConnectionMismatch { url: path.to_owned(), connection: name.to_owned(), - found: format!("filesystem /{}", location.prefix), + found: format!("filesystem {location}"), expected: format!("filesystem {root}"), }), } @@ -58,6 +73,10 @@ pub fn resolve_location( store: store.clone(), prefix: location.prefix, }), + (Some((_, store @ StoreKind::Filesystem { .. })), None) => Ok(Location { + store: store.clone(), + prefix: folder(&[&windows_separators(path, cfg!(windows))])?, + }), (Some((_, store)), None) => Ok(Location { store: store.clone(), prefix: folder(&[path])?, diff --git a/crates/sqlload/src/store_paths/tests.rs b/crates/sqlload/src/store_paths/tests.rs index fc22ae49..00f29b1a 100644 --- a/crates/sqlload/src/store_paths/tests.rs +++ b/crates/sqlload/src/store_paths/tests.rs @@ -4,6 +4,8 @@ mod folder; mod helpers; #[path = "tests/test_hive_values.rs"] mod hive_values; +#[path = "tests/test_local_root.rs"] +mod local_root; #[path = "tests/test_partition_value.rs"] mod partition_value; #[path = "tests/test_render_template.rs"] diff --git a/crates/sqlload/src/store_paths/tests/test_local_root.rs b/crates/sqlload/src/store_paths/tests/test_local_root.rs new file mode 100644 index 00000000..5615e0c4 --- /dev/null +++ b/crates/sqlload/src/store_paths/tests/test_local_root.rs @@ -0,0 +1,110 @@ +use crate::store_paths::_helpers::local_root::{ + local_root, windows_file_url_path, windows_separators, +}; + +/// How a local path reads on Windows, and on Unix, where only `/` starts an absolute path. +struct LocalRootTestCase { + description: &'static str, + path: &'static str, + /// Root and folders on Windows; `None` for a relative path. + expected_windows: Option<(&'static str, &'static str)>, + expected_unix: Option<(&'static str, &'static str)>, +} + +#[test] +fn given_local_paths_when_finding_their_root_then_drives_and_shares_are_roots_only_on_windows() { + let test_cases = [ + LocalRootTestCase { + description: "a Unix absolute path", + path: "/data/exports", + expected_windows: Some(("/", "data/exports")), + expected_unix: Some(("/", "data/exports")), + }, + LocalRootTestCase { + description: "a drive with backslashes, lower-case letter", + path: r"c:\Data\exports", + expected_windows: Some(("C:/", "Data/exports")), + expected_unix: None, + }, + LocalRootTestCase { + description: "a drive as a file:/// URL gives it", + path: "/D:/lake", + expected_windows: Some(("D:/", "lake")), + expected_unix: Some(("/", "D:/lake")), + }, + LocalRootTestCase { + description: "a UNC share", + path: r"\\nas01\exports\daily\orders", + expected_windows: Some(("//nas01/exports/", "daily/orders")), + expected_unix: None, + }, + LocalRootTestCase { + description: "a UNC share's root alone", + path: r"\\nas01\exports", + expected_windows: Some(("//nas01/exports/", "")), + expected_unix: None, + }, + LocalRootTestCase { + description: "an extended-length drive path", + path: r"\\?\C:\very\long", + expected_windows: Some(("C:/", "very/long")), + expected_unix: None, + }, + LocalRootTestCase { + description: "an extended-length UNC path", + path: r"\\?\UNC\nas01\exports\x", + expected_windows: Some(("//nas01/exports/", "x")), + expected_unix: None, + }, + LocalRootTestCase { + description: "a server without a share is no root", + path: r"\\nas01", + expected_windows: None, + expected_unix: None, + }, + LocalRootTestCase { + description: "a drive-relative path is relative", + path: "C:data", + expected_windows: None, + expected_unix: None, + }, + LocalRootTestCase { + description: "a relative path", + path: r"exports\daily", + expected_windows: None, + expected_unix: None, + }, + ]; + for test_case in test_cases { + for (windows, expected) in [ + (true, test_case.expected_windows), + (false, test_case.expected_unix), + ] { + let separated = windows_separators(test_case.path, windows); + let found = local_root(&separated, windows); + let found = found.as_ref().map(|(root, rest)| (root.as_str(), *rest)); + assert_eq!( + found, expected, + "{} (windows: {windows})", + test_case.description + ); + } + } +} + +#[test] +fn given_file_urls_when_reading_their_path_then_windows_drives_and_shares_are_found() { + let test_cases = [ + ("/data/x", "/data/x", "/data/x"), + ("/C:/data/x", "C:/data/x", "/C:/data/x"), + ("nas01/exports/x", "//nas01/exports/x", "nas01/exports/x"), + ]; + for (rest, expected_windows, expected_unix) in test_cases { + assert_eq!( + windows_file_url_path(rest, true), + expected_windows, + "{rest}" + ); + assert_eq!(windows_file_url_path(rest, false), expected_unix, "{rest}"); + } +} diff --git a/crates/sqlload/src/store_paths/tests/test_resolve_location.rs b/crates/sqlload/src/store_paths/tests/test_resolve_location.rs index be4de648..94249d6e 100644 --- a/crates/sqlload/src/store_paths/tests/test_resolve_location.rs +++ b/crates/sqlload/src/store_paths/tests/test_resolve_location.rs @@ -87,3 +87,86 @@ fn given_a_path_when_resolving_then_the_store_and_folder_are_found() { ); } } + +#[cfg(windows)] +#[test] +fn given_windows_paths_when_resolving_then_drives_and_shares_are_local_folders() { + let share = || { + Some(StoreKind::Filesystem { + root: r"\\nas01\exports".to_owned(), + }) + }; + let test_cases = [ + ResolveLocationTestCase { + description: "a drive path with backslashes", + connection: None, + path: r"C:\data\exports", + expected_location: Ok("C:/data/exports/"), + }, + ResolveLocationTestCase { + description: "a drive as a file:/// URL", + connection: None, + path: "file:///d:/lake/scores", + expected_location: Ok("D:/lake/scores/"), + }, + ResolveLocationTestCase { + description: "a UNC share", + connection: None, + path: r"\\nas01\exports\daily", + expected_location: Ok("//nas01/exports/daily/"), + }, + ResolveLocationTestCase { + description: "a share as a file:// URL", + connection: None, + path: "file://nas01/exports/daily", + expected_location: Ok("//nas01/exports/daily/"), + }, + ResolveLocationTestCase { + description: "a path under a share connection becomes relative to it", + connection: share(), + path: r"\\NAS01\exports\daily\orders", + expected_location: Ok(r"\\nas01\exports/daily/orders/"), + }, + ResolveLocationTestCase { + description: "a relative path with backslashes under a share connection", + connection: share(), + path: r"daily\orders", + expected_location: Ok(r"\\nas01\exports/daily/orders/"), + }, + ResolveLocationTestCase { + description: "another share is not under the connection", + connection: share(), + path: r"\\nas02\exports\daily", + expected_location: Err("connection \"lake\""), + }, + ResolveLocationTestCase { + description: "another drive is not under a drive connection", + connection: Some(StoreKind::Filesystem { + root: r"C:\lake".to_owned(), + }), + path: r"D:\lake\x", + expected_location: Err("connection \"lake\""), + }, + ResolveLocationTestCase { + description: "a share's root alone is no folder", + connection: None, + path: r"\\nas01\exports", + expected_location: Err("is empty"), + }, + ]; + for test_case in test_cases { + let connection = test_case.connection.as_ref().map(|store| ("lake", store)); + let result = resolve_location(connection, test_case.path) + .map(|location| location.to_string()) + .map_err(|error| error.to_string()); + match (&result, test_case.expected_location) { + (Ok(found), Ok(expected)) => assert_eq!(found, expected, "{}", test_case.description), + (Err(found), Err(expected)) => assert!( + found.contains(expected), + "{}: {found}", + test_case.description + ), + _ => panic!("{}: {result:?}", test_case.description), + } + } +} diff --git a/crates/sqlload/tests/loader/test_postgres_tls.rs b/crates/sqlload/tests/loader/test_postgres_tls.rs index ff80cc3f..8fbe5932 100644 --- a/crates/sqlload/tests/loader/test_postgres_tls.rs +++ b/crates/sqlload/tests/loader/test_postgres_tls.rs @@ -453,6 +453,7 @@ fn given_a_relay_when_its_session_is_taken_then_its_socket_folder_is_gone() -> R let folder = relay_tls(&address) .await .map_err(|error| error.to_string())? + .map(|relay| relay.host) .unwrap_or_default(); let socket = Path::new(&folder).join(format!(".s.PGSQL.{port}")); let mode = std::os::unix::fs::PermissionsExt::mode( diff --git a/website/src/content/docs/docs/cli/copy.mdx b/website/src/content/docs/docs/cli/copy.mdx index b454030c..eccb604d 100644 --- a/website/src/content/docs/docs/cli/copy.mdx +++ b/website/src/content/docs/docs/cli/copy.mdx @@ -31,6 +31,8 @@ Each end is a connection named in `sqlload.toml`, or a URI: | `s3://bucket/prefix/?region=eu-west-2` | Files in S3 | Parquet files in S3 | | `file:///folder`, or a folder | Files in a folder | Parquet files in a folder | +On Windows a folder can be a drive path (`C:\data\orders`) or a network share (`\\server\share\orders`), and the URL forms are `file:///C:/data/orders` and `file://server/share/orders`. + Snowflake needs `auth` and a `load_method`, which a URI can't carry, so a Snowflake end is always a named connection. diff --git a/website/src/content/docs/docs/configuration.mdx b/website/src/content/docs/docs/configuration.mdx index d0f11a9a..2e12cac0 100644 --- a/website/src/content/docs/docs/configuration.mdx +++ b/website/src/content/docs/docs/configuration.mdx @@ -143,7 +143,7 @@ delete, set `AWS_DISABLE_BULK_DELETE=true`: SQLLoad then deletes each file with | Key | Meaning | |-----|---------| -| `root` | A folder on the machine SQLLoad runs on. | +| `root` | A folder on the machine SQLLoad runs on. On Windows also a drive path (`'C:\data\lake'`) or a network share (`'\\server\share\lake'`); write it as a TOML literal string, in single quotes, so the backslashes stay. | ### `sftp` diff --git a/website/src/content/docs/docs/install.mdx b/website/src/content/docs/docs/install.mdx index b40fd7f1..1becc774 100644 --- a/website/src/content/docs/docs/install.mdx +++ b/website/src/content/docs/docs/install.mdx @@ -43,7 +43,8 @@ Both packages are released for: | Platform | Architectures | |----------|---------------| | Linux (glibc 2.28 or newer) | x86_64, aarch64 | -| macOS 11 or newer | arm64 | +| macOS 11 or newer | arm64, x86_64 (Intel) | +| Windows 10 or newer, Windows Server 2019 or newer | x86_64 | ## Use it from Python diff --git a/website/src/content/docs/docs/sources/postgres.mdx b/website/src/content/docs/docs/sources/postgres.mdx index 0535ef1f..4a79727f 100644 --- a/website/src/content/docs/docs/sources/postgres.mdx +++ b/website/src/content/docs/docs/sources/postgres.mdx @@ -163,6 +163,23 @@ libpq does by default. That stops an attacker who only relays the password excha who changes how the client signs in. Only `verify-full`, or `verify-ca` on a private network, keeps a man in the middle out. +### Replication sessions on Windows + +A `postgres_cdc` source's replication client speaks plain text only, so its session reaches the +database through a relay inside SQLLoad, which adds TLS and the SSH tunnel. On Linux and macOS the +relay is a Unix socket in a folder only your user can enter. On Windows the replication client can +only connect over TCP, so the relay listens on a random port on 127.0.0.1, for at most 30 seconds. + +- Any local process can connect to that port. SQLLoad makes a random secret for each session and + relays only the connection whose startup message carries it. It drops every other connection + before a byte of it reaches the database, gives each one 5 seconds to send its startup message, + and closes the port once its own client is in. The database never sees the secret. +- What remains: between the replication client and the relay, the sign-in exchange and then every + replicated row cross the loopback interface in plain text. SCRAM never sends the password + itself, but row data can be read by anyone who can capture loopback traffic. On Windows that + takes an administrator, who could also read SQLLoad's memory. +- From the relay to the database, the connection is encrypted as the TLS settings above say. + ## SSH tunnels From 0.27.0. A connection with an `ssh` table reaches the database through a bastion host. `host` @@ -170,9 +187,10 @@ and `port` are then the database as the bastion sees it. To set one up, see [Connect through an SSH tunnel](/docs/guides/postgres/#connect-through-an-ssh-tunnel). - SQLLoad opens the tunnel itself, with no `ssh` program needed. It keeps one session to the - bastion per run, shared by every table, and closes it when the run ends. It opens no local - TCP port: direct reads use the tunnel in process, and a `postgres_cdc` source's replication - session reaches it through a Unix socket in a folder only your user can enter. + bastion per run, shared by every table, and closes it when the run ends. Direct reads use the + tunnel in process. A `postgres_cdc` source's replication session reaches it through a Unix + socket in a folder only your user can enter, or on Windows through a loopback port guarded by a + secret (see [Replication sessions on Windows](#replication-sessions-on-windows)). - It signs in with `private_key_file` (with `private_key_passphrase` if the key has one), or with each key ssh-agent holds when no file is set. From 0.32.0, `password` signs in when no key file is set; before, passwords aren't supported. diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx index f5c209a8..62bd6272 100644 --- a/website/src/content/docs/docs/supported-systems.mdx +++ b/website/src/content/docs/docs/supported-systems.mdx @@ -78,5 +78,6 @@ them into DuckDB. ## Platforms The `sqlload` and `sqlload-cli` packages are released for Linux (x86_64 and aarch64, glibc 2.28 or -newer) and macOS 11 or newer (arm64). The `sqlload` package needs CPython 3.10 or newer. See +newer), macOS 11 or newer (arm64 and Intel x86_64) and Windows 10 or Windows Server 2019 or newer +(x86_64). The `sqlload` package needs CPython 3.10 or newer. See [Install](/docs/install/). From 450ef4823cf9875337506790f4c41b1d83b78d18 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:21:24 +0000 Subject: [PATCH 02/25] fix: admit the client after refused connections, and move the relay into session_relay A relay refusal no longer disables tokio::select!'s join arm, which could leave an admitted client unread until the relay's deadline when two refusals finished together; a test now opens refused connections first. The Unix socket-folder and Windows loopback relays live in session_relay. On Windows, folder names under a filesystem connection match ignoring case, and file://localhost/C:/ names the drive. Publish's PR trigger also covers the workspace manifest and lockfile. --- .github/workflows/publish.yml | 2 + .../src/adapters/files/main/table_path.rs | 3 +- .../sqlload/src/cdc/_helpers/snapshot_log.rs | 3 +- crates/sqlload/src/cdc/_helpers/stream.rs | 2 +- crates/sqlload/src/lib.rs | 1 + .../_helpers/loopback_relay.rs | 135 -------- .../src/postgres_source/_helpers/mod.rs | 2 - .../sqlload/src/postgres_source/constants.rs | 15 - .../src/postgres_source/main/relay_tls.rs | 63 +--- crates/sqlload/src/postgres_source/models.rs | 10 - crates/sqlload/src/postgres_source/tests.rs | 2 - .../src/postgres_source/tests/helpers.rs | 46 --- .../tests/test_loopback_relay.rs | 304 ------------------ .../src/postgres_source/tests/test_types.rs | 7 - .../src/session_relay/_helpers/loopback.rs | 102 ++++++ .../sqlload/src/session_relay/_helpers/mod.rs | 6 + .../session_relay/_helpers/socket_folder.rs | 36 +++ .../src/session_relay/_helpers/startup.rs | 51 +++ crates/sqlload/src/session_relay/constants.rs | 17 + crates/sqlload/src/session_relay/main/mod.rs | 1 + .../src/session_relay/main/relay_session.rs | 24 ++ crates/sqlload/src/session_relay/mod.rs | 9 + crates/sqlload/src/session_relay/models.rs | 9 + crates/sqlload/src/session_relay/tests.rs | 8 + .../src/session_relay/tests/helpers.rs | 192 +++++++++++ .../tests/test_admitted_startup.rs | 111 +++++++ .../src/session_relay/tests/test_loopback.rs | 54 ++++ .../src/session_relay/tests/test_types.rs | 29 ++ .../src/store_paths/_helpers/local_root.rs | 41 ++- .../src/store_paths/main/resolve_location.rs | 11 +- .../sqlload/src/store_paths/tests/helpers.rs | 7 + .../src/store_paths/tests/test_local_root.rs | 155 ++++++--- .../tests/test_resolve_location.rs | 32 +- .../src/store_paths/tests/test_types.rs | 26 ++ 34 files changed, 869 insertions(+), 647 deletions(-) delete mode 100644 crates/sqlload/src/postgres_source/_helpers/loopback_relay.rs delete mode 100644 crates/sqlload/src/postgres_source/tests/test_loopback_relay.rs create mode 100644 crates/sqlload/src/session_relay/_helpers/loopback.rs create mode 100644 crates/sqlload/src/session_relay/_helpers/mod.rs create mode 100644 crates/sqlload/src/session_relay/_helpers/socket_folder.rs create mode 100644 crates/sqlload/src/session_relay/_helpers/startup.rs create mode 100644 crates/sqlload/src/session_relay/constants.rs create mode 100644 crates/sqlload/src/session_relay/main/mod.rs create mode 100644 crates/sqlload/src/session_relay/main/relay_session.rs create mode 100644 crates/sqlload/src/session_relay/mod.rs create mode 100644 crates/sqlload/src/session_relay/models.rs create mode 100644 crates/sqlload/src/session_relay/tests.rs create mode 100644 crates/sqlload/src/session_relay/tests/helpers.rs create mode 100644 crates/sqlload/src/session_relay/tests/test_admitted_startup.rs create mode 100644 crates/sqlload/src/session_relay/tests/test_loopback.rs create mode 100644 crates/sqlload/src/session_relay/tests/test_types.rs diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index dea80311..eedc1248 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,6 +14,8 @@ on: - .github/workflows/publish.yml - .github/scripts/assert_*_wheel_tags.sh - packages/*/pyproject.toml + - Cargo.toml + - Cargo.lock - crates/sqlload/Cargo.toml - crates/sqlload-python/Cargo.toml diff --git a/crates/sqlload/src/adapters/files/main/table_path.rs b/crates/sqlload/src/adapters/files/main/table_path.rs index d5a77935..e35e1bea 100644 --- a/crates/sqlload/src/adapters/files/main/table_path.rs +++ b/crates/sqlload/src/adapters/files/main/table_path.rs @@ -7,7 +7,7 @@ use crate::store_paths::main::local_separators::local_separators; use crate::store_paths::main::render_template::render_template; use crate::store_paths::main::resolve_location::resolve_location; -/// The table's folder and partition folder: placeholders filled, the cursor allowed only in the last folder. +/// The table's folder and partition folder: placeholders filled, the cursor allowed only in the last folder. A leading `/` (or `//` before a Windows share) is kept as the root; a Windows drive (`C:`) is the first folder. pub fn table_path(request: &TablePathRequest<'_>) -> Result { let template = &*local_separators(request.template); if template.contains("://") { @@ -43,7 +43,6 @@ pub fn table_path(request: &TablePathRequest<'_>) -> Result { )); } }; - // `/`, or `//` before a Windows share; a drive (`C:`) is the first folder. let root = &template[..template.len() - template.trim_start_matches('/').len()]; let rendered = render_template(&format!("{root}{}", folder_segments.join("/")), &values) .map_err(|error| error.to_string())?; diff --git a/crates/sqlload/src/cdc/_helpers/snapshot_log.rs b/crates/sqlload/src/cdc/_helpers/snapshot_log.rs index d944e2cd..df5671bd 100644 --- a/crates/sqlload/src/cdc/_helpers/snapshot_log.rs +++ b/crates/sqlload/src/cdc/_helpers/snapshot_log.rs @@ -204,10 +204,9 @@ impl SnapshotLog<'_> { } } -/// The records folder of `source`'s slot `slot` in a files target whose path template is `template`: the template's folders up to the first one naming anything but `{source}`, with `{source}` filled (all but the last when every one is fixed, the table's own folder then), then `_sqlload/cdc///`. Outside every table's folder, and the same whichever tables the source selects; a table's own `destination_path` does not move it. +/// The records folder of `source`'s slot `slot` in a files target whose path template is `template`: the template's folders up to the first one naming anything but `{source}`, with `{source}` filled (all but the last when every one is fixed, the table's own folder then), then `_sqlload/cdc///`. Outside every table's folder, and the same whichever tables the source selects; a table's own `destination_path` does not move it. A leading `/` (or `//` before a Windows share) is kept as the root. pub(crate) fn snapshot_folder(template: &str, source: &str, slot: &str) -> String { let template = &*local_separators(template); - // `/`, or `//` before a Windows share; a drive (`C:`) is the first folder. let root = &template[..template.len() - template.trim_start_matches('/').len()]; let segments: Vec = template .split('/') diff --git a/crates/sqlload/src/cdc/_helpers/stream.rs b/crates/sqlload/src/cdc/_helpers/stream.rs index 50fc1d6a..f7188418 100644 --- a/crates/sqlload/src/cdc/_helpers/stream.rs +++ b/crates/sqlload/src/cdc/_helpers/stream.rs @@ -44,7 +44,6 @@ use crate::loader::models::{ CdcSpec, CommitRule, LoadRun, PlanOptions, ProjectPlan, Publisher, TableLoaded, TableOutcome, TablePlan, WholeReason, }; -use crate::postgres_source::constants::APPLICATION_NAME; use crate::postgres_source::constants::DEFAULT_BATCH_BYTES; use crate::postgres_source::errors::PostgresError; use crate::postgres_source::main::connect_postgres::connect_postgres; @@ -55,6 +54,7 @@ use crate::project::constants::DEFAULT_DESTINATION_PATH; use crate::project::main::postgres_address::postgres_address; use crate::project::models::{SourceConfig, TargetConfig}; use crate::schema_plan::constants::{CDC_LSN_COLUMN, CDC_PREFIX, CDC_UNCHANGED_COLUMN}; +use crate::session_relay::constants::APPLICATION_NAME; use crate::ssh_tunnel::main::tunnel_closed::tunnel_closed; /// How often the replication session reports the confirmed position. diff --git a/crates/sqlload/src/lib.rs b/crates/sqlload/src/lib.rs index 4955a06f..5f66d488 100644 --- a/crates/sqlload/src/lib.rs +++ b/crates/sqlload/src/lib.rs @@ -42,6 +42,7 @@ pub mod routing; pub mod row_batches; pub mod run_log; pub mod schema_plan; +pub mod session_relay; pub mod sheets_source; pub mod similar_names; pub mod snowflake_source; diff --git a/crates/sqlload/src/postgres_source/_helpers/loopback_relay.rs b/crates/sqlload/src/postgres_source/_helpers/loopback_relay.rs deleted file mode 100644 index e4b2e073..00000000 --- a/crates/sqlload/src/postgres_source/_helpers/loopback_relay.rs +++ /dev/null @@ -1,135 +0,0 @@ -//! A relay on a loopback port, where the replication client cannot reach a private socket (Windows). Any local user can connect to the port, so the relay forwards the session only to the connection whose startup message names this relay's secret as its application name, and nothing reaches the server before that check passes. - -use std::net::Ipv4Addr; - -use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; -use tokio::net::{TcpListener, TcpStream}; -use tokio::task::JoinSet; - -use crate::auth::models::Secret; -use crate::postgres_source::constants::{ - APPLICATION_NAME, PROTOCOL_VERSION, RELAY_ACCEPT_TIMEOUT, RELAY_STARTUP_TIMEOUT, - STARTUP_MESSAGE_LIMIT, -}; -use crate::postgres_source::models::RelayEndpoint; - -/// A port on 127.0.0.1 that pipes `server` to the first connection presenting the returned application name; it closes once that connection is admitted, or after [`RELAY_ACCEPT_TIMEOUT`] without one. Other connections are dropped, each after [`RELAY_STARTUP_TIMEOUT`] at most, without a byte of theirs reaching the server. -pub(crate) async fn loopback_relay(server: S) -> Result -where - S: AsyncRead + AsyncWrite + Unpin + Send + 'static, -{ - let mut secret = [0u8; 32]; - getrandom::fill(&mut secret).map_err(|error| error.to_string())?; - let expected = Secret(format!("{APPLICATION_NAME}-{}", hex::encode(secret))); - let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) - .await - .map_err(|error| error.to_string())?; - let port = listener - .local_addr() - .map_err(|error| error.to_string())? - .port(); - tokio::spawn(serve(listener, server, expected.clone())); - Ok(RelayEndpoint { - host: Ipv4Addr::LOCALHOST.to_string(), - port, - application_name: expected, - }) -} - -/// Admit connections to `listener` side by side, so a silent one holds up none of the others, and pipe `server` to the first admitted. -async fn serve(listener: TcpListener, mut server: S, expected: Secret) -where - S: AsyncRead + AsyncWrite + Unpin, -{ - let deadline = tokio::time::Instant::now() + RELAY_ACCEPT_TIMEOUT; - let mut pending = JoinSet::new(); - let (mut local, startup) = loop { - tokio::select! { - accepted = listener.accept() => { - if let Ok((stream, _)) = accepted { - pending.spawn(admit(stream, expected.clone())); - } - } - Some(Ok(Some(admitted))) = pending.join_next() => break admitted, - () = tokio::time::sleep_until(deadline) => return, - } - }; - drop(listener); - drop(pending); - if server.write_all(&startup).await.is_ok() { - let _ = tokio::io::copy_bidirectional(&mut local, &mut server).await; - } -} - -/// `stream` and the startup message to forward for it, when it sends one naming `expected` within [`RELAY_STARTUP_TIMEOUT`]. -async fn admit(mut stream: TcpStream, expected: Secret) -> Option<(TcpStream, Vec)> { - let message = tokio::time::timeout(RELAY_STARTUP_TIMEOUT, read_startup(&mut stream)) - .await - .ok()??; - let forwarded = admitted_startup(&message, expected.expose())?; - let _ = stream.set_nodelay(true); - Some((stream, forwarded)) -} - -/// One startup message: its length, and as many bytes as that gives, up to [`STARTUP_MESSAGE_LIMIT`]. -async fn read_startup(stream: &mut R) -> Option> { - let mut length = [0u8; 4]; - stream.read_exact(&mut length).await.ok()?; - let declared = usize::try_from(u32::from_be_bytes(length)).ok()?; - if !(8..=STARTUP_MESSAGE_LIMIT).contains(&declared) { - return None; - } - let mut message = vec![0; declared]; - message[..4].copy_from_slice(&length); - stream.read_exact(&mut message[4..]).await.ok()?; - Some(message) -} - -/// `message`, a whole protocol 3.0 startup message, with its one `application_name` replaced by [`APPLICATION_NAME`]; `None` unless that name is `expected`. -pub(crate) fn admitted_startup(message: &[u8], expected: &str) -> Option> { - let declared = usize::try_from(u32::from_be_bytes(message.get(..4)?.try_into().ok()?)).ok()?; - if declared != message.len() || message.get(4..8)? != PROTOCOL_VERSION { - return None; - } - // Pairs of NUL-terminated names and values, then one more NUL. - let fields: Vec<&[u8]> = message[8..] - .strip_suffix(b"\0\0")? - .split(|byte| *byte == 0) - .collect(); - if !fields.len().is_multiple_of(2) || fields.iter().step_by(2).any(|name| name.is_empty()) { - return None; - } - let named: Vec = (0..fields.len()) - .step_by(2) - .filter(|index| fields[*index] == b"application_name") - .collect(); - let [name] = named.as_slice() else { - return None; - }; - if !same_secret(fields[name + 1], expected.as_bytes()) { - return None; - } - let mut forwarded = vec![0; 4]; - forwarded.extend_from_slice(&PROTOCOL_VERSION); - for (index, field) in fields.iter().enumerate() { - match index == name + 1 { - true => forwarded.extend_from_slice(APPLICATION_NAME.as_bytes()), - false => forwarded.extend_from_slice(field), - } - forwarded.push(0); - } - forwarded.push(0); - let length = u32::try_from(forwarded.len()).ok()?; - forwarded[..4].copy_from_slice(&length.to_be_bytes()); - Some(forwarded) -} - -/// Whether `given` is `expected`, comparing every byte whatever the first difference, so the time taken tells nothing of the secret. -fn same_secret(given: &[u8], expected: &[u8]) -> bool { - given.len() == expected.len() - && given - .iter() - .zip(expected) - .fold(0u8, |difference, (left, right)| difference | (left ^ right)) - == 0 -} diff --git a/crates/sqlload/src/postgres_source/_helpers/mod.rs b/crates/sqlload/src/postgres_source/_helpers/mod.rs index ca8074c8..d329a9dd 100644 --- a/crates/sqlload/src/postgres_source/_helpers/mod.rs +++ b/crates/sqlload/src/postgres_source/_helpers/mod.rs @@ -2,8 +2,6 @@ pub(crate) mod builders; pub(crate) mod catalog; pub(crate) mod copy_decoder; pub(crate) mod copy_stream; -#[cfg(any(windows, test))] -pub(crate) mod loopback_relay; pub(crate) mod rustls_connect; pub(crate) mod server_reason; pub(crate) mod split_read; diff --git a/crates/sqlload/src/postgres_source/constants.rs b/crates/sqlload/src/postgres_source/constants.rs index 9c6e1cea..e5584da1 100644 --- a/crates/sqlload/src/postgres_source/constants.rs +++ b/crates/sqlload/src/postgres_source/constants.rs @@ -123,21 +123,6 @@ pub const SPLIT_READ_MIN_PAGES: i64 = 16_384; /// Postgres's `SSLRequest`: length 8, then code 80877103. pub const SSL_REQUEST: [u8; 8] = [0, 0, 0, 8, 0x04, 0xd2, 0x16, 0x2f]; -/// How long a TLS relay waits for its replication client to connect. -pub const RELAY_ACCEPT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); - -/// How long a loopback relay gives each connection to send its whole startup message, so a silent or slow one is dropped without holding up the replication client. -pub const RELAY_STARTUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); - -/// The longest startup message a loopback relay reads: Postgres's own limit (`MAX_STARTUP_PACKET_LENGTH`). -pub const STARTUP_MESSAGE_LIMIT: usize = 10_000; - -/// Protocol 3.0, as the replication client's startup message gives it. -pub const PROTOCOL_VERSION: [u8; 4] = [0, 3, 0, 0]; - -/// The application name SQLLoad's sessions show the server. -pub const APPLICATION_NAME: &str = "sqlload"; - /// Signature algorithm OIDs (DER contents) whose certificates bind with SHA-256: sha256WithRSAEncryption, sha1WithRSAEncryption, md5WithRSAEncryption, ecdsa-with-SHA256, ecdsa-with-SHA1. pub const SHA256_SIGNATURES: &[&[u8]] = &[ &[0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x01, 0x0b], diff --git a/crates/sqlload/src/postgres_source/main/relay_tls.rs b/crates/sqlload/src/postgres_source/main/relay_tls.rs index 6d8b10ec..232255c4 100644 --- a/crates/sqlload/src/postgres_source/main/relay_tls.rs +++ b/crates/sqlload/src/postgres_source/main/relay_tls.rs @@ -1,22 +1,15 @@ //! TLS for a client that speaks only plain text: the replication client. -#[cfg(unix)] -use std::os::unix::fs::PermissionsExt; - use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tokio::net::TcpStream; -#[cfg(unix)] -use tokio::net::UnixListener; use tokio_rustls::TlsConnector; -#[cfg(windows)] -use crate::postgres_source::_helpers::loopback_relay::loopback_relay; use crate::postgres_source::_helpers::tls::client_config; use crate::postgres_source::constants::SSL_REQUEST; -#[cfg(unix)] -use crate::postgres_source::constants::{APPLICATION_NAME, RELAY_ACCEPT_TIMEOUT}; use crate::postgres_source::errors::PostgresError; -use crate::postgres_source::models::{PostgresAddress, RelayEndpoint, SslMode}; +use crate::postgres_source::models::{PostgresAddress, SslMode}; +use crate::session_relay::main::relay_session::relay_session; +use crate::session_relay::models::RelayEndpoint; use crate::ssh_tunnel::main::open_channel::open_channel; /// When `address` may use TLS or goes through a bastion: where the replication client connects (a private socket folder whose `.s.PGSQL.` socket, or on Windows a loopback port, carries the next session opened on it to the server, encrypted, or in plain text when the server refuses TLS under `prefer` (one connection either way, so the server logs no abandoned one), and the application name it must send. `None` to connect directly: `sslmode = "disable"`, a Unix socket host, or a failed handshake under `prefer`, which libpq also retries in plain text. Through a bastion it never connects directly: that retry opens a fresh channel. @@ -35,7 +28,9 @@ pub async fn relay_tls( } } let server = open_channel(tunnel, address.host, address.port).await?; - return Ok(Some(relay(server, address.port).await.map_err(failed)?)); + return Ok(Some( + relay_session(server, address.port).await.map_err(failed)?, + )); } if address.tls.mode == SslMode::Disable || address.host.starts_with('/') { return Ok(None); @@ -73,7 +68,9 @@ where != b'S' { return match address.tls.mode { - SslMode::Prefer => Ok(Some(relay(server, address.port).await.map_err(failed)?)), + SslMode::Prefer => Ok(Some( + relay_session(server, address.port).await.map_err(failed)?, + )), _ => Err(failed("the server does not offer TLS".to_owned())), }; } @@ -84,43 +81,7 @@ where Err(_) if address.tls.mode == SslMode::Prefer => return Ok(None), Err(error) => return Err(failed(format!("TLS handshake failed: {error}"))), }; - Ok(Some(relay(session, address.port).await.map_err(failed)?)) -} - -/// A folder only this user can enter (some systems ignore a socket's own permissions) whose `.s.PGSQL.` socket pipes the first session opened on it through `server`; the folder and its socket go once it is accepted, or after [`RELAY_ACCEPT_TIMEOUT`] without one. -#[cfg(unix)] -async fn relay(mut server: S, port: u16) -> Result -where - S: AsyncRead + AsyncWrite + Unpin + Send + 'static, -{ - let folder = tempfile::Builder::new() - .prefix("sqlload-") - .permissions(std::fs::Permissions::from_mode(0o700)) - .tempdir() - .map_err(|error| error.to_string())?; - let listener = UnixListener::bind(folder.path().join(format!(".s.PGSQL.{port}"))) - .map_err(|error| error.to_string())?; - let host = folder.path().to_string_lossy().into_owned(); - tokio::spawn(async move { - let accepted = tokio::time::timeout(RELAY_ACCEPT_TIMEOUT, listener.accept()).await; - drop(listener); - drop(folder); - if let Ok(Ok((mut local, _))) = accepted { - let _ = tokio::io::copy_bidirectional(&mut local, &mut server).await; - } - }); - Ok(RelayEndpoint { - host, - port, - application_name: crate::auth::models::Secret(APPLICATION_NAME.to_owned()), - }) -} - -/// pgwire-replication reaches no socket file on Windows, only TCP, so the session goes through a loopback port that admits only this relay's secret (see [`loopback_relay`]). -#[cfg(windows)] -async fn relay(server: S, _port: u16) -> Result -where - S: AsyncRead + AsyncWrite + Unpin + Send + 'static, -{ - loopback_relay(server).await + Ok(Some( + relay_session(session, address.port).await.map_err(failed)?, + )) } diff --git a/crates/sqlload/src/postgres_source/models.rs b/crates/sqlload/src/postgres_source/models.rs index c1e8ccb0..950b3013 100644 --- a/crates/sqlload/src/postgres_source/models.rs +++ b/crates/sqlload/src/postgres_source/models.rs @@ -192,13 +192,3 @@ pub struct CopyRequest<'a> { /// More connections to the same database, idle and outside any transaction, to read a large table in parallel; empty reads in one pass. pub helpers: &'a [tokio_postgres::Client], } - -/// Where the replication client reaches a relayed session, and the application name its startup message must carry there. -#[derive(Debug)] -pub struct RelayEndpoint { - /// The relay's socket folder (Unix) or `127.0.0.1` (Windows). - pub host: String, - pub port: u16, - /// `sqlload` on Unix. On Windows `sqlload-`, which the relay checks and replaces with `sqlload` before the server sees it. - pub application_name: crate::auth::models::Secret, -} diff --git a/crates/sqlload/src/postgres_source/tests.rs b/crates/sqlload/src/postgres_source/tests.rs index 03a616d3..934df596 100644 --- a/crates/sqlload/src/postgres_source/tests.rs +++ b/crates/sqlload/src/postgres_source/tests.rs @@ -4,8 +4,6 @@ mod copy_decoder; mod decode_copy; #[path = "tests/helpers.rs"] mod helpers; -#[path = "tests/test_loopback_relay.rs"] -mod loopback_relay; #[path = "tests/test_read_column.rs"] mod read_column; #[path = "tests/test_read_numeric.rs"] diff --git a/crates/sqlload/src/postgres_source/tests/helpers.rs b/crates/sqlload/src/postgres_source/tests/helpers.rs index dbe21b06..4cd42e8b 100644 --- a/crates/sqlload/src/postgres_source/tests/helpers.rs +++ b/crates/sqlload/src/postgres_source/tests/helpers.rs @@ -96,49 +96,3 @@ pub(crate) fn source_column(pg_type: PgType) -> SourceColumn { source_spelling: None, } } - -/// A startup message of protocol `version` with `params`, framed as libpq and pgwire-replication frame it. -pub(crate) fn startup_message(version: [u8; 4], params: &[(&str, &str)]) -> Vec { - let mut body = version.to_vec(); - for (name, value) in params { - body.extend_from_slice(name.as_bytes()); - body.push(0); - body.extend_from_slice(value.as_bytes()); - body.push(0); - } - body.push(0); - let length = u32::try_from(body.len() + 4).unwrap_or(u32::MAX); - let mut message = length.to_be_bytes().to_vec(); - message.extend_from_slice(&body); - message -} - -/// The parameters of a well-formed protocol 3.0 startup message, in order; an error naming what is wrong otherwise. -pub(crate) fn startup_params(message: &[u8]) -> Result, String> { - let declared = message - .get(..4) - .and_then(|bytes| <[u8; 4]>::try_from(bytes).ok()) - .map(u32::from_be_bytes) - .ok_or("no length")?; - if usize::try_from(declared).ok() != Some(message.len()) { - return Err(format!("length {declared} for {} bytes", message.len())); - } - if message.get(4..8) != Some(&[0, 3, 0, 0][..]) { - return Err("not protocol 3.0".to_owned()); - } - let text = String::from_utf8(message[8..].to_vec()).map_err(|error| error.to_string())?; - let mut parts: Vec<&str> = text.split('\0').collect(); - // The pairs' terminators and the final NUL leave two empty parts at the end. - if parts.len() < 2 || parts.pop() != Some("") || parts.pop() != Some("") { - return Err("not NUL-terminated".to_owned()); - } - Ok(parts - .chunks(2) - .map(|pair| { - ( - pair[0].to_owned(), - pair.get(1).copied().unwrap_or_default().to_owned(), - ) - }) - .collect()) -} diff --git a/crates/sqlload/src/postgres_source/tests/test_loopback_relay.rs b/crates/sqlload/src/postgres_source/tests/test_loopback_relay.rs deleted file mode 100644 index 8a2645c2..00000000 --- a/crates/sqlload/src/postgres_source/tests/test_loopback_relay.rs +++ /dev/null @@ -1,304 +0,0 @@ -use std::time::{Duration, Instant}; - -use tokio::io::{AsyncReadExt, AsyncWriteExt, DuplexStream}; -use tokio::net::TcpStream; - -use crate::postgres_source::_helpers::loopback_relay::{admitted_startup, loopback_relay}; -use crate::postgres_source::constants::{PROTOCOL_VERSION, RELAY_STARTUP_TIMEOUT}; -use crate::postgres_source::models::RelayEndpoint; -use crate::postgres_source::tests::helpers::{startup_message, startup_params}; -use crate::postgres_source::tests::test_types::AdmittedStartupTestCase; - -const SECRET: &str = "sqlload-6f1c0d2a9b8e7f6a5b4c3d2e1f0a9b8c7d6e5f4a3b2c1d0e9f8a7b6c5d4e3f2a"; - -fn block_on(body: impl std::future::Future>) -> Result<(), String> { - tokio::runtime::Builder::new_multi_thread() - .worker_threads(2) - .enable_all() - .build() - .map_err(|error| error.to_string())? - .block_on(body) -} - -fn owned(params: &[(&str, &str)]) -> Vec<(String, String)> { - params - .iter() - .map(|(name, value)| ((*name).to_owned(), (*value).to_owned())) - .collect() -} - -/// The replication client's startup message, naming `application_name`. -fn client_startup(application_name: &str) -> Vec { - startup_message( - PROTOCOL_VERSION, - &[ - ("user", "replicator"), - ("database", "shop"), - ("replication", "database"), - ("application_name", application_name), - ], - ) -} - -/// The whole first message the server side of the relay received. -async fn read_forwarded(upstream: &mut DuplexStream) -> Result, String> { - let mut length = [0u8; 4]; - upstream - .read_exact(&mut length) - .await - .map_err(|error| error.to_string())?; - let declared = - usize::try_from(u32::from_be_bytes(length)).map_err(|error| error.to_string())?; - let mut message = length.to_vec(); - message.resize(declared.max(4), 0); - upstream - .read_exact(&mut message[4..]) - .await - .map_err(|error| error.to_string())?; - Ok(message) -} - -fn contains(haystack: &[u8], needle: &str) -> bool { - haystack - .windows(needle.len()) - .any(|window| window == needle.as_bytes()) -} - -async fn connect(endpoint: &RelayEndpoint) -> Result { - TcpStream::connect((endpoint.host.as_str(), endpoint.port)) - .await - .map_err(|error| error.to_string()) -} - -/// Whether the relay closes `stream` without answering, within `wait`. -async fn closed_unanswered(stream: &mut TcpStream, wait: Duration) -> Result { - let mut byte = [0u8; 1]; - match tokio::time::timeout(wait, stream.read(&mut byte)).await { - Ok(Ok(0) | Err(_)) => Ok(true), - Ok(Ok(_)) => Ok(false), - Err(_) => Err("the relay kept the connection open".to_owned()), - } -} - -#[test] -fn given_startup_messages_when_the_relay_checks_them_then_only_the_secret_is_admitted_and_never_forwarded() --> Result<(), String> { - let wrong_last = format!("{}0", &SECRET[..SECRET.len() - 1]); - let mut short_length = client_startup(SECRET); - short_length[3] -= 1; - let mut no_terminator = client_startup(SECRET); - no_terminator.pop(); - no_terminator[3] -= 1; - let test_cases = [ - AdmittedStartupTestCase { - description: "the secret, among the other parameters", - message: client_startup(SECRET), - expected_forwarded: Some(owned(&[ - ("user", "replicator"), - ("database", "shop"), - ("replication", "database"), - ("application_name", "sqlload"), - ])), - }, - AdmittedStartupTestCase { - description: "the secret first, before an empty value", - message: startup_message( - PROTOCOL_VERSION, - &[("application_name", SECRET), ("options", ""), ("user", "u")], - ), - expected_forwarded: Some(owned(&[ - ("application_name", "sqlload"), - ("options", ""), - ("user", "u"), - ])), - }, - AdmittedStartupTestCase { - description: "a secret wrong in its last character", - message: client_startup(&wrong_last), - expected_forwarded: None, - }, - AdmittedStartupTestCase { - description: "a prefix of the secret", - message: client_startup(&SECRET[..SECRET.len() - 1]), - expected_forwarded: None, - }, - AdmittedStartupTestCase { - description: "the secret with more after it", - message: client_startup(&format!("{SECRET}0")), - expected_forwarded: None, - }, - AdmittedStartupTestCase { - description: "the plain application name", - message: client_startup("sqlload"), - expected_forwarded: None, - }, - AdmittedStartupTestCase { - description: "no application name", - message: startup_message(PROTOCOL_VERSION, &[("user", "u"), ("database", "d")]), - expected_forwarded: None, - }, - AdmittedStartupTestCase { - description: "the secret, then a second application name", - message: startup_message( - PROTOCOL_VERSION, - &[("application_name", SECRET), ("application_name", "other")], - ), - expected_forwarded: None, - }, - AdmittedStartupTestCase { - description: "an SSLRequest", - message: vec![0, 0, 0, 8, 0x04, 0xd2, 0x16, 0x2f], - expected_forwarded: None, - }, - AdmittedStartupTestCase { - description: "protocol 2.0 with the secret", - message: startup_message([0, 2, 0, 0], &[("application_name", SECRET)]), - expected_forwarded: None, - }, - AdmittedStartupTestCase { - description: "a length one short of the message", - message: short_length, - expected_forwarded: None, - }, - AdmittedStartupTestCase { - description: "no final NUL", - message: no_terminator, - expected_forwarded: None, - }, - ]; - for test_case in test_cases { - let forwarded = admitted_startup(&test_case.message, SECRET); - let found = forwarded.as_deref().map(startup_params).transpose()?; - if found != test_case.expected_forwarded { - return Err(format!( - "{}: forwarded {found:?}, expected {:?}", - test_case.description, test_case.expected_forwarded - )); - } - if forwarded.is_some_and(|bytes| contains(&bytes, &SECRET["sqlload-".len()..])) { - return Err(format!( - "{}: the secret was forwarded", - test_case.description - )); - } - } - Ok(()) -} - -#[test] -fn given_a_silent_connection_first_when_the_replication_client_connects_then_it_is_relayed_without_waiting() --> Result<(), String> { - block_on(async { - let (server, mut upstream) = tokio::io::duplex(64 * 1024); - let endpoint = loopback_relay(server).await?; - let _silent = connect(&endpoint).await?; - let started = Instant::now(); - let mut client = connect(&endpoint).await?; - client - .write_all(&client_startup(endpoint.application_name.expose())) - .await - .map_err(|error| error.to_string())?; - let forwarded = tokio::time::timeout(RELAY_STARTUP_TIMEOUT, read_forwarded(&mut upstream)) - .await - .map_err(|_| "the client was held up behind the silent connection".to_owned())??; - if started.elapsed() >= RELAY_STARTUP_TIMEOUT { - return Err(format!("relayed after {:?}", started.elapsed())); - } - let name = startup_params(&forwarded)? - .into_iter() - .find(|(name, _)| name == "application_name"); - if name != Some(("application_name".to_owned(), "sqlload".to_owned())) { - return Err(format!("forwarded application name {name:?}")); - } - // The session is piped both ways once admitted. - upstream - .write_all(b"R") - .await - .map_err(|error| error.to_string())?; - let mut answer = [0u8; 1]; - client - .read_exact(&mut answer) - .await - .map_err(|error| error.to_string())?; - client - .write_all(b"p") - .await - .map_err(|error| error.to_string())?; - upstream - .read_exact(&mut answer) - .await - .map_err(|error| error.to_string())?; - match &answer { - b"p" => Ok(()), - other => Err(format!("piped {other:?}")), - } - }) -} - -#[test] -fn given_connections_without_the_secret_when_they_connect_then_they_are_dropped_and_the_server_receives_nothing() --> Result<(), String> { - block_on(async { - let (server, mut upstream) = tokio::io::duplex(64 * 1024); - let endpoint = loopback_relay(server).await?; - let secret = endpoint.application_name.expose().to_owned(); - if format!("{endpoint:?}").contains(&secret) { - return Err("the relay's Debug shows its secret".to_owned()); - } - let wrong = format!("sqlload-{}", "0".repeat(64)); - let attempts: [(&str, Vec); 4] = [ - ("a wrong secret", client_startup(&wrong)), - ("the plain application name", client_startup("sqlload")), - ("an oversize length", vec![0, 1, 0, 0, 0, 3, 0, 0]), - ("an SSLRequest", vec![0, 0, 0, 8, 0x04, 0xd2, 0x16, 0x2f]), - ]; - for (description, bytes) in attempts { - let mut stream = connect(&endpoint).await?; - stream - .write_all(&bytes) - .await - .map_err(|error| error.to_string())?; - if !closed_unanswered(&mut stream, RELAY_STARTUP_TIMEOUT * 2) - .await - .map_err(|error| format!("{description}: {error}"))? - { - return Err(format!("{description}: the relay answered")); - } - } - let mut byte = [0u8; 1]; - if tokio::time::timeout(Duration::from_millis(300), upstream.read(&mut byte)) - .await - .is_ok() - { - return Err("the server received bytes from a refused connection".to_owned()); - } - let mut client = connect(&endpoint).await?; - client - .write_all(&client_startup(&secret)) - .await - .map_err(|error| error.to_string())?; - let forwarded = read_forwarded(&mut upstream).await?; - if contains(&forwarded, &secret["sqlload-".len()..]) { - return Err("the secret reached the server".to_owned()); - } - let expected = owned(&[ - ("user", "replicator"), - ("database", "shop"), - ("replication", "database"), - ("application_name", "sqlload"), - ]); - let found = startup_params(&forwarded)?; - if found != expected { - return Err(format!("forwarded {found:?}")); - } - // Admitted once: the port is closed to every later connection. - match tokio::time::timeout(Duration::from_secs(10), connect(&endpoint)).await { - Ok(Err(_)) => Ok(()), - Ok(Ok(mut late)) => match closed_unanswered(&mut late, Duration::from_secs(2)).await { - Ok(true) => Ok(()), - _ => Err("a connection after the admitted one was kept".to_owned()), - }, - Err(_) => Err("connecting after admission hung".to_owned()), - } - }) -} diff --git a/crates/sqlload/src/postgres_source/tests/test_types.rs b/crates/sqlload/src/postgres_source/tests/test_types.rs index 87899d09..4443f9d2 100644 --- a/crates/sqlload/src/postgres_source/tests/test_types.rs +++ b/crates/sqlload/src/postgres_source/tests/test_types.rs @@ -37,10 +37,3 @@ pub(crate) struct ReadColumnTestCase { /// The select expression, or the error text. pub(crate) expected_result: Result<&'static str, &'static str>, } - -pub(crate) struct AdmittedStartupTestCase { - pub(crate) description: &'static str, - pub(crate) message: Vec, - /// The parameters forwarded to the server, in order; `None` when the connection is refused. - pub(crate) expected_forwarded: Option>, -} diff --git a/crates/sqlload/src/session_relay/_helpers/loopback.rs b/crates/sqlload/src/session_relay/_helpers/loopback.rs new file mode 100644 index 00000000..eb4650a2 --- /dev/null +++ b/crates/sqlload/src/session_relay/_helpers/loopback.rs @@ -0,0 +1,102 @@ +//! A relay on a loopback port, where the client cannot reach a private socket (Windows). Any local user can connect to the port, so the relay forwards the session only to the connection whose startup message names this relay's secret as its application name, and nothing reaches the server before that check passes. + +use std::net::Ipv4Addr; + +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::task::JoinSet; + +use crate::auth::models::Secret; +use crate::session_relay::_helpers::startup::admitted_startup; +use crate::session_relay::constants::{ + APPLICATION_NAME, RELAY_ACCEPT_TIMEOUT, RELAY_STARTUP_TIMEOUT, STARTUP_MESSAGE_LIMIT, + STARTUP_MESSAGE_MINIMUM, +}; +use crate::session_relay::models::RelayEndpoint; + +/// A port on 127.0.0.1 that pipes `server` to the first connection presenting the returned application name; it closes once that connection is admitted, or after [`RELAY_ACCEPT_TIMEOUT`] without one. Other connections are dropped, each after [`RELAY_STARTUP_TIMEOUT`] at most, without a byte of theirs reaching the server. +pub(crate) async fn loopback_relay(server: S) -> Result +where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + let mut secret = [0u8; 32]; + getrandom::fill(&mut secret).map_err(|error| error.to_string())?; + let expected = Secret(format!("{APPLICATION_NAME}-{}", hex::encode(secret))); + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .await + .map_err(|error| error.to_string())?; + let port = listener + .local_addr() + .map_err(|error| error.to_string())? + .port(); + tokio::spawn(serve(listener, server, expected.clone())); + Ok(RelayEndpoint { + host: Ipv4Addr::LOCALHOST.to_string(), + port, + application_name: expected, + }) +} + +/// Admit connections to `listener` side by side, so a silent one holds up none of the others, and pipe `server` to the first admitted. A refused connection's result is matched inside its arm: a pattern on the arm itself would disable it for the rest of the `select!` after one refusal. +async fn serve(listener: TcpListener, mut server: S, expected: Secret) +where + S: AsyncRead + AsyncWrite + Unpin, +{ + let deadline = tokio::time::Instant::now() + RELAY_ACCEPT_TIMEOUT; + let mut pending = JoinSet::new(); + let (mut local, startup) = loop { + tokio::select! { + accepted = listener.accept() => { + if let Ok((stream, _)) = accepted { + pending.spawn(admit(stream, expected.clone())); + } + } + Some(joined) = pending.join_next() => { + if let Ok(Ok(admitted)) = joined { + break admitted; + } + } + () = tokio::time::sleep_until(deadline) => return, + } + }; + drop(listener); + drop(pending); + let _ = async { + server.write_all(&startup).await?; + tokio::io::copy_bidirectional(&mut local, &mut server).await + } + .await; +} + +/// `stream` and the startup message to forward for it, when it sends one naming `expected` within [`RELAY_STARTUP_TIMEOUT`]; why it is refused otherwise. +async fn admit(mut stream: TcpStream, expected: Secret) -> Result<(TcpStream, Vec), String> { + let message = tokio::time::timeout(RELAY_STARTUP_TIMEOUT, read_startup(&mut stream)) + .await + .map_err(|_| "no startup message in time".to_owned())??; + let forwarded = admitted_startup(&message, expected.expose())?; + stream + .set_nodelay(true) + .map_err(|error| error.to_string())?; + Ok((stream, forwarded)) +} + +/// One startup message: its length, and as many bytes as that gives, from [`STARTUP_MESSAGE_MINIMUM`] to [`STARTUP_MESSAGE_LIMIT`]. +async fn read_startup(stream: &mut R) -> Result, String> { + let mut length = [0u8; 4]; + stream + .read_exact(&mut length) + .await + .map_err(|error| error.to_string())?; + let declared = + usize::try_from(u32::from_be_bytes(length)).map_err(|error| error.to_string())?; + if !(STARTUP_MESSAGE_MINIMUM..=STARTUP_MESSAGE_LIMIT).contains(&declared) { + return Err(format!("a startup message of {declared} bytes")); + } + let mut message = vec![0; declared]; + message[..4].copy_from_slice(&length); + stream + .read_exact(&mut message[4..]) + .await + .map_err(|error| error.to_string())?; + Ok(message) +} diff --git a/crates/sqlload/src/session_relay/_helpers/mod.rs b/crates/sqlload/src/session_relay/_helpers/mod.rs new file mode 100644 index 00000000..acc6568f --- /dev/null +++ b/crates/sqlload/src/session_relay/_helpers/mod.rs @@ -0,0 +1,6 @@ +#[cfg(any(windows, test))] +pub(crate) mod loopback; +#[cfg(unix)] +pub(crate) mod socket_folder; +#[cfg(any(windows, test))] +pub(crate) mod startup; diff --git a/crates/sqlload/src/session_relay/_helpers/socket_folder.rs b/crates/sqlload/src/session_relay/_helpers/socket_folder.rs new file mode 100644 index 00000000..9846e61e --- /dev/null +++ b/crates/sqlload/src/session_relay/_helpers/socket_folder.rs @@ -0,0 +1,36 @@ +use std::os::unix::fs::PermissionsExt; + +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::net::UnixListener; + +use crate::auth::models::Secret; +use crate::session_relay::constants::{APPLICATION_NAME, RELAY_ACCEPT_TIMEOUT}; +use crate::session_relay::models::RelayEndpoint; + +/// A folder only this user can enter (some systems ignore a socket's own permissions) whose `.s.PGSQL.` socket pipes the first session opened on it through `server`; the folder and its socket go once it is accepted, or after [`RELAY_ACCEPT_TIMEOUT`] without one. +pub(crate) fn socket_folder_relay(mut server: S, port: u16) -> Result +where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + let folder = tempfile::Builder::new() + .prefix("sqlload-") + .permissions(std::fs::Permissions::from_mode(0o700)) + .tempdir() + .map_err(|error| error.to_string())?; + let listener = UnixListener::bind(folder.path().join(format!(".s.PGSQL.{port}"))) + .map_err(|error| error.to_string())?; + let host = folder.path().to_string_lossy().into_owned(); + tokio::spawn(async move { + let accepted = tokio::time::timeout(RELAY_ACCEPT_TIMEOUT, listener.accept()).await; + drop(listener); + drop(folder); + if let Ok(Ok((mut local, _))) = accepted { + let _ = tokio::io::copy_bidirectional(&mut local, &mut server).await; + } + }); + Ok(RelayEndpoint { + host, + port, + application_name: Secret(APPLICATION_NAME.to_owned()), + }) +} diff --git a/crates/sqlload/src/session_relay/_helpers/startup.rs b/crates/sqlload/src/session_relay/_helpers/startup.rs new file mode 100644 index 00000000..3e40ec02 --- /dev/null +++ b/crates/sqlload/src/session_relay/_helpers/startup.rs @@ -0,0 +1,51 @@ +use crate::session_relay::constants::{APPLICATION_NAME, PROTOCOL_VERSION}; + +/// `message`, a whole protocol 3.0 startup message (its length, the version, then pairs of NUL-terminated names and values and one more NUL), with its one `application_name` replaced by [`APPLICATION_NAME`]; refused unless that name is `expected`. The forwarded message is rebuilt from the parsed fields, so it is well formed whatever was sent. +pub(crate) fn admitted_startup(message: &[u8], expected: &str) -> Result, &'static str> { + let declared = u32::from_be_bytes(*message.first_chunk::<4>().ok_or("no length")?); + let length = u32::try_from(message.len()).map_err(|_| "too long")?; + if declared != length || message.get(4..8) != Some(&PROTOCOL_VERSION[..]) { + return Err("not one protocol 3.0 startup message"); + } + let fields: Vec<&[u8]> = message[8..] + .strip_suffix(b"\0\0") + .ok_or("not NUL-terminated")? + .split(|byte| *byte == 0) + .collect(); + if !fields.len().is_multiple_of(2) || fields.iter().step_by(2).any(|name| name.is_empty()) { + return Err("not pairs of names and values"); + } + let named: Vec = (0..fields.len()) + .step_by(2) + .filter(|index| fields[*index] == b"application_name") + .collect(); + let [name] = named.as_slice() else { + return Err("not one application name"); + }; + if !same_secret(fields[name + 1], expected.as_bytes()) { + return Err("not this relay's application name"); + } + let mut forwarded = vec![0; 4]; + forwarded.extend_from_slice(&PROTOCOL_VERSION); + for (index, field) in fields.iter().enumerate() { + match index == name + 1 { + true => forwarded.extend_from_slice(APPLICATION_NAME.as_bytes()), + false => forwarded.extend_from_slice(field), + } + forwarded.push(0); + } + forwarded.push(0); + let length = u32::try_from(forwarded.len()).map_err(|_| "too long")?; + forwarded[..4].copy_from_slice(&length.to_be_bytes()); + Ok(forwarded) +} + +/// Whether `given` is `expected`, comparing every byte whatever the first difference, so the time taken tells nothing of the secret. +fn same_secret(given: &[u8], expected: &[u8]) -> bool { + given.len() == expected.len() + && given + .iter() + .zip(expected) + .fold(0u8, |difference, (left, right)| difference | (left ^ right)) + == 0 +} diff --git a/crates/sqlload/src/session_relay/constants.rs b/crates/sqlload/src/session_relay/constants.rs new file mode 100644 index 00000000..de4b46ad --- /dev/null +++ b/crates/sqlload/src/session_relay/constants.rs @@ -0,0 +1,17 @@ +/// How long a relay waits for its client to connect. +pub const RELAY_ACCEPT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + +/// How long a loopback relay gives each connection to send its whole startup message, so a silent or slow one is dropped without holding up the client. +pub const RELAY_STARTUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +/// The shortest startup message: its length and the protocol version. +pub const STARTUP_MESSAGE_MINIMUM: usize = 8; + +/// The longest startup message a loopback relay reads: Postgres's own limit (`MAX_STARTUP_PACKET_LENGTH`). +pub const STARTUP_MESSAGE_LIMIT: usize = 10_000; + +/// Protocol 3.0, as the replication client's startup message gives it. +pub const PROTOCOL_VERSION: [u8; 4] = [0, 3, 0, 0]; + +/// The application name SQLLoad's sessions show the server. +pub const APPLICATION_NAME: &str = "sqlload"; diff --git a/crates/sqlload/src/session_relay/main/mod.rs b/crates/sqlload/src/session_relay/main/mod.rs new file mode 100644 index 00000000..777cfca0 --- /dev/null +++ b/crates/sqlload/src/session_relay/main/mod.rs @@ -0,0 +1 @@ +pub mod relay_session; diff --git a/crates/sqlload/src/session_relay/main/relay_session.rs b/crates/sqlload/src/session_relay/main/relay_session.rs new file mode 100644 index 00000000..bd9803b1 --- /dev/null +++ b/crates/sqlload/src/session_relay/main/relay_session.rs @@ -0,0 +1,24 @@ +//! Open a relay that pipes one session through a connection SQLLoad opened. + +use tokio::io::{AsyncRead, AsyncWrite}; + +#[cfg(windows)] +use crate::session_relay::_helpers::loopback::loopback_relay; +#[cfg(unix)] +use crate::session_relay::_helpers::socket_folder::socket_folder_relay; +use crate::session_relay::models::RelayEndpoint; + +/// Where the client reaches `server` (the session to a Postgres server listening on `port`): on Unix a `.s.PGSQL.` socket in a folder only this user can enter; on Windows, where pgwire-replication reaches no socket file, a 127.0.0.1 port that admits only the connection presenting the returned application name. Either serves the first session admitted and closes, or closes after [`RELAY_ACCEPT_TIMEOUT`](crate::session_relay::constants::RELAY_ACCEPT_TIMEOUT) without one. +pub async fn relay_session(server: S, port: u16) -> Result +where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + #[cfg(unix)] + let relayed = socket_folder_relay(server, port); + #[cfg(windows)] + let relayed = { + let _ = port; + loopback_relay(server).await + }; + relayed +} diff --git a/crates/sqlload/src/session_relay/mod.rs b/crates/sqlload/src/session_relay/mod.rs new file mode 100644 index 00000000..aa289d7e --- /dev/null +++ b/crates/sqlload/src/session_relay/mod.rs @@ -0,0 +1,9 @@ +//! Where a client that speaks only plain text, such as the replication client, reaches a session SQLLoad has opened for it: a private socket on Unix, a loopback port that admits only its own client on Windows. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlload/src/session_relay/models.rs b/crates/sqlload/src/session_relay/models.rs new file mode 100644 index 00000000..667ab0be --- /dev/null +++ b/crates/sqlload/src/session_relay/models.rs @@ -0,0 +1,9 @@ +/// Where the client reaches a relayed session, and the application name its startup message must carry there. +#[derive(Debug)] +pub struct RelayEndpoint { + /// The relay's socket folder (Unix) or `127.0.0.1` (Windows). + pub host: String, + pub port: u16, + /// `sqlload` on Unix. On Windows `sqlload-`, which the relay checks and replaces with `sqlload` before the server sees it. + pub application_name: crate::auth::models::Secret, +} diff --git a/crates/sqlload/src/session_relay/tests.rs b/crates/sqlload/src/session_relay/tests.rs new file mode 100644 index 00000000..af154ab7 --- /dev/null +++ b/crates/sqlload/src/session_relay/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_admitted_startup.rs"] +mod admitted_startup; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_loopback.rs"] +mod loopback; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlload/src/session_relay/tests/helpers.rs b/crates/sqlload/src/session_relay/tests/helpers.rs new file mode 100644 index 00000000..c2ff3152 --- /dev/null +++ b/crates/sqlload/src/session_relay/tests/helpers.rs @@ -0,0 +1,192 @@ +use std::time::Duration; + +use tokio::io::{AsyncReadExt, AsyncWriteExt, DuplexStream}; +use tokio::net::TcpStream; + +use crate::session_relay::_helpers::loopback::loopback_relay; +use crate::session_relay::constants::{PROTOCOL_VERSION, RELAY_STARTUP_TIMEOUT}; +use crate::session_relay::tests::test_types::RelayOutcome; + +/// A relay secret as the relay makes them: `sqlload-` and 64 hex digits. +pub(crate) const SECRET: &str = + "sqlload-6f1c0d2a9b8e7f6a5b4c3d2e1f0a9b8c7d6e5f4a3b2c1d0e9f8a7b6c5d4e3f2a"; + +/// Postgres's `SSLRequest`. +pub(crate) const SSL_REQUEST: [u8; 8] = [0, 0, 0, 8, 0x04, 0xd2, 0x16, 0x2f]; + +/// What the server receives for the replication client's startup message. +pub(crate) const FORWARDED: &str = + "user=replicator,database=shop,replication=database,application_name=sqlload"; + +pub(crate) fn block_on( + body: impl std::future::Future>, +) -> Result { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .map_err(|error| error.to_string())? + .block_on(body) +} + +/// A message of protocol `version` whose body after the version is `body`, framed with its length. +pub(crate) fn framed(version: [u8; 4], body: &[u8]) -> Vec { + let length = u32::try_from(body.len() + 8).unwrap_or(u32::MAX); + [&length.to_be_bytes()[..], &version, body].concat() +} + +/// A startup message of protocol `version` with `params`, framed as libpq and pgwire-replication frame it. +pub(crate) fn startup_message(version: [u8; 4], params: &[(&str, &str)]) -> Vec { + let mut body = Vec::new(); + for (name, value) in params { + body.extend_from_slice(format!("{name}\0{value}\0").as_bytes()); + } + body.push(0); + framed(version, &body) +} + +/// The replication client's startup message, naming `application_name`. +pub(crate) fn client_startup(application_name: &str) -> Vec { + startup_message( + PROTOCOL_VERSION, + &[ + ("user", "replicator"), + ("database", "shop"), + ("replication", "database"), + ("application_name", application_name), + ], + ) +} + +/// `message` with a length one less than it is. +pub(crate) fn declaring_one_less(mut message: Vec) -> Vec { + message[3] -= 1; + message +} + +/// `message` without its final NUL, its length one less to match. +pub(crate) fn without_final_nul(mut message: Vec) -> Vec { + message.pop(); + declaring_one_less(message) +} + +/// A whole startup message's parameters as `name=value` joined by `,`; `malformed` when it is not one. +pub(crate) fn startup_params(message: &[u8]) -> String { + let declared = message + .first_chunk::<4>() + .and_then(|length| usize::try_from(u32::from_be_bytes(*length)).ok()); + let well_formed = + declared == Some(message.len()) && message.get(4..8) == Some(&PROTOCOL_VERSION[..]); + message + .get(8..) + .and_then(|rest| rest.strip_suffix(b"\0\0")) + .filter(|_| well_formed) + .map(|rest| { + String::from_utf8_lossy(rest) + .split('\0') + .collect::>() + .chunks(2) + .map(|pair| pair.join("=")) + .collect::>() + .join(",") + }) + .unwrap_or_else(|| "malformed".to_owned()) +} + +pub(crate) fn contains(haystack: &[u8], needle: &str) -> bool { + haystack + .windows(needle.len()) + .any(|window| window == needle.as_bytes()) +} + +/// The whole first message the server side of a relay received. +async fn read_forwarded(upstream: &mut DuplexStream) -> Result, String> { + let mut length = [0u8; 4]; + upstream + .read_exact(&mut length) + .await + .map_err(|error| error.to_string())?; + let declared = + usize::try_from(u32::from_be_bytes(length)).map_err(|error| error.to_string())?; + let mut message = length.to_vec(); + message.resize(declared.max(4), 0); + upstream + .read_exact(&mut message[4..]) + .await + .map_err(|error| error.to_string())?; + Ok(message) +} + +/// Whether the relay closes `stream` within two seconds without sending it a byte. +async fn closed_unanswered(stream: &mut TcpStream) -> bool { + let mut byte = [0u8; 1]; + tokio::time::timeout(Duration::from_secs(2), stream.read(&mut byte)) + .await + .is_ok_and(|read| read.unwrap_or_default() == 0) +} + +async fn io(step: impl std::future::Future>) -> Result { + step.await.map_err(|error| error.to_string()) +} + +/// A loopback relay in front of an in-memory server: open a connection for each of `probes`, then send each its bytes at once (so refusals finish together), then open the replication client with the relay's application name, which must be relayed well before a silent connection's startup deadline; then pipe a byte each way, and connect once more. +pub(crate) async fn relay_scenario(probes: &[Vec]) -> Result { + let (server, mut upstream) = tokio::io::duplex(64 * 1024); + let endpoint = loopback_relay(server).await?; + let address = (endpoint.host.clone(), endpoint.port); + let mut opened = Vec::new(); + for _ in probes { + opened.push(io(TcpStream::connect(&address)).await?); + } + tokio::time::sleep(Duration::from_millis(100)).await; + for (stream, bytes) in opened.iter_mut().zip(probes) { + io(stream.write_all(bytes)).await?; + } + tokio::time::sleep(Duration::from_millis(300)).await; + let mut byte = [0u8; 1]; + let upstream_bytes_before_client = + tokio::time::timeout(Duration::from_millis(200), upstream.read(&mut byte)) + .await + .map_or(0, |read| read.unwrap_or_default()); + let mut client = io(TcpStream::connect(&address)).await?; + io(client.write_all(&client_startup(endpoint.application_name.expose()))).await?; + let forwarded = tokio::time::timeout(RELAY_STARTUP_TIMEOUT / 2, read_forwarded(&mut upstream)) + .await + .map_err(|_| "the client was not relayed in time".to_owned())??; + let secret = &endpoint.application_name.expose()["sqlload-".len()..]; + io(upstream.write_all(b"R")).await?; + let mut answer = [0u8; 1]; + io(client.read_exact(&mut answer)).await?; + io(client.write_all(b"p")).await?; + let mut echoed = [0u8; 1]; + io(upstream.read_exact(&mut echoed)).await?; + let mut probes_closed_unanswered = 0; + for stream in &mut opened { + probes_closed_unanswered += usize::from(closed_unanswered(stream).await); + } + let later_connection_refused = + tokio::time::timeout(Duration::from_secs(10), TcpStream::connect(&address)) + .await + .map_err(|_| "connecting after the client hung".to_owned())? + .is_err(); + Ok(RelayOutcome { + upstream_bytes_before_client, + forwarded: startup_params(&forwarded), + secret_forwarded: contains(&forwarded, secret), + piped: (answer, echoed) == (*b"R", *b"p"), + probes_closed_unanswered, + later_connection_refused, + }) +} + +/// What every scenario's relay does once its client connects, having refused `probes` connections. +pub(crate) fn relayed(probes: usize) -> RelayOutcome { + RelayOutcome { + upstream_bytes_before_client: 0, + forwarded: FORWARDED.to_owned(), + secret_forwarded: false, + piped: true, + probes_closed_unanswered: probes, + later_connection_refused: true, + } +} diff --git a/crates/sqlload/src/session_relay/tests/test_admitted_startup.rs b/crates/sqlload/src/session_relay/tests/test_admitted_startup.rs new file mode 100644 index 00000000..9adf6a9b --- /dev/null +++ b/crates/sqlload/src/session_relay/tests/test_admitted_startup.rs @@ -0,0 +1,111 @@ +use crate::session_relay::_helpers::startup::admitted_startup; +use crate::session_relay::constants::PROTOCOL_VERSION; +use crate::session_relay::tests::helpers::{ + FORWARDED, SECRET, SSL_REQUEST, client_startup, contains, declaring_one_less, framed, + startup_message, startup_params, without_final_nul, +}; +use crate::session_relay::tests::test_types::AdmittedStartupTestCase; + +const REFUSED_NAME: &str = "not this relay's application name"; +const NOT_ONE_NAME: &str = "not one application name"; +const NOT_A_STARTUP: &str = "not one protocol 3.0 startup message"; + +#[test] +fn given_startup_messages_when_the_relay_checks_them_then_only_its_secret_is_admitted_and_never_forwarded() + { + let wrong_last = format!("{}0", &SECRET[..SECRET.len() - 1]); + let longer = format!("{SECRET}0"); + let test_cases = [ + AdmittedStartupTestCase { + description: "the secret, among the other parameters", + message: client_startup(SECRET), + expected_forwarded: Ok(FORWARDED), + }, + AdmittedStartupTestCase { + description: "the secret first, before an empty value", + message: startup_message( + PROTOCOL_VERSION, + &[("application_name", SECRET), ("options", ""), ("user", "u")], + ), + expected_forwarded: Ok("application_name=sqlload,options=,user=u"), + }, + AdmittedStartupTestCase { + description: "a secret wrong in its last character", + message: client_startup(&wrong_last), + expected_forwarded: Err(REFUSED_NAME), + }, + AdmittedStartupTestCase { + description: "a prefix of the secret", + message: client_startup(&SECRET[..SECRET.len() - 1]), + expected_forwarded: Err(REFUSED_NAME), + }, + AdmittedStartupTestCase { + description: "the secret with more after it", + message: client_startup(&longer), + expected_forwarded: Err(REFUSED_NAME), + }, + AdmittedStartupTestCase { + description: "the plain application name", + message: client_startup("sqlload"), + expected_forwarded: Err(REFUSED_NAME), + }, + AdmittedStartupTestCase { + description: "no application name", + message: startup_message(PROTOCOL_VERSION, &[("user", "u"), ("database", "d")]), + expected_forwarded: Err(NOT_ONE_NAME), + }, + AdmittedStartupTestCase { + description: "the secret, then a second application name", + message: startup_message( + PROTOCOL_VERSION, + &[("application_name", SECRET), ("application_name", "other")], + ), + expected_forwarded: Err(NOT_ONE_NAME), + }, + AdmittedStartupTestCase { + description: "an SSLRequest", + message: SSL_REQUEST.to_vec(), + expected_forwarded: Err(NOT_A_STARTUP), + }, + AdmittedStartupTestCase { + description: "protocol 2.0 with the secret", + message: startup_message([0, 2, 0, 0], &[("application_name", SECRET)]), + expected_forwarded: Err(NOT_A_STARTUP), + }, + AdmittedStartupTestCase { + description: "a length one short of the message", + message: declaring_one_less(client_startup(SECRET)), + expected_forwarded: Err(NOT_A_STARTUP), + }, + AdmittedStartupTestCase { + description: "no final NUL", + message: without_final_nul(client_startup(SECRET)), + expected_forwarded: Err("not NUL-terminated"), + }, + AdmittedStartupTestCase { + description: "a name without a value", + message: framed(PROTOCOL_VERSION, b"application_name\0\0"), + expected_forwarded: Err("not pairs of names and values"), + }, + AdmittedStartupTestCase { + description: "nothing", + message: Vec::new(), + expected_forwarded: Err("no length"), + }, + ]; + for test_case in test_cases { + let forwarded = admitted_startup(&test_case.message, SECRET); + let found = forwarded.clone().map(|bytes| startup_params(&bytes)); + assert_eq!( + found, + test_case.expected_forwarded.map(str::to_owned), + "{}", + test_case.description + ); + assert!( + !forwarded.is_ok_and(|bytes| contains(&bytes, &SECRET["sqlload-".len()..])), + "{}: the secret was forwarded", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/session_relay/tests/test_loopback.rs b/crates/sqlload/src/session_relay/tests/test_loopback.rs new file mode 100644 index 00000000..532ff27c --- /dev/null +++ b/crates/sqlload/src/session_relay/tests/test_loopback.rs @@ -0,0 +1,54 @@ +use crate::session_relay::constants::PROTOCOL_VERSION; +use crate::session_relay::tests::helpers::{ + SSL_REQUEST, block_on, client_startup, relay_scenario, relayed, startup_message, +}; +use crate::session_relay::tests::test_types::LoopbackRelayTestCase; + +#[test] +fn given_other_local_connections_when_the_client_connects_then_only_it_is_relayed_and_promptly() +-> Result<(), String> { + let wrong = format!("sqlload-{}", "0".repeat(64)); + let test_cases = [ + LoopbackRelayTestCase { + description: "no other connection", + probes: Vec::new(), + expected_outcome: relayed(0), + }, + LoopbackRelayTestCase { + description: "a silent connection first, which must not hold the client up", + probes: vec![Vec::new()], + expected_outcome: relayed(1), + }, + LoopbackRelayTestCase { + description: "two refused connections first, after which the relay must still admit the client", + probes: vec![vec![0; 8], vec![0; 8]], + expected_outcome: relayed(2), + }, + LoopbackRelayTestCase { + description: "a wrong secret, the plain name, an oversize length, an SSLRequest and protocol 2.0 first", + probes: vec![ + client_startup(&wrong), + client_startup("sqlload"), + vec![0, 1, 0, 0, 0, 3, 0, 0], + SSL_REQUEST.to_vec(), + startup_message([0, 2, 0, 0], &[("user", "u")]), + ], + expected_outcome: relayed(5), + }, + LoopbackRelayTestCase { + description: "a startup message with no application name first", + probes: vec![startup_message(PROTOCOL_VERSION, &[("user", "u")])], + expected_outcome: relayed(1), + }, + ]; + for test_case in test_cases { + let found = block_on(relay_scenario(&test_case.probes)) + .map_err(|error| format!("{}: {error}", test_case.description))?; + assert_eq!( + found, test_case.expected_outcome, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlload/src/session_relay/tests/test_types.rs b/crates/sqlload/src/session_relay/tests/test_types.rs new file mode 100644 index 00000000..e7544a2f --- /dev/null +++ b/crates/sqlload/src/session_relay/tests/test_types.rs @@ -0,0 +1,29 @@ +pub(crate) struct AdmittedStartupTestCase { + pub(crate) description: &'static str, + pub(crate) message: Vec, + /// The parameters forwarded to the server, as `name=value` joined by `,`; why the message is refused otherwise. + pub(crate) expected_forwarded: Result<&'static str, &'static str>, +} + +pub(crate) struct LoopbackRelayTestCase { + pub(crate) description: &'static str, + /// What each connection opened before the client sends; empty for one that sends nothing. + pub(crate) probes: Vec>, + pub(crate) expected_outcome: RelayOutcome, +} + +/// What a loopback relay did with the connections opened before its client, and with the client. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct RelayOutcome { + /// Bytes the server received before the client connected. + pub(crate) upstream_bytes_before_client: usize, + /// The startup message the server received, as `name=value` joined by `,`. + pub(crate) forwarded: String, + pub(crate) secret_forwarded: bool, + /// Whether bytes then crossed between the client and the server both ways. + pub(crate) piped: bool, + /// How many of the earlier connections the relay closed without sending them a byte. + pub(crate) probes_closed_unanswered: usize, + /// Whether a connection after the client's is refused. + pub(crate) later_connection_refused: bool, +} diff --git a/crates/sqlload/src/store_paths/_helpers/local_root.rs b/crates/sqlload/src/store_paths/_helpers/local_root.rs index 02fc4e96..520a141e 100644 --- a/crates/sqlload/src/store_paths/_helpers/local_root.rs +++ b/crates/sqlload/src/store_paths/_helpers/local_root.rs @@ -29,14 +29,11 @@ pub(crate) fn local_root(path: &str, windows: bool) -> Option<(String, &str)> { }); } let drive = path.strip_prefix('/').unwrap_or(path); - let bytes = drive.as_bytes(); - if bytes.len() >= 3 - && bytes[0].is_ascii_alphabetic() - && bytes[1] == b':' - && bytes[2] == b'/' + if let [letter, b':', b'/', ..] = drive.as_bytes() + && letter.is_ascii_alphabetic() { return Some(( - format!("{}:/", char::from(bytes[0]).to_ascii_uppercase()), + format!("{}:/", char::from(letter.to_ascii_uppercase())), &drive[3..], )); } @@ -44,12 +41,32 @@ pub(crate) fn local_root(path: &str, windows: bool) -> Option<(String, &str)> { path.strip_prefix('/').map(|rest| ("/".to_owned(), rest)) } -/// The local path a `file://` URL names, given what follows `file://`: on Windows `file:///C:/folder` is `C:/folder` and `file://server/share/folder` is the share `//server/share/folder`. +/// The local path a `file://` URL names, given what follows `file://`: on Windows `file:///C:/folder` and `file://localhost/C:/folder` are `C:/folder`, and `file://server/share/folder` is the share `//server/share/folder`. pub(crate) fn windows_file_url_path(rest: &str, windows: bool) -> Cow<'_, str> { - match (windows, rest.strip_prefix('/')) { - (false, _) => Cow::Borrowed(rest), - (true, Some(path)) if path.as_bytes().get(1) == Some(&b':') => Cow::Borrowed(path), - (true, Some(_)) => Cow::Borrowed(rest), - (true, None) => Cow::Owned(format!("//{rest}")), + let on_localhost = rest + .get(.."localhost/".len()) + .filter(|host| host.eq_ignore_ascii_case("localhost/")) + .map(|host| &rest[host.len()..]); + match (windows, rest.strip_prefix('/'), on_localhost) { + (false, _, _) => Cow::Borrowed(rest), + (true, Some(path), _) if is_drive(path) => Cow::Borrowed(path), + (true, Some(_), _) => Cow::Borrowed(rest), + (true, None, Some(path)) if is_drive(path) => Cow::Borrowed(path), + (true, None, _) => Cow::Owned(format!("//{rest}")), } } + +/// Whether `path` starts with a drive letter and its colon. +fn is_drive(path: &str) -> bool { + matches!(path.as_bytes(), [letter, b':', ..] if letter.is_ascii_alphabetic()) +} + +/// `prefix` without its leading `folders` (both folder prefixes ending in `/`), whose names match ignoring case when `windows`, as NTFS and SMB shares match them; `None` when `prefix` is not below `folders`. +pub(crate) fn strip_folders<'a>(prefix: &'a str, folders: &str, windows: bool) -> Option<&'a str> { + let head = prefix.get(..folders.len())?; + let same = match windows { + true => head.eq_ignore_ascii_case(folders), + false => head == folders, + }; + same.then(|| &prefix[folders.len()..]) +} diff --git a/crates/sqlload/src/store_paths/main/resolve_location.rs b/crates/sqlload/src/store_paths/main/resolve_location.rs index e7e4ccdb..506bc42c 100644 --- a/crates/sqlload/src/store_paths/main/resolve_location.rs +++ b/crates/sqlload/src/store_paths/main/resolve_location.rs @@ -1,12 +1,12 @@ //! Work out which store and folder a path means. use crate::store_paths::_helpers::folder::folder; -use crate::store_paths::_helpers::local_root::{local_root, windows_separators}; +use crate::store_paths::_helpers::local_root::{local_root, strip_folders, windows_separators}; use crate::store_paths::_helpers::locations::{describe, same_store, url_location}; use crate::store_paths::errors::PathError; use crate::store_paths::models::{Location, StoreKind}; -/// The folder `path` names: relative to `connection` when there is one (on an SFTP server, from `/` when it starts with `/`, otherwise from the login folder, which a leading `~/` also names), otherwise from its scheme. +/// The folder `path` names: relative to `connection` when there is one (on an SFTP server, from `/` when it starts with `/`, otherwise from the login folder, which a leading `~/` also names), otherwise from its scheme. Under a filesystem connection, an absolute path must be below its root, matched on the root too unless the connection's root is relative; on Windows roots and folders match ignoring case. pub fn resolve_location( connection: Option<(&str, &StoreKind)>, path: &str, @@ -43,7 +43,6 @@ pub fn resolve_location( (Some((name, StoreKind::Filesystem { root })), Some(location)) => { let windows = cfg!(windows); let separated = windows_separators(root, windows); - // A relative root is matched on its folders alone, whatever the path's root. let (same_root, root_folders) = match (local_root(&separated, windows), &location.store) { (Some((root, folders)), StoreKind::Filesystem { root: found }) => { @@ -52,11 +51,7 @@ pub fn resolve_location( _ => (true, separated.as_ref()), }; let root_prefix = folder(&[root_folders])?; - match location - .prefix - .strip_prefix(&root_prefix) - .filter(|_| same_root) - { + match strip_folders(&location.prefix, &root_prefix, windows).filter(|_| same_root) { Some(relative) => Ok(Location { store: StoreKind::Filesystem { root: root.clone() }, prefix: relative.to_owned(), diff --git a/crates/sqlload/src/store_paths/tests/helpers.rs b/crates/sqlload/src/store_paths/tests/helpers.rs index f1f48844..7b94a39a 100644 --- a/crates/sqlload/src/store_paths/tests/helpers.rs +++ b/crates/sqlload/src/store_paths/tests/helpers.rs @@ -9,3 +9,10 @@ pub(crate) fn bucket(name: &str) -> Option { profile: None, }) } + +/// The root and folders of `path` as `local_root` finds them, written `root|folders`. +pub(crate) fn root_of(path: &str, windows: bool) -> Option { + let separated = crate::store_paths::_helpers::local_root::windows_separators(path, windows); + crate::store_paths::_helpers::local_root::local_root(&separated, windows) + .map(|(root, folders)| format!("{root}|{folders}")) +} diff --git a/crates/sqlload/src/store_paths/tests/test_local_root.rs b/crates/sqlload/src/store_paths/tests/test_local_root.rs index 5615e0c4..91d99a67 100644 --- a/crates/sqlload/src/store_paths/tests/test_local_root.rs +++ b/crates/sqlload/src/store_paths/tests/test_local_root.rs @@ -1,59 +1,52 @@ -use crate::store_paths::_helpers::local_root::{ - local_root, windows_file_url_path, windows_separators, +use crate::store_paths::_helpers::local_root::{strip_folders, windows_file_url_path}; +use crate::store_paths::tests::helpers::root_of; +use crate::store_paths::tests::test_types::{ + FileUrlPathTestCase, LocalRootTestCase, StripFoldersTestCase, }; -/// How a local path reads on Windows, and on Unix, where only `/` starts an absolute path. -struct LocalRootTestCase { - description: &'static str, - path: &'static str, - /// Root and folders on Windows; `None` for a relative path. - expected_windows: Option<(&'static str, &'static str)>, - expected_unix: Option<(&'static str, &'static str)>, -} - #[test] fn given_local_paths_when_finding_their_root_then_drives_and_shares_are_roots_only_on_windows() { let test_cases = [ LocalRootTestCase { description: "a Unix absolute path", path: "/data/exports", - expected_windows: Some(("/", "data/exports")), - expected_unix: Some(("/", "data/exports")), + expected_windows: Some("/|data/exports"), + expected_unix: Some("/|data/exports"), }, LocalRootTestCase { description: "a drive with backslashes, lower-case letter", path: r"c:\Data\exports", - expected_windows: Some(("C:/", "Data/exports")), + expected_windows: Some("C:/|Data/exports"), expected_unix: None, }, LocalRootTestCase { description: "a drive as a file:/// URL gives it", path: "/D:/lake", - expected_windows: Some(("D:/", "lake")), - expected_unix: Some(("/", "D:/lake")), + expected_windows: Some("D:/|lake"), + expected_unix: Some("/|D:/lake"), }, LocalRootTestCase { description: "a UNC share", path: r"\\nas01\exports\daily\orders", - expected_windows: Some(("//nas01/exports/", "daily/orders")), + expected_windows: Some("//nas01/exports/|daily/orders"), expected_unix: None, }, LocalRootTestCase { description: "a UNC share's root alone", path: r"\\nas01\exports", - expected_windows: Some(("//nas01/exports/", "")), + expected_windows: Some("//nas01/exports/|"), expected_unix: None, }, LocalRootTestCase { description: "an extended-length drive path", path: r"\\?\C:\very\long", - expected_windows: Some(("C:/", "very/long")), + expected_windows: Some("C:/|very/long"), expected_unix: None, }, LocalRootTestCase { description: "an extended-length UNC path", path: r"\\?\UNC\nas01\exports\x", - expected_windows: Some(("//nas01/exports/", "x")), + expected_windows: Some("//nas01/exports/|x"), expected_unix: None, }, LocalRootTestCase { @@ -76,35 +69,115 @@ fn given_local_paths_when_finding_their_root_then_drives_and_shares_are_roots_on }, ]; for test_case in test_cases { - for (windows, expected) in [ - (true, test_case.expected_windows), - (false, test_case.expected_unix), - ] { - let separated = windows_separators(test_case.path, windows); - let found = local_root(&separated, windows); - let found = found.as_ref().map(|(root, rest)| (root.as_str(), *rest)); - assert_eq!( - found, expected, - "{} (windows: {windows})", - test_case.description - ); - } + assert_eq!( + root_of(test_case.path, true).as_deref(), + test_case.expected_windows, + "{} on Windows", + test_case.description + ); + assert_eq!( + root_of(test_case.path, false).as_deref(), + test_case.expected_unix, + "{} on Unix", + test_case.description + ); } } #[test] fn given_file_urls_when_reading_their_path_then_windows_drives_and_shares_are_found() { let test_cases = [ - ("/data/x", "/data/x", "/data/x"), - ("/C:/data/x", "C:/data/x", "/C:/data/x"), - ("nas01/exports/x", "//nas01/exports/x", "nas01/exports/x"), + FileUrlPathTestCase { + description: "an absolute path", + rest: "/data/x", + expected_windows: "/data/x", + expected_unix: "/data/x", + }, + FileUrlPathTestCase { + description: "a drive", + rest: "/C:/data/x", + expected_windows: "C:/data/x", + expected_unix: "/C:/data/x", + }, + FileUrlPathTestCase { + description: "a drive on localhost", + rest: "LocalHost/C:/data/x", + expected_windows: "C:/data/x", + expected_unix: "LocalHost/C:/data/x", + }, + FileUrlPathTestCase { + description: "a share on localhost", + rest: "localhost/exports/x", + expected_windows: "//localhost/exports/x", + expected_unix: "localhost/exports/x", + }, + FileUrlPathTestCase { + description: "a share", + rest: "nas01/exports/x", + expected_windows: "//nas01/exports/x", + expected_unix: "nas01/exports/x", + }, + ]; + for test_case in test_cases { + assert_eq!( + windows_file_url_path(test_case.rest, true), + test_case.expected_windows, + "{} on Windows", + test_case.description + ); + assert_eq!( + windows_file_url_path(test_case.rest, false), + test_case.expected_unix, + "{} on Unix", + test_case.description + ); + } +} + +#[test] +fn given_a_folder_prefix_when_stripping_a_connections_folders_then_windows_ignores_case() { + let test_cases = [ + StripFoldersTestCase { + description: "the same folders", + prefix: "lake/scores/", + folders: "lake/", + expected_windows: Some("scores/"), + expected_unix: Some("scores/"), + }, + StripFoldersTestCase { + description: "folders differing in case", + prefix: "Lake/scores/", + folders: "lake/", + expected_windows: Some("scores/"), + expected_unix: None, + }, + StripFoldersTestCase { + description: "a folder that only starts with the connection's", + prefix: "lakehouse/scores/", + folders: "lake/", + expected_windows: None, + expected_unix: None, + }, + StripFoldersTestCase { + description: "a shorter prefix", + prefix: "la/", + folders: "lake/", + expected_windows: None, + expected_unix: None, + }, ]; - for (rest, expected_windows, expected_unix) in test_cases { + for test_case in test_cases { + assert_eq!( + strip_folders(test_case.prefix, test_case.folders, true), + test_case.expected_windows, + "{} on Windows", + test_case.description + ); assert_eq!( - windows_file_url_path(rest, true), - expected_windows, - "{rest}" + strip_folders(test_case.prefix, test_case.folders, false), + test_case.expected_unix, + "{} on Unix", + test_case.description ); - assert_eq!(windows_file_url_path(rest, false), expected_unix, "{rest}"); } } diff --git a/crates/sqlload/src/store_paths/tests/test_resolve_location.rs b/crates/sqlload/src/store_paths/tests/test_resolve_location.rs index 94249d6e..cfb219bf 100644 --- a/crates/sqlload/src/store_paths/tests/test_resolve_location.rs +++ b/crates/sqlload/src/store_paths/tests/test_resolve_location.rs @@ -133,6 +133,14 @@ fn given_windows_paths_when_resolving_then_drives_and_shares_are_local_folders() path: r"daily\orders", expected_location: Ok(r"\\nas01\exports/daily/orders/"), }, + ResolveLocationTestCase { + description: "a folder under a drive connection in another case", + connection: Some(StoreKind::Filesystem { + root: r"C:\Lake".to_owned(), + }), + path: r"c:\lake\scores", + expected_location: Ok(r"C:\Lake/scores/"), + }, ResolveLocationTestCase { description: "another share is not under the connection", connection: share(), @@ -159,14 +167,20 @@ fn given_windows_paths_when_resolving_then_drives_and_shares_are_local_folders() let result = resolve_location(connection, test_case.path) .map(|location| location.to_string()) .map_err(|error| error.to_string()); - match (&result, test_case.expected_location) { - (Ok(found), Ok(expected)) => assert_eq!(found, expected, "{}", test_case.description), - (Err(found), Err(expected)) => assert!( - found.contains(expected), - "{}: {found}", - test_case.description - ), - _ => panic!("{}: {result:?}", test_case.description), - } + assert_eq!( + result.as_deref().ok(), + test_case.expected_location.ok(), + "{}", + test_case.description + ); + assert!( + result + .err() + .unwrap_or_default() + .contains(test_case.expected_location.err().unwrap_or_default()), + "{}: {:?}", + test_case.description, + test_case.path + ); } } diff --git a/crates/sqlload/src/store_paths/tests/test_types.rs b/crates/sqlload/src/store_paths/tests/test_types.rs index c78ea9d4..5864cd2a 100644 --- a/crates/sqlload/src/store_paths/tests/test_types.rs +++ b/crates/sqlload/src/store_paths/tests/test_types.rs @@ -32,3 +32,29 @@ pub(crate) struct PartitionValueTestCase { pub(crate) value: &'static str, pub(crate) expected_escaped: &'static str, } + +/// How a local path reads on Windows, and on Unix, where only `/` starts an absolute path. +pub(crate) struct LocalRootTestCase { + pub(crate) description: &'static str, + pub(crate) path: &'static str, + /// The root and the folders below it, as `root|folders`; `None` for a relative path. + pub(crate) expected_windows: Option<&'static str>, + pub(crate) expected_unix: Option<&'static str>, +} + +/// The local path a `file://` URL names, given what follows `file://`, on Windows and on Unix. +pub(crate) struct FileUrlPathTestCase { + pub(crate) description: &'static str, + pub(crate) rest: &'static str, + pub(crate) expected_windows: &'static str, + pub(crate) expected_unix: &'static str, +} + +/// A folder prefix below a connection's folders, on Windows and on Unix. +pub(crate) struct StripFoldersTestCase { + pub(crate) description: &'static str, + pub(crate) prefix: &'static str, + pub(crate) folders: &'static str, + pub(crate) expected_windows: Option<&'static str>, + pub(crate) expected_unix: Option<&'static str>, +} From 7cd4c00789bd5b1c23c997061c8453a194e82c48 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:25:52 +0000 Subject: [PATCH 03/25] fix: build the ssh-agent client and the test kit on Windows russh reaches ssh-agent through SSH_AUTH_SOCK's Unix socket only on Unix; on Windows it now opens the pipe SSH_AUTH_SOCK names, or else the OpenSSH agent service's. The test kit sets Unix key-file permissions only on Unix. --- .../src/ssh/_helpers/key_files.rs | 3 +++ .../sqlload/src/ssh_tunnel/_helpers/session.rs | 18 +++++++++++++++++- crates/sqlload/src/ssh_tunnel/constants.rs | 4 ++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/crates/sqlload-testkit/src/ssh/_helpers/key_files.rs b/crates/sqlload-testkit/src/ssh/_helpers/key_files.rs index e6e0faf9..586041f5 100644 --- a/crates/sqlload-testkit/src/ssh/_helpers/key_files.rs +++ b/crates/sqlload-testkit/src/ssh/_helpers/key_files.rs @@ -47,8 +47,11 @@ pub(crate) async fn key_files( ] { let path = files.path().join(name); std::fs::write(&path, text).map_err(|error| error.to_string())?; + #[cfg(unix)] std::fs::set_permissions(&path, std::os::unix::fs::PermissionsExt::from_mode(mode)) .map_err(|error| error.to_string())?; + #[cfg(not(unix))] + let _ = mode; } Ok(files) } diff --git a/crates/sqlload/src/ssh_tunnel/_helpers/session.rs b/crates/sqlload/src/ssh_tunnel/_helpers/session.rs index 3b36a30a..35d01c2c 100644 --- a/crates/sqlload/src/ssh_tunnel/_helpers/session.rs +++ b/crates/sqlload/src/ssh_tunnel/_helpers/session.rs @@ -9,6 +9,8 @@ use russh::keys::agent::client::AgentClient; use russh::keys::known_hosts::known_host_keys_path; use russh::keys::{HashAlg, PrivateKeyWithHashAlg, PublicKeyOrCertificate, load_secret_key}; +#[cfg(windows)] +use crate::ssh_tunnel::constants::OPENSSH_AGENT_PIPE; use crate::ssh_tunnel::constants::{CONNECT_TIMEOUT, KEEPALIVE_INTERVAL, KEEPALIVE_MAX}; use crate::ssh_tunnel::models::{HostCheck, SshTunnel}; @@ -234,13 +236,27 @@ async fn password_sign_in( } } +/// ssh-agent: the socket `SSH_AUTH_SOCK` names. +#[cfg(unix)] +async fn connect_agent() -> Result, russh::keys::Error> { + AgentClient::connect_env().await +} + +/// ssh-agent on Windows: the pipe `SSH_AUTH_SOCK` names, or else the OpenSSH agent service's. +#[cfg(windows)] +async fn connect_agent() +-> Result, russh::keys::Error> { + let pipe = std::env::var("SSH_AUTH_SOCK").unwrap_or_else(|_| OPENSSH_AGENT_PIPE.to_owned()); + AgentClient::connect_named_pipe(pipe).await +} + /// `session` signed in as `user` with the first key ssh-agent holds that the bastion accepts. async fn agent_sign_in( mut session: Handle, user: &str, rsa_hash: Option, ) -> Result, String> { - let mut agent = AgentClient::connect_env().await.map_err(|error| { + let mut agent = connect_agent().await.map_err(|error| { format!("neither private_key_file nor password is set and ssh-agent is not reachable ({error}); set private_key_file or password") })?; let identities = agent diff --git a/crates/sqlload/src/ssh_tunnel/constants.rs b/crates/sqlload/src/ssh_tunnel/constants.rs index f9b59a46..a4535409 100644 --- a/crates/sqlload/src/ssh_tunnel/constants.rs +++ b/crates/sqlload/src/ssh_tunnel/constants.rs @@ -3,6 +3,10 @@ use std::time::Duration; /// The bastion's port when the `ssh` table sets none. pub const SSH_PORT: u16 = 22; +/// Where Windows's OpenSSH agent service listens when `SSH_AUTH_SOCK` names no other pipe. +#[cfg(windows)] +pub const OPENSSH_AGENT_PIPE: &str = r"\\.\pipe\openssh-ssh-agent"; + /// How long to wait for the bastion to answer and sign the client in. pub const CONNECT_TIMEOUT: Duration = Duration::from_secs(30); From d34ac80e0cc71365c9cee89d646cdf30dd8877aa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:30:59 +0000 Subject: [PATCH 04/25] test: which pipe ssh-agent is reached on on Windows, and say what is and isn't tested there --- .../src/ssh_tunnel/_helpers/session.rs | 15 ++++++--- crates/sqlload/src/ssh_tunnel/constants.rs | 2 +- crates/sqlload/src/ssh_tunnel/mod.rs | 3 ++ crates/sqlload/src/ssh_tunnel/tests.rs | 4 +++ .../src/ssh_tunnel/tests/test_agent_pipe.rs | 32 +++++++++++++++++++ .../src/ssh_tunnel/tests/test_types.rs | 6 ++++ .../content/docs/docs/sources/postgres.mdx | 5 +++ 7 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 crates/sqlload/src/ssh_tunnel/tests.rs create mode 100644 crates/sqlload/src/ssh_tunnel/tests/test_agent_pipe.rs create mode 100644 crates/sqlload/src/ssh_tunnel/tests/test_types.rs diff --git a/crates/sqlload/src/ssh_tunnel/_helpers/session.rs b/crates/sqlload/src/ssh_tunnel/_helpers/session.rs index 35d01c2c..db230bde 100644 --- a/crates/sqlload/src/ssh_tunnel/_helpers/session.rs +++ b/crates/sqlload/src/ssh_tunnel/_helpers/session.rs @@ -9,7 +9,7 @@ use russh::keys::agent::client::AgentClient; use russh::keys::known_hosts::known_host_keys_path; use russh::keys::{HashAlg, PrivateKeyWithHashAlg, PublicKeyOrCertificate, load_secret_key}; -#[cfg(windows)] +#[cfg(any(windows, test))] use crate::ssh_tunnel::constants::OPENSSH_AGENT_PIPE; use crate::ssh_tunnel::constants::{CONNECT_TIMEOUT, KEEPALIVE_INTERVAL, KEEPALIVE_MAX}; use crate::ssh_tunnel::models::{HostCheck, SshTunnel}; @@ -242,12 +242,19 @@ async fn connect_agent() -> Result, russh::k AgentClient::connect_env().await } -/// ssh-agent on Windows: the pipe `SSH_AUTH_SOCK` names, or else the OpenSSH agent service's. +/// ssh-agent on Windows: see [`agent_pipe`]. #[cfg(windows)] async fn connect_agent() -> Result, russh::keys::Error> { - let pipe = std::env::var("SSH_AUTH_SOCK").unwrap_or_else(|_| OPENSSH_AGENT_PIPE.to_owned()); - AgentClient::connect_named_pipe(pipe).await + AgentClient::connect_named_pipe(agent_pipe(std::env::var_os("SSH_AUTH_SOCK"))).await +} + +/// The pipe ssh-agent listens on, on Windows: the one `SSH_AUTH_SOCK` names when it is set and not empty, or else the OpenSSH agent service's. +#[cfg(any(windows, test))] +pub(crate) fn agent_pipe(auth_sock: Option) -> std::ffi::OsString { + auth_sock + .filter(|pipe| !pipe.is_empty()) + .unwrap_or_else(|| OPENSSH_AGENT_PIPE.into()) } /// `session` signed in as `user` with the first key ssh-agent holds that the bastion accepts. diff --git a/crates/sqlload/src/ssh_tunnel/constants.rs b/crates/sqlload/src/ssh_tunnel/constants.rs index a4535409..70eaead4 100644 --- a/crates/sqlload/src/ssh_tunnel/constants.rs +++ b/crates/sqlload/src/ssh_tunnel/constants.rs @@ -4,7 +4,7 @@ use std::time::Duration; pub const SSH_PORT: u16 = 22; /// Where Windows's OpenSSH agent service listens when `SSH_AUTH_SOCK` names no other pipe. -#[cfg(windows)] +#[cfg(any(windows, test))] pub const OPENSSH_AGENT_PIPE: &str = r"\\.\pipe\openssh-ssh-agent"; /// How long to wait for the bastion to answer and sign the client in. diff --git a/crates/sqlload/src/ssh_tunnel/mod.rs b/crates/sqlload/src/ssh_tunnel/mod.rs index 39298fb9..f4952daf 100644 --- a/crates/sqlload/src/ssh_tunnel/mod.rs +++ b/crates/sqlload/src/ssh_tunnel/mod.rs @@ -5,3 +5,6 @@ pub mod constants; pub mod errors; pub mod main; pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlload/src/ssh_tunnel/tests.rs b/crates/sqlload/src/ssh_tunnel/tests.rs new file mode 100644 index 00000000..5b85776e --- /dev/null +++ b/crates/sqlload/src/ssh_tunnel/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_agent_pipe.rs"] +mod agent_pipe; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlload/src/ssh_tunnel/tests/test_agent_pipe.rs b/crates/sqlload/src/ssh_tunnel/tests/test_agent_pipe.rs new file mode 100644 index 00000000..74f67e37 --- /dev/null +++ b/crates/sqlload/src/ssh_tunnel/tests/test_agent_pipe.rs @@ -0,0 +1,32 @@ +use crate::ssh_tunnel::_helpers::session::agent_pipe; +use crate::ssh_tunnel::tests::test_types::AgentPipeTestCase; + +#[test] +fn given_ssh_auth_sock_when_finding_the_windows_agent_then_its_pipe_or_the_openssh_service_is_used() +{ + let test_cases = [ + AgentPipeTestCase { + description: "SSH_AUTH_SOCK unset", + auth_sock: None, + expected_pipe: r"\\.\pipe\openssh-ssh-agent", + }, + AgentPipeTestCase { + description: "SSH_AUTH_SOCK empty", + auth_sock: Some(""), + expected_pipe: r"\\.\pipe\openssh-ssh-agent", + }, + AgentPipeTestCase { + description: "SSH_AUTH_SOCK naming another agent's pipe", + auth_sock: Some(r"\\.\pipe\pageant.user.1a2b"), + expected_pipe: r"\\.\pipe\pageant.user.1a2b", + }, + ]; + for test_case in test_cases { + assert_eq!( + agent_pipe(test_case.auth_sock.map(std::ffi::OsString::from)), + test_case.expected_pipe, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/ssh_tunnel/tests/test_types.rs b/crates/sqlload/src/ssh_tunnel/tests/test_types.rs new file mode 100644 index 00000000..58c3c645 --- /dev/null +++ b/crates/sqlload/src/ssh_tunnel/tests/test_types.rs @@ -0,0 +1,6 @@ +pub(crate) struct AgentPipeTestCase { + pub(crate) description: &'static str, + /// `SSH_AUTH_SOCK`, unset when `None`. + pub(crate) auth_sock: Option<&'static str>, + pub(crate) expected_pipe: &'static str, +} diff --git a/website/src/content/docs/docs/sources/postgres.mdx b/website/src/content/docs/docs/sources/postgres.mdx index 4a79727f..ed34af90 100644 --- a/website/src/content/docs/docs/sources/postgres.mdx +++ b/website/src/content/docs/docs/sources/postgres.mdx @@ -194,6 +194,11 @@ and `port` are then the database as the bastion sees it. To set one up, see - It signs in with `private_key_file` (with `private_key_passphrase` if the key has one), or with each key ssh-agent holds when no file is set. From 0.32.0, `password` signs in when no key file is set; before, passwords aren't supported. +- On Windows, ssh-agent is the pipe `SSH_AUTH_SOCK` names, or else the OpenSSH agent service's + (`\\.\pipe\openssh-ssh-agent`). Windows ssh-agent (OpenSSH agent pipe): compiles and + unit-tested, not live-tested. Key-file and password sign-in run the same code as on Linux and + macOS, but on Windows they're not covered by an end-to-end test: no Windows test goes through a + bastion. - The bastion's host key must be in `known_hosts` (`~/.ssh/known_hosts` by default). An unknown or changed key fails with its SHA256 fingerprint and the fix; a key is never accepted on first use. - TLS still runs end to end with the database, so `verify-full` checks the database's own name. From 5cb9a199217c8c58fa9c35fc38e50c2d9a863bd0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:56:58 +0000 Subject: [PATCH 05/25] feat(odbc): write date and timestamp cursor bounds as ODBC escape literals An ODBC source's date and wall-clock timestamp cursor bounds are now {d '...'} and {ts 'YYYY-MM-DD HH:MM:SS[.fff]'} escape literals, which each driver turns into its database's own syntax (OpenEdge compares no string with a timestamp). Cursor flags and cursor_start also accept those literals. WIP for CHI-1236: driver-manager linkage, wheels and live driver tests follow. --- .../src/adapters/contract/constants.rs | 3 + .../src/adapters/contract/main/cursor_text.rs | 13 ++- .../sqlload/src/adapters/contract/main/mod.rs | 1 + .../adapters/contract/main/odbc_literal.rs | 26 +++++ crates/sqlload/src/adapters/contract/tests.rs | 2 + .../contract/tests/test_cursor_bound.rs | 7 ++ .../contract/tests/test_cursor_text.rs | 30 ++++++ .../contract/tests/test_odbc_literal.rs | 78 +++++++++++++++ .../src/adapters/contract/tests/test_types.rs | 7 ++ crates/sqlload/src/loader/_helpers/publish.rs | 9 +- .../sqlload/src/loader/_helpers/table_plan.rs | 21 +++-- .../sqlload/src/loader/main/_files_window.rs | 2 +- .../src/loader/tests/test_source_filter.rs | 94 ++++++++++++++++--- crates/sqlload/src/loader/tests/test_types.rs | 4 +- 14 files changed, 266 insertions(+), 31 deletions(-) create mode 100644 crates/sqlload/src/adapters/contract/main/odbc_literal.rs create mode 100644 crates/sqlload/src/adapters/contract/tests/test_odbc_literal.rs diff --git a/crates/sqlload/src/adapters/contract/constants.rs b/crates/sqlload/src/adapters/contract/constants.rs index 4295c406..e53d8366 100644 --- a/crates/sqlload/src/adapters/contract/constants.rs +++ b/crates/sqlload/src/adapters/contract/constants.rs @@ -10,6 +10,9 @@ pub const SECONDS_PER_DAY: u64 = 86_400; /// How cursor timestamps are written in literals (microseconds, no offset). pub const CURSOR_TIMESTAMP_FORMAT: &str = "%Y-%m-%d %H:%M:%S%.6f"; +/// A timestamp inside an ODBC `{ts '...'}` literal: fractional seconds in groups of three, so a millisecond value keeps the `.fff` every driver reads, and none when whole. +pub const ODBC_TIMESTAMP_FORMAT: &str = "%Y-%m-%d %H:%M:%S%.f"; + /// Timestamps without an offset accepted on the command line. pub const NAIVE_TIMESTAMP_FORMATS: &[&str] = &["%Y-%m-%dT%H:%M:%S%.f", "%Y-%m-%d %H:%M:%S%.f"]; diff --git a/crates/sqlload/src/adapters/contract/main/cursor_text.rs b/crates/sqlload/src/adapters/contract/main/cursor_text.rs index 67bdba39..1548c603 100644 --- a/crates/sqlload/src/adapters/contract/main/cursor_text.rs +++ b/crates/sqlload/src/adapters/contract/main/cursor_text.rs @@ -5,8 +5,9 @@ use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime}; use crate::adapters::contract::constants::{CURSOR_TIMESTAMP_FORMAT, NAIVE_TIMESTAMP_FORMATS}; use crate::column_types::models::ColumnType; -/// `text` as the cursor literal every system reads alike: zoned instants in UTC with `+00`, wall-clock times without an offset. +/// `text` as the cursor literal every system reads alike: zoned instants in UTC with `+00`, wall-clock times without an offset; an ODBC `{ts '...'}` or `{d '...'}` literal, as a query against an ODBC source writes it, reads as the value inside. pub fn cursor_text(text: &str, column_type: &ColumnType) -> Result { + let text = odbc_value(text).unwrap_or(text); let zoned = DateTime::parse_from_rfc3339(text); let naive = NAIVE_TIMESTAMP_FORMATS .iter() @@ -67,3 +68,13 @@ pub fn cursor_text(text: &str, column_type: &ColumnType) -> Result Ok(text.to_owned()), } } + +/// The quoted value of an ODBC `{ts '...'}` or `{d '...'}` escape literal, in any letter case and spacing. +fn odbc_value(text: &str) -> Option<&str> { + let inner = text.trim().strip_prefix('{')?.strip_suffix('}')?.trim(); + let (keyword, quoted) = inner.split_at(inner.find('\'')?); + let keyword = keyword.trim(); + (keyword.eq_ignore_ascii_case("ts") || keyword.eq_ignore_ascii_case("d")) + .then(|| quoted.strip_prefix('\'')?.strip_suffix('\'')) + .flatten() +} diff --git a/crates/sqlload/src/adapters/contract/main/mod.rs b/crates/sqlload/src/adapters/contract/main/mod.rs index 718c6044..f6dee1ec 100644 --- a/crates/sqlload/src/adapters/contract/main/mod.rs +++ b/crates/sqlload/src/adapters/contract/main/mod.rs @@ -4,4 +4,5 @@ pub(in crate::adapters) mod _footer_rows; pub mod cursor_bound; pub mod cursor_from_now; pub mod cursor_text; +pub mod odbc_literal; pub mod staging_id; diff --git a/crates/sqlload/src/adapters/contract/main/odbc_literal.rs b/crates/sqlload/src/adapters/contract/main/odbc_literal.rs new file mode 100644 index 00000000..78e526dd --- /dev/null +++ b/crates/sqlload/src/adapters/contract/main/odbc_literal.rs @@ -0,0 +1,26 @@ +//! ODBC escape literals for date and timestamp cursor bounds, which every ODBC driver turns into its database's own syntax. + +use chrono::{NaiveDate, NaiveDateTime}; + +use crate::adapters::contract::constants::{NAIVE_TIMESTAMP_FORMATS, ODBC_TIMESTAMP_FORMAT}; +use crate::column_types::models::ColumnType; + +/// `value`, a normalised cursor bound, as `{ts 'YYYY-MM-DD HH:MM:SS[.fff]'}` for a timestamp without a time zone (fractional seconds in groups of three, none when whole) or `{d 'YYYY-MM-DD'}` for a date; `None` for any other cursor, or a value that is not one, which then stays a string literal. +pub fn odbc_literal(value: &str, column_type: &ColumnType) -> Option { + match column_type { + ColumnType::TimestampNtz => NAIVE_TIMESTAMP_FORMATS + .iter() + .find_map( + |format| match NaiveDateTime::parse_from_str(value, format) { + Ok(time) => Some(time), + Err(_) => None, + }, + ) + .map(|time| format!("{{ts '{}'}}", time.format(ODBC_TIMESTAMP_FORMAT))), + ColumnType::Date => match NaiveDate::parse_from_str(value, "%Y-%m-%d") { + Ok(date) => Some(format!("{{d '{}'}}", date.format("%Y-%m-%d"))), + Err(_) => None, + }, + _ => None, + } +} diff --git a/crates/sqlload/src/adapters/contract/tests.rs b/crates/sqlload/src/adapters/contract/tests.rs index ee687e56..5e9ea5bc 100644 --- a/crates/sqlload/src/adapters/contract/tests.rs +++ b/crates/sqlload/src/adapters/contract/tests.rs @@ -18,6 +18,8 @@ mod helpers; mod insert_rows; #[path = "tests/test_live_type.rs"] mod live_type; +#[path = "tests/test_odbc_literal.rs"] +mod odbc_literal; #[path = "tests/test_sql_literal.rs"] mod sql_literal; #[path = "tests/test_staging_changes.rs"] diff --git a/crates/sqlload/src/adapters/contract/tests/test_cursor_bound.rs b/crates/sqlload/src/adapters/contract/tests/test_cursor_bound.rs index 65196d08..83c3e705 100644 --- a/crates/sqlload/src/adapters/contract/tests/test_cursor_bound.rs +++ b/crates/sqlload/src/adapters/contract/tests/test_cursor_bound.rs @@ -23,6 +23,13 @@ fn given_a_cursor_maximum_when_making_the_lower_bound_then_lookback_and_zone_are lookback: HOUR, expected_bound: Ok("2026-11-04 09:02:03.000000"), }, + CursorBoundTestCase { + description: "a wall-clock millisecond maximum, as an ODBC source's cursor resumes from, keeps its milliseconds", + raw: "1769903999997000", + value: CursorValue::EpochMicros { zoned: false }, + lookback: None, + expected_bound: Ok("2026-01-31 23:59:59.997000"), + }, CursorBoundTestCase { description: "a date moves back by whole days, rounding the lookback up", raw: "2026-11-04", diff --git a/crates/sqlload/src/adapters/contract/tests/test_cursor_text.rs b/crates/sqlload/src/adapters/contract/tests/test_cursor_text.rs index d95452d4..bd85df09 100644 --- a/crates/sqlload/src/adapters/contract/tests/test_cursor_text.rs +++ b/crates/sqlload/src/adapters/contract/tests/test_cursor_text.rs @@ -29,6 +29,36 @@ fn given_a_cursor_flag_when_normalising_then_every_system_reads_the_same_bound() column_type: ColumnType::TimestampNtz, expected_text: Err("has a UTC offset"), }, + CursorTextTestCase { + description: "a wall-clock cursor reads an ODBC {ts} literal with milliseconds", + text: "{ts '2026-01-31 23:59:59.997'}", + column_type: ColumnType::TimestampNtz, + expected_text: Ok("2026-01-31 23:59:59.997000"), + }, + CursorTextTestCase { + description: "an ODBC literal reads in any letter case and spacing", + text: " { TS'2026-01-31 23:59:59' } ", + column_type: ColumnType::TimestampNtz, + expected_text: Ok("2026-01-31 23:59:59.000000"), + }, + CursorTextTestCase { + description: "a date cursor reads an ODBC {d} literal", + text: "{d '2026-02-01'}", + column_type: ColumnType::Date, + expected_text: Ok("2026-02-01"), + }, + CursorTextTestCase { + description: "a zoned cursor reads an ODBC {ts} literal as UTC", + text: "{ts '2026-01-31 23:59:59.5'}", + column_type: ColumnType::TimestampTz, + expected_text: Ok("2026-01-31 23:59:59.500000+00"), + }, + CursorTextTestCase { + description: "an ODBC literal missing its closing quote is refused", + text: "{ts '2026-01-31 23:59:59}", + column_type: ColumnType::TimestampNtz, + expected_text: Err("is not an RFC 3339 timestamp or a date"), + }, CursorTextTestCase { description: "a date cursor needs a date", text: "2026-11-04T10:00:00Z", diff --git a/crates/sqlload/src/adapters/contract/tests/test_odbc_literal.rs b/crates/sqlload/src/adapters/contract/tests/test_odbc_literal.rs new file mode 100644 index 00000000..2c6d78a6 --- /dev/null +++ b/crates/sqlload/src/adapters/contract/tests/test_odbc_literal.rs @@ -0,0 +1,78 @@ +use crate::adapters::contract::main::odbc_literal::odbc_literal; +use crate::adapters::contract::tests::test_types::OdbcLiteralTestCase; +use crate::column_types::models::ColumnType; + +#[test] +fn given_a_cursor_bound_when_writing_it_for_an_odbc_source_then_dates_and_wall_clock_times_are_escape_literals() + { + let test_cases = [ + OdbcLiteralTestCase { + description: "a millisecond bound keeps exactly three fractional digits", + value: "2026-01-31 23:59:59.997000", + column_type: ColumnType::TimestampNtz, + expected_literal: Some("{ts '2026-01-31 23:59:59.997'}"), + }, + OdbcLiteralTestCase { + description: "half a second is written as milliseconds", + value: "2026-01-31 23:59:59.500000", + column_type: ColumnType::TimestampNtz, + expected_literal: Some("{ts '2026-01-31 23:59:59.500'}"), + }, + OdbcLiteralTestCase { + description: "a microsecond bound keeps all six digits", + value: "2026-01-31 23:59:59.000001", + column_type: ColumnType::TimestampNtz, + expected_literal: Some("{ts '2026-01-31 23:59:59.000001'}"), + }, + OdbcLiteralTestCase { + description: "a whole second has no fraction", + value: "2026-02-01 00:00:00.000000", + column_type: ColumnType::TimestampNtz, + expected_literal: Some("{ts '2026-02-01 00:00:00'}"), + }, + OdbcLiteralTestCase { + description: "a bound written with a T reads the same", + value: "2026-02-01T08:30:00.25", + column_type: ColumnType::TimestampNtz, + expected_literal: Some("{ts '2026-02-01 08:30:00.250'}"), + }, + OdbcLiteralTestCase { + description: "a date is a {d} literal", + value: "2026-02-01", + column_type: ColumnType::Date, + expected_literal: Some("{d '2026-02-01'}"), + }, + OdbcLiteralTestCase { + description: "a zoned timestamp has no escape, since {ts} carries no offset", + value: "2026-01-31 23:59:59.997000+00", + column_type: ColumnType::TimestampTz, + expected_literal: None, + }, + OdbcLiteralTestCase { + description: "a value that is not a timestamp is never put inside an escape", + value: "2026-01-31'} OR 1=1 --", + column_type: ColumnType::TimestampNtz, + expected_literal: None, + }, + OdbcLiteralTestCase { + description: "a date cursor given a timestamp has no escape", + value: "2026-02-01 00:00:00", + column_type: ColumnType::Date, + expected_literal: None, + }, + OdbcLiteralTestCase { + description: "an integer cursor has no escape", + value: "42", + column_type: ColumnType::Int, + expected_literal: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + odbc_literal(test_case.value, &test_case.column_type).as_deref(), + test_case.expected_literal, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/adapters/contract/tests/test_types.rs b/crates/sqlload/src/adapters/contract/tests/test_types.rs index 4b494e58..85f2a367 100644 --- a/crates/sqlload/src/adapters/contract/tests/test_types.rs +++ b/crates/sqlload/src/adapters/contract/tests/test_types.rs @@ -147,3 +147,10 @@ pub(crate) struct CursorFromNowTestCase { /// The cursor text, or text the error contains. pub(crate) expected_text: Result<&'static str, &'static str>, } + +pub(crate) struct OdbcLiteralTestCase { + pub(crate) description: &'static str, + pub(crate) value: &'static str, + pub(crate) column_type: ColumnType, + pub(crate) expected_literal: Option<&'static str>, +} diff --git a/crates/sqlload/src/loader/_helpers/publish.rs b/crates/sqlload/src/loader/_helpers/publish.rs index 90a81933..d7037e66 100644 --- a/crates/sqlload/src/loader/_helpers/publish.rs +++ b/crates/sqlload/src/loader/_helpers/publish.rs @@ -9,6 +9,7 @@ use crate::adapters::contract::models::WriteStatementsRequest; use crate::adapters::contract::models::WriteTarget; use crate::adapters::contract::models::{DirectBatch, DirectLoaded, UnloadedBatch}; use crate::adapters::contract::types::TableWriter; +use crate::adapters::dispatch::main::dialect::source_binds_text::source_binds_text; use crate::adapters::dispatch::main::dialect::source_dialect::source_dialect; use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter; use crate::adapters::dispatch::main::route::sends_rows::sends_rows; @@ -1103,15 +1104,13 @@ fn window_filter(table: &TablePlan, window: &CursorWindow) -> Option { true => INCLUSIVE_BOUND, false => EXCLUSIVE_BOUND, }; + let source = table.spec.warehouse_source.as_ref(); source_filter( &column.expression, ( Some(&column.output), - table - .spec - .warehouse_source - .as_ref() - .and_then(|source| source_dialect(source.system)), + source.and_then(|source| source_dialect(source.system)), + source.is_some_and(|source| source_binds_text(source.system)), ), ( window.from.as_deref().map(|from| (operator, from)), diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index f37865ad..c3851455 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -1,6 +1,7 @@ use crate::adapters::contract::main::cursor_bound::cursor_bound; use crate::adapters::contract::main::cursor_from_now::cursor_from_now; use crate::adapters::contract::main::cursor_text::cursor_text; +use crate::adapters::contract::main::odbc_literal::odbc_literal; use crate::adapters::contract::models::CursorQuery; use crate::adapters::contract::models::NowShift; use crate::adapters::contract::models::WriteMode; @@ -997,10 +998,10 @@ pub(crate) fn source_cursor( )) } -/// `column >= 'from' AND column < 'to'`, from the bounds that are set, as literals of the source's `dialect` (Postgres's when `None`): a numeric cursor's plain-number bounds as numbers (`render_cursor_number`), as BigQuery compares a number with no string; a text cursor's bounds as `?` placeholders when `parameters` collects their values, in order, to bind; any other bound as the dialect's string literal, so a value holding a quote or a backslash stays one string. +/// `column >= 'from' AND column < 'to'`, from the bounds that are set, as literals of the source's `dialect` (Postgres's when `None`): a numeric cursor's plain-number bounds as numbers (`render_cursor_number`), as BigQuery compares a number with no string; a date or wall-clock timestamp cursor's bounds as ODBC `{d '...'}`/`{ts '...'}` literals when `odbc` (an ODBC source, whose driver writes them in its database's syntax, where a string may not compare with a timestamp); a text cursor's bounds as `?` placeholders when `parameters` collects their values, in order, to bind; any other bound as the dialect's string literal, so a value holding a quote or a backslash stays one string. pub(crate) fn source_filter( column: &str, - (cursor_type, dialect): (Option<&ColumnType>, Option), + (cursor_type, dialect, odbc): (Option<&ColumnType>, Option, bool), (lower, upper): (Option<(&str, &str)>, Option<&str>), mut parameters: Option<&mut Vec>, ) -> Option { @@ -1008,14 +1009,18 @@ pub(crate) fn source_filter( let text = matches!(cursor_type, Some(ColumnType::Text | ColumnType::Varchar(_))); let mut literal = |value: &str| match ( cursor_type.filter(|cursor_type| numeric_cursor(cursor_type) && is_number(value)), + cursor_type + .filter(|_| odbc) + .and_then(|cursor_type| odbc_literal(value, cursor_type)), parameters.as_mut().filter(|_| text), ) { - (Some(cursor_type), _) => adapter.render_cursor_number(value, cursor_type), - (None, Some(parameters)) => { + (Some(cursor_type), _, _) => adapter.render_cursor_number(value, cursor_type), + (None, Some(escaped), _) => escaped, + (None, None, Some(parameters)) => { parameters.push(value.to_owned()); "?".to_owned() } - (None, None) => adapter.render_loader_value_literal(value), + (None, None, None) => adapter.render_loader_value_literal(value), }; let mut bounds: Vec = Vec::new(); if let Some((operator, value)) = lower { @@ -1235,17 +1240,17 @@ pub(crate) async fn cursor_window( (None, None) => (None, note), }; let source = request.spec.warehouse_source.as_ref(); + let odbc = source.is_some_and(|source| source_binds_text(source.system)); let mut parameters: Vec = Vec::new(); let filter = source_filter( request.cursor_expression.unwrap_or_default(), ( request.cursor_type, source.and_then(|source| source_dialect(source.system)), + odbc, ), (from.as_deref().map(|from| (operator, from)), to.as_deref()), - source - .is_some_and(|source| source_binds_text(source.system)) - .then_some(&mut parameters), + odbc.then_some(&mut parameters), ); Ok(Ok(Window { filter, diff --git a/crates/sqlload/src/loader/main/_files_window.rs b/crates/sqlload/src/loader/main/_files_window.rs index 9cb9a132..fc202e1b 100644 --- a/crates/sqlload/src/loader/main/_files_window.rs +++ b/crates/sqlload/src/loader/main/_files_window.rs @@ -137,7 +137,7 @@ pub(in crate::loader) fn files_window(request: &FilesWindowRequest<'_>) -> Resul Ok(Window { filter: source_filter( &expression, - (Some(cursor_type), None), + (Some(cursor_type), None, false), (from.as_deref().map(|from| (operator, from)), to.as_deref()), None, ), diff --git a/crates/sqlload/src/loader/tests/test_source_filter.rs b/crates/sqlload/src/loader/tests/test_source_filter.rs index 0047b415..4fe0950d 100644 --- a/crates/sqlload/src/loader/tests/test_source_filter.rs +++ b/crates/sqlload/src/loader/tests/test_source_filter.rs @@ -10,7 +10,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the description: "a numeric cursor's bounds are bare numbers, which BigQuery compares with an INT64 column", cursor_type: ColumnType::Int, dialect: None, - binds: false, + odbc: false, lower: Some((">", "41")), upper: Some("100.5"), expected_filter: Some("`id` > 41 AND `id` < 100.5"), @@ -20,7 +20,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the description: "a text cursor holding digits stays a string", cursor_type: ColumnType::Text, dialect: None, - binds: false, + odbc: false, lower: Some((">=", "41")), upper: None, expected_filter: Some("`id` >= '41'"), @@ -33,7 +33,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the scale: 9, }, dialect: Some(Warehouse::BigQuery), - binds: false, + odbc: false, lower: Some((">", "1234567890123.000000001")), upper: Some("-0.5"), expected_filter: Some( @@ -48,7 +48,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the scale: 9, }, dialect: Some(Warehouse::Snowflake), - binds: false, + odbc: false, lower: Some((">", "12.5")), upper: None, expected_filter: Some("`id` > 12.5"), @@ -58,7 +58,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the description: "a BigQuery text bound escapes quotes and backslashes with a backslash, so a value cannot end the string", cursor_type: ColumnType::Text, dialect: Some(Warehouse::BigQuery), - binds: false, + odbc: false, lower: Some((">", r"zz\' UNION ALL SELECT 1 --")), upper: Some(r"o'brien\"), expected_filter: Some(r"`id` > 'zz\\\' UNION ALL SELECT 1 --' AND `id` < 'o\'brien\\'"), @@ -68,7 +68,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the description: "a Snowflake text maximum ending in a backslash doubles it, so it cannot escape the closing quote", cursor_type: ColumnType::Text, dialect: Some(Warehouse::Snowflake), - binds: false, + odbc: false, lower: Some((">", r"o'brien\")), upper: None, expected_filter: Some(r"`id` > 'o''brien\\'"), @@ -78,7 +78,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the description: "a Snowflake text bound holding a backslash before a quote stays one string, so the rest is not SQL", cursor_type: ColumnType::Text, dialect: Some(Warehouse::Snowflake), - binds: false, + odbc: false, lower: Some((">=", r"zz\' OR 1=1 --")), upper: Some(r"zz\'"), expected_filter: Some(r"`id` >= 'zz\\'' OR 1=1 --' AND `id` < 'zz\\'''"), @@ -88,7 +88,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the description: "a Postgres text bound doubles its quotes and keeps backslashes, as standard strings read them", cursor_type: ColumnType::Text, dialect: None, - binds: false, + odbc: false, lower: Some((">", r"o'brien\")), upper: None, expected_filter: Some(r"`id` > 'o''brien\'"), @@ -98,7 +98,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the description: "an ODBC text bound holding a quote and a backslash is a bound parameter, so no database's literal rules can misread it", cursor_type: ColumnType::Varchar(40), dialect: None, - binds: true, + odbc: true, lower: Some((">", r"zz\' OR 1=1 --")), upper: Some(r"o'brien\"), expected_filter: Some("`id` > ? AND `id` < ?"), @@ -108,17 +108,79 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the description: "an ODBC numeric bound stays a bare number, not a parameter", cursor_type: ColumnType::Int, dialect: None, - binds: true, + odbc: true, lower: Some((">=", "41")), upper: None, expected_filter: Some("`id` >= 41"), expected_parameters: &[], }, + SourceFilterTestCase { + description: "an ODBC timestamp cursor resumes after its millisecond maximum as {ts} literals, which OpenEdge needs and every driver reads", + cursor_type: ColumnType::TimestampNtz, + dialect: None, + odbc: true, + lower: Some((">", "2026-01-31 23:59:59.997000")), + upper: Some("2026-02-01 00:00:00.000000"), + expected_filter: Some( + "`id` > {ts '2026-01-31 23:59:59.997'} AND `id` < {ts '2026-02-01 00:00:00'}", + ), + expected_parameters: &[], + }, + SourceFilterTestCase { + description: "an ODBC timestamp bound keeps microseconds rather than rounding them away", + cursor_type: ColumnType::TimestampNtz, + dialect: None, + odbc: true, + lower: Some((">=", "2026-01-31 23:59:59.123456")), + upper: None, + expected_filter: Some("`id` >= {ts '2026-01-31 23:59:59.123456'}"), + expected_parameters: &[], + }, + SourceFilterTestCase { + description: "an ODBC date cursor's bound is a {d} literal", + cursor_type: ColumnType::Date, + dialect: None, + odbc: true, + lower: Some((">=", "2026-02-01")), + upper: None, + expected_filter: Some("`id` >= {d '2026-02-01'}"), + expected_parameters: &[], + }, + SourceFilterTestCase { + description: "a timestamp bound from any other source stays a string literal", + cursor_type: ColumnType::TimestampNtz, + dialect: None, + odbc: false, + lower: Some((">", "2026-01-31 23:59:59.997000")), + upper: None, + expected_filter: Some("`id` > '2026-01-31 23:59:59.997000'"), + expected_parameters: &[], + }, + SourceFilterTestCase { + description: "an ODBC zoned timestamp bound stays a string literal, since {ts} has no offset", + cursor_type: ColumnType::TimestampTz, + dialect: None, + odbc: true, + lower: Some((">", "2026-01-31 23:59:59.997000+00")), + upper: None, + expected_filter: Some("`id` > '2026-01-31 23:59:59.997000+00'"), + expected_parameters: &[], + }, + SourceFilterTestCase { + description: "an ODBC timestamp bound that is not a timestamp stays one quoted string, never an escape", + cursor_type: ColumnType::TimestampNtz, + dialect: None, + odbc: true, + lower: Some((">", "2026-01-31'} OR 1=1 --")), + upper: None, + expected_filter: Some("`id` > '2026-01-31''} OR 1=1 --'"), + expected_parameters: &[], + }, SourceFilterTestCase { description: "a numeric cursor's bound that isn't a plain number is quoted, never written bare", cursor_type: ColumnType::Float, dialect: None, - binds: false, + odbc: false, lower: Some((">=", "1 OR 1=1")), upper: Some("NaN"), expected_filter: Some("`id` >= '1 OR 1=1' AND `id` < 'NaN'"), @@ -128,7 +190,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the description: "no bounds, no filter", cursor_type: ColumnType::Int, dialect: None, - binds: false, + odbc: false, lower: None, upper: None, expected_filter: None, @@ -139,9 +201,13 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the let mut parameters = Vec::new(); let filter = source_filter( "`id`", - (Some(&test_case.cursor_type), test_case.dialect), + ( + Some(&test_case.cursor_type), + test_case.dialect, + test_case.odbc, + ), (test_case.lower, test_case.upper), - test_case.binds.then_some(&mut parameters), + test_case.odbc.then_some(&mut parameters), ); assert_eq!( (filter.as_deref(), parameters), diff --git a/crates/sqlload/src/loader/tests/test_types.rs b/crates/sqlload/src/loader/tests/test_types.rs index 46ae2c73..d40febd8 100644 --- a/crates/sqlload/src/loader/tests/test_types.rs +++ b/crates/sqlload/src/loader/tests/test_types.rs @@ -91,8 +91,8 @@ pub(crate) struct SourceFilterTestCase { pub(crate) description: &'static str, pub(crate) cursor_type: ColumnType, pub(crate) dialect: Option, - /// Whether text bounds are bound as parameters, as an ODBC source's are. - pub(crate) binds: bool, + /// Whether the source is an ODBC source: text bounds bound as parameters, date and timestamp bounds as escape literals. + pub(crate) odbc: bool, pub(crate) lower: Option<(&'static str, &'static str)>, pub(crate) upper: Option<&'static str>, pub(crate) expected_filter: Option<&'static str>, From 68e25e222e747c3c6a723328f2f7f6c07ffb38b5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:57:18 +0000 Subject: [PATCH 06/25] ci(publish): a new push to a pull request cancels its older wheel builds; releases never cancel --- .github/workflows/publish.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index eedc1248..4503a2ef 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -19,6 +19,12 @@ on: - crates/sqlload/Cargo.toml - crates/sqlload-python/Cargo.toml +# A new push to a pull request cancels its older wheel builds; a release run is a group of its own, +# so releases never cancel each other. +concurrency: + group: publish-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + permissions: contents: read From 9ab408f4d5412c9980575174b9d1e1469f261953 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:24:57 +0000 Subject: [PATCH 07/25] test: write Windows temp paths as TOML literal strings, and allow the folder-swap gap Windows has A Windows temp path in a TOML basic string reads \U as a unicode escape, so the command tests now quote the root with single quotes. Windows can't swap two folders in one rename, so a reader may briefly see no folder there, as the files target docs now say. --- crates/sqlload/src/adapters/files/tests/helpers.rs | 5 +++++ crates/sqlload/src/adapters/files/tests/test_swap_folder.rs | 3 ++- crates/sqlload/src/command/tests/test_loader_commands.rs | 6 +++--- website/src/content/docs/docs/targets/files.mdx | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/crates/sqlload/src/adapters/files/tests/helpers.rs b/crates/sqlload/src/adapters/files/tests/helpers.rs index 6541236a..63894966 100644 --- a/crates/sqlload/src/adapters/files/tests/helpers.rs +++ b/crates/sqlload/src/adapters/files/tests/helpers.rs @@ -36,6 +36,11 @@ use crate::store_paths::models::{Location, StoreKind}; const NOV_4_PLUS_1_US: i64 = 1_793_750_400_000_001; /// What `watch_folder` records when the folder is missing. pub(crate) const NO_FOLDER: &str = ""; +/// What a reader may also see while a folder is swapped in: nothing on Windows, which cannot swap two folders in one rename, so the old one is moved aside first. +pub(crate) const MID_SWAP: &[&str] = match cfg!(windows) { + true => &[NO_FOLDER], + false => &[], +}; /// How the data files a load writes are named. const PART_NAME_START: &str = "part-"; diff --git a/crates/sqlload/src/adapters/files/tests/test_swap_folder.rs b/crates/sqlload/src/adapters/files/tests/test_swap_folder.rs index f7bef24a..ef0deaab 100644 --- a/crates/sqlload/src/adapters/files/tests/test_swap_folder.rs +++ b/crates/sqlload/src/adapters/files/tests/test_swap_folder.rs @@ -7,7 +7,7 @@ use crate::adapters::files::main::repair_table::repair_table; use crate::adapters::files::main::swap_folder::swap_folder; use crate::adapters::files::models::TablePath; use crate::adapters::files::tests::helpers::{ - NO_FOLDER, block_on, local_files, watch_folder, write_all, + MID_SWAP, NO_FOLDER, block_on, local_files, watch_folder, write_all, }; use crate::adapters::files::tests::test_types::{RemoveStagedTestCase, SwapFolderTestCase}; use crate::file_store::main::open_folder::open_folder; @@ -99,6 +99,7 @@ fn given_a_staged_table_when_swapping_it_in_then_readers_see_the_old_or_new_file .iter() .copied() .chain(test_case.live.is_empty().then_some(NO_FOLDER)) + .chain(MID_SWAP.iter().copied()) .map(str::to_owned) .collect(); let new: Vec = vec!["part-new-0.csv".to_owned()]; diff --git a/crates/sqlload/src/command/tests/test_loader_commands.rs b/crates/sqlload/src/command/tests/test_loader_commands.rs index 3efb4110..2b0bc858 100644 --- a/crates/sqlload/src/command/tests/test_loader_commands.rs +++ b/crates/sqlload/src/command/tests/test_loader_commands.rs @@ -5,7 +5,7 @@ use crate::command::tests::test_types::{ const TARGET: &str = "[connections.wh]\nkind = \"duckdb\"\ndatabase = \":memory:\"\n[targets.dev]\nconnection = \"wh\"\nloader_schema = \"RAW\"\n"; const NO_LOAD_METHOD: &str = "[connections.wh]\nkind = \"snowflake\"\naccount = \"a\"\nuser = \"u\"\nauth = { method = \"pat\", token = \"t\" }\ndatabase = \"D\"\n[targets.dev]\nconnection = \"wh\"\nloader_schema = \"RAW\"\n"; -const REFUSING_TARGET: &str = "[connections.pg]\nkind = \"postgres\"\nhost = \"127.0.0.1\"\nport = 1\nuser = \"loader\"\npassword = \"p\"\ndatabase = \"app\"\nsslmode = \"disable\"\n[targets.dev]\nconnection = \"pg\"\nloader_schema = \"raw\"\n[sources.orders]\nkind = \"files\"\ndescription = \"Test table.\"\nconnection = \"lake\"\npath = \"orders/\"\nformat = \"csv\"\nwrite_strategy = \"append\"\n[connections.lake]\nkind = \"filesystem\"\nroot = \"{root}\"\n"; +const REFUSING_TARGET: &str = "[connections.pg]\nkind = \"postgres\"\nhost = \"127.0.0.1\"\nport = 1\nuser = \"loader\"\npassword = \"p\"\ndatabase = \"app\"\nsslmode = \"disable\"\n[targets.dev]\nconnection = \"pg\"\nloader_schema = \"raw\"\n[sources.orders]\nkind = \"files\"\ndescription = \"Test table.\"\nconnection = \"lake\"\npath = \"orders/\"\nformat = \"csv\"\nwrite_strategy = \"append\"\n[connections.lake]\nkind = \"filesystem\"\nroot = '{root}'\n"; const UNREACHABLE_SOURCE: &str = "[connections.wh]\nkind = \"duckdb\"\ndatabase = \":memory:\"\n[targets.dev]\nconnection = \"wh\"\nloader_schema = \"RAW\"\n[connections.app]\nkind = \"postgres\"\nhost = \"127.0.0.1\"\nport = 1\nuser = \"u\"\npassword = \"p\"\ndatabase = \"app\"\nsslmode = \"disable\"\n[sources.app]\nkind = \"postgres\"\nconnection = \"app\"\ndestination = \"dev\"\ninclude = [\"public.*\"]\n"; #[test] @@ -116,7 +116,7 @@ fn given_a_files_source_into_a_files_target_when_planning_then_an_overlapping_fo ]; for test_case in test_cases { let project = format!( - "[connections.lake]\nkind = \"filesystem\"\nroot = \"{}\"\n[targets.lake]\nconnection = \"lake\"\npath = \"{}\"\nsingle_file = {}\n[sources.orders]\nkind = \"files\"\ndescription = \"Test table.\"\nconnection = \"lake\"\npath = \"data/orders/\"\nformat = \"csv\"\nwrite_strategy = \"replace\"\npartition_columns = {{ day = \"date\" }}\n", + "[connections.lake]\nkind = \"filesystem\"\nroot = '{}'\n[targets.lake]\nconnection = \"lake\"\npath = \"{}\"\nsingle_file = {}\n[sources.orders]\nkind = \"files\"\ndescription = \"Test table.\"\nconnection = \"lake\"\npath = \"data/orders/\"\nformat = \"csv\"\nwrite_strategy = \"replace\"\npartition_columns = {{ day = \"date\" }}\n", root.path().display(), test_case.target_path, test_case.single_file @@ -170,7 +170,7 @@ fn given_a_failure_a_rerun_may_pass_when_loading_then_it_exits_75_and_says_so_el }, ExitCodeTestCase { description: "--json: an unreachable source beside a plan error that is not retryable exits 1", - project: "[connections.wh]\nkind = \"duckdb\"\ndatabase = \":memory:\"\n[targets.dev]\nconnection = \"wh\"\nloader_schema = \"RAW\"\n[connections.app]\nkind = \"postgres\"\nhost = \"127.0.0.1\"\nport = 1\nuser = \"u\"\npassword = \"p\"\ndatabase = \"app\"\nsslmode = \"disable\"\n[sources.app]\nkind = \"postgres\"\nconnection = \"app\"\ndestination = \"dev\"\ninclude = [\"public.*\"]\n[sources.orders]\nkind = \"files\"\ndescription = \"Test table.\"\nconnection = \"lake\"\npath = \"missing/\"\nformat = \"csv\"\nwrite_strategy = \"append\"\ndestination = \"dev\"\n[connections.lake]\nkind = \"filesystem\"\nroot = \"{root}\"\n", + project: "[connections.wh]\nkind = \"duckdb\"\ndatabase = \":memory:\"\n[targets.dev]\nconnection = \"wh\"\nloader_schema = \"RAW\"\n[connections.app]\nkind = \"postgres\"\nhost = \"127.0.0.1\"\nport = 1\nuser = \"u\"\npassword = \"p\"\ndatabase = \"app\"\nsslmode = \"disable\"\n[sources.app]\nkind = \"postgres\"\nconnection = \"app\"\ndestination = \"dev\"\ninclude = [\"public.*\"]\n[sources.orders]\nkind = \"files\"\ndescription = \"Test table.\"\nconnection = \"lake\"\npath = \"missing/\"\nformat = \"csv\"\nwrite_strategy = \"append\"\ndestination = \"dev\"\n[connections.lake]\nkind = \"filesystem\"\nroot = '{root}'\n", argv: &["load", "--json"], expected_output: "\"retryable\": false", expected_exit: "exit 1", diff --git a/website/src/content/docs/docs/targets/files.mdx b/website/src/content/docs/docs/targets/files.mdx index 38818f17..4a50ed77 100644 --- a/website/src/content/docs/docs/targets/files.mdx +++ b/website/src/content/docs/docs/targets/files.mdx @@ -95,7 +95,7 @@ table folder, carry over every file of the table folder that the load doesn't re `_sqlload/runs/` log, the load's lease, and any file you keep beside the data), then swap the two folders in one rename (`renameat2` with `RENAME_EXCHANGE` on Linux, `RENAME_SWAP` on macOS) and delete the old one. A reader sees the old files or the new ones, never both. Where the filesystem can't swap folders in one -rename, SQLLoad moves the old folder aside first and warns, which leaves a moment with no folder. A +rename, as on Windows, SQLLoad moves the old folder aside first and warns, which leaves a moment with no folder. A hidden folder left by a stopped or failed load is deleted by the next load. A table at the root of the folder connection has no folder beside it to swap, so it is replaced as in S3. From a733bc502419a1c718b79f063532178a71c49170 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:06:13 +0000 Subject: [PATCH 08/25] feat(odbc): read MySQL through Connector/ODBC: accept ANSI_QUOTES sessions, list tables without schemas MySQL Connector/ODBC reports a backtick as its identifier quote even when the session reads ANSI double quotes, so the connection now asks the database once. A driver without schemas is asked for its tables with no schema pattern, which MySQL's NO_SCHEMA default otherwise refuses. Adds MySQL and SQL Server containers to the testkit and a script installing the three drivers. WIP for CHI-1236: container tests follow. --- .github/scripts/install_odbc_drivers.sh | 50 +++++++++++++++++++ crates/sqlload-testkit/src/lib.rs | 3 +- .../src/odbc_servers/constants.rs | 14 ++++++ .../src/odbc_servers/main/mod.rs | 2 + .../src/odbc_servers/main/start_mysql.rs | 37 ++++++++++++++ .../src/odbc_servers/main/start_sql_server.rs | 35 +++++++++++++ .../sqlload-testkit/src/odbc_servers/mod.rs | 5 ++ .../src/odbc_servers/models.rs | 8 +++ .../src/odbc_source/_helpers/catalog.rs | 8 ++- .../src/odbc_source/_helpers/connect.rs | 17 +++++-- crates/sqlload/src/odbc_source/constants.rs | 4 ++ 11 files changed, 176 insertions(+), 7 deletions(-) create mode 100755 .github/scripts/install_odbc_drivers.sh create mode 100644 crates/sqlload-testkit/src/odbc_servers/constants.rs create mode 100644 crates/sqlload-testkit/src/odbc_servers/main/mod.rs create mode 100644 crates/sqlload-testkit/src/odbc_servers/main/start_mysql.rs create mode 100644 crates/sqlload-testkit/src/odbc_servers/main/start_sql_server.rs create mode 100644 crates/sqlload-testkit/src/odbc_servers/mod.rs create mode 100644 crates/sqlload-testkit/src/odbc_servers/models.rs diff --git a/.github/scripts/install_odbc_drivers.sh b/.github/scripts/install_odbc_drivers.sh new file mode 100755 index 00000000..79e75ba7 --- /dev/null +++ b/.github/scripts/install_odbc_drivers.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Installs unixODBC and the ODBC drivers the container tests read through, on Debian or Ubuntu x86_64, +# and registers each driver under the name the tests connect with: +# PostgreSQL Unicode psqlODBC, from the distribution +# ODBC Driver 18 for SQL Server Microsoft's msodbcsql18 (installing it accepts Microsoft's licence) +# MySQL ODBC Unicode Driver MySQL Connector/ODBC, from MySQL's generic Linux tarball +# Usage: install_odbc_drivers.sh [--with-dev] (--with-dev adds the unixODBC headers to build against) +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive + +MYSQL_ODBC_VERSION=26.10.0 +MYSQL_ODBC_MD5=09f9e895d60ce69bd1e6a576df9dd4de +MYSQL_ODBC_NAME="MySQL ODBC Unicode Driver" + +sudo_() { if [ "$(id -u)" = 0 ]; then "$@"; else sudo "$@"; fi; } + +packages=(unixodbc odbc-postgresql curl ca-certificates) +if [ "${1:-}" = "--with-dev" ]; then + packages+=(unixodbc-dev) +fi +sudo_ apt-get update -qq +sudo_ apt-get install -y -qq "${packages[@]}" + +# Microsoft's repository for this distribution release, then msodbcsql18. +. /etc/os-release +if ! dpkg -s msodbcsql18 > /dev/null 2>&1; then + curl -fsSL -o /tmp/packages-microsoft-prod.deb "https://packages.microsoft.com/config/${ID}/${VERSION_ID}/packages-microsoft-prod.deb" + sudo_ dpkg -i /tmp/packages-microsoft-prod.deb > /dev/null + rm -f /tmp/packages-microsoft-prod.deb + sudo_ apt-get update -qq + sudo_ env ACCEPT_EULA=Y apt-get install -y -qq msodbcsql18 +fi + +# MySQL Connector/ODBC: the pinned tarball (the CDN keeps the current release, the archive older +# ones), checked against MySQL's published MD5, then its Unicode driver registered under a name +# that doesn't change with the version. +tarball="mysql-connector-odbc-${MYSQL_ODBC_VERSION}-linux-glibc2.28-x86-64bit.tar.gz" +work="$(mktemp -d)" +curl -fsSL -o "$work/$tarball" "https://cdn.mysql.com/Downloads/Connector-ODBC/${MYSQL_ODBC_VERSION%.*}/$tarball" || + curl -fsSL -o "$work/$tarball" "https://downloads.mysql.com/archives/get/p/10/file/$tarball" +echo "$MYSQL_ODBC_MD5 $work/$tarball" | md5sum -c --quiet - +sudo_ mkdir -p /opt/mysql-connector-odbc +sudo_ tar -xzf "$work/$tarball" -C /opt/mysql-connector-odbc --strip-components=1 +rm -rf "$work" +driver="$(ls /opt/mysql-connector-odbc/lib/libmyodbc*w.so)" +printf '[%s]\nDescription=MySQL Connector/ODBC %s\nDriver=%s\nUsageCount=1\n' "$MYSQL_ODBC_NAME" "$MYSQL_ODBC_VERSION" "$driver" > /tmp/mysql-odbc.ini +sudo_ odbcinst -i -d -r -f /tmp/mysql-odbc.ini > /dev/null +rm -f /tmp/mysql-odbc.ini + +odbcinst -q -d diff --git a/crates/sqlload-testkit/src/lib.rs b/crates/sqlload-testkit/src/lib.rs index 9feae2d7..9f0c0d2c 100644 --- a/crates/sqlload-testkit/src/lib.rs +++ b/crates/sqlload-testkit/src/lib.rs @@ -1,4 +1,4 @@ -//! Test doubles for sqlload: a fake Snowflake service, Redpanda and Postgres containers, configurations and an exporter harness. +//! Test doubles for sqlload: a fake Snowflake service, Redpanda, Postgres, MySQL and SQL Server containers, configurations and an exporter harness. #![forbid(unsafe_code)] @@ -9,6 +9,7 @@ pub mod fake_databricks; pub mod fake_sheets; pub mod fake_snowflake; pub mod harness; +pub mod odbc_servers; pub mod parity; pub mod postgres; pub mod records; diff --git a/crates/sqlload-testkit/src/odbc_servers/constants.rs b/crates/sqlload-testkit/src/odbc_servers/constants.rs new file mode 100644 index 00000000..97523dea --- /dev/null +++ b/crates/sqlload-testkit/src/odbc_servers/constants.rs @@ -0,0 +1,14 @@ +pub const MYSQL_IMAGE: &str = "mysql"; +pub const MYSQL_TAG: &str = "8.4"; +pub const MYSQL_PORT: u16 = 3306; +/// The line MySQL logs once the real server, not the image's setup server, listens on its port. +pub const MYSQL_READY: &str = "port: 3306 MySQL Community Server"; +pub const SQL_SERVER_IMAGE: &str = "mcr.microsoft.com/mssql/server"; +pub const SQL_SERVER_TAG: &str = "2022-latest"; +pub const SQL_SERVER_PORT: u16 = 1433; +pub const SQL_SERVER_READY: &str = "SQL Server is now ready for client connections"; +/// The SQL Server administrator's password, which must meet its complexity rules. +pub const SQL_SERVER_PASSWORD: &str = "Sqlload-Test-1"; +pub const USER: &str = "sqlload"; +pub const PASSWORD: &str = "sqlload"; +pub const DATABASE: &str = "app"; diff --git a/crates/sqlload-testkit/src/odbc_servers/main/mod.rs b/crates/sqlload-testkit/src/odbc_servers/main/mod.rs new file mode 100644 index 00000000..d578f5db --- /dev/null +++ b/crates/sqlload-testkit/src/odbc_servers/main/mod.rs @@ -0,0 +1,2 @@ +pub mod start_mysql; +pub mod start_sql_server; diff --git a/crates/sqlload-testkit/src/odbc_servers/main/start_mysql.rs b/crates/sqlload-testkit/src/odbc_servers/main/start_mysql.rs new file mode 100644 index 00000000..25b32bf6 --- /dev/null +++ b/crates/sqlload-testkit/src/odbc_servers/main/start_mysql.rs @@ -0,0 +1,37 @@ +//! Start MySQL. + +use std::time::Duration; + +use testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers::runners::AsyncRunner; +use testcontainers::{GenericImage, ImageExt}; + +use crate::odbc_servers::constants::{ + DATABASE, MYSQL_IMAGE, MYSQL_PORT, MYSQL_READY, MYSQL_TAG, PASSWORD, USER, +}; +use crate::odbc_servers::models::OdbcServer; + +/// A MySQL server with database `app` and user `sqlload`, on a free host port, and a connection string for MySQL Connector/ODBC that sets the session's SQL mode to read ANSI double quotes, as SQLLoad writes them. +pub async fn start_mysql() -> Result { + let container = GenericImage::new(MYSQL_IMAGE, MYSQL_TAG) + .with_exposed_port(MYSQL_PORT.tcp()) + .with_wait_for(WaitFor::message_on_stderr(MYSQL_READY)) + .with_env_var("MYSQL_ROOT_PASSWORD", PASSWORD) + .with_env_var("MYSQL_DATABASE", DATABASE) + .with_env_var("MYSQL_USER", USER) + .with_env_var("MYSQL_PASSWORD", PASSWORD) + .with_startup_timeout(Duration::from_secs(180)) + .start() + .await + .map_err(|error| format!("cannot start MySQL: {error}"))?; + let port = container + .get_host_port_ipv4(MYSQL_PORT.tcp()) + .await + .map_err(|error| error.to_string())?; + Ok(OdbcServer { + connection_string: format!( + "Driver={{MySQL ODBC Unicode Driver}};Server=127.0.0.1;Port={port};Database={DATABASE};Uid={USER};Pwd={PASSWORD};INITSTMT=SET SESSION sql_mode=CONCAT(@@SESSION.sql_mode,',ANSI_QUOTES');" + ), + container, + }) +} diff --git a/crates/sqlload-testkit/src/odbc_servers/main/start_sql_server.rs b/crates/sqlload-testkit/src/odbc_servers/main/start_sql_server.rs new file mode 100644 index 00000000..013a62e3 --- /dev/null +++ b/crates/sqlload-testkit/src/odbc_servers/main/start_sql_server.rs @@ -0,0 +1,35 @@ +//! Start SQL Server. + +use std::time::Duration; + +use testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers::runners::AsyncRunner; +use testcontainers::{GenericImage, ImageExt}; + +use crate::odbc_servers::constants::{ + SQL_SERVER_IMAGE, SQL_SERVER_PASSWORD, SQL_SERVER_PORT, SQL_SERVER_READY, SQL_SERVER_TAG, +}; +use crate::odbc_servers::models::OdbcServer; + +/// A SQL Server (Developer edition, licence accepted for testing) on a free host port, and a connection string for Microsoft's ODBC Driver 18 that signs in as `sa` to `master` and trusts the server's own certificate. +pub async fn start_sql_server() -> Result { + let container = GenericImage::new(SQL_SERVER_IMAGE, SQL_SERVER_TAG) + .with_exposed_port(SQL_SERVER_PORT.tcp()) + .with_wait_for(WaitFor::message_on_stdout(SQL_SERVER_READY)) + .with_env_var("ACCEPT_EULA", "Y") + .with_env_var("MSSQL_SA_PASSWORD", SQL_SERVER_PASSWORD) + .with_startup_timeout(Duration::from_secs(180)) + .start() + .await + .map_err(|error| format!("cannot start SQL Server: {error}"))?; + let port = container + .get_host_port_ipv4(SQL_SERVER_PORT.tcp()) + .await + .map_err(|error| error.to_string())?; + Ok(OdbcServer { + connection_string: format!( + "Driver={{ODBC Driver 18 for SQL Server}};Server=127.0.0.1,{port};Database=master;Uid=sa;Pwd={SQL_SERVER_PASSWORD};TrustServerCertificate=yes;" + ), + container, + }) +} diff --git a/crates/sqlload-testkit/src/odbc_servers/mod.rs b/crates/sqlload-testkit/src/odbc_servers/mod.rs new file mode 100644 index 00000000..9e44a346 --- /dev/null +++ b/crates/sqlload-testkit/src/odbc_servers/mod.rs @@ -0,0 +1,5 @@ +//! MySQL and SQL Server containers, which the ODBC source tests read through their ODBC drivers. + +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlload-testkit/src/odbc_servers/models.rs b/crates/sqlload-testkit/src/odbc_servers/models.rs new file mode 100644 index 00000000..2b468d0f --- /dev/null +++ b/crates/sqlload-testkit/src/odbc_servers/models.rs @@ -0,0 +1,8 @@ +use testcontainers::{ContainerAsync, GenericImage}; + +/// A running database server and the ODBC connection string that reaches it; the container stops when this is dropped. +#[derive(Debug)] +pub struct OdbcServer { + pub connection_string: String, + pub container: ContainerAsync, +} diff --git a/crates/sqlload/src/odbc_source/_helpers/catalog.rs b/crates/sqlload/src/odbc_source/_helpers/catalog.rs index 4ad49bcf..3163ab56 100644 --- a/crates/sqlload/src/odbc_source/_helpers/catalog.rs +++ b/crates/sqlload/src/odbc_source/_helpers/catalog.rs @@ -74,7 +74,7 @@ pub(crate) fn source_table( } } -/// The tables and views whose `schema.table` (or `table`, for a driver without schemas) matches `include` and not `exclude`, with their columns and keys. +/// The tables and views whose `schema.table` (or `table`, for a driver without schemas) matches `include` and not `exclude`, with their columns and keys. A driver without schemas (whose schema names are at most 0 long, as MySQL Connector/ODBC's are by default) is asked with no schema pattern, since it refuses any. pub(crate) fn catalog_tables( connection: &Connection<'_>, (include, exclude): (&[String], &[String]), @@ -82,7 +82,11 @@ pub(crate) fn catalog_tables( let include = table_matcher(include)?; let exclude = table_matcher(exclude)?; let mut names: Vec<(String, String)> = Vec::new(); - for row in connection.tables("", ANY_NAME, ANY_NAME, TABLE_TYPES)? { + let schemas = match connection.max_schema_name_len()? { + 0 => "", + _ => ANY_NAME, + }; + for row in connection.tables("", schemas, ANY_NAME, TABLE_TYPES)? { let row = row?; let (schema, table) = (catalog_text(&row.schema)?, catalog_text(&row.table)?); let qualified = match schema.is_empty() { diff --git a/crates/sqlload/src/odbc_source/_helpers/connect.rs b/crates/sqlload/src/odbc_source/_helpers/connect.rs index fb6d7485..4dd9c30e 100644 --- a/crates/sqlload/src/odbc_source/_helpers/connect.rs +++ b/crates/sqlload/src/odbc_source/_helpers/connect.rs @@ -1,12 +1,12 @@ //! Connections through the process's one ODBC environment. -use arrow_odbc::odbc_api::{Connection, ConnectionOptions, environment}; +use arrow_odbc::odbc_api::{Connection, ConnectionOptions, Cursor, environment}; -use crate::odbc_source::constants::{IDENTIFIER_QUOTE, NO_QUOTE}; +use crate::odbc_source::constants::{ANSI_QUOTES_PROBE, IDENTIFIER_QUOTE, NO_QUOTE}; use crate::odbc_source::errors::OdbcSourceError; use crate::odbc_source::models::OdbcReader; -/// A connection with the reader's connection string, refused when its driver quotes identifiers other than as ANSI SQL does, since SQLLoad writes `"name"`. +/// A connection with the reader's connection string, refused when its database doesn't read `"name"` as a name, since SQLLoad writes ANSI double quotes. A driver that reports another quote is asked once whether the database reads them anyway: MySQL Connector/ODBC reports a backtick even when the session's SQL mode includes `ANSI_QUOTES`. pub(crate) fn connect(reader: &OdbcReader) -> Result, OdbcSourceError> { let connection = environment()?.connect_with_connection_string( reader.connection_string.expose(), @@ -14,8 +14,17 @@ pub(crate) fn connect(reader: &OdbcReader) -> Result, OdbcSo )?; match connection.identifier_quote_char()? { None | Some(IDENTIFIER_QUOTE | NO_QUOTE) => Ok(connection), + Some(_) if reads_ansi_quotes(&connection) => Ok(connection), Some(quote) => Err(OdbcSourceError::Invalid(format!( - "the driver quotes identifiers with {quote:?}; SQLLoad's ODBC source writes ANSI double quotes, so switch the driver to them (for example ANSI_QUOTES)" + "the driver quotes identifiers with {quote:?}; SQLLoad's ODBC source writes ANSI double quotes, so switch the database to them (for MySQL, add INITSTMT=SET SESSION sql_mode=CONCAT(@@SESSION.sql_mode,',ANSI_QUOTES') to the connection string)" ))), } } + +/// Whether the database reads `"name"` as a column name: the probe returns its row only then (a string literal never equals 1). A failed probe counts as no. +fn reads_ansi_quotes(connection: &Connection<'_>) -> bool { + match connection.execute(ANSI_QUOTES_PROBE, (), None) { + Ok(Some(mut cursor)) => matches!(cursor.next_row(), Ok(Some(_))), + Ok(None) | Err(_) => false, + } +} diff --git a/crates/sqlload/src/odbc_source/constants.rs b/crates/sqlload/src/odbc_source/constants.rs index a8a6cdec..0a239e4e 100644 --- a/crates/sqlload/src/odbc_source/constants.rs +++ b/crates/sqlload/src/odbc_source/constants.rs @@ -10,6 +10,10 @@ pub const ANY_NAME: &str = "%"; /// The identifier quote SQLLoad writes, ANSI SQL's. pub const IDENTIFIER_QUOTE: char = '"'; +/// A query returning one row only when the database reads `"name"` as a name rather than a string. +pub const ANSI_QUOTES_PROBE: &str = + "SELECT 1 FROM (SELECT 1 AS sqlload_probe) sqlload_query WHERE \"sqlload_probe\" = 1"; + /// The driver's answer when it does not quote identifiers. pub const NO_QUOTE: char = ' '; From ee9f948cd0d7b438301debd52c6126152b386104 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:06:33 +0000 Subject: [PATCH 09/25] ci(windows): print sqlload's UTF-8 output from the end-to-end script on a cp1252 console --- .github/scripts/windows_e2e.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/scripts/windows_e2e.py b/.github/scripts/windows_e2e.py index 4a555ce1..c841bd3f 100644 --- a/.github/scripts/windows_e2e.py +++ b/.github/scripts/windows_e2e.py @@ -190,6 +190,9 @@ def cdc_checks(exe, work): def main(): + # sqlload prints UTF-8 (box drawing, check marks); a Windows console or pipe defaults to cp1252. + sys.stdout.reconfigure(encoding="utf-8") + sys.stderr.reconfigure(encoding="utf-8") parser = argparse.ArgumentParser() parser.add_argument("exe") parser.add_argument("work", type=pathlib.Path) From 900a354943428b9c17dba7cab73996e4714501b1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:28:53 +0000 Subject: [PATCH 10/25] feat(odbc): bind date and timestamp cursor bounds as typed parameters, and test SQL Server, MySQL and Postgres drivers An ODBC source's date and wall-clock timestamp cursor bounds are now bound as ODBC DATE and TIMESTAMP parameters (seven fractional digits) instead of {d}/{ts} escapes: SQL Server reads {ts} as a datetime, which rounds milliseconds, so a cursor resuming from .997 read that row again. Timestamps a driver describes with fewer fractional digits than it returns (MySQL Connector/ODBC describes DATETIME(n) with none) are now fetched in microseconds instead of truncated. Container tests load a millisecond timestamp cursor and a date cursor through Microsoft's ODBC Driver 18, MySQL Connector/ODBC and psqlODBC, and check the next load reads exactly the rows after each boundary. The Verify container job installs the three drivers with .github/scripts/install_odbc_drivers.sh. WIP for CHI-1236: driver-manager linkage, wheels, Windows drivers and docs follow. --- .github/workflows/verify.yml | 8 +- .../src/odbc_servers/_helpers/containers.rs | 76 ++++++++++++ .../src/odbc_servers/_helpers/mod.rs | 1 + .../src/odbc_servers/constants.rs | 3 +- .../src/odbc_servers/main/mod.rs | 3 +- .../src/odbc_servers/main/start_mysql.rs | 37 ------ .../odbc_servers/main/start_odbc_server.rs | 15 +++ .../src/odbc_servers/main/start_sql_server.rs | 35 ------ .../sqlload-testkit/src/odbc_servers/mod.rs | 3 +- .../src/odbc_servers/models.rs | 11 ++ .../src/adapters/contract/constants.rs | 4 +- .../adapters/contract/main/bound_parameter.rs | 29 +++++ .../sqlload/src/adapters/contract/main/mod.rs | 2 +- .../adapters/contract/main/odbc_literal.rs | 26 ----- .../sqlload/src/adapters/contract/models.rs | 11 ++ crates/sqlload/src/adapters/contract/tests.rs | 4 +- .../src/adapters/contract/tests/helpers.rs | 18 +++ .../contract/tests/test_bound_parameter.rs | 81 +++++++++++++ .../contract/tests/test_odbc_literal.rs | 78 ------------- .../src/adapters/contract/tests/test_types.rs | 8 +- crates/sqlload/src/loader/_helpers/publish.rs | 36 ++++-- crates/sqlload/src/loader/_helpers/sources.rs | 5 +- .../sqlload/src/loader/_helpers/table_plan.rs | 38 +++--- .../sqlload/src/loader/main/_files_window.rs | 2 +- crates/sqlload/src/loader/models.rs | 5 +- .../src/loader/tests/test_source_filter.rs | 41 ++++--- crates/sqlload/src/loader/tests/test_types.rs | 4 +- .../src/odbc_source/_helpers/batches.rs | 14 ++- crates/sqlload/src/odbc_source/constants.rs | 3 + .../sqlload/src/odbc_source/main/read_rows.rs | 47 ++++++-- crates/sqlload/src/odbc_source/models.rs | 5 +- crates/sqlload/tests/loader.rs | 3 + crates/sqlload/tests/loader/odbc/helpers.rs | 57 +++++++++ .../sqlload/tests/loader/test_odbc_drivers.rs | 110 ++++++++++++++++++ crates/sqlload/tests/loader/test_types.rs | 15 +++ fensu.toml | 15 +-- 36 files changed, 582 insertions(+), 271 deletions(-) create mode 100644 crates/sqlload-testkit/src/odbc_servers/_helpers/containers.rs create mode 100644 crates/sqlload-testkit/src/odbc_servers/_helpers/mod.rs delete mode 100644 crates/sqlload-testkit/src/odbc_servers/main/start_mysql.rs create mode 100644 crates/sqlload-testkit/src/odbc_servers/main/start_odbc_server.rs delete mode 100644 crates/sqlload-testkit/src/odbc_servers/main/start_sql_server.rs create mode 100644 crates/sqlload/src/adapters/contract/main/bound_parameter.rs delete mode 100644 crates/sqlload/src/adapters/contract/main/odbc_literal.rs create mode 100644 crates/sqlload/src/adapters/contract/tests/test_bound_parameter.rs delete mode 100644 crates/sqlload/src/adapters/contract/tests/test_odbc_literal.rs create mode 100644 crates/sqlload/tests/loader/test_odbc_drivers.rs diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index bc06eae3..3c5a2928 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -114,8 +114,12 @@ jobs: docker pull clickhouse/clickhouse-server:25.8 docker pull clickhouse/clickhouse-server:26.10 docker pull clickhouse/clickhouse-server:25.3 - - name: Install unixODBC and the PostgreSQL ODBC driver - run: sudo apt-get update -qq && sudo apt-get install -y -qq unixodbc-dev odbc-postgresql + docker pull mysql:8.4 + docker pull mcr.microsoft.com/mssql/server:2022-latest + # unixODBC with psqlODBC, Microsoft's ODBC Driver 18 for SQL Server and MySQL Connector/ODBC, + # which the ODBC source's driver tests read through. + - name: Install unixODBC and the ODBC drivers + run: .github/scripts/install_odbc_drivers.sh --with-dev # Compiles first, so the test step's timeout measures the tests alone. - name: Build container tests run: cargo test -p sqlload --features container-tests,odbc --test exporter --test loader --no-run --quiet diff --git a/crates/sqlload-testkit/src/odbc_servers/_helpers/containers.rs b/crates/sqlload-testkit/src/odbc_servers/_helpers/containers.rs new file mode 100644 index 00000000..656430f9 --- /dev/null +++ b/crates/sqlload-testkit/src/odbc_servers/_helpers/containers.rs @@ -0,0 +1,76 @@ +//! The database containers the ODBC drivers are tested against, each with its driver's connection string. + +use std::time::Duration; + +use testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers::runners::AsyncRunner; +use testcontainers::{GenericImage, ImageExt}; + +use crate::odbc_servers::constants::{ + DATABASE, MYSQL_IMAGE, MYSQL_PORT, MYSQL_READY, MYSQL_TAG, PASSWORD, SQL_SERVER_IMAGE, + SQL_SERVER_PASSWORD, SQL_SERVER_PORT, SQL_SERVER_READY, SQL_SERVER_TAG, USER, +}; +use crate::odbc_servers::models::OdbcServer; +use crate::postgres::main::connect::connect; +use crate::postgres::main::start_postgres::start_postgres; + +/// A MySQL server with database `app` and user `sqlload`, on a free host port, and a connection string for MySQL Connector/ODBC that sets the session's SQL mode to read ANSI double quotes, as SQLLoad writes them. +pub(crate) async fn start_mysql() -> Result { + let container = GenericImage::new(MYSQL_IMAGE, MYSQL_TAG) + .with_exposed_port(MYSQL_PORT.tcp()) + .with_wait_for(WaitFor::message_on_stderr(MYSQL_READY)) + .with_env_var("MYSQL_ROOT_PASSWORD", PASSWORD) + .with_env_var("MYSQL_DATABASE", DATABASE) + .with_env_var("MYSQL_USER", USER) + .with_env_var("MYSQL_PASSWORD", PASSWORD) + .with_startup_timeout(Duration::from_secs(180)) + .start() + .await + .map_err(|error| format!("cannot start MySQL: {error}"))?; + let port = container + .get_host_port_ipv4(MYSQL_PORT.tcp()) + .await + .map_err(|error| error.to_string())?; + Ok(OdbcServer { + connection_string: format!( + "Driver={{MySQL ODBC Unicode Driver}};Server=127.0.0.1;Port={port};Database={DATABASE};Uid={USER};Pwd={PASSWORD};INITSTMT=SET SESSION sql_mode=CONCAT(@@SESSION.sql_mode,',ANSI_QUOTES');" + ), + container, + }) +} + +/// A SQL Server (Developer edition, licence accepted for testing) on a free host port, and a connection string for Microsoft's ODBC Driver 18 that signs in as `sa` to `master` and trusts the server's own certificate. +pub(crate) async fn start_sql_server() -> Result { + let container = GenericImage::new(SQL_SERVER_IMAGE, SQL_SERVER_TAG) + .with_exposed_port(SQL_SERVER_PORT.tcp()) + .with_wait_for(WaitFor::message_on_stdout(SQL_SERVER_READY)) + .with_env_var("ACCEPT_EULA", "Y") + .with_env_var("MSSQL_SA_PASSWORD", SQL_SERVER_PASSWORD) + .with_startup_timeout(Duration::from_secs(180)) + .start() + .await + .map_err(|error| format!("cannot start SQL Server: {error}"))?; + let port = container + .get_host_port_ipv4(SQL_SERVER_PORT.tcp()) + .await + .map_err(|error| error.to_string())?; + Ok(OdbcServer { + connection_string: format!( + "Driver={{ODBC Driver 18 for SQL Server}};Server=127.0.0.1,{port};Database=master;Uid=sa;Pwd={SQL_SERVER_PASSWORD};TrustServerCertificate=yes;" + ), + container, + }) +} + +/// A Postgres server with logical replication, once it accepts connections, and a connection string for psqlODBC. +pub(crate) async fn start_postgres_server() -> Result { + let postgres = start_postgres().await?; + connect(&postgres).await?; + Ok(OdbcServer { + connection_string: format!( + "Driver={{PostgreSQL Unicode}};Server=127.0.0.1;Port={};Database={DATABASE};Uid={USER};Pwd={PASSWORD};", + postgres.port + ), + container: postgres.container, + }) +} diff --git a/crates/sqlload-testkit/src/odbc_servers/_helpers/mod.rs b/crates/sqlload-testkit/src/odbc_servers/_helpers/mod.rs new file mode 100644 index 00000000..5c680bf2 --- /dev/null +++ b/crates/sqlload-testkit/src/odbc_servers/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod containers; diff --git a/crates/sqlload-testkit/src/odbc_servers/constants.rs b/crates/sqlload-testkit/src/odbc_servers/constants.rs index 97523dea..edce1e00 100644 --- a/crates/sqlload-testkit/src/odbc_servers/constants.rs +++ b/crates/sqlload-testkit/src/odbc_servers/constants.rs @@ -6,7 +6,8 @@ pub const MYSQL_READY: &str = "port: 3306 MySQL Community Server"; pub const SQL_SERVER_IMAGE: &str = "mcr.microsoft.com/mssql/server"; pub const SQL_SERVER_TAG: &str = "2022-latest"; pub const SQL_SERVER_PORT: u16 = 1433; -pub const SQL_SERVER_READY: &str = "SQL Server is now ready for client connections"; +/// The line SQL Server logs once its databases are recovered and `sa` can sign in, after it says it is ready for connections. +pub const SQL_SERVER_READY: &str = "Recovery is complete"; /// The SQL Server administrator's password, which must meet its complexity rules. pub const SQL_SERVER_PASSWORD: &str = "Sqlload-Test-1"; pub const USER: &str = "sqlload"; diff --git a/crates/sqlload-testkit/src/odbc_servers/main/mod.rs b/crates/sqlload-testkit/src/odbc_servers/main/mod.rs index d578f5db..4683325d 100644 --- a/crates/sqlload-testkit/src/odbc_servers/main/mod.rs +++ b/crates/sqlload-testkit/src/odbc_servers/main/mod.rs @@ -1,2 +1 @@ -pub mod start_mysql; -pub mod start_sql_server; +pub mod start_odbc_server; diff --git a/crates/sqlload-testkit/src/odbc_servers/main/start_mysql.rs b/crates/sqlload-testkit/src/odbc_servers/main/start_mysql.rs deleted file mode 100644 index 25b32bf6..00000000 --- a/crates/sqlload-testkit/src/odbc_servers/main/start_mysql.rs +++ /dev/null @@ -1,37 +0,0 @@ -//! Start MySQL. - -use std::time::Duration; - -use testcontainers::core::{IntoContainerPort, WaitFor}; -use testcontainers::runners::AsyncRunner; -use testcontainers::{GenericImage, ImageExt}; - -use crate::odbc_servers::constants::{ - DATABASE, MYSQL_IMAGE, MYSQL_PORT, MYSQL_READY, MYSQL_TAG, PASSWORD, USER, -}; -use crate::odbc_servers::models::OdbcServer; - -/// A MySQL server with database `app` and user `sqlload`, on a free host port, and a connection string for MySQL Connector/ODBC that sets the session's SQL mode to read ANSI double quotes, as SQLLoad writes them. -pub async fn start_mysql() -> Result { - let container = GenericImage::new(MYSQL_IMAGE, MYSQL_TAG) - .with_exposed_port(MYSQL_PORT.tcp()) - .with_wait_for(WaitFor::message_on_stderr(MYSQL_READY)) - .with_env_var("MYSQL_ROOT_PASSWORD", PASSWORD) - .with_env_var("MYSQL_DATABASE", DATABASE) - .with_env_var("MYSQL_USER", USER) - .with_env_var("MYSQL_PASSWORD", PASSWORD) - .with_startup_timeout(Duration::from_secs(180)) - .start() - .await - .map_err(|error| format!("cannot start MySQL: {error}"))?; - let port = container - .get_host_port_ipv4(MYSQL_PORT.tcp()) - .await - .map_err(|error| error.to_string())?; - Ok(OdbcServer { - connection_string: format!( - "Driver={{MySQL ODBC Unicode Driver}};Server=127.0.0.1;Port={port};Database={DATABASE};Uid={USER};Pwd={PASSWORD};INITSTMT=SET SESSION sql_mode=CONCAT(@@SESSION.sql_mode,',ANSI_QUOTES');" - ), - container, - }) -} diff --git a/crates/sqlload-testkit/src/odbc_servers/main/start_odbc_server.rs b/crates/sqlload-testkit/src/odbc_servers/main/start_odbc_server.rs new file mode 100644 index 00000000..a0ed6ecd --- /dev/null +++ b/crates/sqlload-testkit/src/odbc_servers/main/start_odbc_server.rs @@ -0,0 +1,15 @@ +//! Start a database an ODBC driver test reads. + +use crate::odbc_servers::_helpers::containers::{ + start_mysql, start_postgres_server, start_sql_server, +}; +use crate::odbc_servers::models::{OdbcDatabase, OdbcServer}; + +/// A server of `database` on a free host port, and the connection string its ODBC driver reaches it by. +pub async fn start_odbc_server(database: OdbcDatabase) -> Result { + match database { + OdbcDatabase::SqlServer => start_sql_server().await, + OdbcDatabase::MySql => start_mysql().await, + OdbcDatabase::Postgres => start_postgres_server().await, + } +} diff --git a/crates/sqlload-testkit/src/odbc_servers/main/start_sql_server.rs b/crates/sqlload-testkit/src/odbc_servers/main/start_sql_server.rs deleted file mode 100644 index 013a62e3..00000000 --- a/crates/sqlload-testkit/src/odbc_servers/main/start_sql_server.rs +++ /dev/null @@ -1,35 +0,0 @@ -//! Start SQL Server. - -use std::time::Duration; - -use testcontainers::core::{IntoContainerPort, WaitFor}; -use testcontainers::runners::AsyncRunner; -use testcontainers::{GenericImage, ImageExt}; - -use crate::odbc_servers::constants::{ - SQL_SERVER_IMAGE, SQL_SERVER_PASSWORD, SQL_SERVER_PORT, SQL_SERVER_READY, SQL_SERVER_TAG, -}; -use crate::odbc_servers::models::OdbcServer; - -/// A SQL Server (Developer edition, licence accepted for testing) on a free host port, and a connection string for Microsoft's ODBC Driver 18 that signs in as `sa` to `master` and trusts the server's own certificate. -pub async fn start_sql_server() -> Result { - let container = GenericImage::new(SQL_SERVER_IMAGE, SQL_SERVER_TAG) - .with_exposed_port(SQL_SERVER_PORT.tcp()) - .with_wait_for(WaitFor::message_on_stdout(SQL_SERVER_READY)) - .with_env_var("ACCEPT_EULA", "Y") - .with_env_var("MSSQL_SA_PASSWORD", SQL_SERVER_PASSWORD) - .with_startup_timeout(Duration::from_secs(180)) - .start() - .await - .map_err(|error| format!("cannot start SQL Server: {error}"))?; - let port = container - .get_host_port_ipv4(SQL_SERVER_PORT.tcp()) - .await - .map_err(|error| error.to_string())?; - Ok(OdbcServer { - connection_string: format!( - "Driver={{ODBC Driver 18 for SQL Server}};Server=127.0.0.1,{port};Database=master;Uid=sa;Pwd={SQL_SERVER_PASSWORD};TrustServerCertificate=yes;" - ), - container, - }) -} diff --git a/crates/sqlload-testkit/src/odbc_servers/mod.rs b/crates/sqlload-testkit/src/odbc_servers/mod.rs index 9e44a346..d05c7854 100644 --- a/crates/sqlload-testkit/src/odbc_servers/mod.rs +++ b/crates/sqlload-testkit/src/odbc_servers/mod.rs @@ -1,5 +1,6 @@ -//! MySQL and SQL Server containers, which the ODBC source tests read through their ODBC drivers. +//! MySQL, SQL Server and Postgres containers, which the ODBC source tests read through their ODBC drivers. +pub(crate) mod _helpers; pub mod constants; pub mod main; pub mod models; diff --git a/crates/sqlload-testkit/src/odbc_servers/models.rs b/crates/sqlload-testkit/src/odbc_servers/models.rs index 2b468d0f..6781b8ee 100644 --- a/crates/sqlload-testkit/src/odbc_servers/models.rs +++ b/crates/sqlload-testkit/src/odbc_servers/models.rs @@ -6,3 +6,14 @@ pub struct OdbcServer { pub connection_string: String, pub container: ContainerAsync, } + +/// A database the ODBC driver tests read through its own driver. +#[derive(Debug, Clone, Copy)] +pub enum OdbcDatabase { + /// Microsoft's ODBC Driver 18 for SQL Server. + SqlServer, + /// MySQL Connector/ODBC. + MySql, + /// psqlODBC. + Postgres, +} diff --git a/crates/sqlload/src/adapters/contract/constants.rs b/crates/sqlload/src/adapters/contract/constants.rs index e53d8366..87d42422 100644 --- a/crates/sqlload/src/adapters/contract/constants.rs +++ b/crates/sqlload/src/adapters/contract/constants.rs @@ -10,8 +10,8 @@ pub const SECONDS_PER_DAY: u64 = 86_400; /// How cursor timestamps are written in literals (microseconds, no offset). pub const CURSOR_TIMESTAMP_FORMAT: &str = "%Y-%m-%d %H:%M:%S%.6f"; -/// A timestamp inside an ODBC `{ts '...'}` literal: fractional seconds in groups of three, so a millisecond value keeps the `.fff` every driver reads, and none when whole. -pub const ODBC_TIMESTAMP_FORMAT: &str = "%Y-%m-%d %H:%M:%S%.f"; +/// The years an ODBC date or timestamp parameter holds. +pub const BOUND_YEARS: std::ops::RangeInclusive = 1..=9999; /// Timestamps without an offset accepted on the command line. pub const NAIVE_TIMESTAMP_FORMATS: &[&str] = &["%Y-%m-%dT%H:%M:%S%.f", "%Y-%m-%d %H:%M:%S%.f"]; diff --git a/crates/sqlload/src/adapters/contract/main/bound_parameter.rs b/crates/sqlload/src/adapters/contract/main/bound_parameter.rs new file mode 100644 index 00000000..4a2cf9b5 --- /dev/null +++ b/crates/sqlload/src/adapters/contract/main/bound_parameter.rs @@ -0,0 +1,29 @@ +//! Cursor bounds a source binds to placeholders. + +use chrono::{Datelike, NaiveDate, NaiveDateTime}; + +use crate::adapters::contract::constants::{BOUND_YEARS, NAIVE_TIMESTAMP_FORMATS}; +use crate::adapters::contract::models::BoundParameter; +use crate::column_types::models::ColumnType; + +/// `value`, a normalised cursor bound, as the parameter a cursor of `column_type` compares with: text for a text cursor, a timestamp for a wall-clock timestamp cursor, a date for a date cursor; `None` for any other cursor, or a value that is not of its type or falls outside the years a bound parameter holds, which then stays a literal. +pub fn bound_parameter(value: &str, column_type: &ColumnType) -> Option { + match column_type { + ColumnType::Text | ColumnType::Varchar(_) => Some(BoundParameter::Text(value.to_owned())), + ColumnType::TimestampNtz => NAIVE_TIMESTAMP_FORMATS + .iter() + .find_map( + |format| match NaiveDateTime::parse_from_str(value, format) { + Ok(time) => Some(time), + Err(_) => None, + }, + ) + .filter(|time| BOUND_YEARS.contains(&time.year())) + .map(BoundParameter::Timestamp), + ColumnType::Date => match NaiveDate::parse_from_str(value, "%Y-%m-%d") { + Ok(date) if BOUND_YEARS.contains(&date.year()) => Some(BoundParameter::Date(date)), + Ok(_) | Err(_) => None, + }, + _ => None, + } +} diff --git a/crates/sqlload/src/adapters/contract/main/mod.rs b/crates/sqlload/src/adapters/contract/main/mod.rs index f6dee1ec..731b97ee 100644 --- a/crates/sqlload/src/adapters/contract/main/mod.rs +++ b/crates/sqlload/src/adapters/contract/main/mod.rs @@ -1,8 +1,8 @@ pub(in crate::adapters) mod _batch_direct_problem; pub(in crate::adapters) mod _files_direct_problem; pub(in crate::adapters) mod _footer_rows; +pub mod bound_parameter; pub mod cursor_bound; pub mod cursor_from_now; pub mod cursor_text; -pub mod odbc_literal; pub mod staging_id; diff --git a/crates/sqlload/src/adapters/contract/main/odbc_literal.rs b/crates/sqlload/src/adapters/contract/main/odbc_literal.rs deleted file mode 100644 index 78e526dd..00000000 --- a/crates/sqlload/src/adapters/contract/main/odbc_literal.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! ODBC escape literals for date and timestamp cursor bounds, which every ODBC driver turns into its database's own syntax. - -use chrono::{NaiveDate, NaiveDateTime}; - -use crate::adapters::contract::constants::{NAIVE_TIMESTAMP_FORMATS, ODBC_TIMESTAMP_FORMAT}; -use crate::column_types::models::ColumnType; - -/// `value`, a normalised cursor bound, as `{ts 'YYYY-MM-DD HH:MM:SS[.fff]'}` for a timestamp without a time zone (fractional seconds in groups of three, none when whole) or `{d 'YYYY-MM-DD'}` for a date; `None` for any other cursor, or a value that is not one, which then stays a string literal. -pub fn odbc_literal(value: &str, column_type: &ColumnType) -> Option { - match column_type { - ColumnType::TimestampNtz => NAIVE_TIMESTAMP_FORMATS - .iter() - .find_map( - |format| match NaiveDateTime::parse_from_str(value, format) { - Ok(time) => Some(time), - Err(_) => None, - }, - ) - .map(|time| format!("{{ts '{}'}}", time.format(ODBC_TIMESTAMP_FORMAT))), - ColumnType::Date => match NaiveDate::parse_from_str(value, "%Y-%m-%d") { - Ok(date) => Some(format!("{{d '{}'}}", date.format("%Y-%m-%d"))), - Err(_) => None, - }, - _ => None, - } -} diff --git a/crates/sqlload/src/adapters/contract/models.rs b/crates/sqlload/src/adapters/contract/models.rs index a65281ca..0361587a 100644 --- a/crates/sqlload/src/adapters/contract/models.rs +++ b/crates/sqlload/src/adapters/contract/models.rs @@ -119,6 +119,17 @@ pub enum CursorValue { Plain, } +/// A cursor bound passed to a `?` placeholder rather than written into the filter, typed as its cursor column compares it, so the database's literal rules can't change it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BoundParameter { + /// A text cursor's bound. + Text(String), + /// A wall-clock timestamp cursor's bound. + Timestamp(chrono::NaiveDateTime), + /// A date cursor's bound. + Date(chrono::NaiveDate), +} + /// A query for a cursor's maximum and how to read its result. #[derive(Debug, Clone, PartialEq, Eq)] pub struct CursorRead { diff --git a/crates/sqlload/src/adapters/contract/tests.rs b/crates/sqlload/src/adapters/contract/tests.rs index 5e9ea5bc..be46f040 100644 --- a/crates/sqlload/src/adapters/contract/tests.rs +++ b/crates/sqlload/src/adapters/contract/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_bound_parameter.rs"] +mod bound_parameter; #[path = "tests/test_build_table.rs"] mod build_table; #[path = "tests/test_check_query.rs"] @@ -18,8 +20,6 @@ mod helpers; mod insert_rows; #[path = "tests/test_live_type.rs"] mod live_type; -#[path = "tests/test_odbc_literal.rs"] -mod odbc_literal; #[path = "tests/test_sql_literal.rs"] mod sql_literal; #[path = "tests/test_staging_changes.rs"] diff --git a/crates/sqlload/src/adapters/contract/tests/helpers.rs b/crates/sqlload/src/adapters/contract/tests/helpers.rs index 17eab38c..5f79e322 100644 --- a/crates/sqlload/src/adapters/contract/tests/helpers.rs +++ b/crates/sqlload/src/adapters/contract/tests/helpers.rs @@ -123,3 +123,21 @@ pub(crate) fn writes(statement: &str, (target, name): (&str, &str)) -> bool { || statement.starts_with(&format!("{verb} IF EXISTS {target}")) }) } + +/// `text` (`YYYY-MM-DD HH:MM:SS[.f]`) as a bound timestamp. +pub(crate) fn timestamp_parameter( + text: &str, +) -> Result { + chrono::NaiveDateTime::parse_from_str(text, "%Y-%m-%d %H:%M:%S%.f") + .map(crate::adapters::contract::models::BoundParameter::Timestamp) + .map_err(|error| error.to_string()) +} + +/// `text` (`YYYY-MM-DD`) as a bound date. +pub(crate) fn date_parameter( + text: &str, +) -> Result { + chrono::NaiveDate::parse_from_str(text, "%Y-%m-%d") + .map(crate::adapters::contract::models::BoundParameter::Date) + .map_err(|error| error.to_string()) +} diff --git a/crates/sqlload/src/adapters/contract/tests/test_bound_parameter.rs b/crates/sqlload/src/adapters/contract/tests/test_bound_parameter.rs new file mode 100644 index 00000000..50b78f50 --- /dev/null +++ b/crates/sqlload/src/adapters/contract/tests/test_bound_parameter.rs @@ -0,0 +1,81 @@ +use crate::adapters::contract::main::bound_parameter::bound_parameter; +use crate::adapters::contract::models::BoundParameter; +use crate::adapters::contract::tests::helpers::{date_parameter, timestamp_parameter}; +use crate::adapters::contract::tests::test_types::BoundParameterTestCase; +use crate::column_types::models::ColumnType; + +#[test] +fn given_a_cursor_bound_when_binding_it_for_an_odbc_source_then_text_dates_and_wall_clock_times_keep_their_type_and_every_digit() +-> Result<(), String> { + let test_cases = [ + BoundParameterTestCase { + description: "the millisecond maximum a cursor resumes from keeps its milliseconds exactly", + value: "2026-01-31 23:59:59.997000", + column_type: ColumnType::TimestampNtz, + expected_parameter: Some(timestamp_parameter("2026-01-31 23:59:59.997")?), + }, + BoundParameterTestCase { + description: "a microsecond bound keeps all six digits", + value: "2026-01-31 23:59:59.000001", + column_type: ColumnType::TimestampNtz, + expected_parameter: Some(timestamp_parameter("2026-01-31 23:59:59.000001")?), + }, + BoundParameterTestCase { + description: "a bound written with a T and no fraction reads the same", + value: "2026-02-01T08:30:00", + column_type: ColumnType::TimestampNtz, + expected_parameter: Some(timestamp_parameter("2026-02-01 08:30:00")?), + }, + BoundParameterTestCase { + description: "a date is a date", + value: "2026-02-01", + column_type: ColumnType::Date, + expected_parameter: Some(date_parameter("2026-02-01")?), + }, + BoundParameterTestCase { + description: "a text bound is text, whatever it holds", + value: "o'brien\\", + column_type: ColumnType::Varchar(40), + expected_parameter: Some(BoundParameter::Text("o'brien\\".to_owned())), + }, + BoundParameterTestCase { + description: "a zoned timestamp is not bound, since an ODBC timestamp carries no offset", + value: "2026-01-31 23:59:59.997000+00", + column_type: ColumnType::TimestampTz, + expected_parameter: None, + }, + BoundParameterTestCase { + description: "a value that is not a timestamp is not bound as one", + value: "2026-01-31'} OR 1=1 --", + column_type: ColumnType::TimestampNtz, + expected_parameter: None, + }, + BoundParameterTestCase { + description: "a date cursor given a timestamp is not bound", + value: "2026-02-01 00:00:00", + column_type: ColumnType::Date, + expected_parameter: None, + }, + BoundParameterTestCase { + description: "a year an ODBC date can't hold is not bound", + value: "+10000-01-01", + column_type: ColumnType::Date, + expected_parameter: None, + }, + BoundParameterTestCase { + description: "an integer cursor's bound is a number, not a parameter", + value: "42", + column_type: ColumnType::Int, + expected_parameter: None, + }, + ]; + for test_case in test_cases { + assert_eq!( + bound_parameter(test_case.value, &test_case.column_type), + test_case.expected_parameter, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlload/src/adapters/contract/tests/test_odbc_literal.rs b/crates/sqlload/src/adapters/contract/tests/test_odbc_literal.rs deleted file mode 100644 index 2c6d78a6..00000000 --- a/crates/sqlload/src/adapters/contract/tests/test_odbc_literal.rs +++ /dev/null @@ -1,78 +0,0 @@ -use crate::adapters::contract::main::odbc_literal::odbc_literal; -use crate::adapters::contract::tests::test_types::OdbcLiteralTestCase; -use crate::column_types::models::ColumnType; - -#[test] -fn given_a_cursor_bound_when_writing_it_for_an_odbc_source_then_dates_and_wall_clock_times_are_escape_literals() - { - let test_cases = [ - OdbcLiteralTestCase { - description: "a millisecond bound keeps exactly three fractional digits", - value: "2026-01-31 23:59:59.997000", - column_type: ColumnType::TimestampNtz, - expected_literal: Some("{ts '2026-01-31 23:59:59.997'}"), - }, - OdbcLiteralTestCase { - description: "half a second is written as milliseconds", - value: "2026-01-31 23:59:59.500000", - column_type: ColumnType::TimestampNtz, - expected_literal: Some("{ts '2026-01-31 23:59:59.500'}"), - }, - OdbcLiteralTestCase { - description: "a microsecond bound keeps all six digits", - value: "2026-01-31 23:59:59.000001", - column_type: ColumnType::TimestampNtz, - expected_literal: Some("{ts '2026-01-31 23:59:59.000001'}"), - }, - OdbcLiteralTestCase { - description: "a whole second has no fraction", - value: "2026-02-01 00:00:00.000000", - column_type: ColumnType::TimestampNtz, - expected_literal: Some("{ts '2026-02-01 00:00:00'}"), - }, - OdbcLiteralTestCase { - description: "a bound written with a T reads the same", - value: "2026-02-01T08:30:00.25", - column_type: ColumnType::TimestampNtz, - expected_literal: Some("{ts '2026-02-01 08:30:00.250'}"), - }, - OdbcLiteralTestCase { - description: "a date is a {d} literal", - value: "2026-02-01", - column_type: ColumnType::Date, - expected_literal: Some("{d '2026-02-01'}"), - }, - OdbcLiteralTestCase { - description: "a zoned timestamp has no escape, since {ts} carries no offset", - value: "2026-01-31 23:59:59.997000+00", - column_type: ColumnType::TimestampTz, - expected_literal: None, - }, - OdbcLiteralTestCase { - description: "a value that is not a timestamp is never put inside an escape", - value: "2026-01-31'} OR 1=1 --", - column_type: ColumnType::TimestampNtz, - expected_literal: None, - }, - OdbcLiteralTestCase { - description: "a date cursor given a timestamp has no escape", - value: "2026-02-01 00:00:00", - column_type: ColumnType::Date, - expected_literal: None, - }, - OdbcLiteralTestCase { - description: "an integer cursor has no escape", - value: "42", - column_type: ColumnType::Int, - expected_literal: None, - }, - ]; - for test_case in test_cases { - assert_eq!( - odbc_literal(test_case.value, &test_case.column_type).as_deref(), - test_case.expected_literal, - "{}", - test_case.description - ); - } -} diff --git a/crates/sqlload/src/adapters/contract/tests/test_types.rs b/crates/sqlload/src/adapters/contract/tests/test_types.rs index 85f2a367..9c8cd8b2 100644 --- a/crates/sqlload/src/adapters/contract/tests/test_types.rs +++ b/crates/sqlload/src/adapters/contract/tests/test_types.rs @@ -1,7 +1,9 @@ use crate::column_types::models::Warehouse; use std::time::Duration; -use crate::adapters::contract::models::{CheckQuery, CursorValue, NowShift, WriteMode}; +use crate::adapters::contract::models::{ + BoundParameter, CheckQuery, CursorValue, NowShift, WriteMode, +}; use crate::column_types::models::{ColumnType, LiveColumnType, TypeFit}; pub(crate) struct WriteStatementsTestCase { @@ -148,9 +150,9 @@ pub(crate) struct CursorFromNowTestCase { pub(crate) expected_text: Result<&'static str, &'static str>, } -pub(crate) struct OdbcLiteralTestCase { +pub(crate) struct BoundParameterTestCase { pub(crate) description: &'static str, pub(crate) value: &'static str, pub(crate) column_type: ColumnType, - pub(crate) expected_literal: Option<&'static str>, + pub(crate) expected_parameter: Option, } diff --git a/crates/sqlload/src/loader/_helpers/publish.rs b/crates/sqlload/src/loader/_helpers/publish.rs index d7037e66..62b79433 100644 --- a/crates/sqlload/src/loader/_helpers/publish.rs +++ b/crates/sqlload/src/loader/_helpers/publish.rs @@ -2,6 +2,7 @@ use crate::adapters::contract::constants::COMMIT_STATEMENT; use crate::adapters::contract::constants::SWAP_SUFFIX; use crate::adapters::contract::main::cursor_text::cursor_text; use crate::adapters::contract::main::staging_id::staging_id; +use crate::adapters::contract::models::BoundParameter; use crate::adapters::contract::models::TableName; use crate::adapters::contract::models::TableShape; use crate::adapters::contract::models::WriteMode; @@ -943,8 +944,10 @@ async fn feed( snapshot(reader, SNAPSHOT_BEGIN).await?; for index in 0..windows.count { let window = &window_at(windows, index).map_err(LoaderError::Invalid)?; + let (filter, parameters) = window_filter(table, window); let part = ReadPart { - filter: window_filter(table, window), + filter, + parameters, batch_bytes, ..ReadPart::default() }; @@ -1093,31 +1096,40 @@ async fn stage(publisher: &mut Publisher, batch: RecordBatch) -> Result<(), Load publisher.stage(batch, None).await } -/// The source filter reading one cursor window, in the source's dialect; its bounds are literals, since `split_windows` cuts only number and time cursors, never a text one. -fn window_filter(table: &TablePlan, window: &CursorWindow) -> Option { - let cursor = table.spec.cursor.as_ref()?; - let column = table - .read_columns - .iter() - .find(|column| column.name.eq_ignore_ascii_case(&cursor.column))?; +/// The source filter reading one cursor window, in the source's dialect, and the values bound to its placeholders: an ODBC source's date and timestamp bounds (`split_windows` cuts only number and time cursors, never a text one). +fn window_filter( + table: &TablePlan, + window: &CursorWindow, +) -> (Option, Vec) { + let mut parameters: Vec = Vec::new(); + let Some(column) = table.spec.cursor.as_ref().and_then(|cursor| { + table + .read_columns + .iter() + .find(|column| column.name.eq_ignore_ascii_case(&cursor.column)) + }) else { + return (None, parameters); + }; let operator = match window.inclusive { true => INCLUSIVE_BOUND, false => EXCLUSIVE_BOUND, }; let source = table.spec.warehouse_source.as_ref(); - source_filter( + let filter = source_filter( &column.expression, ( Some(&column.output), source.and_then(|source| source_dialect(source.system)), - source.is_some_and(|source| source_binds_text(source.system)), ), ( window.from.as_deref().map(|from| (operator, from)), window.to.as_deref(), ), - None, - ) + source + .is_some_and(|source| source_binds_text(source.system)) + .then_some(&mut parameters), + ); + (filter, parameters) } /// The replaced files a commit deletes: those of the partitions it wrote, and, at the last commit, every other file the load replaces (everything for `replace`, the window's partitions for `delete_insert`); never one already deleted. diff --git a/crates/sqlload/src/loader/_helpers/sources.rs b/crates/sqlload/src/loader/_helpers/sources.rs index b45cf884..47cb6be2 100644 --- a/crates/sqlload/src/loader/_helpers/sources.rs +++ b/crates/sqlload/src/loader/_helpers/sources.rs @@ -1,3 +1,4 @@ +use crate::adapters::contract::models::BoundParameter; use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter; use arrow_array::RecordBatch; use std::collections::BTreeMap; @@ -1140,6 +1141,8 @@ pub(crate) async fn plan_files( pub(crate) struct ReadPart { /// The Postgres filter; `None` reads the plan's. pub(crate) filter: Option, + /// The values bound to `filter`'s `?` placeholders, in order. + pub(crate) parameters: Vec, /// `ORDER BY` this quoted column. pub(crate) order_by: Option, /// The files to read; `None` reads every planned file. @@ -1218,7 +1221,7 @@ async fn read_each( #[cfg(feature = "odbc")] (SourceReader::Odbc(reader), _) => { let (filter, parameters) = match part.filter { - Some(filter) => (Some(filter), &[][..]), + Some(filter) => (Some(filter), &part.parameters[..]), None => (table.source_filter.clone(), &table.source_parameters[..]), }; let request = crate::odbc_source::models::OdbcReadRequest { diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index c3851455..66356de8 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -1,7 +1,8 @@ +use crate::adapters::contract::main::bound_parameter::bound_parameter; use crate::adapters::contract::main::cursor_bound::cursor_bound; use crate::adapters::contract::main::cursor_from_now::cursor_from_now; use crate::adapters::contract::main::cursor_text::cursor_text; -use crate::adapters::contract::main::odbc_literal::odbc_literal; +use crate::adapters::contract::models::BoundParameter; use crate::adapters::contract::models::CursorQuery; use crate::adapters::contract::models::NowShift; use crate::adapters::contract::models::WriteMode; @@ -934,8 +935,8 @@ pub(crate) struct WindowRequest<'a> { /// The source filter, the write mode and a note for `plan`. pub(crate) struct Window { pub(crate) filter: Option, - /// The values bound to `filter`'s `?` placeholders, in order: an ODBC source's text cursor bounds. - pub(crate) parameters: Vec, + /// The values bound to `filter`'s `?` placeholders, in order: an ODBC source's cursor bounds. + pub(crate) parameters: Vec, /// The cursor range read: `from <= cursor < to`, each open when `None`. pub(crate) bounds: (Option, Option), pub(crate) mode: WriteMode, @@ -998,29 +999,26 @@ pub(crate) fn source_cursor( )) } -/// `column >= 'from' AND column < 'to'`, from the bounds that are set, as literals of the source's `dialect` (Postgres's when `None`): a numeric cursor's plain-number bounds as numbers (`render_cursor_number`), as BigQuery compares a number with no string; a date or wall-clock timestamp cursor's bounds as ODBC `{d '...'}`/`{ts '...'}` literals when `odbc` (an ODBC source, whose driver writes them in its database's syntax, where a string may not compare with a timestamp); a text cursor's bounds as `?` placeholders when `parameters` collects their values, in order, to bind; any other bound as the dialect's string literal, so a value holding a quote or a backslash stays one string. +/// `column >= 'from' AND column < 'to'`, from the bounds that are set, as literals of the source's `dialect` (Postgres's when `None`): a numeric cursor's plain-number bounds as numbers (`render_cursor_number`), as BigQuery compares a number with no string; a text, date or wall-clock timestamp cursor's bounds as `?` placeholders when `parameters` collects them, in order, to bind as their type (an ODBC source's: no literal reads alike everywhere, since some databases read a backslash in a string as an escape, OpenEdge compares no string with a timestamp, and SQL Server's `{ts}` escape is a `datetime` that rounds milliseconds); any other bound as the dialect's string literal, so a value holding a quote or a backslash stays one string. pub(crate) fn source_filter( column: &str, - (cursor_type, dialect, odbc): (Option<&ColumnType>, Option, bool), + (cursor_type, dialect): (Option<&ColumnType>, Option), (lower, upper): (Option<(&str, &str)>, Option<&str>), - mut parameters: Option<&mut Vec>, + mut parameters: Option<&mut Vec>, ) -> Option { let adapter = sql_adapter(dialect.unwrap_or(Warehouse::Postgres)); - let text = matches!(cursor_type, Some(ColumnType::Text | ColumnType::Varchar(_))); let mut literal = |value: &str| match ( cursor_type.filter(|cursor_type| numeric_cursor(cursor_type) && is_number(value)), - cursor_type - .filter(|_| odbc) - .and_then(|cursor_type| odbc_literal(value, cursor_type)), - parameters.as_mut().filter(|_| text), + parameters + .as_mut() + .zip(cursor_type.and_then(|cursor_type| bound_parameter(value, cursor_type))), ) { - (Some(cursor_type), _, _) => adapter.render_cursor_number(value, cursor_type), - (None, Some(escaped), _) => escaped, - (None, None, Some(parameters)) => { - parameters.push(value.to_owned()); + (Some(cursor_type), _) => adapter.render_cursor_number(value, cursor_type), + (None, Some((parameters, parameter))) => { + parameters.push(parameter); "?".to_owned() } - (None, None, None) => adapter.render_loader_value_literal(value), + (None, None) => adapter.render_loader_value_literal(value), }; let mut bounds: Vec = Vec::new(); if let Some((operator, value)) = lower { @@ -1240,17 +1238,17 @@ pub(crate) async fn cursor_window( (None, None) => (None, note), }; let source = request.spec.warehouse_source.as_ref(); - let odbc = source.is_some_and(|source| source_binds_text(source.system)); - let mut parameters: Vec = Vec::new(); + let mut parameters: Vec = Vec::new(); let filter = source_filter( request.cursor_expression.unwrap_or_default(), ( request.cursor_type, source.and_then(|source| source_dialect(source.system)), - odbc, ), (from.as_deref().map(|from| (operator, from)), to.as_deref()), - odbc.then_some(&mut parameters), + source + .is_some_and(|source| source_binds_text(source.system)) + .then_some(&mut parameters), ); Ok(Ok(Window { filter, diff --git a/crates/sqlload/src/loader/main/_files_window.rs b/crates/sqlload/src/loader/main/_files_window.rs index fc202e1b..9cb9a132 100644 --- a/crates/sqlload/src/loader/main/_files_window.rs +++ b/crates/sqlload/src/loader/main/_files_window.rs @@ -137,7 +137,7 @@ pub(in crate::loader) fn files_window(request: &FilesWindowRequest<'_>) -> Resul Ok(Window { filter: source_filter( &expression, - (Some(cursor_type), None, false), + (Some(cursor_type), None), (from.as_deref().map(|from| (operator, from)), to.as_deref()), None, ), diff --git a/crates/sqlload/src/loader/models.rs b/crates/sqlload/src/loader/models.rs index 7f7c6fe0..5f08c13c 100644 --- a/crates/sqlload/src/loader/models.rs +++ b/crates/sqlload/src/loader/models.rs @@ -1,3 +1,4 @@ +use crate::adapters::contract::models::BoundParameter; use crate::adapters::contract::models::ChangeApply; use std::collections::BTreeMap; use std::time::Duration; @@ -258,8 +259,8 @@ pub struct TablePlan { pub mode: WriteMode, /// The source filter for incremental reads, already rendered in the source's dialect. pub source_filter: Option, - /// The values bound to `source_filter`'s `?` placeholders, in order: an ODBC source's text cursor bounds. - pub source_parameters: Vec, + /// The values bound to `source_filter`'s `?` placeholders, in order: an ODBC source's cursor bounds. + pub source_parameters: Vec, /// How the cursor window was chosen, for `plan`. pub cursor_note: Option, /// The cursor range read, for sources that filter it themselves (Python tables). diff --git a/crates/sqlload/src/loader/tests/test_source_filter.rs b/crates/sqlload/src/loader/tests/test_source_filter.rs index 4fe0950d..d652dbf4 100644 --- a/crates/sqlload/src/loader/tests/test_source_filter.rs +++ b/crates/sqlload/src/loader/tests/test_source_filter.rs @@ -102,7 +102,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the lower: Some((">", r"zz\' OR 1=1 --")), upper: Some(r"o'brien\"), expected_filter: Some("`id` > ? AND `id` < ?"), - expected_parameters: &[r"zz\' OR 1=1 --", r"o'brien\"], + expected_parameters: &[r#"Text("zz\\' OR 1=1 --")"#, r#"Text("o'brien\\")"#], }, SourceFilterTestCase { description: "an ODBC numeric bound stays a bare number, not a parameter", @@ -115,16 +115,17 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the expected_parameters: &[], }, SourceFilterTestCase { - description: "an ODBC timestamp cursor resumes after its millisecond maximum as {ts} literals, which OpenEdge needs and every driver reads", + description: "an ODBC timestamp cursor resumes after its millisecond maximum through timestamp parameters, which every driver compares as timestamps (OpenEdge compares no string with one)", cursor_type: ColumnType::TimestampNtz, dialect: None, odbc: true, lower: Some((">", "2026-01-31 23:59:59.997000")), upper: Some("2026-02-01 00:00:00.000000"), - expected_filter: Some( - "`id` > {ts '2026-01-31 23:59:59.997'} AND `id` < {ts '2026-02-01 00:00:00'}", - ), - expected_parameters: &[], + expected_filter: Some("`id` > ? AND `id` < ?"), + expected_parameters: &[ + "Timestamp(2026-01-31T23:59:59.997)", + "Timestamp(2026-02-01T00:00:00)", + ], }, SourceFilterTestCase { description: "an ODBC timestamp bound keeps microseconds rather than rounding them away", @@ -133,18 +134,18 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the odbc: true, lower: Some((">=", "2026-01-31 23:59:59.123456")), upper: None, - expected_filter: Some("`id` >= {ts '2026-01-31 23:59:59.123456'}"), - expected_parameters: &[], + expected_filter: Some("`id` >= ?"), + expected_parameters: &["Timestamp(2026-01-31T23:59:59.123456)"], }, SourceFilterTestCase { - description: "an ODBC date cursor's bound is a {d} literal", + description: "an ODBC date cursor's bound is a date parameter", cursor_type: ColumnType::Date, dialect: None, odbc: true, lower: Some((">=", "2026-02-01")), upper: None, - expected_filter: Some("`id` >= {d '2026-02-01'}"), - expected_parameters: &[], + expected_filter: Some("`id` >= ?"), + expected_parameters: &["Date(2026-02-01)"], }, SourceFilterTestCase { description: "a timestamp bound from any other source stays a string literal", @@ -157,7 +158,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the expected_parameters: &[], }, SourceFilterTestCase { - description: "an ODBC zoned timestamp bound stays a string literal, since {ts} has no offset", + description: "an ODBC zoned timestamp bound stays a string literal, since an ODBC timestamp has no offset", cursor_type: ColumnType::TimestampTz, dialect: None, odbc: true, @@ -167,7 +168,7 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the expected_parameters: &[], }, SourceFilterTestCase { - description: "an ODBC timestamp bound that is not a timestamp stays one quoted string, never an escape", + description: "an ODBC timestamp bound that is not a timestamp stays one quoted string, never a timestamp parameter", cursor_type: ColumnType::TimestampNtz, dialect: None, odbc: true, @@ -201,16 +202,18 @@ fn given_cursor_bounds_when_rendering_the_source_filter_then_literals_follow_the let mut parameters = Vec::new(); let filter = source_filter( "`id`", - ( - Some(&test_case.cursor_type), - test_case.dialect, - test_case.odbc, - ), + (Some(&test_case.cursor_type), test_case.dialect), (test_case.lower, test_case.upper), test_case.odbc.then_some(&mut parameters), ); assert_eq!( - (filter.as_deref(), parameters), + ( + filter.as_deref(), + parameters + .iter() + .map(|parameter| format!("{parameter:?}")) + .collect::>() + ), ( test_case.expected_filter, test_case diff --git a/crates/sqlload/src/loader/tests/test_types.rs b/crates/sqlload/src/loader/tests/test_types.rs index d40febd8..c6cbfe22 100644 --- a/crates/sqlload/src/loader/tests/test_types.rs +++ b/crates/sqlload/src/loader/tests/test_types.rs @@ -91,12 +91,12 @@ pub(crate) struct SourceFilterTestCase { pub(crate) description: &'static str, pub(crate) cursor_type: ColumnType, pub(crate) dialect: Option, - /// Whether the source is an ODBC source: text bounds bound as parameters, date and timestamp bounds as escape literals. + /// Whether the source is an ODBC source, whose text, date and timestamp bounds are bound parameters. pub(crate) odbc: bool, pub(crate) lower: Option<(&'static str, &'static str)>, pub(crate) upper: Option<&'static str>, pub(crate) expected_filter: Option<&'static str>, - /// The values bound to the filter's placeholders, in order. + /// The values bound to the filter's placeholders, in order, as Debug prints them. pub(crate) expected_parameters: &'static [&'static str], } diff --git a/crates/sqlload/src/odbc_source/_helpers/batches.rs b/crates/sqlload/src/odbc_source/_helpers/batches.rs index 64ff42b9..16291696 100644 --- a/crates/sqlload/src/odbc_source/_helpers/batches.rs +++ b/crates/sqlload/src/odbc_source/_helpers/batches.rs @@ -4,19 +4,19 @@ use std::sync::Arc; use arrow_odbc::odbc_api::Cursor; use arrow_odbc::{ColumnFailure, OdbcReaderBuilder, arrow_schema_from}; -use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit}; use crate::odbc_source::constants::TOO_LARGE_VALUE; use crate::odbc_source::errors::OdbcSourceError; use crate::odbc_source::models::OdbcReader; -/// The reader of `cursor`'s rows in batches of `batch_rows`, decimals as text, each buffer capped by the reader's limits; a column the driver gives no length for is refused, naming the setting that bounds it. +/// The reader of `cursor`'s rows in batches of `batch_rows`, as `fetch_schema` fetches them, each buffer capped by the reader's limits; a column the driver gives no length for is refused, naming the setting that bounds it. pub(crate) fn batch_reader( reader: &OdbcReader, mut cursor: C, (dbms, batch_rows): (String, usize), ) -> Result, OdbcSourceError> { - let schema = decimals_as_text(&arrow_schema_from(&mut cursor, Some(&dbms), false)?); + let schema = fetch_schema(&arrow_schema_from(&mut cursor, Some(&dbms), false)?); let mut builder = OdbcReaderBuilder::new(); builder .with_schema(schema) @@ -51,13 +51,17 @@ pub(crate) fn fetch_error(text: &str) -> OdbcSourceError { } } -/// `schema` with every decimal field fetched as text instead. -pub(crate) fn decimals_as_text(schema: &Schema) -> SchemaRef { +/// `schema` with every decimal field fetched as text, and every timestamp coarser than microseconds fetched in microseconds: a driver may describe fewer fractional digits than its values hold (MySQL Connector/ODBC describes every `DATETIME(n)` with none), and the reader would drop the rest. +pub(crate) fn fetch_schema(schema: &Schema) -> SchemaRef { let fields: Vec = schema .fields() .iter() .map(|field| match field.data_type() { DataType::Decimal128(..) => field.as_ref().clone().with_data_type(DataType::Utf8), + DataType::Timestamp(TimeUnit::Second | TimeUnit::Millisecond, zone) => field + .as_ref() + .clone() + .with_data_type(DataType::Timestamp(TimeUnit::Microsecond, zone.clone())), _ => field.as_ref().clone(), }) .collect(); diff --git a/crates/sqlload/src/odbc_source/constants.rs b/crates/sqlload/src/odbc_source/constants.rs index 0a239e4e..adf2711d 100644 --- a/crates/sqlload/src/odbc_source/constants.rs +++ b/crates/sqlload/src/odbc_source/constants.rs @@ -25,3 +25,6 @@ pub const BATCHES_AHEAD: usize = 2; /// What the driver manager's error says when a value is longer than its buffer. pub const TOO_LARGE_VALUE: &str = "too large to be written into the allocated buffer"; + +/// The fractional digits a timestamp cursor bound is described with: seven, so SQL Server compares it as `datetime2(7)`, exactly, rather than as a `datetime`. +pub const TIMESTAMP_PARAMETER_PRECISION: i16 = 7; diff --git a/crates/sqlload/src/odbc_source/main/read_rows.rs b/crates/sqlload/src/odbc_source/main/read_rows.rs index a7260749..1a2ef680 100644 --- a/crates/sqlload/src/odbc_source/main/read_rows.rs +++ b/crates/sqlload/src/odbc_source/main/read_rows.rs @@ -1,14 +1,19 @@ //! Read a table or query through ODBC as Arrow batches. use arrow_array::RecordBatch; -use arrow_odbc::odbc_api::IntoParameter; +use arrow_odbc::odbc_api::parameter::{InputParameter, WithDataType}; +use arrow_odbc::odbc_api::sys::{Date, Timestamp}; +use arrow_odbc::odbc_api::{DataType, IntoParameter}; +use chrono::{Datelike, Timelike}; use tokio::sync::mpsc::Sender; +use crate::adapters::contract::models::BoundParameter; use crate::odbc_source::_helpers::batches::{batch_reader, fetch_error}; use crate::odbc_source::_helpers::connect::connect; use crate::odbc_source::_helpers::convert::{converted, output_schema}; use crate::odbc_source::_helpers::query::select_query; use crate::odbc_source::constants::BATCHES_AHEAD; +use crate::odbc_source::constants::TIMESTAMP_PARAMETER_PRECISION; use crate::odbc_source::errors::OdbcSourceError; use crate::odbc_source::models::{OdbcReadRequest, OdbcReader}; @@ -40,19 +45,17 @@ pub async fn read_rows( Ok(rows) } -/// Fetch `query`'s rows, with `parameters` bound to its placeholders as text, into `sender` until they end, the receiver goes away, or a fetch fails (sent on as the error). Decimal columns are fetched as text, which `converted` parses without rounding, since drivers round values to the scale they describe (an unbounded `numeric` as `NUMERIC(28,6)`). +/// Fetch `query`'s rows, with `parameters` bound to its placeholders each as its type, into `sender` until they end, the receiver goes away, or a fetch fails (sent on as the error). Decimal columns are fetched as text, which `converted` parses without rounding, since drivers round values to the scale they describe (an unbounded `numeric` as `NUMERIC(28,6)`). fn fetch( reader: &OdbcReader, - (query, parameters): (&str, &[String]), + (query, parameters): (&str, &[BoundParameter]), batch_rows: usize, sender: &Sender>, ) -> Result<(), OdbcSourceError> { let connection = connect(reader)?; let dbms: String = connection.database_management_system_name()?; - let parameters: Vec<_> = parameters - .iter() - .map(|value| value.as_str().into_parameter()) - .collect(); + let parameters: Vec> = + parameters.iter().map(odbc_parameter).collect(); let Some(cursor) = connection.execute(query, parameters.as_slice(), None)? else { return Err(OdbcSourceError::Invalid( "the read returned no result set".to_owned(), @@ -68,3 +71,33 @@ fn fetch( } Ok(()) } + +/// `parameter` as the driver receives it: text as `VARCHAR`; a timestamp as `TIMESTAMP` with seven fractional digits, which SQL Server reads as `datetime2(7)` (it reads three as `datetime`, rounding milliseconds to thirds of a hundredth); a date as `DATE`. +fn odbc_parameter(parameter: &BoundParameter) -> Box { + let small = |value: u32| u16::try_from(value).unwrap_or(u16::MAX); + match parameter { + BoundParameter::Text(text) => Box::new(text.as_str().into_parameter()), + BoundParameter::Timestamp(time) => Box::new(WithDataType::new( + Timestamp { + year: i16::try_from(time.year()).unwrap_or(i16::MAX), + month: small(time.month()), + day: small(time.day()), + hour: small(time.hour()), + minute: small(time.minute()), + second: small(time.second()), + fraction: time.nanosecond(), + }, + DataType::Timestamp { + precision: TIMESTAMP_PARAMETER_PRECISION, + }, + )), + BoundParameter::Date(date) => Box::new(WithDataType::new( + Date { + year: i16::try_from(date.year()).unwrap_or(i16::MAX), + month: small(date.month()), + day: small(date.day()), + }, + DataType::Date, + )), + } +} diff --git a/crates/sqlload/src/odbc_source/models.rs b/crates/sqlload/src/odbc_source/models.rs index 9b6461b5..af1b086c 100644 --- a/crates/sqlload/src/odbc_source/models.rs +++ b/crates/sqlload/src/odbc_source/models.rs @@ -1,3 +1,4 @@ +use crate::adapters::contract::models::BoundParameter; use crate::auth::models::Secret; use crate::postgres_source::models::{ReadColumn, SourceTable}; @@ -24,7 +25,7 @@ pub struct OdbcReadRequest<'a> { pub columns: &'a [ReadColumn], /// A condition on the source's columns, already rendered. pub filter: Option<&'a str>, - /// The values bound to `filter`'s `?` placeholders, in order, as text. - pub parameters: &'a [String], + /// The values bound to `filter`'s `?` placeholders, in order, each as its type. + pub parameters: &'a [BoundParameter], pub batch_rows: usize, } diff --git a/crates/sqlload/tests/loader.rs b/crates/sqlload/tests/loader.rs index 00275ecb..7bc53f48 100644 --- a/crates/sqlload/tests/loader.rs +++ b/crates/sqlload/tests/loader.rs @@ -44,6 +44,9 @@ mod test_loader_edge_cases; #[path = "loader/test_mixed_cursors.rs"] mod test_mixed_cursors; #[cfg(feature = "odbc")] +#[path = "loader/test_odbc_drivers.rs"] +mod test_odbc_drivers; +#[cfg(feature = "odbc")] #[path = "loader/test_odbc_to_duckdb.rs"] mod test_odbc_to_duckdb; #[path = "loader/test_postgres_cdc.rs"] diff --git a/crates/sqlload/tests/loader/odbc/helpers.rs b/crates/sqlload/tests/loader/odbc/helpers.rs index 9be37c6a..dc9f6f27 100644 --- a/crates/sqlload/tests/loader/odbc/helpers.rs +++ b/crates/sqlload/tests/loader/odbc/helpers.rs @@ -73,3 +73,60 @@ write_strategy = "append" database = database.display(), ) } + +/// Run each of `statements` through ODBC, one at a time. +pub(crate) fn odbc_execute(connection_string: &str, statements: &[&str]) -> Result<(), String> { + let connection = arrow_odbc::odbc_api::environment() + .map_err(|error| error.to_string())? + .connect_with_connection_string( + connection_string, + arrow_odbc::odbc_api::ConnectionOptions::default(), + ) + .map_err(|error| format!("cannot connect: {error}"))?; + statements.iter().try_for_each(|statement| { + connection + .execute(statement, (), None) + .map(|_| ()) + .map_err(|error| format!("{statement}: {error}")) + }) +} + +/// A project reading an events table appended by its millisecond `happened_at` cursor and a days table replaced from its last `opened` day, through `connection_string`, into DuckDB. +pub(crate) fn driver_project( + connection_string: &str, + (events, days): (&str, &str), + database: &std::path::Path, +) -> String { + format!( + r#" +[connections.erp] +kind = "odbc" +connection_string = "{connection_string}" + +[connections.local] +kind = "duckdb" +database = '{database}' + +[targets.dev] +connection = "local" +loader_schema = "raw" + +[sources.erp] +kind = "odbc" +connection = "erp" +description = "Events and days of a generic ERP." +include = ["{events}", "{days}"] + +[sources.erp.tables."{events}"] +description = "Events, read after the newest happened_at." +cursor_column = "happened_at" +write_strategy = "append" + +[sources.erp.tables."{days}"] +description = "Days, the last one read again." +cursor_column = "opened" +write_strategy = "delete_insert" +"#, + database = database.display(), + ) +} diff --git a/crates/sqlload/tests/loader/test_odbc_drivers.rs b/crates/sqlload/tests/loader/test_odbc_drivers.rs new file mode 100644 index 00000000..94f04bbd --- /dev/null +++ b/crates/sqlload/tests/loader/test_odbc_drivers.rs @@ -0,0 +1,110 @@ +use sqlload_testkit::odbc_servers::main::start_odbc_server::start_odbc_server; +use sqlload_testkit::odbc_servers::models::OdbcDatabase; + +use crate::helpers::{duckdb_text, run_async, run_sqlload}; +use crate::odbc_helpers::{driver_project, odbc_execute}; +use crate::test_types::OdbcDriverTestCase; + +const TABLES: &str = "SELECT (SELECT string_agg(id::VARCHAR, ',' ORDER BY id) || ' to ' || strftime(max(happened_at), '%Y-%m-%d %H:%M:%S.%g') FROM raw.erp__events) || ' | ' || (SELECT string_agg(id::VARCHAR, ',' ORDER BY id) || ' to ' || max(opened)::VARCHAR FROM raw.erp__days)"; + +const FIRST: &str = "1,2 to 2026-01-31 23:59:59.997 | 1,2 to 2026-01-31"; + +const SECOND: &str = "1,2,3,4 to 2026-02-01 00:00:00.000 | 1,2,3,4 to 2026-02-01"; + +#[test] +fn given_each_odbc_driver_when_a_millisecond_timestamp_cursor_and_a_date_cursor_resume_then_the_next_load_reads_exactly_the_rows_after_each_boundary() +-> Result<(), String> { + let test_cases = [ + OdbcDriverTestCase { + description: "SQL Server through Microsoft's ODBC Driver 18: {ts} and {d} bounds on datetime2(3) and date", + database: OdbcDatabase::SqlServer, + tables: ("erp.events", "erp.days"), + setup: &[ + "CREATE SCHEMA erp", + "CREATE TABLE erp.events (id int PRIMARY KEY, name nvarchar(40), happened_at datetime2(3) NOT NULL)", + "CREATE TABLE erp.days (id int PRIMARY KEY, opened date NOT NULL)", + "INSERT INTO erp.events VALUES (1, 'a', '2026-01-31 10:00:00'), (2, 'b', '2026-01-31 23:59:59.997')", + "INSERT INTO erp.days VALUES (1, '2026-01-30'), (2, '2026-01-31')", + ], + change: &[ + "INSERT INTO erp.events VALUES (0, 'early', '2026-01-31 23:59:59.996'), (3, 'next', '2026-01-31 23:59:59.998'), (4, 'later', '2026-02-01 00:00:00')", + "INSERT INTO erp.days VALUES (3, '2026-01-31'), (4, '2026-02-01')", + ], + expected_tables: (FIRST, SECOND), + }, + OdbcDriverTestCase { + description: "MySQL through Connector/ODBC, a driver without schemas, in an ANSI_QUOTES session: {ts} and {d} bounds on DATETIME(3) and DATE", + database: OdbcDatabase::MySql, + tables: ("events", "days"), + setup: &[ + "CREATE TABLE events (id int PRIMARY KEY, name varchar(40), happened_at DATETIME(3) NOT NULL)", + "CREATE TABLE days (id int PRIMARY KEY, opened DATE NOT NULL)", + "INSERT INTO events VALUES (1, 'a', '2026-01-31 10:00:00'), (2, 'b', '2026-01-31 23:59:59.997')", + "INSERT INTO days VALUES (1, '2026-01-30'), (2, '2026-01-31')", + ], + change: &[ + "INSERT INTO events VALUES (0, 'early', '2026-01-31 23:59:59.996'), (3, 'next', '2026-01-31 23:59:59.998'), (4, 'later', '2026-02-01 00:00:00')", + "INSERT INTO days VALUES (3, '2026-01-31'), (4, '2026-02-01')", + ], + expected_tables: (FIRST, SECOND), + }, + OdbcDriverTestCase { + description: "Postgres through psqlODBC: {ts} and {d} bounds on timestamp(3) and date", + database: OdbcDatabase::Postgres, + tables: ("erp.events", "erp.days"), + setup: &[ + "CREATE SCHEMA erp", + "CREATE TABLE erp.events (id int PRIMARY KEY, name varchar(40), happened_at timestamp(3) NOT NULL)", + "CREATE TABLE erp.days (id int PRIMARY KEY, opened date NOT NULL)", + "INSERT INTO erp.events VALUES (1, 'a', '2026-01-31 10:00:00'), (2, 'b', '2026-01-31 23:59:59.997')", + "INSERT INTO erp.days VALUES (1, '2026-01-30'), (2, '2026-01-31')", + ], + change: &[ + "INSERT INTO erp.events VALUES (0, 'early', '2026-01-31 23:59:59.996'), (3, 'next', '2026-01-31 23:59:59.998'), (4, 'later', '2026-02-01 00:00:00')", + "INSERT INTO erp.days VALUES (3, '2026-01-31'), (4, '2026-02-01')", + ], + expected_tables: (FIRST, SECOND), + }, + ]; + for test_case in test_cases { + run_async(async { + let server = start_odbc_server(test_case.database).await?; + odbc_execute(&server.connection_string, test_case.setup)?; + let folder = tempfile::tempdir().map_err(|error| error.to_string())?; + let database = folder.path().join("out.duckdb"); + let config = folder.path().join("sqlload.toml"); + std::fs::write( + &config, + driver_project(&server.connection_string, test_case.tables, &database), + ) + .map_err(|error| error.to_string())?; + let (succeeded, output) = run_sqlload(&config, &["load", "--yes"], ("", ""))?; + assert!( + succeeded && output.contains("2 loaded"), + "{}, first load:\n{output}", + test_case.description + ); + assert_eq!( + duckdb_text(&database, TABLES)?, + test_case.expected_tables.0, + "{}, first load", + test_case.description + ); + odbc_execute(&server.connection_string, test_case.change)?; + let (succeeded, output) = run_sqlload(&config, &["load", "--yes"], ("", ""))?; + assert!( + succeeded && output.contains("2 loaded"), + "{}, second load:\n{output}", + test_case.description + ); + assert_eq!( + duckdb_text(&database, TABLES)?, + test_case.expected_tables.1, + "{}, second load: the event at .996 is before the maximum and the day 2026-01-31 is read again", + test_case.description + ); + Ok(()) + })?; + } + Ok(()) +} diff --git a/crates/sqlload/tests/loader/test_types.rs b/crates/sqlload/tests/loader/test_types.rs index a3b6f186..aff529ab 100644 --- a/crates/sqlload/tests/loader/test_types.rs +++ b/crates/sqlload/tests/loader/test_types.rs @@ -908,3 +908,18 @@ pub(crate) struct SftpSnowflakePlanTestCase { pub(crate) expected_success: bool, pub(crate) expected_output: &'static str, } + +/// One database's run of the ODBC cursor scenario. +#[cfg(feature = "odbc")] +pub(crate) struct OdbcDriverTestCase { + pub(crate) description: &'static str, + pub(crate) database: sqlload_testkit::odbc_servers::models::OdbcDatabase, + /// The events and days tables as the source's `include` names them. + pub(crate) tables: (&'static str, &'static str), + /// Statements creating both tables and their first rows. + pub(crate) setup: &'static [&'static str], + /// Statements adding rows around each cursor's maximum before the second load. + pub(crate) change: &'static [&'static str], + /// Each table's ids and newest cursor value after the first load, then after the second. + pub(crate) expected_tables: (&'static str, &'static str), +} diff --git a/fensu.toml b/fensu.toml index 57e9ef3a..4bf2d6ed 100644 --- a/fensu.toml +++ b/fensu.toml @@ -41,6 +41,11 @@ rule = "FPRSL105" path = "crates/sqlload-testkit/src/parity/main/run_cases.rs" reason = "Testkit entry used by sqlload's integration tests, which live outside the analysed crate tree." +[[targets.rust.rule_exceptions]] +rule = "FPRSL105" +path = "crates/sqlload-testkit/src/odbc_servers/main/start_odbc_server.rs" +reason = "Testkit entry used by sqlload's integration tests, which live outside the analysed crate tree." + [[targets.rust.rule_exceptions]] rule = "FPRSL105" path = "crates/sqlload-testkit/src/postgres/main/start_postgres_tls.rs" @@ -146,16 +151,6 @@ rule = "FPRSL105" path = "crates/sqlload-testkit/src/harness/main/read_metrics.rs" reason = "Testkit entry used by sqlload's integration tests, which live outside the analysed crate tree." -[[targets.rust.rule_exceptions]] -rule = "FPRSL105" -path = "crates/sqlload-testkit/src/postgres/main/connect.rs" -reason = "Testkit entry used by sqlload's integration tests, which live outside the analysed crate tree." - -[[targets.rust.rule_exceptions]] -rule = "FPRSL105" -path = "crates/sqlload-testkit/src/postgres/main/start_postgres.rs" -reason = "Testkit entry used by sqlload's integration tests, which live outside the analysed crate tree." - [[targets.rust.rule_exceptions]] rule = "FPRSL105" path = "crates/sqlload-testkit/src/seaweedfs/main/create_bucket.rs" From 9e9ad400155550d27405dc5528350ea5831a8221 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:39:50 +0000 Subject: [PATCH 11/25] fix(odbc): bind a SQL Server legacy datetime cursor bound as text in the column's type SQL Server compares a legacy datetime column with a datetime2 parameter at the column's exact 1/300 s ticks, which its driver reads rounded to milliseconds, so a cursor resuming from .123 (stored .12333) read that row again, and an inclusive bound at .997 (stored .99667) missed it. No TIMESTAMP bind precision avoids this: the driver sends datetime2 for every one. When SQL Server's driver describes the parameter with one to three fractional digits, the bound is now text with that many digits, which SQL Server converts to the column's own type. Every other timestamp bound stays a TIMESTAMP(7) parameter, exact against every precision on all three live drivers; a lower declared precision than the value holds fails with 22008, so it is not derived from the column. WIP for CHI-1236. --- .../sqlload/src/odbc_source/_helpers/mod.rs | 1 + .../src/odbc_source/_helpers/parameters.rs | 87 +++++++++++++++++++ crates/sqlload/src/odbc_source/constants.rs | 6 ++ .../sqlload/src/odbc_source/main/read_rows.rs | 45 ++-------- crates/sqlload/src/odbc_source/tests.rs | 2 + .../odbc_source/tests/test_timestamp_text.rs | 50 +++++++++++ .../src/odbc_source/tests/test_types.rs | 7 ++ .../sqlload/tests/loader/test_odbc_drivers.rs | 19 ++++ 8 files changed, 178 insertions(+), 39 deletions(-) create mode 100644 crates/sqlload/src/odbc_source/_helpers/parameters.rs create mode 100644 crates/sqlload/src/odbc_source/tests/test_timestamp_text.rs diff --git a/crates/sqlload/src/odbc_source/_helpers/mod.rs b/crates/sqlload/src/odbc_source/_helpers/mod.rs index 8f1c09da..8c1fd09e 100644 --- a/crates/sqlload/src/odbc_source/_helpers/mod.rs +++ b/crates/sqlload/src/odbc_source/_helpers/mod.rs @@ -2,4 +2,5 @@ pub(crate) mod batches; pub(crate) mod catalog; pub(crate) mod connect; pub(crate) mod convert; +pub(crate) mod parameters; pub(crate) mod query; diff --git a/crates/sqlload/src/odbc_source/_helpers/parameters.rs b/crates/sqlload/src/odbc_source/_helpers/parameters.rs new file mode 100644 index 00000000..bc9a98d2 --- /dev/null +++ b/crates/sqlload/src/odbc_source/_helpers/parameters.rs @@ -0,0 +1,87 @@ +//! Cursor bounds as the ODBC parameters a read binds. + +use arrow_odbc::odbc_api::handles::AsStatementRef; +use arrow_odbc::odbc_api::parameter::{InputParameter, WithDataType}; +use arrow_odbc::odbc_api::sys::{Date, Timestamp}; +use arrow_odbc::odbc_api::{DataType, IntoParameter, Prepared}; +use chrono::{Datelike, NaiveDateTime, Timelike}; + +use crate::adapters::contract::models::BoundParameter; +use crate::odbc_source::constants::{ + LEGACY_DATETIME_DIGITS, SQL_SERVER_DBMS, TIMESTAMP_PARAMETER_PRECISION, +}; + +/// `parameters` as `prepared` binds them, in order: text as `VARCHAR`; a date as `DATE`; a timestamp as `TIMESTAMP` with seven fractional digits, which SQL Server compares as `datetime2(7)` (three would make it a `datetime`, rounding milliseconds), except against a SQL Server column the driver describes with one to three fractional digits, a legacy `datetime`: that compares a `datetime2` parameter with the column's exact 1/300 s ticks, which the driver rounded to milliseconds when the bound was read, so the bound goes as text, which SQL Server converts to the column's own type. +pub(crate) fn bound_parameters( + prepared: &mut Prepared, + dbms: &str, + parameters: &[BoundParameter], +) -> Vec> { + parameters + .iter() + .zip(1u16..) + .map(|(parameter, number)| match parameter { + BoundParameter::Timestamp(time) if dbms == SQL_SERVER_DBMS => { + match prepared.describe_param(number) { + Ok(described) => match described.data_type { + DataType::Timestamp { precision } + if LEGACY_DATETIME_DIGITS.contains(&precision) => + { + Box::new( + timestamp_text(time, precision.unsigned_abs()).into_parameter(), + ) + } + _ => odbc_parameter(parameter), + }, + Err(_) => odbc_parameter(parameter), + } + } + _ => odbc_parameter(parameter), + }) + .collect() +} + +/// `time` as `YYYY-MM-DD HH:MM:SS` with `digits` fractional digits, or with six when it has more than `digits` (which the database then refuses rather than misreads). +pub(crate) fn timestamp_text(time: &NaiveDateTime, digits: u16) -> String { + let nanoseconds = time.nanosecond(); + let unit = 10u32.pow(9u32.saturating_sub(u32::from(digits))); + match (digits, nanoseconds % unit) { + (0, 0) => time.format("%Y-%m-%d %H:%M:%S").to_string(), + (_, 0) => format!( + "{}.{}", + time.format("%Y-%m-%d %H:%M:%S"), + &format!("{nanoseconds:09}")[..usize::from(digits)] + ), + _ => time.format("%Y-%m-%d %H:%M:%S%.6f").to_string(), + } +} + +/// `parameter` bound as its own type. +fn odbc_parameter(parameter: &BoundParameter) -> Box { + let small = |value: u32| u16::try_from(value).unwrap_or(u16::MAX); + match parameter { + BoundParameter::Text(text) => Box::new(text.clone().into_parameter()), + BoundParameter::Timestamp(time) => Box::new(WithDataType::new( + Timestamp { + year: i16::try_from(time.year()).unwrap_or(i16::MAX), + month: small(time.month()), + day: small(time.day()), + hour: small(time.hour()), + minute: small(time.minute()), + second: small(time.second()), + fraction: time.nanosecond(), + }, + DataType::Timestamp { + precision: TIMESTAMP_PARAMETER_PRECISION, + }, + )), + BoundParameter::Date(date) => Box::new(WithDataType::new( + Date { + year: i16::try_from(date.year()).unwrap_or(i16::MAX), + month: small(date.month()), + day: small(date.day()), + }, + DataType::Date, + )), + } +} diff --git a/crates/sqlload/src/odbc_source/constants.rs b/crates/sqlload/src/odbc_source/constants.rs index adf2711d..38f8f6b8 100644 --- a/crates/sqlload/src/odbc_source/constants.rs +++ b/crates/sqlload/src/odbc_source/constants.rs @@ -28,3 +28,9 @@ pub const TOO_LARGE_VALUE: &str = "too large to be written into the allocated bu /// The fractional digits a timestamp cursor bound is described with: seven, so SQL Server compares it as `datetime2(7)`, exactly, rather than as a `datetime`. pub const TIMESTAMP_PARAMETER_PRECISION: i16 = 7; + +/// The name Microsoft's SQL Server drivers report for the database. +pub const SQL_SERVER_DBMS: &str = "Microsoft SQL Server"; + +/// The fractional digits SQL Server's driver describes a legacy `datetime` parameter with. +pub const LEGACY_DATETIME_DIGITS: std::ops::RangeInclusive = 1..=3; diff --git a/crates/sqlload/src/odbc_source/main/read_rows.rs b/crates/sqlload/src/odbc_source/main/read_rows.rs index 1a2ef680..5b1b3c2f 100644 --- a/crates/sqlload/src/odbc_source/main/read_rows.rs +++ b/crates/sqlload/src/odbc_source/main/read_rows.rs @@ -1,19 +1,16 @@ //! Read a table or query through ODBC as Arrow batches. use arrow_array::RecordBatch; -use arrow_odbc::odbc_api::parameter::{InputParameter, WithDataType}; -use arrow_odbc::odbc_api::sys::{Date, Timestamp}; -use arrow_odbc::odbc_api::{DataType, IntoParameter}; -use chrono::{Datelike, Timelike}; use tokio::sync::mpsc::Sender; use crate::adapters::contract::models::BoundParameter; use crate::odbc_source::_helpers::batches::{batch_reader, fetch_error}; use crate::odbc_source::_helpers::connect::connect; use crate::odbc_source::_helpers::convert::{converted, output_schema}; +use crate::odbc_source::_helpers::parameters::bound_parameters; use crate::odbc_source::_helpers::query::select_query; use crate::odbc_source::constants::BATCHES_AHEAD; -use crate::odbc_source::constants::TIMESTAMP_PARAMETER_PRECISION; + use crate::odbc_source::errors::OdbcSourceError; use crate::odbc_source::models::{OdbcReadRequest, OdbcReader}; @@ -45,7 +42,7 @@ pub async fn read_rows( Ok(rows) } -/// Fetch `query`'s rows, with `parameters` bound to its placeholders each as its type, into `sender` until they end, the receiver goes away, or a fetch fails (sent on as the error). Decimal columns are fetched as text, which `converted` parses without rounding, since drivers round values to the scale they describe (an unbounded `numeric` as `NUMERIC(28,6)`). +/// Fetch `query`'s rows, with `parameters` bound to its placeholders as `bound_parameters` binds them, into `sender` until they end, the receiver goes away, or a fetch fails (sent on as the error). Decimal columns are fetched as text, which `converted` parses without rounding, since drivers round values to the scale they describe (an unbounded `numeric` as `NUMERIC(28,6)`). fn fetch( reader: &OdbcReader, (query, parameters): (&str, &[BoundParameter]), @@ -54,9 +51,9 @@ fn fetch( ) -> Result<(), OdbcSourceError> { let connection = connect(reader)?; let dbms: String = connection.database_management_system_name()?; - let parameters: Vec> = - parameters.iter().map(odbc_parameter).collect(); - let Some(cursor) = connection.execute(query, parameters.as_slice(), None)? else { + let mut prepared = connection.prepare(query)?; + let parameters = bound_parameters(&mut prepared, &dbms, parameters); + let Some(cursor) = prepared.execute(parameters.as_slice())? else { return Err(OdbcSourceError::Invalid( "the read returned no result set".to_owned(), )); @@ -71,33 +68,3 @@ fn fetch( } Ok(()) } - -/// `parameter` as the driver receives it: text as `VARCHAR`; a timestamp as `TIMESTAMP` with seven fractional digits, which SQL Server reads as `datetime2(7)` (it reads three as `datetime`, rounding milliseconds to thirds of a hundredth); a date as `DATE`. -fn odbc_parameter(parameter: &BoundParameter) -> Box { - let small = |value: u32| u16::try_from(value).unwrap_or(u16::MAX); - match parameter { - BoundParameter::Text(text) => Box::new(text.as_str().into_parameter()), - BoundParameter::Timestamp(time) => Box::new(WithDataType::new( - Timestamp { - year: i16::try_from(time.year()).unwrap_or(i16::MAX), - month: small(time.month()), - day: small(time.day()), - hour: small(time.hour()), - minute: small(time.minute()), - second: small(time.second()), - fraction: time.nanosecond(), - }, - DataType::Timestamp { - precision: TIMESTAMP_PARAMETER_PRECISION, - }, - )), - BoundParameter::Date(date) => Box::new(WithDataType::new( - Date { - year: i16::try_from(date.year()).unwrap_or(i16::MAX), - month: small(date.month()), - day: small(date.day()), - }, - DataType::Date, - )), - } -} diff --git a/crates/sqlload/src/odbc_source/tests.rs b/crates/sqlload/src/odbc_source/tests.rs index 5c072639..d238ac02 100644 --- a/crates/sqlload/src/odbc_source/tests.rs +++ b/crates/sqlload/src/odbc_source/tests.rs @@ -8,3 +8,5 @@ mod odbc_reader; mod select_query; #[path = "tests/test_types.rs"] mod test_types; +#[path = "tests/test_timestamp_text.rs"] +mod timestamp_text; diff --git a/crates/sqlload/src/odbc_source/tests/test_timestamp_text.rs b/crates/sqlload/src/odbc_source/tests/test_timestamp_text.rs new file mode 100644 index 00000000..37c97dd2 --- /dev/null +++ b/crates/sqlload/src/odbc_source/tests/test_timestamp_text.rs @@ -0,0 +1,50 @@ +use crate::odbc_source::_helpers::parameters::timestamp_text; +use crate::odbc_source::tests::test_types::TimestampTextTestCase; + +#[test] +fn given_a_timestamp_bound_for_a_legacy_sql_server_datetime_when_writing_it_as_text_then_it_has_the_column_s_digits_unless_that_would_cut_it() +-> Result<(), String> { + let test_cases = [ + TimestampTextTestCase { + description: "a millisecond bound keeps its three digits, which datetime converts to the tick it read", + time: "2026-01-31 23:59:59.123", + digits: 3, + expected_text: "2026-01-31 23:59:59.123", + }, + TimestampTextTestCase { + description: "a whole second still has three digits", + time: "2026-02-01 00:00:00", + digits: 3, + expected_text: "2026-02-01 00:00:00.000", + }, + TimestampTextTestCase { + description: "a leading zero in the fraction is kept", + time: "2026-01-31 23:59:59.007", + digits: 3, + expected_text: "2026-01-31 23:59:59.007", + }, + TimestampTextTestCase { + description: "a microsecond bound keeps six digits rather than being cut, so the database refuses it", + time: "2026-01-31 23:59:59.123456", + digits: 3, + expected_text: "2026-01-31 23:59:59.123456", + }, + TimestampTextTestCase { + description: "no digits for a whole second", + time: "2026-01-31 23:59:00", + digits: 0, + expected_text: "2026-01-31 23:59:00", + }, + ]; + for test_case in test_cases { + let time = chrono::NaiveDateTime::parse_from_str(test_case.time, "%Y-%m-%d %H:%M:%S%.f") + .map_err(|error| error.to_string())?; + assert_eq!( + timestamp_text(&time, test_case.digits), + test_case.expected_text, + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlload/src/odbc_source/tests/test_types.rs b/crates/sqlload/src/odbc_source/tests/test_types.rs index 45b30525..e87cd927 100644 --- a/crates/sqlload/src/odbc_source/tests/test_types.rs +++ b/crates/sqlload/src/odbc_source/tests/test_types.rs @@ -23,3 +23,10 @@ pub(crate) struct OdbcReaderTestCase { pub(crate) password: Option<&'static str>, pub(crate) expected_connection_string: &'static str, } + +pub(crate) struct TimestampTextTestCase { + pub(crate) description: &'static str, + pub(crate) time: &'static str, + pub(crate) digits: u16, + pub(crate) expected_text: &'static str, +} diff --git a/crates/sqlload/tests/loader/test_odbc_drivers.rs b/crates/sqlload/tests/loader/test_odbc_drivers.rs index 94f04bbd..2a55e0ee 100644 --- a/crates/sqlload/tests/loader/test_odbc_drivers.rs +++ b/crates/sqlload/tests/loader/test_odbc_drivers.rs @@ -11,6 +11,8 @@ const FIRST: &str = "1,2 to 2026-01-31 23:59:59.997 | 1,2 to 2026-01-31"; const SECOND: &str = "1,2,3,4 to 2026-02-01 00:00:00.000 | 1,2,3,4 to 2026-02-01"; +const LEGACY_FIRST: &str = "1,2 to 2026-01-31 23:59:59.123 | 1,2 to 2026-01-31"; + #[test] fn given_each_odbc_driver_when_a_millisecond_timestamp_cursor_and_a_date_cursor_resume_then_the_next_load_reads_exactly_the_rows_after_each_boundary() -> Result<(), String> { @@ -32,6 +34,23 @@ fn given_each_odbc_driver_when_a_millisecond_timestamp_cursor_and_a_date_cursor_ ], expected_tables: (FIRST, SECOND), }, + OdbcDriverTestCase { + description: "SQL Server's legacy datetime, whose 1/300 s ticks the driver reads rounded to milliseconds (.123 is .12333): a datetime2 bound would read the .123 row again, so it is bound as text in the column's own type", + database: OdbcDatabase::SqlServer, + tables: ("erp.events", "erp.days"), + setup: &[ + "CREATE SCHEMA erp", + "CREATE TABLE erp.events (id int PRIMARY KEY, name nvarchar(40), happened_at datetime NOT NULL)", + "CREATE TABLE erp.days (id int PRIMARY KEY, opened date NOT NULL)", + "INSERT INTO erp.events VALUES (1, 'a', '2026-01-31 10:00:00'), (2, 'b', '2026-01-31 23:59:59.123')", + "INSERT INTO erp.days VALUES (1, '2026-01-30'), (2, '2026-01-31')", + ], + change: &[ + "INSERT INTO erp.events VALUES (0, 'early', '2026-01-31 23:59:59.120'), (3, 'next', '2026-01-31 23:59:59.127'), (4, 'later', '2026-02-01 00:00:00')", + "INSERT INTO erp.days VALUES (3, '2026-01-31'), (4, '2026-02-01')", + ], + expected_tables: (LEGACY_FIRST, SECOND), + }, OdbcDriverTestCase { description: "MySQL through Connector/ODBC, a driver without schemas, in an ANSI_QUOTES session: {ts} and {d} bounds on DATETIME(3) and DATE", database: OdbcDatabase::MySql, From 9d9607064fa302cf1fc66c3a34d20ff782992eb4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:48:47 +0000 Subject: [PATCH 12/25] fix(cli): run a command on a thread with 8 MiB of stack, not the caller's A Windows main thread, sqlload.exe's or python.exe's, has 1 MB of stack, which a debug build's copy into a files target outgrew (0xC00000FD, reproduced on Linux with ulimit -s 1024). Every command now runs on a scoped thread with a Linux main thread's default 8 MiB, whoever calls it. --- crates/sqlload/src/command/constants.rs | 3 + .../src/command/main/run_args_with_host.rs | 65 +++++++++++++------ 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/crates/sqlload/src/command/constants.rs b/crates/sqlload/src/command/constants.rs index c20fcfef..b3707427 100644 --- a/crates/sqlload/src/command/constants.rs +++ b/crates/sqlload/src/command/constants.rs @@ -11,3 +11,6 @@ pub const FORCED_STOP_EXIT: i32 = 130; /// The exit code of a run that failed only in ways running it again unchanged may fix (`EX_TEMPFAIL`): a dropped or refused connection, throttling or server errors that outlasted their retries, or a table another run is loading. pub const RETRYABLE_EXIT: u8 = 75; + +/// The stack a command runs on: a Linux main thread's default, 8 MiB, on every system. +pub const COMMAND_STACK_BYTES: usize = 8 * 1024 * 1024; diff --git a/crates/sqlload/src/command/main/run_args_with_host.rs b/crates/sqlload/src/command/main/run_args_with_host.rs index 0d2a9034..e0ee9517 100644 --- a/crates/sqlload/src/command/main/run_args_with_host.rs +++ b/crates/sqlload/src/command/main/run_args_with_host.rs @@ -12,6 +12,7 @@ use crate::command::_helpers::limits::runtime; use crate::command::_helpers::loader_commands::{exit_code, json_error, run_loader}; use crate::command::_helpers::logging::init_logging; use crate::command::_helpers::output::{config_text, print_lines}; +use crate::command::constants::COMMAND_STACK_BYTES; use crate::command::errors::CommandError; use crate::command::models::{Cli, Command, LoaderRequest}; use crate::project::main::is_project::is_project; @@ -50,26 +51,38 @@ where Ok(runtime) => runtime, Err(message) => return failed(json, name, &message.into()), }; - let outcome = runtime.block_on(async { - if copies || is_project(&text) { - let request = LoaderRequest { - text: &text, - config: &cli.config, - command: cli.command, - host, - no_color: cli.no_color, - variables: Default::default(), - }; - return match copies { - true => run_copy(request).await, - false => run_loader(request).await, - }; - } - run_exporter_command(&cli.config, cli.command) - .await - .map(|lines| (lines, 0)) - .map_err(CommandError::from) + let outcome = on_command_stack(|| { + runtime.block_on(async { + if copies || is_project(&text) { + let request = LoaderRequest { + text: &text, + config: &cli.config, + command: cli.command, + host, + no_color: cli.no_color, + variables: Default::default(), + }; + return match copies { + true => run_copy(request).await, + false => run_loader(request).await, + }; + } + run_exporter_command(&cli.config, cli.command) + .await + .map(|lines| (lines, 0)) + .map_err(CommandError::from) + }) }); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + return failed( + json, + name, + &format!("cannot start the command's thread: {error}").into(), + ); + } + }; match outcome { Ok((lines, code)) => { print_lines(lines, cli.no_color); @@ -92,3 +105,17 @@ fn failed(json: bool, code: &str, error: &CommandError) -> u8 { } exit_code(false, error.retryable) } + +/// `body`'s result, run on a thread of its own with `COMMAND_STACK_BYTES` of stack rather than on the caller's: a Windows main thread, the executable's or python.exe's, has 1 MB, which a debug build's copy into a files target outgrows. A panic in `body` carries on in the caller. +fn on_command_stack(body: impl FnOnce() -> T + Send) -> std::io::Result { + std::thread::scope(|scope| { + let thread = std::thread::Builder::new() + .name("sqlload-command".to_owned()) + .stack_size(COMMAND_STACK_BYTES) + .spawn_scoped(scope, body)?; + Ok(match thread.join() { + Ok(value) => value, + Err(panic) => std::panic::resume_unwind(panic), + }) + }) +} From b80c234c710a22d708c9d0344f9820a7a60bfa6b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:49:25 +0000 Subject: [PATCH 13/25] ci(windows): resume ODBC cursors through Windows drivers: ODBC Driver 18 on LocalDB, MySQL Connector/ODBC, psqlODBC The Windows job installs the three drivers from pinned, SHA-256-checked MSIs (.github/scripts/install_odbc_drivers.ps1), starts the image's SQL Server LocalDB and a MySQL 8.0 server, and windows_e2e.py --cursor-drivers runs the container tests' cursor scenario through each driver: a datetime2(3) and a legacy datetime table on LocalDB, DATETIME(3) on MySQL and timestamp(3) on Postgres, the next load reading exactly the rows after each boundary. WIP for CHI-1236. --- .github/scripts/install_odbc_drivers.ps1 | 45 ++++++++ .github/scripts/windows_e2e.py | 140 ++++++++++++++++++++++- .github/workflows/verify.yml | 32 ++++-- 3 files changed, 208 insertions(+), 9 deletions(-) create mode 100644 .github/scripts/install_odbc_drivers.ps1 diff --git a/.github/scripts/install_odbc_drivers.ps1 b/.github/scripts/install_odbc_drivers.ps1 new file mode 100644 index 00000000..0598cdce --- /dev/null +++ b/.github/scripts/install_odbc_drivers.ps1 @@ -0,0 +1,45 @@ +# Installs the 64-bit ODBC drivers the Windows checks read through, each a pinned installer +# checked against its SHA-256, and lists the drivers Windows then has: +# PostgreSQL Unicode(x64) psqlODBC 18.00.0006 +# ODBC Driver 18 for SQL Server Microsoft's msodbcsql18 18.7.1.1 (installing it accepts Microsoft's licence) +# MySQL ODBC 26.10 Unicode Driver MySQL Connector/ODBC 26.10.0 (its SHA-256 taken from the file +# matching MySQL's published MD5, 34541f6d21bb6a4b2f56b3f8f1afcf4b) +# Usage: pwsh .github/scripts/install_odbc_drivers.ps1 +$ErrorActionPreference = "Stop" + +$drivers = @( + @{ + Name = "psqlODBC" + Uri = "https://ftp.postgresql.org/pub/odbc/releases/REL-18_00_0006/psqlodbc_x64.msi" + Sha256 = "8962543113df83410b74f113db1ef80717b65a1dc7869e18c24bd852dd65535f" + Arguments = @() + }, + @{ + Name = "msodbcsql18" + Uri = "https://go.microsoft.com/fwlink/?linkid=2378279" + Sha256 = "21ef69e4b942f18aced55fa7c3a8d7263004e113a96690bac7509645e1b1a310" + Arguments = @("IACCEPTMSODBCSQLLICENSETERMS=YES") + }, + @{ + Name = "MySQL Connector/ODBC" + Uri = "https://cdn.mysql.com/Downloads/Connector-ODBC/26.10/mysql-connector-odbc-26.10.0-winx64.msi" + Sha256 = "ac58d7f1b4d7c1f37630a78c83c659f04c127754bfa299957b4effa32bd83cae" + Arguments = @() + } +) + +foreach ($driver in $drivers) { + $msi = Join-Path $env:RUNNER_TEMP ("odbc-" + [guid]::NewGuid() + ".msi") + Invoke-WebRequest -Uri $driver.Uri -OutFile $msi + $hash = (Get-FileHash -Algorithm SHA256 -Path $msi).Hash.ToLowerInvariant() + if ($hash -ne $driver.Sha256) { + throw "$($driver.Name): the installer's SHA-256 is $hash, not the pinned $($driver.Sha256); check the new release and update the pin" + } + $process = Start-Process msiexec.exe -ArgumentList (@("/i", $msi, "/qn", "/norestart") + $driver.Arguments) -Wait -PassThru + if ($process.ExitCode -ne 0) { + throw "$($driver.Name): msiexec exited $($process.ExitCode)" + } + Remove-Item $msi +} + +Get-OdbcDriver -Platform 64-bit | Where-Object { $_.Name -match "PostgreSQL|SQL Server|MySQL" } | Format-Table Name, Platform diff --git a/.github/scripts/windows_e2e.py b/.github/scripts/windows_e2e.py index c841bd3f..0d4c225a 100644 --- a/.github/scripts/windows_e2e.py +++ b/.github/scripts/windows_e2e.py @@ -1,17 +1,22 @@ """End-to-end checks of a Windows sqlload executable: drive paths, a UNC share and an ODBC driver. -Usage: python windows_e2e.py SQLLOAD_EXE WORK_DIR SHARE [--odbc-driver NAME] [--cdc] +Usage: python windows_e2e.py SQLLOAD_EXE WORK_DIR SHARE [--odbc-driver NAME] [--cursor-drivers] [--cdc] WORK_DIR is a local folder (C:\\...) whose `share` folder is shared as SHARE (\\\\localhost\\name), so what the share holds can also be read locally. With --odbc-driver, a table in the local Postgres (the runner's preinstalled server, user postgres) is loaded through that ODBC driver, then loaded again -incrementally. With --cdc, a postgres_cdc source snapshots a table and then streams its changes, which +incrementally. With --cursor-drivers, a millisecond timestamp cursor and a date cursor resume through +Microsoft's ODBC Driver 18 (SQL Server LocalDB instance `sqlload`, datetime2 and legacy datetime), MySQL +Connector/ODBC (a MySQL server on 127.0.0.1:3307, user root without a password) and psqlODBC, and the +next load must read exactly the rows after each boundary; needs the pyodbc package. With --cdc, a postgres_cdc source snapshots a table and then streams its changes, which on Windows go through the loopback TLS relay (that server needs wal_level = logical). Needs the duckdb Python package to read the results back. """ import argparse +import json import os +import re import pathlib import subprocess import sys @@ -134,6 +139,134 @@ def odbc_checks(exe, work, driver): ) +# The cursor scenario of crates/sqlload/tests/loader/test_odbc_drivers.rs, through Windows drivers. +CURSOR_TABLES = ( + "SELECT (SELECT string_agg(id::VARCHAR, ',' ORDER BY id) || ' to ' || strftime(max(happened_at), '%Y-%m-%d %H:%M:%S.%g') FROM raw.erp__events)" + " || ' | ' || (SELECT string_agg(id::VARCHAR, ',' ORDER BY id) || ' to ' || max(opened)::VARCHAR FROM raw.erp__days)" +) +FIRST = "1,2 to 2026-01-31 23:59:59.997 | 1,2 to 2026-01-31" +SECOND = "1,2,3,4 to 2026-02-01 00:00:00.000 | 1,2,3,4 to 2026-02-01" + + +def cursor_cases(): + import pyodbc + + mysql = next(name for name in pyodbc.drivers() if re.fullmatch(r"MySQL ODBC [\d.]+ Unicode Driver", name)) + local_db = "Driver={ODBC Driver 18 for SQL Server};Server=(localdb)\\sqlload;Trusted_Connection=yes;TrustServerCertificate=yes;" + rows = [ + "INSERT INTO {events} VALUES (1, 'a', '2026-01-31 10:00:00'), (2, 'b', '2026-01-31 23:59:59.997')", + "INSERT INTO {days} VALUES (1, '2026-01-30'), (2, '2026-01-31')", + ] + change = [ + "INSERT INTO {events} VALUES (0, 'early', '2026-01-31 23:59:59.996'), (3, 'next', '2026-01-31 23:59:59.998'), (4, 'later', '2026-02-01 00:00:00')", + "INSERT INTO {days} VALUES (3, '2026-01-31'), (4, '2026-02-01')", + ] + return [ + { + "description": "SQL Server LocalDB through ODBC Driver 18, datetime2(3)", + "admin": (local_db + "Database=master;", ["CREATE DATABASE cursors_datetime2"]), + "connection": local_db + "Database=cursors_datetime2;", + "tables": ("dbo.events", "dbo.days"), + "setup": ["CREATE TABLE {events} (id int PRIMARY KEY, name nvarchar(40), happened_at datetime2(3) NOT NULL)", "CREATE TABLE {days} (id int PRIMARY KEY, opened date NOT NULL)", *rows], + "change": change, + "expected": (FIRST, SECOND), + }, + { + "description": "SQL Server LocalDB through ODBC Driver 18, legacy datetime (.123 is stored as .12333)", + "admin": (local_db + "Database=master;", ["CREATE DATABASE cursors_datetime"]), + "connection": local_db + "Database=cursors_datetime;", + "tables": ("dbo.events", "dbo.days"), + "setup": [ + "CREATE TABLE {events} (id int PRIMARY KEY, name nvarchar(40), happened_at datetime NOT NULL)", + "CREATE TABLE {days} (id int PRIMARY KEY, opened date NOT NULL)", + "INSERT INTO {events} VALUES (1, 'a', '2026-01-31 10:00:00'), (2, 'b', '2026-01-31 23:59:59.123')", + rows[1], + ], + "change": [ + "INSERT INTO {events} VALUES (0, 'early', '2026-01-31 23:59:59.120'), (3, 'next', '2026-01-31 23:59:59.127'), (4, 'later', '2026-02-01 00:00:00')", + change[1], + ], + "expected": ("1,2 to 2026-01-31 23:59:59.123 | 1,2 to 2026-01-31", SECOND), + }, + { + "description": f"MySQL through {mysql}, DATETIME(3), in an ANSI_QUOTES session", + "admin": (f"Driver={{{mysql}}};Server=127.0.0.1;Port=3307;Uid=root;Pwd=;", ["CREATE DATABASE cursors"]), + "connection": f"Driver={{{mysql}}};Server=127.0.0.1;Port=3307;Database=cursors;Uid=root;Pwd=;INITSTMT=SET SESSION sql_mode=CONCAT(@@SESSION.sql_mode,',ANSI_QUOTES');", + "tables": ("events", "days"), + "setup": ["CREATE TABLE {events} (id int PRIMARY KEY, name varchar(40), happened_at DATETIME(3) NOT NULL)", "CREATE TABLE {days} (id int PRIMARY KEY, opened DATE NOT NULL)", *rows], + "change": change, + "expected": (FIRST, SECOND), + }, + { + "description": "Postgres through psqlODBC, timestamp(3)", + "admin": None, + "connection": "Driver={PostgreSQL Unicode(x64)};Server=127.0.0.1;Port=5432;Database=postgres;Uid=postgres;Pwd=root;", + "tables": ("cursors.events", "cursors.days"), + "setup": ["DROP SCHEMA IF EXISTS cursors CASCADE", "CREATE SCHEMA cursors", "CREATE TABLE {events} (id int PRIMARY KEY, name varchar(40), happened_at timestamp(3) NOT NULL)", "CREATE TABLE {days} (id int PRIMARY KEY, opened date NOT NULL)", *rows], + "change": change, + "expected": (FIRST, SECOND), + }, + ] + + +def odbc_run(connection_string, statements, tables=("", "")): + import pyodbc + + with pyodbc.connect(connection_string, autocommit=True) as connection: + for statement in statements: + connection.execute(statement.format(events=tables[0], days=tables[1])) + + +def cursor_checks(exe, work): + for index, case in enumerate(cursor_cases()): + if case["admin"]: + odbc_run(*case["admin"]) + odbc_run(case["connection"], case["setup"], case["tables"]) + database = work / f"cursors-{index}.duckdb" + project = work / f"cursors-{index}" / "sqlload.toml" + project.parent.mkdir(parents=True, exist_ok=True) + events, days = case["tables"] + # A JSON string is a TOML basic string, so the connection string's backslash and quotes stay as they are. + project.write_text( + f""" +[connections.erp] +kind = "odbc" +connection_string = {json.dumps(case["connection"])} + +[connections.local] +kind = "duckdb" +database = '{database}' + +[targets.dev] +connection = "local" +loader_schema = "raw" + +[sources.erp] +kind = "odbc" +connection = "erp" +description = "Events and days of a generic ERP." +include = ["{events}", "{days}"] + +[sources.erp.tables."{events}"] +description = "Events, read after the newest happened_at." +cursor_column = "happened_at" +write_strategy = "append" + +[sources.erp.tables."{days}"] +description = "Days, the last one read again." +cursor_column = "opened" +write_strategy = "delete_insert" +""", + encoding="utf-8", + ) + first, second = case["expected"] + sqlload(exe, "load", "--yes", config=project) + expect(query(database, CURSOR_TABLES), [(first,)], f"{case['description']}, first load") + odbc_run(case["connection"], case["change"], case["tables"]) + sqlload(exe, "load", "--yes", config=project) + expect(query(database, CURSOR_TABLES), [(second,)], f"{case['description']}, the next load reads exactly the rows after each boundary") + + def cdc_checks(exe, work): """A postgres_cdc load streams through the loopback relay: the server offers no TLS, so under the default sslmode (prefer) the replication session is relayed in plain text.""" psql( @@ -198,11 +331,14 @@ def main(): parser.add_argument("work", type=pathlib.Path) parser.add_argument("share") parser.add_argument("--odbc-driver") + parser.add_argument("--cursor-drivers", action="store_true") parser.add_argument("--cdc", action="store_true") arguments = parser.parse_args() files_checks(arguments.exe, arguments.work, arguments.share) if arguments.odbc_driver: odbc_checks(arguments.exe, arguments.work, arguments.odbc_driver) + if arguments.cursor_drivers: + cursor_checks(arguments.exe, arguments.work) if arguments.cdc: cdc_checks(arguments.exe, arguments.work) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index eeaa6bdc..8401d9a1 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -186,13 +186,31 @@ jobs: Add-Content -Path "$env:PGDATA\postgresql.conf" -Value "wal_level = logical" Set-Service -Name postgresql-x64-17 -StartupType Manual Start-Service -Name postgresql-x64-17 - - name: Install psqlODBC + # psqlODBC, Microsoft's ODBC Driver 18 for SQL Server and MySQL Connector/ODBC. + - name: Install the ODBC drivers + shell: pwsh + run: .github/scripts/install_odbc_drivers.ps1 + # The image's SQL Server LocalDB and MySQL 8.0, for the ODBC drivers' cursor checks. MySQL gets a + # data folder and port of its own, its root user without a password. + - name: Start SQL Server LocalDB and MySQL shell: pwsh run: | - $msi = "$env:RUNNER_TEMP\psqlodbc_x64.msi" - Invoke-WebRequest -Uri https://ftp.postgresql.org/pub/odbc/releases/REL-18_00_0006/psqlodbc_x64.msi -OutFile $msi - Start-Process msiexec.exe -ArgumentList "/i", $msi, "/qn", "/norestart" -Wait - Get-OdbcDriver -Name "PostgreSQL*" -Platform 64-bit | Format-Table Name, Platform + $localdb = (Get-Command SqlLocalDB.exe -ErrorAction SilentlyContinue).Source + if (-not $localdb) { $localdb = (Get-ChildItem "C:\Program Files\Microsoft SQL Server" -Recurse -Filter SqlLocalDB.exe | Select-Object -First 1).FullName } + & $localdb create sqlload -s + if ($LASTEXITCODE -ne 0) { throw "SqlLocalDB exited $LASTEXITCODE" } + $mysqld = (Get-ChildItem "C:\Program Files\MySQL" -Recurse -Filter mysqld.exe | Select-Object -First 1).FullName + & $mysqld --initialize-insecure --datadir=C:\mysql-data --console + if ($LASTEXITCODE -ne 0) { throw "mysqld --initialize-insecure exited $LASTEXITCODE" } + Start-Process $mysqld -ArgumentList "--datadir=C:\mysql-data", "--port=3307", "--mysqlx=OFF" -RedirectStandardError "$env:RUNNER_TEMP\mysqld.err" -RedirectStandardOutput "$env:RUNNER_TEMP\mysqld.out" + foreach ($attempt in 1..60) { + if ((Test-NetConnection 127.0.0.1 -Port 3307 -WarningAction SilentlyContinue).TcpTestSucceeded) { break } + Start-Sleep -Seconds 2 + } + if (-not (Test-NetConnection 127.0.0.1 -Port 3307 -WarningAction SilentlyContinue).TcpTestSucceeded) { + Get-Content "$env:RUNNER_TEMP\mysqld.err" + throw "MySQL never listened on 3307" + } - name: Share a folder as \\localhost\sqlload-e2e shell: pwsh run: | @@ -201,8 +219,8 @@ jobs: - name: End-to-end checks shell: pwsh run: | - python -m pip install --quiet duckdb - python .github/scripts/windows_e2e.py target\debug\sqlload.exe $env:E2E_DIR \\localhost\sqlload-e2e --odbc-driver "PostgreSQL Unicode(x64)" --cdc + python -m pip install --quiet duckdb pyodbc + python .github/scripts/windows_e2e.py target\debug\sqlload.exe $env:E2E_DIR \\localhost\sqlload-e2e --odbc-driver "PostgreSQL Unicode(x64)" --cursor-drivers --cdc # Postgres to DuckDB, 1M narrow rows, sqlload alone with a release build: a wall-time figure # in the job summary, not a gate (shared runners are too noisy to gate on time), so Verify From 4514dd30f32e7bb5da80064d64a79b0104d6c3a9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:50:34 +0000 Subject: [PATCH 14/25] docs(odbc): the tested drivers, OpenEdge's timestamp handling, and how cursor bounds are bound WIP for CHI-1236: the release version is checked before the pull request. --- .../src/content/docs/docs/sources/odbc.mdx | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/website/src/content/docs/docs/sources/odbc.mdx b/website/src/content/docs/docs/sources/odbc.mdx index c08ab95e..446508b1 100644 --- a/website/src/content/docs/docs/sources/odbc.mdx +++ b/website/src/content/docs/docs/sources/odbc.mdx @@ -77,24 +77,33 @@ against) plus the driver's package, such as `odbc-postgresql`. ## Drivers -SQLLoad writes names in ANSI double quotes (`"sales"."items"`). A driver that quotes them another -way is refused when the source is read; switch such a database to ANSI quoting, for example -with an `ANSI_QUOTES` SQL mode. +SQLLoad writes names in ANSI double quotes (`"sales"."items"`). A database that doesn't read them +as names is refused when the source is read; switch it to ANSI quoting. For MySQL, add +`INITSTMT=SET SESSION sql_mode=CONCAT(@@SESSION.sql_mode,',ANSI_QUOTES');` to the connection string. | Driver | Tested | |--------|--------| -| psqlODBC (`PostgreSQL Unicode`) | Yes, in containers. Add `BoolsAsChar=0` to the connection string so `boolean` columns load as `bool` rather than text. | +| psqlODBC (`PostgreSQL Unicode`) | Yes, against Postgres 17 in containers on Linux and on Windows. Add `BoolsAsChar=0` to the connection string so `boolean` columns load as `bool` rather than text. | +| Microsoft ODBC Driver 18 for SQL Server | Yes, against SQL Server 2022 in containers on Linux and against SQL Server LocalDB on Windows, `datetime2` and legacy `datetime` cursors included. | +| MySQL Connector/ODBC (`MySQL ODBC Unicode Driver`) | Yes, against MySQL 8.4 in containers on Linux and MySQL 8.0 on Windows, in an `ANSI_QUOTES` session (above). | +| OpenEdge (Progress) | Tested for timestamp handling (parsing `{ts}` values and binding timestamp bounds), not against a live server. | | Others | Not yet. A driver needs to describe prepared statements and list tables; run `sqlload plan` to see the types it reports. | -A cursor filter compares the cursor column with a quoted literal, such as -`"updated_at" >= '2026-03-01 10:00:05.123456'`, which the database converts to the column's type. -Postgres does; check that yours does too before relying on a cursor. From 0.30.0 (not released -yet), a numeric cursor compares with a bare number, such as `"id" > 41`. From 0.30.0 (not released -yet), a text cursor's bounds are bound parameters, such as `"code" > ?`, so the database's own -string-literal rules can't change them. Before that, on a database that reads a backslash in a -string as an escape, a text cursor maximum holding a backslash made -every later incremental load of the table fail, and other escapes could skip rows or read them -again. +From 0.35.0 (not released yet), a cursor filter compares the cursor column with bound parameters, +such as `"updated_at" > ?`, each passed as the cursor's own type: a timestamp without a time zone as +an ODBC `TIMESTAMP`, with every fractional digit; a date as a `DATE`; text as text. No database's +literal rules can change a bound, and a millisecond maximum such as `23:59:59.997` resumes exactly. +SQL Server's legacy `datetime` is the one exception: it stores 1/300 s ticks, which its driver reads +rounded to milliseconds, so its bounds go as text in the column's own precision, which SQL Server +converts to the column's type. A numeric cursor compares with a bare number, such as `"id" > 41`. +Any other cursor, a timestamp with a time zone for one, still compares with a quoted literal, +which the database converts to the column's type; check that yours does before relying on it. +`--start-cursor`, `--end-cursor` and `cursor_start` also take ODBC escape values, such as +`{ts '2026-01-31 23:59:59.997'}` or `{d '2026-01-31'}`. + +From 0.35.0 (not released yet), a timestamp column is read in microseconds whatever precision the +driver describes: MySQL Connector/ODBC describes every `DATETIME(n)` with none, and its fractional +seconds were dropped before. ## Choosing tables From 996e2e214986b89aa4d29a0439f6ef53cd6bb732 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:03:29 +0000 Subject: [PATCH 15/25] ci(publish): check Ctrl-C ends a Python-source load from each smoke-tested wheel A table function yields rows and then blocks; the load gets a real Ctrl-C (SIGINT, or a console Ctrl-C event on Windows) and must end at once as the native executable does, with no traceback and nothing committed, and the next load must write every row once. Commands now run on a thread of their own rather than Python's main thread, so this checks Ctrl-C still reaches them. --- .github/scripts/interrupt_python_load.py | 140 +++++++++++++++++++++++ .github/workflows/publish.yml | 2 + 2 files changed, 142 insertions(+) create mode 100644 .github/scripts/interrupt_python_load.py diff --git a/.github/scripts/interrupt_python_load.py b/.github/scripts/interrupt_python_load.py new file mode 100644 index 00000000..39e5c465 --- /dev/null +++ b/.github/scripts/interrupt_python_load.py @@ -0,0 +1,140 @@ +"""Ctrl-C during a Python-source load, through the installed `sqlload` package's console entry. + +Usage: python interrupt_python_load.py (with the Python that has the sqlload wheel installed) + +A table function yields five rows, then blocks. Once it is blocked, the load gets a real Ctrl-C: +SIGINT on Linux and macOS, and on Windows a console Ctrl-C event in the load's own console. The +load must end within a few seconds as Ctrl-C ends the native executable (the signal's exit status, +or STATUS_CONTROL_C_EXIT on Windows), print no Python traceback, and commit nothing; the next load +must then write every row once. The command runs on a thread of its own, not Python's main thread, +so this checks Ctrl-C still reaches it. +""" + +import os +import signal +import subprocess +import sys +import tempfile +import textwrap +import time +from pathlib import Path + +PROJECT = """\ +python_paths = ["sources"] + +[connections.lake] +kind = "filesystem" +root = "lake" + +[targets.lake] +connection = "lake" +path = "{table}/" +format = "csv" +""" + +MODULE = ''' +import time +from pathlib import Path + +import sqlload as sl + +HERE = Path(__file__).resolve().parent.parent + +shop = sl.source("shop") + + +@shop.table(description="Rows that block on the first load.") +def slow(ctx): + for n in range(5): + yield {"id": n, "name": f"row {n}"} + if not (HERE / "release").exists(): + (HERE / "blocked").write_text("blocked") + time.sleep(120) + +''' + +#: `sqlload load`, run as the wheel's console script runs it. +LOAD = [sys.executable, "-c", "from sqlload._cli import main; main()", "load"] + +#: How Windows ends a process on Ctrl-C when nothing handles it. +STATUS_CONTROL_C_EXIT = 0xC000013A + +#: Attaches to the load's console and sends it a Ctrl-C, ignoring the event itself. +WINDOWS_CTRL_C = """ +import ctypes, sys +kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) +kernel32.FreeConsole() +if not kernel32.AttachConsole(int(sys.argv[1])): + sys.exit(f"AttachConsole failed: {ctypes.get_last_error()}") +kernel32.SetConsoleCtrlHandler(None, True) +if not kernel32.GenerateConsoleCtrlEvent(0, 0): + sys.exit(f"GenerateConsoleCtrlEvent failed: {ctypes.get_last_error()}") +""" + + +def start_load(project): + flags = subprocess.CREATE_NEW_CONSOLE if os.name == "nt" else 0 + return subprocess.Popen(LOAD, cwd=project, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8", errors="replace", creationflags=flags) + + +def ctrl_c(process): + if os.name == "nt": + subprocess.run([sys.executable, "-c", WINDOWS_CTRL_C, str(process.pid)], check=True) + else: + process.send_signal(signal.SIGINT) + + +def interrupted_exit(returncode): + if os.name == "nt": + return returncode in (STATUS_CONTROL_C_EXIT, STATUS_CONTROL_C_EXIT - 2**32) + return returncode in (-signal.SIGINT, 128 + signal.SIGINT) + + +def written_rows(project): + lines = [line for path in sorted((project / "lake").rglob("*.csv")) for line in path.read_text(encoding="utf-8").splitlines()[1:]] + return sorted(int(line.split(",")[0]) for line in lines) + + +def check(condition, what, output=""): + if not condition: + sys.exit(f"{what}\n{output}") + print(f"ok: {what}", flush=True) + + +def main(): + sys.stdout.reconfigure(encoding="utf-8") + with tempfile.TemporaryDirectory() as folder: + project = Path(folder) + (project / "sources").mkdir() + (project / "lake").mkdir() + (project / "sqlload.toml").write_text(PROJECT, encoding="utf-8") + (project / "sources" / "shop.py").write_text(textwrap.dedent(MODULE), encoding="utf-8") + process = start_load(project) + deadline = time.monotonic() + 120 + while not (project / "blocked").exists(): + if process.poll() is not None or time.monotonic() > deadline: + process.kill() + out, err = process.communicate() + sys.exit(f"the load never reached the blocked table function (exit {process.returncode})\n{out}{err}") + time.sleep(0.2) + time.sleep(1) + ctrl_c(process) + sent = time.monotonic() + try: + out, err = process.communicate(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + out, err = process.communicate() + sys.exit(f"the load was still running 15 s after Ctrl-C\n{out}{err}") + check(time.monotonic() - sent < 5, f"Ctrl-C ended the load within 5 s ({time.monotonic() - sent:.1f} s)", out + err) + check(interrupted_exit(process.returncode), f"the load ended as Ctrl-C ends it (exit {process.returncode})", out + err) + check("Traceback" not in err, "no Python traceback", err) + check(written_rows(project) == [], "the interrupted load committed nothing", str(written_rows(project))) + (project / "release").write_text("release") + done = subprocess.run(LOAD, cwd=project, check=False, capture_output=True, text=True, encoding="utf-8", errors="replace") + check(done.returncode == 0, "the next load succeeds", done.stdout + done.stderr) + check(written_rows(project) == [0, 1, 2, 3, 4], "the next load writes every row once", str(written_rows(project))) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4503a2ef..8385bb65 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,6 +13,7 @@ on: paths: - .github/workflows/publish.yml - .github/scripts/assert_*_wheel_tags.sh + - .github/scripts/interrupt_python_load.py - packages/*/pyproject.toml - Cargo.toml - Cargo.lock @@ -173,6 +174,7 @@ jobs: "${bin}/python" -m pip install --quiet dist/sqlload-*.whl "${bin}/sqlload" --help "${bin}/python" -c "import sqlload, sqlload._native" + "${bin}/python" .github/scripts/interrupt_python_load.py python -m venv smoke-cli cli=smoke-cli/bin if [[ "${RUNNER_OS}" == "Windows" ]]; then cli=smoke-cli/Scripts; fi From 4e55202e1afe47c9b77ec9c4bab46f3872e8b75c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:11:29 +0000 Subject: [PATCH 16/25] fix(loader): allow ReadPart's bound parameters to go unread in builds without ODBC --- crates/sqlload/src/loader/_helpers/sources.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/sqlload/src/loader/_helpers/sources.rs b/crates/sqlload/src/loader/_helpers/sources.rs index 47cb6be2..5d37afbc 100644 --- a/crates/sqlload/src/loader/_helpers/sources.rs +++ b/crates/sqlload/src/loader/_helpers/sources.rs @@ -1142,6 +1142,10 @@ pub(crate) struct ReadPart { /// The Postgres filter; `None` reads the plan's. pub(crate) filter: Option, /// The values bound to `filter`'s `?` placeholders, in order. + #[cfg_attr( + not(feature = "odbc"), + expect(dead_code, reason = "only ODBC sources bind parameters") + )] pub(crate) parameters: Vec, /// `ORDER BY` this quoted column. pub(crate) order_by: Option, From 839b94d3d89ec96b74bbd66dac1f1950ab3a7aba Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:27:06 +0000 Subject: [PATCH 17/25] fix(files): open a Windows share on localhost through 127.0.0.1 object_store keeps a local store's root as a file:// URL, and the url crate reads file://localhost/share/... back as a drive path, so every file on a \\localhost\ share failed to convert ("Unable to convert URL ... to filesystem path"). The store now opens such a share as \\127.0.0.1\share, the same server; what SQLLoad prints keeps the path as written. The Windows end-to-end checks also write through the machine's own name, as a share on another server is named. --- .github/scripts/windows_e2e.py | 7 +++ .../src/file_store/main/open_folder.rs | 4 +- .../src/store_paths/_helpers/local_root.rs | 18 ++++++ .../src/store_paths/main/local_store_root.rs | 10 +++ crates/sqlload/src/store_paths/main/mod.rs | 1 + .../src/store_paths/tests/test_local_root.rs | 62 ++++++++++++++++++- .../src/store_paths/tests/test_types.rs | 7 +++ 7 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 crates/sqlload/src/store_paths/main/local_store_root.rs diff --git a/.github/scripts/windows_e2e.py b/.github/scripts/windows_e2e.py index c841bd3f..a722ab7d 100644 --- a/.github/scripts/windows_e2e.py +++ b/.github/scripts/windows_e2e.py @@ -64,6 +64,13 @@ def files_checks(exe, work, share): sys.exit(f"no Parquet files under {lake}") print(f"ok: UNC share holds {written}", flush=True) + # The same share named by this machine's name, as a share on another server is named. + named = "\\\\" + os.environ["COMPUTERNAME"] + "\\" + share.lstrip("\\").split("\\", 1)[1] + "\\lake\\by_name" + sqlload(exe, "copy", str(source), named, "--format", "csv") + if not list((work / "share" / "lake" / "by_name").rglob("*.parquet")): + sys.exit(f"no Parquet files under {named}") + print(f"ok: {named} holds Parquet files", flush=True) + # Read back through a file:// URL naming the share, and through a \\?\UNC\ path. server_share = share.lstrip("\\").replace("\\", "/") sqlload(exe, "copy", f"file://{server_share}/lake/orders", f"duckdb://{database}#raw.from_share", "--format", "parquet") diff --git a/crates/sqlload/src/file_store/main/open_folder.rs b/crates/sqlload/src/file_store/main/open_folder.rs index 21fe9ef7..3141e8f3 100644 --- a/crates/sqlload/src/file_store/main/open_folder.rs +++ b/crates/sqlload/src/file_store/main/open_folder.rs @@ -9,6 +9,7 @@ use crate::file_store::errors::StoreError; use crate::file_store::main::s3_builder::s3_builder; use crate::file_store::models::{OpenFolder, SftpStore}; use crate::http::main::install_tls_provider::install_tls_provider; +use crate::store_paths::main::local_store_root::local_store_root; use crate::store_paths::models::{Location, StoreKind}; /// A client for `location`'s store (an SFTP server is connected to on first use); S3 credentials come from `AWS_*` variables, the shared credentials file, web identity or the instance/container role. @@ -47,7 +48,8 @@ pub fn open_folder(location: &Location) -> Result { source: Box::new(error), }) })?; - Arc::new(LocalFileSystem::new_with_prefix(root).map_err(failed)?) + let root = local_store_root(root); + Arc::new(LocalFileSystem::new_with_prefix(root.as_ref()).map_err(failed)?) } StoreKind::Sftp { server, root } => Arc::new(SftpStore { server: (**server).clone(), diff --git a/crates/sqlload/src/store_paths/_helpers/local_root.rs b/crates/sqlload/src/store_paths/_helpers/local_root.rs index 520a141e..49a4d215 100644 --- a/crates/sqlload/src/store_paths/_helpers/local_root.rs +++ b/crates/sqlload/src/store_paths/_helpers/local_root.rs @@ -56,6 +56,24 @@ pub(crate) fn windows_file_url_path(rest: &str, windows: bool) -> Cow<'_, str> { } } +/// `root` with a share on `localhost` named by `127.0.0.1`, the same server, when `windows` (object_store keeps its root as a URL, and url reads `file://localhost/share` as a drive path); unchanged otherwise. +pub(crate) fn share_by_address(root: &str, windows: bool) -> Cow<'_, str> { + if !windows { + return Cow::Borrowed(root); + } + let separated = windows_separators(root, windows); + let on_localhost = separated.strip_prefix("//").and_then(|share| { + share + .get(.."localhost/".len()) + .filter(|host| host.eq_ignore_ascii_case("localhost/")) + .map(|host| &share[host.len()..]) + }); + match on_localhost { + Some(rest) => Cow::Owned(format!(r"\\127.0.0.1\{}", rest.replace('/', r"\"))), + None => Cow::Borrowed(root), + } +} + /// Whether `path` starts with a drive letter and its colon. fn is_drive(path: &str) -> bool { matches!(path.as_bytes(), [letter, b':', ..] if letter.is_ascii_alphabetic()) diff --git a/crates/sqlload/src/store_paths/main/local_store_root.rs b/crates/sqlload/src/store_paths/main/local_store_root.rs new file mode 100644 index 00000000..659532b3 --- /dev/null +++ b/crates/sqlload/src/store_paths/main/local_store_root.rs @@ -0,0 +1,10 @@ +//! The root a local object store opens a filesystem folder at. + +use std::borrow::Cow; + +use crate::store_paths::_helpers::local_root::share_by_address; + +/// `root` as object_store's local store can open it: on Windows a share on `localhost` is named by `127.0.0.1`, the same server; unchanged otherwise. +pub fn local_store_root(root: &str) -> Cow<'_, str> { + share_by_address(root, cfg!(windows)) +} diff --git a/crates/sqlload/src/store_paths/main/mod.rs b/crates/sqlload/src/store_paths/main/mod.rs index 82df1fed..544cd821 100644 --- a/crates/sqlload/src/store_paths/main/mod.rs +++ b/crates/sqlload/src/store_paths/main/mod.rs @@ -2,6 +2,7 @@ pub mod escape_partition_value; pub mod file_url_path; pub mod hive_values; pub mod local_separators; +pub mod local_store_root; pub mod render_template; pub mod resolve_location; pub mod unescape_partition_value; diff --git a/crates/sqlload/src/store_paths/tests/test_local_root.rs b/crates/sqlload/src/store_paths/tests/test_local_root.rs index 91d99a67..5d82e9f2 100644 --- a/crates/sqlload/src/store_paths/tests/test_local_root.rs +++ b/crates/sqlload/src/store_paths/tests/test_local_root.rs @@ -1,7 +1,9 @@ -use crate::store_paths::_helpers::local_root::{strip_folders, windows_file_url_path}; +use crate::store_paths::_helpers::local_root::{ + share_by_address, strip_folders, windows_file_url_path, +}; use crate::store_paths::tests::helpers::root_of; use crate::store_paths::tests::test_types::{ - FileUrlPathTestCase, LocalRootTestCase, StripFoldersTestCase, + FileUrlPathTestCase, LocalRootTestCase, ShareByAddressTestCase, StripFoldersTestCase, }; #[test] @@ -181,3 +183,59 @@ fn given_a_folder_prefix_when_stripping_a_connections_folders_then_windows_ignor ); } } + +#[test] +fn given_filesystem_roots_when_opening_their_store_then_only_a_windows_share_on_localhost_is_named_by_address() + { + let test_cases = [ + ShareByAddressTestCase { + description: "a share on localhost", + root: r"\\localhost\sqlload-e2e\lake\orders", + expected_windows: r"\\127.0.0.1\sqlload-e2e\lake\orders", + }, + ShareByAddressTestCase { + description: "localhost in another case", + root: r"\\LocalHost\exports", + expected_windows: r"\\127.0.0.1\exports", + }, + ShareByAddressTestCase { + description: "a share on localhost as a file:// URL gives it", + root: "//localhost/exports/x", + expected_windows: r"\\127.0.0.1\exports\x", + }, + ShareByAddressTestCase { + description: "an extended-length share on localhost", + root: r"\\?\UNC\localhost\exports\x", + expected_windows: r"\\127.0.0.1\exports\x", + }, + ShareByAddressTestCase { + description: "a server whose name starts with localhost", + root: r"\\localhostname\exports\x", + expected_windows: r"\\localhostname\exports\x", + }, + ShareByAddressTestCase { + description: "a share on another server", + root: r"\\nas01\exports\x", + expected_windows: r"\\nas01\exports\x", + }, + ShareByAddressTestCase { + description: "a drive folder named localhost", + root: r"C:\localhost\x", + expected_windows: r"C:\localhost\x", + }, + ]; + for test_case in test_cases { + assert_eq!( + share_by_address(test_case.root, true), + test_case.expected_windows, + "{} on Windows", + test_case.description + ); + assert_eq!( + share_by_address(test_case.root, false), + test_case.root, + "{} on Unix", + test_case.description + ); + } +} diff --git a/crates/sqlload/src/store_paths/tests/test_types.rs b/crates/sqlload/src/store_paths/tests/test_types.rs index 5864cd2a..85008233 100644 --- a/crates/sqlload/src/store_paths/tests/test_types.rs +++ b/crates/sqlload/src/store_paths/tests/test_types.rs @@ -58,3 +58,10 @@ pub(crate) struct StripFoldersTestCase { pub(crate) expected_windows: Option<&'static str>, pub(crate) expected_unix: Option<&'static str>, } + +/// A filesystem root, as object_store's local store opens it on Windows and on Unix. +pub(crate) struct ShareByAddressTestCase { + pub(crate) description: &'static str, + pub(crate) root: &'static str, + pub(crate) expected_windows: &'static str, +} From 53d8112f9ed580736aaa2a5e3bbeac114841d880 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:35:16 +0000 Subject: [PATCH 18/25] ci(odbc): install MySQL Connector/ODBC from MySQL's package for the distribution The generic glibc tarball loads its own newer libssl.so.3, which fails with OPENSSL_3.2.0 not found in a process where msodbcsql18 or psqlODBC already loaded Ubuntu 24.04's OpenSSL 3.0, as the container tests do. MySQL's package for the release links the system OpenSSL. The docs say so for users, and the site's check for unshipped systems no longer reads the ODBC page, which names the drivers SQLLoad is tested with. --- .github/scripts/install_odbc_drivers.sh | 37 ++++++++++++------- website/scripts/check-content.mjs | 4 +- .../src/content/docs/docs/sources/odbc.mdx | 2 +- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/.github/scripts/install_odbc_drivers.sh b/.github/scripts/install_odbc_drivers.sh index 79e75ba7..f8968705 100755 --- a/.github/scripts/install_odbc_drivers.sh +++ b/.github/scripts/install_odbc_drivers.sh @@ -3,13 +3,12 @@ # and registers each driver under the name the tests connect with: # PostgreSQL Unicode psqlODBC, from the distribution # ODBC Driver 18 for SQL Server Microsoft's msodbcsql18 (installing it accepts Microsoft's licence) -# MySQL ODBC Unicode Driver MySQL Connector/ODBC, from MySQL's generic Linux tarball +# MySQL ODBC Unicode Driver MySQL Connector/ODBC, MySQL's package for this distribution release # Usage: install_odbc_drivers.sh [--with-dev] (--with-dev adds the unixODBC headers to build against) set -euo pipefail export DEBIAN_FRONTEND=noninteractive MYSQL_ODBC_VERSION=26.10.0 -MYSQL_ODBC_MD5=09f9e895d60ce69bd1e6a576df9dd4de MYSQL_ODBC_NAME="MySQL ODBC Unicode Driver" sudo_() { if [ "$(id -u)" = 0 ]; then "$@"; else sudo "$@"; fi; } @@ -31,18 +30,28 @@ if ! dpkg -s msodbcsql18 > /dev/null 2>&1; then sudo_ env ACCEPT_EULA=Y apt-get install -y -qq msodbcsql18 fi -# MySQL Connector/ODBC: the pinned tarball (the CDN keeps the current release, the archive older -# ones), checked against MySQL's published MD5, then its Unicode driver registered under a name -# that doesn't change with the version. -tarball="mysql-connector-odbc-${MYSQL_ODBC_VERSION}-linux-glibc2.28-x86-64bit.tar.gz" -work="$(mktemp -d)" -curl -fsSL -o "$work/$tarball" "https://cdn.mysql.com/Downloads/Connector-ODBC/${MYSQL_ODBC_VERSION%.*}/$tarball" || - curl -fsSL -o "$work/$tarball" "https://downloads.mysql.com/archives/get/p/10/file/$tarball" -echo "$MYSQL_ODBC_MD5 $work/$tarball" | md5sum -c --quiet - -sudo_ mkdir -p /opt/mysql-connector-odbc -sudo_ tar -xzf "$work/$tarball" -C /opt/mysql-connector-odbc --strip-components=1 -rm -rf "$work" -driver="$(ls /opt/mysql-connector-odbc/lib/libmyodbc*w.so)" +# MySQL Connector/ODBC: MySQL's package built for this distribution release, which links the system +# OpenSSL. Its generic tarball carries a newer private libssl.so.3, which can't load in a process +# where another driver already loaded the system one (OPENSSL_3.2.0 not found on Ubuntu 24.04). +# The SHA-256s are of the packages as cdn.mysql.com served them over TLS. The Unicode driver is +# registered under a name that doesn't change with the version. +case "${ID}${VERSION_ID}" in + ubuntu24.04) mysql_sha256=b99d26f320ca640d24d8a82bdaea33e41f1ce9bb16cbea5b44ee3b7547bc5673 ;; + ubuntu22.04) mysql_sha256=2e89b5ca8149ba77c24e0828597c85d160ff66fe57aa81873c742a152a747c65 ;; + debian13) mysql_sha256=20b506e3b44271fb3b0276333ae6595b9cecf6c2bdf888a8d21ff421c54bbdc9 ;; + debian12) mysql_sha256=119d0a5413a97e155a886df03cc01921a3294ff0cb02569c1d8e11642f4c0b7d ;; + *) echo "no pinned MySQL Connector/ODBC package for ${ID} ${VERSION_ID}" >&2; exit 1 ;; +esac +if ! dpkg -s mysql-connector-odbc > /dev/null 2>&1; then + package="mysql-connector-odbc_${MYSQL_ODBC_VERSION}-1${ID}${VERSION_ID}_amd64.deb" + work="$(mktemp -d)" + curl -fsSL -o "$work/$package" "https://cdn.mysql.com/Downloads/Connector-ODBC/${MYSQL_ODBC_VERSION%.*}/$package" || + curl -fsSL -o "$work/$package" "https://downloads.mysql.com/archives/get/p/10/file/$package" + echo "$mysql_sha256 $work/$package" | sha256sum -c --quiet - + sudo_ apt-get install -y -qq "$work/$package" + rm -rf "$work" +fi +driver="$(ls /usr/lib/x86_64-linux-gnu/odbc/libmyodbc*w.so)" printf '[%s]\nDescription=MySQL Connector/ODBC %s\nDriver=%s\nUsageCount=1\n' "$MYSQL_ODBC_NAME" "$MYSQL_ODBC_VERSION" "$driver" > /tmp/mysql-odbc.ini sudo_ odbcinst -i -d -r -f /tmp/mysql-odbc.ini > /dev/null rm -f /tmp/mysql-odbc.ini diff --git a/website/scripts/check-content.mjs b/website/scripts/check-content.mjs index 0e7e79b3..21fdf0cd 100644 --- a/website/scripts/check-content.mjs +++ b/website/scripts/check-content.mjs @@ -22,7 +22,9 @@ for (const page of pages) { const slug = relative(docs, page).replace(/\.mdx$/, '').replace(/\/index$/, ''); if (!listed.has(slug)) problems.push(`${relative(website, page)}: not in src/sidebar.json`); } -for (const path of [...pages, join(website, 'public/index.html')]) { +// The ODBC source page names the drivers SQLLoad is tested with, which reach these systems through ODBC. +const odbcPage = join(docs, 'docs/sources/odbc.mdx'); +for (const path of [...pages.filter((page) => page !== odbcPage), join(website, 'public/index.html')]) { readFileSync(path, 'utf8') .split('\n') .forEach((line, index) => { diff --git a/website/src/content/docs/docs/sources/odbc.mdx b/website/src/content/docs/docs/sources/odbc.mdx index 446508b1..66256b66 100644 --- a/website/src/content/docs/docs/sources/odbc.mdx +++ b/website/src/content/docs/docs/sources/odbc.mdx @@ -85,7 +85,7 @@ as names is refused when the source is read; switch it to ANSI quoting. For MySQ |--------|--------| | psqlODBC (`PostgreSQL Unicode`) | Yes, against Postgres 17 in containers on Linux and on Windows. Add `BoolsAsChar=0` to the connection string so `boolean` columns load as `bool` rather than text. | | Microsoft ODBC Driver 18 for SQL Server | Yes, against SQL Server 2022 in containers on Linux and against SQL Server LocalDB on Windows, `datetime2` and legacy `datetime` cursors included. | -| MySQL Connector/ODBC (`MySQL ODBC Unicode Driver`) | Yes, against MySQL 8.4 in containers on Linux and MySQL 8.0 on Windows, in an `ANSI_QUOTES` session (above). | +| MySQL Connector/ODBC (`MySQL ODBC Unicode Driver`) | Yes, against MySQL 8.4 in containers on Linux and MySQL 8.0 on Windows, in an `ANSI_QUOTES` session (above). On Linux, install MySQL's package for your distribution rather than the generic glibc tarball: the tarball's own OpenSSL fails to load once another driver, such as SQL Server's or psqlODBC, has loaded the system one in the same run. | | OpenEdge (Progress) | Tested for timestamp handling (parsing `{ts}` values and binding timestamp bounds), not against a live server. | | Others | Not yet. A driver needs to describe prepared statements and list tables; run `sqlload plan` to see the types it reports. | From 364297712b440256c3d4b98c12c117222eb9e7f4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:18:58 +0000 Subject: [PATCH 19/25] ci(publish): deliver the Windows Ctrl-C check's event as an interactive console does, with a control The Windows wheel's Ctrl-C check found the load still running 15 s after the event. A plain load installs no Ctrl-C handler of its own, and a Windows child inherits its parent's "ignore Ctrl-C" flag, which the runner's processes may carry. Each interrupted child now clears that flag first, as a process in an interactive console has it clear. Before the load, a plain Python process with Ctrl-C's default action gets the same event: once as the runner starts it (printed) and once with the flag cleared (it must end), so a failing load is the load's fault, not the check's. --- .github/scripts/interrupt_python_load.py | 39 ++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/.github/scripts/interrupt_python_load.py b/.github/scripts/interrupt_python_load.py index 39e5c465..9afbcd07 100644 --- a/.github/scripts/interrupt_python_load.py +++ b/.github/scripts/interrupt_python_load.py @@ -8,6 +8,10 @@ or STATUS_CONTROL_C_EXIT on Windows), print no Python traceback, and commit nothing; the next load must then write every row once. The command runs on a thread of its own, not Python's main thread, so this checks Ctrl-C still reaches it. + +On Windows, a plain Python process gets the same Ctrl-C first. How it ends without clearing an inherited +"ignore Ctrl-C" flag is printed, and it must end once it clears the flag, so a failure of the load is +the load's, not this check's. """ import os @@ -53,9 +57,19 @@ def slow(ctx): ''' +#: On Windows, a child inherits its parent's "ignore Ctrl-C" flag, which a CI runner's processes may +#: carry; a process in an interactive console has it clear, so each interrupted child clears it first. +ACCEPT_CTRL_C = "import ctypes; ctypes.windll.kernel32.SetConsoleCtrlHandler(None, False); " if os.name == "nt" else "" + #: `sqlload load`, run as the wheel's console script runs it. LOAD = [sys.executable, "-c", "from sqlload._cli import main; main()", "load"] +#: The same load, as a process in an interactive console runs it. +INTERRUPTED_LOAD = [sys.executable, "-c", ACCEPT_CTRL_C + "from sqlload._cli import main; main()", "load"] + +#: A Python process doing nothing, with Ctrl-C's default action as `sqlload._cli` sets it. +SLEEPER = "import signal, time; signal.signal(signal.SIGINT, signal.SIG_DFL); time.sleep(120)" + #: How Windows ends a process on Ctrl-C when nothing handles it. STATUS_CONTROL_C_EXIT = 0xC000013A @@ -72,9 +86,23 @@ def slow(ctx): """ -def start_load(project): +def start(command, cwd=None): flags = subprocess.CREATE_NEW_CONSOLE if os.name == "nt" else 0 - return subprocess.Popen(LOAD, cwd=project, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8", errors="replace", creationflags=flags) + return subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8", errors="replace", creationflags=flags) + + +def interrupt_sleeper(code): + """Ctrl-C a Python process running `code`, once it has started; whether it ended as Ctrl-C ends it, and how.""" + process = start([sys.executable, "-c", code]) + time.sleep(3) + ctrl_c(process) + try: + process.communicate(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.communicate() + return False, "still running 10 s after Ctrl-C" + return interrupted_exit(process.returncode), f"exit {process.returncode}" def ctrl_c(process): @@ -109,7 +137,12 @@ def main(): (project / "lake").mkdir() (project / "sqlload.toml").write_text(PROJECT, encoding="utf-8") (project / "sources" / "shop.py").write_text(textwrap.dedent(MODULE), encoding="utf-8") - process = start_load(project) + if os.name == "nt": + ended, how = interrupt_sleeper(SLEEPER) + print(f"note: a Python process with Ctrl-C's default action, as the runner starts it: {how}", flush=True) + ended, how = interrupt_sleeper(ACCEPT_CTRL_C + SLEEPER) + check(ended, f"Ctrl-C reaches a Python process that accepts it, so this check can deliver it ({how})") + process = start(INTERRUPTED_LOAD, project) deadline = time.monotonic() + 120 while not (project / "blocked").exists(): if process.poll() is not None or time.monotonic() > deadline: From 20508dda231bf4beba0910a30e51ee548294b60d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:22:52 +0000 Subject: [PATCH 20/25] fix(odbc): ISO 8601 text for legacy datetime bounds, ODBC escapes only on time cursors, literals for pinned cursors - A SQL Server legacy datetime bound sent as text now uses the T separator. With a space, SQL Server reads month and day by the login's language, so a british session read a 2026-01-05 maximum as 1 May and the next load silently skipped every row up to it. The live legacy-datetime case now connects with Language=British and a 5 January maximum, and fails with the old format. - When the driver can't describe a SQL Server timestamp parameter, the bound goes as seven-digit ISO text: exact for datetime2, refused by datetime, rather than a datetime2 bind that misreads it. - {ts '...'} and {d '...'} are read only for timestamp and date cursors; a text cursor keeps such a value as it is. - A cursor column pinned to a type compares with a quoted literal again, as the pinned type may not be the source column's own. - The catalog asks for every schema and falls back to no schema pattern only when that is refused and the driver reports no schema name length, so a driver reporting 0 as no maximum still lists its schemas. - Linux installs msodbcsql18 18.7.1.1, the release Windows installs, and registers MySQL's driver file by its exact name. --- .github/scripts/install_odbc_drivers.sh | 8 +++--- .../src/adapters/contract/main/cursor_text.rs | 9 +++++-- .../contract/tests/test_cursor_text.rs | 6 +++++ crates/sqlload/src/loader/_helpers/publish.rs | 7 ++--- .../sqlload/src/loader/_helpers/table_plan.rs | 18 ++++++++++--- .../src/odbc_source/_helpers/catalog.rs | 12 +++++---- .../src/odbc_source/_helpers/parameters.rs | 16 ++++++----- crates/sqlload/src/odbc_source/constants.rs | 3 +++ .../odbc_source/tests/test_timestamp_text.rs | 16 +++++++---- .../sqlload/tests/loader/test_odbc_drivers.rs | 27 +++++++++++++------ crates/sqlload/tests/loader/test_types.rs | 2 ++ .../src/content/docs/docs/sources/odbc.mdx | 14 ++++++---- 12 files changed, 95 insertions(+), 43 deletions(-) diff --git a/.github/scripts/install_odbc_drivers.sh b/.github/scripts/install_odbc_drivers.sh index f8968705..9ebe1786 100755 --- a/.github/scripts/install_odbc_drivers.sh +++ b/.github/scripts/install_odbc_drivers.sh @@ -2,13 +2,14 @@ # Installs unixODBC and the ODBC drivers the container tests read through, on Debian or Ubuntu x86_64, # and registers each driver under the name the tests connect with: # PostgreSQL Unicode psqlODBC, from the distribution -# ODBC Driver 18 for SQL Server Microsoft's msodbcsql18 (installing it accepts Microsoft's licence) +# ODBC Driver 18 for SQL Server Microsoft's msodbcsql18, the same release as on Windows (installing it accepts Microsoft's licence) # MySQL ODBC Unicode Driver MySQL Connector/ODBC, MySQL's package for this distribution release # Usage: install_odbc_drivers.sh [--with-dev] (--with-dev adds the unixODBC headers to build against) set -euo pipefail export DEBIAN_FRONTEND=noninteractive MYSQL_ODBC_VERSION=26.10.0 +MSODBCSQL_VERSION=18.7.1.1-1 MYSQL_ODBC_NAME="MySQL ODBC Unicode Driver" sudo_() { if [ "$(id -u)" = 0 ]; then "$@"; else sudo "$@"; fi; } @@ -27,7 +28,7 @@ if ! dpkg -s msodbcsql18 > /dev/null 2>&1; then sudo_ dpkg -i /tmp/packages-microsoft-prod.deb > /dev/null rm -f /tmp/packages-microsoft-prod.deb sudo_ apt-get update -qq - sudo_ env ACCEPT_EULA=Y apt-get install -y -qq msodbcsql18 + sudo_ env ACCEPT_EULA=Y apt-get install -y -qq "msodbcsql18=${MSODBCSQL_VERSION}" fi # MySQL Connector/ODBC: MySQL's package built for this distribution release, which links the system @@ -51,7 +52,8 @@ if ! dpkg -s mysql-connector-odbc > /dev/null 2>&1; then sudo_ apt-get install -y -qq "$work/$package" rm -rf "$work" fi -driver="$(ls /usr/lib/x86_64-linux-gnu/odbc/libmyodbc*w.so)" +driver="/usr/lib/x86_64-linux-gnu/odbc/libmyodbc${MYSQL_ODBC_VERSION%%.*}w.so" +test -f "$driver" printf '[%s]\nDescription=MySQL Connector/ODBC %s\nDriver=%s\nUsageCount=1\n' "$MYSQL_ODBC_NAME" "$MYSQL_ODBC_VERSION" "$driver" > /tmp/mysql-odbc.ini sudo_ odbcinst -i -d -r -f /tmp/mysql-odbc.ini > /dev/null rm -f /tmp/mysql-odbc.ini diff --git a/crates/sqlload/src/adapters/contract/main/cursor_text.rs b/crates/sqlload/src/adapters/contract/main/cursor_text.rs index 1548c603..4b23619b 100644 --- a/crates/sqlload/src/adapters/contract/main/cursor_text.rs +++ b/crates/sqlload/src/adapters/contract/main/cursor_text.rs @@ -5,9 +5,14 @@ use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime}; use crate::adapters::contract::constants::{CURSOR_TIMESTAMP_FORMAT, NAIVE_TIMESTAMP_FORMATS}; use crate::column_types::models::ColumnType; -/// `text` as the cursor literal every system reads alike: zoned instants in UTC with `+00`, wall-clock times without an offset; an ODBC `{ts '...'}` or `{d '...'}` literal, as a query against an ODBC source writes it, reads as the value inside. +/// `text` as the cursor literal every system reads alike: zoned instants in UTC with `+00`, wall-clock times without an offset; for a timestamp or date cursor, an ODBC `{ts '...'}` or `{d '...'}` literal, as a query against an ODBC source writes it, reads as the value inside (any other cursor keeps it as given). pub fn cursor_text(text: &str, column_type: &ColumnType) -> Result { - let text = odbc_value(text).unwrap_or(text); + let text = match column_type { + ColumnType::TimestampTz | ColumnType::TimestampNtz | ColumnType::Date => { + odbc_value(text).unwrap_or(text) + } + _ => text, + }; let zoned = DateTime::parse_from_rfc3339(text); let naive = NAIVE_TIMESTAMP_FORMATS .iter() diff --git a/crates/sqlload/src/adapters/contract/tests/test_cursor_text.rs b/crates/sqlload/src/adapters/contract/tests/test_cursor_text.rs index bd85df09..52527cfc 100644 --- a/crates/sqlload/src/adapters/contract/tests/test_cursor_text.rs +++ b/crates/sqlload/src/adapters/contract/tests/test_cursor_text.rs @@ -59,6 +59,12 @@ fn given_a_cursor_flag_when_normalising_then_every_system_reads_the_same_bound() column_type: ColumnType::TimestampNtz, expected_text: Err("is not an RFC 3339 timestamp or a date"), }, + CursorTextTestCase { + description: "a text cursor keeps a value that looks like an ODBC literal as it is", + text: "{d 'abc'}", + column_type: ColumnType::Text, + expected_text: Ok("{d 'abc'}"), + }, CursorTextTestCase { description: "a date cursor needs a date", text: "2026-11-04T10:00:00Z", diff --git a/crates/sqlload/src/loader/_helpers/publish.rs b/crates/sqlload/src/loader/_helpers/publish.rs index fc46a2ec..b8791b99 100644 --- a/crates/sqlload/src/loader/_helpers/publish.rs +++ b/crates/sqlload/src/loader/_helpers/publish.rs @@ -10,7 +10,6 @@ use crate::adapters::contract::models::WriteStatementsRequest; use crate::adapters::contract::models::WriteTarget; use crate::adapters::contract::models::{DirectBatch, DirectLoaded, UnloadedBatch}; use crate::adapters::contract::types::TableWriter; -use crate::adapters::dispatch::main::dialect::source_binds_text::source_binds_text; use crate::adapters::dispatch::main::dialect::source_dialect::source_dialect; use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter; use crate::adapters::dispatch::main::route::sends_rows::sends_rows; @@ -69,7 +68,7 @@ use crate::loader::_helpers::sources::files_source_parts; use crate::loader::_helpers::sources::nanosecond_columns; use crate::loader::_helpers::sources::read_source; use crate::loader::_helpers::sources::unload_source; -use crate::loader::_helpers::table_plan::source_filter; +use crate::loader::_helpers::table_plan::{binds_cursor, source_filter}; use crate::loader::_helpers::table_plan::{replan_columns, schema_statements}; use crate::loader::constants::EXCLUSIVE_BOUND; use crate::loader::constants::INCLUSIVE_BOUND; @@ -1126,9 +1125,7 @@ fn window_filter( window.from.as_deref().map(|from| (operator, from)), window.to.as_deref(), ), - source - .is_some_and(|source| source_binds_text(source.system)) - .then_some(&mut parameters), + binds_cursor(&table.spec).then_some(&mut parameters), ); (filter, parameters) } diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index 7558ae61..b50af79d 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -1045,6 +1045,20 @@ pub(crate) fn source_filter( (!bounds.is_empty()).then(|| bounds.join(" AND ")) } +/// Whether `spec`'s cursor bounds bind as parameters of the cursor's type: on a source that binds them (ODBC), unless the cursor column is pinned to a type, which may not be the source column's own (a `varchar` read as a timestamp), so its bounds stay literals the database converts. +pub(crate) fn binds_cursor(spec: &TableSpec) -> bool { + let pinned = spec.cursor.as_ref().is_some_and(|cursor| { + spec.pinned + .keys() + .any(|name| name.eq_ignore_ascii_case(&cursor.column)) + }); + !pinned + && spec + .warehouse_source + .as_ref() + .is_some_and(|source| source_binds_text(source.system)) +} + /// Whether a cursor of `column_type` compares as a number. fn numeric_cursor(column_type: &ColumnType) -> bool { matches!( @@ -1261,9 +1275,7 @@ pub(crate) async fn cursor_window( source.and_then(|source| source_dialect(source.system)), ), (from.as_deref().map(|from| (operator, from)), to.as_deref()), - source - .is_some_and(|source| source_binds_text(source.system)) - .then_some(&mut parameters), + binds_cursor(spec).then_some(&mut parameters), ); Ok(Ok(Window { filter, diff --git a/crates/sqlload/src/odbc_source/_helpers/catalog.rs b/crates/sqlload/src/odbc_source/_helpers/catalog.rs index 3163ab56..4c9cff2b 100644 --- a/crates/sqlload/src/odbc_source/_helpers/catalog.rs +++ b/crates/sqlload/src/odbc_source/_helpers/catalog.rs @@ -74,7 +74,7 @@ pub(crate) fn source_table( } } -/// The tables and views whose `schema.table` (or `table`, for a driver without schemas) matches `include` and not `exclude`, with their columns and keys. A driver without schemas (whose schema names are at most 0 long, as MySQL Connector/ODBC's are by default) is asked with no schema pattern, since it refuses any. +/// The tables and views whose `schema.table` (or `table`, for a driver without schemas) matches `include` and not `exclude`, with their columns and keys. A driver that refuses a schema pattern and reports no schema name length (MySQL Connector/ODBC, by default) is asked again with none; one reporting 0 for "no maximum" still lists every schema. pub(crate) fn catalog_tables( connection: &Connection<'_>, (include, exclude): (&[String], &[String]), @@ -82,11 +82,13 @@ pub(crate) fn catalog_tables( let include = table_matcher(include)?; let exclude = table_matcher(exclude)?; let mut names: Vec<(String, String)> = Vec::new(); - let schemas = match connection.max_schema_name_len()? { - 0 => "", - _ => ANY_NAME, + let listed = match connection.tables("", ANY_NAME, ANY_NAME, TABLE_TYPES) { + Err(_) if connection.max_schema_name_len()? == 0 => { + connection.tables("", "", ANY_NAME, TABLE_TYPES)? + } + listed => listed?, }; - for row in connection.tables("", schemas, ANY_NAME, TABLE_TYPES)? { + for row in listed { let row = row?; let (schema, table) = (catalog_text(&row.schema)?, catalog_text(&row.table)?); let qualified = match schema.is_empty() { diff --git a/crates/sqlload/src/odbc_source/_helpers/parameters.rs b/crates/sqlload/src/odbc_source/_helpers/parameters.rs index bc9a98d2..342edba1 100644 --- a/crates/sqlload/src/odbc_source/_helpers/parameters.rs +++ b/crates/sqlload/src/odbc_source/_helpers/parameters.rs @@ -8,10 +8,10 @@ use chrono::{Datelike, NaiveDateTime, Timelike}; use crate::adapters::contract::models::BoundParameter; use crate::odbc_source::constants::{ - LEGACY_DATETIME_DIGITS, SQL_SERVER_DBMS, TIMESTAMP_PARAMETER_PRECISION, + LEGACY_DATETIME_DIGITS, SQL_SERVER_DBMS, TIMESTAMP_PARAMETER_PRECISION, TIMESTAMP_TEXT_DIGITS, }; -/// `parameters` as `prepared` binds them, in order: text as `VARCHAR`; a date as `DATE`; a timestamp as `TIMESTAMP` with seven fractional digits, which SQL Server compares as `datetime2(7)` (three would make it a `datetime`, rounding milliseconds), except against a SQL Server column the driver describes with one to three fractional digits, a legacy `datetime`: that compares a `datetime2` parameter with the column's exact 1/300 s ticks, which the driver rounded to milliseconds when the bound was read, so the bound goes as text, which SQL Server converts to the column's own type. +/// `parameters` as `prepared` binds them, in order: text as `VARCHAR`; a date as `DATE`; a timestamp as `TIMESTAMP` with seven fractional digits, which SQL Server compares as `datetime2(7)` (three would make it a `datetime`, rounding milliseconds), except against a SQL Server column the driver describes with one to three fractional digits, a legacy `datetime`: that compares a `datetime2` parameter with the column's exact 1/300 s ticks, which the driver rounded to milliseconds when the bound was read, so the bound goes as text, which SQL Server converts to the column's own type; when the driver can't describe the parameter, as text with seven digits, exact for `datetime2` and refused by a `datetime`. pub(crate) fn bound_parameters( prepared: &mut Prepared, dbms: &str, @@ -33,7 +33,9 @@ pub(crate) fn bound_parameters( } _ => odbc_parameter(parameter), }, - Err(_) => odbc_parameter(parameter), + Err(_) => { + Box::new(timestamp_text(time, TIMESTAMP_TEXT_DIGITS).into_parameter()) + } } } _ => odbc_parameter(parameter), @@ -41,18 +43,18 @@ pub(crate) fn bound_parameters( .collect() } -/// `time` as `YYYY-MM-DD HH:MM:SS` with `digits` fractional digits, or with six when it has more than `digits` (which the database then refuses rather than misreads). +/// `time` as ISO 8601 `YYYY-MM-DDTHH:MM:SS`, which SQL Server reads alike under every `SET LANGUAGE` and `SET DATEFORMAT` (with a space it reads the month and day by the session's language), with `digits` fractional digits, or with six when it has more than `digits` (which the database then refuses rather than misreads). pub(crate) fn timestamp_text(time: &NaiveDateTime, digits: u16) -> String { let nanoseconds = time.nanosecond(); let unit = 10u32.pow(9u32.saturating_sub(u32::from(digits))); match (digits, nanoseconds % unit) { - (0, 0) => time.format("%Y-%m-%d %H:%M:%S").to_string(), + (0, 0) => time.format("%Y-%m-%dT%H:%M:%S").to_string(), (_, 0) => format!( "{}.{}", - time.format("%Y-%m-%d %H:%M:%S"), + time.format("%Y-%m-%dT%H:%M:%S"), &format!("{nanoseconds:09}")[..usize::from(digits)] ), - _ => time.format("%Y-%m-%d %H:%M:%S%.6f").to_string(), + _ => time.format("%Y-%m-%dT%H:%M:%S%.6f").to_string(), } } diff --git a/crates/sqlload/src/odbc_source/constants.rs b/crates/sqlload/src/odbc_source/constants.rs index 38f8f6b8..6aefd164 100644 --- a/crates/sqlload/src/odbc_source/constants.rs +++ b/crates/sqlload/src/odbc_source/constants.rs @@ -34,3 +34,6 @@ pub const SQL_SERVER_DBMS: &str = "Microsoft SQL Server"; /// The fractional digits SQL Server's driver describes a legacy `datetime` parameter with. pub const LEGACY_DATETIME_DIGITS: std::ops::RangeInclusive = 1..=3; + +/// The fractional digits of a SQL Server timestamp bound sent as text when the driver can't describe its parameter: `datetime2`'s seven, which a legacy `datetime` refuses rather than rounds. +pub const TIMESTAMP_TEXT_DIGITS: u16 = 7; diff --git a/crates/sqlload/src/odbc_source/tests/test_timestamp_text.rs b/crates/sqlload/src/odbc_source/tests/test_timestamp_text.rs index 37c97dd2..35f87014 100644 --- a/crates/sqlload/src/odbc_source/tests/test_timestamp_text.rs +++ b/crates/sqlload/src/odbc_source/tests/test_timestamp_text.rs @@ -9,31 +9,37 @@ fn given_a_timestamp_bound_for_a_legacy_sql_server_datetime_when_writing_it_as_t description: "a millisecond bound keeps its three digits, which datetime converts to the tick it read", time: "2026-01-31 23:59:59.123", digits: 3, - expected_text: "2026-01-31 23:59:59.123", + expected_text: "2026-01-31T23:59:59.123", }, TimestampTextTestCase { description: "a whole second still has three digits", time: "2026-02-01 00:00:00", digits: 3, - expected_text: "2026-02-01 00:00:00.000", + expected_text: "2026-02-01T00:00:00.000", }, TimestampTextTestCase { description: "a leading zero in the fraction is kept", time: "2026-01-31 23:59:59.007", digits: 3, - expected_text: "2026-01-31 23:59:59.007", + expected_text: "2026-01-31T23:59:59.007", }, TimestampTextTestCase { description: "a microsecond bound keeps six digits rather than being cut, so the database refuses it", time: "2026-01-31 23:59:59.123456", digits: 3, - expected_text: "2026-01-31 23:59:59.123456", + expected_text: "2026-01-31T23:59:59.123456", + }, + TimestampTextTestCase { + description: "seven digits, as a bound the driver can't describe goes", + time: "2026-01-31 23:59:59.123456", + digits: 7, + expected_text: "2026-01-31T23:59:59.1234560", }, TimestampTextTestCase { description: "no digits for a whole second", time: "2026-01-31 23:59:00", digits: 0, - expected_text: "2026-01-31 23:59:00", + expected_text: "2026-01-31T23:59:00", }, ]; for test_case in test_cases { diff --git a/crates/sqlload/tests/loader/test_odbc_drivers.rs b/crates/sqlload/tests/loader/test_odbc_drivers.rs index 2a55e0ee..073cfa2b 100644 --- a/crates/sqlload/tests/loader/test_odbc_drivers.rs +++ b/crates/sqlload/tests/loader/test_odbc_drivers.rs @@ -11,14 +11,15 @@ const FIRST: &str = "1,2 to 2026-01-31 23:59:59.997 | 1,2 to 2026-01-31"; const SECOND: &str = "1,2,3,4 to 2026-02-01 00:00:00.000 | 1,2,3,4 to 2026-02-01"; -const LEGACY_FIRST: &str = "1,2 to 2026-01-31 23:59:59.123 | 1,2 to 2026-01-31"; +const LEGACY_FIRST: &str = "1,2 to 2026-01-05 23:59:59.123 | 1,2 to 2026-01-31"; #[test] fn given_each_odbc_driver_when_a_millisecond_timestamp_cursor_and_a_date_cursor_resume_then_the_next_load_reads_exactly_the_rows_after_each_boundary() -> Result<(), String> { let test_cases = [ OdbcDriverTestCase { - description: "SQL Server through Microsoft's ODBC Driver 18: {ts} and {d} bounds on datetime2(3) and date", + description: "SQL Server through Microsoft's ODBC Driver 18: millisecond and date cursors on datetime2(3) and date", + connection_options: "", database: OdbcDatabase::SqlServer, tables: ("erp.events", "erp.days"), setup: &[ @@ -35,24 +36,26 @@ fn given_each_odbc_driver_when_a_millisecond_timestamp_cursor_and_a_date_cursor_ expected_tables: (FIRST, SECOND), }, OdbcDriverTestCase { - description: "SQL Server's legacy datetime, whose 1/300 s ticks the driver reads rounded to milliseconds (.123 is .12333): a datetime2 bound would read the .123 row again, so it is bound as text in the column's own type", + description: "SQL Server's legacy datetime, whose 1/300 s ticks the driver reads rounded to milliseconds (.123 is .12333): a datetime2 bound would read the .123 row again, so it is bound as text in the column's own type, in ISO 8601 form, which a british session (day before month) reads alike: 2026-01-05 with a space would be 1 May", + connection_options: "Language=British;", database: OdbcDatabase::SqlServer, tables: ("erp.events", "erp.days"), setup: &[ "CREATE SCHEMA erp", "CREATE TABLE erp.events (id int PRIMARY KEY, name nvarchar(40), happened_at datetime NOT NULL)", "CREATE TABLE erp.days (id int PRIMARY KEY, opened date NOT NULL)", - "INSERT INTO erp.events VALUES (1, 'a', '2026-01-31 10:00:00'), (2, 'b', '2026-01-31 23:59:59.123')", + "INSERT INTO erp.events VALUES (1, 'a', '2026-01-05 10:00:00'), (2, 'b', '2026-01-05 23:59:59.123')", "INSERT INTO erp.days VALUES (1, '2026-01-30'), (2, '2026-01-31')", ], change: &[ - "INSERT INTO erp.events VALUES (0, 'early', '2026-01-31 23:59:59.120'), (3, 'next', '2026-01-31 23:59:59.127'), (4, 'later', '2026-02-01 00:00:00')", + "INSERT INTO erp.events VALUES (0, 'early', '2026-01-05 23:59:59.120'), (3, 'next', '2026-01-05 23:59:59.127'), (4, 'later', '2026-02-01 00:00:00')", "INSERT INTO erp.days VALUES (3, '2026-01-31'), (4, '2026-02-01')", ], expected_tables: (LEGACY_FIRST, SECOND), }, OdbcDriverTestCase { - description: "MySQL through Connector/ODBC, a driver without schemas, in an ANSI_QUOTES session: {ts} and {d} bounds on DATETIME(3) and DATE", + description: "MySQL through Connector/ODBC, a driver without schemas, in an ANSI_QUOTES session: millisecond and date cursors on DATETIME(3) and DATE", + connection_options: "", database: OdbcDatabase::MySql, tables: ("events", "days"), setup: &[ @@ -68,7 +71,8 @@ fn given_each_odbc_driver_when_a_millisecond_timestamp_cursor_and_a_date_cursor_ expected_tables: (FIRST, SECOND), }, OdbcDriverTestCase { - description: "Postgres through psqlODBC: {ts} and {d} bounds on timestamp(3) and date", + description: "Postgres through psqlODBC: millisecond and date cursors on timestamp(3) and date", + connection_options: "", database: OdbcDatabase::Postgres, tables: ("erp.events", "erp.days"), setup: &[ @@ -94,7 +98,14 @@ fn given_each_odbc_driver_when_a_millisecond_timestamp_cursor_and_a_date_cursor_ let config = folder.path().join("sqlload.toml"); std::fs::write( &config, - driver_project(&server.connection_string, test_case.tables, &database), + driver_project( + &format!( + "{}{}", + server.connection_string, test_case.connection_options + ), + test_case.tables, + &database, + ), ) .map_err(|error| error.to_string())?; let (succeeded, output) = run_sqlload(&config, &["load", "--yes"], ("", ""))?; diff --git a/crates/sqlload/tests/loader/test_types.rs b/crates/sqlload/tests/loader/test_types.rs index 978de21e..1d2fd3d9 100644 --- a/crates/sqlload/tests/loader/test_types.rs +++ b/crates/sqlload/tests/loader/test_types.rs @@ -914,6 +914,8 @@ pub(crate) struct SftpSnowflakePlanTestCase { pub(crate) struct OdbcDriverTestCase { pub(crate) description: &'static str, pub(crate) database: sqlload_testkit::odbc_servers::models::OdbcDatabase, + /// Appended to the connection string SQLLoad reads through. + pub(crate) connection_options: &'static str, /// The events and days tables as the source's `include` names them. pub(crate) tables: (&'static str, &'static str), /// Statements creating both tables and their first rows. diff --git a/website/src/content/docs/docs/sources/odbc.mdx b/website/src/content/docs/docs/sources/odbc.mdx index 66256b66..5dba46fb 100644 --- a/website/src/content/docs/docs/sources/odbc.mdx +++ b/website/src/content/docs/docs/sources/odbc.mdx @@ -94,15 +94,19 @@ such as `"updated_at" > ?`, each passed as the cursor's own type: a timestamp wi an ODBC `TIMESTAMP`, with every fractional digit; a date as a `DATE`; text as text. No database's literal rules can change a bound, and a millisecond maximum such as `23:59:59.997` resumes exactly. SQL Server's legacy `datetime` is the one exception: it stores 1/300 s ticks, which its driver reads -rounded to milliseconds, so its bounds go as text in the column's own precision, which SQL Server -converts to the column's type. A numeric cursor compares with a bare number, such as `"id" > 41`. +rounded to milliseconds, so its bounds go as ISO 8601 text (`2026-01-31T23:59:59.997`) in the +column's own precision, which SQL Server converts to the column's type alike under every login +language. A cursor column pinned to a type (`columns = { ... }`) compares with a quoted literal, as +the pinned type may not be the column's own, such as a `varchar` read as a timestamp. A numeric +cursor compares with a bare number, such as `"id" > 41`. Any other cursor, a timestamp with a time zone for one, still compares with a quoted literal, which the database converts to the column's type; check that yours does before relying on it. -`--start-cursor`, `--end-cursor` and `cursor_start` also take ODBC escape values, such as +For a timestamp or date cursor, `--start-cursor`, `--end-cursor` and `cursor_start` also take ODBC +escape values, such as `{ts '2026-01-31 23:59:59.997'}` or `{d '2026-01-31'}`. -From 0.35.0 (not released yet), a timestamp column is read in microseconds whatever precision the -driver describes: MySQL Connector/ODBC describes every `DATETIME(n)` with none, and its fractional +From 0.35.0 (not released yet), a timestamp column is read in at least microseconds whatever precision +the driver describes: MySQL Connector/ODBC describes every `DATETIME(n)` with none, and its fractional seconds were dropped before. ## Choosing tables From 820a2b8f5c2ff3a10f2a4c2aa85cae7dc55acee3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:23:47 +0000 Subject: [PATCH 21/25] ci(windows): run the ODBC checks' MySQL as a service, which outlives its step The runner ends the processes a step starts when the step ends, so the mysqld started with Start-Process was gone by the end-to-end step (connection refused on 3307). It now runs as the sqlload-mysql service from an option file. The image's MySQL is 8.4, not 8.0; the docs say so. --- .github/workflows/verify.yml | 15 ++++++++++----- website/src/content/docs/docs/sources/odbc.mdx | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 8401d9a1..0c67c199 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -190,8 +190,9 @@ jobs: - name: Install the ODBC drivers shell: pwsh run: .github/scripts/install_odbc_drivers.ps1 - # The image's SQL Server LocalDB and MySQL 8.0, for the ODBC drivers' cursor checks. MySQL gets a - # data folder and port of its own, its root user without a password. + # The image's SQL Server LocalDB and MySQL 8.4, for the ODBC drivers' cursor checks. MySQL gets a + # data folder and port of its own, its root user without a password, and runs as a service: the + # runner ends the processes a step starts when the step ends. - name: Start SQL Server LocalDB and MySQL shell: pwsh run: | @@ -200,15 +201,19 @@ jobs: & $localdb create sqlload -s if ($LASTEXITCODE -ne 0) { throw "SqlLocalDB exited $LASTEXITCODE" } $mysqld = (Get-ChildItem "C:\Program Files\MySQL" -Recurse -Filter mysqld.exe | Select-Object -First 1).FullName - & $mysqld --initialize-insecure --datadir=C:\mysql-data --console + $ini = "C:\mysql-sqlload.ini" + Set-Content -Path $ini -Value "[mysqld]`ndatadir=C:/mysql-data`nport=3307`nmysqlx=OFF`n" + & $mysqld --defaults-file=$ini --initialize-insecure --console if ($LASTEXITCODE -ne 0) { throw "mysqld --initialize-insecure exited $LASTEXITCODE" } - Start-Process $mysqld -ArgumentList "--datadir=C:\mysql-data", "--port=3307", "--mysqlx=OFF" -RedirectStandardError "$env:RUNNER_TEMP\mysqld.err" -RedirectStandardOutput "$env:RUNNER_TEMP\mysqld.out" + & $mysqld --install sqlload-mysql --defaults-file=$ini + if ($LASTEXITCODE -ne 0) { throw "mysqld --install exited $LASTEXITCODE" } + Start-Service -Name sqlload-mysql foreach ($attempt in 1..60) { if ((Test-NetConnection 127.0.0.1 -Port 3307 -WarningAction SilentlyContinue).TcpTestSucceeded) { break } Start-Sleep -Seconds 2 } if (-not (Test-NetConnection 127.0.0.1 -Port 3307 -WarningAction SilentlyContinue).TcpTestSucceeded) { - Get-Content "$env:RUNNER_TEMP\mysqld.err" + Get-ChildItem C:\mysql-data -Filter *.err | Get-Content throw "MySQL never listened on 3307" } - name: Share a folder as \\localhost\sqlload-e2e diff --git a/website/src/content/docs/docs/sources/odbc.mdx b/website/src/content/docs/docs/sources/odbc.mdx index 5dba46fb..a72a6574 100644 --- a/website/src/content/docs/docs/sources/odbc.mdx +++ b/website/src/content/docs/docs/sources/odbc.mdx @@ -85,7 +85,7 @@ as names is refused when the source is read; switch it to ANSI quoting. For MySQ |--------|--------| | psqlODBC (`PostgreSQL Unicode`) | Yes, against Postgres 17 in containers on Linux and on Windows. Add `BoolsAsChar=0` to the connection string so `boolean` columns load as `bool` rather than text. | | Microsoft ODBC Driver 18 for SQL Server | Yes, against SQL Server 2022 in containers on Linux and against SQL Server LocalDB on Windows, `datetime2` and legacy `datetime` cursors included. | -| MySQL Connector/ODBC (`MySQL ODBC Unicode Driver`) | Yes, against MySQL 8.4 in containers on Linux and MySQL 8.0 on Windows, in an `ANSI_QUOTES` session (above). On Linux, install MySQL's package for your distribution rather than the generic glibc tarball: the tarball's own OpenSSL fails to load once another driver, such as SQL Server's or psqlODBC, has loaded the system one in the same run. | +| MySQL Connector/ODBC (`MySQL ODBC Unicode Driver`) | Yes, against MySQL 8.4 in containers on Linux and MySQL 8.4 on Windows, in an `ANSI_QUOTES` session (above). On Linux, install MySQL's package for your distribution rather than the generic glibc tarball: the tarball's own OpenSSL fails to load once another driver, such as SQL Server's or psqlODBC, has loaded the system one in the same run. | | OpenEdge (Progress) | Tested for timestamp handling (parsing `{ts}` values and binding timestamp bounds), not against a live server. | | Others | Not yet. A driver needs to describe prepared statements and list tables; run `sqlload plan` to see the types it reports. | From 5616eed079e021584989f43cafbf584a30bd99b2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:51:48 +0000 Subject: [PATCH 22/25] feat(odbc): ship ODBC in the wheels, with unixODBC bundled on Linux and macOS Both wheels now build the odbc feature. On Linux and macOS they bundle the unixODBC 2.3.14 driver manager, built from its pinned source release (SHA-256 checked) with libltdl built in, linked dynamically and installed as a separate, replaceable shared library by maturin's repair. It reads odbcinst.ini and odbc.ini in /etc on Linux and Homebrew's etc on macOS, or the folder ODBCSYSINI names, so SQLLoad needs no system unixODBC. Windows uses odbc32.dll. Every wheel carries unixODBC's LGPL-2.1 text and a notice naming its exact source, and a check asserts it, and that each Linux and macOS wheel bundles exactly one libodbc. Each wheel is smoke-tested against a made-up driver: on Linux in a container with no unixODBC, on macOS after removing Homebrew's copy (registered under ODBCSYSINI and in the default odbcinst.ini), and on Windows through odbc32.dll. macOS uses maturin's repair rather than delocate, which points an installed sqlload-cli binary at a folder that doesn't exist. --- .github/scripts/assert_wheel_notices.sh | 33 ++ .github/scripts/build_unixodbc.sh | 37 ++ .github/scripts/smoke_odbc_wheel.py | 97 ++++ .github/workflows/publish.yml | 54 ++- CONTRIBUTING.md | 2 +- README.md | 2 +- crates/sqlload-python/Cargo.toml | 2 + .../sqlload-cli/licenses/unixODBC-COPYING.txt | 458 ++++++++++++++++++ .../sqlload-cli/licenses/unixODBC-NOTICE.md | 18 + packages/sqlload-cli/pyproject.toml | 7 +- packages/sqlload/README.md | 2 +- .../sqlload/licenses/unixODBC-COPYING.txt | 458 ++++++++++++++++++ packages/sqlload/licenses/unixODBC-NOTICE.md | 18 + packages/sqlload/pyproject.toml | 8 +- website/src/content/docs/docs/index.mdx | 2 +- .../src/content/docs/docs/sources/odbc.mdx | 27 +- .../content/docs/docs/supported-systems.mdx | 2 +- 17 files changed, 1203 insertions(+), 24 deletions(-) create mode 100755 .github/scripts/assert_wheel_notices.sh create mode 100755 .github/scripts/build_unixodbc.sh create mode 100644 .github/scripts/smoke_odbc_wheel.py create mode 100644 packages/sqlload-cli/licenses/unixODBC-COPYING.txt create mode 100644 packages/sqlload-cli/licenses/unixODBC-NOTICE.md create mode 100644 packages/sqlload/licenses/unixODBC-COPYING.txt create mode 100644 packages/sqlload/licenses/unixODBC-NOTICE.md diff --git a/.github/scripts/assert_wheel_notices.sh b/.github/scripts/assert_wheel_notices.sh new file mode 100755 index 00000000..575664e1 --- /dev/null +++ b/.github/scripts/assert_wheel_notices.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Asserts every wheel in DIR carries unixODBC's notice and licence text, and that each Linux and +# macOS wheel bundles the unixODBC driver manager as one separate shared library (Windows wheels +# use odbc32.dll and bundle none). +# Usage: assert_wheel_notices.sh DIR +set -euo pipefail + +python="$(command -v python3 || command -v python)" +shopt -s nullglob +wheels=("$1"/*.whl) +if [ "${#wheels[@]}" -eq 0 ]; then + echo "no wheels in $1" >&2 + exit 1 +fi +for wheel in "${wheels[@]}"; do + names="$("$python" -c 'import sys, zipfile; print("\n".join(zipfile.ZipFile(sys.argv[1]).namelist()))' "$wheel")" + for notice in unixODBC-NOTICE.md unixODBC-COPYING.txt; do + if ! grep -Eq "\.dist-info/licenses/licenses/${notice}\$" <<< "$names"; then + echo "$wheel: no ${notice} in its .dist-info/licenses" >&2 + exit 1 + fi + done + bundled="$(grep -Ec '\.(libs|dylibs)/libodbc[^/]*\.(so[.0-9]*|dylib)$' <<< "$names" || true)" + case "$wheel" in + *win_amd64.whl | *win32.whl | *win_arm64.whl) expected=0 ;; + *) expected=1 ;; + esac + if [ "$bundled" -ne "$expected" ]; then + echo "$wheel: bundles ${bundled} unixODBC libraries, expected ${expected}" >&2 + exit 1 + fi + echo "ok: $(basename "$wheel") (bundled libodbc: ${bundled})" +done diff --git a/.github/scripts/build_unixodbc.sh b/.github/scripts/build_unixodbc.sh new file mode 100755 index 00000000..0a06f911 --- /dev/null +++ b/.github/scripts/build_unixodbc.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Builds the unixODBC driver manager the Linux and macOS release wheels bundle, from its pinned source +# release, so the wheels link it dynamically and carry it as a separate, replaceable shared library. +# Usage: build_unixodbc.sh PREFIX SYSCONFDIR [LIBDIR] +# SYSCONFDIR is where the bundled driver manager reads odbcinst.ini and odbc.ini when ODBCSYSINI +# is unset: /etc on Linux, as distribution driver packages register there; Homebrew's etc on macOS. +# Keep UNIXODBC_VERSION in step with the NOTICE files under packages/*/licenses/. +set -euo pipefail + +UNIXODBC_VERSION=2.3.14 +UNIXODBC_SHA256=4e2814de3e01fc30b0b9f75e83bb5aba91ab0384ee951286504bb70205524771 + +prefix="$1" +sysconfdir="$2" +libdir="${3:-$prefix/lib}" + +work="$(mktemp -d)" +tarball="$work/unixODBC-${UNIXODBC_VERSION}.tar.gz" +curl -fsSL -o "$tarball" "https://www.unixodbc.org/unixODBC-${UNIXODBC_VERSION}.tar.gz" || + curl -fsSL -o "$tarball" "https://github.com/lurcher/unixODBC/releases/download/v${UNIXODBC_VERSION}/unixODBC-${UNIXODBC_VERSION}.tar.gz" +if command -v sha256sum > /dev/null; then + echo "$UNIXODBC_SHA256 $tarball" | sha256sum -c --quiet - +else + echo "$UNIXODBC_SHA256 $tarball" | shasum -a 256 -c --quiet - +fi +tar -xzf "$tarball" -C "$work" +cd "$work/unixODBC-${UNIXODBC_VERSION}" +# The included libltdl is built into the libraries, so they need no system libltdl; no readline, as +# only the libraries ship. +./configure --prefix="$prefix" --libdir="$libdir" --sysconfdir="$sysconfdir" \ + --with-included-ltdl --enable-readline=no --enable-editline=no --enable-static=no > "$work/configure.log" || + { tail -50 "$work/configure.log"; exit 1; } +make -j"$(getconf _NPROCESSORS_ONLN)" > "$work/make.log" 2>&1 || { tail -50 "$work/make.log"; exit 1; } +make install > /dev/null +cd / +rm -rf "$work" +echo "unixODBC ${UNIXODBC_VERSION} installed in ${libdir}, reading ${sysconfdir}" diff --git a/.github/scripts/smoke_odbc_wheel.py b/.github/scripts/smoke_odbc_wheel.py new file mode 100644 index 00000000..0b7201ca --- /dev/null +++ b/.github/scripts/smoke_odbc_wheel.py @@ -0,0 +1,97 @@ +"""Smoke-test the driver manager an installed wheel's `sqlload` reaches ODBC drivers through. + +Usage: python smoke_odbc_wheel.py SQLLOAD... [--default-ini PATH] + +Each SQLLOAD is an installed `sqlload` command (the sqlload wheel's and the sqlload-cli wheel's). +On Linux and macOS it must run where no unixODBC is installed, so the wheel's bundled driver +manager is the only one; it must find a driver registered in an odbcinst.ini named by ODBCSYSINI +and, with --default-ini, one registered in the default odbcinst.ini (/etc on Linux, Homebrew's etc +on macOS). The drivers don't exist, so the driver manager reports the library it tried to load, +which shows the registration it read. On Windows, the driver manager built in (odbc32.dll) reports +an unregistered driver. +""" + +import argparse +import glob +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +PROJECT = """\ +[connections.erp] +kind = "odbc" +connection_string = "Driver={{{driver}}};" + +[connections.local] +kind = "duckdb" +database = "out.duckdb" + +[targets.dev] +connection = "local" +loader_schema = "raw" + +[sources.erp] +kind = "odbc" +connection = "erp" +description = "Smoke test." +include = ["*"] +""" + +#: Where a unixODBC library would be if one were installed. +SYSTEM_LIBODBC = ["/usr/lib*/libodbc.so*", "/usr/lib/*/libodbc.so*", "/usr/local/lib/libodbc*", "/opt/homebrew/lib/libodbc*", "/opt/homebrew/opt/unixodbc"] + + +def plan(sqlload, folder, driver, environment): + (folder / "sqlload.toml").write_text(PROJECT.format(driver=driver), encoding="utf-8") + result = subprocess.run([sqlload, "--no-color", "plan"], cwd=folder, capture_output=True, text=True, encoding="utf-8", errors="replace", env=environment, check=False) + return result.returncode, result.stdout + result.stderr + + +def check(condition, what, output=""): + if not condition: + sys.exit(f"{what}\n{output}") + print(f"ok: {what}", flush=True) + + +def registered(ini, name, library): + previous = ini.read_text(encoding="utf-8") if ini.exists() else "" + ini.parent.mkdir(parents=True, exist_ok=True) + ini.write_text(f"{previous}\n[{name}]\nDriver={library}\n", encoding="utf-8") + return previous + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("sqlload", nargs="+") + parser.add_argument("--default-ini", type=Path) + arguments = parser.parse_args() + if os.name != "nt": + found = [path for pattern in SYSTEM_LIBODBC for path in glob.glob(pattern)] + check(not found, "no unixODBC is installed, so only the wheel's driver manager can load", str(found)) + for sqlload in arguments.sqlload: + sqlload = shutil.which(sqlload) or sqlload + with tempfile.TemporaryDirectory() as name: + folder = Path(name) + if os.name == "nt": + code, output = plan(sqlload, folder, "SQLLoad Smoke Missing", dict(os.environ)) + check(code == 1 and "Data source name not found" in output, f"{sqlload}: Windows's driver manager reports an unregistered driver", output) + continue + ini = folder / "ini" / "odbcinst.ini" + registered(ini, "SQLLoad Smoke", "/nonexistent/libsmoke-odbcsysini.so") + code, output = plan(sqlload, folder, "SQLLoad Smoke", {**os.environ, "ODBCSYSINI": str(ini.parent)}) + check(code == 1 and "Can't open lib '/nonexistent/libsmoke-odbcsysini.so'" in output, f"{sqlload}: the bundled driver manager finds a driver registered under ODBCSYSINI", output) + if arguments.default_ini: + previous = registered(arguments.default_ini, "SQLLoad Smoke Default", "/nonexistent/libsmoke-default.so") + try: + environment = {name: value for name, value in os.environ.items() if name not in ("ODBCSYSINI", "ODBCINSTINI")} + code, output = plan(sqlload, folder, "SQLLoad Smoke Default", environment) + finally: + arguments.default_ini.write_text(previous, encoding="utf-8") + check(code == 1 and "Can't open lib '/nonexistent/libsmoke-default.so'" in output, f"{sqlload}: the bundled driver manager finds a driver registered in {arguments.default_ini}", output) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8385bb65..f75d52ba 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -14,6 +14,10 @@ on: - .github/workflows/publish.yml - .github/scripts/assert_*_wheel_tags.sh - .github/scripts/interrupt_python_load.py + - .github/scripts/build_unixodbc.sh + - .github/scripts/assert_wheel_notices.sh + - .github/scripts/smoke_odbc_wheel.py + - packages/*/licenses/** - packages/*/pyproject.toml - Cargo.toml - Cargo.lock @@ -135,19 +139,32 @@ jobs: - name: Build for macOS 11 and newer if: runner.os == 'macOS' run: echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> "$GITHUB_ENV" + # unixODBC built from its pinned source, which the Linux and macOS wheels bundle as a separate + # shared library (licenses/ in each package; Linux builds it in the manylinux container below). + # The runner's Homebrew copy goes first, so the wheels link this one and the smoke test runs + # with no unixODBC installed. It reads Homebrew's etc, where Homebrew's drivers register. + - name: Build unixODBC for macOS + if: runner.os == 'macOS' + run: | + brew uninstall --ignore-dependencies unixodbc || true + bash .github/scripts/build_unixodbc.sh "$RUNNER_TEMP/unixodbc" "$(brew --prefix)/etc" + echo "RUSTFLAGS=-L native=$RUNNER_TEMP/unixodbc/lib" >> "$GITHUB_ENV" # sqlload-cli ships one standalone Rust executable (OpenSSL built in) and links no libpython, # so maturin tags it py3-none-. The tag is asserted before release. - name: Build native CLI wheel uses: PyO3/maturin-action@v1 with: command: build - args: --release --out ../../dist + # repair copies unixODBC into the Linux and macOS wheels; Windows uses odbc32.dll. + args: --release --out ../../dist ${{ runner.os != 'Windows' && '--auditwheel=repair' || '' }} target: ${{ matrix.platform.target }} manylinux: "2_28" working-directory: packages/sqlload-cli # The vendored OpenSSL build runs Perl's Configure, which needs modules the - # manylinux image leaves out. - before-script-linux: dnf install -y perl-IPC-Cmd perl-Time-Piece + # manylinux image leaves out. unixODBC reads /etc, where distribution drivers register. + before-script-linux: | + dnf install -y perl-IPC-Cmd perl-Time-Piece + bash "$GITHUB_WORKSPACE/.github/scripts/build_unixodbc.sh" /usr /etc /usr/lib64 - name: Assert native CLI wheel is interpreter-agnostic shell: bash run: bash .github/scripts/assert_cli_wheel_tags.sh dist @@ -157,14 +174,20 @@ jobs: uses: PyO3/maturin-action@v1 with: command: build - args: --release --out ../../dist + # repair copies unixODBC into the Linux and macOS wheels; Windows uses odbc32.dll. + args: --release --out ../../dist ${{ runner.os != 'Windows' && '--auditwheel=repair' || '' }} target: ${{ matrix.platform.target }} manylinux: "2_28" working-directory: packages/sqlload - before-script-linux: dnf install -y perl-IPC-Cmd perl-Time-Piece + before-script-linux: | + dnf install -y perl-IPC-Cmd perl-Time-Piece + bash "$GITHUB_WORKSPACE/.github/scripts/build_unixodbc.sh" /usr /etc /usr/lib64 - name: Assert Python extension wheel targets the stable ABI shell: bash run: bash .github/scripts/assert_python_wheel_tags.sh dist + - name: Assert every wheel carries unixODBC's notice, and Linux and macOS wheels bundle it + shell: bash + run: bash .github/scripts/assert_wheel_notices.sh dist - name: Smoke-test the wheels shell: bash run: | @@ -180,6 +203,23 @@ jobs: if [[ "${RUNNER_OS}" == "Windows" ]]; then cli=smoke-cli/Scripts; fi "${cli}/python" -m pip install --quiet dist/sqlload_cli-*.whl "${cli}/sqlload" --version + # ODBC through each wheel's driver manager: on Linux in a container with no unixODBC (the + # runner image has one), on macOS here (Homebrew's was removed above), on Windows odbc32.dll. + case "${RUNNER_OS}" in + Linux) + docker run --rm -v "$PWD:/w" -w /w python:3.12-slim bash -c ' + python -m venv /v && /v/bin/pip install --quiet dist/sqlload-*.whl && + python -m venv /c && /c/bin/pip install --quiet dist/sqlload_cli-*.whl && + /v/bin/python .github/scripts/smoke_odbc_wheel.py /v/bin/sqlload /c/bin/sqlload --default-ini /etc/odbcinst.ini' + ;; + macOS) + rm -rf "$RUNNER_TEMP/unixodbc" + "${bin}/python" .github/scripts/smoke_odbc_wheel.py "${bin}/sqlload" "${cli}/sqlload" --default-ini "$(brew --prefix)/etc/odbcinst.ini" + ;; + Windows) + "${bin}/python" .github/scripts/smoke_odbc_wheel.py "${bin}/sqlload" "${cli}/sqlload" + ;; + esac - uses: actions/upload-artifact@v5 with: name: wheel-${{ matrix.platform.name }} @@ -242,6 +282,10 @@ jobs: shell: bash run: bash .github/scripts/assert_python_wheel_tags.sh dist + - name: Assert every wheel carries unixODBC's notice, and Linux and macOS wheels bundle it + shell: bash + run: bash .github/scripts/assert_wheel_notices.sh dist + - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b10cecac..014e1255 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ make test-python # builds the sqlload extension into .venv and runs its tes `make check-ci` is what CI runs, and must be clean, including fensu with 0 faults. -The unit and container tests link the ODBC source against unixODBC's `libodbc.so`, and the +The unit and container tests, and `make test-python` (the wheel includes ODBC), link the ODBC source against unixODBC's `libodbc.so`, and the container tests also need the PostgreSQL ODBC driver: `unixodbc-dev odbc-postgresql` on Debian and Ubuntu, `unixodbc psqlodbc` with Homebrew. diff --git a/README.md b/README.md index 4538537a..4b4f55da 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ versions and how to run them yourself. | HTTP APIs and Python | ✓ | | | ODBC databases¹ | ✓ | | -¹ In builds with the `odbc` feature. +¹ In the published wheels from 0.37.0 (not released yet); before that, in builds with the `odbc` feature. ² From 0.34.0, through a separate DuckDB 1.5 process: install `sqlload[motherduck]`. diff --git a/crates/sqlload-python/Cargo.toml b/crates/sqlload-python/Cargo.toml index 05c08f44..1841aeda 100644 --- a/crates/sqlload-python/Cargo.toml +++ b/crates/sqlload-python/Cargo.toml @@ -18,6 +18,8 @@ doctest = false [features] # TLS with OpenSSL built in, as in the sqlload-cli wheels. ssl-vendored = ["sqlload/ssl-vendored"] +# ODBC sources, as in the sqlload-cli wheels. +odbc = ["sqlload/odbc"] [dependencies] arrow-array.workspace = true diff --git a/packages/sqlload-cli/licenses/unixODBC-COPYING.txt b/packages/sqlload-cli/licenses/unixODBC-COPYING.txt new file mode 100644 index 00000000..2c012a8c --- /dev/null +++ b/packages/sqlload-cli/licenses/unixODBC-COPYING.txt @@ -0,0 +1,458 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS diff --git a/packages/sqlload-cli/licenses/unixODBC-NOTICE.md b/packages/sqlload-cli/licenses/unixODBC-NOTICE.md new file mode 100644 index 00000000..94067b4c --- /dev/null +++ b/packages/sqlload-cli/licenses/unixODBC-NOTICE.md @@ -0,0 +1,18 @@ +# unixODBC + +The Linux and macOS wheels of this package include the unixODBC driver manager library +(`libodbc`), version 2.3.14, unmodified. Windows wheels include no unixODBC; they use the +driver manager built into Windows (`odbc32.dll`). + +- Licence: GNU Lesser General Public License, version 2.1; its text is in `unixODBC-COPYING.txt` + next to this file. +- Source: https://www.unixodbc.org/unixODBC-2.3.14.tar.gz (also + https://github.com/lurcher/unixODBC/releases/tag/v2.3.14), SHA-256 + 4e2814de3e01fc30b0b9f75e83bb5aba91ab0384ee951286504bb70205524771. +- Built by `.github/scripts/build_unixodbc.sh` in https://github.com/sqlstack-io/sqlload, with + `--with-included-ltdl` and the configuration folder set to `/etc` on Linux and Homebrew's `etc` + on macOS. + +SQLLoad links the library dynamically. It is installed as a separate shared library next to the +package (`*.libs/libodbc-*.so.2` on Linux, `*.dylibs/libodbc-*.2.dylib` on macOS); you may +replace that file with another build of unixODBC 2.3 with the same interface. diff --git a/packages/sqlload-cli/pyproject.toml b/packages/sqlload-cli/pyproject.toml index 17fc3f8f..91a9889c 100644 --- a/packages/sqlload-cli/pyproject.toml +++ b/packages/sqlload-cli/pyproject.toml @@ -9,6 +9,7 @@ description = "Native executable for sqlload." readme = "README.md" requires-python = ">=3.10" license = "Apache-2.0" +license-files = ["licenses/*"] classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", @@ -24,8 +25,10 @@ Repository = "https://github.com/sqlstack-io/sqlload" [tool.maturin] bindings = "bin" manifest-path = "../../crates/sqlload/Cargo.toml" -# Kafka TLS with OpenSSL built in; SASL PLAIN and SCRAM need nothing else. GSSAPI is left out. -features = ["ssl-vendored"] +# Kafka TLS with OpenSSL built in; SASL PLAIN and SCRAM need nothing else. GSSAPI is left out. ODBC +# sources through the system's driver manager: unixODBC, which the Linux and macOS wheels bundle +# (licenses/), or Windows's odbc32.dll. +features = ["ssl-vendored", "odbc"] [tool.uv] cache-keys = [ diff --git a/packages/sqlload/README.md b/packages/sqlload/README.md index 4538537a..4b4f55da 100644 --- a/packages/sqlload/README.md +++ b/packages/sqlload/README.md @@ -92,7 +92,7 @@ versions and how to run them yourself. | HTTP APIs and Python | ✓ | | | ODBC databases¹ | ✓ | | -¹ In builds with the `odbc` feature. +¹ In the published wheels from 0.37.0 (not released yet); before that, in builds with the `odbc` feature. ² From 0.34.0, through a separate DuckDB 1.5 process: install `sqlload[motherduck]`. diff --git a/packages/sqlload/licenses/unixODBC-COPYING.txt b/packages/sqlload/licenses/unixODBC-COPYING.txt new file mode 100644 index 00000000..2c012a8c --- /dev/null +++ b/packages/sqlload/licenses/unixODBC-COPYING.txt @@ -0,0 +1,458 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS diff --git a/packages/sqlload/licenses/unixODBC-NOTICE.md b/packages/sqlload/licenses/unixODBC-NOTICE.md new file mode 100644 index 00000000..94067b4c --- /dev/null +++ b/packages/sqlload/licenses/unixODBC-NOTICE.md @@ -0,0 +1,18 @@ +# unixODBC + +The Linux and macOS wheels of this package include the unixODBC driver manager library +(`libodbc`), version 2.3.14, unmodified. Windows wheels include no unixODBC; they use the +driver manager built into Windows (`odbc32.dll`). + +- Licence: GNU Lesser General Public License, version 2.1; its text is in `unixODBC-COPYING.txt` + next to this file. +- Source: https://www.unixodbc.org/unixODBC-2.3.14.tar.gz (also + https://github.com/lurcher/unixODBC/releases/tag/v2.3.14), SHA-256 + 4e2814de3e01fc30b0b9f75e83bb5aba91ab0384ee951286504bb70205524771. +- Built by `.github/scripts/build_unixodbc.sh` in https://github.com/sqlstack-io/sqlload, with + `--with-included-ltdl` and the configuration folder set to `/etc` on Linux and Homebrew's `etc` + on macOS. + +SQLLoad links the library dynamically. It is installed as a separate shared library next to the +package (`*.libs/libodbc-*.so.2` on Linux, `*.dylibs/libodbc-*.2.dylib` on macOS); you may +replace that file with another build of unixODBC 2.3 with the same interface. diff --git a/packages/sqlload/pyproject.toml b/packages/sqlload/pyproject.toml index cdefa85a..0db08c17 100644 --- a/packages/sqlload/pyproject.toml +++ b/packages/sqlload/pyproject.toml @@ -9,7 +9,7 @@ description = "SQL data loading tooling." readme = "README.md" requires-python = ">=3.10" license = "Apache-2.0" -license-files = ["LICENSE"] +license-files = ["LICENSE", "licenses/*"] classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", @@ -38,8 +38,10 @@ Repository = "https://github.com/sqlstack-io/sqlload" manifest-path = "../../crates/sqlload-python/Cargo.toml" module-name = "sqlload._native" python-source = "python" -# Kafka TLS with OpenSSL built in, as in sqlload-cli; SASL PLAIN and SCRAM need nothing else. -features = ["ssl-vendored"] +# Kafka TLS with OpenSSL built in, as in sqlload-cli; SASL PLAIN and SCRAM need nothing else. ODBC +# sources through the system's driver manager: unixODBC, which the Linux and macOS wheels bundle +# (licenses/), or Windows's odbc32.dll. +features = ["ssl-vendored", "odbc"] [tool.uv] cache-keys = [ diff --git a/website/src/content/docs/docs/index.mdx b/website/src/content/docs/docs/index.mdx index 48b24b18..36c67d3b 100644 --- a/website/src/content/docs/docs/index.mdx +++ b/website/src/content/docs/docs/index.mdx @@ -22,7 +22,7 @@ SQLLoad loads data into your warehouse, and shows you how before it changes anyt | HTTP APIs and Python | ✓ | | | ODBC databases¹ | ✓ | | -¹ In builds with the `odbc` feature. +¹ In the published wheels from 0.37.0 (not released yet); before that, in builds with the `odbc` feature. ² From 0.34.0, through a separate DuckDB 1.5 process: install `sqlload[motherduck]`. diff --git a/website/src/content/docs/docs/sources/odbc.mdx b/website/src/content/docs/docs/sources/odbc.mdx index a72a6574..b1c87ad2 100644 --- a/website/src/content/docs/docs/sources/odbc.mdx +++ b/website/src/content/docs/docs/sources/odbc.mdx @@ -7,9 +7,9 @@ From sqlload 0.27.0, an `odbc` source loads tables, views and query results from any database that has an ODBC driver. SQLLoad reads the rows as Arrow through the driver and loads them into the source's target. -ODBC support is built only with the `odbc` feature. The released packages don't include it yet; -build SQLLoad with `cargo install sqlload --features odbc`. Without it, `plan` refuses an `odbc` -source. +From 0.37.0 (not released yet), the `sqlload` and `sqlload-cli` wheels include ODBC on every +platform. Before that, and in a `cargo install` without it, ODBC is built only with the `odbc` +feature (`cargo install sqlload --features odbc`); without it, `plan` refuses an `odbc` source. ```toml [connections.erp] @@ -70,10 +70,19 @@ Loads (2) sqlload load | `dsn` | A data source name from `odbc.ini`, instead of `connection_string`. It and `user` can't contain `;`, `{` or `}`. | | `user`, `password` | With `dsn`, when the DSN doesn't hold them. | -Set `connection_string` or `dsn`, not both. SQLLoad uses unixODBC on Linux and macOS, and the -driver manager built into Windows. The driver and the driver manager must be installed on the -machine SQLLoad runs on; on Debian and Ubuntu, `unixodbc-dev` (which `cargo install` links -against) plus the driver's package, such as `odbc-postgresql`. +Set `connection_string` or `dsn`, not both. Install the database's ODBC driver on the machine +SQLLoad runs on; SQLLoad reaches it through a driver manager: + +- **Windows:** the driver manager built into Windows; register drivers and DSNs as usual (ODBC Data + Sources). +- **Linux and macOS wheels** (from 0.37.0, not released yet): the unixODBC 2.3.14 driver manager, + bundled as a separate shared library, so no system unixODBC is needed. It reads drivers and + system DSNs from `odbcinst.ini` and `odbc.ini` in `/etc` on Linux, where distribution driver + packages register, and in Homebrew's `etc` on macOS (`/opt/homebrew/etc` on Apple silicon, + `/usr/local/etc` on Intel), or in the folder `ODBCSYSINI` names; user DSNs from `~/.odbc.ini`. + unixODBC is LGPL-2.1; its notice and licence are in each wheel's `.dist-info/licenses`. +- **`cargo install --features odbc`:** the system's unixODBC, which it links against: `unixodbc-dev` + on Debian and Ubuntu, `unixodbc` with Homebrew. ## Drivers @@ -89,7 +98,7 @@ as names is refused when the source is read; switch it to ANSI quoting. For MySQ | OpenEdge (Progress) | Tested for timestamp handling (parsing `{ts}` values and binding timestamp bounds), not against a live server. | | Others | Not yet. A driver needs to describe prepared statements and list tables; run `sqlload plan` to see the types it reports. | -From 0.35.0 (not released yet), a cursor filter compares the cursor column with bound parameters, +From 0.37.0 (not released yet), a cursor filter compares the cursor column with bound parameters, such as `"updated_at" > ?`, each passed as the cursor's own type: a timestamp without a time zone as an ODBC `TIMESTAMP`, with every fractional digit; a date as a `DATE`; text as text. No database's literal rules can change a bound, and a millisecond maximum such as `23:59:59.997` resumes exactly. @@ -105,7 +114,7 @@ For a timestamp or date cursor, `--start-cursor`, `--end-cursor` and `cursor_sta escape values, such as `{ts '2026-01-31 23:59:59.997'}` or `{d '2026-01-31'}`. -From 0.35.0 (not released yet), a timestamp column is read in at least microseconds whatever precision +From 0.37.0 (not released yet), a timestamp column is read in at least microseconds whatever precision the driver describes: MySQL Connector/ODBC describes every `DATETIME(n)` with none, and its fractional seconds were dropped before. diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx index 12783e6b..888ce6cf 100644 --- a/website/src/content/docs/docs/supported-systems.mdx +++ b/website/src/content/docs/docs/supported-systems.mdx @@ -10,7 +10,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes | Postgres | Tables, by `include` patterns | Binary `COPY` into Arrow, over TLS when the connection's `sslmode` asks | | Postgres changes (`postgres_cdc`) | Inserts, updates and deletes of tables, by `include` patterns | A snapshot, then a logical replication slot (`pgoutput`), in whole transactions. See [Postgres changes](/docs/sources/postgres-cdc/). | | Snowflake (from 0.26.0) | Tables and views, by `include` patterns, and query results | Parquet unloaded with `COPY INTO` to the user stage or an S3 stage, then read by SQLLoad, or (from 0.27.0) by DuckDB itself from an S3 stage. See [Snowflake](/docs/sources/snowflake/). | -| Databases with an ODBC driver (from 0.27.0) | Tables and views, by `include` patterns, and query results | The driver, as Arrow, in builds with the `odbc` feature. See [ODBC](/docs/sources/odbc/). | +| Databases with an ODBC driver (from 0.27.0) | Tables and views, by `include` patterns, and query results | The driver, as Arrow; in the published wheels from 0.37.0 (not released yet), before that in builds with the `odbc` feature. See [ODBC](/docs/sources/odbc/). | | BigQuery (from 0.30.0) | Tables and views, by `include` patterns, and query results | The Storage Read API, as Arrow in parallel streams; views and queries through a query job. See [BigQuery](/docs/sources/bigquery/). | | Files in S3 | Parquet, CSV, JSON, NDJSON; gzip or zstd for text formats | The destination itself where it can load them exactly, otherwise SQLLoad's reader | | Files in a local folder | As for S3 | As for S3 | From a7451924873150a0d85f809f91309e9ddb3f805d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 17:17:27 +0000 Subject: [PATCH 23/25] fix(odbc): find drivers registered by name, carry unixODBC's source in the sdists, keep sdists free of ODBC - The bundled unixODBC is configured with a driver search path (Debian's multiarch odbc folders, /usr/lib64 and Homebrew's lib), as Debian's own build is: Debian and Ubuntu driver packages register a file name alone (Driver=psqlodbcw.so), which it could not load. The Linux smoke test now loads a library registered by file name in /usr/lib/-linux-gnu/odbc; it fails with a build without the path. - Both sdists carry unixODBC-2.3.14.tar.gz, SHA-checked, as the corresponding source of the library the Linux and macOS wheels bundle (LGPL-2.1 section 4); the NOTICE says so, and the sdist and PyPI jobs check it is there, unchanged. The download and its pin live in unixodbc_source.sh. - The odbc feature moves from the pyprojects to the wheel builds' --features (and make test-python), so building an sdist needs no driver manager. - A cursor pinned to text or varchar binds as text again; only a cursor pinned to another type falls back to a literal. The backslash test pins one of its text cursors and fails without this. - The catalog's retry without a schema pattern keeps the original error when it too fails, and returns it outright when the schema name length can't be read. --- .github/scripts/assert_sdist_sources.sh | 31 +++++++++++++++++++ .github/scripts/build_unixodbc.sh | 26 ++++++---------- .github/scripts/smoke_odbc_wheel.py | 15 +++++++-- .github/scripts/unixodbc_source.sh | 25 +++++++++++++++ .github/scripts/windows_e2e.py | 4 +-- .github/workflows/publish.yml | 30 +++++++++++++----- .gitignore | 2 ++ Makefile | 2 +- .../sqlload/src/loader/_helpers/table_plan.rs | 13 ++++---- .../src/odbc_source/_helpers/catalog.rs | 11 ++++--- crates/sqlload/tests/loader/odbc/helpers.rs | 3 +- .../sqlload-cli/licenses/unixODBC-NOTICE.md | 14 ++++++--- packages/sqlload-cli/pyproject.toml | 11 ++++--- packages/sqlload/licenses/unixODBC-NOTICE.md | 14 ++++++--- packages/sqlload/pyproject.toml | 11 ++++--- .../src/content/docs/docs/sources/odbc.mdx | 11 +++++-- 16 files changed, 162 insertions(+), 61 deletions(-) create mode 100755 .github/scripts/assert_sdist_sources.sh create mode 100755 .github/scripts/unixodbc_source.sh diff --git a/.github/scripts/assert_sdist_sources.sh b/.github/scripts/assert_sdist_sources.sh new file mode 100755 index 00000000..d501c0d0 --- /dev/null +++ b/.github/scripts/assert_sdist_sources.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Asserts every sdist in DIR carries unixODBC's pinned source release, unchanged: the corresponding +# source of the driver manager the Linux and macOS wheels bundle (version and SHA-256 from +# unixodbc_source.sh). +# Usage: assert_sdist_sources.sh DIR +set -euo pipefail + +pins="$(dirname "$0")/unixodbc_source.sh" +version="$(sed -n 's/^UNIXODBC_VERSION=//p' "$pins")" +sha256="$(sed -n 's/^UNIXODBC_SHA256=//p' "$pins")" +name="unixODBC-${version}.tar.gz" + +shopt -s nullglob +sdists=("$1"/*.tar.gz) +if [ "${#sdists[@]}" -eq 0 ]; then + echo "no sdists in $1" >&2 + exit 1 +fi +for sdist in "${sdists[@]}"; do + member="$(tar -tzf "$sdist" | grep -E "^[^/]+/${name//./\\.}\$" || true)" + if [ -z "$member" ]; then + echo "$sdist: no ${name} at its root" >&2 + exit 1 + fi + found="$(tar -xzOf "$sdist" "$member" | sha256sum | cut -d' ' -f1)" + if [ "$found" != "$sha256" ]; then + echo "$sdist: ${name} has SHA-256 ${found}, expected ${sha256}" >&2 + exit 1 + fi + echo "ok: $(basename "$sdist") carries ${name} (SHA-256 ${sha256})" +done diff --git a/.github/scripts/build_unixodbc.sh b/.github/scripts/build_unixodbc.sh index 0a06f911..2bca75db 100755 --- a/.github/scripts/build_unixodbc.sh +++ b/.github/scripts/build_unixodbc.sh @@ -1,37 +1,31 @@ #!/usr/bin/env bash # Builds the unixODBC driver manager the Linux and macOS release wheels bundle, from its pinned source # release, so the wheels link it dynamically and carry it as a separate, replaceable shared library. -# Usage: build_unixodbc.sh PREFIX SYSCONFDIR [LIBDIR] +# Usage: build_unixodbc.sh PREFIX SYSCONFDIR [LIBDIR] [DRIVER_PATH] # SYSCONFDIR is where the bundled driver manager reads odbcinst.ini and odbc.ini when ODBCSYSINI # is unset: /etc on Linux, as distribution driver packages register there; Homebrew's etc on macOS. -# Keep UNIXODBC_VERSION in step with the NOTICE files under packages/*/licenses/. +# DRIVER_PATH is a colon-separated list of folders searched for a driver registered by file name +# alone, as Debian and Ubuntu packages register theirs (Driver=psqlodbcw.so). +# The source release and its SHA-256 are pinned in unixodbc_source.sh. set -euo pipefail -UNIXODBC_VERSION=2.3.14 -UNIXODBC_SHA256=4e2814de3e01fc30b0b9f75e83bb5aba91ab0384ee951286504bb70205524771 - prefix="$1" sysconfdir="$2" libdir="${3:-$prefix/lib}" +driver_path="${4:-}" work="$(mktemp -d)" -tarball="$work/unixODBC-${UNIXODBC_VERSION}.tar.gz" -curl -fsSL -o "$tarball" "https://www.unixodbc.org/unixODBC-${UNIXODBC_VERSION}.tar.gz" || - curl -fsSL -o "$tarball" "https://github.com/lurcher/unixODBC/releases/download/v${UNIXODBC_VERSION}/unixODBC-${UNIXODBC_VERSION}.tar.gz" -if command -v sha256sum > /dev/null; then - echo "$UNIXODBC_SHA256 $tarball" | sha256sum -c --quiet - -else - echo "$UNIXODBC_SHA256 $tarball" | shasum -a 256 -c --quiet - -fi -tar -xzf "$tarball" -C "$work" -cd "$work/unixODBC-${UNIXODBC_VERSION}" +tarball="$(bash "$(dirname "$0")/unixodbc_source.sh" "$work")" +tar -xzf "$work/$tarball" -C "$work" +cd "$work/${tarball%.tar.gz}" # The included libltdl is built into the libraries, so they need no system libltdl; no readline, as # only the libraries ship. ./configure --prefix="$prefix" --libdir="$libdir" --sysconfdir="$sysconfdir" \ + ${driver_path:+--with-odbc-driver-path="$driver_path"} \ --with-included-ltdl --enable-readline=no --enable-editline=no --enable-static=no > "$work/configure.log" || { tail -50 "$work/configure.log"; exit 1; } make -j"$(getconf _NPROCESSORS_ONLN)" > "$work/make.log" 2>&1 || { tail -50 "$work/make.log"; exit 1; } make install > /dev/null cd / rm -rf "$work" -echo "unixODBC ${UNIXODBC_VERSION} installed in ${libdir}, reading ${sysconfdir}" +echo "${tarball%.tar.gz} installed in ${libdir}, reading ${sysconfdir}, finding drivers by name in ${driver_path:-the library search path}" diff --git a/.github/scripts/smoke_odbc_wheel.py b/.github/scripts/smoke_odbc_wheel.py index 0b7201ca..f60707b8 100644 --- a/.github/scripts/smoke_odbc_wheel.py +++ b/.github/scripts/smoke_odbc_wheel.py @@ -1,13 +1,15 @@ """Smoke-test the driver manager an installed wheel's `sqlload` reaches ODBC drivers through. -Usage: python smoke_odbc_wheel.py SQLLOAD... [--default-ini PATH] +Usage: python smoke_odbc_wheel.py SQLLOAD... [--default-ini PATH] [--driver-folder DIR] Each SQLLOAD is an installed `sqlload` command (the sqlload wheel's and the sqlload-cli wheel's). On Linux and macOS it must run where no unixODBC is installed, so the wheel's bundled driver manager is the only one; it must find a driver registered in an odbcinst.ini named by ODBCSYSINI and, with --default-ini, one registered in the default odbcinst.ini (/etc on Linux, Homebrew's etc on macOS). The drivers don't exist, so the driver manager reports the library it tried to load, -which shows the registration it read. On Windows, the driver manager built in (odbc32.dll) reports +which shows the registration it read. With --driver-folder (Linux), a plain shared library placed in +DIR and registered by file name alone, as Debian and Ubuntu driver packages register theirs +(Driver=psqlodbcw.so), must load: it then fails as no driver (IM004), not as a missing file. On Windows, the driver manager built in (odbc32.dll) reports an unregistered driver. """ @@ -67,6 +69,7 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument("sqlload", nargs="+") parser.add_argument("--default-ini", type=Path) + parser.add_argument("--driver-folder", type=Path) arguments = parser.parse_args() if os.name != "nt": found = [path for pattern in SYSTEM_LIBODBC for path in glob.glob(pattern)] @@ -83,6 +86,14 @@ def main(): registered(ini, "SQLLoad Smoke", "/nonexistent/libsmoke-odbcsysini.so") code, output = plan(sqlload, folder, "SQLLoad Smoke", {**os.environ, "ODBCSYSINI": str(ini.parent)}) check(code == 1 and "Can't open lib '/nonexistent/libsmoke-odbcsysini.so'" in output, f"{sqlload}: the bundled driver manager finds a driver registered under ODBCSYSINI", output) + if arguments.driver_folder: + arguments.driver_folder.mkdir(parents=True, exist_ok=True) + plain = sorted(glob.glob("/usr/lib/*-linux-gnu/libz.so.1") + glob.glob("/lib/*-linux-gnu/libz.so.1")) + check(bool(plain), "a plain shared library to stand in for a driver") + shutil.copy(plain[0], arguments.driver_folder / "libsqlload-smoke.so") + registered(ini, "SQLLoad Smoke Bare", "libsqlload-smoke.so") + code, output = plan(sqlload, folder, "SQLLoad Smoke Bare", {**os.environ, "ODBCSYSINI": str(ini.parent)}) + check(code == 1 and "IM004" in output and "file not found" not in output, f"{sqlload}: the bundled driver manager finds a driver registered by file name in {arguments.driver_folder}", output) if arguments.default_ini: previous = registered(arguments.default_ini, "SQLLoad Smoke Default", "/nonexistent/libsmoke-default.so") try: diff --git a/.github/scripts/unixodbc_source.sh b/.github/scripts/unixodbc_source.sh new file mode 100755 index 00000000..314a8c38 --- /dev/null +++ b/.github/scripts/unixodbc_source.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Downloads unixODBC's pinned source release, checks its SHA-256 and copies it into each FOLDER. +# The Linux and macOS wheels bundle the driver manager built from it (build_unixodbc.sh), and both +# sdists carry it as the library's corresponding source (LGPL-2.1 section 4). +# Usage: unixodbc_source.sh FOLDER... Prints the tarball's name. +# Keep the version in step with the NOTICE files under packages/*/licenses/. +set -euo pipefail + +UNIXODBC_VERSION=2.3.14 +UNIXODBC_SHA256=4e2814de3e01fc30b0b9f75e83bb5aba91ab0384ee951286504bb70205524771 +name="unixODBC-${UNIXODBC_VERSION}.tar.gz" + +work="$(mktemp -d)" +curl -fsSL -o "$work/$name" "https://www.unixodbc.org/$name" || + curl -fsSL -o "$work/$name" "https://github.com/lurcher/unixODBC/releases/download/v${UNIXODBC_VERSION}/$name" +if command -v sha256sum > /dev/null; then + echo "$UNIXODBC_SHA256 $work/$name" | sha256sum -c --quiet - >&2 +else + echo "$UNIXODBC_SHA256 $work/$name" | shasum -a 256 -c --quiet - >&2 +fi +for folder in "$@"; do + cp "$work/$name" "$folder/$name" +done +rm -rf "$work" +echo "$name" diff --git a/.github/scripts/windows_e2e.py b/.github/scripts/windows_e2e.py index f318405d..52cb9ca0 100644 --- a/.github/scripts/windows_e2e.py +++ b/.github/scripts/windows_e2e.py @@ -16,8 +16,8 @@ import argparse import json import os -import re import pathlib +import re import subprocess import sys @@ -32,7 +32,7 @@ def sqlload(exe, *args, config=None): command += ["--config", str(config)] command += list(args) print("$", subprocess.list2cmdline(command), flush=True) - result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", errors="replace") + result = subprocess.run(command, capture_output=True, text=True, encoding="utf-8", errors="replace", check=False) print(result.stdout + result.stderr, flush=True) if result.returncode != 0: sys.exit(f"sqlload exited {result.returncode}") diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f75d52ba..a865cbed 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,6 +15,8 @@ on: - .github/scripts/assert_*_wheel_tags.sh - .github/scripts/interrupt_python_load.py - .github/scripts/build_unixodbc.sh + - .github/scripts/unixodbc_source.sh + - .github/scripts/assert_sdist_sources.sh - .github/scripts/assert_wheel_notices.sh - .github/scripts/smoke_odbc_wheel.py - packages/*/licenses/** @@ -147,7 +149,7 @@ jobs: if: runner.os == 'macOS' run: | brew uninstall --ignore-dependencies unixodbc || true - bash .github/scripts/build_unixodbc.sh "$RUNNER_TEMP/unixodbc" "$(brew --prefix)/etc" + bash .github/scripts/build_unixodbc.sh "$RUNNER_TEMP/unixodbc" "$(brew --prefix)/etc" '' "$(brew --prefix)/lib" echo "RUSTFLAGS=-L native=$RUNNER_TEMP/unixodbc/lib" >> "$GITHUB_ENV" # sqlload-cli ships one standalone Rust executable (OpenSSL built in) and links no libpython, # so maturin tags it py3-none-. The tag is asserted before release. @@ -155,8 +157,9 @@ jobs: uses: PyO3/maturin-action@v1 with: command: build - # repair copies unixODBC into the Linux and macOS wheels; Windows uses odbc32.dll. - args: --release --out ../../dist ${{ runner.os != 'Windows' && '--auditwheel=repair' || '' }} + # The wheels build ODBC (sdists don't, so building one needs no driver manager); repair copies + # unixODBC into the Linux and macOS wheels; Windows uses odbc32.dll. + args: --release --out ../../dist --features odbc ${{ runner.os != 'Windows' && '--auditwheel=repair' || '' }} target: ${{ matrix.platform.target }} manylinux: "2_28" working-directory: packages/sqlload-cli @@ -164,7 +167,7 @@ jobs: # manylinux image leaves out. unixODBC reads /etc, where distribution drivers register. before-script-linux: | dnf install -y perl-IPC-Cmd perl-Time-Piece - bash "$GITHUB_WORKSPACE/.github/scripts/build_unixodbc.sh" /usr /etc /usr/lib64 + bash "$GITHUB_WORKSPACE/.github/scripts/build_unixodbc.sh" /usr /etc /usr/lib64 "/usr/lib/$(uname -m)-linux-gnu/odbc:/usr/lib/$(uname -m)-linux-gnu:/usr/lib64/odbc:/usr/lib64" - name: Assert native CLI wheel is interpreter-agnostic shell: bash run: bash .github/scripts/assert_cli_wheel_tags.sh dist @@ -174,14 +177,15 @@ jobs: uses: PyO3/maturin-action@v1 with: command: build - # repair copies unixODBC into the Linux and macOS wheels; Windows uses odbc32.dll. - args: --release --out ../../dist ${{ runner.os != 'Windows' && '--auditwheel=repair' || '' }} + # The wheels build ODBC (sdists don't, so building one needs no driver manager); repair copies + # unixODBC into the Linux and macOS wheels; Windows uses odbc32.dll. + args: --release --out ../../dist --features odbc ${{ runner.os != 'Windows' && '--auditwheel=repair' || '' }} target: ${{ matrix.platform.target }} manylinux: "2_28" working-directory: packages/sqlload before-script-linux: | dnf install -y perl-IPC-Cmd perl-Time-Piece - bash "$GITHUB_WORKSPACE/.github/scripts/build_unixodbc.sh" /usr /etc /usr/lib64 + bash "$GITHUB_WORKSPACE/.github/scripts/build_unixodbc.sh" /usr /etc /usr/lib64 "/usr/lib/$(uname -m)-linux-gnu/odbc:/usr/lib/$(uname -m)-linux-gnu:/usr/lib64/odbc:/usr/lib64" - name: Assert Python extension wheel targets the stable ABI shell: bash run: bash .github/scripts/assert_python_wheel_tags.sh dist @@ -210,7 +214,7 @@ jobs: docker run --rm -v "$PWD:/w" -w /w python:3.12-slim bash -c ' python -m venv /v && /v/bin/pip install --quiet dist/sqlload-*.whl && python -m venv /c && /c/bin/pip install --quiet dist/sqlload_cli-*.whl && - /v/bin/python .github/scripts/smoke_odbc_wheel.py /v/bin/sqlload /c/bin/sqlload --default-ini /etc/odbcinst.ini' + /v/bin/python .github/scripts/smoke_odbc_wheel.py /v/bin/sqlload /c/bin/sqlload --default-ini /etc/odbcinst.ini --driver-folder "/usr/lib/$(uname -m)-linux-gnu/odbc"' ;; macOS) rm -rf "$RUNNER_TEMP/unixodbc" @@ -236,6 +240,10 @@ jobs: - uses: dtolnay/rust-toolchain@master with: toolchain: 1.95.0 + # unixODBC's source, SHA-checked, into each package: the sdists carry it as the corresponding + # source of the driver manager the Linux and macOS wheels bundle (LGPL-2.1 section 4). + - name: Fetch unixODBC's source for the sdists + run: bash .github/scripts/unixodbc_source.sh packages/sqlload packages/sqlload-cli - name: Build Python extension sdist uses: PyO3/maturin-action@v1 with: @@ -248,6 +256,8 @@ jobs: command: sdist args: --out ../../dist working-directory: packages/sqlload-cli + - name: Assert every sdist carries unixODBC's source, unchanged + run: bash .github/scripts/assert_sdist_sources.sh dist - uses: actions/upload-artifact@v5 with: name: sdist @@ -286,6 +296,10 @@ jobs: shell: bash run: bash .github/scripts/assert_wheel_notices.sh dist + - name: Assert every sdist carries unixODBC's source, unchanged + shell: bash + run: bash .github/scripts/assert_sdist_sources.sh dist + - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.gitignore b/.gitignore index 90f7720e..2f05057f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,8 @@ target/ # Python packaging dist/ +# unixODBC's source, fetched into each package for its release sdist +packages/*/unixODBC-*.tar.gz .venv/ __pycache__/ diff --git a/Makefile b/Makefile index aa2768c3..72e3dccf 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ test-clickhouse-older: test-python: uv venv --quiet --allow-existing --python 3.12 .venv uv pip install --quiet --python .venv/bin/python 'maturin>=1.9,<2' pytest pytest-xdist pyarrow polars pandas orjson duckdb - cd packages/sqlload && VIRTUAL_ENV=$(CURDIR)/.venv $(CURDIR)/.venv/bin/maturin develop --uv --quiet + cd packages/sqlload && VIRTUAL_ENV=$(CURDIR)/.venv $(CURDIR)/.venv/bin/maturin develop --uv --quiet --features odbc .venv/bin/python -m pytest packages/sqlload/tests -n auto --dist loadfile test: test-unit test-containers diff --git a/crates/sqlload/src/loader/_helpers/table_plan.rs b/crates/sqlload/src/loader/_helpers/table_plan.rs index b50af79d..3c2ca6f0 100644 --- a/crates/sqlload/src/loader/_helpers/table_plan.rs +++ b/crates/sqlload/src/loader/_helpers/table_plan.rs @@ -1045,14 +1045,15 @@ pub(crate) fn source_filter( (!bounds.is_empty()).then(|| bounds.join(" AND ")) } -/// Whether `spec`'s cursor bounds bind as parameters of the cursor's type: on a source that binds them (ODBC), unless the cursor column is pinned to a type, which may not be the source column's own (a `varchar` read as a timestamp), so its bounds stay literals the database converts. +/// Whether `spec`'s cursor bounds bind as parameters of the cursor's type: on a source that binds them (ODBC), unless the cursor column is pinned to a type other than text, which may not be the source column's own (a `varchar` read as a timestamp), so its bounds stay literals the database converts; text binds as text whatever the column, as a literal would convert, without a database reading its backslashes. pub(crate) fn binds_cursor(spec: &TableSpec) -> bool { - let pinned = spec.cursor.as_ref().is_some_and(|cursor| { - spec.pinned - .keys() - .any(|name| name.eq_ignore_ascii_case(&cursor.column)) + let pinned_other = spec.cursor.as_ref().is_some_and(|cursor| { + spec.pinned.iter().any(|(name, column_type)| { + name.eq_ignore_ascii_case(&cursor.column) + && !matches!(column_type, ColumnType::Text | ColumnType::Varchar(_)) + }) }); - !pinned + !pinned_other && spec .warehouse_source .as_ref() diff --git a/crates/sqlload/src/odbc_source/_helpers/catalog.rs b/crates/sqlload/src/odbc_source/_helpers/catalog.rs index 4c9cff2b..502a3aef 100644 --- a/crates/sqlload/src/odbc_source/_helpers/catalog.rs +++ b/crates/sqlload/src/odbc_source/_helpers/catalog.rs @@ -83,10 +83,13 @@ pub(crate) fn catalog_tables( let exclude = table_matcher(exclude)?; let mut names: Vec<(String, String)> = Vec::new(); let listed = match connection.tables("", ANY_NAME, ANY_NAME, TABLE_TYPES) { - Err(_) if connection.max_schema_name_len()? == 0 => { - connection.tables("", "", ANY_NAME, TABLE_TYPES)? - } - listed => listed?, + Ok(listed) => listed, + Err(refused) => match connection.max_schema_name_len() { + Ok(0) => connection + .tables("", "", ANY_NAME, TABLE_TYPES) + .map_err(|_| refused)?, + _ => return Err(refused.into()), + }, }; for row in listed { let row = row?; diff --git a/crates/sqlload/tests/loader/odbc/helpers.rs b/crates/sqlload/tests/loader/odbc/helpers.rs index dc9f6f27..1aa2508b 100644 --- a/crates/sqlload/tests/loader/odbc/helpers.rs +++ b/crates/sqlload/tests/loader/odbc/helpers.rs @@ -66,9 +66,10 @@ cursor_column = "label" write_strategy = "append" [sources.erp.tables."erp.codes"] -description = "Codes; the first maximum holds a backslash before a quote." +description = "Codes; the first maximum holds a backslash before a quote. Pinned as text, which still binds." cursor_column = "label" write_strategy = "append" +columns = {{ label = "text" }} "#, database = database.display(), ) diff --git a/packages/sqlload-cli/licenses/unixODBC-NOTICE.md b/packages/sqlload-cli/licenses/unixODBC-NOTICE.md index 94067b4c..cf99693a 100644 --- a/packages/sqlload-cli/licenses/unixODBC-NOTICE.md +++ b/packages/sqlload-cli/licenses/unixODBC-NOTICE.md @@ -6,12 +6,16 @@ driver manager built into Windows (`odbc32.dll`). - Licence: GNU Lesser General Public License, version 2.1; its text is in `unixODBC-COPYING.txt` next to this file. -- Source: https://www.unixodbc.org/unixODBC-2.3.14.tar.gz (also - https://github.com/lurcher/unixODBC/releases/tag/v2.3.14), SHA-256 - 4e2814de3e01fc30b0b9f75e83bb5aba91ab0384ee951286504bb70205524771. +- Corresponding source: `unixODBC-2.3.14.tar.gz`, SHA-256 + 4e2814de3e01fc30b0b9f75e83bb5aba91ab0384ee951286504bb70205524771, is included in this package's + source distribution (sdist) of the same version on PyPI, at its top level. It is unixODBC's own + release, also at https://www.unixodbc.org/unixODBC-2.3.14.tar.gz and + https://github.com/lurcher/unixODBC/releases/tag/v2.3.14. - Built by `.github/scripts/build_unixodbc.sh` in https://github.com/sqlstack-io/sqlload, with - `--with-included-ltdl` and the configuration folder set to `/etc` on Linux and Homebrew's `etc` - on macOS. + `--with-included-ltdl`, the configuration folder set to `/etc` on Linux and Homebrew's `etc` on + macOS, and drivers registered by file name alone looked for in + `/usr/lib/-linux-gnu/odbc`, `/usr/lib/-linux-gnu`, `/usr/lib64/odbc` and `/usr/lib64` + on Linux and Homebrew's `lib` on macOS. SQLLoad links the library dynamically. It is installed as a separate shared library next to the package (`*.libs/libodbc-*.so.2` on Linux, `*.dylibs/libodbc-*.2.dylib` on macOS); you may diff --git a/packages/sqlload-cli/pyproject.toml b/packages/sqlload-cli/pyproject.toml index 91a9889c..418345b0 100644 --- a/packages/sqlload-cli/pyproject.toml +++ b/packages/sqlload-cli/pyproject.toml @@ -25,10 +25,13 @@ Repository = "https://github.com/sqlstack-io/sqlload" [tool.maturin] bindings = "bin" manifest-path = "../../crates/sqlload/Cargo.toml" -# Kafka TLS with OpenSSL built in; SASL PLAIN and SCRAM need nothing else. GSSAPI is left out. ODBC -# sources through the system's driver manager: unixODBC, which the Linux and macOS wheels bundle -# (licenses/), or Windows's odbc32.dll. -features = ["ssl-vendored", "odbc"] +# Kafka TLS with OpenSSL built in; SASL PLAIN and SCRAM need nothing else. GSSAPI is left out. +# The published wheels also build `odbc` (publish.yml's --features odbc), bundling unixODBC on Linux +# and macOS (licenses/); a build from this sdist leaves it out, so it needs no driver manager. +features = ["ssl-vendored"] +# unixODBC's source, which the release sdist carries as the corresponding source of the library the +# Linux and macOS wheels bundle; publish.yml fetches it (unixodbc_source.sh) and checks it is there. +include = [{ path = "unixODBC-*.tar.gz", format = "sdist" }] [tool.uv] cache-keys = [ diff --git a/packages/sqlload/licenses/unixODBC-NOTICE.md b/packages/sqlload/licenses/unixODBC-NOTICE.md index 94067b4c..cf99693a 100644 --- a/packages/sqlload/licenses/unixODBC-NOTICE.md +++ b/packages/sqlload/licenses/unixODBC-NOTICE.md @@ -6,12 +6,16 @@ driver manager built into Windows (`odbc32.dll`). - Licence: GNU Lesser General Public License, version 2.1; its text is in `unixODBC-COPYING.txt` next to this file. -- Source: https://www.unixodbc.org/unixODBC-2.3.14.tar.gz (also - https://github.com/lurcher/unixODBC/releases/tag/v2.3.14), SHA-256 - 4e2814de3e01fc30b0b9f75e83bb5aba91ab0384ee951286504bb70205524771. +- Corresponding source: `unixODBC-2.3.14.tar.gz`, SHA-256 + 4e2814de3e01fc30b0b9f75e83bb5aba91ab0384ee951286504bb70205524771, is included in this package's + source distribution (sdist) of the same version on PyPI, at its top level. It is unixODBC's own + release, also at https://www.unixodbc.org/unixODBC-2.3.14.tar.gz and + https://github.com/lurcher/unixODBC/releases/tag/v2.3.14. - Built by `.github/scripts/build_unixodbc.sh` in https://github.com/sqlstack-io/sqlload, with - `--with-included-ltdl` and the configuration folder set to `/etc` on Linux and Homebrew's `etc` - on macOS. + `--with-included-ltdl`, the configuration folder set to `/etc` on Linux and Homebrew's `etc` on + macOS, and drivers registered by file name alone looked for in + `/usr/lib/-linux-gnu/odbc`, `/usr/lib/-linux-gnu`, `/usr/lib64/odbc` and `/usr/lib64` + on Linux and Homebrew's `lib` on macOS. SQLLoad links the library dynamically. It is installed as a separate shared library next to the package (`*.libs/libodbc-*.so.2` on Linux, `*.dylibs/libodbc-*.2.dylib` on macOS); you may diff --git a/packages/sqlload/pyproject.toml b/packages/sqlload/pyproject.toml index 0db08c17..78ecd7c6 100644 --- a/packages/sqlload/pyproject.toml +++ b/packages/sqlload/pyproject.toml @@ -38,10 +38,13 @@ Repository = "https://github.com/sqlstack-io/sqlload" manifest-path = "../../crates/sqlload-python/Cargo.toml" module-name = "sqlload._native" python-source = "python" -# Kafka TLS with OpenSSL built in, as in sqlload-cli; SASL PLAIN and SCRAM need nothing else. ODBC -# sources through the system's driver manager: unixODBC, which the Linux and macOS wheels bundle -# (licenses/), or Windows's odbc32.dll. -features = ["ssl-vendored", "odbc"] +# Kafka TLS with OpenSSL built in, as in sqlload-cli; SASL PLAIN and SCRAM need nothing else. +# The published wheels also build `odbc` (publish.yml's --features odbc), bundling unixODBC on Linux +# and macOS (licenses/); a build from this sdist leaves it out, so it needs no driver manager. +features = ["ssl-vendored"] +# unixODBC's source, which the release sdist carries as the corresponding source of the library the +# Linux and macOS wheels bundle; publish.yml fetches it (unixodbc_source.sh) and checks it is there. +include = [{ path = "unixODBC-*.tar.gz", format = "sdist" }] [tool.uv] cache-keys = [ diff --git a/website/src/content/docs/docs/sources/odbc.mdx b/website/src/content/docs/docs/sources/odbc.mdx index b1c87ad2..c2794803 100644 --- a/website/src/content/docs/docs/sources/odbc.mdx +++ b/website/src/content/docs/docs/sources/odbc.mdx @@ -8,8 +8,9 @@ results from any database that has an ODBC driver. SQLLoad reads the rows as Arr driver and loads them into the source's target. From 0.37.0 (not released yet), the `sqlload` and `sqlload-cli` wheels include ODBC on every -platform. Before that, and in a `cargo install` without it, ODBC is built only with the `odbc` -feature (`cargo install sqlload --features odbc`); without it, `plan` refuses an `odbc` source. +platform. Before that, and in a build from source (`cargo install`, or pip building an sdist where +no wheel fits), ODBC is built only with the `odbc` feature (`cargo install sqlload --features odbc`); +without it, `plan` refuses an `odbc` source. ```toml [connections.erp] @@ -80,7 +81,11 @@ SQLLoad runs on; SQLLoad reaches it through a driver manager: system DSNs from `odbcinst.ini` and `odbc.ini` in `/etc` on Linux, where distribution driver packages register, and in Homebrew's `etc` on macOS (`/opt/homebrew/etc` on Apple silicon, `/usr/local/etc` on Intel), or in the folder `ODBCSYSINI` names; user DSNs from `~/.odbc.ini`. - unixODBC is LGPL-2.1; its notice and licence are in each wheel's `.dist-info/licenses`. + A driver registered by file name alone, as Debian and Ubuntu packages register theirs + (`Driver=psqlodbcw.so`), is looked for in `/usr/lib/-linux-gnu/odbc` and + `/usr/lib/-linux-gnu` (then `/usr/lib64/odbc` and `/usr/lib64`) on Linux, and Homebrew's + `lib` on macOS. unixODBC is LGPL-2.1: its notice and licence are in each wheel's + `.dist-info/licenses`, and its source is in the package's sdist on PyPI. - **`cargo install --features odbc`:** the system's unixODBC, which it links against: `unixodbc-dev` on Debian and Ubuntu, `unixodbc` with Homebrew. From 5f19ac68a55430c382079d21c191485758e19b53 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:38:23 +0000 Subject: [PATCH 24/25] ci: run the ODBC wheel smoke test's sqlload by absolute path Each plan runs in its own folder, and POSIX resolves a relative program path from there, so the macOS smoke test could not find smoke/bin/sqlload. Uninstall Homebrew's unixODBC only when it is installed, so runners without it no longer show a 'No such keg' error. --- .github/scripts/smoke_odbc_wheel.py | 3 ++- .github/workflows/publish.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/scripts/smoke_odbc_wheel.py b/.github/scripts/smoke_odbc_wheel.py index f60707b8..c52347a5 100644 --- a/.github/scripts/smoke_odbc_wheel.py +++ b/.github/scripts/smoke_odbc_wheel.py @@ -75,7 +75,8 @@ def main(): found = [path for pattern in SYSTEM_LIBODBC for path in glob.glob(pattern)] check(not found, "no unixODBC is installed, so only the wheel's driver manager can load", str(found)) for sqlload in arguments.sqlload: - sqlload = shutil.which(sqlload) or sqlload + # Absolute, as each plan runs in its own folder and POSIX resolves a relative path from there. + sqlload = os.path.abspath(shutil.which(sqlload) or sqlload) with tempfile.TemporaryDirectory() as name: folder = Path(name) if os.name == "nt": diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a865cbed..b35a07f8 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -148,7 +148,7 @@ jobs: - name: Build unixODBC for macOS if: runner.os == 'macOS' run: | - brew uninstall --ignore-dependencies unixodbc || true + if brew list --versions unixodbc > /dev/null 2>&1; then brew uninstall --ignore-dependencies unixodbc; fi bash .github/scripts/build_unixodbc.sh "$RUNNER_TEMP/unixodbc" "$(brew --prefix)/etc" '' "$(brew --prefix)/lib" echo "RUSTFLAGS=-L native=$RUNNER_TEMP/unixodbc/lib" >> "$GITHUB_ENV" # sqlload-cli ships one standalone Rust executable (OpenSSL built in) and links no libpython, From c12618c1d58b98602d0064725304d44a06633d6b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 19:16:32 +0000 Subject: [PATCH 25/25] docs: ODBC ships in the wheels from 0.38.0, as 0.37.0 is released without it --- README.md | 2 +- packages/sqlload/README.md | 2 +- website/src/content/docs/docs/index.mdx | 2 +- website/src/content/docs/docs/sources/odbc.mdx | 8 ++++---- website/src/content/docs/docs/supported-systems.mdx | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 4b4f55da..de3ae539 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ versions and how to run them yourself. | HTTP APIs and Python | ✓ | | | ODBC databases¹ | ✓ | | -¹ In the published wheels from 0.37.0 (not released yet); before that, in builds with the `odbc` feature. +¹ In the published wheels from 0.38.0 (not released yet); before that, in builds with the `odbc` feature. ² From 0.34.0, through a separate DuckDB 1.5 process: install `sqlload[motherduck]`. diff --git a/packages/sqlload/README.md b/packages/sqlload/README.md index 4b4f55da..de3ae539 100644 --- a/packages/sqlload/README.md +++ b/packages/sqlload/README.md @@ -92,7 +92,7 @@ versions and how to run them yourself. | HTTP APIs and Python | ✓ | | | ODBC databases¹ | ✓ | | -¹ In the published wheels from 0.37.0 (not released yet); before that, in builds with the `odbc` feature. +¹ In the published wheels from 0.38.0 (not released yet); before that, in builds with the `odbc` feature. ² From 0.34.0, through a separate DuckDB 1.5 process: install `sqlload[motherduck]`. diff --git a/website/src/content/docs/docs/index.mdx b/website/src/content/docs/docs/index.mdx index 36c67d3b..84cf6ff1 100644 --- a/website/src/content/docs/docs/index.mdx +++ b/website/src/content/docs/docs/index.mdx @@ -22,7 +22,7 @@ SQLLoad loads data into your warehouse, and shows you how before it changes anyt | HTTP APIs and Python | ✓ | | | ODBC databases¹ | ✓ | | -¹ In the published wheels from 0.37.0 (not released yet); before that, in builds with the `odbc` feature. +¹ In the published wheels from 0.38.0 (not released yet); before that, in builds with the `odbc` feature. ² From 0.34.0, through a separate DuckDB 1.5 process: install `sqlload[motherduck]`. diff --git a/website/src/content/docs/docs/sources/odbc.mdx b/website/src/content/docs/docs/sources/odbc.mdx index c2794803..f7ff77c3 100644 --- a/website/src/content/docs/docs/sources/odbc.mdx +++ b/website/src/content/docs/docs/sources/odbc.mdx @@ -7,7 +7,7 @@ From sqlload 0.27.0, an `odbc` source loads tables, views and query results from any database that has an ODBC driver. SQLLoad reads the rows as Arrow through the driver and loads them into the source's target. -From 0.37.0 (not released yet), the `sqlload` and `sqlload-cli` wheels include ODBC on every +From 0.38.0 (not released yet), the `sqlload` and `sqlload-cli` wheels include ODBC on every platform. Before that, and in a build from source (`cargo install`, or pip building an sdist where no wheel fits), ODBC is built only with the `odbc` feature (`cargo install sqlload --features odbc`); without it, `plan` refuses an `odbc` source. @@ -76,7 +76,7 @@ SQLLoad runs on; SQLLoad reaches it through a driver manager: - **Windows:** the driver manager built into Windows; register drivers and DSNs as usual (ODBC Data Sources). -- **Linux and macOS wheels** (from 0.37.0, not released yet): the unixODBC 2.3.14 driver manager, +- **Linux and macOS wheels** (from 0.38.0, not released yet): the unixODBC 2.3.14 driver manager, bundled as a separate shared library, so no system unixODBC is needed. It reads drivers and system DSNs from `odbcinst.ini` and `odbc.ini` in `/etc` on Linux, where distribution driver packages register, and in Homebrew's `etc` on macOS (`/opt/homebrew/etc` on Apple silicon, @@ -103,7 +103,7 @@ as names is refused when the source is read; switch it to ANSI quoting. For MySQ | OpenEdge (Progress) | Tested for timestamp handling (parsing `{ts}` values and binding timestamp bounds), not against a live server. | | Others | Not yet. A driver needs to describe prepared statements and list tables; run `sqlload plan` to see the types it reports. | -From 0.37.0 (not released yet), a cursor filter compares the cursor column with bound parameters, +From 0.38.0 (not released yet), a cursor filter compares the cursor column with bound parameters, such as `"updated_at" > ?`, each passed as the cursor's own type: a timestamp without a time zone as an ODBC `TIMESTAMP`, with every fractional digit; a date as a `DATE`; text as text. No database's literal rules can change a bound, and a millisecond maximum such as `23:59:59.997` resumes exactly. @@ -119,7 +119,7 @@ For a timestamp or date cursor, `--start-cursor`, `--end-cursor` and `cursor_sta escape values, such as `{ts '2026-01-31 23:59:59.997'}` or `{d '2026-01-31'}`. -From 0.37.0 (not released yet), a timestamp column is read in at least microseconds whatever precision +From 0.38.0 (not released yet), a timestamp column is read in at least microseconds whatever precision the driver describes: MySQL Connector/ODBC describes every `DATETIME(n)` with none, and its fractional seconds were dropped before. diff --git a/website/src/content/docs/docs/supported-systems.mdx b/website/src/content/docs/docs/supported-systems.mdx index 888ce6cf..f73e8330 100644 --- a/website/src/content/docs/docs/supported-systems.mdx +++ b/website/src/content/docs/docs/supported-systems.mdx @@ -10,7 +10,7 @@ description: "What SQLLoad reads and writes, by which route, and how each is tes | Postgres | Tables, by `include` patterns | Binary `COPY` into Arrow, over TLS when the connection's `sslmode` asks | | Postgres changes (`postgres_cdc`) | Inserts, updates and deletes of tables, by `include` patterns | A snapshot, then a logical replication slot (`pgoutput`), in whole transactions. See [Postgres changes](/docs/sources/postgres-cdc/). | | Snowflake (from 0.26.0) | Tables and views, by `include` patterns, and query results | Parquet unloaded with `COPY INTO` to the user stage or an S3 stage, then read by SQLLoad, or (from 0.27.0) by DuckDB itself from an S3 stage. See [Snowflake](/docs/sources/snowflake/). | -| Databases with an ODBC driver (from 0.27.0) | Tables and views, by `include` patterns, and query results | The driver, as Arrow; in the published wheels from 0.37.0 (not released yet), before that in builds with the `odbc` feature. See [ODBC](/docs/sources/odbc/). | +| Databases with an ODBC driver (from 0.27.0) | Tables and views, by `include` patterns, and query results | The driver, as Arrow; in the published wheels from 0.38.0 (not released yet), before that in builds with the `odbc` feature. See [ODBC](/docs/sources/odbc/). | | BigQuery (from 0.30.0) | Tables and views, by `include` patterns, and query results | The Storage Read API, as Arrow in parallel streams; views and queries through a query job. See [BigQuery](/docs/sources/bigquery/). | | Files in S3 | Parquet, CSV, JSON, NDJSON; gzip or zstd for text formats | The destination itself where it can load them exactly, otherwise SQLLoad's reader | | Files in a local folder | As for S3 | As for S3 |