Merge fix/bundled-duckdb-json (41547ab): compile DuckDB's json extension into bundled builds

Author: kvlonge

.github/workflows/build-distributions.yml

@@ -34,6 +34,11 @@ on:
         required: false
         type: number
         default: 0
+      platforms:
+        description: "Comma-separated wheel platforms to build, such as linux-x86_64; empty builds all"
+        required: false
+        type: string
+        default: ""
     outputs:
       sha:
         description: "Commit the distributions were built from"
@@ -49,6 +54,7 @@ jobs:
     outputs:
       sha: ${{ steps.source.outputs.sha }}
       tree: ${{ steps.source.outputs.tree }}
+      platforms: ${{ steps.platforms.outputs.platforms }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -61,6 +67,28 @@ jobs:
           echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
           echo "tree=${tree}" >> "${GITHUB_OUTPUT}"
           echo "Distributions for commit ${sha} with tree ${tree}" >> "${GITHUB_STEP_SUMMARY}"
+      # The wheel platform matrix. DuckDB's C++ needs aligned deallocation, available from macOS 10.13.
+      - name: Choose the wheel platforms
+        id: platforms
+        env:
+          ONLY: ${{ inputs.platforms }}
+        run: |
+          all='[
+            {"name": "linux-x86_64", "runner": "ubuntu-24.04", "target": "x86_64"},
+            {"name": "linux-aarch64", "runner": "ubuntu-24.04-arm", "target": "aarch64"},
+            {"name": "macos-x86_64", "runner": "macos-15-intel", "target": "x86_64", "macos-target": "10.15"},
+            {"name": "macos-arm64", "runner": "macos-15", "target": "aarch64", "macos-target": "11.0"},
+            {"name": "windows-x86_64", "runner": "windows-2025", "target": "x86_64"}
+          ]'
+          chosen="$(jq -c --arg only "${ONLY}" \
+            'if $only == "" then . else [.[] | select(.name as $name | $only | split(",") | index($name))] end' \
+            <<< "${all}")"
+          if [[ "${chosen}" == "[]" ]]; then
+            echo "No wheel platform is named ${ONLY}" >&2
+            exit 1
+          fi
+          echo "platforms=${chosen}" >> "${GITHUB_OUTPUT}"
+          echo "Wheel platforms: $(jq -r 'map(.name) | join(", ")' <<< "${chosen}")" >> "${GITHUB_STEP_SUMMARY}"
       - name: Download the prebuilt manifest
         if: inputs.prebuilt-run-id != ''
         uses: actions/download-artifact@v4
@@ -92,25 +120,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform:
-          - name: linux-x86_64
-            runner: ubuntu-24.04
-            target: x86_64
-          - name: linux-aarch64
-            runner: ubuntu-24.04-arm
-            target: aarch64
-          # DuckDB's C++ needs aligned deallocation, available from macOS 10.13.
-          - name: macos-x86_64
-            runner: macos-15-intel
-            target: x86_64
-            macos-target: "10.15"
-          - name: macos-arm64
-            runner: macos-15
-            target: aarch64
-            macos-target: "11.0"
-          - name: windows-x86_64
-            runner: windows-2025
-            target: x86_64
+        platform: ${{ fromJSON(needs.source.outputs.platforms) }}
     steps:
       - uses: actions/checkout@v4
         if: inputs.prebuilt-run-id == ''

.github/workflows/distributions.yml

@@ -1,7 +1,12 @@
 name: Distributions
 
-# Builds and smoke tests every wheel and sdist when a pull request changes what goes into them.
+# Builds and smoke tests every wheel and sdist when a pull request changes what goes into them. On
+# every push to main it builds and smoke tests the linux-x86_64 wheels only, and runs the command
+# line's convert tests with DuckDB compiled in, so a distribution-only failure shows at the landing
+# rather than at the next release. Neither blocks a landing.
 on:
+  push:
+    branches: [main]
   pull_request:
     paths:
       - "python/**"
@@ -25,3 +30,23 @@ jobs:
   distributions:
     name: Distributions
     uses: ./.github/workflows/build-distributions.yml
+    with:
+      platforms: ${{ github.event_name == 'push' && 'linux-x86_64' || '' }}
+      sdist: ${{ github.event_name != 'push' }}
+
+  # The command line built with `bundled` alone, as the sqlquack wheel compiles DuckDB in: no system
+  # DuckDB and none of the `extensions` feature's extra extensions. It runs only the zone-free convert
+  # test until the bundled build compiles in ICU; then it runs the full convert test.
+  bundled-convert:
+    name: Convert with DuckDB compiled in
+    runs-on: ubuntu-24.04
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install the pinned Rust toolchain
+        run: rustup toolchain install
+      - uses: Swatinem/rust-cache@v2
+      - name: Run the convert tests
+        run: >-
+          cargo test --locked -p sqlquack --features bundled --test cli
+          given_zone_free_snowflake_sql_when_converting_then_the_checking_engine_binds_it

Makefile

@@ -30,7 +30,7 @@ PYTHON_TYPED := scripts tests/unit/scripts python/src $(PYTHON_API_TESTS)
 PYTHON_TESTS := tests/unit/scripts $(PYTHON_API_TESTS)
 endif
 
-.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage depth-probe depth-probe-debug gate-build polyglot-guard engine-free dialect-builds generate-cases generated-cases generate-syntax-cases syntax-cases dialect-cases corpus-rerecord generate-referee-cases referee-cases referee-pending translation-files translation-results help pending check-syntax record-syntax check-convert record-convert sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-module python-api python-wheel verify-recorded evidence-clone evidence-refresh evidence-pin evidence-pin-check clutter-guard polyglot-duckshop census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure
+.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage depth-probe depth-probe-debug gate-build polyglot-guard engine-free bundled-json dialect-builds generate-cases generated-cases generate-syntax-cases syntax-cases dialect-cases corpus-rerecord generate-referee-cases referee-cases referee-pending translation-files translation-results help pending check-syntax record-syntax check-convert record-convert sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-module python-api python-wheel verify-recorded evidence-clone evidence-refresh evidence-pin evidence-pin-check clutter-guard polyglot-duckshop census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure
 
 format:
 	cargo fmt --all
@@ -116,6 +116,14 @@ engine-free:
 	    echo "sqlquack-convert reaches DuckDB's engine" >&2; exit 1; \
 	  fi
 
+# Fails when DuckDB compiled into a distribution lacks its json extension, which the checking
+# engine's helper macros need: the sqlquack wheel and the sqlquack-cli wheel both bundle DuckDB.
+bundled-json:
+	@features="$$(cargo tree --locked -p sqlquack-blocks --features bundled -e features -i duckdb --prefix none)" || exit 1; \
+	  grep -q '^duckdb feature "json"' <<<"$$features" || { \
+	    echo "sqlquack-blocks' bundled feature compiles DuckDB without its json extension" >&2; exit 1; \
+	  }
+
 # The crates whose features include dialects, as consumers build them one dialect at a time.
 DIALECT_CRATES ?= sqlquack-semantics sqlquack-parse sqlquack-fingerprint sqlquack-format sqlquack-lint \
   sqlquack-convert sqlquack-diagnostics
@@ -199,7 +207,7 @@ referee-pending:
 	uv run python -m scripts.write_referee_pending $(REFEREE_ARGS)
 
 # The static gate, the first step of `gate`, and the Python API's tests.
-check-ci: rust-lint python-check fensu syntax-coverage depth-probe evidence-pin-check clutter-guard polyglot-guard engine-free dialect-builds generated-cases syntax-cases dialect-cases referee-cases translation-files python-api distribution-notices ## The static gate CI runs.
+check-ci: rust-lint python-check fensu syntax-coverage depth-probe evidence-pin-check clutter-guard polyglot-guard engine-free bundled-json dialect-builds generated-cases syntax-cases dialect-cases referee-cases translation-files python-api distribution-notices ## The static gate CI runs.
 
 # Each distribution ships the licence and third-party notices; its copies must match the root files.
 distribution-notices:

crates/sqlquack-blocks/Cargo.toml

@@ -15,11 +15,13 @@ rust-version.workspace = true
 [features]
 default = []
 engine = ["dep:duckdb"]
-# Compiles DuckDB into the binary, for distributions that must not need a system DuckDB.
-bundled = ["engine", "duckdb/bundled"]
-# DuckDB's parquet and json extensions compiled in, so `sqlquack run` reads Parquet fixtures and
-# JSON columns without downloading an extension.
-extensions = ["bundled", "duckdb/parquet", "duckdb/json"]
+# Compiles DuckDB into the binary, for distributions that must not need a system DuckDB. Its json
+# extension comes with it: the checking engine's helper macros call json functions, so without it
+# every conversion fails its check.
+bundled = ["engine", "duckdb/bundled", "duckdb/json"]
+# DuckDB's parquet extension compiled in as well, so `sqlquack run` reads Parquet fixtures without
+# downloading an extension.
+extensions = ["bundled", "duckdb/parquet"]
 
 [dependencies]
 duckdb = { workspace = true, optional = true }

crates/sqlquack/tests/cli.rs

@@ -2,6 +2,8 @@
 mod helpers;
 #[path = "cli/test_builtin_policy.rs"]
 mod test_builtin_policy;
+#[path = "cli/test_bundled.rs"]
+mod test_bundled;
 #[path = "cli/test_check.rs"]
 mod test_check;
 #[path = "cli/test_compare.rs"]

crates/sqlquack/tests/cli/test_bundled.rs

@@ -0,0 +1,40 @@
+use crate::helpers::{convert_outcome, run_convert};
+use crate::test_types::ConvertTestCase;
+
+// The distributions workflow also runs this test with DuckDB compiled in (`--features bundled`), as
+// the wheels build it. It holds only zone-free statements: the compiled-in DuckDB lacks ICU until
+// the bundled build gains it, and then that job also runs the full convert test in test_convert.rs.
+#[test]
+fn given_zone_free_snowflake_sql_when_converting_then_the_checking_engine_binds_it() {
+    let test_cases = [
+        ConvertTestCase {
+            description: "a constant converts once the checking engine binds it",
+            sql: "SELECT 1 AS total",
+            flags: &[],
+            mention: "sqlquack convert: done, converted under utc-v1",
+            expected_outcome: convert_outcome(0, true),
+        },
+        ConvertTestCase {
+            description: "a VARIANT path converts with DuckDB's json functions",
+            sql: "SELECT PARSE_JSON('{\"total\": 3}'):total AS total",
+            flags: &[],
+            mention: "sqlquack convert: done, converted under utc-v1",
+            expected_outcome: convert_outcome(0, true),
+        },
+        ConvertTestCase {
+            description: "a statement refused for its own reason still names it",
+            sql: "WITH totals AS (SELECT 1 AS \"orderTotal\") SELECT ordertotal FROM totals",
+            flags: &[],
+            mention: " --> <stdin>:1:",
+            expected_outcome: convert_outcome(1, false),
+        },
+    ];
+    for test_case in test_cases {
+        assert_eq!(
+            run_convert(test_case.sql, test_case.flags, test_case.mention),
+            test_case.expected_outcome,
+            "{}",
+            test_case.description
+        );
+    }
+}