feat: add a ClickHouse adapter and fix CLI rename, errors and test functions

.github/workflows/verify.yml

@@ -23,6 +23,7 @@ jobs:
       compiler_differential: ${{ steps.scope.outputs.compiler_differential }}
       duckdb: ${{ steps.scope.outputs.duckdb }}
       postgres: ${{ steps.scope.outputs.postgres }}
+      clickhouse: ${{ steps.scope.outputs.clickhouse }}
       performance: ${{ steps.scope.outputs.performance }}
       format_performance: ${{ steps.scope.outputs.format_performance }}
       rules_performance: ${{ steps.scope.outputs.rules_performance }}
@@ -50,11 +51,12 @@ jobs:
               echo "checks=$6"
               echo "wheels=$7"
               echo "compiler_differential=$8"
+              echo "clickhouse=$9"
             } >> "${GITHUB_OUTPUT}"
           }
 
           if [[ "${EVENT_NAME}" != "pull_request" && "${EVENT_NAME}" != "merge_group" ]]; then
-            write_scope true true true true true true true true
+            write_scope true true true true true true true true true
             exit 0
           fi
 
@@ -66,7 +68,7 @@ jobs:
           if [[ -z "${BASE_SHA}" || -z "${HEAD_SHA}" ]] || \
              ! git diff --name-only -z "${BASE_SHA}...${HEAD_SHA}" > "${changed_paths}"; then
             echo "Could not diff ${BASE_SHA:-<none>}...${HEAD_SHA:-<none>}; running every suite" >&2
-            write_scope true true true true true true true true
+            write_scope true true true true true true true true true
             exit 0
           fi
 
@@ -112,12 +114,13 @@ jobs:
                 && "${HEAD_REF}" == release-please--branches--* ]] && release_metadata_only; then
             echo "Release Please pull request changes only release metadata; skipping the suites" \
               | tee -a "${GITHUB_STEP_SUMMARY}"
-            write_scope false false false false false false false false
+            write_scope false false false false false false false false false
             exit 0
           fi
 
           duckdb=false
           postgres=false
+          clickhouse=false
           performance=false
           format_performance=false
           rules_performance=false
@@ -182,6 +185,9 @@ jobs:
               tests/e2e/src/sqlbuild/cli/commands/main/postgres/*|tests/integration/src/sqlbuild/adapters/postgres/*|tests/integration/src/sqlbuild/runtime/execution_history/*)
                 postgres=true
                 ;;
+              src/sqlbuild/adapters/clickhouse/*|tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/*|tests/integration/src/sqlbuild/adapters/clickhouse/*)
+                clickhouse=true
+                ;;
               tests/e2e/*)
                 duckdb=true
                 ;;
@@ -201,7 +207,7 @@ jobs:
           done < "${changed_paths}"
 
           if [[ "${changed}" != "true" ]]; then
-            write_scope true true true true true true true true
+            write_scope true true true true true true true true true
             exit 0
           fi
 
@@ -213,6 +219,7 @@ jobs:
           if [[ "${broad}" == "true" ]]; then
             duckdb=true
             postgres=true
+            clickhouse=true
             format_performance=true
             compiler_differential=true
           fi
@@ -224,7 +231,8 @@ jobs:
             "${rules_performance}" \
             "${checks}" \
             "${wheels}" \
-            "${compiler_differential}"
+            "${compiler_differential}" \
+            "${clickhouse}"
 
   lockfile:
     name: Lockfile consistency
@@ -853,6 +861,39 @@ jobs:
           -m "real_warehouse and postgres"
           -vv --color=yes -n auto --dist loadfile
 
+  clickhouse:
+    name: ClickHouse integration and E2E tests
+    needs: [changes, lockfile]
+    if: needs.changes.outputs.clickhouse == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.12.3"
+          enable-cache: true
+          cache-dependency-glob: |
+            uv.lock
+            Cargo.lock
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - name: Install Python
+        run: uv python install 3.12
+      - name: Install dependencies
+        run: uv sync --locked --all-groups --all-extras
+      - name: Verify Docker
+        run: docker info
+      - name: Run ClickHouse tests
+        env:
+          PYTHONUNBUFFERED: "1"
+          TESTCONTAINERS_RYUK_DISABLED: "true"
+        run: >-
+          uv run pytest tests
+          -m "real_warehouse and clickhouse"
+          -vv --color=yes -n auto --dist loadfile
+
   wheels:
     name: Wheels
     needs: [changes]
@@ -891,6 +932,7 @@ jobs:
         cache_compile_performance,
         rules_performance,
         postgres,
+        clickhouse,
         wheels,
       ]
     runs-on: ubuntu-24.04
@@ -913,6 +955,7 @@ jobs:
           CACHE_COMPILE_PERFORMANCE_RESULT: ${{ needs.cache_compile_performance.result }}
           RULES_PERFORMANCE_RESULT: ${{ needs.rules_performance.result }}
           POSTGRES_RESULT: ${{ needs.postgres.result }}
+          CLICKHOUSE_RESULT: ${{ needs.clickhouse.result }}
           WHEELS_RESULT: ${{ needs.wheels.result }}
         run: |
           if [[ "${CHANGES_RESULT}" != "success" || "${LOCKFILE_RESULT}" != "success" ]]; then
@@ -937,6 +980,7 @@ jobs:
             "${CACHE_COMPILE_PERFORMANCE_RESULT}" \
             "${RULES_PERFORMANCE_RESULT}" \
             "${POSTGRES_RESULT}" \
+            "${CLICKHOUSE_RESULT}" \
             "${WHEELS_RESULT}"; do
             if [[ "${result}" != "success" && "${result}" != "skipped" ]]; then
               echo "A selected verification suite did not pass: ${result}" >&2

Makefile

@@ -1,6 +1,6 @@
 SHELL := /bin/bash
 
-.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-python-boundary check-release-gate \
+.PHONY: verify verify-quick verify-pg coverage cli-preview rust-check check-native-layering check-python-boundary check-release-gate check-docs-skill \
 	check-e2e-shards test-e2e-duckdb test-e2e-duckdb-build-core \
 	test-e2e-duckdb-build-incremental \
 	test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-inspection \
@@ -153,6 +153,7 @@ E2E_DUCKDB_CLI_PATHS := \
 	tests/e2e/src/sqlbuild/cli/commands/main/audit \
 	tests/e2e/src/sqlbuild/cli/commands/main/bigquery \
 	tests/e2e/src/sqlbuild/cli/commands/main/check \
+	tests/e2e/src/sqlbuild/cli/commands/main/clickhouse \
 	tests/e2e/src/sqlbuild/cli/commands/main/compile \
 	tests/e2e/src/sqlbuild/cli/commands/main/databricks \
 	tests/e2e/src/sqlbuild/cli/commands/main/dbt \
@@ -459,6 +460,16 @@ test-real-postgres:
 	exit $$status
 
 
+test-real-clickhouse:
+	@mkdir -p /tmp/opencode
+	@log="/tmp/opencode/test-real-clickhouse-$$(date +%Y%m%d-%H%M%S).log"; \
+	echo "Logging to $$log"; \
+	env PYTHONUNBUFFERED=1 TESTCONTAINERS_RYUK_DISABLED=true SQLBUILD_CONCURRENCY=$(SQLBUILD_CONCURRENCY) uv run pytest tests -m "real_warehouse and clickhouse" -vv --color=yes -n auto --dist loadfile 2>&1 | tee "$$log"; \
+	status=$${PIPESTATUS[0]}; \
+	echo "TEST_REAL_CLICKHOUSE_EXIT=$$status (log: $$log)" | tee -a "$$log"; \
+	exit $$status
+
+
 test-real-sqlserver:
 	@mkdir -p /tmp/opencode
 	@log="/tmp/opencode/test-real-sqlserver-$$(date +%Y%m%d-%H%M%S).log"; \
@@ -626,7 +637,7 @@ verify-quick:
 	exit $$status
 
 
-check-ci: check-e2e-shards check-native-layering check-python-boundary check-release-gate rust-lint
+check-ci: check-e2e-shards check-native-layering check-python-boundary check-release-gate check-docs-skill rust-lint
 	uv run ruff format --check .
 	uv run ruff check .
 	uv run ty check src tests
@@ -646,6 +657,10 @@ check-release-gate:
 	uv run python -m scripts.check_release_gate
 
 
+check-docs-skill:
+	uv run python -m scripts.check_docs_skill
+
+
 check-pr-metadata:
 	@test -n "$(PR_TITLE)" || { echo "PR_TITLE is required" >&2; exit 2; }
 	@test -n "$(PR_BODY_FILE)" || { echo "PR_BODY_FILE is required" >&2; exit 2; }

crates/sqlbuild-analysis/src/compiler/_helpers/sql_tests/planning.rs

@@ -1455,8 +1455,21 @@ fn analyze_and_resolve_sql(
             replacements.insert((function_name.as_str(), referenced_name.as_str()), target);
         }
     }
+    let relations_resolved =
+        replace_relation_markers(request.query_sql, &replacements, request.patterns);
+    let functions_resolved = resolve_function_calls(
+        &resolve_function_calls(
+            &relations_resolved,
+            request.functions,
+            false,
+            request.patterns,
+        )?,
+        request.functions,
+        true,
+        request.patterns,
+    )?;
     let cte_body_sql = strip_statement_terminators(
-        &replace_relation_markers(request.query_sql, &replacements, request.patterns),
+        &functions_resolved,
         SliceDialect::new(Some(request.dialect_name)),
     )
     .to_string();

crates/sqlbuild-analysis/src/compiler/tests.rs

@@ -1,5 +1,7 @@
 #[path = "tests/helpers.rs"]
 mod helpers;
+#[path = "tests/planning/helpers.rs"]
+mod planning_helpers;
 #[path = "tests/test_scenario_expected_columns.rs"]
 mod scenario_expected_columns;
 #[path = "tests/test_scenario_helper_ctes.rs"]

crates/sqlbuild-analysis/src/compiler/tests/helpers.rs

@@ -1559,7 +1559,7 @@ pub(super) fn snowflake_function_synonyms_stay_as_authored() -> bool {
         && !sql.contains("SUBSTRING")
 }
 
-fn generic_lexical_syntax() -> Value {
+pub(crate) fn generic_lexical_syntax() -> Value {
     json!({
         "backslashEscapeQuotes": [],
         "escapeStringPrefix": false,

crates/sqlbuild-analysis/src/compiler/tests/planning/helpers.rs

@@ -0,0 +1,31 @@
+use serde_json::{Value, json};
+
+use crate::compiler::tests::helpers::{generic_lexical_syntax, plan_json};
+
+/// The rendered SQL of a model test whose real model calls a UDF in its projection.
+pub(crate) fn projected_udf_call_sql(dialect: &str) -> String {
+    let request = json!({
+        "lexicalSyntax": generic_lexical_syntax(),
+        "models": [
+            {"name": "orders", "querySql": "SELECT o.order_id, __udf(\"is_open\")(o.status) AS is_open FROM __source(\"raw_orders\") AS o"}
+        ],
+        "functions": [{"name": "is_open", "udfPrefix": "analytics__is_open", "udfSuffix": ""}],
+        "tests": [{
+            "name": "orders_case", "fileLabel": "tests/orders.sql",
+            "payload": {
+                "kind": "model",
+                "authoredCtes": [{"name": "__source__raw_orders", "sqlBody": "SELECT 1 AS order_id, 'open' AS status"}],
+                "expectedCtes": [{"name": "__expected__orders", "sqlBody": "SELECT 1 AS order_id, TRUE AS is_open"}],
+                "expectedModelNames": ["orders"]
+            }
+        }],
+        "sqlAnalysisEnabled": true, "sqlAnalysisDialect": dialect
+    });
+    let response: Value =
+        serde_json::from_str(&plan_json(&request.to_string()).expect("planning succeeds"))
+            .expect("valid JSON");
+    response["artifacts"][0]["sql"]
+        .as_str()
+        .expect("rendered SQL")
+        .to_owned()
+}

crates/sqlbuild-analysis/src/compiler/tests/test_sql_test_planning.rs

@@ -17,9 +17,10 @@ use crate::compiler::tests::helpers::{
     unicode_cte_after_leading_with_preserves_identifier,
     unresolved_reference_fast_rejection_preserves_warning, upstream_fallback_resolves,
 };
+use crate::compiler::tests::planning_helpers::projected_udf_call_sql;
 use crate::compiler::tests::test_types::{
     GlueErrorProjectionTestCase, GluePlanParityTestCase, GluePlannerErrorTestCase,
-    SqlTestPlanningTestCase,
+    ProjectedUdfTestCase, SqlTestPlanningTestCase,
 };
 
 /// A mocked model test, a test missing its source mock and a UDF test over two models.
@@ -324,3 +325,33 @@ fn given_plan_warnings_when_projecting_errors_then_each_distinct_error_message_i
         );
     }
 }
+
+#[test]
+fn given_real_model_calling_udf_in_projection_when_planning_then_call_targets_the_function() {
+    let test_cases = [
+        ProjectedUdfTestCase {
+            description: "DuckDB falls back to textual resolution",
+            dialect: "duckdb",
+            expected_call: "analytics__is_open(o.status)",
+        },
+        ProjectedUdfTestCase {
+            description: "ClickHouse parses the marker as a parametric call",
+            dialect: "clickhouse",
+            expected_call: "analytics__is_open(o.status)",
+        },
+    ];
+
+    for test_case in test_cases {
+        let sql = projected_udf_call_sql(test_case.dialect);
+
+        assert_eq!(
+            (
+                sql.contains(test_case.expected_call),
+                sql.contains("__udf(")
+            ),
+            (true, false),
+            "{}: {sql}",
+            test_case.description
+        );
+    }
+}

crates/sqlbuild-analysis/src/compiler/tests/test_types.rs

@@ -227,3 +227,9 @@ pub(crate) struct ScenarioHelperShadowTestCase {
     pub(crate) check_sql: &'static str,
     pub(crate) expected_rejected: bool,
 }
+
+pub(super) struct ProjectedUdfTestCase {
+    pub(super) description: &'static str,
+    pub(super) dialect: &'static str,
+    pub(super) expected_call: &'static str,
+}

crates/sqlbuild-analysis/src/planner_identity/constants.rs

@@ -1,7 +1,7 @@
 //! Planner identity constants shared with the Python planner.
 
 /// Model config keys whose values participate in version identity, in Python's insertion order after `cursor_inputs`.
-pub const VERSION_IDENTITY_CONFIG_KEYS: [&str; 34] = [
+pub const VERSION_IDENTITY_CONFIG_KEYS: [&str; 39] = [
     "append_cursor_inclusive",
     "batch_size",
     "check_columns",
@@ -36,6 +36,11 @@ pub const VERSION_IDENTITY_CONFIG_KEYS: [&str; 34] = [
     "updated_at",
     "valid_from_column",
     "valid_to_column",
+    "engine",
+    "order_by",
+    "partition_by",
+    "primary_key",
+    "table_settings",
 ];
 /// Model config key of cursor inputs, recorded before every other identity key.
 pub const CURSOR_INPUTS_CONFIG_KEY: &str = "cursor_inputs";

crates/sqlbuild-analysis/src/planner_strategy/constants.rs

@@ -86,3 +86,15 @@ pub const PERMANENT_TABLE_TYPE: &str = "permanent";
 pub const TRANSIENT_TABLE_DDL: &str = "TRANSIENT TABLE";
 /// The permanent table DDL replacing the transient keyword.
 pub const PERMANENT_TABLE_DDL: &str = "TABLE";
+/// The warning code of an incremental model whose table keeps an outdated physical layout.
+pub const PHYSICAL_LAYOUT_PENDING_WARNING_CODE: &str = "PHYSICAL_LAYOUT_PENDING";
+/// Model config keys of a table's physical layout, in the order warnings name them.
+pub const PHYSICAL_LAYOUT_CONFIG_KEYS: [&str; 5] = [
+    "engine",
+    "order_by",
+    "partition_by",
+    "primary_key",
+    "table_settings",
+];
+/// The version metadata member holding the model config.
+pub const METADATA_CONFIG_MEMBER: &str = "config";

crates/sqlbuild-analysis/src/planner_strategy/main/mod.rs

@@ -13,6 +13,7 @@ pub mod microbatch_limit_warning;
 pub mod microbatch_lookback;
 pub mod model_plan_action;
 pub mod model_warnings;
+pub mod physical_layout_warning;
 pub mod schema_actions;
 pub mod singular_attachment;
 pub mod table_type_ddl;