diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs index 0500b3c29b..46bc71a1ad 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs @@ -434,7 +434,7 @@ fn check_engine(config: &ConfigView<'_, N>) -> Check { _ => Err(ValidationStop::Error( config .config_error(format!( - "{ENGINE_KEY} '{}' must be a single-server MergeTree-family table engine", + "{ENGINE_KEY} '{}' must be a non-replicated MergeTree-family table engine", value.python_str() )) .with_help(model_header_help( diff --git a/crates/sqlbuild-model-config/src/model_validation/constants.rs b/crates/sqlbuild-model-config/src/model_validation/constants.rs index fdef85e14e..74d56179a8 100644 --- a/crates/sqlbuild-model-config/src/model_validation/constants.rs +++ b/crates/sqlbuild-model-config/src/model_validation/constants.rs @@ -300,5 +300,5 @@ pub const PHYSICAL_STORAGE_KEYS: [&str; 5] = [ pub const PHYSICAL_STORAGE_MATERIALIZATIONS: [&str; 2] = ["table", "incremental"]; /// The name suffix every supported table engine shares. pub const ENGINE_FAMILY_SUFFIX: &str = "MergeTree"; -/// Engine name prefixes of replicated and shared engines, which need a cluster. -pub const UNSUPPORTED_ENGINE_PREFIXES: [&str; 2] = ["Replicated", "Shared"]; +/// Engine name prefixes of replicated engines, which need a cluster. +pub const UNSUPPORTED_ENGINE_PREFIXES: [&str; 1] = ["Replicated"]; diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs b/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs index 06366bc6d4..c367ec9999 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs @@ -50,6 +50,40 @@ fn given_physical_layout_keys_when_validating_then_shapes_and_materializations_a query_sql: "select 1", expected_outcome: "model 'orders_daily': order_by is only valid for table and incremental models", }, + ValidateTestCase { + description: "a shared engine", + config: table(vec![("engine", Value::Str("SharedMergeTree"))]), + references: &[], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "a shared replacing engine with arguments", + config: table(vec![( + "engine", + Value::Str("SharedReplacingMergeTree(updated_at)"), + )]), + references: &[], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "a replicated replacing engine", + config: table(vec![( + "engine", + Value::Str("ReplicatedReplacingMergeTree(updated_at)"), + )]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': engine 'ReplicatedReplacingMergeTree(updated_at)' must be a non-replicated MergeTree-family table engine", + }, + ValidateTestCase { + description: "a shared engine outside the MergeTree family", + config: table(vec![("engine", Value::Str("SharedSet"))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': engine 'SharedSet' must be a non-replicated MergeTree-family table engine", + }, ValidateTestCase { description: "a replicated engine", config: table(vec![( @@ -58,14 +92,14 @@ fn given_physical_layout_keys_when_validating_then_shapes_and_materializations_a )]), references: &[], query_sql: "select 1", - expected_outcome: "model 'orders_daily': engine 'ReplicatedMergeTree('/tables/orders', '{replica}')' must be a single-server MergeTree-family table engine", + expected_outcome: "model 'orders_daily': engine 'ReplicatedMergeTree('/tables/orders', '{replica}')' must be a non-replicated MergeTree-family table engine", }, ValidateTestCase { description: "an engine outside the MergeTree family", config: table(vec![("engine", Value::Str("Log"))]), references: &[], query_sql: "select 1", - expected_outcome: "model 'orders_daily': engine 'Log' must be a single-server MergeTree-family table engine", + expected_outcome: "model 'orders_daily': engine 'Log' must be a non-replicated MergeTree-family table engine", }, ValidateTestCase { description: "a blank sorting expression", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md index b8bdd8f42d..83c01eecc8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md @@ -113,13 +113,15 @@ MODEL ( | Field | Value | Default | |-------|-------|---------| -| `engine` | A single-server MergeTree-family engine, with its arguments | `MergeTree` | +| `engine` | A non-replicated MergeTree-family engine, with its arguments | `MergeTree` | | `order_by` | A sorting expression, or a list of them | `tuple()` (unsorted) | | `partition_by` | A partitioning expression | No partitioning | | `primary_key` | A prefix of `order_by` | The sorting key | | `table_settings` | Setting names mapped to strings, integers or booleans | None | -Quote expressions that are not plain column names. Compile rejects malformed values, replicated and shared engines, and these fields on views and snapshots; other adapters reject them with `A011` when planning. +Quote expressions that are not plain column names. Compile rejects malformed values, `Replicated*` engines, and these fields on views and snapshots; other adapters reject them with `A011` when planning. + +ClickHouse Cloud's `Shared*` engines (`SharedMergeTree`, `SharedReplacingMergeTree` and the other `SharedMergeTree` variants) are accepted and handled like their `MergeTree` counterparts. Support is best effort: SQLBuild is not tested on ClickHouse Cloud. ClickHouse Cloud already creates a `MergeTree` table as `SharedMergeTree`, so naming the plain engine works there too. The fields are part of the model's version, so `sqb plan` reports a changed layout as a config change. A table model is rebuilt with the new layout. An incremental model keeps writing into its existing table with the old layout, and `sqb plan` warns with `PHYSICAL_LAYOUT_PENDING` until a full refresh (`sqb build --full-refresh`, with `--start-cursor-ts` and `--end-cursor-ts` for a cursor model) rebuilds the table with the new layout. @@ -181,7 +183,7 @@ ClickHouse returns a type's default value (`0`, the epoch, an empty string) from ## Limits -- Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. +- Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. ClickHouse Cloud and its `Shared*` engines are best effort and untested. - No transactions. Each step above is designed so a rerun converges instead. - Concurrent `sqb` processes writing the same model can lose rows; see [One writer per model](#one-writer-per-model). - Snapshots always use an unsorted `MergeTree` table. diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/test_table_layout.py b/tests/unit/src/sqlbuild/adapters/clickhouse/test_table_layout.py index ff381b1932..3305b4054f 100644 --- a/tests/unit/src/sqlbuild/adapters/clickhouse/test_table_layout.py +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/test_table_layout.py @@ -39,6 +39,18 @@ layout={"order_by": "order_id"}, expected_clauses="ENGINE = MergeTree ORDER BY (order_id)", ), + TableLayoutTestCase( + description="a shared engine", + layout={"engine": "SharedMergeTree", "partition_by": "toYYYYMM(ordered_at)"}, + expected_clauses=( + "ENGINE = SharedMergeTree ORDER BY tuple() PARTITION BY toYYYYMM(ordered_at)" + ), + ), + TableLayoutTestCase( + description="a shared replacing engine with its version column", + layout={"engine": "SharedReplacingMergeTree(version)", "order_by": ["customer_id"]}, + expected_clauses="ENGINE = SharedReplacingMergeTree(version) ORDER BY (customer_id)", + ), ], ids=lambda case: case.description, ) diff --git a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx index 5aa6becff3..3876b673df 100644 --- a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx +++ b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx @@ -100,13 +100,15 @@ MODEL ( | Field | Value | Default | |-------|-------|---------| -| `engine` | A single-server MergeTree-family engine, with its arguments | `MergeTree` | +| `engine` | A non-replicated MergeTree-family engine, with its arguments | `MergeTree` | | `order_by` | A sorting expression, or a list of them | `tuple()` (unsorted) | | `partition_by` | A partitioning expression | No partitioning | | `primary_key` | A prefix of `order_by` | The sorting key | | `table_settings` | Setting names mapped to strings, integers or booleans | None | -Quote expressions that are not plain column names. Compile rejects malformed values, replicated and shared engines, and these fields on views and snapshots; other adapters reject them with `A011` when planning. +Quote expressions that are not plain column names. Compile rejects malformed values, `Replicated*` engines, and these fields on views and snapshots; other adapters reject them with `A011` when planning. + +ClickHouse Cloud's `Shared*` engines (`SharedMergeTree`, `SharedReplacingMergeTree` and the other `SharedMergeTree` variants) are accepted and handled like their `MergeTree` counterparts. Support is best effort: SQLBuild is not tested on ClickHouse Cloud. ClickHouse Cloud already creates a `MergeTree` table as `SharedMergeTree`, so naming the plain engine works there too. The fields are part of the model's version, so `sqb plan` reports a changed layout as a config change. A table model is rebuilt with the new layout. An incremental model keeps writing into its existing table with the old layout, and `sqb plan` warns with `PHYSICAL_LAYOUT_PENDING` until a full refresh (`sqb build --full-refresh`, with `--start-cursor-ts` and `--end-cursor-ts` for a cursor model) rebuilds the table with the new layout. @@ -168,7 +170,7 @@ ClickHouse returns a type's default value (`0`, the epoch, an empty string) from ## Limits -- Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. +- Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. ClickHouse Cloud and its `Shared*` engines are best effort and untested. - No transactions. Each step above is designed so a rerun converges instead. - Concurrent `sqb` processes writing the same model can lose rows; see [One writer per model](#one-writer-per-model). - Snapshots always use an unsorted `MergeTree` table.