From 293215f3169abe1e374fcddd6e4537477ac2f43c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:15:24 +0000 Subject: [PATCH 01/10] feat(config): add consumer_group_identity for default Kafka consumer groups A project can set consumer_group_identity in sqltide_project.toml to use another value in place of name when deriving default Kafka consumer groups. Per-source consumer_group still wins, and leaving the key unset keeps today's groups and DDL unchanged. The value travels through the authored and effective configuration, the compile inputs and the compiled project to the managed source realization request. The shape baseline records the new trailing defaulted fields. --- crates/sqltide-clickhouse/src/realization.rs | 62 +++++++++++++++---- crates/sqltide-discovery/src/config.rs | 18 +++++- crates/sqltide-discovery/src/effective.rs | 2 + .../sqltide-python/src/clickhouse_adapter.rs | 3 + crates/sqltide-python/src/compilation.rs | 4 ++ crates/sqltide-python/src/config.rs | 8 +++ crates/sqltide-python/src/project_analysis.rs | 4 ++ crates/sqltide-python/src/realization.rs | 4 ++ .../src/shapes/adapter_models.rs | 1 + .../src/shapes/compiler_project.rs | 2 + .../src/shapes/discovery_models.rs | 2 + scripts/shape_surface/baseline.json | 10 +-- src/sqltide/_native.pyi | 5 ++ .../sqltide/compiler/discovery/_test_types.py | 7 +++ .../discovery/test_project_configuration.py | 57 +++++++++++++++++ 15 files changed, 171 insertions(+), 18 deletions(-) diff --git a/crates/sqltide-clickhouse/src/realization.rs b/crates/sqltide-clickhouse/src/realization.rs index c40bff38..93e7f449 100644 --- a/crates/sqltide-clickhouse/src/realization.rs +++ b/crates/sqltide-clickhouse/src/realization.rs @@ -49,6 +49,9 @@ pub struct ManagedSourceRequest<'a> { pub format: &'a str, pub consumer_group: Option<&'a str>, pub project_name: Option<&'a str>, + /// The project's `consumer_group_identity`, which replaces `project_name` in the default + /// consumer group when set. + pub consumer_group_identity: Option<&'a str>, pub target_name: Option<&'a str>, pub setting_keys: Vec<&'a str>, } @@ -99,18 +102,11 @@ pub fn realize_managed_source( let kafka_table_name: String = format!("{KAFKA_TABLE_PREFIX}{}", request.logical_name); let consumer_group: String = match request.consumer_group { Some(group) if !group.is_empty() => group.to_owned(), - // The pre-rename group, so existing sources resume from their committed offsets. - _ => { - let segments: Vec<&str> = [ - request.project_name, - request.target_name, - Some(request.logical_name), - ] - .into_iter() - .flatten() - .collect(); - format!("streambuild_{}", segments.join("_")) - } + _ => default_consumer_group( + request.consumer_group_identity.or(request.project_name), + request.target_name, + request.logical_name, + ), }; Ok(ManagedSourceRealization { raw_table_name: format!("{RAW_TABLE_PREFIX}{}", request.logical_name), @@ -121,6 +117,22 @@ pub fn realize_managed_source( }) } +/// The consumer group a Kafka source uses when it declares none: the project identity (its +/// `consumer_group_identity`, else its name), target and logical name. The `streambuild_` +/// prefix is the pre-rename one, so existing sources resume from their committed offsets. +#[must_use] +pub fn default_consumer_group( + identity: Option<&str>, + target_name: Option<&str>, + logical_name: &str, +) -> String { + let segments: Vec<&str> = [identity, target_name, Some(logical_name)] + .into_iter() + .flatten() + .collect(); + format!("streambuild_{}", segments.join("_")) +} + fn landing_query(kafka_table_name: &str) -> String { format!( "SELECT\n _key AS kafka_key,\n message AS kafka_value,\n _topic AS kafka_topic,\n \ @@ -183,6 +195,7 @@ mod tests { format: "JSONAsString", consumer_group, project_name: Some("shop"), + consumer_group_identity: None, target_name: None, setting_keys: vec!["kafka_num_consumers"], } @@ -204,6 +217,31 @@ mod tests { ); } + #[test] + fn consumer_group_identity_replaces_the_project_name_in_the_default_group_only() { + let mut renamed = request(None); + renamed.project_name = Some("storefront"); + renamed.target_name = Some("dev"); + assert_eq!( + realize_managed_source(&renamed).unwrap().consumer_group, + "streambuild_storefront_dev_orders" + ); + renamed.consumer_group_identity = Some("shop"); + assert_eq!( + realize_managed_source(&renamed).unwrap().consumer_group, + "streambuild_shop_dev_orders" + ); + renamed.consumer_group = Some("orders-reader"); + assert_eq!( + realize_managed_source(&renamed).unwrap().consumer_group, + "orders-reader" + ); + assert_eq!( + default_consumer_group(None, None, "orders"), + "streambuild_orders" + ); + } + #[test] fn rejects_unsupported_kinds_formats_and_typed_setting_overrides() { let mut bad = request(None); diff --git a/crates/sqltide-discovery/src/config.rs b/crates/sqltide-discovery/src/config.rs index 0f6fb1e3..d434c59b 100644 --- a/crates/sqltide-discovery/src/config.rs +++ b/crates/sqltide-discovery/src/config.rs @@ -16,8 +16,9 @@ use crate::retention::{ use crate::text::python_strip; use crate::value::{Kind, Value}; -const PROJECT_CONFIG_KEYS: [&str; 15] = [ +const PROJECT_CONFIG_KEYS: [&str; 16] = [ "name", + "consumer_group_identity", "adapter", "default_target", "connection", @@ -116,6 +117,9 @@ pub type Pairs = Vec<(String, usize)>; #[derive(Clone, Debug, PartialEq)] pub struct ProjectConfig { pub name: String, + /// Replaces `name` in default Kafka consumer group names, so a renamed project keeps + /// consuming from its committed offsets. + pub consumer_group_identity: Option, pub adapter: String, pub default_target: String, pub connection: Pairs, @@ -337,6 +341,17 @@ impl Parser<'_> { return self .error("project.name must be a committed literal and cannot be interpolated"); } + let consumer_group_identity: Option = + self.optional_non_empty(payload, "consumer_group_identity", "project")?; + if consumer_group_identity + .as_deref() + .is_some_and(|identity| identity.contains(INTERPOLATION_TOKEN_START)) + { + return self.error( + "project.consumer_group_identity must be a committed literal and cannot be \ + interpolated", + ); + } let adapter: String = self .optional_non_empty(payload, "adapter", "project")? .unwrap_or_else(|| DEFAULT_ADAPTER_NAME.to_owned()); @@ -354,6 +369,7 @@ impl Parser<'_> { let dependencies: Dependencies = self.dependencies(field(payload, "dependencies"))?; Ok(ProjectConfig { name, + consumer_group_identity, adapter, default_target, connection, diff --git a/crates/sqltide-discovery/src/effective.rs b/crates/sqltide-discovery/src/effective.rs index a324777a..f1796398 100644 --- a/crates/sqltide-discovery/src/effective.rs +++ b/crates/sqltide-discovery/src/effective.rs @@ -26,6 +26,7 @@ pub struct Naming { #[derive(Clone, Debug, PartialEq)] pub struct EffectiveConfig { pub name: String, + pub consumer_group_identity: Option, pub adapter: String, pub target_name: String, pub production_target: bool, @@ -163,6 +164,7 @@ pub fn resolve_effective_config( let project_sensors = project_side.map(|target| target.sensors); Ok(EffectiveConfig { name: project_config.name.clone(), + consumer_group_identity: project_config.consumer_group_identity.clone(), adapter, target_name, production_target: project_target.is_some_and(|target| target.production), diff --git a/crates/sqltide-python/src/clickhouse_adapter.rs b/crates/sqltide-python/src/clickhouse_adapter.rs index e3418ff1..0c4631ff 100644 --- a/crates/sqltide-python/src/clickhouse_adapter.rs +++ b/crates/sqltide-python/src/clickhouse_adapter.rs @@ -196,6 +196,8 @@ fn realize_clickhouse_source<'py>(request: &Bound<'py, PyAny>) -> PyResult = request.getattr("consumer_group")?.extract()?; let project_name: Option = request.getattr("project_name")?.extract()?; + let consumer_group_identity: Option = + request.getattr("consumer_group_identity")?.extract()?; let target_name: Option = request.getattr("target_name")?.extract()?; let setting_keys: Vec = request .getattr("settings")? @@ -211,6 +213,7 @@ fn realize_clickhouse_source<'py>(request: &Bound<'py, PyAny>) -> PyResult Compiler<'py> { "allowed_cross_pipeline_references", inputs.getattr("allowed_cross_pipeline_references")?, ), + ( + "consumer_group_identity", + inputs.getattr("consumer_group_identity")?, + ), ], )?; self.validate_relation_names(&project)?; diff --git a/crates/sqltide-python/src/config.rs b/crates/sqltide-python/src/config.rs index 25c60832..f1a3407a 100644 --- a/crates/sqltide-python/src/config.rs +++ b/crates/sqltide-python/src/config.rs @@ -298,6 +298,10 @@ impl<'py> Document<'py> { ], )?, ), + ( + "consumer_group_identity", + self.object(config.consumer_group_identity.as_deref())?, + ), ], ) } @@ -1082,6 +1086,10 @@ impl<'py> Document<'py> { ], )?, ), + ( + "consumer_group_identity", + self.object(config.consumer_group_identity.as_deref())?, + ), ], ) } diff --git a/crates/sqltide-python/src/project_analysis.rs b/crates/sqltide-python/src/project_analysis.rs index 8ac821ad..030cde8b 100644 --- a/crates/sqltide-python/src/project_analysis.rs +++ b/crates/sqltide-python/src/project_analysis.rs @@ -347,6 +347,10 @@ fn build_compile_inputs<'py>( "allowed_cross_pipeline_references", allowed_cross_pipeline_references, ), + ( + "consumer_group_identity", + optional(&configuration, "consumer_group_identity")?, + ), ], ) } diff --git a/crates/sqltide-python/src/realization.rs b/crates/sqltide-python/src/realization.rs index d05a4f0a..0aa93959 100644 --- a/crates/sqltide-python/src/realization.rs +++ b/crates/sqltide-python/src/realization.rs @@ -291,6 +291,10 @@ impl<'py> Realizer<'py> { ("consumer_group", kafka.getattr("consumer_group")?), ("format", kafka.getattr("format")?), ("project_name", project.getattr("project_name")?), + ( + "consumer_group_identity", + project.getattr("consumer_group_identity")?, + ), ("target_name", project.getattr("target_name")?), ("ttl", ttl), ("settings", settings), diff --git a/crates/sqltide-python/src/shapes/adapter_models.rs b/crates/sqltide-python/src/shapes/adapter_models.rs index 2fe68840..f9dd6635 100644 --- a/crates/sqltide-python/src/shapes/adapter_models.rs +++ b/crates/sqltide-python/src/shapes/adapter_models.rs @@ -130,6 +130,7 @@ shapes! { ttl(default = Given::None), settings(default = Given::EmptyTuple), naming_macro_fingerprint(default = Given::None), + consumer_group_identity(default = Given::None), } /// Logical adopted-source fields needed for adapter realization. diff --git a/crates/sqltide-python/src/shapes/compiler_project.rs b/crates/sqltide-python/src/shapes/compiler_project.rs index b263e8fd..ccd522a9 100644 --- a/crates/sqltide-python/src/shapes/compiler_project.rs +++ b/crates/sqltide-python/src/shapes/compiler_project.rs @@ -463,6 +463,7 @@ shapes! { coerce = Coerce::Enum("ModelReferenceScope"), ), allowed_cross_pipeline_references(default = Given::EmptyTuple, coerce = Coerce::Tuple), + consumer_group_identity(default = Given::None), } /// Current whole-project compile output consumed by every command. @@ -484,6 +485,7 @@ shapes! { coerce = Coerce::Enum("ModelReferenceScope"), ), allowed_cross_pipeline_references(default = Given::EmptyTuple, coerce = Coerce::Tuple), + consumer_group_identity(default = Given::None), } } diff --git a/crates/sqltide-python/src/shapes/discovery_models.rs b/crates/sqltide-python/src/shapes/discovery_models.rs index 605df845..1d167d43 100644 --- a/crates/sqltide-python/src/shapes/discovery_models.rs +++ b/crates/sqltide-python/src/shapes/discovery_models.rs @@ -271,6 +271,7 @@ shapes! { build(default = Given::Factory(factory::)), ui(default = Given::Factory(factory::)), dependencies(default = Given::Factory(factory::)), + consumer_group_identity(default = Given::None), } /// Optional local TOML overrides before invocation resolution. @@ -303,6 +304,7 @@ shapes! { ui(default = Given::Factory(factory::)), production_target(default = Given::Bool(false)), dependencies(default = Given::Factory(factory::)), + consumer_group_identity(default = Given::None), } /// Kafka engine settings for a landing step. diff --git a/scripts/shape_surface/baseline.json b/scripts/shape_surface/baseline.json index 511c7d93..96ec6b41 100644 --- a/scripts/shape_surface/baseline.json +++ b/scripts/shape_surface/baseline.json @@ -60,7 +60,7 @@ "sqltide.adapter.models:AdapterIdentity": {"fields": [["name", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["name"], "module": "sqltide.adapter.models", "mro": ["sqltide.adapter.models.AdapterIdentity", "builtins.object"], "qualname": "AdapterIdentity"}, "sqltide.adapter.models:AdapterInvocationRecord": {"fields": [["invocation_id", false, true, true], ["project_identity", false, true, true], ["target_identity", false, true, true], ["command", false, true, true], ["mode", false, true, true], ["outcome", false, true, true], ["exit_code", false, true, true], ["materialized_outcome", false, true, true], ["deployment_id", false, true, true], ["workflow_id", false, true, true], ["selected_node_count", false, true, true], ["started_at", false, true, true], ["completed_at", false, true, true], ["duration_ms", false, true, true], ["error_message", false, true, true], ["summary_json", false, true, true], ["tool_version", false, true, true], ["artifact_project_dir", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["invocation_id", "project_identity", "target_identity", "command", "mode", "outcome", "exit_code", "materialized_outcome", "deployment_id", "workflow_id", "selected_node_count", "started_at", "completed_at", "duration_ms", "error_message", "summary_json", "tool_version", "artifact_project_dir"], "module": "sqltide.adapter.models", "mro": ["sqltide.adapter.models.AdapterInvocationRecord", "builtins.object"], "qualname": "AdapterInvocationRecord"}, "sqltide.adapter.models:AdapterManagedSource": {"fields": [["source_kind", false, true, true], ["name", false, true, true], ["columns", false, true, true], ["broker_list", false, true, true], ["topic", false, true, true], ["consumer_group", false, true, true], ["format", false, true, true], ["settings", true, true, true], ["naming_macro_fingerprint", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["source_kind", "name", "columns", "broker_list", "topic", "consumer_group", "format", "settings", "naming_macro_fingerprint"], "module": "sqltide.adapter.models", "mro": ["sqltide.adapter.models.AdapterManagedSource", "builtins.object"], "qualname": "AdapterManagedSource"}, -"sqltide.adapter.models:AdapterManagedSourceRealizationRequest": {"fields": [["logical_name", false, true, true], ["source_kind", false, true, true], ["broker_list", false, true, true], ["topic", false, true, true], ["consumer_group", false, true, true], ["format", false, true, true], ["project_name", true, true, true], ["target_name", true, true, true], ["ttl", true, true, true], ["settings", true, true, true], ["naming_macro_fingerprint", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["logical_name", "source_kind", "broker_list", "topic", "consumer_group", "format", "project_name", "target_name", "ttl", "settings", "naming_macro_fingerprint"], "module": "sqltide.adapter.models", "mro": ["sqltide.adapter.models.AdapterManagedSourceRealizationRequest", "builtins.object"], "qualname": "AdapterManagedSourceRealizationRequest"}, +"sqltide.adapter.models:AdapterManagedSourceRealizationRequest": {"fields": [["logical_name", false, true, true], ["source_kind", false, true, true], ["broker_list", false, true, true], ["topic", false, true, true], ["consumer_group", false, true, true], ["format", false, true, true], ["project_name", true, true, true], ["target_name", true, true, true], ["ttl", true, true, true], ["settings", true, true, true], ["naming_macro_fingerprint", true, true, true], ["consumer_group_identity", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["logical_name", "source_kind", "broker_list", "topic", "consumer_group", "format", "project_name", "target_name", "ttl", "settings", "naming_macro_fingerprint", "consumer_group_identity"], "module": "sqltide.adapter.models", "mro": ["sqltide.adapter.models.AdapterManagedSourceRealizationRequest", "builtins.object"], "qualname": "AdapterManagedSourceRealizationRequest"}, "sqltide.adapter.models:AdapterManifest": {"fields": [["manifest_id", false, true, true], ["invocation_id", false, true, true], ["project_identity", false, true, true], ["target_name", false, true, true], ["target_database", false, true, true], ["is_production", false, true, true], ["project_revision", false, true, true], ["manifest_fingerprint", false, true, true], ["manifest_version", false, true, true], ["pipelines", false, true, true], ["resources", false, true, true], ["tool_version", false, true, true], ["published_at", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["manifest_id", "invocation_id", "project_identity", "target_name", "target_database", "is_production", "project_revision", "manifest_fingerprint", "manifest_version", "pipelines", "resources", "tool_version", "published_at"], "module": "sqltide.adapter.models", "mro": ["sqltide.adapter.models.AdapterManifest", "builtins.object"], "qualname": "AdapterManifest"}, "sqltide.adapter.models:AdapterManifestResource": {"fields": [["pipeline_name", false, true, true], ["logical_type", false, true, true], ["logical_name", false, true, true], ["resource_role", false, true, true], ["resource_database", false, true, true], ["resource_name", false, true, true], ["resource_kind", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["pipeline_name", "logical_type", "logical_name", "resource_role", "resource_database", "resource_name", "resource_kind"], "module": "sqltide.adapter.models", "mro": ["sqltide.adapter.models.AdapterManifestResource", "builtins.object"], "qualname": "AdapterManifestResource"}, "sqltide.adapter.models:AdapterManifestSnapshot": {"fields": [["status", false, true, true], ["manifests", false, true, true], ["warning", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["status", "manifests", "warning"], "module": "sqltide.adapter.models", "mro": ["sqltide.adapter.models.AdapterManifestSnapshot", "builtins.object"], "qualname": "AdapterManifestSnapshot"}, @@ -311,10 +311,10 @@ "sqltide.compiler.compile.exceptions:TransformSqlUntypedProjectionError": {"doc": "Raised when an outermost projection is not an explicitly typed alias.", "instance": {"args": ["probe-transform_name"], "repr": "TransformSqlUntypedProjectionError('probe-transform_name')", "str": "Transform 'probe-transform_name' has an invalid output contract at column 3. Expected outermost projection 3 to use an explicit typed alias like CAST(expr AS Type) AS name or expr::Type AS name, but found `probe-projection_sql`."}, "kind": "exception", "module": "sqltide.compiler.compile.exceptions", "mro": ["sqltide.compiler.compile.exceptions.TransformSqlUntypedProjectionError", "sqltide.compiler.compile.exceptions.TransformSqlContractError", "builtins.Exception", "builtins.BaseException", "builtins.object"], "probe": {"args": ["probe-transform_name", 3, "probe-projection_sql"], "kwargs": {}}, "qualname": "TransformSqlUntypedProjectionError"}, "sqltide.compiler.compile.exceptions:TransformTtlUnknownColumnError": {"doc": "Raised when a TTL expression references unknown derived columns.", "instance": {"args": ["probe-transform_name"], "repr": "TransformTtlUnknownColumnError('probe-transform_name')", "str": "Transform 'probe-transform_name' has an invalid TTL expression `probe-expression`. Referenced output columns not found in the derived schema: unknown_column_names-a, unknown_column_names-b. Available columns: available_column_names-a, available_column_names-b."}, "kind": "exception", "module": "sqltide.compiler.compile.exceptions", "mro": ["sqltide.compiler.compile.exceptions.TransformTtlUnknownColumnError", "sqltide.compiler.compile.exceptions.TransformSqlContractError", "builtins.Exception", "builtins.BaseException", "builtins.object"], "probe": {"args": ["probe-transform_name", "probe-expression", {"tuple": ["unknown_column_names-a", "unknown_column_names-b"]}, {"tuple": ["available_column_names-a", "available_column_names-b"]}], "kwargs": {}}, "qualname": "TransformTtlUnknownColumnError"}, "sqltide.compiler.compile.models:Column": {"fields": [["name", false, true, true], ["type", false, true, true], ["default", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["name", "type", "default"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.Column", "builtins.object"], "qualname": "Column"}, -"sqltide.compiler.compile.models:CompileProjectInputs": {"fields": [["discovered_inputs", false, true, true], ["adapter_profile", false, true, true], ["effective_target", false, true, true], ["variables", false, true, true], ["macro_registry", false, true, true], ["macro_context", false, true, true], ["pipelines", false, true, true], ["tests", false, true, true], ["audits", false, true, true], ["sources", true, true, true], ["virtual_environments", true, true, true], ["project_name", true, true, true], ["target_name", true, true, true], ["model_reference_scope", true, true, true], ["allowed_cross_pipeline_references", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["discovered_inputs", "adapter_profile", "effective_target", "variables", "macro_registry", "macro_context", "pipelines", "tests", "audits", "sources", "virtual_environments", "project_name", "target_name", "model_reference_scope", "allowed_cross_pipeline_references"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompileProjectInputs", "builtins.object"], "qualname": "CompileProjectInputs"}, +"sqltide.compiler.compile.models:CompileProjectInputs": {"fields": [["discovered_inputs", false, true, true], ["adapter_profile", false, true, true], ["effective_target", false, true, true], ["variables", false, true, true], ["macro_registry", false, true, true], ["macro_context", false, true, true], ["pipelines", false, true, true], ["tests", false, true, true], ["audits", false, true, true], ["sources", true, true, true], ["virtual_environments", true, true, true], ["project_name", true, true, true], ["target_name", true, true, true], ["model_reference_scope", true, true, true], ["allowed_cross_pipeline_references", true, true, true], ["consumer_group_identity", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["discovered_inputs", "adapter_profile", "effective_target", "variables", "macro_registry", "macro_context", "pipelines", "tests", "audits", "sources", "virtual_environments", "project_name", "target_name", "model_reference_scope", "allowed_cross_pipeline_references", "consumer_group_identity"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompileProjectInputs", "builtins.object"], "qualname": "CompileProjectInputs"}, "sqltide.compiler.compile.models:CompiledModel": {"fields": [["key", false, true, true], ["pipeline_name", false, true, true], ["relation_name", false, true, true], ["kind", false, true, true], ["sql_analysis", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["key", "pipeline_name", "relation_name", "kind", "sql_analysis"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompiledModel", "builtins.object"], "qualname": "CompiledModel"}, "sqltide.compiler.compile.models:CompiledPipeline": {"fields": [["pipeline", false, true, true], ["project", false, true, true], ["file_path", false, true, true], ["effective_replay_lineage_mode", false, true, true], ["source", false, true, true], ["models", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["pipeline", "project", "file_path", "effective_replay_lineage_mode", "source", "models"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompiledPipeline", "builtins.object"], "qualname": "CompiledPipeline"}, -"sqltide.compiler.compile.models:CompiledProject": {"fields": [["sources", false, true, true], ["models", false, true, true], ["pipelines", false, true, true], ["tests", false, true, true], ["test_cases", false, true, true], ["audits", false, true, true], ["macro_registry", true, true, true], ["macro_context", true, true, true], ["project_name", true, true, true], ["target_name", true, true, true], ["production_target", true, true, true], ["destruction_relation_drop_size_limit", true, true, true], ["model_reference_scope", true, true, true], ["allowed_cross_pipeline_references", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["sources", "models", "pipelines", "tests", "test_cases", "audits", "macro_registry", "macro_context", "project_name", "target_name", "production_target", "destruction_relation_drop_size_limit", "model_reference_scope", "allowed_cross_pipeline_references"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompiledProject", "builtins.object"], "qualname": "CompiledProject"}, +"sqltide.compiler.compile.models:CompiledProject": {"fields": [["sources", false, true, true], ["models", false, true, true], ["pipelines", false, true, true], ["tests", false, true, true], ["test_cases", false, true, true], ["audits", false, true, true], ["macro_registry", true, true, true], ["macro_context", true, true, true], ["project_name", true, true, true], ["target_name", true, true, true], ["production_target", true, true, true], ["destruction_relation_drop_size_limit", true, true, true], ["model_reference_scope", true, true, true], ["allowed_cross_pipeline_references", true, true, true], ["consumer_group_identity", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["sources", "models", "pipelines", "tests", "test_cases", "audits", "macro_registry", "macro_context", "project_name", "target_name", "production_target", "destruction_relation_drop_size_limit", "model_reference_scope", "allowed_cross_pipeline_references", "consumer_group_identity"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompiledProject", "builtins.object"], "qualname": "CompiledProject"}, "sqltide.compiler.compile.models:CompiledSource": {"fields": [["key", false, true, true], ["source", false, true, true], ["effective_replay_lineage_mode", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["key", "source", "effective_replay_lineage_mode"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompiledSource", "builtins.object"], "qualname": "CompiledSource"}, "sqltide.compiler.compile.models:CompiledTableModel": {"fields": [["key", false, true, true], ["pipeline_name", false, true, true], ["relation_name", false, true, true], ["kind", false, true, true], ["sql_analysis", false, true, true], ["transform", false, true, true], ["preserves_required_lineage", false, true, true], ["replay_anchor_eligible", false, true, true], ["effective_bounded_replay_fallback", false, true, true], ["replay_on_change", true, true, true], ["retention", true, true, true], ["retention_applied", true, true, true], ["on_row_error", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["key", "pipeline_name", "relation_name", "kind", "sql_analysis", "transform", "preserves_required_lineage", "replay_anchor_eligible", "effective_bounded_replay_fallback", "replay_on_change", "retention", "retention_applied", "on_row_error"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompiledTableModel", "sqltide.compiler.compile.models.CompiledModel", "builtins.object"], "qualname": "CompiledTableModel"}, "sqltide.compiler.compile.models:CompiledViewModel": {"fields": [["key", false, true, true], ["pipeline_name", false, true, true], ["relation_name", false, true, true], ["kind", false, true, true], ["sql_analysis", false, true, true], ["view", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["key", "pipeline_name", "relation_name", "kind", "sql_analysis", "view"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompiledViewModel", "sqltide.compiler.compile.models.CompiledModel", "builtins.object"], "qualname": "CompiledViewModel"}, @@ -364,7 +364,7 @@ "sqltide.compiler.discovery.models:AuditDefaults": {"fields": [["severity", true, true, true], ["cadence_seconds", true, true, true], ["warmup_seconds", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["severity", "cadence_seconds", "warmup_seconds"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.AuditDefaults", "builtins.object"], "qualname": "AuditDefaults"}, "sqltide.compiler.discovery.models:AuditSchedulerConfig": {"fields": [["enabled", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["enabled"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.AuditSchedulerConfig", "builtins.object"], "qualname": "AuditSchedulerConfig"}, "sqltide.compiler.discovery.models:AuditSchedulerOverride": {"fields": [["enabled", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["enabled"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.AuditSchedulerOverride", "builtins.object"], "qualname": "AuditSchedulerOverride"}, -"sqltide.compiler.discovery.models:AuthoredProjectConfig": {"fields": [["name", false, true, true], ["adapter", false, true, true], ["default_target", false, true, true], ["connection", false, true, true], ["variables", false, true, true], ["targets", false, true, true], ["defaults", true, true, true], ["naming", true, true, true], ["audit_scheduler", true, true, true], ["sensors", true, true, true], ["build", true, true, true], ["ui", true, true, true], ["dependencies", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["name", "adapter", "default_target", "connection", "variables", "targets", "defaults", "naming", "audit_scheduler", "sensors", "build", "ui", "dependencies"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.AuthoredProjectConfig", "builtins.object"], "qualname": "AuthoredProjectConfig"}, +"sqltide.compiler.discovery.models:AuthoredProjectConfig": {"fields": [["name", false, true, true], ["adapter", false, true, true], ["default_target", false, true, true], ["connection", false, true, true], ["variables", false, true, true], ["targets", false, true, true], ["defaults", true, true, true], ["naming", true, true, true], ["audit_scheduler", true, true, true], ["sensors", true, true, true], ["build", true, true, true], ["ui", true, true, true], ["dependencies", true, true, true], ["consumer_group_identity", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["name", "adapter", "default_target", "connection", "variables", "targets", "defaults", "naming", "audit_scheduler", "sensors", "build", "ui", "dependencies", "consumer_group_identity"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.AuthoredProjectConfig", "builtins.object"], "qualname": "AuthoredProjectConfig"}, "sqltide.compiler.discovery.models:BuildConfig": {"fields": [["max_pipelines", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["max_pipelines"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.BuildConfig", "builtins.object"], "qualname": "BuildConfig"}, "sqltide.compiler.discovery.models:DeploymentReadinessDefaults": {"fields": [["maximum_lag_seconds", true, true, true], ["minimum_staged_row_ratio", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["maximum_lag_seconds", "minimum_staged_row_ratio"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.DeploymentReadinessDefaults", "builtins.object"], "qualname": "DeploymentReadinessDefaults"}, "sqltide.compiler.discovery.models:DestructionConfig": {"fields": [["max_table_size_to_drop_bytes", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["max_table_size_to_drop_bytes"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.DestructionConfig", "builtins.object"], "qualname": "DestructionConfig"}, @@ -372,7 +372,7 @@ "sqltide.compiler.discovery.models:DiscoveredProjectFile": {"fields": [["file_path", false, true, true], ["relative_path", false, true, true], ["contents", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["file_path", "relative_path", "contents"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.DiscoveredProjectFile", "builtins.object"], "qualname": "DiscoveredProjectFile"}, "sqltide.compiler.discovery.models:DiscoveredProjectInputs": {"fields": [["project_dir", false, true, true], ["loaded_project", false, true, true], ["source_files", false, true, true], ["pipeline_directories", false, true, true], ["model_files", false, true, true], ["test_files", false, true, true], ["audit_files", false, true, true], ["macro_files", false, true, true], ["access_file", true, true, true], ["constant_files", true, true, true], ["enum_files", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["project_dir", "loaded_project", "source_files", "pipeline_directories", "model_files", "test_files", "audit_files", "macro_files", "access_file", "constant_files", "enum_files"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.DiscoveredProjectInputs", "builtins.object"], "qualname": "DiscoveredProjectInputs"}, "sqltide.compiler.discovery.models:DiscoveredSourceFile": {"fields": [["source_file", false, true, true], ["sources", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["source_file", "sources"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.DiscoveredSourceFile", "builtins.object"], "qualname": "DiscoveredSourceFile"}, -"sqltide.compiler.discovery.models:EffectiveProjectConfiguration": {"fields": [["name", false, true, true], ["adapter", false, true, true], ["target_name", false, true, true], ["database", false, true, true], ["connection", false, true, true], ["variables", false, true, true], ["defaults", false, true, true], ["naming", true, true, true], ["audit_scheduler", true, true, true], ["sensors", true, true, true], ["build", true, true, true], ["destruction", true, true, true], ["ui", true, true, true], ["production_target", true, true, true], ["dependencies", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["name", "adapter", "target_name", "database", "connection", "variables", "defaults", "naming", "audit_scheduler", "sensors", "build", "destruction", "ui", "production_target", "dependencies"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.EffectiveProjectConfiguration", "builtins.object"], "qualname": "EffectiveProjectConfiguration"}, +"sqltide.compiler.discovery.models:EffectiveProjectConfiguration": {"fields": [["name", false, true, true], ["adapter", false, true, true], ["target_name", false, true, true], ["database", false, true, true], ["connection", false, true, true], ["variables", false, true, true], ["defaults", false, true, true], ["naming", true, true, true], ["audit_scheduler", true, true, true], ["sensors", true, true, true], ["build", true, true, true], ["destruction", true, true, true], ["ui", true, true, true], ["production_target", true, true, true], ["dependencies", true, true, true], ["consumer_group_identity", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["name", "adapter", "target_name", "database", "connection", "variables", "defaults", "naming", "audit_scheduler", "sensors", "build", "destruction", "ui", "production_target", "dependencies", "consumer_group_identity"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.EffectiveProjectConfiguration", "builtins.object"], "qualname": "EffectiveProjectConfiguration"}, "sqltide.compiler.discovery.models:ExecutionSettings": {"fields": [["replay", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["replay"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.ExecutionSettings", "builtins.object"], "qualname": "ExecutionSettings"}, "sqltide.compiler.discovery.models:ExternalTableSourceStep": {"fields": [["name", false, true, true], ["kind", false, true, true], ["table_name", false, true, true], ["replay_boundary", false, true, true], ["freshness", true, true, true], ["name_origin", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["name", "kind", "table_name", "replay_boundary", "freshness", "name_origin"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.ExternalTableSourceStep", "builtins.object"], "qualname": "ExternalTableSourceStep"}, "sqltide.compiler.discovery.models:KafkaLandingStep": {"fields": [["name", false, true, true], ["kafka", false, true, true], ["replay_boundary", true, true, true], ["freshness", true, true, true], ["name_origin", true, true, true], ["naming_macro", true, true, true], ["naming_macro_fingerprint", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["name", "kafka", "replay_boundary", "freshness", "name_origin", "naming_macro", "naming_macro_fingerprint"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.KafkaLandingStep", "builtins.object"], "qualname": "KafkaLandingStep"}, diff --git a/src/sqltide/_native.pyi b/src/sqltide/_native.pyi index 79910231..7c993267 100644 --- a/src/sqltide/_native.pyi +++ b/src/sqltide/_native.pyi @@ -850,6 +850,7 @@ class CompileProjectInputs: target_name: str | None = None model_reference_scope: ModelReferenceScope | str = ModelReferenceScope.PROJECT allowed_cross_pipeline_references: tuple[AllowedCrossPipelineReference, ...] = () + consumer_group_identity: str | None = None @final @dataclass(frozen=True) @@ -868,6 +869,7 @@ class CompiledProject: destruction_relation_drop_size_limit: int | None = None model_reference_scope: ModelReferenceScope | str = ModelReferenceScope.PROJECT allowed_cross_pipeline_references: tuple[AllowedCrossPipelineReference, ...] = () + consumer_group_identity: str | None = None @final @dataclass(frozen=True) @@ -3413,6 +3415,7 @@ class AuthoredProjectConfig: build: BuildConfig = ... ui: UiConfig = ... dependencies: ProjectDependencies = ... + consumer_group_identity: str | None = None @final @dataclass(frozen=True, repr=False) @@ -3450,6 +3453,7 @@ class EffectiveProjectConfiguration: ui: UiConfig = ... production_target: bool = False dependencies: ProjectDependencies = ... + consumer_group_identity: str | None = None @final @dataclass(frozen=True, repr=False) @@ -3753,6 +3757,7 @@ class AdapterManagedSourceRealizationRequest: ttl: str | None = None settings: tuple[tuple[str, str], ...] = () naming_macro_fingerprint: str | None = None + consumer_group_identity: str | None = None @final @dataclass(frozen=True) diff --git a/tests/unit/src/sqltide/compiler/discovery/_test_types.py b/tests/unit/src/sqltide/compiler/discovery/_test_types.py index 323771dc..c93a9589 100644 --- a/tests/unit/src/sqltide/compiler/discovery/_test_types.py +++ b/tests/unit/src/sqltide/compiler/discovery/_test_types.py @@ -350,6 +350,13 @@ class ProjectProductionTargetTestCase: expected_production_target: bool +@dataclass(frozen=True) +class ProjectConsumerGroupIdentityTestCase: + description: str + identity_toml: str + expected_identity: str | None + + @dataclass(frozen=True) class ModelStepErrorTestCase: description: str diff --git a/tests/unit/src/sqltide/compiler/discovery/test_project_configuration.py b/tests/unit/src/sqltide/compiler/discovery/test_project_configuration.py index 0618988e..875e4cf6 100644 --- a/tests/unit/src/sqltide/compiler/discovery/test_project_configuration.py +++ b/tests/unit/src/sqltide/compiler/discovery/test_project_configuration.py @@ -33,6 +33,7 @@ ProjectBuildLimitResolutionTestCase, ProjectConfigurationErrorTestCase, ProjectConnectionSettingsTestCase, + ProjectConsumerGroupIdentityTestCase, ProjectDependencyScopeTestCase, ProjectDeploymentReadinessDefaultsTestCase, ProjectDestructionLimitResolutionTestCase, @@ -158,6 +159,46 @@ def test_given_committed_production_target_when_resolving_then_classification_is assert effective.production_target is test_case.expected_production_target +@pytest.mark.parametrize( + "test_case", + [ + ProjectConsumerGroupIdentityTestCase( + description="leaves the consumer group identity unset by default", + identity_toml="", + expected_identity=None, + ), + ProjectConsumerGroupIdentityTestCase( + description="retains a committed consumer group identity", + identity_toml='consumer_group_identity = "shop"\n', + expected_identity="shop", + ), + ], + ids=lambda case: case.description, +) +def test_given_consumer_group_identity_when_resolving_then_it_reaches_effective_configuration( + test_case: ProjectConsumerGroupIdentityTestCase, + tmp_path: Path, +) -> None: + write_project_toml( + project_dir=tmp_path, + contents=( + f'name = "storefront"\n{test_case.identity_toml}default_target = "dev"\n' + '[targets.dev]\ndatabase = "storefront_dev"\n' + ), + ) + loaded: LoadedProjectConfiguration = load_project_configuration(project_dir=tmp_path) + + effective: EffectiveProjectConfiguration = resolve_effective_project_configuration( + loaded=loaded, + selected_target="dev", + cli_variables={}, + environment={}, + ) + + assert loaded.project.consumer_group_identity == test_case.expected_identity + assert effective.consumer_group_identity == test_case.expected_identity + + @pytest.mark.parametrize( "test_case", [ @@ -991,6 +1032,22 @@ def test_given_absent_local_toml_when_loading_then_returns_typed_defaults( project_contents=('name = "${project_name}"\ndefault_target = "dev"\n[targets.dev]\n'), expected_error_fragment="project.name must be a committed literal", ), + ProjectConfigurationErrorTestCase( + description="rejects interpolation of the consumer group identity", + project_contents=( + 'name = "analytics"\nconsumer_group_identity = "${identity}"\n' + 'default_target = "dev"\n[targets.dev]\n' + ), + expected_error_fragment="project.consumer_group_identity must be a committed literal", + ), + ProjectConfigurationErrorTestCase( + description="rejects an empty consumer group identity", + project_contents=( + 'name = "analytics"\nconsumer_group_identity = " "\n' + 'default_target = "dev"\n[targets.dev]\n' + ), + expected_error_fragment="project.consumer_group_identity must be a non-empty string", + ), ProjectConfigurationErrorTestCase( description="rejects invalid deployment readiness lag", project_contents=( From b3606e57f621d3d5bf60230c1b83af3b149a4fc2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:15:31 +0000 Subject: [PATCH 02/10] feat(cli): add sqt rename-project sqt rename-project NEW_NAME previews a project rename for the selected target and applies it with --apply: - keeps every Kafka consumer group: when a managed source uses the default group, consumer_group_identity is set to the old name, and the command re-derives every group before and after and refuses any change - copies the target's manifest history to the new identity under the target mutation lock, skipping manifests already copied, so destroy planning still finds relations recorded before the rename - moves project role assignments in the control store in one audited transaction - edits sqltide_project.toml last, splicing value spans so comments and formatting are kept; a failed write moves the role assignments back It refuses when the name is unchanged or invalid, a run holds the target mutation lock, the new name already has its own manifest history, or both names hold the same active role assignment. --- .../sqltide-cli/src/argv/command_table.json | 149 ++++ crates/sqltide-clickhouse/src/metadata.rs | 32 + crates/sqltide-config/src/lib.rs | 1 + crates/sqltide-config/src/project_rename.rs | 141 ++++ crates/sqltide-config/src/tests.rs | 3 + .../src/tests/test_project_rename.rs | 83 +++ crates/sqltide-python/src/cli/dispatch.rs | 22 + crates/sqltide-python/src/cli/invocation.rs | 3 +- crates/sqltide-python/src/cli/mod.rs | 2 +- .../sqltide-python/src/control_store/mod.rs | 180 +++++ .../sqltide-python/src/control_store/sql.rs | 2 + crates/sqltide-python/src/lib.rs | 2 + crates/sqltide-python/src/realization.rs | 38 ++ .../src/rename_project_command.rs | 636 ++++++++++++++++++ .../sqltide-python/src/shapes/cli_service.rs | 2 + scripts/shape_surface/baseline.json | 4 +- src/sqltide/_native.pyi | 13 + .../src/sqltide/cli/test_help_baseline.py | 7 +- 18 files changed, 1315 insertions(+), 5 deletions(-) create mode 100644 crates/sqltide-config/src/project_rename.rs create mode 100644 crates/sqltide-config/src/tests/test_project_rename.rs create mode 100644 crates/sqltide-python/src/rename_project_command.rs diff --git a/crates/sqltide-cli/src/argv/command_table.json b/crates/sqltide-cli/src/argv/command_table.json index c78e90e4..43a556bf 100644 --- a/crates/sqltide-cli/src/argv/command_table.json +++ b/crates/sqltide-cli/src/argv/command_table.json @@ -3373,6 +3373,151 @@ ] } }, + { + "name": "rename-project", + "parser": { + "prog": "sqt rename-project", + "description": "Rename the project while keeping its Kafka consumer groups, manifest history and project role grants. Dry-run by default; use --apply to rename.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "new_name", + "default": null, + "required": true, + "help": "New project name", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--project-dir" + ], + "dest": "project_dir", + "default": null, + "type": "Path", + "help": "Path to the project root containing pipelines/", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--host" + ], + "dest": "host", + "default": null, + "help": "ClickHouse host", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--port" + ], + "dest": "port", + "default": null, + "type": "int", + "help": "ClickHouse HTTP port", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--username" + ], + "dest": "username", + "default": null, + "help": "ClickHouse username", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--password" + ], + "dest": "password", + "default": null, + "help": "ClickHouse password", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--database" + ], + "dest": "database", + "default": null, + "help": "Target ClickHouse database", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--apply" + ], + "dest": "apply", + "nargs": 0, + "const": true, + "default": false, + "help": "Apply the rename (dry-run by default)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--control-store-url" + ], + "dest": "control_store_url", + "default": null, + "help": "SQLite/PostgreSQL account control-store URL", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--target" + ], + "dest": "target", + "default": null, + "help": "Named project target to use", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--vars" + ], + "dest": "vars", + "default": {}, + "type": "parse_cli_vars", + "help": "Project variable overrides as one JSON object", + "group": 1 + } + ] + } + }, { "name": "janitor", "parser": { @@ -3929,6 +4074,10 @@ "metavar": "reconcile", "help": "Reconcile live metadata baseline" }, + { + "metavar": "rename-project", + "help": "Rename the project and keep its live state" + }, { "metavar": "janitor", "help": "Clean up stale deployment artifacts" diff --git a/crates/sqltide-clickhouse/src/metadata.rs b/crates/sqltide-clickhouse/src/metadata.rs index e5e26fd7..40599303 100644 --- a/crates/sqltide-clickhouse/src/metadata.rs +++ b/crates/sqltide-clickhouse/src/metadata.rs @@ -715,6 +715,38 @@ pub fn manifest_history_query( ) } +/// Copy one project target's manifest history from `from_identity` to `to_identity`, row for +/// row, skipping manifests the new identity already holds, so applying it twice is a no-op. +#[must_use] +pub fn manifest_identity_copy( + database: &str, + from_identity: &str, + to_identity: &str, + target_name: &str, + target_database: &str, +) -> String { + let table: &str = MANIFESTS_TABLE.0; + let target: String = format!( + "manifest.target_name = {} AND manifest.target_database = {}", + text_literal(target_name), + text_literal(target_database) + ); + format!( + "INSERT INTO {database}.{table} (manifest_id, invocation_id, project_identity, \ + target_name, target_database, is_production, project_revision, manifest_fingerprint, \ + manifest_version, pipelines, resources, tool_version, published_at) SELECT \ + manifest.manifest_id, manifest.invocation_id, {to}, manifest.target_name, \ + manifest.target_database, manifest.is_production, manifest.project_revision, \ + manifest.manifest_fingerprint, manifest.manifest_version, manifest.pipelines, \ + manifest.resources, manifest.tool_version, manifest.published_at FROM \ + {database}.{table} AS manifest WHERE manifest.project_identity = {from} AND {target} \ + AND manifest.manifest_id NOT IN (SELECT manifest.manifest_id FROM {database}.{table} \ + AS manifest WHERE manifest.project_identity = {to} AND {target});", + from = text_literal(from_identity), + to = text_literal(to_identity), + ) +} + /// One append containing a complete immutable manifest. #[must_use] pub fn manifest_publication(database: &str, manifest: &Manifest) -> String { diff --git a/crates/sqltide-config/src/lib.rs b/crates/sqltide-config/src/lib.rs index 4e41bea3..7a638b42 100644 --- a/crates/sqltide-config/src/lib.rs +++ b/crates/sqltide-config/src/lib.rs @@ -7,6 +7,7 @@ pub mod errors; pub mod models; pub mod pipelines; pub mod project; +pub mod project_rename; pub mod toml; pub mod yaml; diff --git a/crates/sqltide-config/src/project_rename.rs b/crates/sqltide-config/src/project_rename.rs new file mode 100644 index 00000000..48869b04 --- /dev/null +++ b/crates/sqltide-config/src/project_rename.rs @@ -0,0 +1,141 @@ +//! In-place edits of a committed project config for `sqt rename-project`: a new `name`, and a +//! `consumer_group_identity` when the rename has to keep existing Kafka consumer groups. +//! +//! Edits splice the parsed value spans into the original text, so every other byte (comments, +//! spacing, key order, line endings) stays exactly as authored. + +use std::fmt::Write as _; +use std::ops::Range; + +use toml_edit::{Document, Item, Table}; + +pub const PROJECT_NAME_KEY: &str = "name"; +pub const CONSUMER_GROUP_IDENTITY_KEY: &str = "consumer_group_identity"; +const IDENTITY_COMMENT: &str = + "# Keeps the Kafka consumer groups this project used before it was renamed."; + +/// The top-level `name` and `consumer_group_identity` a project config declares. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct ProjectIdentityKeys { + pub name: Option, + pub consumer_group_identity: Option, +} + +fn parse(text: &str) -> Result, String> { + Document::parse(text).map_err(|error| format!("project config is not valid TOML: {error}")) +} + +fn string_at(root: &Table, key: &str) -> Result)>, String> { + let Some(item) = root.get(key) else { + return Ok(None); + }; + let value = match item { + Item::Value(value) => value, + _ => return Err(format!("project config key '{key}' must be a string")), + }; + let Some(text) = value.as_str() else { + return Err(format!("project config key '{key}' must be a string")); + }; + let span: Range = value + .span() + .ok_or_else(|| format!("project config key '{key}' has no source position"))?; + Ok(Some((text.to_owned(), span))) +} + +/// The top-level identity keys of a project config text. +/// +/// # Errors +/// A message when the text is not TOML or either key is not a string. +pub fn project_identity_keys(text: &str) -> Result { + let document: Document<&str> = parse(text)?; + let root: &Table = document.as_table(); + Ok(ProjectIdentityKeys { + name: string_at(root, PROJECT_NAME_KEY)?.map(|(value, _)| value), + consumer_group_identity: string_at(root, CONSUMER_GROUP_IDENTITY_KEY)? + .map(|(value, _)| value), + }) +} + +/// `value` as a TOML basic string. +#[must_use] +pub fn toml_basic_string(value: &str) -> String { + let mut quoted: String = String::with_capacity(value.len() + 2); + quoted.push('"'); + for character in value.chars() { + match character { + '"' => quoted.push_str("\\\""), + '\\' => quoted.push_str("\\\\"), + '\n' => quoted.push_str("\\n"), + '\t' => quoted.push_str("\\t"), + '\r' => quoted.push_str("\\r"), + character if character.is_control() => { + let _ = write!(quoted, "\\u{:04X}", u32::from(character)); + } + character => quoted.push(character), + } + } + quoted.push('"'); + quoted +} + +/// The project config text with `name` set to `new_name` and, when given, a +/// `consumer_group_identity` line added after the `name` line. Everything else is unchanged. +/// +/// # Errors +/// A message when the text is not TOML, has no string `name`, already declares a +/// `consumer_group_identity` while one is to be added, or does not read back as intended. +pub fn rename_project_text( + text: &str, + new_name: &str, + consumer_group_identity: Option<&str>, +) -> Result { + let document: Document<&str> = parse(text)?; + let root: &Table = document.as_table(); + let (_, name_span) = string_at(root, PROJECT_NAME_KEY)? + .ok_or_else(|| "project config does not declare a string 'name'".to_owned())?; + if consumer_group_identity.is_some() && root.contains_key(CONSUMER_GROUP_IDENTITY_KEY) { + return Err(format!( + "project config already declares '{CONSUMER_GROUP_IDENTITY_KEY}'" + )); + } + let mut edited: String = String::with_capacity(text.len() + 128); + edited.push_str(&text[..name_span.start]); + edited.push_str(&toml_basic_string(new_name)); + let rest: &str = &text[name_span.end..]; + match consumer_group_identity { + None => edited.push_str(rest), + Some(identity) => { + let (line_rest, after) = match rest.find('\n') { + Some(index) => rest.split_at(index + 1), + None => (rest, ""), + }; + let newline: &str = if line_rest.ends_with("\r\n") { + "\r\n" + } else { + "\n" + }; + edited.push_str(line_rest); + if !line_rest.ends_with('\n') { + edited.push_str(newline); + } + edited.push_str(IDENTITY_COMMENT); + edited.push_str(newline); + edited.push_str(CONSUMER_GROUP_IDENTITY_KEY); + edited.push_str(" = "); + edited.push_str(&toml_basic_string(identity)); + edited.push_str(newline); + edited.push_str(after); + } + } + let read_back: ProjectIdentityKeys = project_identity_keys(&edited)?; + let expected_identity: Option = match consumer_group_identity { + Some(identity) => Some(identity.to_owned()), + None => string_at(root, CONSUMER_GROUP_IDENTITY_KEY)?.map(|(value, _)| value), + }; + if read_back.name.as_deref() != Some(new_name) + || read_back.consumer_group_identity != expected_identity + { + return Err("project config edit did not read back as intended".to_owned()); + } + Ok(edited) +} diff --git a/crates/sqltide-config/src/tests.rs b/crates/sqltide-config/src/tests.rs index bb5f066d..1fd18ac2 100644 --- a/crates/sqltide-config/src/tests.rs +++ b/crates/sqltide-config/src/tests.rs @@ -6,3 +6,6 @@ mod toml; #[path = "tests/test_pipelines.rs"] mod pipelines; + +#[path = "tests/test_project_rename.rs"] +mod project_rename; diff --git a/crates/sqltide-config/src/tests/test_project_rename.rs b/crates/sqltide-config/src/tests/test_project_rename.rs new file mode 100644 index 00000000..de4b4622 --- /dev/null +++ b/crates/sqltide-config/src/tests/test_project_rename.rs @@ -0,0 +1,83 @@ +//! Project config edits for `sqt rename-project` keep every authored byte they do not change. + +use crate::project_rename::{ + ProjectIdentityKeys, project_identity_keys, rename_project_text, toml_basic_string, +}; + +const AUTHORED: &str = "# Shop analytics project.\n\ +name = \"shop\" # the project identity\n\ +default_target = \"dev\"\n\ +\n\ +[targets.dev]\n\ +# Local development database.\n\ +database = \"shop_dev\"\n"; + +#[test] +fn renames_in_place_and_keeps_comments_and_spacing() { + let edited = rename_project_text(AUTHORED, "storefront", None).unwrap(); + assert_eq!( + edited, + AUTHORED.replace("name = \"shop\"", "name = \"storefront\"") + ); +} + +#[test] +fn adds_the_consumer_group_identity_after_the_name_line() { + let edited = rename_project_text(AUTHORED, "storefront", Some("shop")).unwrap(); + assert_eq!( + edited, + "# Shop analytics project.\n\ +name = \"storefront\" # the project identity\n\ +# Keeps the Kafka consumer groups this project used before it was renamed.\n\ +consumer_group_identity = \"shop\"\n\ +default_target = \"dev\"\n\ +\n\ +[targets.dev]\n\ +# Local development database.\n\ +database = \"shop_dev\"\n" + ); + assert_eq!( + project_identity_keys(&edited).unwrap(), + ProjectIdentityKeys { + name: Some("storefront".to_owned()), + consumer_group_identity: Some("shop".to_owned()), + } + ); +} + +#[test] +fn keeps_crlf_line_endings_and_a_final_line_without_newline() { + let edited = rename_project_text("name = 'shop'\r\nx = 1\r\n", "store", Some("shop")).unwrap(); + assert_eq!( + edited, + "name = \"store\"\r\n# Keeps the Kafka consumer groups this project used before it was \ + renamed.\r\nconsumer_group_identity = \"shop\"\r\nx = 1\r\n" + ); + let edited = rename_project_text("x = 1\nname = \"shop\"", "store", Some("shop")).unwrap(); + assert!(edited.ends_with("name = \"store\"\n# Keeps the Kafka consumer groups this project used before it was renamed.\nconsumer_group_identity = \"shop\"\n")); +} + +#[test] +fn keeps_an_existing_consumer_group_identity() { + let text = "name = \"storefront\"\nconsumer_group_identity = \"shop\"\n"; + assert_eq!( + rename_project_text(text, "outlet", None).unwrap(), + "name = \"outlet\"\nconsumer_group_identity = \"shop\"\n" + ); + assert_eq!( + rename_project_text(text, "outlet", Some("shop")).unwrap_err(), + "project config already declares 'consumer_group_identity'" + ); +} + +#[test] +fn refuses_documents_without_a_string_name() { + assert!(rename_project_text("default_target = \"dev\"\n", "x", None).is_err()); + assert!(rename_project_text("name = 1\n", "x", None).is_err()); + assert!(rename_project_text("name = \n", "x", None).is_err()); +} + +#[test] +fn quotes_names_as_basic_strings() { + assert_eq!(toml_basic_string("a\"b\\c\u{1}"), "\"a\\\"b\\\\c\\u0001\""); +} diff --git a/crates/sqltide-python/src/cli/dispatch.rs b/crates/sqltide-python/src/cli/dispatch.rs index b9a91e10..f0d233fb 100644 --- a/crates/sqltide-python/src/cli/dispatch.rs +++ b/crates/sqltide-python/src/cli/dispatch.rs @@ -538,6 +538,28 @@ pub(crate) fn dispatch_cli_command<'py>( ("adapter_profile", adapter_profile), ], ), + "rename-project" => { + let project_dir: Bound<'py, PyAny> = project_dir(py, invocation)?; + build( + py, + NATIVE, + "run_rename_project", + vec![ + ("pipelines_root", require_pipelines_root(py, invocation)?), + ("project_dir", project_dir), + ("new_name", args.getattr("new_name")?), + ("database", database), + ( + "control_store_url", + getattr_none(args, "control_store_url")?, + ), + ("apply", boolean(py, flag(args, "apply")?)), + ("client", client), + ("loaded_project", loaded_project), + ("adapter_profile", adapter_profile), + ], + ) + } "doctor" => call_handler( py, handlers, diff --git a/crates/sqltide-python/src/cli/invocation.rs b/crates/sqltide-python/src/cli/invocation.rs index 087ff840..450c7cbb 100644 --- a/crates/sqltide-python/src/cli/invocation.rs +++ b/crates/sqltide-python/src/cli/invocation.rs @@ -12,7 +12,7 @@ const DEV_CLI_VARIABLES_ENV_VAR: &str = "SQLTIDE_INTERNAL_CLI_VARS"; const PROJECT_FILE: &str = "sqltide_project.toml"; const LEGACY_PROJECT_FILE: &str = "streambuild_project.toml"; const PIPELINES_DIRECTORY: &str = "pipelines"; -const COMMANDS_REQUIRING_PIPELINES_ROOT: [&str; 13] = [ +const COMMANDS_REQUIRING_PIPELINES_ROOT: [&str; 14] = [ "dev", "discover", "compile", @@ -25,6 +25,7 @@ const COMMANDS_REQUIRING_PIPELINES_ROOT: [&str; 13] = [ "reset-target", "deployment", "reconcile", + "rename-project", "audit", ]; const COMMANDS_WITHOUT_ADAPTER_CONNECTION: [&str; 4] = ["discover", "compile", "lint", "format"]; diff --git a/crates/sqltide-python/src/cli/mod.rs b/crates/sqltide-python/src/cli/mod.rs index f1e8d394..09d8dd4a 100644 --- a/crates/sqltide-python/src/cli/mod.rs +++ b/crates/sqltide-python/src/cli/mod.rs @@ -10,7 +10,7 @@ mod entry; pub(crate) mod invocation; pub(crate) mod output; mod playground; -mod python; +pub(crate) mod python; mod readiness; pub(crate) mod render; mod sql; diff --git a/crates/sqltide-python/src/control_store/mod.rs b/crates/sqltide-python/src/control_store/mod.rs index fc5751ed..3b6eb0e4 100644 --- a/crates/sqltide-python/src/control_store/mod.rs +++ b/crates/sqltide-python/src/control_store/mod.rs @@ -1685,7 +1685,187 @@ impl ControlStore { } } +/// What renaming a project would do to its role assignments. +pub(crate) struct ProjectRenamePreview { + /// Assignments, active or revoked, still recorded under the old project name. + pub(crate) moving: usize, + /// Assignments already recorded under the new project name. + pub(crate) existing: usize, + /// `user role scope` of each active assignment both names hold. + pub(crate) conflicts: Vec, +} + impl ControlStore { + fn project_rename_preview( + transaction: &Transaction<'_>, + from_project: &str, + to_project: &str, + ) -> PyResult { + let py: Python<'_> = transaction.py(); + let rows_for = |project: &str| -> PyResult> { + transaction + .rows( + &sql::assignments_where( + " WHERE sqltide_project_role_assignments.project_name = :project_name", + ), + &[("project_name", object(py, project)?)], + )? + .iter() + .map(|row| { + Ok(( + column(row, "user_id")?.str()?.to_string(), + column(row, "role_name")?.str()?.to_string(), + column(row, "assignment_scope")?.str()?.to_string(), + column(row, "revoked_at")?.is_none(), + )) + }) + .collect() + }; + let moving = rows_for(from_project)?; + let existing = rows_for(to_project)?; + let mut conflicts: Vec = moving + .iter() + .filter(|(user, role, scope, active)| { + *active + && existing + .iter() + .any(|(other_user, other_role, other_scope, other_active)| { + *other_active + && other_user == user + && other_role == role + && other_scope == scope + }) + }) + .map(|(user, role, scope, _)| { + let scope: &str = if scope.is_empty() { + "all targets" + } else { + scope + }; + format!("user {user} role '{role}' ({scope})") + }) + .collect(); + conflicts.sort(); + Ok(ProjectRenamePreview { + moving: moving.len(), + existing: existing.len(), + conflicts, + }) + } + + /// What moving `from_project`'s role assignments to `to_project` would change. + pub(crate) fn preview_project_rename( + &self, + py: Python<'_>, + from_project: &str, + to_project: &str, + ) -> PyResult { + self.transaction(py, |transaction| { + Self::project_rename_preview(transaction, from_project, to_project) + }) + } + + /// Move every role assignment, active or revoked, from `from_project` to `to_project` in one + /// transaction and audit it; the moved assignment ids. Refuses when both names hold the + /// same active assignment. + pub(crate) fn rename_project( + &self, + py: Python<'_>, + from_project: &str, + to_project: &str, + ) -> PyResult> { + let now: Any<'_> = now(py)?; + let moved: Vec = self.transaction(py, |transaction| { + let preview = Self::project_rename_preview(transaction, from_project, to_project)?; + if !preview.conflicts.is_empty() { + return Err(error( + py, + "AccountConflictError", + format!( + "Project '{to_project}' already holds role assignments that project \ + '{from_project}' also holds: {}", + preview.conflicts.join(", ") + ), + )); + } + let ids: Vec = transaction + .rows( + &sql::assignments_where( + " WHERE sqltide_project_role_assignments.project_name = :project_name \ + ORDER BY sqltide_project_role_assignments.id", + ), + &[("project_name", object(py, from_project)?)], + )? + .iter() + .map(|row| Ok(column(row, "id")?.str()?.to_string())) + .collect::>()?; + if ids.is_empty() { + return Ok(ids); + } + transaction.execute( + sql::RENAME_PROJECT_ASSIGNMENTS, + &[ + ("to_project", object(py, to_project)?), + ("from_project", object(py, from_project)?), + ], + )?; + let details: Bound<'_, PyDict> = PyDict::new(py); + details.set_item("fromProjectName", from_project)?; + details.set_item("toProjectName", to_project)?; + details.set_item("assignmentCount", ids.len())?; + self.audit( + transaction, + "project.renamed", + &none(py), + &none(py), + details, + &now, + )?; + Ok(ids) + })?; + if !moved.is_empty() { + self.invalidate(); + } + Ok(moved) + } + + /// Return assignments `rename_project` moved to `to_project` back to `from_project`. + pub(crate) fn restore_project_rename( + &self, + py: Python<'_>, + assignment_ids: &[String], + from_project: &str, + to_project: &str, + ) -> PyResult<()> { + let now: Any<'_> = now(py)?; + self.transaction(py, |transaction| { + for id in assignment_ids { + transaction.execute( + sql::MOVE_PROJECT_ASSIGNMENT, + &[ + ("to_project", object(py, from_project)?), + ("id", object(py, id)?), + ("from_project", object(py, to_project)?), + ], + )?; + } + let details: Bound<'_, PyDict> = PyDict::new(py); + details.set_item("fromProjectName", to_project)?; + details.set_item("toProjectName", from_project)?; + details.set_item("assignmentCount", assignment_ids.len())?; + self.audit( + transaction, + "project.rename_reverted", + &none(py), + &none(py), + details, + &now, + ) + })?; + self.invalidate(); + Ok(()) + } + /// The identity a competing writer linked, or a real conflict. fn provisioned_by_competing_writer<'py>( &self, diff --git a/crates/sqltide-python/src/control_store/sql.rs b/crates/sqltide-python/src/control_store/sql.rs index 83e44527..2d582ef6 100644 --- a/crates/sqltide-python/src/control_store/sql.rs +++ b/crates/sqltide-python/src/control_store/sql.rs @@ -104,6 +104,8 @@ pub(crate) const USER_ID: &str = pub(crate) const DELETE_USER_ROLE: &str = "DELETE FROM sqltide_user_roles WHERE sqltide_user_roles.user_id = :user_id AND sqltide_user_roles.role_name = :role_name"; pub(crate) const INSERT_ASSIGNMENT: &str = "INSERT INTO sqltide_project_role_assignments (id, user_id, project_name, role_name, target_name, assignment_scope, assigned_by, assigned_at, revoked_by, revoked_at, active_key) VALUES (:id, :user_id, :project_name, :role_name, :target_name, :assignment_scope, :assigned_by, :assigned_at, :revoked_by, :revoked_at, :active_key)"; pub(crate) const REVOKE_ASSIGNMENT: &str = "UPDATE sqltide_project_role_assignments SET revoked_by=:revoked_by, revoked_at=:revoked_at, active_key=:active_key WHERE sqltide_project_role_assignments.id = :id"; +pub(crate) const RENAME_PROJECT_ASSIGNMENTS: &str = "UPDATE sqltide_project_role_assignments SET project_name=:to_project WHERE sqltide_project_role_assignments.project_name = :from_project"; +pub(crate) const MOVE_PROJECT_ASSIGNMENT: &str = "UPDATE sqltide_project_role_assignments SET project_name=:to_project WHERE sqltide_project_role_assignments.id = :id AND sqltide_project_role_assignments.project_name = :from_project"; pub(crate) const INSERT_AUDIT: &str = "INSERT INTO sqltide_account_audit_log (operation, actor_user_id, affected_user_id, occurred_at, details_json) VALUES (:operation, :actor_user_id, :affected_user_id, :occurred_at, :details_json)"; pub(crate) const AUDIT_RECORDS: &str = "SELECT sqltide_account_audit_log.id, sqltide_account_audit_log.operation, sqltide_account_audit_log.actor_user_id, sqltide_account_audit_log.affected_user_id, sqltide_account_audit_log.occurred_at, sqltide_account_audit_log.details_json FROM sqltide_account_audit_log ORDER BY sqltide_account_audit_log.id"; pub(crate) const PRUNE_SESSIONS: &str = "DELETE FROM sqltide_sessions WHERE sqltide_sessions.expires_at <= :now OR sqltide_sessions.revoked_at IS NOT NULL"; diff --git a/crates/sqltide-python/src/lib.rs b/crates/sqltide-python/src/lib.rs index f6b95da6..c5fe1da6 100644 --- a/crates/sqltide-python/src/lib.rs +++ b/crates/sqltide-python/src/lib.rs @@ -70,6 +70,7 @@ mod promotion_command; mod realization; mod reconcile_command; mod recovery; +mod rename_project_command; mod row_error_policy; mod rules; mod run_events; @@ -172,6 +173,7 @@ fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { deployment_command::register(module)?; promotion_command::register(module)?; reconcile_command::register(module)?; + rename_project_command::register(module)?; maintenance_command::register(module)?; sql_tests_command::register(module)?; audit_execution::register(module)?; diff --git a/crates/sqltide-python/src/realization.rs b/crates/sqltide-python/src/realization.rs index 0aa93959..26ab096b 100644 --- a/crates/sqltide-python/src/realization.rs +++ b/crates/sqltide-python/src/realization.rs @@ -953,6 +953,44 @@ impl<'py> Realizer<'py> { } } +/// `(source name, consumer group)` for every managed source the adapter realizes for `project`, +/// in source order, with the project name and consumer group identity replaced as given. +pub(crate) fn managed_source_consumer_groups<'py>( + project: &Bound<'py, PyAny>, + adapter_profile: &Bound<'py, PyAny>, + project_name: &str, + consumer_group_identity: Option<&str>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let realizer: Realizer<'py> = Realizer::import(py)?; + let project: Bound<'py, PyAny> = realizer.replace( + project.clone(), + &[ + ("project_name", project_name.into_pyobject(py)?.into_any()), + ( + "consumer_group_identity", + consumer_group_identity.into_pyobject(py)?.into_any(), + ), + ], + )?; + let realize_source: Bound<'py, PyAny> = adapter_profile.getattr("realize_source")?; + let mut groups: Vec<(String, String)> = Vec::new(); + for source in realizer.items(&project.getattr("sources")?)? { + let request: Bound<'py, PyAny> = realizer.source_request(&source, &project)?; + let realization: Bound<'py, PyAny> = + realizer.call(&realize_source, &[("request", request)])?; + for resource in realizer.items(&realization.getattr("resources")?)? { + if resource.is_instance(&realizer.managed_source)? { + groups.push(( + source.getattr("key")?.getattr("name")?.extract()?, + resource.getattr("consumer_group")?.extract()?, + )); + } + } + } + Ok(groups) +} + /// Map one complete logical project to adapter resources and planner desired state through /// the adapter profile's connection-free callbacks. #[pyfunction] diff --git a/crates/sqltide-python/src/rename_project_command.rs b/crates/sqltide-python/src/rename_project_command.rs new file mode 100644 index 00000000..7c2f1f3e --- /dev/null +++ b/crates/sqltide-python/src/rename_project_command.rs @@ -0,0 +1,636 @@ +//! `sqt rename-project`: rename a project while its Kafka consumer groups, manifest history and +//! project role assignments carry over to the new name. +//! +//! Every step is idempotent and the project config is edited last, so a failure leaves the old +//! name fully working and running the command again completes the rename. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use pyo3::prelude::*; +use pyo3::types::{PyList, PyTuple}; +use sqltide_clickhouse::metadata::manifest_identity_copy; +use sqltide_clickhouse::resources::database_scoped_consumer_group; +use sqltide_config::project::PROJECT_CONFIG_FILE_NAME; +use sqltide_config::project_rename::{ + ProjectIdentityKeys, project_identity_keys, rename_project_text, +}; + +use crate::cli::python::{print, user_error}; +use crate::control_store::{ControlStore, ProjectRenamePreview, default_control_store_url}; +use crate::dev_server::support::attr; +use crate::handling::within; +use crate::py_kwargs; +use crate::realization::managed_source_consumer_groups; +use crate::workflow_statements::numbered; + +const NATIVE: &str = "sqltide._native"; +const ADAPTER_EXCEPTIONS: &str = "sqltide.adapter.exceptions"; +const MUTATION_LOCK: &str = "sqltide.executor.workflow.main.target_mutation_lock"; +const WAREHOUSE_WORKFLOW: &str = "sqltide.executor.workflow.main._execute_warehouse_workflow"; +const CONTROL_STORE_ENV_VAR: &str = "SQLTIDE_CONTROL_STORE_URL"; +const SQLITE_FILE_PREFIX: &str = "sqlite:///"; +const INTERPOLATION_TOKEN_START: &str = "${"; + +type Any<'py> = Bound<'py, PyAny>; + +/// The manifest history of one project target under the old and new identities. +struct ManifestPlan { + from_identity: String, + to_identity: String, + target_name: String, + database: String, + /// Manifests recorded under the old identity and not yet under the new one. + to_copy: usize, + /// Manifests recorded under the old identity. + recorded: usize, +} + +/// The role assignments the rename moves, and where they live. +struct RolePlan<'py> { + store: Bound<'py, ControlStore>, + url: String, + preview: ProjectRenamePreview, +} + +/// Everything the rename would change, computed without writing anything. +struct RenamePlan<'py> { + config_path: PathBuf, + current_name: String, + new_name: String, + /// The identity to add to the project config, when existing groups need it. + added_identity: Option, + /// `(source, database-scoped group)` for every managed source, unchanged by the rename. + groups: Vec<(String, String)>, + manifests: ManifestPlan, + roles: Option>, + other_targets: Vec, + edited_config: String, +} + +fn fail(py: Python<'_>, message: &str) -> PyErr { + user_error(py, message, None) +} + +/// The new name, held to the rules the project config applies to `name`. +fn validated_name(py: Python<'_>, new_name: &str, current_name: &str) -> PyResult { + if new_name.trim().is_empty() { + return Err(fail(py, "The new project name must be a non-empty string")); + } + if new_name.trim() != new_name { + return Err(fail( + py, + "The new project name must not start or end with whitespace", + )); + } + if new_name.contains(INTERPOLATION_TOKEN_START) { + return Err(fail( + py, + "The new project name must be a committed literal and cannot be interpolated", + )); + } + if new_name == current_name { + return Err(fail( + py, + &format!("The project is already named '{current_name}'"), + )); + } + Ok(new_name.to_owned()) +} + +/// The manifest ids one identity holds for the target, or the reason they cannot be read. +fn manifest_ids( + client: &Any<'_>, + database: &str, + identity: &str, + target_name: &str, +) -> PyResult> { + let py: Python<'_> = client.py(); + let snapshot: Any<'_> = client.call_method( + "load_manifests", + (), + Some(&py_kwargs!(py; + database = database, + project_identity = identity, + target_name = target_name, + target_database = database, + )?), + )?; + let unavailable: Any<'_> = + attr(py, "sqltide.adapter.types", "AdapterOptionalStateStatus")?.getattr("UNAVAILABLE")?; + if snapshot.getattr("status")?.eq(unavailable)? { + let warning: Any<'_> = snapshot.getattr("warning")?; + return Err(fail( + py, + &format!( + "Manifest history is unavailable, so the rename cannot carry it over: {}", + if warning.is_none() { + "no detail".to_owned() + } else { + warning.str()?.to_string() + } + ), + )); + } + snapshot + .getattr("manifests")? + .try_iter()? + .map(|manifest| manifest?.getattr("manifest_id")?.extract()) + .collect() +} + +fn manifest_plan( + client: &Any<'_>, + from_identity: &str, + to_identity: &str, + target_name: &str, + database: &str, +) -> PyResult { + let py: Python<'_> = client.py(); + let recorded: BTreeSet = manifest_ids(client, database, from_identity, target_name)?; + let copied: BTreeSet = manifest_ids(client, database, to_identity, target_name)?; + let foreign: Vec<&String> = copied.difference(&recorded).collect(); + if !foreign.is_empty() { + return Err(fail( + py, + &format!( + "Target '{target_name}' (database '{database}') already records {} manifest(s) \ + for project '{to_identity}' that did not come from '{from_identity}'; another \ + project already uses that name here. Choose a different name.", + foreign.len() + ), + )); + } + Ok(ManifestPlan { + from_identity: from_identity.to_owned(), + to_identity: to_identity.to_owned(), + target_name: target_name.to_owned(), + database: database.to_owned(), + to_copy: recorded.difference(&copied).count(), + recorded: recorded.len(), + }) +} + +/// The control store the project uses: `--control-store-url`, then the environment, then the +/// project-local store when it exists. +fn control_store_url( + py: Python<'_>, + project_dir: &Any<'_>, + option: &Any<'_>, +) -> PyResult> { + if option.is_truthy()? { + return Ok(Some(option.extract()?)); + } + let environment: Any<'_> = attr(py, "os", "environ")?; + let configured: Any<'_> = environment.call_method1("get", (CONTROL_STORE_ENV_VAR,))?; + if configured.is_truthy()? { + return Ok(Some(configured.extract()?)); + } + let local: String = default_control_store_url(project_dir)?; + let path: &str = local.strip_prefix(SQLITE_FILE_PREFIX).unwrap_or(&local); + Ok(Path::new(path).exists().then_some(local)) +} + +fn role_plan<'py>( + py: Python<'py>, + url: Option, + from_project: &str, + to_project: &str, +) -> PyResult>> { + let Some(url) = url else { + return Ok(None); + }; + let store: Bound<'py, ControlStore> = attr(py, NATIVE, "ControlStore")? + .call((), Some(&py_kwargs!(py; url = &url)?))? + .cast_into()?; + let preview: ProjectRenamePreview = + store + .get() + .preview_project_rename(py, from_project, to_project)?; + if !preview.conflicts.is_empty() { + return Err(fail( + py, + &format!( + "Project '{to_project}' already holds role assignments that '{from_project}' \ + also holds: {}. Revoke one of each pair, then run the rename again.", + preview.conflicts.join(", ") + ), + )); + } + Ok(Some(RolePlan { + store, + url, + preview, + })) +} + +fn other_targets(loaded_project: &Any<'_>, target_name: &str) -> PyResult> { + let configuration: Any<'_> = loaded_project.getattr("configuration")?; + if configuration.is_none() { + return Ok(Vec::new()); + } + let mut names: Vec = Vec::new(); + for entry in configuration + .getattr("project")? + .getattr("targets")? + .try_iter()? + { + let name: String = entry?.get_item(0)?.extract()?; + if name != target_name { + names.push(name); + } + } + Ok(names) +} + +#[allow(clippy::too_many_arguments)] +fn plan_rename<'py>( + py: Python<'py>, + project_dir: &Any<'py>, + new_name: &str, + database: &str, + control_store_option: &Any<'py>, + client: &Any<'py>, + loaded_project: &Any<'py>, + analysis: &Any<'py>, + adapter_profile: &Any<'py>, +) -> PyResult> { + let config_path: PathBuf = project_dir + .call_method1("__truediv__", (PROJECT_CONFIG_FILE_NAME,))? + .extract()?; + let config_text: String = std::fs::read_to_string(&config_path).map_err(|error| { + fail( + py, + &format!("Could not read {}: {error}", config_path.display()), + ) + })?; + let keys: ProjectIdentityKeys = + project_identity_keys(&config_text).map_err(|message| fail(py, &message))?; + let compiled: Any<'py> = analysis.getattr("compiled_project")?; + let current_name: String = compiled.getattr("project_name")?.extract()?; + let new_name: String = validated_name(py, new_name, ¤t_name)?; + let identity: Option = compiled.getattr("consumer_group_identity")?.extract()?; + if identity + != keys + .consumer_group_identity + .map(|value| value.trim().to_owned()) + { + return Err(fail( + py, + &format!( + "The consumer group identity in {} does not match the loaded project; check \ + the file and run the rename again", + config_path.display() + ), + )); + } + let before: Vec<(String, String)> = managed_source_consumer_groups( + &compiled, + adapter_profile, + ¤t_name, + identity.as_deref(), + )?; + let renamed = |identity: Option<&str>| { + managed_source_consumer_groups(&compiled, adapter_profile, &new_name, identity) + }; + // Keep the old name in the groups only when some group would otherwise change. + let added_identity: Option = if identity.is_none() && renamed(None)? != before { + Some(current_name.clone()) + } else { + None + }; + let after: Vec<(String, String)> = renamed(identity.as_deref().or(added_identity.as_deref()))?; + let changed: Vec = before + .iter() + .zip(&after) + .filter(|(old, new)| old != new) + .map(|((source, old), (_, new))| format!("{source}: '{old}' -> '{new}'")) + .collect(); + if !changed.is_empty() || before.len() != after.len() { + return Err(fail( + py, + &format!( + "Renaming would change Kafka consumer groups, so sources would not resume from \ + their committed offsets: {}", + changed.join("; ") + ), + )); + } + let groups: Vec<(String, String)> = before + .into_iter() + .map(|(source, group)| (source, database_scoped_consumer_group(&group, database))) + .collect(); + let target_name: String = compiled.getattr("target_name")?.extract()?; + let from_identity: String = attr(py, NATIVE, "resolve_manifest_project_identity")? + .call((), Some(&py_kwargs!(py; analysis = analysis)?))? + .extract()?; + let manifests: ManifestPlan = + manifest_plan(client, &from_identity, &new_name, &target_name, database)?; + let roles: Option> = role_plan( + py, + control_store_url(py, project_dir, control_store_option)?, + ¤t_name, + &new_name, + )?; + let edited_config: String = + rename_project_text(&config_text, &new_name, added_identity.as_deref()) + .map_err(|message| fail(py, &message))?; + Ok(RenamePlan { + config_path, + current_name, + new_name, + added_identity, + groups, + manifests, + roles, + other_targets: other_targets(loaded_project, &target_name)?, + edited_config, + }) +} + +fn quoted(value: &str) -> String { + sqltide_config::project_rename::toml_basic_string(value) +} + +fn render_plan(plan: &RenamePlan<'_>, applied: bool) -> String { + let manifests: &ManifestPlan = &plan.manifests; + let mut lines: Vec = vec![ + if applied { + "Project Renamed".to_owned() + } else { + "Rename Project Preview".to_owned() + }, + format!("Project: {} -> {}", plan.current_name, plan.new_name), + format!( + "Target: {} (database {})", + manifests.target_name, manifests.database + ), + format!("Config: {}", plan.config_path.display()), + format!(" name = {}", quoted(&plan.new_name)), + ]; + if let Some(identity) = &plan.added_identity { + lines.push(format!( + " consumer_group_identity = {} (added so existing Kafka consumer groups stay the same)", + quoted(identity) + )); + } + lines.push(format!( + "Kafka consumer groups: {} unchanged", + plan.groups.len() + )); + lines.extend( + plan.groups + .iter() + .map(|(source, group)| format!(" - {source}: {group}")), + ); + lines.push(format!( + "Manifest history: {} of {} manifest(s) {} from '{}' to '{}'", + manifests.to_copy, + manifests.recorded, + if applied { "copied" } else { "to copy" }, + manifests.from_identity, + manifests.to_identity + )); + match &plan.roles { + None => lines.push("Role assignments: no control store in use".to_owned()), + Some(roles) => lines.push(format!( + "Role assignments: {} {} from '{}' to '{}'{} (control store {})", + roles.preview.moving, + if applied { "moved" } else { "to move" }, + plan.current_name, + plan.new_name, + if roles.preview.existing > 0 { + format!( + "; {} already recorded under '{}'", + roles.preview.existing, plan.new_name + ) + } else { + String::new() + }, + redacted_url(&roles.url) + )), + } + if !plan.other_targets.is_empty() { + lines.push(format!( + "Other targets keep manifest history under '{}': {}", + manifests.from_identity, + plan.other_targets.join(", ") + )); + } + lines.push( + "Unchanged: run history and audit and check status follow the project directory name, \ + not the project name." + .to_owned(), + ); + lines.push(if applied { + "Restart any running `sqt dev` server to pick up the new name.".to_owned() + } else { + "Dry run: nothing was changed. Run again with --apply to rename.".to_owned() + }); + lines.join("\n") +} + +/// A control store URL without its password. +fn redacted_url(url: &str) -> String { + let Some((scheme, rest)) = url.split_once("://") else { + return url.to_owned(); + }; + let Some((credentials, host)) = rest.rsplit_once('@') else { + return url.to_owned(); + }; + match credentials.split_once(':') { + Some((user, _)) => format!("{scheme}://{user}:***@{host}"), + None => url.to_owned(), + } +} + +/// Copy the manifest history under the target mutation lock; refuse while a run holds it. +fn copy_manifests(client: &Any<'_>, plan: &ManifestPlan) -> PyResult { + let py: Python<'_> = client.py(); + if !client + .call_method1("database_exists", (&plan.database,))? + .is_truthy()? + { + // Never built: no manifests to copy and no run that could hold the lock. + return Ok(0); + } + let lock: Any<'_> = attr(py, MUTATION_LOCK, "target_mutation_lock")?.call( + (), + Some(&py_kwargs!(py; connection = client, database = &plan.database)?), + )?; + let copied = within(&lock, || { + let fresh: ManifestPlan = manifest_plan( + client, + &plan.from_identity, + &plan.to_identity, + &plan.target_name, + &plan.database, + )?; + if fresh.to_copy == 0 { + return Ok(0); + } + let statement: String = manifest_identity_copy( + &plan.database, + &plan.from_identity, + &plan.to_identity, + &plan.target_name, + &plan.database, + ); + let rendered: Any<'_> = PyTuple::new(py, [statement])?.into_any(); + let statements: Vec> = + numbered(&rendered, 1, "copy_manifest_history", "FINALIZATION")?; + attr(py, WAREHOUSE_WORKFLOW, "execute_warehouse_workflow")?.call( + (), + Some(&py_kwargs!(py; statements = PyTuple::new(py, statements)?, connection = client)?), + )?; + let after: ManifestPlan = manifest_plan( + client, + &plan.from_identity, + &plan.to_identity, + &plan.target_name, + &plan.database, + )?; + if after.to_copy != 0 { + return Err(fail( + py, + &format!( + "{} manifest(s) were not copied to '{}'; run the rename again", + after.to_copy, plan.to_identity + ), + )); + } + Ok(fresh.to_copy) + }); + copied.map_err(|error| { + let locked: bool = attr(py, ADAPTER_EXCEPTIONS, "AdapterTargetMutationLockError") + .and_then(|class| error.value(py).is_instance(&class)) + .unwrap_or(false); + if !locked { + return error; + } + let refused: PyErr = fail( + py, + &format!( + "A run is active on target database '{}' ({}). Wait for it to finish, then \ + run the rename again.", + plan.database, + error.value(py) + ), + ); + refused.set_cause(py, Some(error)); + refused + }) +} + +/// Write the edited config next to the original and move it into place. +fn write_config(path: &Path, text: &str) -> std::io::Result<()> { + let staged: PathBuf = path.with_file_name(format!( + ".{}.rename-project", + path.file_name() + .and_then(|name| name.to_str()) + .unwrap_or("project") + )); + std::fs::write(&staged, text)?; + if let Ok(metadata) = std::fs::metadata(path) { + std::fs::set_permissions(&staged, metadata.permissions())?; + } + std::fs::rename(&staged, path).inspect_err(|_| { + let _ = std::fs::remove_file(&staged); + }) +} + +fn apply_rename(py: Python<'_>, client: &Any<'_>, plan: &RenamePlan<'_>) -> PyResult<()> { + copy_manifests(client, &plan.manifests)?; + let moved: Vec = match &plan.roles { + Some(roles) => roles + .store + .get() + .rename_project(py, &plan.current_name, &plan.new_name)?, + None => Vec::new(), + }; + if let Err(error) = write_config(&plan.config_path, &plan.edited_config) { + let mut message: String = format!( + "Could not write {}: {error}. The project keeps its old name", + plan.config_path.display() + ); + if let Some(roles) = &plan.roles + && !moved.is_empty() + && let Err(restore) = roles.store.get().restore_project_rename( + py, + &moved, + &plan.current_name, + &plan.new_name, + ) + { + message.push_str(&format!( + "; its role assignments stayed under '{}' ({restore}), so run the rename \ + again to finish", + plan.new_name + )); + } + return Err(fail(py, &message)); + } + Ok(()) +} + +/// Preview the rename of the project, or apply it with `apply`. +#[pyfunction] +#[pyo3(signature = ( + *, pipelines_root, project_dir, new_name, database, control_store_url, apply, client, + loaded_project, adapter_profile, +))] +#[allow(clippy::too_many_arguments)] +fn run_rename_project<'py>( + py: Python<'py>, + pipelines_root: Any<'py>, + project_dir: Any<'py>, + new_name: String, + database: Any<'py>, + control_store_url: Any<'py>, + apply: bool, + client: Any<'py>, + loaded_project: Any<'py>, + adapter_profile: Any<'py>, +) -> PyResult { + let analysis: Any<'py> = attr(py, NATIVE, "analyze_project")?.call( + (), + Some(&py_kwargs!(py; + pipelines_root = pipelines_root, + loaded_project = &loaded_project, + adapter_profile = &adapter_profile, + )?), + )?; + let pipelines: Any<'py> = PyList::new( + py, + analysis + .getattr("compile_inputs")? + .getattr("pipelines")? + .try_iter()? + .collect::>>()?, + )? + .into_any(); + let database: String = + crate::cli::invocation::resolve_default_database(py, &pipelines, database)?.extract()?; + let plan: RenamePlan<'py> = plan_rename( + py, + &project_dir, + &new_name, + &database, + &control_store_url, + &client, + &loaded_project, + &analysis, + &adapter_profile, + )?; + if !apply { + print(py, &render_plan(&plan, false))?; + return Ok(0); + } + apply_rename(py, &client, &plan)?; + print(py, &render_plan(&plan, true))?; + Ok(0) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_rename_project, module)?) +} diff --git a/crates/sqltide-python/src/shapes/cli_service.rs b/crates/sqltide-python/src/shapes/cli_service.rs index df3acc9b..952c44ae 100644 --- a/crates/sqltide-python/src/shapes/cli_service.rs +++ b/crates/sqltide-python/src/shapes/cli_service.rs @@ -573,6 +573,7 @@ const ENUMS: [EnumSpec; 14] = [ ("DEPLOYMENT", "deployment"), ("AUDIT", "audit"), ("RECONCILE", "reconcile"), + ("RENAME_PROJECT", "rename-project"), ("JANITOR", "janitor"), ("DOCTOR", "doctor"), ("REPAIR", "repair"), @@ -995,6 +996,7 @@ fn constants(module: &Bound<'_, PyModule>) -> PyResult<()> { "RESET_TARGET", "DEPLOYMENT", "RECONCILE", + "RENAME_PROJECT", "AUDIT", ], )?, diff --git a/scripts/shape_surface/baseline.json b/scripts/shape_surface/baseline.json index 96ec6b41..5983b31d 100644 --- a/scripts/shape_surface/baseline.json +++ b/scripts/shape_surface/baseline.json @@ -218,7 +218,7 @@ "sqltide.cli.dev.models:DevCommandOptions": {"fields": [["pipelines_root", false, true, true], ["database", false, true, true], ["host", false, true, true], ["port", false, true, true], ["selected_target", true, true, true], ["cli_variables", true, true, true], ["environment", true, true, true], ["connection_host", true, true, true], ["connection_port", true, true, true], ["connection_username", true, true, true], ["connection_password", true, true, true], ["auth_settings", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["pipelines_root", "database", "host", "port", "selected_target", "cli_variables", "environment", "connection_host", "connection_port", "connection_username", "connection_password", "auth_settings"], "module": "sqltide.cli.dev.models", "mro": ["sqltide.cli.dev.models.DevCommandOptions", "builtins.object"], "qualname": "DevCommandOptions"}, "sqltide.cli.entry.constants:ACTIVE_VIEW_SUBCOMMAND": {"kind": "constant", "type": "CliSubcommand", "value": "CliSubcommand.ACTIVE_VIEW"}, "sqltide.cli.entry.constants:AFFIRMATIVE_RESPONSES": {"kind": "constant", "type": "frozenset", "value": ["\"y\"", "\"yes\""]}, -"sqltide.cli.entry.constants:COMMANDS_REQUIRING_PIPELINES_ROOT": {"kind": "constant", "type": "frozenset", "value": ["\"CliCommand.AUDIT\"", "\"CliCommand.BUILD\"", "\"CliCommand.COMPILE\"", "\"CliCommand.DEPLOYMENT\"", "\"CliCommand.DESTROY\"", "\"CliCommand.DEV\"", "\"CliCommand.DISCOVER\"", "\"CliCommand.FORMAT\"", "\"CliCommand.LINT\"", "\"CliCommand.PLAN\"", "\"CliCommand.RECONCILE\"", "\"CliCommand.RESET_TARGET\"", "\"CliCommand.TEST\""]}, +"sqltide.cli.entry.constants:COMMANDS_REQUIRING_PIPELINES_ROOT": {"kind": "constant", "type": "frozenset", "value": ["\"CliCommand.AUDIT\"", "\"CliCommand.BUILD\"", "\"CliCommand.COMPILE\"", "\"CliCommand.DEPLOYMENT\"", "\"CliCommand.DESTROY\"", "\"CliCommand.DEV\"", "\"CliCommand.DISCOVER\"", "\"CliCommand.FORMAT\"", "\"CliCommand.LINT\"", "\"CliCommand.PLAN\"", "\"CliCommand.RECONCILE\"", "\"CliCommand.RENAME_PROJECT\"", "\"CliCommand.RESET_TARGET\"", "\"CliCommand.TEST\""]}, "sqltide.cli.entry.constants:COMMANDS_WITHOUT_ADAPTER_CONNECTION": {"kind": "constant", "type": "frozenset", "value": ["\"CliCommand.COMPILE\"", "\"CliCommand.DISCOVER\"", "\"CliCommand.FORMAT\"", "\"CliCommand.LINT\""]}, "sqltide.cli.entry.constants:DEV_CLI_VARIABLES_ENV_VAR": {"kind": "constant", "type": "str", "value": "SQLTIDE_INTERNAL_CLI_VARS"}, "sqltide.cli.entry.constants:DIRECT_ONLY_COMMANDS": {"kind": "constant", "type": "frozenset", "value": []}, @@ -232,7 +232,7 @@ "sqltide.cli.entry.models:ResolvedCliInvocation": {"fields": [["args", false, true, true], ["project_dir", false, true, true], ["pipelines_root", false, true, true], ["database", false, true, true], ["adapter", false, true, true], ["loaded_project", false, true, true], ["connection", false, true, true], ["cli_variables", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["args", "project_dir", "pipelines_root", "database", "adapter", "loaded_project", "connection", "cli_variables"], "module": "sqltide.cli.entry.models", "mro": ["sqltide.cli.entry.models.ResolvedCliInvocation", "builtins.object"], "qualname": "ResolvedCliInvocation"}, "sqltide.cli.entry.models:ResolvedCliProjectConfig": {"fields": [["connection", false, true, true], ["default_database", false, true, true], ["adapter_name", false, true, true], ["loaded_project", false, true, true], ["raw_connection", true, true, true], ["variables", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["connection", "default_database", "adapter_name", "loaded_project", "raw_connection", "variables"], "module": "sqltide.cli.entry.models", "mro": ["sqltide.cli.entry.models.ResolvedCliProjectConfig", "builtins.object"], "qualname": "ResolvedCliProjectConfig"}, "sqltide.cli.entry.models:ResolvedInvocationConnection": {"fields": [["connection", false, true, true], ["close_after_command", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["connection", "close_after_command"], "module": "sqltide.cli.entry.models", "mro": ["sqltide.cli.entry.models.ResolvedInvocationConnection", "builtins.object"], "qualname": "ResolvedInvocationConnection"}, -"sqltide.cli.entry.types:CliCommand": {"kind": "enum", "members": [["ADMIN", "admin"], ["PLAYGROUND", "playground"], ["DISCOVER", "discover"], ["DEV", "dev"], ["COMPILE", "compile"], ["LINT", "lint"], ["FORMAT", "format"], ["TEST", "test"], ["PLAN", "plan"], ["BUILD", "build"], ["DESTROY", "destroy"], ["RESET_TARGET", "reset-target"], ["DEPLOYMENT", "deployment"], ["AUDIT", "audit"], ["RECONCILE", "reconcile"], ["JANITOR", "janitor"], ["DOCTOR", "doctor"], ["REPAIR", "repair"]], "module": "sqltide.cli.entry.types", "mro": ["sqltide.cli.entry.types.CliCommand", "enum.StrEnum", "builtins.str", "enum.ReprEnum", "enum.Enum", "builtins.object"], "qualname": "CliCommand"}, +"sqltide.cli.entry.types:CliCommand": {"kind": "enum", "members": [["ADMIN", "admin"], ["PLAYGROUND", "playground"], ["DISCOVER", "discover"], ["DEV", "dev"], ["COMPILE", "compile"], ["LINT", "lint"], ["FORMAT", "format"], ["TEST", "test"], ["PLAN", "plan"], ["BUILD", "build"], ["DESTROY", "destroy"], ["RESET_TARGET", "reset-target"], ["DEPLOYMENT", "deployment"], ["AUDIT", "audit"], ["RECONCILE", "reconcile"], ["RENAME_PROJECT", "rename-project"], ["JANITOR", "janitor"], ["DOCTOR", "doctor"], ["REPAIR", "repair"]], "module": "sqltide.cli.entry.types", "mro": ["sqltide.cli.entry.types.CliCommand", "enum.StrEnum", "builtins.str", "enum.ReprEnum", "enum.Enum", "builtins.object"], "qualname": "CliCommand"}, "sqltide.cli.entry.types:CliSubcommand": {"kind": "enum", "members": [["ACTIVE_VIEW", "active-view"], ["DIFF", "diff"], ["LIST", "list"], ["SHOW", "show"], ["AUDIT", "audit"], ["PROMOTE", "promote"], ["ROLLBACK", "rollback"]], "module": "sqltide.cli.entry.types", "mro": ["sqltide.cli.entry.types.CliSubcommand", "enum.StrEnum", "builtins.str", "enum.ReprEnum", "enum.Enum", "builtins.object"], "qualname": "CliSubcommand"}, "sqltide.cli.plan.constants:DIRECT_MODE_LABEL": {"kind": "constant", "type": "str", "value": "direct"}, "sqltide.cli.plan.models:PlanCommandOptions": {"fields": [["pipelines_root", false, true, true], ["database", false, true, true], ["selectors", false, true, true], ["deployment_id", false, true, true], ["full_refresh", false, true, true], ["start_time", false, true, true], ["json_output", false, true, true], ["verbose", false, true, true], ["changed", true, true, true], ["include_missing_upstream", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["pipelines_root", "database", "selectors", "deployment_id", "full_refresh", "start_time", "json_output", "verbose", "changed", "include_missing_upstream"], "module": "sqltide.cli.plan.models", "mro": ["sqltide.cli.plan.models.PlanCommandOptions", "builtins.object"], "qualname": "PlanCommandOptions"}, diff --git a/src/sqltide/_native.pyi b/src/sqltide/_native.pyi index 7c993267..79122b52 100644 --- a/src/sqltide/_native.pyi +++ b/src/sqltide/_native.pyi @@ -2413,6 +2413,7 @@ class CliCommand(StrEnum): DEPLOYMENT = "deployment" AUDIT = "audit" RECONCILE = "reconcile" + RENAME_PROJECT = "rename-project" JANITOR = "janitor" DOCTOR = "doctor" REPAIR = "repair" @@ -7168,6 +7169,18 @@ def run_deployment_promotion( json_output: bool, client: AdapterConnection, ) -> int: ... +def run_rename_project( + *, + pipelines_root: Path, + project_dir: Path, + new_name: str, + database: str | None, + control_store_url: str | None, + apply: bool, + client: AdapterConnection, + loaded_project: LoadedProject | None, + adapter_profile: CompilerAdapterProfile, +) -> int: ... def run_reconcile( *, pipelines_root: Path, diff --git a/tests/unit/src/sqltide/cli/test_help_baseline.py b/tests/unit/src/sqltide/cli/test_help_baseline.py index a2067c2d..c418e654 100644 --- a/tests/unit/src/sqltide/cli/test_help_baseline.py +++ b/tests/unit/src/sqltide/cli/test_help_baseline.py @@ -13,7 +13,7 @@ CliHelpBaselineTestCase( description="captures root help", argv=("--help",), - expected_sha256="b079994e78490e17030dd3f51fe5ec6b0549faab2c0f7030c5c9321c0c72f4bd", + expected_sha256="0f958a0704f3fcd2e684d9ac738c547d0785cc4be3aefe90d8291306b78de0ae", ), CliHelpBaselineTestCase( description="captures admin help", @@ -105,6 +105,11 @@ argv=("reconcile", "--help"), expected_sha256="fc01e63a3e0a2dff1d689e074284bacae7d43d68fa2043bd6c32ca3f3afb9652", ), + CliHelpBaselineTestCase( + description="captures rename-project help", + argv=("rename-project", "--help"), + expected_sha256="7ec7020376f7af22c1b7561474d34cf379d74d325b50e259247fa57fc7f881eb", + ), CliHelpBaselineTestCase( description="captures janitor help", argv=("janitor", "--help"), From c0570d5e33bf82b88e59198dfb42354c867d85dc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:15:38 +0000 Subject: [PATCH 03/10] feat(plan): warn when a Kafka table consumes as an undeclared consumer group When a managed Kafka table's live kafka_group_name differs from the declared group, sqt plan (direct, virtual and mixed) prints a warning naming both groups and noting that offsets would not carry over if the table were recreated; --json lists it under consumer_group_warnings. The UI's source health shows the same warning. The check reuses the catalog the plan already read and changes no plan or build decision. --- crates/sqltide-executor/src/sources.rs | 76 +++++++++++ crates/sqltide-python/src/build_entry.rs | 21 ++- crates/sqltide-python/src/dev_server/state.rs | 18 ++- crates/sqltide-python/src/plan_warnings.rs | 120 +++++++++++++++++- ui/src/lib/api/_helpers/mapping.ts | 3 +- ui/src/lib/domain/types.ts | 10 ++ .../components/source-sidebar.svelte | 11 ++ ui/tests/lib/api/_helpers/mapping.test.ts | 28 +++- 8 files changed, 276 insertions(+), 11 deletions(-) diff --git a/crates/sqltide-executor/src/sources.rs b/crates/sqltide-executor/src/sources.rs index edd72b96..5b8a233a 100644 --- a/crates/sqltide-executor/src/sources.rs +++ b/crates/sqltide-executor/src/sources.rs @@ -540,6 +540,55 @@ fn settings_drift(resource: &Resource, relation: &Relation) -> Vec { .collect() } +/// A Kafka table consuming as a different consumer group than the project declares. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ConsumerGroupDrift { + pub table_name: String, + pub live_group: String, + pub declared_group: String, +} + +impl ConsumerGroupDrift { + /// The warning `sqt plan` and the UI show for this drift. + #[must_use] + pub fn message(&self) -> String { + format!( + "{} consumes as Kafka consumer group '{}' but the project declares '{}'. Recreating \ + it would consume as '{}', and the offsets committed for '{}' would not carry over.", + self.table_name, + self.live_group, + self.declared_group, + self.declared_group, + self.live_group + ) + } +} + +/// The drift of a live Kafka table's consumer group from `declared_group`, scoped to +/// `database` as ClickHouse consumes it. `None` when they match or the live group is unknown +/// or masked. +#[must_use] +pub fn consumer_group_drift( + table_name: &str, + declared_group: &str, + database: &str, + live_settings: &[(String, String)], +) -> Option { + let declared_group: String = database_scoped_consumer_group(declared_group, database); + let live_group: &str = live_settings + .iter() + .find(|(name, _)| name == GROUP_NAME_SETTING) + .map(|(_, value)| unquoted(value))?; + if live_group == declared_group || live_group == MASKED_SETTING_VALUE { + return None; + } + Some(ConsumerGroupDrift { + table_name: table_name.to_owned(), + live_group: live_group.to_owned(), + declared_group, + }) +} + fn actual_settings(relation: &Relation) -> HashMap { relation .settings @@ -557,6 +606,33 @@ fn unquoted(value: &str) -> &str { mod tests { use super::*; + #[test] + fn consumer_group_drift_names_both_groups_and_ignores_matching_or_masked_groups() { + let live = |group: &str| vec![("kafka_group_name".to_owned(), format!("'{group}'"))]; + assert_eq!( + consumer_group_drift("kafka__orders", "g", "shop-dev", &live("g_shop_dev")), + None + ); + assert_eq!( + consumer_group_drift("kafka__orders", "g", "shop_dev", &live("[HIDDEN]")), + None + ); + assert_eq!( + consumer_group_drift("kafka__orders", "g", "shop_dev", &[]), + None + ); + let drift = consumer_group_drift("kafka__orders", "new", "shop_dev", &live("old_shop_dev")) + .unwrap(); + assert_eq!(drift.live_group, "old_shop_dev"); + assert_eq!(drift.declared_group, "new_shop_dev"); + assert_eq!( + drift.message(), + "kafka__orders consumes as Kafka consumer group 'old_shop_dev' but the project \ + declares 'new_shop_dev'. Recreating it would consume as 'new_shop_dev', and the \ + offsets committed for 'old_shop_dev' would not carry over." + ); + } + fn kafka(name: &str, topic: &str) -> Resource { Resource::ManagedSource { name: name.to_owned(), diff --git a/crates/sqltide-python/src/build_entry.rs b/crates/sqltide-python/src/build_entry.rs index 033e50b2..dc168c86 100644 --- a/crates/sqltide-python/src/build_entry.rs +++ b/crates/sqltide-python/src/build_entry.rs @@ -386,7 +386,7 @@ fn run_plan<'py>( py, "prepare_build_workflow", &[ - ("analysis", analysis), + ("analysis", analysis.clone()), ("options", preparation_options), ("client", client.clone()), ("adapter_profile", adapter_profile.clone()), @@ -423,14 +423,21 @@ fn run_plan<'py>( ], )?; } + let plan_json: Value<'py> = if mixed { + preparation.getattr("plan_json")? + } else { + preparation.getattr("workflow")?.getattr("plan_json")? + }; + let (plan_text, plan_json) = crate::plan_warnings::with_consumer_group_drift_warnings( + &analysis, + &preparation, + preparation.getattr("plan_text")?, + plan_json, + )?; let rendered: Value<'py> = if options.getattr("json_output")?.is_truthy()? { - if mixed { - preparation.getattr("plan_json")? - } else { - preparation.getattr("workflow")?.getattr("plan_json")? - } + plan_json } else { - preparation.getattr("plan_text")?.add("\n")? + plan_text.add("\n")? }; let arguments: Bound<'py, PyDict> = PyDict::new(py); arguments.set_item("end", "")?; diff --git a/crates/sqltide-python/src/dev_server/state.rs b/crates/sqltide-python/src/dev_server/state.rs index dc6befdc..23956ef0 100644 --- a/crates/sqltide-python/src/dev_server/state.rs +++ b/crates/sqltide-python/src/dev_server/state.rs @@ -962,6 +962,7 @@ fn partitions<'py>(warehouse: Option>, kafka_lag: &Any<'py>) -> PyResul fn source_states<'py>( analysis: &Any<'py>, connection: &Any<'py>, + catalog: &Any<'py>, database: &str, stats: &Any<'py>, extents: &Bound<'py, PyDict>, @@ -1001,6 +1002,20 @@ fn source_states<'py>( } } let throughputs = throughputs(connection, database, &newest_ages)?; + let drifts: Bound<'py, PyDict> = PyDict::new(py); + for (source_name, drift) in + crate::plan_warnings::consumer_group_drifts(analysis, catalog, database)? + { + drifts.set_item( + source_name, + py_kwargs!(py; + tableName = &drift.table_name, + liveGroup = &drift.live_group, + declaredGroup = &drift.declared_group, + message = drift.message(), + )?, + )?; + } let states: Bound<'py, PyDict> = PyDict::new(py); for item in inputs { let throughput = get(throughputs.as_any(), &item.relation_name)?; @@ -1016,6 +1031,7 @@ fn source_states<'py>( throughput = &throughput, rowsPerSecond = rows_per_second(&throughput)?, partitions = partitions(warehouse.get_item(&item.relation_name)?, &item.kafka_lag)?, + consumerGroupDrift = drifts.get_item(item.source.getattr("key")?.getattr("name")?)?, )?; states.set_item(item.source.getattr("key")?.getattr("name")?, state)?; } @@ -1103,7 +1119,7 @@ pub(crate) fn build_state_payload<'py>( capturedAt = &captured, warehouseHealth = warehouse_health_payload(&health, &captured, adapter_name, &database_value)?, models = model_states(&analysis, &catalog, &baselines, &stats, &extents, &captured_at, &activity)?, - sources = source_states(&analysis, &connection, &database, &stats, &extents, &captured_at, &reader, &warehouse)?, + sources = source_states(&analysis, &connection, &catalog, &database, &stats, &extents, &captured_at, &reader, &warehouse)?, )? .into_any()) } diff --git a/crates/sqltide-python/src/plan_warnings.rs b/crates/sqltide-python/src/plan_warnings.rs index 15b7ddcd..8dfaf501 100644 --- a/crates/sqltide-python/src/plan_warnings.rs +++ b/crates/sqltide-python/src/plan_warnings.rs @@ -1,11 +1,17 @@ //! Planner warnings that need a point-in-time warehouse read: rebuild subtrees whose replay -//! source is empty. Counts go through the Python adapter connection. +//! source is empty, and Kafka tables consuming as a different consumer group than the project +//! declares. Reads go through the Python adapter connection. use pyo3::prelude::*; use pyo3::types::{PyDict, PyInt}; +use sqltide_executor::sources::{ConsumerGroupDrift, consumer_group_drift}; +use sqltide_planner::cli_style::CliStyle; use crate::direct_build::{Value, build, class, items, replace, text, tuple}; +const ADAPTER_MODELS: &str = "sqltide.adapter.models"; +const CONSUMER_GROUP_DRIFT: &str = "consumer_group_drift"; + const ADAPTER_EXCEPTIONS: &str = "sqltide.adapter.exceptions"; const PLANNER_MODELS: &str = "sqltide.compiler.planner.models"; const EMPTY_REPLAY_SOURCE: &str = "empty_replay_source"; @@ -134,6 +140,118 @@ fn add_empty_replay_source_warnings<'py>( ) } +/// `(source name, drift)` for every realized managed source whose live table in `database` +/// consumes as a different consumer group than the project declares, by source name. +pub(crate) fn consumer_group_drifts( + analysis: &Value<'_>, + catalog: &Value<'_>, + database: &str, +) -> PyResult> { + let py: Python<'_> = analysis.py(); + let managed_source: Value<'_> = class(py, ADAPTER_MODELS, "AdapterManagedSource")?; + let mut drifts: Vec<(String, ConsumerGroupDrift)> = Vec::new(); + for entry in analysis + .getattr("realized_project")? + .getattr("resources_by_logical_key")? + .call_method0("items")? + .try_iter()? + { + let entry: Value<'_> = entry?; + let source_name: String = entry.get_item(0)?.getattr("name")?.extract()?; + for resource in items(&entry.get_item(1)?)? { + if !resource.is_instance(&managed_source)? { + continue; + } + let table_name: String = resource.getattr("name")?.extract()?; + let relation: Value<'_> = catalog.call_method1("relation", (&table_name,))?; + if relation.is_none() { + continue; + } + let settings: Vec<(String, String)> = relation.getattr("settings")?.extract()?; + let declared: String = resource.getattr("consumer_group")?.extract()?; + if let Some(drift) = consumer_group_drift(&table_name, &declared, database, &settings) { + drifts.push((source_name.clone(), drift)); + } + } + } + drifts.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(drifts) +} + +/// The plan text and JSON with a consumer group drift warning for each Kafka table that +/// consumes as a different group than the project declares, read from the catalogs the plan +/// already loaded; unchanged when none does. +pub(crate) fn with_consumer_group_drift_warnings<'py>( + analysis: &Value<'py>, + preparation: &Value<'py>, + plan_text: Value<'py>, + plan_json: Value<'py>, +) -> PyResult<(Value<'py>, Value<'py>)> { + let py: Python<'py> = analysis.py(); + let previews: Vec> = if preparation.hasattr("virtual")? { + vec![ + preparation.getattr("virtual")?.getattr("preview")?, + preparation.getattr("direct")?.getattr("preview")?, + ] + } else { + vec![preparation.getattr("preview")?] + }; + let mut drifts: Vec<(String, ConsumerGroupDrift)> = Vec::new(); + for preview in previews { + let catalog: Value<'py> = if preview.hasattr("warehouse_snapshot")? { + preview.getattr("warehouse_snapshot")?.getattr("catalog")? + } else { + preview.getattr("target_catalog")? + }; + if catalog.is_none() { + continue; + } + let database: String = preview.getattr("database")?.extract()?; + for drift in consumer_group_drifts(analysis, &catalog, &database)? { + if !drifts.contains(&drift) { + drifts.push(drift); + } + } + } + if drifts.is_empty() { + return Ok((plan_text, plan_json)); + } + let style = CliStyle { + use_color: crate::cli::style::terminal_style(py)?.use_color, + }; + let mut lines: Vec = vec![plan_text.extract()?, String::new()]; + for (_, drift) in &drifts { + lines.push(style.warning(&format!("warning {}", drift.message()))); + } + let json = py.import("json")?; + let payload: Value<'py> = json.getattr("loads")?.call1((&plan_json,))?; + let mut warnings: Vec> = Vec::new(); + for (source_name, drift) in &drifts { + let warning: Bound<'py, PyDict> = PyDict::new(py); + warning.set_item("warning_code", CONSUMER_GROUP_DRIFT)?; + warning.set_item("message", drift.message())?; + warning.set_item("source_name", source_name)?; + warning.set_item("table_name", &drift.table_name)?; + warning.set_item("live_consumer_group", &drift.live_group)?; + warning.set_item("declared_consumer_group", &drift.declared_group)?; + warnings.push(warning.into_any()); + } + payload.set_item( + "consumer_group_warnings", + pyo3::types::PyList::new(py, warnings)?, + )?; + let keywords: Bound<'py, PyDict> = PyDict::new(py); + keywords.set_item("indent", 2)?; + let plan_json: String = json + .getattr("dumps")? + .call((payload,), Some(&keywords))? + .extract()?; + Ok(( + lines.join("\n").into_pyobject(py)?.into_any(), + format!("{plan_json}\n").into_pyobject(py)?.into_any(), + )) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(add_empty_replay_source_warnings, module)?) } diff --git a/ui/src/lib/api/_helpers/mapping.ts b/ui/src/lib/api/_helpers/mapping.ts index 589195d0..6a894a62 100644 --- a/ui/src/lib/api/_helpers/mapping.ts +++ b/ui/src/lib/api/_helpers/mapping.ts @@ -233,7 +233,8 @@ function sourceFromServer(source: Payload, live: Payload): Source { rows: (live.rows as number) ?? 0, partitions: ((live.partitions ?? []) as Payload[]).map(partitionFromServer), throughput: buckets, - throughputWindowSeconds: (throughput?.windowSeconds as number | null) ?? null + throughputWindowSeconds: (throughput?.windowSeconds as number | null) ?? null, + consumerGroupDrift: (live.consumerGroupDrift as Source['live']['consumerGroupDrift']) ?? null } }; } diff --git a/ui/src/lib/domain/types.ts b/ui/src/lib/domain/types.ts index 5f2bf964..3cb32a67 100644 --- a/ui/src/lib/domain/types.ts +++ b/ui/src/lib/domain/types.ts @@ -50,6 +50,14 @@ export type PartitionState = { newestEventAt: string; }; +/** A live Kafka table consuming as a different consumer group than the project declares. */ +type ConsumerGroupDrift = { + tableName: string; + liveGroup: string; + declaredGroup: string; + message: string; +}; + type SourceLiveState = { rowsPerSecond: number; /** @@ -70,6 +78,8 @@ type SourceLiveState = { throughput: number[]; /** The window the throughput buckets cover, from the server query. */ throughputWindowSeconds: number | null; + /** Set when the live Kafka table's consumer group differs from the declared one. */ + consumerGroupDrift: ConsumerGroupDrift | null; }; export type Source = { diff --git a/ui/src/lib/source-browser/components/source-sidebar.svelte b/ui/src/lib/source-browser/components/source-sidebar.svelte index f6640b59..26b98a17 100644 --- a/ui/src/lib/source-browser/components/source-sidebar.svelte +++ b/ui/src/lib/source-browser/components/source-sidebar.svelte @@ -24,6 +24,17 @@ {#if source.live.throughputWindowSeconds}
last {formatDuration(source.live.throughputWindowSeconds)}
{/if} + {#if source.live.consumerGroupDrift} +
+
Consumer group drift
+
{source.live.consumerGroupDrift.message}
+
+ {/if} diff --git a/ui/tests/lib/api/_helpers/mapping.test.ts b/ui/tests/lib/api/_helpers/mapping.test.ts index aa5d4e7b..15a5f3e2 100644 --- a/ui/tests/lib/api/_helpers/mapping.test.ts +++ b/ui/tests/lib/api/_helpers/mapping.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; import { planFromServer, projectFromServer } from '$lib/api/_helpers/mapping'; -import type { Project } from '$lib/domain/types'; +import type { Project, Source } from '$lib/domain/types'; import type { Plan } from '$lib/planning/types'; describe('plan mapping', () => { @@ -134,6 +134,32 @@ describe('project mapping', () => { expect(project.models[0]?.status).toBe('unknown'); }); + it('given consumer group drift when mapped then both groups and the warning stay intact', () => { + const drift: Source['live']['consumerGroupDrift'] = { + tableName: 'kafka__orders', + liveGroup: 'streambuild_shop_dev_orders_shop_dev', + declaredGroup: 'streambuild_storefront_dev_orders_shop_dev', + message: 'offsets would not carry over' + }; + const project: Project = projectFromServer( + { + project: {}, + sources: [{ name: 'orders' }, { name: 'customers' }], + pipelines: [], + models: [], + audits: [], + tests: [], + macros: [] + }, + { + capturedAt: '2026-08-23T10:00:00Z', + sources: { orders: { consumerGroupDrift: drift }, customers: {} } + } + ); + + expect(project.sources.map((source) => source.live.consumerGroupDrift)).toEqual([drift, null]); + }); + it('given warehouse health when mapped then zero and unknown measurements remain distinct', () => { const project: Project = projectFromServer( { From fc7848f8088109abf81925beb606827df34d81d7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:15:39 +0000 Subject: [PATCH 04/10] test(rename): cover project rename and consumer group drift on ClickHouse Builds a direct project with a managed Kafka source on a real ClickHouse target and checks that a rename keeps the Kafka table, consumer group and plan unchanged, copies manifest history visible to destroy planning, writes nothing on a dry run, refuses an unchanged name, completes after a lost config edit without duplicating manifests, refuses foreign manifest history and conflicting role grants, moves role grants, and reports consumer group drift in plan text, plan JSON and the UI state payload. --- .../sqltide/cli/rename_project/__init__.py | 1 + .../cli/rename_project/main/__init__.py | 1 + .../cli/rename_project/main/_test_types.py | 41 ++ .../cli/rename_project/main/helpers.py | 132 +++++++ .../main/test_rename_project.py | 363 ++++++++++++++++++ 5 files changed, 538 insertions(+) create mode 100644 tests/integration/src/sqltide/cli/rename_project/__init__.py create mode 100644 tests/integration/src/sqltide/cli/rename_project/main/__init__.py create mode 100644 tests/integration/src/sqltide/cli/rename_project/main/_test_types.py create mode 100644 tests/integration/src/sqltide/cli/rename_project/main/helpers.py create mode 100644 tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py diff --git a/tests/integration/src/sqltide/cli/rename_project/__init__.py b/tests/integration/src/sqltide/cli/rename_project/__init__.py new file mode 100644 index 00000000..609b6983 --- /dev/null +++ b/tests/integration/src/sqltide/cli/rename_project/__init__.py @@ -0,0 +1 @@ +"""Project rename integration checks.""" diff --git a/tests/integration/src/sqltide/cli/rename_project/main/__init__.py b/tests/integration/src/sqltide/cli/rename_project/main/__init__.py new file mode 100644 index 00000000..3cedb5b2 --- /dev/null +++ b/tests/integration/src/sqltide/cli/rename_project/main/__init__.py @@ -0,0 +1 @@ +"""Project rename entrypoint integration tests.""" diff --git a/tests/integration/src/sqltide/cli/rename_project/main/_test_types.py b/tests/integration/src/sqltide/cli/rename_project/main/_test_types.py new file mode 100644 index 00000000..2ef3ba8f --- /dev/null +++ b/tests/integration/src/sqltide/cli/rename_project/main/_test_types.py @@ -0,0 +1,41 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ProjectRenameTestCase: + description: str + old_name: str + new_name: str + expected_config: str + + +@dataclass(frozen=True) +class ProjectRenameConflictTestCase: + description: str + new_name: str + conflicting_manifest_id: str + expected_error_fragment: str + + +@dataclass(frozen=True) +class RoleAssignmentRenameTestCase: + description: str + new_name: str + role_name: str + expected_moved_roles: tuple[str, ...] + + +@dataclass(frozen=True) +class RoleAssignmentConflictTestCase: + description: str + new_name: str + role_name: str + expected_error_fragment: str + + +@dataclass(frozen=True) +class ConsumerGroupDriftTestCase: + description: str + renamed_config: str + expected_live_group_prefix: str + expected_declared_group_prefix: str diff --git a/tests/integration/src/sqltide/cli/rename_project/main/helpers.py b/tests/integration/src/sqltide/cli/rename_project/main/helpers.py new file mode 100644 index 00000000..1afb2204 --- /dev/null +++ b/tests/integration/src/sqltide/cli/rename_project/main/helpers.py @@ -0,0 +1,132 @@ +"""Build one managed Kafka project and run `sqt` against a real ClickHouse target.""" + +from pathlib import Path + +import pytest +from clickhouse_connect.driver.client import Client + +from sqltide._native import run_cli +from tests.integration.src.sqltide.conftest import ClickHouseConnectionSettings + +PROJECT_CONFIG: str = ( + "# Shop analytics project.\n" + 'name = "shop" # project identity\n' + 'default_target = "test"\n' + "\n" + "[defaults]\n" + 'pipeline_mode = "direct"\n' + "\n" + "[targets.test]\n" + "# Overridden with --database in tests.\n" + 'database = "analytics"\n' +) +SOURCE_CONFIG: str = ( + "sources:\n" + " - name: orders\n" + " kind: kafka\n" + " broker_list: kafka.example.com:9092\n" + " topic: shop.orders\n" + " replay_boundary:\n" + " mode: offsets\n" +) +MODEL_SQL: str = ( + "MODEL (\n" + ' engine "MergeTree",\n' + ' order_by ["order_id"],\n' + ");\n" + "\n" + "SELECT\n" + " CAST(JSONExtractString(kafka_value, 'order_id') AS String) AS order_id,\n" + " CAST(_replay_landed_at AS DateTime64(3)) AS _replay_landed_at\n" + 'FROM __ref("orders")\n' +) + + +def write_shop_project(project_dir: Path) -> Path: + """Write a direct-mode project with one managed Kafka source and one model.""" + project_dir.mkdir(parents=True, exist_ok=True) + (project_dir / "sqltide_project.toml").write_text(PROJECT_CONFIG, encoding="utf-8") + (project_dir / "sources").mkdir() + (project_dir / "sources" / "orders.yml").write_text(SOURCE_CONFIG, encoding="utf-8") + pipeline_dir: Path = project_dir / "pipelines" / "pl__orders" + pipeline_dir.mkdir(parents=True) + (pipeline_dir / "orders_clean.sql").write_text(MODEL_SQL, encoding="utf-8") + return project_dir + + +def connection_arguments( + *, settings: ClickHouseConnectionSettings, project_dir: Path, database: str +) -> tuple[str, ...]: + return ( + "--project-dir", + str(project_dir), + "--host", + settings.host, + "--port", + str(settings.port), + "--username", + settings.username, + "--password", + settings.password, + "--database", + database, + ) + + +def built_shop_project( + *, + tmp_path: Path, + settings: ClickHouseConnectionSettings, + database: str, + monkeypatch: pytest.MonkeyPatch, +) -> tuple[Path, tuple[str, ...]]: + """A built shop project and the connection arguments every command takes.""" + monkeypatch.delenv("SQLTIDE_CONTROL_STORE_URL", raising=False) + project_dir: Path = write_shop_project(tmp_path / "shop") + arguments: tuple[str, ...] = connection_arguments( + settings=settings, project_dir=project_dir, database=database + ) + assert run_cli(("sqt", "build", "--auto-approve", *arguments)) == 0 + return project_dir, arguments + + +def plan_json(arguments: tuple[str, ...], capsys: pytest.CaptureFixture[str]) -> str: + """The `sqt plan --json` output for the project.""" + _ = capsys.readouterr() + assert run_cli(("sqt", "plan", "--json", *arguments)) == 0 + return capsys.readouterr().out + + +def manifest_ids(client: Client, *, database: str, identity: str) -> tuple[str, ...]: + return tuple( + str(row[0]) + for row in client.query( + f"SELECT manifest_id FROM {database}._sqltide_manifests " + "WHERE project_identity = {identity:String} ORDER BY manifest_id", + parameters={"identity": identity}, + ).result_rows + ) + + +def kafka_group_name(client: Client, *, database: str) -> str: + """The consumer group the live Kafka table for `orders` consumes as.""" + definition: str = str( + client.query( + "SELECT create_table_query FROM system.tables " + "WHERE database = {database:String} AND name = 'kafka__orders'", + parameters={"database": database}, + ).result_rows[0][0] + ) + return definition.split("kafka_group_name = '", 1)[1].split("'", 1)[0] + + +def catalog_definitions(client: Client, *, database: str) -> tuple[tuple[str, str], ...]: + return tuple( + (str(row[0]), str(row[1])) + for row in client.query( + "SELECT name, create_table_query FROM system.tables " + "WHERE database = {database:String} AND NOT startsWith(name, '_sqltide') " + "ORDER BY name", + parameters={"database": database}, + ).result_rows + ) diff --git a/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py b/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py new file mode 100644 index 00000000..7e4ce7d8 --- /dev/null +++ b/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py @@ -0,0 +1,363 @@ +import json +from collections.abc import Mapping +from pathlib import Path +from typing import cast +from uuid import UUID + +import pytest +from clickhouse_connect.driver.client import Client + +import sqltide._native as _native +from sqltide._native import ( + ControlStore, + analyze_project, + build_compiler_adapter_profile, + build_state_payload, + run_cli, +) +from sqltide.adapter.classes.adapter_connection import AdapterConnection +from sqltide.adapter.models import AdapterManifestSnapshot +from sqltide.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter +from sqltide.auth.models import ProjectRoleAssignment +from tests.integration.src.sqltide.cli.helpers import build_managed_clickhouse_client +from tests.integration.src.sqltide.cli.rename_project.main._test_types import ( + ConsumerGroupDriftTestCase, + ProjectRenameConflictTestCase, + ProjectRenameTestCase, + RoleAssignmentConflictTestCase, + RoleAssignmentRenameTestCase, +) +from tests.integration.src.sqltide.cli.rename_project.main.helpers import ( + PROJECT_CONFIG, + built_shop_project, + catalog_definitions, + kafka_group_name, + manifest_ids, + plan_json, +) +from tests.integration.src.sqltide.conftest import ClickHouseConnectionSettings + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_case", + [ + ProjectRenameTestCase( + description="keeps groups, plan and manifest history; repeats are no-ops", + old_name="shop", + new_name="storefront", + expected_config=PROJECT_CONFIG.replace( + 'name = "shop" # project identity\n', + 'name = "storefront" # project identity\n' + "# Keeps the Kafka consumer groups this project used before it was renamed.\n" + 'consumer_group_identity = "shop"\n', + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_built_project_when_renaming_then_live_state_carries_over( + test_case: ProjectRenameTestCase, + tmp_path: Path, + clickhouse_connection_settings: ClickHouseConnectionSettings, + clickhouse_client: Client, + clickhouse_database: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir, arguments = built_shop_project( + tmp_path=tmp_path, + settings=clickhouse_connection_settings, + database=clickhouse_database, + monkeypatch=monkeypatch, + ) + config_path: Path = project_dir / "sqltide_project.toml" + plan_before: str = plan_json(arguments, capsys) + catalog_before: tuple[tuple[str, str], ...] = catalog_definitions( + clickhouse_client, database=clickhouse_database + ) + group_before: str = kafka_group_name(clickhouse_client, database=clickhouse_database) + recorded: tuple[str, ...] = manifest_ids( + clickhouse_client, database=clickhouse_database, identity=test_case.old_name + ) + + assert run_cli(("sqt", "rename-project", test_case.new_name, *arguments)) == 0 + dry_run: str = capsys.readouterr().out + assert config_path.read_text(encoding="utf-8") == PROJECT_CONFIG + assert ( + manifest_ids(clickhouse_client, database=clickhouse_database, identity=test_case.new_name) + == () + ) + assert "Kafka consumer groups: 1 unchanged" in dry_run + assert f"{len(recorded)} of {len(recorded)} manifest(s) to copy" in dry_run + assert "Dry run: nothing was changed." in dry_run + + assert run_cli(("sqt", "rename-project", test_case.new_name, "--apply", *arguments)) == 0 + renamed_config: str = config_path.read_text(encoding="utf-8") + assert renamed_config == test_case.expected_config + assert kafka_group_name(clickhouse_client, database=clickhouse_database) == group_before + assert catalog_definitions(clickhouse_client, database=clickhouse_database) == catalog_before + assert plan_json(arguments, capsys) == plan_before + connection: AdapterConnection = build_managed_clickhouse_client( + clickhouse_connection_settings, database=clickhouse_database + ) + destroy_history: AdapterManifestSnapshot = connection.load_manifests( + database=clickhouse_database, + project_identity=test_case.new_name, + target_name="test", + target_database=clickhouse_database, + ) + assert tuple(sorted(manifest.manifest_id for manifest in destroy_history.manifests)) == ( + recorded + ) + + assert run_cli(("sqt", "rename-project", test_case.new_name, "--apply", *arguments)) == 1 + assert "already named" in capsys.readouterr().err + config_path.write_text(PROJECT_CONFIG, encoding="utf-8") + assert run_cli(("sqt", "rename-project", test_case.new_name, "--apply", *arguments)) == 0 + assert f"0 of {len(recorded)} manifest(s) copied" in capsys.readouterr().out + assert config_path.read_text(encoding="utf-8") == renamed_config + assert ( + manifest_ids(clickhouse_client, database=clickhouse_database, identity=test_case.new_name) + == recorded + ) + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_case", + [ + ProjectRenameConflictTestCase( + description="refuses when the new name already has its own manifest history", + new_name="storefront", + conflicting_manifest_id="manifest-from-another-project", + expected_error_fragment="another project already uses that name here", + ) + ], + ids=lambda case: case.description, +) +def test_given_new_name_with_foreign_manifests_when_renaming_then_rename_is_refused( + test_case: ProjectRenameConflictTestCase, + tmp_path: Path, + clickhouse_connection_settings: ClickHouseConnectionSettings, + clickhouse_client: Client, + clickhouse_database: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir, arguments = built_shop_project( + tmp_path=tmp_path, + settings=clickhouse_connection_settings, + database=clickhouse_database, + monkeypatch=monkeypatch, + ) + clickhouse_client.command( + f"INSERT INTO {clickhouse_database}._sqltide_manifests " + "(manifest_id, project_identity, target_name, target_database, manifest_version, " + "published_at) VALUES ({id:String}, {identity:String}, 'test', {database:String}, 1, " + "now64(6))", + parameters={ + "id": test_case.conflicting_manifest_id, + "identity": test_case.new_name, + "database": clickhouse_database, + }, + ) + _ = capsys.readouterr() + + assert run_cli(("sqt", "rename-project", test_case.new_name, "--apply", *arguments)) == 1 + + assert test_case.expected_error_fragment in capsys.readouterr().err + assert (project_dir / "sqltide_project.toml").read_text(encoding="utf-8") == PROJECT_CONFIG + assert manifest_ids( + clickhouse_client, database=clickhouse_database, identity=test_case.new_name + ) == (test_case.conflicting_manifest_id,) + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_case", + [ + RoleAssignmentRenameTestCase( + description="moves project role assignments to the new name", + new_name="storefront", + role_name="operator", + expected_moved_roles=("operator",), + ) + ], + ids=lambda case: case.description, +) +def test_given_project_role_grants_when_renaming_then_grants_follow_the_project( + test_case: RoleAssignmentRenameTestCase, + tmp_path: Path, + clickhouse_connection_settings: ClickHouseConnectionSettings, + clickhouse_database: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, arguments = built_shop_project( + tmp_path=tmp_path, + settings=clickhouse_connection_settings, + database=clickhouse_database, + monkeypatch=monkeypatch, + ) + url: str = f"sqlite:///{tmp_path / 'control.db'}" + store: ControlStore = ControlStore(url=url) + user_id: UUID = store.create_user(username="ada").user_id + store.grant_project_role( + user_id=user_id, + project_name="shop", + role_name=test_case.role_name, + target_name=None, + actor_user_id=None, + ) + + assert ( + run_cli( + ( + "sqt", + "rename-project", + test_case.new_name, + "--apply", + "--control-store-url", + url, + *arguments, + ) + ) + == 0 + ) + + moved: tuple[ProjectRoleAssignment, ...] = store.list_project_role_assignments( + user_id=user_id, project_name=test_case.new_name + ) + assert tuple(assignment.role_name for assignment in moved) == test_case.expected_moved_roles + assert store.list_project_role_assignments(user_id=user_id, project_name="shop") == () + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_case", + [ + RoleAssignmentConflictTestCase( + description="refuses when both names hold the same active role", + new_name="storefront", + role_name="operator", + expected_error_fragment="already holds role assignments", + ) + ], + ids=lambda case: case.description, +) +def test_given_conflicting_role_grants_when_renaming_then_rename_is_refused( + test_case: RoleAssignmentConflictTestCase, + tmp_path: Path, + clickhouse_connection_settings: ClickHouseConnectionSettings, + clickhouse_client: Client, + clickhouse_database: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir, arguments = built_shop_project( + tmp_path=tmp_path, + settings=clickhouse_connection_settings, + database=clickhouse_database, + monkeypatch=monkeypatch, + ) + url: str = f"sqlite:///{tmp_path / 'control.db'}" + store: ControlStore = ControlStore(url=url) + user_id: UUID = store.create_user(username="ada").user_id + for project_name in ("shop", test_case.new_name): + store.grant_project_role( + user_id=user_id, + project_name=project_name, + role_name=test_case.role_name, + target_name=None, + actor_user_id=None, + ) + _ = capsys.readouterr() + + assert ( + run_cli( + ( + "sqt", + "rename-project", + test_case.new_name, + "--apply", + "--control-store-url", + url, + *arguments, + ) + ) + == 1 + ) + + assert test_case.expected_error_fragment in capsys.readouterr().err + assert (project_dir / "sqltide_project.toml").read_text(encoding="utf-8") == PROJECT_CONFIG + assert ( + manifest_ids(clickhouse_client, database=clickhouse_database, identity=test_case.new_name) + == () + ) + assert len(store.list_project_role_assignments(user_id=user_id, project_name="shop")) == 1 + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_case", + [ + ConsumerGroupDriftTestCase( + description="plan and source health warn when the project name changes by hand", + renamed_config=PROJECT_CONFIG.replace('name = "shop"', 'name = "storefront"'), + expected_live_group_prefix="streambuild_shop_test_orders_", + expected_declared_group_prefix="streambuild_storefront_test_orders_", + ) + ], + ids=lambda case: case.description, +) +def test_given_live_group_differs_when_planning_then_drift_is_reported_without_blocking( + test_case: ConsumerGroupDriftTestCase, + tmp_path: Path, + clickhouse_connection_settings: ClickHouseConnectionSettings, + clickhouse_client: Client, + clickhouse_database: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir, arguments = built_shop_project( + tmp_path=tmp_path, + settings=clickhouse_connection_settings, + database=clickhouse_database, + monkeypatch=monkeypatch, + ) + (project_dir / "sqltide_project.toml").write_text(test_case.renamed_config, encoding="utf-8") + live_group: str = f"{test_case.expected_live_group_prefix}{clickhouse_database}" + declared_group: str = f"{test_case.expected_declared_group_prefix}{clickhouse_database}" + _ = capsys.readouterr() + + assert run_cli(("sqt", "plan", *arguments)) == 0 + plan_text: str = capsys.readouterr().out + warnings: list[dict[str, str]] = json.loads(plan_json(arguments, capsys))[ + "consumer_group_warnings" + ] + connection: AdapterConnection = build_managed_clickhouse_client( + clickhouse_connection_settings, database=clickhouse_database + ) + payload: dict[str, object] = build_state_payload( + analysis=analyze_project( + pipelines_root=project_dir / "pipelines", + loaded_project=_native.load_project_input_for_path(path=project_dir), + adapter_profile=build_compiler_adapter_profile(ClickHouseAdapter()), + ), + connection=connection, + database=clickhouse_database, + ) + + assert ( + f"consumes as Kafka consumer group '{live_group}' but the project declares " + f"'{declared_group}'" + ) in plan_text + assert "would not carry over" in plan_text + assert [ + (warning["live_consumer_group"], warning["declared_consumer_group"]) for warning in warnings + ] == [(live_group, declared_group)] + sources: Mapping[str, Mapping[str, Mapping[str, str]]] = cast( + Mapping[str, Mapping[str, Mapping[str, str]]], payload["sources"] + ) + assert sources["orders"]["consumerGroupDrift"]["liveGroup"] == live_group + assert kafka_group_name(clickhouse_client, database=clickhouse_database) == live_group From 45e9dd88ef54bab8e029582e373a57c0f94f7ff4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:15:39 +0000 Subject: [PATCH 05/10] docs: document project renames and consumer_group_identity Add the sqt rename-project reference, a Project identity section with consumer_group_identity and Renaming a project to the project configuration guide, and the consumer group drift warning to the plan and Sources pages. --- website/src/content/docs/docs/cli/index.mdx | 1 + website/src/content/docs/docs/cli/plan.mdx | 5 ++ .../content/docs/docs/cli/rename-project.mdx | 63 +++++++++++++++++++ .../docs/concepts/project-configuration.mdx | 40 ++++++++++++ website/src/content/docs/docs/ui/sources.mdx | 4 ++ website/src/navigation.mjs | 2 +- 6 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 website/src/content/docs/docs/cli/rename-project.mdx diff --git a/website/src/content/docs/docs/cli/index.mdx b/website/src/content/docs/docs/cli/index.mdx index 53c71db8..46cbc6e1 100644 --- a/website/src/content/docs/docs/cli/index.mdx +++ b/website/src/content/docs/docs/cli/index.mdx @@ -19,6 +19,7 @@ SQLTide 0.60.0. Run from inside a project or pass `--project-dir PATH`. | Inspect stable bindings | `sqt doctor` | Read-only warehouse inspection | | Repair one stable view | `sqt repair active-view` | Explicit mutation | | Adopt compatible metadata baseline | `sqt reconcile` | Preview; `--apply` confirms and writes metadata | +| Rename the project | `sqt rename-project` | Preview; `--apply` copies metadata, moves role grants and edits `sqltide_project.toml` | | Clean deployment physicals | `sqt janitor` | Preview; `--apply` drops without another prompt | | Destroy/reset owned relations | `sqt destroy`, `sqt reset-target` | Interactive frozen-plan review and typed challenges | | Manage accounts | `sqt admin` | Control-store mutation; no warehouse required | diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index 93390167..fcba6441 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -23,6 +23,11 @@ The effective project mode chooses the planner: - **Virtual environments:** plan changed deployment objects, replay roots, and policies against published and staged state. +In both modes, a warning follows the plan when a live Kafka table consumes as a different consumer +group than the project declares, for example after `name` was edited by hand. It names both groups +and notes that offsets would not carry over if the table were recreated. `--json` lists these +warnings under `consumer_group_warnings`. The warning does not change or block the plan. + Use `--deployment-id` in virtual-environment mode to inspect exact deployment-specific physical names. Its timestamp prefix also fixes the deployment creation timestamp, so unchanged warehouse state and the same explicit identity produce deterministic workflow bytes. If omitted, `plan` diff --git a/website/src/content/docs/docs/cli/rename-project.mdx b/website/src/content/docs/docs/cli/rename-project.mdx new file mode 100644 index 00000000..ba5521ab --- /dev/null +++ b/website/src/content/docs/docs/cli/rename-project.mdx @@ -0,0 +1,63 @@ +--- +title: "sqt rename-project" +description: "Rename a project while keeping its Kafka consumer groups, manifest history and role grants." +--- + +Preview a project rename for the selected target: + +```bash +sqt rename-project storefront +sqt rename-project storefront --target prod +``` + +The default is a dry run. It prints the configuration edit, the Kafka consumer groups that stay the +same, the number of manifests to copy and the number of project role assignments to move, and +writes nothing. + +Apply the rename explicitly: + +```bash +sqt rename-project storefront --apply +``` + +`--apply` does not prompt. It runs these steps in order: + +1. Copies the target's manifest history from the old name to the new one under the target mutation + lock, so [`sqt destroy`](/docs/cli/destruction/) keeps finding relations recorded before the + rename. +2. Moves project role assignments, active and revoked, from the old name to the new one in the + control store. +3. Edits `sqltide_project.toml` in place: sets `name` and, when a managed Kafka source uses the + default consumer group, adds `consumer_group_identity` set to the old name. Comments and + formatting are kept. + +The configuration is edited last, so a failure leaves the old name working, and running the same +command again completes the rename. Steps that already ran are not repeated: manifests already +copied are skipped and assignments already moved are not moved again. + +The command refuses to run when: + +- the new name equals the current name, is empty or blank-padded, or contains `${` +- a build or other run holds the target mutation lock +- the target already records manifests for the new name that did not come from the old name +- both names hold the same active role assignment for the same user, role and target scope +- the rename would change any Kafka consumer group + +## Options + +| Option | Purpose | +| --- | --- | +| `NEW_NAME` | The new project name | +| `--apply` | Apply the rename; without it the command is a dry run | +| `--target` | Named project target whose warehouse history is copied | +| `--database` | Target database override | +| `--control-store-url` | Account control store; defaults to `SQLTIDE_CONTROL_STORE_URL`, then the project's `.sqltide/control.db` when it exists | +| `--project-dir`, `--host`, `--port`, `--username`, `--password`, `--vars` | As for other commands | + +Manifest history is copied for the selected target only. Other targets keep their history under +the old name, so `sqt destroy` on those targets does not see relations recorded there before the +rename. The dry run lists those targets. + +Restart a running `sqt dev` server after a rename. See +[Renaming a project](/docs/concepts/project-configuration/#renaming-a-project) for what a rename +changes. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index fba69904..bbc9c40e 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -64,6 +64,46 @@ production = true This immutable classification adds the exact `PRODUCTION` challenge to target reset. It is not accepted in `sqltide_local.toml`, so a local override cannot weaken or add production status. +## Project identity + +`name` identifies the project in three places that outlive a single run: + +- the default Kafka consumer group of every managed source without its own `consumer_group`, + `streambuild___` plus the target database suffix +- the manifest history `sqt destroy` uses to find relations recorded by earlier builds +- the project role assignments the UI grants to users + +Set `consumer_group_identity` to use another value in place of `name` in default consumer groups: + +```toml +name = "storefront" +consumer_group_identity = "shop" +``` + +It must be a non-empty literal string. A source's own `consumer_group` still takes precedence. +When the key is unset, default consumer groups use `name`, exactly as before. + +### Renaming a project + +Rename a project with [`sqt rename-project`](/docs/cli/rename-project/) rather than by editing +`name`. The command: + +- keeps every Kafka consumer group: when a managed source uses the default group, it sets + `consumer_group_identity` to the old name, so sources resume from their committed offsets +- copies the target's manifest history to the new name, so `sqt destroy` still finds relations + recorded before the rename +- moves project role assignments in the control store to the new name +- edits `name` in `sqltide_project.toml`, keeping comments and formatting + +Editing `name` by hand changes the default consumer group of every managed source that has no +`consumer_group`. `sqt plan` and the source health panel in the UI then warn that the live Kafka +table consumes as a different group than the project declares, and that its committed offsets +would not carry over if the table were recreated. The warning does not block a plan or build. + +Renaming the project directory is a separate change. Run history, invocations, and audit and check +status are keyed by the directory name, not by `name`, so a new directory name starts them afresh. +No command migrates that history. + Commands that connect to ClickHouse require `host`, `port`, `username`, and `password` after all configuration layers are merged. SQLTide supplies no connection defaults. Use `password = ""` when the ClickHouse account intentionally has an empty password. diff --git a/website/src/content/docs/docs/ui/sources.mdx b/website/src/content/docs/docs/ui/sources.mdx index e5bb1660..bf81a976 100644 --- a/website/src/content/docs/docs/ui/sources.mdx +++ b/website/src/content/docs/docs/ui/sources.mdx @@ -17,6 +17,10 @@ The Sources index summarizes every authored source. Open a managed Kafka source - the effective consumer group - landed, committed, and broker-end offsets by partition +When the live Kafka table consumes as a different consumer group than the project declares, the +source shows a consumer group drift warning naming both groups. Lag is read for the declared group. +See [Project identity](/docs/concepts/project-configuration/#project-identity). + Kafka lag and last arrival are different signals: - **Kafka lag** is the broker high-water offset minus the effective consumer group's committed diff --git a/website/src/navigation.mjs b/website/src/navigation.mjs index d3fd55e9..dfea6441 100644 --- a/website/src/navigation.mjs +++ b/website/src/navigation.mjs @@ -4,7 +4,7 @@ export const sidebar = [ group('Build a pipeline', 'concepts/', [['project-configuration', 'Project configuration'], ['sql-rules', 'SQL Rules and formatting'], ['pipelines', 'Sources and pipelines'], ['models', 'Typed SQL models'], ['retention', 'Retention policies'], ['declarations', 'Constants and enums'], ['macros', 'Python macros'], ['testing', 'SQL testing'], ['audits', 'Data-quality audits']]), group('Operate and deploy', 'concepts/', [['direct-mode', 'Direct builds'], ['deployments', 'Staged deployments'], ['replay-lineage', 'Replay and lineage'], ['row-error-quarantine', 'Row-error quarantine'], ['destructive-operations', 'Destructive operations'], ['architecture', 'ClickHouse architecture'], ['authentication-and-access', 'Authentication'], ['access-control', 'Access control'], ['sensors', 'Sensors'], ['providers', 'Providers']]), group('Development UI', 'ui/', [['overview', 'Overview'], ['pipelines', 'Pipelines'], ['warehouse-health', 'Warehouse health'], ['sources', 'Sources'], ['topics', 'Topics'], ['messages', 'Messages'], ['plans', 'Plans'], ['runs', 'Runs'], ['quality', 'Quality']]), - group('CLI reference', 'cli/', [['index', 'Command overview'], ...['discover', 'compile', 'plan', 'build', 'dev', 'test', 'audit', 'deployment', 'doctor', 'repair', 'reconcile', 'janitor', 'destruction', 'admin'].map(name => [name, name === 'destruction' ? 'sqt destroy / reset-target' : `sqt ${name}`])]), + group('CLI reference', 'cli/', [['index', 'Command overview'], ...['discover', 'compile', 'plan', 'build', 'dev', 'test', 'audit', 'deployment', 'doctor', 'repair', 'reconcile', 'rename-project', 'janitor', 'destruction', 'admin'].map(name => [name, name === 'destruction' ? 'sqt destroy / reset-target' : `sqt ${name}`])]), group('Examples', 'examples/', [['orders-demo', 'Kafka commerce demo'], ['staged-workflow', 'Stage a pipeline change']]), ]; export const docsRoutes = sidebar.flatMap(group => group.items.map(item => `/${item.slug.replace(/\/$/, '').replace(/\/index$/, '')}/`)); From 9086b652bae75a929c409c00cd1baf8084f9c87e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:27:38 +0000 Subject: [PATCH 06/10] feat(cli): complete a project rename per target with --from Each target has its own warehouse and usually its own UI host and control store, so a rename must be completable on the remaining targets after sqltide_project.toml already carries the new name. sqt rename-project NEW_NAME --from OLD_NAME leaves the config unchanged, checks that the target's Kafka consumer groups stay the same, copies the target's manifest history from OLD_NAME and moves the control store's role assignments from OLD_NAME, with the same lock, dry-run default, idempotence and conflict rules as the first run. It refuses when NEW_NAME is not the configured name, when the config is still named OLD_NAME, or when OLD_NAME equals the new name. The first run now names the --from command for the targets still to complete. --- .../sqltide-cli/src/argv/command_table.json | 11 ++ crates/sqltide-python/src/cli/dispatch.rs | 1 + .../src/rename_project_command.rs | 141 +++++++++++++++--- src/sqltide/_native.pyi | 1 + .../src/sqltide/cli/test_help_baseline.py | 2 +- 5 files changed, 131 insertions(+), 25 deletions(-) diff --git a/crates/sqltide-cli/src/argv/command_table.json b/crates/sqltide-cli/src/argv/command_table.json index 43a556bf..ccf40b8b 100644 --- a/crates/sqltide-cli/src/argv/command_table.json +++ b/crates/sqltide-cli/src/argv/command_table.json @@ -3484,6 +3484,17 @@ "help": "Apply the rename (dry-run by default)", "group": 1 }, + { + "kind": "store", + "options": [ + "--from" + ], + "dest": "from_name", + "default": null, + "help": "Previous project name; complete a rename on this target after sqltide_project.toml already carries the new name", + "metavar": "OLD_NAME", + "group": 1 + }, { "kind": "store", "options": [ diff --git a/crates/sqltide-python/src/cli/dispatch.rs b/crates/sqltide-python/src/cli/dispatch.rs index f0d233fb..ff2ec801 100644 --- a/crates/sqltide-python/src/cli/dispatch.rs +++ b/crates/sqltide-python/src/cli/dispatch.rs @@ -548,6 +548,7 @@ pub(crate) fn dispatch_cli_command<'py>( ("pipelines_root", require_pipelines_root(py, invocation)?), ("project_dir", project_dir), ("new_name", args.getattr("new_name")?), + ("from_name", getattr_none(args, "from_name")?), ("database", database), ( "control_store_url", diff --git a/crates/sqltide-python/src/rename_project_command.rs b/crates/sqltide-python/src/rename_project_command.rs index 7c2f1f3e..a212ce13 100644 --- a/crates/sqltide-python/src/rename_project_command.rs +++ b/crates/sqltide-python/src/rename_project_command.rs @@ -56,8 +56,11 @@ struct RolePlan<'py> { /// Everything the rename would change, computed without writing anything. struct RenamePlan<'py> { config_path: PathBuf, + /// The name the warehouse and control store still record. current_name: String, new_name: String, + /// Whether this run completes an earlier rename (`--from`) on another target. + completing: bool, /// The identity to add to the project config, when existing groups need it. added_identity: Option, /// `(source, database-scoped group)` for every managed source, unchanged by the rename. @@ -65,7 +68,8 @@ struct RenamePlan<'py> { manifests: ManifestPlan, roles: Option>, other_targets: Vec, - edited_config: String, + /// The edited project config, or `None` when the config already carries the new name. + edited_config: Option, } fn fail(py: Python<'_>, message: &str) -> PyErr { @@ -243,11 +247,57 @@ fn other_targets(loaded_project: &Any<'_>, target_name: &str) -> PyResult, + from_name: &str, + new_name: &str, + config_name: &str, +) -> PyResult<(String, String)> { + if from_name.trim().is_empty() || from_name.trim() != from_name { + return Err(fail( + py, + "--from must be the previous project name, a non-empty string without surrounding \ + whitespace", + )); + } + if from_name == new_name { + return Err(fail( + py, + &format!( + "--from names the previous project name and cannot be the new name '{new_name}'" + ), + )); + } + if from_name == config_name { + return Err(fail( + py, + &format!( + "sqltide_project.toml is still named '{config_name}'. Run `sqt rename-project \ + {new_name}` without --from first; use --from only on the remaining targets \ + after the config is renamed" + ), + )); + } + if new_name != config_name { + return Err(fail( + py, + &format!( + "With --from, the new name must be the name sqltide_project.toml already \ + carries ('{config_name}'), not '{new_name}'" + ), + )); + } + Ok((from_name.to_owned(), new_name.to_owned())) +} + #[allow(clippy::too_many_arguments)] fn plan_rename<'py>( py: Python<'py>, project_dir: &Any<'py>, new_name: &str, + from_name: Option<&str>, database: &str, control_store_option: &Any<'py>, client: &Any<'py>, @@ -267,8 +317,15 @@ fn plan_rename<'py>( let keys: ProjectIdentityKeys = project_identity_keys(&config_text).map_err(|message| fail(py, &message))?; let compiled: Any<'py> = analysis.getattr("compiled_project")?; - let current_name: String = compiled.getattr("project_name")?.extract()?; - let new_name: String = validated_name(py, new_name, ¤t_name)?; + let config_name: String = compiled.getattr("project_name")?.extract()?; + let (current_name, new_name): (String, String) = match from_name { + Some(from_name) => completed_names(py, from_name, new_name, &config_name)?, + None => ( + config_name.clone(), + validated_name(py, new_name, &config_name)?, + ), + }; + let completing: bool = from_name.is_some(); let identity: Option = compiled.getattr("consumer_group_identity")?.extract()?; if identity != keys @@ -293,12 +350,14 @@ fn plan_rename<'py>( let renamed = |identity: Option<&str>| { managed_source_consumer_groups(&compiled, adapter_profile, &new_name, identity) }; - // Keep the old name in the groups only when some group would otherwise change. - let added_identity: Option = if identity.is_none() && renamed(None)? != before { - Some(current_name.clone()) - } else { - None - }; + // Keep the old name in the groups only when some group would otherwise change. A completing + // run never edits the config, so its identity must already keep the groups. + let added_identity: Option = + if !completing && identity.is_none() && renamed(None)? != before { + Some(current_name.clone()) + } else { + None + }; let after: Vec<(String, String)> = renamed(identity.as_deref().or(added_identity.as_deref()))?; let changed: Vec = before .iter() @@ -311,8 +370,16 @@ fn plan_rename<'py>( py, &format!( "Renaming would change Kafka consumer groups, so sources would not resume from \ - their committed offsets: {}", - changed.join("; ") + their committed offsets: {}{}", + changed.join("; "), + if completing { + format!( + ". Set consumer_group_identity = {} in sqltide_project.toml first", + quoted(¤t_name) + ) + } else { + String::new() + } ), )); } @@ -321,9 +388,13 @@ fn plan_rename<'py>( .map(|(source, group)| (source, database_scoped_consumer_group(&group, database))) .collect(); let target_name: String = compiled.getattr("target_name")?.extract()?; - let from_identity: String = attr(py, NATIVE, "resolve_manifest_project_identity")? - .call((), Some(&py_kwargs!(py; analysis = analysis)?))? - .extract()?; + let from_identity: String = if completing { + current_name.clone() + } else { + attr(py, NATIVE, "resolve_manifest_project_identity")? + .call((), Some(&py_kwargs!(py; analysis = analysis)?))? + .extract()? + }; let manifests: ManifestPlan = manifest_plan(client, &from_identity, &new_name, &target_name, database)?; let roles: Option> = role_plan( @@ -332,13 +403,19 @@ fn plan_rename<'py>( ¤t_name, &new_name, )?; - let edited_config: String = - rename_project_text(&config_text, &new_name, added_identity.as_deref()) - .map_err(|message| fail(py, &message))?; + let edited_config: Option = if completing { + None + } else { + Some( + rename_project_text(&config_text, &new_name, added_identity.as_deref()) + .map_err(|message| fail(py, &message))?, + ) + }; Ok(RenamePlan { config_path, current_name, new_name, + completing, added_identity, groups, manifests, @@ -365,9 +442,17 @@ fn render_plan(plan: &RenamePlan<'_>, applied: bool) -> String { "Target: {} (database {})", manifests.target_name, manifests.database ), - format!("Config: {}", plan.config_path.display()), - format!(" name = {}", quoted(&plan.new_name)), ]; + if plan.completing { + lines.push(format!( + "Config: {} already named {}; unchanged", + plan.config_path.display(), + quoted(&plan.new_name) + )); + } else { + lines.push(format!("Config: {}", plan.config_path.display())); + lines.push(format!(" name = {}", quoted(&plan.new_name))); + } if let Some(identity) = &plan.added_identity { lines.push(format!( " consumer_group_identity = {} (added so existing Kafka consumer groups stay the same)", @@ -410,10 +495,13 @@ fn render_plan(plan: &RenamePlan<'_>, applied: bool) -> String { redacted_url(&roles.url) )), } - if !plan.other_targets.is_empty() { + if !plan.other_targets.is_empty() && !plan.completing { lines.push(format!( - "Other targets keep manifest history under '{}': {}", - manifests.from_identity, + "Other targets keep manifest history and role grants under '{}' until you run \ + `sqt rename-project {} --from {} --target ` there: {}", + plan.current_name, + plan.new_name, + plan.current_name, plan.other_targets.join(", ") )); } @@ -548,7 +636,10 @@ fn apply_rename(py: Python<'_>, client: &Any<'_>, plan: &RenamePlan<'_>) -> PyRe .rename_project(py, &plan.current_name, &plan.new_name)?, None => Vec::new(), }; - if let Err(error) = write_config(&plan.config_path, &plan.edited_config) { + let Some(edited_config) = &plan.edited_config else { + return Ok(()); + }; + if let Err(error) = write_config(&plan.config_path, edited_config) { let mut message: String = format!( "Could not write {}: {error}. The project keeps its old name", plan.config_path.display() @@ -577,7 +668,7 @@ fn apply_rename(py: Python<'_>, client: &Any<'_>, plan: &RenamePlan<'_>) -> PyRe #[pyfunction] #[pyo3(signature = ( *, pipelines_root, project_dir, new_name, database, control_store_url, apply, client, - loaded_project, adapter_profile, + loaded_project, adapter_profile, from_name = None, ))] #[allow(clippy::too_many_arguments)] fn run_rename_project<'py>( @@ -591,6 +682,7 @@ fn run_rename_project<'py>( client: Any<'py>, loaded_project: Any<'py>, adapter_profile: Any<'py>, + from_name: Option, ) -> PyResult { let analysis: Any<'py> = attr(py, NATIVE, "analyze_project")?.call( (), @@ -615,6 +707,7 @@ fn run_rename_project<'py>( py, &project_dir, &new_name, + from_name.as_deref(), &database, &control_store_url, &client, diff --git a/src/sqltide/_native.pyi b/src/sqltide/_native.pyi index 79122b52..bb2cef1c 100644 --- a/src/sqltide/_native.pyi +++ b/src/sqltide/_native.pyi @@ -7180,6 +7180,7 @@ def run_rename_project( client: AdapterConnection, loaded_project: LoadedProject | None, adapter_profile: CompilerAdapterProfile, + from_name: str | None = None, ) -> int: ... def run_reconcile( *, diff --git a/tests/unit/src/sqltide/cli/test_help_baseline.py b/tests/unit/src/sqltide/cli/test_help_baseline.py index c418e654..ce7bd2bc 100644 --- a/tests/unit/src/sqltide/cli/test_help_baseline.py +++ b/tests/unit/src/sqltide/cli/test_help_baseline.py @@ -108,7 +108,7 @@ CliHelpBaselineTestCase( description="captures rename-project help", argv=("rename-project", "--help"), - expected_sha256="7ec7020376f7af22c1b7561474d34cf379d74d325b50e259247fa57fc7f881eb", + expected_sha256="e5ed515f1e543aa5a750757a63811df4e532d6bc90c21d6ea7e50da214d80d84", ), CliHelpBaselineTestCase( description="captures janitor help", From 1aeeb97af3c3ddca112a0d460a033f4a0633e8bc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:27:38 +0000 Subject: [PATCH 07/10] test(rename): cover completing a rename on a second target with --from --- .../cli/rename_project/main/_test_types.py | 8 + .../cli/rename_project/main/helpers.py | 61 +++++- .../main/test_rename_project.py | 173 ++++++++++++++++++ 3 files changed, 239 insertions(+), 3 deletions(-) diff --git a/tests/integration/src/sqltide/cli/rename_project/main/_test_types.py b/tests/integration/src/sqltide/cli/rename_project/main/_test_types.py index 2ef3ba8f..3e1abe6e 100644 --- a/tests/integration/src/sqltide/cli/rename_project/main/_test_types.py +++ b/tests/integration/src/sqltide/cli/rename_project/main/_test_types.py @@ -39,3 +39,11 @@ class ConsumerGroupDriftTestCase: renamed_config: str expected_live_group_prefix: str expected_declared_group_prefix: str + + +@dataclass(frozen=True) +class MultiTargetRenameTestCase: + description: str + new_name: str + role_name: str + expected_config: str diff --git a/tests/integration/src/sqltide/cli/rename_project/main/helpers.py b/tests/integration/src/sqltide/cli/rename_project/main/helpers.py index 1afb2204..8f8ba203 100644 --- a/tests/integration/src/sqltide/cli/rename_project/main/helpers.py +++ b/tests/integration/src/sqltide/cli/rename_project/main/helpers.py @@ -1,11 +1,14 @@ """Build one managed Kafka project and run `sqt` against a real ClickHouse target.""" from pathlib import Path +from uuid import UUID import pytest from clickhouse_connect.driver.client import Client -from sqltide._native import run_cli +from sqltide._native import ControlStore, run_cli +from sqltide.adapter.models import AdapterManifestSnapshot +from tests.integration.src.sqltide.cli.helpers import build_managed_clickhouse_client from tests.integration.src.sqltide.conftest import ClickHouseConnectionSettings PROJECT_CONFIG: str = ( @@ -20,6 +23,19 @@ "# Overridden with --database in tests.\n" 'database = "analytics"\n' ) +TWO_TARGET_PROJECT_CONFIG: str = ( + 'name = "shop"\n' + 'default_target = "east"\n' + "\n" + "[defaults]\n" + 'pipeline_mode = "direct"\n' + "\n" + "[targets.east]\n" + 'database = "shop_east"\n' + "\n" + "[targets.west]\n" + 'database = "shop_west"\n' +) SOURCE_CONFIG: str = ( "sources:\n" " - name: orders\n" @@ -42,10 +58,10 @@ ) -def write_shop_project(project_dir: Path) -> Path: +def write_shop_project(project_dir: Path, project_config: str = PROJECT_CONFIG) -> Path: """Write a direct-mode project with one managed Kafka source and one model.""" project_dir.mkdir(parents=True, exist_ok=True) - (project_dir / "sqltide_project.toml").write_text(PROJECT_CONFIG, encoding="utf-8") + (project_dir / "sqltide_project.toml").write_text(project_config, encoding="utf-8") (project_dir / "sources").mkdir() (project_dir / "sources" / "orders.yml").write_text(SOURCE_CONFIG, encoding="utf-8") pipeline_dir: Path = project_dir / "pipelines" / "pl__orders" @@ -97,6 +113,45 @@ def plan_json(arguments: tuple[str, ...], capsys: pytest.CaptureFixture[str]) -> return capsys.readouterr().out +def target_arguments( + *, + settings: ClickHouseConnectionSettings, + project_dir: Path, + target: str, + database: str, +) -> tuple[str, ...]: + """Connection arguments for one target with its own database.""" + return ( + *connection_arguments(settings=settings, project_dir=project_dir, database=database), + "--target", + target, + ) + + +def destroy_history_ids( + *, settings: ClickHouseConnectionSettings, database: str, identity: str, target: str +) -> tuple[str, ...]: + """The manifest ids destroy planning reads for one project target.""" + snapshot: AdapterManifestSnapshot = build_managed_clickhouse_client( + settings, database=database + ).load_manifests( + database=database, + project_identity=identity, + target_name=target, + target_database=database, + ) + return tuple(sorted(manifest.manifest_id for manifest in snapshot.manifests)) + + +def role_names(store: ControlStore, *, user_id: UUID, project_name: str) -> tuple[str, ...]: + return tuple( + assignment.role_name + for assignment in store.list_project_role_assignments( + user_id=user_id, project_name=project_name + ) + ) + + def manifest_ids(client: Client, *, database: str, identity: str) -> tuple[str, ...]: return tuple( str(row[0]) diff --git a/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py b/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py index 7e4ce7d8..eb32ead7 100644 --- a/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py +++ b/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py @@ -22,6 +22,7 @@ from tests.integration.src.sqltide.cli.helpers import build_managed_clickhouse_client from tests.integration.src.sqltide.cli.rename_project.main._test_types import ( ConsumerGroupDriftTestCase, + MultiTargetRenameTestCase, ProjectRenameConflictTestCase, ProjectRenameTestCase, RoleAssignmentConflictTestCase, @@ -29,11 +30,16 @@ ) from tests.integration.src.sqltide.cli.rename_project.main.helpers import ( PROJECT_CONFIG, + TWO_TARGET_PROJECT_CONFIG, built_shop_project, catalog_definitions, + destroy_history_ids, kafka_group_name, manifest_ids, plan_json, + role_names, + target_arguments, + write_shop_project, ) from tests.integration.src.sqltide.conftest import ClickHouseConnectionSettings @@ -361,3 +367,170 @@ def test_given_live_group_differs_when_planning_then_drift_is_reported_without_b ) assert sources["orders"]["consumerGroupDrift"]["liveGroup"] == live_group assert kafka_group_name(clickhouse_client, database=clickhouse_database) == live_group + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_case", + [ + MultiTargetRenameTestCase( + description="renames on one target, then completes with --from on the other", + new_name="storefront", + role_name="operator", + expected_config=TWO_TARGET_PROJECT_CONFIG.replace( + 'name = "shop"\n', + 'name = "storefront"\n' + "# Keeps the Kafka consumer groups this project used before it was renamed.\n" + 'consumer_group_identity = "shop"\n', + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_two_targets_when_completing_rename_with_from_then_each_target_carries_over( + test_case: MultiTargetRenameTestCase, + tmp_path: Path, + clickhouse_connection_settings: ClickHouseConnectionSettings, + clickhouse_client: Client, + clickhouse_database: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + request: pytest.FixtureRequest, +) -> None: + monkeypatch.delenv("SQLTIDE_CONTROL_STORE_URL", raising=False) + west_database: str = f"{clickhouse_database}_west" + request.addfinalizer( + lambda: clickhouse_client.command(f"DROP DATABASE IF EXISTS {west_database} SYNC") + ) + project_dir: Path = write_shop_project(tmp_path / "shop", TWO_TARGET_PROJECT_CONFIG) + config_path: Path = project_dir / "sqltide_project.toml" + databases: dict[str, str] = {"east": clickhouse_database, "west": west_database} + stores: dict[str, ControlStore] = { + target: ControlStore(url=f"sqlite:///{tmp_path / target / 'control.db'}") + for target in databases + } + build_arguments: dict[str, tuple[str, ...]] = { + target: target_arguments( + settings=clickhouse_connection_settings, + project_dir=project_dir, + target=target, + database=database, + ) + for target, database in databases.items() + } + arguments: dict[str, tuple[str, ...]] = { + target: (*build_arguments[target], "--control-store-url", str(stores[target].url)) + for target in databases + } + user_ids: dict[str, UUID] = { + target: store.create_user(username="ada").user_id for target, store in stores.items() + } + _ = [ + store.grant_project_role( + user_id=user_ids[target], + project_name="shop", + role_name=test_case.role_name, + target_name=target, + actor_user_id=None, + ) + for target, store in stores.items() + ] + assert [ + run_cli(("sqt", "build", "--auto-approve", *build_arguments[target])) + for target in databases + ] == [0, 0] + groups: dict[str, str] = { + target: kafka_group_name(clickhouse_client, database=database) + for target, database in databases.items() + } + recorded: dict[str, tuple[str, ...]] = { + target: manifest_ids(clickhouse_client, database=database, identity="shop") + for target, database in databases.items() + } + + assert ( + run_cli(("sqt", "rename-project", test_case.new_name, "--from", "shop", *arguments["west"])) + == 1 + ) + assert "sqltide_project.toml is still named 'shop'" in capsys.readouterr().err + assert ( + run_cli(("sqt", "rename-project", test_case.new_name, "--apply", *arguments["east"])) == 0 + ) + renamed_config: str = config_path.read_text(encoding="utf-8") + assert run_cli(("sqt", "rename-project", "outlet", "--from", "shop", *arguments["west"])) == 1 + assert "the new name must be the name sqltide_project.toml already carries" in ( + capsys.readouterr().err + ) + assert ( + run_cli( + ( + "sqt", + "rename-project", + test_case.new_name, + "--from", + test_case.new_name, + *arguments["west"], + ) + ) + == 1 + ) + assert "cannot be the new name 'storefront'" in capsys.readouterr().err + assert ( + run_cli(("sqt", "rename-project", test_case.new_name, "--from", "shop", *arguments["west"])) + == 0 + ) + dry_run: str = capsys.readouterr().out + assert ( + manifest_ids(clickhouse_client, database=west_database, identity=test_case.new_name) == () + ) + assert ( + run_cli( + ( + "sqt", + "rename-project", + test_case.new_name, + "--from", + "shop", + "--apply", + *arguments["west"], + ) + ) + == 0 + ) + assert ( + run_cli( + ( + "sqt", + "rename-project", + test_case.new_name, + "--from", + "shop", + "--apply", + *arguments["west"], + ) + ) + == 0 + ) + + assert renamed_config == test_case.expected_config + assert config_path.read_text(encoding="utf-8") == renamed_config + assert 'already named "storefront"; unchanged' in dry_run + assert "Kafka consumer groups: 1 unchanged" in dry_run + assert { + target: kafka_group_name(clickhouse_client, database=database) + for target, database in databases.items() + } == groups + destroy_history: dict[str, tuple[str, ...]] = { + target: destroy_history_ids( + settings=clickhouse_connection_settings, + database=database, + identity=test_case.new_name, + target=target, + ) + for target, database in databases.items() + } + assert destroy_history == recorded + assert { + target: role_names(store, user_id=user_ids[target], project_name=test_case.new_name) + for target, store in stores.items() + } == {target: (test_case.role_name,) for target in databases} From 5e037d16211c92c8a28f3f866b1a065908bf6c76 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:27:38 +0000 Subject: [PATCH 08/10] docs: describe completing a project rename on each target Document --from and the multi-target procedure, and note the dry run's control store edge cases. --- website/src/content/docs/docs/cli/index.mdx | 2 +- .../content/docs/docs/cli/rename-project.mdx | 38 +++++++++++++++++-- .../docs/concepts/project-configuration.mdx | 4 ++ 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/website/src/content/docs/docs/cli/index.mdx b/website/src/content/docs/docs/cli/index.mdx index 46cbc6e1..30ea282f 100644 --- a/website/src/content/docs/docs/cli/index.mdx +++ b/website/src/content/docs/docs/cli/index.mdx @@ -19,7 +19,7 @@ SQLTide 0.60.0. Run from inside a project or pass `--project-dir PATH`. | Inspect stable bindings | `sqt doctor` | Read-only warehouse inspection | | Repair one stable view | `sqt repair active-view` | Explicit mutation | | Adopt compatible metadata baseline | `sqt reconcile` | Preview; `--apply` confirms and writes metadata | -| Rename the project | `sqt rename-project` | Preview; `--apply` copies metadata, moves role grants and edits `sqltide_project.toml` | +| Rename the project | `sqt rename-project` | Preview; `--apply` copies metadata, moves role grants and edits `sqltide_project.toml`; `--from` completes other targets | | Clean deployment physicals | `sqt janitor` | Preview; `--apply` drops without another prompt | | Destroy/reset owned relations | `sqt destroy`, `sqt reset-target` | Interactive frozen-plan review and typed challenges | | Manage accounts | `sqt admin` | Control-store mutation; no warehouse required | diff --git a/website/src/content/docs/docs/cli/rename-project.mdx b/website/src/content/docs/docs/cli/rename-project.mdx index ba5521ab..0d4b5424 100644 --- a/website/src/content/docs/docs/cli/rename-project.mdx +++ b/website/src/content/docs/docs/cli/rename-project.mdx @@ -12,7 +12,9 @@ sqt rename-project storefront --target prod The default is a dry run. It prints the configuration edit, the Kafka consumer groups that stay the same, the number of manifests to copy and the number of project role assignments to move, and -writes nothing. +writes nothing to the warehouse or the project config. Opening a configured control store that has +never been initialized creates its tables, and a legacy `.streambuild/` state directory is moved to +`.sqltide/`, as every command does. Apply the rename explicitly: @@ -49,14 +51,42 @@ The command refuses to run when: | --- | --- | | `NEW_NAME` | The new project name | | `--apply` | Apply the rename; without it the command is a dry run | +| `--from OLD_NAME` | Complete a rename on another target after `sqltide_project.toml` already carries `NEW_NAME` | | `--target` | Named project target whose warehouse history is copied | | `--database` | Target database override | | `--control-store-url` | Account control store; defaults to `SQLTIDE_CONTROL_STORE_URL`, then the project's `.sqltide/control.db` when it exists | | `--project-dir`, `--host`, `--port`, `--username`, `--password`, `--vars` | As for other commands | -Manifest history is copied for the selected target only. Other targets keep their history under -the old name, so `sqt destroy` on those targets does not see relations recorded there before the -rename. The dry run lists those targets. +## Projects with several targets + +Each target has its own warehouse, and usually its own UI host and control store. The first run +renames the project on one target and edits the configuration; every other target then completes +the rename with `--from`: + +1. On one target, rename and commit the configuration change: + + ```bash + sqt rename-project storefront --target dev --apply + ``` + +2. On each other target, typically on that target's own host so its control store is used, complete + the rename from the committed configuration: + + ```bash + sqt rename-project storefront --from shop --target prod + sqt rename-project storefront --from shop --target prod --apply + ``` + +With `--from`, the command leaves `sqltide_project.toml` unchanged. It checks that the target's +Kafka consumer groups stay the same, copies the target's manifest history from `OLD_NAME`, and moves +the control store's role assignments from `OLD_NAME`, under the same lock, dry-run default, conflict +rules and idempotence as the first run. It refuses when `NEW_NAME` is not the name the configuration +carries, when the configuration is still named `OLD_NAME` (run without `--from` first), or when +`OLD_NAME` equals the new name. + +Until a target is completed, it keeps its manifest history and role assignments under the old +name: `sqt destroy` there does not see relations recorded before the rename, and the target's UI +grants still name the old project. The first run's output lists the targets still to complete. Restart a running `sqt dev` server after a rename. See [Renaming a project](/docs/concepts/project-configuration/#renaming-a-project) for what a rename diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index bbc9c40e..bbfbd358 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -95,6 +95,10 @@ Rename a project with [`sqt rename-project`](/docs/cli/rename-project/) rather t - moves project role assignments in the control store to the new name - edits `name` in `sqltide_project.toml`, keeping comments and formatting +A project with several targets is renamed once and then completed on each other target with +`sqt rename-project NEW_NAME --from OLD_NAME --target `, usually on that target's own +host. See [projects with several targets](/docs/cli/rename-project/#projects-with-several-targets). + Editing `name` by hand changes the default consumer group of every managed source that has no `consumer_group`. `sqt plan` and the source health panel in the UI then warn that the live Kafka table consumes as a different group than the project declares, and that its committed offsets From 24607473d5e58fd4e3f7dbd60c45154f77d4f994 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:21:47 +0000 Subject: [PATCH 09/10] fix(rename): address review findings for project renames - --from merges the old manifest history into a target that has already built under the new name instead of refusing it; the first run still refuses a new name with foreign manifest history - --from refuses when nothing on the target records OLD_NAME (no manifests, no role assignments), which usually means a mistyped name - a rename refuses while a managed Kafka table's live consumer group already differs from the declared one, instead of pinning consumer_group_identity to a name the live tables do not use - the control store URL is read from the invocation's environment - plan JSON, and the published plan artifact build shares, always carry consumer_group_warnings (an empty list when there is no drift), added without re-rendering the rest of the plan - the plan output and docs say role grants move for the whole control store, what that means for targets on the old config, that two projects with one name cannot share a store, and that connecting renames legacy metadata tables during a dry run --- crates/sqltide-python/src/build_entry.rs | 6 +- crates/sqltide-python/src/build_workflow.rs | 38 +++- crates/sqltide-python/src/cli/dispatch.rs | 1 + crates/sqltide-python/src/plan_warnings.rs | 194 ++++++++++++------ .../src/rename_project_command.rs | 97 ++++++++- src/sqltide/_native.pyi | 1 + .../main/test_rename_project.py | 65 +++++- website/src/content/docs/docs/cli/plan.mdx | 3 +- .../content/docs/docs/cli/rename-project.mdx | 33 ++- 9 files changed, 348 insertions(+), 90 deletions(-) diff --git a/crates/sqltide-python/src/build_entry.rs b/crates/sqltide-python/src/build_entry.rs index dc168c86..09fe8fdd 100644 --- a/crates/sqltide-python/src/build_entry.rs +++ b/crates/sqltide-python/src/build_entry.rs @@ -428,11 +428,9 @@ fn run_plan<'py>( } else { preparation.getattr("workflow")?.getattr("plan_json")? }; - let (plan_text, plan_json) = crate::plan_warnings::with_consumer_group_drift_warnings( - &analysis, - &preparation, + let plan_text: Value<'py> = crate::plan_warnings::with_consumer_group_drift_text( preparation.getattr("plan_text")?, - plan_json, + &plan_json, )?; let rendered: Value<'py> = if options.getattr("json_output")?.is_truthy()? { plan_json diff --git a/crates/sqltide-python/src/build_workflow.rs b/crates/sqltide-python/src/build_workflow.rs index 6931a8ef..b67b4856 100644 --- a/crates/sqltide-python/src/build_workflow.rs +++ b/crates/sqltide-python/src/build_workflow.rs @@ -232,6 +232,13 @@ fn mixed_preparation<'py>( DIRECT, json_loads(py, &direct.getattr("workflow")?.getattr("plan_json")?)?, )?; + // Both phases read the same target, so the direct phase's list stands for the whole plan. + payload.set_item( + "consumer_group_warnings", + payload.get_item(DIRECT)?.map_or(Ok(None), |direct| { + direct.get_item("consumer_group_warnings").map(Some) + })?, + )?; let plan_json: String = json_dumps(payload.as_any(), Some(2), false)?; let mut requirements: Vec> = items(&virtual_.getattr("protection_requirements")?)?; requirements.extend(items(&direct.getattr("protection_requirements")?)?); @@ -300,7 +307,22 @@ fn prepare_direct<'py>( ("row_error_policy_changes", policy_changes.clone()), ], )?; - let plan_json: Value<'py> = native( + let drift_warnings = crate::plan_warnings::consumer_group_warning_items( + analysis, + &preview.getattr("warehouse_snapshot")?.getattr("catalog")?, + &preview.getattr("database")?, + )?; + let with_warnings = |plan_json: Value<'py>| -> PyResult> { + Ok(PyString::new( + py, + &crate::plan_warnings::with_consumer_group_warnings_json( + &plan_json.extract::()?, + &drift_warnings, + )?, + ) + .into_any()) + }; + let plan_json: Value<'py> = with_warnings(native( py, "render_direct_plan_json", &[ @@ -308,7 +330,7 @@ fn prepare_direct<'py>( ("adapter_name", adapter_name.clone()), ("row_error_policy_changes", policy_changes.clone()), ], - )?; + )?)?; let mut workflow: Value<'py> = native( py, "assemble_direct_build_workflow", @@ -320,7 +342,7 @@ fn prepare_direct<'py>( ], )?; if truthy(&workflow, "recreated_sources")? { - let plan_json: Value<'py> = native( + let plan_json: Value<'py> = with_warnings(native( py, "render_direct_plan_json", &[ @@ -329,7 +351,7 @@ fn prepare_direct<'py>( ("source_recreations", workflow.getattr("recreated_sources")?), ("row_error_policy_changes", policy_changes.clone()), ], - )?; + )?)?; let template: Value<'py> = replace(&workflow.getattr("template")?, &[("plan_json", plan_json)])?; workflow = replace(&workflow, &[("template", template)])?; @@ -468,6 +490,14 @@ fn prepare_virtual<'py>( ("row_error_policy_changes", policy_changes), ], )?; + payload.set_item( + "consumer_group_warnings", + crate::plan_warnings::consumer_group_warning_items( + analysis, + &preview.getattr("target_catalog")?, + &preview.getattr("database")?, + )?, + )?; let plan_json: String = json_dumps(&payload, Some(2), false)? + "\n"; let workflow: Value<'py> = native( py, diff --git a/crates/sqltide-python/src/cli/dispatch.rs b/crates/sqltide-python/src/cli/dispatch.rs index ff2ec801..6d2b1290 100644 --- a/crates/sqltide-python/src/cli/dispatch.rs +++ b/crates/sqltide-python/src/cli/dispatch.rs @@ -549,6 +549,7 @@ pub(crate) fn dispatch_cli_command<'py>( ("project_dir", project_dir), ("new_name", args.getattr("new_name")?), ("from_name", getattr_none(args, "from_name")?), + ("environment", invocation.connection.environment.clone()), ("database", database), ( "control_store_url", diff --git a/crates/sqltide-python/src/plan_warnings.rs b/crates/sqltide-python/src/plan_warnings.rs index 8dfaf501..b8a1d191 100644 --- a/crates/sqltide-python/src/plan_warnings.rs +++ b/crates/sqltide-python/src/plan_warnings.rs @@ -3,7 +3,7 @@ //! declares. Reads go through the Python adapter connection. use pyo3::prelude::*; -use pyo3::types::{PyDict, PyInt}; +use pyo3::types::{PyDict, PyInt, PyList}; use sqltide_executor::sources::{ConsumerGroupDrift, consumer_group_drift}; use sqltide_planner::cli_style::CliStyle; @@ -11,6 +11,7 @@ use crate::direct_build::{Value, build, class, items, replace, text, tuple}; const ADAPTER_MODELS: &str = "sqltide.adapter.models"; const CONSUMER_GROUP_DRIFT: &str = "consumer_group_drift"; +const CONSUMER_GROUP_WARNINGS_KEY: &str = "consumer_group_warnings"; const ADAPTER_EXCEPTIONS: &str = "sqltide.adapter.exceptions"; const PLANNER_MODELS: &str = "sqltide.compiler.planner.models"; @@ -178,55 +179,22 @@ pub(crate) fn consumer_group_drifts( Ok(drifts) } -/// The plan text and JSON with a consumer group drift warning for each Kafka table that -/// consumes as a different group than the project declares, read from the catalogs the plan -/// already loaded; unchanged when none does. -pub(crate) fn with_consumer_group_drift_warnings<'py>( +/// The `consumer_group_warnings` entries for one plan: each managed Kafka table in the +/// catalog the plan read that consumes as a different group than the project declares. Empty +/// when there is none or the plan read no catalog. +pub(crate) fn consumer_group_warning_items<'py>( analysis: &Value<'py>, - preparation: &Value<'py>, - plan_text: Value<'py>, - plan_json: Value<'py>, -) -> PyResult<(Value<'py>, Value<'py>)> { + catalog: &Value<'py>, + database: &Value<'py>, +) -> PyResult> { let py: Python<'py> = analysis.py(); - let previews: Vec> = if preparation.hasattr("virtual")? { - vec![ - preparation.getattr("virtual")?.getattr("preview")?, - preparation.getattr("direct")?.getattr("preview")?, - ] - } else { - vec![preparation.getattr("preview")?] - }; - let mut drifts: Vec<(String, ConsumerGroupDrift)> = Vec::new(); - for preview in previews { - let catalog: Value<'py> = if preview.hasattr("warehouse_snapshot")? { - preview.getattr("warehouse_snapshot")?.getattr("catalog")? - } else { - preview.getattr("target_catalog")? - }; - if catalog.is_none() { - continue; - } - let database: String = preview.getattr("database")?.extract()?; - for drift in consumer_group_drifts(analysis, &catalog, &database)? { - if !drifts.contains(&drift) { - drifts.push(drift); - } - } - } - if drifts.is_empty() { - return Ok((plan_text, plan_json)); + let items: Bound<'py, PyList> = PyList::empty(py); + if catalog.is_none() || database.is_none() { + return Ok(items); } - let style = CliStyle { - use_color: crate::cli::style::terminal_style(py)?.use_color, - }; - let mut lines: Vec = vec![plan_text.extract()?, String::new()]; - for (_, drift) in &drifts { - lines.push(style.warning(&format!("warning {}", drift.message()))); - } - let json = py.import("json")?; - let payload: Value<'py> = json.getattr("loads")?.call1((&plan_json,))?; - let mut warnings: Vec> = Vec::new(); - for (source_name, drift) in &drifts { + for (source_name, drift) in + consumer_group_drifts(analysis, catalog, &database.extract::()?)? + { let warning: Bound<'py, PyDict> = PyDict::new(py); warning.set_item("warning_code", CONSUMER_GROUP_DRIFT)?; warning.set_item("message", drift.message())?; @@ -234,24 +202,132 @@ pub(crate) fn with_consumer_group_drift_warnings<'py>( warning.set_item("table_name", &drift.table_name)?; warning.set_item("live_consumer_group", &drift.live_group)?; warning.set_item("declared_consumer_group", &drift.declared_group)?; - warnings.push(warning.into_any()); + items.append(warning)?; } - payload.set_item( - "consumer_group_warnings", - pyo3::types::PyList::new(py, warnings)?, - )?; - let keywords: Bound<'py, PyDict> = PyDict::new(py); - keywords.set_item("indent", 2)?; - let plan_json: String = json + Ok(items) +} + +/// A rendered plan JSON object with `consumer_group_warnings` appended as its last member and +/// every other byte unchanged. +pub(crate) fn with_consumer_group_warnings_json( + plan_json: &str, + warnings: &Bound<'_, PyList>, +) -> PyResult { + let py: Python<'_> = warnings.py(); + let indent: usize = top_level_indent(plan_json); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + if indent > 0 { + keywords.set_item("indent", indent)?; + } + let value: String = py + .import("json")? .getattr("dumps")? - .call((payload,), Some(&keywords))? + .call((warnings,), Some(&keywords))? .extract()?; - Ok(( - lines.join("\n").into_pyobject(py)?.into_any(), - format!("{plan_json}\n").into_pyobject(py)?.into_any(), + Ok(with_top_level_key( + plan_json, + CONSUMER_GROUP_WARNINGS_KEY, + &value, )) } +/// The plan text followed by a warning line for each consumer group drift the plan's JSON +/// records under `consumer_group_warnings`; unchanged when there is none. +pub(crate) fn with_consumer_group_drift_text<'py>( + plan_text: Value<'py>, + plan_json: &Value<'py>, +) -> PyResult> { + let py: Python<'py> = plan_text.py(); + let payload: Value<'py> = py.import("json")?.getattr("loads")?.call1((plan_json,))?; + let mut messages: Vec = Vec::new(); + let mut lists: Vec> = vec![payload.clone()]; + for mode in ["virtual", "direct"] { + if let Some(nested) = payload.downcast::()?.get_item(mode)? { + lists.push(nested); + } + } + for holder in lists { + let Ok(holder) = holder.downcast_into::() else { + continue; + }; + let Some(warnings) = holder.get_item(CONSUMER_GROUP_WARNINGS_KEY)? else { + continue; + }; + for warning in warnings.try_iter()? { + let message: String = warning?.get_item("message")?.extract()?; + if !messages.contains(&message) { + messages.push(message); + } + } + } + if messages.is_empty() { + return Ok(plan_text); + } + let style = CliStyle { + use_color: crate::cli::style::terminal_style(py)?.use_color, + }; + let mut lines: Vec = vec![plan_text.extract()?, String::new()]; + for message in &messages { + lines.push(style.warning(&format!("warning {message}"))); + } + Ok(lines.join("\n").into_pyobject(py)?.into_any()) +} + +/// The indentation of a JSON object's top-level keys: the leading spaces of its second line, or +/// 0 when the object is on one line. +fn top_level_indent(json: &str) -> usize { + json.lines() + .nth(1) + .map_or(0, |line| line.len() - line.trim_start_matches(' ').len()) +} + +/// `json`, a JSON object, with `"key": value` appended as its last member. Every existing byte +/// is kept; `value` is already rendered at the object's indentation for its first line, and +/// its later lines are shifted one level so they nest under the key. +fn with_top_level_key(json: &str, key: &str, value: &str) -> String { + let Some(end) = json.rfind('}') else { + return json.to_owned(); + }; + let indent: usize = top_level_indent(json); + let head: &str = json[..end].trim_end(); + let tail: &str = &json[end..]; + let separator: &str = if head.ends_with('{') { "" } else { "," }; + let nested: String = value.replace('\n', &format!("\n{}", " ".repeat(indent))); + if indent == 0 { + let space: &str = if separator.is_empty() { "" } else { " " }; + return format!("{head}{separator}{space}\"{key}\": {nested}{tail}"); + } + format!( + "{head}{separator}\n{}\"{key}\": {nested}\n{tail}", + " ".repeat(indent) + ) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(add_empty_replay_source_warnings, module)?) } + +#[cfg(test)] +mod tests { + use super::with_top_level_key; + + #[test] + fn appends_a_key_and_keeps_every_existing_byte() { + assert_eq!( + with_top_level_key("{\n \"mode\": \"direct\"\n}\n", "warnings", "[]"), + "{\n \"mode\": \"direct\",\n \"warnings\": []\n}\n" + ); + assert_eq!( + with_top_level_key( + "{\n \"mode\": \"direct\"\n}\n", + "warnings", + "[\n {\n \"a\": 1\n }\n]" + ), + "{\n \"mode\": \"direct\",\n \"warnings\": [\n {\n \"a\": 1\n }\n ]\n}\n" + ); + assert_eq!( + with_top_level_key("{\"mode\": \"direct\"}", "warnings", "[]"), + "{\"mode\": \"direct\", \"warnings\": []}" + ); + } +} diff --git a/crates/sqltide-python/src/rename_project_command.rs b/crates/sqltide-python/src/rename_project_command.rs index a212ce13..f8efd68c 100644 --- a/crates/sqltide-python/src/rename_project_command.rs +++ b/crates/sqltide-python/src/rename_project_command.rs @@ -44,6 +44,9 @@ struct ManifestPlan { to_copy: usize, /// Manifests recorded under the old identity. recorded: usize, + /// Whether the new identity may already hold manifests of its own, as it does once a + /// target being completed with `--from` has built under the new name. + allow_existing: bool, } /// The role assignments the rename moves, and where they live. @@ -149,12 +152,13 @@ fn manifest_plan( to_identity: &str, target_name: &str, database: &str, + allow_existing: bool, ) -> PyResult { let py: Python<'_> = client.py(); let recorded: BTreeSet = manifest_ids(client, database, from_identity, target_name)?; let copied: BTreeSet = manifest_ids(client, database, to_identity, target_name)?; let foreign: Vec<&String> = copied.difference(&recorded).collect(); - if !foreign.is_empty() { + if !foreign.is_empty() && !allow_existing { return Err(fail( py, &format!( @@ -172,20 +176,20 @@ fn manifest_plan( database: database.to_owned(), to_copy: recorded.difference(&copied).count(), recorded: recorded.len(), + allow_existing, }) } -/// The control store the project uses: `--control-store-url`, then the environment, then the -/// project-local store when it exists. +/// The control store the project uses: `--control-store-url`, then `SQLTIDE_CONTROL_STORE_URL` +/// in the invocation's environment, then the project-local store when it exists. fn control_store_url( - py: Python<'_>, project_dir: &Any<'_>, option: &Any<'_>, + environment: &Any<'_>, ) -> PyResult> { if option.is_truthy()? { return Ok(Some(option.extract()?)); } - let environment: Any<'_> = attr(py, "os", "environ")?; let configured: Any<'_> = environment.call_method1("get", (CONTROL_STORE_ENV_VAR,))?; if configured.is_truthy()? { return Ok(Some(configured.extract()?)); @@ -300,6 +304,7 @@ fn plan_rename<'py>( from_name: Option<&str>, database: &str, control_store_option: &Any<'py>, + environment: &Any<'py>, client: &Any<'py>, loaded_project: &Any<'py>, analysis: &Any<'py>, @@ -341,6 +346,7 @@ fn plan_rename<'py>( ), )); } + refuse_live_group_drift(py, client, analysis, database)?; let before: Vec<(String, String)> = managed_source_consumer_groups( &compiled, adapter_profile, @@ -395,14 +401,31 @@ fn plan_rename<'py>( .call((), Some(&py_kwargs!(py; analysis = analysis)?))? .extract()? }; - let manifests: ManifestPlan = - manifest_plan(client, &from_identity, &new_name, &target_name, database)?; + let manifests: ManifestPlan = manifest_plan( + client, + &from_identity, + &new_name, + &target_name, + database, + completing, + )?; let roles: Option> = role_plan( py, - control_store_url(py, project_dir, control_store_option)?, + control_store_url(project_dir, control_store_option, environment)?, ¤t_name, &new_name, )?; + let moving_roles: usize = roles.as_ref().map_or(0, |roles| roles.preview.moving); + if completing && manifests.recorded == 0 && moving_roles == 0 { + return Err(fail( + py, + &format!( + "Nothing on target '{target_name}' records project '{current_name}': it has no \ + manifests in database '{database}' and no role assignments in the control \ + store. Check the name passed to --from." + ), + )); + } let edited_config: Option = if completing { None } else { @@ -425,6 +448,47 @@ fn plan_rename<'py>( }) } +/// Refuse when a managed Kafka table on the target already consumes as a different consumer +/// group than the project declares: pinning `consumer_group_identity` to the current name would +/// then keep a group the live tables do not use. +fn refuse_live_group_drift( + py: Python<'_>, + client: &Any<'_>, + analysis: &Any<'_>, + database: &str, +) -> PyResult<()> { + if !client + .call_method1("database_exists", (database,))? + .is_truthy()? + { + return Ok(()); + } + let catalog: Any<'_> = client.call_method1("load_catalog", (database,))?; + let drifts = crate::plan_warnings::consumer_group_drifts(analysis, &catalog, database)?; + if drifts.is_empty() { + return Ok(()); + } + let tables: Vec = drifts + .iter() + .map(|(_, drift)| { + format!( + "{} consumes as '{}' but the project declares '{}'", + drift.table_name, drift.live_group, drift.declared_group + ) + }) + .collect(); + Err(fail( + py, + &format!( + "Kafka consumer groups on database '{database}' already differ from the project \ + declaration: {}. `sqt plan` shows this as a consumer group drift warning. Resolve \ + the drift before renaming, for example by restoring the name or \ + consumer_group_identity the live tables were created with.", + tables.join("; ") + ), + )) +} + fn quoted(value: &str) -> String { sqltide_config::project_rename::toml_basic_string(value) } @@ -497,13 +561,18 @@ fn render_plan(plan: &RenamePlan<'_>, applied: bool) -> String { } if !plan.other_targets.is_empty() && !plan.completing { lines.push(format!( - "Other targets keep manifest history and role grants under '{}' until you run \ + "Other targets keep manifest history under '{}' until you run \ `sqt rename-project {} --from {} --target ` there: {}", plan.current_name, plan.new_name, plan.current_name, plan.other_targets.join(", ") )); + lines.push( + "Role grants move for this whole control store. If other targets share it, their \ + UI users lose those grants until those targets run the renamed config." + .to_owned(), + ); } lines.push( "Unchanged: run history and audit and check status follow the project directory name, \ @@ -553,6 +622,7 @@ fn copy_manifests(client: &Any<'_>, plan: &ManifestPlan) -> PyResult { &plan.to_identity, &plan.target_name, &plan.database, + plan.allow_existing, )?; if fresh.to_copy == 0 { return Ok(0); @@ -577,6 +647,7 @@ fn copy_manifests(client: &Any<'_>, plan: &ManifestPlan) -> PyResult { &plan.to_identity, &plan.target_name, &plan.database, + plan.allow_existing, )?; if after.to_copy != 0 { return Err(fail( @@ -668,7 +739,7 @@ fn apply_rename(py: Python<'_>, client: &Any<'_>, plan: &RenamePlan<'_>) -> PyRe #[pyfunction] #[pyo3(signature = ( *, pipelines_root, project_dir, new_name, database, control_store_url, apply, client, - loaded_project, adapter_profile, from_name = None, + loaded_project, adapter_profile, from_name = None, environment = None, ))] #[allow(clippy::too_many_arguments)] fn run_rename_project<'py>( @@ -683,7 +754,12 @@ fn run_rename_project<'py>( loaded_project: Any<'py>, adapter_profile: Any<'py>, from_name: Option, + environment: Option>, ) -> PyResult { + let environment: Any<'py> = match environment { + Some(environment) if environment.is_truthy()? => environment, + _ => pyo3::types::PyDict::new(py).into_any(), + }; let analysis: Any<'py> = attr(py, NATIVE, "analyze_project")?.call( (), Some(&py_kwargs!(py; @@ -710,6 +786,7 @@ fn run_rename_project<'py>( from_name.as_deref(), &database, &control_store_url, + &environment, &client, &loaded_project, &analysis, diff --git a/src/sqltide/_native.pyi b/src/sqltide/_native.pyi index bb2cef1c..6c3038ea 100644 --- a/src/sqltide/_native.pyi +++ b/src/sqltide/_native.pyi @@ -7181,6 +7181,7 @@ def run_rename_project( loaded_project: LoadedProject | None, adapter_profile: CompilerAdapterProfile, from_name: str | None = None, + environment: Mapping[str, str] | None = None, ) -> int: ... def run_reconcile( *, diff --git a/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py b/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py index eb32ead7..4a5ddaea 100644 --- a/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py +++ b/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py @@ -475,13 +475,25 @@ def test_given_two_targets_when_completing_rename_with_from_then_each_target_car == 1 ) assert "cannot be the new name 'storefront'" in capsys.readouterr().err + assert run_cli(("sqt", "build", "--auto-approve", *build_arguments["west"])) == 0 + built_under_new_name: tuple[str, ...] = manifest_ids( + clickhouse_client, database=west_database, identity=test_case.new_name + ) + assert ( + run_cli( + ("sqt", "rename-project", test_case.new_name, "--from", "shopp", *arguments["west"]) + ) + == 1 + ) + assert "Nothing on target 'west' records project 'shopp'" in capsys.readouterr().err assert ( run_cli(("sqt", "rename-project", test_case.new_name, "--from", "shop", *arguments["west"])) == 0 ) dry_run: str = capsys.readouterr().out assert ( - manifest_ids(clickhouse_client, database=west_database, identity=test_case.new_name) == () + manifest_ids(clickhouse_client, database=west_database, identity=test_case.new_name) + == built_under_new_name ) assert ( run_cli( @@ -529,8 +541,57 @@ def test_given_two_targets_when_completing_rename_with_from_then_each_target_car ) for target, database in databases.items() } - assert destroy_history == recorded + assert len(built_under_new_name) == 1 + assert destroy_history == { + "east": recorded["east"], + "west": tuple(sorted(recorded["west"] + built_under_new_name)), + } assert { target: role_names(store, user_id=user_ids[target], project_name=test_case.new_name) for target, store in stores.items() } == {target: (test_case.role_name,) for target in databases} + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_case", + [ + ConsumerGroupDriftTestCase( + description="refuses to rename while live groups differ from the declaration", + renamed_config=PROJECT_CONFIG.replace('name = "shop"', 'name = "storefront"'), + expected_live_group_prefix="streambuild_shop_test_orders_", + expected_declared_group_prefix="streambuild_storefront_test_orders_", + ) + ], + ids=lambda case: case.description, +) +def test_given_live_group_drift_when_renaming_then_rename_is_refused( + test_case: ConsumerGroupDriftTestCase, + tmp_path: Path, + clickhouse_connection_settings: ClickHouseConnectionSettings, + clickhouse_client: Client, + clickhouse_database: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir, arguments = built_shop_project( + tmp_path=tmp_path, + settings=clickhouse_connection_settings, + database=clickhouse_database, + monkeypatch=monkeypatch, + ) + config_path: Path = project_dir / "sqltide_project.toml" + config_path.write_text(test_case.renamed_config, encoding="utf-8") + _ = capsys.readouterr() + + assert run_cli(("sqt", "rename-project", "outlet", "--apply", *arguments)) == 1 + + error: str = capsys.readouterr().err + assert ( + f"kafka__orders consumes as '{test_case.expected_live_group_prefix}{clickhouse_database}' " + "but the project declares " + f"'{test_case.expected_declared_group_prefix}{clickhouse_database}'" + ) in error + assert "consumer group drift warning" in error + assert config_path.read_text(encoding="utf-8") == test_case.renamed_config + assert manifest_ids(clickhouse_client, database=clickhouse_database, identity="outlet") == () diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index fcba6441..c8afac73 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -26,7 +26,8 @@ The effective project mode chooses the planner: In both modes, a warning follows the plan when a live Kafka table consumes as a different consumer group than the project declares, for example after `name` was edited by hand. It names both groups and notes that offsets would not carry over if the table were recreated. `--json` lists these -warnings under `consumer_group_warnings`. The warning does not change or block the plan. +warnings under `consumer_group_warnings`, an empty list when there are none; the same key is in +the published plan artifact. The warning does not change or block the plan. Use `--deployment-id` in virtual-environment mode to inspect exact deployment-specific physical names. Its timestamp prefix also fixes the deployment creation timestamp, so unchanged warehouse diff --git a/website/src/content/docs/docs/cli/rename-project.mdx b/website/src/content/docs/docs/cli/rename-project.mdx index 0d4b5424..a5b97273 100644 --- a/website/src/content/docs/docs/cli/rename-project.mdx +++ b/website/src/content/docs/docs/cli/rename-project.mdx @@ -12,9 +12,10 @@ sqt rename-project storefront --target prod The default is a dry run. It prints the configuration edit, the Kafka consumer groups that stay the same, the number of manifests to copy and the number of project role assignments to move, and -writes nothing to the warehouse or the project config. Opening a configured control store that has -never been initialized creates its tables, and a legacy `.streambuild/` state directory is moved to -`.sqltide/`, as every command does. +writes nothing to the warehouse or the project config. As with every command, a few legacy +migrations can still run: connecting renames legacy `_streambuild_*` metadata tables to +`_sqltide_*`, opening a configured control store that has never been initialized creates its +tables, and a legacy `.streambuild/` state directory is moved to `.sqltide/`. Apply the rename explicitly: @@ -44,6 +45,9 @@ The command refuses to run when: - the target already records manifests for the new name that did not come from the old name - both names hold the same active role assignment for the same user, role and target scope - the rename would change any Kafka consumer group +- a managed Kafka table on the target already consumes as a different consumer group than the + project declares, which `sqt plan` reports as consumer group drift; resolve that first, for + example by restoring the name or `consumer_group_identity` the live tables were created with ## Options @@ -80,13 +84,22 @@ the rename with `--from`: With `--from`, the command leaves `sqltide_project.toml` unchanged. It checks that the target's Kafka consumer groups stay the same, copies the target's manifest history from `OLD_NAME`, and moves the control store's role assignments from `OLD_NAME`, under the same lock, dry-run default, conflict -rules and idempotence as the first run. It refuses when `NEW_NAME` is not the name the configuration -carries, when the configuration is still named `OLD_NAME` (run without `--from` first), or when -`OLD_NAME` equals the new name. - -Until a target is completed, it keeps its manifest history and role assignments under the old -name: `sqt destroy` there does not see relations recorded before the rename, and the target's UI -grants still name the old project. The first run's output lists the targets still to complete. +rules and idempotence as the first run. The target may already have been built under the new name: +its own new manifests are kept and the old history is merged in, so `sqt destroy` sees both. It +refuses when `NEW_NAME` is not the name the configuration carries, when the configuration is still +named `OLD_NAME` (run without `--from` first), when `OLD_NAME` equals the new name, or when nothing +on the target records `OLD_NAME`: no manifests and no role assignments, which usually means a +mistyped name. + +Until a target is completed, it keeps its manifest history under the old name, so `sqt destroy` +there does not see relations recorded before the rename. The first run's output lists the targets +still to complete. + +Role assignments move for the whole control store a run uses. When several targets share one +control store, the first run moves every target's grants, and UI users of targets still running +the old configuration lose those grants until the targets get the renamed configuration. Role +assignments are keyed by project name only, so two projects with the same name in one shared +control store cannot be told apart. Restart a running `sqt dev` server after a rename. See [Renaming a project](/docs/concepts/project-configuration/#renaming-a-project) for what a rename From fa27e1c112a9a3a58631804eba4a28c153ebbe06 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:30:14 +0000 Subject: [PATCH 10/10] fix(rename): refuse --from with another project's history Completing a rename with --from OLD_NAME merged whatever history the target recorded under OLD_NAME. With two projects in one database, --from the other project's name copied its manifests, and destroy planning then treated its pipelines as retired pipelines of this project. Completing now requires consumer_group_identity, when set, to equal OLD_NAME, and otherwise the newest OLD_NAME manifest on the target to share a pipeline with the compiled project. The no-history refusal also names the legitimate case of a target never built under OLD_NAME, and the shared control store note is only printed when a control store is in use. --- .../src/rename_project_command.rs | 107 ++++++++++++++--- .../cli/rename_project/main/_test_types.py | 8 ++ .../cli/rename_project/main/helpers.py | 25 ++++ .../main/test_rename_project.py | 109 ++++++++++++++++++ .../content/docs/docs/cli/rename-project.mdx | 10 +- 5 files changed, 242 insertions(+), 17 deletions(-) diff --git a/crates/sqltide-python/src/rename_project_command.rs b/crates/sqltide-python/src/rename_project_command.rs index f8efd68c..de95eb7e 100644 --- a/crates/sqltide-python/src/rename_project_command.rs +++ b/crates/sqltide-python/src/rename_project_command.rs @@ -44,6 +44,8 @@ struct ManifestPlan { to_copy: usize, /// Manifests recorded under the old identity. recorded: usize, + /// The pipelines the old identity's newest manifest on the target names. + latest_pipelines: Vec, /// Whether the new identity may already hold manifests of its own, as it does once a /// target being completed with `--from` has built under the new name. allow_existing: bool, @@ -105,13 +107,14 @@ fn validated_name(py: Python<'_>, new_name: &str, current_name: &str) -> PyResul Ok(new_name.to_owned()) } -/// The manifest ids one identity holds for the target, or the reason they cannot be read. -fn manifest_ids( +/// The manifest ids one identity holds for the target, and the pipelines its newest manifest +/// names, or the reason they cannot be read. +fn manifest_history( client: &Any<'_>, database: &str, identity: &str, target_name: &str, -) -> PyResult> { +) -> PyResult<(BTreeSet, Vec)> { let py: Python<'_> = client.py(); let snapshot: Any<'_> = client.call_method( "load_manifests", @@ -139,11 +142,20 @@ fn manifest_ids( ), )); } - snapshot + let manifests: Vec> = snapshot .getattr("manifests")? .try_iter()? - .map(|manifest| manifest?.getattr("manifest_id")?.extract()) - .collect() + .collect::>()?; + // Newest first, as the manifest history query orders it. + let latest_pipelines: Vec = match manifests.first() { + Some(latest) => latest.getattr("pipelines")?.extract()?, + None => Vec::new(), + }; + let ids: BTreeSet = manifests + .iter() + .map(|manifest| manifest.getattr("manifest_id")?.extract()) + .collect::>()?; + Ok((ids, latest_pipelines)) } fn manifest_plan( @@ -155,8 +167,9 @@ fn manifest_plan( allow_existing: bool, ) -> PyResult { let py: Python<'_> = client.py(); - let recorded: BTreeSet = manifest_ids(client, database, from_identity, target_name)?; - let copied: BTreeSet = manifest_ids(client, database, to_identity, target_name)?; + let (recorded, latest_pipelines) = + manifest_history(client, database, from_identity, target_name)?; + let (copied, _) = manifest_history(client, database, to_identity, target_name)?; let foreign: Vec<&String> = copied.difference(&recorded).collect(); if !foreign.is_empty() && !allow_existing { return Err(fail( @@ -176,6 +189,7 @@ fn manifest_plan( database: database.to_owned(), to_copy: recorded.difference(&copied).count(), recorded: recorded.len(), + latest_pipelines, allow_existing, }) } @@ -422,10 +436,21 @@ fn plan_rename<'py>( &format!( "Nothing on target '{target_name}' records project '{current_name}': it has no \ manifests in database '{database}' and no role assignments in the control \ - store. Check the name passed to --from." + store. Either '{current_name}' is mistyped, or this target was never built \ + under it and this control store holds no grants for it, in which case there \ + is nothing to complete here." ), )); } + if completing { + refuse_foreign_history( + py, + &compiled, + identity.as_deref(), + ¤t_name, + &manifests, + )?; + } let edited_config: Option = if completing { None } else { @@ -448,6 +473,58 @@ fn plan_rename<'py>( }) } +/// Refuse to complete a rename from a name whose history on the target belongs to another +/// project: merging it would make destroy planning treat that project's pipelines as this +/// project's retired ones. A `consumer_group_identity` must equal OLD_NAME; without one, the +/// newest OLD_NAME manifest must share a pipeline with the compiled project. +fn refuse_foreign_history( + py: Python<'_>, + compiled: &Any<'_>, + identity: Option<&str>, + from_name: &str, + manifests: &ManifestPlan, +) -> PyResult<()> { + let foreign = |detail: String| -> PyErr { + fail( + py, + &format!( + "The history of '{from_name}' on target '{}' does not look like this project's: \ + {detail}. Pass the name this project had before the rename to --from.", + manifests.target_name + ), + ) + }; + if let Some(identity) = identity { + if identity != from_name { + return Err(foreign(format!( + "sqltide_project.toml keeps consumer_group_identity = {}, not {}", + quoted(identity), + quoted(from_name) + ))); + } + return Ok(()); + } + if manifests.recorded == 0 { + return Ok(()); + } + let current: Vec = compiled + .getattr("pipelines")? + .try_iter()? + .map(|pipeline| pipeline?.getattr("pipeline")?.getattr("name")?.extract()) + .collect::>()?; + if manifests + .latest_pipelines + .iter() + .any(|name| current.contains(name)) + { + return Ok(()); + } + Err(foreign(format!( + "its latest manifest names pipelines {} and none of them is in this project", + manifests.latest_pipelines.join(", ") + ))) +} + /// Refuse when a managed Kafka table on the target already consumes as a different consumer /// group than the project declares: pinning `consumer_group_identity` to the current name would /// then keep a group the live tables do not use. @@ -568,11 +645,13 @@ fn render_plan(plan: &RenamePlan<'_>, applied: bool) -> String { plan.current_name, plan.other_targets.join(", ") )); - lines.push( - "Role grants move for this whole control store. If other targets share it, their \ - UI users lose those grants until those targets run the renamed config." - .to_owned(), - ); + if plan.roles.is_some() { + lines.push( + "Role grants move for this whole control store. If other targets share it, \ + their UI users lose those grants until those targets run the renamed config." + .to_owned(), + ); + } } lines.push( "Unchanged: run history and audit and check status follow the project directory name, \ diff --git a/tests/integration/src/sqltide/cli/rename_project/main/_test_types.py b/tests/integration/src/sqltide/cli/rename_project/main/_test_types.py index 3e1abe6e..0787e853 100644 --- a/tests/integration/src/sqltide/cli/rename_project/main/_test_types.py +++ b/tests/integration/src/sqltide/cli/rename_project/main/_test_types.py @@ -47,3 +47,11 @@ class MultiTargetRenameTestCase: new_name: str role_name: str expected_config: str + + +@dataclass(frozen=True) +class ForeignHistoryTestCase: + description: str + new_name: str + foreign_name: str + expected_error_fragments: tuple[str, ...] diff --git a/tests/integration/src/sqltide/cli/rename_project/main/helpers.py b/tests/integration/src/sqltide/cli/rename_project/main/helpers.py index 8f8ba203..8e89b1f8 100644 --- a/tests/integration/src/sqltide/cli/rename_project/main/helpers.py +++ b/tests/integration/src/sqltide/cli/rename_project/main/helpers.py @@ -70,6 +70,31 @@ def write_shop_project(project_dir: Path, project_config: str = PROJECT_CONFIG) return project_dir +def write_project( + project_dir: Path, *, name: str, source: str, consumer_group: str, database: str +) -> Path: + """Write a direct-mode project whose one Kafka source declares its own consumer group.""" + project_dir.mkdir(parents=True, exist_ok=True) + (project_dir / "sqltide_project.toml").write_text( + f'name = "{name}"\ndefault_target = "test"\n\n[defaults]\npipeline_mode = "direct"\n\n' + f'[targets.test]\ndatabase = "{database}"\n', + encoding="utf-8", + ) + (project_dir / "sources").mkdir() + (project_dir / "sources" / f"{source}.yml").write_text( + SOURCE_CONFIG.replace("name: orders", f"name: {source}") + .replace("shop.orders", f"{name}.{source}") + .replace(" topic:", f" consumer_group: {consumer_group}\n topic:"), + encoding="utf-8", + ) + pipeline_dir: Path = project_dir / "pipelines" / f"pl__{source}" + pipeline_dir.mkdir(parents=True) + (pipeline_dir / f"{source}_clean.sql").write_text( + MODEL_SQL.replace('__ref("orders")', f'__ref("{source}")'), encoding="utf-8" + ) + return project_dir + + def connection_arguments( *, settings: ClickHouseConnectionSettings, project_dir: Path, database: str ) -> tuple[str, ...]: diff --git a/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py b/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py index 4a5ddaea..36b1f47a 100644 --- a/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py +++ b/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py @@ -22,6 +22,7 @@ from tests.integration.src.sqltide.cli.helpers import build_managed_clickhouse_client from tests.integration.src.sqltide.cli.rename_project.main._test_types import ( ConsumerGroupDriftTestCase, + ForeignHistoryTestCase, MultiTargetRenameTestCase, ProjectRenameConflictTestCase, ProjectRenameTestCase, @@ -33,12 +34,14 @@ TWO_TARGET_PROJECT_CONFIG, built_shop_project, catalog_definitions, + connection_arguments, destroy_history_ids, kafka_group_name, manifest_ids, plan_json, role_names, target_arguments, + write_project, write_shop_project, ) from tests.integration.src.sqltide.conftest import ClickHouseConnectionSettings @@ -595,3 +598,109 @@ def test_given_live_group_drift_when_renaming_then_rename_is_refused( assert "consumer group drift warning" in error assert config_path.read_text(encoding="utf-8") == test_case.renamed_config assert manifest_ids(clickhouse_client, database=clickhouse_database, identity="outlet") == () + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_case", + [ + ForeignHistoryTestCase( + description="refuses --from another project's name in the same database", + new_name="storefront", + foreign_name="billing", + expected_error_fragments=( + "The history of 'billing' on target 'test' does not look like this project's", + "pl__invoices", + 'consumer_group_identity = "shop", not "billing"', + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_other_project_in_database_when_completing_from_its_name_then_it_is_refused( + test_case: ForeignHistoryTestCase, + tmp_path: Path, + clickhouse_connection_settings: ClickHouseConnectionSettings, + clickhouse_client: Client, + clickhouse_database: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.delenv("SQLTIDE_CONTROL_STORE_URL", raising=False) + projects: dict[str, Path] = { + name: write_project( + tmp_path / name, + name=name, + source=source, + consumer_group=f"{name}-{source}", + database=clickhouse_database, + ) + for name, source in (("shop", "orders"), (test_case.foreign_name, "invoices")) + } + arguments: dict[str, tuple[str, ...]] = { + name: connection_arguments( + settings=clickhouse_connection_settings, + project_dir=project_dir, + database=clickhouse_database, + ) + for name, project_dir in projects.items() + } + assert [run_cli(("sqt", "build", "--auto-approve", *arguments[name])) for name in projects] == [ + 0, + 0, + ] + shop_history: tuple[str, ...] = manifest_ids( + clickhouse_client, database=clickhouse_database, identity="shop" + ) + config_path: Path = projects["shop"] / "sqltide_project.toml" + assert ( + run_cli(("sqt", "rename-project", test_case.new_name, "--apply", *arguments["shop"])) == 0 + ) + renamed_config: str = config_path.read_text(encoding="utf-8") + _ = capsys.readouterr() + + from_foreign: tuple[str, ...] = ( + "sqt", + "rename-project", + test_case.new_name, + "--from", + test_case.foreign_name, + "--apply", + *arguments["shop"], + ) + assert run_cli(from_foreign) == 1 + pipeline_error: str = capsys.readouterr().err + assert ( + run_cli( + ( + "sqt", + "rename-project", + test_case.new_name, + "--from", + "shop", + "--apply", + *arguments["shop"], + ) + ) + == 0 + ) + config_path.write_text( + renamed_config.replace( + f'name = "{test_case.new_name}"\n', + f'name = "{test_case.new_name}"\nconsumer_group_identity = "shop"\n', + ), + encoding="utf-8", + ) + _ = capsys.readouterr() + assert run_cli(from_foreign) == 1 + identity_error: str = capsys.readouterr().err + + assert "consumer_group_identity" not in renamed_config + assert test_case.expected_error_fragments[0] in pipeline_error + assert test_case.expected_error_fragments[1] in pipeline_error + assert test_case.expected_error_fragments[0] in identity_error + assert test_case.expected_error_fragments[2] in identity_error + assert ( + manifest_ids(clickhouse_client, database=clickhouse_database, identity=test_case.new_name) + == shop_history + ) diff --git a/website/src/content/docs/docs/cli/rename-project.mdx b/website/src/content/docs/docs/cli/rename-project.mdx index a5b97273..5ea12ea0 100644 --- a/website/src/content/docs/docs/cli/rename-project.mdx +++ b/website/src/content/docs/docs/cli/rename-project.mdx @@ -87,9 +87,13 @@ the control store's role assignments from `OLD_NAME`, under the same lock, dry-r rules and idempotence as the first run. The target may already have been built under the new name: its own new manifests are kept and the old history is merged in, so `sqt destroy` sees both. It refuses when `NEW_NAME` is not the name the configuration carries, when the configuration is still -named `OLD_NAME` (run without `--from` first), when `OLD_NAME` equals the new name, or when nothing -on the target records `OLD_NAME`: no manifests and no role assignments, which usually means a -mistyped name. +named `OLD_NAME` (run without `--from` first), when `OLD_NAME` equals the new name, when nothing +on the target records `OLD_NAME` (a mistyped name, or a target never built under it with no grants +in this control store), or when the history of `OLD_NAME` does not look like this project's: a +`consumer_group_identity` that differs from `OLD_NAME`, or a newest `OLD_NAME` manifest that shares +no pipeline with the project. The last check keeps another project's history in the same database +from being merged in, where `sqt destroy` would treat its pipelines as retired ones of this +project. Until a target is completed, it keeps its manifest history under the old name, so `sqt destroy` there does not see relations recorded before the rename. The first run's output lists the targets