diff --git a/crates/sqltide-cli/src/argv/command_table.json b/crates/sqltide-cli/src/argv/command_table.json index c78e90e4..ccf40b8b 100644 --- a/crates/sqltide-cli/src/argv/command_table.json +++ b/crates/sqltide-cli/src/argv/command_table.json @@ -3373,6 +3373,162 @@ ] } }, + { + "name": "rename-project", + "parser": { + "prog": "sqt rename-project", + "description": "Rename the project while keeping its Kafka consumer groups, manifest history and project role grants. Dry-run by default; use --apply to rename.", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [], + "dest": "new_name", + "default": null, + "required": true, + "help": "New project name", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--project-dir" + ], + "dest": "project_dir", + "default": null, + "type": "Path", + "help": "Path to the project root containing pipelines/", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--host" + ], + "dest": "host", + "default": null, + "help": "ClickHouse host", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--port" + ], + "dest": "port", + "default": null, + "type": "int", + "help": "ClickHouse HTTP port", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--username" + ], + "dest": "username", + "default": null, + "help": "ClickHouse username", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--password" + ], + "dest": "password", + "default": null, + "help": "ClickHouse password", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--database" + ], + "dest": "database", + "default": null, + "help": "Target ClickHouse database", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--apply" + ], + "dest": "apply", + "nargs": 0, + "const": true, + "default": false, + "help": "Apply the rename (dry-run by default)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--from" + ], + "dest": "from_name", + "default": null, + "help": "Previous project name; complete a rename on this target after sqltide_project.toml already carries the new name", + "metavar": "OLD_NAME", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--control-store-url" + ], + "dest": "control_store_url", + "default": null, + "help": "SQLite/PostgreSQL account control-store URL", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--target" + ], + "dest": "target", + "default": null, + "help": "Named project target to use", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--vars" + ], + "dest": "vars", + "default": {}, + "type": "parse_cli_vars", + "help": "Project variable overrides as one JSON object", + "group": 1 + } + ] + } + }, { "name": "janitor", "parser": { @@ -3929,6 +4085,10 @@ "metavar": "reconcile", "help": "Reconcile live metadata baseline" }, + { + "metavar": "rename-project", + "help": "Rename the project and keep its live state" + }, { "metavar": "janitor", "help": "Clean up stale deployment artifacts" diff --git a/crates/sqltide-clickhouse/src/metadata.rs b/crates/sqltide-clickhouse/src/metadata.rs index e5e26fd7..40599303 100644 --- a/crates/sqltide-clickhouse/src/metadata.rs +++ b/crates/sqltide-clickhouse/src/metadata.rs @@ -715,6 +715,38 @@ pub fn manifest_history_query( ) } +/// Copy one project target's manifest history from `from_identity` to `to_identity`, row for +/// row, skipping manifests the new identity already holds, so applying it twice is a no-op. +#[must_use] +pub fn manifest_identity_copy( + database: &str, + from_identity: &str, + to_identity: &str, + target_name: &str, + target_database: &str, +) -> String { + let table: &str = MANIFESTS_TABLE.0; + let target: String = format!( + "manifest.target_name = {} AND manifest.target_database = {}", + text_literal(target_name), + text_literal(target_database) + ); + format!( + "INSERT INTO {database}.{table} (manifest_id, invocation_id, project_identity, \ + target_name, target_database, is_production, project_revision, manifest_fingerprint, \ + manifest_version, pipelines, resources, tool_version, published_at) SELECT \ + manifest.manifest_id, manifest.invocation_id, {to}, manifest.target_name, \ + manifest.target_database, manifest.is_production, manifest.project_revision, \ + manifest.manifest_fingerprint, manifest.manifest_version, manifest.pipelines, \ + manifest.resources, manifest.tool_version, manifest.published_at FROM \ + {database}.{table} AS manifest WHERE manifest.project_identity = {from} AND {target} \ + AND manifest.manifest_id NOT IN (SELECT manifest.manifest_id FROM {database}.{table} \ + AS manifest WHERE manifest.project_identity = {to} AND {target});", + from = text_literal(from_identity), + to = text_literal(to_identity), + ) +} + /// One append containing a complete immutable manifest. #[must_use] pub fn manifest_publication(database: &str, manifest: &Manifest) -> String { diff --git a/crates/sqltide-clickhouse/src/realization.rs b/crates/sqltide-clickhouse/src/realization.rs index c40bff38..93e7f449 100644 --- a/crates/sqltide-clickhouse/src/realization.rs +++ b/crates/sqltide-clickhouse/src/realization.rs @@ -49,6 +49,9 @@ pub struct ManagedSourceRequest<'a> { pub format: &'a str, pub consumer_group: Option<&'a str>, pub project_name: Option<&'a str>, + /// The project's `consumer_group_identity`, which replaces `project_name` in the default + /// consumer group when set. + pub consumer_group_identity: Option<&'a str>, pub target_name: Option<&'a str>, pub setting_keys: Vec<&'a str>, } @@ -99,18 +102,11 @@ pub fn realize_managed_source( let kafka_table_name: String = format!("{KAFKA_TABLE_PREFIX}{}", request.logical_name); let consumer_group: String = match request.consumer_group { Some(group) if !group.is_empty() => group.to_owned(), - // The pre-rename group, so existing sources resume from their committed offsets. - _ => { - let segments: Vec<&str> = [ - request.project_name, - request.target_name, - Some(request.logical_name), - ] - .into_iter() - .flatten() - .collect(); - format!("streambuild_{}", segments.join("_")) - } + _ => default_consumer_group( + request.consumer_group_identity.or(request.project_name), + request.target_name, + request.logical_name, + ), }; Ok(ManagedSourceRealization { raw_table_name: format!("{RAW_TABLE_PREFIX}{}", request.logical_name), @@ -121,6 +117,22 @@ pub fn realize_managed_source( }) } +/// The consumer group a Kafka source uses when it declares none: the project identity (its +/// `consumer_group_identity`, else its name), target and logical name. The `streambuild_` +/// prefix is the pre-rename one, so existing sources resume from their committed offsets. +#[must_use] +pub fn default_consumer_group( + identity: Option<&str>, + target_name: Option<&str>, + logical_name: &str, +) -> String { + let segments: Vec<&str> = [identity, target_name, Some(logical_name)] + .into_iter() + .flatten() + .collect(); + format!("streambuild_{}", segments.join("_")) +} + fn landing_query(kafka_table_name: &str) -> String { format!( "SELECT\n _key AS kafka_key,\n message AS kafka_value,\n _topic AS kafka_topic,\n \ @@ -183,6 +195,7 @@ mod tests { format: "JSONAsString", consumer_group, project_name: Some("shop"), + consumer_group_identity: None, target_name: None, setting_keys: vec!["kafka_num_consumers"], } @@ -204,6 +217,31 @@ mod tests { ); } + #[test] + fn consumer_group_identity_replaces_the_project_name_in_the_default_group_only() { + let mut renamed = request(None); + renamed.project_name = Some("storefront"); + renamed.target_name = Some("dev"); + assert_eq!( + realize_managed_source(&renamed).unwrap().consumer_group, + "streambuild_storefront_dev_orders" + ); + renamed.consumer_group_identity = Some("shop"); + assert_eq!( + realize_managed_source(&renamed).unwrap().consumer_group, + "streambuild_shop_dev_orders" + ); + renamed.consumer_group = Some("orders-reader"); + assert_eq!( + realize_managed_source(&renamed).unwrap().consumer_group, + "orders-reader" + ); + assert_eq!( + default_consumer_group(None, None, "orders"), + "streambuild_orders" + ); + } + #[test] fn rejects_unsupported_kinds_formats_and_typed_setting_overrides() { let mut bad = request(None); diff --git a/crates/sqltide-config/src/lib.rs b/crates/sqltide-config/src/lib.rs index 4e41bea3..7a638b42 100644 --- a/crates/sqltide-config/src/lib.rs +++ b/crates/sqltide-config/src/lib.rs @@ -7,6 +7,7 @@ pub mod errors; pub mod models; pub mod pipelines; pub mod project; +pub mod project_rename; pub mod toml; pub mod yaml; diff --git a/crates/sqltide-config/src/project_rename.rs b/crates/sqltide-config/src/project_rename.rs new file mode 100644 index 00000000..48869b04 --- /dev/null +++ b/crates/sqltide-config/src/project_rename.rs @@ -0,0 +1,141 @@ +//! In-place edits of a committed project config for `sqt rename-project`: a new `name`, and a +//! `consumer_group_identity` when the rename has to keep existing Kafka consumer groups. +//! +//! Edits splice the parsed value spans into the original text, so every other byte (comments, +//! spacing, key order, line endings) stays exactly as authored. + +use std::fmt::Write as _; +use std::ops::Range; + +use toml_edit::{Document, Item, Table}; + +pub const PROJECT_NAME_KEY: &str = "name"; +pub const CONSUMER_GROUP_IDENTITY_KEY: &str = "consumer_group_identity"; +const IDENTITY_COMMENT: &str = + "# Keeps the Kafka consumer groups this project used before it was renamed."; + +/// The top-level `name` and `consumer_group_identity` a project config declares. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct ProjectIdentityKeys { + pub name: Option, + pub consumer_group_identity: Option, +} + +fn parse(text: &str) -> Result, String> { + Document::parse(text).map_err(|error| format!("project config is not valid TOML: {error}")) +} + +fn string_at(root: &Table, key: &str) -> Result)>, String> { + let Some(item) = root.get(key) else { + return Ok(None); + }; + let value = match item { + Item::Value(value) => value, + _ => return Err(format!("project config key '{key}' must be a string")), + }; + let Some(text) = value.as_str() else { + return Err(format!("project config key '{key}' must be a string")); + }; + let span: Range = value + .span() + .ok_or_else(|| format!("project config key '{key}' has no source position"))?; + Ok(Some((text.to_owned(), span))) +} + +/// The top-level identity keys of a project config text. +/// +/// # Errors +/// A message when the text is not TOML or either key is not a string. +pub fn project_identity_keys(text: &str) -> Result { + let document: Document<&str> = parse(text)?; + let root: &Table = document.as_table(); + Ok(ProjectIdentityKeys { + name: string_at(root, PROJECT_NAME_KEY)?.map(|(value, _)| value), + consumer_group_identity: string_at(root, CONSUMER_GROUP_IDENTITY_KEY)? + .map(|(value, _)| value), + }) +} + +/// `value` as a TOML basic string. +#[must_use] +pub fn toml_basic_string(value: &str) -> String { + let mut quoted: String = String::with_capacity(value.len() + 2); + quoted.push('"'); + for character in value.chars() { + match character { + '"' => quoted.push_str("\\\""), + '\\' => quoted.push_str("\\\\"), + '\n' => quoted.push_str("\\n"), + '\t' => quoted.push_str("\\t"), + '\r' => quoted.push_str("\\r"), + character if character.is_control() => { + let _ = write!(quoted, "\\u{:04X}", u32::from(character)); + } + character => quoted.push(character), + } + } + quoted.push('"'); + quoted +} + +/// The project config text with `name` set to `new_name` and, when given, a +/// `consumer_group_identity` line added after the `name` line. Everything else is unchanged. +/// +/// # Errors +/// A message when the text is not TOML, has no string `name`, already declares a +/// `consumer_group_identity` while one is to be added, or does not read back as intended. +pub fn rename_project_text( + text: &str, + new_name: &str, + consumer_group_identity: Option<&str>, +) -> Result { + let document: Document<&str> = parse(text)?; + let root: &Table = document.as_table(); + let (_, name_span) = string_at(root, PROJECT_NAME_KEY)? + .ok_or_else(|| "project config does not declare a string 'name'".to_owned())?; + if consumer_group_identity.is_some() && root.contains_key(CONSUMER_GROUP_IDENTITY_KEY) { + return Err(format!( + "project config already declares '{CONSUMER_GROUP_IDENTITY_KEY}'" + )); + } + let mut edited: String = String::with_capacity(text.len() + 128); + edited.push_str(&text[..name_span.start]); + edited.push_str(&toml_basic_string(new_name)); + let rest: &str = &text[name_span.end..]; + match consumer_group_identity { + None => edited.push_str(rest), + Some(identity) => { + let (line_rest, after) = match rest.find('\n') { + Some(index) => rest.split_at(index + 1), + None => (rest, ""), + }; + let newline: &str = if line_rest.ends_with("\r\n") { + "\r\n" + } else { + "\n" + }; + edited.push_str(line_rest); + if !line_rest.ends_with('\n') { + edited.push_str(newline); + } + edited.push_str(IDENTITY_COMMENT); + edited.push_str(newline); + edited.push_str(CONSUMER_GROUP_IDENTITY_KEY); + edited.push_str(" = "); + edited.push_str(&toml_basic_string(identity)); + edited.push_str(newline); + edited.push_str(after); + } + } + let read_back: ProjectIdentityKeys = project_identity_keys(&edited)?; + let expected_identity: Option = match consumer_group_identity { + Some(identity) => Some(identity.to_owned()), + None => string_at(root, CONSUMER_GROUP_IDENTITY_KEY)?.map(|(value, _)| value), + }; + if read_back.name.as_deref() != Some(new_name) + || read_back.consumer_group_identity != expected_identity + { + return Err("project config edit did not read back as intended".to_owned()); + } + Ok(edited) +} diff --git a/crates/sqltide-config/src/tests.rs b/crates/sqltide-config/src/tests.rs index bb5f066d..1fd18ac2 100644 --- a/crates/sqltide-config/src/tests.rs +++ b/crates/sqltide-config/src/tests.rs @@ -6,3 +6,6 @@ mod toml; #[path = "tests/test_pipelines.rs"] mod pipelines; + +#[path = "tests/test_project_rename.rs"] +mod project_rename; diff --git a/crates/sqltide-config/src/tests/test_project_rename.rs b/crates/sqltide-config/src/tests/test_project_rename.rs new file mode 100644 index 00000000..de4b4622 --- /dev/null +++ b/crates/sqltide-config/src/tests/test_project_rename.rs @@ -0,0 +1,83 @@ +//! Project config edits for `sqt rename-project` keep every authored byte they do not change. + +use crate::project_rename::{ + ProjectIdentityKeys, project_identity_keys, rename_project_text, toml_basic_string, +}; + +const AUTHORED: &str = "# Shop analytics project.\n\ +name = \"shop\" # the project identity\n\ +default_target = \"dev\"\n\ +\n\ +[targets.dev]\n\ +# Local development database.\n\ +database = \"shop_dev\"\n"; + +#[test] +fn renames_in_place_and_keeps_comments_and_spacing() { + let edited = rename_project_text(AUTHORED, "storefront", None).unwrap(); + assert_eq!( + edited, + AUTHORED.replace("name = \"shop\"", "name = \"storefront\"") + ); +} + +#[test] +fn adds_the_consumer_group_identity_after_the_name_line() { + let edited = rename_project_text(AUTHORED, "storefront", Some("shop")).unwrap(); + assert_eq!( + edited, + "# Shop analytics project.\n\ +name = \"storefront\" # the project identity\n\ +# Keeps the Kafka consumer groups this project used before it was renamed.\n\ +consumer_group_identity = \"shop\"\n\ +default_target = \"dev\"\n\ +\n\ +[targets.dev]\n\ +# Local development database.\n\ +database = \"shop_dev\"\n" + ); + assert_eq!( + project_identity_keys(&edited).unwrap(), + ProjectIdentityKeys { + name: Some("storefront".to_owned()), + consumer_group_identity: Some("shop".to_owned()), + } + ); +} + +#[test] +fn keeps_crlf_line_endings_and_a_final_line_without_newline() { + let edited = rename_project_text("name = 'shop'\r\nx = 1\r\n", "store", Some("shop")).unwrap(); + assert_eq!( + edited, + "name = \"store\"\r\n# Keeps the Kafka consumer groups this project used before it was \ + renamed.\r\nconsumer_group_identity = \"shop\"\r\nx = 1\r\n" + ); + let edited = rename_project_text("x = 1\nname = \"shop\"", "store", Some("shop")).unwrap(); + assert!(edited.ends_with("name = \"store\"\n# Keeps the Kafka consumer groups this project used before it was renamed.\nconsumer_group_identity = \"shop\"\n")); +} + +#[test] +fn keeps_an_existing_consumer_group_identity() { + let text = "name = \"storefront\"\nconsumer_group_identity = \"shop\"\n"; + assert_eq!( + rename_project_text(text, "outlet", None).unwrap(), + "name = \"outlet\"\nconsumer_group_identity = \"shop\"\n" + ); + assert_eq!( + rename_project_text(text, "outlet", Some("shop")).unwrap_err(), + "project config already declares 'consumer_group_identity'" + ); +} + +#[test] +fn refuses_documents_without_a_string_name() { + assert!(rename_project_text("default_target = \"dev\"\n", "x", None).is_err()); + assert!(rename_project_text("name = 1\n", "x", None).is_err()); + assert!(rename_project_text("name = \n", "x", None).is_err()); +} + +#[test] +fn quotes_names_as_basic_strings() { + assert_eq!(toml_basic_string("a\"b\\c\u{1}"), "\"a\\\"b\\\\c\\u0001\""); +} diff --git a/crates/sqltide-discovery/src/config.rs b/crates/sqltide-discovery/src/config.rs index 0f6fb1e3..d434c59b 100644 --- a/crates/sqltide-discovery/src/config.rs +++ b/crates/sqltide-discovery/src/config.rs @@ -16,8 +16,9 @@ use crate::retention::{ use crate::text::python_strip; use crate::value::{Kind, Value}; -const PROJECT_CONFIG_KEYS: [&str; 15] = [ +const PROJECT_CONFIG_KEYS: [&str; 16] = [ "name", + "consumer_group_identity", "adapter", "default_target", "connection", @@ -116,6 +117,9 @@ pub type Pairs = Vec<(String, usize)>; #[derive(Clone, Debug, PartialEq)] pub struct ProjectConfig { pub name: String, + /// Replaces `name` in default Kafka consumer group names, so a renamed project keeps + /// consuming from its committed offsets. + pub consumer_group_identity: Option, pub adapter: String, pub default_target: String, pub connection: Pairs, @@ -337,6 +341,17 @@ impl Parser<'_> { return self .error("project.name must be a committed literal and cannot be interpolated"); } + let consumer_group_identity: Option = + self.optional_non_empty(payload, "consumer_group_identity", "project")?; + if consumer_group_identity + .as_deref() + .is_some_and(|identity| identity.contains(INTERPOLATION_TOKEN_START)) + { + return self.error( + "project.consumer_group_identity must be a committed literal and cannot be \ + interpolated", + ); + } let adapter: String = self .optional_non_empty(payload, "adapter", "project")? .unwrap_or_else(|| DEFAULT_ADAPTER_NAME.to_owned()); @@ -354,6 +369,7 @@ impl Parser<'_> { let dependencies: Dependencies = self.dependencies(field(payload, "dependencies"))?; Ok(ProjectConfig { name, + consumer_group_identity, adapter, default_target, connection, diff --git a/crates/sqltide-discovery/src/effective.rs b/crates/sqltide-discovery/src/effective.rs index a324777a..f1796398 100644 --- a/crates/sqltide-discovery/src/effective.rs +++ b/crates/sqltide-discovery/src/effective.rs @@ -26,6 +26,7 @@ pub struct Naming { #[derive(Clone, Debug, PartialEq)] pub struct EffectiveConfig { pub name: String, + pub consumer_group_identity: Option, pub adapter: String, pub target_name: String, pub production_target: bool, @@ -163,6 +164,7 @@ pub fn resolve_effective_config( let project_sensors = project_side.map(|target| target.sensors); Ok(EffectiveConfig { name: project_config.name.clone(), + consumer_group_identity: project_config.consumer_group_identity.clone(), adapter, target_name, production_target: project_target.is_some_and(|target| target.production), diff --git a/crates/sqltide-executor/src/sources.rs b/crates/sqltide-executor/src/sources.rs index edd72b96..5b8a233a 100644 --- a/crates/sqltide-executor/src/sources.rs +++ b/crates/sqltide-executor/src/sources.rs @@ -540,6 +540,55 @@ fn settings_drift(resource: &Resource, relation: &Relation) -> Vec { .collect() } +/// A Kafka table consuming as a different consumer group than the project declares. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ConsumerGroupDrift { + pub table_name: String, + pub live_group: String, + pub declared_group: String, +} + +impl ConsumerGroupDrift { + /// The warning `sqt plan` and the UI show for this drift. + #[must_use] + pub fn message(&self) -> String { + format!( + "{} consumes as Kafka consumer group '{}' but the project declares '{}'. Recreating \ + it would consume as '{}', and the offsets committed for '{}' would not carry over.", + self.table_name, + self.live_group, + self.declared_group, + self.declared_group, + self.live_group + ) + } +} + +/// The drift of a live Kafka table's consumer group from `declared_group`, scoped to +/// `database` as ClickHouse consumes it. `None` when they match or the live group is unknown +/// or masked. +#[must_use] +pub fn consumer_group_drift( + table_name: &str, + declared_group: &str, + database: &str, + live_settings: &[(String, String)], +) -> Option { + let declared_group: String = database_scoped_consumer_group(declared_group, database); + let live_group: &str = live_settings + .iter() + .find(|(name, _)| name == GROUP_NAME_SETTING) + .map(|(_, value)| unquoted(value))?; + if live_group == declared_group || live_group == MASKED_SETTING_VALUE { + return None; + } + Some(ConsumerGroupDrift { + table_name: table_name.to_owned(), + live_group: live_group.to_owned(), + declared_group, + }) +} + fn actual_settings(relation: &Relation) -> HashMap { relation .settings @@ -557,6 +606,33 @@ fn unquoted(value: &str) -> &str { mod tests { use super::*; + #[test] + fn consumer_group_drift_names_both_groups_and_ignores_matching_or_masked_groups() { + let live = |group: &str| vec![("kafka_group_name".to_owned(), format!("'{group}'"))]; + assert_eq!( + consumer_group_drift("kafka__orders", "g", "shop-dev", &live("g_shop_dev")), + None + ); + assert_eq!( + consumer_group_drift("kafka__orders", "g", "shop_dev", &live("[HIDDEN]")), + None + ); + assert_eq!( + consumer_group_drift("kafka__orders", "g", "shop_dev", &[]), + None + ); + let drift = consumer_group_drift("kafka__orders", "new", "shop_dev", &live("old_shop_dev")) + .unwrap(); + assert_eq!(drift.live_group, "old_shop_dev"); + assert_eq!(drift.declared_group, "new_shop_dev"); + assert_eq!( + drift.message(), + "kafka__orders consumes as Kafka consumer group 'old_shop_dev' but the project \ + declares 'new_shop_dev'. Recreating it would consume as 'new_shop_dev', and the \ + offsets committed for 'old_shop_dev' would not carry over." + ); + } + fn kafka(name: &str, topic: &str) -> Resource { Resource::ManagedSource { name: name.to_owned(), diff --git a/crates/sqltide-python/src/build_entry.rs b/crates/sqltide-python/src/build_entry.rs index 033e50b2..09fe8fdd 100644 --- a/crates/sqltide-python/src/build_entry.rs +++ b/crates/sqltide-python/src/build_entry.rs @@ -386,7 +386,7 @@ fn run_plan<'py>( py, "prepare_build_workflow", &[ - ("analysis", analysis), + ("analysis", analysis.clone()), ("options", preparation_options), ("client", client.clone()), ("adapter_profile", adapter_profile.clone()), @@ -423,14 +423,19 @@ fn run_plan<'py>( ], )?; } + let plan_json: Value<'py> = if mixed { + preparation.getattr("plan_json")? + } else { + preparation.getattr("workflow")?.getattr("plan_json")? + }; + let plan_text: Value<'py> = crate::plan_warnings::with_consumer_group_drift_text( + preparation.getattr("plan_text")?, + &plan_json, + )?; let rendered: Value<'py> = if options.getattr("json_output")?.is_truthy()? { - if mixed { - preparation.getattr("plan_json")? - } else { - preparation.getattr("workflow")?.getattr("plan_json")? - } + plan_json } else { - preparation.getattr("plan_text")?.add("\n")? + plan_text.add("\n")? }; let arguments: Bound<'py, PyDict> = PyDict::new(py); arguments.set_item("end", "")?; diff --git a/crates/sqltide-python/src/build_workflow.rs b/crates/sqltide-python/src/build_workflow.rs index 6931a8ef..b67b4856 100644 --- a/crates/sqltide-python/src/build_workflow.rs +++ b/crates/sqltide-python/src/build_workflow.rs @@ -232,6 +232,13 @@ fn mixed_preparation<'py>( DIRECT, json_loads(py, &direct.getattr("workflow")?.getattr("plan_json")?)?, )?; + // Both phases read the same target, so the direct phase's list stands for the whole plan. + payload.set_item( + "consumer_group_warnings", + payload.get_item(DIRECT)?.map_or(Ok(None), |direct| { + direct.get_item("consumer_group_warnings").map(Some) + })?, + )?; let plan_json: String = json_dumps(payload.as_any(), Some(2), false)?; let mut requirements: Vec> = items(&virtual_.getattr("protection_requirements")?)?; requirements.extend(items(&direct.getattr("protection_requirements")?)?); @@ -300,7 +307,22 @@ fn prepare_direct<'py>( ("row_error_policy_changes", policy_changes.clone()), ], )?; - let plan_json: Value<'py> = native( + let drift_warnings = crate::plan_warnings::consumer_group_warning_items( + analysis, + &preview.getattr("warehouse_snapshot")?.getattr("catalog")?, + &preview.getattr("database")?, + )?; + let with_warnings = |plan_json: Value<'py>| -> PyResult> { + Ok(PyString::new( + py, + &crate::plan_warnings::with_consumer_group_warnings_json( + &plan_json.extract::()?, + &drift_warnings, + )?, + ) + .into_any()) + }; + let plan_json: Value<'py> = with_warnings(native( py, "render_direct_plan_json", &[ @@ -308,7 +330,7 @@ fn prepare_direct<'py>( ("adapter_name", adapter_name.clone()), ("row_error_policy_changes", policy_changes.clone()), ], - )?; + )?)?; let mut workflow: Value<'py> = native( py, "assemble_direct_build_workflow", @@ -320,7 +342,7 @@ fn prepare_direct<'py>( ], )?; if truthy(&workflow, "recreated_sources")? { - let plan_json: Value<'py> = native( + let plan_json: Value<'py> = with_warnings(native( py, "render_direct_plan_json", &[ @@ -329,7 +351,7 @@ fn prepare_direct<'py>( ("source_recreations", workflow.getattr("recreated_sources")?), ("row_error_policy_changes", policy_changes.clone()), ], - )?; + )?)?; let template: Value<'py> = replace(&workflow.getattr("template")?, &[("plan_json", plan_json)])?; workflow = replace(&workflow, &[("template", template)])?; @@ -468,6 +490,14 @@ fn prepare_virtual<'py>( ("row_error_policy_changes", policy_changes), ], )?; + payload.set_item( + "consumer_group_warnings", + crate::plan_warnings::consumer_group_warning_items( + analysis, + &preview.getattr("target_catalog")?, + &preview.getattr("database")?, + )?, + )?; let plan_json: String = json_dumps(&payload, Some(2), false)? + "\n"; let workflow: Value<'py> = native( py, diff --git a/crates/sqltide-python/src/cli/dispatch.rs b/crates/sqltide-python/src/cli/dispatch.rs index b9a91e10..6d2b1290 100644 --- a/crates/sqltide-python/src/cli/dispatch.rs +++ b/crates/sqltide-python/src/cli/dispatch.rs @@ -538,6 +538,30 @@ pub(crate) fn dispatch_cli_command<'py>( ("adapter_profile", adapter_profile), ], ), + "rename-project" => { + let project_dir: Bound<'py, PyAny> = project_dir(py, invocation)?; + build( + py, + NATIVE, + "run_rename_project", + vec![ + ("pipelines_root", require_pipelines_root(py, invocation)?), + ("project_dir", project_dir), + ("new_name", args.getattr("new_name")?), + ("from_name", getattr_none(args, "from_name")?), + ("environment", invocation.connection.environment.clone()), + ("database", database), + ( + "control_store_url", + getattr_none(args, "control_store_url")?, + ), + ("apply", boolean(py, flag(args, "apply")?)), + ("client", client), + ("loaded_project", loaded_project), + ("adapter_profile", adapter_profile), + ], + ) + } "doctor" => call_handler( py, handlers, diff --git a/crates/sqltide-python/src/cli/invocation.rs b/crates/sqltide-python/src/cli/invocation.rs index 087ff840..450c7cbb 100644 --- a/crates/sqltide-python/src/cli/invocation.rs +++ b/crates/sqltide-python/src/cli/invocation.rs @@ -12,7 +12,7 @@ const DEV_CLI_VARIABLES_ENV_VAR: &str = "SQLTIDE_INTERNAL_CLI_VARS"; const PROJECT_FILE: &str = "sqltide_project.toml"; const LEGACY_PROJECT_FILE: &str = "streambuild_project.toml"; const PIPELINES_DIRECTORY: &str = "pipelines"; -const COMMANDS_REQUIRING_PIPELINES_ROOT: [&str; 13] = [ +const COMMANDS_REQUIRING_PIPELINES_ROOT: [&str; 14] = [ "dev", "discover", "compile", @@ -25,6 +25,7 @@ const COMMANDS_REQUIRING_PIPELINES_ROOT: [&str; 13] = [ "reset-target", "deployment", "reconcile", + "rename-project", "audit", ]; const COMMANDS_WITHOUT_ADAPTER_CONNECTION: [&str; 4] = ["discover", "compile", "lint", "format"]; diff --git a/crates/sqltide-python/src/cli/mod.rs b/crates/sqltide-python/src/cli/mod.rs index f1e8d394..09d8dd4a 100644 --- a/crates/sqltide-python/src/cli/mod.rs +++ b/crates/sqltide-python/src/cli/mod.rs @@ -10,7 +10,7 @@ mod entry; pub(crate) mod invocation; pub(crate) mod output; mod playground; -mod python; +pub(crate) mod python; mod readiness; pub(crate) mod render; mod sql; diff --git a/crates/sqltide-python/src/clickhouse_adapter.rs b/crates/sqltide-python/src/clickhouse_adapter.rs index e3418ff1..0c4631ff 100644 --- a/crates/sqltide-python/src/clickhouse_adapter.rs +++ b/crates/sqltide-python/src/clickhouse_adapter.rs @@ -196,6 +196,8 @@ fn realize_clickhouse_source<'py>(request: &Bound<'py, PyAny>) -> PyResult = request.getattr("consumer_group")?.extract()?; let project_name: Option = request.getattr("project_name")?.extract()?; + let consumer_group_identity: Option = + request.getattr("consumer_group_identity")?.extract()?; let target_name: Option = request.getattr("target_name")?.extract()?; let setting_keys: Vec = request .getattr("settings")? @@ -211,6 +213,7 @@ fn realize_clickhouse_source<'py>(request: &Bound<'py, PyAny>) -> PyResult Compiler<'py> { "allowed_cross_pipeline_references", inputs.getattr("allowed_cross_pipeline_references")?, ), + ( + "consumer_group_identity", + inputs.getattr("consumer_group_identity")?, + ), ], )?; self.validate_relation_names(&project)?; diff --git a/crates/sqltide-python/src/config.rs b/crates/sqltide-python/src/config.rs index 25c60832..f1a3407a 100644 --- a/crates/sqltide-python/src/config.rs +++ b/crates/sqltide-python/src/config.rs @@ -298,6 +298,10 @@ impl<'py> Document<'py> { ], )?, ), + ( + "consumer_group_identity", + self.object(config.consumer_group_identity.as_deref())?, + ), ], ) } @@ -1082,6 +1086,10 @@ impl<'py> Document<'py> { ], )?, ), + ( + "consumer_group_identity", + self.object(config.consumer_group_identity.as_deref())?, + ), ], ) } diff --git a/crates/sqltide-python/src/control_store/mod.rs b/crates/sqltide-python/src/control_store/mod.rs index fc5751ed..3b6eb0e4 100644 --- a/crates/sqltide-python/src/control_store/mod.rs +++ b/crates/sqltide-python/src/control_store/mod.rs @@ -1685,7 +1685,187 @@ impl ControlStore { } } +/// What renaming a project would do to its role assignments. +pub(crate) struct ProjectRenamePreview { + /// Assignments, active or revoked, still recorded under the old project name. + pub(crate) moving: usize, + /// Assignments already recorded under the new project name. + pub(crate) existing: usize, + /// `user role scope` of each active assignment both names hold. + pub(crate) conflicts: Vec, +} + impl ControlStore { + fn project_rename_preview( + transaction: &Transaction<'_>, + from_project: &str, + to_project: &str, + ) -> PyResult { + let py: Python<'_> = transaction.py(); + let rows_for = |project: &str| -> PyResult> { + transaction + .rows( + &sql::assignments_where( + " WHERE sqltide_project_role_assignments.project_name = :project_name", + ), + &[("project_name", object(py, project)?)], + )? + .iter() + .map(|row| { + Ok(( + column(row, "user_id")?.str()?.to_string(), + column(row, "role_name")?.str()?.to_string(), + column(row, "assignment_scope")?.str()?.to_string(), + column(row, "revoked_at")?.is_none(), + )) + }) + .collect() + }; + let moving = rows_for(from_project)?; + let existing = rows_for(to_project)?; + let mut conflicts: Vec = moving + .iter() + .filter(|(user, role, scope, active)| { + *active + && existing + .iter() + .any(|(other_user, other_role, other_scope, other_active)| { + *other_active + && other_user == user + && other_role == role + && other_scope == scope + }) + }) + .map(|(user, role, scope, _)| { + let scope: &str = if scope.is_empty() { + "all targets" + } else { + scope + }; + format!("user {user} role '{role}' ({scope})") + }) + .collect(); + conflicts.sort(); + Ok(ProjectRenamePreview { + moving: moving.len(), + existing: existing.len(), + conflicts, + }) + } + + /// What moving `from_project`'s role assignments to `to_project` would change. + pub(crate) fn preview_project_rename( + &self, + py: Python<'_>, + from_project: &str, + to_project: &str, + ) -> PyResult { + self.transaction(py, |transaction| { + Self::project_rename_preview(transaction, from_project, to_project) + }) + } + + /// Move every role assignment, active or revoked, from `from_project` to `to_project` in one + /// transaction and audit it; the moved assignment ids. Refuses when both names hold the + /// same active assignment. + pub(crate) fn rename_project( + &self, + py: Python<'_>, + from_project: &str, + to_project: &str, + ) -> PyResult> { + let now: Any<'_> = now(py)?; + let moved: Vec = self.transaction(py, |transaction| { + let preview = Self::project_rename_preview(transaction, from_project, to_project)?; + if !preview.conflicts.is_empty() { + return Err(error( + py, + "AccountConflictError", + format!( + "Project '{to_project}' already holds role assignments that project \ + '{from_project}' also holds: {}", + preview.conflicts.join(", ") + ), + )); + } + let ids: Vec = transaction + .rows( + &sql::assignments_where( + " WHERE sqltide_project_role_assignments.project_name = :project_name \ + ORDER BY sqltide_project_role_assignments.id", + ), + &[("project_name", object(py, from_project)?)], + )? + .iter() + .map(|row| Ok(column(row, "id")?.str()?.to_string())) + .collect::>()?; + if ids.is_empty() { + return Ok(ids); + } + transaction.execute( + sql::RENAME_PROJECT_ASSIGNMENTS, + &[ + ("to_project", object(py, to_project)?), + ("from_project", object(py, from_project)?), + ], + )?; + let details: Bound<'_, PyDict> = PyDict::new(py); + details.set_item("fromProjectName", from_project)?; + details.set_item("toProjectName", to_project)?; + details.set_item("assignmentCount", ids.len())?; + self.audit( + transaction, + "project.renamed", + &none(py), + &none(py), + details, + &now, + )?; + Ok(ids) + })?; + if !moved.is_empty() { + self.invalidate(); + } + Ok(moved) + } + + /// Return assignments `rename_project` moved to `to_project` back to `from_project`. + pub(crate) fn restore_project_rename( + &self, + py: Python<'_>, + assignment_ids: &[String], + from_project: &str, + to_project: &str, + ) -> PyResult<()> { + let now: Any<'_> = now(py)?; + self.transaction(py, |transaction| { + for id in assignment_ids { + transaction.execute( + sql::MOVE_PROJECT_ASSIGNMENT, + &[ + ("to_project", object(py, from_project)?), + ("id", object(py, id)?), + ("from_project", object(py, to_project)?), + ], + )?; + } + let details: Bound<'_, PyDict> = PyDict::new(py); + details.set_item("fromProjectName", to_project)?; + details.set_item("toProjectName", from_project)?; + details.set_item("assignmentCount", assignment_ids.len())?; + self.audit( + transaction, + "project.rename_reverted", + &none(py), + &none(py), + details, + &now, + ) + })?; + self.invalidate(); + Ok(()) + } + /// The identity a competing writer linked, or a real conflict. fn provisioned_by_competing_writer<'py>( &self, diff --git a/crates/sqltide-python/src/control_store/sql.rs b/crates/sqltide-python/src/control_store/sql.rs index 83e44527..2d582ef6 100644 --- a/crates/sqltide-python/src/control_store/sql.rs +++ b/crates/sqltide-python/src/control_store/sql.rs @@ -104,6 +104,8 @@ pub(crate) const USER_ID: &str = pub(crate) const DELETE_USER_ROLE: &str = "DELETE FROM sqltide_user_roles WHERE sqltide_user_roles.user_id = :user_id AND sqltide_user_roles.role_name = :role_name"; pub(crate) const INSERT_ASSIGNMENT: &str = "INSERT INTO sqltide_project_role_assignments (id, user_id, project_name, role_name, target_name, assignment_scope, assigned_by, assigned_at, revoked_by, revoked_at, active_key) VALUES (:id, :user_id, :project_name, :role_name, :target_name, :assignment_scope, :assigned_by, :assigned_at, :revoked_by, :revoked_at, :active_key)"; pub(crate) const REVOKE_ASSIGNMENT: &str = "UPDATE sqltide_project_role_assignments SET revoked_by=:revoked_by, revoked_at=:revoked_at, active_key=:active_key WHERE sqltide_project_role_assignments.id = :id"; +pub(crate) const RENAME_PROJECT_ASSIGNMENTS: &str = "UPDATE sqltide_project_role_assignments SET project_name=:to_project WHERE sqltide_project_role_assignments.project_name = :from_project"; +pub(crate) const MOVE_PROJECT_ASSIGNMENT: &str = "UPDATE sqltide_project_role_assignments SET project_name=:to_project WHERE sqltide_project_role_assignments.id = :id AND sqltide_project_role_assignments.project_name = :from_project"; pub(crate) const INSERT_AUDIT: &str = "INSERT INTO sqltide_account_audit_log (operation, actor_user_id, affected_user_id, occurred_at, details_json) VALUES (:operation, :actor_user_id, :affected_user_id, :occurred_at, :details_json)"; pub(crate) const AUDIT_RECORDS: &str = "SELECT sqltide_account_audit_log.id, sqltide_account_audit_log.operation, sqltide_account_audit_log.actor_user_id, sqltide_account_audit_log.affected_user_id, sqltide_account_audit_log.occurred_at, sqltide_account_audit_log.details_json FROM sqltide_account_audit_log ORDER BY sqltide_account_audit_log.id"; pub(crate) const PRUNE_SESSIONS: &str = "DELETE FROM sqltide_sessions WHERE sqltide_sessions.expires_at <= :now OR sqltide_sessions.revoked_at IS NOT NULL"; diff --git a/crates/sqltide-python/src/dev_server/state.rs b/crates/sqltide-python/src/dev_server/state.rs index dc6befdc..23956ef0 100644 --- a/crates/sqltide-python/src/dev_server/state.rs +++ b/crates/sqltide-python/src/dev_server/state.rs @@ -962,6 +962,7 @@ fn partitions<'py>(warehouse: Option>, kafka_lag: &Any<'py>) -> PyResul fn source_states<'py>( analysis: &Any<'py>, connection: &Any<'py>, + catalog: &Any<'py>, database: &str, stats: &Any<'py>, extents: &Bound<'py, PyDict>, @@ -1001,6 +1002,20 @@ fn source_states<'py>( } } let throughputs = throughputs(connection, database, &newest_ages)?; + let drifts: Bound<'py, PyDict> = PyDict::new(py); + for (source_name, drift) in + crate::plan_warnings::consumer_group_drifts(analysis, catalog, database)? + { + drifts.set_item( + source_name, + py_kwargs!(py; + tableName = &drift.table_name, + liveGroup = &drift.live_group, + declaredGroup = &drift.declared_group, + message = drift.message(), + )?, + )?; + } let states: Bound<'py, PyDict> = PyDict::new(py); for item in inputs { let throughput = get(throughputs.as_any(), &item.relation_name)?; @@ -1016,6 +1031,7 @@ fn source_states<'py>( throughput = &throughput, rowsPerSecond = rows_per_second(&throughput)?, partitions = partitions(warehouse.get_item(&item.relation_name)?, &item.kafka_lag)?, + consumerGroupDrift = drifts.get_item(item.source.getattr("key")?.getattr("name")?)?, )?; states.set_item(item.source.getattr("key")?.getattr("name")?, state)?; } @@ -1103,7 +1119,7 @@ pub(crate) fn build_state_payload<'py>( capturedAt = &captured, warehouseHealth = warehouse_health_payload(&health, &captured, adapter_name, &database_value)?, models = model_states(&analysis, &catalog, &baselines, &stats, &extents, &captured_at, &activity)?, - sources = source_states(&analysis, &connection, &database, &stats, &extents, &captured_at, &reader, &warehouse)?, + sources = source_states(&analysis, &connection, &catalog, &database, &stats, &extents, &captured_at, &reader, &warehouse)?, )? .into_any()) } diff --git a/crates/sqltide-python/src/lib.rs b/crates/sqltide-python/src/lib.rs index f6b95da6..c5fe1da6 100644 --- a/crates/sqltide-python/src/lib.rs +++ b/crates/sqltide-python/src/lib.rs @@ -70,6 +70,7 @@ mod promotion_command; mod realization; mod reconcile_command; mod recovery; +mod rename_project_command; mod row_error_policy; mod rules; mod run_events; @@ -172,6 +173,7 @@ fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { deployment_command::register(module)?; promotion_command::register(module)?; reconcile_command::register(module)?; + rename_project_command::register(module)?; maintenance_command::register(module)?; sql_tests_command::register(module)?; audit_execution::register(module)?; diff --git a/crates/sqltide-python/src/plan_warnings.rs b/crates/sqltide-python/src/plan_warnings.rs index 15b7ddcd..b8a1d191 100644 --- a/crates/sqltide-python/src/plan_warnings.rs +++ b/crates/sqltide-python/src/plan_warnings.rs @@ -1,11 +1,18 @@ //! Planner warnings that need a point-in-time warehouse read: rebuild subtrees whose replay -//! source is empty. Counts go through the Python adapter connection. +//! source is empty, and Kafka tables consuming as a different consumer group than the project +//! declares. Reads go through the Python adapter connection. use pyo3::prelude::*; -use pyo3::types::{PyDict, PyInt}; +use pyo3::types::{PyDict, PyInt, PyList}; +use sqltide_executor::sources::{ConsumerGroupDrift, consumer_group_drift}; +use sqltide_planner::cli_style::CliStyle; use crate::direct_build::{Value, build, class, items, replace, text, tuple}; +const ADAPTER_MODELS: &str = "sqltide.adapter.models"; +const CONSUMER_GROUP_DRIFT: &str = "consumer_group_drift"; +const CONSUMER_GROUP_WARNINGS_KEY: &str = "consumer_group_warnings"; + const ADAPTER_EXCEPTIONS: &str = "sqltide.adapter.exceptions"; const PLANNER_MODELS: &str = "sqltide.compiler.planner.models"; const EMPTY_REPLAY_SOURCE: &str = "empty_replay_source"; @@ -134,6 +141,193 @@ fn add_empty_replay_source_warnings<'py>( ) } +/// `(source name, drift)` for every realized managed source whose live table in `database` +/// consumes as a different consumer group than the project declares, by source name. +pub(crate) fn consumer_group_drifts( + analysis: &Value<'_>, + catalog: &Value<'_>, + database: &str, +) -> PyResult> { + let py: Python<'_> = analysis.py(); + let managed_source: Value<'_> = class(py, ADAPTER_MODELS, "AdapterManagedSource")?; + let mut drifts: Vec<(String, ConsumerGroupDrift)> = Vec::new(); + for entry in analysis + .getattr("realized_project")? + .getattr("resources_by_logical_key")? + .call_method0("items")? + .try_iter()? + { + let entry: Value<'_> = entry?; + let source_name: String = entry.get_item(0)?.getattr("name")?.extract()?; + for resource in items(&entry.get_item(1)?)? { + if !resource.is_instance(&managed_source)? { + continue; + } + let table_name: String = resource.getattr("name")?.extract()?; + let relation: Value<'_> = catalog.call_method1("relation", (&table_name,))?; + if relation.is_none() { + continue; + } + let settings: Vec<(String, String)> = relation.getattr("settings")?.extract()?; + let declared: String = resource.getattr("consumer_group")?.extract()?; + if let Some(drift) = consumer_group_drift(&table_name, &declared, database, &settings) { + drifts.push((source_name.clone(), drift)); + } + } + } + drifts.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(drifts) +} + +/// The `consumer_group_warnings` entries for one plan: each managed Kafka table in the +/// catalog the plan read that consumes as a different group than the project declares. Empty +/// when there is none or the plan read no catalog. +pub(crate) fn consumer_group_warning_items<'py>( + analysis: &Value<'py>, + catalog: &Value<'py>, + database: &Value<'py>, +) -> PyResult> { + let py: Python<'py> = analysis.py(); + let items: Bound<'py, PyList> = PyList::empty(py); + if catalog.is_none() || database.is_none() { + return Ok(items); + } + for (source_name, drift) in + consumer_group_drifts(analysis, catalog, &database.extract::()?)? + { + let warning: Bound<'py, PyDict> = PyDict::new(py); + warning.set_item("warning_code", CONSUMER_GROUP_DRIFT)?; + warning.set_item("message", drift.message())?; + warning.set_item("source_name", source_name)?; + warning.set_item("table_name", &drift.table_name)?; + warning.set_item("live_consumer_group", &drift.live_group)?; + warning.set_item("declared_consumer_group", &drift.declared_group)?; + items.append(warning)?; + } + Ok(items) +} + +/// A rendered plan JSON object with `consumer_group_warnings` appended as its last member and +/// every other byte unchanged. +pub(crate) fn with_consumer_group_warnings_json( + plan_json: &str, + warnings: &Bound<'_, PyList>, +) -> PyResult { + let py: Python<'_> = warnings.py(); + let indent: usize = top_level_indent(plan_json); + let keywords: Bound<'_, PyDict> = PyDict::new(py); + if indent > 0 { + keywords.set_item("indent", indent)?; + } + let value: String = py + .import("json")? + .getattr("dumps")? + .call((warnings,), Some(&keywords))? + .extract()?; + Ok(with_top_level_key( + plan_json, + CONSUMER_GROUP_WARNINGS_KEY, + &value, + )) +} + +/// The plan text followed by a warning line for each consumer group drift the plan's JSON +/// records under `consumer_group_warnings`; unchanged when there is none. +pub(crate) fn with_consumer_group_drift_text<'py>( + plan_text: Value<'py>, + plan_json: &Value<'py>, +) -> PyResult> { + let py: Python<'py> = plan_text.py(); + let payload: Value<'py> = py.import("json")?.getattr("loads")?.call1((plan_json,))?; + let mut messages: Vec = Vec::new(); + let mut lists: Vec> = vec![payload.clone()]; + for mode in ["virtual", "direct"] { + if let Some(nested) = payload.downcast::()?.get_item(mode)? { + lists.push(nested); + } + } + for holder in lists { + let Ok(holder) = holder.downcast_into::() else { + continue; + }; + let Some(warnings) = holder.get_item(CONSUMER_GROUP_WARNINGS_KEY)? else { + continue; + }; + for warning in warnings.try_iter()? { + let message: String = warning?.get_item("message")?.extract()?; + if !messages.contains(&message) { + messages.push(message); + } + } + } + if messages.is_empty() { + return Ok(plan_text); + } + let style = CliStyle { + use_color: crate::cli::style::terminal_style(py)?.use_color, + }; + let mut lines: Vec = vec![plan_text.extract()?, String::new()]; + for message in &messages { + lines.push(style.warning(&format!("warning {message}"))); + } + Ok(lines.join("\n").into_pyobject(py)?.into_any()) +} + +/// The indentation of a JSON object's top-level keys: the leading spaces of its second line, or +/// 0 when the object is on one line. +fn top_level_indent(json: &str) -> usize { + json.lines() + .nth(1) + .map_or(0, |line| line.len() - line.trim_start_matches(' ').len()) +} + +/// `json`, a JSON object, with `"key": value` appended as its last member. Every existing byte +/// is kept; `value` is already rendered at the object's indentation for its first line, and +/// its later lines are shifted one level so they nest under the key. +fn with_top_level_key(json: &str, key: &str, value: &str) -> String { + let Some(end) = json.rfind('}') else { + return json.to_owned(); + }; + let indent: usize = top_level_indent(json); + let head: &str = json[..end].trim_end(); + let tail: &str = &json[end..]; + let separator: &str = if head.ends_with('{') { "" } else { "," }; + let nested: String = value.replace('\n', &format!("\n{}", " ".repeat(indent))); + if indent == 0 { + let space: &str = if separator.is_empty() { "" } else { " " }; + return format!("{head}{separator}{space}\"{key}\": {nested}{tail}"); + } + format!( + "{head}{separator}\n{}\"{key}\": {nested}\n{tail}", + " ".repeat(indent) + ) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(add_empty_replay_source_warnings, module)?) } + +#[cfg(test)] +mod tests { + use super::with_top_level_key; + + #[test] + fn appends_a_key_and_keeps_every_existing_byte() { + assert_eq!( + with_top_level_key("{\n \"mode\": \"direct\"\n}\n", "warnings", "[]"), + "{\n \"mode\": \"direct\",\n \"warnings\": []\n}\n" + ); + assert_eq!( + with_top_level_key( + "{\n \"mode\": \"direct\"\n}\n", + "warnings", + "[\n {\n \"a\": 1\n }\n]" + ), + "{\n \"mode\": \"direct\",\n \"warnings\": [\n {\n \"a\": 1\n }\n ]\n}\n" + ); + assert_eq!( + with_top_level_key("{\"mode\": \"direct\"}", "warnings", "[]"), + "{\"mode\": \"direct\", \"warnings\": []}" + ); + } +} diff --git a/crates/sqltide-python/src/project_analysis.rs b/crates/sqltide-python/src/project_analysis.rs index 8ac821ad..030cde8b 100644 --- a/crates/sqltide-python/src/project_analysis.rs +++ b/crates/sqltide-python/src/project_analysis.rs @@ -347,6 +347,10 @@ fn build_compile_inputs<'py>( "allowed_cross_pipeline_references", allowed_cross_pipeline_references, ), + ( + "consumer_group_identity", + optional(&configuration, "consumer_group_identity")?, + ), ], ) } diff --git a/crates/sqltide-python/src/realization.rs b/crates/sqltide-python/src/realization.rs index d05a4f0a..26ab096b 100644 --- a/crates/sqltide-python/src/realization.rs +++ b/crates/sqltide-python/src/realization.rs @@ -291,6 +291,10 @@ impl<'py> Realizer<'py> { ("consumer_group", kafka.getattr("consumer_group")?), ("format", kafka.getattr("format")?), ("project_name", project.getattr("project_name")?), + ( + "consumer_group_identity", + project.getattr("consumer_group_identity")?, + ), ("target_name", project.getattr("target_name")?), ("ttl", ttl), ("settings", settings), @@ -949,6 +953,44 @@ impl<'py> Realizer<'py> { } } +/// `(source name, consumer group)` for every managed source the adapter realizes for `project`, +/// in source order, with the project name and consumer group identity replaced as given. +pub(crate) fn managed_source_consumer_groups<'py>( + project: &Bound<'py, PyAny>, + adapter_profile: &Bound<'py, PyAny>, + project_name: &str, + consumer_group_identity: Option<&str>, +) -> PyResult> { + let py: Python<'py> = project.py(); + let realizer: Realizer<'py> = Realizer::import(py)?; + let project: Bound<'py, PyAny> = realizer.replace( + project.clone(), + &[ + ("project_name", project_name.into_pyobject(py)?.into_any()), + ( + "consumer_group_identity", + consumer_group_identity.into_pyobject(py)?.into_any(), + ), + ], + )?; + let realize_source: Bound<'py, PyAny> = adapter_profile.getattr("realize_source")?; + let mut groups: Vec<(String, String)> = Vec::new(); + for source in realizer.items(&project.getattr("sources")?)? { + let request: Bound<'py, PyAny> = realizer.source_request(&source, &project)?; + let realization: Bound<'py, PyAny> = + realizer.call(&realize_source, &[("request", request)])?; + for resource in realizer.items(&realization.getattr("resources")?)? { + if resource.is_instance(&realizer.managed_source)? { + groups.push(( + source.getattr("key")?.getattr("name")?.extract()?, + resource.getattr("consumer_group")?.extract()?, + )); + } + } + } + Ok(groups) +} + /// Map one complete logical project to adapter resources and planner desired state through /// the adapter profile's connection-free callbacks. #[pyfunction] diff --git a/crates/sqltide-python/src/rename_project_command.rs b/crates/sqltide-python/src/rename_project_command.rs new file mode 100644 index 00000000..de95eb7e --- /dev/null +++ b/crates/sqltide-python/src/rename_project_command.rs @@ -0,0 +1,885 @@ +//! `sqt rename-project`: rename a project while its Kafka consumer groups, manifest history and +//! project role assignments carry over to the new name. +//! +//! Every step is idempotent and the project config is edited last, so a failure leaves the old +//! name fully working and running the command again completes the rename. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +use pyo3::prelude::*; +use pyo3::types::{PyList, PyTuple}; +use sqltide_clickhouse::metadata::manifest_identity_copy; +use sqltide_clickhouse::resources::database_scoped_consumer_group; +use sqltide_config::project::PROJECT_CONFIG_FILE_NAME; +use sqltide_config::project_rename::{ + ProjectIdentityKeys, project_identity_keys, rename_project_text, +}; + +use crate::cli::python::{print, user_error}; +use crate::control_store::{ControlStore, ProjectRenamePreview, default_control_store_url}; +use crate::dev_server::support::attr; +use crate::handling::within; +use crate::py_kwargs; +use crate::realization::managed_source_consumer_groups; +use crate::workflow_statements::numbered; + +const NATIVE: &str = "sqltide._native"; +const ADAPTER_EXCEPTIONS: &str = "sqltide.adapter.exceptions"; +const MUTATION_LOCK: &str = "sqltide.executor.workflow.main.target_mutation_lock"; +const WAREHOUSE_WORKFLOW: &str = "sqltide.executor.workflow.main._execute_warehouse_workflow"; +const CONTROL_STORE_ENV_VAR: &str = "SQLTIDE_CONTROL_STORE_URL"; +const SQLITE_FILE_PREFIX: &str = "sqlite:///"; +const INTERPOLATION_TOKEN_START: &str = "${"; + +type Any<'py> = Bound<'py, PyAny>; + +/// The manifest history of one project target under the old and new identities. +struct ManifestPlan { + from_identity: String, + to_identity: String, + target_name: String, + database: String, + /// Manifests recorded under the old identity and not yet under the new one. + to_copy: usize, + /// Manifests recorded under the old identity. + recorded: usize, + /// The pipelines the old identity's newest manifest on the target names. + latest_pipelines: Vec, + /// Whether the new identity may already hold manifests of its own, as it does once a + /// target being completed with `--from` has built under the new name. + allow_existing: bool, +} + +/// The role assignments the rename moves, and where they live. +struct RolePlan<'py> { + store: Bound<'py, ControlStore>, + url: String, + preview: ProjectRenamePreview, +} + +/// Everything the rename would change, computed without writing anything. +struct RenamePlan<'py> { + config_path: PathBuf, + /// The name the warehouse and control store still record. + current_name: String, + new_name: String, + /// Whether this run completes an earlier rename (`--from`) on another target. + completing: bool, + /// The identity to add to the project config, when existing groups need it. + added_identity: Option, + /// `(source, database-scoped group)` for every managed source, unchanged by the rename. + groups: Vec<(String, String)>, + manifests: ManifestPlan, + roles: Option>, + other_targets: Vec, + /// The edited project config, or `None` when the config already carries the new name. + edited_config: Option, +} + +fn fail(py: Python<'_>, message: &str) -> PyErr { + user_error(py, message, None) +} + +/// The new name, held to the rules the project config applies to `name`. +fn validated_name(py: Python<'_>, new_name: &str, current_name: &str) -> PyResult { + if new_name.trim().is_empty() { + return Err(fail(py, "The new project name must be a non-empty string")); + } + if new_name.trim() != new_name { + return Err(fail( + py, + "The new project name must not start or end with whitespace", + )); + } + if new_name.contains(INTERPOLATION_TOKEN_START) { + return Err(fail( + py, + "The new project name must be a committed literal and cannot be interpolated", + )); + } + if new_name == current_name { + return Err(fail( + py, + &format!("The project is already named '{current_name}'"), + )); + } + Ok(new_name.to_owned()) +} + +/// The manifest ids one identity holds for the target, and the pipelines its newest manifest +/// names, or the reason they cannot be read. +fn manifest_history( + client: &Any<'_>, + database: &str, + identity: &str, + target_name: &str, +) -> PyResult<(BTreeSet, Vec)> { + let py: Python<'_> = client.py(); + let snapshot: Any<'_> = client.call_method( + "load_manifests", + (), + Some(&py_kwargs!(py; + database = database, + project_identity = identity, + target_name = target_name, + target_database = database, + )?), + )?; + let unavailable: Any<'_> = + attr(py, "sqltide.adapter.types", "AdapterOptionalStateStatus")?.getattr("UNAVAILABLE")?; + if snapshot.getattr("status")?.eq(unavailable)? { + let warning: Any<'_> = snapshot.getattr("warning")?; + return Err(fail( + py, + &format!( + "Manifest history is unavailable, so the rename cannot carry it over: {}", + if warning.is_none() { + "no detail".to_owned() + } else { + warning.str()?.to_string() + } + ), + )); + } + let manifests: Vec> = snapshot + .getattr("manifests")? + .try_iter()? + .collect::>()?; + // Newest first, as the manifest history query orders it. + let latest_pipelines: Vec = match manifests.first() { + Some(latest) => latest.getattr("pipelines")?.extract()?, + None => Vec::new(), + }; + let ids: BTreeSet = manifests + .iter() + .map(|manifest| manifest.getattr("manifest_id")?.extract()) + .collect::>()?; + Ok((ids, latest_pipelines)) +} + +fn manifest_plan( + client: &Any<'_>, + from_identity: &str, + to_identity: &str, + target_name: &str, + database: &str, + allow_existing: bool, +) -> PyResult { + let py: Python<'_> = client.py(); + let (recorded, latest_pipelines) = + manifest_history(client, database, from_identity, target_name)?; + let (copied, _) = manifest_history(client, database, to_identity, target_name)?; + let foreign: Vec<&String> = copied.difference(&recorded).collect(); + if !foreign.is_empty() && !allow_existing { + return Err(fail( + py, + &format!( + "Target '{target_name}' (database '{database}') already records {} manifest(s) \ + for project '{to_identity}' that did not come from '{from_identity}'; another \ + project already uses that name here. Choose a different name.", + foreign.len() + ), + )); + } + Ok(ManifestPlan { + from_identity: from_identity.to_owned(), + to_identity: to_identity.to_owned(), + target_name: target_name.to_owned(), + database: database.to_owned(), + to_copy: recorded.difference(&copied).count(), + recorded: recorded.len(), + latest_pipelines, + allow_existing, + }) +} + +/// The control store the project uses: `--control-store-url`, then `SQLTIDE_CONTROL_STORE_URL` +/// in the invocation's environment, then the project-local store when it exists. +fn control_store_url( + project_dir: &Any<'_>, + option: &Any<'_>, + environment: &Any<'_>, +) -> PyResult> { + if option.is_truthy()? { + return Ok(Some(option.extract()?)); + } + let configured: Any<'_> = environment.call_method1("get", (CONTROL_STORE_ENV_VAR,))?; + if configured.is_truthy()? { + return Ok(Some(configured.extract()?)); + } + let local: String = default_control_store_url(project_dir)?; + let path: &str = local.strip_prefix(SQLITE_FILE_PREFIX).unwrap_or(&local); + Ok(Path::new(path).exists().then_some(local)) +} + +fn role_plan<'py>( + py: Python<'py>, + url: Option, + from_project: &str, + to_project: &str, +) -> PyResult>> { + let Some(url) = url else { + return Ok(None); + }; + let store: Bound<'py, ControlStore> = attr(py, NATIVE, "ControlStore")? + .call((), Some(&py_kwargs!(py; url = &url)?))? + .cast_into()?; + let preview: ProjectRenamePreview = + store + .get() + .preview_project_rename(py, from_project, to_project)?; + if !preview.conflicts.is_empty() { + return Err(fail( + py, + &format!( + "Project '{to_project}' already holds role assignments that '{from_project}' \ + also holds: {}. Revoke one of each pair, then run the rename again.", + preview.conflicts.join(", ") + ), + )); + } + Ok(Some(RolePlan { + store, + url, + preview, + })) +} + +fn other_targets(loaded_project: &Any<'_>, target_name: &str) -> PyResult> { + let configuration: Any<'_> = loaded_project.getattr("configuration")?; + if configuration.is_none() { + return Ok(Vec::new()); + } + let mut names: Vec = Vec::new(); + for entry in configuration + .getattr("project")? + .getattr("targets")? + .try_iter()? + { + let name: String = entry?.get_item(0)?.extract()?; + if name != target_name { + names.push(name); + } + } + Ok(names) +} + +/// The old and new names when completing an earlier rename on another target: the config +/// must already carry `new_name`, and `from_name` must be the name it replaced. +fn completed_names( + py: Python<'_>, + from_name: &str, + new_name: &str, + config_name: &str, +) -> PyResult<(String, String)> { + if from_name.trim().is_empty() || from_name.trim() != from_name { + return Err(fail( + py, + "--from must be the previous project name, a non-empty string without surrounding \ + whitespace", + )); + } + if from_name == new_name { + return Err(fail( + py, + &format!( + "--from names the previous project name and cannot be the new name '{new_name}'" + ), + )); + } + if from_name == config_name { + return Err(fail( + py, + &format!( + "sqltide_project.toml is still named '{config_name}'. Run `sqt rename-project \ + {new_name}` without --from first; use --from only on the remaining targets \ + after the config is renamed" + ), + )); + } + if new_name != config_name { + return Err(fail( + py, + &format!( + "With --from, the new name must be the name sqltide_project.toml already \ + carries ('{config_name}'), not '{new_name}'" + ), + )); + } + Ok((from_name.to_owned(), new_name.to_owned())) +} + +#[allow(clippy::too_many_arguments)] +fn plan_rename<'py>( + py: Python<'py>, + project_dir: &Any<'py>, + new_name: &str, + from_name: Option<&str>, + database: &str, + control_store_option: &Any<'py>, + environment: &Any<'py>, + client: &Any<'py>, + loaded_project: &Any<'py>, + analysis: &Any<'py>, + adapter_profile: &Any<'py>, +) -> PyResult> { + let config_path: PathBuf = project_dir + .call_method1("__truediv__", (PROJECT_CONFIG_FILE_NAME,))? + .extract()?; + let config_text: String = std::fs::read_to_string(&config_path).map_err(|error| { + fail( + py, + &format!("Could not read {}: {error}", config_path.display()), + ) + })?; + let keys: ProjectIdentityKeys = + project_identity_keys(&config_text).map_err(|message| fail(py, &message))?; + let compiled: Any<'py> = analysis.getattr("compiled_project")?; + let config_name: String = compiled.getattr("project_name")?.extract()?; + let (current_name, new_name): (String, String) = match from_name { + Some(from_name) => completed_names(py, from_name, new_name, &config_name)?, + None => ( + config_name.clone(), + validated_name(py, new_name, &config_name)?, + ), + }; + let completing: bool = from_name.is_some(); + let identity: Option = compiled.getattr("consumer_group_identity")?.extract()?; + if identity + != keys + .consumer_group_identity + .map(|value| value.trim().to_owned()) + { + return Err(fail( + py, + &format!( + "The consumer group identity in {} does not match the loaded project; check \ + the file and run the rename again", + config_path.display() + ), + )); + } + refuse_live_group_drift(py, client, analysis, database)?; + let before: Vec<(String, String)> = managed_source_consumer_groups( + &compiled, + adapter_profile, + ¤t_name, + identity.as_deref(), + )?; + let renamed = |identity: Option<&str>| { + managed_source_consumer_groups(&compiled, adapter_profile, &new_name, identity) + }; + // Keep the old name in the groups only when some group would otherwise change. A completing + // run never edits the config, so its identity must already keep the groups. + let added_identity: Option = + if !completing && identity.is_none() && renamed(None)? != before { + Some(current_name.clone()) + } else { + None + }; + let after: Vec<(String, String)> = renamed(identity.as_deref().or(added_identity.as_deref()))?; + let changed: Vec = before + .iter() + .zip(&after) + .filter(|(old, new)| old != new) + .map(|((source, old), (_, new))| format!("{source}: '{old}' -> '{new}'")) + .collect(); + if !changed.is_empty() || before.len() != after.len() { + return Err(fail( + py, + &format!( + "Renaming would change Kafka consumer groups, so sources would not resume from \ + their committed offsets: {}{}", + changed.join("; "), + if completing { + format!( + ". Set consumer_group_identity = {} in sqltide_project.toml first", + quoted(¤t_name) + ) + } else { + String::new() + } + ), + )); + } + let groups: Vec<(String, String)> = before + .into_iter() + .map(|(source, group)| (source, database_scoped_consumer_group(&group, database))) + .collect(); + let target_name: String = compiled.getattr("target_name")?.extract()?; + let from_identity: String = if completing { + current_name.clone() + } else { + attr(py, NATIVE, "resolve_manifest_project_identity")? + .call((), Some(&py_kwargs!(py; analysis = analysis)?))? + .extract()? + }; + let manifests: ManifestPlan = manifest_plan( + client, + &from_identity, + &new_name, + &target_name, + database, + completing, + )?; + let roles: Option> = role_plan( + py, + control_store_url(project_dir, control_store_option, environment)?, + ¤t_name, + &new_name, + )?; + let moving_roles: usize = roles.as_ref().map_or(0, |roles| roles.preview.moving); + if completing && manifests.recorded == 0 && moving_roles == 0 { + return Err(fail( + py, + &format!( + "Nothing on target '{target_name}' records project '{current_name}': it has no \ + manifests in database '{database}' and no role assignments in the control \ + store. Either '{current_name}' is mistyped, or this target was never built \ + under it and this control store holds no grants for it, in which case there \ + is nothing to complete here." + ), + )); + } + if completing { + refuse_foreign_history( + py, + &compiled, + identity.as_deref(), + ¤t_name, + &manifests, + )?; + } + let edited_config: Option = if completing { + None + } else { + Some( + rename_project_text(&config_text, &new_name, added_identity.as_deref()) + .map_err(|message| fail(py, &message))?, + ) + }; + Ok(RenamePlan { + config_path, + current_name, + new_name, + completing, + added_identity, + groups, + manifests, + roles, + other_targets: other_targets(loaded_project, &target_name)?, + edited_config, + }) +} + +/// Refuse to complete a rename from a name whose history on the target belongs to another +/// project: merging it would make destroy planning treat that project's pipelines as this +/// project's retired ones. A `consumer_group_identity` must equal OLD_NAME; without one, the +/// newest OLD_NAME manifest must share a pipeline with the compiled project. +fn refuse_foreign_history( + py: Python<'_>, + compiled: &Any<'_>, + identity: Option<&str>, + from_name: &str, + manifests: &ManifestPlan, +) -> PyResult<()> { + let foreign = |detail: String| -> PyErr { + fail( + py, + &format!( + "The history of '{from_name}' on target '{}' does not look like this project's: \ + {detail}. Pass the name this project had before the rename to --from.", + manifests.target_name + ), + ) + }; + if let Some(identity) = identity { + if identity != from_name { + return Err(foreign(format!( + "sqltide_project.toml keeps consumer_group_identity = {}, not {}", + quoted(identity), + quoted(from_name) + ))); + } + return Ok(()); + } + if manifests.recorded == 0 { + return Ok(()); + } + let current: Vec = compiled + .getattr("pipelines")? + .try_iter()? + .map(|pipeline| pipeline?.getattr("pipeline")?.getattr("name")?.extract()) + .collect::>()?; + if manifests + .latest_pipelines + .iter() + .any(|name| current.contains(name)) + { + return Ok(()); + } + Err(foreign(format!( + "its latest manifest names pipelines {} and none of them is in this project", + manifests.latest_pipelines.join(", ") + ))) +} + +/// Refuse when a managed Kafka table on the target already consumes as a different consumer +/// group than the project declares: pinning `consumer_group_identity` to the current name would +/// then keep a group the live tables do not use. +fn refuse_live_group_drift( + py: Python<'_>, + client: &Any<'_>, + analysis: &Any<'_>, + database: &str, +) -> PyResult<()> { + if !client + .call_method1("database_exists", (database,))? + .is_truthy()? + { + return Ok(()); + } + let catalog: Any<'_> = client.call_method1("load_catalog", (database,))?; + let drifts = crate::plan_warnings::consumer_group_drifts(analysis, &catalog, database)?; + if drifts.is_empty() { + return Ok(()); + } + let tables: Vec = drifts + .iter() + .map(|(_, drift)| { + format!( + "{} consumes as '{}' but the project declares '{}'", + drift.table_name, drift.live_group, drift.declared_group + ) + }) + .collect(); + Err(fail( + py, + &format!( + "Kafka consumer groups on database '{database}' already differ from the project \ + declaration: {}. `sqt plan` shows this as a consumer group drift warning. Resolve \ + the drift before renaming, for example by restoring the name or \ + consumer_group_identity the live tables were created with.", + tables.join("; ") + ), + )) +} + +fn quoted(value: &str) -> String { + sqltide_config::project_rename::toml_basic_string(value) +} + +fn render_plan(plan: &RenamePlan<'_>, applied: bool) -> String { + let manifests: &ManifestPlan = &plan.manifests; + let mut lines: Vec = vec![ + if applied { + "Project Renamed".to_owned() + } else { + "Rename Project Preview".to_owned() + }, + format!("Project: {} -> {}", plan.current_name, plan.new_name), + format!( + "Target: {} (database {})", + manifests.target_name, manifests.database + ), + ]; + if plan.completing { + lines.push(format!( + "Config: {} already named {}; unchanged", + plan.config_path.display(), + quoted(&plan.new_name) + )); + } else { + lines.push(format!("Config: {}", plan.config_path.display())); + lines.push(format!(" name = {}", quoted(&plan.new_name))); + } + if let Some(identity) = &plan.added_identity { + lines.push(format!( + " consumer_group_identity = {} (added so existing Kafka consumer groups stay the same)", + quoted(identity) + )); + } + lines.push(format!( + "Kafka consumer groups: {} unchanged", + plan.groups.len() + )); + lines.extend( + plan.groups + .iter() + .map(|(source, group)| format!(" - {source}: {group}")), + ); + lines.push(format!( + "Manifest history: {} of {} manifest(s) {} from '{}' to '{}'", + manifests.to_copy, + manifests.recorded, + if applied { "copied" } else { "to copy" }, + manifests.from_identity, + manifests.to_identity + )); + match &plan.roles { + None => lines.push("Role assignments: no control store in use".to_owned()), + Some(roles) => lines.push(format!( + "Role assignments: {} {} from '{}' to '{}'{} (control store {})", + roles.preview.moving, + if applied { "moved" } else { "to move" }, + plan.current_name, + plan.new_name, + if roles.preview.existing > 0 { + format!( + "; {} already recorded under '{}'", + roles.preview.existing, plan.new_name + ) + } else { + String::new() + }, + redacted_url(&roles.url) + )), + } + if !plan.other_targets.is_empty() && !plan.completing { + lines.push(format!( + "Other targets keep manifest history under '{}' until you run \ + `sqt rename-project {} --from {} --target ` there: {}", + plan.current_name, + plan.new_name, + plan.current_name, + plan.other_targets.join(", ") + )); + if plan.roles.is_some() { + lines.push( + "Role grants move for this whole control store. If other targets share it, \ + their UI users lose those grants until those targets run the renamed config." + .to_owned(), + ); + } + } + lines.push( + "Unchanged: run history and audit and check status follow the project directory name, \ + not the project name." + .to_owned(), + ); + lines.push(if applied { + "Restart any running `sqt dev` server to pick up the new name.".to_owned() + } else { + "Dry run: nothing was changed. Run again with --apply to rename.".to_owned() + }); + lines.join("\n") +} + +/// A control store URL without its password. +fn redacted_url(url: &str) -> String { + let Some((scheme, rest)) = url.split_once("://") else { + return url.to_owned(); + }; + let Some((credentials, host)) = rest.rsplit_once('@') else { + return url.to_owned(); + }; + match credentials.split_once(':') { + Some((user, _)) => format!("{scheme}://{user}:***@{host}"), + None => url.to_owned(), + } +} + +/// Copy the manifest history under the target mutation lock; refuse while a run holds it. +fn copy_manifests(client: &Any<'_>, plan: &ManifestPlan) -> PyResult { + let py: Python<'_> = client.py(); + if !client + .call_method1("database_exists", (&plan.database,))? + .is_truthy()? + { + // Never built: no manifests to copy and no run that could hold the lock. + return Ok(0); + } + let lock: Any<'_> = attr(py, MUTATION_LOCK, "target_mutation_lock")?.call( + (), + Some(&py_kwargs!(py; connection = client, database = &plan.database)?), + )?; + let copied = within(&lock, || { + let fresh: ManifestPlan = manifest_plan( + client, + &plan.from_identity, + &plan.to_identity, + &plan.target_name, + &plan.database, + plan.allow_existing, + )?; + if fresh.to_copy == 0 { + return Ok(0); + } + let statement: String = manifest_identity_copy( + &plan.database, + &plan.from_identity, + &plan.to_identity, + &plan.target_name, + &plan.database, + ); + let rendered: Any<'_> = PyTuple::new(py, [statement])?.into_any(); + let statements: Vec> = + numbered(&rendered, 1, "copy_manifest_history", "FINALIZATION")?; + attr(py, WAREHOUSE_WORKFLOW, "execute_warehouse_workflow")?.call( + (), + Some(&py_kwargs!(py; statements = PyTuple::new(py, statements)?, connection = client)?), + )?; + let after: ManifestPlan = manifest_plan( + client, + &plan.from_identity, + &plan.to_identity, + &plan.target_name, + &plan.database, + plan.allow_existing, + )?; + if after.to_copy != 0 { + return Err(fail( + py, + &format!( + "{} manifest(s) were not copied to '{}'; run the rename again", + after.to_copy, plan.to_identity + ), + )); + } + Ok(fresh.to_copy) + }); + copied.map_err(|error| { + let locked: bool = attr(py, ADAPTER_EXCEPTIONS, "AdapterTargetMutationLockError") + .and_then(|class| error.value(py).is_instance(&class)) + .unwrap_or(false); + if !locked { + return error; + } + let refused: PyErr = fail( + py, + &format!( + "A run is active on target database '{}' ({}). Wait for it to finish, then \ + run the rename again.", + plan.database, + error.value(py) + ), + ); + refused.set_cause(py, Some(error)); + refused + }) +} + +/// Write the edited config next to the original and move it into place. +fn write_config(path: &Path, text: &str) -> std::io::Result<()> { + let staged: PathBuf = path.with_file_name(format!( + ".{}.rename-project", + path.file_name() + .and_then(|name| name.to_str()) + .unwrap_or("project") + )); + std::fs::write(&staged, text)?; + if let Ok(metadata) = std::fs::metadata(path) { + std::fs::set_permissions(&staged, metadata.permissions())?; + } + std::fs::rename(&staged, path).inspect_err(|_| { + let _ = std::fs::remove_file(&staged); + }) +} + +fn apply_rename(py: Python<'_>, client: &Any<'_>, plan: &RenamePlan<'_>) -> PyResult<()> { + copy_manifests(client, &plan.manifests)?; + let moved: Vec = match &plan.roles { + Some(roles) => roles + .store + .get() + .rename_project(py, &plan.current_name, &plan.new_name)?, + None => Vec::new(), + }; + let Some(edited_config) = &plan.edited_config else { + return Ok(()); + }; + if let Err(error) = write_config(&plan.config_path, edited_config) { + let mut message: String = format!( + "Could not write {}: {error}. The project keeps its old name", + plan.config_path.display() + ); + if let Some(roles) = &plan.roles + && !moved.is_empty() + && let Err(restore) = roles.store.get().restore_project_rename( + py, + &moved, + &plan.current_name, + &plan.new_name, + ) + { + message.push_str(&format!( + "; its role assignments stayed under '{}' ({restore}), so run the rename \ + again to finish", + plan.new_name + )); + } + return Err(fail(py, &message)); + } + Ok(()) +} + +/// Preview the rename of the project, or apply it with `apply`. +#[pyfunction] +#[pyo3(signature = ( + *, pipelines_root, project_dir, new_name, database, control_store_url, apply, client, + loaded_project, adapter_profile, from_name = None, environment = None, +))] +#[allow(clippy::too_many_arguments)] +fn run_rename_project<'py>( + py: Python<'py>, + pipelines_root: Any<'py>, + project_dir: Any<'py>, + new_name: String, + database: Any<'py>, + control_store_url: Any<'py>, + apply: bool, + client: Any<'py>, + loaded_project: Any<'py>, + adapter_profile: Any<'py>, + from_name: Option, + environment: Option>, +) -> PyResult { + let environment: Any<'py> = match environment { + Some(environment) if environment.is_truthy()? => environment, + _ => pyo3::types::PyDict::new(py).into_any(), + }; + let analysis: Any<'py> = attr(py, NATIVE, "analyze_project")?.call( + (), + Some(&py_kwargs!(py; + pipelines_root = pipelines_root, + loaded_project = &loaded_project, + adapter_profile = &adapter_profile, + )?), + )?; + let pipelines: Any<'py> = PyList::new( + py, + analysis + .getattr("compile_inputs")? + .getattr("pipelines")? + .try_iter()? + .collect::>>()?, + )? + .into_any(); + let database: String = + crate::cli::invocation::resolve_default_database(py, &pipelines, database)?.extract()?; + let plan: RenamePlan<'py> = plan_rename( + py, + &project_dir, + &new_name, + from_name.as_deref(), + &database, + &control_store_url, + &environment, + &client, + &loaded_project, + &analysis, + &adapter_profile, + )?; + if !apply { + print(py, &render_plan(&plan, false))?; + return Ok(0); + } + apply_rename(py, &client, &plan)?; + print(py, &render_plan(&plan, true))?; + Ok(0) +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(run_rename_project, module)?) +} diff --git a/crates/sqltide-python/src/shapes/adapter_models.rs b/crates/sqltide-python/src/shapes/adapter_models.rs index 2fe68840..f9dd6635 100644 --- a/crates/sqltide-python/src/shapes/adapter_models.rs +++ b/crates/sqltide-python/src/shapes/adapter_models.rs @@ -130,6 +130,7 @@ shapes! { ttl(default = Given::None), settings(default = Given::EmptyTuple), naming_macro_fingerprint(default = Given::None), + consumer_group_identity(default = Given::None), } /// Logical adopted-source fields needed for adapter realization. diff --git a/crates/sqltide-python/src/shapes/cli_service.rs b/crates/sqltide-python/src/shapes/cli_service.rs index df3acc9b..952c44ae 100644 --- a/crates/sqltide-python/src/shapes/cli_service.rs +++ b/crates/sqltide-python/src/shapes/cli_service.rs @@ -573,6 +573,7 @@ const ENUMS: [EnumSpec; 14] = [ ("DEPLOYMENT", "deployment"), ("AUDIT", "audit"), ("RECONCILE", "reconcile"), + ("RENAME_PROJECT", "rename-project"), ("JANITOR", "janitor"), ("DOCTOR", "doctor"), ("REPAIR", "repair"), @@ -995,6 +996,7 @@ fn constants(module: &Bound<'_, PyModule>) -> PyResult<()> { "RESET_TARGET", "DEPLOYMENT", "RECONCILE", + "RENAME_PROJECT", "AUDIT", ], )?, diff --git a/crates/sqltide-python/src/shapes/compiler_project.rs b/crates/sqltide-python/src/shapes/compiler_project.rs index b263e8fd..ccd522a9 100644 --- a/crates/sqltide-python/src/shapes/compiler_project.rs +++ b/crates/sqltide-python/src/shapes/compiler_project.rs @@ -463,6 +463,7 @@ shapes! { coerce = Coerce::Enum("ModelReferenceScope"), ), allowed_cross_pipeline_references(default = Given::EmptyTuple, coerce = Coerce::Tuple), + consumer_group_identity(default = Given::None), } /// Current whole-project compile output consumed by every command. @@ -484,6 +485,7 @@ shapes! { coerce = Coerce::Enum("ModelReferenceScope"), ), allowed_cross_pipeline_references(default = Given::EmptyTuple, coerce = Coerce::Tuple), + consumer_group_identity(default = Given::None), } } diff --git a/crates/sqltide-python/src/shapes/discovery_models.rs b/crates/sqltide-python/src/shapes/discovery_models.rs index 605df845..1d167d43 100644 --- a/crates/sqltide-python/src/shapes/discovery_models.rs +++ b/crates/sqltide-python/src/shapes/discovery_models.rs @@ -271,6 +271,7 @@ shapes! { build(default = Given::Factory(factory::)), ui(default = Given::Factory(factory::)), dependencies(default = Given::Factory(factory::)), + consumer_group_identity(default = Given::None), } /// Optional local TOML overrides before invocation resolution. @@ -303,6 +304,7 @@ shapes! { ui(default = Given::Factory(factory::)), production_target(default = Given::Bool(false)), dependencies(default = Given::Factory(factory::)), + consumer_group_identity(default = Given::None), } /// Kafka engine settings for a landing step. diff --git a/scripts/shape_surface/baseline.json b/scripts/shape_surface/baseline.json index 511c7d93..5983b31d 100644 --- a/scripts/shape_surface/baseline.json +++ b/scripts/shape_surface/baseline.json @@ -60,7 +60,7 @@ "sqltide.adapter.models:AdapterIdentity": {"fields": [["name", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["name"], "module": "sqltide.adapter.models", "mro": ["sqltide.adapter.models.AdapterIdentity", "builtins.object"], "qualname": "AdapterIdentity"}, "sqltide.adapter.models:AdapterInvocationRecord": {"fields": [["invocation_id", false, true, true], ["project_identity", false, true, true], ["target_identity", false, true, true], ["command", false, true, true], ["mode", false, true, true], ["outcome", false, true, true], ["exit_code", false, true, true], ["materialized_outcome", false, true, true], ["deployment_id", false, true, true], ["workflow_id", false, true, true], ["selected_node_count", false, true, true], ["started_at", false, true, true], ["completed_at", false, true, true], ["duration_ms", false, true, true], ["error_message", false, true, true], ["summary_json", false, true, true], ["tool_version", false, true, true], ["artifact_project_dir", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["invocation_id", "project_identity", "target_identity", "command", "mode", "outcome", "exit_code", "materialized_outcome", "deployment_id", "workflow_id", "selected_node_count", "started_at", "completed_at", "duration_ms", "error_message", "summary_json", "tool_version", "artifact_project_dir"], "module": "sqltide.adapter.models", "mro": ["sqltide.adapter.models.AdapterInvocationRecord", "builtins.object"], "qualname": "AdapterInvocationRecord"}, "sqltide.adapter.models:AdapterManagedSource": {"fields": [["source_kind", false, true, true], ["name", false, true, true], ["columns", false, true, true], ["broker_list", false, true, true], ["topic", false, true, true], ["consumer_group", false, true, true], ["format", false, true, true], ["settings", true, true, true], ["naming_macro_fingerprint", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["source_kind", "name", "columns", "broker_list", "topic", "consumer_group", "format", "settings", "naming_macro_fingerprint"], "module": "sqltide.adapter.models", "mro": ["sqltide.adapter.models.AdapterManagedSource", "builtins.object"], "qualname": "AdapterManagedSource"}, -"sqltide.adapter.models:AdapterManagedSourceRealizationRequest": {"fields": [["logical_name", false, true, true], ["source_kind", false, true, true], ["broker_list", false, true, true], ["topic", false, true, true], ["consumer_group", false, true, true], ["format", false, true, true], ["project_name", true, true, true], ["target_name", true, true, true], ["ttl", true, true, true], ["settings", true, true, true], ["naming_macro_fingerprint", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["logical_name", "source_kind", "broker_list", "topic", "consumer_group", "format", "project_name", "target_name", "ttl", "settings", "naming_macro_fingerprint"], "module": "sqltide.adapter.models", "mro": ["sqltide.adapter.models.AdapterManagedSourceRealizationRequest", "builtins.object"], "qualname": "AdapterManagedSourceRealizationRequest"}, +"sqltide.adapter.models:AdapterManagedSourceRealizationRequest": {"fields": [["logical_name", false, true, true], ["source_kind", false, true, true], ["broker_list", false, true, true], ["topic", false, true, true], ["consumer_group", false, true, true], ["format", false, true, true], ["project_name", true, true, true], ["target_name", true, true, true], ["ttl", true, true, true], ["settings", true, true, true], ["naming_macro_fingerprint", true, true, true], ["consumer_group_identity", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["logical_name", "source_kind", "broker_list", "topic", "consumer_group", "format", "project_name", "target_name", "ttl", "settings", "naming_macro_fingerprint", "consumer_group_identity"], "module": "sqltide.adapter.models", "mro": ["sqltide.adapter.models.AdapterManagedSourceRealizationRequest", "builtins.object"], "qualname": "AdapterManagedSourceRealizationRequest"}, "sqltide.adapter.models:AdapterManifest": {"fields": [["manifest_id", false, true, true], ["invocation_id", false, true, true], ["project_identity", false, true, true], ["target_name", false, true, true], ["target_database", false, true, true], ["is_production", false, true, true], ["project_revision", false, true, true], ["manifest_fingerprint", false, true, true], ["manifest_version", false, true, true], ["pipelines", false, true, true], ["resources", false, true, true], ["tool_version", false, true, true], ["published_at", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["manifest_id", "invocation_id", "project_identity", "target_name", "target_database", "is_production", "project_revision", "manifest_fingerprint", "manifest_version", "pipelines", "resources", "tool_version", "published_at"], "module": "sqltide.adapter.models", "mro": ["sqltide.adapter.models.AdapterManifest", "builtins.object"], "qualname": "AdapterManifest"}, "sqltide.adapter.models:AdapterManifestResource": {"fields": [["pipeline_name", false, true, true], ["logical_type", false, true, true], ["logical_name", false, true, true], ["resource_role", false, true, true], ["resource_database", false, true, true], ["resource_name", false, true, true], ["resource_kind", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["pipeline_name", "logical_type", "logical_name", "resource_role", "resource_database", "resource_name", "resource_kind"], "module": "sqltide.adapter.models", "mro": ["sqltide.adapter.models.AdapterManifestResource", "builtins.object"], "qualname": "AdapterManifestResource"}, "sqltide.adapter.models:AdapterManifestSnapshot": {"fields": [["status", false, true, true], ["manifests", false, true, true], ["warning", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["status", "manifests", "warning"], "module": "sqltide.adapter.models", "mro": ["sqltide.adapter.models.AdapterManifestSnapshot", "builtins.object"], "qualname": "AdapterManifestSnapshot"}, @@ -218,7 +218,7 @@ "sqltide.cli.dev.models:DevCommandOptions": {"fields": [["pipelines_root", false, true, true], ["database", false, true, true], ["host", false, true, true], ["port", false, true, true], ["selected_target", true, true, true], ["cli_variables", true, true, true], ["environment", true, true, true], ["connection_host", true, true, true], ["connection_port", true, true, true], ["connection_username", true, true, true], ["connection_password", true, true, true], ["auth_settings", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["pipelines_root", "database", "host", "port", "selected_target", "cli_variables", "environment", "connection_host", "connection_port", "connection_username", "connection_password", "auth_settings"], "module": "sqltide.cli.dev.models", "mro": ["sqltide.cli.dev.models.DevCommandOptions", "builtins.object"], "qualname": "DevCommandOptions"}, "sqltide.cli.entry.constants:ACTIVE_VIEW_SUBCOMMAND": {"kind": "constant", "type": "CliSubcommand", "value": "CliSubcommand.ACTIVE_VIEW"}, "sqltide.cli.entry.constants:AFFIRMATIVE_RESPONSES": {"kind": "constant", "type": "frozenset", "value": ["\"y\"", "\"yes\""]}, -"sqltide.cli.entry.constants:COMMANDS_REQUIRING_PIPELINES_ROOT": {"kind": "constant", "type": "frozenset", "value": ["\"CliCommand.AUDIT\"", "\"CliCommand.BUILD\"", "\"CliCommand.COMPILE\"", "\"CliCommand.DEPLOYMENT\"", "\"CliCommand.DESTROY\"", "\"CliCommand.DEV\"", "\"CliCommand.DISCOVER\"", "\"CliCommand.FORMAT\"", "\"CliCommand.LINT\"", "\"CliCommand.PLAN\"", "\"CliCommand.RECONCILE\"", "\"CliCommand.RESET_TARGET\"", "\"CliCommand.TEST\""]}, +"sqltide.cli.entry.constants:COMMANDS_REQUIRING_PIPELINES_ROOT": {"kind": "constant", "type": "frozenset", "value": ["\"CliCommand.AUDIT\"", "\"CliCommand.BUILD\"", "\"CliCommand.COMPILE\"", "\"CliCommand.DEPLOYMENT\"", "\"CliCommand.DESTROY\"", "\"CliCommand.DEV\"", "\"CliCommand.DISCOVER\"", "\"CliCommand.FORMAT\"", "\"CliCommand.LINT\"", "\"CliCommand.PLAN\"", "\"CliCommand.RECONCILE\"", "\"CliCommand.RENAME_PROJECT\"", "\"CliCommand.RESET_TARGET\"", "\"CliCommand.TEST\""]}, "sqltide.cli.entry.constants:COMMANDS_WITHOUT_ADAPTER_CONNECTION": {"kind": "constant", "type": "frozenset", "value": ["\"CliCommand.COMPILE\"", "\"CliCommand.DISCOVER\"", "\"CliCommand.FORMAT\"", "\"CliCommand.LINT\""]}, "sqltide.cli.entry.constants:DEV_CLI_VARIABLES_ENV_VAR": {"kind": "constant", "type": "str", "value": "SQLTIDE_INTERNAL_CLI_VARS"}, "sqltide.cli.entry.constants:DIRECT_ONLY_COMMANDS": {"kind": "constant", "type": "frozenset", "value": []}, @@ -232,7 +232,7 @@ "sqltide.cli.entry.models:ResolvedCliInvocation": {"fields": [["args", false, true, true], ["project_dir", false, true, true], ["pipelines_root", false, true, true], ["database", false, true, true], ["adapter", false, true, true], ["loaded_project", false, true, true], ["connection", false, true, true], ["cli_variables", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["args", "project_dir", "pipelines_root", "database", "adapter", "loaded_project", "connection", "cli_variables"], "module": "sqltide.cli.entry.models", "mro": ["sqltide.cli.entry.models.ResolvedCliInvocation", "builtins.object"], "qualname": "ResolvedCliInvocation"}, "sqltide.cli.entry.models:ResolvedCliProjectConfig": {"fields": [["connection", false, true, true], ["default_database", false, true, true], ["adapter_name", false, true, true], ["loaded_project", false, true, true], ["raw_connection", true, true, true], ["variables", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["connection", "default_database", "adapter_name", "loaded_project", "raw_connection", "variables"], "module": "sqltide.cli.entry.models", "mro": ["sqltide.cli.entry.models.ResolvedCliProjectConfig", "builtins.object"], "qualname": "ResolvedCliProjectConfig"}, "sqltide.cli.entry.models:ResolvedInvocationConnection": {"fields": [["connection", false, true, true], ["close_after_command", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["connection", "close_after_command"], "module": "sqltide.cli.entry.models", "mro": ["sqltide.cli.entry.models.ResolvedInvocationConnection", "builtins.object"], "qualname": "ResolvedInvocationConnection"}, -"sqltide.cli.entry.types:CliCommand": {"kind": "enum", "members": [["ADMIN", "admin"], ["PLAYGROUND", "playground"], ["DISCOVER", "discover"], ["DEV", "dev"], ["COMPILE", "compile"], ["LINT", "lint"], ["FORMAT", "format"], ["TEST", "test"], ["PLAN", "plan"], ["BUILD", "build"], ["DESTROY", "destroy"], ["RESET_TARGET", "reset-target"], ["DEPLOYMENT", "deployment"], ["AUDIT", "audit"], ["RECONCILE", "reconcile"], ["JANITOR", "janitor"], ["DOCTOR", "doctor"], ["REPAIR", "repair"]], "module": "sqltide.cli.entry.types", "mro": ["sqltide.cli.entry.types.CliCommand", "enum.StrEnum", "builtins.str", "enum.ReprEnum", "enum.Enum", "builtins.object"], "qualname": "CliCommand"}, +"sqltide.cli.entry.types:CliCommand": {"kind": "enum", "members": [["ADMIN", "admin"], ["PLAYGROUND", "playground"], ["DISCOVER", "discover"], ["DEV", "dev"], ["COMPILE", "compile"], ["LINT", "lint"], ["FORMAT", "format"], ["TEST", "test"], ["PLAN", "plan"], ["BUILD", "build"], ["DESTROY", "destroy"], ["RESET_TARGET", "reset-target"], ["DEPLOYMENT", "deployment"], ["AUDIT", "audit"], ["RECONCILE", "reconcile"], ["RENAME_PROJECT", "rename-project"], ["JANITOR", "janitor"], ["DOCTOR", "doctor"], ["REPAIR", "repair"]], "module": "sqltide.cli.entry.types", "mro": ["sqltide.cli.entry.types.CliCommand", "enum.StrEnum", "builtins.str", "enum.ReprEnum", "enum.Enum", "builtins.object"], "qualname": "CliCommand"}, "sqltide.cli.entry.types:CliSubcommand": {"kind": "enum", "members": [["ACTIVE_VIEW", "active-view"], ["DIFF", "diff"], ["LIST", "list"], ["SHOW", "show"], ["AUDIT", "audit"], ["PROMOTE", "promote"], ["ROLLBACK", "rollback"]], "module": "sqltide.cli.entry.types", "mro": ["sqltide.cli.entry.types.CliSubcommand", "enum.StrEnum", "builtins.str", "enum.ReprEnum", "enum.Enum", "builtins.object"], "qualname": "CliSubcommand"}, "sqltide.cli.plan.constants:DIRECT_MODE_LABEL": {"kind": "constant", "type": "str", "value": "direct"}, "sqltide.cli.plan.models:PlanCommandOptions": {"fields": [["pipelines_root", false, true, true], ["database", false, true, true], ["selectors", false, true, true], ["deployment_id", false, true, true], ["full_refresh", false, true, true], ["start_time", false, true, true], ["json_output", false, true, true], ["verbose", false, true, true], ["changed", true, true, true], ["include_missing_upstream", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["pipelines_root", "database", "selectors", "deployment_id", "full_refresh", "start_time", "json_output", "verbose", "changed", "include_missing_upstream"], "module": "sqltide.cli.plan.models", "mro": ["sqltide.cli.plan.models.PlanCommandOptions", "builtins.object"], "qualname": "PlanCommandOptions"}, @@ -311,10 +311,10 @@ "sqltide.compiler.compile.exceptions:TransformSqlUntypedProjectionError": {"doc": "Raised when an outermost projection is not an explicitly typed alias.", "instance": {"args": ["probe-transform_name"], "repr": "TransformSqlUntypedProjectionError('probe-transform_name')", "str": "Transform 'probe-transform_name' has an invalid output contract at column 3. Expected outermost projection 3 to use an explicit typed alias like CAST(expr AS Type) AS name or expr::Type AS name, but found `probe-projection_sql`."}, "kind": "exception", "module": "sqltide.compiler.compile.exceptions", "mro": ["sqltide.compiler.compile.exceptions.TransformSqlUntypedProjectionError", "sqltide.compiler.compile.exceptions.TransformSqlContractError", "builtins.Exception", "builtins.BaseException", "builtins.object"], "probe": {"args": ["probe-transform_name", 3, "probe-projection_sql"], "kwargs": {}}, "qualname": "TransformSqlUntypedProjectionError"}, "sqltide.compiler.compile.exceptions:TransformTtlUnknownColumnError": {"doc": "Raised when a TTL expression references unknown derived columns.", "instance": {"args": ["probe-transform_name"], "repr": "TransformTtlUnknownColumnError('probe-transform_name')", "str": "Transform 'probe-transform_name' has an invalid TTL expression `probe-expression`. Referenced output columns not found in the derived schema: unknown_column_names-a, unknown_column_names-b. Available columns: available_column_names-a, available_column_names-b."}, "kind": "exception", "module": "sqltide.compiler.compile.exceptions", "mro": ["sqltide.compiler.compile.exceptions.TransformTtlUnknownColumnError", "sqltide.compiler.compile.exceptions.TransformSqlContractError", "builtins.Exception", "builtins.BaseException", "builtins.object"], "probe": {"args": ["probe-transform_name", "probe-expression", {"tuple": ["unknown_column_names-a", "unknown_column_names-b"]}, {"tuple": ["available_column_names-a", "available_column_names-b"]}], "kwargs": {}}, "qualname": "TransformTtlUnknownColumnError"}, "sqltide.compiler.compile.models:Column": {"fields": [["name", false, true, true], ["type", false, true, true], ["default", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["name", "type", "default"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.Column", "builtins.object"], "qualname": "Column"}, -"sqltide.compiler.compile.models:CompileProjectInputs": {"fields": [["discovered_inputs", false, true, true], ["adapter_profile", false, true, true], ["effective_target", false, true, true], ["variables", false, true, true], ["macro_registry", false, true, true], ["macro_context", false, true, true], ["pipelines", false, true, true], ["tests", false, true, true], ["audits", false, true, true], ["sources", true, true, true], ["virtual_environments", true, true, true], ["project_name", true, true, true], ["target_name", true, true, true], ["model_reference_scope", true, true, true], ["allowed_cross_pipeline_references", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["discovered_inputs", "adapter_profile", "effective_target", "variables", "macro_registry", "macro_context", "pipelines", "tests", "audits", "sources", "virtual_environments", "project_name", "target_name", "model_reference_scope", "allowed_cross_pipeline_references"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompileProjectInputs", "builtins.object"], "qualname": "CompileProjectInputs"}, +"sqltide.compiler.compile.models:CompileProjectInputs": {"fields": [["discovered_inputs", false, true, true], ["adapter_profile", false, true, true], ["effective_target", false, true, true], ["variables", false, true, true], ["macro_registry", false, true, true], ["macro_context", false, true, true], ["pipelines", false, true, true], ["tests", false, true, true], ["audits", false, true, true], ["sources", true, true, true], ["virtual_environments", true, true, true], ["project_name", true, true, true], ["target_name", true, true, true], ["model_reference_scope", true, true, true], ["allowed_cross_pipeline_references", true, true, true], ["consumer_group_identity", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["discovered_inputs", "adapter_profile", "effective_target", "variables", "macro_registry", "macro_context", "pipelines", "tests", "audits", "sources", "virtual_environments", "project_name", "target_name", "model_reference_scope", "allowed_cross_pipeline_references", "consumer_group_identity"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompileProjectInputs", "builtins.object"], "qualname": "CompileProjectInputs"}, "sqltide.compiler.compile.models:CompiledModel": {"fields": [["key", false, true, true], ["pipeline_name", false, true, true], ["relation_name", false, true, true], ["kind", false, true, true], ["sql_analysis", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["key", "pipeline_name", "relation_name", "kind", "sql_analysis"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompiledModel", "builtins.object"], "qualname": "CompiledModel"}, "sqltide.compiler.compile.models:CompiledPipeline": {"fields": [["pipeline", false, true, true], ["project", false, true, true], ["file_path", false, true, true], ["effective_replay_lineage_mode", false, true, true], ["source", false, true, true], ["models", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["pipeline", "project", "file_path", "effective_replay_lineage_mode", "source", "models"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompiledPipeline", "builtins.object"], "qualname": "CompiledPipeline"}, -"sqltide.compiler.compile.models:CompiledProject": {"fields": [["sources", false, true, true], ["models", false, true, true], ["pipelines", false, true, true], ["tests", false, true, true], ["test_cases", false, true, true], ["audits", false, true, true], ["macro_registry", true, true, true], ["macro_context", true, true, true], ["project_name", true, true, true], ["target_name", true, true, true], ["production_target", true, true, true], ["destruction_relation_drop_size_limit", true, true, true], ["model_reference_scope", true, true, true], ["allowed_cross_pipeline_references", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["sources", "models", "pipelines", "tests", "test_cases", "audits", "macro_registry", "macro_context", "project_name", "target_name", "production_target", "destruction_relation_drop_size_limit", "model_reference_scope", "allowed_cross_pipeline_references"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompiledProject", "builtins.object"], "qualname": "CompiledProject"}, +"sqltide.compiler.compile.models:CompiledProject": {"fields": [["sources", false, true, true], ["models", false, true, true], ["pipelines", false, true, true], ["tests", false, true, true], ["test_cases", false, true, true], ["audits", false, true, true], ["macro_registry", true, true, true], ["macro_context", true, true, true], ["project_name", true, true, true], ["target_name", true, true, true], ["production_target", true, true, true], ["destruction_relation_drop_size_limit", true, true, true], ["model_reference_scope", true, true, true], ["allowed_cross_pipeline_references", true, true, true], ["consumer_group_identity", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["sources", "models", "pipelines", "tests", "test_cases", "audits", "macro_registry", "macro_context", "project_name", "target_name", "production_target", "destruction_relation_drop_size_limit", "model_reference_scope", "allowed_cross_pipeline_references", "consumer_group_identity"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompiledProject", "builtins.object"], "qualname": "CompiledProject"}, "sqltide.compiler.compile.models:CompiledSource": {"fields": [["key", false, true, true], ["source", false, true, true], ["effective_replay_lineage_mode", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["key", "source", "effective_replay_lineage_mode"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompiledSource", "builtins.object"], "qualname": "CompiledSource"}, "sqltide.compiler.compile.models:CompiledTableModel": {"fields": [["key", false, true, true], ["pipeline_name", false, true, true], ["relation_name", false, true, true], ["kind", false, true, true], ["sql_analysis", false, true, true], ["transform", false, true, true], ["preserves_required_lineage", false, true, true], ["replay_anchor_eligible", false, true, true], ["effective_bounded_replay_fallback", false, true, true], ["replay_on_change", true, true, true], ["retention", true, true, true], ["retention_applied", true, true, true], ["on_row_error", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["key", "pipeline_name", "relation_name", "kind", "sql_analysis", "transform", "preserves_required_lineage", "replay_anchor_eligible", "effective_bounded_replay_fallback", "replay_on_change", "retention", "retention_applied", "on_row_error"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompiledTableModel", "sqltide.compiler.compile.models.CompiledModel", "builtins.object"], "qualname": "CompiledTableModel"}, "sqltide.compiler.compile.models:CompiledViewModel": {"fields": [["key", false, true, true], ["pipeline_name", false, true, true], ["relation_name", false, true, true], ["kind", false, true, true], ["sql_analysis", false, true, true], ["view", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["key", "pipeline_name", "relation_name", "kind", "sql_analysis", "view"], "module": "sqltide.compiler.compile.models", "mro": ["sqltide.compiler.compile.models.CompiledViewModel", "sqltide.compiler.compile.models.CompiledModel", "builtins.object"], "qualname": "CompiledViewModel"}, @@ -364,7 +364,7 @@ "sqltide.compiler.discovery.models:AuditDefaults": {"fields": [["severity", true, true, true], ["cadence_seconds", true, true, true], ["warmup_seconds", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["severity", "cadence_seconds", "warmup_seconds"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.AuditDefaults", "builtins.object"], "qualname": "AuditDefaults"}, "sqltide.compiler.discovery.models:AuditSchedulerConfig": {"fields": [["enabled", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["enabled"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.AuditSchedulerConfig", "builtins.object"], "qualname": "AuditSchedulerConfig"}, "sqltide.compiler.discovery.models:AuditSchedulerOverride": {"fields": [["enabled", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["enabled"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.AuditSchedulerOverride", "builtins.object"], "qualname": "AuditSchedulerOverride"}, -"sqltide.compiler.discovery.models:AuthoredProjectConfig": {"fields": [["name", false, true, true], ["adapter", false, true, true], ["default_target", false, true, true], ["connection", false, true, true], ["variables", false, true, true], ["targets", false, true, true], ["defaults", true, true, true], ["naming", true, true, true], ["audit_scheduler", true, true, true], ["sensors", true, true, true], ["build", true, true, true], ["ui", true, true, true], ["dependencies", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["name", "adapter", "default_target", "connection", "variables", "targets", "defaults", "naming", "audit_scheduler", "sensors", "build", "ui", "dependencies"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.AuthoredProjectConfig", "builtins.object"], "qualname": "AuthoredProjectConfig"}, +"sqltide.compiler.discovery.models:AuthoredProjectConfig": {"fields": [["name", false, true, true], ["adapter", false, true, true], ["default_target", false, true, true], ["connection", false, true, true], ["variables", false, true, true], ["targets", false, true, true], ["defaults", true, true, true], ["naming", true, true, true], ["audit_scheduler", true, true, true], ["sensors", true, true, true], ["build", true, true, true], ["ui", true, true, true], ["dependencies", true, true, true], ["consumer_group_identity", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["name", "adapter", "default_target", "connection", "variables", "targets", "defaults", "naming", "audit_scheduler", "sensors", "build", "ui", "dependencies", "consumer_group_identity"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.AuthoredProjectConfig", "builtins.object"], "qualname": "AuthoredProjectConfig"}, "sqltide.compiler.discovery.models:BuildConfig": {"fields": [["max_pipelines", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["max_pipelines"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.BuildConfig", "builtins.object"], "qualname": "BuildConfig"}, "sqltide.compiler.discovery.models:DeploymentReadinessDefaults": {"fields": [["maximum_lag_seconds", true, true, true], ["minimum_staged_row_ratio", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["maximum_lag_seconds", "minimum_staged_row_ratio"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.DeploymentReadinessDefaults", "builtins.object"], "qualname": "DeploymentReadinessDefaults"}, "sqltide.compiler.discovery.models:DestructionConfig": {"fields": [["max_table_size_to_drop_bytes", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["max_table_size_to_drop_bytes"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.DestructionConfig", "builtins.object"], "qualname": "DestructionConfig"}, @@ -372,7 +372,7 @@ "sqltide.compiler.discovery.models:DiscoveredProjectFile": {"fields": [["file_path", false, true, true], ["relative_path", false, true, true], ["contents", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["file_path", "relative_path", "contents"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.DiscoveredProjectFile", "builtins.object"], "qualname": "DiscoveredProjectFile"}, "sqltide.compiler.discovery.models:DiscoveredProjectInputs": {"fields": [["project_dir", false, true, true], ["loaded_project", false, true, true], ["source_files", false, true, true], ["pipeline_directories", false, true, true], ["model_files", false, true, true], ["test_files", false, true, true], ["audit_files", false, true, true], ["macro_files", false, true, true], ["access_file", true, true, true], ["constant_files", true, true, true], ["enum_files", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["project_dir", "loaded_project", "source_files", "pipeline_directories", "model_files", "test_files", "audit_files", "macro_files", "access_file", "constant_files", "enum_files"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.DiscoveredProjectInputs", "builtins.object"], "qualname": "DiscoveredProjectInputs"}, "sqltide.compiler.discovery.models:DiscoveredSourceFile": {"fields": [["source_file", false, true, true], ["sources", false, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["source_file", "sources"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.DiscoveredSourceFile", "builtins.object"], "qualname": "DiscoveredSourceFile"}, -"sqltide.compiler.discovery.models:EffectiveProjectConfiguration": {"fields": [["name", false, true, true], ["adapter", false, true, true], ["target_name", false, true, true], ["database", false, true, true], ["connection", false, true, true], ["variables", false, true, true], ["defaults", false, true, true], ["naming", true, true, true], ["audit_scheduler", true, true, true], ["sensors", true, true, true], ["build", true, true, true], ["destruction", true, true, true], ["ui", true, true, true], ["production_target", true, true, true], ["dependencies", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["name", "adapter", "target_name", "database", "connection", "variables", "defaults", "naming", "audit_scheduler", "sensors", "build", "destruction", "ui", "production_target", "dependencies"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.EffectiveProjectConfiguration", "builtins.object"], "qualname": "EffectiveProjectConfiguration"}, +"sqltide.compiler.discovery.models:EffectiveProjectConfiguration": {"fields": [["name", false, true, true], ["adapter", false, true, true], ["target_name", false, true, true], ["database", false, true, true], ["connection", false, true, true], ["variables", false, true, true], ["defaults", false, true, true], ["naming", true, true, true], ["audit_scheduler", true, true, true], ["sensors", true, true, true], ["build", true, true, true], ["destruction", true, true, true], ["ui", true, true, true], ["production_target", true, true, true], ["dependencies", true, true, true], ["consumer_group_identity", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["name", "adapter", "target_name", "database", "connection", "variables", "defaults", "naming", "audit_scheduler", "sensors", "build", "destruction", "ui", "production_target", "dependencies", "consumer_group_identity"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.EffectiveProjectConfiguration", "builtins.object"], "qualname": "EffectiveProjectConfiguration"}, "sqltide.compiler.discovery.models:ExecutionSettings": {"fields": [["replay", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["replay"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.ExecutionSettings", "builtins.object"], "qualname": "ExecutionSettings"}, "sqltide.compiler.discovery.models:ExternalTableSourceStep": {"fields": [["name", false, true, true], ["kind", false, true, true], ["table_name", false, true, true], ["replay_boundary", false, true, true], ["freshness", true, true, true], ["name_origin", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["name", "kind", "table_name", "replay_boundary", "freshness", "name_origin"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.ExternalTableSourceStep", "builtins.object"], "qualname": "ExternalTableSourceStep"}, "sqltide.compiler.discovery.models:KafkaLandingStep": {"fields": [["name", false, true, true], ["kafka", false, true, true], ["replay_boundary", true, true, true], ["freshness", true, true, true], ["name_origin", true, true, true], ["naming_macro", true, true, true], ["naming_macro_fingerprint", true, true, true]], "frozen": true, "hashable": true, "kind": "dataclass", "match_args": ["name", "kafka", "replay_boundary", "freshness", "name_origin", "naming_macro", "naming_macro_fingerprint"], "module": "sqltide.compiler.discovery.models", "mro": ["sqltide.compiler.discovery.models.KafkaLandingStep", "builtins.object"], "qualname": "KafkaLandingStep"}, diff --git a/src/sqltide/_native.pyi b/src/sqltide/_native.pyi index 79910231..6c3038ea 100644 --- a/src/sqltide/_native.pyi +++ b/src/sqltide/_native.pyi @@ -850,6 +850,7 @@ class CompileProjectInputs: target_name: str | None = None model_reference_scope: ModelReferenceScope | str = ModelReferenceScope.PROJECT allowed_cross_pipeline_references: tuple[AllowedCrossPipelineReference, ...] = () + consumer_group_identity: str | None = None @final @dataclass(frozen=True) @@ -868,6 +869,7 @@ class CompiledProject: destruction_relation_drop_size_limit: int | None = None model_reference_scope: ModelReferenceScope | str = ModelReferenceScope.PROJECT allowed_cross_pipeline_references: tuple[AllowedCrossPipelineReference, ...] = () + consumer_group_identity: str | None = None @final @dataclass(frozen=True) @@ -2411,6 +2413,7 @@ class CliCommand(StrEnum): DEPLOYMENT = "deployment" AUDIT = "audit" RECONCILE = "reconcile" + RENAME_PROJECT = "rename-project" JANITOR = "janitor" DOCTOR = "doctor" REPAIR = "repair" @@ -3413,6 +3416,7 @@ class AuthoredProjectConfig: build: BuildConfig = ... ui: UiConfig = ... dependencies: ProjectDependencies = ... + consumer_group_identity: str | None = None @final @dataclass(frozen=True, repr=False) @@ -3450,6 +3454,7 @@ class EffectiveProjectConfiguration: ui: UiConfig = ... production_target: bool = False dependencies: ProjectDependencies = ... + consumer_group_identity: str | None = None @final @dataclass(frozen=True, repr=False) @@ -3753,6 +3758,7 @@ class AdapterManagedSourceRealizationRequest: ttl: str | None = None settings: tuple[tuple[str, str], ...] = () naming_macro_fingerprint: str | None = None + consumer_group_identity: str | None = None @final @dataclass(frozen=True) @@ -7163,6 +7169,20 @@ def run_deployment_promotion( json_output: bool, client: AdapterConnection, ) -> int: ... +def run_rename_project( + *, + pipelines_root: Path, + project_dir: Path, + new_name: str, + database: str | None, + control_store_url: str | None, + apply: bool, + client: AdapterConnection, + loaded_project: LoadedProject | None, + adapter_profile: CompilerAdapterProfile, + from_name: str | None = None, + environment: Mapping[str, str] | None = None, +) -> int: ... def run_reconcile( *, pipelines_root: Path, diff --git a/tests/integration/src/sqltide/cli/rename_project/__init__.py b/tests/integration/src/sqltide/cli/rename_project/__init__.py new file mode 100644 index 00000000..609b6983 --- /dev/null +++ b/tests/integration/src/sqltide/cli/rename_project/__init__.py @@ -0,0 +1 @@ +"""Project rename integration checks.""" diff --git a/tests/integration/src/sqltide/cli/rename_project/main/__init__.py b/tests/integration/src/sqltide/cli/rename_project/main/__init__.py new file mode 100644 index 00000000..3cedb5b2 --- /dev/null +++ b/tests/integration/src/sqltide/cli/rename_project/main/__init__.py @@ -0,0 +1 @@ +"""Project rename entrypoint integration tests.""" diff --git a/tests/integration/src/sqltide/cli/rename_project/main/_test_types.py b/tests/integration/src/sqltide/cli/rename_project/main/_test_types.py new file mode 100644 index 00000000..0787e853 --- /dev/null +++ b/tests/integration/src/sqltide/cli/rename_project/main/_test_types.py @@ -0,0 +1,57 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ProjectRenameTestCase: + description: str + old_name: str + new_name: str + expected_config: str + + +@dataclass(frozen=True) +class ProjectRenameConflictTestCase: + description: str + new_name: str + conflicting_manifest_id: str + expected_error_fragment: str + + +@dataclass(frozen=True) +class RoleAssignmentRenameTestCase: + description: str + new_name: str + role_name: str + expected_moved_roles: tuple[str, ...] + + +@dataclass(frozen=True) +class RoleAssignmentConflictTestCase: + description: str + new_name: str + role_name: str + expected_error_fragment: str + + +@dataclass(frozen=True) +class ConsumerGroupDriftTestCase: + description: str + renamed_config: str + expected_live_group_prefix: str + expected_declared_group_prefix: str + + +@dataclass(frozen=True) +class MultiTargetRenameTestCase: + description: str + new_name: str + role_name: str + expected_config: str + + +@dataclass(frozen=True) +class ForeignHistoryTestCase: + description: str + new_name: str + foreign_name: str + expected_error_fragments: tuple[str, ...] diff --git a/tests/integration/src/sqltide/cli/rename_project/main/helpers.py b/tests/integration/src/sqltide/cli/rename_project/main/helpers.py new file mode 100644 index 00000000..8e89b1f8 --- /dev/null +++ b/tests/integration/src/sqltide/cli/rename_project/main/helpers.py @@ -0,0 +1,212 @@ +"""Build one managed Kafka project and run `sqt` against a real ClickHouse target.""" + +from pathlib import Path +from uuid import UUID + +import pytest +from clickhouse_connect.driver.client import Client + +from sqltide._native import ControlStore, run_cli +from sqltide.adapter.models import AdapterManifestSnapshot +from tests.integration.src.sqltide.cli.helpers import build_managed_clickhouse_client +from tests.integration.src.sqltide.conftest import ClickHouseConnectionSettings + +PROJECT_CONFIG: str = ( + "# Shop analytics project.\n" + 'name = "shop" # project identity\n' + 'default_target = "test"\n' + "\n" + "[defaults]\n" + 'pipeline_mode = "direct"\n' + "\n" + "[targets.test]\n" + "# Overridden with --database in tests.\n" + 'database = "analytics"\n' +) +TWO_TARGET_PROJECT_CONFIG: str = ( + 'name = "shop"\n' + 'default_target = "east"\n' + "\n" + "[defaults]\n" + 'pipeline_mode = "direct"\n' + "\n" + "[targets.east]\n" + 'database = "shop_east"\n' + "\n" + "[targets.west]\n" + 'database = "shop_west"\n' +) +SOURCE_CONFIG: str = ( + "sources:\n" + " - name: orders\n" + " kind: kafka\n" + " broker_list: kafka.example.com:9092\n" + " topic: shop.orders\n" + " replay_boundary:\n" + " mode: offsets\n" +) +MODEL_SQL: str = ( + "MODEL (\n" + ' engine "MergeTree",\n' + ' order_by ["order_id"],\n' + ");\n" + "\n" + "SELECT\n" + " CAST(JSONExtractString(kafka_value, 'order_id') AS String) AS order_id,\n" + " CAST(_replay_landed_at AS DateTime64(3)) AS _replay_landed_at\n" + 'FROM __ref("orders")\n' +) + + +def write_shop_project(project_dir: Path, project_config: str = PROJECT_CONFIG) -> Path: + """Write a direct-mode project with one managed Kafka source and one model.""" + project_dir.mkdir(parents=True, exist_ok=True) + (project_dir / "sqltide_project.toml").write_text(project_config, encoding="utf-8") + (project_dir / "sources").mkdir() + (project_dir / "sources" / "orders.yml").write_text(SOURCE_CONFIG, encoding="utf-8") + pipeline_dir: Path = project_dir / "pipelines" / "pl__orders" + pipeline_dir.mkdir(parents=True) + (pipeline_dir / "orders_clean.sql").write_text(MODEL_SQL, encoding="utf-8") + return project_dir + + +def write_project( + project_dir: Path, *, name: str, source: str, consumer_group: str, database: str +) -> Path: + """Write a direct-mode project whose one Kafka source declares its own consumer group.""" + project_dir.mkdir(parents=True, exist_ok=True) + (project_dir / "sqltide_project.toml").write_text( + f'name = "{name}"\ndefault_target = "test"\n\n[defaults]\npipeline_mode = "direct"\n\n' + f'[targets.test]\ndatabase = "{database}"\n', + encoding="utf-8", + ) + (project_dir / "sources").mkdir() + (project_dir / "sources" / f"{source}.yml").write_text( + SOURCE_CONFIG.replace("name: orders", f"name: {source}") + .replace("shop.orders", f"{name}.{source}") + .replace(" topic:", f" consumer_group: {consumer_group}\n topic:"), + encoding="utf-8", + ) + pipeline_dir: Path = project_dir / "pipelines" / f"pl__{source}" + pipeline_dir.mkdir(parents=True) + (pipeline_dir / f"{source}_clean.sql").write_text( + MODEL_SQL.replace('__ref("orders")', f'__ref("{source}")'), encoding="utf-8" + ) + return project_dir + + +def connection_arguments( + *, settings: ClickHouseConnectionSettings, project_dir: Path, database: str +) -> tuple[str, ...]: + return ( + "--project-dir", + str(project_dir), + "--host", + settings.host, + "--port", + str(settings.port), + "--username", + settings.username, + "--password", + settings.password, + "--database", + database, + ) + + +def built_shop_project( + *, + tmp_path: Path, + settings: ClickHouseConnectionSettings, + database: str, + monkeypatch: pytest.MonkeyPatch, +) -> tuple[Path, tuple[str, ...]]: + """A built shop project and the connection arguments every command takes.""" + monkeypatch.delenv("SQLTIDE_CONTROL_STORE_URL", raising=False) + project_dir: Path = write_shop_project(tmp_path / "shop") + arguments: tuple[str, ...] = connection_arguments( + settings=settings, project_dir=project_dir, database=database + ) + assert run_cli(("sqt", "build", "--auto-approve", *arguments)) == 0 + return project_dir, arguments + + +def plan_json(arguments: tuple[str, ...], capsys: pytest.CaptureFixture[str]) -> str: + """The `sqt plan --json` output for the project.""" + _ = capsys.readouterr() + assert run_cli(("sqt", "plan", "--json", *arguments)) == 0 + return capsys.readouterr().out + + +def target_arguments( + *, + settings: ClickHouseConnectionSettings, + project_dir: Path, + target: str, + database: str, +) -> tuple[str, ...]: + """Connection arguments for one target with its own database.""" + return ( + *connection_arguments(settings=settings, project_dir=project_dir, database=database), + "--target", + target, + ) + + +def destroy_history_ids( + *, settings: ClickHouseConnectionSettings, database: str, identity: str, target: str +) -> tuple[str, ...]: + """The manifest ids destroy planning reads for one project target.""" + snapshot: AdapterManifestSnapshot = build_managed_clickhouse_client( + settings, database=database + ).load_manifests( + database=database, + project_identity=identity, + target_name=target, + target_database=database, + ) + return tuple(sorted(manifest.manifest_id for manifest in snapshot.manifests)) + + +def role_names(store: ControlStore, *, user_id: UUID, project_name: str) -> tuple[str, ...]: + return tuple( + assignment.role_name + for assignment in store.list_project_role_assignments( + user_id=user_id, project_name=project_name + ) + ) + + +def manifest_ids(client: Client, *, database: str, identity: str) -> tuple[str, ...]: + return tuple( + str(row[0]) + for row in client.query( + f"SELECT manifest_id FROM {database}._sqltide_manifests " + "WHERE project_identity = {identity:String} ORDER BY manifest_id", + parameters={"identity": identity}, + ).result_rows + ) + + +def kafka_group_name(client: Client, *, database: str) -> str: + """The consumer group the live Kafka table for `orders` consumes as.""" + definition: str = str( + client.query( + "SELECT create_table_query FROM system.tables " + "WHERE database = {database:String} AND name = 'kafka__orders'", + parameters={"database": database}, + ).result_rows[0][0] + ) + return definition.split("kafka_group_name = '", 1)[1].split("'", 1)[0] + + +def catalog_definitions(client: Client, *, database: str) -> tuple[tuple[str, str], ...]: + return tuple( + (str(row[0]), str(row[1])) + for row in client.query( + "SELECT name, create_table_query FROM system.tables " + "WHERE database = {database:String} AND NOT startsWith(name, '_sqltide') " + "ORDER BY name", + parameters={"database": database}, + ).result_rows + ) diff --git a/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py b/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py new file mode 100644 index 00000000..36b1f47a --- /dev/null +++ b/tests/integration/src/sqltide/cli/rename_project/main/test_rename_project.py @@ -0,0 +1,706 @@ +import json +from collections.abc import Mapping +from pathlib import Path +from typing import cast +from uuid import UUID + +import pytest +from clickhouse_connect.driver.client import Client + +import sqltide._native as _native +from sqltide._native import ( + ControlStore, + analyze_project, + build_compiler_adapter_profile, + build_state_payload, + run_cli, +) +from sqltide.adapter.classes.adapter_connection import AdapterConnection +from sqltide.adapter.models import AdapterManifestSnapshot +from sqltide.adapters.clickhouse.classes.clickhouse_adapter import ClickHouseAdapter +from sqltide.auth.models import ProjectRoleAssignment +from tests.integration.src.sqltide.cli.helpers import build_managed_clickhouse_client +from tests.integration.src.sqltide.cli.rename_project.main._test_types import ( + ConsumerGroupDriftTestCase, + ForeignHistoryTestCase, + MultiTargetRenameTestCase, + ProjectRenameConflictTestCase, + ProjectRenameTestCase, + RoleAssignmentConflictTestCase, + RoleAssignmentRenameTestCase, +) +from tests.integration.src.sqltide.cli.rename_project.main.helpers import ( + PROJECT_CONFIG, + TWO_TARGET_PROJECT_CONFIG, + built_shop_project, + catalog_definitions, + connection_arguments, + destroy_history_ids, + kafka_group_name, + manifest_ids, + plan_json, + role_names, + target_arguments, + write_project, + write_shop_project, +) +from tests.integration.src.sqltide.conftest import ClickHouseConnectionSettings + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_case", + [ + ProjectRenameTestCase( + description="keeps groups, plan and manifest history; repeats are no-ops", + old_name="shop", + new_name="storefront", + expected_config=PROJECT_CONFIG.replace( + 'name = "shop" # project identity\n', + 'name = "storefront" # project identity\n' + "# Keeps the Kafka consumer groups this project used before it was renamed.\n" + 'consumer_group_identity = "shop"\n', + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_built_project_when_renaming_then_live_state_carries_over( + test_case: ProjectRenameTestCase, + tmp_path: Path, + clickhouse_connection_settings: ClickHouseConnectionSettings, + clickhouse_client: Client, + clickhouse_database: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir, arguments = built_shop_project( + tmp_path=tmp_path, + settings=clickhouse_connection_settings, + database=clickhouse_database, + monkeypatch=monkeypatch, + ) + config_path: Path = project_dir / "sqltide_project.toml" + plan_before: str = plan_json(arguments, capsys) + catalog_before: tuple[tuple[str, str], ...] = catalog_definitions( + clickhouse_client, database=clickhouse_database + ) + group_before: str = kafka_group_name(clickhouse_client, database=clickhouse_database) + recorded: tuple[str, ...] = manifest_ids( + clickhouse_client, database=clickhouse_database, identity=test_case.old_name + ) + + assert run_cli(("sqt", "rename-project", test_case.new_name, *arguments)) == 0 + dry_run: str = capsys.readouterr().out + assert config_path.read_text(encoding="utf-8") == PROJECT_CONFIG + assert ( + manifest_ids(clickhouse_client, database=clickhouse_database, identity=test_case.new_name) + == () + ) + assert "Kafka consumer groups: 1 unchanged" in dry_run + assert f"{len(recorded)} of {len(recorded)} manifest(s) to copy" in dry_run + assert "Dry run: nothing was changed." in dry_run + + assert run_cli(("sqt", "rename-project", test_case.new_name, "--apply", *arguments)) == 0 + renamed_config: str = config_path.read_text(encoding="utf-8") + assert renamed_config == test_case.expected_config + assert kafka_group_name(clickhouse_client, database=clickhouse_database) == group_before + assert catalog_definitions(clickhouse_client, database=clickhouse_database) == catalog_before + assert plan_json(arguments, capsys) == plan_before + connection: AdapterConnection = build_managed_clickhouse_client( + clickhouse_connection_settings, database=clickhouse_database + ) + destroy_history: AdapterManifestSnapshot = connection.load_manifests( + database=clickhouse_database, + project_identity=test_case.new_name, + target_name="test", + target_database=clickhouse_database, + ) + assert tuple(sorted(manifest.manifest_id for manifest in destroy_history.manifests)) == ( + recorded + ) + + assert run_cli(("sqt", "rename-project", test_case.new_name, "--apply", *arguments)) == 1 + assert "already named" in capsys.readouterr().err + config_path.write_text(PROJECT_CONFIG, encoding="utf-8") + assert run_cli(("sqt", "rename-project", test_case.new_name, "--apply", *arguments)) == 0 + assert f"0 of {len(recorded)} manifest(s) copied" in capsys.readouterr().out + assert config_path.read_text(encoding="utf-8") == renamed_config + assert ( + manifest_ids(clickhouse_client, database=clickhouse_database, identity=test_case.new_name) + == recorded + ) + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_case", + [ + ProjectRenameConflictTestCase( + description="refuses when the new name already has its own manifest history", + new_name="storefront", + conflicting_manifest_id="manifest-from-another-project", + expected_error_fragment="another project already uses that name here", + ) + ], + ids=lambda case: case.description, +) +def test_given_new_name_with_foreign_manifests_when_renaming_then_rename_is_refused( + test_case: ProjectRenameConflictTestCase, + tmp_path: Path, + clickhouse_connection_settings: ClickHouseConnectionSettings, + clickhouse_client: Client, + clickhouse_database: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir, arguments = built_shop_project( + tmp_path=tmp_path, + settings=clickhouse_connection_settings, + database=clickhouse_database, + monkeypatch=monkeypatch, + ) + clickhouse_client.command( + f"INSERT INTO {clickhouse_database}._sqltide_manifests " + "(manifest_id, project_identity, target_name, target_database, manifest_version, " + "published_at) VALUES ({id:String}, {identity:String}, 'test', {database:String}, 1, " + "now64(6))", + parameters={ + "id": test_case.conflicting_manifest_id, + "identity": test_case.new_name, + "database": clickhouse_database, + }, + ) + _ = capsys.readouterr() + + assert run_cli(("sqt", "rename-project", test_case.new_name, "--apply", *arguments)) == 1 + + assert test_case.expected_error_fragment in capsys.readouterr().err + assert (project_dir / "sqltide_project.toml").read_text(encoding="utf-8") == PROJECT_CONFIG + assert manifest_ids( + clickhouse_client, database=clickhouse_database, identity=test_case.new_name + ) == (test_case.conflicting_manifest_id,) + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_case", + [ + RoleAssignmentRenameTestCase( + description="moves project role assignments to the new name", + new_name="storefront", + role_name="operator", + expected_moved_roles=("operator",), + ) + ], + ids=lambda case: case.description, +) +def test_given_project_role_grants_when_renaming_then_grants_follow_the_project( + test_case: RoleAssignmentRenameTestCase, + tmp_path: Path, + clickhouse_connection_settings: ClickHouseConnectionSettings, + clickhouse_database: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _, arguments = built_shop_project( + tmp_path=tmp_path, + settings=clickhouse_connection_settings, + database=clickhouse_database, + monkeypatch=monkeypatch, + ) + url: str = f"sqlite:///{tmp_path / 'control.db'}" + store: ControlStore = ControlStore(url=url) + user_id: UUID = store.create_user(username="ada").user_id + store.grant_project_role( + user_id=user_id, + project_name="shop", + role_name=test_case.role_name, + target_name=None, + actor_user_id=None, + ) + + assert ( + run_cli( + ( + "sqt", + "rename-project", + test_case.new_name, + "--apply", + "--control-store-url", + url, + *arguments, + ) + ) + == 0 + ) + + moved: tuple[ProjectRoleAssignment, ...] = store.list_project_role_assignments( + user_id=user_id, project_name=test_case.new_name + ) + assert tuple(assignment.role_name for assignment in moved) == test_case.expected_moved_roles + assert store.list_project_role_assignments(user_id=user_id, project_name="shop") == () + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_case", + [ + RoleAssignmentConflictTestCase( + description="refuses when both names hold the same active role", + new_name="storefront", + role_name="operator", + expected_error_fragment="already holds role assignments", + ) + ], + ids=lambda case: case.description, +) +def test_given_conflicting_role_grants_when_renaming_then_rename_is_refused( + test_case: RoleAssignmentConflictTestCase, + tmp_path: Path, + clickhouse_connection_settings: ClickHouseConnectionSettings, + clickhouse_client: Client, + clickhouse_database: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir, arguments = built_shop_project( + tmp_path=tmp_path, + settings=clickhouse_connection_settings, + database=clickhouse_database, + monkeypatch=monkeypatch, + ) + url: str = f"sqlite:///{tmp_path / 'control.db'}" + store: ControlStore = ControlStore(url=url) + user_id: UUID = store.create_user(username="ada").user_id + for project_name in ("shop", test_case.new_name): + store.grant_project_role( + user_id=user_id, + project_name=project_name, + role_name=test_case.role_name, + target_name=None, + actor_user_id=None, + ) + _ = capsys.readouterr() + + assert ( + run_cli( + ( + "sqt", + "rename-project", + test_case.new_name, + "--apply", + "--control-store-url", + url, + *arguments, + ) + ) + == 1 + ) + + assert test_case.expected_error_fragment in capsys.readouterr().err + assert (project_dir / "sqltide_project.toml").read_text(encoding="utf-8") == PROJECT_CONFIG + assert ( + manifest_ids(clickhouse_client, database=clickhouse_database, identity=test_case.new_name) + == () + ) + assert len(store.list_project_role_assignments(user_id=user_id, project_name="shop")) == 1 + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_case", + [ + ConsumerGroupDriftTestCase( + description="plan and source health warn when the project name changes by hand", + renamed_config=PROJECT_CONFIG.replace('name = "shop"', 'name = "storefront"'), + expected_live_group_prefix="streambuild_shop_test_orders_", + expected_declared_group_prefix="streambuild_storefront_test_orders_", + ) + ], + ids=lambda case: case.description, +) +def test_given_live_group_differs_when_planning_then_drift_is_reported_without_blocking( + test_case: ConsumerGroupDriftTestCase, + tmp_path: Path, + clickhouse_connection_settings: ClickHouseConnectionSettings, + clickhouse_client: Client, + clickhouse_database: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir, arguments = built_shop_project( + tmp_path=tmp_path, + settings=clickhouse_connection_settings, + database=clickhouse_database, + monkeypatch=monkeypatch, + ) + (project_dir / "sqltide_project.toml").write_text(test_case.renamed_config, encoding="utf-8") + live_group: str = f"{test_case.expected_live_group_prefix}{clickhouse_database}" + declared_group: str = f"{test_case.expected_declared_group_prefix}{clickhouse_database}" + _ = capsys.readouterr() + + assert run_cli(("sqt", "plan", *arguments)) == 0 + plan_text: str = capsys.readouterr().out + warnings: list[dict[str, str]] = json.loads(plan_json(arguments, capsys))[ + "consumer_group_warnings" + ] + connection: AdapterConnection = build_managed_clickhouse_client( + clickhouse_connection_settings, database=clickhouse_database + ) + payload: dict[str, object] = build_state_payload( + analysis=analyze_project( + pipelines_root=project_dir / "pipelines", + loaded_project=_native.load_project_input_for_path(path=project_dir), + adapter_profile=build_compiler_adapter_profile(ClickHouseAdapter()), + ), + connection=connection, + database=clickhouse_database, + ) + + assert ( + f"consumes as Kafka consumer group '{live_group}' but the project declares " + f"'{declared_group}'" + ) in plan_text + assert "would not carry over" in plan_text + assert [ + (warning["live_consumer_group"], warning["declared_consumer_group"]) for warning in warnings + ] == [(live_group, declared_group)] + sources: Mapping[str, Mapping[str, Mapping[str, str]]] = cast( + Mapping[str, Mapping[str, Mapping[str, str]]], payload["sources"] + ) + assert sources["orders"]["consumerGroupDrift"]["liveGroup"] == live_group + assert kafka_group_name(clickhouse_client, database=clickhouse_database) == live_group + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_case", + [ + MultiTargetRenameTestCase( + description="renames on one target, then completes with --from on the other", + new_name="storefront", + role_name="operator", + expected_config=TWO_TARGET_PROJECT_CONFIG.replace( + 'name = "shop"\n', + 'name = "storefront"\n' + "# Keeps the Kafka consumer groups this project used before it was renamed.\n" + 'consumer_group_identity = "shop"\n', + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_two_targets_when_completing_rename_with_from_then_each_target_carries_over( + test_case: MultiTargetRenameTestCase, + tmp_path: Path, + clickhouse_connection_settings: ClickHouseConnectionSettings, + clickhouse_client: Client, + clickhouse_database: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + request: pytest.FixtureRequest, +) -> None: + monkeypatch.delenv("SQLTIDE_CONTROL_STORE_URL", raising=False) + west_database: str = f"{clickhouse_database}_west" + request.addfinalizer( + lambda: clickhouse_client.command(f"DROP DATABASE IF EXISTS {west_database} SYNC") + ) + project_dir: Path = write_shop_project(tmp_path / "shop", TWO_TARGET_PROJECT_CONFIG) + config_path: Path = project_dir / "sqltide_project.toml" + databases: dict[str, str] = {"east": clickhouse_database, "west": west_database} + stores: dict[str, ControlStore] = { + target: ControlStore(url=f"sqlite:///{tmp_path / target / 'control.db'}") + for target in databases + } + build_arguments: dict[str, tuple[str, ...]] = { + target: target_arguments( + settings=clickhouse_connection_settings, + project_dir=project_dir, + target=target, + database=database, + ) + for target, database in databases.items() + } + arguments: dict[str, tuple[str, ...]] = { + target: (*build_arguments[target], "--control-store-url", str(stores[target].url)) + for target in databases + } + user_ids: dict[str, UUID] = { + target: store.create_user(username="ada").user_id for target, store in stores.items() + } + _ = [ + store.grant_project_role( + user_id=user_ids[target], + project_name="shop", + role_name=test_case.role_name, + target_name=target, + actor_user_id=None, + ) + for target, store in stores.items() + ] + assert [ + run_cli(("sqt", "build", "--auto-approve", *build_arguments[target])) + for target in databases + ] == [0, 0] + groups: dict[str, str] = { + target: kafka_group_name(clickhouse_client, database=database) + for target, database in databases.items() + } + recorded: dict[str, tuple[str, ...]] = { + target: manifest_ids(clickhouse_client, database=database, identity="shop") + for target, database in databases.items() + } + + assert ( + run_cli(("sqt", "rename-project", test_case.new_name, "--from", "shop", *arguments["west"])) + == 1 + ) + assert "sqltide_project.toml is still named 'shop'" in capsys.readouterr().err + assert ( + run_cli(("sqt", "rename-project", test_case.new_name, "--apply", *arguments["east"])) == 0 + ) + renamed_config: str = config_path.read_text(encoding="utf-8") + assert run_cli(("sqt", "rename-project", "outlet", "--from", "shop", *arguments["west"])) == 1 + assert "the new name must be the name sqltide_project.toml already carries" in ( + capsys.readouterr().err + ) + assert ( + run_cli( + ( + "sqt", + "rename-project", + test_case.new_name, + "--from", + test_case.new_name, + *arguments["west"], + ) + ) + == 1 + ) + assert "cannot be the new name 'storefront'" in capsys.readouterr().err + assert run_cli(("sqt", "build", "--auto-approve", *build_arguments["west"])) == 0 + built_under_new_name: tuple[str, ...] = manifest_ids( + clickhouse_client, database=west_database, identity=test_case.new_name + ) + assert ( + run_cli( + ("sqt", "rename-project", test_case.new_name, "--from", "shopp", *arguments["west"]) + ) + == 1 + ) + assert "Nothing on target 'west' records project 'shopp'" in capsys.readouterr().err + assert ( + run_cli(("sqt", "rename-project", test_case.new_name, "--from", "shop", *arguments["west"])) + == 0 + ) + dry_run: str = capsys.readouterr().out + assert ( + manifest_ids(clickhouse_client, database=west_database, identity=test_case.new_name) + == built_under_new_name + ) + assert ( + run_cli( + ( + "sqt", + "rename-project", + test_case.new_name, + "--from", + "shop", + "--apply", + *arguments["west"], + ) + ) + == 0 + ) + assert ( + run_cli( + ( + "sqt", + "rename-project", + test_case.new_name, + "--from", + "shop", + "--apply", + *arguments["west"], + ) + ) + == 0 + ) + + assert renamed_config == test_case.expected_config + assert config_path.read_text(encoding="utf-8") == renamed_config + assert 'already named "storefront"; unchanged' in dry_run + assert "Kafka consumer groups: 1 unchanged" in dry_run + assert { + target: kafka_group_name(clickhouse_client, database=database) + for target, database in databases.items() + } == groups + destroy_history: dict[str, tuple[str, ...]] = { + target: destroy_history_ids( + settings=clickhouse_connection_settings, + database=database, + identity=test_case.new_name, + target=target, + ) + for target, database in databases.items() + } + assert len(built_under_new_name) == 1 + assert destroy_history == { + "east": recorded["east"], + "west": tuple(sorted(recorded["west"] + built_under_new_name)), + } + assert { + target: role_names(store, user_id=user_ids[target], project_name=test_case.new_name) + for target, store in stores.items() + } == {target: (test_case.role_name,) for target in databases} + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_case", + [ + ConsumerGroupDriftTestCase( + description="refuses to rename while live groups differ from the declaration", + renamed_config=PROJECT_CONFIG.replace('name = "shop"', 'name = "storefront"'), + expected_live_group_prefix="streambuild_shop_test_orders_", + expected_declared_group_prefix="streambuild_storefront_test_orders_", + ) + ], + ids=lambda case: case.description, +) +def test_given_live_group_drift_when_renaming_then_rename_is_refused( + test_case: ConsumerGroupDriftTestCase, + tmp_path: Path, + clickhouse_connection_settings: ClickHouseConnectionSettings, + clickhouse_client: Client, + clickhouse_database: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir, arguments = built_shop_project( + tmp_path=tmp_path, + settings=clickhouse_connection_settings, + database=clickhouse_database, + monkeypatch=monkeypatch, + ) + config_path: Path = project_dir / "sqltide_project.toml" + config_path.write_text(test_case.renamed_config, encoding="utf-8") + _ = capsys.readouterr() + + assert run_cli(("sqt", "rename-project", "outlet", "--apply", *arguments)) == 1 + + error: str = capsys.readouterr().err + assert ( + f"kafka__orders consumes as '{test_case.expected_live_group_prefix}{clickhouse_database}' " + "but the project declares " + f"'{test_case.expected_declared_group_prefix}{clickhouse_database}'" + ) in error + assert "consumer group drift warning" in error + assert config_path.read_text(encoding="utf-8") == test_case.renamed_config + assert manifest_ids(clickhouse_client, database=clickhouse_database, identity="outlet") == () + + +@pytest.mark.integration +@pytest.mark.parametrize( + "test_case", + [ + ForeignHistoryTestCase( + description="refuses --from another project's name in the same database", + new_name="storefront", + foreign_name="billing", + expected_error_fragments=( + "The history of 'billing' on target 'test' does not look like this project's", + "pl__invoices", + 'consumer_group_identity = "shop", not "billing"', + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_other_project_in_database_when_completing_from_its_name_then_it_is_refused( + test_case: ForeignHistoryTestCase, + tmp_path: Path, + clickhouse_connection_settings: ClickHouseConnectionSettings, + clickhouse_client: Client, + clickhouse_database: str, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.delenv("SQLTIDE_CONTROL_STORE_URL", raising=False) + projects: dict[str, Path] = { + name: write_project( + tmp_path / name, + name=name, + source=source, + consumer_group=f"{name}-{source}", + database=clickhouse_database, + ) + for name, source in (("shop", "orders"), (test_case.foreign_name, "invoices")) + } + arguments: dict[str, tuple[str, ...]] = { + name: connection_arguments( + settings=clickhouse_connection_settings, + project_dir=project_dir, + database=clickhouse_database, + ) + for name, project_dir in projects.items() + } + assert [run_cli(("sqt", "build", "--auto-approve", *arguments[name])) for name in projects] == [ + 0, + 0, + ] + shop_history: tuple[str, ...] = manifest_ids( + clickhouse_client, database=clickhouse_database, identity="shop" + ) + config_path: Path = projects["shop"] / "sqltide_project.toml" + assert ( + run_cli(("sqt", "rename-project", test_case.new_name, "--apply", *arguments["shop"])) == 0 + ) + renamed_config: str = config_path.read_text(encoding="utf-8") + _ = capsys.readouterr() + + from_foreign: tuple[str, ...] = ( + "sqt", + "rename-project", + test_case.new_name, + "--from", + test_case.foreign_name, + "--apply", + *arguments["shop"], + ) + assert run_cli(from_foreign) == 1 + pipeline_error: str = capsys.readouterr().err + assert ( + run_cli( + ( + "sqt", + "rename-project", + test_case.new_name, + "--from", + "shop", + "--apply", + *arguments["shop"], + ) + ) + == 0 + ) + config_path.write_text( + renamed_config.replace( + f'name = "{test_case.new_name}"\n', + f'name = "{test_case.new_name}"\nconsumer_group_identity = "shop"\n', + ), + encoding="utf-8", + ) + _ = capsys.readouterr() + assert run_cli(from_foreign) == 1 + identity_error: str = capsys.readouterr().err + + assert "consumer_group_identity" not in renamed_config + assert test_case.expected_error_fragments[0] in pipeline_error + assert test_case.expected_error_fragments[1] in pipeline_error + assert test_case.expected_error_fragments[0] in identity_error + assert test_case.expected_error_fragments[2] in identity_error + assert ( + manifest_ids(clickhouse_client, database=clickhouse_database, identity=test_case.new_name) + == shop_history + ) diff --git a/tests/unit/src/sqltide/cli/test_help_baseline.py b/tests/unit/src/sqltide/cli/test_help_baseline.py index a2067c2d..ce7bd2bc 100644 --- a/tests/unit/src/sqltide/cli/test_help_baseline.py +++ b/tests/unit/src/sqltide/cli/test_help_baseline.py @@ -13,7 +13,7 @@ CliHelpBaselineTestCase( description="captures root help", argv=("--help",), - expected_sha256="b079994e78490e17030dd3f51fe5ec6b0549faab2c0f7030c5c9321c0c72f4bd", + expected_sha256="0f958a0704f3fcd2e684d9ac738c547d0785cc4be3aefe90d8291306b78de0ae", ), CliHelpBaselineTestCase( description="captures admin help", @@ -105,6 +105,11 @@ argv=("reconcile", "--help"), expected_sha256="fc01e63a3e0a2dff1d689e074284bacae7d43d68fa2043bd6c32ca3f3afb9652", ), + CliHelpBaselineTestCase( + description="captures rename-project help", + argv=("rename-project", "--help"), + expected_sha256="e5ed515f1e543aa5a750757a63811df4e532d6bc90c21d6ea7e50da214d80d84", + ), CliHelpBaselineTestCase( description="captures janitor help", argv=("janitor", "--help"), diff --git a/tests/unit/src/sqltide/compiler/discovery/_test_types.py b/tests/unit/src/sqltide/compiler/discovery/_test_types.py index 323771dc..c93a9589 100644 --- a/tests/unit/src/sqltide/compiler/discovery/_test_types.py +++ b/tests/unit/src/sqltide/compiler/discovery/_test_types.py @@ -350,6 +350,13 @@ class ProjectProductionTargetTestCase: expected_production_target: bool +@dataclass(frozen=True) +class ProjectConsumerGroupIdentityTestCase: + description: str + identity_toml: str + expected_identity: str | None + + @dataclass(frozen=True) class ModelStepErrorTestCase: description: str diff --git a/tests/unit/src/sqltide/compiler/discovery/test_project_configuration.py b/tests/unit/src/sqltide/compiler/discovery/test_project_configuration.py index 0618988e..875e4cf6 100644 --- a/tests/unit/src/sqltide/compiler/discovery/test_project_configuration.py +++ b/tests/unit/src/sqltide/compiler/discovery/test_project_configuration.py @@ -33,6 +33,7 @@ ProjectBuildLimitResolutionTestCase, ProjectConfigurationErrorTestCase, ProjectConnectionSettingsTestCase, + ProjectConsumerGroupIdentityTestCase, ProjectDependencyScopeTestCase, ProjectDeploymentReadinessDefaultsTestCase, ProjectDestructionLimitResolutionTestCase, @@ -158,6 +159,46 @@ def test_given_committed_production_target_when_resolving_then_classification_is assert effective.production_target is test_case.expected_production_target +@pytest.mark.parametrize( + "test_case", + [ + ProjectConsumerGroupIdentityTestCase( + description="leaves the consumer group identity unset by default", + identity_toml="", + expected_identity=None, + ), + ProjectConsumerGroupIdentityTestCase( + description="retains a committed consumer group identity", + identity_toml='consumer_group_identity = "shop"\n', + expected_identity="shop", + ), + ], + ids=lambda case: case.description, +) +def test_given_consumer_group_identity_when_resolving_then_it_reaches_effective_configuration( + test_case: ProjectConsumerGroupIdentityTestCase, + tmp_path: Path, +) -> None: + write_project_toml( + project_dir=tmp_path, + contents=( + f'name = "storefront"\n{test_case.identity_toml}default_target = "dev"\n' + '[targets.dev]\ndatabase = "storefront_dev"\n' + ), + ) + loaded: LoadedProjectConfiguration = load_project_configuration(project_dir=tmp_path) + + effective: EffectiveProjectConfiguration = resolve_effective_project_configuration( + loaded=loaded, + selected_target="dev", + cli_variables={}, + environment={}, + ) + + assert loaded.project.consumer_group_identity == test_case.expected_identity + assert effective.consumer_group_identity == test_case.expected_identity + + @pytest.mark.parametrize( "test_case", [ @@ -991,6 +1032,22 @@ def test_given_absent_local_toml_when_loading_then_returns_typed_defaults( project_contents=('name = "${project_name}"\ndefault_target = "dev"\n[targets.dev]\n'), expected_error_fragment="project.name must be a committed literal", ), + ProjectConfigurationErrorTestCase( + description="rejects interpolation of the consumer group identity", + project_contents=( + 'name = "analytics"\nconsumer_group_identity = "${identity}"\n' + 'default_target = "dev"\n[targets.dev]\n' + ), + expected_error_fragment="project.consumer_group_identity must be a committed literal", + ), + ProjectConfigurationErrorTestCase( + description="rejects an empty consumer group identity", + project_contents=( + 'name = "analytics"\nconsumer_group_identity = " "\n' + 'default_target = "dev"\n[targets.dev]\n' + ), + expected_error_fragment="project.consumer_group_identity must be a non-empty string", + ), ProjectConfigurationErrorTestCase( description="rejects invalid deployment readiness lag", project_contents=( diff --git a/ui/src/lib/api/_helpers/mapping.ts b/ui/src/lib/api/_helpers/mapping.ts index 589195d0..6a894a62 100644 --- a/ui/src/lib/api/_helpers/mapping.ts +++ b/ui/src/lib/api/_helpers/mapping.ts @@ -233,7 +233,8 @@ function sourceFromServer(source: Payload, live: Payload): Source { rows: (live.rows as number) ?? 0, partitions: ((live.partitions ?? []) as Payload[]).map(partitionFromServer), throughput: buckets, - throughputWindowSeconds: (throughput?.windowSeconds as number | null) ?? null + throughputWindowSeconds: (throughput?.windowSeconds as number | null) ?? null, + consumerGroupDrift: (live.consumerGroupDrift as Source['live']['consumerGroupDrift']) ?? null } }; } diff --git a/ui/src/lib/domain/types.ts b/ui/src/lib/domain/types.ts index 5f2bf964..3cb32a67 100644 --- a/ui/src/lib/domain/types.ts +++ b/ui/src/lib/domain/types.ts @@ -50,6 +50,14 @@ export type PartitionState = { newestEventAt: string; }; +/** A live Kafka table consuming as a different consumer group than the project declares. */ +type ConsumerGroupDrift = { + tableName: string; + liveGroup: string; + declaredGroup: string; + message: string; +}; + type SourceLiveState = { rowsPerSecond: number; /** @@ -70,6 +78,8 @@ type SourceLiveState = { throughput: number[]; /** The window the throughput buckets cover, from the server query. */ throughputWindowSeconds: number | null; + /** Set when the live Kafka table's consumer group differs from the declared one. */ + consumerGroupDrift: ConsumerGroupDrift | null; }; export type Source = { diff --git a/ui/src/lib/source-browser/components/source-sidebar.svelte b/ui/src/lib/source-browser/components/source-sidebar.svelte index f6640b59..26b98a17 100644 --- a/ui/src/lib/source-browser/components/source-sidebar.svelte +++ b/ui/src/lib/source-browser/components/source-sidebar.svelte @@ -24,6 +24,17 @@ {#if source.live.throughputWindowSeconds}
last {formatDuration(source.live.throughputWindowSeconds)}
{/if} + {#if source.live.consumerGroupDrift} +
+
Consumer group drift
+
{source.live.consumerGroupDrift.message}
+
+ {/if} diff --git a/ui/tests/lib/api/_helpers/mapping.test.ts b/ui/tests/lib/api/_helpers/mapping.test.ts index aa5d4e7b..15a5f3e2 100644 --- a/ui/tests/lib/api/_helpers/mapping.test.ts +++ b/ui/tests/lib/api/_helpers/mapping.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; import { planFromServer, projectFromServer } from '$lib/api/_helpers/mapping'; -import type { Project } from '$lib/domain/types'; +import type { Project, Source } from '$lib/domain/types'; import type { Plan } from '$lib/planning/types'; describe('plan mapping', () => { @@ -134,6 +134,32 @@ describe('project mapping', () => { expect(project.models[0]?.status).toBe('unknown'); }); + it('given consumer group drift when mapped then both groups and the warning stay intact', () => { + const drift: Source['live']['consumerGroupDrift'] = { + tableName: 'kafka__orders', + liveGroup: 'streambuild_shop_dev_orders_shop_dev', + declaredGroup: 'streambuild_storefront_dev_orders_shop_dev', + message: 'offsets would not carry over' + }; + const project: Project = projectFromServer( + { + project: {}, + sources: [{ name: 'orders' }, { name: 'customers' }], + pipelines: [], + models: [], + audits: [], + tests: [], + macros: [] + }, + { + capturedAt: '2026-08-23T10:00:00Z', + sources: { orders: { consumerGroupDrift: drift }, customers: {} } + } + ); + + expect(project.sources.map((source) => source.live.consumerGroupDrift)).toEqual([drift, null]); + }); + it('given warehouse health when mapped then zero and unknown measurements remain distinct', () => { const project: Project = projectFromServer( { diff --git a/website/src/content/docs/docs/cli/index.mdx b/website/src/content/docs/docs/cli/index.mdx index 53c71db8..30ea282f 100644 --- a/website/src/content/docs/docs/cli/index.mdx +++ b/website/src/content/docs/docs/cli/index.mdx @@ -19,6 +19,7 @@ SQLTide 0.60.0. Run from inside a project or pass `--project-dir PATH`. | Inspect stable bindings | `sqt doctor` | Read-only warehouse inspection | | Repair one stable view | `sqt repair active-view` | Explicit mutation | | Adopt compatible metadata baseline | `sqt reconcile` | Preview; `--apply` confirms and writes metadata | +| Rename the project | `sqt rename-project` | Preview; `--apply` copies metadata, moves role grants and edits `sqltide_project.toml`; `--from` completes other targets | | Clean deployment physicals | `sqt janitor` | Preview; `--apply` drops without another prompt | | Destroy/reset owned relations | `sqt destroy`, `sqt reset-target` | Interactive frozen-plan review and typed challenges | | Manage accounts | `sqt admin` | Control-store mutation; no warehouse required | diff --git a/website/src/content/docs/docs/cli/plan.mdx b/website/src/content/docs/docs/cli/plan.mdx index 93390167..c8afac73 100644 --- a/website/src/content/docs/docs/cli/plan.mdx +++ b/website/src/content/docs/docs/cli/plan.mdx @@ -23,6 +23,12 @@ The effective project mode chooses the planner: - **Virtual environments:** plan changed deployment objects, replay roots, and policies against published and staged state. +In both modes, a warning follows the plan when a live Kafka table consumes as a different consumer +group than the project declares, for example after `name` was edited by hand. It names both groups +and notes that offsets would not carry over if the table were recreated. `--json` lists these +warnings under `consumer_group_warnings`, an empty list when there are none; the same key is in +the published plan artifact. The warning does not change or block the plan. + Use `--deployment-id` in virtual-environment mode to inspect exact deployment-specific physical names. Its timestamp prefix also fixes the deployment creation timestamp, so unchanged warehouse state and the same explicit identity produce deterministic workflow bytes. If omitted, `plan` diff --git a/website/src/content/docs/docs/cli/rename-project.mdx b/website/src/content/docs/docs/cli/rename-project.mdx new file mode 100644 index 00000000..5ea12ea0 --- /dev/null +++ b/website/src/content/docs/docs/cli/rename-project.mdx @@ -0,0 +1,110 @@ +--- +title: "sqt rename-project" +description: "Rename a project while keeping its Kafka consumer groups, manifest history and role grants." +--- + +Preview a project rename for the selected target: + +```bash +sqt rename-project storefront +sqt rename-project storefront --target prod +``` + +The default is a dry run. It prints the configuration edit, the Kafka consumer groups that stay the +same, the number of manifests to copy and the number of project role assignments to move, and +writes nothing to the warehouse or the project config. As with every command, a few legacy +migrations can still run: connecting renames legacy `_streambuild_*` metadata tables to +`_sqltide_*`, opening a configured control store that has never been initialized creates its +tables, and a legacy `.streambuild/` state directory is moved to `.sqltide/`. + +Apply the rename explicitly: + +```bash +sqt rename-project storefront --apply +``` + +`--apply` does not prompt. It runs these steps in order: + +1. Copies the target's manifest history from the old name to the new one under the target mutation + lock, so [`sqt destroy`](/docs/cli/destruction/) keeps finding relations recorded before the + rename. +2. Moves project role assignments, active and revoked, from the old name to the new one in the + control store. +3. Edits `sqltide_project.toml` in place: sets `name` and, when a managed Kafka source uses the + default consumer group, adds `consumer_group_identity` set to the old name. Comments and + formatting are kept. + +The configuration is edited last, so a failure leaves the old name working, and running the same +command again completes the rename. Steps that already ran are not repeated: manifests already +copied are skipped and assignments already moved are not moved again. + +The command refuses to run when: + +- the new name equals the current name, is empty or blank-padded, or contains `${` +- a build or other run holds the target mutation lock +- the target already records manifests for the new name that did not come from the old name +- both names hold the same active role assignment for the same user, role and target scope +- the rename would change any Kafka consumer group +- a managed Kafka table on the target already consumes as a different consumer group than the + project declares, which `sqt plan` reports as consumer group drift; resolve that first, for + example by restoring the name or `consumer_group_identity` the live tables were created with + +## Options + +| Option | Purpose | +| --- | --- | +| `NEW_NAME` | The new project name | +| `--apply` | Apply the rename; without it the command is a dry run | +| `--from OLD_NAME` | Complete a rename on another target after `sqltide_project.toml` already carries `NEW_NAME` | +| `--target` | Named project target whose warehouse history is copied | +| `--database` | Target database override | +| `--control-store-url` | Account control store; defaults to `SQLTIDE_CONTROL_STORE_URL`, then the project's `.sqltide/control.db` when it exists | +| `--project-dir`, `--host`, `--port`, `--username`, `--password`, `--vars` | As for other commands | + +## Projects with several targets + +Each target has its own warehouse, and usually its own UI host and control store. The first run +renames the project on one target and edits the configuration; every other target then completes +the rename with `--from`: + +1. On one target, rename and commit the configuration change: + + ```bash + sqt rename-project storefront --target dev --apply + ``` + +2. On each other target, typically on that target's own host so its control store is used, complete + the rename from the committed configuration: + + ```bash + sqt rename-project storefront --from shop --target prod + sqt rename-project storefront --from shop --target prod --apply + ``` + +With `--from`, the command leaves `sqltide_project.toml` unchanged. It checks that the target's +Kafka consumer groups stay the same, copies the target's manifest history from `OLD_NAME`, and moves +the control store's role assignments from `OLD_NAME`, under the same lock, dry-run default, conflict +rules and idempotence as the first run. The target may already have been built under the new name: +its own new manifests are kept and the old history is merged in, so `sqt destroy` sees both. It +refuses when `NEW_NAME` is not the name the configuration carries, when the configuration is still +named `OLD_NAME` (run without `--from` first), when `OLD_NAME` equals the new name, when nothing +on the target records `OLD_NAME` (a mistyped name, or a target never built under it with no grants +in this control store), or when the history of `OLD_NAME` does not look like this project's: a +`consumer_group_identity` that differs from `OLD_NAME`, or a newest `OLD_NAME` manifest that shares +no pipeline with the project. The last check keeps another project's history in the same database +from being merged in, where `sqt destroy` would treat its pipelines as retired ones of this +project. + +Until a target is completed, it keeps its manifest history under the old name, so `sqt destroy` +there does not see relations recorded before the rename. The first run's output lists the targets +still to complete. + +Role assignments move for the whole control store a run uses. When several targets share one +control store, the first run moves every target's grants, and UI users of targets still running +the old configuration lose those grants until the targets get the renamed configuration. Role +assignments are keyed by project name only, so two projects with the same name in one shared +control store cannot be told apart. + +Restart a running `sqt dev` server after a rename. See +[Renaming a project](/docs/concepts/project-configuration/#renaming-a-project) for what a rename +changes. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index fba69904..bbfbd358 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -64,6 +64,50 @@ production = true This immutable classification adds the exact `PRODUCTION` challenge to target reset. It is not accepted in `sqltide_local.toml`, so a local override cannot weaken or add production status. +## Project identity + +`name` identifies the project in three places that outlive a single run: + +- the default Kafka consumer group of every managed source without its own `consumer_group`, + `streambuild___` plus the target database suffix +- the manifest history `sqt destroy` uses to find relations recorded by earlier builds +- the project role assignments the UI grants to users + +Set `consumer_group_identity` to use another value in place of `name` in default consumer groups: + +```toml +name = "storefront" +consumer_group_identity = "shop" +``` + +It must be a non-empty literal string. A source's own `consumer_group` still takes precedence. +When the key is unset, default consumer groups use `name`, exactly as before. + +### Renaming a project + +Rename a project with [`sqt rename-project`](/docs/cli/rename-project/) rather than by editing +`name`. The command: + +- keeps every Kafka consumer group: when a managed source uses the default group, it sets + `consumer_group_identity` to the old name, so sources resume from their committed offsets +- copies the target's manifest history to the new name, so `sqt destroy` still finds relations + recorded before the rename +- moves project role assignments in the control store to the new name +- edits `name` in `sqltide_project.toml`, keeping comments and formatting + +A project with several targets is renamed once and then completed on each other target with +`sqt rename-project NEW_NAME --from OLD_NAME --target `, usually on that target's own +host. See [projects with several targets](/docs/cli/rename-project/#projects-with-several-targets). + +Editing `name` by hand changes the default consumer group of every managed source that has no +`consumer_group`. `sqt plan` and the source health panel in the UI then warn that the live Kafka +table consumes as a different group than the project declares, and that its committed offsets +would not carry over if the table were recreated. The warning does not block a plan or build. + +Renaming the project directory is a separate change. Run history, invocations, and audit and check +status are keyed by the directory name, not by `name`, so a new directory name starts them afresh. +No command migrates that history. + Commands that connect to ClickHouse require `host`, `port`, `username`, and `password` after all configuration layers are merged. SQLTide supplies no connection defaults. Use `password = ""` when the ClickHouse account intentionally has an empty password. diff --git a/website/src/content/docs/docs/ui/sources.mdx b/website/src/content/docs/docs/ui/sources.mdx index e5bb1660..bf81a976 100644 --- a/website/src/content/docs/docs/ui/sources.mdx +++ b/website/src/content/docs/docs/ui/sources.mdx @@ -17,6 +17,10 @@ The Sources index summarizes every authored source. Open a managed Kafka source - the effective consumer group - landed, committed, and broker-end offsets by partition +When the live Kafka table consumes as a different consumer group than the project declares, the +source shows a consumer group drift warning naming both groups. Lag is read for the declared group. +See [Project identity](/docs/concepts/project-configuration/#project-identity). + Kafka lag and last arrival are different signals: - **Kafka lag** is the broker high-water offset minus the effective consumer group's committed diff --git a/website/src/navigation.mjs b/website/src/navigation.mjs index d3fd55e9..dfea6441 100644 --- a/website/src/navigation.mjs +++ b/website/src/navigation.mjs @@ -4,7 +4,7 @@ export const sidebar = [ group('Build a pipeline', 'concepts/', [['project-configuration', 'Project configuration'], ['sql-rules', 'SQL Rules and formatting'], ['pipelines', 'Sources and pipelines'], ['models', 'Typed SQL models'], ['retention', 'Retention policies'], ['declarations', 'Constants and enums'], ['macros', 'Python macros'], ['testing', 'SQL testing'], ['audits', 'Data-quality audits']]), group('Operate and deploy', 'concepts/', [['direct-mode', 'Direct builds'], ['deployments', 'Staged deployments'], ['replay-lineage', 'Replay and lineage'], ['row-error-quarantine', 'Row-error quarantine'], ['destructive-operations', 'Destructive operations'], ['architecture', 'ClickHouse architecture'], ['authentication-and-access', 'Authentication'], ['access-control', 'Access control'], ['sensors', 'Sensors'], ['providers', 'Providers']]), group('Development UI', 'ui/', [['overview', 'Overview'], ['pipelines', 'Pipelines'], ['warehouse-health', 'Warehouse health'], ['sources', 'Sources'], ['topics', 'Topics'], ['messages', 'Messages'], ['plans', 'Plans'], ['runs', 'Runs'], ['quality', 'Quality']]), - group('CLI reference', 'cli/', [['index', 'Command overview'], ...['discover', 'compile', 'plan', 'build', 'dev', 'test', 'audit', 'deployment', 'doctor', 'repair', 'reconcile', 'janitor', 'destruction', 'admin'].map(name => [name, name === 'destruction' ? 'sqt destroy / reset-target' : `sqt ${name}`])]), + group('CLI reference', 'cli/', [['index', 'Command overview'], ...['discover', 'compile', 'plan', 'build', 'dev', 'test', 'audit', 'deployment', 'doctor', 'repair', 'reconcile', 'rename-project', 'janitor', 'destruction', 'admin'].map(name => [name, name === 'destruction' ? 'sqt destroy / reset-target' : `sqt ${name}`])]), group('Examples', 'examples/', [['orders-demo', 'Kafka commerce demo'], ['staged-workflow', 'Stage a pipeline change']]), ]; export const docsRoutes = sidebar.flatMap(group => group.items.map(item => `/${item.slug.replace(/\/$/, '').replace(/\/index$/, '')}/`));