feat: create the Recall database automatically and lead the Recall docs with the intended workflow

crates/sqlbuild-cli/src/argv/command_table.json

@@ -7027,7 +7027,7 @@
              "nargs": 0,
              "const": true,
              "default": false,
-             "help": "print the SQL that creates the Recall store or brings it up to this version",
+             "help": "print the SQL an administrator runs when the writer may not create or upgrade the Recall store",
              "mutex_group": 0,
              "group": 1
             },

crates/sqlbuild-cli/src/dispatch/main/closing_status.rs

@@ -1,13 +1,14 @@
 //! The closing line after an incomplete-delivery notice, so the command's own outcome is the last word.
 
-/// `sqb <command>`'s outcome by exit code, naming the only thing a preceding notice affects.
+/// `sqb <command>`'s outcome by exit code, naming the only thing a preceding notice affects; with nothing `affected`, as after a purely informational notice, success is reported plainly.
 #[must_use]
-pub fn closing_status(command: &str, exit_code: Option<i64>, affected: &str) -> String {
-    match exit_code {
-        Some(0) => {
+pub fn closing_status(command: &str, exit_code: Option<i64>, affected: Option<&str>) -> String {
+    match (exit_code, affected) {
+        (Some(0), Some(affected)) => {
             format!("sqb {command} completed successfully; only its {affected} is affected.\n")
         }
-        Some(code) => format!("sqb {command} failed with exit code {code}.\n"),
-        None => format!("sqb {command} failed.\n"),
+        (Some(0), None) => format!("sqb {command} completed successfully.\n"),
+        (Some(code), _) => format!("sqb {command} failed with exit code {code}.\n"),
+        (None, _) => format!("sqb {command} failed.\n"),
     }
 }

crates/sqlbuild-cli/src/dispatch/tests/test_closing_status.rs

@@ -7,23 +7,38 @@ fn given_command_outcome_when_closing_a_notice_then_the_outcome_is_the_last_line
         ClosingStatusTestCase {
             description: "success names the only affected output",
             exit_code: Some(0),
+            affected: Some("lifecycle event export"),
             expected_status: "sqb audit completed successfully; only its lifecycle event export \
                               is affected.\n",
         },
         ClosingStatusTestCase {
             description: "failure reports the exit code",
             exit_code: Some(2),
+            affected: Some("lifecycle event export"),
             expected_status: "sqb audit failed with exit code 2.\n",
         },
         ClosingStatusTestCase {
             description: "unknown exit code reports failure",
             exit_code: None,
+            affected: Some("lifecycle event export"),
             expected_status: "sqb audit failed.\n",
         },
+        ClosingStatusTestCase {
+            description: "success after an informational notice names nothing affected",
+            exit_code: Some(0),
+            affected: None,
+            expected_status: "sqb audit completed successfully.\n",
+        },
+        ClosingStatusTestCase {
+            description: "failure after an informational notice still reports the exit code",
+            exit_code: Some(1),
+            affected: None,
+            expected_status: "sqb audit failed with exit code 1.\n",
+        },
     ];
     for test_case in test_cases {
         assert_eq!(
-            closing_status("audit", test_case.exit_code, "lifecycle event export"),
+            closing_status("audit", test_case.exit_code, test_case.affected),
             test_case.expected_status,
             "{}",
             test_case.description

crates/sqlbuild-cli/src/dispatch/tests/test_types.rs

@@ -23,5 +23,6 @@ pub(super) struct EventExportWarningTestCase {
 pub(super) struct ClosingStatusTestCase {
     pub(super) description: &'static str,
     pub(super) exit_code: Option<i64>,
+    pub(super) affected: Option<&'static str>,
     pub(super) expected_status: &'static str,
 }

crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs

@@ -211,7 +211,7 @@ fn warn_incomplete_export(
         return Ok(());
     };
     let closing: String = if command_succeeded {
-        closing_status(command, Some(0), LIFECYCLE_EVENT_EXPORT)
+        closing_status(command, Some(0), Some(LIFECYCLE_EVENT_EXPORT))
     } else {
         String::new()
     };

crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_store.rs

@@ -2,6 +2,7 @@
 
 use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::path::{Path, PathBuf};
+use std::sync::{Mutex, PoisonError};
 use std::time::{Duration, Instant, SystemTime};
 
 use pyo3::prelude::Python;
@@ -25,9 +26,12 @@ use sqlbuild_recall::schema::constants::{
 };
 use sqlbuild_recall::schema::main::add_column_statement::add_column_statement;
 use sqlbuild_recall::schema::main::clickhouse_relation::clickhouse_relation;
+use sqlbuild_recall::schema::main::create_database_statement::create_database_statement;
+use sqlbuild_recall::schema::main::ddl_statements::ddl_statements;
 use sqlbuild_recall::schema::main::render_script::render_script;
 use sqlbuild_recall::schema::main::retention_fix_statements::retention_fix_statements;
 use sqlbuild_recall::schema::main::schema_age::schema_age;
+use sqlbuild_recall::schema::main::store_version_with_views::store_version_with_views;
 use sqlbuild_recall::schema::main::table_columns::table_columns;
 use sqlbuild_recall::schema::main::table_spec::table_spec;
 use sqlbuild_recall::schema::main::table_statements::table_statements;
@@ -38,6 +42,7 @@ use sqlbuild_recall::schema::main::view_statements::view_statements;
 use sqlbuild_recall::schema::models::{
     ColumnSpec, DdlOptions, Dialect, ObservedSchema, SchemaAge, StoreVersion, Table, VersionPlan,
 };
+use sqlbuild_recall::settings::constants::CLICKHOUSE_DATABASE_KEY;
 use sqlbuild_recall::store::constants::LAST_INVOCATION_SELECTOR;
 use sqlbuild_recall::store::errors::{StoreError, StoreErrorKind};
 use sqlbuild_recall::store::models::{
@@ -49,7 +54,8 @@ use crate::bindings::_helpers::recall::clickhouse_calls::{
     self as calls, ConnectionRole, QueryStatistics, StoreHandle,
 };
 use crate::bindings::_helpers::recall::constants::{
-    READ_ONLY_QUERIES, READONLY_SETTING, SECONDS_PER_DAY, UNKNOWN_TABLE_ERROR,
+    READ_ONLY_QUERIES, READONLY_SETTING, SECONDS_PER_DAY, SERVER_DEFAULT_DATABASE,
+    UNKNOWN_TABLE_ERROR,
 };
 
 /// Where a ClickHouse store lives and how long it keeps history.
@@ -90,12 +96,16 @@ pub(crate) struct ClickHouseStoreOpener {
     pub(crate) location: ClickHouseLocation,
     pub(crate) retention_notice: fn(&Path, &str) -> bool,
     pub(crate) project_dir: PathBuf,
+    /// What opening did that nothing is wrong with, such as creating the database, kept until the session takes it even when a later step fails.
+    pub(crate) information: Mutex<Vec<String>>,
 }
 
 /// The Recall relations present: tables with their `CREATE` queries, views, and the stored schema version.
 struct StoreState {
     tables: Vec<(String, String)>,
     views: HashSet<String>,
+    /// The views' `CREATE` queries, which show what an older version wrote when columns cannot.
+    view_definitions: Vec<String>,
     version: i64,
     /// Seconds since the newest version row was recorded.
     version_age_seconds: i64,
@@ -113,6 +123,7 @@ struct ClickHouseStore {
     connection: Option<StoreHandle>,
     columns: HashMap<Table, Vec<StoredColumn>>,
     notices: Vec<String>,
+    information: Vec<String>,
 }
 
 fn text(value: Option<&RecallValue>) -> String {
@@ -158,10 +169,12 @@ impl ClickHouseStore {
         let relations: QueryResult = self.query(py, &sql.relations, "inspect the Recall store")?;
         let mut tables: Vec<(String, String)> = Vec::new();
         let mut views: HashSet<String> = HashSet::new();
+        let mut view_definitions: Vec<String> = Vec::new();
         for row in &relations.rows {
             let name = text(row.first());
             if integer(row.get(1)) == 1 {
                 views.insert(name);
+                view_definitions.push(text(row.get(2)));
             } else {
                 tables.push((name, text(row.get(2))));
             }
@@ -182,16 +195,29 @@ impl ClickHouseStore {
         Ok(StoreState {
             tables,
             views,
+            view_definitions,
             version,
             version_age_seconds,
         })
     }
 
-    /// The recorded version, or without one how the tables, read by `read_columns`, and the views compare with this build's.
+    /// The recorded version, or without one how the tables, read by `read_columns`, and the views compare with this build's; either is lowered when a view definition shows an older version wrote it.
     fn store_version(
         &self,
         py: Python<'_>,
         state: &StoreState,
+    ) -> Result<StoreVersion, StoreError> {
+        let definitions: Vec<&str> = state.view_definitions.iter().map(String::as_str).collect();
+        Ok(store_version_with_views(
+            self.columns_version(py, state)?,
+            &definitions,
+        ))
+    }
+
+    fn columns_version(
+        &self,
+        py: Python<'_>,
+        state: &StoreState,
     ) -> Result<StoreVersion, StoreError> {
         if state.version > 0 {
             return Ok(StoreVersion::Recorded {
@@ -235,7 +261,7 @@ impl ClickHouseStore {
         )))
     }
 
-    fn require_database(&self, py: Python<'_>) -> Result<(), StoreError> {
+    fn database_exists(&self, py: Python<'_>) -> Result<bool, StoreError> {
         let found = self.query(
             py,
             &format!(
@@ -244,17 +270,35 @@ impl ClickHouseStore {
             ),
             "inspect the Recall store",
         )?;
-        if integer(found.rows.first().and_then(|row| row.first())) == 0 {
-            return Err(StoreError {
-                kind: StoreErrorKind::NotFound,
-                message: format!(
-                    "ClickHouse database `{}` does not exist; Recall creates its tables only in an \
-                     existing database",
+        Ok(integer(found.rows.first().and_then(|row| row.first())) > 0)
+    }
+
+    /// Create the configured database when it is missing, saying so; an existing one, whatever its engine, is left alone. A user who may not create it gets the statement to hand to an administrator.
+    fn ensure_database(&mut self, py: Python<'_>) -> Result<(), StoreError> {
+        if self.database_exists(py)? {
+            return Ok(());
+        }
+        let Some(statement) = create_database_statement(&self.location.options) else {
+            return Ok(());
+        };
+        match self.command(py, &statement, "create the Recall database") {
+            Ok(()) => {
+                self.information.push(format!(
+                    "created ClickHouse database `{}` for the store",
                     self.location.database()
+                ));
+                Ok(())
+            }
+            Err(error) if error.kind == StoreErrorKind::Permission => Err(StoreError {
+                kind: StoreErrorKind::Permission,
+                message: format!(
+                    "{} ({})",
+                    missing_database_reason(self.location.database()),
+                    error.message
                 ),
-            });
+            }),
+            Err(error) => Err(error),
         }
-        Ok(())
     }
 
     fn read_columns(&mut self, py: Python<'_>) -> Result<(), StoreError> {
@@ -304,7 +348,7 @@ impl ClickHouseStore {
         observed
     }
 
-    /// Create a store whose tables are missing. On existing tables add missing columns, create missing views, and replace views on a version upgrade; when retention removed the version row, the views' columns tell whether the store is older, current or newer, and a newer store's views are never replaced. A user who may not run that DDL writes the columns that exist and gets the statements an administrator must run. A version is recorded only once the views are settled, never lower than the store holds, and again before retention removes the newest row.
+    /// Create a store whose tables are missing. On existing tables add missing columns, create missing views, and replace views on a version upgrade; when retention removed the version row, the views' columns tell whether the store is older, current or newer, and a newer store's views are never replaced; views whose definitions show an older version wrote them make the store older either way. A user who may not run that DDL writes the columns that exist and gets the statements an administrator must run. A version is recorded only once the views are settled, never lower than the store holds, and again before retention removes the newest row.
     fn ensure_schema(
         &mut self,
         py: Python<'_>,
@@ -372,7 +416,7 @@ impl ClickHouseStore {
         envelope: &RowEnvelope,
         views: &HashSet<String>,
     ) -> Result<(), StoreError> {
-        self.require_database(py)?;
+        self.ensure_database(py)?;
         for statement in table_statements(Dialect::ClickHouse, &self.location.options) {
             self.command(py, &statement, "create the Recall tables")?;
         }
@@ -591,15 +635,17 @@ impl RecallStoreOpener for ClickHouseStoreOpener {
         Python::attach(|py| {
             let mut store = ClickHouseStore {
                 location: self.location.clone(),
-                connection: Some(calls::connect(
-                    py,
-                    (&self.location.config, ConnectionRole::Writer),
-                    &self.location.connect_action(),
-                )?),
+                connection: Some(connect_store(py, &self.location, ConnectionRole::Writer)?),
                 columns: HashMap::new(),
                 notices: Vec::new(),
+                information: Vec::new(),
             };
-            match store.ensure_schema(py, envelope) {
+            let ensured = store.ensure_schema(py, envelope);
+            self.information
+                .lock()
+                .unwrap_or_else(PoisonError::into_inner)
+                .append(&mut store.information);
+            match ensured {
                 Ok(ReadyStore { tables, behind }) => {
                     if !behind.is_empty()
                         && (self.retention_notice)(
@@ -627,6 +673,15 @@ impl RecallStoreOpener for ClickHouseStoreOpener {
         })
     }
 
+    fn take_information(&self) -> Vec<String> {
+        std::mem::take(
+            &mut *self
+                .information
+                .lock()
+                .unwrap_or_else(PoisonError::into_inner),
+        )
+    }
+
     fn batch_rows(&self) -> usize {
         BATCH_ROWS
     }
@@ -652,6 +707,7 @@ impl ClickHouseReader {
             )?),
             columns: HashMap::new(),
             notices: Vec::new(),
+            information: Vec::new(),
         };
         Ok(Self { store })
     }
@@ -683,15 +739,26 @@ impl ClickHouseReader {
     ) -> Result<Vec<String>, StoreError> {
         let mut store = ClickHouseStore {
             location: location.clone(),
-            connection: Some(calls::connect(
-                py,
-                (&location.config, ConnectionRole::Reader),
-                &location.connect_action(),
-            )?),
+            connection: Some(connect_store(py, location, ConnectionRole::Reader)?),
             columns: HashMap::new(),
             notices: Vec::new(),
+            information: Vec::new(),
         };
-        let observed = store.require_database(py).and_then(|()| {
+        match store.database_exists(py) {
+            Ok(true) => {}
+            Ok(false) => {
+                store.close();
+                return Ok(create_database_statement(&location.options)
+                    .into_iter()
+                    .chain(ddl_statements(Dialect::ClickHouse, &location.options))
+                    .collect());
+            }
+            Err(error) => {
+                store.close();
+                return Err(error);
+            }
+        }
+        let observed = {
             let state: StoreState = store.state(py)?;
             store.read_columns(py)?;
             let version: i64 = if state.tables.len() < ALL_TABLES.len() {
@@ -700,7 +767,7 @@ impl ClickHouseReader {
                 observed_version(store.store_version(py, &state)?)
             };
             Ok(store.observed(&state.views, version))
-        });
+        };
         store.close();
         Ok(upgrade_statements(
             Dialect::ClickHouse,
@@ -925,6 +992,39 @@ impl ClickHouseReader {
     }
 }
 
+/// Why a store cannot be created when its database is missing and the writer may not create it.
+pub(crate) fn missing_database_reason(database: &str) -> String {
+    format!("ClickHouse database `{database}` does not exist, and this user may not create it")
+}
+
+/// Connect for `role`. When the configured database does not exist yet, connect without it, so that it can be created or reported; every statement names the database itself.
+fn connect_store(
+    py: Python<'_>,
+    location: &ClickHouseLocation,
+    role: ConnectionRole,
+) -> Result<StoreHandle, StoreError> {
+    match calls::connect(py, (&location.config, role), &location.connect_action()) {
+        Err(error) if error.kind == StoreErrorKind::NotFound => {
+            let config: Vec<(String, ConfigValue)> = location
+                .config
+                .iter()
+                .map(|(key, value)| {
+                    if key == CLICKHOUSE_DATABASE_KEY {
+                        (
+                            key.clone(),
+                            ConfigValue::String(SERVER_DEFAULT_DATABASE.to_owned()),
+                        )
+                    } else {
+                        (key.clone(), value.clone())
+                    }
+                })
+                .collect();
+            calls::connect(py, (&config, role), &location.connect_action())
+        }
+        connected => connected,
+    }
+}
+
 /// The version `upgrade_statements` compares with: a store without a version row whose views are current or newer needs no view replacement.
 fn observed_version(store: StoreVersion) -> i64 {
     match store {