From 221b3bf1292ee5fab11cc80e47ade715ec787cf4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 02:31:38 +0100 Subject: [PATCH 01/19] feat: add the Recall core crate with records, schema, settings and the local SQLite store --- Cargo.lock | 89 +++- Cargo.toml | 5 + crates/sqlbuild-config/src/lib.rs | 1 + .../src/project/_helpers/documents.rs | 10 + .../src/project/_helpers/targets.rs | 9 +- .../sqlbuild-config/src/project/constants.rs | 3 +- .../src/recall/_helpers/mod.rs | 1 + .../src/recall/_helpers/values.rs | 52 +++ .../sqlbuild-config/src/recall/constants.rs | 20 + crates/sqlbuild-config/src/recall/main/mod.rs | 1 + .../src/recall/main/recall_section.rs | 61 +++ crates/sqlbuild-config/src/recall/mod.rs | 9 + crates/sqlbuild-config/src/recall/models.rs | 11 + crates/sqlbuild-config/src/recall/tests.rs | 2 + .../src/recall/tests/test_recall_section.rs | 84 ++++ crates/sqlbuild-recall/Cargo.toml | 21 + crates/sqlbuild-recall/src/lib.rs | 9 + .../src/records/_helpers/events.rs | 99 +++++ .../src/records/_helpers/ids.rs | 54 +++ .../src/records/_helpers/mod.rs | 4 + .../src/records/_helpers/rows.rs | 45 ++ .../src/records/_helpers/time.rs | 101 +++++ .../sqlbuild-recall/src/records/constants.rs | 48 +++ crates/sqlbuild-recall/src/records/errors.rs | 23 + .../src/records/main/artifact_rows.rs | 42 ++ .../src/records/main/conflict_row.rs | 26 ++ .../src/records/main/delivery_row.rs | 30 ++ .../main/diagnostic_input_from_json.rs | 51 +++ .../src/records/main/diagnostic_row.rs | 53 +++ .../src/records/main/event_row.rs | 22 + .../src/records/main/format_timestamp.rs | 11 + .../src/records/main/heartbeat_row.rs | 27 ++ .../src/records/main/invocation_row.rs | 74 ++++ .../src/records/main/meta_row.rs | 24 ++ .../sqlbuild-recall/src/records/main/mod.rs | 13 + .../src/records/main/output_row.rs | 29 ++ .../src/records/main/parse_timestamp.rs | 17 + .../src/records/main/sha256_hex.rs | 14 + crates/sqlbuild-recall/src/records/mod.rs | 10 + crates/sqlbuild-recall/src/records/models.rs | 165 ++++++++ crates/sqlbuild-recall/src/records/tests.rs | 2 + .../src/records/tests/test_records.rs | 135 ++++++ .../src/schema/_helpers/clickhouse_views.rs | 352 ++++++++++++++++ .../src/schema/_helpers/mod.rs | 3 + .../src/schema/_helpers/rendering.rs | 88 ++++ .../src/schema/_helpers/sqlite_views.rs | 360 ++++++++++++++++ .../sqlbuild-recall/src/schema/constants.rs | 340 +++++++++++++++ .../src/schema/main/add_column_statement.rs | 37 ++ .../src/schema/main/ddl_script.rs | 19 + crates/sqlbuild-recall/src/schema/main/mod.rs | 7 + .../src/schema/main/sqlite_retention_sql.rs | 59 +++ .../src/schema/main/table_columns.rs | 11 + .../src/schema/main/table_spec.rs | 24 ++ .../src/schema/main/table_statements.rs | 84 ++++ .../src/schema/main/view_statements.rs | 38 ++ crates/sqlbuild-recall/src/schema/mod.rs | 9 + crates/sqlbuild-recall/src/schema/models.rs | 111 +++++ crates/sqlbuild-recall/src/schema/tests.rs | 2 + .../src/schema/tests/test_ddl.rs | 129 ++++++ .../src/settings/_helpers/documents.rs | 73 ++++ .../src/settings/_helpers/layers.rs | 117 ++++++ .../src/settings/_helpers/mod.rs | 2 + .../sqlbuild-recall/src/settings/constants.rs | 14 + crates/sqlbuild-recall/src/settings/errors.rs | 23 + .../sqlbuild-recall/src/settings/main/mod.rs | 1 + .../settings/main/resolve_recall_settings.rs | 68 +++ crates/sqlbuild-recall/src/settings/mod.rs | 10 + crates/sqlbuild-recall/src/settings/models.rs | 37 ++ crates/sqlbuild-recall/src/settings/tests.rs | 2 + .../src/settings/tests/test_resolve.rs | 101 +++++ .../sqlbuild-recall/src/store/_helpers/mod.rs | 3 + .../src/store/_helpers/rows.rs | 138 ++++++ .../src/store/_helpers/schema.rs | 80 ++++ .../src/store/_helpers/sqlite.rs | 99 +++++ crates/sqlbuild-recall/src/store/constants.rs | 14 + crates/sqlbuild-recall/src/store/errors.rs | 43 ++ .../src/store/main/describe_store.rs | 49 +++ .../src/store/main/ensure_sqlite_schema.rs | 57 +++ .../src/store/main/insert_rows.rs | 18 + .../src/store/main/maintain_sqlite_store.rs | 174 ++++++++ crates/sqlbuild-recall/src/store/main/mod.rs | 8 + .../src/store/main/open_sqlite_store.rs | 124 ++++++ .../src/store/main/query_store.rs | 54 +++ .../src/store/main/read_output_transcript.rs | 78 ++++ .../src/store/main/resolve_invocation.rs | 62 +++ crates/sqlbuild-recall/src/store/mod.rs | 10 + crates/sqlbuild-recall/src/store/models.rs | 99 +++++ crates/sqlbuild-recall/src/store/tests.rs | 8 + .../src/store/tests/helpers.rs | 98 +++++ .../src/store/tests/test_maintenance.rs | 110 +++++ .../src/store/tests/test_store.rs | 134 ++++++ .../src/store/tests/test_views.rs | 392 ++++++++++++++++++ .../src/writer/_helpers/delivery.rs | 253 +++++++++++ .../src/writer/_helpers/mod.rs | 3 + .../src/writer/_helpers/queue.rs | 275 ++++++++++++ .../src/writer/_helpers/session.rs | 193 +++++++++ .../sqlbuild-recall/src/writer/constants.rs | 17 + crates/sqlbuild-recall/src/writer/main/mod.rs | 1 + .../src/writer/main/start_recall_session.rs | 42 ++ crates/sqlbuild-recall/src/writer/mod.rs | 9 + crates/sqlbuild-recall/src/writer/models.rs | 130 ++++++ crates/sqlbuild-recall/src/writer/tests.rs | 4 + .../src/writer/tests/test_queue.rs | 88 ++++ .../src/writer/tests/test_session.rs | 123 ++++++ fensu.toml | 1 + .../compiler/failure/target-unknown-key.json | 2 +- 106 files changed, 6616 insertions(+), 5 deletions(-) create mode 100644 crates/sqlbuild-config/src/recall/_helpers/mod.rs create mode 100644 crates/sqlbuild-config/src/recall/_helpers/values.rs create mode 100644 crates/sqlbuild-config/src/recall/constants.rs create mode 100644 crates/sqlbuild-config/src/recall/main/mod.rs create mode 100644 crates/sqlbuild-config/src/recall/main/recall_section.rs create mode 100644 crates/sqlbuild-config/src/recall/mod.rs create mode 100644 crates/sqlbuild-config/src/recall/models.rs create mode 100644 crates/sqlbuild-config/src/recall/tests.rs create mode 100644 crates/sqlbuild-config/src/recall/tests/test_recall_section.rs create mode 100644 crates/sqlbuild-recall/Cargo.toml create mode 100644 crates/sqlbuild-recall/src/lib.rs create mode 100644 crates/sqlbuild-recall/src/records/_helpers/events.rs create mode 100644 crates/sqlbuild-recall/src/records/_helpers/ids.rs create mode 100644 crates/sqlbuild-recall/src/records/_helpers/mod.rs create mode 100644 crates/sqlbuild-recall/src/records/_helpers/rows.rs create mode 100644 crates/sqlbuild-recall/src/records/_helpers/time.rs create mode 100644 crates/sqlbuild-recall/src/records/constants.rs create mode 100644 crates/sqlbuild-recall/src/records/errors.rs create mode 100644 crates/sqlbuild-recall/src/records/main/artifact_rows.rs create mode 100644 crates/sqlbuild-recall/src/records/main/conflict_row.rs create mode 100644 crates/sqlbuild-recall/src/records/main/delivery_row.rs create mode 100644 crates/sqlbuild-recall/src/records/main/diagnostic_input_from_json.rs create mode 100644 crates/sqlbuild-recall/src/records/main/diagnostic_row.rs create mode 100644 crates/sqlbuild-recall/src/records/main/event_row.rs create mode 100644 crates/sqlbuild-recall/src/records/main/format_timestamp.rs create mode 100644 crates/sqlbuild-recall/src/records/main/heartbeat_row.rs create mode 100644 crates/sqlbuild-recall/src/records/main/invocation_row.rs create mode 100644 crates/sqlbuild-recall/src/records/main/meta_row.rs create mode 100644 crates/sqlbuild-recall/src/records/main/mod.rs create mode 100644 crates/sqlbuild-recall/src/records/main/output_row.rs create mode 100644 crates/sqlbuild-recall/src/records/main/parse_timestamp.rs create mode 100644 crates/sqlbuild-recall/src/records/main/sha256_hex.rs create mode 100644 crates/sqlbuild-recall/src/records/mod.rs create mode 100644 crates/sqlbuild-recall/src/records/models.rs create mode 100644 crates/sqlbuild-recall/src/records/tests.rs create mode 100644 crates/sqlbuild-recall/src/records/tests/test_records.rs create mode 100644 crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs create mode 100644 crates/sqlbuild-recall/src/schema/_helpers/mod.rs create mode 100644 crates/sqlbuild-recall/src/schema/_helpers/rendering.rs create mode 100644 crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs create mode 100644 crates/sqlbuild-recall/src/schema/constants.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/add_column_statement.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/ddl_script.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/mod.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/table_columns.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/table_spec.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/table_statements.rs create mode 100644 crates/sqlbuild-recall/src/schema/main/view_statements.rs create mode 100644 crates/sqlbuild-recall/src/schema/mod.rs create mode 100644 crates/sqlbuild-recall/src/schema/models.rs create mode 100644 crates/sqlbuild-recall/src/schema/tests.rs create mode 100644 crates/sqlbuild-recall/src/schema/tests/test_ddl.rs create mode 100644 crates/sqlbuild-recall/src/settings/_helpers/documents.rs create mode 100644 crates/sqlbuild-recall/src/settings/_helpers/layers.rs create mode 100644 crates/sqlbuild-recall/src/settings/_helpers/mod.rs create mode 100644 crates/sqlbuild-recall/src/settings/constants.rs create mode 100644 crates/sqlbuild-recall/src/settings/errors.rs create mode 100644 crates/sqlbuild-recall/src/settings/main/mod.rs create mode 100644 crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs create mode 100644 crates/sqlbuild-recall/src/settings/mod.rs create mode 100644 crates/sqlbuild-recall/src/settings/models.rs create mode 100644 crates/sqlbuild-recall/src/settings/tests.rs create mode 100644 crates/sqlbuild-recall/src/settings/tests/test_resolve.rs create mode 100644 crates/sqlbuild-recall/src/store/_helpers/mod.rs create mode 100644 crates/sqlbuild-recall/src/store/_helpers/rows.rs create mode 100644 crates/sqlbuild-recall/src/store/_helpers/schema.rs create mode 100644 crates/sqlbuild-recall/src/store/_helpers/sqlite.rs create mode 100644 crates/sqlbuild-recall/src/store/constants.rs create mode 100644 crates/sqlbuild-recall/src/store/errors.rs create mode 100644 crates/sqlbuild-recall/src/store/main/describe_store.rs create mode 100644 crates/sqlbuild-recall/src/store/main/ensure_sqlite_schema.rs create mode 100644 crates/sqlbuild-recall/src/store/main/insert_rows.rs create mode 100644 crates/sqlbuild-recall/src/store/main/maintain_sqlite_store.rs create mode 100644 crates/sqlbuild-recall/src/store/main/mod.rs create mode 100644 crates/sqlbuild-recall/src/store/main/open_sqlite_store.rs create mode 100644 crates/sqlbuild-recall/src/store/main/query_store.rs create mode 100644 crates/sqlbuild-recall/src/store/main/read_output_transcript.rs create mode 100644 crates/sqlbuild-recall/src/store/main/resolve_invocation.rs create mode 100644 crates/sqlbuild-recall/src/store/mod.rs create mode 100644 crates/sqlbuild-recall/src/store/models.rs create mode 100644 crates/sqlbuild-recall/src/store/tests.rs create mode 100644 crates/sqlbuild-recall/src/store/tests/helpers.rs create mode 100644 crates/sqlbuild-recall/src/store/tests/test_maintenance.rs create mode 100644 crates/sqlbuild-recall/src/store/tests/test_store.rs create mode 100644 crates/sqlbuild-recall/src/store/tests/test_views.rs create mode 100644 crates/sqlbuild-recall/src/writer/_helpers/delivery.rs create mode 100644 crates/sqlbuild-recall/src/writer/_helpers/mod.rs create mode 100644 crates/sqlbuild-recall/src/writer/_helpers/queue.rs create mode 100644 crates/sqlbuild-recall/src/writer/_helpers/session.rs create mode 100644 crates/sqlbuild-recall/src/writer/constants.rs create mode 100644 crates/sqlbuild-recall/src/writer/main/mod.rs create mode 100644 crates/sqlbuild-recall/src/writer/main/start_recall_session.rs create mode 100644 crates/sqlbuild-recall/src/writer/mod.rs create mode 100644 crates/sqlbuild-recall/src/writer/models.rs create mode 100644 crates/sqlbuild-recall/src/writer/tests.rs create mode 100644 crates/sqlbuild-recall/src/writer/tests/test_queue.rs create mode 100644 crates/sqlbuild-recall/src/writer/tests/test_session.rs diff --git a/Cargo.lock b/Cargo.lock index 37ac81b26c..011fa5984f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -303,6 +303,18 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.5.0" @@ -328,6 +340,12 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "generic-array" version = "0.14.7" @@ -357,7 +375,7 @@ checksum = "823645bc6404ae2915707777061a47d3a031a9ee0bff51b34ec973df3d8d2990" dependencies = [ "compact_str", "get-size-derive2", - "hashbrown", + "hashbrown 0.17.1", "ordermap", "smallvec", "thin-vec", @@ -407,12 +425,30 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + [[package]] name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "heck" version = "0.5.0" @@ -426,7 +462,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.17.1", ] [[package]] @@ -496,6 +532,17 @@ dependencies = [ "cc", ] +[[package]] +name = "libsqlite3-sys" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -655,6 +702,12 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + [[package]] name = "polyglot-sql" version = "0.13.9" @@ -982,6 +1035,20 @@ dependencies = [ "get-size2", ] +[[package]] +name = "rusqlite" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -1265,6 +1332,18 @@ dependencies = [ "stacker", ] +[[package]] +name = "sqlbuild-recall" +version = "0.158.2" +dependencies = [ + "rusqlite", + "serde_json", + "sha2", + "sqlbuild-config", + "sqlbuild-core", + "tempfile", +] + [[package]] name = "sqlbuild-refactor" version = "0.160.1" @@ -1578,6 +1657,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index 5dace8bcc8..3dec1b100d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/sqlbuild-cache", "crates/sqlbuild-sqltext", "crates/sqlbuild-config", + "crates/sqlbuild-recall", "crates/sqlbuild-discovery", "crates/sqlbuild-scopes", "crates/sqlbuild-model-config", @@ -32,6 +33,7 @@ order = [ "sqlbuild-cache", "sqlbuild-sqltext", "sqlbuild-config", + "sqlbuild-recall", "sqlbuild-model-config", "sqlbuild-discovery", "sqlbuild-render", @@ -86,6 +88,8 @@ ruff_text_size = "=0.0.4" # Unix only: `sqb` shares the compute log prune lock with Python's `filelock` through `flock`. # Fast deterministic hasher for hot in-memory maps whose iteration order never reaches output. rustc-hash = "2" +# Recall's local store: SQLite compiled in, so every build has the same engine version. +rusqlite = { version = "0.37", features = ["bundled"] } rustix = { version = "1.1.4", default-features = false, features = ["fs", "std"] } saphyr-parser = "0.1" regex = "1" @@ -106,6 +110,7 @@ sqlbuild-refactor = { path = "crates/sqlbuild-refactor" } sqlbuild-rules = { path = "crates/sqlbuild-rules" } sqlbuild-model-config = { path = "crates/sqlbuild-model-config" } sqlbuild-pysource = { path = "crates/sqlbuild-pysource" } +sqlbuild-recall = { path = "crates/sqlbuild-recall" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } sqlbuild-state = { path = "crates/sqlbuild-state" } diff --git a/crates/sqlbuild-config/src/lib.rs b/crates/sqlbuild-config/src/lib.rs index da69b29e36..66fb1a8c59 100644 --- a/crates/sqlbuild-config/src/lib.rs +++ b/crates/sqlbuild-config/src/lib.rs @@ -10,6 +10,7 @@ pub mod effective; pub mod errors; pub mod models; pub mod project; +pub mod recall; pub mod toml; pub mod workspace; pub mod yaml; diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index 80ff474882..5690123c50 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -30,6 +30,8 @@ use crate::project::constants::{ use crate::project::errors::{ConfigFailure, FailureClass, LoadFailure, MessagePiece}; use crate::project::models::{Built, ConfigFile, ConfigPaths}; use crate::project::types::{Entries, Fields, Loaded}; +use crate::recall::constants::RECALL_SECTION; +use crate::recall::main::recall_section::recall_section; use crate::toml::main::load_toml::load_toml; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_core::constants::SETTINGS_SECTION; @@ -270,6 +272,10 @@ fn project_fields(payload: &Entries) -> Loaded { "event_exporters was replaced by sinks.lifecycle", )); } + if let Some(recall) = get(payload, RECALL_SECTION) { + recall_section(recall, RECALL_SECTION) + .map_err(|message| ConfigFailure::at(PROJECT_FILE, message))?; + } let mut fields = identity(payload)?; fields.extend(sections(payload)?); fields.push(("targets", targets(payload)?)); @@ -331,6 +337,10 @@ fn local_settings(payload: Option<&ConfigValue>) -> Loaded { } fn local_fields(payload: &Entries) -> Loaded { + if let Some(recall) = get(payload, RECALL_SECTION) { + recall_section(recall, RECALL_SECTION) + .map_err(|message| ConfigFailure::at(LOCAL_FILE, message))?; + } let mut fields: Fields = vec![ ("target", text_or_none(optional_str(payload, "target")?)), ("adapter", text_or_none(optional_str(payload, "adapter")?)), diff --git a/crates/sqlbuild-config/src/project/_helpers/targets.rs b/crates/sqlbuild-config/src/project/_helpers/targets.rs index 63c3c214b4..2ffa133ab5 100644 --- a/crates/sqlbuild-config/src/project/_helpers/targets.rs +++ b/crates/sqlbuild-config/src/project/_helpers/targets.rs @@ -21,6 +21,8 @@ use crate::project::constants::{ use crate::project::errors::ConfigFailure; use crate::project::models::{Built, ConfigFile}; use crate::project::types::{Entries, Fields, Loaded}; +use crate::recall::constants::RECALL_SECTION; +use crate::recall::main::recall_section::recall_section; /// Policy fields of a target, with the enum each projects to. const TARGET_POLICIES: [(&str, &str); 3] = [ @@ -293,7 +295,12 @@ fn validate_target_keys(mapping: &Entries, target: TargetPlace<'_>) -> Result<() label: &label, }; reject_removed_virtual_keys(mapping, &["state", "changes_only"], place)?; - validate_allowed_keys(mapping, &TARGET_KEYS, place) + validate_allowed_keys(mapping, &TARGET_KEYS, place)?; + if let Some(recall) = get(mapping, RECALL_SECTION) { + recall_section(recall, &format!("{label}.{RECALL_SECTION}")) + .map_err(|message| ConfigFailure::at(target.file, message))?; + } + Ok(()) } fn clone_mapping<'a>(mapping: &'a Entries, target: TargetPlace<'_>) -> Loaded<&'a Entries> { diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index dc1c783554..286b6732f0 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -169,7 +169,7 @@ pub(crate) const MODEL_HEADER_KEYS: [&str; 73] = [ "valid_from_column", "valid_to_column", ]; -pub(crate) const TARGET_KEYS: [&str; 18] = [ +pub(crate) const TARGET_KEYS: [&str; 19] = [ "connection", "vars", "database", @@ -188,6 +188,7 @@ pub(crate) const TARGET_KEYS: [&str; 18] = [ "execution_limits", "diff", "warehouses", + "recall", ]; pub(crate) const CLONE_KEYS: [&str; 2] = ["allow_as_clone_origin", "allow_as_clone_destination"]; pub(crate) const EXECUTION_LIMIT_KEYS: [&str; 5] = [ diff --git a/crates/sqlbuild-config/src/recall/_helpers/mod.rs b/crates/sqlbuild-config/src/recall/_helpers/mod.rs new file mode 100644 index 0000000000..4b9235503b --- /dev/null +++ b/crates/sqlbuild-config/src/recall/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod values; diff --git a/crates/sqlbuild-config/src/recall/_helpers/values.rs b/crates/sqlbuild-config/src/recall/_helpers/values.rs new file mode 100644 index 0000000000..daf30896a5 --- /dev/null +++ b/crates/sqlbuild-config/src/recall/_helpers/values.rs @@ -0,0 +1,52 @@ +//! Recall setting value parsing: durations, retention days and strings. + +use crate::models::ConfigValue; +use crate::recall::constants::{ + DECIMAL_RADIX, HOURS_PER_DAY, MAX_RETENTION_DAYS, MAX_SHUTDOWN_TIMEOUT_MS, MILLIS_PER_SECOND, +}; + +/// The unsigned decimal before `unit`, `None` when it is not one or overflows. +fn amount(text: &str, unit: &str) -> Option { + let digits = text.strip_suffix(unit)?; + if digits.is_empty() { + return None; + } + digits.bytes().try_fold(0_u64, |total, byte| { + let digit = char::from(byte).to_digit(DECIMAL_RADIX)?; + total + .checked_mul(u64::from(DECIMAL_RADIX))? + .checked_add(u64::from(digit)) + }) +} + +/// Whole days of `30d` or `720h`, within the supported range. +pub(crate) fn retention_days(text: &str) -> Option { + let days = amount(text, "d").or_else(|| { + amount(text, "h") + .filter(|hours| hours % HOURS_PER_DAY == 0) + .map(|hours| hours / HOURS_PER_DAY) + })?; + if days == 0 || days > MAX_RETENTION_DAYS { + return None; + } + Some(u32::try_from(days).unwrap_or(u32::MAX)) +} + +/// Milliseconds of `3s` or `500ms`, within the supported range. +pub(crate) fn timeout_ms(text: &str) -> Option { + let millis = amount(text, "ms") + .or_else(|| amount(text, "s").and_then(|seconds| seconds.checked_mul(MILLIS_PER_SECOND)))?; + (millis <= MAX_SHUTDOWN_TIMEOUT_MS).then_some(millis) +} + +/// A trimmed non-empty string, or the message for `label.key`. +pub(crate) fn non_empty<'a>( + value: &'a ConfigValue, + label: &str, + key: &str, +) -> Result<&'a str, String> { + match value { + ConfigValue::String(text) if !text.trim().is_empty() => Ok(text.trim()), + _ => Err(format!("{label}.{key} must be a non-empty string")), + } +} diff --git a/crates/sqlbuild-config/src/recall/constants.rs b/crates/sqlbuild-config/src/recall/constants.rs new file mode 100644 index 0000000000..205579f213 --- /dev/null +++ b/crates/sqlbuild-config/src/recall/constants.rs @@ -0,0 +1,20 @@ +//! Recall setting keys and value vocabularies. + +pub const RECALL_SECTION: &str = "recall"; +/// Keys `[recall]` and `targets..recall` accept, sorted. +pub const RECALL_KEYS: [&str; 5] = [ + "connection", + "diagnostics_min_severity", + "enabled", + "retention", + "shutdown_timeout", +]; +/// The diagnostic severities a Recall store can keep from, least severe first. +pub const RECALL_SEVERITIES: [&str; 6] = ["trace", "debug", "info", "warning", "error", "critical"]; +/// The longest shutdown flush budget, in milliseconds. +pub const MAX_SHUTDOWN_TIMEOUT_MS: u64 = 600_000; +/// The longest retention, in days. +pub const MAX_RETENTION_DAYS: u64 = 3_650; +pub(crate) const DECIMAL_RADIX: u32 = 10; +pub(crate) const HOURS_PER_DAY: u64 = 24; +pub(crate) const MILLIS_PER_SECOND: u64 = 1_000; diff --git a/crates/sqlbuild-config/src/recall/main/mod.rs b/crates/sqlbuild-config/src/recall/main/mod.rs new file mode 100644 index 0000000000..619cdce30e --- /dev/null +++ b/crates/sqlbuild-config/src/recall/main/mod.rs @@ -0,0 +1 @@ +pub mod recall_section; diff --git a/crates/sqlbuild-config/src/recall/main/recall_section.rs b/crates/sqlbuild-config/src/recall/main/recall_section.rs new file mode 100644 index 0000000000..212632400c --- /dev/null +++ b/crates/sqlbuild-config/src/recall/main/recall_section.rs @@ -0,0 +1,61 @@ +//! `recall_section`: validate one Recall settings table. + +use crate::models::ConfigValue; +use crate::recall::_helpers::values::{non_empty, retention_days, timeout_ms}; +use crate::recall::constants::{MAX_RETENTION_DAYS, RECALL_KEYS, RECALL_SEVERITIES}; +use crate::recall::models::RecallSection; + +/// The settings of `value`, or the message naming its first problem at `label`. +pub fn recall_section(value: &ConfigValue, label: &str) -> Result { + let Some(entries) = value.as_map() else { + return Err(format!("{label} must be a table")); + }; + let mut section = RecallSection::default(); + for (key, value) in entries { + let ConfigValue::String(key) = key else { + return Err(format!("{label} keys must be strings")); + }; + match key.as_str() { + "enabled" => match value { + ConfigValue::Bool(flag) => section.enabled = Some(*flag), + _ => return Err(format!("{label}.enabled must be true or false")), + }, + "connection" => section.connection = Some(non_empty(value, label, key)?.to_owned()), + "retention" => { + let text = non_empty(value, label, key)?; + section.retention_days = Some(retention_days(text).ok_or_else(|| { + format!( + "{label}.retention must be a whole number of days such as '30d', \ + at most {MAX_RETENTION_DAYS}d" + ) + })?); + } + "shutdown_timeout" => { + let text = non_empty(value, label, key)?; + section.shutdown_timeout_ms = Some(timeout_ms(text).ok_or_else(|| { + format!( + "{label}.shutdown_timeout must be a duration such as '3s' or '500ms', \ + at most 600s" + ) + })?); + } + "diagnostics_min_severity" => { + let text = non_empty(value, label, key)?; + if !RECALL_SEVERITIES.contains(&text) { + return Err(format!( + "{label}.diagnostics_min_severity must be one of: {}", + RECALL_SEVERITIES.join(", ") + )); + } + section.diagnostics_min_severity = Some(text.to_owned()); + } + other => { + return Err(format!( + "{label} contains unknown key(s): {other}. Allowed keys: {}", + RECALL_KEYS.join(", ") + )); + } + } + } + Ok(section) +} diff --git a/crates/sqlbuild-config/src/recall/mod.rs b/crates/sqlbuild-config/src/recall/mod.rs new file mode 100644 index 0000000000..97389af3ff --- /dev/null +++ b/crates/sqlbuild-config/src/recall/mod.rs @@ -0,0 +1,9 @@ +//! The `[recall]` section and `targets..recall` tables. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-config/src/recall/models.rs b/crates/sqlbuild-config/src/recall/models.rs new file mode 100644 index 0000000000..4edf3e6900 --- /dev/null +++ b/crates/sqlbuild-config/src/recall/models.rs @@ -0,0 +1,11 @@ +//! One validated Recall settings table; absent keys inherit. + +/// The settings one `[recall]` or `targets..recall` table sets. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RecallSection { + pub enabled: Option, + pub connection: Option, + pub retention_days: Option, + pub shutdown_timeout_ms: Option, + pub diagnostics_min_severity: Option, +} diff --git a/crates/sqlbuild-config/src/recall/tests.rs b/crates/sqlbuild-config/src/recall/tests.rs new file mode 100644 index 0000000000..992dd65d8d --- /dev/null +++ b/crates/sqlbuild-config/src/recall/tests.rs @@ -0,0 +1,2 @@ +#[path = "tests/test_recall_section.rs"] +mod recall_section; diff --git a/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs b/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs new file mode 100644 index 0000000000..b57776f3b2 --- /dev/null +++ b/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs @@ -0,0 +1,84 @@ +use crate::models::ConfigValue; +use crate::recall::main::recall_section::recall_section; +use crate::recall::models::RecallSection; + +fn table(entries: &[(&str, ConfigValue)]) -> ConfigValue { + ConfigValue::Map( + entries + .iter() + .map(|(key, value)| (ConfigValue::String((*key).to_owned()), value.clone())) + .collect(), + ) +} + +fn text(value: &str) -> ConfigValue { + ConfigValue::String(value.to_owned()) +} + +#[test] +fn given_every_key_when_validated_then_settings_are_typed() { + let section = recall_section( + &table(&[ + ("enabled", ConfigValue::Bool(false)), + ("connection", text(" history ")), + ("retention", text("720h")), + ("shutdown_timeout", text("1500ms")), + ("diagnostics_min_severity", text("debug")), + ]), + "recall", + ); + assert_eq!( + section, + Ok(RecallSection { + enabled: Some(false), + connection: Some("history".to_owned()), + retention_days: Some(30), + shutdown_timeout_ms: Some(1500), + diagnostics_min_severity: Some("debug".to_owned()), + }) + ); +} + +#[test] +fn given_invalid_values_when_validated_then_the_message_names_the_key() { + let cases: [(&str, ConfigValue, &str); 6] = [ + ( + "unknown key", + table(&[("mode", text("x"))]), + "recall contains unknown key(s): mode", + ), + ( + "bad retention", + table(&[("retention", text("30x"))]), + "recall.retention must be", + ), + ( + "hours not days", + table(&[("retention", text("25h"))]), + "recall.retention must be", + ), + ( + "zero days", + table(&[("retention", text("0d"))]), + "recall.retention must be", + ), + ( + "timeout too long", + table(&[("shutdown_timeout", text("601s"))]), + "recall.shutdown_timeout must be", + ), + ( + "severity", + table(&[("diagnostics_min_severity", text("loud"))]), + "recall.diagnostics_min_severity must be one of", + ), + ]; + for (description, value, expected) in cases { + let message = recall_section(&value, "recall").expect_err(description); + assert!(message.starts_with(expected), "{description}: {message}"); + } + assert_eq!( + recall_section(&text("x"), "targets.dev.recall"), + Err("targets.dev.recall must be a table".to_owned()) + ); +} diff --git a/crates/sqlbuild-recall/Cargo.toml b/crates/sqlbuild-recall/Cargo.toml new file mode 100644 index 0000000000..a53553ecc6 --- /dev/null +++ b/crates/sqlbuild-recall/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "sqlbuild-recall" +version.workspace = true +description = "Recall: invocation history records, their SQLite and ClickHouse schema, settings and the local store." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +rusqlite.workspace = true +serde_json.workspace = true +sha2.workspace = true +sqlbuild-config.workspace = true +sqlbuild-core.workspace = true + +[dev-dependencies] +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-recall/src/lib.rs b/crates/sqlbuild-recall/src/lib.rs new file mode 100644 index 0000000000..53e5776303 --- /dev/null +++ b/crates/sqlbuild-recall/src/lib.rs @@ -0,0 +1,9 @@ +//! Recall: the records of every invocation, their SQLite and ClickHouse schema and views, retention, the `[recall]` settings, the local SQLite store and its delivery thread. + +#![forbid(unsafe_code)] + +pub mod records; +pub mod schema; +pub mod settings; +pub mod store; +pub mod writer; diff --git a/crates/sqlbuild-recall/src/records/_helpers/events.rs b/crates/sqlbuild-recall/src/records/_helpers/events.rs new file mode 100644 index 0000000000..7e139cc29f --- /dev/null +++ b/crates/sqlbuild-recall/src/records/_helpers/events.rs @@ -0,0 +1,99 @@ +//! The envelope and promoted payload columns of a lifecycle event row. + +use serde_json::{Map, Value}; +use sqlbuild_core::event_export::main::lifecycle_export_policy::lifecycle_export_policy; +use sqlbuild_core::lifecycle_events::main::catalog::catalog; + +use crate::records::_helpers::rows::{optional_text, text}; +use crate::records::constants::{ + CORRELATION_FIELDS, PROMOTED_INTEGER_FIELDS, PROMOTED_REAL_FIELDS, PROMOTED_TEXT_FIELDS, +}; +use crate::records::errors::RecordError; +use crate::records::models::RecallValue; + +/// One named column value. +pub(crate) type Column = (&'static str, RecallValue); + +fn required_text<'a>(event: &'a Map, field: &str) -> Result<&'a str, RecordError> { + event + .get(field) + .and_then(Value::as_str) + .ok_or_else(|| RecordError::new(format!("lifecycle event {field} must be a string"))) +} + +fn is_terminal(event_type: &str, schema_version: i64) -> bool { + catalog(schema_version) + .into_iter() + .flatten() + .any(|definition| definition.event_type == event_type && definition.terminal) +} + +/// The event JSON object. +pub(crate) fn event_object(event_json: &str) -> Result, RecordError> { + match serde_json::from_str(event_json) { + Ok(Value::Object(event)) => Ok(event), + Ok(_) => Err(RecordError::new("lifecycle event must be a JSON object")), + Err(error) => Err(RecordError::new(format!( + "lifecycle event is not JSON: {error}" + ))), + } +} + +/// Identity, type, export policy and correlation columns. +pub(crate) fn envelope_columns(event: &Map) -> Result, RecordError> { + let event_type: &str = required_text(event, "event_type")?; + let schema_version: i64 = event + .get("schema_version") + .and_then(Value::as_i64) + .unwrap_or_default(); + let policy = lifecycle_export_policy(event_type, is_terminal(event_type, schema_version)); + let mut columns: Vec = vec![ + ("event_id", text(required_text(event, "event_id")?)), + ( + "invocation_sequence", + event + .get("invocation_sequence") + .and_then(Value::as_i64) + .map_or(RecallValue::Null, RecallValue::Integer), + ), + ("occurred_at", text(required_text(event, "occurred_at")?)), + ("event_type", text(event_type)), + ("event_kind", text(policy.kind)), + ("severity", text(policy.severity.value())), + ("producer", text(required_text(event, "producer")?)), + ("event_schema_version", RecallValue::Integer(schema_version)), + ]; + columns.extend(CORRELATION_FIELDS.iter().map(|field| { + ( + *field, + optional_text(event.get(*field).and_then(Value::as_str)), + ) + })); + Ok(columns) +} + +/// Payload fields promoted to columns, then the payload itself. +pub(crate) fn payload_columns(payload: &Map) -> Vec { + let mut columns: Vec = PROMOTED_TEXT_FIELDS + .iter() + .map(|field| { + ( + *field, + optional_text(payload.get(*field).and_then(Value::as_str)), + ) + }) + .collect(); + columns.extend(PROMOTED_INTEGER_FIELDS.iter().map(|field| { + let value = payload.get(*field).and_then(Value::as_i64); + ( + *field, + value.map_or(RecallValue::Null, RecallValue::Integer), + ) + })); + columns.extend(PROMOTED_REAL_FIELDS.iter().map(|field| { + let value = payload.get(*field).and_then(Value::as_f64); + (*field, value.map_or(RecallValue::Null, RecallValue::Real)) + })); + columns.push(("payload", text(Value::Object(payload.clone()).to_string()))); + columns +} diff --git a/crates/sqlbuild-recall/src/records/_helpers/ids.rs b/crates/sqlbuild-recall/src/records/_helpers/ids.rs new file mode 100644 index 0000000000..903dc88854 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/_helpers/ids.rs @@ -0,0 +1,54 @@ +//! Deterministic record IDs and content digests. + +use sha2::{Digest, Sha256}; + +use crate::records::constants::{DIAGNOSTIC_RECORD_ID_INFIX, OUTPUT_RECORD_ID_INFIX}; +use crate::records::models::RecallValue; +use crate::schema::constants::UNHASHED_COLUMNS; + +/// `:command-output:`, the command-output sink record ID scheme. +pub(crate) fn output_record_id(invocation_id: &str, sequence: u64) -> String { + format!("{invocation_id}:{OUTPUT_RECORD_ID_INFIX}:{sequence}") +} + +/// `:diagnostic:`. +pub(crate) fn diagnostic_record_id(invocation_id: &str, sequence: u64) -> String { + format!("{invocation_id}:{DIAGNOSTIC_RECORD_ID_INFIX}:{sequence}") +} + +fn framed(tag: &[u8; 1], bytes: &[u8]) -> Vec { + let mut frame: Vec = Vec::with_capacity(bytes.len() + 9); + frame.extend_from_slice(tag); + frame.extend_from_slice(&(bytes.len() as u64).to_le_bytes()); + frame.extend_from_slice(bytes); + frame +} + +fn encoded(value: &RecallValue) -> Vec { + match value { + RecallValue::Null => b"0".to_vec(), + RecallValue::Integer(number) => [b"i".as_slice(), &number.to_le_bytes()].concat(), + RecallValue::Real(number) => [b"r".as_slice(), &number.to_bits().to_le_bytes()].concat(), + RecallValue::Text(text) => framed(b"t", text.as_bytes()), + RecallValue::Blob(data) => framed(b"b", data), + } +} + +/// Hex SHA-256 over every named value except `ingested_at` and `content_sha256`, by column name. +pub(crate) fn content_sha256(values: &[(&str, RecallValue)]) -> String { + let mut sorted: Vec<&(&str, RecallValue)> = values + .iter() + .filter(|(name, _)| !UNHASHED_COLUMNS.contains(name)) + .collect(); + sorted.sort_by(|left, right| left.0.cmp(right.0)); + let mut hasher = Sha256::new(); + for (name, value) in sorted { + hasher.update(framed(b"n", name.as_bytes())); + hasher.update(encoded(value)); + } + hasher + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} diff --git a/crates/sqlbuild-recall/src/records/_helpers/mod.rs b/crates/sqlbuild-recall/src/records/_helpers/mod.rs new file mode 100644 index 0000000000..eaf86fcc66 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod events; +pub(crate) mod ids; +pub(crate) mod rows; +pub(crate) mod time; diff --git a/crates/sqlbuild-recall/src/records/_helpers/rows.rs b/crates/sqlbuild-recall/src/records/_helpers/rows.rs new file mode 100644 index 0000000000..6cf27c265f --- /dev/null +++ b/crates/sqlbuild-recall/src/records/_helpers/rows.rs @@ -0,0 +1,45 @@ +//! Assemble a row: the dataset's values, the common columns and the content digest. + +use crate::records::_helpers::ids::content_sha256; +use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; +use crate::schema::constants::RECALL_SCHEMA_VERSION; +use crate::schema::models::Table; + +pub(crate) fn text(value: impl Into) -> RecallValue { + RecallValue::Text(value.into()) +} + +pub(crate) fn optional_text(value: Option<&str>) -> RecallValue { + value.map_or(RecallValue::Null, |value| { + RecallValue::Text(value.to_owned()) + }) +} + +pub(crate) fn integer(value: u64) -> RecallValue { + RecallValue::Integer(i64::try_from(value).unwrap_or(i64::MAX)) +} + +pub(crate) fn boolean(value: bool) -> RecallValue { + RecallValue::Integer(i64::from(value)) +} + +/// `values` plus the envelope columns and `content_sha256`. +pub(crate) fn finish_row( + table: Table, + envelope: &RowEnvelope, + mut values: Vec<(&'static str, RecallValue)>, +) -> RecallRow { + values.push(("invocation_id", text(&envelope.invocation_id))); + values.push(( + "invocation_started_at", + text(&envelope.invocation_started_at), + )); + values.push(( + "schema_version", + RecallValue::Integer(RECALL_SCHEMA_VERSION), + )); + values.push(("producer_version", text(&envelope.producer_version))); + let digest: String = content_sha256(&values); + values.push(("content_sha256", RecallValue::Text(digest))); + RecallRow { table, values } +} diff --git a/crates/sqlbuild-recall/src/records/_helpers/time.rs b/crates/sqlbuild-recall/src/records/_helpers/time.rs new file mode 100644 index 0000000000..d32586c82e --- /dev/null +++ b/crates/sqlbuild-recall/src/records/_helpers/time.rs @@ -0,0 +1,101 @@ +//! UTC timestamps as Recall stores them: `YYYY-MM-DDTHH:MM:SS.ffffffZ`. + +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::records::constants::{ + DECIMAL_RADIX, FEBRUARY, MARCH_BASED_JANUARY_INDEX, MICROS_PER_SECOND, SECONDS_PER_DAY, +}; + +/// Days since 1970-01-01 of a proleptic Gregorian date. +fn days_from_civil(year: i64, month: i64, day: i64) -> i64 { + let year = if month <= FEBRUARY { year - 1 } else { year }; + let era = year.div_euclid(400); + let year_of_era = year - era * 400; + let month_index = (month + 9) % 12; + let day_of_year = (153 * month_index + 2) / 5 + day - 1; + let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + era * 146_097 + day_of_era - 719_468 +} + +/// The date of a day count since 1970-01-01. +fn civil_from_days(days: i64) -> (i64, i64, i64) { + let shifted = days + 719_468; + let era = shifted.div_euclid(146_097); + let day_of_era = shifted - era * 146_097; + let year_of_era = + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_index + 2) / 5 + 1; + let month = if month_index < MARCH_BASED_JANUARY_INDEX { + month_index + 3 + } else { + month_index - 9 + }; + let year = year_of_era + era * 400 + i64::from(month <= FEBRUARY); + (year, month, day) +} + +/// Microseconds since the epoch as a stored timestamp. +pub(crate) fn format_micros(micros: i64) -> String { + let seconds = micros.div_euclid(MICROS_PER_SECOND); + let fraction = micros.rem_euclid(MICROS_PER_SECOND); + let days = seconds.div_euclid(SECONDS_PER_DAY); + let second_of_day = seconds.rem_euclid(SECONDS_PER_DAY); + let (year, month, day) = civil_from_days(days); + format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}.{fraction:06}Z", + second_of_day / 3600, + second_of_day % 3600 / 60, + second_of_day % 60 + ) +} + +/// Microseconds since the epoch of a system time. +pub(crate) fn system_micros(time: SystemTime) -> i64 { + match time.duration_since(UNIX_EPOCH) { + Ok(elapsed) => i64::try_from(elapsed.as_micros()).unwrap_or(i64::MAX), + Err(before) => -i64::try_from(before.duration().as_micros()).unwrap_or(i64::MAX), + } +} + +fn digits(text: &str) -> Option { + if text.is_empty() { + return None; + } + text.bytes().try_fold(0_i64, |total, byte| { + let digit = char::from(byte).to_digit(DECIMAL_RADIX)?; + total + .checked_mul(i64::from(DECIMAL_RADIX))? + .checked_add(i64::from(digit)) + }) +} + +/// Microseconds since the epoch of an ISO-8601 timestamp in UTC (`Z` or `+00:00`) or naive UTC. +pub(crate) fn parse_micros(text: &str) -> Option { + let text = text + .strip_suffix('Z') + .or_else(|| text.strip_suffix("+00:00")) + .unwrap_or(text); + let (date, time) = text.split_once(['T', ' '])?; + let mut date_parts = date.split('-'); + let year = digits(date_parts.next()?)?; + let month = digits(date_parts.next()?)?; + let day = digits(date_parts.next()?)?; + let (clock, fraction) = time.split_once('.').unwrap_or((time, "")); + let mut clock_parts = clock.split(':'); + let hour = digits(clock_parts.next()?)?; + let minute = digits(clock_parts.next()?)?; + let second = digits(clock_parts.next()?)?; + if date_parts.next().is_some() || clock_parts.next().is_some() { + return None; + } + let micros = if fraction.is_empty() { + 0 + } else { + let padded: String = fraction.chars().chain("000000".chars()).take(6).collect(); + digits(&padded)? + }; + let days = days_from_civil(year, month, day); + Some((days * SECONDS_PER_DAY + hour * 3600 + minute * 60 + second) * MICROS_PER_SECOND + micros) +} diff --git a/crates/sqlbuild-recall/src/records/constants.rs b/crates/sqlbuild-recall/src/records/constants.rs new file mode 100644 index 0000000000..69f783962c --- /dev/null +++ b/crates/sqlbuild-recall/src/records/constants.rs @@ -0,0 +1,48 @@ +//! Record vocabularies and limits. + +/// The largest output chunk, in bytes. +pub const MAX_OUTPUT_CHUNK_BYTES: usize = 64 * 1024; +/// The largest external context, in bytes of JSON. +pub const MAX_EXTERNAL_CONTEXT_BYTES: usize = 16 * 1024; +/// The output record ID infix, shared with command-output sink records. +pub const OUTPUT_RECORD_ID_INFIX: &str = "command-output"; +pub const DIAGNOSTIC_RECORD_ID_INFIX: &str = "diagnostic"; +/// Artifact digests are content digests with this prefix, as statement `sql_digest` without it. +pub const ARTIFACT_DIGEST_PREFIX: &str = "sha256:"; +pub const SQL_ARTIFACT_KIND: &str = "sql"; +pub const SQL_MEDIA_TYPE: &str = "text/x-sql"; +pub const EXECUTED_SQL_ROLE: &str = "executed_sql"; +/// Delivery stream names, one `recall_delivery` row each. +pub const DELIVERY_STREAMS: [&str; 5] = + ["invocation", "output", "events", "diagnostics", "artifacts"]; +/// Diagnostic severities, least severe first. +pub const SEVERITY_ORDER: [&str; 6] = ["trace", "debug", "info", "warning", "error", "critical"]; +/// Event payload fields promoted to their own `recall_events` columns. +pub const PROMOTED_TEXT_FIELDS: [&str; 11] = [ + "command", + "resource_kind", + "resource_name", + "error_code", + "error_type", + "query_id", + "sql_digest", + "intent", + "statement_kind", + "purpose", + "outcome", +]; +pub const PROMOTED_INTEGER_FIELDS: [&str; 3] = ["exit_code", "row_count", "affected_rows"]; +pub const PROMOTED_REAL_FIELDS: [&str; 1] = ["duration_ms"]; +pub const CORRELATION_FIELDS: [&str; 5] = [ + "run_id", + "resource_id", + "resource_attempt_id", + "operation_id", + "statement_id", +]; +pub(crate) const MICROS_PER_SECOND: i64 = 1_000_000; +pub(crate) const SECONDS_PER_DAY: i64 = 86_400; +pub(crate) const DECIMAL_RADIX: u32 = 10; +pub(crate) const FEBRUARY: i64 = 2; +/// In a year counted from March, January is month index 10. +pub(crate) const MARCH_BASED_JANUARY_INDEX: i64 = 10; diff --git a/crates/sqlbuild-recall/src/records/errors.rs b/crates/sqlbuild-recall/src/records/errors.rs new file mode 100644 index 0000000000..e0f42c5b60 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/errors.rs @@ -0,0 +1,23 @@ +//! Why an input cannot become a Recall record. + +/// A rejected record input, with a message naming the problem. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RecordError { + pub message: String, +} + +impl RecordError { + pub(crate) fn new(message: impl Into) -> Self { + Self { + message: message.into(), + } + } +} + +impl std::fmt::Display for RecordError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.message) + } +} + +impl std::error::Error for RecordError {} diff --git a/crates/sqlbuild-recall/src/records/main/artifact_rows.rs b/crates/sqlbuild-recall/src/records/main/artifact_rows.rs new file mode 100644 index 0000000000..22379043b5 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/artifact_rows.rs @@ -0,0 +1,42 @@ +//! `artifact_rows`: the content row of one artifact and the reference row naming it. + +use crate::records::_helpers::rows::{finish_row, integer, optional_text, text}; +use crate::records::models::{ArtifactInput, RecallRow, RecallValue, RowEnvelope}; +use crate::schema::models::Table; + +/// `[recall_artifacts row, recall_artifact_refs row]`, written in one batch so retention never sees the content without its reference. +#[must_use] +pub fn artifact_rows(envelope: &RowEnvelope, artifact: &ArtifactInput) -> [RecallRow; 2] { + [ + finish_row( + Table::Artifacts, + envelope, + vec![ + ("digest", text(&artifact.digest)), + ("kind", text(&artifact.kind)), + ("media_type", text(&artifact.media_type)), + ("size", integer(artifact.content.len() as u64)), + ("content", RecallValue::Blob(artifact.content.clone())), + ("seen_at", text(&artifact.seen_at)), + ], + ), + finish_row( + Table::ArtifactRefs, + envelope, + vec![ + ("digest", text(&artifact.digest)), + ("ref_id", text(&artifact.ref_id)), + ( + "statement_id", + optional_text(artifact.statement_id.as_deref()), + ), + ( + "resource_id", + optional_text(artifact.resource_id.as_deref()), + ), + ("role", text(&artifact.role)), + ("referenced_at", text(&artifact.seen_at)), + ], + ), + ] +} diff --git a/crates/sqlbuild-recall/src/records/main/conflict_row.rs b/crates/sqlbuild-recall/src/records/main/conflict_row.rs new file mode 100644 index 0000000000..89265416b2 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/conflict_row.rs @@ -0,0 +1,26 @@ +//! `conflict_row`: evidence that a record key was reused with different content. + +use crate::records::_helpers::rows::{finish_row, text}; +use crate::records::models::{RecallRow, RowEnvelope}; +use crate::schema::models::Table; + +/// A `recall_conflicts` row; the first stored row is kept and the rejected digest recorded. +#[must_use] +pub fn conflict_row( + envelope: &RowEnvelope, + (dataset, record_key): (&str, &str), + (existing_sha256, rejected_sha256): (&str, &str), + detected_at: &str, +) -> RecallRow { + finish_row( + Table::Conflicts, + envelope, + vec![ + ("dataset", text(dataset)), + ("record_key", text(record_key)), + ("existing_sha256", text(existing_sha256)), + ("rejected_sha256", text(rejected_sha256)), + ("detected_at", text(detected_at)), + ], + ) +} diff --git a/crates/sqlbuild-recall/src/records/main/delivery_row.rs b/crates/sqlbuild-recall/src/records/main/delivery_row.rs new file mode 100644 index 0000000000..950768aa11 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/delivery_row.rs @@ -0,0 +1,30 @@ +//! `delivery_row`: the final accounting of one stream, written last. + +use crate::records::_helpers::rows::{boolean, finish_row, integer, text}; +use crate::records::models::{DeliveryCounts, RecallRow, RowEnvelope}; +use crate::schema::models::Table; + +/// The delivery row of `stream`. +#[must_use] +pub fn delivery_row( + envelope: &RowEnvelope, + stream: &str, + (counts, flush_complete): (&DeliveryCounts, bool), + written_at: &str, +) -> RecallRow { + finish_row( + Table::Delivery, + envelope, + vec![ + ("stream", text(stream)), + ("accepted", integer(counts.accepted)), + ("delivered", integer(counts.delivered)), + ("filtered", integer(counts.filtered)), + ("dropped", integer(counts.dropped)), + ("failed", integer(counts.failed)), + ("conflicts", integer(counts.conflicts)), + ("flush_complete", boolean(flush_complete)), + ("written_at", text(written_at)), + ], + ) +} diff --git a/crates/sqlbuild-recall/src/records/main/diagnostic_input_from_json.rs b/crates/sqlbuild-recall/src/records/main/diagnostic_input_from_json.rs new file mode 100644 index 0000000000..8d6ba82341 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/diagnostic_input_from_json.rs @@ -0,0 +1,51 @@ +//! `diagnostic_input_from_json`: a diagnostic from `diagnostic_log_to_json` text. + +use serde_json::{Map, Value}; + +use crate::records::errors::RecordError; +use crate::records::models::{Correlations, DiagnosticInput}; + +fn text(object: &Map, field: &str) -> Result { + object + .get(field) + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| RecordError::new(format!("diagnostic {field} must be a string"))) +} + +fn optional(object: &Map, field: &str) -> Option { + object.get(field).and_then(Value::as_str).map(str::to_owned) +} + +/// The diagnostic encoded in `json`, numbered `sequence` within its invocation. +pub fn diagnostic_input_from_json( + json: &str, + sequence: u64, +) -> Result { + let parsed: Value = serde_json::from_str(json) + .map_err(|error| RecordError::new(format!("diagnostic is not JSON: {error}")))?; + let Value::Object(object) = parsed else { + return Err(RecordError::new("diagnostic must be a JSON object")); + }; + let fields: String = object + .get("fields") + .filter(|value| value.is_object()) + .map_or_else(|| "{}".to_owned(), Value::to_string); + Ok(DiagnosticInput { + sequence, + occurred_at: text(&object, "occurred_at")?, + severity: text(&object, "severity")?, + logger: text(&object, "logger")?, + source: text(&object, "source")?, + message: text(&object, "message")?, + fields, + log_stream_id: optional(&object, "log_stream_id"), + correlations: Correlations { + run_id: optional(&object, "run_id"), + resource_id: optional(&object, "resource_id"), + resource_attempt_id: optional(&object, "resource_attempt_id"), + operation_id: optional(&object, "operation_id"), + statement_id: optional(&object, "statement_id"), + }, + }) +} diff --git a/crates/sqlbuild-recall/src/records/main/diagnostic_row.rs b/crates/sqlbuild-recall/src/records/main/diagnostic_row.rs new file mode 100644 index 0000000000..c839d2d5a3 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/diagnostic_row.rs @@ -0,0 +1,53 @@ +//! `diagnostic_row`: the `recall_diagnostics` row of one diagnostic. + +use crate::records::_helpers::ids::diagnostic_record_id; +use crate::records::_helpers::rows::{finish_row, integer, optional_text, text}; +use crate::records::models::{DiagnosticInput, RecallRow, RowEnvelope}; +use crate::schema::models::Table; + +/// The row of `diagnostic`, keyed by invocation and sequence. +#[must_use] +pub fn diagnostic_row(envelope: &RowEnvelope, diagnostic: &DiagnosticInput) -> RecallRow { + let correlations = &diagnostic.correlations; + finish_row( + Table::Diagnostics, + envelope, + vec![ + ( + "record_id", + text(diagnostic_record_id( + &envelope.invocation_id, + diagnostic.sequence, + )), + ), + ("sequence", integer(diagnostic.sequence)), + ("occurred_at", text(&diagnostic.occurred_at)), + ("severity", text(&diagnostic.severity)), + ("logger", text(&diagnostic.logger)), + ("source", text(&diagnostic.source)), + ("message", text(&diagnostic.message)), + ("fields", text(&diagnostic.fields)), + ( + "log_stream_id", + optional_text(diagnostic.log_stream_id.as_deref()), + ), + ("run_id", optional_text(correlations.run_id.as_deref())), + ( + "resource_id", + optional_text(correlations.resource_id.as_deref()), + ), + ( + "resource_attempt_id", + optional_text(correlations.resource_attempt_id.as_deref()), + ), + ( + "operation_id", + optional_text(correlations.operation_id.as_deref()), + ), + ( + "statement_id", + optional_text(correlations.statement_id.as_deref()), + ), + ], + ) +} diff --git a/crates/sqlbuild-recall/src/records/main/event_row.rs b/crates/sqlbuild-recall/src/records/main/event_row.rs new file mode 100644 index 0000000000..de0b2f69ad --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/event_row.rs @@ -0,0 +1,22 @@ +//! `event_row`: the `recall_events` row of one lifecycle event's canonical JSON. + +use serde_json::{Map, Value}; + +use crate::records::_helpers::events::{envelope_columns, event_object, payload_columns}; +use crate::records::_helpers::rows::finish_row; +use crate::records::errors::RecordError; +use crate::records::models::{RecallRow, RowEnvelope}; +use crate::schema::models::Table; + +/// The row of one event, as `lifecycle_event_to_json` encodes it. +pub fn event_row(envelope: &RowEnvelope, event_json: &str) -> Result { + let event: Map = event_object(event_json)?; + let mut columns = envelope_columns(&event)?; + let empty: Map = Map::new(); + let payload = event + .get("payload") + .and_then(Value::as_object) + .unwrap_or(&empty); + columns.extend(payload_columns(payload)); + Ok(finish_row(Table::Events, envelope, columns)) +} diff --git a/crates/sqlbuild-recall/src/records/main/format_timestamp.rs b/crates/sqlbuild-recall/src/records/main/format_timestamp.rs new file mode 100644 index 0000000000..2b3a8b5012 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/format_timestamp.rs @@ -0,0 +1,11 @@ +//! `format_timestamp`: a system time as Recall stores it. + +use std::time::SystemTime; + +use crate::records::_helpers::time::{format_micros, system_micros}; + +/// `YYYY-MM-DDTHH:MM:SS.ffffffZ` in UTC. +#[must_use] +pub fn format_timestamp(time: SystemTime) -> String { + format_micros(system_micros(time)) +} diff --git a/crates/sqlbuild-recall/src/records/main/heartbeat_row.rs b/crates/sqlbuild-recall/src/records/main/heartbeat_row.rs new file mode 100644 index 0000000000..340c09365f --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/heartbeat_row.rs @@ -0,0 +1,27 @@ +//! `heartbeat_row`: one liveness fact of a running invocation. + +use crate::records::_helpers::rows::{finish_row, integer, optional_text, text}; +use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; +use crate::schema::models::Table; + +/// A heartbeat at `beat_at` that declares the invocation abandoned after `stale_at`. +#[must_use] +pub fn heartbeat_row( + envelope: &RowEnvelope, + (beat_at, stale_at): (&str, &str), + (host_token, pid): (Option<&str>, Option), +) -> RecallRow { + finish_row( + Table::Heartbeats, + envelope, + vec![ + ("beat_at", text(beat_at)), + ("stale_at", text(stale_at)), + ("host_token", optional_text(host_token)), + ( + "pid", + pid.map_or(RecallValue::Null, |pid| integer(u64::from(pid))), + ), + ], + ) +} diff --git a/crates/sqlbuild-recall/src/records/main/invocation_row.rs b/crates/sqlbuild-recall/src/records/main/invocation_row.rs new file mode 100644 index 0000000000..53c678fd43 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/invocation_row.rs @@ -0,0 +1,74 @@ +//! `invocation_row`: the `recall_invocations` context row. + +use crate::records::_helpers::rows::{boolean, finish_row, integer, optional_text, text}; +use crate::records::models::{InvocationContext, RecallRow, RecallValue, RowEnvelope}; +use crate::schema::models::Table; + +fn json_list(items: &[String]) -> RecallValue { + text(serde_json::Value::from(items.to_vec()).to_string()) +} + +/// The context row of one invocation. +#[must_use] +pub fn invocation_row(envelope: &RowEnvelope, context: &InvocationContext) -> RecallRow { + finish_row( + Table::Invocations, + envelope, + vec![ + ("command", optional_text(context.command.as_deref())), + ("subcommand", optional_text(context.subcommand.as_deref())), + ("selection", json_list(&context.selection)), + ("exclusion", json_list(&context.exclusion)), + ( + "project_name", + optional_text(context.project_name.as_deref()), + ), + ( + "project_dir_hash", + optional_text(context.project_dir_hash.as_deref()), + ), + ("target", optional_text(context.target.as_deref())), + ("adapter", optional_text(context.adapter.as_deref())), + ( + "destination_database", + optional_text(context.destination_database.as_deref()), + ), + ( + "destination_schema", + optional_text(context.destination_schema.as_deref()), + ), + ( + "destination_fingerprint", + optional_text(context.destination_fingerprint.as_deref()), + ), + ( + "sqlbuild_version", + optional_text(context.sqlbuild_version.as_deref()), + ), + ( + "git_revision", + optional_text(context.git_revision.as_deref()), + ), + ( + "git_dirty", + context.git_dirty.map_or(RecallValue::Null, boolean), + ), + ("host_token", optional_text(context.host_token.as_deref())), + ( + "pid", + context + .pid + .map_or(RecallValue::Null, |pid| integer(u64::from(pid))), + ), + ( + "external_context", + optional_text(context.external_context.as_deref()), + ), + ( + "recall_filter_active", + boolean(context.recall_filter_active), + ), + ("argv_digest", optional_text(context.argv_digest.as_deref())), + ], + ) +} diff --git a/crates/sqlbuild-recall/src/records/main/meta_row.rs b/crates/sqlbuild-recall/src/records/main/meta_row.rs new file mode 100644 index 0000000000..ecf40303f5 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/meta_row.rs @@ -0,0 +1,24 @@ +//! `meta_row`: one insert-only store metadata fact. + +use crate::records::_helpers::rows::{finish_row, text}; +use crate::records::models::{RecallRow, RowEnvelope}; +use crate::schema::models::Table; + +/// A `recall_meta` row. +#[must_use] +pub fn meta_row( + envelope: &RowEnvelope, + (component, key, value): (&str, &str, &str), + recorded_at: &str, +) -> RecallRow { + finish_row( + Table::Meta, + envelope, + vec![ + ("component", text(component)), + ("key", text(key)), + ("value", text(value)), + ("recorded_at", text(recorded_at)), + ], + ) +} diff --git a/crates/sqlbuild-recall/src/records/main/mod.rs b/crates/sqlbuild-recall/src/records/main/mod.rs new file mode 100644 index 0000000000..f1d7c2d3f7 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/mod.rs @@ -0,0 +1,13 @@ +pub mod artifact_rows; +pub mod conflict_row; +pub mod delivery_row; +pub mod diagnostic_input_from_json; +pub mod diagnostic_row; +pub mod event_row; +pub mod format_timestamp; +pub mod heartbeat_row; +pub mod invocation_row; +pub mod meta_row; +pub mod output_row; +pub mod parse_timestamp; +pub mod sha256_hex; diff --git a/crates/sqlbuild-recall/src/records/main/output_row.rs b/crates/sqlbuild-recall/src/records/main/output_row.rs new file mode 100644 index 0000000000..9d312eecc6 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/output_row.rs @@ -0,0 +1,29 @@ +//! `output_row`: the `recall_output` row of one chunk. + +use crate::records::_helpers::ids::output_record_id; +use crate::records::_helpers::rows::{finish_row, integer, text}; +use crate::records::models::{OutputChunk, RecallRow, RecallValue, RowEnvelope}; +use crate::schema::models::Table; + +/// The row of `chunk`, keyed by invocation and sequence. +#[must_use] +pub fn output_row(envelope: &RowEnvelope, chunk: &OutputChunk) -> RecallRow { + finish_row( + Table::Output, + envelope, + vec![ + ( + "record_id", + text(output_record_id(&envelope.invocation_id, chunk.sequence)), + ), + ("sequence", integer(chunk.sequence)), + ("stream", text(chunk.stream.as_str())), + ("occurred_at", text(&chunk.occurred_at)), + ("data", RecallValue::Blob(chunk.data.clone())), + ("byte_offset", integer(chunk.byte_offset)), + ("byte_count", integer(chunk.data.len() as u64)), + ("priority", text("output")), + ("dropped_before", integer(chunk.dropped_before)), + ], + ) +} diff --git a/crates/sqlbuild-recall/src/records/main/parse_timestamp.rs b/crates/sqlbuild-recall/src/records/main/parse_timestamp.rs new file mode 100644 index 0000000000..c5a2470f0d --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/parse_timestamp.rs @@ -0,0 +1,17 @@ +//! `parse_timestamp`: a stored or ISO-8601 UTC timestamp as a system time. + +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use crate::records::_helpers::time::parse_micros; + +/// The time of `text`, or `None` when it is not a UTC ISO-8601 timestamp. +#[must_use] +pub fn parse_timestamp(text: &str) -> Option { + let micros = parse_micros(text)?; + let magnitude = Duration::from_micros(micros.unsigned_abs()); + if micros >= 0 { + UNIX_EPOCH.checked_add(magnitude) + } else { + UNIX_EPOCH.checked_sub(magnitude) + } +} diff --git a/crates/sqlbuild-recall/src/records/main/sha256_hex.rs b/crates/sqlbuild-recall/src/records/main/sha256_hex.rs new file mode 100644 index 0000000000..f9fe2b38fb --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/sha256_hex.rs @@ -0,0 +1,14 @@ +//! `sha256_hex`: the lowercase hexadecimal SHA-256 of bytes. + +use sha2::{Digest, Sha256}; + +/// Lowercase hex SHA-256 of `data`. +#[must_use] +pub fn sha256_hex(data: &[u8]) -> String { + let digest = Sha256::digest(data); + let mut text = String::with_capacity(64); + for byte in digest { + text.push_str(&format!("{byte:02x}")); + } + text +} diff --git a/crates/sqlbuild-recall/src/records/mod.rs b/crates/sqlbuild-recall/src/records/mod.rs new file mode 100644 index 0000000000..bfc1b59309 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/mod.rs @@ -0,0 +1,10 @@ +//! Recall records: deterministic IDs, content digests and the rows each dataset writes. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-recall/src/records/models.rs b/crates/sqlbuild-recall/src/records/models.rs new file mode 100644 index 0000000000..a4b1c5d06f --- /dev/null +++ b/crates/sqlbuild-recall/src/records/models.rs @@ -0,0 +1,165 @@ +//! Recall record inputs and the rows they become. + +use crate::schema::models::Table; + +/// One stored value. +#[derive(Clone, Debug, PartialEq)] +pub enum RecallValue { + Null, + Integer(i64), + Real(f64), + Text(String), + Blob(Vec), +} + +/// One row bound for a Recall table: named values, without `ingested_at`, which the store sets. +#[derive(Clone, Debug, PartialEq)] +pub struct RecallRow { + pub table: Table, + pub values: Vec<(&'static str, RecallValue)>, +} + +impl RecallRow { + /// The value of `column`, if the row has it. + #[must_use] + pub fn value(&self, column: &str) -> Option<&RecallValue> { + self.values + .iter() + .find(|(name, _)| *name == column) + .map(|(_, value)| value) + } + + /// The row's content digest. + #[must_use] + pub fn content_sha256(&self) -> Option<&str> { + match self.value("content_sha256") { + Some(RecallValue::Text(text)) => Some(text), + _ => None, + } + } +} + +/// What every row of one invocation carries. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RowEnvelope { + pub invocation_id: String, + pub invocation_started_at: String, + pub producer_version: String, +} + +/// A process output stream. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum OutputStream { + Stdout, + Stderr, +} + +impl OutputStream { + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::Stdout => "stdout", + Self::Stderr => "stderr", + } + } + + /// The stream named `name`. + #[must_use] + pub fn from_name(name: &str) -> Option { + match name { + "stdout" => Some(Self::Stdout), + "stderr" => Some(Self::Stderr), + _ => None, + } + } +} + +/// One exact-byte output chunk. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OutputChunk { + /// Per invocation, from 0, shared by both streams. + pub sequence: u64, + pub stream: OutputStream, + pub occurred_at: String, + pub data: Vec, + /// Bytes of the same stream before this chunk. + pub byte_offset: u64, + /// Output records the queue dropped just before this one. + pub dropped_before: u64, +} + +/// The context of one invocation, written once when it starts. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct InvocationContext { + pub command: Option, + pub subcommand: Option, + pub selection: Vec, + pub exclusion: Vec, + pub project_name: Option, + pub project_dir_hash: Option, + pub target: Option, + pub adapter: Option, + pub destination_database: Option, + pub destination_schema: Option, + pub destination_fingerprint: Option, + pub sqlbuild_version: Option, + pub git_revision: Option, + pub git_dirty: Option, + pub host_token: Option, + pub pid: Option, + /// JSON object text, at most 16 KiB. + pub external_context: Option, + pub recall_filter_active: bool, + pub argv_digest: Option, +} + +/// The correlation identifiers of a fact. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Correlations { + pub run_id: Option, + pub resource_id: Option, + pub resource_attempt_id: Option, + pub operation_id: Option, + pub statement_id: Option, +} + +/// One structured diagnostic. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DiagnosticInput { + pub sequence: u64, + pub occurred_at: String, + pub severity: String, + pub logger: String, + pub source: String, + pub message: String, + /// JSON object text. + pub fields: String, + pub log_stream_id: Option, + pub correlations: Correlations, +} + +/// One captured artifact and the reference that names it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ArtifactInput { + /// `sha256:` of `content`. + pub digest: String, + pub kind: String, + pub media_type: String, + pub content: Vec, + pub seen_at: String, + pub ref_id: String, + pub role: String, + pub statement_id: Option, + pub resource_id: Option, +} + +/// Delivery accounting of one stream. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DeliveryCounts { + pub accepted: u64, + pub delivered: u64, + pub filtered: u64, + pub dropped: u64, + pub failed: u64, + pub conflicts: u64, +} diff --git a/crates/sqlbuild-recall/src/records/tests.rs b/crates/sqlbuild-recall/src/records/tests.rs new file mode 100644 index 0000000000..540eed7214 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/tests.rs @@ -0,0 +1,2 @@ +#[path = "tests/test_records.rs"] +mod records; diff --git a/crates/sqlbuild-recall/src/records/tests/test_records.rs b/crates/sqlbuild-recall/src/records/tests/test_records.rs new file mode 100644 index 0000000000..48eec2925d --- /dev/null +++ b/crates/sqlbuild-recall/src/records/tests/test_records.rs @@ -0,0 +1,135 @@ +use std::time::{Duration, UNIX_EPOCH}; + +use crate::records::_helpers::ids::content_sha256; +use crate::records::_helpers::ids::diagnostic_record_id; +use crate::records::_helpers::ids::output_record_id; +use crate::records::main::diagnostic_input_from_json::diagnostic_input_from_json; +use crate::records::main::event_row::event_row; +use crate::records::main::format_timestamp::format_timestamp; +use crate::records::main::output_row::output_row; +use crate::records::main::parse_timestamp::parse_timestamp; +use crate::records::models::{OutputChunk, OutputStream, RecallValue, RowEnvelope}; + +fn envelope() -> RowEnvelope { + RowEnvelope { + invocation_id: "inv1".to_owned(), + invocation_started_at: "2026-10-19T00:00:00.000000Z".to_owned(), + producer_version: "0.0.0".to_owned(), + } +} + +#[test] +fn given_sequences_when_ids_are_built_then_they_follow_the_record_id_scheme() { + assert_eq!(output_record_id("abc", 7), "abc:command-output:7"); + assert_eq!(diagnostic_record_id("abc", 0), "abc:diagnostic:0"); +} + +#[test] +fn given_times_when_formatted_and_parsed_then_they_round_trip() { + let cases = [ + (0_u64, "1970-01-01T00:00:00.000000Z"), + (951_782_400_123_456, "2000-02-29T00:00:00.123456Z"), + (1_792_281_600_000_001, "2026-10-18T00:00:00.000001Z"), + ]; + for (micros, expected) in cases { + let time = UNIX_EPOCH + Duration::from_micros(micros); + assert_eq!(format_timestamp(time), expected); + assert_eq!(parse_timestamp(expected), Some(time)); + } + assert_eq!( + parse_timestamp("2026-10-18T00:00:00+00:00"), + parse_timestamp("2026-10-18T00:00:00.000000Z") + ); + assert_eq!(parse_timestamp("yesterday"), None); +} + +#[test] +fn given_equal_and_different_content_when_hashed_then_only_content_matters() { + let base = vec![ + ("a", RecallValue::Text("x".to_owned())), + ("b", RecallValue::Integer(1)), + ]; + let reordered = vec![ + ("b", RecallValue::Integer(1)), + ("a", RecallValue::Text("x".to_owned())), + ("ingested_at", RecallValue::Text("later".to_owned())), + ]; + let changed = vec![ + ("a", RecallValue::Blob(b"x".to_vec())), + ("b", RecallValue::Integer(1)), + ]; + assert_eq!(content_sha256(&base), content_sha256(&reordered)); + assert_ne!(content_sha256(&base), content_sha256(&changed)); +} + +#[test] +fn given_a_chunk_when_rowed_then_bytes_and_offsets_are_kept() { + let row = output_row( + &envelope(), + &OutputChunk { + sequence: 3, + stream: OutputStream::Stderr, + occurred_at: "2026-10-19T00:00:01.000000Z".to_owned(), + data: vec![0, 255, 10], + byte_offset: 12, + dropped_before: 1, + }, + ); + assert_eq!( + row.value("data"), + Some(&RecallValue::Blob(vec![0, 255, 10])) + ); + assert_eq!(row.value("byte_count"), Some(&RecallValue::Integer(3))); + assert_eq!( + row.value("record_id"), + Some(&RecallValue::Text("inv1:command-output:3".to_owned())) + ); + assert!( + row.content_sha256() + .is_some_and(|digest| digest.len() == 64) + ); +} + +#[test] +fn given_lifecycle_json_when_rowed_then_dimensions_are_promoted() { + let json = r#"{"event_id":"e1","event_type":"statement_failed","schema_version":2, + "producer":"sqlbuild","producer_version":"1","occurred_at":"2026-10-19T00:00:02.000000Z", + "invocation_id":"inv1","invocation_sequence":4,"statement_id":"s1", + "payload":{"error_type":"ValueError","duration_ms":1.5,"sql_digest":"ab","purpose":"audit"}}"#; + let row = event_row(&envelope(), json).expect("valid event"); + assert_eq!( + row.value("event_kind"), + Some(&RecallValue::Text("statement".to_owned())) + ); + assert_eq!( + row.value("severity"), + Some(&RecallValue::Text("error".to_owned())) + ); + assert_eq!( + row.value("statement_id"), + Some(&RecallValue::Text("s1".to_owned())) + ); + assert_eq!(row.value("duration_ms"), Some(&RecallValue::Real(1.5))); + assert_eq!( + row.value("purpose"), + Some(&RecallValue::Text("audit".to_owned())) + ); + assert_eq!( + row.value("invocation_sequence"), + Some(&RecallValue::Integer(4)) + ); + assert!(event_row(&envelope(), "[]").is_err()); + assert!(event_row(&envelope(), r#"{"event_type":"x"}"#).is_err()); +} + +#[test] +fn given_diagnostic_json_when_parsed_then_correlations_are_kept() { + let json = r#"{"occurred_at":"2026-10-19T00:00:00.000000Z","severity":"warning", + "logger":"sqlbuild.x","source":"python","message":"careful","fields":{"k":1}, + "run_id":"r1"}"#; + let diagnostic = diagnostic_input_from_json(json, 5).expect("valid diagnostic"); + assert_eq!(diagnostic.sequence, 5); + assert_eq!(diagnostic.fields, r#"{"k":1}"#); + assert_eq!(diagnostic.correlations.run_id.as_deref(), Some("r1")); + assert!(diagnostic_input_from_json(r#"{"severity":"info"}"#, 0).is_err()); +} diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs new file mode 100644 index 0000000000..8fdce6aad8 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -0,0 +1,352 @@ +//! The bodies of the supported views in ClickHouse: rows deduplicated by key, within retention. + +use crate::schema::_helpers::rendering::clickhouse_table; +use crate::schema::constants::OUTPUT_TAIL_RECORDS; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::{DdlOptions, Table}; + +/// `table`'s rows within retention, first ingested copy per key. +fn deduplicated(table: Table, options: &DdlOptions) -> String { + let spec = table_spec(table); + format!( + "(SELECT * FROM {name} WHERE {expiry} >= now64(6) - INTERVAL {days} DAY \ + ORDER BY ingested_at ASC LIMIT 1 BY {key})", + name = clickhouse_table(spec.name, options), + expiry = spec.expiry_column, + days = options.retention_days, + key = spec.key.join(", "), + ) +} + +fn invocations(options: &DdlOptions) -> String { + let context = deduplicated(Table::Invocations, options); + let events = deduplicated(Table::Events, options); + let beats = deduplicated(Table::Heartbeats, options); + format!( + "WITH +ids AS ( + SELECT invocation_id, min(invocation_started_at) AS invocation_started_at + FROM ( + SELECT invocation_id, invocation_started_at FROM {context} + UNION ALL SELECT invocation_id, invocation_started_at FROM {beats} + UNION ALL SELECT invocation_id, invocation_started_at FROM {events} + WHERE event_type IN ('invocation_started', 'invocation_completed', 'invocation_failed') + ) + GROUP BY invocation_id +), +started AS ( + SELECT invocation_id, min(occurred_at) AS started_at, any(command) AS command + FROM {events} WHERE event_type = 'invocation_started' GROUP BY invocation_id +), +terminals AS ( + SELECT invocation_id, + argMax(event_type, tuple(invocation_sequence, occurred_at)) AS event_type, + argMax(occurred_at, tuple(invocation_sequence, occurred_at)) AS occurred_at, + argMax(exit_code, tuple(invocation_sequence, occurred_at)) AS exit_code, + argMax(duration_ms, tuple(invocation_sequence, occurred_at)) AS duration_ms, + argMax(error_type, tuple(invocation_sequence, occurred_at)) AS error_type, + argMax(error_code, tuple(invocation_sequence, occurred_at)) AS error_code + FROM {events} + WHERE event_type IN ('invocation_completed', 'invocation_failed') + GROUP BY invocation_id +), +beats AS ( + SELECT invocation_id, max(beat_at) AS last_heartbeat_at, max(stale_at) AS stale_at + FROM {beats} GROUP BY invocation_id +) +SELECT + ids.invocation_id AS invocation_id, + ids.invocation_started_at AS invocation_started_at, + coalesce(started.started_at, ids.invocation_started_at) AS started_at, + multiIf( + terminals.event_type = 'invocation_completed', 'completed', + terminals.event_type = 'invocation_failed', 'failed', + beats.stale_at IS NULL, 'unknown', + beats.stale_at >= now64(6), 'running', + 'abandoned' + ) AS status, + if(terminals.event_type IS NOT NULL, terminals.occurred_at, + if(beats.stale_at < now64(6), beats.last_heartbeat_at, NULL)) AS ended_at, + if(terminals.event_type IS NOT NULL, terminals.duration_ms, + if(beats.stale_at < now64(6), + toFloat64(dateDiff('millisecond', started_at, beats.last_heartbeat_at)), NULL)) + AS duration_ms, + terminals.exit_code AS exit_code, + terminals.error_type AS error_type, + terminals.error_code AS error_code, + coalesce(context.command, started.command) AS command, + context.subcommand AS subcommand, + context.target AS target, + context.adapter AS adapter, + context.project_name AS project_name, + context.selection AS selection, + context.exclusion AS exclusion, + context.sqlbuild_version AS sqlbuild_version, + context.git_revision AS git_revision, + context.git_dirty AS git_dirty, + context.host_token AS host_token, + context.pid AS pid, + context.external_context AS external_context, + context.recall_filter_active AS recall_filter_active, + beats.last_heartbeat_at AS last_heartbeat_at +FROM ids +LEFT JOIN {context} AS context ON context.invocation_id = ids.invocation_id +LEFT JOIN started ON started.invocation_id = ids.invocation_id +LEFT JOIN terminals ON terminals.invocation_id = ids.invocation_id +LEFT JOIN beats ON beats.invocation_id = ids.invocation_id +SETTINGS join_use_nulls = 1" + ) +} + +fn statements(options: &DdlOptions) -> String { + let events = deduplicated(Table::Events, options); + let artifacts = clickhouse_table(table_spec(Table::Artifacts).name, options); + format!( + "WITH +facts AS ( + SELECT * FROM {events} WHERE statement_id IS NOT NULL AND startsWith(event_type, 'statement_') +), +firsts AS ( + SELECT statement_id, + argMin(invocation_id, tuple(invocation_sequence, occurred_at)) AS invocation_id, + argMin(run_id, tuple(invocation_sequence, occurred_at)) AS run_id, + argMin(resource_id, tuple(invocation_sequence, occurred_at)) AS resource_id, + argMin(resource_attempt_id, tuple(invocation_sequence, occurred_at)) AS resource_attempt_id, + argMin(operation_id, tuple(invocation_sequence, occurred_at)) AS operation_id, + argMin(intent, tuple(invocation_sequence, occurred_at)) AS intent, + argMin(statement_kind, tuple(invocation_sequence, occurred_at)) AS statement_kind, + argMin(purpose, tuple(invocation_sequence, occurred_at)) AS purpose, + argMin(sql_digest, tuple(invocation_sequence, occurred_at)) AS sql_digest, + minIf(occurred_at, event_type = 'statement_started') AS started_at, + minIf(occurred_at, event_type = 'statement_submitted') AS submitted_at, + max(query_id) AS any_query_id + FROM facts GROUP BY statement_id +), +terminals AS ( + SELECT statement_id, + argMax(event_type, tuple(invocation_sequence, occurred_at)) AS event_type, + argMax(occurred_at, tuple(invocation_sequence, occurred_at)) AS ended_at, + argMax(query_id, tuple(invocation_sequence, occurred_at)) AS query_id, + argMax(error_type, tuple(invocation_sequence, occurred_at)) AS error_type, + argMax(error_code, tuple(invocation_sequence, occurred_at)) AS error_code, + argMax(duration_ms, tuple(invocation_sequence, occurred_at)) AS duration_ms, + argMax(row_count, tuple(invocation_sequence, occurred_at)) AS row_count, + argMax(affected_rows, tuple(invocation_sequence, occurred_at)) AS affected_rows + FROM facts + WHERE event_type IN ('statement_completed', 'statement_failed') + GROUP BY statement_id +), +resources AS ( + SELECT invocation_id, resource_id, min(resource_name) AS resource_name, + min(resource_kind) AS resource_kind + FROM {events} + WHERE resource_id IS NOT NULL AND resource_name IS NOT NULL + GROUP BY invocation_id, resource_id +) +SELECT + firsts.invocation_id AS invocation_id, + firsts.statement_id AS statement_id, + firsts.run_id AS run_id, + firsts.resource_id AS resource_id, + resources.resource_name AS resource_name, + resources.resource_kind AS resource_kind, + firsts.resource_attempt_id AS resource_attempt_id, + firsts.operation_id AS operation_id, + firsts.intent AS intent, + firsts.statement_kind AS statement_kind, + firsts.purpose AS purpose, + coalesce(terminals.query_id, firsts.any_query_id) AS query_id, + firsts.started_at AS started_at, + firsts.submitted_at AS submitted_at, + terminals.ended_at AS ended_at, + multiIf( + terminals.event_type = 'statement_completed', 'completed', + terminals.event_type = 'statement_failed', 'failed', + 'no_terminal' + ) AS outcome, + terminals.error_type AS error_type, + terminals.error_code AS error_code, + terminals.duration_ms AS duration_ms, + terminals.row_count AS row_count, + terminals.affected_rows AS affected_rows, + firsts.sql_digest AS sql_digest, + concat('sha256:', firsts.sql_digest) IN (SELECT digest FROM {artifacts}) AS sql_available +FROM firsts +LEFT JOIN terminals ON terminals.statement_id = firsts.statement_id +LEFT JOIN resources + ON resources.invocation_id = firsts.invocation_id + AND resources.resource_id = firsts.resource_id +SETTINGS join_use_nulls = 1" + ) +} + +fn audit_results(options: &DdlOptions) -> String { + let events = deduplicated(Table::Events, options); + format!( + "SELECT + invocation_id, + invocation_started_at, + event_id, + occurred_at, + run_id, + resource_id, + resource_attempt_id, + JSONExtract(payload, 'audit_name', 'Nullable(String)') AS audit_name, + JSONExtract(payload, 'audit_definition_name', 'Nullable(String)') AS audit_definition_name, + if(JSONHas(payload, 'target_name'), concat( + if(JSONHas(payload, 'target_database'), concat(JSONExtractString(payload, 'target_database'), '.'), ''), + if(JSONHas(payload, 'target_schema'), concat(JSONExtractString(payload, 'target_schema'), '.'), ''), + JSONExtractString(payload, 'target_name')), NULL) AS target_relation, + JSONExtract(payload, 'attached_target_kind', 'Nullable(String)') AS attached_target_kind, + JSONExtract(payload, 'attached_target_name', 'Nullable(String)') AS attached_target_name, + JSONExtract(payload, 'attached_column_name', 'Nullable(String)') AS column, + outcome, + JSONExtract(payload, 'severity', 'Nullable(String)') AS severity, + JSONExtract(payload, 'violation_count', 'Nullable(Int64)') AS violation_count, + JSONExtract(payload, 'measured_value', 'Nullable(Float64)') AS measured_value, + JSONExtract(payload, 'evaluation_mode', 'Nullable(String)') AS evaluation_mode, + JSONExtract(payload, 'execution_error', 'Nullable(String)') AS execution_error, + sql_digest, + JSONExtract(payload, 'sample_digest', 'Nullable(String)') AS sample_digest +FROM {events} +WHERE event_type = 'audit_completed'" + ) +} + +fn output_invocations(options: &DdlOptions) -> String { + let output = deduplicated(Table::Output, options); + let delivery = deduplicated(Table::Delivery, options); + let context = deduplicated(Table::Invocations, options); + format!( + "WITH +received AS ( + SELECT invocation_id, + min(invocation_started_at) AS invocation_started_at, + count() AS chunks, + sum(byte_count) AS bytes, + sumIf(byte_count, stream = 'stdout') AS stdout_bytes, + sumIf(byte_count, stream = 'stderr') AS stderr_bytes, + max(sequence) AS max_sequence, + sum(dropped_before) AS queue_drops + FROM {output} GROUP BY invocation_id +), +delivery AS (SELECT * FROM {delivery} WHERE stream = 'output'), +ids AS ( + SELECT DISTINCT invocation_id, invocation_started_at + FROM ( + SELECT invocation_id, invocation_started_at FROM received + UNION ALL SELECT invocation_id, invocation_started_at FROM delivery + ) +), +counted AS ( + SELECT + ids.invocation_id AS invocation_id, + ids.invocation_started_at AS invocation_started_at, + coalesce(received.chunks, 0) AS chunks, + coalesce(received.bytes, 0) AS bytes, + coalesce(received.stdout_bytes, 0) AS stdout_bytes, + coalesce(received.stderr_bytes, 0) AS stderr_bytes, + received.max_sequence AS max_sequence, + coalesce(received.max_sequence + 1 - received.chunks, 0) AS missing_sequences, + greatest(coalesce(received.queue_drops, 0), coalesce(delivery.dropped, 0)) AS reported_drops, + coalesce(delivery.filtered, 0) AS filtered, + coalesce(delivery.failed, 0) AS failed, + coalesce(context.recall_filter_active, 0) AS filter_active, + coalesce(delivery.flush_complete, 0) AS flush_complete, + delivery.invocation_id IS NOT NULL AS delivery_reported, + coalesce(delivery.delivered, 0) AS delivered + FROM ids + LEFT JOIN received ON received.invocation_id = ids.invocation_id + LEFT JOIN delivery ON delivery.invocation_id = ids.invocation_id + LEFT JOIN {context} AS context ON context.invocation_id = ids.invocation_id + SETTINGS join_use_nulls = 1 +) +SELECT + invocation_id, invocation_started_at, chunks, bytes, stdout_bytes, stderr_bytes, + max_sequence, missing_sequences, reported_drops, filtered, failed, filter_active, + flush_complete, + multiIf( + missing_sequences = 0, NULL, + filtered >= missing_sequences, 'filtered', + filtered + reported_drops + failed >= missing_sequences, 'dropped', + 'unexplained' + ) AS gap_reason, + toUInt8(delivery_reported AND flush_complete = 1 AND missing_sequences = 0 + AND reported_drops = 0 AND filtered = 0 AND failed = 0 AND chunks >= delivered) + AS complete +FROM counted" + ) +} + +fn output_tail(options: &DdlOptions) -> String { + let output = deduplicated(Table::Output, options); + format!( + "SELECT invocation_id, invocation_started_at, sequence, stream, occurred_at, byte_count, + toValidUTF8(data) AS text +FROM {output} +ORDER BY invocation_id, sequence DESC +LIMIT {OUTPUT_TAIL_RECORDS} BY invocation_id" + ) +} + +fn output_transcripts(options: &DdlOptions) -> String { + let output = deduplicated(Table::Output, options); + format!( + "SELECT + invocation_id, + min(invocation_started_at) AS invocation_started_at, + arrayStringConcat(arrayMap(chunk -> chunk.2, arraySort(chunk -> chunk.1, + groupArrayIf(tuple(sequence, data), stream = 'stdout')))) AS stdout, + arrayStringConcat(arrayMap(chunk -> chunk.2, arraySort(chunk -> chunk.1, + groupArrayIf(tuple(sequence, data), stream = 'stderr')))) AS stderr, + arrayStringConcat(arrayMap(chunk -> chunk.2, arraySort(chunk -> chunk.1, + groupArray(tuple(sequence, data))))) AS combined +FROM {output} +GROUP BY invocation_id" + ) +} + +fn passthrough(table: Table, options: &DdlOptions) -> String { + format!("SELECT * FROM {}", deduplicated(table, options)) +} + +fn artifacts(options: &DdlOptions) -> String { + let refs = deduplicated(Table::ArtifactRefs, options); + let stored = deduplicated(Table::Artifacts, options); + format!( + "SELECT + refs.invocation_id AS invocation_id, + refs.invocation_started_at AS invocation_started_at, + refs.digest AS digest, + refs.ref_id AS ref_id, + refs.role AS role, + refs.statement_id AS statement_id, + refs.resource_id AS resource_id, + artifact.kind AS kind, + artifact.media_type AS media_type, + artifact.size AS size, + artifact.digest IS NOT NULL AS available, + if(artifact.digest IS NULL, 'expired_or_not_captured', NULL) AS unavailable_reason, + if(startsWith(artifact.media_type, 'text/'), toValidUTF8(artifact.content), NULL) AS text, + artifact.content AS content +FROM {refs} AS refs +LEFT JOIN {stored} AS artifact ON artifact.digest = refs.digest +SETTINGS join_use_nulls = 1" + ) +} + +/// Each supported view's name and ClickHouse body, in creation order. +pub(crate) fn clickhouse_view_bodies(options: &DdlOptions) -> Vec<(&'static str, String)> { + vec![ + ("invocations", invocations(options)), + ("statements", statements(options)), + ("audit_results", audit_results(options)), + ("output_invocations", output_invocations(options)), + ("output_tail", output_tail(options)), + ("output_transcripts", output_transcripts(options)), + ("events", passthrough(Table::Events, options)), + ("diagnostics", passthrough(Table::Diagnostics, options)), + ("artifacts", artifacts(options)), + ("conflicts", passthrough(Table::Conflicts, options)), + ] +} diff --git a/crates/sqlbuild-recall/src/schema/_helpers/mod.rs b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs new file mode 100644 index 0000000000..8e71b76aae --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod clickhouse_views; +pub(crate) mod rendering; +pub(crate) mod sqlite_views; diff --git a/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs b/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs new file mode 100644 index 0000000000..658aef267d --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs @@ -0,0 +1,88 @@ +//! Column, table and identifier rendering shared by the DDL statements. + +use crate::schema::constants::COMMON_COLUMNS; +use crate::schema::models::{ColumnSpec, ColumnType, DdlOptions, TableSpec}; + +/// The table's own columns followed by the common columns. +pub(crate) fn all_columns(spec: &TableSpec) -> Vec { + spec.columns + .iter() + .chain(COMMON_COLUMNS.iter()) + .copied() + .collect() +} + +pub(crate) fn sqlite_type(column_type: ColumnType) -> &'static str { + match column_type { + ColumnType::Text | ColumnType::Timestamp | ColumnType::Json => "TEXT", + ColumnType::Integer | ColumnType::Boolean => "INTEGER", + ColumnType::Real => "REAL", + ColumnType::Blob => "BLOB", + } +} + +pub(crate) fn clickhouse_type(column: &ColumnSpec) -> String { + let base: &str = match column.column_type { + ColumnType::Text | ColumnType::Json | ColumnType::Blob => "String", + ColumnType::Integer => "Int64", + ColumnType::Real => "Float64", + ColumnType::Boolean => "UInt8", + ColumnType::Timestamp => "DateTime64(6, 'UTC')", + }; + if column.nullable { + format!("Nullable({base})") + } else { + base.to_owned() + } +} + +/// `"name"` for SQLite. +pub(crate) fn sqlite_identifier(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +/// `` `name` `` for ClickHouse. +pub(crate) fn clickhouse_identifier(name: &str) -> String { + format!("`{}`", name.replace('\\', "\\\\").replace('`', "\\`")) +} + +/// A ClickHouse table or view name, qualified by the configured database. +pub(crate) fn clickhouse_table(name: &str, options: &DdlOptions) -> String { + match &options.database { + Some(database) => format!( + "{}.{}", + clickhouse_identifier(database), + clickhouse_identifier(name) + ), + None => clickhouse_identifier(name), + } +} + +/// ` ON CLUSTER `c`` when a cluster is configured. +pub(crate) fn on_cluster(options: &DdlOptions) -> String { + options + .cluster + .as_deref() + .map(|cluster| format!(" ON CLUSTER {}", clickhouse_identifier(cluster))) + .unwrap_or_default() +} + +/// The TTL expression of a table: expiry column plus retention, artifacts outliving references. +pub(crate) fn clickhouse_ttl(spec: &TableSpec, options: &DdlOptions) -> String { + let days: u32 = options.retention_days.saturating_add(spec.ttl_extra_days); + format!("{} + INTERVAL {days} DAY", spec.expiry_column) +} + +/// The engine clause, replicated with one shard and every replica on a cluster. +pub(crate) fn clickhouse_engine(spec: &TableSpec, options: &DdlOptions) -> String { + let version: &str = spec.clickhouse_version.unwrap_or_default(); + if options.cluster.is_some() { + let separator: &str = if version.is_empty() { "" } else { ", " }; + format!( + "Replicated{}('/clickhouse/tables/{{uuid}}/{{shard}}', '{{replica}}'{separator}{version})", + spec.clickhouse_engine + ) + } else { + format!("{}({version})", spec.clickhouse_engine) + } +} diff --git a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs new file mode 100644 index 0000000000..76615bcb83 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs @@ -0,0 +1,360 @@ +//! The bodies of the supported views in SQLite. + +use crate::schema::constants::OUTPUT_TAIL_RECORDS; + +/// The current time as stored timestamps spell it. +const NOW: &str = "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')"; + +const STATEMENTS: &str = "WITH facts AS ( + SELECT * + FROM recall_events + WHERE statement_id IS NOT NULL AND event_type LIKE 'statement!_%' ESCAPE '!' +), +firsts AS ( + SELECT invocation_id, statement_id, run_id, resource_id, resource_attempt_id, operation_id, + intent, statement_kind, purpose, sql_digest, + ROW_NUMBER() OVER ( + PARTITION BY statement_id ORDER BY invocation_sequence, occurred_at, event_id + ) AS position + FROM facts +), +started AS ( + SELECT statement_id, MIN(occurred_at) AS started_at + FROM facts WHERE event_type = 'statement_started' GROUP BY statement_id +), +submitted AS ( + SELECT statement_id, MIN(occurred_at) AS submitted_at + FROM facts WHERE event_type = 'statement_submitted' GROUP BY statement_id +), +query_ids AS ( + SELECT statement_id, MAX(query_id) AS query_id FROM facts GROUP BY statement_id +), +terminals AS ( + SELECT statement_id, event_type, occurred_at, query_id, error_type, error_code, duration_ms, + row_count, affected_rows, + ROW_NUMBER() OVER ( + PARTITION BY statement_id + ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC + ) AS position + FROM facts + WHERE event_type IN ('statement_completed', 'statement_failed') +), +resources AS ( + SELECT invocation_id, resource_id, MIN(resource_name) AS resource_name, + MIN(resource_kind) AS resource_kind + FROM recall_events + WHERE resource_id IS NOT NULL AND resource_name IS NOT NULL + GROUP BY invocation_id, resource_id +) +SELECT + firsts.invocation_id, + firsts.statement_id, + firsts.run_id, + firsts.resource_id, + resources.resource_name, + resources.resource_kind, + firsts.resource_attempt_id, + firsts.operation_id, + firsts.intent, + firsts.statement_kind, + firsts.purpose, + COALESCE(terminals.query_id, query_ids.query_id) AS query_id, + started.started_at, + submitted.submitted_at, + terminals.occurred_at AS ended_at, + CASE terminals.event_type + WHEN 'statement_completed' THEN 'completed' + WHEN 'statement_failed' THEN 'failed' + ELSE 'no_terminal' + END AS outcome, + terminals.error_type, + terminals.error_code, + terminals.duration_ms, + terminals.row_count, + terminals.affected_rows, + firsts.sql_digest, + EXISTS ( + SELECT 1 FROM recall_artifacts AS artifact + WHERE artifact.digest = 'sha256:' || firsts.sql_digest + ) AS sql_available +FROM firsts +LEFT JOIN started ON started.statement_id = firsts.statement_id +LEFT JOIN submitted ON submitted.statement_id = firsts.statement_id +LEFT JOIN query_ids ON query_ids.statement_id = firsts.statement_id +LEFT JOIN terminals + ON terminals.statement_id = firsts.statement_id AND terminals.position = 1 +LEFT JOIN resources + ON resources.invocation_id = firsts.invocation_id + AND resources.resource_id = firsts.resource_id +WHERE firsts.position = 1"; + +const AUDIT_RESULTS: &str = "SELECT + invocation_id, + invocation_started_at, + event_id, + occurred_at, + run_id, + resource_id, + resource_attempt_id, + json_extract(payload, '$.audit_name') AS audit_name, + json_extract(payload, '$.audit_definition_name') AS audit_definition_name, + CASE WHEN json_extract(payload, '$.target_name') IS NULL THEN NULL ELSE + COALESCE(json_extract(payload, '$.target_database') || '.', '') + || COALESCE(json_extract(payload, '$.target_schema') || '.', '') + || json_extract(payload, '$.target_name') + END AS target_relation, + json_extract(payload, '$.attached_target_kind') AS attached_target_kind, + json_extract(payload, '$.attached_target_name') AS attached_target_name, + json_extract(payload, '$.attached_column_name') AS \"column\", + outcome, + json_extract(payload, '$.severity') AS severity, + json_extract(payload, '$.violation_count') AS violation_count, + json_extract(payload, '$.measured_value') AS measured_value, + json_extract(payload, '$.evaluation_mode') AS evaluation_mode, + json_extract(payload, '$.execution_error') AS execution_error, + sql_digest, + json_extract(payload, '$.sample_digest') AS sample_digest +FROM recall_events +WHERE event_type = 'audit_completed'"; + +const OUTPUT_INVOCATIONS: &str = "WITH received AS ( + SELECT invocation_id, + MIN(invocation_started_at) AS invocation_started_at, + COUNT(*) AS chunks, + SUM(byte_count) AS bytes, + SUM(CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END) AS stdout_bytes, + SUM(CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END) AS stderr_bytes, + MAX(sequence) AS max_sequence, + SUM(dropped_before) AS queue_drops + FROM recall_output + GROUP BY invocation_id +), +delivery AS ( + SELECT * FROM recall_delivery WHERE stream = 'output' +), +ids AS ( + SELECT invocation_id, invocation_started_at FROM received + UNION SELECT invocation_id, invocation_started_at FROM delivery +), +counted AS ( + SELECT + ids.invocation_id, + ids.invocation_started_at, + COALESCE(received.chunks, 0) AS chunks, + COALESCE(received.bytes, 0) AS bytes, + COALESCE(received.stdout_bytes, 0) AS stdout_bytes, + COALESCE(received.stderr_bytes, 0) AS stderr_bytes, + received.max_sequence, + COALESCE(received.max_sequence + 1 - received.chunks, 0) AS missing_sequences, + MAX(COALESCE(received.queue_drops, 0), COALESCE(delivery.dropped, 0)) AS reported_drops, + COALESCE(delivery.filtered, 0) AS filtered, + COALESCE(delivery.failed, 0) AS failed, + COALESCE(context.recall_filter_active, 0) AS filter_active, + COALESCE(delivery.flush_complete, 0) AS flush_complete, + delivery.invocation_id IS NOT NULL AS delivery_reported, + COALESCE(delivery.delivered, 0) AS delivered + FROM ids + LEFT JOIN received ON received.invocation_id = ids.invocation_id + LEFT JOIN delivery ON delivery.invocation_id = ids.invocation_id + LEFT JOIN recall_invocations AS context ON context.invocation_id = ids.invocation_id +) +SELECT + invocation_id, + invocation_started_at, + chunks, + bytes, + stdout_bytes, + stderr_bytes, + max_sequence, + missing_sequences, + reported_drops, + filtered, + failed, + filter_active, + flush_complete, + CASE + WHEN missing_sequences = 0 THEN NULL + WHEN filtered >= missing_sequences THEN 'filtered' + WHEN filtered + reported_drops + failed >= missing_sequences THEN 'dropped' + ELSE 'unexplained' + END AS gap_reason, + CASE + WHEN delivery_reported AND flush_complete = 1 AND missing_sequences = 0 + AND reported_drops = 0 AND filtered = 0 AND failed = 0 AND chunks >= delivered + THEN 1 ELSE 0 + END AS complete +FROM counted"; + +const OUTPUT_TRANSCRIPTS: &str = "SELECT + invocation_id, + MIN(invocation_started_at) AS invocation_started_at, + CAST(COALESCE( + group_concat(CASE WHEN stream = 'stdout' THEN data END, x'' ORDER BY sequence), x'' + ) AS BLOB) AS stdout, + CAST(COALESCE( + group_concat(CASE WHEN stream = 'stderr' THEN data END, x'' ORDER BY sequence), x'' + ) AS BLOB) AS stderr, + CAST(COALESCE(group_concat(data, x'' ORDER BY sequence), x'') AS BLOB) AS combined +FROM recall_output +GROUP BY invocation_id"; + +const EVENTS: &str = "SELECT * FROM recall_events"; + +const DIAGNOSTICS: &str = "SELECT * FROM recall_diagnostics"; + +const ARTIFACTS: &str = "SELECT + refs.invocation_id, + refs.invocation_started_at, + refs.digest, + refs.ref_id, + refs.role, + refs.statement_id, + refs.resource_id, + artifact.kind, + artifact.media_type, + artifact.size, + artifact.digest IS NOT NULL AS available, + CASE WHEN artifact.digest IS NULL THEN 'expired_or_not_captured' END AS unavailable_reason, + CASE WHEN artifact.media_type LIKE 'text/%' THEN CAST(artifact.content AS TEXT) END AS text, + artifact.content +FROM recall_artifact_refs AS refs +LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest"; + +const CONFLICTS: &str = "SELECT * FROM recall_conflicts"; + +fn invocations() -> String { + format!( + "WITH ids AS ( + SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at + FROM ( + SELECT invocation_id, invocation_started_at FROM recall_invocations + UNION ALL SELECT invocation_id, invocation_started_at FROM recall_heartbeats + UNION ALL SELECT invocation_id, invocation_started_at FROM recall_events + WHERE event_type IN ('invocation_started', 'invocation_completed', 'invocation_failed') + ) + GROUP BY invocation_id +), +started AS ( + SELECT invocation_id, MIN(occurred_at) AS started_at, MIN(command) AS command + FROM recall_events + WHERE event_type = 'invocation_started' + GROUP BY invocation_id +), +terminals AS ( + SELECT invocation_id, event_type, occurred_at, exit_code, duration_ms, error_type, error_code, + ROW_NUMBER() OVER ( + PARTITION BY invocation_id + ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC + ) AS position + FROM recall_events + WHERE event_type IN ('invocation_completed', 'invocation_failed') +), +beats AS ( + SELECT invocation_id, MAX(beat_at) AS last_heartbeat_at, MAX(stale_at) AS stale_at + FROM recall_heartbeats + GROUP BY invocation_id +), +projected AS ( + SELECT + ids.invocation_id, + ids.invocation_started_at, + COALESCE(started.started_at, ids.invocation_started_at) AS started_at, + CASE + WHEN terminals.event_type = 'invocation_completed' THEN 'completed' + WHEN terminals.event_type = 'invocation_failed' THEN 'failed' + WHEN beats.stale_at IS NULL THEN 'unknown' + WHEN beats.stale_at >= {NOW} THEN 'running' + ELSE 'abandoned' + END AS status, + terminals.event_type AS terminal_event, + terminals.occurred_at AS terminal_at, + terminals.duration_ms AS terminal_duration_ms, + terminals.exit_code, + terminals.error_type, + terminals.error_code, + COALESCE(context.command, started.command) AS command, + context.subcommand, + context.target, + context.adapter, + context.project_name, + context.selection, + context.exclusion, + context.sqlbuild_version, + context.git_revision, + context.git_dirty, + context.host_token, + context.pid, + context.external_context, + context.recall_filter_active, + beats.last_heartbeat_at + FROM ids + LEFT JOIN recall_invocations AS context ON context.invocation_id = ids.invocation_id + LEFT JOIN started ON started.invocation_id = ids.invocation_id + LEFT JOIN terminals + ON terminals.invocation_id = ids.invocation_id AND terminals.position = 1 + LEFT JOIN beats ON beats.invocation_id = ids.invocation_id +) +SELECT + invocation_id, + invocation_started_at, + started_at, + status, + CASE + WHEN terminal_event IS NOT NULL THEN terminal_at + WHEN status = 'abandoned' THEN last_heartbeat_at + END AS ended_at, + CASE + WHEN terminal_event IS NOT NULL THEN terminal_duration_ms + WHEN status = 'abandoned' THEN ROUND( + (unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3 + ) + END AS duration_ms, + exit_code, + error_type, + error_code, + command, + subcommand, + target, + adapter, + project_name, + selection, + exclusion, + sqlbuild_version, + git_revision, + git_dirty, + host_token, + pid, + external_context, + recall_filter_active, + last_heartbeat_at +FROM projected" + ) +} + +fn output_tail() -> String { + format!( + "SELECT invocation_id, invocation_started_at, sequence, stream, occurred_at, byte_count, + CAST(data AS TEXT) AS text +FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY invocation_id ORDER BY sequence DESC) AS position + FROM recall_output +) +WHERE position <= {OUTPUT_TAIL_RECORDS}" + ) +} + +/// Each supported view's name and SQLite body, in creation order. +pub(crate) fn sqlite_view_bodies() -> Vec<(&'static str, String)> { + vec![ + ("invocations", invocations()), + ("statements", STATEMENTS.to_owned()), + ("audit_results", AUDIT_RESULTS.to_owned()), + ("output_invocations", OUTPUT_INVOCATIONS.to_owned()), + ("output_tail", output_tail()), + ("output_transcripts", OUTPUT_TRANSCRIPTS.to_owned()), + ("events", EVENTS.to_owned()), + ("diagnostics", DIAGNOSTICS.to_owned()), + ("artifacts", ARTIFACTS.to_owned()), + ("conflicts", CONFLICTS.to_owned()), + ] +} diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs new file mode 100644 index 0000000000..ca4959c160 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -0,0 +1,340 @@ +//! Recall table definitions, view names and schema versions. + +use crate::schema::models::{ColumnSpec, ColumnType, Table, TableSpec}; + +/// The Recall schema version this build writes. Raise it when a column or view is added. +pub const RECALL_SCHEMA_VERSION: i64 = 1; +/// The oldest SQLite library the store's views run on (ordered `group_concat`). +pub const MINIMUM_SQLITE_VERSION: (u32, u32) = (3, 44); +/// The `recall_meta` component of schema rows. +pub const META_SCHEMA_COMPONENT: &str = "schema"; +pub const META_SCHEMA_VERSION_KEY: &str = "schema_version"; +/// The `recall_meta` component of maintenance claims. +pub const META_MAINTENANCE_COMPONENT: &str = "maintenance"; +pub const META_MAINTENANCE_CLAIM_KEY: &str = "claim"; + +/// Columns every Recall row carries, after the table's own columns. +pub const COMMON_COLUMNS: [ColumnSpec; 6] = [ + ColumnSpec::required("invocation_id", ColumnType::Text), + ColumnSpec::required("invocation_started_at", ColumnType::Timestamp), + ColumnSpec::required("schema_version", ColumnType::Integer), + ColumnSpec::required("producer_version", ColumnType::Text), + ColumnSpec::required("ingested_at", ColumnType::Timestamp), + ColumnSpec::required("content_sha256", ColumnType::Text), +]; +/// Common columns left out of a row's content digest. +pub const UNHASHED_COLUMNS: [&str; 2] = ["ingested_at", "content_sha256"]; + +const CORRELATIONS: [ColumnSpec; 5] = [ + ColumnSpec::optional("run_id", ColumnType::Text), + ColumnSpec::optional("resource_id", ColumnType::Text), + ColumnSpec::optional("resource_attempt_id", ColumnType::Text), + ColumnSpec::optional("operation_id", ColumnType::Text), + ColumnSpec::optional("statement_id", ColumnType::Text), +]; + +const INVOCATION_COLUMNS: [ColumnSpec; 19] = [ + ColumnSpec::optional("command", ColumnType::Text), + ColumnSpec::optional("subcommand", ColumnType::Text), + ColumnSpec::optional("selection", ColumnType::Json), + ColumnSpec::optional("exclusion", ColumnType::Json), + ColumnSpec::optional("project_name", ColumnType::Text), + ColumnSpec::optional("project_dir_hash", ColumnType::Text), + ColumnSpec::optional("target", ColumnType::Text), + ColumnSpec::optional("adapter", ColumnType::Text), + ColumnSpec::optional("destination_database", ColumnType::Text), + ColumnSpec::optional("destination_schema", ColumnType::Text), + ColumnSpec::optional("destination_fingerprint", ColumnType::Text), + ColumnSpec::optional("sqlbuild_version", ColumnType::Text), + ColumnSpec::optional("git_revision", ColumnType::Text), + ColumnSpec::optional("git_dirty", ColumnType::Boolean), + ColumnSpec::optional("host_token", ColumnType::Text), + ColumnSpec::optional("pid", ColumnType::Integer), + ColumnSpec::optional("external_context", ColumnType::Json), + ColumnSpec::optional("recall_filter_active", ColumnType::Boolean), + ColumnSpec::optional("argv_digest", ColumnType::Text), +]; + +const OUTPUT_COLUMNS: [ColumnSpec; 9] = [ + ColumnSpec::required("record_id", ColumnType::Text), + ColumnSpec::required("sequence", ColumnType::Integer), + ColumnSpec::required("stream", ColumnType::Text), + ColumnSpec::required("occurred_at", ColumnType::Timestamp), + ColumnSpec::required("data", ColumnType::Blob), + ColumnSpec::required("byte_offset", ColumnType::Integer), + ColumnSpec::required("byte_count", ColumnType::Integer), + ColumnSpec::optional("priority", ColumnType::Text), + ColumnSpec::required("dropped_before", ColumnType::Integer), +]; + +const EVENT_COLUMNS: [ColumnSpec; 29] = [ + ColumnSpec::required("event_id", ColumnType::Text), + ColumnSpec::optional("invocation_sequence", ColumnType::Integer), + ColumnSpec::required("occurred_at", ColumnType::Timestamp), + ColumnSpec::required("event_type", ColumnType::Text), + ColumnSpec::required("event_kind", ColumnType::Text), + ColumnSpec::required("severity", ColumnType::Text), + ColumnSpec::required("producer", ColumnType::Text), + ColumnSpec::required("event_schema_version", ColumnType::Integer), + CORRELATIONS[0], + CORRELATIONS[1], + CORRELATIONS[2], + CORRELATIONS[3], + CORRELATIONS[4], + ColumnSpec::optional("command", ColumnType::Text), + ColumnSpec::optional("resource_kind", ColumnType::Text), + ColumnSpec::optional("resource_name", ColumnType::Text), + ColumnSpec::optional("error_code", ColumnType::Text), + ColumnSpec::optional("error_type", ColumnType::Text), + ColumnSpec::optional("duration_ms", ColumnType::Real), + ColumnSpec::optional("exit_code", ColumnType::Integer), + ColumnSpec::optional("query_id", ColumnType::Text), + ColumnSpec::optional("sql_digest", ColumnType::Text), + ColumnSpec::optional("intent", ColumnType::Text), + ColumnSpec::optional("statement_kind", ColumnType::Text), + ColumnSpec::optional("purpose", ColumnType::Text), + ColumnSpec::optional("outcome", ColumnType::Text), + ColumnSpec::optional("row_count", ColumnType::Integer), + ColumnSpec::optional("affected_rows", ColumnType::Integer), + ColumnSpec::required("payload", ColumnType::Json), +]; + +const DIAGNOSTIC_COLUMNS: [ColumnSpec; 14] = [ + ColumnSpec::required("record_id", ColumnType::Text), + ColumnSpec::required("sequence", ColumnType::Integer), + ColumnSpec::required("occurred_at", ColumnType::Timestamp), + ColumnSpec::required("severity", ColumnType::Text), + ColumnSpec::required("logger", ColumnType::Text), + ColumnSpec::required("source", ColumnType::Text), + ColumnSpec::required("message", ColumnType::Text), + ColumnSpec::required("fields", ColumnType::Json), + ColumnSpec::optional("log_stream_id", ColumnType::Text), + CORRELATIONS[0], + CORRELATIONS[1], + CORRELATIONS[2], + CORRELATIONS[3], + CORRELATIONS[4], +]; + +const ARTIFACT_COLUMNS: [ColumnSpec; 6] = [ + ColumnSpec::required("digest", ColumnType::Text), + ColumnSpec::required("kind", ColumnType::Text), + ColumnSpec::required("media_type", ColumnType::Text), + ColumnSpec::required("size", ColumnType::Integer), + ColumnSpec::required("content", ColumnType::Blob), + ColumnSpec::required("seen_at", ColumnType::Timestamp), +]; + +const ARTIFACT_REF_COLUMNS: [ColumnSpec; 6] = [ + ColumnSpec::required("digest", ColumnType::Text), + ColumnSpec::required("ref_id", ColumnType::Text), + ColumnSpec::optional("statement_id", ColumnType::Text), + ColumnSpec::optional("resource_id", ColumnType::Text), + ColumnSpec::required("role", ColumnType::Text), + ColumnSpec::required("referenced_at", ColumnType::Timestamp), +]; + +const HEARTBEAT_COLUMNS: [ColumnSpec; 4] = [ + ColumnSpec::required("beat_at", ColumnType::Timestamp), + ColumnSpec::required("stale_at", ColumnType::Timestamp), + ColumnSpec::optional("host_token", ColumnType::Text), + ColumnSpec::optional("pid", ColumnType::Integer), +]; + +const DELIVERY_COLUMNS: [ColumnSpec; 9] = [ + ColumnSpec::required("stream", ColumnType::Text), + ColumnSpec::required("accepted", ColumnType::Integer), + ColumnSpec::required("delivered", ColumnType::Integer), + ColumnSpec::required("filtered", ColumnType::Integer), + ColumnSpec::required("dropped", ColumnType::Integer), + ColumnSpec::required("failed", ColumnType::Integer), + ColumnSpec::required("conflicts", ColumnType::Integer), + ColumnSpec::required("flush_complete", ColumnType::Boolean), + ColumnSpec::required("written_at", ColumnType::Timestamp), +]; + +const META_COLUMNS: [ColumnSpec; 4] = [ + ColumnSpec::required("component", ColumnType::Text), + ColumnSpec::required("key", ColumnType::Text), + ColumnSpec::required("value", ColumnType::Text), + ColumnSpec::required("recorded_at", ColumnType::Timestamp), +]; + +const CONFLICT_COLUMNS: [ColumnSpec; 5] = [ + ColumnSpec::required("dataset", ColumnType::Text), + ColumnSpec::required("record_key", ColumnType::Text), + ColumnSpec::required("existing_sha256", ColumnType::Text), + ColumnSpec::required("rejected_sha256", ColumnType::Text), + ColumnSpec::required("detected_at", ColumnType::Timestamp), +]; + +pub(crate) const INVOCATIONS_TABLE: TableSpec = TableSpec { + name: "recall_invocations", + clickhouse_version: None, + columns: &INVOCATION_COLUMNS, + key: &["invocation_id"], + sqlite_indexes: &[("started", &["invocation_started_at"])], + clickhouse_engine: "ReplacingMergeTree", + clickhouse_order_by: &["invocation_started_at", "invocation_id"], + ttl_extra_days: 0, + expiry_column: "invocation_started_at", +}; +pub(crate) const OUTPUT_TABLE: TableSpec = TableSpec { + name: "recall_output", + clickhouse_version: None, + columns: &OUTPUT_COLUMNS, + key: &["invocation_id", "sequence"], + sqlite_indexes: &[("expiry", &["invocation_started_at"])], + clickhouse_engine: "ReplacingMergeTree", + clickhouse_order_by: &["invocation_id", "sequence"], + ttl_extra_days: 0, + expiry_column: "invocation_started_at", +}; +pub(crate) const EVENTS_TABLE: TableSpec = TableSpec { + name: "recall_events", + clickhouse_version: None, + columns: &EVENT_COLUMNS, + key: &["event_id"], + sqlite_indexes: &[ + ("invocation", &["invocation_id", "invocation_sequence"]), + ("type", &["event_type", "occurred_at"]), + ("statement", &["statement_id"]), + ("expiry", &["invocation_started_at"]), + ], + clickhouse_engine: "ReplacingMergeTree", + clickhouse_order_by: &["invocation_id", "event_id"], + ttl_extra_days: 0, + expiry_column: "invocation_started_at", +}; +pub(crate) const DIAGNOSTICS_TABLE: TableSpec = TableSpec { + name: "recall_diagnostics", + clickhouse_version: None, + columns: &DIAGNOSTIC_COLUMNS, + key: &["invocation_id", "sequence"], + sqlite_indexes: &[ + ("severity", &["severity"]), + ("expiry", &["invocation_started_at"]), + ], + clickhouse_engine: "ReplacingMergeTree", + clickhouse_order_by: &["invocation_id", "sequence"], + ttl_extra_days: 0, + expiry_column: "invocation_started_at", +}; +pub(crate) const ARTIFACTS_TABLE: TableSpec = TableSpec { + name: "recall_artifacts", + clickhouse_version: Some("seen_at"), + columns: &ARTIFACT_COLUMNS, + key: &["digest"], + sqlite_indexes: &[("seen", &["seen_at"])], + clickhouse_engine: "ReplacingMergeTree", + clickhouse_order_by: &["digest"], + ttl_extra_days: ARTIFACT_REFRESH_DAYS, + expiry_column: "seen_at", +}; +pub(crate) const ARTIFACT_REFS_TABLE: TableSpec = TableSpec { + name: "recall_artifact_refs", + clickhouse_version: None, + columns: &ARTIFACT_REF_COLUMNS, + key: &["invocation_id", "digest", "ref_id"], + sqlite_indexes: &[("digest", &["digest"])], + clickhouse_engine: "MergeTree", + clickhouse_order_by: &["invocation_id", "digest", "ref_id"], + ttl_extra_days: 0, + expiry_column: "invocation_started_at", +}; +pub(crate) const HEARTBEATS_TABLE: TableSpec = TableSpec { + name: "recall_heartbeats", + clickhouse_version: None, + columns: &HEARTBEAT_COLUMNS, + key: &["invocation_id", "beat_at"], + sqlite_indexes: &[], + clickhouse_engine: "MergeTree", + clickhouse_order_by: &["invocation_id", "beat_at"], + ttl_extra_days: 0, + expiry_column: "invocation_started_at", +}; +pub(crate) const DELIVERY_TABLE: TableSpec = TableSpec { + name: "recall_delivery", + clickhouse_version: None, + columns: &DELIVERY_COLUMNS, + key: &["invocation_id", "stream"], + sqlite_indexes: &[], + clickhouse_engine: "MergeTree", + clickhouse_order_by: &["invocation_id", "stream"], + ttl_extra_days: 0, + expiry_column: "invocation_started_at", +}; +pub(crate) const META_TABLE: TableSpec = TableSpec { + name: "recall_meta", + clickhouse_version: None, + columns: &META_COLUMNS, + key: &["component", "key", "recorded_at"], + sqlite_indexes: &[], + clickhouse_engine: "MergeTree", + clickhouse_order_by: &["component", "key", "recorded_at"], + ttl_extra_days: 0, + expiry_column: "recorded_at", +}; +pub(crate) const CONFLICTS_TABLE: TableSpec = TableSpec { + name: "recall_conflicts", + clickhouse_version: None, + columns: &CONFLICT_COLUMNS, + key: &["dataset", "record_key", "rejected_sha256"], + sqlite_indexes: &[("invocation", &["invocation_id"])], + clickhouse_engine: "MergeTree", + clickhouse_order_by: &["dataset", "record_key", "rejected_sha256"], + ttl_extra_days: 0, + expiry_column: "invocation_started_at", +}; + +/// Every table, in creation order. +pub const ALL_TABLES: [Table; 10] = [ + Table::Meta, + Table::Invocations, + Table::Output, + Table::Events, + Table::Diagnostics, + Table::Artifacts, + Table::ArtifactRefs, + Table::Heartbeats, + Table::Delivery, + Table::Conflicts, +]; +/// Tables whose rows belong to one invocation and expire with it. +pub const INVOCATION_TABLES: [Table; 8] = [ + Table::Invocations, + Table::Output, + Table::Events, + Table::Diagnostics, + Table::ArtifactRefs, + Table::Heartbeats, + Table::Delivery, + Table::Conflicts, +]; +/// The supported views, in creation order. +pub const VIEW_NAMES: [&str; 10] = [ + "invocations", + "statements", + "audit_results", + "output_invocations", + "output_tail", + "output_transcripts", + "events", + "diagnostics", + "artifacts", + "conflicts", +]; +/// The number of records `output_tail` keeps per invocation. +pub const OUTPUT_TAIL_RECORDS: u32 = 200; +/// Invocations deleted per maintenance transaction. +pub const RETENTION_BATCH_INVOCATIONS: u32 = 200; +/// ClickHouse re-inserts an artifact whose newest copy is older than this. +pub const ARTIFACT_REFRESH_DAYS: u32 = 7; +/// Tables whose expiry index finds candidate invocations for retention. +pub(crate) const RETENTION_CANDIDATE_TABLES: [Table; 5] = [ + Table::Invocations, + Table::Heartbeats, + Table::Events, + Table::Output, + Table::Diagnostics, +]; diff --git a/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs b/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs new file mode 100644 index 0000000000..cea0815488 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs @@ -0,0 +1,37 @@ +//! `add_column_statement`: the additive upgrade of one table by one column. + +use crate::schema::_helpers::rendering::{ + clickhouse_identifier, clickhouse_table, clickhouse_type, on_cluster, sqlite_identifier, + sqlite_type, +}; +use crate::schema::models::{ColumnSpec, DdlOptions, Dialect, TableSpec}; + +/// `ALTER TABLE … ADD COLUMN`; added columns are always nullable so older rows stay valid. +#[must_use] +pub fn add_column_statement( + dialect: Dialect, + options: &DdlOptions, + (spec, column): (&TableSpec, &ColumnSpec), +) -> String { + match dialect { + Dialect::Sqlite => format!( + "ALTER TABLE {} ADD COLUMN {} {}", + sqlite_identifier(spec.name), + sqlite_identifier(column.name), + sqlite_type(column.column_type) + ), + Dialect::ClickHouse => { + let nullable: ColumnSpec = ColumnSpec { + nullable: true, + ..*column + }; + format!( + "ALTER TABLE {}{} ADD COLUMN IF NOT EXISTS {} {}", + clickhouse_table(spec.name, options), + on_cluster(options), + clickhouse_identifier(column.name), + clickhouse_type(&nullable) + ) + } + } +} diff --git a/crates/sqlbuild-recall/src/schema/main/ddl_script.rs b/crates/sqlbuild-recall/src/schema/main/ddl_script.rs new file mode 100644 index 0000000000..42ea6a014c --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/ddl_script.rs @@ -0,0 +1,19 @@ +//! `ddl_script`: the complete ordered script an administrator runs to create a Recall store. + +use crate::schema::main::table_statements::table_statements; +use crate::schema::main::view_statements::view_statements; +use crate::schema::models::{DdlOptions, Dialect}; + +/// Every table, index and view statement, each terminated by `;`. +#[must_use] +pub fn ddl_script(dialect: Dialect, options: &DdlOptions) -> String { + let mut statements: Vec = table_statements(dialect, options); + statements.extend( + view_statements(dialect, options) + .into_iter() + .map(|(_, statement)| statement), + ); + let mut script: String = statements.join(";\n\n"); + script.push_str(";\n"); + script +} diff --git a/crates/sqlbuild-recall/src/schema/main/mod.rs b/crates/sqlbuild-recall/src/schema/main/mod.rs new file mode 100644 index 0000000000..288b8f6614 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/mod.rs @@ -0,0 +1,7 @@ +pub mod add_column_statement; +pub mod ddl_script; +pub mod sqlite_retention_sql; +pub mod table_columns; +pub mod table_spec; +pub mod table_statements; +pub mod view_statements; diff --git a/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs b/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs new file mode 100644 index 0000000000..267f17d543 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs @@ -0,0 +1,59 @@ +//! `sqlite_retention_sql`: the deletes SQLite maintenance runs, all rows of an invocation together. + +use crate::schema::_helpers::rendering::sqlite_identifier; +use crate::schema::constants::{ + INVOCATION_TABLES, META_MAINTENANCE_COMPONENT, RETENTION_CANDIDATE_TABLES, +}; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::SqliteRetentionSql; + +fn expired_invocations() -> String { + let candidates: Vec = RETENTION_CANDIDATE_TABLES + .iter() + .map(|table| { + format!( + "SELECT invocation_id FROM {} WHERE invocation_started_at < ?1", + sqlite_identifier(table_spec(*table).name) + ) + }) + .collect(); + format!( + "SELECT DISTINCT invocation_id FROM (\n {}\n)\n\ + WHERE invocation_id NOT IN (\n\ + \x20 SELECT beat.invocation_id FROM recall_heartbeats AS beat\n\ + \x20 WHERE beat.stale_at >= ?2 AND NOT EXISTS (\n\ + \x20 SELECT 1 FROM recall_events AS terminal\n\ + \x20 WHERE terminal.invocation_id = beat.invocation_id\n\ + \x20 AND terminal.event_type IN ('invocation_completed', 'invocation_failed')\n\ + \x20 )\n\ + )\nLIMIT ?3", + candidates.join("\n UNION ALL ") + ) +} + +/// The maintenance statements; running invocations are never deleted. +#[must_use] +pub fn sqlite_retention_sql() -> SqliteRetentionSql { + SqliteRetentionSql { + expired_invocations: expired_invocations(), + delete_invocations: INVOCATION_TABLES + .iter() + .map(|table| { + ( + *table, + format!( + "DELETE FROM {} WHERE invocation_id IN (SELECT value FROM json_each(?1))", + sqlite_identifier(table_spec(*table).name) + ), + ) + }) + .collect(), + delete_orphan_artifacts: "DELETE FROM recall_artifacts WHERE seen_at < ?1 AND NOT EXISTS (\ + SELECT 1 FROM recall_artifact_refs AS ref WHERE ref.digest = recall_artifacts.digest)" + .to_owned(), + delete_old_claims: format!( + "DELETE FROM recall_meta WHERE component = '{META_MAINTENANCE_COMPONENT}' \ + AND recorded_at < ?1" + ), + } +} diff --git a/crates/sqlbuild-recall/src/schema/main/table_columns.rs b/crates/sqlbuild-recall/src/schema/main/table_columns.rs new file mode 100644 index 0000000000..87d842c064 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/table_columns.rs @@ -0,0 +1,11 @@ +//! `table_columns`: every column of one table, its own then the common columns. + +use crate::schema::_helpers::rendering::all_columns; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::{ColumnSpec, Table}; + +/// The table's columns in DDL order. +#[must_use] +pub fn table_columns(table: Table) -> Vec { + all_columns(table_spec(table)) +} diff --git a/crates/sqlbuild-recall/src/schema/main/table_spec.rs b/crates/sqlbuild-recall/src/schema/main/table_spec.rs new file mode 100644 index 0000000000..5fed770f82 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/table_spec.rs @@ -0,0 +1,24 @@ +//! `table_spec`: the definition of one Recall table. + +use crate::schema::constants::{ + ARTIFACT_REFS_TABLE, ARTIFACTS_TABLE, CONFLICTS_TABLE, DELIVERY_TABLE, DIAGNOSTICS_TABLE, + EVENTS_TABLE, HEARTBEATS_TABLE, INVOCATIONS_TABLE, META_TABLE, OUTPUT_TABLE, +}; +use crate::schema::models::{Table, TableSpec}; + +/// The physical definition of `table`. +#[must_use] +pub fn table_spec(table: Table) -> &'static TableSpec { + match table { + Table::Invocations => &INVOCATIONS_TABLE, + Table::Output => &OUTPUT_TABLE, + Table::Events => &EVENTS_TABLE, + Table::Diagnostics => &DIAGNOSTICS_TABLE, + Table::Artifacts => &ARTIFACTS_TABLE, + Table::ArtifactRefs => &ARTIFACT_REFS_TABLE, + Table::Heartbeats => &HEARTBEATS_TABLE, + Table::Delivery => &DELIVERY_TABLE, + Table::Meta => &META_TABLE, + Table::Conflicts => &CONFLICTS_TABLE, + } +} diff --git a/crates/sqlbuild-recall/src/schema/main/table_statements.rs b/crates/sqlbuild-recall/src/schema/main/table_statements.rs new file mode 100644 index 0000000000..621f8e1f12 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/table_statements.rs @@ -0,0 +1,84 @@ +//! `table_statements`: idempotent `CREATE TABLE` and index statements of every Recall table. + +use crate::schema::_helpers::rendering::{ + all_columns, clickhouse_engine, clickhouse_identifier, clickhouse_table, clickhouse_ttl, + clickhouse_type, on_cluster, sqlite_identifier, sqlite_type, +}; +use crate::schema::constants::ALL_TABLES; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::{DdlOptions, Dialect, TableSpec}; + +fn sqlite_table(spec: &TableSpec) -> Vec { + let mut definitions: Vec = all_columns(spec) + .iter() + .map(|column| { + let not_null: &str = if column.nullable { "" } else { " NOT NULL" }; + format!( + " {} {}{not_null}", + sqlite_identifier(column.name), + sqlite_type(column.column_type) + ) + }) + .collect(); + let key: Vec = spec + .key + .iter() + .map(|name| sqlite_identifier(name)) + .collect(); + definitions.push(format!(" UNIQUE ({})", key.join(", "))); + let mut statements: Vec = vec![format!( + "CREATE TABLE IF NOT EXISTS {} (\n{}\n)", + sqlite_identifier(spec.name), + definitions.join(",\n") + )]; + for (suffix, columns) in spec.sqlite_indexes { + let columns: Vec = columns.iter().map(|name| sqlite_identifier(name)).collect(); + statements.push(format!( + "CREATE INDEX IF NOT EXISTS {} ON {} ({})", + sqlite_identifier(&format!("{}_{suffix}", spec.name)), + sqlite_identifier(spec.name), + columns.join(", ") + )); + } + statements +} + +fn clickhouse_table_statement(spec: &TableSpec, options: &DdlOptions) -> String { + let columns: Vec = all_columns(spec) + .iter() + .map(|column| { + format!( + " {} {}", + clickhouse_identifier(column.name), + clickhouse_type(column) + ) + }) + .collect(); + format!( + "CREATE TABLE IF NOT EXISTS {name}{cluster} (\n{columns}\n)\nENGINE = {engine}\n\ + PARTITION BY toDate({expiry})\nORDER BY ({order})\nTTL {ttl}\n\ + SETTINGS ttl_only_drop_parts = 1, non_replicated_deduplication_window = 1000", + name = clickhouse_table(spec.name, options), + cluster = on_cluster(options), + columns = columns.join(",\n"), + engine = clickhouse_engine(spec, options), + expiry = spec.expiry_column, + order = spec.clickhouse_order_by.join(", "), + ttl = clickhouse_ttl(spec, options), + ) +} + +/// Every table's creation statements, in dependency order; safe to rerun. +#[must_use] +pub fn table_statements(dialect: Dialect, options: &DdlOptions) -> Vec { + ALL_TABLES + .iter() + .flat_map(|table| { + let spec: &TableSpec = table_spec(*table); + match dialect { + Dialect::Sqlite => sqlite_table(spec), + Dialect::ClickHouse => vec![clickhouse_table_statement(spec, options)], + } + }) + .collect() +} diff --git a/crates/sqlbuild-recall/src/schema/main/view_statements.rs b/crates/sqlbuild-recall/src/schema/main/view_statements.rs new file mode 100644 index 0000000000..ce34504a3c --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/view_statements.rs @@ -0,0 +1,38 @@ +//! `view_statements`: the statements that create the supported views. + +use crate::schema::_helpers::clickhouse_views::clickhouse_view_bodies; +use crate::schema::_helpers::rendering::{clickhouse_table, on_cluster, sqlite_identifier}; +use crate::schema::_helpers::sqlite_views::sqlite_view_bodies; +use crate::schema::models::{DdlOptions, Dialect}; + +/// Each view's name and creation statement. SQLite views are created only when missing; the store drops them first when the schema version rises. ClickHouse views are replaced. +#[must_use] +pub fn view_statements(dialect: Dialect, options: &DdlOptions) -> Vec<(&'static str, String)> { + match dialect { + Dialect::Sqlite => sqlite_view_bodies() + .into_iter() + .map(|(name, body)| { + ( + name, + format!( + "CREATE VIEW IF NOT EXISTS {} AS\n{body}", + sqlite_identifier(name) + ), + ) + }) + .collect(), + Dialect::ClickHouse => clickhouse_view_bodies(options) + .into_iter() + .map(|(name, body)| { + ( + name, + format!( + "CREATE OR REPLACE VIEW {}{} AS\n{body}", + clickhouse_table(name, options), + on_cluster(options) + ), + ) + }) + .collect(), + } +} diff --git a/crates/sqlbuild-recall/src/schema/mod.rs b/crates/sqlbuild-recall/src/schema/mod.rs new file mode 100644 index 0000000000..55009b62ab --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/mod.rs @@ -0,0 +1,9 @@ +//! Recall's physical schema: tables, supported views, DDL and retention SQL for each dialect. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-recall/src/schema/models.rs b/crates/sqlbuild-recall/src/schema/models.rs new file mode 100644 index 0000000000..526db74cf9 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/models.rs @@ -0,0 +1,111 @@ +//! The physical shape of Recall tables and the options their DDL is rendered with. + +/// The SQL dialect a Recall store speaks. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Dialect { + Sqlite, + ClickHouse, +} + +/// The logical type of one Recall column. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ColumnType { + Text, + Integer, + Real, + Blob, + /// ISO-8601 UTC text with microseconds in SQLite, `DateTime64(6, 'UTC')` in ClickHouse. + Timestamp, + /// JSON text. + Json, + /// 0 or 1. + Boolean, +} + +/// One column: its name, type and whether it may be null. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ColumnSpec { + pub name: &'static str, + pub column_type: ColumnType, + pub nullable: bool, +} + +impl ColumnSpec { + /// A column that is never null. + #[must_use] + pub const fn required(name: &'static str, column_type: ColumnType) -> Self { + Self { + name, + column_type, + nullable: false, + } + } + + /// A column that may be null. + #[must_use] + pub const fn optional(name: &'static str, column_type: ColumnType) -> Self { + Self { + name, + column_type, + nullable: true, + } + } +} + +/// One Recall table. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TableSpec { + pub name: &'static str, + /// Columns specific to this table; every table also has the common columns. + pub columns: &'static [ColumnSpec], + /// The deterministic identity of a row; a second row with this key is a duplicate. + pub key: &'static [&'static str], + /// SQLite secondary indexes: name suffix and columns. + pub sqlite_indexes: &'static [(&'static str, &'static [&'static str])], + /// ClickHouse engine without arguments: `ReplacingMergeTree` or `MergeTree`. + pub clickhouse_engine: &'static str, + /// ClickHouse `ORDER BY` columns. + pub clickhouse_order_by: &'static [&'static str], + /// Days ClickHouse keeps rows beyond retention; artifacts outlive their references. + pub ttl_extra_days: u32, + /// The `ReplacingMergeTree` version column: the newest copy survives merges. + pub clickhouse_version: Option<&'static str>, + /// The timestamp column partitions and TTL derive from. + pub expiry_column: &'static str, +} + +/// The tables Recall writes. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum Table { + Invocations, + Output, + Events, + Diagnostics, + Artifacts, + ArtifactRefs, + Heartbeats, + Delivery, + Meta, + Conflicts, +} + +/// What DDL is rendered with: database (ClickHouse), retention and cluster. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DdlOptions { + pub database: Option, + pub retention_days: u32, + pub cluster: Option, +} + +/// The statements SQLite maintenance runs, with their positional parameters. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SqliteRetentionSql { + /// `?1` cutoff, `?2` now, `?3` limit: expired invocation IDs that are not running. + pub expired_invocations: String, + /// Per table, `?1` a JSON array of invocation IDs to delete. + pub delete_invocations: Vec<(Table, String)>, + /// `?1` cutoff: artifacts no remaining reference names. + pub delete_orphan_artifacts: String, + /// `?1` cutoff: maintenance claims older than retention. + pub delete_old_claims: String, +} diff --git a/crates/sqlbuild-recall/src/schema/tests.rs b/crates/sqlbuild-recall/src/schema/tests.rs new file mode 100644 index 0000000000..3492109aaa --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/tests.rs @@ -0,0 +1,2 @@ +#[path = "tests/test_ddl.rs"] +mod ddl; diff --git a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs new file mode 100644 index 0000000000..82daf72aee --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs @@ -0,0 +1,129 @@ +use crate::schema::constants::{ALL_TABLES, VIEW_NAMES}; +use crate::schema::main::add_column_statement::add_column_statement; +use crate::schema::main::ddl_script::ddl_script; +use crate::schema::main::sqlite_retention_sql::sqlite_retention_sql; +use crate::schema::main::table_spec::table_spec; +use crate::schema::main::table_statements::table_statements; +use crate::schema::main::view_statements::view_statements; +use crate::schema::models::{ColumnSpec, ColumnType, DdlOptions, Dialect, Table}; + +fn options(cluster: Option<&str>) -> DdlOptions { + DdlOptions { + database: Some("observability".to_owned()), + retention_days: 14, + cluster: cluster.map(str::to_owned), + } +} + +#[test] +fn given_sqlite_script_when_run_twice_then_every_table_and_view_exists() { + let connection = rusqlite::Connection::open_in_memory().expect("memory database"); + let script = ddl_script(Dialect::Sqlite, &options(None)); + connection.execute_batch(&script).expect("first run"); + connection + .execute_batch(&script) + .expect("rerun is idempotent"); + let count: i64 = connection + .query_row( + "SELECT COUNT(*) FROM sqlite_schema WHERE type IN ('table', 'view')", + [], + |row| row.get(0), + ) + .expect("count"); + assert_eq!(count as usize, ALL_TABLES.len() + VIEW_NAMES.len()); + for view in VIEW_NAMES { + connection + .prepare(&format!("SELECT * FROM \"{view}\"")) + .unwrap_or_else(|error| panic!("{view}: {error}")); + } + let retention = sqlite_retention_sql(); + connection + .prepare(&retention.expired_invocations) + .expect("expired query prepares"); + for (_, delete) in &retention.delete_invocations { + connection.prepare(delete).expect("delete prepares"); + } + connection + .prepare(&retention.delete_orphan_artifacts) + .expect("artifact delete prepares"); +} + +#[test] +fn given_clickhouse_options_when_rendered_then_engines_ttl_and_partitions_follow_the_table() { + let statements = table_statements(Dialect::ClickHouse, &options(None)); + assert_eq!(statements.len(), ALL_TABLES.len()); + let output = statements + .iter() + .find(|statement| statement.contains("`recall_output`")) + .expect("output table"); + assert!(output.starts_with("CREATE TABLE IF NOT EXISTS `observability`.`recall_output` (")); + assert!(output.contains("ENGINE = ReplacingMergeTree()")); + assert!( + statements + .iter() + .any(|statement| statement.contains("ENGINE = ReplacingMergeTree(seen_at)")) + ); + assert!(output.contains("PARTITION BY toDate(invocation_started_at)")); + assert!(output.contains("ORDER BY (invocation_id, sequence)")); + assert!(output.contains("TTL invocation_started_at + INTERVAL 14 DAY")); + assert!(output.contains("`data` String")); + assert!(output.contains("`priority` Nullable(String)")); + let artifacts = statements + .iter() + .find(|statement| statement.contains("`recall_artifacts`")) + .expect("artifacts table"); + assert!(artifacts.contains("TTL seen_at + INTERVAL 21 DAY")); + let heartbeats = statements + .iter() + .find(|statement| statement.contains("`recall_heartbeats`")) + .expect("heartbeat table"); + assert!(heartbeats.contains("ENGINE = MergeTree()")); +} + +#[test] +fn given_a_cluster_when_rendered_then_tables_are_replicated_on_cluster() { + let statements = table_statements(Dialect::ClickHouse, &options(Some("analytics"))); + assert!( + statements + .iter() + .all(|statement| statement.contains(" ON CLUSTER `analytics` (")) + ); + assert!(statements[1].contains( + "ReplicatedReplacingMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')" + )); + let views = view_statements(Dialect::ClickHouse, &options(Some("analytics"))); + assert_eq!(views.len(), VIEW_NAMES.len()); + assert!(views.iter().all( + |(_, statement)| statement.starts_with("CREATE OR REPLACE VIEW") + && statement.contains("ON CLUSTER `analytics`") + )); +} + +#[test] +fn given_clickhouse_views_when_rendered_then_rows_are_deduplicated_within_retention() { + for (name, statement) in view_statements(Dialect::ClickHouse, &options(None)) { + assert!( + statement.contains("now64(6) - INTERVAL 14 DAY") && statement.contains("LIMIT 1 BY"), + "{name} must deduplicate and filter by retention" + ); + } +} + +#[test] +fn given_a_new_column_when_upgraded_then_the_statement_is_additive_and_nullable() { + let column = ColumnSpec { + name: "new_field", + column_type: ColumnType::Integer, + nullable: false, + }; + let spec = table_spec(Table::Events); + assert_eq!( + add_column_statement(Dialect::Sqlite, &options(None), (spec, &column)), + "ALTER TABLE \"recall_events\" ADD COLUMN \"new_field\" INTEGER" + ); + assert_eq!( + add_column_statement(Dialect::ClickHouse, &options(None), (spec, &column)), + "ALTER TABLE `observability`.`recall_events` ADD COLUMN IF NOT EXISTS `new_field` \ + Nullable(Int64)" + ); +} diff --git a/crates/sqlbuild-recall/src/settings/_helpers/documents.rs b/crates/sqlbuild-recall/src/settings/_helpers/documents.rs new file mode 100644 index 0000000000..1fb6e4e52d --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/_helpers/documents.rs @@ -0,0 +1,73 @@ +//! Reading the project and local configuration documents for Recall. + +use std::path::Path; + +use sqlbuild_config::models::ConfigValue; +use sqlbuild_config::toml::main::load_toml::load_toml; + +use crate::settings::errors::SettingsError; + +/// The root table of a TOML file, `None` when it does not exist. +pub(crate) fn read_document(path: &Path) -> Result, SettingsError> { + let text = match std::fs::read_to_string(path) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(SettingsError::new(format!( + "cannot read {}: {error}", + path.display() + ))); + } + }; + load_toml(&text).map(Some).map_err(|error| { + SettingsError::new(format!( + "{} is not valid TOML: {}", + path.display(), + error.message + )) + }) +} + +/// The value at a dotted path of tables. +pub(crate) fn lookup<'a>( + document: Option<&'a ConfigValue>, + path: &[&str], +) -> Option<&'a ConfigValue> { + let mut value: &ConfigValue = document?; + for key in path { + value = value.get(key)?; + } + Some(value) +} + +pub(crate) fn string_at(document: Option<&ConfigValue>, path: &[&str]) -> Option { + match lookup(document, path)? { + ConfigValue::String(text) if !text.trim().is_empty() => Some(text.trim().to_owned()), + _ => None, + } +} + +/// Replace `${env:NAME}` with the environment variable, failing when it is unset. +pub(crate) fn expand_environment(text: &str, label: &str) -> Result { + let mut expanded = String::with_capacity(text.len()); + let mut rest = text; + while let Some(start) = rest.find("${env:") { + expanded.push_str(&rest[..start]); + let after = &rest[start + "${env:".len()..]; + let Some(end) = after.find('}') else { + return Err(SettingsError::new(format!( + "{label} has an unterminated ${{env:...}} reference" + ))); + }; + let name = &after[..end]; + let value = std::env::var(name).map_err(|_| { + SettingsError::new(format!( + "{label} references unset environment variable {name}" + )) + })?; + expanded.push_str(&value); + rest = &after[end + 1..]; + } + expanded.push_str(rest); + Ok(expanded) +} diff --git a/crates/sqlbuild-recall/src/settings/_helpers/layers.rs b/crates/sqlbuild-recall/src/settings/_helpers/layers.rs new file mode 100644 index 0000000000..381ab4de55 --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/_helpers/layers.rs @@ -0,0 +1,117 @@ +//! Layering Recall settings tables and locating the store connection. + +use std::path::{Path, PathBuf}; + +use sqlbuild_config::models::ConfigValue; +use sqlbuild_config::recall::main::recall_section::recall_section; +use sqlbuild_config::recall::models::RecallSection; + +use crate::settings::_helpers::documents::{expand_environment, lookup}; +use crate::settings::constants::{ + CONNECTION_ADAPTER_KEY, DEFAULT_STORE_DIRECTORY, DEFAULT_STORE_FILE_NAME, SQLITE_ADAPTER, + SQLITE_PATH_KEY, +}; +use crate::settings::errors::SettingsError; +use crate::settings::models::StoreLocation; + +pub(crate) fn section( + document: Option<&ConfigValue>, + path: &[&str], + label: &str, +) -> Result { + match lookup(document, path) { + None => Ok(RecallSection::default()), + Some(value) => recall_section(value, label).map_err(SettingsError::new), + } +} + +pub(crate) fn overlay(base: RecallSection, layer: RecallSection) -> RecallSection { + RecallSection { + enabled: layer.enabled.or(base.enabled), + connection: layer.connection.or(base.connection), + retention_days: layer.retention_days.or(base.retention_days), + shutdown_timeout_ms: layer.shutdown_timeout_ms.or(base.shutdown_timeout_ms), + diagnostics_min_severity: layer + .diagnostics_min_severity + .or(base.diagnostics_min_severity), + } +} + +/// The named connection's entries, local keys overriding project keys. +fn connection_entries( + documents: [Option<&ConfigValue>; 2], + name: &str, +) -> Option> { + let mut merged: Option> = None; + for document in documents { + let Some(entries) = lookup(document, &["connections", name]).and_then(ConfigValue::as_map) + else { + continue; + }; + let target = merged.get_or_insert_with(Vec::new); + for (key, value) in entries { + let ConfigValue::String(key) = key else { + continue; + }; + target.retain(|(existing, _)| existing != key); + target.push((key.clone(), value.clone())); + } + } + merged +} + +pub(crate) fn store_location( + project_dir: &Path, + documents: [Option<&ConfigValue>; 2], + (connection, project_adapter): (Option<&str>, Option<&str>), +) -> Result { + let Some(name) = connection else { + return Ok(StoreLocation::Sqlite { + path: project_dir + .join(DEFAULT_STORE_DIRECTORY) + .join(DEFAULT_STORE_FILE_NAME), + }); + }; + let entries = connection_entries(documents, name).ok_or_else(|| { + SettingsError::new(format!( + "recall.connection names unknown connection '{name}'; define [connections.{name}]" + )) + })?; + let value = |key: &str| { + entries + .iter() + .find(|(candidate, _)| candidate == key) + .map(|(_, value)| value) + }; + let adapter: String = match value(CONNECTION_ADAPTER_KEY) { + Some(ConfigValue::String(adapter)) => adapter.trim().to_owned(), + Some(_) => { + return Err(SettingsError::new(format!( + "connections.{name}.adapter must be a string" + ))); + } + None => project_adapter.unwrap_or_default().to_owned(), + }; + if adapter != SQLITE_ADAPTER { + return Ok(StoreLocation::Unsupported { + connection: name.to_owned(), + adapter, + }); + } + let Some(ConfigValue::String(path)) = value(SQLITE_PATH_KEY) else { + return Err(SettingsError::new(format!( + "connections.{name}.path must name the SQLite file Recall writes" + ))); + }; + let path: PathBuf = PathBuf::from(expand_environment( + path.trim(), + &format!("connections.{name}.path"), + )?); + Ok(StoreLocation::Sqlite { + path: if path.is_absolute() { + path + } else { + project_dir.join(path) + }, + }) +} diff --git a/crates/sqlbuild-recall/src/settings/_helpers/mod.rs b/crates/sqlbuild-recall/src/settings/_helpers/mod.rs new file mode 100644 index 0000000000..3e9da610d7 --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod documents; +pub(crate) mod layers; diff --git a/crates/sqlbuild-recall/src/settings/constants.rs b/crates/sqlbuild-recall/src/settings/constants.rs new file mode 100644 index 0000000000..516b902392 --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/constants.rs @@ -0,0 +1,14 @@ +//! Built-in Recall defaults. + +/// The local store, relative to the project directory. +pub const DEFAULT_STORE_DIRECTORY: &str = "logs"; +pub const DEFAULT_STORE_FILE_NAME: &str = "recall.sqlite3"; +pub const DEFAULT_RETENTION_DAYS: u32 = 30; +pub const DEFAULT_SHUTDOWN_TIMEOUT_MS: u64 = 3_000; +pub const DEFAULT_DIAGNOSTICS_MIN_SEVERITY: &str = "info"; +/// The connection key naming a connection's adapter; never passed to `connect()`. +pub const CONNECTION_ADAPTER_KEY: &str = "adapter"; +pub const SQLITE_ADAPTER: &str = "sqlite"; +pub const SQLITE_PATH_KEY: &str = "path"; +pub const PROJECT_FILE_NAME: &str = "sqlbuild_project.toml"; +pub const LOCAL_FILE_NAME: &str = "sqlbuild_local.toml"; diff --git a/crates/sqlbuild-recall/src/settings/errors.rs b/crates/sqlbuild-recall/src/settings/errors.rs new file mode 100644 index 0000000000..fe4baa26dd --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/errors.rs @@ -0,0 +1,23 @@ +//! Why Recall settings cannot be resolved. + +/// A settings problem, with a message naming the setting. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SettingsError { + pub message: String, +} + +impl SettingsError { + pub(crate) fn new(message: impl Into) -> Self { + Self { + message: message.into(), + } + } +} + +impl std::fmt::Display for SettingsError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.message) + } +} + +impl std::error::Error for SettingsError {} diff --git a/crates/sqlbuild-recall/src/settings/main/mod.rs b/crates/sqlbuild-recall/src/settings/main/mod.rs new file mode 100644 index 0000000000..189f053d5e --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/main/mod.rs @@ -0,0 +1 @@ +pub mod resolve_recall_settings; diff --git a/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs b/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs new file mode 100644 index 0000000000..010834ad1d --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs @@ -0,0 +1,68 @@ +//! `resolve_recall_settings`: the effective Recall settings of one invocation. + +use sqlbuild_config::models::ConfigValue; +use sqlbuild_config::recall::constants::RECALL_SECTION; + +use crate::settings::_helpers::documents::{read_document, string_at}; +use crate::settings::_helpers::layers::{overlay, section, store_location}; +use crate::settings::constants::{ + DEFAULT_DIAGNOSTICS_MIN_SEVERITY, DEFAULT_RETENTION_DAYS, DEFAULT_SHUTDOWN_TIMEOUT_MS, + LOCAL_FILE_NAME, PROJECT_FILE_NAME, +}; +use crate::settings::errors::SettingsError; +use crate::settings::models::{RecallSettings, SettingsRequest}; + +/// The effective settings: `targets..recall` over `[recall]` over the built-in local store, local configuration over project configuration at each level. +pub fn resolve_recall_settings(request: &SettingsRequest) -> Result { + let project = read_document(&request.project_dir.join(PROJECT_FILE_NAME))?; + let local = read_document(&request.project_dir.join(LOCAL_FILE_NAME))?; + let documents: [Option<&ConfigValue>; 2] = [project.as_ref(), local.as_ref()]; + let target: Option = request + .selected_target + .clone() + .or_else(|| string_at(local.as_ref(), &["target"])) + .or_else(|| string_at(project.as_ref(), &["default_target"])); + let mut effective = section(project.as_ref(), &[RECALL_SECTION], RECALL_SECTION)?; + effective = overlay( + effective, + section(local.as_ref(), &[RECALL_SECTION], RECALL_SECTION)?, + ); + if let Some(target) = target.as_deref() { + let label = format!("targets.{target}.recall"); + for document in documents { + effective = overlay( + effective, + section(document, &["targets", target, RECALL_SECTION], &label)?, + ); + } + } + let project_adapter: Option = string_at(local.as_ref(), &["adapter"]) + .or_else(|| string_at(project.as_ref(), &["adapter"])); + let target_field = |key: &str| -> Option { + let target = target.as_deref()?; + string_at(local.as_ref(), &["targets", target, key]) + .or_else(|| string_at(project.as_ref(), &["targets", target, key])) + }; + let store = store_location( + &request.project_dir, + documents, + (effective.connection.as_deref(), project_adapter.as_deref()), + )?; + Ok(RecallSettings { + enabled: effective.enabled.unwrap_or(true), + target: target.clone(), + store, + connection: effective.connection, + retention_days: effective.retention_days.unwrap_or(DEFAULT_RETENTION_DAYS), + shutdown_timeout_ms: effective + .shutdown_timeout_ms + .unwrap_or(DEFAULT_SHUTDOWN_TIMEOUT_MS), + diagnostics_min_severity: effective + .diagnostics_min_severity + .unwrap_or_else(|| DEFAULT_DIAGNOSTICS_MIN_SEVERITY.to_owned()), + project_name: string_at(project.as_ref(), &["name"]), + project_adapter, + destination_database: target_field("database"), + destination_schema: target_field("schema"), + }) +} diff --git a/crates/sqlbuild-recall/src/settings/mod.rs b/crates/sqlbuild-recall/src/settings/mod.rs new file mode 100644 index 0000000000..69fa1e7523 --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/mod.rs @@ -0,0 +1,10 @@ +//! Effective Recall settings: `[recall]`, `targets..recall` and the store connection. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-recall/src/settings/models.rs b/crates/sqlbuild-recall/src/settings/models.rs new file mode 100644 index 0000000000..d0f3bbbba5 --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/models.rs @@ -0,0 +1,37 @@ +//! Resolved Recall settings for one invocation. + +use std::path::PathBuf; + +/// Where Recall writes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StoreLocation { + /// A SQLite file, written natively. + Sqlite { path: PathBuf }, + /// A connection whose adapter this build cannot write Recall rows to yet. + Unsupported { connection: String, adapter: String }, +} + +/// The effective settings: target, store and policies. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RecallSettings { + pub enabled: bool, + /// The effective target; `None` for commands without one. + pub target: Option, + pub store: StoreLocation, + /// The configured connection name, `None` for the built-in local store. + pub connection: Option, + pub retention_days: u32, + pub shutdown_timeout_ms: u64, + pub diagnostics_min_severity: String, + pub project_name: Option, + pub project_adapter: Option, + pub destination_database: Option, + pub destination_schema: Option, +} + +/// What selects the settings: the project directory and the `--target` option. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SettingsRequest { + pub project_dir: PathBuf, + pub selected_target: Option, +} diff --git a/crates/sqlbuild-recall/src/settings/tests.rs b/crates/sqlbuild-recall/src/settings/tests.rs new file mode 100644 index 0000000000..8766dedceb --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/tests.rs @@ -0,0 +1,2 @@ +#[path = "tests/test_resolve.rs"] +mod resolve; diff --git a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs new file mode 100644 index 0000000000..b570f5fc86 --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs @@ -0,0 +1,101 @@ +use std::path::Path; + +use crate::settings::main::resolve_recall_settings::resolve_recall_settings; +use crate::settings::models::{SettingsRequest, StoreLocation}; + +fn project(files: &[(&str, &str)]) -> tempfile::TempDir { + let directory = tempfile::tempdir().expect("temporary directory"); + for (name, text) in files { + std::fs::write(directory.path().join(name), text).expect("write config"); + } + directory +} + +fn request(directory: &Path, target: Option<&str>) -> SettingsRequest { + SettingsRequest { + project_dir: directory.to_path_buf(), + selected_target: target.map(str::to_owned), + } +} + +#[test] +fn given_no_recall_section_when_resolved_then_local_sqlite_is_on_by_default() { + let directory = project(&[( + "sqlbuild_project.toml", + "name = \"shop\"\nadapter = \"duckdb\"\ndefault_target = \"dev\"\n[targets.dev]\nschema = \"analytics\"\n", + )]); + let settings = resolve_recall_settings(&request(directory.path(), None)).expect("settings"); + assert!(settings.enabled); + assert_eq!(settings.target.as_deref(), Some("dev")); + assert_eq!( + settings.store, + StoreLocation::Sqlite { + path: directory.path().join("logs").join("recall.sqlite3") + } + ); + assert_eq!(settings.retention_days, 30); + assert_eq!(settings.shutdown_timeout_ms, 3000); + assert_eq!(settings.diagnostics_min_severity, "info"); + assert_eq!(settings.destination_schema.as_deref(), Some("analytics")); + assert_eq!(settings.project_adapter.as_deref(), Some("duckdb")); +} + +#[test] +fn given_layers_when_resolved_then_target_over_local_over_project() { + let directory = project(&[ + ( + "sqlbuild_project.toml", + "name = \"shop\"\nadapter = \"duckdb\"\n\ + [recall]\nretention = \"10d\"\nconnection = \"history\"\n\ + [connections.history]\nadapter = \"sqlite\"\npath = \"state/history.sqlite3\"\n\ + [connections.central]\nadapter = \"clickhouse\"\nhost = \"localhost\"\n\ + [targets.dev]\n[targets.dev.recall]\nenabled = false\n\ + [targets.prod]\n[targets.prod.recall]\nconnection = \"central\"\n", + ), + ( + "sqlbuild_local.toml", + "[recall]\nshutdown_timeout = \"1s\"\n[connections.history]\npath = \"/tmp/opencode/elsewhere.sqlite3\"\n", + ), + ]); + let dev = resolve_recall_settings(&request(directory.path(), Some("dev"))).expect("dev"); + assert!(!dev.enabled); + assert_eq!(dev.retention_days, 10); + assert_eq!(dev.shutdown_timeout_ms, 1000); + assert_eq!( + dev.store, + StoreLocation::Sqlite { + path: "/tmp/opencode/elsewhere.sqlite3".into() + } + ); + let prod = resolve_recall_settings(&request(directory.path(), Some("prod"))).expect("prod"); + assert!(prod.enabled); + assert_eq!( + prod.store, + StoreLocation::Unsupported { + connection: "central".to_owned(), + adapter: "clickhouse".to_owned() + } + ); +} + +#[test] +fn given_invalid_settings_when_resolved_then_the_error_names_the_setting() { + let unknown = project(&[( + "sqlbuild_project.toml", + "name = \"x\"\nadapter = \"duckdb\"\n[recall]\nconnection = \"missing\"\n", + )]); + let error = resolve_recall_settings(&request(unknown.path(), None)).expect_err("unknown"); + assert!( + error.message.contains("unknown connection 'missing'"), + "{error}" + ); + let bad = project(&[( + "sqlbuild_project.toml", + "name = \"x\"\nadapter = \"duckdb\"\n[targets.dev.recall]\nretention = \"soon\"\n", + )]); + let error = resolve_recall_settings(&request(bad.path(), Some("dev"))).expect_err("bad"); + assert!( + error.message.starts_with("targets.dev.recall.retention"), + "{error}" + ); +} diff --git a/crates/sqlbuild-recall/src/store/_helpers/mod.rs b/crates/sqlbuild-recall/src/store/_helpers/mod.rs new file mode 100644 index 0000000000..434bbaef58 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod rows; +pub(crate) mod schema; +pub(crate) mod sqlite; diff --git a/crates/sqlbuild-recall/src/store/_helpers/rows.rs b/crates/sqlbuild-recall/src/store/_helpers/rows.rs new file mode 100644 index 0000000000..400bb157ee --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/rows.rs @@ -0,0 +1,138 @@ +//! Idempotent row writes: insert once, compare content on a duplicate key, record conflicts. + +use rusqlite::{Connection, OptionalExtension}; + +use crate::records::main::conflict_row::conflict_row; +use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::Table; +use crate::store::_helpers::sqlite::{SqlValue, checked}; +use crate::store::errors::StoreError; +use crate::store::models::{ConflictEvidence, InsertOutcome}; + +fn key_text(value: Option<&RecallValue>) -> String { + match value { + Some(RecallValue::Text(text)) => text.clone(), + Some(RecallValue::Integer(number)) => number.to_string(), + Some(RecallValue::Real(number)) => number.to_string(), + Some(RecallValue::Blob(_)) => "".to_owned(), + Some(RecallValue::Null) | None => String::new(), + } +} + +fn text_of(row: &RecallRow, column: &str) -> String { + key_text(row.value(column)) +} + +/// Insert one row; `Some(evidence)` when its key holds different content. +fn write_row( + connection: &Connection, + row: &RecallRow, + ingested_at: &str, +) -> Result, StoreError> { + let spec = table_spec(row.table); + let ingested: RecallValue = RecallValue::Text(ingested_at.to_owned()); + let mut names: Vec<&str> = row.values.iter().map(|(name, _)| *name).collect(); + names.push("ingested_at"); + let placeholders: Vec = (1..=names.len()).map(|index| format!("?{index}")).collect(); + let sql = format!( + "INSERT INTO \"{}\" ({}) VALUES ({}) ON CONFLICT DO NOTHING", + spec.name, + names.join(", "), + placeholders.join(", ") + ); + let parameters: Vec> = row + .values + .iter() + .map(|(_, value)| SqlValue(value)) + .chain(std::iter::once(SqlValue(&ingested))) + .collect(); + let mut statement = checked(connection.prepare_cached(&sql), "prepare Recall insert")?; + let changed: usize = checked( + statement.execute(rusqlite::params_from_iter(parameters.iter())), + "write Recall rows", + )?; + if changed > 0 { + return Ok(None); + } + let condition: Vec = spec + .key + .iter() + .enumerate() + .map(|(index, name)| format!("{name} = ?{}", index + 1)) + .collect(); + let lookup = format!( + "SELECT content_sha256 FROM \"{}\" WHERE {}", + spec.name, + condition.join(" AND ") + ); + let null = RecallValue::Null; + let key_values: Vec> = spec + .key + .iter() + .map(|name| SqlValue(row.value(name).unwrap_or(&null))) + .collect(); + let mut statement = checked(connection.prepare_cached(&lookup), "prepare Recall lookup")?; + let existing: Option = checked( + statement + .query_row(rusqlite::params_from_iter(key_values.iter()), |found| { + found.get(0) + }) + .optional(), + "read stored Recall row", + )?; + let rejected: &str = row.content_sha256().unwrap_or_default(); + match existing { + Some(existing) if existing != rejected => Ok(Some(ConflictEvidence { + table: row.table, + record_key: spec + .key + .iter() + .map(|name| key_text(row.value(name))) + .collect::>() + .join(":"), + existing_sha256: existing, + rejected_sha256: rejected.to_owned(), + })), + _ => Ok(None), + } +} + +/// Write `rows` on the current transaction; conflicts become `recall_conflicts` rows. +pub(crate) fn write_rows( + connection: &Connection, + rows: &[RecallRow], + ingested_at: &str, +) -> Result { + let mut outcome = InsertOutcome::default(); + for row in rows { + match write_row(connection, row, ingested_at)? { + None => match outcome + .delivered + .iter_mut() + .find(|(seen, _)| *seen == row.table) + { + Some((_, count)) => *count += 1, + None => outcome.delivered.push((row.table, 1)), + }, + Some(evidence) => { + let envelope = RowEnvelope { + invocation_id: text_of(row, "invocation_id"), + invocation_started_at: text_of(row, "invocation_started_at"), + producer_version: text_of(row, "producer_version"), + }; + let conflict = conflict_row( + &envelope, + (table_spec(row.table).name, &evidence.record_key), + (&evidence.existing_sha256, &evidence.rejected_sha256), + ingested_at, + ); + if row.table != Table::Conflicts { + write_row(connection, &conflict, ingested_at)?; + } + outcome.conflicts.push(evidence); + } + } + } + Ok(outcome) +} diff --git a/crates/sqlbuild-recall/src/store/_helpers/schema.rs b/crates/sqlbuild-recall/src/store/_helpers/schema.rs new file mode 100644 index 0000000000..0a95844fcb --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/schema.rs @@ -0,0 +1,80 @@ +//! Schema state checks and additive column upgrades of a SQLite store. + +use std::collections::HashSet; + +use rusqlite::Connection; + +use crate::schema::constants::{ALL_TABLES, RECALL_SCHEMA_VERSION, VIEW_NAMES}; +use crate::schema::main::add_column_statement::add_column_statement; +use crate::schema::main::table_columns::table_columns; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::{DdlOptions, Dialect}; +use crate::store::_helpers::sqlite::{checked, stored_schema_version}; +use crate::store::errors::StoreError; + +pub(crate) fn sqlite_options() -> DdlOptions { + DdlOptions { + database: None, + retention_days: 0, + cluster: None, + } +} + +/// Whether every table and view exists at this build's schema version, read without a lock. +pub(crate) fn current(connection: &Connection) -> Result { + if stored_schema_version(connection)?.is_none_or(|version| version < RECALL_SCHEMA_VERSION) { + return Ok(false); + } + let present: i64 = checked( + connection.query_row( + "SELECT COUNT(*) FROM sqlite_schema WHERE type IN ('table', 'view') \ + AND name IN (SELECT value FROM json_each(?1))", + [expected_names()], + |row| row.get(0), + ), + "inspect Recall store", + )?; + Ok(present == i64::try_from(ALL_TABLES.len() + VIEW_NAMES.len()).unwrap_or(i64::MAX)) +} + +fn expected_names() -> String { + let names: Vec<&str> = ALL_TABLES + .iter() + .map(|table| table_spec(*table).name) + .chain(VIEW_NAMES) + .collect(); + serde_json::Value::from(names).to_string() +} + +pub(crate) fn add_missing_columns(connection: &Connection) -> Result<(), StoreError> { + let options = sqlite_options(); + for table in ALL_TABLES { + let spec = table_spec(table); + let mut statement = checked( + connection.prepare(&format!( + "SELECT name FROM pragma_table_xinfo('{}')", + spec.name + )), + "inspect Recall table", + )?; + let existing: HashSet = checked( + statement + .query_map([], |row| row.get::<_, String>(0)) + .and_then(Iterator::collect), + "inspect Recall table", + )?; + for column in table_columns(table) { + if !existing.contains(column.name) { + checked( + connection.execute_batch(&add_column_statement( + Dialect::Sqlite, + &options, + (spec, &column), + )), + "add Recall column", + )?; + } + } + } + Ok(()) +} diff --git a/crates/sqlbuild-recall/src/store/_helpers/sqlite.rs b/crates/sqlbuild-recall/src/store/_helpers/sqlite.rs new file mode 100644 index 0000000000..b188ed849a --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/sqlite.rs @@ -0,0 +1,99 @@ +//! rusqlite plumbing: error kinds, value conversion and transactions. + +use rusqlite::types::{ToSqlOutput, Value, ValueRef}; +use rusqlite::{Connection, ErrorCode}; + +use crate::records::models::RecallValue; +use crate::store::errors::{StoreError, StoreErrorKind}; + +/// The store error of a rusqlite failure, with what was being done. +pub(crate) fn store_error(error: &rusqlite::Error, doing: &str) -> StoreError { + let kind = match error.sqlite_error_code() { + Some(ErrorCode::ReadOnly | ErrorCode::CannotOpen | ErrorCode::PermissionDenied) => { + StoreErrorKind::Permission + } + Some(ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) => StoreErrorKind::Busy, + _ => StoreErrorKind::Other, + }; + StoreError::new(kind, format!("{doing}: {error}")) +} + +/// A closure's `rusqlite::Result` as a store result. +pub(crate) fn checked(result: rusqlite::Result, doing: &str) -> Result { + result.map_err(|error| store_error(&error, doing)) +} + +pub(crate) fn to_sql(value: &RecallValue) -> ToSqlOutput<'_> { + match value { + RecallValue::Null => ToSqlOutput::Owned(Value::Null), + RecallValue::Integer(number) => ToSqlOutput::Owned(Value::Integer(*number)), + RecallValue::Real(number) => ToSqlOutput::Owned(Value::Real(*number)), + RecallValue::Text(text) => ToSqlOutput::Borrowed(ValueRef::Text(text.as_bytes())), + RecallValue::Blob(data) => ToSqlOutput::Borrowed(ValueRef::Blob(data)), + } +} + +pub(crate) fn from_sql(value: ValueRef<'_>) -> RecallValue { + match value { + ValueRef::Null => RecallValue::Null, + ValueRef::Integer(number) => RecallValue::Integer(number), + ValueRef::Real(number) => RecallValue::Real(number), + ValueRef::Text(bytes) => RecallValue::Text(String::from_utf8_lossy(bytes).into_owned()), + ValueRef::Blob(bytes) => RecallValue::Blob(bytes.to_vec()), + } +} + +/// Run `work` inside `BEGIN IMMEDIATE … COMMIT`, rolling back when it fails. +pub(crate) fn immediate( + connection: &Connection, + doing: &str, + work: impl FnOnce(&Connection) -> Result, +) -> Result { + checked(connection.execute_batch("BEGIN IMMEDIATE"), doing)?; + match work(connection) { + Ok(value) => { + if let Err(error) = connection.execute_batch("COMMIT") { + drop(connection.execute_batch("ROLLBACK")); + return Err(store_error(&error, doing)); + } + Ok(value) + } + Err(error) => { + drop(connection.execute_batch("ROLLBACK")); + Err(error) + } + } +} + +/// The highest schema version recorded in `recall_meta`, `None` before the first setup. +pub(crate) fn stored_schema_version(connection: &Connection) -> Result, StoreError> { + let exists: bool = checked( + connection.query_row( + "SELECT EXISTS (SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = 'recall_meta')", + [], + |row| row.get(0), + ), + "inspect Recall store", + )?; + if !exists { + return Ok(None); + } + checked( + connection.query_row( + "SELECT MAX(CAST(value AS INTEGER)) FROM recall_meta \ + WHERE component = 'schema' AND key = 'schema_version'", + [], + |row| row.get(0), + ), + "read Recall schema version", + ) +} + +/// A borrowed Recall value bound as a statement parameter. +pub(crate) struct SqlValue<'a>(pub(crate) &'a RecallValue); + +impl rusqlite::ToSql for SqlValue<'_> { + fn to_sql(&self) -> rusqlite::Result> { + Ok(to_sql(self.0)) + } +} diff --git a/crates/sqlbuild-recall/src/store/constants.rs b/crates/sqlbuild-recall/src/store/constants.rs new file mode 100644 index 0000000000..fc5d1f1a7d --- /dev/null +++ b/crates/sqlbuild-recall/src/store/constants.rs @@ -0,0 +1,14 @@ +//! SQLite store pragmas and maintenance budgets. + +pub const BUSY_TIMEOUT_MS: u64 = 5_000; +/// `journal_size_limit`: the WAL is truncated to this size after checkpoints. +pub const JOURNAL_SIZE_LIMIT_BYTES: u64 = 64 * 1024 * 1024; +/// Maintenance runs at most once per this many seconds. +pub const MAINTENANCE_INTERVAL_SECONDS: u64 = 24 * 60 * 60; +/// Total time one maintenance run may spend deleting. +pub const MAINTENANCE_BUDGET_MS: u64 = 2_000; +/// Pages released by one `incremental_vacuum`. +pub const INCREMENTAL_VACUUM_PAGES: u32 = 2_048; +pub const META_MAINTENANCE_OVERDUE_KEY: &str = "overdue"; +/// The selector naming the most recently started invocation. +pub const LAST_INVOCATION_SELECTOR: &str = "last"; diff --git a/crates/sqlbuild-recall/src/store/errors.rs b/crates/sqlbuild-recall/src/store/errors.rs new file mode 100644 index 0000000000..4959ea152e --- /dev/null +++ b/crates/sqlbuild-recall/src/store/errors.rs @@ -0,0 +1,43 @@ +//! Why a Recall store operation failed. + +/// What kind of failure a store reported. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StoreErrorKind { + /// The file or directory cannot be created or written. + Permission, + /// Another writer holds the lock past the busy timeout; retrying may succeed. + Busy, + /// The SQLite library is older than the store needs. + Version, + /// A query was rejected: not read-only, or invalid SQL. + Query, + /// No invocation matches. + NotFound, + /// More than one invocation matches. + Ambiguous, + Other, +} + +/// A store failure with its kind and message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StoreError { + pub kind: StoreErrorKind, + pub message: String, +} + +impl StoreError { + pub(crate) fn new(kind: StoreErrorKind, message: impl Into) -> Self { + Self { + kind, + message: message.into(), + } + } +} + +impl std::fmt::Display for StoreError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.message) + } +} + +impl std::error::Error for StoreError {} diff --git a/crates/sqlbuild-recall/src/store/main/describe_store.rs b/crates/sqlbuild-recall/src/store/main/describe_store.rs new file mode 100644 index 0000000000..a23de3603a --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/describe_store.rs @@ -0,0 +1,49 @@ +//! `describe_store`: the store's views and tables with their columns. + +use crate::store::_helpers::sqlite::checked; +use crate::store::errors::StoreError; +use crate::store::models::{DescribedColumn, DescribedRelation, SqliteStore}; + +/// Views first, then `recall_*` tables, each sorted by name. +pub fn describe_store(store: &SqliteStore) -> Result, StoreError> { + let mut statement = checked( + store.connection.prepare( + "SELECT name, type FROM sqlite_schema WHERE type IN ('table', 'view') \ + AND name NOT LIKE 'sqlite!_%' ESCAPE '!' ORDER BY type = 'table', name", + ), + "describe Recall store", + )?; + let relations: Vec<(String, String)> = checked( + statement + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .and_then(Iterator::collect), + "describe Recall store", + )?; + let mut described = Vec::with_capacity(relations.len()); + for (name, kind) in relations { + let mut columns = checked( + store + .connection + .prepare("SELECT name, type, \"notnull\" FROM pragma_table_xinfo(?1)"), + "describe Recall relation", + )?; + let columns: Vec = checked( + columns + .query_map([&name], |row| { + Ok(DescribedColumn { + name: row.get(0)?, + declared_type: row.get(1)?, + nullable: row.get::<_, i64>(2)? == 0, + }) + }) + .and_then(Iterator::collect), + "describe Recall relation", + )?; + described.push(DescribedRelation { + name, + kind, + columns, + }); + } + Ok(described) +} diff --git a/crates/sqlbuild-recall/src/store/main/ensure_sqlite_schema.rs b/crates/sqlbuild-recall/src/store/main/ensure_sqlite_schema.rs new file mode 100644 index 0000000000..01ad4d8695 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/ensure_sqlite_schema.rs @@ -0,0 +1,57 @@ +//! `ensure_sqlite_schema`: create and additively upgrade the store's tables and views. + +use crate::records::main::meta_row::meta_row; +use crate::records::models::RowEnvelope; +use crate::schema::constants::{ + META_SCHEMA_COMPONENT, META_SCHEMA_VERSION_KEY, RECALL_SCHEMA_VERSION, +}; +use crate::schema::main::table_statements::table_statements; +use crate::schema::main::view_statements::view_statements; +use crate::schema::models::Dialect; +use crate::store::_helpers::rows::write_rows; +use crate::store::_helpers::schema::{add_missing_columns, current, sqlite_options}; +use crate::store::_helpers::sqlite::{checked, immediate, stored_schema_version}; +use crate::store::errors::StoreError; +use crate::store::models::SqliteStore; + +/// Create missing tables, indexes and views, add missing columns and record the schema version, in one `BEGIN IMMEDIATE` transaction. A store written by a newer SQLBuild keeps its views. +pub fn ensure_sqlite_schema( + store: &SqliteStore, + envelope: &RowEnvelope, + recorded_at: &str, +) -> Result<(), StoreError> { + if current(&store.connection)? { + return Ok(()); + } + let options = sqlite_options(); + immediate(&store.connection, "create Recall tables", |connection| { + for statement in table_statements(Dialect::Sqlite, &options) { + checked(connection.execute_batch(&statement), "create Recall table")?; + } + add_missing_columns(connection)?; + let stored: Option = stored_schema_version(connection)?; + let upgrade: bool = stored.is_none_or(|version| version < RECALL_SCHEMA_VERSION); + for (name, statement) in view_statements(Dialect::Sqlite, &options) { + if upgrade { + checked( + connection.execute_batch(&format!("DROP VIEW IF EXISTS \"{name}\"")), + "replace Recall view", + )?; + } + checked(connection.execute_batch(&statement), "create Recall view")?; + } + if upgrade { + let version = RECALL_SCHEMA_VERSION.to_string(); + write_rows( + connection, + &[meta_row( + envelope, + (META_SCHEMA_COMPONENT, META_SCHEMA_VERSION_KEY, &version), + recorded_at, + )], + recorded_at, + )?; + } + Ok(()) + }) +} diff --git a/crates/sqlbuild-recall/src/store/main/insert_rows.rs b/crates/sqlbuild-recall/src/store/main/insert_rows.rs new file mode 100644 index 0000000000..aaec9c77e8 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/insert_rows.rs @@ -0,0 +1,18 @@ +//! `insert_rows`: write one batch atomically and idempotently. + +use crate::records::models::RecallRow; +use crate::store::_helpers::rows::write_rows; +use crate::store::_helpers::sqlite::immediate; +use crate::store::errors::StoreError; +use crate::store::models::{InsertOutcome, SqliteStore}; + +/// Write `rows` in one transaction. A row whose key exists with equal content is a no-op; one with different content keeps the stored row and is reported and recorded as a conflict. +pub fn insert_rows( + store: &SqliteStore, + rows: &[RecallRow], + ingested_at: &str, +) -> Result { + immediate(&store.connection, "write Recall rows", |connection| { + write_rows(connection, rows, ingested_at) + }) +} diff --git a/crates/sqlbuild-recall/src/store/main/maintain_sqlite_store.rs b/crates/sqlbuild-recall/src/store/main/maintain_sqlite_store.rs new file mode 100644 index 0000000000..0961d2a5cc --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/maintain_sqlite_store.rs @@ -0,0 +1,174 @@ +//! `maintain_sqlite_store`: claimed, budgeted retention deletes and space reclamation. + +use std::time::{Duration, Instant, SystemTime}; + +use rusqlite::Connection; + +use crate::records::main::format_timestamp::format_timestamp; +use crate::records::main::meta_row::meta_row; +use crate::records::main::parse_timestamp::parse_timestamp; +use crate::schema::constants::{ + META_MAINTENANCE_CLAIM_KEY, META_MAINTENANCE_COMPONENT, RETENTION_BATCH_INVOCATIONS, +}; +use crate::schema::main::sqlite_retention_sql::sqlite_retention_sql; +use crate::store::_helpers::rows::write_rows; +use crate::store::_helpers::sqlite::{checked, immediate}; +use crate::store::constants::{ + INCREMENTAL_VACUUM_PAGES, MAINTENANCE_INTERVAL_SECONDS, META_MAINTENANCE_OVERDUE_KEY, +}; +use crate::store::errors::StoreError; +use crate::store::models::{MaintenanceOutcome, MaintenanceRequest, SqliteStore}; + +fn latest(connection: &Connection, key: &str) -> Result, StoreError> { + checked( + connection.query_row( + "SELECT MAX(recorded_at) FROM recall_meta WHERE component = ?1 AND key = ?2", + [META_MAINTENANCE_COMPONENT, key], + |row| row.get(0), + ), + "read Recall maintenance state", + ) +} + +/// Whether this process should maintain now: no recent claim, or the last run left work. +fn due(connection: &Connection, now: SystemTime) -> Result { + let Some(claim) = latest(connection, META_MAINTENANCE_CLAIM_KEY)? else { + return Ok(true); + }; + if latest(connection, META_MAINTENANCE_OVERDUE_KEY)?.is_some_and(|overdue| overdue >= claim) { + return Ok(true); + } + let claimed: SystemTime = parse_timestamp(&claim).unwrap_or(SystemTime::UNIX_EPOCH); + Ok(now + .duration_since(claimed) + .is_ok_and(|elapsed| elapsed >= Duration::from_secs(MAINTENANCE_INTERVAL_SECONDS))) +} + +/// Delete invocations started before the retention cutoff, at most once per interval across processes, in short batches within the budget, then reclaim pages and checkpoint the WAL. +pub fn maintain_sqlite_store( + store: &SqliteStore, + request: &MaintenanceRequest, +) -> Result { + let started = Instant::now(); + let now_text: String = format_timestamp(request.now); + let claimed: bool = immediate( + &store.connection, + "claim Recall maintenance", + |connection| { + if !due(connection, request.now)? { + return Ok(false); + } + write_rows( + connection, + &[meta_row( + &request.envelope, + ( + META_MAINTENANCE_COMPONENT, + META_MAINTENANCE_CLAIM_KEY, + &request.owner, + ), + &now_text, + )], + &now_text, + )?; + Ok(true) + }, + )?; + if !claimed { + return Ok(MaintenanceOutcome::default()); + } + let retention = Duration::from_secs(u64::from(request.retention_days) * 86_400); + let cutoff: String = format_timestamp( + request + .now + .checked_sub(retention) + .unwrap_or(SystemTime::UNIX_EPOCH), + ); + let sql = sqlite_retention_sql(); + let mut outcome = MaintenanceOutcome { + ran: true, + ..MaintenanceOutcome::default() + }; + loop { + if started.elapsed() >= request.budget { + break; + } + let deleted: usize = + immediate(&store.connection, "apply Recall retention", |connection| { + let mut statement = checked( + connection.prepare_cached(&sql.expired_invocations), + "find expired Recall invocations", + )?; + let ids: Vec = checked( + statement + .query_map( + rusqlite::params![cutoff, now_text, RETENTION_BATCH_INVOCATIONS], + |row| row.get(0), + ) + .and_then(Iterator::collect), + "find expired Recall invocations", + )?; + if ids.is_empty() { + return Ok(0); + } + let encoded: String = serde_json::Value::from(ids.clone()).to_string(); + for (_, delete) in &sql.delete_invocations { + checked( + connection.execute(delete, [&encoded]), + "delete expired Recall rows", + )?; + } + Ok(ids.len()) + })?; + if deleted == 0 { + outcome.complete = true; + break; + } + outcome.deleted_invocations += deleted as u64; + } + outcome.deleted_artifacts = checked( + store + .connection + .execute(&sql.delete_orphan_artifacts, [&cutoff]), + "delete unreferenced Recall artifacts", + )? as u64; + checked( + store.connection.execute(&sql.delete_old_claims, [&cutoff]), + "delete old Recall maintenance claims", + )?; + if !outcome.complete { + immediate( + &store.connection, + "record overdue maintenance", + |connection| { + write_rows( + connection, + &[meta_row( + &request.envelope, + ( + META_MAINTENANCE_COMPONENT, + META_MAINTENANCE_OVERDUE_KEY, + &request.owner, + ), + &now_text, + )], + &now_text, + ) + .map(drop) + }, + )?; + } + checked( + store.connection.execute_batch(&format!( + "PRAGMA incremental_vacuum({INCREMENTAL_VACUUM_PAGES})" + )), + "reclaim Recall pages", + )?; + checked( + store + .connection + .query_row("PRAGMA wal_checkpoint(PASSIVE)", [], |_| Ok(())), + "checkpoint Recall WAL", + )?; + Ok(outcome) +} diff --git a/crates/sqlbuild-recall/src/store/main/mod.rs b/crates/sqlbuild-recall/src/store/main/mod.rs new file mode 100644 index 0000000000..0605e2bff9 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/mod.rs @@ -0,0 +1,8 @@ +pub mod describe_store; +pub mod ensure_sqlite_schema; +pub mod insert_rows; +pub mod maintain_sqlite_store; +pub mod open_sqlite_store; +pub mod query_store; +pub mod read_output_transcript; +pub mod resolve_invocation; diff --git a/crates/sqlbuild-recall/src/store/main/open_sqlite_store.rs b/crates/sqlbuild-recall/src/store/main/open_sqlite_store.rs new file mode 100644 index 0000000000..e2a102e774 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/open_sqlite_store.rs @@ -0,0 +1,124 @@ +//! `open_sqlite_store`: open, or create, a SQLite Recall store with its pragmas. + +use std::path::Path; +use std::time::Duration; + +use rusqlite::{Connection, OpenFlags}; + +use crate::schema::constants::MINIMUM_SQLITE_VERSION; +use crate::store::_helpers::sqlite::{checked, store_error}; +use crate::store::constants::{BUSY_TIMEOUT_MS, JOURNAL_SIZE_LIMIT_BYTES}; +use crate::store::errors::{StoreError, StoreErrorKind}; +use crate::store::models::{OpenMode, SqliteStore}; + +fn check_version() -> Result<(), StoreError> { + let version: i32 = rusqlite::version_number(); + let (major, minor) = MINIMUM_SQLITE_VERSION; + let minimum = i32::try_from(major * 1_000_000 + minor * 1_000).unwrap_or(i32::MAX); + if version < minimum { + return Err(StoreError::new( + StoreErrorKind::Version, + format!( + "Recall needs SQLite {major}.{minor} or newer; this build has {}", + rusqlite::version() + ), + )); + } + Ok(()) +} + +fn create_parent(path: &Path) -> Result<(), StoreError> { + let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + else { + return Ok(()); + }; + std::fs::create_dir_all(parent).map_err(|error| { + let kind = if error.kind() == std::io::ErrorKind::PermissionDenied + || error.kind() == std::io::ErrorKind::ReadOnlyFilesystem + { + StoreErrorKind::Permission + } else { + StoreErrorKind::Other + }; + StoreError::new( + kind, + format!("cannot create directory {}: {error}", parent.display()), + ) + }) +} + +/// The open store at `path`. Writers create the file and its directory and switch it to WAL; a new file gets incremental auto-vacuum before any table exists. +pub fn open_sqlite_store(path: &Path, mode: OpenMode) -> Result { + check_version()?; + let connection: Connection = match mode { + OpenMode::ReadWrite => { + create_parent(path)?; + checked( + Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_WRITE + | OpenFlags::SQLITE_OPEN_CREATE + | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ), + &format!("cannot open {}", path.display()), + )? + } + OpenMode::ReadOnly => { + if !path.exists() { + return Err(StoreError::new( + StoreErrorKind::NotFound, + format!("no Recall store at {}", path.display()), + )); + } + checked( + Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ), + &format!("cannot open {}", path.display()), + )? + } + }; + checked( + connection.busy_timeout(Duration::from_millis(BUSY_TIMEOUT_MS)), + "set busy timeout", + )?; + if mode == OpenMode::ReadWrite { + let page_count: i64 = checked( + connection.query_row("PRAGMA page_count", [], |row| row.get(0)), + &format!("cannot read {}", path.display()), + )?; + if page_count == 0 { + checked( + connection.execute_batch("PRAGMA auto_vacuum = INCREMENTAL"), + "enable incremental vacuum", + )?; + } + let journal: String = connection + .query_row("PRAGMA journal_mode = WAL", [], |row| row.get(0)) + .map_err(|error| store_error(&error, &format!("cannot write {}", path.display())))?; + if !journal.eq_ignore_ascii_case("wal") { + return Err(StoreError::new( + StoreErrorKind::Permission, + format!("cannot switch {} to WAL mode", path.display()), + )); + } + checked( + connection.execute_batch(&format!( + "PRAGMA synchronous = NORMAL; PRAGMA journal_size_limit = {JOURNAL_SIZE_LIMIT_BYTES};" + )), + "set store pragmas", + )?; + } else { + checked( + connection.execute_batch("PRAGMA query_only = ON"), + "open store read-only", + )?; + } + Ok(SqliteStore { + connection, + path: path.to_path_buf(), + }) +} diff --git a/crates/sqlbuild-recall/src/store/main/query_store.rs b/crates/sqlbuild-recall/src/store/main/query_store.rs new file mode 100644 index 0000000000..26461a3ac5 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/query_store.rs @@ -0,0 +1,54 @@ +//! `query_store`: run one read-only query against a store. + +use crate::store::_helpers::sqlite::{checked, from_sql}; +use crate::store::errors::{StoreError, StoreErrorKind}; +use crate::store::models::{QueryResult, SqliteStore}; + +/// The columns and at most `limit` rows of `sql`; statements that write are rejected. +pub fn query_store( + store: &SqliteStore, + sql: &str, + limit: Option, +) -> Result { + let mut statement = store.connection.prepare(sql).map_err(|error| { + StoreError::new( + StoreErrorKind::Query, + format!("invalid Recall query: {error}"), + ) + })?; + if !statement.readonly() { + return Err(StoreError::new( + StoreErrorKind::Query, + "Recall queries must be read-only SELECT statements", + )); + } + let columns: Vec = statement + .column_names() + .into_iter() + .map(str::to_owned) + .collect(); + let width: usize = columns.len(); + let mut rows = checked(statement.query([]), "run Recall query")?; + let mut result = QueryResult { + columns, + rows: Vec::new(), + truncated: false, + }; + while let Some(row) = rows.next().map_err(|error| { + StoreError::new( + StoreErrorKind::Query, + format!("Recall query failed: {error}"), + ) + })? { + if limit.is_some_and(|limit| result.rows.len() >= limit) { + result.truncated = true; + break; + } + let mut values = Vec::with_capacity(width); + for index in 0..width { + values.push(from_sql(checked(row.get_ref(index), "read Recall row")?)); + } + result.rows.push(values); + } + Ok(result) +} diff --git a/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs b/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs new file mode 100644 index 0000000000..705968bd5d --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs @@ -0,0 +1,78 @@ +//! `read_output_transcript`: one invocation's exact output bytes and their completeness. + +use rusqlite::OptionalExtension; + +use crate::records::models::OutputStream; +use crate::store::_helpers::sqlite::checked; +use crate::store::errors::StoreError; +use crate::store::models::{OutputTranscript, SqliteStore}; + +/// The bytes of `stream`, or both streams interleaved by sequence when `None`. +pub fn read_output_transcript( + store: &SqliteStore, + invocation_id: &str, + stream: Option, +) -> Result { + let mut statement = checked( + store.connection.prepare( + "SELECT data FROM recall_output WHERE invocation_id = ?1 \ + AND (?2 IS NULL OR stream = ?2) ORDER BY sequence", + ), + "read Recall output", + )?; + let chunks: Vec> = checked( + statement + .query_map( + rusqlite::params![invocation_id, stream.map(OutputStream::as_str)], + |row| row.get(0), + ) + .and_then(Iterator::collect), + "read Recall output", + )?; + let mut transcript = OutputTranscript { + invocation_id: invocation_id.to_owned(), + data: chunks.concat(), + ..OutputTranscript::default() + }; + transcript.status = checked( + store + .connection + .query_row( + "SELECT status FROM invocations WHERE invocation_id = ?1", + [invocation_id], + |row| row.get(0), + ) + .optional(), + "read Recall invocation", + )?; + let completeness: Option<(i64, i64, i64, bool, bool, Option)> = checked( + store + .connection + .query_row( + "SELECT missing_sequences, reported_drops, filtered, flush_complete, complete, \ + gap_reason FROM output_invocations WHERE invocation_id = ?1", + [invocation_id], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + )) + }, + ) + .optional(), + "read Recall output completeness", + )?; + if let Some((missing, drops, filtered, flushed, complete, reason)) = completeness { + transcript.missing_sequences = missing; + transcript.reported_drops = drops; + transcript.filtered = filtered; + transcript.flush_complete = flushed; + transcript.complete = complete; + transcript.gap_reason = reason; + } + Ok(transcript) +} diff --git a/crates/sqlbuild-recall/src/store/main/resolve_invocation.rs b/crates/sqlbuild-recall/src/store/main/resolve_invocation.rs new file mode 100644 index 0000000000..71245acf0b --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/resolve_invocation.rs @@ -0,0 +1,62 @@ +//! `resolve_invocation`: the invocation an ID, unique prefix or `last` names. + +use crate::store::_helpers::sqlite::checked; +use crate::store::constants::LAST_INVOCATION_SELECTOR as LAST; +use crate::store::errors::{StoreError, StoreErrorKind}; +use crate::store::models::SqliteStore; + +/// The full invocation ID of `selector`; `last` is the most recently started invocation. +pub fn resolve_invocation(store: &SqliteStore, selector: &str) -> Result { + let candidates: Vec = if selector == LAST { + let mut statement = checked( + store.connection.prepare( + "SELECT invocation_id FROM invocations \ + ORDER BY invocation_started_at DESC, invocation_id DESC LIMIT 1", + ), + "find the last invocation", + )?; + checked( + statement + .query_map([], |row| row.get(0)) + .and_then(Iterator::collect), + "find the last invocation", + )? + } else { + let mut statement = checked( + store.connection.prepare( + "SELECT invocation_id FROM invocations \ + WHERE substr(invocation_id, 1, length(?1)) = ?1 \ + ORDER BY invocation_id LIMIT 3", + ), + "find the invocation", + )?; + checked( + statement + .query_map([selector], |row| row.get(0)) + .and_then(Iterator::collect), + "find the invocation", + )? + }; + if let Some(exact) = candidates.iter().find(|candidate| *candidate == selector) { + return Ok(exact.clone()); + } + match candidates.as_slice() { + [] => Err(StoreError::new( + StoreErrorKind::NotFound, + if selector == LAST { + "no invocations are recorded yet".to_owned() + } else { + format!("no recorded invocation matches '{selector}'") + }, + )), + [only] => Ok(only.clone()), + several => Err(StoreError::new( + StoreErrorKind::Ambiguous, + format!( + "'{selector}' matches several invocations: {}{}", + several[..2].join(", "), + if several.len() > 2 { ", …" } else { "" } + ), + )), + } +} diff --git a/crates/sqlbuild-recall/src/store/mod.rs b/crates/sqlbuild-recall/src/store/mod.rs new file mode 100644 index 0000000000..0126341c03 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/mod.rs @@ -0,0 +1,10 @@ +//! The native SQLite Recall store: automatic DDL, idempotent inserts, maintenance and reads. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-recall/src/store/models.rs b/crates/sqlbuild-recall/src/store/models.rs new file mode 100644 index 0000000000..975c096ecc --- /dev/null +++ b/crates/sqlbuild-recall/src/store/models.rs @@ -0,0 +1,99 @@ +//! An open SQLite Recall store and the outcomes of its operations. + +use std::path::PathBuf; +use std::time::{Duration, SystemTime}; + +use crate::records::models::{RecallValue, RowEnvelope}; +use crate::schema::models::Table; + +/// One open store file. Only one thread uses a store. +#[derive(Debug)] +pub struct SqliteStore { + pub(crate) connection: rusqlite::Connection, + pub path: PathBuf, +} + +/// Whether a store is opened for writing, creating it when missing, or only for reading. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OpenMode { + ReadWrite, + ReadOnly, +} + +/// A row whose key was already stored with different content; the first row is kept. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConflictEvidence { + pub table: Table, + pub record_key: String, + pub existing_sha256: String, + pub rejected_sha256: String, +} + +/// What one insert batch did, per table. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct InsertOutcome { + /// Rows written or already present with equal content, by table. + pub delivered: Vec<(Table, u64)>, + pub conflicts: Vec, +} + +/// What a maintenance run is allowed to do. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MaintenanceRequest { + pub now: SystemTime, + pub retention_days: u32, + pub owner: String, + pub budget: Duration, + pub envelope: RowEnvelope, +} + +/// What a maintenance run did. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MaintenanceOutcome { + /// False when another process maintained the store within the interval. + pub ran: bool, + pub deleted_invocations: u64, + pub deleted_artifacts: u64, + /// False when the budget ran out with expired invocations left; the next run is overdue. + pub complete: bool, +} + +/// The rows of a read-only query. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct QueryResult { + pub columns: Vec, + pub rows: Vec>, + /// More rows existed beyond the limit. + pub truncated: bool, +} + +/// One column of a table or view, as the store describes it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DescribedColumn { + pub name: String, + pub declared_type: String, + pub nullable: bool, +} + +/// One table or view of the store. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DescribedRelation { + pub name: String, + /// `table` or `view`. + pub kind: String, + pub columns: Vec, +} + +/// One invocation's stored output and how complete it is. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct OutputTranscript { + pub invocation_id: String, + pub data: Vec, + pub status: Option, + pub missing_sequences: i64, + pub reported_drops: i64, + pub filtered: i64, + pub flush_complete: bool, + pub complete: bool, + pub gap_reason: Option, +} diff --git a/crates/sqlbuild-recall/src/store/tests.rs b/crates/sqlbuild-recall/src/store/tests.rs new file mode 100644 index 0000000000..807f02a975 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_maintenance.rs"] +mod maintenance; +#[path = "tests/test_store.rs"] +mod store; +#[path = "tests/test_views.rs"] +mod views; diff --git a/crates/sqlbuild-recall/src/store/tests/helpers.rs b/crates/sqlbuild-recall/src/store/tests/helpers.rs new file mode 100644 index 0000000000..24a3853e58 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/tests/helpers.rs @@ -0,0 +1,98 @@ +use std::time::{Duration, SystemTime}; + +use tempfile::TempDir; + +use crate::records::main::event_row::event_row; +use crate::records::main::format_timestamp::format_timestamp; +use crate::records::main::output_row::output_row; +use crate::records::models::{OutputChunk, OutputStream, RecallRow, RecallValue, RowEnvelope}; +use crate::store::main::ensure_sqlite_schema::ensure_sqlite_schema; +use crate::store::main::insert_rows::insert_rows; +use crate::store::main::open_sqlite_store::open_sqlite_store; +use crate::store::main::query_store::query_store; +use crate::store::models::{OpenMode, SqliteStore}; + +pub(crate) const NOW: &str = "2026-10-19T12:00:00.000000Z"; + +pub(crate) fn envelope(invocation_id: &str, started_at: &str) -> RowEnvelope { + RowEnvelope { + invocation_id: invocation_id.to_owned(), + invocation_started_at: started_at.to_owned(), + producer_version: "test".to_owned(), + } +} + +pub(crate) fn new_store() -> (TempDir, SqliteStore) { + let directory = tempfile::tempdir().expect("temporary directory"); + let store = open_sqlite_store( + &directory.path().join("logs/recall.sqlite3"), + OpenMode::ReadWrite, + ) + .expect("store opens"); + ensure_sqlite_schema(&store, &envelope("setup", NOW), NOW).expect("schema"); + (directory, store) +} + +pub(crate) fn write(store: &SqliteStore, rows: &[RecallRow]) { + insert_rows(store, rows, NOW).expect("rows written"); +} + +pub(crate) fn chunk( + envelope: &RowEnvelope, + (sequence, stream, data): (u64, OutputStream, &[u8]), +) -> RecallRow { + output_row( + envelope, + &OutputChunk { + sequence, + stream, + occurred_at: NOW.to_owned(), + data: data.to_vec(), + byte_offset: 0, + dropped_before: 0, + }, + ) +} + +pub(crate) fn event( + envelope: &RowEnvelope, + (event_id, event_type, sequence): (&str, &str, i64), + correlations: &str, + payload: &str, +) -> RecallRow { + let json = format!( + r#"{{"event_id":"{event_id}","event_type":"{event_type}","schema_version":2, + "producer":"sqlbuild","producer_version":"1","occurred_at":"{NOW}", + "invocation_id":"{}","invocation_sequence":{sequence}{correlations},"payload":{payload}}}"#, + envelope.invocation_id + ); + event_row(envelope, &json).expect("event row") +} + +pub(crate) fn rows(store: &SqliteStore, sql: &str) -> Vec> { + query_store(store, sql, None).expect("query").rows +} + +pub(crate) fn text(value: &str) -> RecallValue { + RecallValue::Text(value.to_owned()) +} + +pub(crate) fn shifted(seconds: i64) -> String { + let now = crate::records::main::parse_timestamp::parse_timestamp(NOW).expect("now"); + let moved: SystemTime = if seconds >= 0 { + now + Duration::from_secs(seconds.unsigned_abs()) + } else { + now - Duration::from_secs(seconds.unsigned_abs()) + }; + format_timestamp(moved) +} + +pub(crate) fn wall_clock(seconds_from_now: i64) -> String { + let now = SystemTime::now(); + let moved = if seconds_from_now >= 0 { + now + Duration::from_secs(seconds_from_now.unsigned_abs()) + } else { + now - Duration::from_secs(seconds_from_now.unsigned_abs()) + }; + format_timestamp(moved) +} diff --git a/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs b/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs new file mode 100644 index 0000000000..b1c3187d79 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs @@ -0,0 +1,110 @@ +use std::time::Duration; + +use crate::records::main::artifact_rows::artifact_rows; +use crate::records::main::heartbeat_row::heartbeat_row; +use crate::records::main::invocation_row::invocation_row; +use crate::records::main::parse_timestamp::parse_timestamp; +use crate::records::models::{ArtifactInput, InvocationContext, OutputStream, RecallValue}; +use crate::store::main::maintain_sqlite_store::maintain_sqlite_store; +use crate::store::models::{MaintenanceOutcome, MaintenanceRequest}; +use crate::store::tests::helpers::{NOW, chunk, envelope, new_store, rows, shifted, text, write}; + +fn request(now: &str, budget: Duration) -> MaintenanceRequest { + MaintenanceRequest { + now: parse_timestamp(now).expect("now"), + retention_days: 30, + owner: "host:1".to_owned(), + budget, + envelope: envelope("maintainer", now), + } +} + +fn artifact(digest: &str, seen_at: &str) -> ArtifactInput { + ArtifactInput { + digest: digest.to_owned(), + kind: "sql".to_owned(), + media_type: "text/x-sql".to_owned(), + content: b"select 1".to_vec(), + seen_at: seen_at.to_owned(), + ref_id: digest.to_owned(), + role: "executed_sql".to_owned(), + statement_id: None, + resource_id: None, + } +} + +#[test] +fn given_expired_recent_and_running_invocations_when_maintained_then_only_expired_rows_go() { + let (_directory, store) = new_store(); + let old = envelope("old", &shifted(-40 * 86_400)); + let recent = envelope("recent", &shifted(-86_400)); + let old_running = envelope("old_running", &shifted(-40 * 86_400)); + let mut batch = Vec::new(); + for env in [&old, &recent, &old_running] { + batch.push(invocation_row(env, &InvocationContext::default())); + batch.push(chunk(env, (0, OutputStream::Stdout, b"x"))); + } + batch.push(heartbeat_row( + &old_running, + (&shifted(-10), &shifted(80)), + (None, None), + )); + batch.extend(artifact_rows( + &old, + &artifact("sha256:old", &shifted(-40 * 86_400)), + )); + batch.extend(artifact_rows( + &recent, + &artifact("sha256:recent", &shifted(-40 * 86_400)), + )); + write(&store, &batch); + let outcome = + maintain_sqlite_store(&store, &request(NOW, Duration::from_secs(5))).expect("maintenance"); + assert_eq!( + outcome, + MaintenanceOutcome { + ran: true, + deleted_invocations: 1, + deleted_artifacts: 1, + complete: true + } + ); + assert_eq!( + rows( + &store, + "SELECT invocation_id FROM recall_output ORDER BY invocation_id" + ), + vec![vec![text("old_running")], vec![text("recent")]] + ); + assert_eq!( + rows(&store, "SELECT digest FROM recall_artifacts"), + vec![vec![text("sha256:recent")]] + ); + let again = maintain_sqlite_store(&store, &request(&shifted(3600), Duration::from_secs(5))) + .expect("second maintenance"); + assert!(!again.ran, "a claim within 24 hours stops another run"); + let next_day = + maintain_sqlite_store(&store, &request(&shifted(90_000), Duration::from_secs(5))) + .expect("next day"); + assert!(next_day.ran); +} + +#[test] +fn given_no_budget_when_maintained_then_the_next_run_is_overdue() { + let (_directory, store) = new_store(); + let old = envelope("old", &shifted(-40 * 86_400)); + write( + &store, + &[invocation_row(&old, &InvocationContext::default())], + ); + let starved = maintain_sqlite_store(&store, &request(NOW, Duration::ZERO)).expect("starved"); + assert!(starved.ran && !starved.complete); + let overdue = maintain_sqlite_store(&store, &request(&shifted(60), Duration::from_secs(5))) + .expect("overdue run"); + assert!(overdue.ran && overdue.complete); + assert_eq!(overdue.deleted_invocations, 1); + assert_eq!( + rows(&store, "SELECT COUNT(*) FROM recall_invocations"), + vec![vec![RecallValue::Integer(0)]] + ); +} diff --git a/crates/sqlbuild-recall/src/store/tests/test_store.rs b/crates/sqlbuild-recall/src/store/tests/test_store.rs new file mode 100644 index 0000000000..6a93e16f9f --- /dev/null +++ b/crates/sqlbuild-recall/src/store/tests/test_store.rs @@ -0,0 +1,134 @@ +use crate::records::main::invocation_row::invocation_row; +use crate::records::main::meta_row::meta_row; +use crate::records::models::{InvocationContext, OutputStream, RecallValue}; +use crate::store::errors::StoreErrorKind; +use crate::store::main::describe_store::describe_store; +use crate::store::main::ensure_sqlite_schema::ensure_sqlite_schema; +use crate::store::main::open_sqlite_store::open_sqlite_store; +use crate::store::main::query_store::query_store; +use crate::store::main::read_output_transcript::read_output_transcript; +use crate::store::main::resolve_invocation::resolve_invocation; +use crate::store::models::OpenMode; +use crate::store::tests::helpers::{NOW, chunk, envelope, new_store, rows, shifted, write}; + +#[test] +fn given_ids_when_resolved_then_prefixes_last_and_ambiguity_are_exact() { + let (_directory, store) = new_store(); + write( + &store, + &[ + invocation_row( + &envelope("abc123", &shifted(-20)), + &InvocationContext::default(), + ), + invocation_row( + &envelope("abd456", &shifted(-10)), + &InvocationContext::default(), + ), + ], + ); + assert_eq!(resolve_invocation(&store, "abc").as_deref(), Ok("abc123")); + assert_eq!(resolve_invocation(&store, "last").as_deref(), Ok("abd456")); + assert_eq!( + resolve_invocation(&store, "ab").map_err(|error| error.kind), + Err(StoreErrorKind::Ambiguous) + ); + assert_eq!( + resolve_invocation(&store, "zz").map_err(|error| error.kind), + Err(StoreErrorKind::NotFound) + ); +} + +#[test] +fn given_queries_when_run_then_writes_are_rejected_and_truncation_reported() { + let (_directory, store) = new_store(); + let env = envelope("q", NOW); + for sequence in 0..3 { + write( + &store, + &[chunk(&env, (sequence, OutputStream::Stdout, b"line\n"))], + ); + } + let limited = query_store( + &store, + "SELECT sequence FROM recall_output ORDER BY sequence", + Some(2), + ) + .expect("query"); + assert_eq!(limited.rows.len(), 2); + assert!(limited.truncated); + assert_eq!(limited.columns, vec!["sequence".to_owned()]); + let rejected = + query_store(&store, "DELETE FROM recall_output", None).expect_err("write rejected"); + assert_eq!(rejected.kind, StoreErrorKind::Query); + let invalid = query_store(&store, "SELEC nope", None).expect_err("invalid sql"); + assert_eq!(invalid.kind, StoreErrorKind::Query); + let transcript = read_output_transcript(&store, "q", None).expect("transcript"); + assert_eq!(transcript.data, b"line\nline\nline\n".to_vec()); + assert!(!transcript.complete); +} + +#[test] +fn given_a_newer_schema_version_when_ensured_then_views_are_kept() { + let (directory, store) = new_store(); + write( + &store, + &[meta_row( + &envelope("newer", NOW), + ("schema", "schema_version", "99"), + NOW, + )], + ); + store + .connection + .execute_batch("DROP VIEW conflicts; CREATE VIEW conflicts AS SELECT 'newer' AS marker") + .expect("newer view"); + drop(store); + let path = directory.path().join("logs/recall.sqlite3"); + let reopened = open_sqlite_store(&path, OpenMode::ReadWrite).expect("reopen"); + ensure_sqlite_schema(&reopened, &envelope("older", NOW), NOW).expect("ensure"); + assert_eq!( + rows(&reopened, "SELECT marker FROM conflicts"), + vec![vec![RecallValue::Text("newer".to_owned())]] + ); + let relations = describe_store(&reopened).expect("describe"); + assert_eq!(relations[0].kind, "view"); + assert!(relations.iter().any(|relation| { + relation.name == "recall_output" + && relation + .columns + .iter() + .any(|column| column.name == "data" && !column.nullable) + })); + let read_only = open_sqlite_store(&path, OpenMode::ReadOnly).expect("read-only open"); + assert!(query_store(&read_only, "SELECT * FROM invocations", None).is_ok()); +} + +#[cfg(unix)] +#[test] +fn given_a_read_only_directory_when_opened_then_the_error_is_a_permission_failure() { + use std::os::unix::fs::PermissionsExt; + let directory = tempfile::tempdir().expect("temporary directory"); + let locked = directory.path().join("locked"); + std::fs::create_dir(&locked).expect("directory"); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o555)).expect("chmod"); + let result = open_sqlite_store(&locked.join("logs/recall.sqlite3"), OpenMode::ReadWrite); + let direct = open_sqlite_store(&locked.join("recall.sqlite3"), OpenMode::ReadWrite); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)).expect("restore"); + if nix_is_root() { + return; + } + assert_eq!( + result.map(|_| ()).map_err(|error| error.kind), + Err(StoreErrorKind::Permission) + ); + assert_eq!( + direct.map(|_| ()).map_err(|error| error.kind), + Err(StoreErrorKind::Permission) + ); +} + +#[cfg(unix)] +fn nix_is_root() -> bool { + std::env::var("USER").is_ok_and(|user| user == "root") +} diff --git a/crates/sqlbuild-recall/src/store/tests/test_views.rs b/crates/sqlbuild-recall/src/store/tests/test_views.rs new file mode 100644 index 0000000000..b0dd05a34b --- /dev/null +++ b/crates/sqlbuild-recall/src/store/tests/test_views.rs @@ -0,0 +1,392 @@ +use crate::records::main::artifact_rows::artifact_rows; +use crate::records::main::delivery_row::delivery_row; +use crate::records::main::heartbeat_row::heartbeat_row; +use crate::records::main::invocation_row::invocation_row; +use crate::records::models::{ + ArtifactInput, DeliveryCounts, InvocationContext, OutputStream, RecallValue, +}; +use crate::store::tests::helpers::{ + NOW, chunk, envelope, event, new_store, rows, text, wall_clock, write, +}; + +fn delivered(filtered: u64, dropped: u64, delivered: u64) -> DeliveryCounts { + DeliveryCounts { + accepted: delivered + filtered + dropped, + delivered, + filtered, + dropped, + ..DeliveryCounts::default() + } +} + +#[test] +fn given_output_with_and_without_gaps_when_projected_then_completeness_is_exact() { + let (_directory, store) = new_store(); + let complete = envelope("complete", NOW); + let gap = envelope("gap", NOW); + let filtered = envelope("filtered", NOW); + let undelivered = envelope("no_delivery", NOW); + for env in [&complete, &gap, &filtered, &undelivered] { + let sequences: &[u64] = + if env.invocation_id == "complete" || env.invocation_id == "no_delivery" { + &[0, 1, 2] + } else { + &[0, 2] + }; + for sequence in sequences { + write( + &store, + &[chunk(env, (*sequence, OutputStream::Stdout, b"x"))], + ); + } + } + write( + &store, + &[ + delivery_row(&complete, "output", (&delivered(0, 0, 3), true), NOW), + delivery_row(&gap, "output", (&delivered(0, 0, 2), true), NOW), + delivery_row(&filtered, "output", (&delivered(1, 0, 2), true), NOW), + ], + ); + let projected = rows( + &store, + "SELECT invocation_id, chunks, missing_sequences, filtered, gap_reason, complete \ + FROM output_invocations ORDER BY invocation_id", + ); + assert_eq!( + projected, + vec![ + vec![ + text("complete"), + RecallValue::Integer(3), + RecallValue::Integer(0), + RecallValue::Integer(0), + RecallValue::Null, + RecallValue::Integer(1) + ], + vec![ + text("filtered"), + RecallValue::Integer(2), + RecallValue::Integer(1), + RecallValue::Integer(1), + text("filtered"), + RecallValue::Integer(0) + ], + vec![ + text("gap"), + RecallValue::Integer(2), + RecallValue::Integer(1), + RecallValue::Integer(0), + text("unexplained"), + RecallValue::Integer(0) + ], + vec![ + text("no_delivery"), + RecallValue::Integer(3), + RecallValue::Integer(0), + RecallValue::Integer(0), + RecallValue::Null, + RecallValue::Integer(0) + ], + ] + ); +} + +#[test] +fn given_binary_interleaved_output_when_reconstructed_then_transcripts_are_byte_exact() { + let (_directory, store) = new_store(); + let env = envelope("binary", NOW); + let pieces: [(OutputStream, &[u8]); 4] = [ + (OutputStream::Stdout, b"line one\n\x00\xff"), + (OutputStream::Stderr, b"warn \xe2\x9c\x93\n"), + (OutputStream::Stdout, b"\xc3\x28 tail"), + (OutputStream::Stderr, b"\x00end"), + ]; + // Written out of order: the view orders by sequence, not by insertion. + for (sequence, (stream, data)) in pieces.iter().enumerate().rev() { + write(&store, &[chunk(&env, (sequence as u64, *stream, data))]); + } + let transcript = rows( + &store, + "SELECT stdout, stderr, combined FROM output_transcripts WHERE invocation_id = 'binary'", + ); + let stdout: Vec = [pieces[0].1, pieces[2].1].concat(); + let stderr: Vec = [pieces[1].1, pieces[3].1].concat(); + let combined: Vec = pieces.iter().flat_map(|(_, data)| data.to_vec()).collect(); + assert_eq!( + transcript, + vec![vec![ + RecallValue::Blob(stdout), + RecallValue::Blob(stderr), + RecallValue::Blob(combined) + ]] + ); + let tail = rows( + &store, + "SELECT COUNT(*) FROM output_tail WHERE invocation_id = 'binary'", + ); + assert_eq!(tail, vec![vec![RecallValue::Integer(4)]]); +} + +#[test] +fn given_duplicates_and_conflicts_when_written_then_first_row_is_kept_and_conflicts_reported() { + let (_directory, store) = new_store(); + let env = envelope("dup", NOW); + let original = chunk(&env, (0, OutputStream::Stdout, b"first")); + let conflicting = chunk(&env, (0, OutputStream::Stdout, b"second")); + write(&store, std::slice::from_ref(&original)); + let duplicate = crate::store::main::insert_rows::insert_rows(&store, &[original], NOW) + .expect("duplicate is a no-op"); + assert!(duplicate.conflicts.is_empty()); + let conflict = crate::store::main::insert_rows::insert_rows(&store, &[conflicting], NOW) + .expect("conflict does not fail"); + assert_eq!(conflict.conflicts.len(), 1); + assert_eq!(conflict.conflicts[0].record_key, "dup:0"); + assert_eq!( + rows(&store, "SELECT data FROM recall_output"), + vec![vec![RecallValue::Blob(b"first".to_vec())]] + ); + assert_eq!( + rows( + &store, + "SELECT invocation_id, dataset, record_key FROM conflicts" + ), + vec![vec![text("dup"), text("recall_output"), text("dup:0")]] + ); +} + +#[test] +fn given_terminals_and_heartbeats_when_projected_then_status_never_claims_false_success() { + let (_directory, store) = new_store(); + let done = envelope("done", NOW); + let failed = envelope("failed_late", NOW); + let running = envelope("running", NOW); + let abandoned = envelope("abandoned", "2026-10-01T12:00:00.000000Z"); + let unknown = envelope("unknown", NOW); + let mut batch = vec![ + event( + &done, + ("d0", "invocation_started", 0), + "", + r#"{"command":"build"}"#, + ), + event( + &done, + ("d1", "invocation_completed", 9), + "", + r#"{"command":"build","duration_ms":12.5,"exit_code":0}"#, + ), + event( + &failed, + ("f1", "invocation_completed", 3), + "", + r#"{"command":"build","exit_code":0}"#, + ), + event( + &failed, + ("f2", "invocation_failed", 4), + "", + r#"{"command":"build","exit_code":1}"#, + ), + event( + &unknown, + ("u0", "invocation_started", 0), + "", + r#"{"command":"run"}"#, + ), + invocation_row( + &running, + &InvocationContext { + command: Some("build".to_owned()), + target: Some("dev".to_owned()), + ..InvocationContext::default() + }, + ), + heartbeat_row( + &running, + (&wall_clock(-1), &wall_clock(90)), + (None, Some(1)), + ), + ]; + batch.push(heartbeat_row( + &abandoned, + ("2026-10-01T12:00:30.000000Z", "2026-10-01T12:02:00.000000Z"), + (None, None), + )); + batch.push(heartbeat_row( + &abandoned, + ("2026-10-01T12:00:00.000000Z", "2026-10-01T12:01:30.000000Z"), + (None, None), + )); + write(&store, &batch); + let projected = rows( + &store, + "SELECT invocation_id, status, ended_at, duration_ms, exit_code, command, target \ + FROM invocations ORDER BY invocation_id", + ); + assert_eq!( + projected, + vec![ + vec![ + text("abandoned"), + text("abandoned"), + text("2026-10-01T12:00:30.000000Z"), + RecallValue::Real(30000.0), + RecallValue::Null, + RecallValue::Null, + RecallValue::Null + ], + vec![ + text("done"), + text("completed"), + text(NOW), + RecallValue::Real(12.5), + RecallValue::Integer(0), + text("build"), + RecallValue::Null + ], + vec![ + text("failed_late"), + text("failed"), + text(NOW), + RecallValue::Null, + RecallValue::Integer(1), + RecallValue::Null, + RecallValue::Null + ], + vec![ + text("running"), + text("running"), + RecallValue::Null, + RecallValue::Null, + RecallValue::Null, + text("build"), + text("dev") + ], + vec![ + text("unknown"), + text("unknown"), + RecallValue::Null, + RecallValue::Null, + RecallValue::Null, + text("run"), + RecallValue::Null + ], + ] + ); +} + +#[test] +fn given_statement_facts_when_projected_then_failures_and_missing_terminals_stay_visible() { + let (_directory, store) = new_store(); + let env = envelope("stmts", NOW); + let sql = "select 1"; + let digest = crate::records::main::sha256_hex::sha256_hex(sql.as_bytes()); + let payload = format!( + r#"{{"intent":"execute","statement_kind":"SELECT","purpose":"state_write","sql_digest":"{digest}"}}"# + ); + let failed_payload = format!( + r#"{{"intent":"execute","statement_kind":"SELECT","sql_digest":"{digest}","error_type":"DatabaseError","error_code":"42P01","query_id":"q9"}}"# + ); + let mut batch = vec![ + event( + &env, + ("r0", "resource_attempt_started", 0), + r#","run_id":"run","resource_id":"orders","resource_attempt_id":"a1""#, + r#"{"attempt_number":1,"resource_kind":"model","resource_name":"orders"}"#, + ), + event( + &env, + ("s0", "statement_started", 1), + r#","statement_id":"s1","resource_id":"orders""#, + &payload, + ), + event( + &env, + ("s1", "statement_failed", 2), + r#","statement_id":"s1","resource_id":"orders""#, + &failed_payload, + ), + event( + &env, + ("s2", "statement_started", 3), + r#","statement_id":"s2""#, + r#"{"intent":"query","statement_kind":"SELECT"}"#, + ), + ]; + batch.extend(artifact_rows( + &env, + &ArtifactInput { + digest: format!("sha256:{digest}"), + kind: "sql".to_owned(), + media_type: "text/x-sql".to_owned(), + content: sql.as_bytes().to_vec(), + seen_at: NOW.to_owned(), + ref_id: "s1".to_owned(), + role: "executed_sql".to_owned(), + statement_id: Some("s1".to_owned()), + resource_id: Some("orders".to_owned()), + }, + )); + write(&store, &batch); + let projected = rows( + &store, + "SELECT statement_id, resource_name, purpose, outcome, error_code, query_id, sql_available \ + FROM statements ORDER BY statement_id", + ); + assert_eq!( + projected, + vec![ + vec![ + text("s1"), + text("orders"), + text("state_write"), + text("failed"), + text("42P01"), + text("q9"), + RecallValue::Integer(1) + ], + vec![ + text("s2"), + RecallValue::Null, + RecallValue::Null, + text("no_terminal"), + RecallValue::Null, + RecallValue::Null, + RecallValue::Integer(0) + ], + ] + ); + assert_eq!( + rows(&store, "SELECT available, text FROM artifacts"), + vec![vec![RecallValue::Integer(1), text(sql)]] + ); +} + +#[test] +fn given_audit_completed_facts_when_projected_then_audit_results_carry_outcome_and_target() { + let (_directory, store) = new_store(); + let env = envelope("audits", NOW); + write( + &store, + &[event( + &env, + ("a0", "audit_completed", 1), + r#","run_id":"run""#, + r#"{"audit_name":"revenue_positive","audit_definition_name":"positive","evaluation_mode":"violations","outcome":"warn","severity":"warn","run_scope_phase":"model","attachment_kind":"column","target_schema":"analytics","target_name":"daily_revenue","attached_column_name":"total_revenue_cents","violation_count":3,"sql_digest":"ab"}"#, + )], + ); + assert_eq!( + rows( + &store, + "SELECT audit_name, target_relation, \"column\", outcome, violation_count FROM audit_results" + ), + vec![vec![ + text("revenue_positive"), + text("analytics.daily_revenue"), + text("total_revenue_cents"), + text("warn"), + RecallValue::Integer(3) + ]] + ); +} diff --git a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs new file mode 100644 index 0000000000..2f8d00872d --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs @@ -0,0 +1,253 @@ +//! The delivery thread: table setup, batched writes with retries, heartbeats, the final delivery rows and maintenance. It alone opens and uses the store connection. + +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::time::{Duration, Instant, SystemTime}; + +use crate::records::main::delivery_row::delivery_row; +use crate::records::main::format_timestamp::format_timestamp; +use crate::records::main::heartbeat_row::heartbeat_row; +use crate::records::models::RecallRow; +use crate::schema::models::Table; +use crate::store::constants::MAINTENANCE_BUDGET_MS; +use crate::store::errors::{StoreError, StoreErrorKind}; +use crate::store::main::ensure_sqlite_schema::ensure_sqlite_schema; +use crate::store::main::insert_rows::insert_rows; +use crate::store::main::maintain_sqlite_store::maintain_sqlite_store; +use crate::store::main::open_sqlite_store::open_sqlite_store; +use crate::store::models::{InsertOutcome, MaintenanceRequest, OpenMode, SqliteStore}; +use crate::writer::_helpers::queue::{Queued, WriterState}; +use crate::writer::constants::{ + BATCH_ROWS, BATCH_WINDOW_MS, HEARTBEAT_STALE_INTERVALS, MAX_RETRY_BACKOFF_MS, + OUTPUT_COALESCE_MS, RUNNING_RETRY_LIMIT_MS, +}; +use crate::writer::models::{DeliveryStream, SessionConfig, SetupOutcome}; + +/// State shared by producers, the session handle and the delivery thread. +#[derive(Debug)] +pub(crate) struct Shared { + pub(crate) state: Mutex, + pub(crate) wake: Condvar, + pub(crate) setup: Mutex>, + pub(crate) setup_done: Condvar, +} + +pub(crate) fn locked(mutex: &Mutex) -> MutexGuard<'_, WriterState> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn publish_setup(shared: &Shared, outcome: SetupOutcome) { + let mut setup = shared + .setup + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *setup = Some(outcome); + shared.setup_done.notify_all(); +} + +fn open_and_setup(config: &SessionConfig) -> Result { + let started = Instant::now(); + loop { + let attempt = + open_sqlite_store(&config.store_path, OpenMode::ReadWrite).and_then(|store| { + let now = format_timestamp(SystemTime::now()); + ensure_sqlite_schema(&store, &config.envelope, &now).map(|()| store) + }); + match attempt { + Err(error) + if error.kind == StoreErrorKind::Busy + && started.elapsed() < Duration::from_millis(RUNNING_RETRY_LIMIT_MS) => + { + std::thread::sleep(Duration::from_millis(50)); + } + other => return other, + } + } +} + +/// Write one batch, retrying a busy store with backoff until `give_up`. +fn write_batch( + store: &SqliteStore, + rows: &[RecallRow], + give_up: Instant, +) -> Result { + let mut backoff = Duration::from_millis(10); + loop { + let now = format_timestamp(SystemTime::now()); + match insert_rows(store, rows, &now) { + Err(error) if error.kind == StoreErrorKind::Busy && Instant::now() < give_up => { + std::thread::sleep(backoff.min(give_up.saturating_duration_since(Instant::now()))); + backoff = (backoff * 2).min(Duration::from_millis(MAX_RETRY_BACKOFF_MS)); + } + other => return other, + } + } +} + +fn heartbeat(config: &SessionConfig, now: SystemTime) -> RecallRow { + let stale = now + config.heartbeat_interval * HEARTBEAT_STALE_INTERVALS; + heartbeat_row( + &config.envelope, + (&format_timestamp(now), &format_timestamp(stale)), + (config.context.host_token.as_deref(), config.context.pid), + ) +} + +pub(crate) fn stream_of(table: Table) -> DeliveryStream { + match table { + Table::Output => DeliveryStream::Output, + Table::Events => DeliveryStream::Events, + Table::Diagnostics => DeliveryStream::Diagnostics, + Table::Artifacts | Table::ArtifactRefs => DeliveryStream::Artifacts, + _ => DeliveryStream::Invocation, + } +} + +/// Wait until a batch is due: 500 rows, the batch window, closing, or a timed duty. +fn next_batch(shared: &Shared, next_heartbeat: Instant) -> (Vec, bool) { + let mut state = locked(&shared.state); + let mut first_seen: Option = None; + loop { + let now = Instant::now(); + if state + .output_due(Duration::from_millis(OUTPUT_COALESCE_MS)) + .is_some_and(|due| due <= now) + || (state.closing && state.has_pending_output()) + { + state.flush_output(); + } + if state.queued() > 0 && first_seen.is_none() { + first_seen = Some(now); + } + let window_over = first_seen + .is_some_and(|seen| now.duration_since(seen) >= Duration::from_millis(BATCH_WINDOW_MS)); + if state.queued() >= BATCH_ROWS || (state.queued() > 0 && (window_over || state.closing)) { + return (state.take_batch(BATCH_ROWS), state.closing); + } + if state.closing && state.queued() == 0 && !state.has_pending_output() { + return (Vec::new(), true); + } + if now >= next_heartbeat { + return (Vec::new(), state.closing); + } + let mut wait_until = next_heartbeat; + if let Some(due) = state.output_due(Duration::from_millis(OUTPUT_COALESCE_MS)) { + wait_until = wait_until.min(due); + } + if let Some(seen) = first_seen { + wait_until = wait_until.min(seen + Duration::from_millis(BATCH_WINDOW_MS)); + } + let timeout = wait_until + .saturating_duration_since(now) + .max(Duration::from_millis(1)); + state = shared + .wake + .wait_timeout(state, timeout) + .unwrap_or_else(std::sync::PoisonError::into_inner) + .0; + } +} + +fn degrade_and_drain(shared: &Shared, reason: String) { + let mut state = locked(&shared.state); + state.degraded.get_or_insert(reason); + state.rejecting = true; + state.drop_all(); + state.finished = true; + shared.wake.notify_all(); +} + +fn final_rows(config: &SessionConfig, state: &WriterState) -> Vec { + let now = format_timestamp(SystemTime::now()); + let complete = state.degraded.is_none(); + DeliveryStream::ALL + .iter() + .map(|stream| { + let counts = state.counts[stream.index()]; + let flushed = complete && counts.dropped == 0 && counts.failed == 0; + delivery_row(&config.envelope, stream.as_str(), (&counts, flushed), &now) + }) + .collect() +} + +/// The delivery thread body. +pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { + let store = match open_and_setup(config) { + Ok(store) => { + publish_setup(shared, SetupOutcome::Ready); + store + } + Err(error) => { + let outcome = if error.kind == StoreErrorKind::Permission { + SetupOutcome::PermissionDenied { + reason: error.message.clone(), + } + } else { + SetupOutcome::Failed { + reason: error.message.clone(), + } + }; + publish_setup(shared, outcome); + degrade_and_drain(shared, error.message); + return; + } + }; + let mut next_heartbeat = Instant::now(); + loop { + let (batch, closing) = next_batch(shared, next_heartbeat); + let mut rows: Vec = batch + .iter() + .flat_map(|record| record.rows.clone()) + .collect(); + if Instant::now() >= next_heartbeat && !closing { + rows.push(heartbeat(config, SystemTime::now())); + next_heartbeat = Instant::now() + config.heartbeat_interval; + } + if !rows.is_empty() { + let give_up = { + let state = locked(&shared.state); + state.deadline.unwrap_or_else(|| { + Instant::now() + Duration::from_millis(RUNNING_RETRY_LIMIT_MS) + }) + }; + let written = write_batch(&store, &rows, give_up); + let mut state = locked(&shared.state); + match &written { + Ok(outcome) => state.account(&batch, Ok(outcome)), + Err(error) => state.account(&batch, Err(&error.message)), + } + } + let mut state = locked(&shared.state); + let expired = state + .deadline + .is_some_and(|deadline| Instant::now() >= deadline); + if expired { + state.flush_output(); + state.drop_all(); + } + if (closing || expired) && state.queued() == 0 && !state.has_pending_output() { + state.rejecting = true; + let rows = final_rows(config, &state); + drop(state); + let give_up = Instant::now() + Duration::from_millis(500); + if let Err(error) = write_batch(&store, &rows, give_up) { + locked(&shared.state).degraded.get_or_insert(error.message); + } + break; + } + } + if config.maintenance { + let request = MaintenanceRequest { + now: SystemTime::now(), + retention_days: config.retention_days, + owner: config.maintenance_owner.clone(), + budget: Duration::from_millis(MAINTENANCE_BUDGET_MS), + envelope: config.envelope.clone(), + }; + drop(maintain_sqlite_store(&store, &request)); + } + let mut state = locked(&shared.state); + state.finished = true; + shared.wake.notify_all(); +} diff --git a/crates/sqlbuild-recall/src/writer/_helpers/mod.rs b/crates/sqlbuild-recall/src/writer/_helpers/mod.rs new file mode 100644 index 0000000000..66aa5e748e --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod delivery; +pub(crate) mod queue; +pub(crate) mod session; diff --git a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs new file mode 100644 index 0000000000..3d9ddf0df9 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs @@ -0,0 +1,275 @@ +//! The bounded priority queue and the output coalescer, guarded together by one mutex. + +use std::collections::{HashSet, VecDeque}; +use std::time::{Duration, Instant, SystemTime}; + +use crate::records::constants::MAX_OUTPUT_CHUNK_BYTES; +use crate::records::main::format_timestamp::format_timestamp; +use crate::records::main::output_row::output_row; +use crate::records::models::{ + DeliveryCounts, OutputChunk, OutputStream, RecallRow, RecallValue, RowEnvelope, +}; +use crate::store::models::InsertOutcome; +use crate::writer::_helpers::delivery::stream_of; +use crate::writer::constants::{QUEUE_BYTE_CAPACITY, QUEUE_RECORD_CAPACITY}; +use crate::writer::models::{DeliveryStream, Priority}; + +/// One queued record: the rows written together and where they are accounted. +#[derive(Debug)] +pub(crate) struct Queued { + pub(crate) rows: Vec, + pub(crate) stream: DeliveryStream, + pub(crate) bytes: usize, +} + +/// Output accepted but not yet chunked: contiguous writes to one stream. +#[derive(Debug)] +struct Pending { + stream: OutputStream, + data: Vec, + since: Instant, + occurred_at: String, +} + +/// Everything producers and the delivery thread share. +#[derive(Debug)] +pub(crate) struct WriterState { + pub(crate) envelope: RowEnvelope, + buckets: [VecDeque; Priority::COUNT], + records: usize, + bytes: usize, + pub(crate) counts: [DeliveryCounts; 5], + pending: Option, + next_sequence: u64, + offsets: [u64; 2], + unreported_output_drops: u64, + pub(crate) diagnostic_sequence: u64, + pub(crate) artifacts_seen: HashSet, + pub(crate) closing: bool, + pub(crate) deadline: Option, + pub(crate) finished: bool, + pub(crate) rejecting: bool, + pub(crate) degraded: Option, + capacity: (usize, usize), +} + +fn row_bytes(rows: &[RecallRow]) -> usize { + rows.iter() + .flat_map(|row| row.values.iter()) + .map(|(name, value)| { + name.len() + + match value { + RecallValue::Text(text) => text.len(), + RecallValue::Blob(data) => data.len(), + _ => 8, + } + }) + .sum() +} + +impl WriterState { + pub(crate) fn new(envelope: RowEnvelope) -> Self { + Self::with_capacity(envelope, (QUEUE_RECORD_CAPACITY, QUEUE_BYTE_CAPACITY)) + } + + pub(crate) fn with_capacity(envelope: RowEnvelope, capacity: (usize, usize)) -> Self { + Self { + envelope, + buckets: Default::default(), + records: 0, + bytes: 0, + counts: [DeliveryCounts::default(); 5], + pending: None, + next_sequence: 0, + offsets: [0; 2], + unreported_output_drops: 0, + diagnostic_sequence: 0, + artifacts_seen: HashSet::new(), + closing: false, + deadline: None, + finished: false, + rejecting: false, + degraded: None, + capacity, + } + } + + pub(crate) fn counts_mut(&mut self, stream: DeliveryStream) -> &mut DeliveryCounts { + &mut self.counts[stream.index()] + } + + fn drop_record(&mut self, record: &Queued) { + self.counts_mut(record.stream).dropped += 1; + if record.stream == DeliveryStream::Output { + self.unreported_output_drops += 1; + } + } + + fn full(&self, bytes: usize) -> bool { + self.records >= self.capacity.0 || self.bytes + bytes > self.capacity.1 + } + + /// Queue `rows`, displacing lower-priority records when full; counts acceptance and drops. + pub(crate) fn push( + &mut self, + rows: Vec, + stream: DeliveryStream, + priority: Priority, + ) { + self.counts_mut(stream).accepted += 1; + let record = Queued { + bytes: row_bytes(&rows), + rows, + stream, + }; + if self.rejecting { + self.drop_record(&record); + return; + } + while self.full(record.bytes) { + let Some(lowest) = self + .buckets + .iter() + .position(|bucket| !bucket.is_empty()) + .filter(|lowest| *lowest < priority as usize) + else { + self.drop_record(&record); + return; + }; + if let Some(displaced) = self.buckets[lowest].pop_front() { + self.records -= 1; + self.bytes -= displaced.bytes; + self.drop_record(&displaced); + } + } + self.records += 1; + self.bytes += record.bytes; + self.buckets[priority as usize].push_back(record); + } + + pub(crate) fn queued(&self) -> usize { + self.records + } + + /// Up to `limit` rows, highest priority first. + pub(crate) fn take_batch(&mut self, limit: usize) -> Vec { + let mut batch: Vec = Vec::new(); + let mut rows = 0; + for bucket in self.buckets.iter_mut().rev() { + while rows < limit { + let Some(record) = bucket.pop_front() else { + break; + }; + rows += record.rows.len(); + self.records -= 1; + self.bytes -= record.bytes; + batch.push(record); + } + } + batch + } + + /// Drop everything queued, as when the store is unusable or the flush budget is spent. + pub(crate) fn drop_all(&mut self) { + let records: Vec = self.buckets.iter_mut().flat_map(std::mem::take).collect(); + for record in &records { + self.drop_record(record); + } + self.records = 0; + self.bytes = 0; + } + + fn emit(&mut self, stream: OutputStream, data: Vec, occurred_at: &str) { + let index = usize::from(stream == OutputStream::Stderr); + let chunk = OutputChunk { + sequence: self.next_sequence, + stream, + occurred_at: occurred_at.to_owned(), + byte_offset: self.offsets[index], + dropped_before: std::mem::take(&mut self.unreported_output_drops), + data, + }; + self.next_sequence += 1; + self.offsets[index] += chunk.data.len() as u64; + let row = output_row(&self.envelope, &chunk); + self.push(vec![row], DeliveryStream::Output, Priority::Output); + } + + /// Accept `data` written to `stream`: a stream switch flushes the other stream first, so the shared sequence preserves interleaving; full chunks are emitted at once. + pub(crate) fn append_output(&mut self, stream: OutputStream, data: &[u8], now: SystemTime) { + if data.is_empty() { + return; + } + if self + .pending + .as_ref() + .is_some_and(|pending| pending.stream != stream) + { + self.flush_output(); + } + let pending = self.pending.get_or_insert_with(|| Pending { + stream, + data: Vec::new(), + since: Instant::now(), + occurred_at: format_timestamp(now), + }); + pending.data.extend_from_slice(data); + while self + .pending + .as_ref() + .is_some_and(|pending| pending.data.len() >= MAX_OUTPUT_CHUNK_BYTES) + { + let Some(mut pending) = self.pending.take() else { + break; + }; + let rest: Vec = pending.data.split_off(MAX_OUTPUT_CHUNK_BYTES); + let occurred_at = pending.occurred_at.clone(); + self.emit(pending.stream, pending.data, &occurred_at); + if !rest.is_empty() { + pending.data = rest; + pending.since = Instant::now(); + self.pending = Some(pending); + } + } + } + + /// Emit pending output as one chunk. + pub(crate) fn flush_output(&mut self) { + if let Some(pending) = self.pending.take() { + self.emit(pending.stream, pending.data, &pending.occurred_at); + } + } + + /// When pending output must be flushed: `None` with nothing pending. + pub(crate) fn output_due(&self, window: Duration) -> Option { + self.pending.as_ref().map(|pending| pending.since + window) + } + + /// Whether any output is pending. + pub(crate) fn has_pending_output(&self) -> bool { + self.pending.is_some() + } + + /// Count a written batch as delivered, less conflicts, or a failed one as failed. + pub(crate) fn account(&mut self, batch: &[Queued], outcome: Result<&InsertOutcome, &str>) { + match outcome { + Ok(outcome) => { + for record in batch { + self.counts_mut(record.stream).delivered += 1; + } + for conflict in &outcome.conflicts { + let stream = stream_of(conflict.table); + let counts = self.counts_mut(stream); + counts.conflicts += 1; + counts.delivered = counts.delivered.saturating_sub(1); + } + } + Err(reason) => { + for record in batch { + self.counts_mut(record.stream).failed += 1; + } + self.degraded.get_or_insert_with(|| reason.to_owned()); + } + } + } +} diff --git a/crates/sqlbuild-recall/src/writer/_helpers/session.rs b/crates/sqlbuild-recall/src/writer/_helpers/session.rs new file mode 100644 index 0000000000..830f0eacae --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/_helpers/session.rs @@ -0,0 +1,193 @@ +//! The session handle: producers queue records; `close` flushes within the shutdown budget. + +use std::time::{Duration, Instant, SystemTime}; + +use crate::records::constants::{ + ARTIFACT_DIGEST_PREFIX, EXECUTED_SQL_ROLE, SEVERITY_ORDER, SQL_ARTIFACT_KIND, SQL_MEDIA_TYPE, +}; +use crate::records::errors::RecordError; +use crate::records::main::artifact_rows::artifact_rows; +use crate::records::main::diagnostic_input_from_json::diagnostic_input_from_json; +use crate::records::main::diagnostic_row::diagnostic_row; +use crate::records::main::event_row::event_row; +use crate::records::main::format_timestamp::format_timestamp; +use crate::records::main::sha256_hex::sha256_hex; +use crate::records::models::{ArtifactInput, OutputStream, RecallValue}; +use crate::store::constants::MAINTENANCE_BUDGET_MS; +use crate::writer::_helpers::delivery::locked; +use crate::writer::models::{ + DeliveryStream, DeliverySummary, Priority, RecallSession, SetupOutcome, +}; + +pub(crate) fn severity_rank(severity: &str) -> usize { + SEVERITY_ORDER + .iter() + .position(|candidate| *candidate == severity) + .unwrap_or(0) +} + +fn event_priority(severity: &str) -> Priority { + match severity { + "error" => Priority::FailedTerminal, + "info" => Priority::Terminal, + _ => Priority::Lifecycle, + } +} + +impl RecallSession { + /// Wait up to `budget` for table setup; `None` while it is still running. + pub fn wait_for_setup(&self, budget: Duration) -> Option { + let deadline = Instant::now() + budget; + let mut setup = self + .shared + .setup + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + while setup.is_none() { + let now = Instant::now(); + if now >= deadline { + return None; + } + setup = self + .shared + .setup_done + .wait_timeout(setup, deadline - now) + .unwrap_or_else(std::sync::PoisonError::into_inner) + .0; + } + setup.clone() + } + + /// The invocation this session records. + #[must_use] + pub fn invocation_id(&self) -> String { + locked(&self.shared.state).envelope.invocation_id.clone() + } + + /// Accept bytes a process stream accepted. + pub fn append_output(&self, stream: OutputStream, data: &[u8]) { + let mut state = locked(&self.shared.state); + state.append_output(stream, data, SystemTime::now()); + self.shared.wake.notify_one(); + } + + /// Queue one lifecycle event from its canonical JSON. + pub fn publish_event(&self, event_json: &str) -> Result<(), RecordError> { + let mut state = locked(&self.shared.state); + let row = event_row(&state.envelope, event_json)?; + let priority = match row.value("severity") { + Some(RecallValue::Text(severity)) => event_priority(severity), + _ => Priority::Lifecycle, + }; + state.push(vec![row], DeliveryStream::Events, priority); + self.shared.wake.notify_one(); + Ok(()) + } + + /// Queue one diagnostic from its canonical JSON, when at or above the minimum severity. + pub fn publish_diagnostic(&self, diagnostic_json: &str) -> Result<(), RecordError> { + let mut state = locked(&self.shared.state); + let sequence = state.diagnostic_sequence; + let diagnostic = diagnostic_input_from_json(diagnostic_json, sequence)?; + let rank = severity_rank(&diagnostic.severity); + if rank < self.min_severity { + return Ok(()); + } + state.diagnostic_sequence += 1; + let priority = if rank >= severity_rank("warning") { + Priority::Diagnostic + } else { + Priority::DebugDiagnostic + }; + let row = diagnostic_row(&state.envelope, &diagnostic); + state.push(vec![row], DeliveryStream::Diagnostics, priority); + self.shared.wake.notify_one(); + Ok(()) + } + + /// Queue the SQL a statement executed, once per digest per invocation. + pub fn publish_sql(&self, sql: &str, statement_id: Option<&str>, resource_id: Option<&str>) { + let digest = format!("{ARTIFACT_DIGEST_PREFIX}{}", sha256_hex(sql.as_bytes())); + let mut state = locked(&self.shared.state); + if !state.artifacts_seen.insert(digest.clone()) { + return; + } + let artifact = ArtifactInput { + ref_id: statement_id.map_or_else(|| digest.clone(), str::to_owned), + digest, + kind: SQL_ARTIFACT_KIND.to_owned(), + media_type: SQL_MEDIA_TYPE.to_owned(), + content: sql.as_bytes().to_vec(), + seen_at: format_timestamp(SystemTime::now()), + role: EXECUTED_SQL_ROLE.to_owned(), + statement_id: statement_id.map(str::to_owned), + resource_id: resource_id.map(str::to_owned), + }; + let rows = artifact_rows(&state.envelope, &artifact).to_vec(); + state.push(rows, DeliveryStream::Artifacts, Priority::Artifact); + self.shared.wake.notify_one(); + } + + /// Count records a replacement sink filtered out of `stream`. + pub fn record_filtered(&self, stream: DeliveryStream, count: u64) { + let mut state = locked(&self.shared.state); + let counts = state.counts_mut(stream); + counts.accepted += count; + counts.filtered += count; + } + + /// Count records a replacement sink failed on in `stream`. + pub fn record_failed(&self, stream: DeliveryStream, count: u64) { + let mut state = locked(&self.shared.state); + let counts = state.counts_mut(stream); + counts.accepted += count; + counts.failed += count; + } + + /// The accounting so far. + #[must_use] + pub fn summary(&self) -> DeliverySummary { + let state = locked(&self.shared.state); + DeliverySummary { + streams: DeliveryStream::ALL + .iter() + .map(|stream| (*stream, state.counts[stream.index()])) + .collect(), + flush_complete: state.finished && state.queued() == 0 && state.degraded.is_none(), + degraded_reason: state.degraded.clone(), + } + } + + /// Flush within the shutdown budget, write the delivery rows and return the accounting. Records still queued at the deadline are counted as dropped. + pub fn close(&self) -> DeliverySummary { + let deadline = Instant::now() + self.shutdown_timeout; + let wait_until = deadline + Duration::from_millis(MAINTENANCE_BUDGET_MS + 500); + { + let mut state = locked(&self.shared.state); + if !state.closing { + state.closing = true; + state.deadline = Some(deadline); + state.flush_output(); + } + self.shared.wake.notify_all(); + while !state.finished { + let now = Instant::now(); + if now >= wait_until { + state.rejecting = true; + state.drop_all(); + state.degraded.get_or_insert_with(|| { + "Recall delivery did not finish within the shutdown timeout".to_owned() + }); + break; + } + state = self + .shared + .wake + .wait_timeout(state, wait_until - now) + .unwrap_or_else(std::sync::PoisonError::into_inner) + .0; + } + } + self.summary() + } +} diff --git a/crates/sqlbuild-recall/src/writer/constants.rs b/crates/sqlbuild-recall/src/writer/constants.rs new file mode 100644 index 0000000000..dd1ba9536b --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/constants.rs @@ -0,0 +1,17 @@ +//! Queue bounds, batching and timing of the Recall writer. + +pub const QUEUE_RECORD_CAPACITY: usize = 10_000; +pub const QUEUE_BYTE_CAPACITY: usize = 32 * 1024 * 1024; +pub const BATCH_ROWS: usize = 500; +pub const BATCH_WINDOW_MS: u64 = 250; +/// Contiguous writes to one stream coalesce into one chunk for at most this long. +pub const OUTPUT_COALESCE_MS: u64 = 250; +pub const DEFAULT_HEARTBEAT_INTERVAL_MS: u64 = 30_000; +/// An invocation is abandoned once its last heartbeat is this many intervals old. +pub const HEARTBEAT_STALE_INTERVALS: u32 = 3; +/// Table setup may delay the command by at most this long. +pub const SETUP_BUDGET_MS: u64 = 2_000; +/// Longest single backoff between retries of a busy write. +pub const MAX_RETRY_BACKOFF_MS: u64 = 500; +/// A batch that stays busy this long while the command runs is counted as failed. +pub const RUNNING_RETRY_LIMIT_MS: u64 = 30_000; diff --git a/crates/sqlbuild-recall/src/writer/main/mod.rs b/crates/sqlbuild-recall/src/writer/main/mod.rs new file mode 100644 index 0000000000..777f9535f8 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/main/mod.rs @@ -0,0 +1 @@ +pub mod start_recall_session; diff --git a/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs b/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs new file mode 100644 index 0000000000..746aa46418 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs @@ -0,0 +1,42 @@ +//! `start_recall_session`: begin recording one invocation. + +use std::sync::{Arc, Condvar, Mutex}; + +use crate::records::main::invocation_row::invocation_row; +use crate::writer::_helpers::delivery::{Shared, deliver, locked}; +use crate::writer::_helpers::queue::WriterState; +use crate::writer::_helpers::session::severity_rank; +use crate::writer::models::{DeliveryStream, Priority, RecallSession, SessionConfig}; + +/// Start the delivery thread and queue the invocation's context row first. +#[must_use] +pub fn start_recall_session(config: SessionConfig) -> RecallSession { + let mut state = WriterState::new(config.envelope.clone()); + state.push( + vec![invocation_row(&config.envelope, &config.context)], + DeliveryStream::Invocation, + Priority::Context, + ); + let shared = Arc::new(Shared { + state: Mutex::new(state), + wake: Condvar::new(), + setup: Mutex::new(None), + setup_done: Condvar::new(), + }); + let session = RecallSession { + shared: Arc::clone(&shared), + shutdown_timeout: config.shutdown_timeout, + min_severity: severity_rank(&config.diagnostics_min_severity), + }; + let spawned = std::thread::Builder::new() + .name("sqlbuild-recall".to_owned()) + .spawn(move || deliver(&shared, &config)); + if let Err(error) = spawned { + let mut state = locked(&session.shared.state); + state.degraded = Some(format!("cannot start the Recall delivery thread: {error}")); + state.rejecting = true; + state.drop_all(); + state.finished = true; + } + session +} diff --git a/crates/sqlbuild-recall/src/writer/mod.rs b/crates/sqlbuild-recall/src/writer/mod.rs new file mode 100644 index 0000000000..9a334c477b --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/mod.rs @@ -0,0 +1,9 @@ +//! The Recall writer: a bounded priority queue fed by producers and drained by one delivery thread that alone owns the store connection. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-recall/src/writer/models.rs b/crates/sqlbuild-recall/src/writer/models.rs new file mode 100644 index 0000000000..4100bb15bb --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/models.rs @@ -0,0 +1,130 @@ +//! Writer configuration, delivery streams, priorities and the delivery summary. + +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use crate::records::models::{DeliveryCounts, InvocationContext, RowEnvelope}; +use crate::writer::_helpers::delivery::Shared; + +/// The datasets whose delivery is accounted separately. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum DeliveryStream { + Invocation, + Output, + Events, + Diagnostics, + Artifacts, +} + +impl DeliveryStream { + pub const ALL: [Self; 5] = [ + Self::Invocation, + Self::Output, + Self::Events, + Self::Diagnostics, + Self::Artifacts, + ]; + + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::Invocation => "invocation", + Self::Output => "output", + Self::Events => "events", + Self::Diagnostics => "diagnostics", + Self::Artifacts => "artifacts", + } + } + + #[must_use] + pub fn index(self) -> usize { + match self { + Self::Invocation => 0, + Self::Output => 1, + Self::Events => 2, + Self::Diagnostics => 3, + Self::Artifacts => 4, + } + } +} + +/// Queue priority, lowest first; a full queue displaces the oldest record of the lowest priority below the arriving one. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Priority { + DebugDiagnostic = 0, + Artifact = 1, + Output = 2, + Diagnostic = 3, + Lifecycle = 4, + Terminal = 5, + FailedTerminal = 6, + Context = 7, +} + +impl Priority { + pub const COUNT: usize = 8; +} + +/// What one writer session needs. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SessionConfig { + pub store_path: PathBuf, + pub envelope: RowEnvelope, + pub context: InvocationContext, + pub retention_days: u32, + pub shutdown_timeout: Duration, + pub diagnostics_min_severity: String, + pub heartbeat_interval: Duration, + pub setup_budget: Duration, + /// Whether this session runs retention maintenance after delivery. + pub maintenance: bool, + pub maintenance_owner: String, +} + +/// How table setup went. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SetupOutcome { + Ready, + /// The store cannot be created or written for lack of permission. + PermissionDenied { + reason: String, + }, + Failed { + reason: String, + }, +} + +/// The delivery accounting of a closed session. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DeliverySummary { + pub streams: Vec<(DeliveryStream, DeliveryCounts)>, + pub flush_complete: bool, + pub degraded_reason: Option, +} + +impl DeliverySummary { + /// Every stream's counts added together. + #[must_use] + pub fn total(&self) -> DeliveryCounts { + self.streams + .iter() + .fold(DeliveryCounts::default(), |mut total, (_, counts)| { + total.accepted += counts.accepted; + total.delivered += counts.delivered; + total.filtered += counts.filtered; + total.dropped += counts.dropped; + total.failed += counts.failed; + total.conflicts += counts.conflicts; + total + }) + } +} + +/// A running Recall session. Producers only touch the queue; the delivery thread writes. +#[derive(Debug)] +pub struct RecallSession { + pub(crate) shared: Arc, + pub(crate) shutdown_timeout: Duration, + pub(crate) min_severity: usize, +} diff --git a/crates/sqlbuild-recall/src/writer/tests.rs b/crates/sqlbuild-recall/src/writer/tests.rs new file mode 100644 index 0000000000..d063ec20fb --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_queue.rs"] +mod queue; +#[path = "tests/test_session.rs"] +mod session; diff --git a/crates/sqlbuild-recall/src/writer/tests/test_queue.rs b/crates/sqlbuild-recall/src/writer/tests/test_queue.rs new file mode 100644 index 0000000000..b643a4d365 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/tests/test_queue.rs @@ -0,0 +1,88 @@ +use std::time::SystemTime; + +use crate::records::main::invocation_row::invocation_row; +use crate::records::models::{InvocationContext, OutputStream, RowEnvelope}; +use crate::writer::_helpers::queue::WriterState; +use crate::writer::models::{DeliveryStream, Priority}; + +fn envelope() -> RowEnvelope { + RowEnvelope { + invocation_id: "inv".to_owned(), + invocation_started_at: "2026-10-19T00:00:00.000000Z".to_owned(), + producer_version: "test".to_owned(), + } +} + +#[test] +fn given_a_full_queue_when_higher_priority_arrives_then_the_oldest_lowest_is_displaced() { + let mut state = WriterState::with_capacity(envelope(), (2, usize::MAX)); + let row = || vec![invocation_row(&envelope(), &InvocationContext::default())]; + state.push( + row(), + DeliveryStream::Diagnostics, + Priority::DebugDiagnostic, + ); + state.push(row(), DeliveryStream::Artifacts, Priority::Artifact); + state.push(row(), DeliveryStream::Events, Priority::FailedTerminal); + state.push( + row(), + DeliveryStream::Diagnostics, + Priority::DebugDiagnostic, + ); + assert_eq!(state.counts[DeliveryStream::Diagnostics.index()].dropped, 2); + assert_eq!(state.counts[DeliveryStream::Artifacts.index()].dropped, 0); + let batch = state.take_batch(10); + let streams: Vec = batch.iter().map(|record| record.stream).collect(); + assert_eq!( + streams, + vec![DeliveryStream::Events, DeliveryStream::Artifacts] + ); +} + +#[test] +fn given_interleaved_and_large_writes_when_chunked_then_sequences_and_offsets_are_exact() { + let mut state = WriterState::new(envelope()); + let big = vec![b'a'; 64 * 1024 + 10]; + state.append_output(OutputStream::Stdout, b"one ", SystemTime::now()); + state.append_output(OutputStream::Stdout, b"two\n", SystemTime::now()); + state.append_output(OutputStream::Stderr, b"err\n", SystemTime::now()); + state.append_output(OutputStream::Stdout, &big, SystemTime::now()); + state.flush_output(); + let rows: Vec<_> = state + .take_batch(100) + .into_iter() + .flat_map(|record| record.rows) + .collect(); + let summary: Vec<(String, String, String)> = rows + .iter() + .map(|row| { + let text = |name: &str| format!("{:?}", row.value(name).expect(name)); + (text("sequence"), text("stream"), text("byte_offset")) + }) + .collect(); + assert_eq!( + summary, + vec![ + ( + "Integer(0)".into(), + "Text(\"stdout\")".into(), + "Integer(0)".into() + ), + ( + "Integer(1)".into(), + "Text(\"stderr\")".into(), + "Integer(0)".into() + ), + ( + "Integer(2)".into(), + "Text(\"stdout\")".into(), + "Integer(8)".into() + ), + ( + "Integer(3)".into(), + "Text(\"stdout\")".into(), + "Integer(65544)".into() + ), + ] + ); +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_session.rs b/crates/sqlbuild-recall/src/writer/tests/test_session.rs new file mode 100644 index 0000000000..7a0a29fb0e --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/tests/test_session.rs @@ -0,0 +1,123 @@ +use std::time::Duration; + +use crate::records::models::{InvocationContext, OutputStream, RecallValue, RowEnvelope}; +use crate::store::main::open_sqlite_store::open_sqlite_store; +use crate::store::main::query_store::query_store; +use crate::store::main::read_output_transcript::read_output_transcript; +use crate::store::models::OpenMode; +use crate::writer::main::start_recall_session::start_recall_session; +use crate::writer::models::{DeliveryStream, SessionConfig, SetupOutcome}; + +fn config(path: std::path::PathBuf, invocation_id: &str) -> SessionConfig { + SessionConfig { + store_path: path, + envelope: RowEnvelope { + invocation_id: invocation_id.to_owned(), + invocation_started_at: "2026-10-19T00:00:00.000000Z".to_owned(), + producer_version: "test".to_owned(), + }, + context: InvocationContext { + command: Some("build".to_owned()), + pid: Some(42), + ..InvocationContext::default() + }, + retention_days: 30, + shutdown_timeout: Duration::from_secs(5), + diagnostics_min_severity: "info".to_owned(), + heartbeat_interval: Duration::from_secs(30), + setup_budget: Duration::from_secs(2), + maintenance: true, + maintenance_owner: "test:42".to_owned(), + } +} + +#[test] +fn given_a_session_when_closed_then_every_record_and_the_delivery_rows_are_written() { + let directory = tempfile::tempdir().expect("temporary directory"); + let path = directory.path().join("logs/recall.sqlite3"); + let session = start_recall_session(config(path.clone(), "inv-1")); + assert_eq!( + session.wait_for_setup(Duration::from_secs(10)), + Some(SetupOutcome::Ready) + ); + let large: Vec = (0..200_000_u32).map(|value| (value % 251) as u8).collect(); + session.append_output(OutputStream::Stdout, b"start\n"); + session.append_output(OutputStream::Stderr, b"\xff\x00warn\n"); + session.append_output(OutputStream::Stdout, &large); + session + .publish_event( + r#"{"event_id":"e1","event_type":"invocation_completed","schema_version":2, + "producer":"sqlbuild","producer_version":"1","occurred_at":"2026-10-19T00:00:01.000000Z", + "invocation_id":"inv-1","invocation_sequence":1,"payload":{"command":"build","exit_code":0,"duration_ms":3.0}}"#, + ) + .expect("event"); + session + .publish_diagnostic(r#"{"occurred_at":"2026-10-19T00:00:00.500000Z","severity":"debug","logger":"l","source":"python","message":"hidden"}"#) + .expect("debug diagnostic"); + session + .publish_diagnostic(r#"{"occurred_at":"2026-10-19T00:00:00.500000Z","severity":"warning","logger":"l","source":"python","message":"kept"}"#) + .expect("warning diagnostic"); + session.publish_sql("select 1", Some("s1"), None); + session.publish_sql("select 1", Some("s2"), None); + let summary = session.close(); + assert_eq!(summary.degraded_reason, None); + let total = summary.total(); + assert_eq!((total.dropped, total.failed), (0, 0)); + let output = summary + .streams + .iter() + .find(|(stream, _)| *stream == DeliveryStream::Output) + .expect("output counts") + .1; + assert_eq!(output.delivered, output.accepted); + let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("read store"); + let transcript = read_output_transcript(&store, "inv-1", None).expect("transcript"); + let expected: Vec = [b"start\n".as_slice(), b"\xff\x00warn\n", &large].concat(); + assert_eq!(transcript.data, expected); + assert!(transcript.complete, "{transcript:?}"); + let stdout = + read_output_transcript(&store, "inv-1", Some(OutputStream::Stdout)).expect("stdout"); + assert_eq!(stdout.data, [b"start\n".as_slice(), &large].concat()); + let facts = query_store( + &store, + "SELECT (SELECT status FROM invocations), (SELECT COUNT(*) FROM recall_diagnostics), \ + (SELECT COUNT(*) FROM recall_artifacts), (SELECT COUNT(*) FROM recall_heartbeats), \ + (SELECT COUNT(*) FROM recall_delivery), (SELECT command FROM recall_invocations)", + None, + ) + .expect("facts"); + assert_eq!( + facts.rows, + vec![vec![ + RecallValue::Text("completed".to_owned()), + RecallValue::Integer(1), + RecallValue::Integer(1), + RecallValue::Integer(1), + RecallValue::Integer(5), + RecallValue::Text("build".to_owned()), + ]] + ); +} + +#[cfg(unix)] +#[test] +fn given_an_unwritable_store_when_started_then_setup_reports_permission_and_close_is_quick() { + use std::os::unix::fs::PermissionsExt; + let directory = tempfile::tempdir().expect("temporary directory"); + std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o555)) + .expect("chmod"); + let session = start_recall_session(config( + directory.path().join("logs/recall.sqlite3"), + "inv-2", + )); + let setup = session.wait_for_setup(Duration::from_secs(10)); + session.append_output(OutputStream::Stdout, b"still printed\n"); + let summary = session.close(); + std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o755)) + .expect("restore"); + assert!( + matches!(setup, Some(SetupOutcome::PermissionDenied { .. })), + "{setup:?}" + ); + assert!(summary.degraded_reason.is_some()); +} diff --git a/fensu.toml b/fensu.toml index c4802e31b1..5dc49662eb 100644 --- a/fensu.toml +++ b/fensu.toml @@ -472,6 +472,7 @@ crate_names = [ "sqlbuild-model-config", "sqlbuild-pysource", "sqlbuild-python", + "sqlbuild-recall", "sqlbuild-refactor", "sqlbuild-render", "sqlbuild-rules", diff --git a/tests/goldens/compiler/failure/target-unknown-key.json b/tests/goldens/compiler/failure/target-unknown-key.json index 238270d0ba..127cffc49b 100644 --- a/tests/goldens/compiler/failure/target-unknown-key.json +++ b/tests/goldens/compiler/failure/target-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: clone, compile_cache, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", + "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: clone, compile_cache, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", "help": "did you mean 'warehouses'?" } ] From 1833703a7f653fbab47b6509bebcebd0f1376d00 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 04:49:13 +0100 Subject: [PATCH 02/19] test: restructure Recall crate tests into table-driven cases --- crates/sqlbuild-config/src/recall/tests.rs | 4 + .../src/recall/tests/helpers.rs | 16 + .../src/recall/tests/test_recall_section.rs | 162 ++++--- .../src/recall/tests/test_types.rs | 15 + crates/sqlbuild-recall/src/records/tests.rs | 4 + .../src/records/tests/helpers.rs | 36 ++ .../src/records/tests/test_records.rs | 339 +++++++++----- .../src/records/tests/test_types.rs | 46 ++ crates/sqlbuild-recall/src/schema/tests.rs | 4 + .../src/schema/tests/helpers.rs | 68 +++ .../src/schema/tests/test_ddl.rs | 250 ++++++----- .../src/schema/tests/test_types.rs | 25 ++ crates/sqlbuild-recall/src/settings/tests.rs | 4 + .../src/settings/tests/helpers.rs | 42 ++ .../src/settings/tests/test_resolve.rs | 196 ++++---- .../src/settings/tests/test_types.rs | 15 + crates/sqlbuild-recall/src/store/tests.rs | 2 + .../src/store/tests/helpers.rs | 179 +++++++- .../src/store/tests/test_maintenance.rs | 164 +++---- .../src/store/tests/test_store.rs | 251 +++++++---- .../src/store/tests/test_types.rs | 52 +++ .../src/store/tests/test_views.rs | 419 ++++++++++-------- crates/sqlbuild-recall/src/writer/tests.rs | 4 + .../src/writer/tests/helpers.rs | 164 +++++++ .../src/writer/tests/test_queue.rs | 159 +++---- .../src/writer/tests/test_session.rs | 173 +++----- .../src/writer/tests/test_types.rs | 38 ++ 27 files changed, 1876 insertions(+), 955 deletions(-) create mode 100644 crates/sqlbuild-config/src/recall/tests/helpers.rs create mode 100644 crates/sqlbuild-config/src/recall/tests/test_types.rs create mode 100644 crates/sqlbuild-recall/src/records/tests/helpers.rs create mode 100644 crates/sqlbuild-recall/src/records/tests/test_types.rs create mode 100644 crates/sqlbuild-recall/src/schema/tests/helpers.rs create mode 100644 crates/sqlbuild-recall/src/schema/tests/test_types.rs create mode 100644 crates/sqlbuild-recall/src/settings/tests/helpers.rs create mode 100644 crates/sqlbuild-recall/src/settings/tests/test_types.rs create mode 100644 crates/sqlbuild-recall/src/store/tests/test_types.rs create mode 100644 crates/sqlbuild-recall/src/writer/tests/helpers.rs create mode 100644 crates/sqlbuild-recall/src/writer/tests/test_types.rs diff --git a/crates/sqlbuild-config/src/recall/tests.rs b/crates/sqlbuild-config/src/recall/tests.rs index 992dd65d8d..403c449344 100644 --- a/crates/sqlbuild-config/src/recall/tests.rs +++ b/crates/sqlbuild-config/src/recall/tests.rs @@ -1,2 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_recall_section.rs"] mod recall_section; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-config/src/recall/tests/helpers.rs b/crates/sqlbuild-config/src/recall/tests/helpers.rs new file mode 100644 index 0000000000..67459d0856 --- /dev/null +++ b/crates/sqlbuild-config/src/recall/tests/helpers.rs @@ -0,0 +1,16 @@ +use crate::models::ConfigValue; + +/// A configuration table holding `entries` in order. +pub(crate) fn table(entries: &[(&str, ConfigValue)]) -> ConfigValue { + ConfigValue::Map( + entries + .iter() + .map(|(key, value)| (ConfigValue::String((*key).to_owned()), value.clone())) + .collect(), + ) +} + +/// A configuration string. +pub(crate) fn text(value: &str) -> ConfigValue { + ConfigValue::String(value.to_owned()) +} diff --git a/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs b/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs index b57776f3b2..2197ec212c 100644 --- a/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs +++ b/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs @@ -1,84 +1,100 @@ use crate::models::ConfigValue; use crate::recall::main::recall_section::recall_section; use crate::recall::models::RecallSection; - -fn table(entries: &[(&str, ConfigValue)]) -> ConfigValue { - ConfigValue::Map( - entries - .iter() - .map(|(key, value)| (ConfigValue::String((*key).to_owned()), value.clone())) - .collect(), - ) -} - -fn text(value: &str) -> ConfigValue { - ConfigValue::String(value.to_owned()) -} +use crate::recall::tests::helpers::{table, text}; +use crate::recall::tests::test_types::{InvalidSectionTestCase, ValidSectionTestCase}; #[test] -fn given_every_key_when_validated_then_settings_are_typed() { - let section = recall_section( - &table(&[ - ("enabled", ConfigValue::Bool(false)), - ("connection", text(" history ")), - ("retention", text("720h")), - ("shutdown_timeout", text("1500ms")), - ("diagnostics_min_severity", text("debug")), - ]), - "recall", - ); - assert_eq!( - section, - Ok(RecallSection { - enabled: Some(false), - connection: Some("history".to_owned()), - retention_days: Some(30), - shutdown_timeout_ms: Some(1500), - diagnostics_min_severity: Some("debug".to_owned()), - }) - ); +fn given_valid_sections_when_validated_then_settings_are_typed() { + let test_cases = [ + ValidSectionTestCase { + description: "every key is typed and trimmed", + value: table(&[ + ("enabled", ConfigValue::Bool(false)), + ("connection", text(" history ")), + ("retention", text("720h")), + ("shutdown_timeout", text("1500ms")), + ("diagnostics_min_severity", text("debug")), + ]), + expected_section: RecallSection { + enabled: Some(false), + connection: Some("history".to_owned()), + retention_days: Some(30), + shutdown_timeout_ms: Some(1500), + diagnostics_min_severity: Some("debug".to_owned()), + }, + }, + ValidSectionTestCase { + description: "an empty table leaves every setting unset", + value: table(&[]), + expected_section: RecallSection::default(), + }, + ]; + + for test_case in test_cases { + assert_eq!( + recall_section(&test_case.value, "recall"), + Ok(test_case.expected_section), + "{}", + test_case.description + ); + } } #[test] -fn given_invalid_values_when_validated_then_the_message_names_the_key() { - let cases: [(&str, ConfigValue, &str); 6] = [ - ( - "unknown key", - table(&[("mode", text("x"))]), - "recall contains unknown key(s): mode", - ), - ( - "bad retention", - table(&[("retention", text("30x"))]), - "recall.retention must be", - ), - ( - "hours not days", - table(&[("retention", text("25h"))]), - "recall.retention must be", - ), - ( - "zero days", - table(&[("retention", text("0d"))]), - "recall.retention must be", - ), - ( - "timeout too long", - table(&[("shutdown_timeout", text("601s"))]), - "recall.shutdown_timeout must be", - ), - ( - "severity", - table(&[("diagnostics_min_severity", text("loud"))]), - "recall.diagnostics_min_severity must be one of", - ), +fn given_invalid_sections_when_validated_then_the_message_names_the_key() { + let test_cases = [ + InvalidSectionTestCase { + description: "unknown keys are listed", + value: table(&[("mode", text("x"))]), + label: "recall", + expected_message_prefix: "recall contains unknown key(s): mode", + }, + InvalidSectionTestCase { + description: "retention needs a known unit", + value: table(&[("retention", text("30x"))]), + label: "recall", + expected_message_prefix: "recall.retention must be", + }, + InvalidSectionTestCase { + description: "retention hours must be whole days", + value: table(&[("retention", text("25h"))]), + label: "recall", + expected_message_prefix: "recall.retention must be", + }, + InvalidSectionTestCase { + description: "retention must be positive", + value: table(&[("retention", text("0d"))]), + label: "recall", + expected_message_prefix: "recall.retention must be", + }, + InvalidSectionTestCase { + description: "shutdown timeout is bounded", + value: table(&[("shutdown_timeout", text("601s"))]), + label: "recall", + expected_message_prefix: "recall.shutdown_timeout must be", + }, + InvalidSectionTestCase { + description: "severity must be known", + value: table(&[("diagnostics_min_severity", text("loud"))]), + label: "recall", + expected_message_prefix: "recall.diagnostics_min_severity must be one of", + }, + InvalidSectionTestCase { + description: "the section must be a table", + value: text("x"), + label: "targets.dev.recall", + expected_message_prefix: "targets.dev.recall must be a table", + }, ]; - for (description, value, expected) in cases { - let message = recall_section(&value, "recall").expect_err(description); - assert!(message.starts_with(expected), "{description}: {message}"); + + for test_case in test_cases { + let message: String = recall_section(&test_case.value, test_case.label) + .expect_err("the settings are invalid"); + assert!( + message.starts_with(test_case.expected_message_prefix), + "{}: {message}", + test_case.description + ); } - assert_eq!( - recall_section(&text("x"), "targets.dev.recall"), - Err("targets.dev.recall must be a table".to_owned()) - ); } diff --git a/crates/sqlbuild-config/src/recall/tests/test_types.rs b/crates/sqlbuild-config/src/recall/tests/test_types.rs new file mode 100644 index 0000000000..e083d04520 --- /dev/null +++ b/crates/sqlbuild-config/src/recall/tests/test_types.rs @@ -0,0 +1,15 @@ +use crate::models::ConfigValue; +use crate::recall::models::RecallSection; + +pub(crate) struct ValidSectionTestCase { + pub(crate) description: &'static str, + pub(crate) value: ConfigValue, + pub(crate) expected_section: RecallSection, +} + +pub(crate) struct InvalidSectionTestCase { + pub(crate) description: &'static str, + pub(crate) value: ConfigValue, + pub(crate) label: &'static str, + pub(crate) expected_message_prefix: &'static str, +} diff --git a/crates/sqlbuild-recall/src/records/tests.rs b/crates/sqlbuild-recall/src/records/tests.rs index 540eed7214..9e55cb0928 100644 --- a/crates/sqlbuild-recall/src/records/tests.rs +++ b/crates/sqlbuild-recall/src/records/tests.rs @@ -1,2 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_records.rs"] mod records; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-recall/src/records/tests/helpers.rs b/crates/sqlbuild-recall/src/records/tests/helpers.rs new file mode 100644 index 0000000000..8e6d26e600 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/tests/helpers.rs @@ -0,0 +1,36 @@ +use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; + +/// The envelope every record test row shares. +pub(crate) fn envelope() -> RowEnvelope { + RowEnvelope { + invocation_id: "inv1".to_owned(), + invocation_started_at: "2026-10-19T00:00:00.000000Z".to_owned(), + producer_version: "0.0.0".to_owned(), + } +} + +/// `columns` of `row` paired with their values, when present. +pub(crate) fn row_columns( + row: &RecallRow, + columns: &[(&'static str, RecallValue)], +) -> Vec<(&'static str, Option)> { + columns + .iter() + .map(|(name, _)| (*name, row.value(name).cloned())) + .collect() +} + +/// `expected` in the shape `row_columns` returns. +pub(crate) fn present( + expected: Vec<(&'static str, RecallValue)>, +) -> Vec<(&'static str, Option)> { + expected + .into_iter() + .map(|(name, value)| (name, Some(value))) + .collect() +} + +/// A text value. +pub(crate) fn text(value: &str) -> RecallValue { + RecallValue::Text(value.to_owned()) +} diff --git a/crates/sqlbuild-recall/src/records/tests/test_records.rs b/crates/sqlbuild-recall/src/records/tests/test_records.rs index 48eec2925d..18bb7c2178 100644 --- a/crates/sqlbuild-recall/src/records/tests/test_records.rs +++ b/crates/sqlbuild-recall/src/records/tests/test_records.rs @@ -8,128 +8,269 @@ use crate::records::main::event_row::event_row; use crate::records::main::format_timestamp::format_timestamp; use crate::records::main::output_row::output_row; use crate::records::main::parse_timestamp::parse_timestamp; -use crate::records::models::{OutputChunk, OutputStream, RecallValue, RowEnvelope}; - -fn envelope() -> RowEnvelope { - RowEnvelope { - invocation_id: "inv1".to_owned(), - invocation_started_at: "2026-10-19T00:00:00.000000Z".to_owned(), - producer_version: "0.0.0".to_owned(), - } -} +use crate::records::models::{OutputChunk, OutputStream, RecallValue}; +use crate::records::tests::helpers::{envelope, present, row_columns, text}; +use crate::records::tests::test_types::{ + ContentHashTestCase, DiagnosticTestCase, ParseTimestampTestCase, RecordIdTestCase, + RejectedInputTestCase, RowColumnsTestCase, TimestampTestCase, +}; #[test] fn given_sequences_when_ids_are_built_then_they_follow_the_record_id_scheme() { - assert_eq!(output_record_id("abc", 7), "abc:command-output:7"); - assert_eq!(diagnostic_record_id("abc", 0), "abc:diagnostic:0"); + let test_cases = [ + RecordIdTestCase { + description: "output chunks are keyed by invocation and sequence", + record_id: output_record_id("abc", 7), + expected_record_id: "abc:command-output:7", + }, + RecordIdTestCase { + description: "diagnostics are keyed by invocation and sequence", + record_id: diagnostic_record_id("abc", 0), + expected_record_id: "abc:diagnostic:0", + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_case.record_id, test_case.expected_record_id, + "{}", + test_case.description + ); + } } #[test] fn given_times_when_formatted_and_parsed_then_they_round_trip() { - let cases = [ - (0_u64, "1970-01-01T00:00:00.000000Z"), - (951_782_400_123_456, "2000-02-29T00:00:00.123456Z"), - (1_792_281_600_000_001, "2026-10-18T00:00:00.000001Z"), + let test_cases = [ + TimestampTestCase { + description: "the epoch", + micros: 0, + expected_text: "1970-01-01T00:00:00.000000Z", + }, + TimestampTestCase { + description: "a leap day with microseconds", + micros: 951_782_400_123_456, + expected_text: "2000-02-29T00:00:00.123456Z", + }, + TimestampTestCase { + description: "one microsecond past midnight", + micros: 1_792_281_600_000_001, + expected_text: "2026-10-18T00:00:00.000001Z", + }, ]; - for (micros, expected) in cases { - let time = UNIX_EPOCH + Duration::from_micros(micros); - assert_eq!(format_timestamp(time), expected); - assert_eq!(parse_timestamp(expected), Some(time)); + + for test_case in test_cases { + let time = UNIX_EPOCH + Duration::from_micros(test_case.micros); + assert_eq!( + ( + format_timestamp(time), + parse_timestamp(test_case.expected_text) + ), + (test_case.expected_text.to_owned(), Some(time)), + "{}", + test_case.description + ); } - assert_eq!( - parse_timestamp("2026-10-18T00:00:00+00:00"), - parse_timestamp("2026-10-18T00:00:00.000000Z") - ); - assert_eq!(parse_timestamp("yesterday"), None); } #[test] -fn given_equal_and_different_content_when_hashed_then_only_content_matters() { - let base = vec![ - ("a", RecallValue::Text("x".to_owned())), - ("b", RecallValue::Integer(1)), - ]; - let reordered = vec![ - ("b", RecallValue::Integer(1)), - ("a", RecallValue::Text("x".to_owned())), - ("ingested_at", RecallValue::Text("later".to_owned())), - ]; - let changed = vec![ - ("a", RecallValue::Blob(b"x".to_vec())), - ("b", RecallValue::Integer(1)), +fn given_timestamp_text_when_parsed_then_offsets_are_accepted_and_garbage_rejected() { + let test_cases = [ + ParseTimestampTestCase { + description: "a zero offset equals Z", + text: "2026-10-18T00:00:00+00:00", + expected_micros: Some(1_792_281_600_000_000), + }, + ParseTimestampTestCase { + description: "missing fractions are zero", + text: "2026-10-18T00:00:00Z", + expected_micros: Some(1_792_281_600_000_000), + }, + ParseTimestampTestCase { + description: "words are not timestamps", + text: "yesterday", + expected_micros: None, + }, ]; - assert_eq!(content_sha256(&base), content_sha256(&reordered)); - assert_ne!(content_sha256(&base), content_sha256(&changed)); + + for test_case in test_cases { + assert_eq!( + parse_timestamp(test_case.text), + test_case + .expected_micros + .map(|micros| UNIX_EPOCH + Duration::from_micros(micros)), + "{}", + test_case.description + ); + } } #[test] -fn given_a_chunk_when_rowed_then_bytes_and_offsets_are_kept() { - let row = output_row( - &envelope(), - &OutputChunk { - sequence: 3, - stream: OutputStream::Stderr, - occurred_at: "2026-10-19T00:00:01.000000Z".to_owned(), - data: vec![0, 255, 10], - byte_offset: 12, - dropped_before: 1, - }, - ); - assert_eq!( - row.value("data"), - Some(&RecallValue::Blob(vec![0, 255, 10])) - ); - assert_eq!(row.value("byte_count"), Some(&RecallValue::Integer(3))); - assert_eq!( - row.value("record_id"), - Some(&RecallValue::Text("inv1:command-output:3".to_owned())) - ); - assert!( - row.content_sha256() - .is_some_and(|digest| digest.len() == 64) - ); +fn given_equal_and_different_content_when_hashed_then_only_content_matters() { + let test_cases = [ + ContentHashTestCase { + description: "column order and ingestion time do not matter", + left: vec![("a", text("x")), ("b", RecallValue::Integer(1))], + right: vec![ + ("b", RecallValue::Integer(1)), + ("a", text("x")), + ("ingested_at", text("later")), + ], + expected_equal: true, + }, + ContentHashTestCase { + description: "value types matter", + left: vec![("a", text("x")), ("b", RecallValue::Integer(1))], + right: vec![ + ("a", RecallValue::Blob(b"x".to_vec())), + ("b", RecallValue::Integer(1)), + ], + expected_equal: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + content_sha256(&test_case.left) == content_sha256(&test_case.right), + test_case.expected_equal, + "{}", + test_case.description + ); + } } #[test] -fn given_lifecycle_json_when_rowed_then_dimensions_are_promoted() { - let json = r#"{"event_id":"e1","event_type":"statement_failed","schema_version":2, +fn given_records_when_rowed_then_columns_are_promoted() { + let lifecycle = r#"{"event_id":"e1","event_type":"statement_failed","schema_version":2, "producer":"sqlbuild","producer_version":"1","occurred_at":"2026-10-19T00:00:02.000000Z", "invocation_id":"inv1","invocation_sequence":4,"statement_id":"s1", "payload":{"error_type":"ValueError","duration_ms":1.5,"sql_digest":"ab","purpose":"audit"}}"#; - let row = event_row(&envelope(), json).expect("valid event"); - assert_eq!( - row.value("event_kind"), - Some(&RecallValue::Text("statement".to_owned())) - ); - assert_eq!( - row.value("severity"), - Some(&RecallValue::Text("error".to_owned())) - ); - assert_eq!( - row.value("statement_id"), - Some(&RecallValue::Text("s1".to_owned())) - ); - assert_eq!(row.value("duration_ms"), Some(&RecallValue::Real(1.5))); - assert_eq!( - row.value("purpose"), - Some(&RecallValue::Text("audit".to_owned())) - ); - assert_eq!( - row.value("invocation_sequence"), - Some(&RecallValue::Integer(4)) - ); - assert!(event_row(&envelope(), "[]").is_err()); - assert!(event_row(&envelope(), r#"{"event_type":"x"}"#).is_err()); + let test_cases = [ + RowColumnsTestCase { + description: "output chunks keep bytes, counts and record ids", + row: output_row( + &envelope(), + &OutputChunk { + sequence: 3, + stream: OutputStream::Stderr, + occurred_at: "2026-10-19T00:00:01.000000Z".to_owned(), + data: vec![0, 255, 10], + byte_offset: 12, + dropped_before: 1, + }, + ), + expected_columns: vec![ + ("data", RecallValue::Blob(vec![0, 255, 10])), + ("byte_count", RecallValue::Integer(3)), + ("byte_offset", RecallValue::Integer(12)), + ("record_id", text("inv1:command-output:3")), + ], + }, + RowColumnsTestCase { + description: "lifecycle events promote kind, severity and payload dimensions", + row: event_row(&envelope(), lifecycle).expect("the fixture is a valid lifecycle event"), + expected_columns: vec![ + ("event_kind", text("statement")), + ("severity", text("error")), + ("statement_id", text("s1")), + ("duration_ms", RecallValue::Real(1.5)), + ("purpose", text("audit")), + ("invocation_sequence", RecallValue::Integer(4)), + ], + }, + ]; + + for test_case in test_cases { + assert_eq!( + row_columns(&test_case.row, &test_case.expected_columns), + present(test_case.expected_columns), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_malformed_lifecycle_json_when_rowed_then_it_is_rejected() { + let test_cases = [ + RejectedInputTestCase { + description: "an array is not an event", + json: "[]", + expected_rejected: true, + }, + RejectedInputTestCase { + description: "an event needs its identity fields", + json: r#"{"event_type":"x"}"#, + expected_rejected: true, + }, + ]; + + for test_case in test_cases { + assert_eq!( + event_row(&envelope(), test_case.json).is_err(), + test_case.expected_rejected, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_diagnostic_json_when_parsed_then_sequence_fields_and_correlations_are_kept() { + let test_cases = [ + DiagnosticTestCase { + description: "fields and run correlation survive", + json: r#"{"occurred_at":"2026-10-19T00:00:00.000000Z","severity":"warning", + "logger":"sqlbuild.x","source":"python","message":"careful","fields":{"k":1}, + "run_id":"r1"}"#, + sequence: 5, + expected_fields: r#"{"k":1}"#, + expected_run_id: Some("r1"), + }, + DiagnosticTestCase { + description: "absent fields become an empty object", + json: r#"{"occurred_at":"2026-10-19T00:00:00.000000Z","severity":"info", + "logger":"sqlbuild.x","source":"python","message":"hello"}"#, + sequence: 0, + expected_fields: "{}", + expected_run_id: None, + }, + ]; + + for test_case in test_cases { + let diagnostic = diagnostic_input_from_json(test_case.json, test_case.sequence) + .expect("the fixture is a valid diagnostic"); + assert_eq!( + ( + diagnostic.sequence, + diagnostic.fields.as_str(), + diagnostic.correlations.run_id.as_deref() + ), + ( + test_case.sequence, + test_case.expected_fields, + test_case.expected_run_id + ), + "{}", + test_case.description + ); + } } #[test] -fn given_diagnostic_json_when_parsed_then_correlations_are_kept() { - let json = r#"{"occurred_at":"2026-10-19T00:00:00.000000Z","severity":"warning", - "logger":"sqlbuild.x","source":"python","message":"careful","fields":{"k":1}, - "run_id":"r1"}"#; - let diagnostic = diagnostic_input_from_json(json, 5).expect("valid diagnostic"); - assert_eq!(diagnostic.sequence, 5); - assert_eq!(diagnostic.fields, r#"{"k":1}"#); - assert_eq!(diagnostic.correlations.run_id.as_deref(), Some("r1")); - assert!(diagnostic_input_from_json(r#"{"severity":"info"}"#, 0).is_err()); +fn given_incomplete_diagnostic_json_when_parsed_then_it_is_rejected() { + let test_cases = [RejectedInputTestCase { + description: "a diagnostic needs a message and logger", + json: r#"{"severity":"info"}"#, + expected_rejected: true, + }]; + + for test_case in test_cases { + assert_eq!( + diagnostic_input_from_json(test_case.json, 0).is_err(), + test_case.expected_rejected, + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-recall/src/records/tests/test_types.rs b/crates/sqlbuild-recall/src/records/tests/test_types.rs new file mode 100644 index 0000000000..86272de086 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/tests/test_types.rs @@ -0,0 +1,46 @@ +use crate::records::models::{RecallRow, RecallValue}; + +pub(crate) struct RecordIdTestCase { + pub(crate) description: &'static str, + pub(crate) record_id: String, + pub(crate) expected_record_id: &'static str, +} + +pub(crate) struct TimestampTestCase { + pub(crate) description: &'static str, + pub(crate) micros: u64, + pub(crate) expected_text: &'static str, +} + +pub(crate) struct ParseTimestampTestCase { + pub(crate) description: &'static str, + pub(crate) text: &'static str, + pub(crate) expected_micros: Option, +} + +pub(crate) struct ContentHashTestCase { + pub(crate) description: &'static str, + pub(crate) left: Vec<(&'static str, RecallValue)>, + pub(crate) right: Vec<(&'static str, RecallValue)>, + pub(crate) expected_equal: bool, +} + +pub(crate) struct RowColumnsTestCase { + pub(crate) description: &'static str, + pub(crate) row: RecallRow, + pub(crate) expected_columns: Vec<(&'static str, RecallValue)>, +} + +pub(crate) struct RejectedInputTestCase { + pub(crate) description: &'static str, + pub(crate) json: &'static str, + pub(crate) expected_rejected: bool, +} + +pub(crate) struct DiagnosticTestCase { + pub(crate) description: &'static str, + pub(crate) json: &'static str, + pub(crate) sequence: u64, + pub(crate) expected_fields: &'static str, + pub(crate) expected_run_id: Option<&'static str>, +} diff --git a/crates/sqlbuild-recall/src/schema/tests.rs b/crates/sqlbuild-recall/src/schema/tests.rs index 3492109aaa..e448cf632c 100644 --- a/crates/sqlbuild-recall/src/schema/tests.rs +++ b/crates/sqlbuild-recall/src/schema/tests.rs @@ -1,2 +1,6 @@ #[path = "tests/test_ddl.rs"] mod ddl; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-recall/src/schema/tests/helpers.rs b/crates/sqlbuild-recall/src/schema/tests/helpers.rs new file mode 100644 index 0000000000..2c8fd299d6 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/tests/helpers.rs @@ -0,0 +1,68 @@ +use crate::schema::constants::VIEW_NAMES; +use crate::schema::main::ddl_script::ddl_script; +use crate::schema::main::sqlite_retention_sql::sqlite_retention_sql; +use crate::schema::main::table_statements::table_statements; +use crate::schema::models::{DdlOptions, Dialect}; + +/// Options rendering into `observability` with 14-day retention, optionally on `cluster`. +pub(crate) fn options(cluster: Option<&str>) -> DdlOptions { + DdlOptions { + database: Some("observability".to_owned()), + retention_days: 14, + cluster: cluster.map(str::to_owned), + } +} + +/// The ClickHouse `CREATE TABLE` statement for `table`. +pub(crate) fn clickhouse_table(table: &str, cluster: Option<&str>) -> String { + let quoted = format!("`{table}`"); + table_statements(Dialect::ClickHouse, &options(cluster)) + .into_iter() + .find(|statement| statement.contains("ed)) + .unwrap_or_default() +} + +/// The `fragments` that `statement` contains, in order. +pub(crate) fn contained<'a>(statement: &str, fragments: &[&'a str]) -> Vec<&'a str> { + fragments + .iter() + .copied() + .filter(|fragment| statement.contains(fragment)) + .collect() +} + +/// Schema objects after running the SQLite script `runs` times, and the statements that fail. +pub(crate) fn sqlite_schema_after(runs: usize) -> (usize, Vec) { + let connection = rusqlite::Connection::open_in_memory().expect("an in-memory database opens"); + let script = ddl_script(Dialect::Sqlite, &options(None)); + let mut failures: Vec = Vec::new(); + for run in 0..runs { + let outcome: rusqlite::Result<()> = connection.execute_batch(&script); + failures.extend(outcome.err().map(|error| format!("run {run}: {error}"))); + } + let retention = sqlite_retention_sql(); + let mut statements: Vec = VIEW_NAMES + .iter() + .map(|view| format!("SELECT * FROM \"{view}\"")) + .collect(); + statements.push(retention.expired_invocations.clone()); + statements.extend( + retention + .delete_invocations + .iter() + .map(|(_, delete)| delete.clone()), + ); + statements.push(retention.delete_orphan_artifacts.clone()); + for statement in statements { + let prepared: rusqlite::Result> = connection.prepare(&statement); + failures.extend(prepared.err().map(|error| format!("{statement}: {error}"))); + } + let count: i64 = connection + .query_row( + "SELECT COUNT(*) FROM sqlite_schema WHERE type IN ('table', 'view')", + [], + |row| row.get(0), + ) + .expect("the count query runs"); + (usize::try_from(count).unwrap_or_default(), failures) +} diff --git a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs index 82daf72aee..b59d10139a 100644 --- a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs +++ b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs @@ -1,129 +1,169 @@ use crate::schema::constants::{ALL_TABLES, VIEW_NAMES}; use crate::schema::main::add_column_statement::add_column_statement; -use crate::schema::main::ddl_script::ddl_script; -use crate::schema::main::sqlite_retention_sql::sqlite_retention_sql; use crate::schema::main::table_spec::table_spec; use crate::schema::main::table_statements::table_statements; use crate::schema::main::view_statements::view_statements; -use crate::schema::models::{ColumnSpec, ColumnType, DdlOptions, Dialect, Table}; +use crate::schema::models::{ColumnSpec, ColumnType, Dialect, Table}; +use crate::schema::tests::helpers::{clickhouse_table, contained, options, sqlite_schema_after}; +use crate::schema::tests::test_types::{ + AddColumnTestCase, FragmentTestCase, SqliteScriptTestCase, StatementSetTestCase, +}; -fn options(cluster: Option<&str>) -> DdlOptions { - DdlOptions { - database: Some("observability".to_owned()), - retention_days: 14, - cluster: cluster.map(str::to_owned), - } -} +const NEW_COLUMN: ColumnSpec = ColumnSpec { + name: "new_field", + column_type: ColumnType::Integer, + nullable: false, +}; #[test] -fn given_sqlite_script_when_run_twice_then_every_table_and_view_exists() { - let connection = rusqlite::Connection::open_in_memory().expect("memory database"); - let script = ddl_script(Dialect::Sqlite, &options(None)); - connection.execute_batch(&script).expect("first run"); - connection - .execute_batch(&script) - .expect("rerun is idempotent"); - let count: i64 = connection - .query_row( - "SELECT COUNT(*) FROM sqlite_schema WHERE type IN ('table', 'view')", - [], - |row| row.get(0), - ) - .expect("count"); - assert_eq!(count as usize, ALL_TABLES.len() + VIEW_NAMES.len()); - for view in VIEW_NAMES { - connection - .prepare(&format!("SELECT * FROM \"{view}\"")) - .unwrap_or_else(|error| panic!("{view}: {error}")); - } - let retention = sqlite_retention_sql(); - connection - .prepare(&retention.expired_invocations) - .expect("expired query prepares"); - for (_, delete) in &retention.delete_invocations { - connection.prepare(delete).expect("delete prepares"); +fn given_sqlite_script_when_run_then_every_table_view_and_retention_statement_prepares() { + let test_cases = [ + SqliteScriptTestCase { + description: "a fresh database gets every object", + runs: 1, + expected_objects: ALL_TABLES.len() + VIEW_NAMES.len(), + expected_failures: Vec::new(), + }, + SqliteScriptTestCase { + description: "rerunning the script is idempotent", + runs: 2, + expected_objects: ALL_TABLES.len() + VIEW_NAMES.len(), + expected_failures: Vec::new(), + }, + ]; + + for test_case in test_cases { + assert_eq!( + sqlite_schema_after(test_case.runs), + (test_case.expected_objects, test_case.expected_failures), + "{}", + test_case.description + ); } - connection - .prepare(&retention.delete_orphan_artifacts) - .expect("artifact delete prepares"); } #[test] -fn given_clickhouse_options_when_rendered_then_engines_ttl_and_partitions_follow_the_table() { - let statements = table_statements(Dialect::ClickHouse, &options(None)); - assert_eq!(statements.len(), ALL_TABLES.len()); - let output = statements - .iter() - .find(|statement| statement.contains("`recall_output`")) - .expect("output table"); - assert!(output.starts_with("CREATE TABLE IF NOT EXISTS `observability`.`recall_output` (")); - assert!(output.contains("ENGINE = ReplacingMergeTree()")); - assert!( - statements - .iter() - .any(|statement| statement.contains("ENGINE = ReplacingMergeTree(seen_at)")) - ); - assert!(output.contains("PARTITION BY toDate(invocation_started_at)")); - assert!(output.contains("ORDER BY (invocation_id, sequence)")); - assert!(output.contains("TTL invocation_started_at + INTERVAL 14 DAY")); - assert!(output.contains("`data` String")); - assert!(output.contains("`priority` Nullable(String)")); - let artifacts = statements - .iter() - .find(|statement| statement.contains("`recall_artifacts`")) - .expect("artifacts table"); - assert!(artifacts.contains("TTL seen_at + INTERVAL 21 DAY")); - let heartbeats = statements - .iter() - .find(|statement| statement.contains("`recall_heartbeats`")) - .expect("heartbeat table"); - assert!(heartbeats.contains("ENGINE = MergeTree()")); -} +fn given_clickhouse_tables_when_rendered_then_engines_ttl_and_partitions_follow_the_table() { + let test_cases = [ + FragmentTestCase { + description: "output is a replacing table partitioned and expired by start time", + statement: clickhouse_table("recall_output", None), + expected_fragments: &[ + "CREATE TABLE IF NOT EXISTS `observability`.`recall_output` (", + "`data` String", + "`priority` Nullable(String)", + "ENGINE = ReplacingMergeTree()", + "PARTITION BY toDate(invocation_started_at)", + "ORDER BY (invocation_id, sequence)", + "TTL invocation_started_at + INTERVAL 14 DAY", + ], + }, + FragmentTestCase { + description: "artifacts keep the latest sighting for an extra week", + statement: clickhouse_table("recall_artifacts", None), + expected_fragments: &[ + "ENGINE = ReplacingMergeTree(seen_at)", + "TTL seen_at + INTERVAL 21 DAY", + ], + }, + FragmentTestCase { + description: "heartbeats are append-only", + statement: clickhouse_table("recall_heartbeats", None), + expected_fragments: &["ENGINE = MergeTree()"], + }, + FragmentTestCase { + description: "clustered replacing tables are replicated", + statement: clickhouse_table("recall_output", Some("analytics")), + expected_fragments: &[ + " ON CLUSTER `analytics` (", + "ReplicatedReplacingMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')", + ], + }, + ]; -#[test] -fn given_a_cluster_when_rendered_then_tables_are_replicated_on_cluster() { - let statements = table_statements(Dialect::ClickHouse, &options(Some("analytics"))); - assert!( - statements - .iter() - .all(|statement| statement.contains(" ON CLUSTER `analytics` (")) - ); - assert!(statements[1].contains( - "ReplicatedReplacingMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')" - )); - let views = view_statements(Dialect::ClickHouse, &options(Some("analytics"))); - assert_eq!(views.len(), VIEW_NAMES.len()); - assert!(views.iter().all( - |(_, statement)| statement.starts_with("CREATE OR REPLACE VIEW") - && statement.contains("ON CLUSTER `analytics`") - )); + for test_case in test_cases { + assert_eq!( + contained(&test_case.statement, test_case.expected_fragments), + test_case.expected_fragments.to_vec(), + "{}", + test_case.description + ); + } } #[test] -fn given_clickhouse_views_when_rendered_then_rows_are_deduplicated_within_retention() { - for (name, statement) in view_statements(Dialect::ClickHouse, &options(None)) { - assert!( - statement.contains("now64(6) - INTERVAL 14 DAY") && statement.contains("LIMIT 1 BY"), - "{name} must deduplicate and filter by retention" +fn given_clickhouse_statement_sets_when_rendered_then_every_statement_has_the_shared_shape() { + let views = |cluster: Option<&str>| -> Vec { + view_statements(Dialect::ClickHouse, &options(cluster)) + .into_iter() + .map(|(_, statement)| statement) + .collect() + }; + let test_cases = [ + StatementSetTestCase { + description: "every clustered table is created on the cluster", + statements: table_statements(Dialect::ClickHouse, &options(Some("analytics"))), + required_fragments: &[" ON CLUSTER `analytics` ("], + expected_count: ALL_TABLES.len(), + }, + StatementSetTestCase { + description: "clustered views are replaced on the cluster", + statements: views(Some("analytics")), + required_fragments: &["CREATE OR REPLACE VIEW", "ON CLUSTER `analytics`"], + expected_count: VIEW_NAMES.len(), + }, + StatementSetTestCase { + description: "views deduplicate rows and filter by retention", + statements: views(None), + required_fragments: &["now64(6) - INTERVAL 14 DAY", "LIMIT 1 BY"], + expected_count: VIEW_NAMES.len(), + }, + ]; + + for test_case in test_cases { + let shaped: usize = test_case + .statements + .iter() + .filter(|statement| { + contained(statement, test_case.required_fragments).len() + == test_case.required_fragments.len() + }) + .count(); + assert_eq!( + (test_case.statements.len(), shaped), + (test_case.expected_count, test_case.expected_count), + "{}", + test_case.description ); } } #[test] fn given_a_new_column_when_upgraded_then_the_statement_is_additive_and_nullable() { - let column = ColumnSpec { - name: "new_field", - column_type: ColumnType::Integer, - nullable: false, - }; let spec = table_spec(Table::Events); - assert_eq!( - add_column_statement(Dialect::Sqlite, &options(None), (spec, &column)), - "ALTER TABLE \"recall_events\" ADD COLUMN \"new_field\" INTEGER" - ); - assert_eq!( - add_column_statement(Dialect::ClickHouse, &options(None), (spec, &column)), - "ALTER TABLE `observability`.`recall_events` ADD COLUMN IF NOT EXISTS `new_field` \ - Nullable(Int64)" - ); + let test_cases = [ + AddColumnTestCase { + description: "SQLite columns are nullable by default", + statement: add_column_statement(Dialect::Sqlite, &options(None), (spec, &NEW_COLUMN)), + expected_statement: "ALTER TABLE \"recall_events\" ADD COLUMN \"new_field\" INTEGER", + }, + AddColumnTestCase { + description: "ClickHouse columns are added idempotently as Nullable", + statement: add_column_statement( + Dialect::ClickHouse, + &options(None), + (spec, &NEW_COLUMN), + ), + expected_statement: "ALTER TABLE `observability`.`recall_events` ADD COLUMN IF NOT \ + EXISTS `new_field` Nullable(Int64)", + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_case.statement, test_case.expected_statement, + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-recall/src/schema/tests/test_types.rs b/crates/sqlbuild-recall/src/schema/tests/test_types.rs new file mode 100644 index 0000000000..e064d5472a --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/tests/test_types.rs @@ -0,0 +1,25 @@ +pub(crate) struct SqliteScriptTestCase { + pub(crate) description: &'static str, + pub(crate) runs: usize, + pub(crate) expected_objects: usize, + pub(crate) expected_failures: Vec, +} + +pub(crate) struct FragmentTestCase { + pub(crate) description: &'static str, + pub(crate) statement: String, + pub(crate) expected_fragments: &'static [&'static str], +} + +pub(crate) struct StatementSetTestCase { + pub(crate) description: &'static str, + pub(crate) statements: Vec, + pub(crate) required_fragments: &'static [&'static str], + pub(crate) expected_count: usize, +} + +pub(crate) struct AddColumnTestCase { + pub(crate) description: &'static str, + pub(crate) statement: String, + pub(crate) expected_statement: &'static str, +} diff --git a/crates/sqlbuild-recall/src/settings/tests.rs b/crates/sqlbuild-recall/src/settings/tests.rs index 8766dedceb..5cf4652f06 100644 --- a/crates/sqlbuild-recall/src/settings/tests.rs +++ b/crates/sqlbuild-recall/src/settings/tests.rs @@ -1,2 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_resolve.rs"] mod resolve; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-recall/src/settings/tests/helpers.rs b/crates/sqlbuild-recall/src/settings/tests/helpers.rs new file mode 100644 index 0000000000..b9dba8056f --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/tests/helpers.rs @@ -0,0 +1,42 @@ +use std::path::PathBuf; + +use crate::settings::main::resolve_recall_settings::resolve_recall_settings; +use crate::settings::models::{RecallSettings, SettingsRequest, StoreLocation}; + +/// The placeholder project directory resolved paths are reported under. +pub(crate) const PROJECT: &str = ""; + +/// Settings resolved for a project holding `files`, debug-printed with its directory as `PROJECT`. +pub(crate) fn resolved(files: &[(&str, &str)], target: Option<&str>) -> Result { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + for (name, text) in files { + std::fs::write(directory.path().join(name), text).expect("test files can be written"); + } + let request = SettingsRequest { + project_dir: directory.path().to_path_buf(), + selected_target: target.map(str::to_owned), + }; + let project: String = directory.path().to_string_lossy().into_owned(); + resolve_recall_settings(&request) + .map(|settings| format!("{settings:?}").replace(&project, PROJECT)) + .map_err(|error| error.message) +} + +/// The default settings for project `shop` on the duckdb adapter. +pub(crate) fn defaults() -> RecallSettings { + RecallSettings { + enabled: true, + target: None, + store: StoreLocation::Sqlite { + path: PathBuf::from(PROJECT).join("logs").join("recall.sqlite3"), + }, + connection: None, + retention_days: 30, + shutdown_timeout_ms: 3000, + diagnostics_min_severity: "info".to_owned(), + project_name: Some("shop".to_owned()), + project_adapter: Some("duckdb".to_owned()), + destination_database: None, + destination_schema: None, + } +} diff --git a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs index b570f5fc86..fc8f5c931e 100644 --- a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs +++ b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs @@ -1,101 +1,115 @@ -use std::path::Path; +use crate::settings::models::{RecallSettings, StoreLocation}; +use crate::settings::tests::helpers::{defaults, resolved}; +use crate::settings::tests::test_types::{ResolveErrorTestCase, ResolveTestCase}; -use crate::settings::main::resolve_recall_settings::resolve_recall_settings; -use crate::settings::models::{SettingsRequest, StoreLocation}; +const DEFAULT_PROJECT: &[(&str, &str)] = &[( + "sqlbuild_project.toml", + "name = \"shop\"\nadapter = \"duckdb\"\ndefault_target = \"dev\"\n[targets.dev]\nschema = \"analytics\"\n", +)]; -fn project(files: &[(&str, &str)]) -> tempfile::TempDir { - let directory = tempfile::tempdir().expect("temporary directory"); - for (name, text) in files { - std::fs::write(directory.path().join(name), text).expect("write config"); - } - directory -} - -fn request(directory: &Path, target: Option<&str>) -> SettingsRequest { - SettingsRequest { - project_dir: directory.to_path_buf(), - selected_target: target.map(str::to_owned), - } -} - -#[test] -fn given_no_recall_section_when_resolved_then_local_sqlite_is_on_by_default() { - let directory = project(&[( +const LAYERED_PROJECT: &[(&str, &str)] = &[ + ( "sqlbuild_project.toml", - "name = \"shop\"\nadapter = \"duckdb\"\ndefault_target = \"dev\"\n[targets.dev]\nschema = \"analytics\"\n", - )]); - let settings = resolve_recall_settings(&request(directory.path(), None)).expect("settings"); - assert!(settings.enabled); - assert_eq!(settings.target.as_deref(), Some("dev")); - assert_eq!( - settings.store, - StoreLocation::Sqlite { - path: directory.path().join("logs").join("recall.sqlite3") - } - ); - assert_eq!(settings.retention_days, 30); - assert_eq!(settings.shutdown_timeout_ms, 3000); - assert_eq!(settings.diagnostics_min_severity, "info"); - assert_eq!(settings.destination_schema.as_deref(), Some("analytics")); - assert_eq!(settings.project_adapter.as_deref(), Some("duckdb")); -} + "name = \"shop\"\nadapter = \"duckdb\"\n\ + [recall]\nretention = \"10d\"\nconnection = \"history\"\n\ + [connections.history]\nadapter = \"sqlite\"\npath = \"state/history.sqlite3\"\n\ + [connections.central]\nadapter = \"clickhouse\"\nhost = \"localhost\"\n\ + [targets.dev]\n[targets.dev.recall]\nenabled = false\n\ + [targets.prod]\n[targets.prod.recall]\nconnection = \"central\"\n", + ), + ( + "sqlbuild_local.toml", + "[recall]\nshutdown_timeout = \"1s\"\n[connections.history]\npath = \"/tmp/opencode/elsewhere.sqlite3\"\n", + ), +]; #[test] -fn given_layers_when_resolved_then_target_over_local_over_project() { - let directory = project(&[ - ( - "sqlbuild_project.toml", - "name = \"shop\"\nadapter = \"duckdb\"\n\ - [recall]\nretention = \"10d\"\nconnection = \"history\"\n\ - [connections.history]\nadapter = \"sqlite\"\npath = \"state/history.sqlite3\"\n\ - [connections.central]\nadapter = \"clickhouse\"\nhost = \"localhost\"\n\ - [targets.dev]\n[targets.dev.recall]\nenabled = false\n\ - [targets.prod]\n[targets.prod.recall]\nconnection = \"central\"\n", - ), - ( - "sqlbuild_local.toml", - "[recall]\nshutdown_timeout = \"1s\"\n[connections.history]\npath = \"/tmp/opencode/elsewhere.sqlite3\"\n", - ), - ]); - let dev = resolve_recall_settings(&request(directory.path(), Some("dev"))).expect("dev"); - assert!(!dev.enabled); - assert_eq!(dev.retention_days, 10); - assert_eq!(dev.shutdown_timeout_ms, 1000); - assert_eq!( - dev.store, - StoreLocation::Sqlite { - path: "/tmp/opencode/elsewhere.sqlite3".into() - } - ); - let prod = resolve_recall_settings(&request(directory.path(), Some("prod"))).expect("prod"); - assert!(prod.enabled); - assert_eq!( - prod.store, - StoreLocation::Unsupported { - connection: "central".to_owned(), - adapter: "clickhouse".to_owned() - } - ); +fn given_project_layers_when_resolved_then_target_over_local_over_project_over_defaults() { + let test_cases = [ + ResolveTestCase { + description: "without a recall section local SQLite is on by default", + files: DEFAULT_PROJECT, + target: None, + expected_settings: RecallSettings { + target: Some("dev".to_owned()), + destination_schema: Some("analytics".to_owned()), + ..defaults() + }, + }, + ResolveTestCase { + description: "the target disables recall and the local file moves the store", + files: LAYERED_PROJECT, + target: Some("dev"), + expected_settings: RecallSettings { + enabled: false, + target: Some("dev".to_owned()), + store: StoreLocation::Sqlite { + path: "/tmp/opencode/elsewhere.sqlite3".into(), + }, + connection: Some("history".to_owned()), + retention_days: 10, + shutdown_timeout_ms: 1000, + ..defaults() + }, + }, + ResolveTestCase { + description: "a non-SQLite connection is reported as unsupported", + files: LAYERED_PROJECT, + target: Some("prod"), + expected_settings: RecallSettings { + target: Some("prod".to_owned()), + store: StoreLocation::Unsupported { + connection: "central".to_owned(), + adapter: "clickhouse".to_owned(), + }, + connection: Some("central".to_owned()), + retention_days: 10, + shutdown_timeout_ms: 1000, + ..defaults() + }, + }, + ]; + + for test_case in test_cases { + assert_eq!( + resolved(test_case.files, test_case.target), + Ok(format!("{:?}", test_case.expected_settings)), + "{}", + test_case.description + ); + } } #[test] fn given_invalid_settings_when_resolved_then_the_error_names_the_setting() { - let unknown = project(&[( - "sqlbuild_project.toml", - "name = \"x\"\nadapter = \"duckdb\"\n[recall]\nconnection = \"missing\"\n", - )]); - let error = resolve_recall_settings(&request(unknown.path(), None)).expect_err("unknown"); - assert!( - error.message.contains("unknown connection 'missing'"), - "{error}" - ); - let bad = project(&[( - "sqlbuild_project.toml", - "name = \"x\"\nadapter = \"duckdb\"\n[targets.dev.recall]\nretention = \"soon\"\n", - )]); - let error = resolve_recall_settings(&request(bad.path(), Some("dev"))).expect_err("bad"); - assert!( - error.message.starts_with("targets.dev.recall.retention"), - "{error}" - ); + let test_cases = [ + ResolveErrorTestCase { + description: "unknown connections are named", + files: &[( + "sqlbuild_project.toml", + "name = \"x\"\nadapter = \"duckdb\"\n[recall]\nconnection = \"missing\"\n", + )], + target: None, + expected_message_fragment: "unknown connection 'missing'", + }, + ResolveErrorTestCase { + description: "target settings are named with their path", + files: &[( + "sqlbuild_project.toml", + "name = \"x\"\nadapter = \"duckdb\"\n[targets.dev.recall]\nretention = \"soon\"\n", + )], + target: Some("dev"), + expected_message_fragment: "targets.dev.recall.retention", + }, + ]; + + for test_case in test_cases { + let message: String = + resolved(test_case.files, test_case.target).expect_err("the settings are invalid"); + assert!( + message.contains(test_case.expected_message_fragment), + "{}: {message}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-recall/src/settings/tests/test_types.rs b/crates/sqlbuild-recall/src/settings/tests/test_types.rs new file mode 100644 index 0000000000..d038a10486 --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/tests/test_types.rs @@ -0,0 +1,15 @@ +use crate::settings::models::RecallSettings; + +pub(crate) struct ResolveTestCase { + pub(crate) description: &'static str, + pub(crate) files: &'static [(&'static str, &'static str)], + pub(crate) target: Option<&'static str>, + pub(crate) expected_settings: RecallSettings, +} + +pub(crate) struct ResolveErrorTestCase { + pub(crate) description: &'static str, + pub(crate) files: &'static [(&'static str, &'static str)], + pub(crate) target: Option<&'static str>, + pub(crate) expected_message_fragment: &'static str, +} diff --git a/crates/sqlbuild-recall/src/store/tests.rs b/crates/sqlbuild-recall/src/store/tests.rs index 807f02a975..5d10c71849 100644 --- a/crates/sqlbuild-recall/src/store/tests.rs +++ b/crates/sqlbuild-recall/src/store/tests.rs @@ -4,5 +4,7 @@ mod helpers; mod maintenance; #[path = "tests/test_store.rs"] mod store; +#[path = "tests/test_types.rs"] +mod test_types; #[path = "tests/test_views.rs"] mod views; diff --git a/crates/sqlbuild-recall/src/store/tests/helpers.rs b/crates/sqlbuild-recall/src/store/tests/helpers.rs index 24a3853e58..6ee7ea78f6 100644 --- a/crates/sqlbuild-recall/src/store/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/store/tests/helpers.rs @@ -1,19 +1,25 @@ -use std::time::{Duration, SystemTime}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tempfile::TempDir; use crate::records::main::event_row::event_row; use crate::records::main::format_timestamp::format_timestamp; use crate::records::main::output_row::output_row; -use crate::records::models::{OutputChunk, OutputStream, RecallRow, RecallValue, RowEnvelope}; +use crate::records::main::parse_timestamp::parse_timestamp; +use crate::records::models::{ + ArtifactInput, DeliveryCounts, OutputChunk, OutputStream, RecallRow, RecallValue, RowEnvelope, +}; +use crate::store::errors::{StoreError, StoreErrorKind}; use crate::store::main::ensure_sqlite_schema::ensure_sqlite_schema; use crate::store::main::insert_rows::insert_rows; +use crate::store::main::maintain_sqlite_store::maintain_sqlite_store; use crate::store::main::open_sqlite_store::open_sqlite_store; use crate::store::main::query_store::query_store; -use crate::store::models::{OpenMode, SqliteStore}; +use crate::store::models::{MaintenanceOutcome, MaintenanceRequest, OpenMode, SqliteStore}; pub(crate) const NOW: &str = "2026-10-19T12:00:00.000000Z"; +/// The envelope of invocation `invocation_id` started at `started_at`. pub(crate) fn envelope(invocation_id: &str, started_at: &str) -> RowEnvelope { RowEnvelope { invocation_id: invocation_id.to_owned(), @@ -22,21 +28,24 @@ pub(crate) fn envelope(invocation_id: &str, started_at: &str) -> RowEnvelope { } } +/// A store with the current schema in a fresh temporary project. pub(crate) fn new_store() -> (TempDir, SqliteStore) { - let directory = tempfile::tempdir().expect("temporary directory"); + let directory = tempfile::tempdir().expect("a temporary directory can be created"); let store = open_sqlite_store( &directory.path().join("logs/recall.sqlite3"), OpenMode::ReadWrite, ) - .expect("store opens"); - ensure_sqlite_schema(&store, &envelope("setup", NOW), NOW).expect("schema"); + .expect("the store opens"); + ensure_sqlite_schema(&store, &envelope("setup", NOW), NOW).expect("the schema can be ensured"); (directory, store) } +/// Insert `rows` at `NOW`. pub(crate) fn write(store: &SqliteStore, rows: &[RecallRow]) { - insert_rows(store, rows, NOW).expect("rows written"); + insert_rows(store, rows, NOW).expect("rows can be inserted"); } +/// One output chunk row. pub(crate) fn chunk( envelope: &RowEnvelope, (sequence, stream, data): (u64, OutputStream, &[u8]), @@ -54,6 +63,15 @@ pub(crate) fn chunk( ) } +/// One `x` stdout chunk per sequence. +pub(crate) fn chunks(envelope: &RowEnvelope, sequences: &[u64]) -> Vec { + sequences + .iter() + .map(|sequence| chunk(envelope, (*sequence, OutputStream::Stdout, b"x"))) + .collect() +} + +/// One lifecycle event row; `correlations` is spliced in as extra JSON members. pub(crate) fn event( envelope: &RowEnvelope, (event_id, event_type, sequence): (&str, &str, i64), @@ -66,33 +84,148 @@ pub(crate) fn event( "invocation_id":"{}","invocation_sequence":{sequence}{correlations},"payload":{payload}}}"#, envelope.invocation_id ); - event_row(envelope, &json).expect("event row") + event_row(envelope, &json).expect("the fixture is a valid lifecycle event") } +/// The result rows of `sql`. pub(crate) fn rows(store: &SqliteStore, sql: &str) -> Vec> { - query_store(store, sql, None).expect("query").rows + query_store(store, sql, None) + .expect("the query is valid") + .rows } +/// The rows `sql` projects after writing each batch in order to a fresh store. +pub(crate) fn projected(batches: &[Vec], sql: &str) -> Vec> { + let (_directory, store) = new_store(); + for batch in batches { + write(&store, batch); + } + rows(&store, sql) +} + +/// A text value. pub(crate) fn text(value: &str) -> RecallValue { RecallValue::Text(value.to_owned()) } +/// An integer value. +pub(crate) fn integer(value: i64) -> RecallValue { + RecallValue::Integer(value) +} + +fn moved(base: SystemTime, seconds: i64) -> String { + let base_seconds: i64 = base.duration_since(UNIX_EPOCH).map_or(0, |elapsed| { + i64::try_from(elapsed.as_secs()).unwrap_or(i64::MAX) + }); + let target: u64 = u64::try_from(base_seconds.saturating_add(seconds)).unwrap_or_default(); + format_timestamp(UNIX_EPOCH + Duration::from_secs(target)) +} + +/// `NOW` moved by `seconds`. pub(crate) fn shifted(seconds: i64) -> String { - let now = crate::records::main::parse_timestamp::parse_timestamp(NOW).expect("now"); - let moved: SystemTime = if seconds >= 0 { - now + Duration::from_secs(seconds.unsigned_abs()) - } else { - now - Duration::from_secs(seconds.unsigned_abs()) - }; - format_timestamp(moved) + moved( + parse_timestamp(NOW).expect("the timestamp is valid"), + seconds, + ) } +/// The real clock moved by `seconds_from_now`. pub(crate) fn wall_clock(seconds_from_now: i64) -> String { - let now = SystemTime::now(); - let moved = if seconds_from_now >= 0 { - now + Duration::from_secs(seconds_from_now.unsigned_abs()) - } else { - now - Duration::from_secs(seconds_from_now.unsigned_abs()) - }; - format_timestamp(moved) + moved(SystemTime::now(), seconds_from_now) +} + +/// Delivery counts with `filtered`, `dropped` and `delivered` rows. +pub(crate) fn delivered(filtered: u64, dropped: u64, delivered: u64) -> DeliveryCounts { + DeliveryCounts { + accepted: delivered + filtered + dropped, + delivered, + filtered, + dropped, + ..DeliveryCounts::default() + } +} + +/// A SQL artifact `digest` referenced by itself and seen at `seen_at`. +pub(crate) fn artifact(digest: &str, seen_at: &str) -> ArtifactInput { + ArtifactInput { + digest: digest.to_owned(), + kind: "sql".to_owned(), + media_type: "text/x-sql".to_owned(), + content: b"select 1".to_vec(), + seen_at: seen_at.to_owned(), + ref_id: digest.to_owned(), + role: "executed_sql".to_owned(), + statement_id: None, + resource_id: None, + } +} + +/// Outcomes of maintenance runs at `(offset seconds, budget seconds)` and the rows `sql` keeps. +pub(crate) fn maintained( + batch: &[RecallRow], + runs: &[(i64, u64)], + sql: &str, +) -> (Vec, Vec>) { + let (_directory, store) = new_store(); + write(&store, batch); + let outcomes: Vec = runs + .iter() + .map(|(offset, budget)| { + let now = shifted(*offset); + let request = MaintenanceRequest { + now: parse_timestamp(&now).expect("the timestamp is valid"), + retention_days: 30, + owner: "host:1".to_owned(), + budget: Duration::from_secs(*budget), + envelope: envelope("maintainer", &now), + }; + maintain_sqlite_store(&store, &request).expect("maintenance succeeds") + }) + .collect(); + (outcomes, rows(&store, sql)) +} + +/// `result` with the error reduced to its kind. +pub(crate) fn error_kind(result: Result) -> Result { + result.map_err(|error| error.kind) +} + +/// A store whose recorded schema version is newer, reopened and ensured by this version. +pub(crate) fn newer_schema_store() -> (TempDir, SqliteStore) { + let (directory, store) = new_store(); + write( + &store, + &[crate::records::main::meta_row::meta_row( + &envelope("newer", NOW), + ("schema", "schema_version", "99"), + NOW, + )], + ); + store + .connection + .execute_batch("DROP VIEW conflicts; CREATE VIEW conflicts AS SELECT 'newer' AS marker") + .expect("the fixture SQL runs"); + drop(store); + let path = directory.path().join("logs/recall.sqlite3"); + let reopened = open_sqlite_store(&path, OpenMode::ReadWrite).expect("the store opens"); + ensure_sqlite_schema(&reopened, &envelope("older", NOW), NOW) + .expect("the schema can be ensured"); + drop(reopened); + let read_only = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); + (directory, read_only) +} + +/// The error kind of opening `relative_path` read-write under a read-only directory. +#[cfg(unix)] +pub(crate) fn open_under_locked_directory(relative_path: &str) -> Result<(), StoreErrorKind> { + use std::os::unix::fs::PermissionsExt; + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let locked = directory.path().join("locked"); + std::fs::create_dir(&locked).expect("the test directory can be created"); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o555)) + .expect("test directory permissions can change"); + let opened = open_sqlite_store(&locked.join(relative_path), OpenMode::ReadWrite); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)) + .expect("test directory permissions can change"); + error_kind(opened.map(drop)) } diff --git a/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs b/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs index b1c3187d79..c5eed7a642 100644 --- a/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs +++ b/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs @@ -1,110 +1,92 @@ -use std::time::Duration; - use crate::records::main::artifact_rows::artifact_rows; use crate::records::main::heartbeat_row::heartbeat_row; use crate::records::main::invocation_row::invocation_row; -use crate::records::main::parse_timestamp::parse_timestamp; -use crate::records::models::{ArtifactInput, InvocationContext, OutputStream, RecallValue}; -use crate::store::main::maintain_sqlite_store::maintain_sqlite_store; -use crate::store::models::{MaintenanceOutcome, MaintenanceRequest}; -use crate::store::tests::helpers::{NOW, chunk, envelope, new_store, rows, shifted, text, write}; +use crate::records::models::{InvocationContext, OutputStream, RecallRow}; +use crate::store::models::MaintenanceOutcome; +use crate::store::tests::helpers::{artifact, chunk, envelope, integer, maintained, shifted, text}; +use crate::store::tests::test_types::MaintenanceTestCase; -fn request(now: &str, budget: Duration) -> MaintenanceRequest { - MaintenanceRequest { - now: parse_timestamp(now).expect("now"), - retention_days: 30, - owner: "host:1".to_owned(), - budget, - envelope: envelope("maintainer", now), - } -} - -fn artifact(digest: &str, seen_at: &str) -> ArtifactInput { - ArtifactInput { - digest: digest.to_owned(), - kind: "sql".to_owned(), - media_type: "text/x-sql".to_owned(), - content: b"select 1".to_vec(), - seen_at: seen_at.to_owned(), - ref_id: digest.to_owned(), - role: "executed_sql".to_owned(), - statement_id: None, - resource_id: None, - } -} +const FORTY_DAYS: i64 = -40 * 86_400; #[test] fn given_expired_recent_and_running_invocations_when_maintained_then_only_expired_rows_go() { - let (_directory, store) = new_store(); - let old = envelope("old", &shifted(-40 * 86_400)); + let old = envelope("old", &shifted(FORTY_DAYS)); let recent = envelope("recent", &shifted(-86_400)); - let old_running = envelope("old_running", &shifted(-40 * 86_400)); - let mut batch = Vec::new(); - for env in [&old, &recent, &old_running] { - batch.push(invocation_row(env, &InvocationContext::default())); - batch.push(chunk(env, (0, OutputStream::Stdout, b"x"))); - } - batch.push(heartbeat_row( + let old_running = envelope("old_running", &shifted(FORTY_DAYS)); + let mut mixed: Vec = [&old, &recent, &old_running] + .into_iter() + .flat_map(|env| { + [ + invocation_row(env, &InvocationContext::default()), + chunk(env, (0, OutputStream::Stdout, b"x")), + ] + }) + .collect(); + mixed.push(heartbeat_row( &old_running, (&shifted(-10), &shifted(80)), (None, None), )); - batch.extend(artifact_rows( + mixed.extend(artifact_rows( &old, - &artifact("sha256:old", &shifted(-40 * 86_400)), + &artifact("sha256:old", &shifted(FORTY_DAYS)), )); - batch.extend(artifact_rows( + mixed.extend(artifact_rows( &recent, - &artifact("sha256:recent", &shifted(-40 * 86_400)), + &artifact("sha256:recent", &shifted(FORTY_DAYS)), )); - write(&store, &batch); - let outcome = - maintain_sqlite_store(&store, &request(NOW, Duration::from_secs(5))).expect("maintenance"); - assert_eq!( - outcome, - MaintenanceOutcome { + let ran = + |deleted_invocations: u64, deleted_artifacts: u64, complete: bool| MaintenanceOutcome { ran: true, - deleted_invocations: 1, - deleted_artifacts: 1, - complete: true - } - ); - assert_eq!( - rows( - &store, - "SELECT invocation_id FROM recall_output ORDER BY invocation_id" - ), - vec![vec![text("old_running")], vec![text("recent")]] - ); - assert_eq!( - rows(&store, "SELECT digest FROM recall_artifacts"), - vec![vec![text("sha256:recent")]] - ); - let again = maintain_sqlite_store(&store, &request(&shifted(3600), Duration::from_secs(5))) - .expect("second maintenance"); - assert!(!again.ran, "a claim within 24 hours stops another run"); - let next_day = - maintain_sqlite_store(&store, &request(&shifted(90_000), Duration::from_secs(5))) - .expect("next day"); - assert!(next_day.ran); -} + deleted_invocations, + deleted_artifacts, + complete, + }; + let test_cases = [ + MaintenanceTestCase { + description: "expired idle invocations go while recent and running ones stay", + batch: mixed.clone(), + runs: &[(0, 5)], + sql: "SELECT invocation_id FROM recall_output ORDER BY invocation_id", + expected_outcomes: vec![ran(1, 1, true)], + expected_rows: vec![vec![text("old_running")], vec![text("recent")]], + }, + MaintenanceTestCase { + description: "artifacts referenced only by expired invocations go", + batch: mixed.clone(), + runs: &[(0, 5)], + sql: "SELECT digest FROM recall_artifacts", + expected_outcomes: vec![ran(1, 1, true)], + expected_rows: vec![vec![text("sha256:recent")]], + }, + MaintenanceTestCase { + description: "a claim stops another run within a day; the next day the stale run expires", + batch: mixed, + runs: &[(0, 5), (3600, 5), (90_000, 5)], + sql: "SELECT COUNT(*) FROM recall_invocations", + expected_outcomes: vec![ + ran(1, 1, true), + MaintenanceOutcome::default(), + ran(1, 0, true), + ], + expected_rows: vec![vec![integer(1)]], + }, + MaintenanceTestCase { + description: "a starved run leaves the next run overdue", + batch: vec![invocation_row(&old, &InvocationContext::default())], + runs: &[(0, 0), (60, 5)], + sql: "SELECT COUNT(*) FROM recall_invocations", + expected_outcomes: vec![ran(0, 0, false), ran(1, 0, true)], + expected_rows: vec![vec![integer(0)]], + }, + ]; -#[test] -fn given_no_budget_when_maintained_then_the_next_run_is_overdue() { - let (_directory, store) = new_store(); - let old = envelope("old", &shifted(-40 * 86_400)); - write( - &store, - &[invocation_row(&old, &InvocationContext::default())], - ); - let starved = maintain_sqlite_store(&store, &request(NOW, Duration::ZERO)).expect("starved"); - assert!(starved.ran && !starved.complete); - let overdue = maintain_sqlite_store(&store, &request(&shifted(60), Duration::from_secs(5))) - .expect("overdue run"); - assert!(overdue.ran && overdue.complete); - assert_eq!(overdue.deleted_invocations, 1); - assert_eq!( - rows(&store, "SELECT COUNT(*) FROM recall_invocations"), - vec![vec![RecallValue::Integer(0)]] - ); + for test_case in test_cases { + assert_eq!( + maintained(&test_case.batch, test_case.runs, test_case.sql), + (test_case.expected_outcomes, test_case.expected_rows), + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-recall/src/store/tests/test_store.rs b/crates/sqlbuild-recall/src/store/tests/test_store.rs index 6a93e16f9f..ed01276652 100644 --- a/crates/sqlbuild-recall/src/store/tests/test_store.rs +++ b/crates/sqlbuild-recall/src/store/tests/test_store.rs @@ -1,15 +1,20 @@ use crate::records::main::invocation_row::invocation_row; -use crate::records::main::meta_row::meta_row; use crate::records::models::{InvocationContext, OutputStream, RecallValue}; use crate::store::errors::StoreErrorKind; use crate::store::main::describe_store::describe_store; -use crate::store::main::ensure_sqlite_schema::ensure_sqlite_schema; -use crate::store::main::open_sqlite_store::open_sqlite_store; use crate::store::main::query_store::query_store; use crate::store::main::read_output_transcript::read_output_transcript; use crate::store::main::resolve_invocation::resolve_invocation; -use crate::store::models::OpenMode; -use crate::store::tests::helpers::{NOW, chunk, envelope, new_store, rows, shifted, write}; +#[cfg(unix)] +use crate::store::tests::helpers::open_under_locked_directory; +use crate::store::tests::helpers::{ + NOW, chunk, envelope, error_kind, new_store, newer_schema_store, rows, shifted, write, +}; +#[cfg(unix)] +use crate::store::tests::test_types::LockedDirectoryTestCase; +use crate::store::tests::test_types::{ + NewerSchemaTestCase, QueryTestCase, ResolveInvocationTestCase, TranscriptTestCase, +}; #[test] fn given_ids_when_resolved_then_prefixes_last_and_ambiguity_are_exact() { @@ -27,16 +32,37 @@ fn given_ids_when_resolved_then_prefixes_last_and_ambiguity_are_exact() { ), ], ); - assert_eq!(resolve_invocation(&store, "abc").as_deref(), Ok("abc123")); - assert_eq!(resolve_invocation(&store, "last").as_deref(), Ok("abd456")); - assert_eq!( - resolve_invocation(&store, "ab").map_err(|error| error.kind), - Err(StoreErrorKind::Ambiguous) - ); - assert_eq!( - resolve_invocation(&store, "zz").map_err(|error| error.kind), - Err(StoreErrorKind::NotFound) - ); + let test_cases = [ + ResolveInvocationTestCase { + description: "a unique prefix resolves", + selector: "abc", + expected_invocation: Ok("abc123"), + }, + ResolveInvocationTestCase { + description: "last is the most recently started invocation", + selector: "last", + expected_invocation: Ok("abd456"), + }, + ResolveInvocationTestCase { + description: "a shared prefix is ambiguous", + selector: "ab", + expected_invocation: Err(StoreErrorKind::Ambiguous), + }, + ResolveInvocationTestCase { + description: "an unknown prefix is not found", + selector: "zz", + expected_invocation: Err(StoreErrorKind::NotFound), + }, + ]; + + for test_case in test_cases { + assert_eq!( + error_kind(resolve_invocation(&store, test_case.selector)), + test_case.expected_invocation.map(str::to_owned), + "{}", + test_case.description + ); + } } #[test] @@ -49,86 +75,137 @@ fn given_queries_when_run_then_writes_are_rejected_and_truncation_reported() { &[chunk(&env, (sequence, OutputStream::Stdout, b"line\n"))], ); } - let limited = query_store( - &store, - "SELECT sequence FROM recall_output ORDER BY sequence", - Some(2), - ) - .expect("query"); - assert_eq!(limited.rows.len(), 2); - assert!(limited.truncated); - assert_eq!(limited.columns, vec!["sequence".to_owned()]); - let rejected = - query_store(&store, "DELETE FROM recall_output", None).expect_err("write rejected"); - assert_eq!(rejected.kind, StoreErrorKind::Query); - let invalid = query_store(&store, "SELEC nope", None).expect_err("invalid sql"); - assert_eq!(invalid.kind, StoreErrorKind::Query); - let transcript = read_output_transcript(&store, "q", None).expect("transcript"); - assert_eq!(transcript.data, b"line\nline\nline\n".to_vec()); - assert!(!transcript.complete); + let test_cases = [ + QueryTestCase { + description: "a limit truncates and is reported", + sql: "SELECT sequence FROM recall_output ORDER BY sequence", + limit: Some(2), + expected_shape: Ok((vec!["sequence".to_owned()], 2, true)), + }, + QueryTestCase { + description: "without a limit every row returns", + sql: "SELECT sequence FROM recall_output ORDER BY sequence", + limit: None, + expected_shape: Ok((vec!["sequence".to_owned()], 3, false)), + }, + QueryTestCase { + description: "writes are rejected", + sql: "DELETE FROM recall_output", + limit: None, + expected_shape: Err(StoreErrorKind::Query), + }, + QueryTestCase { + description: "invalid SQL is a query error", + sql: "SELEC nope", + limit: None, + expected_shape: Err(StoreErrorKind::Query), + }, + ]; + + for test_case in test_cases { + let shape = error_kind(query_store(&store, test_case.sql, test_case.limit)) + .map(|result| (result.columns, result.rows.len(), result.truncated)); + assert_eq!(shape, test_case.expected_shape, "{}", test_case.description); + } } #[test] -fn given_a_newer_schema_version_when_ensured_then_views_are_kept() { - let (directory, store) = new_store(); - write( - &store, - &[meta_row( - &envelope("newer", NOW), - ("schema", "schema_version", "99"), - NOW, - )], - ); - store - .connection - .execute_batch("DROP VIEW conflicts; CREATE VIEW conflicts AS SELECT 'newer' AS marker") - .expect("newer view"); - drop(store); - let path = directory.path().join("logs/recall.sqlite3"); - let reopened = open_sqlite_store(&path, OpenMode::ReadWrite).expect("reopen"); - ensure_sqlite_schema(&reopened, &envelope("older", NOW), NOW).expect("ensure"); - assert_eq!( - rows(&reopened, "SELECT marker FROM conflicts"), - vec![vec![RecallValue::Text("newer".to_owned())]] - ); - let relations = describe_store(&reopened).expect("describe"); - assert_eq!(relations[0].kind, "view"); - assert!(relations.iter().any(|relation| { - relation.name == "recall_output" - && relation - .columns - .iter() - .any(|column| column.name == "data" && !column.nullable) - })); - let read_only = open_sqlite_store(&path, OpenMode::ReadOnly).expect("read-only open"); - assert!(query_store(&read_only, "SELECT * FROM invocations", None).is_ok()); +fn given_output_without_delivery_facts_when_read_then_bytes_return_and_completeness_is_unproven() { + let (_directory, store) = new_store(); + let env = envelope("q", NOW); + for sequence in 0..3 { + write( + &store, + &[chunk(&env, (sequence, OutputStream::Stdout, b"line\n"))], + ); + } + let test_cases = [TranscriptTestCase { + description: "chunks concatenate in sequence order", + invocation_id: "q", + expected_transcript: Ok((b"line\nline\nline\n".to_vec(), false)), + }]; + + for test_case in test_cases { + let transcript = error_kind(read_output_transcript( + &store, + test_case.invocation_id, + None, + )) + .map(|transcript| (transcript.data, transcript.complete)); + assert_eq!( + transcript, test_case.expected_transcript, + "{}", + test_case.description + ); + } } -#[cfg(unix)] #[test] -fn given_a_read_only_directory_when_opened_then_the_error_is_a_permission_failure() { - use std::os::unix::fs::PermissionsExt; - let directory = tempfile::tempdir().expect("temporary directory"); - let locked = directory.path().join("locked"); - std::fs::create_dir(&locked).expect("directory"); - std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o555)).expect("chmod"); - let result = open_sqlite_store(&locked.join("logs/recall.sqlite3"), OpenMode::ReadWrite); - let direct = open_sqlite_store(&locked.join("recall.sqlite3"), OpenMode::ReadWrite); - std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)).expect("restore"); - if nix_is_root() { - return; +fn given_a_newer_schema_version_when_ensured_then_views_are_kept_and_read_only_opens_work() { + let (_directory, store) = newer_schema_store(); + let relations = describe_store(&store).expect("the store can be described"); + let data_nullable: Option = relations + .iter() + .find(|relation| relation.name == "recall_output") + .and_then(|relation| relation.columns.iter().find(|column| column.name == "data")) + .map(|column| column.nullable); + let test_cases = [ + NewerSchemaTestCase { + description: "a newer view definition is not replaced", + sql: "SELECT marker FROM conflicts", + expected_rows: vec![vec![RecallValue::Text("newer".to_owned())]], + expected_first_kind: "view", + expected_data_nullable: Some(false), + }, + NewerSchemaTestCase { + description: "the read-only store answers view queries", + sql: "SELECT COUNT(*) FROM invocations", + expected_rows: vec![vec![RecallValue::Integer(0)]], + expected_first_kind: "view", + expected_data_nullable: Some(false), + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + rows(&store, test_case.sql), + relations[0].kind.as_str(), + data_nullable + ), + ( + test_case.expected_rows, + test_case.expected_first_kind, + test_case.expected_data_nullable + ), + "{}", + test_case.description + ); } - assert_eq!( - result.map(|_| ()).map_err(|error| error.kind), - Err(StoreErrorKind::Permission) - ); - assert_eq!( - direct.map(|_| ()).map_err(|error| error.kind), - Err(StoreErrorKind::Permission) - ); } #[cfg(unix)] -fn nix_is_root() -> bool { - std::env::var("USER").is_ok_and(|user| user == "root") +#[test] +fn given_a_read_only_directory_when_opened_then_the_error_is_a_permission_failure() { + let test_cases = [ + LockedDirectoryTestCase { + description: "a missing logs directory cannot be created", + relative_path: "logs/recall.sqlite3", + expected_kind: Err(StoreErrorKind::Permission), + }, + LockedDirectoryTestCase { + description: "the database file cannot be created", + relative_path: "recall.sqlite3", + expected_kind: Err(StoreErrorKind::Permission), + }, + ]; + + for test_case in test_cases { + assert_eq!( + open_under_locked_directory(test_case.relative_path), + test_case.expected_kind, + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-recall/src/store/tests/test_types.rs b/crates/sqlbuild-recall/src/store/tests/test_types.rs new file mode 100644 index 0000000000..7090695f6a --- /dev/null +++ b/crates/sqlbuild-recall/src/store/tests/test_types.rs @@ -0,0 +1,52 @@ +use crate::records::models::{RecallRow, RecallValue}; +use crate::store::errors::StoreErrorKind; +use crate::store::models::MaintenanceOutcome; + +pub(crate) struct ProjectionTestCase { + pub(crate) description: &'static str, + pub(crate) batches: Vec>, + pub(crate) sql: &'static str, + pub(crate) expected_rows: Vec>, +} + +pub(crate) struct ResolveInvocationTestCase { + pub(crate) description: &'static str, + pub(crate) selector: &'static str, + pub(crate) expected_invocation: Result<&'static str, StoreErrorKind>, +} + +pub(crate) struct QueryTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) limit: Option, + pub(crate) expected_shape: Result<(Vec, usize, bool), StoreErrorKind>, +} + +pub(crate) struct TranscriptTestCase { + pub(crate) description: &'static str, + pub(crate) invocation_id: &'static str, + pub(crate) expected_transcript: Result<(Vec, bool), StoreErrorKind>, +} + +pub(crate) struct NewerSchemaTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_rows: Vec>, + pub(crate) expected_first_kind: &'static str, + pub(crate) expected_data_nullable: Option, +} + +pub(crate) struct LockedDirectoryTestCase { + pub(crate) description: &'static str, + pub(crate) relative_path: &'static str, + pub(crate) expected_kind: Result<(), StoreErrorKind>, +} + +pub(crate) struct MaintenanceTestCase { + pub(crate) description: &'static str, + pub(crate) batch: Vec, + pub(crate) runs: &'static [(i64, u64)], + pub(crate) sql: &'static str, + pub(crate) expected_outcomes: Vec, + pub(crate) expected_rows: Vec>, +} diff --git a/crates/sqlbuild-recall/src/store/tests/test_views.rs b/crates/sqlbuild-recall/src/store/tests/test_views.rs index b0dd05a34b..a6eee47867 100644 --- a/crates/sqlbuild-recall/src/store/tests/test_views.rs +++ b/crates/sqlbuild-recall/src/store/tests/test_views.rs @@ -2,168 +2,181 @@ use crate::records::main::artifact_rows::artifact_rows; use crate::records::main::delivery_row::delivery_row; use crate::records::main::heartbeat_row::heartbeat_row; use crate::records::main::invocation_row::invocation_row; -use crate::records::models::{ - ArtifactInput, DeliveryCounts, InvocationContext, OutputStream, RecallValue, -}; +use crate::records::main::sha256_hex::sha256_hex; +use crate::records::models::{ArtifactInput, InvocationContext, OutputStream, RecallValue}; use crate::store::tests::helpers::{ - NOW, chunk, envelope, event, new_store, rows, text, wall_clock, write, + NOW, chunk, chunks, delivered, envelope, event, integer, projected, text, wall_clock, }; +use crate::store::tests::test_types::ProjectionTestCase; -fn delivered(filtered: u64, dropped: u64, delivered: u64) -> DeliveryCounts { - DeliveryCounts { - accepted: delivered + filtered + dropped, - delivered, - filtered, - dropped, - ..DeliveryCounts::default() - } -} +const BINARY_PIECES: [(OutputStream, &[u8]); 4] = [ + (OutputStream::Stdout, b"line one\n\x00\xff"), + (OutputStream::Stderr, b"warn \xe2\x9c\x93\n"), + (OutputStream::Stdout, b"\xc3\x28 tail"), + (OutputStream::Stderr, b"\x00end"), +]; + +const STATEMENT_SQL: &str = "select 1"; #[test] fn given_output_with_and_without_gaps_when_projected_then_completeness_is_exact() { - let (_directory, store) = new_store(); let complete = envelope("complete", NOW); let gap = envelope("gap", NOW); let filtered = envelope("filtered", NOW); let undelivered = envelope("no_delivery", NOW); - for env in [&complete, &gap, &filtered, &undelivered] { - let sequences: &[u64] = - if env.invocation_id == "complete" || env.invocation_id == "no_delivery" { - &[0, 1, 2] - } else { - &[0, 2] - }; - for sequence in sequences { - write( - &store, - &[chunk(env, (*sequence, OutputStream::Stdout, b"x"))], - ); - } - } - write( - &store, - &[ - delivery_row(&complete, "output", (&delivered(0, 0, 3), true), NOW), - delivery_row(&gap, "output", (&delivered(0, 0, 2), true), NOW), - delivery_row(&filtered, "output", (&delivered(1, 0, 2), true), NOW), - ], - ); - let projected = rows( - &store, - "SELECT invocation_id, chunks, missing_sequences, filtered, gap_reason, complete \ - FROM output_invocations ORDER BY invocation_id", - ); - assert_eq!( - projected, - vec![ + let mut output = chunks(&complete, &[0, 1, 2]); + output.extend(chunks(&gap, &[0, 2])); + output.extend(chunks(&filtered, &[0, 2])); + output.extend(chunks(&undelivered, &[0, 1, 2])); + let deliveries = vec![ + delivery_row(&complete, "output", (&delivered(0, 0, 3), true), NOW), + delivery_row(&gap, "output", (&delivered(0, 0, 2), true), NOW), + delivery_row(&filtered, "output", (&delivered(1, 0, 2), true), NOW), + ]; + let test_cases = [ProjectionTestCase { + description: "only gap-free delivered output is complete and gaps are explained", + batches: vec![output, deliveries], + sql: "SELECT invocation_id, chunks, missing_sequences, filtered, gap_reason, complete \ + FROM output_invocations ORDER BY invocation_id", + expected_rows: vec![ vec![ text("complete"), - RecallValue::Integer(3), - RecallValue::Integer(0), - RecallValue::Integer(0), + integer(3), + integer(0), + integer(0), RecallValue::Null, - RecallValue::Integer(1) + integer(1), ], vec![ text("filtered"), - RecallValue::Integer(2), - RecallValue::Integer(1), - RecallValue::Integer(1), + integer(2), + integer(1), + integer(1), text("filtered"), - RecallValue::Integer(0) + integer(0), ], vec![ text("gap"), - RecallValue::Integer(2), - RecallValue::Integer(1), - RecallValue::Integer(0), + integer(2), + integer(1), + integer(0), text("unexplained"), - RecallValue::Integer(0) + integer(0), ], vec![ text("no_delivery"), - RecallValue::Integer(3), - RecallValue::Integer(0), - RecallValue::Integer(0), + integer(3), + integer(0), + integer(0), RecallValue::Null, - RecallValue::Integer(0) + integer(0), ], - ] - ); + ], + }]; + + for test_case in test_cases { + assert_eq!( + projected(&test_case.batches, test_case.sql), + test_case.expected_rows, + "{}", + test_case.description + ); + } } #[test] fn given_binary_interleaved_output_when_reconstructed_then_transcripts_are_byte_exact() { - let (_directory, store) = new_store(); let env = envelope("binary", NOW); - let pieces: [(OutputStream, &[u8]); 4] = [ - (OutputStream::Stdout, b"line one\n\x00\xff"), - (OutputStream::Stderr, b"warn \xe2\x9c\x93\n"), - (OutputStream::Stdout, b"\xc3\x28 tail"), - (OutputStream::Stderr, b"\x00end"), + let mut reversed: Vec> = (0_u64..) + .zip(BINARY_PIECES) + .map(|(sequence, (stream, data))| vec![chunk(&env, (sequence, stream, data))]) + .collect(); + reversed.reverse(); + let stdout: Vec = [BINARY_PIECES[0].1, BINARY_PIECES[2].1].concat(); + let stderr: Vec = [BINARY_PIECES[1].1, BINARY_PIECES[3].1].concat(); + let combined: Vec = BINARY_PIECES + .iter() + .flat_map(|(_, data)| data.iter().copied()) + .collect(); + let test_cases = [ + ProjectionTestCase { + description: "out-of-order writes reconstruct each stream and the combined order", + batches: reversed.clone(), + sql: "SELECT stdout, stderr, combined FROM output_transcripts \ + WHERE invocation_id = 'binary'", + expected_rows: vec![vec![ + RecallValue::Blob(stdout), + RecallValue::Blob(stderr), + RecallValue::Blob(combined), + ]], + }, + ProjectionTestCase { + description: "the tail lists every chunk", + batches: reversed, + sql: "SELECT COUNT(*) FROM output_tail WHERE invocation_id = 'binary'", + expected_rows: vec![vec![integer(4)]], + }, ]; - // Written out of order: the view orders by sequence, not by insertion. - for (sequence, (stream, data)) in pieces.iter().enumerate().rev() { - write(&store, &[chunk(&env, (sequence as u64, *stream, data))]); + + for test_case in test_cases { + assert_eq!( + projected(&test_case.batches, test_case.sql), + test_case.expected_rows, + "{}", + test_case.description + ); } - let transcript = rows( - &store, - "SELECT stdout, stderr, combined FROM output_transcripts WHERE invocation_id = 'binary'", - ); - let stdout: Vec = [pieces[0].1, pieces[2].1].concat(); - let stderr: Vec = [pieces[1].1, pieces[3].1].concat(); - let combined: Vec = pieces.iter().flat_map(|(_, data)| data.to_vec()).collect(); - assert_eq!( - transcript, - vec![vec![ - RecallValue::Blob(stdout), - RecallValue::Blob(stderr), - RecallValue::Blob(combined) - ]] - ); - let tail = rows( - &store, - "SELECT COUNT(*) FROM output_tail WHERE invocation_id = 'binary'", - ); - assert_eq!(tail, vec![vec![RecallValue::Integer(4)]]); } #[test] fn given_duplicates_and_conflicts_when_written_then_first_row_is_kept_and_conflicts_reported() { - let (_directory, store) = new_store(); let env = envelope("dup", NOW); let original = chunk(&env, (0, OutputStream::Stdout, b"first")); let conflicting = chunk(&env, (0, OutputStream::Stdout, b"second")); - write(&store, std::slice::from_ref(&original)); - let duplicate = crate::store::main::insert_rows::insert_rows(&store, &[original], NOW) - .expect("duplicate is a no-op"); - assert!(duplicate.conflicts.is_empty()); - let conflict = crate::store::main::insert_rows::insert_rows(&store, &[conflicting], NOW) - .expect("conflict does not fail"); - assert_eq!(conflict.conflicts.len(), 1); - assert_eq!(conflict.conflicts[0].record_key, "dup:0"); - assert_eq!( - rows(&store, "SELECT data FROM recall_output"), - vec![vec![RecallValue::Blob(b"first".to_vec())]] - ); - assert_eq!( - rows( - &store, - "SELECT invocation_id, dataset, record_key FROM conflicts" - ), - vec![vec![text("dup"), text("recall_output"), text("dup:0")]] - ); + let writes = vec![ + vec![original.clone()], + vec![original.clone()], + vec![conflicting.clone()], + ]; + let test_cases = [ + ProjectionTestCase { + description: "the first row wins", + batches: writes.clone(), + sql: "SELECT data FROM recall_output", + expected_rows: vec![vec![RecallValue::Blob(b"first".to_vec())]], + }, + ProjectionTestCase { + description: "only the differing rewrite is a conflict", + batches: writes, + sql: "SELECT invocation_id, dataset, record_key FROM conflicts", + expected_rows: vec![vec![text("dup"), text("recall_output"), text("dup:0")]], + }, + ProjectionTestCase { + description: "identical rewrites are not conflicts", + batches: vec![vec![original.clone()], vec![original]], + sql: "SELECT COUNT(*) FROM conflicts", + expected_rows: vec![vec![integer(0)]], + }, + ]; + + for test_case in test_cases { + assert_eq!( + projected(&test_case.batches, test_case.sql), + test_case.expected_rows, + "{}", + test_case.description + ); + } } #[test] fn given_terminals_and_heartbeats_when_projected_then_status_never_claims_false_success() { - let (_directory, store) = new_store(); let done = envelope("done", NOW); let failed = envelope("failed_late", NOW); let running = envelope("running", NOW); let abandoned = envelope("abandoned", "2026-10-01T12:00:00.000000Z"); let unknown = envelope("unknown", NOW); - let mut batch = vec![ + let facts = vec![ event( &done, ("d0", "invocation_started", 0), @@ -207,26 +220,23 @@ fn given_terminals_and_heartbeats_when_projected_then_status_never_claims_false_ (&wall_clock(-1), &wall_clock(90)), (None, Some(1)), ), + heartbeat_row( + &abandoned, + ("2026-10-01T12:00:30.000000Z", "2026-10-01T12:02:00.000000Z"), + (None, None), + ), + heartbeat_row( + &abandoned, + ("2026-10-01T12:00:00.000000Z", "2026-10-01T12:01:30.000000Z"), + (None, None), + ), ]; - batch.push(heartbeat_row( - &abandoned, - ("2026-10-01T12:00:30.000000Z", "2026-10-01T12:02:00.000000Z"), - (None, None), - )); - batch.push(heartbeat_row( - &abandoned, - ("2026-10-01T12:00:00.000000Z", "2026-10-01T12:01:30.000000Z"), - (None, None), - )); - write(&store, &batch); - let projected = rows( - &store, - "SELECT invocation_id, status, ended_at, duration_ms, exit_code, command, target \ - FROM invocations ORDER BY invocation_id", - ); - assert_eq!( - projected, - vec![ + let test_cases = [ProjectionTestCase { + description: "failures outrank completions and stale heartbeats are abandoned", + batches: vec![facts], + sql: "SELECT invocation_id, status, ended_at, duration_ms, exit_code, command, target \ + FROM invocations ORDER BY invocation_id", + expected_rows: vec![ vec![ text("abandoned"), text("abandoned"), @@ -234,25 +244,25 @@ fn given_terminals_and_heartbeats_when_projected_then_status_never_claims_false_ RecallValue::Real(30000.0), RecallValue::Null, RecallValue::Null, - RecallValue::Null + RecallValue::Null, ], vec![ text("done"), text("completed"), text(NOW), RecallValue::Real(12.5), - RecallValue::Integer(0), + integer(0), text("build"), - RecallValue::Null + RecallValue::Null, ], vec![ text("failed_late"), text("failed"), text(NOW), RecallValue::Null, - RecallValue::Integer(1), + integer(1), + RecallValue::Null, RecallValue::Null, - RecallValue::Null ], vec![ text("running"), @@ -261,7 +271,7 @@ fn given_terminals_and_heartbeats_when_projected_then_status_never_claims_false_ RecallValue::Null, RecallValue::Null, text("build"), - text("dev") + text("dev"), ], vec![ text("unknown"), @@ -270,25 +280,32 @@ fn given_terminals_and_heartbeats_when_projected_then_status_never_claims_false_ RecallValue::Null, RecallValue::Null, text("run"), - RecallValue::Null + RecallValue::Null, ], - ] - ); + ], + }]; + + for test_case in test_cases { + assert_eq!( + projected(&test_case.batches, test_case.sql), + test_case.expected_rows, + "{}", + test_case.description + ); + } } #[test] fn given_statement_facts_when_projected_then_failures_and_missing_terminals_stay_visible() { - let (_directory, store) = new_store(); let env = envelope("stmts", NOW); - let sql = "select 1"; - let digest = crate::records::main::sha256_hex::sha256_hex(sql.as_bytes()); + let digest = sha256_hex(STATEMENT_SQL.as_bytes()); let payload = format!( r#"{{"intent":"execute","statement_kind":"SELECT","purpose":"state_write","sql_digest":"{digest}"}}"# ); let failed_payload = format!( r#"{{"intent":"execute","statement_kind":"SELECT","sql_digest":"{digest}","error_type":"DatabaseError","error_code":"42P01","query_id":"q9"}}"# ); - let mut batch = vec![ + let mut facts = vec![ event( &env, ("r0", "resource_attempt_started", 0), @@ -314,13 +331,13 @@ fn given_statement_facts_when_projected_then_failures_and_missing_terminals_stay r#"{"intent":"query","statement_kind":"SELECT"}"#, ), ]; - batch.extend(artifact_rows( + facts.extend(artifact_rows( &env, &ArtifactInput { digest: format!("sha256:{digest}"), kind: "sql".to_owned(), media_type: "text/x-sql".to_owned(), - content: sql.as_bytes().to_vec(), + content: STATEMENT_SQL.as_bytes().to_vec(), seen_at: NOW.to_owned(), ref_id: "s1".to_owned(), role: "executed_sql".to_owned(), @@ -328,65 +345,79 @@ fn given_statement_facts_when_projected_then_failures_and_missing_terminals_stay resource_id: Some("orders".to_owned()), }, )); - write(&store, &batch); - let projected = rows( - &store, - "SELECT statement_id, resource_name, purpose, outcome, error_code, query_id, sql_available \ - FROM statements ORDER BY statement_id", - ); - assert_eq!( - projected, - vec![ - vec![ - text("s1"), - text("orders"), - text("state_write"), - text("failed"), - text("42P01"), - text("q9"), - RecallValue::Integer(1) - ], - vec![ - text("s2"), - RecallValue::Null, - RecallValue::Null, - text("no_terminal"), - RecallValue::Null, - RecallValue::Null, - RecallValue::Integer(0) + let test_cases = [ + ProjectionTestCase { + description: "statements join resources, terminals and their SQL", + batches: vec![facts.clone()], + sql: "SELECT statement_id, resource_name, purpose, outcome, error_code, query_id, \ + sql_available FROM statements ORDER BY statement_id", + expected_rows: vec![ + vec![ + text("s1"), + text("orders"), + text("state_write"), + text("failed"), + text("42P01"), + text("q9"), + integer(1), + ], + vec![ + text("s2"), + RecallValue::Null, + RecallValue::Null, + text("no_terminal"), + RecallValue::Null, + RecallValue::Null, + integer(0), + ], ], - ] - ); - assert_eq!( - rows(&store, "SELECT available, text FROM artifacts"), - vec![vec![RecallValue::Integer(1), text(sql)]] - ); + }, + ProjectionTestCase { + description: "artifacts decode their SQL text", + batches: vec![facts], + sql: "SELECT available, text FROM artifacts", + expected_rows: vec![vec![integer(1), text(STATEMENT_SQL)]], + }, + ]; + + for test_case in test_cases { + assert_eq!( + projected(&test_case.batches, test_case.sql), + test_case.expected_rows, + "{}", + test_case.description + ); + } } #[test] fn given_audit_completed_facts_when_projected_then_audit_results_carry_outcome_and_target() { - let (_directory, store) = new_store(); let env = envelope("audits", NOW); - write( - &store, - &[event( + let test_cases = [ProjectionTestCase { + description: "audit outcomes name the audited relation and column", + batches: vec![vec![event( &env, ("a0", "audit_completed", 1), r#","run_id":"run""#, r#"{"audit_name":"revenue_positive","audit_definition_name":"positive","evaluation_mode":"violations","outcome":"warn","severity":"warn","run_scope_phase":"model","attachment_kind":"column","target_schema":"analytics","target_name":"daily_revenue","attached_column_name":"total_revenue_cents","violation_count":3,"sql_digest":"ab"}"#, - )], - ); - assert_eq!( - rows( - &store, - "SELECT audit_name, target_relation, \"column\", outcome, violation_count FROM audit_results" - ), - vec![vec![ + )]], + sql: "SELECT audit_name, target_relation, \"column\", outcome, violation_count \ + FROM audit_results", + expected_rows: vec![vec![ text("revenue_positive"), text("analytics.daily_revenue"), text("total_revenue_cents"), text("warn"), - RecallValue::Integer(3) - ]] - ); + integer(3), + ]], + }]; + + for test_case in test_cases { + assert_eq!( + projected(&test_case.batches, test_case.sql), + test_case.expected_rows, + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-recall/src/writer/tests.rs b/crates/sqlbuild-recall/src/writer/tests.rs index d063ec20fb..81c66254ff 100644 --- a/crates/sqlbuild-recall/src/writer/tests.rs +++ b/crates/sqlbuild-recall/src/writer/tests.rs @@ -1,4 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_queue.rs"] mod queue; #[path = "tests/test_session.rs"] mod session; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs new file mode 100644 index 0000000000..4f7e1d9899 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -0,0 +1,164 @@ +use std::path::PathBuf; +use std::time::{Duration, SystemTime}; + +use crate::records::main::invocation_row::invocation_row; +use crate::records::models::{InvocationContext, OutputStream, RecallValue, RowEnvelope}; +use crate::store::main::open_sqlite_store::open_sqlite_store; +use crate::store::main::query_store::query_store; +use crate::store::main::read_output_transcript::read_output_transcript; +use crate::store::models::OpenMode; +use crate::writer::_helpers::queue::WriterState; +use crate::writer::main::start_recall_session::start_recall_session; +use crate::writer::models::{DeliveryStream, Priority, SessionConfig, SetupOutcome}; +use crate::writer::tests::test_types::RecordedSession; + +/// The envelope of invocation `invocation_id`. +pub(crate) fn envelope(invocation_id: &str) -> RowEnvelope { + RowEnvelope { + invocation_id: invocation_id.to_owned(), + invocation_started_at: "2026-10-19T00:00:00.000000Z".to_owned(), + producer_version: "test".to_owned(), + } +} + +/// Dropped counts per stream and the streams taken after pushing `pushes` into `capacity`. +pub(crate) fn pushed( + capacity: usize, + pushes: &[(DeliveryStream, Priority)], +) -> (Vec<(DeliveryStream, u64)>, Vec) { + let mut state = WriterState::with_capacity(envelope("inv"), (capacity, usize::MAX)); + for (stream, priority) in pushes { + state.push( + vec![invocation_row( + &envelope("inv"), + &InvocationContext::default(), + )], + *stream, + *priority, + ); + } + let mut streams: Vec = pushes.iter().map(|(stream, _)| *stream).collect(); + streams.dedup(); + let dropped: Vec<(DeliveryStream, u64)> = streams + .iter() + .map(|stream| (*stream, state.counts[stream.index()].dropped)) + .collect(); + let taken: Vec = state + .take_batch(usize::MAX) + .iter() + .map(|record| record.stream) + .collect(); + (dropped, taken) +} + +/// `(sequence, stream, byte_offset)` of each output chunk after `writes` and a flush. +pub(crate) fn chunked(writes: &[(OutputStream, Vec)]) -> Vec<[Option; 3]> { + let mut state = WriterState::new(envelope("inv")); + for (stream, data) in writes { + state.append_output(*stream, data, SystemTime::now()); + } + state.flush_output(); + state + .take_batch(usize::MAX) + .into_iter() + .flat_map(|record| record.rows) + .map(|row| ["sequence", "stream", "byte_offset"].map(|name| row.value(name).cloned())) + .collect() +} + +/// A session configuration writing invocation `invocation_id` to `path`. +pub(crate) fn config(path: PathBuf, invocation_id: &str) -> SessionConfig { + SessionConfig { + store_path: path, + envelope: envelope(invocation_id), + context: InvocationContext { + command: Some("build".to_owned()), + pid: Some(42), + ..InvocationContext::default() + }, + retention_days: 30, + shutdown_timeout: Duration::from_secs(5), + diagnostics_min_severity: "info".to_owned(), + heartbeat_interval: Duration::from_secs(30), + setup_budget: Duration::from_secs(2), + maintenance: true, + maintenance_owner: "test:42".to_owned(), + } +} + +/// Run a session that writes `output`, one completed event, two diagnostics and one SQL twice. +pub(crate) fn recorded_session(output: &[(OutputStream, Vec)]) -> RecordedSession { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("logs/recall.sqlite3"); + let session = start_recall_session(config(path.clone(), "inv-1")); + let setup = session.wait_for_setup(Duration::from_secs(10)); + for (stream, data) in output { + session.append_output(*stream, data); + } + session + .publish_event( + r#"{"event_id":"e1","event_type":"invocation_completed","schema_version":2, + "producer":"sqlbuild","producer_version":"1","occurred_at":"2026-10-19T00:00:01.000000Z", + "invocation_id":"inv-1","invocation_sequence":1,"payload":{"command":"build","exit_code":0,"duration_ms":3.0}}"#, + ) + .expect("the event is valid"); + for severity in ["debug", "warning"] { + session + .publish_diagnostic(&format!( + r#"{{"occurred_at":"2026-10-19T00:00:00.500000Z","severity":"{severity}","logger":"l","source":"python","message":"m"}}"# + )) + .expect("the diagnostic is valid"); + } + session.publish_sql("select 1", Some("s1"), None); + session.publish_sql("select 1", Some("s2"), None); + let summary = session.close(); + let total = summary.total(); + let output_fully_delivered: bool = summary + .streams + .iter() + .find(|(stream, _)| *stream == DeliveryStream::Output) + .is_some_and(|(_, counts)| counts.delivered == counts.accepted); + let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); + let transcript = + read_output_transcript(&store, "inv-1", None).expect("the transcript can be read"); + let stdout = read_output_transcript(&store, "inv-1", Some(OutputStream::Stdout)) + .expect("the transcript can be read"); + let facts = query_store( + &store, + "SELECT (SELECT status FROM invocations), (SELECT COUNT(*) FROM recall_diagnostics), \ + (SELECT COUNT(*) FROM recall_artifacts), (SELECT COUNT(*) FROM recall_heartbeats), \ + (SELECT COUNT(*) FROM recall_delivery), (SELECT command FROM recall_invocations)", + None, + ) + .expect("the query is valid"); + RecordedSession { + setup, + lost: (total.dropped, total.failed), + output_fully_delivered, + transcript: (transcript.data, transcript.complete), + stdout: stdout.data, + facts: facts.rows, + } +} + +/// Whether a store under a read-only directory reports permission at setup and degrades. +#[cfg(unix)] +pub(crate) fn unwritable_session() -> (bool, bool) { + use std::os::unix::fs::PermissionsExt; + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o555)) + .expect("test directory permissions can change"); + let session = start_recall_session(config( + directory.path().join("logs/recall.sqlite3"), + "inv-2", + )); + let setup = session.wait_for_setup(Duration::from_secs(10)); + session.append_output(OutputStream::Stdout, b"still printed\n"); + let summary = session.close(); + std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o755)) + .expect("test directory permissions can change"); + ( + matches!(setup, Some(SetupOutcome::PermissionDenied { .. })), + summary.degraded_reason.is_some(), + ) +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_queue.rs b/crates/sqlbuild-recall/src/writer/tests/test_queue.rs index b643a4d365..def94d1c63 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_queue.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_queue.rs @@ -1,88 +1,91 @@ -use std::time::SystemTime; - -use crate::records::main::invocation_row::invocation_row; -use crate::records::models::{InvocationContext, OutputStream, RowEnvelope}; -use crate::writer::_helpers::queue::WriterState; +use crate::records::models::{OutputStream, RecallValue}; use crate::writer::models::{DeliveryStream, Priority}; - -fn envelope() -> RowEnvelope { - RowEnvelope { - invocation_id: "inv".to_owned(), - invocation_started_at: "2026-10-19T00:00:00.000000Z".to_owned(), - producer_version: "test".to_owned(), - } -} +use crate::writer::tests::helpers::{chunked, pushed}; +use crate::writer::tests::test_types::{ChunkingTestCase, DisplacementTestCase}; #[test] fn given_a_full_queue_when_higher_priority_arrives_then_the_oldest_lowest_is_displaced() { - let mut state = WriterState::with_capacity(envelope(), (2, usize::MAX)); - let row = || vec![invocation_row(&envelope(), &InvocationContext::default())]; - state.push( - row(), - DeliveryStream::Diagnostics, - Priority::DebugDiagnostic, - ); - state.push(row(), DeliveryStream::Artifacts, Priority::Artifact); - state.push(row(), DeliveryStream::Events, Priority::FailedTerminal); - state.push( - row(), - DeliveryStream::Diagnostics, - Priority::DebugDiagnostic, - ); - assert_eq!(state.counts[DeliveryStream::Diagnostics.index()].dropped, 2); - assert_eq!(state.counts[DeliveryStream::Artifacts.index()].dropped, 0); - let batch = state.take_batch(10); - let streams: Vec = batch.iter().map(|record| record.stream).collect(); - assert_eq!( - streams, - vec![DeliveryStream::Events, DeliveryStream::Artifacts] - ); + let test_cases = [ + DisplacementTestCase { + description: "debug diagnostics give way first, artifacts survive", + capacity: 2, + pushes: &[ + (DeliveryStream::Diagnostics, Priority::DebugDiagnostic), + (DeliveryStream::Artifacts, Priority::Artifact), + (DeliveryStream::Events, Priority::FailedTerminal), + (DeliveryStream::Diagnostics, Priority::DebugDiagnostic), + ], + expected_dropped: vec![ + (DeliveryStream::Diagnostics, 2), + (DeliveryStream::Artifacts, 0), + (DeliveryStream::Events, 0), + (DeliveryStream::Diagnostics, 2), + ], + expected_taken: vec![DeliveryStream::Events, DeliveryStream::Artifacts], + }, + DisplacementTestCase { + description: "a queue with room keeps everything in priority order", + capacity: 4, + pushes: &[ + (DeliveryStream::Diagnostics, Priority::DebugDiagnostic), + (DeliveryStream::Events, Priority::FailedTerminal), + ], + expected_dropped: vec![ + (DeliveryStream::Diagnostics, 0), + (DeliveryStream::Events, 0), + ], + expected_taken: vec![DeliveryStream::Events, DeliveryStream::Diagnostics], + }, + ]; + + for test_case in test_cases { + assert_eq!( + pushed(test_case.capacity, test_case.pushes), + (test_case.expected_dropped, test_case.expected_taken), + "{}", + test_case.description + ); + } } #[test] fn given_interleaved_and_large_writes_when_chunked_then_sequences_and_offsets_are_exact() { - let mut state = WriterState::new(envelope()); - let big = vec![b'a'; 64 * 1024 + 10]; - state.append_output(OutputStream::Stdout, b"one ", SystemTime::now()); - state.append_output(OutputStream::Stdout, b"two\n", SystemTime::now()); - state.append_output(OutputStream::Stderr, b"err\n", SystemTime::now()); - state.append_output(OutputStream::Stdout, &big, SystemTime::now()); - state.flush_output(); - let rows: Vec<_> = state - .take_batch(100) - .into_iter() - .flat_map(|record| record.rows) - .collect(); - let summary: Vec<(String, String, String)> = rows - .iter() - .map(|row| { - let text = |name: &str| format!("{:?}", row.value(name).expect(name)); - (text("sequence"), text("stream"), text("byte_offset")) - }) - .collect(); - assert_eq!( - summary, - vec![ - ( - "Integer(0)".into(), - "Text(\"stdout\")".into(), - "Integer(0)".into() - ), - ( - "Integer(1)".into(), - "Text(\"stderr\")".into(), - "Integer(0)".into() - ), - ( - "Integer(2)".into(), - "Text(\"stdout\")".into(), - "Integer(8)".into() - ), - ( - "Integer(3)".into(), - "Text(\"stdout\")".into(), - "Integer(65544)".into() - ), + let chunk = |sequence: i64, stream: &str, offset: i64| { + [ + Some(RecallValue::Integer(sequence)), + Some(RecallValue::Text(stream.to_owned())), + Some(RecallValue::Integer(offset)), ] - ); + }; + let test_cases = [ + ChunkingTestCase { + description: "adjacent writes coalesce, stream switches and 64KiB split chunks", + writes: vec![ + (OutputStream::Stdout, b"one ".to_vec()), + (OutputStream::Stdout, b"two\n".to_vec()), + (OutputStream::Stderr, b"err\n".to_vec()), + (OutputStream::Stdout, vec![b'a'; 64 * 1024 + 10]), + ], + expected_chunks: vec![ + chunk(0, "stdout", 0), + chunk(1, "stderr", 0), + chunk(2, "stdout", 8), + chunk(3, "stdout", 65_544), + ], + }, + ChunkingTestCase { + description: "one small write is one chunk", + writes: vec![(OutputStream::Stderr, b"only\n".to_vec())], + expected_chunks: vec![chunk(0, "stderr", 0)], + }, + ]; + + for test_case in test_cases { + assert_eq!( + chunked(&test_case.writes), + test_case.expected_chunks, + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-recall/src/writer/tests/test_session.rs b/crates/sqlbuild-recall/src/writer/tests/test_session.rs index 7a0a29fb0e..d52e6384e5 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_session.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_session.rs @@ -1,123 +1,68 @@ -use std::time::Duration; - -use crate::records::models::{InvocationContext, OutputStream, RecallValue, RowEnvelope}; -use crate::store::main::open_sqlite_store::open_sqlite_store; -use crate::store::main::query_store::query_store; -use crate::store::main::read_output_transcript::read_output_transcript; -use crate::store::models::OpenMode; -use crate::writer::main::start_recall_session::start_recall_session; -use crate::writer::models::{DeliveryStream, SessionConfig, SetupOutcome}; - -fn config(path: std::path::PathBuf, invocation_id: &str) -> SessionConfig { - SessionConfig { - store_path: path, - envelope: RowEnvelope { - invocation_id: invocation_id.to_owned(), - invocation_started_at: "2026-10-19T00:00:00.000000Z".to_owned(), - producer_version: "test".to_owned(), - }, - context: InvocationContext { - command: Some("build".to_owned()), - pid: Some(42), - ..InvocationContext::default() - }, - retention_days: 30, - shutdown_timeout: Duration::from_secs(5), - diagnostics_min_severity: "info".to_owned(), - heartbeat_interval: Duration::from_secs(30), - setup_budget: Duration::from_secs(2), - maintenance: true, - maintenance_owner: "test:42".to_owned(), - } -} +use crate::records::models::{OutputStream, RecallValue}; +use crate::writer::models::SetupOutcome; +use crate::writer::tests::helpers::recorded_session; +#[cfg(unix)] +use crate::writer::tests::helpers::unwritable_session; +#[cfg(unix)] +use crate::writer::tests::test_types::UnwritableTestCase; +use crate::writer::tests::test_types::{RecordedSession, SessionTestCase}; #[test] fn given_a_session_when_closed_then_every_record_and_the_delivery_rows_are_written() { - let directory = tempfile::tempdir().expect("temporary directory"); - let path = directory.path().join("logs/recall.sqlite3"); - let session = start_recall_session(config(path.clone(), "inv-1")); - assert_eq!( - session.wait_for_setup(Duration::from_secs(10)), - Some(SetupOutcome::Ready) - ); - let large: Vec = (0..200_000_u32).map(|value| (value % 251) as u8).collect(); - session.append_output(OutputStream::Stdout, b"start\n"); - session.append_output(OutputStream::Stderr, b"\xff\x00warn\n"); - session.append_output(OutputStream::Stdout, &large); - session - .publish_event( - r#"{"event_id":"e1","event_type":"invocation_completed","schema_version":2, - "producer":"sqlbuild","producer_version":"1","occurred_at":"2026-10-19T00:00:01.000000Z", - "invocation_id":"inv-1","invocation_sequence":1,"payload":{"command":"build","exit_code":0,"duration_ms":3.0}}"#, - ) - .expect("event"); - session - .publish_diagnostic(r#"{"occurred_at":"2026-10-19T00:00:00.500000Z","severity":"debug","logger":"l","source":"python","message":"hidden"}"#) - .expect("debug diagnostic"); - session - .publish_diagnostic(r#"{"occurred_at":"2026-10-19T00:00:00.500000Z","severity":"warning","logger":"l","source":"python","message":"kept"}"#) - .expect("warning diagnostic"); - session.publish_sql("select 1", Some("s1"), None); - session.publish_sql("select 1", Some("s2"), None); - let summary = session.close(); - assert_eq!(summary.degraded_reason, None); - let total = summary.total(); - assert_eq!((total.dropped, total.failed), (0, 0)); - let output = summary - .streams - .iter() - .find(|(stream, _)| *stream == DeliveryStream::Output) - .expect("output counts") - .1; - assert_eq!(output.delivered, output.accepted); - let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("read store"); - let transcript = read_output_transcript(&store, "inv-1", None).expect("transcript"); - let expected: Vec = [b"start\n".as_slice(), b"\xff\x00warn\n", &large].concat(); - assert_eq!(transcript.data, expected); - assert!(transcript.complete, "{transcript:?}"); - let stdout = - read_output_transcript(&store, "inv-1", Some(OutputStream::Stdout)).expect("stdout"); - assert_eq!(stdout.data, [b"start\n".as_slice(), &large].concat()); - let facts = query_store( - &store, - "SELECT (SELECT status FROM invocations), (SELECT COUNT(*) FROM recall_diagnostics), \ - (SELECT COUNT(*) FROM recall_artifacts), (SELECT COUNT(*) FROM recall_heartbeats), \ - (SELECT COUNT(*) FROM recall_delivery), (SELECT command FROM recall_invocations)", - None, - ) - .expect("facts"); - assert_eq!( - facts.rows, - vec![vec![ - RecallValue::Text("completed".to_owned()), - RecallValue::Integer(1), - RecallValue::Integer(1), - RecallValue::Integer(1), - RecallValue::Integer(5), - RecallValue::Text("build".to_owned()), - ]] - ); + let large: Vec = (0..200_000_u32) + .map(|value| u8::try_from(value % 251).unwrap_or_default()) + .collect(); + let test_cases = [SessionTestCase { + description: "binary output, events, filtered diagnostics and deduplicated SQL land", + output: vec![ + (OutputStream::Stdout, b"start\n".to_vec()), + (OutputStream::Stderr, b"\xff\x00warn\n".to_vec()), + (OutputStream::Stdout, large.clone()), + ], + expected_session: RecordedSession { + setup: Some(SetupOutcome::Ready), + lost: (0, 0), + output_fully_delivered: true, + transcript: ( + [b"start\n".as_slice(), b"\xff\x00warn\n", &large].concat(), + true, + ), + stdout: [b"start\n".as_slice(), &large].concat(), + facts: vec![vec![ + RecallValue::Text("completed".to_owned()), + RecallValue::Integer(1), + RecallValue::Integer(1), + RecallValue::Integer(1), + RecallValue::Integer(5), + RecallValue::Text("build".to_owned()), + ]], + }, + }]; + + for test_case in test_cases { + assert_eq!( + recorded_session(&test_case.output), + test_case.expected_session, + "{}", + test_case.description + ); + } } #[cfg(unix)] #[test] fn given_an_unwritable_store_when_started_then_setup_reports_permission_and_close_is_quick() { - use std::os::unix::fs::PermissionsExt; - let directory = tempfile::tempdir().expect("temporary directory"); - std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o555)) - .expect("chmod"); - let session = start_recall_session(config( - directory.path().join("logs/recall.sqlite3"), - "inv-2", - )); - let setup = session.wait_for_setup(Duration::from_secs(10)); - session.append_output(OutputStream::Stdout, b"still printed\n"); - let summary = session.close(); - std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o755)) - .expect("restore"); - assert!( - matches!(setup, Some(SetupOutcome::PermissionDenied { .. })), - "{setup:?}" - ); - assert!(summary.degraded_reason.is_some()); + let test_cases = [UnwritableTestCase { + description: "permission is reported at setup and the summary is degraded", + expected_permission_and_degraded: (true, true), + }]; + + for test_case in test_cases { + assert_eq!( + unwritable_session(), + test_case.expected_permission_and_degraded, + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs new file mode 100644 index 0000000000..78ba896904 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -0,0 +1,38 @@ +use crate::records::models::{OutputStream, RecallValue}; +use crate::writer::models::{DeliveryStream, Priority, SetupOutcome}; + +/// What one recorded session left in the store. +#[derive(Debug, PartialEq)] +pub(crate) struct RecordedSession { + pub(crate) setup: Option, + pub(crate) lost: (u64, u64), + pub(crate) output_fully_delivered: bool, + pub(crate) transcript: (Vec, bool), + pub(crate) stdout: Vec, + pub(crate) facts: Vec>, +} + +pub(crate) struct DisplacementTestCase { + pub(crate) description: &'static str, + pub(crate) capacity: usize, + pub(crate) pushes: &'static [(DeliveryStream, Priority)], + pub(crate) expected_dropped: Vec<(DeliveryStream, u64)>, + pub(crate) expected_taken: Vec, +} + +pub(crate) struct ChunkingTestCase { + pub(crate) description: &'static str, + pub(crate) writes: Vec<(OutputStream, Vec)>, + pub(crate) expected_chunks: Vec<[Option; 3]>, +} + +pub(crate) struct SessionTestCase { + pub(crate) description: &'static str, + pub(crate) output: Vec<(OutputStream, Vec)>, + pub(crate) expected_session: RecordedSession, +} + +pub(crate) struct UnwritableTestCase { + pub(crate) description: &'static str, + pub(crate) expected_permission_and_degraded: (bool, bool), +} From 77a0265672e3dbd5a56d70a244f3541bdad227c9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 04:49:14 +0100 Subject: [PATCH 03/19] feat: enrich lifecycle events with plan, purpose and invocation context for Recall --- .../src/lifecycle_events/constants.rs | 36 ++++++++++++-- .../lifecycle_events/tests/test_validate.rs | 2 +- .../executor/auditing/main/_execute.py | 25 +++++----- .../executor/build/_helpers/scheduler.py | 27 +++++----- .../executor/build/classes/build_scheduler.py | 3 ++ .../observability/classes/event_dispatcher.py | 49 ++++++++++++++++++- .../classes/resource_attempt_lifecycle.py | 11 ++++- .../classes/statement_lifecycle.py | 18 +++++++ .../runtime/observability/constants.py | 2 + src/sqlbuild/runtime/observability/types.py | 14 ++++++ 10 files changed, 157 insertions(+), 30 deletions(-) diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index cb337572fc..fd5d643514 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -79,7 +79,7 @@ pub(crate) const FORBIDDEN_STATEMENT_PAYLOAD_FIELDS: [&str; 8] = [ "sql", "statement_sql", ]; -pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 51] = [ +pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 55] = [ "adapter", "attached_column_name", "attached_target_kind", @@ -115,8 +115,11 @@ pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 51] = [ "operation_kind", "operation_name", "outcome", + "plan_action", + "plan_reason_code", "planned_cursor_end_exclusive", "planned_cursor_start", + "purpose", "query_id", "resource_kind", "resource_name", @@ -128,11 +131,12 @@ pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 51] = [ "severity", "sql_digest", "statement_kind", + "target", "target_database", "target_name", "target_schema", ]; -pub(crate) const NONNEGATIVE_INTEGER_PAYLOAD_FIELDS: [&str; 25] = [ +pub(crate) const NONNEGATIVE_INTEGER_PAYLOAD_FIELDS: [&str; 26] = [ "affected_rows", "attempt_number", "batch_count", @@ -152,6 +156,7 @@ pub(crate) const NONNEGATIVE_INTEGER_PAYLOAD_FIELDS: [&str; 25] = [ "row_count", "sample_count", "selected_count", + "selection_count", "signal_number", "skipped_count", "succeeded_count", @@ -628,7 +633,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ ]; /// The schema version 2 catalogue in declaration order. pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ - ("invocation_started", &[], &[], &["command"], false), + ( + "invocation_started", + &[], + &[], + &["adapter", "command", "selection_count", "target"], + false, + ), ( "invocation_completed", &[], @@ -699,7 +710,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "resource_attempt_started", &["resource_attempt_id", "resource_id", "run_id"], &[], - &["attempt_number", "resource_kind", "resource_name"], + &[ + "attempt_number", + "plan_action", + "plan_reason_code", + "resource_kind", + "resource_name", + ], false, ), ( @@ -709,6 +726,8 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ &[ "attempt_number", "duration_ms", + "plan_action", + "plan_reason_code", "resource_kind", "resource_name", ], @@ -723,6 +742,8 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "duration_ms", "error_code", "error_type", + "plan_action", + "plan_reason_code", "resource_kind", "resource_name", ], @@ -740,6 +761,8 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ &[ "attempt_number", "duration_ms", + "plan_action", + "plan_reason_code", "resource_kind", "resource_name", "skip_code", @@ -897,6 +920,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "intent", "job_id", "metadata", + "purpose", "query_id", "row_count", "sql_digest", @@ -915,6 +939,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "intent", "job_id", "metadata", + "purpose", "query_id", "sql_digest", "statement_kind", @@ -933,6 +958,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "intent", "job_id", "metadata", + "purpose", "query_id", "row_count", "sql_digest", @@ -952,6 +978,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "intent", "job_id", "metadata", + "purpose", "query_id", "row_count", "sql_digest", @@ -973,6 +1000,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "intent", "job_id", "metadata", + "purpose", "query_id", "row_count", "sql_digest", diff --git a/crates/sqlbuild-core/src/lifecycle_events/tests/test_validate.rs b/crates/sqlbuild-core/src/lifecycle_events/tests/test_validate.rs index 8d98a1c185..6a841b4bbe 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/tests/test_validate.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/tests/test_validate.rs @@ -16,7 +16,7 @@ fn given_lifecycle_event_when_validating_then_first_contract_violation_is_report description: "unexpected payload lists allowed fields", event: event("invocation_started", vec![("x", int(1))]), expected_result: Err( - "payload field(s) not allowed for known event 'invocation_started': x; allowed fields: command", + "payload field(s) not allowed for known event 'invocation_started': x; allowed fields: adapter, command, selection_count, target", ), }, ValidateTestCase { diff --git a/src/sqlbuild/executor/auditing/main/_execute.py b/src/sqlbuild/executor/auditing/main/_execute.py index 716360750f..97be843670 100644 --- a/src/sqlbuild/executor/auditing/main/_execute.py +++ b/src/sqlbuild/executor/auditing/main/_execute.py @@ -9,6 +9,8 @@ from sqlbuild.compiler.auditing.types import AuditRunScope from sqlbuild.executor.auditing._helpers.execution import execute_audit_impl from sqlbuild.executor.auditing.models import AuditExecutionResult +from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle +from sqlbuild.runtime.observability.constants import AUDIT_PURPOSE def execute_audit( @@ -25,14 +27,15 @@ def execute_audit( ) -> AuditExecutionResult: """Execute and evaluate one audit, returning quality failure as result data.""" - return execute_audit_impl( - audit=audit, - adapter=adapter, - connection=connection, - model_locations=model_locations, - seed_locations=seed_locations, - source_map=source_map, - relation_overrides=relation_overrides, - run_scope_phase=run_scope_phase, - quality_scope=quality_scope, - ) + with StatementLifecycle.purpose_scope(AUDIT_PURPOSE): + return execute_audit_impl( + audit=audit, + adapter=adapter, + connection=connection, + model_locations=model_locations, + seed_locations=seed_locations, + source_map=source_map, + relation_overrides=relation_overrides, + run_scope_phase=run_scope_phase, + quality_scope=quality_scope, + ) diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index fcf165264f..18f57fbb91 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -47,6 +47,8 @@ from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.runtime.contracts.types import ExecutionResourceKind +from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle +from sqlbuild.runtime.observability.constants import MODEL_BUILD_PURPOSE type _BuildWorkerResult = ( ModelExecutionResult @@ -91,19 +93,20 @@ def _dispatch_model( ) -> ModelExecutionResult: """Route a model to the correct executor, then report a stalled cursor window as a warning.""" - return with_waiting_on_empty_inputs_warning( - _dispatch_model_entry( - context=context, - promotion_mode=promotion_mode, - snapshots=snapshots, - allow_snapshot_schema_change=allow_snapshot_schema_change, - custom_materializations=custom_materializations, - target=target, - effective_vars=effective_vars, - warehouse_relations=warehouse_relations, - on_progress=on_progress, + with StatementLifecycle.purpose_scope(MODEL_BUILD_PURPOSE): + return with_waiting_on_empty_inputs_warning( + _dispatch_model_entry( + context=context, + promotion_mode=promotion_mode, + snapshots=snapshots, + allow_snapshot_schema_change=allow_snapshot_schema_change, + custom_materializations=custom_materializations, + target=target, + effective_vars=effective_vars, + warehouse_relations=warehouse_relations, + on_progress=on_progress, + ) ) - ) def _dispatch_model_entry( diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 2c9515e434..5ebd3dae3b 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -764,11 +764,14 @@ def _execute_node( | LoadExecutionResult ): resource_kind: str = self._canonical_resource_kind(key=key) + planned: ModelPlanEntry | None = self._indexes.model_entries_by_key.get(key) with ResourceAttemptLifecycle( resource_id=self._canonical_resource_id(key=key), resource_kind=resource_kind, resource_name=key.name, run_id=self._run_id, + plan_action=None if planned is None else str(planned.action), + plan_reason_code=None if planned is None else str(planned.reason), ) as lifecycle: with CostContext.scope( run_id=self._run_id, diff --git a/src/sqlbuild/runtime/observability/classes/event_dispatcher.py b/src/sqlbuild/runtime/observability/classes/event_dispatcher.py index 4bffd0c4ff..f4bbc28bf6 100644 --- a/src/sqlbuild/runtime/observability/classes/event_dispatcher.py +++ b/src/sqlbuild/runtime/observability/classes/event_dispatcher.py @@ -24,9 +24,11 @@ OpaqueLifecycleEvent, ) from sqlbuild.runtime.observability.types import ( + ArtifactSubscriber, DiagnosticRegistration, DiagnosticSubscriber, HealthCallback, + HistorySubscriber, KnownLifecycleSubscriber, LifecycleRegistration, OpaqueLifecycleSubscriber, @@ -55,6 +57,7 @@ def __init__(self, *, health_callback: HealthCallback | None = None) -> None: | tuple[Literal["diagnostic"], DiagnosticLog, bool] ] = deque() self._publishing = False + self._artifact_subscribers: tuple[ArtifactSubscriber, ...] = () @overload def subscribe_lifecycle( @@ -89,6 +92,35 @@ def subscribe_diagnostics(self, subscriber: DiagnosticSubscriber) -> Unsubscribe self._diagnostics = (*self._diagnostics, registration) return partial(self._unsubscribe_diagnostics, token=token) + def subscribe_artifacts(self, subscriber: ArtifactSubscriber) -> Unsubscribe: + """Register a subscriber for executed SQL text; only history stores subscribe.""" + + with self._lock: + self._artifact_subscribers = (*self._artifact_subscribers, subscriber) + return partial(self._unsubscribe_artifacts, subscriber=subscriber) + + def subscribe_history(self, subscriber: HistorySubscriber) -> Unsubscribe: + """Register a history store for every known lifecycle fact and executed SQL text.""" + + lifecycle: Unsubscribe = self.subscribe_lifecycle( + subscriber=subscriber, accepts_opaque=False + ) + artifacts: Unsubscribe = self.subscribe_artifacts(subscriber.record_sql) + return partial(self._unsubscribe_history, unsubscribers=(lifecycle, artifacts)) + + def publish_sql_artifact( + self, *, sql: str, statement_id: str | None, resource_id: str | None + ) -> None: + """Hand executed SQL text to artifact subscribers; it never enters lifecycle payloads.""" + + with self._lock: + subscribers: tuple[ArtifactSubscriber, ...] = self._artifact_subscribers + for subscriber in subscribers: + try: + subscriber(sql=sql, statement_id=statement_id, resource_id=resource_id) + except Exception as error: + self._report_failure(channel="lifecycle", subscriber=subscriber, error=error) + def publish_lifecycle(self, event: LifecycleEvent | OpaqueLifecycleEvent) -> None: """Synchronously publish a lifecycle fact to the current ordered snapshot.""" @@ -184,6 +216,18 @@ def _unsubscribe_lifecycle(self, *, token: object) -> None: registration for registration in self._lifecycle if registration[0] is not token ) + def _unsubscribe_history(self, *, unsubscribers: tuple[Unsubscribe, ...]) -> None: + for unsubscribe in unsubscribers: + unsubscribe() + + def _unsubscribe_artifacts(self, *, subscriber: ArtifactSubscriber) -> None: + with self._lock: + self._artifact_subscribers = tuple( + registered + for registered in self._artifact_subscribers + if registered is not subscriber + ) + def _unsubscribe_diagnostics(self, *, token: object) -> None: with self._lock: self._diagnostics = tuple( @@ -194,7 +238,10 @@ def _report_failure( self, *, channel: Literal["lifecycle", "diagnostic"], - subscriber: KnownLifecycleSubscriber | OpaqueLifecycleSubscriber | DiagnosticSubscriber, + subscriber: KnownLifecycleSubscriber + | OpaqueLifecycleSubscriber + | DiagnosticSubscriber + | ArtifactSubscriber, error: BaseException, ) -> None: callback: HealthCallback | None = self._health_callback diff --git a/src/sqlbuild/runtime/observability/classes/resource_attempt_lifecycle.py b/src/sqlbuild/runtime/observability/classes/resource_attempt_lifecycle.py index 9b0bf9aa84..4d7c7b37e9 100644 --- a/src/sqlbuild/runtime/observability/classes/resource_attempt_lifecycle.py +++ b/src/sqlbuild/runtime/observability/classes/resource_attempt_lifecycle.py @@ -41,8 +41,12 @@ def __init__( attempt_number: int = 1, resource_attempt_id: str | None = None, run_id: str | None = None, + plan_action: str | None = None, + plan_reason_code: str | None = None, ) -> None: self._resource_id: str = resource_id + self._plan_action: str | None = plan_action + self._plan_reason_code: str | None = plan_reason_code self._resource_kind: str = resource_kind self._resource_name: str = resource_name self._attempt_number: int = attempt_number @@ -140,11 +144,16 @@ def __exit__( self._stack.close() def _base_payload(self) -> dict[str, JSONValue]: - return { + payload: dict[str, JSONValue] = { "resource_kind": self._resource_kind, "resource_name": self._resource_name, "attempt_number": self._attempt_number, } + if self._plan_action is not None: + payload["plan_action"] = self._plan_action + if self._plan_reason_code is not None: + payload["plan_reason_code"] = self._plan_reason_code + return payload def _terminal_payload(self) -> dict[str, JSONValue]: payload: dict[str, JSONValue] = self._base_payload() diff --git a/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py b/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py index 33ec8eff2f..85d0901508 100644 --- a/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py +++ b/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py @@ -43,6 +43,7 @@ _STATEMENT_LIFECYCLE_OWNER: ContextVar[StatementLifecycle | None] = ContextVar( "sqlbuild_statement_lifecycle_owner", default=None ) +_STATEMENT_PURPOSE: ContextVar[str | None] = ContextVar("sqlbuild_statement_purpose", default=None) type _ExecutionOwnerKey = tuple[int, int | None] @@ -84,6 +85,7 @@ def __init__( ) self._pending_failure: tuple[BaseException, str | None, str | None] | None = None self._statement_facts: tuple[str, str] | None = None + self._purpose: str | None = _STATEMENT_PURPOSE.get() self._item_deadline: ItemDeadline | None = None self._terminal_query_id: str | None = None @@ -95,6 +97,17 @@ def listener_scope(listener: StatementListener) -> Iterator[None]: with statement_listener_scope(listener): yield + @staticmethod + @contextmanager + def purpose_scope(purpose: str) -> Iterator[None]: + """Label statements run inside the block with why SQLBuild runs them, such as audit.""" + + token: Token[str | None] = _STATEMENT_PURPOSE.set(purpose) + try: + yield + finally: + _STATEMENT_PURPOSE.reset(token) + @staticmethod def discard_failed_attempt() -> None: """Forget a failed attempt deferred to the active statement before an adapter retry.""" @@ -124,6 +137,9 @@ def __enter__(self) -> StatementLifecycle: self._started_monotonic = self._clock() self._execution_owner_key = execution_owner_key self._publish(event_type="statement_started", payload=self._base_payload()) + dispatcher.publish_sql_artifact( + sql=self._sql, statement_id=self._statement_id, resource_id=identity.resource_id + ) self._item_deadline = current_item_deadline() if self._item_deadline is not None: self._item_deadline.statement_started() @@ -303,6 +319,8 @@ def _base_payload(self) -> dict[str, JSONValue]: } if self._batch_size is not None: payload["batch_size"] = self._batch_size + if self._purpose is not None: + payload["purpose"] = self._purpose return payload def _terminal_payload(self) -> dict[str, JSONValue]: diff --git a/src/sqlbuild/runtime/observability/constants.py b/src/sqlbuild/runtime/observability/constants.py index 47ecb4b533..df9f0d299d 100644 --- a/src/sqlbuild/runtime/observability/constants.py +++ b/src/sqlbuild/runtime/observability/constants.py @@ -8,6 +8,8 @@ RENAME_OPERATION_STRATEGY: str = "rename" RETRY_SCHEDULED_EVENT: str = "retry_scheduled" STATEMENT_HEARTBEAT_THRESHOLD_SECONDS: float = 30.0 +MODEL_BUILD_PURPOSE: str = "model_build" +AUDIT_PURPOSE: str = "audit" LIFECYCLE_ENVELOPE_FIELDS_V1: frozenset[str] = frozenset( { "event_id", diff --git a/src/sqlbuild/runtime/observability/types.py b/src/sqlbuild/runtime/observability/types.py index 94cb1e76e9..f2f8f45ac9 100644 --- a/src/sqlbuild/runtime/observability/types.py +++ b/src/sqlbuild/runtime/observability/types.py @@ -18,6 +18,20 @@ type OpaqueLifecycleSubscriber = Callable[[LifecycleEvent | OpaqueLifecycleEvent], None] type DiagnosticSubscriber = Callable[[DiagnosticLog], None] type HealthCallback = Callable[[DispatchFailure], None] + + +class ArtifactSubscriber(Protocol): + def __call__(self, *, sql: str, statement_id: str | None, resource_id: str | None) -> None: ... + + +class HistorySubscriber(Protocol): + def __call__(self, event: "LifecycleEvent") -> None: ... + + def record_sql( + self, *, sql: str, statement_id: str | None, resource_id: str | None + ) -> None: ... + + type LifecycleRegistration = tuple[ object, KnownLifecycleSubscriber | OpaqueLifecycleSubscriber, bool ] From c2daba1055b77414c2176c5ec388fa989154d815 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 04:49:19 +0100 Subject: [PATCH 04/19] feat: record invocations in the local Recall store and add sqb recall query, schema and output --- Cargo.lock | 1 + .../sqlbuild-cli/src/argv/command_table.json | 465 ++++++++++++++++++ crates/sqlbuild-cli/src/argv/constants.rs | 14 +- crates/sqlbuild-cli/src/dispatch/constants.rs | 27 + .../src/dispatch/main/command_dispatch.rs | 5 +- .../src/dispatch/main/invocation_policy.rs | 3 +- crates/sqlbuild-cli/src/dispatch/models.rs | 2 + .../dispatch/tests/test_invocation_policy.rs | 14 + .../src/command_connection/constants.rs | 3 +- crates/sqlbuild-python/Cargo.toml | 1 + .../src/bindings/_helpers/cli/dispatch.rs | 2 + .../src/bindings/_helpers/cli/entry.rs | 55 ++- .../src/bindings/_helpers/cli/registration.rs | 3 +- .../bindings/_helpers/compute_logs/storage.rs | 32 +- .../src/bindings/_helpers/mod.rs | 1 + .../src/bindings/_helpers/recall/command.rs | 327 ++++++++++++ .../src/bindings/_helpers/recall/console.rs | 22 + .../src/bindings/_helpers/recall/constants.rs | 57 +++ .../src/bindings/_helpers/recall/mod.rs | 7 + .../bindings/_helpers/recall/registration.rs | 9 + .../src/bindings/_helpers/recall/rendering.rs | 252 ++++++++++ .../src/bindings/_helpers/recall/session.rs | 380 ++++++++++++++ .../src/bindings/_helpers/recall/sinks.rs | 34 ++ .../bindings/_helpers/records/constants.rs | 2 + fensu.toml | 5 +- src/sqlbuild/_native.pyi | 24 + .../entrypoint/_dispatch_with_compute_logs.py | 4 + .../src/sqlbuild/cli/recall/__init__.py | 0 .../src/sqlbuild/cli/recall/_test_types.py | 74 +++ .../src/sqlbuild/cli/recall/conftest.py | 16 + .../src/sqlbuild/cli/recall/helpers.py | 187 +++++++ .../sqlbuild/cli/recall/test_local_recall.py | 395 +++++++++++++++ 32 files changed, 2406 insertions(+), 17 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs create mode 100644 tests/integration/src/sqlbuild/cli/recall/__init__.py create mode 100644 tests/integration/src/sqlbuild/cli/recall/_test_types.py create mode 100644 tests/integration/src/sqlbuild/cli/recall/conftest.py create mode 100644 tests/integration/src/sqlbuild/cli/recall/helpers.py create mode 100644 tests/integration/src/sqlbuild/cli/recall/test_local_recall.py diff --git a/Cargo.lock b/Cargo.lock index 011fa5984f..32c3e25e2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1323,6 +1323,7 @@ dependencies = [ "sqlbuild-discovery", "sqlbuild-model-config", "sqlbuild-pysource", + "sqlbuild-recall", "sqlbuild-refactor", "sqlbuild-render", "sqlbuild-rules", diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index f0af9a7345..8f0360fb38 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -6722,6 +6722,466 @@ } ] } + }, + { + "name": "recall", + "parser": { + "prog": "sqb recall", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "parsers", + "dest": "recall_command", + "nargs": "A...", + "required": true, + "group": 0, + "parsers": [ + { + "name": "query", + "parser": { + "prog": "sqb recall query", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "mutex_groups": [ + false + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "dest": "recall_sql", + "metavar": "sql", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--backend" + ], + "dest": "recall_backend", + "default": "logs", + "choices": [ + "logs", + "warehouse" + ], + "help": "logs reads the Recall store; warehouse queries the target's connection", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--format" + ], + "dest": "recall_format", + "default": "long", + "choices": [ + "long", + "table", + "json", + "csv" + ], + "mutex_group": 0, + "group": 1 + }, + { + "kind": "store_const", + "options": [ + "--json" + ], + "dest": "recall_format", + "nargs": 0, + "const": "json", + "help": "shorthand for --format json", + "mutex_group": 0, + "group": 1 + }, + { + "kind": "store", + "options": [ + "--limit" + ], + "dest": "recall_limit", + "default": 100, + "type": "int", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--no-limit" + ], + "dest": "recall_no_limit", + "nargs": 0, + "const": true, + "default": false, + "group": 1 + }, + { + "kind": "store", + "options": [ + "--max-cell-bytes" + ], + "dest": "recall_max_cell_bytes", + "type": "int", + "help": "longest cell in text formats, in bytes (default 4096; 0 for no limit)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--target" + ], + "dest": "target", + "help": "target whose Recall store to read (default: the default target)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--project-dir", + "--sqb-project-dir" + ], + "dest": "project_dir", + "default": { + "suppress": true + }, + "help": "SQLBuild project directory (default: the current directory)", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--no-color" + ], + "dest": "no_color", + "nargs": 0, + "const": true, + "default": { + "suppress": true + }, + "help": "disable colored output", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--debug" + ], + "dest": "debug", + "nargs": 0, + "const": true, + "default": { + "suppress": true + }, + "help": "enable verbose output and mirror internal diagnostics to stderr", + "group": 1 + } + ] + } + }, + { + "name": "schema", + "parser": { + "prog": "sqb recall schema", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "mutex_groups": [ + false + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--backend" + ], + "dest": "recall_backend", + "default": "logs", + "choices": [ + "logs", + "warehouse" + ], + "help": "logs reads the Recall store; warehouse queries the target's connection", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--state" + ], + "dest": "recall_state", + "nargs": 0, + "const": true, + "default": false, + "help": "with --backend warehouse, list SQLBuild state tables (the default)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--relation" + ], + "dest": "recall_relation", + "help": "with --backend warehouse, describe one relation", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--format" + ], + "dest": "recall_format", + "default": "long", + "choices": [ + "long", + "table", + "json", + "csv" + ], + "mutex_group": 0, + "group": 1 + }, + { + "kind": "store_const", + "options": [ + "--json" + ], + "dest": "recall_format", + "nargs": 0, + "const": "json", + "help": "shorthand for --format json", + "mutex_group": 0, + "group": 1 + }, + { + "kind": "store", + "options": [ + "--target" + ], + "dest": "target", + "help": "target whose Recall store to read (default: the default target)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--project-dir", + "--sqb-project-dir" + ], + "dest": "project_dir", + "default": { + "suppress": true + }, + "help": "SQLBuild project directory (default: the current directory)", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--no-color" + ], + "dest": "no_color", + "nargs": 0, + "const": true, + "default": { + "suppress": true + }, + "help": "disable colored output", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--debug" + ], + "dest": "debug", + "nargs": 0, + "const": true, + "default": { + "suppress": true + }, + "help": "enable verbose output and mirror internal diagnostics to stderr", + "group": 1 + } + ] + } + }, + { + "name": "output", + "parser": { + "prog": "sqb recall output", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "dest": "recall_invocation", + "nargs": "?", + "default": "last", + "metavar": "invocation", + "help": "invocation ID, unique prefix, or last (default)", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--stream" + ], + "dest": "recall_stream", + "default": "combined", + "choices": [ + "stdout", + "stderr", + "combined" + ], + "group": 1 + }, + { + "kind": "store", + "options": [ + "--tail" + ], + "dest": "recall_tail", + "type": "int", + "help": "print only the last N lines", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--target" + ], + "dest": "target", + "help": "target whose Recall store to read (default: the default target)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--project-dir", + "--sqb-project-dir" + ], + "dest": "project_dir", + "default": { + "suppress": true + }, + "help": "SQLBuild project directory (default: the current directory)", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--no-color" + ], + "dest": "no_color", + "nargs": 0, + "const": true, + "default": { + "suppress": true + }, + "help": "disable colored output", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--debug" + ], + "dest": "debug", + "nargs": 0, + "const": true, + "default": { + "suppress": true + }, + "help": "enable verbose output and mirror internal diagnostics to stderr", + "group": 1 + } + ] + } + } + ] + } + ] + } } ], "choices_help": [ @@ -6739,6 +7199,11 @@ "dest": "mv", "metavar": "mv", "help": "move a model file, optionally renaming it, and update references" + }, + { + "dest": "recall", + "metavar": "recall", + "help": "query the recorded history of past invocations" } ] } diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 2a653e1255..e343239742 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -63,7 +63,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 162] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 174] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -212,6 +212,18 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 162] = [ ("rules_command", NamespaceDefault::None), ("rules_rule_selector", NamespaceDefault::None), ("rules_skills_check", NamespaceDefault::Bool(false)), + ("recall_command", NamespaceDefault::None), + ("recall_sql", NamespaceDefault::None), + ("recall_backend", NamespaceDefault::Text("logs")), + ("recall_format", NamespaceDefault::Text("long")), + ("recall_limit", NamespaceDefault::Int(100)), + ("recall_no_limit", NamespaceDefault::Bool(false)), + ("recall_max_cell_bytes", NamespaceDefault::None), + ("recall_state", NamespaceDefault::Bool(false)), + ("recall_relation", NamespaceDefault::None), + ("recall_invocation", NamespaceDefault::Text("last")), + ("recall_stream", NamespaceDefault::Text("combined")), + ("recall_tail", NamespaceDefault::None), ("format_check", NamespaceDefault::Bool(false)), ("format_diff", NamespaceDefault::Bool(false)), ("format_fixtures_only", NamespaceDefault::Bool(false)), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 16ecc2f8af..e1240666b5 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -10,6 +10,8 @@ pub(crate) const SKILL_EXEMPT_COMMANDS: [&str; 3] = ["init", "playground", "skil pub(crate) const DBT_COMMAND_NAME: &str = "dbt"; pub(crate) const DBT_INIT_SUBCOMMAND: &str = "init"; pub(crate) const DIFF_COMMAND_NAME: &str = "diff"; +/// `sqb recall` reads history and is never recorded itself. +pub(crate) const RECALL_COMMAND_NAME: &str = "recall"; const PROJECT_DIR: Field = field("project_dir", Source::ProjectDir); const SELECT: Field = field("select", Source::Select); @@ -418,6 +420,31 @@ pub(crate) const QUERY_ROUTE: Route = routed( }, ); +const RECALL_ARGUMENTS: [Field; 13] = [ + PROJECT_DIR, + SELECTED_TARGET, + same("recall_command"), + attr("sql", "recall_sql"), + attr("backend", "recall_backend"), + attr("output_format", "recall_format"), + attr("limit", "recall_limit"), + attr("no_limit", "recall_no_limit"), + attr("max_cell_bytes", "recall_max_cell_bytes"), + attr("relation", "recall_relation"), + attr("invocation", "recall_invocation"), + attr("stream", "recall_stream"), + attr("tail", "recall_tail"), +]; +pub(crate) const RECALL_ROUTE: Route = routed( + "run_recall", + "run_recall_command", + &RECALL_ARGUMENTS, + CallShape { + slot_positional: 0, + implementation_keywords: true, + }, +); + const COST_FIELDS: [Field; 11] = [ PROJECT_DIR, attr("selector", "cost_selector"), diff --git a/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs b/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs index 251ef7a369..2a8444ef27 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs @@ -6,8 +6,8 @@ use crate::dispatch::_helpers::subcommands::{ use crate::dispatch::constants::{ AUDIT_ROUTE, BUILD_ROUTE, CHECK_ROUTE, CLONE_ROUTE, COMPILE_ROUTE, CONTRACT_ROUTE, COST_ROUTE, DAG_ROUTE, DEBUG_ROUTE, DIFF_ROUTE, FORMAT_ROUTE, FRESHNESS_ROUTE, INIT_ROUTE, JANITOR_ROUTE, - LINEAGE_ROUTE, LOAD_ROUTE, PLAN_ROUTE, PLAYGROUND_ROUTE, QUERY_ROUTE, REFACTOR_ROUTE, - RULES_ROUTE, SCOPE_ROUTE, SEED_ROUTE, SKILLS_ROUTE, TEST_ROUTE, + LINEAGE_ROUTE, LOAD_ROUTE, PLAN_ROUTE, PLAYGROUND_ROUTE, QUERY_ROUTE, RECALL_COMMAND_NAME, + RECALL_ROUTE, REFACTOR_ROUTE, RULES_ROUTE, SCOPE_ROUTE, SEED_ROUTE, SKILLS_ROUTE, TEST_ROUTE, }; use crate::dispatch::main::unavailable_handler::unavailable_handler; use crate::dispatch::models::{CommandFacts, Dispatch, DispatchFailure, Route}; @@ -54,6 +54,7 @@ pub fn command_dispatch(facts: &CommandFacts) -> Dispatch { ), "diff" => run(&DIFF_ROUTE), "query" => run(&QUERY_ROUTE), + RECALL_COMMAND_NAME => run(&RECALL_ROUTE), "cost" => run(&COST_ROUTE), "debug" => run(&DEBUG_ROUTE), "janitor" => run(&JANITOR_ROUTE), diff --git a/crates/sqlbuild-cli/src/dispatch/main/invocation_policy.rs b/crates/sqlbuild-cli/src/dispatch/main/invocation_policy.rs index 3396d875c2..56c7d09f06 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/invocation_policy.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/invocation_policy.rs @@ -2,7 +2,7 @@ use crate::dispatch::constants::{ DBT_COMMAND_NAME, DBT_INIT_SUBCOMMAND, DIFF_COMMAND_NAME, PROJECT_CREATING_COMMANDS, - SKILL_EXEMPT_COMMANDS, + RECALL_COMMAND_NAME, SKILL_EXEMPT_COMMANDS, }; use crate::dispatch::models::{CommandFacts, InvocationPolicy}; @@ -18,5 +18,6 @@ pub fn invocation_policy(facts: &CommandFacts) -> InvocationPolicy { creates_project, observed: !creates_project && !raw_query_diff, maintains_skills: !SKILL_EXEMPT_COMMANDS.contains(&command) && !raw_query_diff, + recalled: !creates_project && !command.is_empty() && command != RECALL_COMMAND_NAME, } } diff --git a/crates/sqlbuild-cli/src/dispatch/models.rs b/crates/sqlbuild-cli/src/dispatch/models.rs index 9e22a922ce..ecca6454b4 100644 --- a/crates/sqlbuild-cli/src/dispatch/models.rs +++ b/crates/sqlbuild-cli/src/dispatch/models.rs @@ -119,6 +119,8 @@ pub struct InvocationPolicy { pub observed: bool, /// Whether installed SQLBuild skills are checked for freshness first. pub maintains_skills: bool, + /// Whether Recall records the invocation; `sqb recall` never records itself. + pub recalled: bool, } /// What a failed command reports: its fallback error code and the debug log message. diff --git a/crates/sqlbuild-cli/src/dispatch/tests/test_invocation_policy.rs b/crates/sqlbuild-cli/src/dispatch/tests/test_invocation_policy.rs index ce61f40996..2272a40cba 100644 --- a/crates/sqlbuild-cli/src/dispatch/tests/test_invocation_policy.rs +++ b/crates/sqlbuild-cli/src/dispatch/tests/test_invocation_policy.rs @@ -13,6 +13,7 @@ fn given_parsed_command_when_choosing_services_then_matches_python_entry_rules() creates_project: false, observed: true, maintains_skills: true, + recalled: true, }, }, InvocationPolicyTestCase { @@ -22,6 +23,7 @@ fn given_parsed_command_when_choosing_services_then_matches_python_entry_rules() creates_project: true, observed: false, maintains_skills: true, + recalled: false, }, }, InvocationPolicyTestCase { @@ -31,6 +33,7 @@ fn given_parsed_command_when_choosing_services_then_matches_python_entry_rules() creates_project: true, observed: false, maintains_skills: false, + recalled: false, }, }, InvocationPolicyTestCase { @@ -43,6 +46,17 @@ fn given_parsed_command_when_choosing_services_then_matches_python_entry_rules() creates_project: false, observed: false, maintains_skills: false, + recalled: true, + }, + }, + InvocationPolicyTestCase { + description: "recall reads history and is never recorded itself", + facts: facts("recall", Some("query")), + expected_policy: InvocationPolicy { + creates_project: false, + observed: true, + maintains_skills: true, + recalled: false, }, }, ]; diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs index 6c70b9781a..ab3bddc173 100644 --- a/crates/sqlbuild-config/src/command_connection/constants.rs +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -25,7 +25,7 @@ pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; /// Two quotes around at least one character. pub(crate) const MIN_QUOTED_IDENTIFIER_LENGTH: usize = 3; /// The warehouse group each CLI command selects a target's default warehouse from. -pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 29] = [ +pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 30] = [ ("audit", Some("build")), ("build", Some("build")), ("check", Some("build")), @@ -40,6 +40,7 @@ pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 29] = [ ("diff", Some("query")), ("freshness", Some("query")), ("query", Some("query")), + ("recall", Some("query")), ("janitor", Some("query")), ("scenario", Some("query")), ("test", Some("query")), diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 077bf8b416..36a6013447 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -32,6 +32,7 @@ sqlbuild-cursor.workspace = true sqlbuild-discovery.workspace = true sqlbuild-model-config.workspace = true sqlbuild-pysource.workspace = true +sqlbuild-recall.workspace = true sqlbuild-refactor.workspace = true sqlbuild-render.workspace = true sqlbuild-rules.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index da8cc80927..cdc5fde694 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -27,6 +27,7 @@ use crate::bindings::_helpers::plan_command::plan::run_plan_command; use crate::bindings::_helpers::plan_command::rules::run_rules_command; use crate::bindings::_helpers::query_command::dag::run_dag_command; use crate::bindings::_helpers::query_command::query::run_query_command; +use crate::bindings::_helpers::recall::command::run_recall_command; use crate::bindings::_helpers::records::construct::{new_record_with, record_class}; use crate::bindings::_helpers::refactor::refactor_command::run_refactor_command; use crate::bindings::_helpers::scenario_command::capture_command::run_scenario_capture_command; @@ -394,6 +395,7 @@ fn run_implementation<'py>( "run_plan_command" => run_plan_command(&request()?)?, "run_playground_command" => run_playground_command(py, &request()?)?, "run_query_command" => run_query_command(py, options())?, + "run_recall_command" => run_recall_command(py, options())?, "run_refactor_command" => run_refactor_command(py, &request()?)?, "run_rules_command" => run_rules_command(&request()?)?, "run_scenario_capture_command" => run_scenario_capture_command(py, &request()?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index d4500c5832..fd10d50ed5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -27,6 +27,10 @@ use crate::bindings::_helpers::native_host::facade::FacadeHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::output_capture::records::invocation_context; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +use crate::bindings::_helpers::recall::session::{ + InvocationRecall, finish_invocation_recall, start_invocation_recall, +}; +use crate::bindings::_helpers::recall::sinks::RecallLifecycleSink; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::record_class; use crate::bindings::_helpers::records::fields::field as native_field; @@ -58,6 +62,7 @@ struct Invocation<'py> { handlers: Option>, use_color: bool, facts: CommandFacts, + recall: InvocationRecall, } impl<'py> Invocation<'py> { @@ -392,6 +397,34 @@ fn lifecycle_event<'py>( ) } +/// Subscribe the invocation's Recall session to every lifecycle fact and SQL artifact. +fn subscribe_recall(dispatcher: &Bound<'_, PyAny>, recall: &InvocationRecall) -> PyResult<()> { + let Some(session) = recall.session.clone() else { + return Ok(()); + }; + let sink = Bound::new(dispatcher.py(), RecallLifecycleSink { session })?; + dispatcher.call_method1("subscribe_history", (sink,))?; + Ok(()) +} + +/// `target`, `adapter` and `selection_count` of the started invocation, when known. +fn add_invocation_context( + invocation: &Invocation<'_>, + payload: &Bound<'_, PyDict>, +) -> PyResult<()> { + if let Some(target) = &invocation.recall.target { + payload.set_item("target", target)?; + } + if let Some(adapter) = &invocation.recall.adapter { + payload.set_item("adapter", adapter)?; + } + let select: Option> = invocation.args.getattr("select")?.extract()?; + if let Some(select) = select { + payload.set_item("selection_count", select.len())?; + } + Ok(()) +} + /// Dispatch once while publishing canonical invocation lifecycle facts. fn observed_dispatch<'py>( invocation: &Invocation<'py>, @@ -421,8 +454,10 @@ fn observed_dispatch<'py>( dispatcher.call_method1("publish_lifecycle", (event,))?; Ok(()) }; + subscribe_recall(dispatcher, &invocation.recall)?; let payload: Bound<'py, PyDict> = PyDict::new(py); payload.set_item("command", &command)?; + add_invocation_context(invocation, &payload)?; publish(lifecycle_event(py, "invocation_started", payload)?)?; match dispatch_command(&invocation.args, invocation.handlers.clone()) { Ok(exit_code) => { @@ -510,6 +545,16 @@ fn parsed_invocation<'py>( } let facts: CommandFacts = command_facts(&args)?; let policy: InvocationPolicy = invocation_policy(&facts); + let recall: InvocationRecall = if policy.recalled { + start_invocation_recall(py, &args, identity, &facts)? + } else { + InvocationRecall::default() + }; + let recall_sink: Bound<'py, PyAny> = match recall.session.clone() { + Some(session) => Bound::new(py, RecallLifecycleSink { session })?.into_any(), + None => py.None().into_bound(py), + }; + let closure_recall: InvocationRecall = recall.clone(); let retained: (Py, Option>) = (args.clone().unbind(), handlers.map(Bound::unbind)); let operation: Bound<'py, PyCFunction> = PyCFunction::new_closure( @@ -528,11 +573,12 @@ fn parsed_invocation<'py>( .map(|handlers| handlers.bind(py).clone()), use_color, facts: facts.clone(), + recall: closure_recall.clone(), }; Ok(handled_dispatch(&invocation)?.unbind()) }, )?; - call_with( + let dispatched: PyResult> = call_with( &host::entry(py, FacadeHost::DispatchWithComputeLogs)?, &[ ("args", args), @@ -544,8 +590,13 @@ fn parsed_invocation<'py>( .to_owned() .into_any(), ), + ("recall", recall_sink), ], - ) + ); + let finished: PyResult<()> = finish_invocation_recall(py, &recall); + let exit_code: Bound<'py, PyAny> = dispatched?; + finished?; + Ok(exit_code) } /// `argv` without a leading launch record option, after offering the record it names. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 3875d8810a..7f26b84889 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -11,7 +11,7 @@ use crate::bindings::_helpers::{ contract_command, cost_command, cost_report, cost_telemetry, dbt_command, dbt_events, dbt_interop, debug_command, diff_report, event_exporting, execution_history, format_command, freshness, init_command, janitor, load, output_capture, plan_command, presentation_text, - query_command, result_output, scenario_command, seed_command, sql_test_command, + query_command, recall, result_output, scenario_command, seed_command, sql_test_command, workspace_commands, }; @@ -50,6 +50,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { build_command::registration::register(module)?; build_progress::registration::register(module)?; query_command::registration::register(module)?; + recall::registration::register(module)?; scenario_command::registration::register(module)?; contract_command::registration::register(module)?; init_command::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs b/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs index 0a7f1c73ac..210464ed2f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs @@ -4,7 +4,7 @@ use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io::{Seek, Write}; use std::path::PathBuf; -use std::sync::{Mutex, MutexGuard}; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock}; use pyo3::IntoPyObjectExt; use pyo3::exceptions::PyRuntimeError; @@ -27,6 +27,7 @@ use sqlbuild_cli::compute_logs::main::read_stream_chunk::read_stream_chunk; use sqlbuild_cli::compute_logs::main::stream_byte_counts::stream_byte_counts; use sqlbuild_cli::compute_logs::main::validate_capture_id::validate_capture_id; use sqlbuild_cli::compute_logs::models::CaptureDocument; +use sqlbuild_recall::writer::models::RecallSession; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, @@ -37,6 +38,8 @@ use crate::bindings::_helpers::compute_logs::conversion::{ }; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::recall::session::{forward_diagnostic, forward_output}; +use crate::bindings::_helpers::recall::sinks::RecallLifecycleSink; const DIAGNOSTICS_STREAM: usize = 2; @@ -53,6 +56,8 @@ pub(crate) struct LocalFilesystemComputeLogStorage { root: PathBuf, retention_count: Option, state: Mutex, + /// The invocation's Recall session, fed every byte and diagnostic the capture accepts. + recall: OnceLock>, } /// The stdout, stderr and diagnostics byte counts of open streams, as `tell()` reports them. @@ -164,9 +169,15 @@ impl LocalFilesystemComputeLogStorage { root, retention_count, state: Mutex::new(StorageState::default()), + recall: OnceLock::new(), }) } + /// Feed every accepted byte and diagnostic to the invocation's Recall session. + fn attach_recall(&self, recall: PyRef<'_, RecallLifecycleSink>) { + drop(self.recall.set(Arc::clone(&recall.session))); + } + /// Return the resolved configured log root. #[getter] pub(crate) fn root<'py>(&self, py: Python<'py>) -> PyResult> { @@ -210,7 +221,13 @@ impl LocalFilesystemComputeLogStorage { "compute log append data must be bytes", )); }; - self.append_bytes(py, &invocation_id, stream, bytes.as_bytes()) + self.append_bytes(py, &invocation_id, stream, bytes.as_bytes())?; + forward_output( + self.recall.get().map(Arc::as_ref), + &invocation_id, + (stream, bytes.as_bytes()), + ); + Ok(()) } /// Append one validated deterministic UTF-8 diagnostic JSON line. @@ -230,16 +247,15 @@ impl LocalFilesystemComputeLogStorage { } let encoded: Bound<'_, PyAny> = host::entry(py, ExecutorHost::DiagnosticLogToJson)?.call1((record,))?; - let mut line: Vec = encoded - .downcast::()? - .to_str()? - .as_bytes() - .to_vec(); + let text: &str = encoded.downcast::()?.to_str()?; + let mut line: Vec = text.as_bytes().to_vec(); line.push(b'\n'); self.ensure_open(py)?; let invocation_id = capture_id(invocation_id)?; checked(py, validate_capture_id(&invocation_id))?; - self.append_bytes(py, &invocation_id, DIAGNOSTICS_STREAM, &line) + self.append_bytes(py, &invocation_id, DIAGNOSTICS_STREAM, &line)?; + forward_diagnostic(self.recall.get().map(Arc::as_ref), &invocation_id, text); + Ok(()) } /// Read at most the public byte limit from an exact stream offset. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 1ad13fd9e6..d0eed71f8b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -78,6 +78,7 @@ pub(crate) mod presentation_text; pub(crate) mod project_assembly; pub(crate) mod python_values; pub(crate) mod query_command; +pub(crate) mod recall; pub(crate) mod record_behaviours; pub(crate) mod records; pub(crate) mod refactor; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs new file mode 100644 index 0000000000..c7f01ac6fd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs @@ -0,0 +1,327 @@ +//! `sqb recall query | schema | output`. + +use std::path::PathBuf; +use std::time::Instant; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyStringMethods}; +use sqlbuild_recall::records::models::OutputStream; +use sqlbuild_recall::settings::main::resolve_recall_settings::resolve_recall_settings; +use sqlbuild_recall::settings::models::{RecallSettings, SettingsRequest, StoreLocation}; +use sqlbuild_recall::store::errors::{StoreError, StoreErrorKind}; +use sqlbuild_recall::store::main::describe_store::describe_store; +use sqlbuild_recall::store::main::open_sqlite_store::open_sqlite_store; +use sqlbuild_recall::store::main::query_store::query_store; +use sqlbuild_recall::store::main::read_output_transcript::read_output_transcript; +use sqlbuild_recall::store::main::resolve_invocation::resolve_invocation; +use sqlbuild_recall::store::models::{OpenMode, SqliteStore}; + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::query_command::query::run_query_command; +use crate::bindings::_helpers::recall::console::{write_stderr, write_stdout_bytes}; +use crate::bindings::_helpers::recall::constants::{ + DEFAULT_MAX_CELL_BYTES, DEFAULT_QUERY_LIMIT, JSON_FORMAT, LAST_INVOCATION, LONG_FORMAT, + RECALL_AMBIGUOUS_CODE, RECALL_DISABLED_CODE, RECALL_NOT_FOUND_CODE, RECALL_OPTIONS, + RECALL_QUERY_CODE, RECALL_SETTINGS_CODE, RECALL_STORE_CODE, WAREHOUSE_BACKEND, +}; +use crate::bindings::_helpers::recall::rendering::{QueryContext, render_result, render_schema}; +use crate::bindings::_helpers::records::construct::new_error; +use crate::bindings::_helpers::records::ids::ErrorId; + +fn user_error(py: Python<'_>, message: &str, code: &str) -> PyErr { + new_error(py, ErrorId::CliUser, message.to_owned(), code) +} + +fn store_failure(py: Python<'_>, error: &StoreError) -> PyErr { + let code = match error.kind { + StoreErrorKind::NotFound => RECALL_NOT_FOUND_CODE, + StoreErrorKind::Ambiguous => RECALL_AMBIGUOUS_CODE, + StoreErrorKind::Query => RECALL_QUERY_CODE, + _ => RECALL_STORE_CODE, + }; + user_error(py, &error.message, code) +} + +/// The options of one `sqb recall` invocation, as the dispatch route passes them. +struct RecallRequest { + command: Option, + project_dir: Option, + selected_target: Option, + sql: Option, + backend: Option, + format: String, + limit: Option, + no_limit: bool, + max_cell_bytes: Option, + relation: Option, + invocation: Option, + stream: Option, + tail: Option, +} + +fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.to_string_lossy().into_owned())) +} + +fn recall_request(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "run_recall_command", options, (&RECALL_OPTIONS, &[]))?; + let text = |name: &str| optional_text(&arguments.value(name)?); + let number = |name: &str| -> PyResult> { arguments.value(name)?.extract() }; + Ok(RecallRequest { + command: text("recall_command")?, + project_dir: text("project_dir")?.map(PathBuf::from), + selected_target: text("selected_target")?, + sql: text("sql")?, + backend: text("backend")?, + format: text("output_format")?.unwrap_or_else(|| LONG_FORMAT.to_owned()), + limit: number("limit")?, + no_limit: arguments.value("no_limit")?.is_truthy()?, + max_cell_bytes: number("max_cell_bytes")?, + relation: text("relation")?, + invocation: text("invocation")?, + stream: text("stream")?, + tail: number("tail")?, + }) +} + +/// The project directory, target and settings, or the coded failure. +fn settings(py: Python<'_>, recall: &RecallRequest) -> PyResult<(PathBuf, RecallSettings)> { + let raw: PathBuf = match &recall.project_dir { + Some(directory) => directory.clone(), + None => std::env::current_dir()?, + }; + let project_dir = std::fs::canonicalize(&raw).unwrap_or(raw); + let request = SettingsRequest { + project_dir: project_dir.clone(), + selected_target: recall.selected_target.clone(), + }; + let settings = resolve_recall_settings(&request) + .map_err(|error| user_error(py, &error.message, RECALL_SETTINGS_CODE))?; + if !settings.enabled { + let target = settings + .target + .as_deref() + .map_or_else(String::new, |target| format!(" for target `{target}`")); + return Err(user_error( + py, + &format!("Recall is disabled{target}; set enabled = true in its recall settings"), + RECALL_DISABLED_CODE, + )); + } + Ok((project_dir, settings)) +} + +fn open_store(py: Python<'_>, settings: &RecallSettings) -> PyResult<(SqliteStore, String)> { + match &settings.store { + StoreLocation::Sqlite { path } => { + let store = open_sqlite_store(path, OpenMode::ReadOnly).map_err(|error| { + if error.kind == StoreErrorKind::NotFound { + user_error( + py, + &format!( + "no Recall history is recorded yet at {}; run a command first", + path.display() + ), + RECALL_STORE_CODE, + ) + } else { + store_failure(py, &error) + } + })?; + Ok((store, path.display().to_string())) + } + StoreLocation::Unsupported { + connection, + adapter, + } => Err(user_error( + py, + &format!( + "connection `{connection}` uses adapter `{adapter}`, which this version cannot \ + read Recall history from" + ), + RECALL_STORE_CODE, + )), + } +} + +/// `sqb recall query --backend warehouse` and `schema --backend warehouse` run through `sqb query`. +fn warehouse_query( + py: Python<'_>, + recall: &RecallRequest, + project_dir: &std::path::Path, + (sql, limit): (&str, Option), +) -> PyResult { + let options = PyDict::new(py); + options.set_item("project_dir", project_dir)?; + options.set_item("sql", sql)?; + options.set_item("selected_target", recall.selected_target.as_deref())?; + options.set_item("output_format", &recall.format)?; + options.set_item("limit", limit)?; + run_query_command(py, Some(options)) +} + +fn limit(recall: &RecallRequest) -> Option { + if recall.no_limit { + return None; + } + recall.limit.or(Some(DEFAULT_QUERY_LIMIT)) +} + +fn run_query(py: Python<'_>, recall: &RecallRequest) -> PyResult { + let (project_dir, settings) = settings(py, recall)?; + let sql: String = recall.sql.clone().unwrap_or_default(); + if sql.trim().is_empty() { + return Err(user_error( + py, + "recall query SQL must not be empty", + RECALL_QUERY_CODE, + )); + } + let limit = limit(recall); + if limit.is_some_and(|limit| limit < 0) { + return Err(user_error( + py, + "--limit must not be negative", + RECALL_QUERY_CODE, + )); + } + if recall.backend.as_deref() == Some(WAREHOUSE_BACKEND) { + return warehouse_query(py, recall, &project_dir, (&sql, limit)); + } + let (store, path) = open_store(py, &settings)?; + let started = Instant::now(); + let limit = limit.map(|limit| usize::try_from(limit).unwrap_or(usize::MAX)); + let result = query_store(&store, &sql, limit).map_err(|error| store_failure(py, &error))?; + let format: &str = &recall.format; + let max_bytes: Option = match recall.max_cell_bytes { + Some(bytes) if bytes <= 0 => None, + Some(bytes) => Some(usize::try_from(bytes).unwrap_or(usize::MAX)), + None if format == JSON_FORMAT => None, + None => Some(DEFAULT_MAX_CELL_BYTES), + }; + let rendered = render_result( + &result, + format, + max_bytes, + &QueryContext { + target: settings.target.as_deref(), + store_path: &path, + limit, + elapsed_ms: started.elapsed().as_secs_f64() * 1000.0, + }, + ); + write_stdout_bytes(py, rendered.as_bytes())?; + Ok(0) +} + +fn run_schema(py: Python<'_>, recall: &RecallRequest) -> PyResult { + let (project_dir, settings) = settings(py, recall)?; + if recall.backend.as_deref() == Some(WAREHOUSE_BACKEND) { + let sql = match &recall.relation { + Some(relation) => format!( + "SELECT table_schema, table_name, column_name, data_type, is_nullable \ + FROM information_schema.columns WHERE table_name = '{}' \ + ORDER BY table_schema, ordinal_position", + relation.replace('\'', "''") + ), + None => "SELECT table_schema, table_name, table_type FROM information_schema.tables \ + WHERE table_name LIKE '\\_sqlbuild\\_%' ESCAPE '\\' \ + ORDER BY table_schema, table_name" + .to_owned(), + }; + return warehouse_query(py, recall, &project_dir, (&sql, None)); + } + let (store, path) = open_store(py, &settings)?; + let relations = describe_store(&store).map_err(|error| store_failure(py, &error))?; + let json_output: bool = recall.format == JSON_FORMAT; + write_stdout_bytes( + py, + render_schema(&relations, json_output, (&path, settings.retention_days)).as_bytes(), + )?; + Ok(0) +} + +fn tail_lines(data: &[u8], lines: usize) -> &[u8] { + let body = data.strip_suffix(b"\n").unwrap_or(data); + let start = body + .iter() + .enumerate() + .rev() + .filter(|(_, byte)| **byte == b'\n') + .nth(lines.saturating_sub(1)) + .map_or(0, |(index, _)| index + 1); + if lines == 0 { + &data[data.len()..] + } else { + &data[start..] + } +} + +fn run_output(py: Python<'_>, recall: &RecallRequest) -> PyResult { + let (_, settings) = settings(py, recall)?; + let (store, _) = open_store(py, &settings)?; + let selector: &str = recall.invocation.as_deref().unwrap_or(LAST_INVOCATION); + let invocation_id = + resolve_invocation(&store, selector).map_err(|error| store_failure(py, &error))?; + let stream = recall.stream.as_deref().and_then(OutputStream::from_name); + let transcript = read_output_transcript(&store, &invocation_id, stream) + .map_err(|error| store_failure(py, &error))?; + let data: &[u8] = match recall.tail { + Some(lines) => tail_lines(&transcript.data, usize::try_from(lines).unwrap_or(0)), + None => &transcript.data, + }; + write_stdout_bytes(py, data)?; + let mut problems: Vec = Vec::new(); + if transcript.missing_sequences > 0 { + problems.push(format!( + "{} chunk(s) missing ({})", + transcript.missing_sequences, + transcript.gap_reason.as_deref().unwrap_or("unexplained") + )); + } + if transcript.reported_drops > 0 { + problems.push(format!("{} chunk(s) dropped", transcript.reported_drops)); + } + if transcript.filtered > 0 { + problems.push(format!("{} chunk(s) filtered", transcript.filtered)); + } + if !transcript.flush_complete { + problems.push("delivery did not report a complete flush".to_owned()); + } + match transcript.status.as_deref() { + Some("completed" | "failed") | None => {} + Some(status) => problems.push(format!("the invocation is {status}")), + } + if !problems.is_empty() { + write_stderr( + py, + &format!( + "note: output of {invocation_id} may be incomplete: {}\n", + problems.join("; ") + ), + )?; + } + Ok(0) +} + +/// Run one `sqb recall` subcommand and return its exit code. +pub(crate) fn run_recall_command( + py: Python<'_>, + options: Option>, +) -> PyResult { + let recall: RecallRequest = recall_request(py, options)?; + match recall.command.as_deref() { + Some("query") => run_query(py, &recall), + Some("schema") => run_schema(py, &recall), + Some("output") => run_output(py, &recall), + _ => Err(user_error( + py, + "sqb recall requires a subcommand: query, schema or output", + RECALL_QUERY_CODE, + )), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs new file mode 100644 index 0000000000..349d0053fa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs @@ -0,0 +1,22 @@ +//! Recall's writes to the process streams, through Python so captures and tees see them. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyBytes; + +/// Write `data` unchanged to stdout's binary buffer after flushing pending text. +pub(crate) fn write_stdout_bytes(py: Python<'_>, data: &[u8]) -> PyResult<()> { + let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; + stdout.call_method0("flush")?; + let target: Bound<'_, PyAny> = stdout.getattr("buffer").unwrap_or(stdout); + target.call_method1("write", (PyBytes::new(py, data),))?; + target.call_method0("flush")?; + Ok(()) +} + +/// Write `text` to stderr and flush it. +pub(crate) fn write_stderr(py: Python<'_>, text: &str) -> PyResult<()> { + let stderr: Bound<'_, PyAny> = py.import("sys")?.getattr("stderr")?; + stderr.call_method1("write", (text,))?; + stderr.call_method0("flush")?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs new file mode 100644 index 0000000000..b11bc71720 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -0,0 +1,57 @@ +//! Recall binding names, environment variables and error codes. + +/// Seconds between heartbeats; tests shorten it to observe abandoned invocations quickly. +pub(crate) const HEARTBEAT_ENVIRONMENT: &str = "SQLBUILD_RECALL_HEARTBEAT_SECONDS"; +/// Marker files under `target/` limiting store notices to one per interval. +pub(crate) const DDL_NOTICE_PREFIX: &str = ".recall-notice-"; +pub(crate) const NOTICE_INTERVAL_SECONDS: u64 = 24 * 60 * 60; +/// Hex digits of the hostname digest recorded as the host token. +pub(crate) const HOST_TOKEN_LENGTH: usize = 16; +/// The shortest heartbeat interval the test environment variable may request. +pub(crate) const MINIMUM_HEARTBEAT_SECONDS: f64 = 0.01; +/// The compute-log stream index of stdout; every other output index is stderr. +pub(crate) const STDOUT_STREAM_INDEX: usize = 0; +pub(crate) const RECALL_DISABLED_CODE: &str = "C970"; +pub(crate) const RECALL_STORE_CODE: &str = "C971"; +pub(crate) const RECALL_NOT_FOUND_CODE: &str = "C972"; +pub(crate) const RECALL_AMBIGUOUS_CODE: &str = "C973"; +pub(crate) const RECALL_QUERY_CODE: &str = "C974"; +pub(crate) const RECALL_SETTINGS_CODE: &str = "C975"; +pub(crate) const WAREHOUSE_BACKEND: &str = "warehouse"; +/// Default cell width in text formats, in bytes. +pub(crate) const DEFAULT_MAX_CELL_BYTES: usize = 4096; +pub(crate) const LONG_FORMAT: &str = "long"; +pub(crate) const TABLE_FORMAT: &str = "table"; +pub(crate) const CSV_FORMAT: &str = "csv"; +pub(crate) const JSON_FORMAT: &str = "json"; +/// The standard base64 alphabet binary JSON cells are encoded with. +pub(crate) const BASE64_ALPHABET: &[u8; 64] = + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +/// Join keys `sqb recall schema` points out. +pub(crate) const JOIN_KEYS: [&str; 5] = [ + "invocation_id", + "statement_id", + "sql_digest", + "run_id", + "resource_id", +]; +/// Rows `sqb recall query` returns without `--limit` or `--no-limit`. +pub(crate) const DEFAULT_QUERY_LIMIT: i64 = 100; +/// The invocation selector naming the most recently started invocation. +pub(crate) const LAST_INVOCATION: &str = "last"; +/// The keyword arguments the `sqb recall` dispatch route passes. +pub(crate) const RECALL_OPTIONS: [&str; 13] = [ + "project_dir", + "selected_target", + "recall_command", + "sql", + "backend", + "output_format", + "limit", + "no_limit", + "max_cell_bytes", + "relation", + "invocation", + "stream", + "tail", +]; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs new file mode 100644 index 0000000000..c0c5073d63 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod command; +pub(crate) mod console; +pub(crate) mod constants; +pub(crate) mod registration; +pub(crate) mod rendering; +pub(crate) mod session; +pub(crate) mod sinks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs new file mode 100644 index 0000000000..6e7568ce35 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs @@ -0,0 +1,9 @@ +//! Register the native Recall classes. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +use crate::bindings::_helpers::recall::sinks::RecallLifecycleSink; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs new file mode 100644 index 0000000000..28e251b037 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs @@ -0,0 +1,252 @@ +//! `sqb recall query` and `schema` output: text cells, CSV and the JSON document. + +use serde_json::{Map, Value, json}; +use sqlbuild_core::query_command::main::render_query_csv::render_query_csv; +use sqlbuild_core::query_command::main::render_query_long::render_query_long; +use sqlbuild_core::query_command::main::render_query_table::render_query_table; +use sqlbuild_core::query_command::models::QueryDisplay; +use sqlbuild_recall::records::models::RecallValue; +use sqlbuild_recall::store::models::{DescribedColumn, DescribedRelation, QueryResult}; + +use crate::bindings::_helpers::recall::constants::{ + BASE64_ALPHABET, CSV_FORMAT, JOIN_KEYS, JSON_FORMAT, TABLE_FORMAT, +}; + +fn base64(data: &[u8]) -> String { + let mut encoded = String::with_capacity(data.len().div_ceil(3) * 4); + for chunk in data.chunks(3) { + let bytes = [ + chunk[0], + *chunk.get(1).unwrap_or(&0), + *chunk.get(2).unwrap_or(&0), + ]; + let triple = u32::from(bytes[0]) << 16 | u32::from(bytes[1]) << 8 | u32::from(bytes[2]); + for index in 0..4 { + if index <= chunk.len() { + encoded.push(char::from( + BASE64_ALPHABET[(triple >> (18 - 6 * index) & 63) as usize], + )); + } else { + encoded.push('='); + } + } + } + encoded +} + +fn truncated(text: &str, max_bytes: Option) -> String { + let Some(limit) = max_bytes.filter(|limit| text.len() > *limit) else { + return text.to_owned(); + }; + let mut end = limit; + while !text.is_char_boundary(end) { + end -= 1; + } + format!("{}…[+{} bytes]", &text[..end], text.len() - end) +} + +/// A cell as text formats show it; `null` is the text for SQL NULL. +pub(crate) fn display_cell(value: &RecallValue, null: &str, max_bytes: Option) -> String { + match value { + RecallValue::Null => null.to_owned(), + RecallValue::Integer(number) => number.to_string(), + RecallValue::Real(number) => format!("{number:?}"), + RecallValue::Text(text) => truncated(text, max_bytes), + RecallValue::Blob(data) => match std::str::from_utf8(data) { + Ok(text) => truncated(text, max_bytes), + Err(_) => format!("<{} bytes of binary data>", data.len()), + }, + } +} + +fn display_row(row: &[RecallValue], null: &str, max_bytes: Option) -> Vec { + let mut cells: Vec = Vec::with_capacity(row.len()); + for value in row { + cells.push(display_cell(value, null, max_bytes)); + } + cells +} + +fn column_json(column: &DescribedColumn) -> Value { + json!({ + "name": column.name, + "type": column.declared_type, + "nullable": column.nullable, + }) +} + +fn json_cell(value: &RecallValue) -> Value { + match value { + RecallValue::Null => Value::Null, + RecallValue::Integer(number) => json!(number), + RecallValue::Real(number) => json!(number), + RecallValue::Text(text) => json!(text), + RecallValue::Blob(data) => match std::str::from_utf8(data) { + Ok(text) => json!(text), + Err(_) => json!({ "base64": base64(data) }), + }, + } +} + +/// What the JSON document reports besides rows. +pub(crate) struct QueryContext<'a> { + pub(crate) target: Option<&'a str>, + pub(crate) store_path: &'a str, + pub(crate) limit: Option, + pub(crate) elapsed_ms: f64, +} + +/// The rendered result in `format`: `long`, `table`, `csv` or `json`. +pub(crate) fn render_result( + result: &QueryResult, + format: &str, + max_bytes: Option, + context: &QueryContext<'_>, +) -> String { + if format == JSON_FORMAT { + let rows: Vec = result + .rows + .iter() + .map(|row| Value::Array(row.iter().map(json_cell).collect())) + .collect(); + let document = json!({ + "backend": "logs", + "target": context.target, + "store": {"adapter": "sqlite", "database": context.store_path}, + "columns": result.columns, + "rows": rows, + "row_count": result.rows.len(), + "truncated": result.truncated, + "limit": context.limit, + "statistics": { + "elapsed_ms": context.elapsed_ms, + "rows_read": Value::Null, + "bytes_read": Value::Null, + "query_id": Value::Null, + }, + }); + return format!("{document}\n"); + } + let csv: bool = format == CSV_FORMAT; + let null: &str = if csv { "" } else { "NULL" }; + let cells: Vec> = result + .rows + .iter() + .map(|row| display_row(row, null, max_bytes)) + .collect(); + if csv { + return render_query_csv(&result.columns, &cells); + } + let display = QueryDisplay { + columns: result.columns.clone(), + rows: cells, + truncated: result.truncated, + limit: context + .limit + .map(|limit| i64::try_from(limit).unwrap_or(i64::MAX)), + }; + if format == TABLE_FORMAT { + render_query_table(&display) + } else { + render_query_long(&display) + } +} + +fn relation_note(name: &str) -> &'static str { + match name { + "invocations" => { + "one row per invocation; status is completed, failed, running, abandoned or unknown" + } + "statements" => "one row per statement; outcome no_terminal means no terminal fact arrived", + "audit_results" => "one row per evaluated audit", + "output_invocations" => "output completeness per invocation: gaps, drops, filtering, flush", + "output_tail" => "the last 200 output chunks of each invocation as text", + "output_transcripts" => "exact stdout, stderr and combined bytes per invocation", + "events" => "every lifecycle fact", + "diagnostics" => "structured diagnostics at or above the configured severity", + "artifacts" => "captured SQL and other artifacts, by reference", + "conflicts" => "record keys reused with different content; the first row is kept", + _ => "", + } +} + +/// The schema document: relations with columns, join keys and time semantics. +pub(crate) fn render_schema( + relations: &[DescribedRelation], + json_output: bool, + (store_path, retention_days): (&str, u32), +) -> String { + let filter_hint = + "filter invocation_started_at to bound scans; all rows of an invocation expire together"; + if json_output { + let items: Vec = relations + .iter() + .map(|relation| { + let mut item = Map::new(); + item.insert("name".to_owned(), json!(relation.name)); + item.insert("kind".to_owned(), json!(relation.kind)); + item.insert( + "description".to_owned(), + json!(relation_note(&relation.name)), + ); + item.insert( + "join_keys".to_owned(), + json!( + relation + .columns + .iter() + .filter(|column| JOIN_KEYS.contains(&column.name.as_str())) + .map(|column| column.name.clone()) + .collect::>() + ), + ); + item.insert( + "columns".to_owned(), + Value::Array(relation.columns.iter().map(column_json).collect()), + ); + Value::Object(item) + }) + .collect(); + let document = json!({ + "backend": "logs", + "dialect": "sqlite", + "store": {"adapter": "sqlite", "database": store_path}, + "retention_days": retention_days, + "time_semantics": "timestamps are ISO-8601 UTC text with microseconds", + "bounds": filter_hint, + "relations": items, + }); + return format!("{document}\n"); + } + let mut lines: Vec = vec![ + format!("Recall store {store_path} (SQLite dialect, retention {retention_days} days)"), + "Timestamps are ISO-8601 UTC text with microseconds; ".to_owned() + filter_hint + ".", + ]; + for relation in relations { + lines.push(String::new()); + let note = relation_note(&relation.name); + lines.push(if note.is_empty() { + format!("{} ({})", relation.name, relation.kind) + } else { + format!("{} ({}): {note}", relation.name, relation.kind) + }); + for column in &relation.columns { + let key = if JOIN_KEYS.contains(&column.name.as_str()) { + " join key" + } else { + "" + }; + let nullability = if column.nullable { "" } else { " not null" }; + lines.push(format!( + " {} {}{nullability}{key}", + column.name, + if column.declared_type.is_empty() { + "ANY" + } else { + &column.declared_type + } + )); + } + } + format!("{}\n", lines.join("\n")) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs new file mode 100644 index 0000000000..ac73b3dc10 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -0,0 +1,380 @@ +//! The invocation's Recall session: started before the command runs and closed after it returns. + +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, SystemTime}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyStringMethods; +use sqlbuild_cli::dispatch::models::CommandFacts; +use sqlbuild_recall::records::main::format_timestamp::format_timestamp; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use sqlbuild_recall::records::models::{InvocationContext, OutputStream, RowEnvelope}; +use sqlbuild_recall::schema::main::ddl_script::ddl_script; +use sqlbuild_recall::schema::models::{DdlOptions, Dialect}; +use sqlbuild_recall::settings::main::resolve_recall_settings::resolve_recall_settings; +use sqlbuild_recall::settings::models::{RecallSettings, SettingsRequest, StoreLocation}; +use sqlbuild_recall::writer::constants::{DEFAULT_HEARTBEAT_INTERVAL_MS, SETUP_BUDGET_MS}; +use sqlbuild_recall::writer::main::start_recall_session::start_recall_session; +use sqlbuild_recall::writer::models::{ + DeliverySummary, RecallSession, SessionConfig, SetupOutcome, +}; + +use crate::bindings::_helpers::native_host::executor::ExecutorHost; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::output_capture::records::invocation_context; +use crate::bindings::_helpers::recall::console::write_stderr; +use crate::bindings::_helpers::recall::constants::{ + DDL_NOTICE_PREFIX, HEARTBEAT_ENVIRONMENT, HOST_TOKEN_LENGTH, MINIMUM_HEARTBEAT_SECONDS, + NOTICE_INTERVAL_SECONDS, STDOUT_STREAM_INDEX, +}; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::rules::config_json::dumped_json; + +/// What the invocation records: its session, when enabled, and the resolved target and adapter. +#[derive(Clone, Debug, Default)] +pub(crate) struct InvocationRecall { + pub(crate) session: Option>, + pub(crate) target: Option, + pub(crate) adapter: Option, +} + +/// The text of an optional attribute value. +fn text_value(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.to_string_lossy().into_owned())) +} + +fn list_attribute(args: &Bound<'_, PyAny>, name: &str) -> PyResult> { + let value: Bound<'_, PyAny> = args.getattr(name)?; + let mut items: Vec = Vec::new(); + if value.is_none() { + return Ok(items); + } + for item in value.try_iter()? { + items.push(item?.str()?.to_string_lossy().into_owned()); + } + Ok(items) +} + +fn environment(name: &str) -> Option { + std::env::var_os(name).map(|value| value.to_string_lossy().into_owned()) +} + +/// `path`'s text when it exists; Recall metadata is best effort, so unreadable means unknown. +fn file_text(path: &Path) -> Option { + match std::fs::read_to_string(path) { + Ok(text) => Some(text), + Err(error) if error.kind() == ErrorKind::NotFound => None, + Err(_unreadable) => None, + } +} + +fn project_dir(args: &Bound<'_, PyAny>) -> PyResult { + let raw: PathBuf = match text_value(&args.getattr("project_dir")?)? { + Some(directory) => PathBuf::from(directory), + None => std::env::current_dir()?, + }; + Ok(std::fs::canonicalize(&raw).unwrap_or(raw)) +} + +/// The commit `HEAD` names, read from the repository files without running git. +fn git_revision(project_dir: &Path) -> Option { + let git: PathBuf = project_dir + .ancestors() + .map(|directory| directory.join(".git")) + .find(|candidate| candidate.is_dir())?; + let head: String = file_text(&git.join("HEAD"))?; + let Some(reference) = head.trim().strip_prefix("ref: ") else { + return Some(head.trim().to_owned()); + }; + if let Some(revision) = file_text(&git.join(reference)) { + return Some(revision.trim().to_owned()); + } + file_text(&git.join("packed-refs"))? + .lines() + .filter_map(|line| line.split_once(' ')) + .find(|(_, name)| *name == reference) + .map(|(revision, _)| revision.to_owned()) +} + +fn host_token() -> Option { + let name: String = file_text(Path::new("/etc/hostname")) + .or_else(|| environment("COMPUTERNAME")) + .or_else(|| environment("HOSTNAME"))?; + Some(sha256_hex(name.trim().as_bytes())[..HOST_TOKEN_LENGTH].to_owned()) +} + +/// The orchestrator context as JSON; an invalid environment context records none. +fn external_context(py: Python<'_>) -> PyResult> { + let context: Bound<'_, PyAny> = match invocation_context(py, None) { + Ok(context) => context, + Err(error) + if error.is_instance(py, &host::entry(py, ExecutorHost::OutputCaptureInputError)?) => + { + return Ok(None); + } + Err(error) => return Err(error), + }; + if !context.is_truthy()? { + return Ok(None); + } + Ok(dumped_json(&context)?.map(|encoded| encoded.to_string())) +} + +fn heartbeat_interval() -> Duration { + let default = Duration::from_millis(DEFAULT_HEARTBEAT_INTERVAL_MS); + let Some(value) = environment(HEARTBEAT_ENVIRONMENT) else { + return default; + }; + match value.parse::() { + Ok(seconds) if seconds.is_finite() && seconds >= MINIMUM_HEARTBEAT_SECONDS => { + Duration::from_secs_f64(seconds) + } + Ok(_) | Err(_) => default, + } +} + +fn invocation_facts( + py: Python<'_>, + (args, facts): (&Bound<'_, PyAny>, &CommandFacts), + (settings, project_dir): (&RecallSettings, &Path), +) -> PyResult { + let fingerprint_source = format!( + "{}\u{0}{}\u{0}{}", + settings.project_adapter.as_deref().unwrap_or_default(), + settings.destination_database.as_deref().unwrap_or_default(), + settings.destination_schema.as_deref().unwrap_or_default() + ); + Ok(InvocationContext { + command: facts.command.clone(), + subcommand: facts.subcommand.clone(), + selection: list_attribute(args, "select")?, + exclusion: list_attribute(args, "exclude")?, + project_name: settings.project_name.clone(), + project_dir_hash: Some(sha256_hex(project_dir.to_string_lossy().as_bytes())), + target: settings.target.clone(), + adapter: settings.project_adapter.clone(), + destination_database: settings.destination_database.clone(), + destination_schema: settings.destination_schema.clone(), + destination_fingerprint: Some(sha256_hex(fingerprint_source.as_bytes())), + sqlbuild_version: Some(env!("CARGO_PKG_VERSION").to_owned()), + git_revision: git_revision(project_dir), + git_dirty: None, + host_token: host_token(), + pid: Some(std::process::id()), + external_context: external_context(py)?, + recall_filter_active: false, + argv_digest: None, + }) +} + +/// Whether a notice for `store` is due, recording it best effort; a missing marker is due. +fn notice_due(project_dir: &Path, store: &str) -> bool { + let directory: PathBuf = project_dir.join("target"); + let marker: PathBuf = directory.join(format!( + "{DDL_NOTICE_PREFIX}{}", + &sha256_hex(store.as_bytes())[..HOST_TOKEN_LENGTH] + )); + let modified: std::io::Result = + std::fs::metadata(&marker).and_then(|metadata| metadata.modified()); + if let Ok(modified) = modified + && SystemTime::now() + .duration_since(modified) + .is_ok_and(|age| age < Duration::from_secs(NOTICE_INTERVAL_SECONDS)) + { + return false; + } + drop( + std::fs::create_dir_all(&directory) + .and_then(|()| std::fs::write(&marker, format_timestamp(SystemTime::now()))), + ); + true +} + +fn target_phrase(settings: &RecallSettings) -> String { + settings + .target + .as_deref() + .map_or_else(String::new, |target| format!(" for target `{target}`")) +} + +fn report_permission( + py: Python<'_>, + (settings, project_dir): (&RecallSettings, &Path), + reason: &str, +) -> PyResult<()> { + let StoreLocation::Sqlite { path } = &settings.store else { + return Ok(()); + }; + if !notice_due(project_dir, &path.to_string_lossy()) { + return Ok(()); + } + let script = ddl_script( + Dialect::Sqlite, + &DdlOptions { + database: None, + retention_days: settings.retention_days, + cluster: None, + }, + ); + write_stderr( + py, + &format!( + "Recall history is not being written{}: {reason}. An administrator can create the \ + store at {} with:\n\n{script}\n", + target_phrase(settings), + path.display() + ), + ) +} + +fn report_unsupported( + py: Python<'_>, + (settings, project_dir): (&RecallSettings, &Path), + (connection, adapter): (&str, &str), +) -> PyResult<()> { + if !notice_due(project_dir, connection) { + return Ok(()); + } + write_stderr( + py, + &format!( + "Recall history is not being written{}: connection `{connection}` uses adapter \ + `{adapter}`, which this version cannot store Recall history in.\n", + target_phrase(settings) + ), + ) +} + +fn started_session( + py: Python<'_>, + (args, identity, facts): (&Bound<'_, PyAny>, &Bound<'_, PyAny>, &CommandFacts), + (settings, project_dir): (&RecallSettings, &Path), +) -> PyResult>> { + let store_path: PathBuf = match &settings.store { + StoreLocation::Sqlite { path } => path.clone(), + StoreLocation::Unsupported { + connection, + adapter, + } => { + report_unsupported(py, (settings, project_dir), (connection, adapter))?; + return Ok(None); + } + }; + let Some(invocation_id) = text_value(&native_field(identity, "invocation_id")?)? else { + return Ok(None); + }; + let context = invocation_facts(py, (args, facts), (settings, project_dir))?; + let session = start_recall_session(SessionConfig { + store_path, + maintenance_owner: format!( + "{}:{}", + context.host_token.as_deref().unwrap_or("host"), + std::process::id() + ), + envelope: RowEnvelope { + invocation_id, + invocation_started_at: format_timestamp(SystemTime::now()), + producer_version: env!("CARGO_PKG_VERSION").to_owned(), + }, + context, + retention_days: settings.retention_days, + shutdown_timeout: Duration::from_millis(settings.shutdown_timeout_ms), + diagnostics_min_severity: settings.diagnostics_min_severity.clone(), + heartbeat_interval: heartbeat_interval(), + setup_budget: Duration::from_millis(SETUP_BUDGET_MS), + maintenance: true, + }); + let setup = py.detach(|| session.wait_for_setup(Duration::from_millis(SETUP_BUDGET_MS))); + if let Some(SetupOutcome::PermissionDenied { reason }) = &setup { + report_permission(py, (settings, project_dir), reason)?; + } + Ok(Some(Arc::new(session))) +} + +/// Begin recording when Recall is enabled; invalid settings record nothing and the command reports them. +pub(crate) fn start_invocation_recall( + py: Python<'_>, + args: &Bound<'_, PyAny>, + identity: &Bound<'_, PyAny>, + facts: &CommandFacts, +) -> PyResult { + let project_dir: PathBuf = project_dir(args)?; + let request = SettingsRequest { + project_dir: project_dir.clone(), + selected_target: text_value(&args.getattr("target")?)?, + }; + let Ok(settings) = resolve_recall_settings(&request) else { + return Ok(InvocationRecall::default()); + }; + let session: Option> = if settings.enabled && settings.project_name.is_some() + { + started_session(py, (args, identity, facts), (&settings, &project_dir))? + } else { + None + }; + Ok(InvocationRecall { + session, + target: settings.target, + adapter: settings.project_adapter, + }) +} + +/// Close the session: flush within the shutdown budget and warn when history is incomplete. +pub(crate) fn finish_invocation_recall(py: Python<'_>, recall: &InvocationRecall) -> PyResult<()> { + let Some(session) = &recall.session else { + return Ok(()); + }; + let summary: DeliverySummary = py.detach(|| session.close()); + let total = summary.total(); + let permission = matches!( + session.wait_for_setup(Duration::ZERO), + Some(SetupOutcome::PermissionDenied { .. }) + ); + if permission || (total.dropped == 0 && total.failed == 0) { + return Ok(()); + } + let reason = summary + .degraded_reason + .as_deref() + .map_or_else(String::new, |reason| format!(": {reason}")); + write_stderr( + py, + &format!( + "warning: Recall history for this run is incomplete ({} dropped, {} failed){reason}\n", + total.dropped, total.failed + ), + ) +} + +/// Forward bytes a process stream accepted to `session` when it records `invocation_id`. +pub(crate) fn forward_output( + session: Option<&RecallSession>, + invocation_id: &str, + (stream, data): (usize, &[u8]), +) { + let Some(session) = session.filter(|session| session.invocation_id() == invocation_id) else { + return; + }; + let stream = if stream == STDOUT_STREAM_INDEX { + OutputStream::Stdout + } else { + OutputStream::Stderr + }; + session.append_output(stream, data); +} + +/// Forward one diagnostic's JSON to `session` for `invocation_id`; unparsable ones are skipped. +pub(crate) fn forward_diagnostic( + session: Option<&RecallSession>, + invocation_id: &str, + diagnostic_json: &str, +) { + if let Some(session) = session.filter(|session| session.invocation_id() == invocation_id) { + drop(session.publish_diagnostic(diagnostic_json)); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs new file mode 100644 index 0000000000..29a00018e0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs @@ -0,0 +1,34 @@ +//! The native subscriber that publishes every lifecycle fact and artifact into Recall. + +use std::sync::Arc; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; +use pyo3::types::{PyString, PyStringMethods}; +use sqlbuild_recall::writer::models::RecallSession; + +use crate::bindings::_helpers::native_host::executor::ExecutorHost; +use crate::bindings::_helpers::native_host::host; + +/// Publishes lifecycle events and SQL artifacts of the active session; delivery is best effort. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct RecallLifecycleSink { + pub(crate) session: Arc, +} + +#[pymethods] +impl RecallLifecycleSink { + fn __call__(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { + let encoded = host::entry(py, ExecutorHost::LifecycleEventToJson)?.call1((event,))?; + let text = encoded + .downcast::()? + .to_string_lossy() + .into_owned(); + drop(self.session.publish_event(&text)); + Ok(()) + } + + #[pyo3(signature = (*, sql, statement_id=None, resource_id=None))] + fn record_sql(&self, sql: &str, statement_id: Option<&str>, resource_id: Option<&str>) { + self.session.publish_sql(sql, statement_id, resource_id); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 9fe49eb2dd..39f8e07f68 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -941,6 +941,7 @@ const CLI_ENTRYPOINT_HANDLERS: RecordSpec = RecordSpec { FieldSpec::required("run_scope"), FieldSpec::new("run_contract", FieldDefault::None), FieldSpec::new("run_refactor", FieldDefault::None), + FieldSpec::new("run_recall", FieldDefault::None), ], }; const COMMAND_WAREHOUSE_SCOPE: RecordSpec = RecordSpec { @@ -1531,6 +1532,7 @@ const CLI_COMMAND: EnumSpec = EnumSpec { ("CONTRACT", "contract"), ("RENAME", "rename"), ("MV", "mv"), + ("RECALL", "recall"), ], }; const PLAYGROUND_TEMPLATE: EnumSpec = EnumSpec { diff --git a/fensu.toml b/fensu.toml index 5dc49662eb..8a3dfb12ec 100644 --- a/fensu.toml +++ b/fensu.toml @@ -555,7 +555,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=19", - "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=26", + "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=28", "crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=26", @@ -960,6 +960,9 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs=10", + "crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs=1", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index bddfa169bb..2aaac2e353 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3321,6 +3321,14 @@ def invocation_context_from_environment( *, environment: Mapping[str, str] | None = None ) -> Mapping[str, object]: ... +class RecallLifecycleSink: + """The invocation's Recall session as lifecycle and SQL artifact subscribers.""" + + def __call__(self, event: LifecycleEvent) -> None: ... + def record_sql( + self, *, sql: str, statement_id: str | None = None, resource_id: str | None = None + ) -> None: ... + class LocalFilesystemComputeLogStorage: """Ordered fsync capture storage subject to underlying filesystem and hardware guarantees.""" @@ -3331,6 +3339,7 @@ class LocalFilesystemComputeLogStorage: root: Path | None = None, retention_count: int | None = 20, ) -> None: ... + def attach_recall(self, recall: RecallLifecycleSink) -> None: ... @property def root(self) -> Path: ... def start_capture(self, metadata: CaptureMetadata) -> None: ... @@ -3644,6 +3653,18 @@ class CliNamespace: rules_command: str | None rules_rule_selector: str | None rules_skills_check: bool + recall_command: str | None + recall_sql: str | None + recall_backend: str + recall_format: str + recall_limit: int + recall_no_limit: bool + recall_max_cell_bytes: int | None + recall_state: bool + recall_relation: str | None + recall_invocation: str + recall_stream: str + recall_tail: int | None format_check: bool format_diff: bool format_fixtures_only: bool @@ -8721,6 +8742,7 @@ class CliEntrypointHandlers(NativeRecord): run_scope: ScopeCommandHandler run_contract: Callable[..., int] | None run_refactor: Callable[..., int] | None + run_recall: Callable[..., int] | None def __init__( self, run_compile: Callable[[CompileCommandRequest], int], @@ -8755,6 +8777,7 @@ class CliEntrypointHandlers(NativeRecord): run_scope: ScopeCommandHandler, run_contract: Callable[..., int] | None = None, run_refactor: Callable[..., int] | None = None, + run_recall: Callable[..., int] | None = None, ) -> None: ... class CommandWarehouseScope(NativeRecord): @@ -9554,6 +9577,7 @@ class CliCommand(StrEnum): CONTRACT = "contract" RENAME = "rename" MV = "mv" + RECALL = "recall" class ScopeCommandHandler(Protocol): def __call__(self, *, request: ScopeCommandRequest) -> int: ... diff --git a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py index 926b51d22c..c6f6242d35 100644 --- a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py +++ b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py @@ -10,6 +10,7 @@ COMPUTE_LOG_FORMAT_VERSION, CliNamespace, LocalFilesystemComputeLogStorage, + RecallLifecycleSink, supports_color, ) from sqlbuild.cli.commands._helpers.entry.compute_log_diagnostics import ( @@ -31,6 +32,7 @@ def dispatch_with_compute_logs( identity: ExecutionIdentity, operation: Callable[[], int], creates_project: bool, + recall: RecallLifecycleSink | None = None, ) -> int: """Run one composed dispatch while retaining exact process streams best-effort.""" @@ -56,6 +58,8 @@ def dispatch_with_compute_logs( storage: LocalFilesystemComputeLogStorage | None = None try: storage = LocalFilesystemComputeLogStorage(project_dir=project_dir) + if recall is not None: + storage.attach_recall(recall) storage.start_capture(metadata) except Exception as error: failures: list[tuple[Exception, str]] = [(error, "capture_open")] diff --git a/tests/integration/src/sqlbuild/cli/recall/__init__.py b/tests/integration/src/sqlbuild/cli/recall/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py new file mode 100644 index 0000000000..3fc523131b --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -0,0 +1,74 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class RecallOutputTestCase: + description: str + args: tuple[str, ...] + expected_stream: str + + +@dataclass(frozen=True) +class CombinedOutputTestCase: + description: str + expected_order: tuple[bytes, ...] + + +@dataclass(frozen=True) +class RecordedFactsTestCase: + description: str + sql: str + expected_rows: list[list[object]] + + +@dataclass(frozen=True) +class JsonQueryTestCase: + description: str + args: tuple[str, ...] + expected_row_count: int + expected_truncated: bool + expected_columns: list[str] + + +@dataclass(frozen=True) +class TextOutputTestCase: + description: str + args: tuple[str, ...] + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class RecallRejectionTestCase: + description: str + project_suffix: str + args: tuple[str, ...] + expected_code: str + + +@dataclass(frozen=True) +class DisabledRecallTestCase: + description: str + project_suffix: str + expected_store_exists: bool + + +@dataclass(frozen=True) +class ReadOnlyStoreTestCase: + description: str + expected_fragments: tuple[str, ...] + expected_repeat_notice: bool + + +@dataclass(frozen=True) +class AbandonedRunTestCase: + description: str + heartbeat_seconds: str + expected_status: str + expected_output_complete: int + + +@dataclass(frozen=True) +class ConcurrentBuildsTestCase: + description: str + targets: tuple[str, ...] + expected_status: str diff --git a/tests/integration/src/sqlbuild/cli/recall/conftest.py b/tests/integration/src/sqlbuild/cli/recall/conftest.py new file mode 100644 index 0000000000..fef97213ac --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/recall/conftest.py @@ -0,0 +1,16 @@ +"""Built synthetic projects for local Recall tests.""" + +import subprocess +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.recall.helpers import run, write_project + + +@pytest.fixture(name="built_project") +def fixture_built_project(tmp_path: Path) -> tuple[Path, subprocess.CompletedProcess[bytes]]: + project_dir: Path = write_project(tmp_path / "orders_shop") + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + assert built.returncode == 0, built.stderr.decode(errors="replace") + return project_dir, built diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py new file mode 100644 index 0000000000..e7fa13564b --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -0,0 +1,187 @@ +"""Synthetic DuckDB projects and process helpers for local Recall tests.""" + +import json +import os +import signal +import stat +import subprocess +import sys +import time +from pathlib import Path + +PYTHON_CLI: Path = Path(sys.executable).with_name("sqlbuild-python") +EMITTED_STDOUT: bytes = b"binary \x00\xff\xfe chunk\n" +EMITTED_STDERR: bytes = "warning \u2713 from the task\n".encode() +EMITTED_TAIL: bytes = b"after the warning\n" +EMPTY_RESULT: bytes = b'{"rows": []}' +PROJECT_TOML: str = ( + 'name = "orders_shop"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connections.local]\n" + 'database = "warehouse.duckdb"\n\n' + "[targets.dev]\n" + 'connection = "local"\n' + 'schema = "dev"\n' +) +ORDERS_MODEL: str = ( + 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + "SELECT 1 AS order_id, CAST('2026-04-01' AS DATE) AS ordered_on\n" +) +EMIT_TASK: str = ( + '"""A task that writes exact bytes to both streams."""\n\n' + "import os\n" + "import sys\n" + "import time\n\n" + "from sqlbuild.tasks import task\n\n\n" + "@task()\n" + "def emit_bytes(ctx):\n" + ' """Write binary stdout, UTF-8 stderr, then more stdout."""\n\n' + " sys.stdout.flush()\n" + f" sys.stdout.buffer.write({EMITTED_STDOUT!r})\n" + " sys.stdout.buffer.flush()\n" + f" sys.stderr.write({EMITTED_STDERR.decode()!r})\n" + " sys.stderr.flush()\n" + f" sys.stdout.write({EMITTED_TAIL.decode()!r})\n" + " sys.stdout.flush()\n" + ' time.sleep(float(os.environ.get("ORDERS_TASK_SLEEP_SECONDS", "0")))\n' + " return ctx.result()\n" +) + + +def write_project(project_dir: Path, *, extra_toml: str = "", targets: int = 1) -> Path: + """Write a DuckDB project with one model and one byte-emitting task.""" + + toml: str = PROJECT_TOML + extra_toml + for index in range(1, targets): + toml += ( + f"\n[connections.local_{index}]\n" + f'database = "warehouse_{index}.duckdb"\n\n' + f"[targets.dev_{index}]\n" + f'connection = "local_{index}"\n' + 'schema = "dev"\n' + ) + files: dict[str, str] = { + "sqlbuild_project.toml": toml, + "models/orders.sql": ORDERS_MODEL, + "python/tasks/emit.py": EMIT_TASK, + } + for relative, contents in files.items(): + path: Path = project_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + return project_dir + + +def environment(**overrides: str) -> dict[str, str]: + """The process environment for one CLI run.""" + + return {**os.environ, "NO_COLOR": "1", **overrides} + + +def run(project_dir: Path, *args: str, **env: str) -> subprocess.CompletedProcess[bytes]: + """Run the Python CLI once in the project and capture exact streams.""" + + return subprocess.run( + [str(PYTHON_CLI), *args], + cwd=project_dir, + capture_output=True, + check=False, + env=environment(**env), + timeout=600, + ) + + +def start(project_dir: Path, *args: str, **env: str) -> subprocess.Popen[bytes]: + """Start the Python CLI in its own process group.""" + + return subprocess.Popen( + [str(PYTHON_CLI), *args], + cwd=project_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment(**env), + start_new_session=True, + ) + + +def recall_rows(project_dir: Path, sql: str) -> list[list[object]]: + """The rows `sqb recall query --json` returns.""" + + completed: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "query", "--json", "--no-limit", sql + ) + assert completed.returncode == 0, completed.stderr.decode(errors="replace") + return json.loads(completed.stdout)["rows"] + + +def wait_for(predicate_sql: str, project_dir: Path, *, timeout: float) -> list[list[object]]: + """Poll the store until `predicate_sql` returns a row; a store not yet created has none.""" + + deadline: float = time.monotonic() + timeout + rows: list[list[object]] = [] + while time.monotonic() < deadline and not rows: + time.sleep(0.2) + completed: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "query", "--json", predicate_sql + ) + rows = json.loads(completed.stdout or EMPTY_RESULT)["rows"] + return rows + + +def build_with_read_only_logs( + project_dir: Path, +) -> tuple[subprocess.CompletedProcess[bytes], subprocess.CompletedProcess[bytes]]: + """Two builds while the project's `logs` directory cannot be written.""" + + logs: Path = project_dir / "logs" + logs.mkdir() + logs.chmod(stat.S_IRUSR | stat.S_IXUSR) + try: + return run(project_dir, "build"), run(project_dir, "build") + finally: + logs.chmod(stat.S_IRWXU) + + +def killed_build(project_dir: Path, *, heartbeat_seconds: str) -> tuple[str, list[list[object]]]: + """The running invocation a SIGKILLed build left, and the store's rows for it afterwards.""" + + process: subprocess.Popen[bytes] = start( + project_dir, + "build", + SQLBUILD_RECALL_HEARTBEAT_SECONDS=heartbeat_seconds, + ORDERS_TASK_SLEEP_SECONDS="60", + ) + running: list[list[object]] = wait_for( + "SELECT invocation_id FROM invocations WHERE status = 'running' " + "AND invocation_id IN (SELECT invocation_id FROM recall_output)", + project_dir, + timeout=120, + ) + os.killpg(process.pid, signal.SIGKILL) + _ = process.wait(timeout=30) + time.sleep(5 * float(heartbeat_seconds)) + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT i.invocation_id, i.status, i.ended_at IS NOT NULL, o.complete " + "FROM invocations i JOIN output_invocations o USING (invocation_id)", + ) + return str(running[0][0]), recorded + + +def concurrent_builds(project_dir: Path, targets: tuple[str, ...]) -> tuple[list[int], int, int]: + """Exit codes of concurrent builds, reads made meanwhile and unexpected read failures.""" + + _ = run(project_dir, "recall", "query", "SELECT 1") + builds: list[subprocess.Popen[bytes]] = [ + start(project_dir, "build", "--target", target) for target in targets + ] + reads: int = 0 + unexpected_failures: int = 0 + while any(build.poll() is None for build in builds): + reader: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "query", "--json", "SELECT COUNT(*) FROM invocations" + ) + reads += 1 + unexpected_failures += int(reader.returncode != 0 and b"[C971]" not in reader.stderr) + return [build.wait(timeout=600) for build in builds], reads, unexpected_failures diff --git a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py new file mode 100644 index 0000000000..60494c14bb --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py @@ -0,0 +1,395 @@ +import json +import os +import subprocess +from pathlib import Path + +import jsonschema +import pytest + +from tests.integration.src.sqlbuild.cli.recall._test_types import ( + AbandonedRunTestCase, + CombinedOutputTestCase, + ConcurrentBuildsTestCase, + DisabledRecallTestCase, + JsonQueryTestCase, + ReadOnlyStoreTestCase, + RecallOutputTestCase, + RecallRejectionTestCase, + RecordedFactsTestCase, + TextOutputTestCase, +) +from tests.integration.src.sqlbuild.cli.recall.helpers import ( + EMITTED_STDERR, + EMITTED_STDOUT, + EMITTED_TAIL, + build_with_read_only_logs, + concurrent_builds, + killed_build, + recall_rows, + run, + write_project, +) + +RECALL_QUERY_SCHEMA: dict[str, object] = { + "type": "object", + "required": [ + "backend", + "target", + "store", + "columns", + "rows", + "row_count", + "truncated", + "limit", + "statistics", + ], + "additionalProperties": False, + "properties": { + "backend": {"const": "logs"}, + "target": {"type": ["string", "null"]}, + "store": { + "type": "object", + "required": ["adapter", "database"], + "properties": {"adapter": {"const": "sqlite"}, "database": {"type": "string"}}, + }, + "columns": {"type": "array", "items": {"type": "string"}}, + "rows": {"type": "array", "items": {"type": "array"}}, + "row_count": {"type": "integer", "minimum": 0}, + "truncated": {"type": "boolean"}, + "limit": {"type": ["integer", "null"]}, + "statistics": { + "type": "object", + "required": ["elapsed_ms", "rows_read", "bytes_read", "query_id"], + "properties": { + "elapsed_ms": {"type": "number"}, + "rows_read": {"type": "null"}, + "bytes_read": {"type": "null"}, + "query_id": {"type": "null"}, + }, + }, + }, +} +DISABLED_TARGET: str = "\n[targets.dev.recall]\nenabled = false\n" +POSIX_PERMISSIONS: bool = os.name == "posix" and os.geteuid() != 0 + + +@pytest.mark.parametrize( + "test_case", + [ + RecallOutputTestCase( + description="stdout", args=("--stream", "stdout"), expected_stream="stdout" + ), + RecallOutputTestCase( + description="stderr", args=("--stream", "stderr"), expected_stream="stderr" + ), + ], + ids=lambda case: case.description, +) +def test_given_a_build_with_binary_output_when_recalled_then_each_stream_is_byte_exact( + test_case: RecallOutputTestCase, + built_project: tuple[Path, subprocess.CompletedProcess[bytes]], +) -> None: + project_dir, built = built_project + + recalled: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "output", "last", *test_case.args + ) + + assert recalled.returncode == 0, recalled.stderr + assert recalled.stdout == getattr(built, test_case.expected_stream) + assert recalled.stderr == b"" + + +@pytest.mark.parametrize( + "test_case", + [ + CombinedOutputTestCase( + description="stdout, then the stderr warning, then more stdout", + expected_order=(EMITTED_STDOUT, EMITTED_STDERR, EMITTED_TAIL), + ), + ], + ids=lambda case: case.description, +) +def test_given_interleaved_writes_when_recalled_combined_then_order_is_preserved( + test_case: CombinedOutputTestCase, + built_project: tuple[Path, subprocess.CompletedProcess[bytes]], +) -> None: + project_dir, built = built_project + + combined: bytes = run(project_dir, "recall", "output", "last").stdout + transcript: list[list[object]] = recall_rows( + project_dir, + "SELECT hex(stdout), hex(stderr) FROM output_transcripts t " + "JOIN output_invocations o USING (invocation_id) WHERE o.complete = 1", + ) + + positions: list[int] = [combined.index(piece) for piece in test_case.expected_order] + assert positions == sorted(positions) + assert len(combined) == len(built.stdout) + len(built.stderr) + assert transcript == [[built.stdout.hex().upper(), built.stderr.hex().upper()]] + + +@pytest.mark.parametrize( + "test_case", + [ + RecordedFactsTestCase( + description="the invocation carries its command, target and adapter", + sql="SELECT status, command, target, adapter, exit_code FROM invocations", + expected_rows=[["completed", "build", "dev", "duckdb", 0]], + ), + RecordedFactsTestCase( + description="every statement completed and kept its SQL", + sql="SELECT COUNT(*) > 0, SUM(outcome = 'completed') = COUNT(*), " + "SUM(sql_available) = COUNT(*) FROM statements", + expected_rows=[[1, 1, 1]], + ), + RecordedFactsTestCase( + description="the started event names target and adapter", + sql="SELECT json_extract(payload, '$.target'), json_extract(payload, '$.adapter') " + "FROM events WHERE event_type = 'invocation_started'", + expected_rows=[["dev", "duckdb"]], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_build_when_recalled_then_invocation_statements_and_events_are_queryable( + test_case: RecordedFactsTestCase, + built_project: tuple[Path, subprocess.CompletedProcess[bytes]], +) -> None: + project_dir, _ = built_project + + rows: list[list[object]] = recall_rows(project_dir, test_case.sql) + + assert rows == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + JsonQueryTestCase( + description="a limited query reports truncation", + args=( + "--limit", + "1", + "SELECT invocation_id, data FROM recall_output ORDER BY sequence", + ), + expected_row_count=1, + expected_truncated=True, + expected_columns=["invocation_id", "data"], + ), + JsonQueryTestCase( + description="an unlimited query is complete", + args=("--no-limit", "SELECT COUNT(*) AS total FROM invocations"), + expected_row_count=1, + expected_truncated=False, + expected_columns=["total"], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_recall_query_when_rendered_as_json_then_it_matches_the_schema( + test_case: JsonQueryTestCase, + built_project: tuple[Path, subprocess.CompletedProcess[bytes]], +) -> None: + project_dir, _ = built_project + + queried: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "query", "--json", *test_case.args + ) + + document: dict[str, object] = json.loads(queried.stdout) + jsonschema.validate(document, RECALL_QUERY_SCHEMA) + assert document["row_count"] == test_case.expected_row_count + assert document["truncated"] is test_case.expected_truncated + assert document["columns"] == test_case.expected_columns + + +@pytest.mark.parametrize( + "test_case", + [ + TextOutputTestCase( + description="long cells are truncated with the remaining byte count", + args=( + "query", + "--max-cell-bytes", + "8", + "SELECT 'first line' || char(10) || 'second line' AS note", + ), + expected_fragments=("note | first li…[+14 bytes]", "1 row"), + ), + TextOutputTestCase( + description="the schema lists views and join keys", + args=("schema",), + expected_fragments=( + "output_transcripts (view)", + "invocation_id TEXT not null join key", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_recall_text_output_when_rendered_then_cells_and_schema_are_readable( + test_case: TextOutputTestCase, + built_project: tuple[Path, subprocess.CompletedProcess[bytes]], +) -> None: + project_dir, _ = built_project + + rendered: subprocess.CompletedProcess[bytes] = run(project_dir, "recall", *test_case.args) + + stdout: str = rendered.stdout.decode() + assert rendered.returncode == 0 + assert [fragment in stdout for fragment in test_case.expected_fragments] == [ + True for _ in test_case.expected_fragments + ] + + +@pytest.mark.parametrize( + "test_case", + [ + RecallRejectionTestCase( + description="disabled target", + project_suffix=DISABLED_TARGET, + args=("recall", "query", "SELECT 1"), + expected_code="C970", + ), + RecallRejectionTestCase( + description="unknown invocation", + project_suffix="", + args=("recall", "output", "no-such-invocation"), + expected_code="C972", + ), + RecallRejectionTestCase( + description="write statement", + project_suffix="", + args=("recall", "query", "DELETE FROM recall_output"), + expected_code="C974", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_recall_requests_when_run_then_coded_errors_are_reported( + test_case: RecallRejectionTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop", extra_toml=test_case.project_suffix) + _ = run(project_dir, "build") + + rejected: subprocess.CompletedProcess[bytes] = run(project_dir, *test_case.args) + + assert rejected.returncode == 1 + assert f"[{test_case.expected_code}]" in rejected.stderr.decode() + + +@pytest.mark.parametrize( + "test_case", + [ + DisabledRecallTestCase( + description="a disabled target writes no store", + project_suffix=DISABLED_TARGET, + expected_store_exists=False, + ), + DisabledRecallTestCase( + description="the default writes the local store", + project_suffix="", + expected_store_exists=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_recall_settings_when_building_then_the_local_store_follows_them( + test_case: DisabledRecallTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop", extra_toml=test_case.project_suffix) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + assert built.returncode == 0 + assert (project_dir / "logs" / "recall.sqlite3").exists() is test_case.expected_store_exists + + +@pytest.mark.skipif(not POSIX_PERMISSIONS, reason="needs POSIX permissions as a non-root user") +@pytest.mark.parametrize( + "test_case", + [ + ReadOnlyStoreTestCase( + description="the notice carries the DDL once and the build still succeeds", + expected_fragments=( + "Recall history is not being written for target `dev`", + 'CREATE TABLE IF NOT EXISTS "recall_output"', + 'CREATE VIEW IF NOT EXISTS "invocations"', + ), + expected_repeat_notice=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_read_only_logs_directory_when_building_then_ddl_is_printed_and_exit_is_zero( + test_case: ReadOnlyStoreTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + + built, repeated = build_with_read_only_logs(project_dir) + + stderr: str = built.stderr.decode(errors="replace") + assert (built.returncode, repeated.returncode) == (0, 0) + assert [fragment in stderr for fragment in test_case.expected_fragments] == [ + True for _ in test_case.expected_fragments + ] + assert "Completed successfully" in built.stdout.decode(errors="replace") + assert ( + "Recall history is not being written" in repeated.stderr.decode(errors="replace") + ) is test_case.expected_repeat_notice + + +@pytest.mark.parametrize( + "test_case", + [ + AbandonedRunTestCase( + description="a SIGKILLed build is abandoned with incomplete output", + heartbeat_seconds="0.3", + expected_status="abandoned", + expected_output_complete=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_killed_build_when_heartbeats_stop_then_the_invocation_is_abandoned( + test_case: AbandonedRunTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + + running_id, recorded = killed_build(project_dir, heartbeat_seconds=test_case.heartbeat_seconds) + + assert recorded == [ + [running_id, test_case.expected_status, 1, test_case.expected_output_complete] + ] + + +@pytest.mark.parametrize( + "test_case", + [ + ConcurrentBuildsTestCase( + description="four builds and a polling reader share one store", + targets=("dev", "dev_1", "dev_2", "dev_3"), + expected_status="completed", + ), + ], + ids=lambda case: case.description, +) +def test_given_four_concurrent_builds_and_a_reader_when_run_then_every_run_is_recorded( + test_case: ConcurrentBuildsTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop", targets=len(test_case.targets)) + + exit_codes, reads, unexpected_failures = concurrent_builds(project_dir, test_case.targets) + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT i.target, i.status, o.complete FROM invocations i " + "JOIN output_invocations o USING (invocation_id) ORDER BY i.target", + ) + + assert exit_codes == [0 for _ in test_case.targets] + assert reads > 0 + assert unexpected_failures == 0 + assert recorded == [ + [target, test_case.expected_status, 1] for target in sorted(test_case.targets) + ] From c88f529222fc23280cb8c1504f56e0e47cc49be0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:55:54 +0100 Subject: [PATCH 05/19] perf: encode lifecycle events natively for Recall and hash rows without per-value copies --- .../src/bindings/_helpers/recall/constants.rs | 30 +++++ .../src/bindings/_helpers/recall/events.rs | 126 ++++++++++++++++++ .../src/bindings/_helpers/recall/mod.rs | 1 + .../src/bindings/_helpers/recall/sinks.rs | 5 + .../src/records/_helpers/ids.rs | 44 +++--- .../src/records/main/event_object_row.rs | 24 ++++ .../src/records/main/event_row.rs | 14 +- .../sqlbuild-recall/src/records/main/mod.rs | 1 + .../sqlbuild-recall/src/schema/constants.rs | 2 + .../src/writer/_helpers/session.rs | 13 +- 10 files changed, 227 insertions(+), 33 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs create mode 100644 crates/sqlbuild-recall/src/records/main/event_object_row.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs index b11bc71720..a10f4d306d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -55,3 +55,33 @@ pub(crate) const RECALL_OPTIONS: [&str; 13] = [ "stream", "tail", ]; +/// Required text envelope fields of a known lifecycle event. +pub(crate) const EVENT_TEXT_FIELDS: [&str; 5] = [ + "event_id", + "event_type", + "producer", + "producer_version", + "invocation_id", +]; +/// Optional correlation fields of a known lifecycle event. +pub(crate) const EVENT_OPTIONAL_TEXT_FIELDS: [&str; 5] = [ + "run_id", + "resource_id", + "resource_attempt_id", + "operation_id", + "statement_id", +]; +/// The first lifecycle schema version carrying invocation sequence and external context. +pub(crate) const INVOCATION_METADATA_SCHEMA_VERSION: i64 = 2; +/// `datetime` attributes of an ISO-8601 timestamp, most significant first. +pub(crate) const TIMESTAMP_PARTS: [&str; 7] = [ + "year", + "month", + "day", + "hour", + "minute", + "second", + "microsecond", +]; +/// `str(datetime.UTC)`. +pub(crate) const UTC_ZONE_NAME: &str = "UTC"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs new file mode 100644 index 0000000000..eaeddf8947 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs @@ -0,0 +1,126 @@ +//! Lifecycle events as the JSON object `lifecycle_event_to_json` encodes, built natively. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyMapping, PyString, + PyStringMethods, PyTuple, +}; +use serde_json::{Map, Number, Value}; + +use crate::bindings::_helpers::native_host::executor::ExecutorHost; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::recall::constants::{ + EVENT_OPTIONAL_TEXT_FIELDS, EVENT_TEXT_FIELDS, INVOCATION_METADATA_SCHEMA_VERSION, + TIMESTAMP_PARTS, UTC_ZONE_NAME, +}; +use crate::bindings::_helpers::records::fields::field as native_field; + +fn text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.to_string_lossy().into_owned())) +} + +fn json_items(dict: &Bound<'_, PyDict>) -> PyResult> { + let mut entries: Map = Map::new(); + for (key, item) in dict.iter() { + let Ok(key) = key.downcast::() else { + return Ok(None); + }; + let Some(item) = json_value(&item)? else { + return Ok(None); + }; + entries.insert(key.to_str()?.to_owned(), item); + } + Ok(Some(Value::Object(entries))) +} + +/// A lifecycle JSON value as `lifecycle_event_to_json` encodes it; `None` for anything else. +fn json_value(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(Some(Value::Null)); + } + if let Ok(flag) = value.downcast::() { + return Ok(Some(Value::Bool(flag.is_true()))); + } + if value.is_instance_of::() { + return Ok(value + .extract::() + .map_or(None, |number| Some(number.into()))); + } + if value.is_instance_of::() { + return Ok(Number::from_f64(value.extract::()?).map(Value::Number)); + } + if let Ok(text) = value.downcast::() { + return Ok(Some(Value::String(text.to_str()?.to_owned()))); + } + if value.is_instance_of::() || value.is_instance_of::() { + let mut items: Vec = Vec::new(); + for item in value.try_iter()? { + let Some(item) = json_value(&item?)? else { + return Ok(None); + }; + items.push(item); + } + return Ok(Some(Value::Array(items))); + } + if let Ok(dict) = value.downcast::() { + return json_items(dict); + } + if let Ok(mapping) = value.downcast::() { + let dict: Bound<'_, PyDict> = PyDict::new(value.py()); + dict.update(mapping)?; + return json_items(&dict); + } + Ok(None) +} + +/// `occurred_at` as microsecond ISO-8601 with `Z`, when it is a UTC datetime. +fn utc_timestamp(value: &Bound<'_, PyAny>) -> PyResult> { + let zone: Bound<'_, PyAny> = native_field(value, "tzinfo")?; + if text(&zone)?.as_deref() != Some(UTC_ZONE_NAME) { + return Ok(None); + } + let mut parts: [i64; 7] = [0; 7]; + for (part, name) in parts.iter_mut().zip(TIMESTAMP_PARTS) { + *part = native_field(value, name)?.extract()?; + } + let [year, month, day, hour, minute, second, microsecond] = parts; + Ok(Some(format!( + "{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}.{microsecond:06}Z" + ))) +} + +/// The canonical JSON object of a known lifecycle event; `None` when only Python can encode it. +pub(crate) fn event_object(event: &Bound<'_, PyAny>) -> PyResult>> { + if !event.is_instance(&host::entry(event.py(), ExecutorHost::LifecycleEvent)?)? { + return Ok(None); + } + let mut object: Map = Map::new(); + for name in EVENT_TEXT_FIELDS { + object.insert(name.to_owned(), text(&native_field(event, name)?)?.into()); + } + for name in EVENT_OPTIONAL_TEXT_FIELDS { + object.insert(name.to_owned(), text(&native_field(event, name)?)?.into()); + } + let schema_version: i64 = native_field(event, "schema_version")?.extract()?; + object.insert("schema_version".to_owned(), schema_version.into()); + let Some(occurred_at) = utc_timestamp(&native_field(event, "occurred_at")?)? else { + return Ok(None); + }; + object.insert("occurred_at".to_owned(), occurred_at.into()); + let Some(payload) = json_value(&native_field(event, "payload")?)? else { + return Ok(None); + }; + object.insert("payload".to_owned(), payload); + if schema_version >= INVOCATION_METADATA_SCHEMA_VERSION { + let sequence: Option = native_field(event, "invocation_sequence")?.extract()?; + object.insert("invocation_sequence".to_owned(), sequence.into()); + let Some(context) = json_value(&native_field(event, "external_context")?)? else { + return Ok(None); + }; + object.insert("external_context".to_owned(), context); + } + Ok(Some(object)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs index c0c5073d63..e05fcbcf73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod command; pub(crate) mod console; pub(crate) mod constants; +pub(crate) mod events; pub(crate) mod registration; pub(crate) mod rendering; pub(crate) mod session; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs index 29a00018e0..b324b6fc32 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs @@ -8,6 +8,7 @@ use sqlbuild_recall::writer::models::RecallSession; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::recall::events::event_object; /// Publishes lifecycle events and SQL artifacts of the active session; delivery is best effort. #[pyclass(module = "sqlbuild._native", frozen)] @@ -18,6 +19,10 @@ pub(crate) struct RecallLifecycleSink { #[pymethods] impl RecallLifecycleSink { fn __call__(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { + if let Some(object) = event_object(event)? { + drop(self.session.publish_event_object(&object)); + return Ok(()); + } let encoded = host::entry(py, ExecutorHost::LifecycleEventToJson)?.call1((event,))?; let text = encoded .downcast::()? diff --git a/crates/sqlbuild-recall/src/records/_helpers/ids.rs b/crates/sqlbuild-recall/src/records/_helpers/ids.rs index 903dc88854..d98bd32489 100644 --- a/crates/sqlbuild-recall/src/records/_helpers/ids.rs +++ b/crates/sqlbuild-recall/src/records/_helpers/ids.rs @@ -16,37 +16,41 @@ pub(crate) fn diagnostic_record_id(invocation_id: &str, sequence: u64) -> String format!("{invocation_id}:{DIAGNOSTIC_RECORD_ID_INFIX}:{sequence}") } -fn framed(tag: &[u8; 1], bytes: &[u8]) -> Vec { - let mut frame: Vec = Vec::with_capacity(bytes.len() + 9); - frame.extend_from_slice(tag); - frame.extend_from_slice(&(bytes.len() as u64).to_le_bytes()); - frame.extend_from_slice(bytes); - frame +fn framed(hasher: Sha256, tag: &[u8; 1], bytes: &[u8]) -> Sha256 { + hasher + .chain_update(tag) + .chain_update((bytes.len() as u64).to_le_bytes()) + .chain_update(bytes) } -fn encoded(value: &RecallValue) -> Vec { +fn valued(hasher: Sha256, value: &RecallValue) -> Sha256 { match value { - RecallValue::Null => b"0".to_vec(), - RecallValue::Integer(number) => [b"i".as_slice(), &number.to_le_bytes()].concat(), - RecallValue::Real(number) => [b"r".as_slice(), &number.to_bits().to_le_bytes()].concat(), - RecallValue::Text(text) => framed(b"t", text.as_bytes()), - RecallValue::Blob(data) => framed(b"b", data), + RecallValue::Null => hasher.chain_update(b"0"), + RecallValue::Integer(number) => { + hasher.chain_update(b"i").chain_update(number.to_le_bytes()) + } + RecallValue::Real(number) => hasher + .chain_update(b"r") + .chain_update(number.to_bits().to_le_bytes()), + RecallValue::Text(text) => framed(hasher, b"t", text.as_bytes()), + RecallValue::Blob(data) => framed(hasher, b"b", data), } } -/// Hex SHA-256 over every named value except `ingested_at` and `content_sha256`, by column name. +/// Hex SHA-256 over every non-NULL named value except `ingested_at` and `content_sha256`, by column name; a NULL column hashes like an absent one. pub(crate) fn content_sha256(values: &[(&str, RecallValue)]) -> String { let mut sorted: Vec<&(&str, RecallValue)> = values .iter() - .filter(|(name, _)| !UNHASHED_COLUMNS.contains(name)) + .filter(|(name, value)| { + !UNHASHED_COLUMNS.contains(name) && !matches!(value, RecallValue::Null) + }) .collect(); sorted.sort_by(|left, right| left.0.cmp(right.0)); - let mut hasher = Sha256::new(); - for (name, value) in sorted { - hasher.update(framed(b"n", name.as_bytes())); - hasher.update(encoded(value)); - } - hasher + sorted + .into_iter() + .fold(Sha256::new(), |hasher, (name, value)| { + valued(framed(hasher, b"n", name.as_bytes()), value) + }) .finalize() .iter() .map(|byte| format!("{byte:02x}")) diff --git a/crates/sqlbuild-recall/src/records/main/event_object_row.rs b/crates/sqlbuild-recall/src/records/main/event_object_row.rs new file mode 100644 index 0000000000..c67acb614b --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/event_object_row.rs @@ -0,0 +1,24 @@ +//! `event_object_row`: the `recall_events` row of one lifecycle event already decoded to JSON. + +use serde_json::{Map, Value}; + +use crate::records::_helpers::events::{envelope_columns, payload_columns}; +use crate::records::_helpers::rows::finish_row; +use crate::records::errors::RecordError; +use crate::records::models::{RecallRow, RowEnvelope}; +use crate::schema::models::Table; + +/// The row of one event given as the object `lifecycle_event_to_json` would encode. +pub fn event_object_row( + envelope: &RowEnvelope, + event: &Map, +) -> Result { + let mut columns = envelope_columns(event)?; + let empty: Map = Map::new(); + let payload = event + .get("payload") + .and_then(Value::as_object) + .unwrap_or(&empty); + columns.extend(payload_columns(payload)); + Ok(finish_row(Table::Events, envelope, columns)) +} diff --git a/crates/sqlbuild-recall/src/records/main/event_row.rs b/crates/sqlbuild-recall/src/records/main/event_row.rs index de0b2f69ad..53c62883e6 100644 --- a/crates/sqlbuild-recall/src/records/main/event_row.rs +++ b/crates/sqlbuild-recall/src/records/main/event_row.rs @@ -2,21 +2,13 @@ use serde_json::{Map, Value}; -use crate::records::_helpers::events::{envelope_columns, event_object, payload_columns}; -use crate::records::_helpers::rows::finish_row; +use crate::records::_helpers::events::event_object; use crate::records::errors::RecordError; +use crate::records::main::event_object_row::event_object_row; use crate::records::models::{RecallRow, RowEnvelope}; -use crate::schema::models::Table; /// The row of one event, as `lifecycle_event_to_json` encodes it. pub fn event_row(envelope: &RowEnvelope, event_json: &str) -> Result { let event: Map = event_object(event_json)?; - let mut columns = envelope_columns(&event)?; - let empty: Map = Map::new(); - let payload = event - .get("payload") - .and_then(Value::as_object) - .unwrap_or(&empty); - columns.extend(payload_columns(payload)); - Ok(finish_row(Table::Events, envelope, columns)) + event_object_row(envelope, &event) } diff --git a/crates/sqlbuild-recall/src/records/main/mod.rs b/crates/sqlbuild-recall/src/records/main/mod.rs index f1d7c2d3f7..8e8b1bb977 100644 --- a/crates/sqlbuild-recall/src/records/main/mod.rs +++ b/crates/sqlbuild-recall/src/records/main/mod.rs @@ -3,6 +3,7 @@ pub mod conflict_row; pub mod delivery_row; pub mod diagnostic_input_from_json; pub mod diagnostic_row; +pub mod event_object_row; pub mod event_row; pub mod format_timestamp; pub mod heartbeat_row; diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index ca4959c160..f7fed0f760 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -22,6 +22,8 @@ pub const COMMON_COLUMNS: [ColumnSpec; 6] = [ ColumnSpec::required("ingested_at", ColumnType::Timestamp), ColumnSpec::required("content_sha256", ColumnType::Text), ]; +/// The artifact columns its content digest covers: what it is, not who saw it when. +pub const ARTIFACT_CONTENT_COLUMNS: [&str; 5] = ["digest", "kind", "media_type", "size", "content"]; /// Common columns left out of a row's content digest. pub const UNHASHED_COLUMNS: [&str; 2] = ["ingested_at", "content_sha256"]; diff --git a/crates/sqlbuild-recall/src/writer/_helpers/session.rs b/crates/sqlbuild-recall/src/writer/_helpers/session.rs index 830f0eacae..c7e8169227 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/session.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/session.rs @@ -2,6 +2,9 @@ use std::time::{Duration, Instant, SystemTime}; +use serde_json::{Map, Value}; + +use crate::records::_helpers::events::event_object; use crate::records::constants::{ ARTIFACT_DIGEST_PREFIX, EXECUTED_SQL_ROLE, SEVERITY_ORDER, SQL_ARTIFACT_KIND, SQL_MEDIA_TYPE, }; @@ -9,7 +12,7 @@ use crate::records::errors::RecordError; use crate::records::main::artifact_rows::artifact_rows; use crate::records::main::diagnostic_input_from_json::diagnostic_input_from_json; use crate::records::main::diagnostic_row::diagnostic_row; -use crate::records::main::event_row::event_row; +use crate::records::main::event_object_row::event_object_row; use crate::records::main::format_timestamp::format_timestamp; use crate::records::main::sha256_hex::sha256_hex; use crate::records::models::{ArtifactInput, OutputStream, RecallValue}; @@ -73,8 +76,14 @@ impl RecallSession { /// Queue one lifecycle event from its canonical JSON. pub fn publish_event(&self, event_json: &str) -> Result<(), RecordError> { + let event: Map = event_object(event_json)?; + self.publish_event_object(&event) + } + + /// Queue one lifecycle event already decoded to its canonical JSON object. + pub fn publish_event_object(&self, event: &Map) -> Result<(), RecordError> { let mut state = locked(&self.shared.state); - let row = event_row(&state.envelope, event_json)?; + let row = event_object_row(&state.envelope, event)?; let priority = match row.value("severity") { Some(RecallValue::Text(severity)) => event_priority(severity), _ => Priority::Lifecycle, From f40a85f6a17b482fe8d6a7b39b3a152de2a485d9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 05:55:54 +0100 Subject: [PATCH 06/19] fix: share repeated SQL artifacts across invocations without conflicts --- .../src/records/_helpers/rows.rs | 15 +++++-- .../src/store/tests/test_views.rs | 41 ++++++++++++++++- .../src/sqlbuild/cli/recall/_test_types.py | 8 ++++ .../sqlbuild/cli/recall/test_local_recall.py | 44 +++++++++++++++++++ 4 files changed, 104 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-recall/src/records/_helpers/rows.rs b/crates/sqlbuild-recall/src/records/_helpers/rows.rs index 6cf27c265f..a023bd4a3d 100644 --- a/crates/sqlbuild-recall/src/records/_helpers/rows.rs +++ b/crates/sqlbuild-recall/src/records/_helpers/rows.rs @@ -2,7 +2,7 @@ use crate::records::_helpers::ids::content_sha256; use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; -use crate::schema::constants::RECALL_SCHEMA_VERSION; +use crate::schema::constants::{ARTIFACT_CONTENT_COLUMNS, RECALL_SCHEMA_VERSION}; use crate::schema::models::Table; pub(crate) fn text(value: impl Into) -> RecallValue { @@ -23,7 +23,7 @@ pub(crate) fn boolean(value: bool) -> RecallValue { RecallValue::Integer(i64::from(value)) } -/// `values` plus the envelope columns and `content_sha256`. +/// `values` plus the envelope columns and `content_sha256`; an artifact's digest covers only its content, since every invocation that sees it writes the same row. pub(crate) fn finish_row( table: Table, envelope: &RowEnvelope, @@ -39,7 +39,16 @@ pub(crate) fn finish_row( RecallValue::Integer(RECALL_SCHEMA_VERSION), )); values.push(("producer_version", text(&envelope.producer_version))); - let digest: String = content_sha256(&values); + let digest: String = match table { + Table::Artifacts => content_sha256( + &values + .iter() + .filter(|(name, _)| ARTIFACT_CONTENT_COLUMNS.contains(name)) + .cloned() + .collect::>(), + ), + _ => content_sha256(&values), + }; values.push(("content_sha256", RecallValue::Text(digest))); RecallRow { table, values } } diff --git a/crates/sqlbuild-recall/src/store/tests/test_views.rs b/crates/sqlbuild-recall/src/store/tests/test_views.rs index a6eee47867..6875dd5ab2 100644 --- a/crates/sqlbuild-recall/src/store/tests/test_views.rs +++ b/crates/sqlbuild-recall/src/store/tests/test_views.rs @@ -5,7 +5,7 @@ use crate::records::main::invocation_row::invocation_row; use crate::records::main::sha256_hex::sha256_hex; use crate::records::models::{ArtifactInput, InvocationContext, OutputStream, RecallValue}; use crate::store::tests::helpers::{ - NOW, chunk, chunks, delivered, envelope, event, integer, projected, text, wall_clock, + NOW, artifact, chunk, chunks, delivered, envelope, event, integer, projected, text, wall_clock, }; use crate::store::tests::test_types::ProjectionTestCase; @@ -421,3 +421,42 @@ fn given_audit_completed_facts_when_projected_then_audit_results_carry_outcome_a ); } } + +#[test] +fn given_the_same_sql_in_two_invocations_when_written_then_artifacts_are_shared_without_conflicts() +{ + let first = envelope("first", NOW); + let second = envelope("second", &wall_clock(60)); + let seen_twice = || { + let mut rows = artifact_rows(&first, &artifact("sha256:shared", NOW)).to_vec(); + rows.extend(artifact_rows( + &second, + &artifact("sha256:shared", &wall_clock(60)), + )); + vec![rows] + }; + let test_cases = [ + ProjectionTestCase { + description: "a repeated artifact is a duplicate, not a conflict", + batches: seen_twice(), + sql: "SELECT COUNT(*) FROM conflicts", + expected_rows: vec![vec![integer(0)]], + }, + ProjectionTestCase { + description: "one content row serves both invocations' references", + batches: seen_twice(), + sql: "SELECT (SELECT COUNT(*) FROM recall_artifacts), \ + (SELECT COUNT(*) FROM artifacts WHERE available = 1)", + expected_rows: vec![vec![integer(1), integer(2)]], + }, + ]; + + for test_case in test_cases { + assert_eq!( + projected(&test_case.batches, test_case.sql), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 3fc523131b..fdc7af05e0 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -72,3 +72,11 @@ class ConcurrentBuildsTestCase: description: str targets: tuple[str, ...] expected_status: str + + +@dataclass(frozen=True) +class RepeatedBuildTestCase: + description: str + builds: int + sql: str + expected_rows: list[list[object]] diff --git a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py index 60494c14bb..10cfab4017 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py @@ -16,6 +16,7 @@ RecallOutputTestCase, RecallRejectionTestCase, RecordedFactsTestCase, + RepeatedBuildTestCase, TextOutputTestCase, ) from tests.integration.src.sqlbuild.cli.recall.helpers import ( @@ -143,6 +144,16 @@ def test_given_interleaved_writes_when_recalled_combined_then_order_is_preserved "SUM(sql_available) = COUNT(*) FROM statements", expected_rows=[[1, 1, 1]], ), + RecordedFactsTestCase( + description="natively encoded events keep payloads, promotions and UTC timestamps", + sql="SELECT COUNT(*) > 0, " + "SUM(payload = '{}' OR statement_kind IS NULL OR intent IS NULL), " + "SUM(occurred_at NOT GLOB " + "'[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9][0-9][0-9][0-9][0-9][0-9]Z'), " + "SUM(invocation_sequence IS NULL) " + "FROM events WHERE event_type LIKE 'statement_%'", + expected_rows=[[1, 0, 0, 0]], + ), RecordedFactsTestCase( description="the started event names target and adapter", sql="SELECT json_extract(payload, '$.target'), json_extract(payload, '$.adapter') " @@ -393,3 +404,36 @@ def test_given_four_concurrent_builds_and_a_reader_when_run_then_every_run_is_re assert recorded == [ [target, test_case.expected_status, 1] for target in sorted(test_case.targets) ] + + +@pytest.mark.parametrize( + "test_case", + [ + RepeatedBuildTestCase( + description="repeated SQL is shared, never a conflict", + builds=2, + sql="SELECT (SELECT COUNT(*) FROM conflicts), " + "(SELECT SUM(conflicts) FROM recall_delivery)", + expected_rows=[[0, 0]], + ), + RepeatedBuildTestCase( + description="every stream of every build delivers exactly what it accepted", + builds=2, + sql="SELECT COUNT(DISTINCT invocation_id), " + "SUM(accepted != delivered + filtered + dropped + failed), " + "SUM(dropped + failed), MIN(flush_complete) FROM recall_delivery", + expected_rows=[[2, 0, 0, 1]], + ), + ], + ids=lambda case: case.description, +) +def test_given_identical_builds_when_recorded_then_artifacts_and_delivery_counts_stay_exact( + test_case: RepeatedBuildTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + + exit_codes: list[int] = [run(project_dir, "build").returncode for _ in range(test_case.builds)] + rows: list[list[object]] = recall_rows(project_dir, test_case.sql) + + assert exit_codes == [0 for _ in range(test_case.builds)] + assert rows == test_case.expected_rows From 60114459c9aff0fb59076ddfb863b11c26b4540b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 06:39:05 +0100 Subject: [PATCH 07/19] fix: record only observed commands so every recalled invocation has terminal events --- crates/sqlbuild-cli/src/dispatch/main/invocation_policy.rs | 7 ++++--- .../src/dispatch/tests/test_invocation_policy.rs | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/sqlbuild-cli/src/dispatch/main/invocation_policy.rs b/crates/sqlbuild-cli/src/dispatch/main/invocation_policy.rs index 56c7d09f06..e01102eb49 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/invocation_policy.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/invocation_policy.rs @@ -6,7 +6,7 @@ use crate::dispatch::constants::{ }; use crate::dispatch::models::{CommandFacts, InvocationPolicy}; -/// Whether the command creates a project, is observed, and checks skill freshness. +/// Whether the command creates a project, is observed, checks skill freshness and is recorded; only observed commands are recorded, so every recorded invocation has terminal events. #[must_use] pub fn invocation_policy(facts: &CommandFacts) -> InvocationPolicy { let command: &str = facts.command.as_deref().unwrap_or_default(); @@ -14,10 +14,11 @@ pub fn invocation_policy(facts: &CommandFacts) -> InvocationPolicy { || (command == DBT_COMMAND_NAME && facts.subcommand.as_deref() == Some(DBT_INIT_SUBCOMMAND)); let raw_query_diff: bool = command == DIFF_COMMAND_NAME && facts.raw_query_diff; + let observed: bool = !creates_project && !raw_query_diff; InvocationPolicy { creates_project, - observed: !creates_project && !raw_query_diff, + observed, maintains_skills: !SKILL_EXEMPT_COMMANDS.contains(&command) && !raw_query_diff, - recalled: !creates_project && !command.is_empty() && command != RECALL_COMMAND_NAME, + recalled: observed && !command.is_empty() && command != RECALL_COMMAND_NAME, } } diff --git a/crates/sqlbuild-cli/src/dispatch/tests/test_invocation_policy.rs b/crates/sqlbuild-cli/src/dispatch/tests/test_invocation_policy.rs index 2272a40cba..fe47e5f633 100644 --- a/crates/sqlbuild-cli/src/dispatch/tests/test_invocation_policy.rs +++ b/crates/sqlbuild-cli/src/dispatch/tests/test_invocation_policy.rs @@ -37,7 +37,7 @@ fn given_parsed_command_when_choosing_services_then_matches_python_entry_rules() }, }, InvocationPolicyTestCase { - description: "a raw query diff is neither observed nor checks skills", + description: "a raw query diff is not observed, so it is not recorded either", facts: CommandFacts { raw_query_diff: true, ..facts("diff", None) @@ -46,7 +46,7 @@ fn given_parsed_command_when_choosing_services_then_matches_python_entry_rules() creates_project: false, observed: false, maintains_skills: false, - recalled: true, + recalled: false, }, }, InvocationPolicyTestCase { From 6112412fc30ef558463742df6f03a343394f1fe5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 06:39:05 +0100 Subject: [PATCH 08/19] fix: cap Recall's delay by the shutdown budget and report store problems before output Startup never waits on the store: setup runs on the writer thread and retries until the close deadline. The shutdown timeout bounds the whole exit, including busy waits and maintenance, and a second interrupt stops waiting at once. A store file that is not a database is moved aside once and a fresh store started. Closing notices end with the command's own status, and a notice that cannot be written never changes the exit code. Output is forwarded to Recall under the compute-log storage lock, dropped output is counted once, heartbeats and retention use wall-clock time so a suspended run is kept, and oversized SQL is dropped before it is hashed or copied. --- .../src/bindings/_helpers/cli/entry.rs | 6 +- .../bindings/_helpers/compute_logs/storage.rs | 38 +-- .../src/bindings/_helpers/recall/constants.rs | 2 + .../src/bindings/_helpers/recall/session.rs | 180 +++++++++----- .../sqlbuild-recall/src/records/constants.rs | 2 + .../src/records/main/output_row.rs | 3 +- .../src/schema/main/sqlite_retention_sql.rs | 4 +- .../src/store/_helpers/sqlite.rs | 1 + crates/sqlbuild-recall/src/store/constants.rs | 6 + crates/sqlbuild-recall/src/store/errors.rs | 2 + crates/sqlbuild-recall/src/store/main/mod.rs | 3 + .../src/store/main/quarantine_store.rs | 37 +++ .../src/store/main/store_header_is_foreign.rs | 17 ++ .../src/store/main/store_location_problem.rs | 20 ++ .../src/store/tests/test_maintenance.rs | 6 +- .../src/writer/_helpers/delivery.rs | 73 ++++-- .../src/writer/_helpers/queue.rs | 16 +- .../src/writer/_helpers/session.rs | 115 ++++++--- .../sqlbuild-recall/src/writer/constants.rs | 10 + .../src/writer/tests/helpers.rs | 78 +++++++ .../src/writer/tests/test_queue.rs | 28 ++- .../src/writer/tests/test_session.rs | 86 ++++++- .../src/writer/tests/test_types.rs | 26 +++ .../src/sqlbuild/cli/recall/_test_types.py | 35 +++ .../src/sqlbuild/cli/recall/helpers.py | 132 ++++++++++- .../sqlbuild/cli/recall/test_local_recall.py | 221 ++++++++++++++++++ 26 files changed, 1006 insertions(+), 141 deletions(-) create mode 100644 crates/sqlbuild-recall/src/store/main/quarantine_store.rs create mode 100644 crates/sqlbuild-recall/src/store/main/store_header_is_foreign.rs create mode 100644 crates/sqlbuild-recall/src/store/main/store_location_problem.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index fd10d50ed5..5f565c3867 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -593,7 +593,11 @@ fn parsed_invocation<'py>( ("recall", recall_sink), ], ); - let finished: PyResult<()> = finish_invocation_recall(py, &recall); + let exit_status: Option = match &dispatched { + Ok(code) if code.is_instance_of::() => Some(code.extract().unwrap_or(1)), + Ok(_) | Err(_) => None, + }; + let finished: PyResult<()> = finish_invocation_recall(py, &recall, exit_status); let exit_code: Bound<'py, PyAny> = dispatched?; finished?; Ok(exit_code) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs b/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs index 210464ed2f..02bc5c5556 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs @@ -114,16 +114,24 @@ impl LocalFilesystemComputeLogStorage { stream: usize, data: &[u8], ) -> PyResult<()> { - let written: Option> = { - let mut state = self.locked()?; - state - .writers - .get_mut(invocation_id) - .map(|writers| writers[stream].write_all(data)) + let mut state = self.locked()?; + let Some(writers) = state.writers.get_mut(invocation_id) else { + return Err(not_active(py)); }; - match written { - Some(result) => Ok(result?), - None => Err(not_active(py)), + writers[stream].write_all(data)?; + self.forward_to_recall(invocation_id, stream, data); + Ok(()) + } + + /// Hand bytes the capture just accepted to Recall while its lock is held, so both keep one order. + fn forward_to_recall(&self, invocation_id: &str, stream: usize, data: &[u8]) { + let session = self.recall.get().map(Arc::as_ref); + if stream != DIAGNOSTICS_STREAM { + forward_output(session, invocation_id, (stream, data)); + return; + } + if let Ok(line) = std::str::from_utf8(data) { + forward_diagnostic(session, invocation_id, line.trim_end_matches('\n')); } } @@ -221,13 +229,7 @@ impl LocalFilesystemComputeLogStorage { "compute log append data must be bytes", )); }; - self.append_bytes(py, &invocation_id, stream, bytes.as_bytes())?; - forward_output( - self.recall.get().map(Arc::as_ref), - &invocation_id, - (stream, bytes.as_bytes()), - ); - Ok(()) + self.append_bytes(py, &invocation_id, stream, bytes.as_bytes()) } /// Append one validated deterministic UTF-8 diagnostic JSON line. @@ -253,9 +255,7 @@ impl LocalFilesystemComputeLogStorage { self.ensure_open(py)?; let invocation_id = capture_id(invocation_id)?; checked(py, validate_capture_id(&invocation_id))?; - self.append_bytes(py, &invocation_id, DIAGNOSTICS_STREAM, &line)?; - forward_diagnostic(self.recall.get().map(Arc::as_ref), &invocation_id, text); - Ok(()) + self.append_bytes(py, &invocation_id, DIAGNOSTICS_STREAM, &line) } /// Read at most the public byte limit from an exact stream offset. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs index a10f4d306d..f253b88dc7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -85,3 +85,5 @@ pub(crate) const TIMESTAMP_PARTS: [&str; 7] = [ ]; /// `str(datetime.UTC)`. pub(crate) const UTC_ZONE_NAME: &str = "UTC"; +/// How often closing checks for an interrupt while delivery finishes. +pub(crate) const CLOSE_POLL_MS: u64 = 50; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index ac73b3dc10..5db3a210f7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -15,7 +15,10 @@ use sqlbuild_recall::schema::main::ddl_script::ddl_script; use sqlbuild_recall::schema::models::{DdlOptions, Dialect}; use sqlbuild_recall::settings::main::resolve_recall_settings::resolve_recall_settings; use sqlbuild_recall::settings::models::{RecallSettings, SettingsRequest, StoreLocation}; -use sqlbuild_recall::writer::constants::{DEFAULT_HEARTBEAT_INTERVAL_MS, SETUP_BUDGET_MS}; +use sqlbuild_recall::store::main::quarantine_store::quarantine_store; +use sqlbuild_recall::store::main::store_header_is_foreign::store_header_is_foreign; +use sqlbuild_recall::store::main::store_location_problem::store_location_problem; +use sqlbuild_recall::writer::constants::DEFAULT_HEARTBEAT_INTERVAL_MS; use sqlbuild_recall::writer::main::start_recall_session::start_recall_session; use sqlbuild_recall::writer::models::{ DeliverySummary, RecallSession, SessionConfig, SetupOutcome, @@ -26,8 +29,8 @@ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::output_capture::records::invocation_context; use crate::bindings::_helpers::recall::console::write_stderr; use crate::bindings::_helpers::recall::constants::{ - DDL_NOTICE_PREFIX, HEARTBEAT_ENVIRONMENT, HOST_TOKEN_LENGTH, MINIMUM_HEARTBEAT_SECONDS, - NOTICE_INTERVAL_SECONDS, STDOUT_STREAM_INDEX, + CLOSE_POLL_MS, DDL_NOTICE_PREFIX, HEARTBEAT_ENVIRONMENT, HOST_TOKEN_LENGTH, + MINIMUM_HEARTBEAT_SECONDS, NOTICE_INTERVAL_SECONDS, STDOUT_STREAM_INDEX, }; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::rules::config_json::dumped_json; @@ -38,6 +41,10 @@ pub(crate) struct InvocationRecall { pub(crate) session: Option>, pub(crate) target: Option, pub(crate) adapter: Option, + /// The command name, for the closing status line. + pub(crate) command: String, + /// The resolved settings and project directory, for notices at the end of the run. + pub(crate) settings: Option<(RecallSettings, PathBuf)>, } /// The text of an optional attribute value. @@ -202,16 +209,16 @@ fn target_phrase(settings: &RecallSettings) -> String { .map_or_else(String::new, |target| format!(" for target `{target}`")) } -fn report_permission( - py: Python<'_>, +/// The DDL notice for an unwritable store, when one is due for it. +fn permission_notice( (settings, project_dir): (&RecallSettings, &Path), reason: &str, -) -> PyResult<()> { +) -> Option { let StoreLocation::Sqlite { path } = &settings.store else { - return Ok(()); + return None; }; if !notice_due(project_dir, &path.to_string_lossy()) { - return Ok(()); + return None; } let script = ddl_script( Dialect::Sqlite, @@ -221,33 +228,48 @@ fn report_permission( cluster: None, }, ); - write_stderr( - py, - &format!( - "Recall history is not being written{}: {reason}. An administrator can create the \ - store at {} with:\n\n{script}\n", - target_phrase(settings), - path.display() - ), - ) + Some(format!( + "Recall history is not being written{}: {reason}. An administrator can create the \ + store at {} with:\n\n{script}\n", + target_phrase(settings), + path.display() + )) } -fn report_unsupported( - py: Python<'_>, +fn unsupported_notice( (settings, project_dir): (&RecallSettings, &Path), (connection, adapter): (&str, &str), -) -> PyResult<()> { - if !notice_due(project_dir, connection) { - return Ok(()); - } - write_stderr( - py, - &format!( +) -> Option { + notice_due(project_dir, connection).then(|| { + format!( "Recall history is not being written{}: connection `{connection}` uses adapter \ `{adapter}`, which this version cannot store Recall history in.\n", target_phrase(settings) + ) + }) +} + +/// Write `text` to stderr; a notice that cannot be written never changes the command's outcome. +fn notify(py: Python<'_>, text: Option) { + if let Some(text) = text { + drop(write_stderr(py, &text)); + } +} + +/// Move a store whose file is not a database aside before the command prints anything. +fn quarantined_notice(path: &Path) -> Option { + if !store_header_is_foreign(path) { + return None; + } + Some(match quarantine_store(path, SystemTime::now()) { + Ok(moved) => format!( + "Recall: {} was not a valid database; it was moved to {} and a fresh store was \ + started.\n", + path.display(), + moved.display() ), - ) + Err(error) => format!("Recall history is not being written: {error}.\n"), + }) } fn started_session( @@ -261,10 +283,18 @@ fn started_session( connection, adapter, } => { - report_unsupported(py, (settings, project_dir), (connection, adapter))?; + notify( + py, + unsupported_notice((settings, project_dir), (connection, adapter)), + ); return Ok(None); } }; + notify(py, quarantined_notice(&store_path)); + if let Some(reason) = store_location_problem(&store_path) { + notify(py, permission_notice((settings, project_dir), &reason)); + return Ok(None); + } let Some(invocation_id) = text_value(&native_field(identity, "invocation_id")?)? else { return Ok(None); }; @@ -286,17 +316,13 @@ fn started_session( shutdown_timeout: Duration::from_millis(settings.shutdown_timeout_ms), diagnostics_min_severity: settings.diagnostics_min_severity.clone(), heartbeat_interval: heartbeat_interval(), - setup_budget: Duration::from_millis(SETUP_BUDGET_MS), + setup_budget: Duration::ZERO, maintenance: true, }); - let setup = py.detach(|| session.wait_for_setup(Duration::from_millis(SETUP_BUDGET_MS))); - if let Some(SetupOutcome::PermissionDenied { reason }) = &setup { - report_permission(py, (settings, project_dir), reason)?; - } Ok(Some(Arc::new(session))) } -/// Begin recording when Recall is enabled; invalid settings record nothing and the command reports them. +/// Begin recording when Recall is enabled, without waiting on the store; invalid settings record nothing and the command reports them. pub(crate) fn start_invocation_recall( py: Python<'_>, args: &Bound<'_, PyAny>, @@ -319,36 +345,76 @@ pub(crate) fn start_invocation_recall( }; Ok(InvocationRecall { session, - target: settings.target, - adapter: settings.project_adapter, + target: settings.target.clone(), + adapter: settings.project_adapter.clone(), + command: facts.command.clone().unwrap_or_default(), + settings: Some((settings, project_dir)), }) } -/// Close the session: flush within the shutdown budget and warn when history is incomplete. -pub(crate) fn finish_invocation_recall(py: Python<'_>, recall: &InvocationRecall) -> PyResult<()> { +/// The closing sentence after a Recall notice, so the command's own outcome stays the last word. +fn final_status(command: &str, exit_code: Option) -> String { + match exit_code { + Some(0) => { + format!("sqb {command} completed successfully; only its Recall history is affected.\n") + } + Some(code) => format!("sqb {command} failed with exit code {code}.\n"), + None => format!("sqb {command} failed.\n"), + } +} + +/// What Recall must tell the user about this run once delivery has finished. +fn closing_notices(recall: &InvocationRecall, session: &RecallSession) -> Vec { + let mut notices: Vec = session + .notices() + .into_iter() + .map(|notice| format!("Recall: {notice}.\n")) + .collect(); + if let (Some(SetupOutcome::PermissionDenied { reason }), Some((settings, project_dir))) = + (session.wait_for_setup(Duration::ZERO), &recall.settings) + { + notices.extend(permission_notice((settings, project_dir), &reason)); + return notices; + } + let summary: DeliverySummary = session.summary(); + let total = summary.total(); + if total.dropped > 0 || total.failed > 0 { + let reason = summary + .degraded_reason + .as_deref() + .map_or_else(String::new, |reason| format!(": {reason}")); + notices.push(format!( + "warning: Recall history for this run is incomplete ({} dropped, {} failed){reason}\n", + total.dropped, total.failed + )); + } + notices +} + +/// Close the session within its shutdown budget; a further interrupt stops waiting at once. Notices end with the command's own outcome. +pub(crate) fn finish_invocation_recall( + py: Python<'_>, + recall: &InvocationRecall, + exit_code: Option, +) -> PyResult<()> { let Some(session) = &recall.session else { return Ok(()); }; - let summary: DeliverySummary = py.detach(|| session.close()); - let total = summary.total(); - let permission = matches!( - session.wait_for_setup(Duration::ZERO), - Some(SetupOutcome::PermissionDenied { .. }) - ); - if permission || (total.dropped == 0 && total.failed == 0) { - return Ok(()); + session.begin_close(); + while !py.detach(|| session.wait_closed(Duration::from_millis(CLOSE_POLL_MS))) { + if let Err(interrupt) = py.check_signals() { + session.abandon(); + return Err(interrupt); + } } - let reason = summary - .degraded_reason - .as_deref() - .map_or_else(String::new, |reason| format!(": {reason}")); - write_stderr( - py, - &format!( - "warning: Recall history for this run is incomplete ({} dropped, {} failed){reason}\n", - total.dropped, total.failed - ), - ) + let notices: Vec = closing_notices(recall, session); + if !notices.is_empty() { + notify( + py, + Some(notices.concat() + &final_status(&recall.command, exit_code)), + ); + } + Ok(()) } /// Forward bytes a process stream accepted to `session` when it records `invocation_id`. diff --git a/crates/sqlbuild-recall/src/records/constants.rs b/crates/sqlbuild-recall/src/records/constants.rs index 69f783962c..6392d48190 100644 --- a/crates/sqlbuild-recall/src/records/constants.rs +++ b/crates/sqlbuild-recall/src/records/constants.rs @@ -46,3 +46,5 @@ pub(crate) const DECIMAL_RADIX: u32 = 10; pub(crate) const FEBRUARY: i64 = 2; /// In a year counted from March, January is month index 10. pub(crate) const MARCH_BASED_JANUARY_INDEX: i64 = 10; +/// The output column counting chunks dropped just before this one. +pub const DROPPED_BEFORE_COLUMN: &str = "dropped_before"; diff --git a/crates/sqlbuild-recall/src/records/main/output_row.rs b/crates/sqlbuild-recall/src/records/main/output_row.rs index 9d312eecc6..0c4dd22bba 100644 --- a/crates/sqlbuild-recall/src/records/main/output_row.rs +++ b/crates/sqlbuild-recall/src/records/main/output_row.rs @@ -2,6 +2,7 @@ use crate::records::_helpers::ids::output_record_id; use crate::records::_helpers::rows::{finish_row, integer, text}; +use crate::records::constants::DROPPED_BEFORE_COLUMN; use crate::records::models::{OutputChunk, RecallRow, RecallValue, RowEnvelope}; use crate::schema::models::Table; @@ -23,7 +24,7 @@ pub fn output_row(envelope: &RowEnvelope, chunk: &OutputChunk) -> RecallRow { ("byte_offset", integer(chunk.byte_offset)), ("byte_count", integer(chunk.data.len() as u64)), ("priority", text("output")), - ("dropped_before", integer(chunk.dropped_before)), + (DROPPED_BEFORE_COLUMN, integer(chunk.dropped_before)), ], ) } diff --git a/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs b/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs index 267f17d543..1a44c7b2e4 100644 --- a/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs +++ b/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs @@ -21,7 +21,7 @@ fn expired_invocations() -> String { "SELECT DISTINCT invocation_id FROM (\n {}\n)\n\ WHERE invocation_id NOT IN (\n\ \x20 SELECT beat.invocation_id FROM recall_heartbeats AS beat\n\ - \x20 WHERE beat.stale_at >= ?2 AND NOT EXISTS (\n\ + \x20 WHERE (beat.stale_at >= ?2 OR beat.beat_at >= ?1) AND NOT EXISTS (\n\ \x20 SELECT 1 FROM recall_events AS terminal\n\ \x20 WHERE terminal.invocation_id = beat.invocation_id\n\ \x20 AND terminal.event_type IN ('invocation_completed', 'invocation_failed')\n\ @@ -31,7 +31,7 @@ fn expired_invocations() -> String { ) } -/// The maintenance statements; running invocations are never deleted. +/// The maintenance statements; running invocations, and unfinished ones that beat within retention, such as a suspended run, are never deleted. #[must_use] pub fn sqlite_retention_sql() -> SqliteRetentionSql { SqliteRetentionSql { diff --git a/crates/sqlbuild-recall/src/store/_helpers/sqlite.rs b/crates/sqlbuild-recall/src/store/_helpers/sqlite.rs index b188ed849a..ef93fde778 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/sqlite.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/sqlite.rs @@ -13,6 +13,7 @@ pub(crate) fn store_error(error: &rusqlite::Error, doing: &str) -> StoreError { StoreErrorKind::Permission } Some(ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) => StoreErrorKind::Busy, + Some(ErrorCode::NotADatabase | ErrorCode::DatabaseCorrupt) => StoreErrorKind::Corrupt, _ => StoreErrorKind::Other, }; StoreError::new(kind, format!("{doing}: {error}")) diff --git a/crates/sqlbuild-recall/src/store/constants.rs b/crates/sqlbuild-recall/src/store/constants.rs index fc5d1f1a7d..f31cccce15 100644 --- a/crates/sqlbuild-recall/src/store/constants.rs +++ b/crates/sqlbuild-recall/src/store/constants.rs @@ -12,3 +12,9 @@ pub const INCREMENTAL_VACUUM_PAGES: u32 = 2_048; pub const META_MAINTENANCE_OVERDUE_KEY: &str = "overdue"; /// The selector naming the most recently started invocation. pub const LAST_INVOCATION_SELECTOR: &str = "last"; +/// The first bytes of every SQLite database file. +pub const SQLITE_HEADER: &[u8; 16] = b"SQLite format 3\0"; +/// What a store moved aside for corruption is renamed to, before its UTC timestamp. +pub const CORRUPT_SUFFIX: &str = ".corrupt-"; +/// SQLite's companion files, moved along with a corrupt store. +pub const COMPANION_SUFFIXES: [&str; 2] = ["-wal", "-shm"]; diff --git a/crates/sqlbuild-recall/src/store/errors.rs b/crates/sqlbuild-recall/src/store/errors.rs index 4959ea152e..a1422e93b2 100644 --- a/crates/sqlbuild-recall/src/store/errors.rs +++ b/crates/sqlbuild-recall/src/store/errors.rs @@ -7,6 +7,8 @@ pub enum StoreErrorKind { Permission, /// Another writer holds the lock past the busy timeout; retrying may succeed. Busy, + /// The file is not a SQLite database, or its pages are damaged. + Corrupt, /// The SQLite library is older than the store needs. Version, /// A query was rejected: not read-only, or invalid SQL. diff --git a/crates/sqlbuild-recall/src/store/main/mod.rs b/crates/sqlbuild-recall/src/store/main/mod.rs index 0605e2bff9..0afb2ee851 100644 --- a/crates/sqlbuild-recall/src/store/main/mod.rs +++ b/crates/sqlbuild-recall/src/store/main/mod.rs @@ -3,6 +3,9 @@ pub mod ensure_sqlite_schema; pub mod insert_rows; pub mod maintain_sqlite_store; pub mod open_sqlite_store; +pub mod quarantine_store; pub mod query_store; pub mod read_output_transcript; pub mod resolve_invocation; +pub mod store_header_is_foreign; +pub mod store_location_problem; diff --git a/crates/sqlbuild-recall/src/store/main/quarantine_store.rs b/crates/sqlbuild-recall/src/store/main/quarantine_store.rs new file mode 100644 index 0000000000..b90d1ce4b4 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/quarantine_store.rs @@ -0,0 +1,37 @@ +//! `quarantine_store`: move a corrupt store aside so a fresh one can start. + +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use crate::records::main::format_timestamp::format_timestamp; +use crate::store::constants::{COMPANION_SUFFIXES, CORRUPT_SUFFIX}; +use crate::store::errors::{StoreError, StoreErrorKind}; + +fn moved_name(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_owned(); + name.push(suffix); + PathBuf::from(name) +} + +/// Rename `path` and its companion files to `.corrupt-`; returns the new store name. +pub fn quarantine_store(path: &Path, now: SystemTime) -> Result { + let stamp: String = format_timestamp(now).replace(':', ""); + let suffix: String = format!("{CORRUPT_SUFFIX}{stamp}"); + let target: PathBuf = moved_name(path, &suffix); + std::fs::rename(path, &target).map_err(|error| { + StoreError::new( + StoreErrorKind::Corrupt, + format!( + "cannot move the corrupt store {} aside: {error}", + path.display() + ), + ) + })?; + for companion in COMPANION_SUFFIXES { + let source: PathBuf = moved_name(path, companion); + if source.exists() { + drop(std::fs::rename(&source, moved_name(&target, companion))); + } + } + Ok(target) +} diff --git a/crates/sqlbuild-recall/src/store/main/store_header_is_foreign.rs b/crates/sqlbuild-recall/src/store/main/store_header_is_foreign.rs new file mode 100644 index 0000000000..7ce99a60dd --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/store_header_is_foreign.rs @@ -0,0 +1,17 @@ +//! `store_header_is_foreign`: whether a store file holds something other than a SQLite database. + +use std::io::Read; +use std::path::Path; + +use crate::store::constants::SQLITE_HEADER; + +/// Whether `path` exists with content that is not a SQLite database; never opens it with SQLite. +#[must_use] +pub fn store_header_is_foreign(path: &Path) -> bool { + let mut header: Vec = Vec::with_capacity(SQLITE_HEADER.len()); + let read = std::fs::File::open(path).and_then(|file| { + file.take(SQLITE_HEADER.len() as u64) + .read_to_end(&mut header) + }); + read.is_ok_and(|length| length > 0) && header.as_slice() != SQLITE_HEADER.as_slice() +} diff --git a/crates/sqlbuild-recall/src/store/main/store_location_problem.rs b/crates/sqlbuild-recall/src/store/main/store_location_problem.rs new file mode 100644 index 0000000000..cde286eb90 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/store_location_problem.rs @@ -0,0 +1,20 @@ +//! `store_location_problem`: why a store could not be written, judged from file permissions alone. + +use std::path::Path; + +/// Why `path` cannot be created or written, from permission bits only, so it never waits on a lock. +#[must_use] +pub fn store_location_problem(path: &Path) -> Option { + let directory: &Path = path + .ancestors() + .skip(1) + .find(|ancestor| ancestor.is_dir())?; + let directory_readonly: bool = + std::fs::metadata(directory).is_ok_and(|metadata| metadata.permissions().readonly()); + if directory_readonly { + return Some(format!("{} is not writable", directory.display())); + } + let file_readonly: bool = + std::fs::metadata(path).is_ok_and(|metadata| metadata.permissions().readonly()); + file_readonly.then(|| format!("{} is read-only", path.display())) +} diff --git a/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs b/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs index c5eed7a642..c94249656a 100644 --- a/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs +++ b/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs @@ -60,16 +60,16 @@ fn given_expired_recent_and_running_invocations_when_maintained_then_only_expire expected_rows: vec![vec![text("sha256:recent")]], }, MaintenanceTestCase { - description: "a claim stops another run within a day; the next day the stale run expires", + description: "a claim stops another run within a day, and an unfinished run that beat within retention is kept", batch: mixed, runs: &[(0, 5), (3600, 5), (90_000, 5)], sql: "SELECT COUNT(*) FROM recall_invocations", expected_outcomes: vec![ ran(1, 1, true), MaintenanceOutcome::default(), - ran(1, 0, true), + ran(0, 0, true), ], - expected_rows: vec![vec![integer(1)]], + expected_rows: vec![vec![integer(2)]], }, MaintenanceTestCase { description: "a starved run leaves the next run overdue", diff --git a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs index 2f8d00872d..fe33c85235 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs @@ -8,17 +8,19 @@ use crate::records::main::format_timestamp::format_timestamp; use crate::records::main::heartbeat_row::heartbeat_row; use crate::records::models::RecallRow; use crate::schema::models::Table; -use crate::store::constants::MAINTENANCE_BUDGET_MS; +use crate::store::constants::{BUSY_TIMEOUT_MS, MAINTENANCE_BUDGET_MS}; use crate::store::errors::{StoreError, StoreErrorKind}; use crate::store::main::ensure_sqlite_schema::ensure_sqlite_schema; use crate::store::main::insert_rows::insert_rows; use crate::store::main::maintain_sqlite_store::maintain_sqlite_store; use crate::store::main::open_sqlite_store::open_sqlite_store; +use crate::store::main::quarantine_store::quarantine_store; use crate::store::models::{InsertOutcome, MaintenanceRequest, OpenMode, SqliteStore}; use crate::writer::_helpers::queue::{Queued, WriterState}; use crate::writer::constants::{ - BATCH_ROWS, BATCH_WINDOW_MS, HEARTBEAT_STALE_INTERVALS, MAX_RETRY_BACKOFF_MS, - OUTPUT_COALESCE_MS, RUNNING_RETRY_LIMIT_MS, + BATCH_ROWS, BATCH_WINDOW_MS, CLOCK_CHECK_MS, FINAL_ROWS_BUDGET_MS, HEARTBEAT_STALE_INTERVALS, + MAX_RETRY_BACKOFF_MS, MINIMUM_MAINTENANCE_MS, OUTPUT_COALESCE_MS, RUNNING_RETRY_LIMIT_MS, + SETUP_RETRY_MS, }; use crate::writer::models::{DeliveryStream, SessionConfig, SetupOutcome}; @@ -46,20 +48,47 @@ fn publish_setup(shared: &Shared, outcome: SetupOutcome) { shared.setup_done.notify_all(); } -fn open_and_setup(config: &SessionConfig) -> Result { +/// When busy retries stop: the close deadline once closing, else the running retry limit. +fn retry_limit(shared: &Shared, started: Instant) -> Instant { + locked(&shared.state) + .deadline + .unwrap_or_else(|| started + Duration::from_millis(RUNNING_RETRY_LIMIT_MS)) +} + +/// Each SQLite busy wait ends by `give_up`, so no write outlasts the shutdown budget. +fn bound_busy_wait(store: &SqliteStore, give_up: Instant) { + let remaining: Duration = give_up + .saturating_duration_since(Instant::now()) + .min(Duration::from_millis(BUSY_TIMEOUT_MS)); + drop(store.connection.busy_timeout(remaining)); +} + +fn open_and_setup(shared: &Shared, config: &SessionConfig) -> Result { let started = Instant::now(); + let mut quarantined: bool = false; loop { let attempt = open_sqlite_store(&config.store_path, OpenMode::ReadWrite).and_then(|store| { + bound_busy_wait(&store, retry_limit(shared, started)); let now = format_timestamp(SystemTime::now()); ensure_sqlite_schema(&store, &config.envelope, &now).map(|()| store) }); match attempt { Err(error) if error.kind == StoreErrorKind::Busy - && started.elapsed() < Duration::from_millis(RUNNING_RETRY_LIMIT_MS) => + && Instant::now() < retry_limit(shared, started) => { - std::thread::sleep(Duration::from_millis(50)); + std::thread::sleep(Duration::from_millis(SETUP_RETRY_MS)); + } + Err(error) if error.kind == StoreErrorKind::Corrupt && !quarantined => { + quarantined = true; + let moved = quarantine_store(&config.store_path, SystemTime::now())?; + locked(&shared.state).notices.push(format!( + "the Recall store {} was not a valid database; it was moved to {} and a \ + fresh store was started", + config.store_path.display(), + moved.display() + )); } other => return other, } @@ -74,6 +103,7 @@ fn write_batch( ) -> Result { let mut backoff = Duration::from_millis(10); loop { + bound_busy_wait(store, give_up); let now = format_timestamp(SystemTime::now()); match insert_rows(store, rows, &now) { Err(error) if error.kind == StoreErrorKind::Busy && Instant::now() < give_up => { @@ -105,7 +135,7 @@ pub(crate) fn stream_of(table: Table) -> DeliveryStream { } /// Wait until a batch is due: 500 rows, the batch window, closing, or a timed duty. -fn next_batch(shared: &Shared, next_heartbeat: Instant) -> (Vec, bool) { +fn next_batch(shared: &Shared, next_heartbeat: SystemTime) -> (Vec, bool) { let mut state = locked(&shared.state); let mut first_seen: Option = None; loop { @@ -128,10 +158,13 @@ fn next_batch(shared: &Shared, next_heartbeat: Instant) -> (Vec, bool) { if state.closing && state.queued() == 0 && !state.has_pending_output() { return (Vec::new(), true); } - if now >= next_heartbeat { + let until_heartbeat: Duration = next_heartbeat + .duration_since(SystemTime::now()) + .unwrap_or(Duration::ZERO); + if until_heartbeat.is_zero() { return (Vec::new(), state.closing); } - let mut wait_until = next_heartbeat; + let mut wait_until = now + until_heartbeat.min(Duration::from_millis(CLOCK_CHECK_MS)); if let Some(due) = state.output_due(Duration::from_millis(OUTPUT_COALESCE_MS)) { wait_until = wait_until.min(due); } @@ -173,7 +206,7 @@ fn final_rows(config: &SessionConfig, state: &WriterState) -> Vec { /// The delivery thread body. pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { - let store = match open_and_setup(config) { + let store = match open_and_setup(shared, config) { Ok(store) => { publish_setup(shared, SetupOutcome::Ready); store @@ -193,16 +226,17 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { return; } }; - let mut next_heartbeat = Instant::now(); + let mut next_heartbeat: SystemTime = SystemTime::now(); loop { let (batch, closing) = next_batch(shared, next_heartbeat); let mut rows: Vec = batch .iter() .flat_map(|record| record.rows.clone()) .collect(); - if Instant::now() >= next_heartbeat && !closing { - rows.push(heartbeat(config, SystemTime::now())); - next_heartbeat = Instant::now() + config.heartbeat_interval; + let now: SystemTime = SystemTime::now(); + if now >= next_heartbeat && !closing { + rows.push(heartbeat(config, now)); + next_heartbeat = now + config.heartbeat_interval; } if !rows.is_empty() { let give_up = { @@ -230,19 +264,24 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { state.rejecting = true; let rows = final_rows(config, &state); drop(state); - let give_up = Instant::now() + Duration::from_millis(500); + let give_up = (Instant::now() + Duration::from_millis(FINAL_ROWS_BUDGET_MS)) + .min(retry_limit(shared, Instant::now())); if let Err(error) = write_batch(&store, &rows, give_up) { locked(&shared.state).degraded.get_or_insert(error.message); } break; } } - if config.maintenance { + let remaining: Duration = retry_limit(shared, Instant::now()) + .saturating_duration_since(Instant::now()) + .min(Duration::from_millis(MAINTENANCE_BUDGET_MS)); + if config.maintenance && remaining >= Duration::from_millis(MINIMUM_MAINTENANCE_MS) { + bound_busy_wait(&store, Instant::now() + remaining); let request = MaintenanceRequest { now: SystemTime::now(), retention_days: config.retention_days, owner: config.maintenance_owner.clone(), - budget: Duration::from_millis(MAINTENANCE_BUDGET_MS), + budget: remaining, envelope: config.envelope.clone(), }; drop(maintain_sqlite_store(&store, &request)); diff --git a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs index 3d9ddf0df9..f6d713fb7f 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs @@ -3,7 +3,7 @@ use std::collections::{HashSet, VecDeque}; use std::time::{Duration, Instant, SystemTime}; -use crate::records::constants::MAX_OUTPUT_CHUNK_BYTES; +use crate::records::constants::{DROPPED_BEFORE_COLUMN, MAX_OUTPUT_CHUNK_BYTES}; use crate::records::main::format_timestamp::format_timestamp; use crate::records::main::output_row::output_row; use crate::records::models::{ @@ -50,6 +50,8 @@ pub(crate) struct WriterState { pub(crate) finished: bool, pub(crate) rejecting: bool, pub(crate) degraded: Option, + /// Store notices the user should see, such as a corrupt store moved aside. + pub(crate) notices: Vec, capacity: (usize, usize), } @@ -90,6 +92,7 @@ impl WriterState { finished: false, rejecting: false, degraded: None, + notices: Vec::new(), capacity, } } @@ -98,10 +101,19 @@ impl WriterState { &mut self.counts[stream.index()] } + /// Count `record` as dropped; a dropped output chunk hands its own earlier drops to the next chunk, so each drop is reported exactly once. fn drop_record(&mut self, record: &Queued) { self.counts_mut(record.stream).dropped += 1; if record.stream == DeliveryStream::Output { - self.unreported_output_drops += 1; + let carried: u64 = record + .rows + .iter() + .map(|row| match row.value(DROPPED_BEFORE_COLUMN) { + Some(RecallValue::Integer(count)) => u64::try_from(*count).unwrap_or_default(), + _ => 0, + }) + .sum(); + self.unreported_output_drops += 1 + carried; } } diff --git a/crates/sqlbuild-recall/src/writer/_helpers/session.rs b/crates/sqlbuild-recall/src/writer/_helpers/session.rs index c7e8169227..e3cbb18a26 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/session.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/session.rs @@ -1,10 +1,10 @@ //! The session handle: producers queue records; `close` flushes within the shutdown budget. +use std::sync::MutexGuard; use std::time::{Duration, Instant, SystemTime}; use serde_json::{Map, Value}; -use crate::records::_helpers::events::event_object; use crate::records::constants::{ ARTIFACT_DIGEST_PREFIX, EXECUTED_SQL_ROLE, SEVERITY_ORDER, SQL_ARTIFACT_KIND, SQL_MEDIA_TYPE, }; @@ -13,11 +13,13 @@ use crate::records::main::artifact_rows::artifact_rows; use crate::records::main::diagnostic_input_from_json::diagnostic_input_from_json; use crate::records::main::diagnostic_row::diagnostic_row; use crate::records::main::event_object_row::event_object_row; +use crate::records::main::event_row::event_row; use crate::records::main::format_timestamp::format_timestamp; use crate::records::main::sha256_hex::sha256_hex; -use crate::records::models::{ArtifactInput, OutputStream, RecallValue}; -use crate::store::constants::MAINTENANCE_BUDGET_MS; +use crate::records::models::{ArtifactInput, OutputStream, RecallRow, RecallValue}; use crate::writer::_helpers::delivery::locked; +use crate::writer::_helpers::queue::WriterState; +use crate::writer::constants::MAX_ARTIFACT_BYTES; use crate::writer::models::{ DeliveryStream, DeliverySummary, Priority, RecallSession, SetupOutcome, }; @@ -76,21 +78,27 @@ impl RecallSession { /// Queue one lifecycle event from its canonical JSON. pub fn publish_event(&self, event_json: &str) -> Result<(), RecordError> { - let event: Map = event_object(event_json)?; - self.publish_event_object(&event) + let state = locked(&self.shared.state); + let row = event_row(&state.envelope, event_json)?; + self.queue_event(state, row); + Ok(()) } /// Queue one lifecycle event already decoded to its canonical JSON object. pub fn publish_event_object(&self, event: &Map) -> Result<(), RecordError> { - let mut state = locked(&self.shared.state); + let state = locked(&self.shared.state); let row = event_object_row(&state.envelope, event)?; + self.queue_event(state, row); + Ok(()) + } + + fn queue_event(&self, mut state: MutexGuard<'_, WriterState>, row: RecallRow) { let priority = match row.value("severity") { Some(RecallValue::Text(severity)) => event_priority(severity), _ => Priority::Lifecycle, }; state.push(vec![row], DeliveryStream::Events, priority); self.shared.wake.notify_one(); - Ok(()) } /// Queue one diagnostic from its canonical JSON, when at or above the minimum severity. @@ -116,6 +124,13 @@ impl RecallSession { /// Queue the SQL a statement executed, once per digest per invocation. pub fn publish_sql(&self, sql: &str, statement_id: Option<&str>, resource_id: Option<&str>) { + if sql.len() > MAX_ARTIFACT_BYTES { + let mut state = locked(&self.shared.state); + let counts = state.counts_mut(DeliveryStream::Artifacts); + counts.accepted += 1; + counts.dropped += 1; + return; + } let digest = format!("{ARTIFACT_DIGEST_PREFIX}{}", sha256_hex(sql.as_bytes())); let mut state = locked(&self.shared.state); if !state.artifacts_seen.insert(digest.clone()) { @@ -167,36 +182,68 @@ impl RecallSession { } } - /// Flush within the shutdown budget, write the delivery rows and return the accounting. Records still queued at the deadline are counted as dropped. - pub fn close(&self) -> DeliverySummary { - let deadline = Instant::now() + self.shutdown_timeout; - let wait_until = deadline + Duration::from_millis(MAINTENANCE_BUDGET_MS + 500); - { - let mut state = locked(&self.shared.state); - if !state.closing { - state.closing = true; - state.deadline = Some(deadline); - state.flush_output(); + /// Start closing: flush pending output and give delivery the shutdown budget from now. + pub fn begin_close(&self) { + let mut state = locked(&self.shared.state); + if !state.closing { + state.closing = true; + state.deadline = Some(Instant::now() + self.shutdown_timeout); + state.flush_output(); + } + self.shared.wake.notify_all(); + } + + /// Wait up to `slice` for delivery to finish; past the deadline, drop what is left and report done. + pub fn wait_closed(&self, slice: Duration) -> bool { + let mut state = locked(&self.shared.state); + let Some(deadline) = state.deadline else { + return state.finished; + }; + let until: Instant = (Instant::now() + slice).min(deadline); + while !state.finished { + let now = Instant::now(); + if now >= deadline { + state.rejecting = true; + state.drop_all(); + state.degraded.get_or_insert_with(|| { + "Recall delivery did not finish within the shutdown timeout".to_owned() + }); + return true; } - self.shared.wake.notify_all(); - while !state.finished { - let now = Instant::now(); - if now >= wait_until { - state.rejecting = true; - state.drop_all(); - state.degraded.get_or_insert_with(|| { - "Recall delivery did not finish within the shutdown timeout".to_owned() - }); - break; - } - state = self - .shared - .wake - .wait_timeout(state, wait_until - now) - .unwrap_or_else(std::sync::PoisonError::into_inner) - .0; + if now >= until { + return false; } + state = self + .shared + .wake + .wait_timeout(state, until - now) + .unwrap_or_else(std::sync::PoisonError::into_inner) + .0; } + true + } + + /// Stop waiting for delivery at once, as on a second interrupt; what is queued is dropped. + pub fn abandon(&self) { + let mut state = locked(&self.shared.state); + state.rejecting = true; + state.drop_all(); + state + .degraded + .get_or_insert_with(|| "Recall delivery was interrupted".to_owned()); + self.shared.wake.notify_all(); + } + + /// Flush within the shutdown budget, a hard cap on the delay, and return the accounting; records still queued at the deadline are counted as dropped. + pub fn close(&self) -> DeliverySummary { + self.begin_close(); + while !self.wait_closed(self.shutdown_timeout) {} self.summary() } + + /// Store notices the user should see, such as a corrupt store moved aside. + #[must_use] + pub fn notices(&self) -> Vec { + locked(&self.shared.state).notices.clone() + } } diff --git a/crates/sqlbuild-recall/src/writer/constants.rs b/crates/sqlbuild-recall/src/writer/constants.rs index dd1ba9536b..294f347961 100644 --- a/crates/sqlbuild-recall/src/writer/constants.rs +++ b/crates/sqlbuild-recall/src/writer/constants.rs @@ -15,3 +15,13 @@ pub const SETUP_BUDGET_MS: u64 = 2_000; pub const MAX_RETRY_BACKOFF_MS: u64 = 500; /// A batch that stays busy this long while the command runs is counted as failed. pub const RUNNING_RETRY_LIMIT_MS: u64 = 30_000; +/// Pause between setup attempts while another writer holds the store. +pub const SETUP_RETRY_MS: u64 = 50; +/// Longest wait between wall-clock checks, so a resumed process beats promptly. +pub const CLOCK_CHECK_MS: u64 = 1_000; +/// Budget for the final delivery rows, inside the shutdown budget. +pub const FINAL_ROWS_BUDGET_MS: u64 = 500; +/// Maintenance is skipped when less than this remains of the shutdown budget. +pub const MINIMUM_MAINTENANCE_MS: u64 = 200; +/// SQL longer than this is counted as dropped without hashing or copying it. +pub const MAX_ARTIFACT_BYTES: usize = 4 * 1024 * 1024; diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs index 4f7e1d9899..9ab59f7f45 100644 --- a/crates/sqlbuild-recall/src/writer/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -162,3 +162,81 @@ pub(crate) fn unwritable_session() -> (bool, bool) { summary.degraded_reason.is_some(), ) } + +/// Seconds `close` took and whether delivery degraded, while another connection holds the write lock. +pub(crate) fn close_under_held_lock(shutdown: Duration) -> (f64, bool) { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("logs/recall.sqlite3"); + let warm = start_recall_session(config(path.clone(), "warm")); + drop(warm.close()); + let holder = rusqlite::Connection::open(&path).expect("the store opens"); + holder + .execute_batch("BEGIN EXCLUSIVE") + .expect("the write lock is taken"); + let session = start_recall_session(SessionConfig { + shutdown_timeout: shutdown, + ..config(path, "held") + }); + session.append_output(OutputStream::Stdout, b"waiting\n"); + let started = std::time::Instant::now(); + let summary = session.close(); + let elapsed = started.elapsed().as_secs_f64(); + drop(holder); + (elapsed, summary.degraded_reason.is_some()) +} + +/// The notices and resulting store state after starting on a file that is not a database. +pub(crate) fn session_on_corrupt_store() -> (usize, bool, Vec>) { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("recall.sqlite3"); + std::fs::write(&path, b"this is not a database").expect("test files can be written"); + let session = start_recall_session(config(path.clone(), "fresh")); + session.append_output(OutputStream::Stdout, b"kept\n"); + let summary = session.close(); + let moved: bool = std::fs::read_dir(directory.path()) + .expect("the directory lists") + .filter_map(Result::ok) + .any(|entry| entry.file_name().to_string_lossy().contains(".corrupt-")); + let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); + let rows = query_store(&store, "SELECT status FROM invocations", None) + .expect("the query is valid") + .rows; + drop(summary); + (session.notices().len(), moved, rows) +} + +/// Artifact counts after publishing SQL of `length` bytes. +pub(crate) fn artifact_counts(length: usize) -> (u64, u64, u64) { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let session = start_recall_session(config(directory.path().join("recall.sqlite3"), "sql")); + session.publish_sql(&"x".repeat(length), Some("s1"), None); + let summary = session.close(); + let counts = summary + .streams + .iter() + .find(|(stream, _)| *stream == DeliveryStream::Artifacts) + .map(|(_, counts)| *counts) + .unwrap_or_default(); + (counts.accepted, counts.delivered, counts.dropped) +} + +/// Output chunks dropped by a queue of `capacity` records, as counted and as each delivered chunk reports them. +pub(crate) fn output_drops(capacity: usize, chunks: usize) -> (u64, Vec>) { + let mut state = WriterState::with_capacity(envelope("inv"), (capacity, usize::MAX)); + for index in 0..chunks { + let stream = [OutputStream::Stdout, OutputStream::Stderr][index % 2]; + state.append_output(stream, b"x", SystemTime::now()); + } + let mut taken: Vec = state.take_batch(usize::MAX); + state.flush_output(); + taken.extend(state.take_batch(usize::MAX)); + let reported: Vec> = taken + .iter() + .flat_map(|record| record.rows.iter()) + .map(|row| row.value("dropped_before").cloned()) + .collect(); + ( + state.counts[DeliveryStream::Output.index()].dropped, + reported, + ) +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_queue.rs b/crates/sqlbuild-recall/src/writer/tests/test_queue.rs index def94d1c63..745ecf0900 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_queue.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_queue.rs @@ -1,7 +1,9 @@ use crate::records::models::{OutputStream, RecallValue}; use crate::writer::models::{DeliveryStream, Priority}; -use crate::writer::tests::helpers::{chunked, pushed}; -use crate::writer::tests::test_types::{ChunkingTestCase, DisplacementTestCase}; +use crate::writer::tests::helpers::{chunked, output_drops, pushed}; +use crate::writer::tests::test_types::{ + ChunkingTestCase, DisplacementTestCase, OutputDropTestCase, +}; #[test] fn given_a_full_queue_when_higher_priority_arrives_then_the_oldest_lowest_is_displaced() { @@ -89,3 +91,25 @@ fn given_interleaved_and_large_writes_when_chunked_then_sequences_and_offsets_ar ); } } + +#[test] +fn given_a_full_queue_when_output_is_dropped_then_each_drop_is_counted_once() { + let test_cases = [OutputDropTestCase { + description: "two chunks dropped by a full queue are counted once and reported once", + capacity: 1, + chunks: 4, + expected_drops: ( + 2, + vec![Some(RecallValue::Integer(0)), Some(RecallValue::Integer(2))], + ), + }]; + + for test_case in test_cases { + assert_eq!( + output_drops(test_case.capacity, test_case.chunks), + test_case.expected_drops, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_session.rs b/crates/sqlbuild-recall/src/writer/tests/test_session.rs index d52e6384e5..e978877de1 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_session.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_session.rs @@ -1,11 +1,20 @@ +use std::time::Duration; + use crate::records::models::{OutputStream, RecallValue}; use crate::writer::models::SetupOutcome; -use crate::writer::tests::helpers::recorded_session; + #[cfg(unix)] use crate::writer::tests::helpers::unwritable_session; +use crate::writer::tests::helpers::{ + artifact_counts, close_under_held_lock, recorded_session, session_on_corrupt_store, +}; #[cfg(unix)] use crate::writer::tests::test_types::UnwritableTestCase; -use crate::writer::tests::test_types::{RecordedSession, SessionTestCase}; +use crate::writer::tests::test_types::{ + ArtifactSizeTestCase, CorruptStoreTestCase, HeldLockTestCase, RecordedSession, SessionTestCase, +}; + +const CLOSE_SLACK_SECONDS: f64 = 0.3; #[test] fn given_a_session_when_closed_then_every_record_and_the_delivery_rows_are_written() { @@ -66,3 +75,76 @@ fn given_an_unwritable_store_when_started_then_setup_reports_permission_and_clos ); } } + +#[test] +fn given_a_held_write_lock_when_closed_then_the_shutdown_budget_caps_the_delay() { + let test_cases = [HeldLockTestCase { + description: "a locked store costs at most the shutdown budget at exit", + shutdown_ms: 700, + expected_degraded: true, + }]; + + for test_case in test_cases { + let (elapsed, degraded) = + close_under_held_lock(Duration::from_millis(test_case.shutdown_ms)); + let budget: f64 = Duration::from_millis(test_case.shutdown_ms).as_secs_f64(); + assert!( + elapsed <= budget + CLOSE_SLACK_SECONDS, + "{}: {elapsed:.3}s", + test_case.description + ); + assert_eq!( + degraded, test_case.expected_degraded, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_file_that_is_not_a_database_when_started_then_it_is_moved_aside_once() { + let test_cases = [CorruptStoreTestCase { + description: "the corrupt file is kept aside and a fresh store records the run", + expected_notices: 1, + expected_moved: true, + expected_rows: vec![vec![RecallValue::Text("unknown".to_owned())]], + }]; + + for test_case in test_cases { + assert_eq!( + session_on_corrupt_store(), + ( + test_case.expected_notices, + test_case.expected_moved, + test_case.expected_rows + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sql_of_different_sizes_when_published_then_oversized_sql_is_dropped_unhashed() { + let test_cases = [ + ArtifactSizeTestCase { + description: "ordinary SQL is delivered with its reference", + length: 64, + expected_counts: (1, 1, 0), + }, + ArtifactSizeTestCase { + description: "SQL above the artifact limit is counted as dropped", + length: 4 * 1024 * 1024 + 1, + expected_counts: (1, 0, 1), + }, + ]; + + for test_case in test_cases { + assert_eq!( + artifact_counts(test_case.length), + test_case.expected_counts, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs index 78ba896904..a4503c4ff5 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -36,3 +36,29 @@ pub(crate) struct UnwritableTestCase { pub(crate) description: &'static str, pub(crate) expected_permission_and_degraded: (bool, bool), } + +pub(crate) struct HeldLockTestCase { + pub(crate) description: &'static str, + pub(crate) shutdown_ms: u64, + pub(crate) expected_degraded: bool, +} + +pub(crate) struct CorruptStoreTestCase { + pub(crate) description: &'static str, + pub(crate) expected_notices: usize, + pub(crate) expected_moved: bool, + pub(crate) expected_rows: Vec>, +} + +pub(crate) struct ArtifactSizeTestCase { + pub(crate) description: &'static str, + pub(crate) length: usize, + pub(crate) expected_counts: (u64, u64, u64), +} + +pub(crate) struct OutputDropTestCase { + pub(crate) description: &'static str, + pub(crate) capacity: usize, + pub(crate) chunks: usize, + pub(crate) expected_drops: (u64, Vec>), +} diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index fdc7af05e0..34bb86d0ce 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -80,3 +80,38 @@ class RepeatedBuildTestCase: builds: int sql: str expected_rows: list[list[object]] + + +@dataclass(frozen=True) +class RecordedCommandTestCase: + description: str + args: tuple[str, ...] + expected_recorded: list[list[object]] + + +@dataclass(frozen=True) +class HeldLockTestCase: + description: str + shutdown_seconds: float + builds_before_lock: int + expected_exit_code: int + + +@dataclass(frozen=True) +class ConcurrentWritersTestCase: + description: str + lines_per_stream: int + expected_order: list[int] + + +@dataclass(frozen=True) +class CorruptStoreTestCase: + description: str + expected_notice_prefix: str + expected_recorded: list[list[object]] + + +@dataclass(frozen=True) +class FailingStderrTestCase: + description: str + expected_exit_code: int diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index e7fa13564b..115c9571d6 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -2,11 +2,14 @@ import json import os +import re import signal import stat import subprocess import sys import time +from collections.abc import Iterator +from contextlib import contextmanager from pathlib import Path PYTHON_CLI: Path = Path(sys.executable).with_name("sqlbuild-python") @@ -14,6 +17,7 @@ EMITTED_STDERR: bytes = "warning \u2713 from the task\n".encode() EMITTED_TAIL: bytes = b"after the warning\n" EMPTY_RESULT: bytes = b'{"rows": []}' +NUMBERED_LINE: re.Pattern[bytes] = re.compile(rb"^(\d+) std(?:out|err)$", re.MULTILINE) PROJECT_TOML: str = ( 'name = "orders_shop"\n' 'adapter = "duckdb"\n' @@ -30,8 +34,10 @@ ) EMIT_TASK: str = ( '"""A task that writes exact bytes to both streams."""\n\n' + "import itertools\n" "import os\n" "import sys\n" + "import threading\n" "import time\n\n" "from sqlbuild.tasks import task\n\n\n" "@task()\n" @@ -45,7 +51,25 @@ f" sys.stdout.write({EMITTED_TAIL.decode()!r})\n" " sys.stdout.flush()\n" ' time.sleep(float(os.environ.get("ORDERS_TASK_SLEEP_SECONDS", "0")))\n' - " return ctx.result()\n" + ' _write_concurrently(int(os.environ.get("ORDERS_TASK_THREAD_LINES", "0")))\n' + " return ctx.result()\n\n\n" + "def _write_concurrently(lines):\n" + ' """Two threads write numbered lines; the number gives the true global order."""\n\n' + " lock = threading.Lock()\n" + " counter = itertools.count()\n\n" + " def writer(name, stream):\n" + " for _ in range(lines):\n" + " with lock:\n" + ' stream.write(f"{next(counter)} {name}\\n")\n' + " stream.flush()\n\n" + " threads = [\n" + ' threading.Thread(target=writer, args=("stdout", sys.stdout)),\n' + ' threading.Thread(target=writer, args=("stderr", sys.stderr)),\n' + " ]\n" + " for thread in threads:\n" + " thread.start()\n" + " for thread in threads:\n" + " thread.join()\n" ) @@ -185,3 +209,109 @@ def concurrent_builds(project_dir: Path, targets: tuple[str, ...]) -> tuple[list reads += 1 unexpected_failures += int(reader.returncode != 0 and b"[C971]" not in reader.stderr) return [build.wait(timeout=600) for build in builds], reads, unexpected_failures + + +def timed_build(project_dir: Path) -> tuple[subprocess.CompletedProcess[bytes], float, float]: + """A build with the seconds until its first stderr byte and until it exited.""" + + started: float = time.monotonic() + process: subprocess.Popen[bytes] = start(project_dir, "build") + assert process.stderr is not None + first_byte: bytes = process.stderr.read(1) + first_output: float = time.monotonic() - started + stdout, stderr = process.communicate(timeout=600) + return ( + subprocess.CompletedProcess(process.args, process.returncode, stdout, first_byte + stderr), + first_output, + time.monotonic() - started, + ) + + +@contextmanager +def held_store_lock(project_dir: Path) -> Iterator[None]: + """Another process holds the store's write lock while the block runs, creating it if absent.""" + + store: Path = project_dir / "logs" / "recall.sqlite3" + store.parent.mkdir(parents=True, exist_ok=True) + holder: subprocess.Popen[bytes] = subprocess.Popen( + [ + sys.executable, + "-c", + "import sqlite3, sys, time\n" + "connection = sqlite3.connect(sys.argv[1], isolation_level=None)\n" + "connection.execute('BEGIN EXCLUSIVE')\n" + "print('locked', flush=True)\n" + "time.sleep(600)\n", + str(store), + ], + stdout=subprocess.PIPE, + ) + try: + assert holder.stdout is not None + _ = holder.stdout.readline() + yield + finally: + holder.kill() + _ = holder.wait(timeout=30) + + +def build_while_store_locked( + project_dir: Path, +) -> tuple[subprocess.CompletedProcess[bytes], float, float]: + """A timed build while another process holds the store's write lock.""" + + with held_store_lock(project_dir): + return timed_build(project_dir) + + +def compute_log_streams(project_dir: Path, invocation_id: str) -> tuple[bytes, bytes]: + """The stdout and stderr bytes compute logs captured for one invocation.""" + + capture: Path = next((project_dir / "logs").glob(f"*/{invocation_id}")) + return (capture / "stdout.log").read_bytes(), (capture / "stderr.log").read_bytes() + + +def write_corrupt_store(project_dir: Path) -> None: + """Replace the Recall store with a file that is not a database.""" + + store: Path = project_dir / "logs" / "recall.sqlite3" + store.parent.mkdir(parents=True, exist_ok=True) + _ = store.write_bytes(b"this file is not a database\n") + + +FAILING_RECALL_STDERR: str = ( + "import sys\n\n\n" + "class _RejectsRecallNotices:\n" + " def __init__(self, stream):\n" + " self._stream = stream\n\n" + " def write(self, text):\n" + ' if "Recall" in text:\n' + ' raise OSError("stderr rejected the Recall notice")\n' + " return self._stream.write(text)\n\n" + " def __getattr__(self, name):\n" + " return getattr(self._stream, name)\n\n\n" + "sys.stderr = _RejectsRecallNotices(sys.stderr)\n" +) + + +def build_with_failing_recall_notices(project_dir: Path, site_dir: Path) -> int: + """The exit code of a build whose Recall notices fail to write, while the store is locked.""" + + site_dir.mkdir(parents=True, exist_ok=True) + _ = (site_dir / "sitecustomize.py").write_text(FAILING_RECALL_STDERR, encoding="utf-8") + with held_store_lock(project_dir): + return run(project_dir, "build", PYTHONPATH=str(site_dir)).returncode + + +def prepared_project(parent: Path, extra_toml: str, *, builds: int) -> Path: + """A project under `parent`, built `builds` times first.""" + + project_dir: Path = write_project(parent / "orders_shop", extra_toml=extra_toml) + _ = [run(project_dir, "build") for _ in range(builds)] + return project_dir + + +def numbered_lines(output: bytes) -> list[int]: + """The leading numbers of the numbered lines in `output`, in order.""" + + return [int(number) for number in NUMBERED_LINE.findall(output)] diff --git a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py index 10cfab4017..8324b8a399 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py @@ -10,11 +10,16 @@ AbandonedRunTestCase, CombinedOutputTestCase, ConcurrentBuildsTestCase, + ConcurrentWritersTestCase, + CorruptStoreTestCase, DisabledRecallTestCase, + FailingStderrTestCase, + HeldLockTestCase, JsonQueryTestCase, ReadOnlyStoreTestCase, RecallOutputTestCase, RecallRejectionTestCase, + RecordedCommandTestCase, RecordedFactsTestCase, RepeatedBuildTestCase, TextOutputTestCase, @@ -23,11 +28,18 @@ EMITTED_STDERR, EMITTED_STDOUT, EMITTED_TAIL, + build_while_store_locked, + build_with_failing_recall_notices, build_with_read_only_logs, + compute_log_streams, concurrent_builds, killed_build, + numbered_lines, + prepared_project, recall_rows, run, + timed_build, + write_corrupt_store, write_project, ) @@ -437,3 +449,212 @@ def test_given_identical_builds_when_recorded_then_artifacts_and_delivery_counts assert exit_codes == [0 for _ in range(test_case.builds)] assert rows == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + RecordedCommandTestCase( + description="compile", + args=("compile",), + expected_recorded=[["compile", "completed"]], + ), + RecordedCommandTestCase( + description="build", + args=("build",), + expected_recorded=[["build", "completed"]], + ), + RecordedCommandTestCase( + description="plan", + args=("plan",), + expected_recorded=[["plan", "completed"]], + ), + RecordedCommandTestCase( + description="dag", + args=("dag",), + expected_recorded=[["dag", "completed"]], + ), + RecordedCommandTestCase( + description="check", + args=("check",), + expected_recorded=[["check", "completed"]], + ), + RecordedCommandTestCase( + description="test", + args=("test",), + expected_recorded=[["test", "completed"]], + ), + RecordedCommandTestCase( + description="lineage orders", + args=("lineage", "orders"), + expected_recorded=[["lineage", "completed"]], + ), + RecordedCommandTestCase( + description="debug", + args=("debug",), + expected_recorded=[["debug", "completed"]], + ), + RecordedCommandTestCase( + description="query SELECT 1 AS one", + args=("query", "SELECT 1 AS one"), + expected_recorded=[["query", "completed"]], + ), + RecordedCommandTestCase( + description="a raw-query diff is not observed, so it is not recorded", + args=( + "diff", + "--unkeyed", + "--left-query", + "SELECT 1 AS one", + "--right-query", + "SELECT 1 AS one", + ), + expected_recorded=[], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_successful_command_when_recorded_then_it_ends_completed( + test_case: RecordedCommandTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + _ = run(project_dir, "build") + + completed: subprocess.CompletedProcess[bytes] = run(project_dir, *test_case.args) + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT command, status FROM invocations " + f"WHERE command = '{test_case.args[0]}' ORDER BY invocation_started_at DESC LIMIT 1", + ) + + assert completed.returncode == 0, completed.stderr.decode(errors="replace") + assert recorded == test_case.expected_recorded + + +@pytest.mark.parametrize( + "test_case", + [ + HeldLockTestCase( + description="a store locked by another process costs at most the budget", + shutdown_seconds=1.0, + builds_before_lock=1, + expected_exit_code=0, + ), + HeldLockTestCase( + description="a store another process holds before setup never delays the start", + shutdown_seconds=1.0, + builds_before_lock=0, + expected_exit_code=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_locked_store_when_building_then_start_and_exit_each_add_at_most_the_budget( + test_case: HeldLockTestCase, tmp_path: Path +) -> None: + budget: str = f'\n[recall]\nshutdown_timeout = "{int(test_case.shutdown_seconds * 1000)}ms"\n' + baseline_dir: Path = prepared_project(tmp_path / "baseline", budget, builds=1) + project_dir: Path = prepared_project( + tmp_path / "locked", budget, builds=test_case.builds_before_lock + ) + _, baseline_first, baseline_total = timed_build(baseline_dir) + + locked, locked_first, locked_total = build_while_store_locked(project_dir) + + slack: float = 0.75 + assert locked.returncode == test_case.expected_exit_code + assert locked_first <= baseline_first + slack + assert locked_total <= baseline_total + test_case.shutdown_seconds + slack + assert ( + locked.stderr.decode(errors="replace") + .rstrip() + .endswith("completed successfully; only its Recall history is affected.") + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ConcurrentWritersTestCase( + description="two threads writing both streams keep one global order", + lines_per_stream=400, + expected_order=list(range(800)), + ), + ], + ids=lambda case: case.description, +) +def test_given_concurrent_stream_writers_when_recalled_then_order_matches_compute_logs( + test_case: ConcurrentWritersTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + + built: subprocess.CompletedProcess[bytes] = run( + project_dir, "build", ORDERS_TASK_THREAD_LINES=str(test_case.lines_per_stream) + ) + [[invocation_id, stdout_hex, stderr_hex, combined_hex]] = recall_rows( + project_dir, + "SELECT invocation_id, hex(stdout), hex(stderr), hex(combined) FROM output_transcripts", + ) + numbered: list[int] = numbered_lines(bytes.fromhex(str(combined_hex))) + + assert built.returncode == 0, built.stderr.decode(errors="replace") + assert (bytes.fromhex(str(stdout_hex)), bytes.fromhex(str(stderr_hex))) == ( + compute_log_streams(project_dir, str(invocation_id)) + ) + assert numbered == test_case.expected_order + + +@pytest.mark.parametrize( + "test_case", + [ + CorruptStoreTestCase( + description="a store that is not a database is moved aside before any output", + expected_notice_prefix="Recall: ", + expected_recorded=[["build", "completed"]], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_corrupt_store_when_building_then_it_is_moved_aside_and_recording_restarts( + test_case: CorruptStoreTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + write_corrupt_store(project_dir) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + again: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + recorded: list[list[object]] = recall_rows( + project_dir, "SELECT command, status FROM invocations ORDER BY invocation_started_at" + ) + moved: list[Path] = list((project_dir / "logs").glob("recall.sqlite3.corrupt-*")) + first_line: str = built.stderr.decode(errors="replace").splitlines()[0] + + assert (built.returncode, again.returncode) == (0, 0) + assert first_line.startswith(test_case.expected_notice_prefix) + assert str(moved[0].name) in first_line + assert len(moved) == 1 + assert b".corrupt-" not in again.stderr + assert recorded == [*test_case.expected_recorded, *test_case.expected_recorded] + + +@pytest.mark.parametrize( + "test_case", + [ + FailingStderrTestCase( + description="a Recall notice that cannot be written keeps the exit code", + expected_exit_code=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_stderr_when_recall_reports_then_the_exit_code_is_unchanged( + test_case: FailingStderrTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project( + tmp_path / "orders_shop", extra_toml='\n[recall]\nshutdown_timeout = "300ms"\n' + ) + _ = run(project_dir, "build") + + exit_code: int = build_with_failing_recall_notices(project_dir, tmp_path / "site") + + assert exit_code == test_case.expected_exit_code From e20d539a04fecb9837ac7944c999cbdbaeff2660 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 06:39:05 +0100 Subject: [PATCH 09/19] docs: document local Recall history and sqb recall --- .../sqlbuild/references/docs/CONTENTS.md | 2 + .../sqlbuild/references/docs/cli/recall.md | 123 ++++++++++++++++++ .../references/docs/concepts/observability.md | 4 +- .../docs/concepts/observability/recall.md | 121 +++++++++++++++++ website/src/content/docs/docs/cli/recall.mdx | 114 ++++++++++++++++ .../docs/docs/concepts/observability.mdx | 4 +- .../docs/concepts/observability/recall.mdx | 109 ++++++++++++++++ website/src/sidebar.json | 2 + 8 files changed, 477 insertions(+), 2 deletions(-) create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md create mode 100644 website/src/content/docs/docs/cli/recall.mdx create mode 100644 website/src/content/docs/docs/concepts/observability/recall.mdx diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index f007592421..c659d00a80 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -76,6 +76,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in ## Advanced Concepts - [Execution Observability](concepts/observability.md) (`concepts/observability`) - Choose authoritative lifecycle facts, readable logs, or command-output records. +- [Recall](concepts/observability/recall.md) (`concepts/observability/recall`) - The local history of every command: what ran, what it executed, and what it printed. - [Typed Sinks](concepts/observability/sinks.md) (`concepts/observability/sinks`) - Export lifecycle facts and command output through project-owned providers. - [Execution history](concepts/observability/execution-history.md) (`concepts/observability/execution-history`) - Store and query lifecycle events and run history from Python, with SQLite locally or PostgreSQL when deployed. - [Overview](concepts/declaration-scopes.md) (`concepts/declaration-scopes`) - Limit enums, constants, macros, audits, schemas, and named hooks to the parts of a project that use them. @@ -130,6 +131,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [lineage](cli/lineage.md) (`cli/lineage`) - Explore model and column-level dependency graphs from the command line. - [sqb dag](cli/dag.md) (`cli/dag`) - Generate the static DAG artifact for Dagster and other integrations. - [sqb query](cli/query.md) (`cli/query`) - Run ad hoc SQL queries against the project database. +- [sqb recall](cli/recall.md) (`cli/recall`) - Query the recorded history of past invocations. - [debug](cli/debug.md) (`cli/debug`) - Validate project configuration and test the warehouse connection. - [sqb janitor](cli/janitor.md) (`cli/janitor`) - Archive and then delete stale warehouse relations. - [sqb clean](cli/clean.md) (`cli/clean`) - Remove compiled artifacts from the target directory. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md new file mode 100644 index 0000000000..38683138bb --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md @@ -0,0 +1,123 @@ + + +# sqb recall + +> Query the recorded history of past invocations. + +Online: https://sqlbuild.com/docs/cli/recall/ + +## Contents + +- Usage +- sqb recall query +- sqb recall schema +- sqb recall output +- Errors + +Read the local [Recall](../concepts/observability/recall.md) store: past invocations, the +statements they executed, and what they printed. `sqb recall` never records itself. + +## Usage + +```bash +sqb recall query "SELECT command, status, duration_ms FROM invocations ORDER BY invocation_started_at DESC" +sqb recall schema +sqb recall output last +``` + +## sqb recall query + +Run one read-only SQLite `SELECT` against the Recall views. Statements that write, and multiple +statements, are rejected. + +| Flag | Description | +|------|-------------| +| `sql` | The query (positional argument) | +| `--format` | `long` (default), `table`, `json`, or `csv` | +| `--json` | Shorthand for `--format json` | +| `--limit` | Maximum rows to return (default: 100) | +| `--no-limit` | Disable the row limit | +| `--max-cell-bytes` | Longest cell in text formats, in bytes (default: 4096; `0` for no limit) | +| `--target` | Target whose store to read (default: the default target) | +| `--backend` | `logs` (default) reads the Recall store; `warehouse` runs the query on the target's connection | + +The latest invocation in a project named `orders_shop`: + +```bash +sqb recall query --format table \ + "SELECT command, status, exit_code, duration_ms FROM invocations + ORDER BY invocation_started_at DESC LIMIT 1" +``` + +``` +command | status | exit_code | duration_ms +------- | --------- | --------- | ----------- +build | completed | 0 | 752.0 + +1 row +``` + +The slowest statements of past builds, with their SQL: + +```bash +sqb recall query --format table --max-cell-bytes 60 \ + "SELECT s.resource_name, round(s.duration_ms, 1) AS ms, a.text AS sql + FROM statements s JOIN artifacts a USING (invocation_id, statement_id) + WHERE s.statement_kind = 'CREATE' ORDER BY s.duration_ms DESC LIMIT 2" +``` + +``` +resource_name | ms | sql +------------- | --- | ------------------------------------------------------------------------ +orders | 3.6 | CREATE OR REPLACE TABLE dev.orders__staging AS SELECT 1 …[+74 bytes] +NULL | 3.2 | CREATE SCHEMA IF NOT EXISTS dev + +2 rows +``` + +Failed invocations of the last week: + +```bash +sqb recall query \ + "SELECT invocation_id, command, error_code FROM invocations + WHERE status = 'failed' AND invocation_started_at >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-7 days')" +``` + +Timestamps are ISO-8601 UTC text, so they compare correctly as strings. Filter on +`invocation_started_at` to bound a scan. + +## sqb recall schema + +List every Recall view with its columns, types, and join keys. Accepts `--format`, `--json`, and +`--target`. With `--backend warehouse`, it lists SQLBuild's state tables in the target's warehouse +instead, or describes one with `--relation `. + +## sqb recall output + +Print what an invocation wrote, byte for byte. + +```bash +sqb recall output last # the latest invocation, both streams in order +sqb recall output 7607b5b6 --stream stderr # an invocation ID or unique prefix +sqb recall output last --tail 20 # only the last 20 lines +``` + +| Flag | Description | +|------|-------------| +| `invocation` | Invocation ID, unique prefix, or `last` (default) | +| `--stream` | `combined` (default), `stdout`, or `stderr` | +| `--tail` | Print only the last N lines | +| `--target` | Target whose store to read | + +Output that Recall could not record completely is followed by a note on stderr naming the gap. + +## Errors + +| Code | Meaning | +|------|---------| +| `C970` | Recall is disabled for this project or target | +| `C971` | The store cannot be read, or no history is recorded yet | +| `C972` | No invocation matches | +| `C973` | The invocation prefix matches more than one invocation | +| `C974` | The query is invalid or not read-only | +| `C975` | The `[recall]` settings are invalid | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability.md index 36bb9f197d..1129254621 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability.md @@ -14,6 +14,7 @@ whether work started, completed, failed, skipped, or retried. | `LifecycleEvent` | Immutable invocation, run, resource-attempt, operation, statement, retry, and audit facts | Canonical lifecycle evidence once durably stored | | `DiagnosticLog` | Structured framework diagnostics | Explanatory only | | `stdout.log` / `stderr.log` | Exact host-local process output | Troubleshooting transcript | +| [Recall](observability/recall.md) | Local queryable history of invocations, statements, SQL, and output | Troubleshooting history; best effort | | `CommandOutputRecord` | Bounded, ANSI-free remote stdout/stderr chunks | Potentially sensitive and lossy troubleshooting data | | Final JSON output | End-of-command aggregate | Result projection, not event history | @@ -83,4 +84,5 @@ A start without a terminal fact is unknown or presumed lost. Consumers must not or failure. Sink delivery is bounded and best effort; destination failure does not change successful warehouse work, and successful delivery does not prove the command succeeded. -See [Typed Sinks](observability/sinks.md) to export lifecycle and command-output records. +See [Recall](observability/recall.md) for the local history `sqb recall` queries, and +[Typed Sinks](observability/sinks.md) to export lifecycle and command-output records. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md new file mode 100644 index 0000000000..a16fe2ed40 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -0,0 +1,121 @@ + + +# Recall + +> The local history of every command: what ran, what it executed, and what it printed. + +Online: https://sqlbuild.com/docs/concepts/observability/recall/ + +## Contents + +- What is recorded +- Settings +- Retention +- When the store cannot be written +- Querying history +- Relationship to compute logs + +Recall records each `sqb` invocation in a local SQLite store so you can ask what happened after the +fact: which command ran, how it ended, which statements it executed and how long they took, and +exactly what it printed. Recall is on by default and needs no setup. + +## What is recorded + +For every recorded invocation, Recall keeps: + +- the invocation itself: command, arguments, target, adapter, status, exit code, and timing; +- its [lifecycle events](../observability.md): runs, resource attempts, statements, + retries, and audit results; +- the SQL text of every executed statement, stored once per distinct statement and shared across + invocations; +- its stdout and stderr, byte for byte, in the order they were written; +- framework diagnostics at or above `diagnostics_min_severity`. + +Writes happen on a background thread and never fail the command. A command that cannot record all +of its history still succeeds, prints a warning followed by its own final status, and +`output_invocations` reports the gap. An invocation whose process died without finishing has +status `abandoned`. + +Every command is recorded except `sqb recall` itself, commands that create a project (such as +`sqb init`), and raw-query `sqb diff`. A `sqb compile` answered from the launcher's warm cache is not +recorded either. + +The store is `logs/recall.sqlite3` in the project directory. Add `logs/` to `.gitignore`; the store +can contain sensitive SQL and output. + +## Settings + +Configure Recall in `[recall]` in `sqlbuild_project.toml`. A `[targets..recall]` table +overrides individual keys for one target. + +```toml +[recall] +retention = "30d" # days of history to keep; also accepts hours, such as "720h" +shutdown_timeout = "3s" # longest delay Recall may add when a command exits +diagnostics_min_severity = "info" + +[targets.ci.recall] +enabled = false # do not record runs against the ci target +``` + +| Key | Default | Description | +|-----|---------|-------------| +| `enabled` | `true` | Set to `false` to stop recording | +| `connection` | (none) | A `sqlite` connection whose `path` names the store; defaults to `logs/recall.sqlite3` | +| `retention` | `"30d"` | How long invocations are kept, in days (`d`) or hours (`h`), up to 3650 days | +| `shutdown_timeout` | `"3s"` | The cap on time spent flushing history at exit, in `s` or `ms`, up to 10 minutes | +| `diagnostics_min_severity` | `"info"` | `trace`, `debug`, `info`, `warning`, `error`, or `critical` | + +To turn Recall off for the whole project: + +```toml +[recall] +enabled = false +``` + +Recall never delays the start of a command. At exit it waits at most `shutdown_timeout` for queued +history to reach the store; anything left is counted as dropped. A second Ctrl-C stops waiting at +once. + +## Retention + +Commands occasionally prune invocations older than `retention`, with all of their events, output, +and diagnostics. SQL text is removed once no remaining invocation references it. Pruning runs in +the background within the exit budget and never removes an invocation that is still running. + +## When the store cannot be written + +If the store location is not writable, the command still runs and prints a notice on stderr with +the SQL an administrator can use to create the store; the notice comes before the command's own +output whenever the problem is visible from file permissions. The notice +repeats at most once a day. If the store file is not a SQLite database, Recall moves it aside to +`recall.sqlite3.corrupt-`, starts a fresh store, and says so once. + +A `connection` that does not use the `sqlite` adapter is not supported yet; Recall prints a notice +and records nothing. + +## Querying history + +Use [`sqb recall`](../../cli/recall.md) to read the store. Queries are read-only SQLite `SELECT` +statements against these views: + +| View | One row per | +|------|-------------| +| `invocations` | Invocation, with status, exit code, and timing | +| `statements` | Executed statement, with resource, kind, purpose, duration, and row counts | +| `audit_results` | Evaluated audit | +| `events` | Lifecycle event | +| `diagnostics` | Retained diagnostic | +| `artifacts` | Reference from an invocation to captured SQL, with its text | +| `output_invocations` | Invocation's output, with completeness | +| `output_transcripts` | Invocation's stdout, stderr, and combined output | +| `output_tail` | Recent output chunk | +| `conflicts` | Record written twice with different content | + +Run `sqb recall schema` for every column. + +## Relationship to compute logs + +Compute logs (`logs///stdout.log`, `stderr.log`, and `diagnostics.jsonl`) are +still written alongside Recall. Recall stores the same output bytes, captured at the same point, so +`sqb recall output` matches the compute log files. diff --git a/website/src/content/docs/docs/cli/recall.mdx b/website/src/content/docs/docs/cli/recall.mdx new file mode 100644 index 0000000000..dcf8336f49 --- /dev/null +++ b/website/src/content/docs/docs/cli/recall.mdx @@ -0,0 +1,114 @@ +--- +title: "sqb recall" +description: "Query the recorded history of past invocations." +sidebar: + label: "recall" +--- + +Read the local [Recall](/docs/concepts/observability/recall) store: past invocations, the +statements they executed, and what they printed. `sqb recall` never records itself. + +## Usage + +```bash +sqb recall query "SELECT command, status, duration_ms FROM invocations ORDER BY invocation_started_at DESC" +sqb recall schema +sqb recall output last +``` + +## sqb recall query + +Run one read-only SQLite `SELECT` against the Recall views. Statements that write, and multiple +statements, are rejected. + +| Flag | Description | +|------|-------------| +| `sql` | The query (positional argument) | +| `--format` | `long` (default), `table`, `json`, or `csv` | +| `--json` | Shorthand for `--format json` | +| `--limit` | Maximum rows to return (default: 100) | +| `--no-limit` | Disable the row limit | +| `--max-cell-bytes` | Longest cell in text formats, in bytes (default: 4096; `0` for no limit) | +| `--target` | Target whose store to read (default: the default target) | +| `--backend` | `logs` (default) reads the Recall store; `warehouse` runs the query on the target's connection | + +The latest invocation in a project named `orders_shop`: + +```bash +sqb recall query --format table \ + "SELECT command, status, exit_code, duration_ms FROM invocations + ORDER BY invocation_started_at DESC LIMIT 1" +``` + +``` +command | status | exit_code | duration_ms +------- | --------- | --------- | ----------- +build | completed | 0 | 752.0 + +1 row +``` + +The slowest statements of past builds, with their SQL: + +```bash +sqb recall query --format table --max-cell-bytes 60 \ + "SELECT s.resource_name, round(s.duration_ms, 1) AS ms, a.text AS sql + FROM statements s JOIN artifacts a USING (invocation_id, statement_id) + WHERE s.statement_kind = 'CREATE' ORDER BY s.duration_ms DESC LIMIT 2" +``` + +``` +resource_name | ms | sql +------------- | --- | ------------------------------------------------------------------------ +orders | 3.6 | CREATE OR REPLACE TABLE dev.orders__staging AS SELECT 1 …[+74 bytes] +NULL | 3.2 | CREATE SCHEMA IF NOT EXISTS dev + +2 rows +``` + +Failed invocations of the last week: + +```bash +sqb recall query \ + "SELECT invocation_id, command, error_code FROM invocations + WHERE status = 'failed' AND invocation_started_at >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-7 days')" +``` + +Timestamps are ISO-8601 UTC text, so they compare correctly as strings. Filter on +`invocation_started_at` to bound a scan. + +## sqb recall schema + +List every Recall view with its columns, types, and join keys. Accepts `--format`, `--json`, and +`--target`. With `--backend warehouse`, it lists SQLBuild's state tables in the target's warehouse +instead, or describes one with `--relation `. + +## sqb recall output + +Print what an invocation wrote, byte for byte. + +```bash +sqb recall output last # the latest invocation, both streams in order +sqb recall output 7607b5b6 --stream stderr # an invocation ID or unique prefix +sqb recall output last --tail 20 # only the last 20 lines +``` + +| Flag | Description | +|------|-------------| +| `invocation` | Invocation ID, unique prefix, or `last` (default) | +| `--stream` | `combined` (default), `stdout`, or `stderr` | +| `--tail` | Print only the last N lines | +| `--target` | Target whose store to read | + +Output that Recall could not record completely is followed by a note on stderr naming the gap. + +## Errors + +| Code | Meaning | +|------|---------| +| `C970` | Recall is disabled for this project or target | +| `C971` | The store cannot be read, or no history is recorded yet | +| `C972` | No invocation matches | +| `C973` | The invocation prefix matches more than one invocation | +| `C974` | The query is invalid or not read-only | +| `C975` | The `[recall]` settings are invalid | diff --git a/website/src/content/docs/docs/concepts/observability.mdx b/website/src/content/docs/docs/concepts/observability.mdx index d747362233..7819885ff9 100644 --- a/website/src/content/docs/docs/concepts/observability.mdx +++ b/website/src/content/docs/docs/concepts/observability.mdx @@ -11,6 +11,7 @@ whether work started, completed, failed, skipped, or retried. | `LifecycleEvent` | Immutable invocation, run, resource-attempt, operation, statement, retry, and audit facts | Canonical lifecycle evidence once durably stored | | `DiagnosticLog` | Structured framework diagnostics | Explanatory only | | `stdout.log` / `stderr.log` | Exact host-local process output | Troubleshooting transcript | +| [Recall](/docs/concepts/observability/recall) | Local queryable history of invocations, statements, SQL, and output | Troubleshooting history; best effort | | `CommandOutputRecord` | Bounded, ANSI-free remote stdout/stderr chunks | Potentially sensitive and lossy troubleshooting data | | Final JSON output | End-of-command aggregate | Result projection, not event history | @@ -80,4 +81,5 @@ A start without a terminal fact is unknown or presumed lost. Consumers must not or failure. Sink delivery is bounded and best effort; destination failure does not change successful warehouse work, and successful delivery does not prove the command succeeded. -See [Typed Sinks](/docs/concepts/observability/sinks) to export lifecycle and command-output records. +See [Recall](/docs/concepts/observability/recall) for the local history `sqb recall` queries, and +[Typed Sinks](/docs/concepts/observability/sinks) to export lifecycle and command-output records. diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx new file mode 100644 index 0000000000..e539dd4c1f --- /dev/null +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -0,0 +1,109 @@ +--- +title: "Recall" +description: "The local history of every command: what ran, what it executed, and what it printed." +--- + +Recall records each `sqb` invocation in a local SQLite store so you can ask what happened after the +fact: which command ran, how it ended, which statements it executed and how long they took, and +exactly what it printed. Recall is on by default and needs no setup. + +## What is recorded + +For every recorded invocation, Recall keeps: + +- the invocation itself: command, arguments, target, adapter, status, exit code, and timing; +- its [lifecycle events](/docs/concepts/observability): runs, resource attempts, statements, + retries, and audit results; +- the SQL text of every executed statement, stored once per distinct statement and shared across + invocations; +- its stdout and stderr, byte for byte, in the order they were written; +- framework diagnostics at or above `diagnostics_min_severity`. + +Writes happen on a background thread and never fail the command. A command that cannot record all +of its history still succeeds, prints a warning followed by its own final status, and +`output_invocations` reports the gap. An invocation whose process died without finishing has +status `abandoned`. + +Every command is recorded except `sqb recall` itself, commands that create a project (such as +`sqb init`), and raw-query `sqb diff`. A `sqb compile` answered from the launcher's warm cache is not +recorded either. + +The store is `logs/recall.sqlite3` in the project directory. Add `logs/` to `.gitignore`; the store +can contain sensitive SQL and output. + +## Settings + +Configure Recall in `[recall]` in `sqlbuild_project.toml`. A `[targets..recall]` table +overrides individual keys for one target. + +```toml +[recall] +retention = "30d" # days of history to keep; also accepts hours, such as "720h" +shutdown_timeout = "3s" # longest delay Recall may add when a command exits +diagnostics_min_severity = "info" + +[targets.ci.recall] +enabled = false # do not record runs against the ci target +``` + +| Key | Default | Description | +|-----|---------|-------------| +| `enabled` | `true` | Set to `false` to stop recording | +| `connection` | (none) | A `sqlite` connection whose `path` names the store; defaults to `logs/recall.sqlite3` | +| `retention` | `"30d"` | How long invocations are kept, in days (`d`) or hours (`h`), up to 3650 days | +| `shutdown_timeout` | `"3s"` | The cap on time spent flushing history at exit, in `s` or `ms`, up to 10 minutes | +| `diagnostics_min_severity` | `"info"` | `trace`, `debug`, `info`, `warning`, `error`, or `critical` | + +To turn Recall off for the whole project: + +```toml +[recall] +enabled = false +``` + +Recall never delays the start of a command. At exit it waits at most `shutdown_timeout` for queued +history to reach the store; anything left is counted as dropped. A second Ctrl-C stops waiting at +once. + +## Retention + +Commands occasionally prune invocations older than `retention`, with all of their events, output, +and diagnostics. SQL text is removed once no remaining invocation references it. Pruning runs in +the background within the exit budget and never removes an invocation that is still running. + +## When the store cannot be written + +If the store location is not writable, the command still runs and prints a notice on stderr with +the SQL an administrator can use to create the store; the notice comes before the command's own +output whenever the problem is visible from file permissions. The notice +repeats at most once a day. If the store file is not a SQLite database, Recall moves it aside to +`recall.sqlite3.corrupt-`, starts a fresh store, and says so once. + +A `connection` that does not use the `sqlite` adapter is not supported yet; Recall prints a notice +and records nothing. + +## Querying history + +Use [`sqb recall`](/docs/cli/recall) to read the store. Queries are read-only SQLite `SELECT` +statements against these views: + +| View | One row per | +|------|-------------| +| `invocations` | Invocation, with status, exit code, and timing | +| `statements` | Executed statement, with resource, kind, purpose, duration, and row counts | +| `audit_results` | Evaluated audit | +| `events` | Lifecycle event | +| `diagnostics` | Retained diagnostic | +| `artifacts` | Reference from an invocation to captured SQL, with its text | +| `output_invocations` | Invocation's output, with completeness | +| `output_transcripts` | Invocation's stdout, stderr, and combined output | +| `output_tail` | Recent output chunk | +| `conflicts` | Record written twice with different content | + +Run `sqb recall schema` for every column. + +## Relationship to compute logs + +Compute logs (`logs///stdout.log`, `stderr.log`, and `diagnostics.jsonl`) are +still written alongside Recall. Recall stores the same output bytes, captured at the same point, so +`sqb recall output` matches the compute log files. diff --git a/website/src/sidebar.json b/website/src/sidebar.json index 05f688d4f2..770ed3304d 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -136,6 +136,7 @@ "collapsed": true, "items": [ "docs/concepts/observability", + "docs/concepts/observability/recall", "docs/concepts/observability/sinks", "docs/concepts/observability/execution-history" ] @@ -216,6 +217,7 @@ "docs/cli/lineage", "docs/cli/dag", "docs/cli/query", + "docs/cli/recall", "docs/cli/debug", "docs/cli/janitor", "docs/cli/clean", From 5fff545d9ae324df635437c7f8fa84a4f1953080 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 06:57:26 +0100 Subject: [PATCH 10/19] perf: build Recall rows on the writer thread and hash row content with BLAKE3 The command thread now only enqueues each lifecycle event and executed SQL; rows, artifact digests and content hashes are built on the delivery thread, which also deduplicates SQL per invocation. Events that cannot be built are counted as failed. Row content is hashed with BLAKE3 over one encoded buffer, and the column is renamed to content_hash. --- Cargo.lock | 1 + .../src/bindings/_helpers/recall/sinks.rs | 2 +- crates/sqlbuild-recall/Cargo.toml | 1 + .../src/records/_helpers/events.rs | 14 +++ .../src/records/_helpers/ids.rs | 53 ++++++----- .../src/records/_helpers/rows.rs | 10 +- .../sqlbuild-recall/src/records/constants.rs | 2 + .../src/records/main/event_row.rs | 14 --- .../src/records/main/event_severity.rs | 11 +++ .../sqlbuild-recall/src/records/main/mod.rs | 3 +- .../src/records/main/parse_event.rs | 11 +++ crates/sqlbuild-recall/src/records/models.rs | 4 +- .../src/records/tests/helpers.rs | 11 +++ .../src/records/tests/test_records.rs | 7 +- .../sqlbuild-recall/src/schema/constants.rs | 4 +- .../src/store/_helpers/rows.rs | 4 +- .../src/store/tests/helpers.rs | 7 +- .../src/writer/_helpers/delivery.rs | 17 ++-- .../src/writer/_helpers/materialize.rs | 93 +++++++++++++++++++ .../src/writer/_helpers/mod.rs | 1 + .../src/writer/_helpers/queue.rs | 90 ++++++++++++++---- .../src/writer/_helpers/session.rs | 90 +++++++++--------- .../sqlbuild-recall/src/writer/constants.rs | 2 + .../src/writer/tests/helpers.rs | 63 ++++++++++--- .../src/writer/tests/test_session.rs | 48 +++++++++- .../src/writer/tests/test_types.rs | 7 ++ 26 files changed, 421 insertions(+), 149 deletions(-) delete mode 100644 crates/sqlbuild-recall/src/records/main/event_row.rs create mode 100644 crates/sqlbuild-recall/src/records/main/event_severity.rs create mode 100644 crates/sqlbuild-recall/src/records/main/parse_event.rs create mode 100644 crates/sqlbuild-recall/src/writer/_helpers/materialize.rs diff --git a/Cargo.lock b/Cargo.lock index 32c3e25e2e..c48b2bb0a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1337,6 +1337,7 @@ dependencies = [ name = "sqlbuild-recall" version = "0.158.2" dependencies = [ + "blake3", "rusqlite", "serde_json", "sha2", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs index b324b6fc32..d0caa1b737 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs @@ -20,7 +20,7 @@ pub(crate) struct RecallLifecycleSink { impl RecallLifecycleSink { fn __call__(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { if let Some(object) = event_object(event)? { - drop(self.session.publish_event_object(&object)); + self.session.publish_event_object(object); return Ok(()); } let encoded = host::entry(py, ExecutorHost::LifecycleEventToJson)?.call1((event,))?; diff --git a/crates/sqlbuild-recall/Cargo.toml b/crates/sqlbuild-recall/Cargo.toml index a53553ecc6..ca3899fd14 100644 --- a/crates/sqlbuild-recall/Cargo.toml +++ b/crates/sqlbuild-recall/Cargo.toml @@ -8,6 +8,7 @@ publish.workspace = true rust-version.workspace = true [dependencies] +blake3.workspace = true rusqlite.workspace = true serde_json.workspace = true sha2.workspace = true diff --git a/crates/sqlbuild-recall/src/records/_helpers/events.rs b/crates/sqlbuild-recall/src/records/_helpers/events.rs index 7e139cc29f..904f592754 100644 --- a/crates/sqlbuild-recall/src/records/_helpers/events.rs +++ b/crates/sqlbuild-recall/src/records/_helpers/events.rs @@ -39,6 +39,20 @@ pub(crate) fn event_object(event_json: &str) -> Result, Recor } } +/// The export severity of an event's type; `None` without a text `event_type`. +pub(crate) fn severity_of(event: &Map) -> Option<&'static str> { + let event_type: &str = event.get("event_type").and_then(Value::as_str)?; + let schema_version: i64 = event + .get("schema_version") + .and_then(Value::as_i64) + .unwrap_or_default(); + Some( + lifecycle_export_policy(event_type, is_terminal(event_type, schema_version)) + .severity + .value(), + ) +} + /// Identity, type, export policy and correlation columns. pub(crate) fn envelope_columns(event: &Map) -> Result, RecordError> { let event_type: &str = required_text(event, "event_type")?; diff --git a/crates/sqlbuild-recall/src/records/_helpers/ids.rs b/crates/sqlbuild-recall/src/records/_helpers/ids.rs index d98bd32489..97ef2a2302 100644 --- a/crates/sqlbuild-recall/src/records/_helpers/ids.rs +++ b/crates/sqlbuild-recall/src/records/_helpers/ids.rs @@ -1,8 +1,8 @@ //! Deterministic record IDs and content digests. -use sha2::{Digest, Sha256}; - -use crate::records::constants::{DIAGNOSTIC_RECORD_ID_INFIX, OUTPUT_RECORD_ID_INFIX}; +use crate::records::constants::{ + CONTENT_HASH_BUFFER_BYTES, DIAGNOSTIC_RECORD_ID_INFIX, OUTPUT_RECORD_ID_INFIX, +}; use crate::records::models::RecallValue; use crate::schema::constants::UNHASHED_COLUMNS; @@ -16,29 +16,32 @@ pub(crate) fn diagnostic_record_id(invocation_id: &str, sequence: u64) -> String format!("{invocation_id}:{DIAGNOSTIC_RECORD_ID_INFIX}:{sequence}") } -fn framed(hasher: Sha256, tag: &[u8; 1], bytes: &[u8]) -> Sha256 { - hasher - .chain_update(tag) - .chain_update((bytes.len() as u64).to_le_bytes()) - .chain_update(bytes) +fn framed(mut buffer: Vec, tag: u8, bytes: &[u8]) -> Vec { + buffer.push(tag); + buffer.extend_from_slice(&(bytes.len() as u64).to_le_bytes()); + buffer.extend_from_slice(bytes); + buffer } -fn valued(hasher: Sha256, value: &RecallValue) -> Sha256 { +fn valued(mut buffer: Vec, value: &RecallValue) -> Vec { match value { - RecallValue::Null => hasher.chain_update(b"0"), + RecallValue::Null => buffer.push(b'0'), RecallValue::Integer(number) => { - hasher.chain_update(b"i").chain_update(number.to_le_bytes()) + buffer.push(b'i'); + buffer.extend_from_slice(&number.to_le_bytes()); + } + RecallValue::Real(number) => { + buffer.push(b'r'); + buffer.extend_from_slice(&number.to_bits().to_le_bytes()); } - RecallValue::Real(number) => hasher - .chain_update(b"r") - .chain_update(number.to_bits().to_le_bytes()), - RecallValue::Text(text) => framed(hasher, b"t", text.as_bytes()), - RecallValue::Blob(data) => framed(hasher, b"b", data), + RecallValue::Text(text) => return framed(buffer, b't', text.as_bytes()), + RecallValue::Blob(data) => return framed(buffer, b'b', data), } + buffer } -/// Hex SHA-256 over every non-NULL named value except `ingested_at` and `content_sha256`, by column name; a NULL column hashes like an absent one. -pub(crate) fn content_sha256(values: &[(&str, RecallValue)]) -> String { +/// Hex BLAKE3 over every non-NULL named value except `ingested_at` and `content_hash`, by column name; a NULL column hashes like an absent one. +pub(crate) fn content_hash(values: &[(&str, RecallValue)]) -> String { let mut sorted: Vec<&(&str, RecallValue)> = values .iter() .filter(|(name, value)| { @@ -46,13 +49,9 @@ pub(crate) fn content_sha256(values: &[(&str, RecallValue)]) -> String { }) .collect(); sorted.sort_by(|left, right| left.0.cmp(right.0)); - sorted - .into_iter() - .fold(Sha256::new(), |hasher, (name, value)| { - valued(framed(hasher, b"n", name.as_bytes()), value) - }) - .finalize() - .iter() - .map(|byte| format!("{byte:02x}")) - .collect() + let encoded: Vec = sorted.into_iter().fold( + Vec::with_capacity(CONTENT_HASH_BUFFER_BYTES), + |buffer, (name, value)| valued(framed(buffer, b'n', name.as_bytes()), value), + ); + blake3::hash(&encoded).to_hex().to_string() } diff --git a/crates/sqlbuild-recall/src/records/_helpers/rows.rs b/crates/sqlbuild-recall/src/records/_helpers/rows.rs index a023bd4a3d..5ad2768b37 100644 --- a/crates/sqlbuild-recall/src/records/_helpers/rows.rs +++ b/crates/sqlbuild-recall/src/records/_helpers/rows.rs @@ -1,6 +1,6 @@ //! Assemble a row: the dataset's values, the common columns and the content digest. -use crate::records::_helpers::ids::content_sha256; +use crate::records::_helpers::ids::content_hash; use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; use crate::schema::constants::{ARTIFACT_CONTENT_COLUMNS, RECALL_SCHEMA_VERSION}; use crate::schema::models::Table; @@ -23,7 +23,7 @@ pub(crate) fn boolean(value: bool) -> RecallValue { RecallValue::Integer(i64::from(value)) } -/// `values` plus the envelope columns and `content_sha256`; an artifact's digest covers only its content, since every invocation that sees it writes the same row. +/// `values` plus the envelope columns and `content_hash`; an artifact's digest covers only its content, since every invocation that sees it writes the same row. pub(crate) fn finish_row( table: Table, envelope: &RowEnvelope, @@ -40,15 +40,15 @@ pub(crate) fn finish_row( )); values.push(("producer_version", text(&envelope.producer_version))); let digest: String = match table { - Table::Artifacts => content_sha256( + Table::Artifacts => content_hash( &values .iter() .filter(|(name, _)| ARTIFACT_CONTENT_COLUMNS.contains(name)) .cloned() .collect::>(), ), - _ => content_sha256(&values), + _ => content_hash(&values), }; - values.push(("content_sha256", RecallValue::Text(digest))); + values.push(("content_hash", RecallValue::Text(digest))); RecallRow { table, values } } diff --git a/crates/sqlbuild-recall/src/records/constants.rs b/crates/sqlbuild-recall/src/records/constants.rs index 6392d48190..da760bd9a0 100644 --- a/crates/sqlbuild-recall/src/records/constants.rs +++ b/crates/sqlbuild-recall/src/records/constants.rs @@ -48,3 +48,5 @@ pub(crate) const FEBRUARY: i64 = 2; pub(crate) const MARCH_BASED_JANUARY_INDEX: i64 = 10; /// The output column counting chunks dropped just before this one. pub const DROPPED_BEFORE_COLUMN: &str = "dropped_before"; +/// Initial capacity of the buffer a row's content hash is computed over. +pub(crate) const CONTENT_HASH_BUFFER_BYTES: usize = 1_024; diff --git a/crates/sqlbuild-recall/src/records/main/event_row.rs b/crates/sqlbuild-recall/src/records/main/event_row.rs deleted file mode 100644 index 53c62883e6..0000000000 --- a/crates/sqlbuild-recall/src/records/main/event_row.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! `event_row`: the `recall_events` row of one lifecycle event's canonical JSON. - -use serde_json::{Map, Value}; - -use crate::records::_helpers::events::event_object; -use crate::records::errors::RecordError; -use crate::records::main::event_object_row::event_object_row; -use crate::records::models::{RecallRow, RowEnvelope}; - -/// The row of one event, as `lifecycle_event_to_json` encodes it. -pub fn event_row(envelope: &RowEnvelope, event_json: &str) -> Result { - let event: Map = event_object(event_json)?; - event_object_row(envelope, &event) -} diff --git a/crates/sqlbuild-recall/src/records/main/event_severity.rs b/crates/sqlbuild-recall/src/records/main/event_severity.rs new file mode 100644 index 0000000000..2f80356364 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/event_severity.rs @@ -0,0 +1,11 @@ +//! `event_severity`: the export severity of a lifecycle event, without building its row. + +use serde_json::{Map, Value}; + +use crate::records::_helpers::events::severity_of; + +/// The severity the event's row will carry; `None` when the event has no text `event_type`. +#[must_use] +pub fn event_severity(event: &Map) -> Option<&'static str> { + severity_of(event) +} diff --git a/crates/sqlbuild-recall/src/records/main/mod.rs b/crates/sqlbuild-recall/src/records/main/mod.rs index 8e8b1bb977..2e466594b9 100644 --- a/crates/sqlbuild-recall/src/records/main/mod.rs +++ b/crates/sqlbuild-recall/src/records/main/mod.rs @@ -4,11 +4,12 @@ pub mod delivery_row; pub mod diagnostic_input_from_json; pub mod diagnostic_row; pub mod event_object_row; -pub mod event_row; +pub mod event_severity; pub mod format_timestamp; pub mod heartbeat_row; pub mod invocation_row; pub mod meta_row; pub mod output_row; +pub mod parse_event; pub mod parse_timestamp; pub mod sha256_hex; diff --git a/crates/sqlbuild-recall/src/records/main/parse_event.rs b/crates/sqlbuild-recall/src/records/main/parse_event.rs new file mode 100644 index 0000000000..97c4043d18 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/parse_event.rs @@ -0,0 +1,11 @@ +//! `parse_event`: a lifecycle event's canonical JSON as its object. + +use serde_json::{Map, Value}; + +use crate::records::_helpers::events::event_object; +use crate::records::errors::RecordError; + +/// The JSON object of one lifecycle event; an error unless `event_json` is a JSON object. +pub fn parse_event(event_json: &str) -> Result, RecordError> { + event_object(event_json) +} diff --git a/crates/sqlbuild-recall/src/records/models.rs b/crates/sqlbuild-recall/src/records/models.rs index a4b1c5d06f..06e2571a9b 100644 --- a/crates/sqlbuild-recall/src/records/models.rs +++ b/crates/sqlbuild-recall/src/records/models.rs @@ -31,8 +31,8 @@ impl RecallRow { /// The row's content digest. #[must_use] - pub fn content_sha256(&self) -> Option<&str> { - match self.value("content_sha256") { + pub fn content_hash(&self) -> Option<&str> { + match self.value("content_hash") { Some(RecallValue::Text(text)) => Some(text), _ => None, } diff --git a/crates/sqlbuild-recall/src/records/tests/helpers.rs b/crates/sqlbuild-recall/src/records/tests/helpers.rs index 8e6d26e600..e4166567da 100644 --- a/crates/sqlbuild-recall/src/records/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/records/tests/helpers.rs @@ -1,3 +1,6 @@ +use crate::records::errors::RecordError; +use crate::records::main::event_object_row::event_object_row; +use crate::records::main::parse_event::parse_event; use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; /// The envelope every record test row shares. @@ -34,3 +37,11 @@ pub(crate) fn present( pub(crate) fn text(value: &str) -> RecallValue { RecallValue::Text(value.to_owned()) } + +/// The events row of one lifecycle event's canonical JSON, as the delivery thread builds it. +pub(crate) fn event_row( + envelope: &RowEnvelope, + event_json: &str, +) -> Result { + parse_event(event_json).and_then(|event| event_object_row(envelope, &event)) +} diff --git a/crates/sqlbuild-recall/src/records/tests/test_records.rs b/crates/sqlbuild-recall/src/records/tests/test_records.rs index 18bb7c2178..03371d895b 100644 --- a/crates/sqlbuild-recall/src/records/tests/test_records.rs +++ b/crates/sqlbuild-recall/src/records/tests/test_records.rs @@ -1,15 +1,14 @@ use std::time::{Duration, UNIX_EPOCH}; -use crate::records::_helpers::ids::content_sha256; +use crate::records::_helpers::ids::content_hash; use crate::records::_helpers::ids::diagnostic_record_id; use crate::records::_helpers::ids::output_record_id; use crate::records::main::diagnostic_input_from_json::diagnostic_input_from_json; -use crate::records::main::event_row::event_row; use crate::records::main::format_timestamp::format_timestamp; use crate::records::main::output_row::output_row; use crate::records::main::parse_timestamp::parse_timestamp; use crate::records::models::{OutputChunk, OutputStream, RecallValue}; -use crate::records::tests::helpers::{envelope, present, row_columns, text}; +use crate::records::tests::helpers::{envelope, event_row, present, row_columns, text}; use crate::records::tests::test_types::{ ContentHashTestCase, DiagnosticTestCase, ParseTimestampTestCase, RecordIdTestCase, RejectedInputTestCase, RowColumnsTestCase, TimestampTestCase, @@ -131,7 +130,7 @@ fn given_equal_and_different_content_when_hashed_then_only_content_matters() { for test_case in test_cases { assert_eq!( - content_sha256(&test_case.left) == content_sha256(&test_case.right), + content_hash(&test_case.left) == content_hash(&test_case.right), test_case.expected_equal, "{}", test_case.description diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index f7fed0f760..8f801f3a75 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -20,12 +20,12 @@ pub const COMMON_COLUMNS: [ColumnSpec; 6] = [ ColumnSpec::required("schema_version", ColumnType::Integer), ColumnSpec::required("producer_version", ColumnType::Text), ColumnSpec::required("ingested_at", ColumnType::Timestamp), - ColumnSpec::required("content_sha256", ColumnType::Text), + ColumnSpec::required("content_hash", ColumnType::Text), ]; /// The artifact columns its content digest covers: what it is, not who saw it when. pub const ARTIFACT_CONTENT_COLUMNS: [&str; 5] = ["digest", "kind", "media_type", "size", "content"]; /// Common columns left out of a row's content digest. -pub const UNHASHED_COLUMNS: [&str; 2] = ["ingested_at", "content_sha256"]; +pub const UNHASHED_COLUMNS: [&str; 2] = ["ingested_at", "content_hash"]; const CORRELATIONS: [ColumnSpec; 5] = [ ColumnSpec::optional("run_id", ColumnType::Text), diff --git a/crates/sqlbuild-recall/src/store/_helpers/rows.rs b/crates/sqlbuild-recall/src/store/_helpers/rows.rs index 400bb157ee..91ee58ea45 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/rows.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/rows.rs @@ -62,7 +62,7 @@ fn write_row( .map(|(index, name)| format!("{name} = ?{}", index + 1)) .collect(); let lookup = format!( - "SELECT content_sha256 FROM \"{}\" WHERE {}", + "SELECT content_hash FROM \"{}\" WHERE {}", spec.name, condition.join(" AND ") ); @@ -81,7 +81,7 @@ fn write_row( .optional(), "read stored Recall row", )?; - let rejected: &str = row.content_sha256().unwrap_or_default(); + let rejected: &str = row.content_hash().unwrap_or_default(); match existing { Some(existing) if existing != rejected => Ok(Some(ConflictEvidence { table: row.table, diff --git a/crates/sqlbuild-recall/src/store/tests/helpers.rs b/crates/sqlbuild-recall/src/store/tests/helpers.rs index 6ee7ea78f6..8387af1331 100644 --- a/crates/sqlbuild-recall/src/store/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/store/tests/helpers.rs @@ -2,9 +2,10 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tempfile::TempDir; -use crate::records::main::event_row::event_row; +use crate::records::main::event_object_row::event_object_row; use crate::records::main::format_timestamp::format_timestamp; use crate::records::main::output_row::output_row; +use crate::records::main::parse_event::parse_event; use crate::records::main::parse_timestamp::parse_timestamp; use crate::records::models::{ ArtifactInput, DeliveryCounts, OutputChunk, OutputStream, RecallRow, RecallValue, RowEnvelope, @@ -84,7 +85,9 @@ pub(crate) fn event( "invocation_id":"{}","invocation_sequence":{sequence}{correlations},"payload":{payload}}}"#, envelope.invocation_id ); - event_row(envelope, &json).expect("the fixture is a valid lifecycle event") + parse_event(&json) + .and_then(|event| event_object_row(envelope, &event)) + .expect("the fixture is a valid lifecycle event") } /// The result rows of `sql`. diff --git a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs index fe33c85235..91fb00a1e1 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs @@ -16,6 +16,7 @@ use crate::store::main::maintain_sqlite_store::maintain_sqlite_store; use crate::store::main::open_sqlite_store::open_sqlite_store; use crate::store::main::quarantine_store::quarantine_store; use crate::store::models::{InsertOutcome, MaintenanceRequest, OpenMode, SqliteStore}; +use crate::writer::_helpers::materialize::{Built, Materializer}; use crate::writer::_helpers::queue::{Queued, WriterState}; use crate::writer::constants::{ BATCH_ROWS, BATCH_WINDOW_MS, CLOCK_CHECK_MS, FINAL_ROWS_BUDGET_MS, HEARTBEAT_STALE_INTERVALS, @@ -227,12 +228,16 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { } }; let mut next_heartbeat: SystemTime = SystemTime::now(); + let mut materializer = Materializer::new(config.envelope.clone()); loop { let (batch, closing) = next_batch(shared, next_heartbeat); - let mut rows: Vec = batch - .iter() - .flat_map(|record| record.rows.clone()) - .collect(); + let built: Built = materializer.build(batch); + if !built.settled.is_empty() || !built.unbuildable.is_empty() { + let mut state = locked(&shared.state); + state.account(&built.settled, Ok(&InsertOutcome::default())); + state.account_unbuildable(&built.unbuildable); + } + let mut rows: Vec = built.rows; let now: SystemTime = SystemTime::now(); if now >= next_heartbeat && !closing { rows.push(heartbeat(config, now)); @@ -248,8 +253,8 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { let written = write_batch(&store, &rows, give_up); let mut state = locked(&shared.state); match &written { - Ok(outcome) => state.account(&batch, Ok(outcome)), - Err(error) => state.account(&batch, Err(&error.message)), + Ok(outcome) => state.account(&built.written, Ok(outcome)), + Err(error) => state.account(&built.written, Err(&error.message)), } } let mut state = locked(&shared.state); diff --git a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs new file mode 100644 index 0000000000..7038867b34 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs @@ -0,0 +1,93 @@ +//! Rows of queued input, built on the delivery thread so producers only enqueue. + +use std::collections::HashSet; + +use crate::records::constants::{ + ARTIFACT_DIGEST_PREFIX, EXECUTED_SQL_ROLE, SQL_ARTIFACT_KIND, SQL_MEDIA_TYPE, +}; +use crate::records::main::artifact_rows::artifact_rows; +use crate::records::main::event_object_row::event_object_row; +use crate::records::main::format_timestamp::format_timestamp; +use crate::records::main::sha256_hex::sha256_hex; +use crate::records::models::{ArtifactInput, RecallRow, RowEnvelope}; +use crate::writer::_helpers::queue::{Payload, Queued, SqlInput}; +use crate::writer::models::DeliveryStream; + +/// A batch ready to write: its rows, the streams of the records they came from, and records already settled. +#[derive(Debug, Default)] +pub(crate) struct Built { + pub(crate) rows: Vec, + /// One entry per record the rows were built from, for delivery accounting. + pub(crate) written: Vec, + /// Records that are already stored, such as SQL this invocation recorded before. + pub(crate) settled: Vec, + /// Records whose rows could not be built. + pub(crate) unbuildable: Vec, +} + +/// Builds rows for one session; remembers which SQL it has stored. +#[derive(Debug)] +pub(crate) struct Materializer { + envelope: RowEnvelope, + artifacts_seen: HashSet, +} + +impl Materializer { + pub(crate) fn new(envelope: RowEnvelope) -> Self { + Self { + envelope, + artifacts_seen: HashSet::new(), + } + } + + fn sql_rows(&mut self, input: SqlInput) -> Option> { + let digest = format!( + "{ARTIFACT_DIGEST_PREFIX}{}", + sha256_hex(input.sql.as_bytes()) + ); + if !self.artifacts_seen.insert(digest.clone()) { + return None; + } + let artifact = ArtifactInput { + ref_id: input.statement_id.clone().unwrap_or_else(|| digest.clone()), + digest, + kind: SQL_ARTIFACT_KIND.to_owned(), + media_type: SQL_MEDIA_TYPE.to_owned(), + content: input.sql.into_bytes(), + seen_at: format_timestamp(input.seen_at), + role: EXECUTED_SQL_ROLE.to_owned(), + statement_id: input.statement_id, + resource_id: input.resource_id, + }; + Some(artifact_rows(&self.envelope, &artifact).to_vec()) + } + + /// The rows of `batch`, with each record's stream sorted by how it is accounted. + pub(crate) fn build(&mut self, batch: Vec) -> Built { + let mut built = Built::default(); + for record in batch { + let stream: DeliveryStream = record.stream; + match record.payload { + Payload::Rows(rows) => { + built.rows.extend(rows); + built.written.push(stream); + } + Payload::Event(event) => match event_object_row(&self.envelope, &event) { + Ok(row) => { + built.rows.push(row); + built.written.push(stream); + } + Err(_) => built.unbuildable.push(stream), + }, + Payload::Sql(input) => match self.sql_rows(input) { + Some(rows) => { + built.rows.extend(rows); + built.written.push(stream); + } + None => built.settled.push(stream), + }, + } + } + built + } +} diff --git a/crates/sqlbuild-recall/src/writer/_helpers/mod.rs b/crates/sqlbuild-recall/src/writer/_helpers/mod.rs index 66aa5e748e..bcd7ab884f 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod delivery; +pub(crate) mod materialize; pub(crate) mod queue; pub(crate) mod session; diff --git a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs index f6d713fb7f..bf9a856153 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs @@ -1,8 +1,10 @@ //! The bounded priority queue and the output coalescer, guarded together by one mutex. -use std::collections::{HashSet, VecDeque}; +use std::collections::VecDeque; use std::time::{Duration, Instant, SystemTime}; +use serde_json::{Map, Value}; + use crate::records::constants::{DROPPED_BEFORE_COLUMN, MAX_OUTPUT_CHUNK_BYTES}; use crate::records::main::format_timestamp::format_timestamp; use crate::records::main::output_row::output_row; @@ -14,14 +16,49 @@ use crate::writer::_helpers::delivery::stream_of; use crate::writer::constants::{QUEUE_BYTE_CAPACITY, QUEUE_RECORD_CAPACITY}; use crate::writer::models::{DeliveryStream, Priority}; -/// One queued record: the rows written together and where they are accounted. +/// SQL a statement executed, hashed and turned into rows on the delivery thread. +#[derive(Debug)] +pub(crate) struct SqlInput { + pub(crate) sql: String, + pub(crate) statement_id: Option, + pub(crate) resource_id: Option, + pub(crate) seen_at: SystemTime, +} + +/// What a queued record holds: finished rows, or input the delivery thread turns into rows. +#[derive(Debug)] +pub(crate) enum Payload { + Rows(Vec), + Event(Map), + Sql(SqlInput), +} + +/// One queued record: what is written together and where it is accounted. #[derive(Debug)] pub(crate) struct Queued { - pub(crate) rows: Vec, + pub(crate) payload: Payload, pub(crate) stream: DeliveryStream, pub(crate) bytes: usize, } +impl Queued { + /// The finished rows; empty while the delivery thread has yet to build them. + pub(crate) fn rows(&self) -> &[RecallRow] { + match &self.payload { + Payload::Rows(rows) => rows, + Payload::Event(_) | Payload::Sql(_) => &[], + } + } + + fn row_count(&self) -> usize { + match &self.payload { + Payload::Rows(rows) => rows.len(), + Payload::Event(_) => 1, + Payload::Sql(_) => 2, + } + } +} + /// Output accepted but not yet chunked: contiguous writes to one stream. #[derive(Debug)] struct Pending { @@ -44,7 +81,6 @@ pub(crate) struct WriterState { offsets: [u64; 2], unreported_output_drops: u64, pub(crate) diagnostic_sequence: u64, - pub(crate) artifacts_seen: HashSet, pub(crate) closing: bool, pub(crate) deadline: Option, pub(crate) finished: bool, @@ -86,7 +122,6 @@ impl WriterState { offsets: [0; 2], unreported_output_drops: 0, diagnostic_sequence: 0, - artifacts_seen: HashSet::new(), closing: false, deadline: None, finished: false, @@ -106,7 +141,7 @@ impl WriterState { self.counts_mut(record.stream).dropped += 1; if record.stream == DeliveryStream::Output { let carried: u64 = record - .rows + .rows() .iter() .map(|row| match row.value(DROPPED_BEFORE_COLUMN) { Some(RecallValue::Integer(count)) => u64::try_from(*count).unwrap_or_default(), @@ -128,12 +163,20 @@ impl WriterState { stream: DeliveryStream, priority: Priority, ) { - self.counts_mut(stream).accepted += 1; - let record = Queued { - bytes: row_bytes(&rows), - rows, - stream, - }; + let bytes: usize = row_bytes(&rows); + self.push_record( + Queued { + payload: Payload::Rows(rows), + stream, + bytes, + }, + priority, + ); + } + + /// Queue one record of any payload, displacing lower-priority records when full. + pub(crate) fn push_record(&mut self, record: Queued, priority: Priority) { + self.counts_mut(record.stream).accepted += 1; if self.rejecting { self.drop_record(&record); return; @@ -172,7 +215,7 @@ impl WriterState { let Some(record) = bucket.pop_front() else { break; }; - rows += record.rows.len(); + rows += record.row_count(); self.records -= 1; self.bytes -= record.bytes; batch.push(record); @@ -262,12 +305,23 @@ impl WriterState { self.pending.is_some() } + /// Count records the delivery thread could not turn into rows as failed. + pub(crate) fn account_unbuildable(&mut self, streams: &[DeliveryStream]) { + for stream in streams { + self.counts_mut(*stream).failed += 1; + } + } + /// Count a written batch as delivered, less conflicts, or a failed one as failed. - pub(crate) fn account(&mut self, batch: &[Queued], outcome: Result<&InsertOutcome, &str>) { + pub(crate) fn account( + &mut self, + batch: &[DeliveryStream], + outcome: Result<&InsertOutcome, &str>, + ) { match outcome { Ok(outcome) => { - for record in batch { - self.counts_mut(record.stream).delivered += 1; + for stream in batch { + self.counts_mut(*stream).delivered += 1; } for conflict in &outcome.conflicts { let stream = stream_of(conflict.table); @@ -277,8 +331,8 @@ impl WriterState { } } Err(reason) => { - for record in batch { - self.counts_mut(record.stream).failed += 1; + for stream in batch { + self.counts_mut(*stream).failed += 1; } self.degraded.get_or_insert_with(|| reason.to_owned()); } diff --git a/crates/sqlbuild-recall/src/writer/_helpers/session.rs b/crates/sqlbuild-recall/src/writer/_helpers/session.rs index e3cbb18a26..ccc1c5c9cc 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/session.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/session.rs @@ -1,25 +1,19 @@ //! The session handle: producers queue records; `close` flushes within the shutdown budget. -use std::sync::MutexGuard; use std::time::{Duration, Instant, SystemTime}; use serde_json::{Map, Value}; -use crate::records::constants::{ - ARTIFACT_DIGEST_PREFIX, EXECUTED_SQL_ROLE, SEVERITY_ORDER, SQL_ARTIFACT_KIND, SQL_MEDIA_TYPE, -}; +use crate::records::constants::SEVERITY_ORDER; use crate::records::errors::RecordError; -use crate::records::main::artifact_rows::artifact_rows; use crate::records::main::diagnostic_input_from_json::diagnostic_input_from_json; use crate::records::main::diagnostic_row::diagnostic_row; -use crate::records::main::event_object_row::event_object_row; -use crate::records::main::event_row::event_row; -use crate::records::main::format_timestamp::format_timestamp; -use crate::records::main::sha256_hex::sha256_hex; -use crate::records::models::{ArtifactInput, OutputStream, RecallRow, RecallValue}; +use crate::records::main::event_severity::event_severity; +use crate::records::main::parse_event::parse_event; +use crate::records::models::OutputStream; use crate::writer::_helpers::delivery::locked; -use crate::writer::_helpers::queue::WriterState; -use crate::writer::constants::MAX_ARTIFACT_BYTES; +use crate::writer::_helpers::queue::{Payload, Queued, SqlInput}; +use crate::writer::constants::{JSON_SCALAR_BYTES, MAX_ARTIFACT_BYTES}; use crate::writer::models::{ DeliveryStream, DeliverySummary, Priority, RecallSession, SetupOutcome, }; @@ -31,6 +25,23 @@ pub(crate) fn severity_rank(severity: &str) -> usize { .unwrap_or(0) } +/// Roughly how many bytes `event` holds, for the queue's byte capacity. +fn json_bytes(event: &Map) -> usize { + event + .iter() + .map(|(key, value)| key.len() + value_bytes(value)) + .sum() +} + +fn value_bytes(value: &Value) -> usize { + match value { + Value::String(text) => text.len(), + Value::Array(items) => items.iter().map(value_bytes).sum(), + Value::Object(entries) => json_bytes(entries), + Value::Null | Value::Bool(_) | Value::Number(_) => JSON_SCALAR_BYTES, + } +} + fn event_priority(severity: &str) -> Priority { match severity { "error" => Priority::FailedTerminal, @@ -78,26 +89,21 @@ impl RecallSession { /// Queue one lifecycle event from its canonical JSON. pub fn publish_event(&self, event_json: &str) -> Result<(), RecordError> { - let state = locked(&self.shared.state); - let row = event_row(&state.envelope, event_json)?; - self.queue_event(state, row); + self.publish_event_object(parse_event(event_json)?); Ok(()) } - /// Queue one lifecycle event already decoded to its canonical JSON object. - pub fn publish_event_object(&self, event: &Map) -> Result<(), RecordError> { - let state = locked(&self.shared.state); - let row = event_object_row(&state.envelope, event)?; - self.queue_event(state, row); - Ok(()) - } - - fn queue_event(&self, mut state: MutexGuard<'_, WriterState>, row: RecallRow) { - let priority = match row.value("severity") { - Some(RecallValue::Text(severity)) => event_priority(severity), - _ => Priority::Lifecycle, + /// Queue one lifecycle event already decoded to its canonical JSON object; its row is built on the delivery thread. + pub fn publish_event_object(&self, event: Map) { + let priority: Priority = event_severity(&event).map_or(Priority::Lifecycle, event_priority); + let bytes: usize = json_bytes(&event); + let record = Queued { + payload: Payload::Event(event), + stream: DeliveryStream::Events, + bytes, }; - state.push(vec![row], DeliveryStream::Events, priority); + let mut state = locked(&self.shared.state); + state.push_record(record, priority); self.shared.wake.notify_one(); } @@ -131,24 +137,18 @@ impl RecallSession { counts.dropped += 1; return; } - let digest = format!("{ARTIFACT_DIGEST_PREFIX}{}", sha256_hex(sql.as_bytes())); - let mut state = locked(&self.shared.state); - if !state.artifacts_seen.insert(digest.clone()) { - return; - } - let artifact = ArtifactInput { - ref_id: statement_id.map_or_else(|| digest.clone(), str::to_owned), - digest, - kind: SQL_ARTIFACT_KIND.to_owned(), - media_type: SQL_MEDIA_TYPE.to_owned(), - content: sql.as_bytes().to_vec(), - seen_at: format_timestamp(SystemTime::now()), - role: EXECUTED_SQL_ROLE.to_owned(), - statement_id: statement_id.map(str::to_owned), - resource_id: resource_id.map(str::to_owned), + let record = Queued { + bytes: sql.len(), + payload: Payload::Sql(SqlInput { + sql: sql.to_owned(), + statement_id: statement_id.map(str::to_owned), + resource_id: resource_id.map(str::to_owned), + seen_at: SystemTime::now(), + }), + stream: DeliveryStream::Artifacts, }; - let rows = artifact_rows(&state.envelope, &artifact).to_vec(); - state.push(rows, DeliveryStream::Artifacts, Priority::Artifact); + let mut state = locked(&self.shared.state); + state.push_record(record, Priority::Artifact); self.shared.wake.notify_one(); } diff --git a/crates/sqlbuild-recall/src/writer/constants.rs b/crates/sqlbuild-recall/src/writer/constants.rs index 294f347961..f38530229d 100644 --- a/crates/sqlbuild-recall/src/writer/constants.rs +++ b/crates/sqlbuild-recall/src/writer/constants.rs @@ -25,3 +25,5 @@ pub const FINAL_ROWS_BUDGET_MS: u64 = 500; pub const MINIMUM_MAINTENANCE_MS: u64 = 200; /// SQL longer than this is counted as dropped without hashing or copying it. pub const MAX_ARTIFACT_BYTES: usize = 4 * 1024 * 1024; +/// What a JSON scalar counts toward the queue's byte capacity. +pub const JSON_SCALAR_BYTES: usize = 8; diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs index 9ab59f7f45..7d2d6d6400 100644 --- a/crates/sqlbuild-recall/src/writer/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -2,14 +2,18 @@ use std::path::PathBuf; use std::time::{Duration, SystemTime}; use crate::records::main::invocation_row::invocation_row; -use crate::records::models::{InvocationContext, OutputStream, RecallValue, RowEnvelope}; +use crate::records::models::{ + DeliveryCounts, InvocationContext, OutputStream, RecallValue, RowEnvelope, +}; use crate::store::main::open_sqlite_store::open_sqlite_store; use crate::store::main::query_store::query_store; use crate::store::main::read_output_transcript::read_output_transcript; use crate::store::models::OpenMode; use crate::writer::_helpers::queue::WriterState; use crate::writer::main::start_recall_session::start_recall_session; -use crate::writer::models::{DeliveryStream, Priority, SessionConfig, SetupOutcome}; +use crate::writer::models::{ + DeliveryStream, DeliverySummary, Priority, SessionConfig, SetupOutcome, +}; use crate::writer::tests::test_types::RecordedSession; /// The envelope of invocation `invocation_id`. @@ -60,8 +64,8 @@ pub(crate) fn chunked(writes: &[(OutputStream, Vec)]) -> Vec<[Option (usize, bool, Vec>) (session.notices().len(), moved, rows) } -/// Artifact counts after publishing SQL of `length` bytes. -pub(crate) fn artifact_counts(length: usize) -> (u64, u64, u64) { - let directory = tempfile::tempdir().expect("a temporary directory can be created"); - let session = start_recall_session(config(directory.path().join("recall.sqlite3"), "sql")); - session.publish_sql(&"x".repeat(length), Some("s1"), None); - let summary = session.close(); - let counts = summary +fn stream_counts(summary: &DeliverySummary, stream: DeliveryStream) -> DeliveryCounts { + summary .streams .iter() - .find(|(stream, _)| *stream == DeliveryStream::Artifacts) + .find(|(candidate, _)| *candidate == stream) .map(|(_, counts)| *counts) - .unwrap_or_default(); - (counts.accepted, counts.delivered, counts.dropped) + .unwrap_or_default() +} + +/// Artifact counts and stored content rows after publishing the same SQL of `length` bytes `times` times. +pub(crate) fn artifact_counts( + length: usize, + times: usize, +) -> (u64, u64, u64, Vec>) { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("recall.sqlite3"); + let session = start_recall_session(config(path.clone(), "sql")); + let sql: String = "x".repeat(length); + for index in 0..times { + session.publish_sql(&sql, Some(&format!("s{index}")), None); + } + let counts = stream_counts(&session.close(), DeliveryStream::Artifacts); + let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); + let stored = query_store(&store, "SELECT COUNT(*) FROM recall_artifacts", None) + .expect("the query is valid") + .rows; + (counts.accepted, counts.delivered, counts.dropped, stored) +} + +/// Event counts after publishing `events`; rows are built on the delivery thread. +pub(crate) fn event_counts(events: &[&str]) -> (u64, u64, u64) { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let session = start_recall_session(config(directory.path().join("recall.sqlite3"), "events")); + let parsed: Vec> = events + .iter() + .map(|event| serde_json::from_str(event).expect("the fixture is a JSON object")) + .collect(); + parsed + .into_iter() + .for_each(|event| session.publish_event_object(event)); + let counts = stream_counts(&session.close(), DeliveryStream::Events); + (counts.accepted, counts.delivered, counts.failed) } /// Output chunks dropped by a queue of `capacity` records, as counted and as each delivered chunk reports them. @@ -232,7 +265,7 @@ pub(crate) fn output_drops(capacity: usize, chunks: usize) -> (u64, Vec> = taken .iter() - .flat_map(|record| record.rows.iter()) + .flat_map(|record| record.rows().iter()) .map(|row| row.value("dropped_before").cloned()) .collect(); ( diff --git a/crates/sqlbuild-recall/src/writer/tests/test_session.rs b/crates/sqlbuild-recall/src/writer/tests/test_session.rs index e978877de1..f523e5b4cc 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_session.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_session.rs @@ -6,12 +6,14 @@ use crate::writer::models::SetupOutcome; #[cfg(unix)] use crate::writer::tests::helpers::unwritable_session; use crate::writer::tests::helpers::{ - artifact_counts, close_under_held_lock, recorded_session, session_on_corrupt_store, + artifact_counts, close_under_held_lock, event_counts, recorded_session, + session_on_corrupt_store, }; #[cfg(unix)] use crate::writer::tests::test_types::UnwritableTestCase; use crate::writer::tests::test_types::{ - ArtifactSizeTestCase, CorruptStoreTestCase, HeldLockTestCase, RecordedSession, SessionTestCase, + ArtifactSizeTestCase, CorruptStoreTestCase, EventBuildTestCase, HeldLockTestCase, + RecordedSession, SessionTestCase, }; const CLOSE_SLACK_SECONDS: f64 = 0.3; @@ -130,18 +132,54 @@ fn given_sql_of_different_sizes_when_published_then_oversized_sql_is_dropped_unh ArtifactSizeTestCase { description: "ordinary SQL is delivered with its reference", length: 64, - expected_counts: (1, 1, 0), + times: 1, + expected_counts: (1, 1, 0, vec![vec![RecallValue::Integer(1)]]), + }, + ArtifactSizeTestCase { + description: "SQL seen twice is stored once and both records count as delivered", + length: 64, + times: 2, + expected_counts: (2, 2, 0, vec![vec![RecallValue::Integer(1)]]), }, ArtifactSizeTestCase { description: "SQL above the artifact limit is counted as dropped", length: 4 * 1024 * 1024 + 1, - expected_counts: (1, 0, 1), + times: 1, + expected_counts: (1, 0, 1, vec![vec![RecallValue::Integer(0)]]), + }, + ]; + + for test_case in test_cases { + assert_eq!( + artifact_counts(test_case.length, test_case.times), + test_case.expected_counts, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_events_when_built_on_the_delivery_thread_then_unbuildable_ones_count_as_failed() { + let valid: &str = r#"{"event_id":"e1","event_type":"invocation_started","schema_version":2, + "producer":"sqlbuild","producer_version":"1","occurred_at":"2026-10-19T00:00:00.000000Z", + "invocation_id":"events","invocation_sequence":1,"external_context":{},"payload":{}}"#; + let test_cases = [ + EventBuildTestCase { + description: "a complete event is delivered", + events: vec![valid], + expected_counts: (1, 1, 0), + }, + EventBuildTestCase { + description: "an event without an ID is accepted, then counted as failed", + events: vec![valid, r#"{"event_type":"invocation_started"}"#], + expected_counts: (2, 1, 1), }, ]; for test_case in test_cases { assert_eq!( - artifact_counts(test_case.length), + event_counts(&test_case.events), test_case.expected_counts, "{}", test_case.description diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs index a4503c4ff5..25687bdd22 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -53,6 +53,13 @@ pub(crate) struct CorruptStoreTestCase { pub(crate) struct ArtifactSizeTestCase { pub(crate) description: &'static str, pub(crate) length: usize, + pub(crate) times: usize, + pub(crate) expected_counts: (u64, u64, u64, Vec>), +} + +pub(crate) struct EventBuildTestCase { + pub(crate) description: &'static str, + pub(crate) events: Vec<&'static str>, pub(crate) expected_counts: (u64, u64, u64), } From 03049da6bf43152ee09d20ada06f157f72654797 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 07:12:56 +0100 Subject: [PATCH 11/19] fix: move a corrupt Recall store aside only while it is still the file found corrupt Moving a corrupt store aside now takes an exclusive lock on recall.sqlite3.lock, waiting within the startup or shutdown budget, and under it re-checks that the path still names the same file (device and inode) that was found corrupt. A store another process already replaced is used as it is, so concurrent commands move the corrupt file and its companions aside exactly once. --- Cargo.lock | 1 + .../src/bindings/_helpers/recall/constants.rs | 2 + .../src/bindings/_helpers/recall/session.rs | 25 ++++-- crates/sqlbuild-recall/Cargo.toml | 3 + .../src/store/_helpers/identity.rs | 39 +++++++++ .../sqlbuild-recall/src/store/_helpers/mod.rs | 2 + .../src/store/_helpers/quarantine.rs | 72 ++++++++++++++++ crates/sqlbuild-recall/src/store/constants.rs | 4 + crates/sqlbuild-recall/src/store/main/mod.rs | 2 +- .../src/store/main/observe_store_file.rs | 21 +++++ .../src/store/main/quarantine_store.rs | 47 ++++++----- .../src/store/main/store_header_is_foreign.rs | 17 ---- crates/sqlbuild-recall/src/store/models.rs | 26 ++++++ .../src/store/tests/helpers.rs | 84 ++++++++++++++++++- .../src/store/tests/test_store.rs | 39 ++++++++- .../src/store/tests/test_types.rs | 12 ++- .../src/writer/_helpers/delivery.rs | 34 ++++++-- .../docs/concepts/observability/recall.md | 3 +- .../src/sqlbuild/cli/recall/_test_types.py | 9 ++ .../src/sqlbuild/cli/recall/helpers.py | 9 ++ .../sqlbuild/cli/recall/test_local_recall.py | 35 ++++++++ .../docs/concepts/observability/recall.mdx | 3 +- 22 files changed, 424 insertions(+), 65 deletions(-) create mode 100644 crates/sqlbuild-recall/src/store/_helpers/identity.rs create mode 100644 crates/sqlbuild-recall/src/store/_helpers/quarantine.rs create mode 100644 crates/sqlbuild-recall/src/store/main/observe_store_file.rs delete mode 100644 crates/sqlbuild-recall/src/store/main/store_header_is_foreign.rs diff --git a/Cargo.lock b/Cargo.lock index c48b2bb0a7..3c00452915 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1339,6 +1339,7 @@ version = "0.158.2" dependencies = [ "blake3", "rusqlite", + "rustix", "serde_json", "sha2", "sqlbuild-config", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs index f253b88dc7..ff0f8711f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -87,3 +87,5 @@ pub(crate) const TIMESTAMP_PARTS: [&str; 7] = [ pub(crate) const UTC_ZONE_NAME: &str = "UTC"; /// How often closing checks for an interrupt while delivery finishes. pub(crate) const CLOSE_POLL_MS: u64 = 50; +/// Longest wait at startup for another process moving a corrupt store aside; the writer thread retries within the shutdown budget. +pub(crate) const QUARANTINE_START_WAIT_MS: u64 = 200; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index 5db3a210f7..e5c5d7e75a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -3,7 +3,7 @@ use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::time::{Duration, SystemTime}; +use std::time::{Duration, Instant, SystemTime}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyStringMethods; @@ -15,9 +15,10 @@ use sqlbuild_recall::schema::main::ddl_script::ddl_script; use sqlbuild_recall::schema::models::{DdlOptions, Dialect}; use sqlbuild_recall::settings::main::resolve_recall_settings::resolve_recall_settings; use sqlbuild_recall::settings::models::{RecallSettings, SettingsRequest, StoreLocation}; +use sqlbuild_recall::store::main::observe_store_file::observe_store_file; use sqlbuild_recall::store::main::quarantine_store::quarantine_store; -use sqlbuild_recall::store::main::store_header_is_foreign::store_header_is_foreign; use sqlbuild_recall::store::main::store_location_problem::store_location_problem; +use sqlbuild_recall::store::models::QuarantineOutcome; use sqlbuild_recall::writer::constants::DEFAULT_HEARTBEAT_INTERVAL_MS; use sqlbuild_recall::writer::main::start_recall_session::start_recall_session; use sqlbuild_recall::writer::models::{ @@ -30,7 +31,8 @@ use crate::bindings::_helpers::output_capture::records::invocation_context; use crate::bindings::_helpers::recall::console::write_stderr; use crate::bindings::_helpers::recall::constants::{ CLOSE_POLL_MS, DDL_NOTICE_PREFIX, HEARTBEAT_ENVIRONMENT, HOST_TOKEN_LENGTH, - MINIMUM_HEARTBEAT_SECONDS, NOTICE_INTERVAL_SECONDS, STDOUT_STREAM_INDEX, + MINIMUM_HEARTBEAT_SECONDS, NOTICE_INTERVAL_SECONDS, QUARANTINE_START_WAIT_MS, + STDOUT_STREAM_INDEX, }; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::rules::config_json::dumped_json; @@ -258,18 +260,23 @@ fn notify(py: Python<'_>, text: Option) { /// Move a store whose file is not a database aside before the command prints anything. fn quarantined_notice(path: &Path) -> Option { - if !store_header_is_foreign(path) { + let Ok(Some(observed)) = observe_store_file(path) else { + return None; + }; + if !observed.header_foreign { return None; } - Some(match quarantine_store(path, SystemTime::now()) { - Ok(moved) => format!( + let give_up: Instant = Instant::now() + Duration::from_millis(QUARANTINE_START_WAIT_MS); + match quarantine_store(path, &observed, (give_up, SystemTime::now())) { + Ok(QuarantineOutcome::Moved(moved)) => Some(format!( "Recall: {} was not a valid database; it was moved to {} and a fresh store was \ started.\n", path.display(), moved.display() - ), - Err(error) => format!("Recall history is not being written: {error}.\n"), - }) + )), + Ok(QuarantineOutcome::Kept | QuarantineOutcome::LockBusy) => None, + Err(error) => Some(format!("Recall history is not being written: {error}.\n")), + } } fn started_session( diff --git a/crates/sqlbuild-recall/Cargo.toml b/crates/sqlbuild-recall/Cargo.toml index ca3899fd14..5738b05067 100644 --- a/crates/sqlbuild-recall/Cargo.toml +++ b/crates/sqlbuild-recall/Cargo.toml @@ -15,6 +15,9 @@ sha2.workspace = true sqlbuild-config.workspace = true sqlbuild-core.workspace = true +[target.'cfg(unix)'.dependencies] +rustix.workspace = true + [dev-dependencies] tempfile.workspace = true diff --git a/crates/sqlbuild-recall/src/store/_helpers/identity.rs b/crates/sqlbuild-recall/src/store/_helpers/identity.rs new file mode 100644 index 0000000000..d2ed39c27b --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/identity.rs @@ -0,0 +1,39 @@ +//! File identity and SQLite header checks for moving a corrupt store aside. + +use std::fs::{File, Metadata}; +use std::io::Read; + +use crate::store::constants::SQLITE_HEADER; +use crate::store::models::StoreFileIdentity; + +#[cfg(unix)] +pub(crate) fn file_identity(metadata: &Metadata) -> StoreFileIdentity { + use std::os::unix::fs::MetadataExt; + StoreFileIdentity { + volume: metadata.dev(), + file: metadata.ino(), + } +} + +#[cfg(not(unix))] +pub(crate) fn file_identity(metadata: &Metadata) -> StoreFileIdentity { + let modified: u64 = metadata.modified().map_or(0, |time| { + time.duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| { + u64::try_from(elapsed.as_nanos()).unwrap_or(u64::MAX) + }) + }); + StoreFileIdentity { + volume: metadata.len(), + file: modified, + } +} + +/// Whether `file` has content that does not start with the SQLite header. +pub(crate) fn header_is_foreign(file: &File) -> bool { + let mut header: Vec = Vec::with_capacity(SQLITE_HEADER.len()); + let read = file + .take(SQLITE_HEADER.len() as u64) + .read_to_end(&mut header); + read.is_ok_and(|length| length > 0) && header.as_slice() != SQLITE_HEADER.as_slice() +} diff --git a/crates/sqlbuild-recall/src/store/_helpers/mod.rs b/crates/sqlbuild-recall/src/store/_helpers/mod.rs index 434bbaef58..c29e92d7b3 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/mod.rs @@ -1,3 +1,5 @@ +pub(crate) mod identity; +pub(crate) mod quarantine; pub(crate) mod rows; pub(crate) mod schema; pub(crate) mod sqlite; diff --git a/crates/sqlbuild-recall/src/store/_helpers/quarantine.rs b/crates/sqlbuild-recall/src/store/_helpers/quarantine.rs new file mode 100644 index 0000000000..bd84f0bbe4 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/quarantine.rs @@ -0,0 +1,72 @@ +//! The quarantine lock and the checks that keep a corrupt store from being moved aside twice. + +use std::fs::{File, OpenOptions}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use crate::store::constants::{QUARANTINE_LOCK_RETRY_MS, QUARANTINE_LOCK_SUFFIX}; +use crate::store::errors::{StoreError, StoreErrorKind}; +use crate::store::main::observe_store_file::observe_store_file; +use crate::store::models::StoreObservation; + +pub(crate) fn moved_name(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_owned(); + name.push(suffix); + PathBuf::from(name) +} + +pub(crate) fn corrupt(path: &Path, doing: &str, error: &std::io::Error) -> StoreError { + StoreError::new( + StoreErrorKind::Corrupt, + format!( + "cannot {doing} for the corrupt store {}: {error}", + path.display() + ), + ) +} + +/// One non-blocking attempt at the exclusive `flock`; `Ok(false)` while another process holds it. +#[cfg(unix)] +fn try_lock(file: &File) -> std::io::Result { + match rustix::fs::flock(file, rustix::fs::FlockOperation::NonBlockingLockExclusive) { + Ok(()) => Ok(true), + Err(rustix::io::Errno::WOULDBLOCK) => Ok(false), + Err(error) => Err(std::io::Error::from(error)), + } +} + +/// Without `flock`, only the identity re-check guards the move. +#[cfg(not(unix))] +fn try_lock(_file: &File) -> std::io::Result { + Ok(true) +} + +/// The exclusive lock on `.lock`, waiting until `give_up`; `None` when another process holds it that long. +pub(crate) fn quarantine_lock(path: &Path, give_up: Instant) -> Result, StoreError> { + let lock_path: PathBuf = moved_name(path, QUARANTINE_LOCK_SUFFIX); + let file: File = OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(&lock_path) + .map_err(|error| corrupt(path, "open the quarantine lock", &error))?; + loop { + match try_lock(&file) { + Ok(true) => return Ok(Some(file)), + Ok(false) if Instant::now() < give_up => { + std::thread::sleep(Duration::from_millis(QUARANTINE_LOCK_RETRY_MS)); + } + Ok(false) => return Ok(None), + Err(error) => return Err(corrupt(path, "take the quarantine lock", &error)), + } + } +} + +/// Whether the file now at `path` is still the one observed corrupt. +pub(crate) fn still_observed(path: &Path, observed: &StoreObservation) -> bool { + matches!( + observe_store_file(path), + Ok(Some(current)) if current.identity == observed.identity + && (current.header_foreign || !observed.header_foreign) + ) +} diff --git a/crates/sqlbuild-recall/src/store/constants.rs b/crates/sqlbuild-recall/src/store/constants.rs index f31cccce15..4ecb02d394 100644 --- a/crates/sqlbuild-recall/src/store/constants.rs +++ b/crates/sqlbuild-recall/src/store/constants.rs @@ -18,3 +18,7 @@ pub const SQLITE_HEADER: &[u8; 16] = b"SQLite format 3\0"; pub const CORRUPT_SUFFIX: &str = ".corrupt-"; /// SQLite's companion files, moved along with a corrupt store. pub const COMPANION_SUFFIXES: [&str; 2] = ["-wal", "-shm"]; +/// Appended to the store path to name the lock file that serialises moving a corrupt store aside. +pub const QUARANTINE_LOCK_SUFFIX: &str = ".lock"; +/// Pause between attempts to take the quarantine lock. +pub const QUARANTINE_LOCK_RETRY_MS: u64 = 10; diff --git a/crates/sqlbuild-recall/src/store/main/mod.rs b/crates/sqlbuild-recall/src/store/main/mod.rs index 0afb2ee851..d86f08ea49 100644 --- a/crates/sqlbuild-recall/src/store/main/mod.rs +++ b/crates/sqlbuild-recall/src/store/main/mod.rs @@ -2,10 +2,10 @@ pub mod describe_store; pub mod ensure_sqlite_schema; pub mod insert_rows; pub mod maintain_sqlite_store; +pub mod observe_store_file; pub mod open_sqlite_store; pub mod quarantine_store; pub mod query_store; pub mod read_output_transcript; pub mod resolve_invocation; -pub mod store_header_is_foreign; pub mod store_location_problem; diff --git a/crates/sqlbuild-recall/src/store/main/observe_store_file.rs b/crates/sqlbuild-recall/src/store/main/observe_store_file.rs new file mode 100644 index 0000000000..01fb7282d3 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/observe_store_file.rs @@ -0,0 +1,21 @@ +//! `observe_store_file`: which file a store path names and whether it is a SQLite database. + +use std::io::ErrorKind; +use std::path::Path; + +use crate::store::_helpers::identity::{file_identity, header_is_foreign}; +use crate::store::models::StoreObservation; + +/// The identity and header verdict of the file at `path`, read from one open handle; `None` when there is no file. +pub fn observe_store_file(path: &Path) -> std::io::Result> { + let file = match std::fs::File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + let identity = file_identity(&file.metadata()?); + Ok(Some(StoreObservation { + identity, + header_foreign: header_is_foreign(&file), + })) +} diff --git a/crates/sqlbuild-recall/src/store/main/quarantine_store.rs b/crates/sqlbuild-recall/src/store/main/quarantine_store.rs index b90d1ce4b4..4567fc059e 100644 --- a/crates/sqlbuild-recall/src/store/main/quarantine_store.rs +++ b/crates/sqlbuild-recall/src/store/main/quarantine_store.rs @@ -1,37 +1,38 @@ -//! `quarantine_store`: move a corrupt store aside so a fresh one can start. +//! `quarantine_store`: move a corrupt store aside, once, so a fresh one can start. use std::path::{Path, PathBuf}; -use std::time::SystemTime; +use std::time::{Instant, SystemTime}; use crate::records::main::format_timestamp::format_timestamp; +use crate::store::_helpers::quarantine::{corrupt, moved_name, quarantine_lock, still_observed}; use crate::store::constants::{COMPANION_SUFFIXES, CORRUPT_SUFFIX}; -use crate::store::errors::{StoreError, StoreErrorKind}; +use crate::store::errors::StoreError; +use crate::store::models::{QuarantineOutcome, StoreObservation}; -fn moved_name(path: &Path, suffix: &str) -> PathBuf { - let mut name = path.as_os_str().to_owned(); - name.push(suffix); - PathBuf::from(name) -} - -/// Rename `path` and its companion files to `.corrupt-`; returns the new store name. -pub fn quarantine_store(path: &Path, now: SystemTime) -> Result { +/// Under the store's quarantine lock, rename the observed corrupt file and its companions to `.corrupt-`. When another process already replaced it, keep the store that is there. +pub fn quarantine_store( + path: &Path, + observed: &StoreObservation, + (give_up, now): (Instant, SystemTime), +) -> Result { + let Some(lock) = quarantine_lock(path, give_up)? else { + return Ok(QuarantineOutcome::LockBusy); + }; + if !still_observed(path, observed) { + drop(lock); + return Ok(QuarantineOutcome::Kept); + } let stamp: String = format_timestamp(now).replace(':', ""); - let suffix: String = format!("{CORRUPT_SUFFIX}{stamp}"); - let target: PathBuf = moved_name(path, &suffix); - std::fs::rename(path, &target).map_err(|error| { - StoreError::new( - StoreErrorKind::Corrupt, - format!( - "cannot move the corrupt store {} aside: {error}", - path.display() - ), - ) - })?; + let target: PathBuf = moved_name(path, &format!("{CORRUPT_SUFFIX}{stamp}")); for companion in COMPANION_SUFFIXES { let source: PathBuf = moved_name(path, companion); if source.exists() { drop(std::fs::rename(&source, moved_name(&target, companion))); } } - Ok(target) + let moved = std::fs::rename(path, &target) + .map(|()| QuarantineOutcome::Moved(target)) + .map_err(|error| corrupt(path, "move the file aside", &error)); + drop(lock); + moved } diff --git a/crates/sqlbuild-recall/src/store/main/store_header_is_foreign.rs b/crates/sqlbuild-recall/src/store/main/store_header_is_foreign.rs deleted file mode 100644 index 7ce99a60dd..0000000000 --- a/crates/sqlbuild-recall/src/store/main/store_header_is_foreign.rs +++ /dev/null @@ -1,17 +0,0 @@ -//! `store_header_is_foreign`: whether a store file holds something other than a SQLite database. - -use std::io::Read; -use std::path::Path; - -use crate::store::constants::SQLITE_HEADER; - -/// Whether `path` exists with content that is not a SQLite database; never opens it with SQLite. -#[must_use] -pub fn store_header_is_foreign(path: &Path) -> bool { - let mut header: Vec = Vec::with_capacity(SQLITE_HEADER.len()); - let read = std::fs::File::open(path).and_then(|file| { - file.take(SQLITE_HEADER.len() as u64) - .read_to_end(&mut header) - }); - read.is_ok_and(|length| length > 0) && header.as_slice() != SQLITE_HEADER.as_slice() -} diff --git a/crates/sqlbuild-recall/src/store/models.rs b/crates/sqlbuild-recall/src/store/models.rs index 975c096ecc..f501bea5fc 100644 --- a/crates/sqlbuild-recall/src/store/models.rs +++ b/crates/sqlbuild-recall/src/store/models.rs @@ -20,6 +20,32 @@ pub enum OpenMode { ReadOnly, } +/// Which file a path named when it was observed: device and inode on Unix, else size and modification time. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StoreFileIdentity { + pub(crate) volume: u64, + pub(crate) file: u64, +} + +/// One look at the file at a store path. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StoreObservation { + pub identity: StoreFileIdentity, + /// The file has content and does not start with the SQLite header. + pub header_foreign: bool, +} + +/// What moving a corrupt store aside did. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum QuarantineOutcome { + /// The observed file and its companions were renamed to this path. + Moved(PathBuf), + /// The path no longer holds the observed corrupt file, so the store there is used as it is. + Kept, + /// Another process held the quarantine lock past the wait; nothing was moved. + LockBusy, +} + /// A row whose key was already stored with different content; the first row is kept. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ConflictEvidence { diff --git a/crates/sqlbuild-recall/src/store/tests/helpers.rs b/crates/sqlbuild-recall/src/store/tests/helpers.rs index 8387af1331..126a992158 100644 --- a/crates/sqlbuild-recall/src/store/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/store/tests/helpers.rs @@ -14,9 +14,15 @@ use crate::store::errors::{StoreError, StoreErrorKind}; use crate::store::main::ensure_sqlite_schema::ensure_sqlite_schema; use crate::store::main::insert_rows::insert_rows; use crate::store::main::maintain_sqlite_store::maintain_sqlite_store; +use crate::store::main::observe_store_file::observe_store_file; use crate::store::main::open_sqlite_store::open_sqlite_store; +use crate::store::main::quarantine_store::quarantine_store; use crate::store::main::query_store::query_store; -use crate::store::models::{MaintenanceOutcome, MaintenanceRequest, OpenMode, SqliteStore}; +use crate::store::models::{ + MaintenanceOutcome, MaintenanceRequest, OpenMode, QuarantineOutcome, SqliteStore, + StoreObservation, +}; +use crate::store::tests::test_types::QuarantineRace; pub(crate) const NOW: &str = "2026-10-19T12:00:00.000000Z"; @@ -232,3 +238,79 @@ pub(crate) fn open_under_locked_directory(relative_path: &str) -> Result<(), Sto .expect("test directory permissions can change"); error_kind(opened.map(drop)) } + +#[cfg(unix)] +fn corrupt_copies(directory: &std::path::Path) -> usize { + std::fs::read_dir(directory) + .expect("the directory lists") + .filter_map(Result::ok) + .filter(|entry| { + let name = entry.file_name().to_string_lossy().into_owned(); + name.contains(".corrupt-") && !name.ends_with("-wal") && !name.ends_with("-shm") + }) + .count() +} + +#[cfg(unix)] +/// Nobody else acts on the corrupt file. +pub(crate) fn nobody_else( + _path: &std::path::Path, + _observed: &StoreObservation, + _lock: &std::fs::File, +) { +} + +#[cfg(unix)] +/// Another process moves the corrupt file aside and starts a fresh store. +pub(crate) fn already_replaced( + path: &std::path::Path, + observed: &StoreObservation, + _lock: &std::fs::File, +) { + let give_up = std::time::Instant::now() + Duration::from_millis(100); + drop( + quarantine_store(path, observed, (give_up, SystemTime::now())) + .expect("the first move succeeds"), + ); + let fresh = open_sqlite_store(path, OpenMode::ReadWrite).expect("a fresh store opens"); + ensure_sqlite_schema(&fresh, &envelope("fresh", NOW), NOW).expect("the fresh store is set up"); +} + +#[cfg(unix)] +/// Another process holds the quarantine lock. +pub(crate) fn lock_held( + _path: &std::path::Path, + _observed: &StoreObservation, + lock: &std::fs::File, +) { + rustix::fs::flock(lock, rustix::fs::FlockOperation::LockExclusive).expect("the lock is taken"); +} + +#[cfg(unix)] +/// Observe a corrupt store, let `race` happen, then move it aside with the now possibly stale observation: whether it moved, how many copies were moved aside, and whether a SQLite store is at the path. +pub(crate) fn quarantine_after(race: QuarantineRace) -> (bool, usize, bool) { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("recall.sqlite3"); + std::fs::write(&path, b"this is not a database").expect("test files can be written"); + std::fs::write(directory.path().join("recall.sqlite3-wal"), b"stale") + .expect("test files can be written"); + let observed = observe_store_file(&path) + .expect("the file can be read") + .expect("the corrupt file is observed"); + let lock = std::fs::File::create(directory.path().join("recall.sqlite3.lock")) + .expect("the lock file can be created"); + race(&path, &observed, &lock); + let give_up = std::time::Instant::now() + Duration::from_millis(100); + let outcome = quarantine_store(&path, &observed, (give_up, SystemTime::now())) + .expect("the move does not fail"); + let live_store: bool = matches!( + observe_store_file(&path), + Ok(Some(current)) if !current.header_foreign + ) && open_sqlite_store(&path, OpenMode::ReadOnly).is_ok(); + drop(lock); + ( + matches!(outcome, QuarantineOutcome::Moved(_)), + corrupt_copies(directory.path()), + live_store, + ) +} diff --git a/crates/sqlbuild-recall/src/store/tests/test_store.rs b/crates/sqlbuild-recall/src/store/tests/test_store.rs index ed01276652..93783e8114 100644 --- a/crates/sqlbuild-recall/src/store/tests/test_store.rs +++ b/crates/sqlbuild-recall/src/store/tests/test_store.rs @@ -5,13 +5,15 @@ use crate::store::main::describe_store::describe_store; use crate::store::main::query_store::query_store; use crate::store::main::read_output_transcript::read_output_transcript; use crate::store::main::resolve_invocation::resolve_invocation; -#[cfg(unix)] -use crate::store::tests::helpers::open_under_locked_directory; use crate::store::tests::helpers::{ NOW, chunk, envelope, error_kind, new_store, newer_schema_store, rows, shifted, write, }; #[cfg(unix)] -use crate::store::tests::test_types::LockedDirectoryTestCase; +use crate::store::tests::helpers::{ + already_replaced, lock_held, nobody_else, open_under_locked_directory, quarantine_after, +}; +#[cfg(unix)] +use crate::store::tests::test_types::{LockedDirectoryTestCase, QuarantineTestCase}; use crate::store::tests::test_types::{ NewerSchemaTestCase, QueryTestCase, ResolveInvocationTestCase, TranscriptTestCase, }; @@ -209,3 +211,34 @@ fn given_a_read_only_directory_when_opened_then_the_error_is_a_permission_failur ); } } + +#[cfg(unix)] +#[test] +fn given_a_stale_observation_when_quarantining_then_only_the_observed_file_is_moved() { + let test_cases = [ + QuarantineTestCase { + description: "the observed corrupt file is moved aside", + race: nobody_else, + expected_outcome: (true, 1, false), + }, + QuarantineTestCase { + description: "a store another process already replaced is kept", + race: already_replaced, + expected_outcome: (false, 1, true), + }, + QuarantineTestCase { + description: "a held quarantine lock moves nothing", + race: lock_held, + expected_outcome: (false, 0, false), + }, + ]; + + for test_case in test_cases { + assert_eq!( + quarantine_after(test_case.race), + test_case.expected_outcome, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/store/tests/test_types.rs b/crates/sqlbuild-recall/src/store/tests/test_types.rs index 7090695f6a..fa20bb0528 100644 --- a/crates/sqlbuild-recall/src/store/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/store/tests/test_types.rs @@ -1,6 +1,6 @@ use crate::records::models::{RecallRow, RecallValue}; use crate::store::errors::StoreErrorKind; -use crate::store::models::MaintenanceOutcome; +use crate::store::models::{MaintenanceOutcome, StoreObservation}; pub(crate) struct ProjectionTestCase { pub(crate) description: &'static str, @@ -50,3 +50,13 @@ pub(crate) struct MaintenanceTestCase { pub(crate) expected_outcomes: Vec, pub(crate) expected_rows: Vec>, } + +/// What another process does between observing a corrupt store and moving it aside. +pub(crate) type QuarantineRace = fn(&std::path::Path, &StoreObservation, &std::fs::File); + +pub(crate) struct QuarantineTestCase { + pub(crate) description: &'static str, + pub(crate) race: QuarantineRace, + /// Whether the move was made, copies moved aside, and whether a SQLite store is at the path. + pub(crate) expected_outcome: (bool, usize, bool), +} diff --git a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs index 91fb00a1e1..2c75159196 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs @@ -13,9 +13,12 @@ use crate::store::errors::{StoreError, StoreErrorKind}; use crate::store::main::ensure_sqlite_schema::ensure_sqlite_schema; use crate::store::main::insert_rows::insert_rows; use crate::store::main::maintain_sqlite_store::maintain_sqlite_store; +use crate::store::main::observe_store_file::observe_store_file; use crate::store::main::open_sqlite_store::open_sqlite_store; use crate::store::main::quarantine_store::quarantine_store; -use crate::store::models::{InsertOutcome, MaintenanceRequest, OpenMode, SqliteStore}; +use crate::store::models::{ + InsertOutcome, MaintenanceRequest, OpenMode, QuarantineOutcome, SqliteStore, StoreObservation, +}; use crate::writer::_helpers::materialize::{Built, Materializer}; use crate::writer::_helpers::queue::{Queued, WriterState}; use crate::writer::constants::{ @@ -68,6 +71,8 @@ fn open_and_setup(shared: &Shared, config: &SessionConfig) -> Result = + observe_store_file(&config.store_path).unwrap_or(None); let attempt = open_sqlite_store(&config.store_path, OpenMode::ReadWrite).and_then(|store| { bound_busy_wait(&store, retry_limit(shared, started)); @@ -83,13 +88,26 @@ fn open_and_setup(shared: &Shared, config: &SessionConfig) -> Result { quarantined = true; - let moved = quarantine_store(&config.store_path, SystemTime::now())?; - locked(&shared.state).notices.push(format!( - "the Recall store {} was not a valid database; it was moved to {} and a \ - fresh store was started", - config.store_path.display(), - moved.display() - )); + let Some(observed) = observed else { + return Err(error); + }; + let outcome = quarantine_store( + &config.store_path, + &observed, + (retry_limit(shared, started), SystemTime::now()), + )?; + match outcome { + QuarantineOutcome::Moved(moved) => { + locked(&shared.state).notices.push(format!( + "the Recall store {} was not a valid database; it was moved to {} \ + and a fresh store was started", + config.store_path.display(), + moved.display() + )); + } + QuarantineOutcome::Kept => {} + QuarantineOutcome::LockBusy => return Err(error), + } } other => return other, } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index a16fe2ed40..eadd5d8b99 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -89,7 +89,8 @@ If the store location is not writable, the command still runs and prints a notic the SQL an administrator can use to create the store; the notice comes before the command's own output whenever the problem is visible from file permissions. The notice repeats at most once a day. If the store file is not a SQLite database, Recall moves it aside to -`recall.sqlite3.corrupt-`, starts a fresh store, and says so once. +`recall.sqlite3.corrupt-`, starts a fresh store, and says so once. Commands that +find the same corrupt file coordinate through `recall.sqlite3.lock`, so only one of them moves it. A `connection` that does not use the `sqlite` adapter is not supported yet; Recall prints a notice and records nothing. diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 34bb86d0ce..a7a07ff7bc 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -115,3 +115,12 @@ class CorruptStoreTestCase: class FailingStderrTestCase: description: str expected_exit_code: int + + +@dataclass(frozen=True) +class CorruptStoreRaceTestCase: + description: str + targets: tuple[str, ...] + expected_exit_codes: list[int] + expected_moved: int + expected_recorded: list[list[object]] diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index 115c9571d6..5e7d3ad1f1 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -315,3 +315,12 @@ def numbered_lines(output: bytes) -> list[int]: """The leading numbers of the numbered lines in `output`, in order.""" return [int(number) for number in NUMBERED_LINE.findall(output)] + + +def simultaneous_builds(project_dir: Path, targets: tuple[str, ...]) -> list[int]: + """Exit codes of builds of `targets`, each with its own warehouse, started together.""" + + started: list[subprocess.Popen[bytes]] = [ + start(project_dir, "build", "--target", target) for target in targets + ] + return [process.wait(timeout=600) for process in started] diff --git a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py index 8324b8a399..2af9d5b27b 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py @@ -11,6 +11,7 @@ CombinedOutputTestCase, ConcurrentBuildsTestCase, ConcurrentWritersTestCase, + CorruptStoreRaceTestCase, CorruptStoreTestCase, DisabledRecallTestCase, FailingStderrTestCase, @@ -38,6 +39,7 @@ prepared_project, recall_rows, run, + simultaneous_builds, timed_build, write_corrupt_store, write_project, @@ -658,3 +660,36 @@ def test_given_failing_stderr_when_recall_reports_then_the_exit_code_is_unchange exit_code: int = build_with_failing_recall_notices(project_dir, tmp_path / "site") assert exit_code == test_case.expected_exit_code + + +@pytest.mark.parametrize( + "test_case", + [ + CorruptStoreRaceTestCase( + description="builds starting together on a corrupt store move it aside exactly once", + targets=("dev", "dev_1"), + expected_exit_codes=[0, 0], + expected_moved=1, + expected_recorded=[["build", "completed", 2]], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_corrupt_store_when_builds_race_then_one_moves_it_and_both_are_recorded( + test_case: CorruptStoreRaceTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop", targets=len(test_case.targets)) + _ = run(project_dir, "compile") + write_corrupt_store(project_dir) + + exit_codes: list[int] = simultaneous_builds(project_dir, test_case.targets) + moved: list[Path] = list((project_dir / "logs").glob("recall.sqlite3.corrupt-*[0-9Z]")) + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT command, status, COUNT(*) FROM invocations WHERE command = 'build' " + "GROUP BY command, status", + ) + + assert exit_codes == test_case.expected_exit_codes + assert len(moved) == test_case.expected_moved + assert recorded == test_case.expected_recorded diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index e539dd4c1f..189a485b1c 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -77,7 +77,8 @@ If the store location is not writable, the command still runs and prints a notic the SQL an administrator can use to create the store; the notice comes before the command's own output whenever the problem is visible from file permissions. The notice repeats at most once a day. If the store file is not a SQLite database, Recall moves it aside to -`recall.sqlite3.corrupt-`, starts a fresh store, and says so once. +`recall.sqlite3.corrupt-`, starts a fresh store, and says so once. Commands that +find the same corrupt file coordinate through `recall.sqlite3.lock`, so only one of them moves it. A `connection` that does not use the `sqlite` adapter is not supported yet; Recall prints a notice and records nothing. From 74d50a85685c77448963b2bc384c4dedbb95c3d5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 09:17:51 +0100 Subject: [PATCH 12/19] chore: record the Recall crate at the workspace version in Cargo.lock --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 3c00452915..5f17772979 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1335,7 +1335,7 @@ dependencies = [ [[package]] name = "sqlbuild-recall" -version = "0.158.2" +version = "0.159.0" dependencies = [ "blake3", "rusqlite", From e232a1283854770e3cdba2b8663e6a648f4d5698 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 09:50:48 +0100 Subject: [PATCH 13/19] chore: record the Recall crate at the workspace version in Cargo.lock --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 5f17772979..f02dd460d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1335,7 +1335,7 @@ dependencies = [ [[package]] name = "sqlbuild-recall" -version = "0.159.0" +version = "0.160.0" dependencies = [ "blake3", "rusqlite", From 0148dd5af849d24d4f708e57bab5c47c6da2f36b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:43:56 +0100 Subject: [PATCH 14/19] perf: chunk Recall output in one pass and record at most 1 MiB per stream Large writes were split into chunks by repeatedly copying the remaining buffer, quadratic in the write size: a warm dense compile printing 25 MB of JSON spent most of its time there. Chunks are now cut straight from the written bytes. Each stream also records at most 1 MiB per invocation; later output takes one filtered sequence per contiguous run, so output_invocations reports a filtered gap, and the views count trailing gaps from the delivery row. --- .../src/schema/_helpers/clickhouse_views.rs | 5 +- .../src/schema/_helpers/sqlite_views.rs | 5 +- .../src/writer/_helpers/queue.rs | 94 ++++++++++++++----- .../sqlbuild-recall/src/writer/constants.rs | 2 + .../src/writer/tests/helpers.rs | 34 ++++++- .../src/writer/tests/test_queue.rs | 40 +++++++- .../src/writer/tests/test_session.rs | 1 - .../src/writer/tests/test_types.rs | 7 ++ .../docs/concepts/observability/recall.md | 3 +- .../src/sqlbuild/cli/recall/_test_types.py | 7 ++ .../src/sqlbuild/cli/recall/helpers.py | 2 + .../sqlbuild/cli/recall/test_local_recall.py | 32 +++++++ .../docs/concepts/observability/recall.mdx | 3 +- 13 files changed, 203 insertions(+), 32 deletions(-) diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs index 8fdce6aad8..88be727828 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -247,7 +247,10 @@ counted AS ( coalesce(received.stdout_bytes, 0) AS stdout_bytes, coalesce(received.stderr_bytes, 0) AS stderr_bytes, received.max_sequence AS max_sequence, - coalesce(received.max_sequence + 1 - received.chunks, 0) AS missing_sequences, + greatest( + coalesce(received.max_sequence + 1 - received.chunks, 0), + coalesce(toInt64(delivery.accepted), 0) - coalesce(toInt64(received.chunks), 0) + ) AS missing_sequences, greatest(coalesce(received.queue_drops, 0), coalesce(delivery.dropped, 0)) AS reported_drops, coalesce(delivery.filtered, 0) AS filtered, coalesce(delivery.failed, 0) AS failed, diff --git a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs index 76615bcb83..2badca5552 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs @@ -145,7 +145,10 @@ counted AS ( COALESCE(received.stdout_bytes, 0) AS stdout_bytes, COALESCE(received.stderr_bytes, 0) AS stderr_bytes, received.max_sequence, - COALESCE(received.max_sequence + 1 - received.chunks, 0) AS missing_sequences, + MAX( + COALESCE(received.max_sequence + 1 - received.chunks, 0), + COALESCE(delivery.accepted, 0) - COALESCE(received.chunks, 0) + ) AS missing_sequences, MAX(COALESCE(received.queue_drops, 0), COALESCE(delivery.dropped, 0)) AS reported_drops, COALESCE(delivery.filtered, 0) AS filtered, COALESCE(delivery.failed, 0) AS failed, diff --git a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs index bf9a856153..454d564b77 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs @@ -13,7 +13,9 @@ use crate::records::models::{ }; use crate::store::models::InsertOutcome; use crate::writer::_helpers::delivery::stream_of; -use crate::writer::constants::{QUEUE_BYTE_CAPACITY, QUEUE_RECORD_CAPACITY}; +use crate::writer::constants::{ + OUTPUT_STREAM_CAPTURE_BYTES, QUEUE_BYTE_CAPACITY, QUEUE_RECORD_CAPACITY, +}; use crate::writer::models::{DeliveryStream, Priority}; /// SQL a statement executed, hashed and turned into rows on the delivery thread. @@ -79,6 +81,11 @@ pub(crate) struct WriterState { pending: Option, next_sequence: u64, offsets: [u64; 2], + /// Bytes of each stream accepted for recording, pending or emitted. + captured: [usize; 2], + /// The stream whose output past the capture limit the latest sequence stands for. + skipping: Option, + output_limit: usize, unreported_output_drops: u64, pub(crate) diagnostic_sequence: u64, pub(crate) closing: bool, @@ -120,6 +127,9 @@ impl WriterState { pending: None, next_sequence: 0, offsets: [0; 2], + captured: [0; 2], + skipping: None, + output_limit: OUTPUT_STREAM_CAPTURE_BYTES, unreported_output_drops: 0, diagnostic_sequence: 0, closing: false, @@ -132,6 +142,13 @@ impl WriterState { } } + /// The same state recording at most `limit` bytes of each output stream. + #[cfg(test)] + pub(crate) fn with_output_limit(mut self, limit: usize) -> Self { + self.output_limit = limit; + self + } + pub(crate) fn counts_mut(&mut self, stream: DeliveryStream) -> &mut DeliveryCounts { &mut self.counts[stream.index()] } @@ -245,6 +262,7 @@ impl WriterState { data, }; self.next_sequence += 1; + self.skipping = None; self.offsets[index] += chunk.data.len() as u64; let row = output_row(&self.envelope, &chunk); self.push(vec![row], DeliveryStream::Output, Priority::Output); @@ -262,30 +280,60 @@ impl WriterState { { self.flush_output(); } - let pending = self.pending.get_or_insert_with(|| Pending { - stream, - data: Vec::new(), - since: Instant::now(), - occurred_at: format_timestamp(now), - }); - pending.data.extend_from_slice(data); - while self - .pending - .as_ref() - .is_some_and(|pending| pending.data.len() >= MAX_OUTPUT_CHUNK_BYTES) - { - let Some(mut pending) = self.pending.take() else { - break; - }; - let rest: Vec = pending.data.split_off(MAX_OUTPUT_CHUNK_BYTES); - let occurred_at = pending.occurred_at.clone(); - self.emit(pending.stream, pending.data, &occurred_at); - if !rest.is_empty() { - pending.data = rest; - pending.since = Instant::now(); - self.pending = Some(pending); + let index = usize::from(stream == OutputStream::Stderr); + let room: usize = self.output_limit.saturating_sub(self.captured[index]); + let (kept, skipped) = data.split_at(room.min(data.len())); + self.captured[index] += kept.len(); + self.record_output(stream, kept, now); + if !skipped.is_empty() { + self.skip_output(stream); + } + } + + /// Output past the stream's capture limit is not recorded: one sequence stands for each contiguous run of it and is counted as filtered, so readers see a filtered gap. + fn skip_output(&mut self, stream: OutputStream) { + if self.skipping == Some(stream) { + return; + } + self.flush_output(); + self.next_sequence += 1; + self.skipping = Some(stream); + let counts = self.counts_mut(DeliveryStream::Output); + counts.accepted += 1; + counts.filtered += 1; + } + + /// Chunk `data` without copying it more than once: fill the pending chunk, emit full chunks straight from `data`, keep the remainder pending. + fn record_output(&mut self, stream: OutputStream, data: &[u8], now: SystemTime) { + if data.is_empty() { + return; + } + let mut rest: &[u8] = data; + if let Some(pending) = self.pending.as_mut() { + let room: usize = MAX_OUTPUT_CHUNK_BYTES.saturating_sub(pending.data.len()); + let (head, tail) = rest.split_at(room.min(rest.len())); + pending.data.extend_from_slice(head); + rest = tail; + if pending.data.len() >= MAX_OUTPUT_CHUNK_BYTES { + self.flush_output(); } } + if rest.is_empty() { + return; + } + let occurred_at: String = format_timestamp(now); + let (full, remainder) = rest.as_chunks::(); + for chunk in full { + self.emit(stream, chunk.to_vec(), &occurred_at); + } + if !remainder.is_empty() { + self.pending = Some(Pending { + stream, + data: remainder.to_vec(), + since: Instant::now(), + occurred_at, + }); + } } /// Emit pending output as one chunk. diff --git a/crates/sqlbuild-recall/src/writer/constants.rs b/crates/sqlbuild-recall/src/writer/constants.rs index f38530229d..5cb01c891b 100644 --- a/crates/sqlbuild-recall/src/writer/constants.rs +++ b/crates/sqlbuild-recall/src/writer/constants.rs @@ -27,3 +27,5 @@ pub const MINIMUM_MAINTENANCE_MS: u64 = 200; pub const MAX_ARTIFACT_BYTES: usize = 4 * 1024 * 1024; /// What a JSON scalar counts toward the queue's byte capacity. pub const JSON_SCALAR_BYTES: usize = 8; +/// Output of each stream recorded per invocation; later output is counted as filtered. +pub const OUTPUT_STREAM_CAPTURE_BYTES: usize = 1024 * 1024; diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs index 7d2d6d6400..08273a2551 100644 --- a/crates/sqlbuild-recall/src/writer/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -130,8 +130,7 @@ pub(crate) fn recorded_session(output: &[(OutputStream, Vec)]) -> RecordedSe let facts = query_store( &store, "SELECT (SELECT status FROM invocations), (SELECT COUNT(*) FROM recall_diagnostics), \ - (SELECT COUNT(*) FROM recall_artifacts), (SELECT COUNT(*) FROM recall_heartbeats), \ - (SELECT COUNT(*) FROM recall_delivery), (SELECT command FROM recall_invocations)", + (SELECT COUNT(*) FROM recall_artifacts), (SELECT COUNT(*) FROM recall_delivery), (SELECT command FROM recall_invocations)", None, ) .expect("the query is valid"); @@ -273,3 +272,34 @@ pub(crate) fn output_drops(capacity: usize, chunks: usize) -> (u64, Vec (Vec<[Option; 3]>, u64) { + let mut state = WriterState::new(envelope("inv")).with_output_limit(limit); + for (stream, length) in writes { + state.append_output(*stream, &vec![b'x'; *length], SystemTime::now()); + } + state.flush_output(); + let chunks = state + .take_batch(usize::MAX) + .iter() + .flat_map(|record| record.rows().iter()) + .map(|row| ["sequence", "stream", "byte_count"].map(|name| row.value(name).cloned())) + .collect(); + ( + chunks, + state.counts[DeliveryStream::Output.index()].filtered, + ) +} + +/// A chunk's sequence, stream and byte count, as `capped` reports them. +pub(crate) fn chunk_facts(sequence: i64, stream: &str, bytes: i64) -> [Option; 3] { + [ + Some(RecallValue::Integer(sequence)), + Some(RecallValue::Text(stream.to_owned())), + Some(RecallValue::Integer(bytes)), + ] +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_queue.rs b/crates/sqlbuild-recall/src/writer/tests/test_queue.rs index 745ecf0900..d09667dd72 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_queue.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_queue.rs @@ -1,8 +1,8 @@ use crate::records::models::{OutputStream, RecallValue}; use crate::writer::models::{DeliveryStream, Priority}; -use crate::writer::tests::helpers::{chunked, output_drops, pushed}; +use crate::writer::tests::helpers::{capped, chunk_facts, chunked, output_drops, pushed}; use crate::writer::tests::test_types::{ - ChunkingTestCase, DisplacementTestCase, OutputDropTestCase, + CaptureLimitTestCase, ChunkingTestCase, DisplacementTestCase, OutputDropTestCase, }; #[test] @@ -113,3 +113,39 @@ fn given_a_full_queue_when_output_is_dropped_then_each_drop_is_counted_once() { ); } } + +#[test] +fn given_output_past_the_capture_limit_when_appended_then_each_skipped_run_is_one_filtered_sequence() + { + let test_cases = [ + CaptureLimitTestCase { + description: "output within the limit is recorded whole", + limit: 10, + writes: vec![(OutputStream::Stdout, 4), (OutputStream::Stdout, 6)], + expected_outcome: (vec![chunk_facts(0, "stdout", 10)], 0), + }, + CaptureLimitTestCase { + description: "each run past a stream's limit leaves one filtered sequence", + limit: 10, + writes: vec![ + (OutputStream::Stdout, 25), + (OutputStream::Stdout, 3), + (OutputStream::Stderr, 5), + (OutputStream::Stdout, 5), + ], + expected_outcome: ( + vec![chunk_facts(0, "stdout", 10), chunk_facts(2, "stderr", 5)], + 2, + ), + }, + ]; + + for test_case in test_cases { + assert_eq!( + capped(test_case.limit, &test_case.writes), + test_case.expected_outcome, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_session.rs b/crates/sqlbuild-recall/src/writer/tests/test_session.rs index f523e5b4cc..4bb2d460aa 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_session.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_session.rs @@ -43,7 +43,6 @@ fn given_a_session_when_closed_then_every_record_and_the_delivery_rows_are_writt RecallValue::Text("completed".to_owned()), RecallValue::Integer(1), RecallValue::Integer(1), - RecallValue::Integer(1), RecallValue::Integer(5), RecallValue::Text("build".to_owned()), ]], diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs index 25687bdd22..34544bbc49 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -69,3 +69,10 @@ pub(crate) struct OutputDropTestCase { pub(crate) chunks: usize, pub(crate) expected_drops: (u64, Vec>), } + +pub(crate) struct CaptureLimitTestCase { + pub(crate) description: &'static str, + pub(crate) limit: usize, + pub(crate) writes: Vec<(OutputStream, usize)>, + pub(crate) expected_outcome: (Vec<[Option; 3]>, u64), +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index eadd5d8b99..80f2e5727f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -28,7 +28,8 @@ For every recorded invocation, Recall keeps: retries, and audit results; - the SQL text of every executed statement, stored once per distinct statement and shared across invocations; -- its stdout and stderr, byte for byte, in the order they were written; +- its stdout and stderr, byte for byte, in the order they were written, up to 1 MiB of each + stream; later output is not recorded and shows as a `filtered` gap in `output_invocations`; - framework diagnostics at or above `diagnostics_min_severity`. Writes happen on a background thread and never fail the command. A command that cannot record all diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index a7a07ff7bc..69cb980a8b 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -124,3 +124,10 @@ class CorruptStoreRaceTestCase: expected_exit_codes: list[int] expected_moved: int expected_recorded: list[list[object]] + + +@dataclass(frozen=True) +class CaptureLimitTestCase: + description: str + stdout_bytes: int + expected_recorded: list[list[object]] diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index 5e7d3ad1f1..27bc574c26 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -52,6 +52,8 @@ " sys.stdout.flush()\n" ' time.sleep(float(os.environ.get("ORDERS_TASK_SLEEP_SECONDS", "0")))\n' ' _write_concurrently(int(os.environ.get("ORDERS_TASK_THREAD_LINES", "0")))\n' + ' sys.stdout.write("y" * int(os.environ.get("ORDERS_TASK_STDOUT_BYTES", "0")))\n' + " sys.stdout.flush()\n" " return ctx.result()\n\n\n" "def _write_concurrently(lines):\n" ' """Two threads write numbered lines; the number gives the true global order."""\n\n' diff --git a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py index 2af9d5b27b..1cfd125d4f 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py @@ -8,6 +8,7 @@ from tests.integration.src.sqlbuild.cli.recall._test_types import ( AbandonedRunTestCase, + CaptureLimitTestCase, CombinedOutputTestCase, ConcurrentBuildsTestCase, ConcurrentWritersTestCase, @@ -693,3 +694,34 @@ def test_given_a_corrupt_store_when_builds_race_then_one_moves_it_and_both_are_r assert exit_codes == test_case.expected_exit_codes assert len(moved) == test_case.expected_moved assert recorded == test_case.expected_recorded + + +@pytest.mark.parametrize( + "test_case", + [ + CaptureLimitTestCase( + description="stdout past the capture limit is a filtered gap; stderr stays whole", + stdout_bytes=3 * 1024 * 1024, + expected_recorded=[[1024 * 1024, 1, "filtered", 0]], + ), + ], + ids=lambda case: case.description, +) +def test_given_output_past_the_capture_limit_when_recorded_then_the_rest_is_a_filtered_gap( + test_case: CaptureLimitTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + + built: subprocess.CompletedProcess[bytes] = run( + project_dir, "build", ORDERS_TASK_STDOUT_BYTES=str(test_case.stdout_bytes) + ) + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT o.stdout_bytes, instr(CAST(t.stderr AS TEXT), 'from the task') > 0, " + "o.gap_reason, o.complete " + "FROM output_invocations o JOIN output_transcripts t USING (invocation_id)", + ) + + assert built.returncode == 0, built.stderr.decode(errors="replace") + assert len(built.stdout) > test_case.stdout_bytes + assert recorded == test_case.expected_recorded diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 189a485b1c..f7d69a7a13 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -16,7 +16,8 @@ For every recorded invocation, Recall keeps: retries, and audit results; - the SQL text of every executed statement, stored once per distinct statement and shared across invocations; -- its stdout and stderr, byte for byte, in the order they were written; +- its stdout and stderr, byte for byte, in the order they were written, up to 1 MiB of each + stream; later output is not recorded and shows as a `filtered` gap in `output_invocations`; - framework diagnostics at or above `diagnostics_min_severity`. Writes happen on a background thread and never fail the command. A command that cannot record all From 5c226431d301be935cc8362b6235c1680653e705 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:48:13 +0100 Subject: [PATCH 15/19] feat: keep the first and last half of each output stream under [recall] max_output_bytes max_output_bytes (also under targets..recall) bounds the bytes of each output stream Recall records: the first half live and the last half from a ring flushed at close, each under sequences reserved in write order. The local SQLite store defaults to 2 MiB and the default is chosen per store; 0 records everything and invalid values are configuration errors. The left-out middle is one gap row with its byte count: output_invocations reports gap_reason 'truncated' and truncated_bytes, and sqb recall output prints a note in its place. Writers no longer checkpoint as they commit or close: each session runs a passive checkpoint when it starts, beside the command, so exits do not wait on WAL fsyncs. --- .../src/recall/_helpers/values.rs | 8 + .../sqlbuild-config/src/recall/constants.rs | 3 +- .../src/recall/main/recall_section.rs | 10 +- crates/sqlbuild-config/src/recall/models.rs | 2 + .../src/recall/tests/test_recall_section.rs | 22 +++ .../src/bindings/_helpers/recall/rendering.rs | 4 +- .../src/bindings/_helpers/recall/session.rs | 1 + .../sqlbuild-recall/src/records/constants.rs | 4 + .../src/records/main/output_row.rs | 16 +- crates/sqlbuild-recall/src/records/models.rs | 10 ++ .../src/records/tests/test_records.rs | 1 + .../src/schema/_helpers/clickhouse_views.rs | 15 +- .../src/schema/_helpers/sqlite_views.rs | 12 +- .../sqlbuild-recall/src/schema/constants.rs | 4 +- .../src/settings/_helpers/layers.rs | 1 + .../sqlbuild-recall/src/settings/constants.rs | 2 + .../settings/main/resolve_recall_settings.rs | 6 + crates/sqlbuild-recall/src/settings/models.rs | 15 ++ .../src/settings/tests/helpers.rs | 1 + .../src/settings/tests/test_resolve.rs | 27 ++++ .../sqlbuild-recall/src/store/_helpers/mod.rs | 1 + .../src/store/_helpers/truncation.rs | 89 +++++++++++ crates/sqlbuild-recall/src/store/constants.rs | 8 + .../src/store/main/checkpoint_sqlite_store.rs | 15 ++ crates/sqlbuild-recall/src/store/main/mod.rs | 1 + .../src/store/main/open_sqlite_store.rs | 8 +- .../src/store/main/read_output_transcript.rs | 17 ++- crates/sqlbuild-recall/src/store/models.rs | 9 ++ .../src/store/tests/helpers.rs | 1 + .../src/store/tests/test_store.rs | 31 ++++ .../src/store/tests/test_types.rs | 7 + .../src/writer/_helpers/delivery.rs | 2 + .../src/writer/_helpers/mod.rs | 1 + .../src/writer/_helpers/queue.rs | 134 ++++++++++++----- .../src/writer/_helpers/session.rs | 2 +- .../src/writer/_helpers/tail.rs | 139 ++++++++++++++++++ .../sqlbuild-recall/src/writer/constants.rs | 2 - .../src/writer/main/start_recall_session.rs | 3 +- crates/sqlbuild-recall/src/writer/models.rs | 2 + .../src/writer/tests/helpers.rs | 50 +++++-- .../src/writer/tests/test_queue.rs | 67 ++++++--- .../src/writer/tests/test_types.rs | 6 +- .../sqlbuild/references/docs/cli/recall.md | 4 +- .../docs/concepts/observability/recall.md | 26 +++- .../src/sqlbuild/cli/recall/_test_types.py | 18 ++- .../src/sqlbuild/cli/recall/helpers.py | 1 + .../sqlbuild/cli/recall/test_local_recall.py | 118 +++++++++++++-- website/src/content/docs/docs/cli/recall.mdx | 4 +- .../docs/concepts/observability/recall.mdx | 25 +++- 49 files changed, 846 insertions(+), 109 deletions(-) create mode 100644 crates/sqlbuild-recall/src/store/_helpers/truncation.rs create mode 100644 crates/sqlbuild-recall/src/store/main/checkpoint_sqlite_store.rs create mode 100644 crates/sqlbuild-recall/src/writer/_helpers/tail.rs diff --git a/crates/sqlbuild-config/src/recall/_helpers/values.rs b/crates/sqlbuild-config/src/recall/_helpers/values.rs index daf30896a5..07c5c7f842 100644 --- a/crates/sqlbuild-config/src/recall/_helpers/values.rs +++ b/crates/sqlbuild-config/src/recall/_helpers/values.rs @@ -39,6 +39,14 @@ pub(crate) fn timeout_ms(text: &str) -> Option { (millis <= MAX_SHUTDOWN_TIMEOUT_MS).then_some(millis) } +/// A non-negative TOML integer of bytes; `None` for anything else. +pub(crate) fn output_bytes(value: &ConfigValue) -> Option { + match value { + ConfigValue::Integer(bytes) if *bytes >= 0 => Some(bytes.unsigned_abs()), + _ => None, + } +} + /// A trimmed non-empty string, or the message for `label.key`. pub(crate) fn non_empty<'a>( value: &'a ConfigValue, diff --git a/crates/sqlbuild-config/src/recall/constants.rs b/crates/sqlbuild-config/src/recall/constants.rs index 205579f213..3d61cf1666 100644 --- a/crates/sqlbuild-config/src/recall/constants.rs +++ b/crates/sqlbuild-config/src/recall/constants.rs @@ -2,10 +2,11 @@ pub const RECALL_SECTION: &str = "recall"; /// Keys `[recall]` and `targets..recall` accept, sorted. -pub const RECALL_KEYS: [&str; 5] = [ +pub const RECALL_KEYS: [&str; 6] = [ "connection", "diagnostics_min_severity", "enabled", + "max_output_bytes", "retention", "shutdown_timeout", ]; diff --git a/crates/sqlbuild-config/src/recall/main/recall_section.rs b/crates/sqlbuild-config/src/recall/main/recall_section.rs index 212632400c..536f3d4ac9 100644 --- a/crates/sqlbuild-config/src/recall/main/recall_section.rs +++ b/crates/sqlbuild-config/src/recall/main/recall_section.rs @@ -1,7 +1,7 @@ //! `recall_section`: validate one Recall settings table. use crate::models::ConfigValue; -use crate::recall::_helpers::values::{non_empty, retention_days, timeout_ms}; +use crate::recall::_helpers::values::{non_empty, output_bytes, retention_days, timeout_ms}; use crate::recall::constants::{MAX_RETENTION_DAYS, RECALL_KEYS, RECALL_SEVERITIES}; use crate::recall::models::RecallSection; @@ -39,6 +39,14 @@ pub fn recall_section(value: &ConfigValue, label: &str) -> Result { + section.max_output_bytes = Some(output_bytes(value).ok_or_else(|| { + format!( + "{label}.max_output_bytes must be a whole number of bytes, such as \ + 2097152, or 0 to record all output" + ) + })?); + } "diagnostics_min_severity" => { let text = non_empty(value, label, key)?; if !RECALL_SEVERITIES.contains(&text) { diff --git a/crates/sqlbuild-config/src/recall/models.rs b/crates/sqlbuild-config/src/recall/models.rs index 4edf3e6900..6c9e735663 100644 --- a/crates/sqlbuild-config/src/recall/models.rs +++ b/crates/sqlbuild-config/src/recall/models.rs @@ -8,4 +8,6 @@ pub struct RecallSection { pub retention_days: Option, pub shutdown_timeout_ms: Option, pub diagnostics_min_severity: Option, + /// Bytes of each output stream to record; `Some(0)` records everything. + pub max_output_bytes: Option, } diff --git a/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs b/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs index 2197ec212c..b8c144918d 100644 --- a/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs +++ b/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs @@ -15,6 +15,7 @@ fn given_valid_sections_when_validated_then_settings_are_typed() { ("retention", text("720h")), ("shutdown_timeout", text("1500ms")), ("diagnostics_min_severity", text("debug")), + ("max_output_bytes", ConfigValue::Integer(4_194_304)), ]), expected_section: RecallSection { enabled: Some(false), @@ -22,6 +23,15 @@ fn given_valid_sections_when_validated_then_settings_are_typed() { retention_days: Some(30), shutdown_timeout_ms: Some(1500), diagnostics_min_severity: Some("debug".to_owned()), + max_output_bytes: Some(4_194_304), + }, + }, + ValidSectionTestCase { + description: "zero output bytes means record everything", + value: table(&[("max_output_bytes", ConfigValue::Integer(0))]), + expected_section: RecallSection { + max_output_bytes: Some(0), + ..RecallSection::default() }, }, ValidSectionTestCase { @@ -44,6 +54,18 @@ fn given_valid_sections_when_validated_then_settings_are_typed() { #[test] fn given_invalid_sections_when_validated_then_the_message_names_the_key() { let test_cases = [ + InvalidSectionTestCase { + description: "output bytes must not be negative", + value: table(&[("max_output_bytes", ConfigValue::Integer(-1))]), + label: "recall", + expected_message_prefix: "recall.max_output_bytes must be", + }, + InvalidSectionTestCase { + description: "output bytes must be an integer, not a size string", + value: table(&[("max_output_bytes", text("2MiB"))]), + label: "recall", + expected_message_prefix: "recall.max_output_bytes must be", + }, InvalidSectionTestCase { description: "unknown keys are listed", value: table(&[("mode", text("x"))]), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs index 28e251b037..40b51110f2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs @@ -159,7 +159,9 @@ fn relation_note(name: &str) -> &'static str { } "statements" => "one row per statement; outcome no_terminal means no terminal fact arrived", "audit_results" => "one row per evaluated audit", - "output_invocations" => "output completeness per invocation: gaps, drops, filtering, flush", + "output_invocations" => { + "output completeness per invocation: gaps, drops, filtering, truncation, flush" + } "output_tail" => "the last 200 output chunks of each invocation as text", "output_transcripts" => "exact stdout, stderr and combined bytes per invocation", "events" => "every lifecycle fact", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index e5c5d7e75a..7c574a89c4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -322,6 +322,7 @@ fn started_session( retention_days: settings.retention_days, shutdown_timeout: Duration::from_millis(settings.shutdown_timeout_ms), diagnostics_min_severity: settings.diagnostics_min_severity.clone(), + max_output_bytes: settings.max_output_bytes, heartbeat_interval: heartbeat_interval(), setup_budget: Duration::ZERO, maintenance: true, diff --git a/crates/sqlbuild-recall/src/records/constants.rs b/crates/sqlbuild-recall/src/records/constants.rs index da760bd9a0..733d77ae5f 100644 --- a/crates/sqlbuild-recall/src/records/constants.rs +++ b/crates/sqlbuild-recall/src/records/constants.rs @@ -50,3 +50,7 @@ pub(crate) const MARCH_BASED_JANUARY_INDEX: i64 = 10; pub const DROPPED_BEFORE_COLUMN: &str = "dropped_before"; /// Initial capacity of the buffer a row's content hash is computed over. pub(crate) const CONTENT_HASH_BUFFER_BYTES: usize = 1_024; +/// The output column holding the bytes a truncation gap replaces. +pub const OMITTED_BYTES_COLUMN: &str = "omitted_bytes"; +/// The output column holding the sequences a truncation gap accounts for. +pub const OMITTED_SEQUENCES_COLUMN: &str = "omitted_sequences"; diff --git a/crates/sqlbuild-recall/src/records/main/output_row.rs b/crates/sqlbuild-recall/src/records/main/output_row.rs index 0c4dd22bba..de74c9a480 100644 --- a/crates/sqlbuild-recall/src/records/main/output_row.rs +++ b/crates/sqlbuild-recall/src/records/main/output_row.rs @@ -2,7 +2,9 @@ use crate::records::_helpers::ids::output_record_id; use crate::records::_helpers::rows::{finish_row, integer, text}; -use crate::records::constants::DROPPED_BEFORE_COLUMN; +use crate::records::constants::{ + DROPPED_BEFORE_COLUMN, OMITTED_BYTES_COLUMN, OMITTED_SEQUENCES_COLUMN, +}; use crate::records::models::{OutputChunk, RecallRow, RecallValue, RowEnvelope}; use crate::schema::models::Table; @@ -25,6 +27,18 @@ pub fn output_row(envelope: &RowEnvelope, chunk: &OutputChunk) -> RecallRow { ("byte_count", integer(chunk.data.len() as u64)), ("priority", text("output")), (DROPPED_BEFORE_COLUMN, integer(chunk.dropped_before)), + ( + OMITTED_BYTES_COLUMN, + chunk + .omitted + .map_or(RecallValue::Null, |omitted| integer(omitted.bytes)), + ), + ( + OMITTED_SEQUENCES_COLUMN, + chunk + .omitted + .map_or(RecallValue::Null, |omitted| integer(omitted.sequences)), + ), ], ) } diff --git a/crates/sqlbuild-recall/src/records/models.rs b/crates/sqlbuild-recall/src/records/models.rs index 06e2571a9b..40146f65be 100644 --- a/crates/sqlbuild-recall/src/records/models.rs +++ b/crates/sqlbuild-recall/src/records/models.rs @@ -86,6 +86,16 @@ pub struct OutputChunk { pub byte_offset: u64, /// Output records the queue dropped just before this one. pub dropped_before: u64, + /// On the empty chunk standing for output past the stream's limit: the bytes and sequences it replaces. + pub omitted: Option, +} + +/// Output of one stream that was not recorded because it exceeded `max_output_bytes`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct OmittedOutput { + pub bytes: u64, + /// Sequences this gap accounts for, its own included. + pub sequences: u64, } /// The context of one invocation, written once when it starts. diff --git a/crates/sqlbuild-recall/src/records/tests/test_records.rs b/crates/sqlbuild-recall/src/records/tests/test_records.rs index 03371d895b..0b8f7fb19c 100644 --- a/crates/sqlbuild-recall/src/records/tests/test_records.rs +++ b/crates/sqlbuild-recall/src/records/tests/test_records.rs @@ -156,6 +156,7 @@ fn given_records_when_rowed_then_columns_are_promoted() { data: vec![0, 255, 10], byte_offset: 12, dropped_before: 1, + omitted: None, }, ), expected_columns: vec![ diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs index 88be727828..d84e3e1713 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -227,7 +227,9 @@ received AS ( sumIf(byte_count, stream = 'stdout') AS stdout_bytes, sumIf(byte_count, stream = 'stderr') AS stderr_bytes, max(sequence) AS max_sequence, - sum(dropped_before) AS queue_drops + sum(dropped_before) AS queue_drops, + sum(coalesce(omitted_sequences, 1)) AS spans, + sum(coalesce(omitted_bytes, 0)) AS truncated_bytes FROM {output} GROUP BY invocation_id ), delivery AS (SELECT * FROM {delivery} WHERE stream = 'output'), @@ -246,9 +248,10 @@ counted AS ( coalesce(received.bytes, 0) AS bytes, coalesce(received.stdout_bytes, 0) AS stdout_bytes, coalesce(received.stderr_bytes, 0) AS stderr_bytes, + coalesce(received.truncated_bytes, 0) AS truncated_bytes, received.max_sequence AS max_sequence, greatest( - coalesce(received.max_sequence + 1 - received.chunks, 0), + coalesce(toInt64(received.max_sequence) + 1 - toInt64(received.spans), 0), coalesce(toInt64(delivery.accepted), 0) - coalesce(toInt64(received.chunks), 0) ) AS missing_sequences, greatest(coalesce(received.queue_drops, 0), coalesce(delivery.dropped, 0)) AS reported_drops, @@ -266,16 +269,18 @@ counted AS ( ) SELECT invocation_id, invocation_started_at, chunks, bytes, stdout_bytes, stderr_bytes, - max_sequence, missing_sequences, reported_drops, filtered, failed, filter_active, - flush_complete, + truncated_bytes, max_sequence, missing_sequences, reported_drops, filtered, failed, + filter_active, flush_complete, multiIf( + missing_sequences = 0 AND truncated_bytes > 0, 'truncated', missing_sequences = 0, NULL, filtered >= missing_sequences, 'filtered', filtered + reported_drops + failed >= missing_sequences, 'dropped', 'unexplained' ) AS gap_reason, toUInt8(delivery_reported AND flush_complete = 1 AND missing_sequences = 0 - AND reported_drops = 0 AND filtered = 0 AND failed = 0 AND chunks >= delivered) + AND reported_drops = 0 AND filtered = 0 AND failed = 0 AND truncated_bytes = 0 + AND chunks >= delivered) AS complete FROM counted" ) diff --git a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs index 2badca5552..bc7a1922d2 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs @@ -125,7 +125,9 @@ const OUTPUT_INVOCATIONS: &str = "WITH received AS ( SUM(CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END) AS stdout_bytes, SUM(CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END) AS stderr_bytes, MAX(sequence) AS max_sequence, - SUM(dropped_before) AS queue_drops + SUM(dropped_before) AS queue_drops, + SUM(COALESCE(omitted_sequences, 1)) AS spans, + SUM(COALESCE(omitted_bytes, 0)) AS truncated_bytes FROM recall_output GROUP BY invocation_id ), @@ -144,9 +146,10 @@ counted AS ( COALESCE(received.bytes, 0) AS bytes, COALESCE(received.stdout_bytes, 0) AS stdout_bytes, COALESCE(received.stderr_bytes, 0) AS stderr_bytes, + COALESCE(received.truncated_bytes, 0) AS truncated_bytes, received.max_sequence, MAX( - COALESCE(received.max_sequence + 1 - received.chunks, 0), + COALESCE(received.max_sequence + 1 - received.spans, 0), COALESCE(delivery.accepted, 0) - COALESCE(received.chunks, 0) ) AS missing_sequences, MAX(COALESCE(received.queue_drops, 0), COALESCE(delivery.dropped, 0)) AS reported_drops, @@ -168,6 +171,7 @@ SELECT bytes, stdout_bytes, stderr_bytes, + truncated_bytes, max_sequence, missing_sequences, reported_drops, @@ -176,6 +180,7 @@ SELECT filter_active, flush_complete, CASE + WHEN missing_sequences = 0 AND truncated_bytes > 0 THEN 'truncated' WHEN missing_sequences = 0 THEN NULL WHEN filtered >= missing_sequences THEN 'filtered' WHEN filtered + reported_drops + failed >= missing_sequences THEN 'dropped' @@ -183,7 +188,8 @@ SELECT END AS gap_reason, CASE WHEN delivery_reported AND flush_complete = 1 AND missing_sequences = 0 - AND reported_drops = 0 AND filtered = 0 AND failed = 0 AND chunks >= delivered + AND reported_drops = 0 AND filtered = 0 AND failed = 0 AND truncated_bytes = 0 + AND chunks >= delivered THEN 1 ELSE 0 END AS complete FROM counted"; diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index 8f801f3a75..0da7cf7f80 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -57,7 +57,7 @@ const INVOCATION_COLUMNS: [ColumnSpec; 19] = [ ColumnSpec::optional("argv_digest", ColumnType::Text), ]; -const OUTPUT_COLUMNS: [ColumnSpec; 9] = [ +const OUTPUT_COLUMNS: [ColumnSpec; 11] = [ ColumnSpec::required("record_id", ColumnType::Text), ColumnSpec::required("sequence", ColumnType::Integer), ColumnSpec::required("stream", ColumnType::Text), @@ -67,6 +67,8 @@ const OUTPUT_COLUMNS: [ColumnSpec; 9] = [ ColumnSpec::required("byte_count", ColumnType::Integer), ColumnSpec::optional("priority", ColumnType::Text), ColumnSpec::required("dropped_before", ColumnType::Integer), + ColumnSpec::optional("omitted_bytes", ColumnType::Integer), + ColumnSpec::optional("omitted_sequences", ColumnType::Integer), ]; const EVENT_COLUMNS: [ColumnSpec; 29] = [ diff --git a/crates/sqlbuild-recall/src/settings/_helpers/layers.rs b/crates/sqlbuild-recall/src/settings/_helpers/layers.rs index 381ab4de55..5ea24d617e 100644 --- a/crates/sqlbuild-recall/src/settings/_helpers/layers.rs +++ b/crates/sqlbuild-recall/src/settings/_helpers/layers.rs @@ -34,6 +34,7 @@ pub(crate) fn overlay(base: RecallSection, layer: RecallSection) -> RecallSectio diagnostics_min_severity: layer .diagnostics_min_severity .or(base.diagnostics_min_severity), + max_output_bytes: layer.max_output_bytes.or(base.max_output_bytes), } } diff --git a/crates/sqlbuild-recall/src/settings/constants.rs b/crates/sqlbuild-recall/src/settings/constants.rs index 516b902392..76d7d584e6 100644 --- a/crates/sqlbuild-recall/src/settings/constants.rs +++ b/crates/sqlbuild-recall/src/settings/constants.rs @@ -12,3 +12,5 @@ pub const SQLITE_ADAPTER: &str = "sqlite"; pub const SQLITE_PATH_KEY: &str = "path"; pub const PROJECT_FILE_NAME: &str = "sqlbuild_project.toml"; pub const LOCAL_FILE_NAME: &str = "sqlbuild_local.toml"; +/// Bytes of each output stream the local SQLite store keeps per invocation: the first and last MiB. +pub const DEFAULT_SQLITE_MAX_OUTPUT_BYTES: u64 = 2 * 1024 * 1024; diff --git a/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs b/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs index 010834ad1d..a85925af60 100644 --- a/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs +++ b/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs @@ -48,6 +48,11 @@ pub fn resolve_recall_settings(request: &SettingsRequest) -> Result = match effective.max_output_bytes { + Some(0) => None, + Some(bytes) => Some(bytes), + None => store.default_max_output_bytes(), + }; Ok(RecallSettings { enabled: effective.enabled.unwrap_or(true), target: target.clone(), @@ -60,6 +65,7 @@ pub fn resolve_recall_settings(request: &SettingsRequest) -> Result Option { + match self { + Self::Sqlite { .. } => Some(DEFAULT_SQLITE_MAX_OUTPUT_BYTES), + Self::Unsupported { .. } => None, + } + } +} + /// The effective settings: target, store and policies. #[derive(Clone, Debug, PartialEq, Eq)] pub struct RecallSettings { @@ -23,6 +36,8 @@ pub struct RecallSettings { pub retention_days: u32, pub shutdown_timeout_ms: u64, pub diagnostics_min_severity: String, + /// Bytes of each output stream recorded, half from its start and half from its end; `None` records everything. + pub max_output_bytes: Option, pub project_name: Option, pub project_adapter: Option, pub destination_database: Option, diff --git a/crates/sqlbuild-recall/src/settings/tests/helpers.rs b/crates/sqlbuild-recall/src/settings/tests/helpers.rs index b9dba8056f..39ef48f7e4 100644 --- a/crates/sqlbuild-recall/src/settings/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/settings/tests/helpers.rs @@ -34,6 +34,7 @@ pub(crate) fn defaults() -> RecallSettings { retention_days: 30, shutdown_timeout_ms: 3000, diagnostics_min_severity: "info".to_owned(), + max_output_bytes: Some(2 * 1024 * 1024), project_name: Some("shop".to_owned()), project_adapter: Some("duckdb".to_owned()), destination_database: None, diff --git a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs index fc8f5c931e..cd7a225b7b 100644 --- a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs +++ b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs @@ -23,6 +23,12 @@ const LAYERED_PROJECT: &[(&str, &str)] = &[ ), ]; +const OUTPUT_LIMIT_PROJECT: &[(&str, &str)] = &[( + "sqlbuild_project.toml", + "name = \"shop\"\nadapter = \"duckdb\"\n[recall]\nmax_output_bytes = 0\n\ + [targets.dev]\n[targets.dev.recall]\nmax_output_bytes = 8388608\n[targets.prod]\n", +)]; + #[test] fn given_project_layers_when_resolved_then_target_over_local_over_project_over_defaults() { let test_cases = [ @@ -65,6 +71,27 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d connection: Some("central".to_owned()), retention_days: 10, shutdown_timeout_ms: 1000, + max_output_bytes: None, + ..defaults() + }, + }, + ResolveTestCase { + description: "a target raises the output limit over the project's unlimited one", + files: OUTPUT_LIMIT_PROJECT, + target: Some("dev"), + expected_settings: RecallSettings { + target: Some("dev".to_owned()), + max_output_bytes: Some(8_388_608), + ..defaults() + }, + }, + ResolveTestCase { + description: "zero records all output", + files: OUTPUT_LIMIT_PROJECT, + target: Some("prod"), + expected_settings: RecallSettings { + target: Some("prod".to_owned()), + max_output_bytes: None, ..defaults() }, }, diff --git a/crates/sqlbuild-recall/src/store/_helpers/mod.rs b/crates/sqlbuild-recall/src/store/_helpers/mod.rs index c29e92d7b3..c1f9469a7f 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/mod.rs @@ -3,3 +3,4 @@ pub(crate) mod quarantine; pub(crate) mod rows; pub(crate) mod schema; pub(crate) mod sqlite; +pub(crate) mod truncation; diff --git a/crates/sqlbuild-recall/src/store/_helpers/truncation.rs b/crates/sqlbuild-recall/src/store/_helpers/truncation.rs new file mode 100644 index 0000000000..678b9cde3d --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/truncation.rs @@ -0,0 +1,89 @@ +//! The note `sqb recall output` prints where output past `max_output_bytes` was left out. + +use crate::store::constants::{BINARY_UNIT_STEP, BINARY_UNITS, DECIMAL_UNIT_STEP, DECIMAL_UNITS}; +use crate::store::models::StoredChunk; + +/// `23.6 MB`: decimal units with one decimal, whole bytes below a kilobyte. +fn decimal_size(bytes: u64) -> String { + let mut value = bytes as f64; + let mut unit: Option<&str> = None; + for candidate in DECIMAL_UNITS { + if value < DECIMAL_UNIT_STEP { + break; + } + value /= DECIMAL_UNIT_STEP; + unit = Some(candidate); + } + unit.map_or_else( + || format!("{bytes} bytes"), + |unit| format!("{value:.1} {unit}"), + ) +} + +/// `1 MiB`: the largest binary unit dividing `bytes` exactly. +fn binary_size(bytes: u64) -> String { + let mut value = bytes; + let mut unit: Option<&str> = None; + for candidate in BINARY_UNITS { + if value == 0 || !value.is_multiple_of(BINARY_UNIT_STEP) { + break; + } + value /= BINARY_UNIT_STEP; + unit = Some(candidate); + } + unit.map_or_else( + || format!("{bytes} bytes"), + |unit| format!("{value} {unit}"), + ) +} + +/// One line naming the bytes of `stream` left out between the kept `head` and `tail`. +pub(crate) fn truncation_note(stream: &str, omitted: u64, (head, tail): (u64, u64)) -> String { + let kept: String = if head == tail { + format!("kept the first and last {}", binary_size(head)) + } else { + format!( + "kept the first {} and the last {}", + binary_size(head), + binary_size(tail) + ) + }; + format!( + "\u{2026} {} of {stream} not recorded ({kept}). Raise [recall] max_output_bytes to keep \ + more.\n", + decimal_size(omitted) + ) +} + +/// Bytes of `stream` in `chunks`. +fn stream_bytes(chunks: &[StoredChunk], stream: &str) -> u64 { + chunks + .iter() + .filter(|chunk| chunk.stream == stream) + .map(|chunk| chunk.data.len() as u64) + .sum() +} + +/// The chunks' bytes in order, with a note line where output was left out. +pub(crate) fn with_truncation_notes(chunks: &[StoredChunk]) -> Vec { + let mut data: Vec = Vec::with_capacity(chunks.iter().map(|chunk| chunk.data.len()).sum()); + for (position, chunk) in chunks.iter().enumerate() { + data.extend_from_slice(&chunk.data); + let omitted: u64 = chunk + .omitted_bytes + .filter(|bytes| *bytes > 0) + .map_or(0, i64::unsigned_abs); + if omitted == 0 { + continue; + } + if data.last().is_some_and(|byte| *byte != b'\n') { + data.push(b'\n'); + } + let kept = ( + stream_bytes(&chunks[..position], &chunk.stream), + stream_bytes(&chunks[position + 1..], &chunk.stream), + ); + data.extend_from_slice(truncation_note(&chunk.stream, omitted, kept).as_bytes()); + } + data +} diff --git a/crates/sqlbuild-recall/src/store/constants.rs b/crates/sqlbuild-recall/src/store/constants.rs index 4ecb02d394..f7ee4fb4ae 100644 --- a/crates/sqlbuild-recall/src/store/constants.rs +++ b/crates/sqlbuild-recall/src/store/constants.rs @@ -22,3 +22,11 @@ pub const COMPANION_SUFFIXES: [&str; 2] = ["-wal", "-shm"]; pub const QUARANTINE_LOCK_SUFFIX: &str = ".lock"; /// Pause between attempts to take the quarantine lock. pub const QUARANTINE_LOCK_RETRY_MS: u64 = 10; +/// The step between decimal size units. +pub const DECIMAL_UNIT_STEP: f64 = 1000.0; +/// The step between binary size units. +pub const BINARY_UNIT_STEP: u64 = 1024; +/// Decimal size units above bytes, for omitted output. +pub const DECIMAL_UNITS: [&str; 4] = ["kB", "MB", "GB", "TB"]; +/// Binary size units above bytes, for kept output. +pub const BINARY_UNITS: [&str; 4] = ["KiB", "MiB", "GiB", "TiB"]; diff --git a/crates/sqlbuild-recall/src/store/main/checkpoint_sqlite_store.rs b/crates/sqlbuild-recall/src/store/main/checkpoint_sqlite_store.rs new file mode 100644 index 0000000000..d4541ba3bb --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/checkpoint_sqlite_store.rs @@ -0,0 +1,15 @@ +//! `checkpoint_sqlite_store`: copy the WAL into the store without waiting on anyone. + +use crate::store::_helpers::sqlite::checked; +use crate::store::errors::StoreError; +use crate::store::models::SqliteStore; + +/// A passive checkpoint: writers never checkpoint as they commit or close, so each session runs one when it starts, beside the command rather than after it. +pub fn checkpoint_sqlite_store(store: &SqliteStore) -> Result<(), StoreError> { + checked( + store + .connection + .query_row("PRAGMA wal_checkpoint(PASSIVE)", [], |_| Ok(())), + "checkpoint Recall WAL", + ) +} diff --git a/crates/sqlbuild-recall/src/store/main/mod.rs b/crates/sqlbuild-recall/src/store/main/mod.rs index d86f08ea49..584ad7dfc5 100644 --- a/crates/sqlbuild-recall/src/store/main/mod.rs +++ b/crates/sqlbuild-recall/src/store/main/mod.rs @@ -1,3 +1,4 @@ +pub mod checkpoint_sqlite_store; pub mod describe_store; pub mod ensure_sqlite_schema; pub mod insert_rows; diff --git a/crates/sqlbuild-recall/src/store/main/open_sqlite_store.rs b/crates/sqlbuild-recall/src/store/main/open_sqlite_store.rs index e2a102e774..9208bf66cd 100644 --- a/crates/sqlbuild-recall/src/store/main/open_sqlite_store.rs +++ b/crates/sqlbuild-recall/src/store/main/open_sqlite_store.rs @@ -3,6 +3,7 @@ use std::path::Path; use std::time::Duration; +use rusqlite::config::DbConfig; use rusqlite::{Connection, OpenFlags}; use crate::schema::constants::MINIMUM_SQLITE_VERSION; @@ -107,10 +108,15 @@ pub fn open_sqlite_store(path: &Path, mode: OpenMode) -> Result Result { let mut statement = checked( store.connection.prepare( - "SELECT data FROM recall_output WHERE invocation_id = ?1 \ + "SELECT stream, data, omitted_bytes FROM recall_output WHERE invocation_id = ?1 \ AND (?2 IS NULL OR stream = ?2) ORDER BY sequence", ), "read Recall output", )?; - let chunks: Vec> = checked( + let chunks: Vec = checked( statement .query_map( rusqlite::params![invocation_id, stream.map(OutputStream::as_str)], - |row| row.get(0), + |row| { + Ok(StoredChunk { + stream: row.get(0)?, + data: row.get(1)?, + omitted_bytes: row.get(2)?, + }) + }, ) .and_then(Iterator::collect), "read Recall output", )?; let mut transcript = OutputTranscript { invocation_id: invocation_id.to_owned(), - data: chunks.concat(), + data: with_truncation_notes(&chunks), ..OutputTranscript::default() }; transcript.status = checked( diff --git a/crates/sqlbuild-recall/src/store/models.rs b/crates/sqlbuild-recall/src/store/models.rs index f501bea5fc..8440f7aee7 100644 --- a/crates/sqlbuild-recall/src/store/models.rs +++ b/crates/sqlbuild-recall/src/store/models.rs @@ -114,6 +114,7 @@ pub struct DescribedRelation { #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct OutputTranscript { pub invocation_id: String, + /// The recorded bytes in order, with a note line where output past `max_output_bytes` was left out. pub data: Vec, pub status: Option, pub missing_sequences: i64, @@ -123,3 +124,11 @@ pub struct OutputTranscript { pub complete: bool, pub gap_reason: Option, } + +/// One stored output row, as the transcript needs it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct StoredChunk { + pub(crate) stream: String, + pub(crate) data: Vec, + pub(crate) omitted_bytes: Option, +} diff --git a/crates/sqlbuild-recall/src/store/tests/helpers.rs b/crates/sqlbuild-recall/src/store/tests/helpers.rs index 126a992158..24ee0c53c8 100644 --- a/crates/sqlbuild-recall/src/store/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/store/tests/helpers.rs @@ -66,6 +66,7 @@ pub(crate) fn chunk( data: data.to_vec(), byte_offset: 0, dropped_before: 0, + omitted: None, }, ) } diff --git a/crates/sqlbuild-recall/src/store/tests/test_store.rs b/crates/sqlbuild-recall/src/store/tests/test_store.rs index 93783e8114..27f7320f06 100644 --- a/crates/sqlbuild-recall/src/store/tests/test_store.rs +++ b/crates/sqlbuild-recall/src/store/tests/test_store.rs @@ -1,5 +1,6 @@ use crate::records::main::invocation_row::invocation_row; use crate::records::models::{InvocationContext, OutputStream, RecallValue}; +use crate::store::_helpers::truncation::truncation_note; use crate::store::errors::StoreErrorKind; use crate::store::main::describe_store::describe_store; use crate::store::main::query_store::query_store; @@ -16,6 +17,7 @@ use crate::store::tests::helpers::{ use crate::store::tests::test_types::{LockedDirectoryTestCase, QuarantineTestCase}; use crate::store::tests::test_types::{ NewerSchemaTestCase, QueryTestCase, ResolveInvocationTestCase, TranscriptTestCase, + TruncationNoteTestCase, }; #[test] @@ -242,3 +244,32 @@ fn given_a_stale_observation_when_quarantining_then_only_the_observed_file_is_mo ); } } + +#[test] +fn given_truncated_output_when_noted_then_sizes_read_naturally() { + let test_cases = [ + TruncationNoteTestCase { + description: "the default limit keeps equal halves", + omitted: 23_600_000, + kept: (1_048_576, 1_048_576), + expected_note: "\u{2026} 23.6 MB of stdout not recorded (kept the first and last \ + 1 MiB). Raise [recall] max_output_bytes to keep more.\n", + }, + TruncationNoteTestCase { + description: "an odd limit keeps one more byte at the start", + omitted: 999, + kept: (3, 2), + expected_note: "\u{2026} 999 bytes of stdout not recorded (kept the first 3 bytes \ + and the last 2 bytes). Raise [recall] max_output_bytes to keep more.\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + truncation_note("stdout", test_case.omitted, test_case.kept), + test_case.expected_note, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/store/tests/test_types.rs b/crates/sqlbuild-recall/src/store/tests/test_types.rs index fa20bb0528..dffbb076a4 100644 --- a/crates/sqlbuild-recall/src/store/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/store/tests/test_types.rs @@ -60,3 +60,10 @@ pub(crate) struct QuarantineTestCase { /// Whether the move was made, copies moved aside, and whether a SQLite store is at the path. pub(crate) expected_outcome: (bool, usize, bool), } + +pub(crate) struct TruncationNoteTestCase { + pub(crate) description: &'static str, + pub(crate) omitted: u64, + pub(crate) kept: (u64, u64), + pub(crate) expected_note: &'static str, +} diff --git a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs index 2c75159196..8a358137f9 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs @@ -10,6 +10,7 @@ use crate::records::models::RecallRow; use crate::schema::models::Table; use crate::store::constants::{BUSY_TIMEOUT_MS, MAINTENANCE_BUDGET_MS}; use crate::store::errors::{StoreError, StoreErrorKind}; +use crate::store::main::checkpoint_sqlite_store::checkpoint_sqlite_store; use crate::store::main::ensure_sqlite_schema::ensure_sqlite_schema; use crate::store::main::insert_rows::insert_rows; use crate::store::main::maintain_sqlite_store::maintain_sqlite_store; @@ -245,6 +246,7 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { return; } }; + drop(checkpoint_sqlite_store(&store)); let mut next_heartbeat: SystemTime = SystemTime::now(); let mut materializer = Materializer::new(config.envelope.clone()); loop { diff --git a/crates/sqlbuild-recall/src/writer/_helpers/mod.rs b/crates/sqlbuild-recall/src/writer/_helpers/mod.rs index bcd7ab884f..2115f0b0b2 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod delivery; pub(crate) mod materialize; pub(crate) mod queue; pub(crate) mod session; +pub(crate) mod tail; diff --git a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs index 454d564b77..ed710a05bf 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs @@ -13,9 +13,8 @@ use crate::records::models::{ }; use crate::store::models::InsertOutcome; use crate::writer::_helpers::delivery::stream_of; -use crate::writer::constants::{ - OUTPUT_STREAM_CAPTURE_BYTES, QUEUE_BYTE_CAPACITY, QUEUE_RECORD_CAPACITY, -}; +use crate::writer::_helpers::tail::{TailRing, TailSegment}; +use crate::writer::constants::{QUEUE_BYTE_CAPACITY, QUEUE_RECORD_CAPACITY}; use crate::writer::models::{DeliveryStream, Priority}; /// SQL a statement executed, hashed and turned into rows on the delivery thread. @@ -81,11 +80,14 @@ pub(crate) struct WriterState { pending: Option, next_sequence: u64, offsets: [u64; 2], - /// Bytes of each stream accepted for recording, pending or emitted. + /// Bytes of each stream recorded from its start, pending or emitted. captured: [usize; 2], - /// The stream whose output past the capture limit the latest sequence stands for. - skipping: Option, - output_limit: usize, + /// Bytes of each stream recorded from its start; `None` records everything. + head_limit: Option, + /// The end of each stream past its head limit, kept until the session closes. + tails: [TailRing; 2], + /// The stream whose newest tail segment holds the latest sequence, so it may grow. + tail_open: Option, unreported_output_drops: u64, pub(crate) diagnostic_sequence: u64, pub(crate) closing: bool, @@ -128,8 +130,9 @@ impl WriterState { next_sequence: 0, offsets: [0; 2], captured: [0; 2], - skipping: None, - output_limit: OUTPUT_STREAM_CAPTURE_BYTES, + head_limit: None, + tails: Default::default(), + tail_open: None, unreported_output_drops: 0, diagnostic_sequence: 0, closing: false, @@ -142,10 +145,15 @@ impl WriterState { } } - /// The same state recording at most `limit` bytes of each output stream. - #[cfg(test)] - pub(crate) fn with_output_limit(mut self, limit: usize) -> Self { - self.output_limit = limit; + /// The same state recording at most `limit` bytes of each output stream: the first half and the last half; `None` records everything. + pub(crate) fn with_output_limit(mut self, limit: Option) -> Self { + let Some(limit) = limit.map(|limit| usize::try_from(limit).unwrap_or(usize::MAX)) else { + self.head_limit = None; + return self; + }; + let tail: usize = limit / 2; + self.head_limit = Some(limit - tail); + self.tails = [TailRing::new(tail), TailRing::new(tail)]; self } @@ -251,6 +259,12 @@ impl WriterState { self.bytes = 0; } + fn push_chunk(&mut self, mut chunk: OutputChunk) { + chunk.dropped_before = std::mem::take(&mut self.unreported_output_drops); + let row = output_row(&self.envelope, &chunk); + self.push(vec![row], DeliveryStream::Output, Priority::Output); + } + fn emit(&mut self, stream: OutputStream, data: Vec, occurred_at: &str) { let index = usize::from(stream == OutputStream::Stderr); let chunk = OutputChunk { @@ -258,17 +272,17 @@ impl WriterState { stream, occurred_at: occurred_at.to_owned(), byte_offset: self.offsets[index], - dropped_before: std::mem::take(&mut self.unreported_output_drops), + dropped_before: 0, data, + omitted: None, }; self.next_sequence += 1; - self.skipping = None; + self.tail_open = None; self.offsets[index] += chunk.data.len() as u64; - let row = output_row(&self.envelope, &chunk); - self.push(vec![row], DeliveryStream::Output, Priority::Output); + self.push_chunk(chunk); } - /// Accept `data` written to `stream`: a stream switch flushes the other stream first, so the shared sequence preserves interleaving; full chunks are emitted at once. + /// Accept `data` written to `stream`: a stream switch flushes the other stream first, so the shared sequence preserves interleaving; full chunks are emitted at once. Past the head limit, bytes go to the stream's tail ring. pub(crate) fn append_output(&mut self, stream: OutputStream, data: &[u8], now: SystemTime) { if data.is_empty() { return; @@ -281,26 +295,80 @@ impl WriterState { self.flush_output(); } let index = usize::from(stream == OutputStream::Stderr); - let room: usize = self.output_limit.saturating_sub(self.captured[index]); - let (kept, skipped) = data.split_at(room.min(data.len())); - self.captured[index] += kept.len(); - self.record_output(stream, kept, now); - if !skipped.is_empty() { - self.skip_output(stream); + let Some(head_limit) = self.head_limit else { + self.record_output(stream, data, now); + return; + }; + let room: usize = head_limit.saturating_sub(self.captured[index]); + let (head, rest) = data.split_at(room.min(data.len())); + self.captured[index] += head.len(); + self.record_output(stream, head, now); + if !rest.is_empty() { + self.tail_output(stream, rest, now); + } + if self.closing { + self.flush_tails(); } } - /// Output past the stream's capture limit is not recorded: one sequence stands for each contiguous run of it and is counted as filtered, so readers see a filtered gap. - fn skip_output(&mut self, stream: OutputStream) { - if self.skipping == Some(stream) { - return; + /// Keep the newest bytes of `stream` past its head limit in its ring, each run under a sequence reserved now so the global order holds; what the ring cannot hold is counted as omitted. + fn tail_output(&mut self, stream: OutputStream, data: &[u8], now: SystemTime) { + let index = usize::from(stream == OutputStream::Stderr); + let occurred_at: String = format_timestamp(now); + if !self.tails[index].active() { + self.flush_output(); + self.tails[index].activate( + self.next_sequence, + self.offsets[index], + occurred_at.clone(), + ); + self.next_sequence += 1; + self.tail_open = None; + } + let capacity: usize = self.tails[index].capacity(); + let skipped: usize = data.len().saturating_sub(capacity); + let mut rest: &[u8] = &data[skipped..]; + if skipped > 0 { + self.tails[index].evict_to(0); + self.tails[index].omit(skipped); + self.offsets[index] += skipped as u64; + self.tail_open = None; + } + self.tails[index].evict_to(capacity - rest.len()); + while !rest.is_empty() { + let room: usize = if self.tail_open == Some(stream) { + self.tails[index].room(MAX_OUTPUT_CHUNK_BYTES) + } else { + 0 + }; + if room == 0 { + self.tails[index].open(TailSegment::new( + self.next_sequence, + self.offsets[index], + occurred_at.clone(), + )); + self.next_sequence += 1; + self.tail_open = Some(stream); + continue; + } + let (taken, after) = rest.split_at(room.min(rest.len())); + self.tails[index].extend(taken); + self.offsets[index] += taken.len() as u64; + rest = after; } + } + + /// Queue each stream's gap and kept tail, as when the session closes. + pub(crate) fn flush_tails(&mut self) { self.flush_output(); - self.next_sequence += 1; - self.skipping = Some(stream); - let counts = self.counts_mut(DeliveryStream::Output); - counts.accepted += 1; - counts.filtered += 1; + for stream in [OutputStream::Stdout, OutputStream::Stderr] { + let index = usize::from(stream == OutputStream::Stderr); + let chunks: Vec = self.tails[index].take(stream); + for chunk in chunks { + self.push_chunk(chunk); + } + } + self.tail_open = None; } /// Chunk `data` without copying it more than once: fill the pending chunk, emit full chunks straight from `data`, keep the remainder pending. diff --git a/crates/sqlbuild-recall/src/writer/_helpers/session.rs b/crates/sqlbuild-recall/src/writer/_helpers/session.rs index ccc1c5c9cc..1fdfcde0b5 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/session.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/session.rs @@ -188,7 +188,7 @@ impl RecallSession { if !state.closing { state.closing = true; state.deadline = Some(Instant::now() + self.shutdown_timeout); - state.flush_output(); + state.flush_tails(); } self.shared.wake.notify_all(); } diff --git a/crates/sqlbuild-recall/src/writer/_helpers/tail.rs b/crates/sqlbuild-recall/src/writer/_helpers/tail.rs new file mode 100644 index 0000000000..b4fe8b5571 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/_helpers/tail.rs @@ -0,0 +1,139 @@ +//! The bounded ring holding the end of one output stream past its head limit until the session closes. + +use std::collections::VecDeque; + +use crate::records::models::{OmittedOutput, OutputChunk, OutputStream}; + +/// Bytes kept from the end of a stream, with the sequence reserved when they were written. +#[derive(Debug)] +pub(crate) struct TailSegment { + pub(crate) sequence: u64, + pub(crate) byte_offset: u64, + pub(crate) occurred_at: String, + pub(crate) data: Vec, + /// Bytes at the front already evicted. + start: usize, +} + +impl TailSegment { + pub(crate) fn new(sequence: u64, byte_offset: u64, occurred_at: String) -> Self { + Self { + sequence, + byte_offset, + occurred_at, + data: Vec::new(), + start: 0, + } + } + + fn kept(&self) -> usize { + self.data.len() - self.start + } +} + +/// One stream's tail: at most `capacity` bytes, the reserved gap sequence and what was left out. +#[derive(Debug, Default)] +pub(crate) struct TailRing { + capacity: usize, + segments: VecDeque, + bytes: usize, + /// The gap's sequence, stream offset and time, reserved when the stream passed its head limit. + marker: Option<(u64, u64, String)>, + omitted_bytes: u64, + evicted_sequences: u64, +} + +impl TailRing { + pub(crate) fn new(capacity: usize) -> Self { + Self { + capacity, + ..Self::default() + } + } + + pub(crate) fn capacity(&self) -> usize { + self.capacity + } + + /// Whether the stream has passed its head limit. + pub(crate) fn active(&self) -> bool { + self.marker.is_some() + } + + /// Reserve the gap's position, right after the head. + pub(crate) fn activate(&mut self, sequence: u64, byte_offset: u64, occurred_at: String) { + self.marker = Some((sequence, byte_offset, occurred_at)); + } + + /// Leave out `bytes` that never entered the ring. + pub(crate) fn omit(&mut self, bytes: usize) { + self.omitted_bytes += bytes as u64; + } + + /// Evict from the front until at most `limit` bytes remain. + pub(crate) fn evict_to(&mut self, limit: usize) { + while self.bytes > limit { + let Some(front) = self.segments.front_mut() else { + break; + }; + let removed: usize = (self.bytes - limit).min(front.kept()); + front.start += removed; + front.byte_offset += removed as u64; + self.bytes -= removed; + self.omitted_bytes += removed as u64; + if front.kept() == 0 { + drop(self.segments.pop_front()); + self.evicted_sequences += 1; + } + } + } + + /// Room left in the newest segment for contiguous bytes, below `chunk_bytes`. + pub(crate) fn room(&self, chunk_bytes: usize) -> usize { + self.segments + .back() + .map_or(0, |segment| chunk_bytes.saturating_sub(segment.data.len())) + } + + pub(crate) fn open(&mut self, segment: TailSegment) { + self.segments.push_back(segment); + } + + /// Append to the newest segment; the caller checked `room`. + pub(crate) fn extend(&mut self, data: &[u8]) { + if let Some(segment) = self.segments.back_mut() { + segment.data.extend_from_slice(data); + self.bytes += data.len(); + } + } + + /// The gap chunk and the kept tail chunks of `stream`, leaving the ring empty and inactive. + pub(crate) fn take(&mut self, stream: OutputStream) -> Vec { + let Some((sequence, byte_offset, occurred_at)) = self.marker.take() else { + return Vec::new(); + }; + let mut chunks: Vec = vec![OutputChunk { + sequence, + stream, + occurred_at, + data: Vec::new(), + byte_offset, + dropped_before: 0, + omitted: Some(OmittedOutput { + bytes: std::mem::take(&mut self.omitted_bytes), + sequences: std::mem::take(&mut self.evicted_sequences) + 1, + }), + }]; + chunks.extend(self.segments.drain(..).map(|segment| OutputChunk { + sequence: segment.sequence, + stream, + occurred_at: segment.occurred_at, + data: segment.data[segment.start..].to_vec(), + byte_offset: segment.byte_offset, + dropped_before: 0, + omitted: None, + })); + self.bytes = 0; + chunks + } +} diff --git a/crates/sqlbuild-recall/src/writer/constants.rs b/crates/sqlbuild-recall/src/writer/constants.rs index 5cb01c891b..f38530229d 100644 --- a/crates/sqlbuild-recall/src/writer/constants.rs +++ b/crates/sqlbuild-recall/src/writer/constants.rs @@ -27,5 +27,3 @@ pub const MINIMUM_MAINTENANCE_MS: u64 = 200; pub const MAX_ARTIFACT_BYTES: usize = 4 * 1024 * 1024; /// What a JSON scalar counts toward the queue's byte capacity. pub const JSON_SCALAR_BYTES: usize = 8; -/// Output of each stream recorded per invocation; later output is counted as filtered. -pub const OUTPUT_STREAM_CAPTURE_BYTES: usize = 1024 * 1024; diff --git a/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs b/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs index 746aa46418..ea1cb1f2fb 100644 --- a/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs +++ b/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs @@ -11,7 +11,8 @@ use crate::writer::models::{DeliveryStream, Priority, RecallSession, SessionConf /// Start the delivery thread and queue the invocation's context row first. #[must_use] pub fn start_recall_session(config: SessionConfig) -> RecallSession { - let mut state = WriterState::new(config.envelope.clone()); + let mut state = + WriterState::new(config.envelope.clone()).with_output_limit(config.max_output_bytes); state.push( vec![invocation_row(&config.envelope, &config.context)], DeliveryStream::Invocation, diff --git a/crates/sqlbuild-recall/src/writer/models.rs b/crates/sqlbuild-recall/src/writer/models.rs index 4100bb15bb..f298403e67 100644 --- a/crates/sqlbuild-recall/src/writer/models.rs +++ b/crates/sqlbuild-recall/src/writer/models.rs @@ -75,6 +75,8 @@ pub struct SessionConfig { pub retention_days: u32, pub shutdown_timeout: Duration, pub diagnostics_min_severity: String, + /// Bytes of each output stream recorded, the first and last half; `None` records everything. + pub max_output_bytes: Option, pub heartbeat_interval: Duration, pub setup_budget: Duration, /// Whether this session runs retention maintenance after delivery. diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs index 08273a2551..eddddbd71d 100644 --- a/crates/sqlbuild-recall/src/writer/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -83,6 +83,7 @@ pub(crate) fn config(path: PathBuf, invocation_id: &str) -> SessionConfig { retention_days: 30, shutdown_timeout: Duration::from_secs(5), diagnostics_min_severity: "info".to_owned(), + max_output_bytes: None, heartbeat_interval: Duration::from_secs(30), setup_budget: Duration::from_secs(2), maintenance: true, @@ -273,33 +274,50 @@ pub(crate) fn output_drops(capacity: usize, chunks: usize) -> (u64, Vec (Vec<[Option; 3]>, u64) { + limit: Option, + writes: &[(OutputStream, &[u8])], +) -> Vec<[Option; 6]> { let mut state = WriterState::new(envelope("inv")).with_output_limit(limit); - for (stream, length) in writes { - state.append_output(*stream, &vec![b'x'; *length], SystemTime::now()); + for (stream, data) in writes { + state.append_output(*stream, data, SystemTime::now()); } - state.flush_output(); - let chunks = state + state.flush_tails(); + let mut rows: Vec<[Option; 6]> = state .take_batch(usize::MAX) .iter() .flat_map(|record| record.rows().iter()) - .map(|row| ["sequence", "stream", "byte_count"].map(|name| row.value(name).cloned())) + .map(|row| { + [ + "sequence", + "stream", + "byte_offset", + "data", + "omitted_bytes", + "omitted_sequences", + ] + .map(|name| row.value(name).cloned()) + }) .collect(); - ( - chunks, - state.counts[DeliveryStream::Output.index()].filtered, - ) + rows.sort_by_key(|row| format!("{:020?}", row[0])); + rows } -/// A chunk's sequence, stream and byte count, as `capped` reports them. -pub(crate) fn chunk_facts(sequence: i64, stream: &str, bytes: i64) -> [Option; 3] { +/// A row as `capped` reports it; `omitted` is set on a truncation gap. +pub(crate) fn chunk_facts( + (sequence, stream, offset): (i64, &str, i64), + data: &[u8], + omitted: Option<(i64, i64)>, +) -> [Option; 6] { [ Some(RecallValue::Integer(sequence)), Some(RecallValue::Text(stream.to_owned())), - Some(RecallValue::Integer(bytes)), + Some(RecallValue::Integer(offset)), + Some(RecallValue::Blob(data.to_vec())), + Some(omitted.map_or(RecallValue::Null, |(bytes, _)| RecallValue::Integer(bytes))), + Some(omitted.map_or(RecallValue::Null, |(_, sequences)| { + RecallValue::Integer(sequences) + })), ] } diff --git a/crates/sqlbuild-recall/src/writer/tests/test_queue.rs b/crates/sqlbuild-recall/src/writer/tests/test_queue.rs index d09667dd72..5b74ba43bb 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_queue.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_queue.rs @@ -115,35 +115,68 @@ fn given_a_full_queue_when_output_is_dropped_then_each_drop_is_counted_once() { } #[test] -fn given_output_past_the_capture_limit_when_appended_then_each_skipped_run_is_one_filtered_sequence() - { +fn given_output_past_the_limit_when_closed_then_head_and_tail_are_exact_around_one_gap() { let test_cases = [ CaptureLimitTestCase { - description: "output within the limit is recorded whole", - limit: 10, - writes: vec![(OutputStream::Stdout, 4), (OutputStream::Stdout, 6)], - expected_outcome: (vec![chunk_facts(0, "stdout", 10)], 0), + description: "output within the first half is one chunk", + limit: Some(8), + writes: vec![(OutputStream::Stdout, b"01"), (OutputStream::Stdout, b"23")], + expected_rows: vec![chunk_facts((0, "stdout", 0), b"0123", None)], }, CaptureLimitTestCase { - description: "each run past a stream's limit leaves one filtered sequence", - limit: 10, + description: "output within the limit is kept whole around an empty gap", + limit: Some(8), writes: vec![ - (OutputStream::Stdout, 25), - (OutputStream::Stdout, 3), - (OutputStream::Stderr, 5), - (OutputStream::Stdout, 5), + (OutputStream::Stdout, b"0123"), + (OutputStream::Stdout, b"45"), ], - expected_outcome: ( - vec![chunk_facts(0, "stdout", 10), chunk_facts(2, "stderr", 5)], - 2, - ), + expected_rows: vec![ + chunk_facts((0, "stdout", 0), b"0123", None), + chunk_facts((1, "stdout", 4), b"", Some((0, 1))), + chunk_facts((2, "stdout", 4), b"45", None), + ], + }, + CaptureLimitTestCase { + description: "the first and last halves are kept around a gap counting the middle", + limit: Some(8), + writes: vec![(OutputStream::Stdout, b"0123456789")], + expected_rows: vec![ + chunk_facts((0, "stdout", 0), b"0123", None), + chunk_facts((1, "stdout", 4), b"", Some((2, 1))), + chunk_facts((2, "stdout", 6), b"6789", None), + ], + }, + CaptureLimitTestCase { + description: "interleaved stderr keeps its place around the kept stdout tail", + limit: Some(8), + writes: vec![ + (OutputStream::Stdout, b"0123"), + (OutputStream::Stdout, b"ab"), + (OutputStream::Stderr, b"E1"), + (OutputStream::Stdout, b"cdef"), + (OutputStream::Stdout, b"gh"), + (OutputStream::Stderr, b"E2"), + ], + expected_rows: vec![ + chunk_facts((0, "stdout", 0), b"0123", None), + chunk_facts((1, "stdout", 4), b"", Some((4, 2))), + chunk_facts((3, "stderr", 0), b"E1", None), + chunk_facts((4, "stdout", 8), b"efgh", None), + chunk_facts((5, "stderr", 2), b"E2", None), + ], + }, + CaptureLimitTestCase { + description: "no limit records everything", + limit: None, + writes: vec![(OutputStream::Stdout, b"0123456789")], + expected_rows: vec![chunk_facts((0, "stdout", 0), b"0123456789", None)], }, ]; for test_case in test_cases { assert_eq!( capped(test_case.limit, &test_case.writes), - test_case.expected_outcome, + test_case.expected_rows, "{}", test_case.description ); diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs index 34544bbc49..3aaa2c7048 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -72,7 +72,7 @@ pub(crate) struct OutputDropTestCase { pub(crate) struct CaptureLimitTestCase { pub(crate) description: &'static str, - pub(crate) limit: usize, - pub(crate) writes: Vec<(OutputStream, usize)>, - pub(crate) expected_outcome: (Vec<[Option; 3]>, u64), + pub(crate) limit: Option, + pub(crate) writes: Vec<(OutputStream, &'static [u8])>, + pub(crate) expected_rows: Vec<[Option; 6]>, } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md index 38683138bb..0abc6ee443 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md @@ -109,7 +109,9 @@ sqb recall output last --tail 20 # only the last 20 lines | `--tail` | Print only the last N lines | | `--target` | Target whose store to read | -Output that Recall could not record completely is followed by a note on stderr naming the gap. +Where output past [`max_output_bytes`](../concepts/observability/recall.md#output-limit) was left +out, a note line marks the gap in place. Output that Recall could not record completely for any +other reason is followed by a note on stderr naming the gap. ## Errors diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index 80f2e5727f..eb4a8a3705 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -10,6 +10,7 @@ Online: https://sqlbuild.com/docs/concepts/observability/recall/ - What is recorded - Settings +- Output limit - Retention - When the store cannot be written - Querying history @@ -28,8 +29,8 @@ For every recorded invocation, Recall keeps: retries, and audit results; - the SQL text of every executed statement, stored once per distinct statement and shared across invocations; -- its stdout and stderr, byte for byte, in the order they were written, up to 1 MiB of each - stream; later output is not recorded and shows as a `filtered` gap in `output_invocations`; +- its stdout and stderr, byte for byte, in the order they were written, up to + [`max_output_bytes`](#output-limit) of each stream; - framework diagnostics at or above `diagnostics_min_severity`. Writes happen on a background thread and never fail the command. A command that cannot record all @@ -54,6 +55,7 @@ overrides individual keys for one target. retention = "30d" # days of history to keep; also accepts hours, such as "720h" shutdown_timeout = "3s" # longest delay Recall may add when a command exits diagnostics_min_severity = "info" +max_output_bytes = 2097152 # bytes of each output stream to keep; 0 keeps everything [targets.ci.recall] enabled = false # do not record runs against the ci target @@ -66,6 +68,7 @@ enabled = false # do not record runs against the ci target | `retention` | `"30d"` | How long invocations are kept, in days (`d`) or hours (`h`), up to 3650 days | | `shutdown_timeout` | `"3s"` | The cap on time spent flushing history at exit, in `s` or `ms`, up to 10 minutes | | `diagnostics_min_severity` | `"info"` | `trace`, `debug`, `info`, `warning`, `error`, or `critical` | +| `max_output_bytes` | `2097152` for the local store | Bytes of stdout and of stderr kept per invocation, an integer; `0` keeps everything. See [Output limit](#output-limit) | To turn Recall off for the whole project: @@ -78,6 +81,25 @@ Recall never delays the start of a command. At exit it waits at most `shutdown_t history to reach the store; anything left is counted as dropped. A second Ctrl-C stops waiting at once. +## Output limit + +Recall keeps at most `max_output_bytes` of each output stream per invocation: the first half and +the last half, byte for byte. The local SQLite store defaults to 2 MiB, so a command that prints +more keeps its first and last MiB of stdout and of stderr. Set `max_output_bytes = 0` to keep +everything, or a larger number to keep more; a `[targets..recall]` table can set it for one +target. The command itself prints nothing about the limit. + +The middle that was left out is one gap: `output_invocations` reports `gap_reason = 'truncated'`, +the left-out byte count in `truncated_bytes`, and `complete = 0`. `sqb recall output` prints a +note in its place: + +``` +… 23.6 MB of stdout not recorded (kept the first and last 1 MiB). Raise [recall] max_output_bytes to keep more. +``` + +The end of a stream is held in memory until the command exits, so a process that is killed keeps +only the first half. + ## Retention Commands occasionally prune invocations older than `retention`, with all of their events, output, diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 69cb980a8b..53a79a8232 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -129,5 +129,21 @@ class CorruptStoreRaceTestCase: @dataclass(frozen=True) class CaptureLimitTestCase: description: str + extra_toml: str stdout_bytes: int - expected_recorded: list[list[object]] + expected_kept: tuple[int, int] + expected_gap: list[object] + + +@dataclass(frozen=True) +class TruncationNoteTestCase: + description: str + stdout_bytes: int + expected_kept: int + + +@dataclass(frozen=True) +class InvalidOutputLimitTestCase: + description: str + value: str + expected_message: bytes diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index 27bc574c26..423fdcdf46 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -16,6 +16,7 @@ EMITTED_STDOUT: bytes = b"binary \x00\xff\xfe chunk\n" EMITTED_STDERR: bytes = "warning \u2713 from the task\n".encode() EMITTED_TAIL: bytes = b"after the warning\n" +MEBIBYTE: int = 1024 * 1024 EMPTY_RESULT: bytes = b'{"rows": []}' NUMBERED_LINE: re.Pattern[bytes] = re.compile(rb"^(\d+) std(?:out|err)$", re.MULTILINE) PROJECT_TOML: str = ( diff --git a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py index 1cfd125d4f..bb4af78561 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py @@ -17,6 +17,7 @@ DisabledRecallTestCase, FailingStderrTestCase, HeldLockTestCase, + InvalidOutputLimitTestCase, JsonQueryTestCase, ReadOnlyStoreTestCase, RecallOutputTestCase, @@ -25,11 +26,13 @@ RecordedFactsTestCase, RepeatedBuildTestCase, TextOutputTestCase, + TruncationNoteTestCase, ) from tests.integration.src.sqlbuild.cli.recall.helpers import ( EMITTED_STDERR, EMITTED_STDOUT, EMITTED_TAIL, + MEBIBYTE, build_while_store_locked, build_with_failing_recall_notices, build_with_read_only_logs, @@ -700,28 +703,123 @@ def test_given_a_corrupt_store_when_builds_race_then_one_moves_it_and_both_are_r "test_case", [ CaptureLimitTestCase( - description="stdout past the capture limit is a filtered gap; stderr stays whole", - stdout_bytes=3 * 1024 * 1024, - expected_recorded=[[1024 * 1024, 1, "filtered", 0]], + description="under the default limit stdout is byte-exact", + extra_toml="", + stdout_bytes=1536 * 1024, + expected_kept=(MEBIBYTE * 1024, 0), + expected_gap=[None, 1], + ), + CaptureLimitTestCase( + description="past the default limit the first and last MiB are kept exactly", + extra_toml="", + stdout_bytes=3 * MEBIBYTE, + expected_kept=(MEBIBYTE, MEBIBYTE), + expected_gap=["truncated", 0], + ), + CaptureLimitTestCase( + description="a larger target limit keeps everything", + extra_toml="\n[targets.dev.recall]\nmax_output_bytes = 8388608\n", + stdout_bytes=3 * MEBIBYTE, + expected_kept=(MEBIBYTE * 1024, 0), + expected_gap=[None, 1], + ), + CaptureLimitTestCase( + description="zero records all output", + extra_toml="\n[recall]\nmax_output_bytes = 0\n", + stdout_bytes=3 * MEBIBYTE, + expected_kept=(MEBIBYTE * 1024, 0), + expected_gap=[None, 1], ), ], ids=lambda case: case.description, ) -def test_given_output_past_the_capture_limit_when_recorded_then_the_rest_is_a_filtered_gap( +def test_given_an_output_limit_when_recorded_then_kept_bytes_are_exact_around_one_gap( test_case: CaptureLimitTestCase, tmp_path: Path ) -> None: - project_dir: Path = write_project(tmp_path / "orders_shop") + project_dir: Path = write_project(tmp_path / "orders_shop", extra_toml=test_case.extra_toml) built: subprocess.CompletedProcess[bytes] = run( project_dir, "build", ORDERS_TASK_STDOUT_BYTES=str(test_case.stdout_bytes) ) - recorded: list[list[object]] = recall_rows( + [[stdout_hex, gap_reason, complete, truncated, warning_first]] = recall_rows( project_dir, - "SELECT o.stdout_bytes, instr(CAST(t.stderr AS TEXT), 'from the task') > 0, " - "o.gap_reason, o.complete " + "SELECT hex(t.stdout), o.gap_reason, o.complete, o.truncated_bytes, " + "instr(CAST(t.combined AS TEXT), 'from the task') < instr(t.combined, 'yyyy') " "FROM output_invocations o JOIN output_transcripts t USING (invocation_id)", ) + head, tail = test_case.expected_kept + expected: bytes = built.stdout[:head] + built.stdout[len(built.stdout) - tail :] assert built.returncode == 0, built.stderr.decode(errors="replace") - assert len(built.stdout) > test_case.stdout_bytes - assert recorded == test_case.expected_recorded + assert bytes.fromhex(str(stdout_hex)) == expected + assert [gap_reason, complete] == test_case.expected_gap + assert truncated == len(built.stdout) - len(expected) + assert warning_first == 1 + assert b"max_output_bytes" not in built.stdout + built.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + TruncationNoteTestCase( + description="the note sits where stdout was left out", + stdout_bytes=3 * MEBIBYTE, + expected_kept=MEBIBYTE, + ), + ], + ids=lambda case: case.description, +) +def test_given_truncated_output_when_shown_then_a_note_marks_the_gap( + test_case: TruncationNoteTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + built: subprocess.CompletedProcess[bytes] = run( + project_dir, "build", ORDERS_TASK_STDOUT_BYTES=str(test_case.stdout_bytes) + ) + + shown: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "output", "last", "--stream", "stdout" + ) + omitted: int = len(built.stdout) - 2 * test_case.expected_kept + note: bytes = ( + f"\u2026 {omitted / 1_000_000:.1f} MB of stdout not recorded (kept the first and last " + "1 MiB). Raise [recall] max_output_bytes to keep more.\n" + ).encode() + + assert shown.returncode == 0, shown.stderr.decode(errors="replace") + assert shown.stdout == ( + built.stdout[: test_case.expected_kept] + + b"\n" + + note + + built.stdout[len(built.stdout) - test_case.expected_kept :] + ) + assert shown.stderr == b"" + + +@pytest.mark.parametrize( + "test_case", + [ + InvalidOutputLimitTestCase( + description="a negative limit is a configuration error", + value="-1", + expected_message=b"error[D001]", + ), + InvalidOutputLimitTestCase( + description="a size string is a configuration error", + value='"2MiB"', + expected_message=b"recall.max_output_bytes must be a whole number of bytes", + ), + ], + ids=lambda case: case.description, +) +def test_given_an_invalid_output_limit_when_running_then_the_command_fails_with_its_code( + test_case: InvalidOutputLimitTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project( + tmp_path / "orders_shop", extra_toml=f"\n[recall]\nmax_output_bytes = {test_case.value}\n" + ) + + completed: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + assert completed.returncode != 0 + assert test_case.expected_message in completed.stderr diff --git a/website/src/content/docs/docs/cli/recall.mdx b/website/src/content/docs/docs/cli/recall.mdx index dcf8336f49..3879c74d27 100644 --- a/website/src/content/docs/docs/cli/recall.mdx +++ b/website/src/content/docs/docs/cli/recall.mdx @@ -100,7 +100,9 @@ sqb recall output last --tail 20 # only the last 20 lines | `--tail` | Print only the last N lines | | `--target` | Target whose store to read | -Output that Recall could not record completely is followed by a note on stderr naming the gap. +Where output past [`max_output_bytes`](/docs/concepts/observability/recall#output-limit) was left +out, a note line marks the gap in place. Output that Recall could not record completely for any +other reason is followed by a note on stderr naming the gap. ## Errors diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index f7d69a7a13..959319b3b8 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -16,8 +16,8 @@ For every recorded invocation, Recall keeps: retries, and audit results; - the SQL text of every executed statement, stored once per distinct statement and shared across invocations; -- its stdout and stderr, byte for byte, in the order they were written, up to 1 MiB of each - stream; later output is not recorded and shows as a `filtered` gap in `output_invocations`; +- its stdout and stderr, byte for byte, in the order they were written, up to + [`max_output_bytes`](#output-limit) of each stream; - framework diagnostics at or above `diagnostics_min_severity`. Writes happen on a background thread and never fail the command. A command that cannot record all @@ -42,6 +42,7 @@ overrides individual keys for one target. retention = "30d" # days of history to keep; also accepts hours, such as "720h" shutdown_timeout = "3s" # longest delay Recall may add when a command exits diagnostics_min_severity = "info" +max_output_bytes = 2097152 # bytes of each output stream to keep; 0 keeps everything [targets.ci.recall] enabled = false # do not record runs against the ci target @@ -54,6 +55,7 @@ enabled = false # do not record runs against the ci target | `retention` | `"30d"` | How long invocations are kept, in days (`d`) or hours (`h`), up to 3650 days | | `shutdown_timeout` | `"3s"` | The cap on time spent flushing history at exit, in `s` or `ms`, up to 10 minutes | | `diagnostics_min_severity` | `"info"` | `trace`, `debug`, `info`, `warning`, `error`, or `critical` | +| `max_output_bytes` | `2097152` for the local store | Bytes of stdout and of stderr kept per invocation, an integer; `0` keeps everything. See [Output limit](#output-limit) | To turn Recall off for the whole project: @@ -66,6 +68,25 @@ Recall never delays the start of a command. At exit it waits at most `shutdown_t history to reach the store; anything left is counted as dropped. A second Ctrl-C stops waiting at once. +## Output limit + +Recall keeps at most `max_output_bytes` of each output stream per invocation: the first half and +the last half, byte for byte. The local SQLite store defaults to 2 MiB, so a command that prints +more keeps its first and last MiB of stdout and of stderr. Set `max_output_bytes = 0` to keep +everything, or a larger number to keep more; a `[targets..recall]` table can set it for one +target. The command itself prints nothing about the limit. + +The middle that was left out is one gap: `output_invocations` reports `gap_reason = 'truncated'`, +the left-out byte count in `truncated_bytes`, and `complete = 0`. `sqb recall output` prints a +note in its place: + +``` +… 23.6 MB of stdout not recorded (kept the first and last 1 MiB). Raise [recall] max_output_bytes to keep more. +``` + +The end of a stream is held in memory until the command exits, so a process that is killed keeps +only the first half. + ## Retention Commands occasionally prune invocations older than `retention`, with all of their events, output, From 2dc9fa26d5178f731e28d858d75522cf9353ac14 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 12:56:03 +0100 Subject: [PATCH 16/19] chore: record the Recall crate at the workspace version in Cargo.lock --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index f02dd460d4..3fc8f13229 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1335,7 +1335,7 @@ dependencies = [ [[package]] name = "sqlbuild-recall" -version = "0.160.0" +version = "0.160.1" dependencies = [ "blake3", "rusqlite", From 657236d1f04626f67596bb1fca9625206bc02360 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:55:12 +0100 Subject: [PATCH 17/19] fix: keep undecodable arguments from changing a recorded command's outcome The invocation_started payload extracted --select as UTF-8 strings, so a selector holding a lone surrogate raised UnicodeEncodeError before the command ran, replacing the error the command reports. selection_count now counts the selectors without decoding them. Recall records such text with \xNN and \uXXXX escapes instead of replacement characters, and its lifecycle and SQL subscriber drops any encoding failure rather than raising it into the command. --- .../src/bindings/_helpers/cli/entry.rs | 6 +-- .../src/bindings/_helpers/recall/events.rs | 15 ++++-- .../src/bindings/_helpers/recall/session.rs | 6 +-- .../src/bindings/_helpers/recall/sinks.rs | 51 +++++++++++++++---- .../src/sqlbuild/cli/recall/_test_types.py | 8 +++ .../sqlbuild/cli/recall/test_local_recall.py | 33 ++++++++++++ 6 files changed, 101 insertions(+), 18 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 5f565c3867..1d6ec0a116 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -418,9 +418,9 @@ fn add_invocation_context( if let Some(adapter) = &invocation.recall.adapter { payload.set_item("adapter", adapter)?; } - let select: Option> = invocation.args.getattr("select")?.extract()?; - if let Some(select) = select { - payload.set_item("selection_count", select.len())?; + let select: Bound<'_, PyAny> = invocation.args.getattr("select")?; + if !select.is_none() { + payload.set_item("selection_count", select.len()?)?; } Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs index eaeddf8947..fb3cfd60fb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs @@ -7,6 +7,7 @@ use pyo3::types::{ }; use serde_json::{Map, Number, Value}; +use crate::bindings::_helpers::discovery::snapshot::escaped; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::recall::constants::{ @@ -15,11 +16,19 @@ use crate::bindings::_helpers::recall::constants::{ }; use crate::bindings::_helpers::records::fields::field as native_field; +/// The text of `text`; lone surrogates, which UTF-8 cannot hold, are kept as `\\xNN` or `\\uXXXX` escapes. +pub(crate) fn escaped_text(text: &Bound<'_, PyString>) -> PyResult { + match text.to_str() { + Ok(valid) => Ok(valid.to_owned()), + Err(_) => escaped(text), + } +} + fn text(value: &Bound<'_, PyAny>) -> PyResult> { if value.is_none() { return Ok(None); } - Ok(Some(value.str()?.to_string_lossy().into_owned())) + escaped_text(&value.str()?).map(Some) } fn json_items(dict: &Bound<'_, PyDict>) -> PyResult> { @@ -31,7 +40,7 @@ fn json_items(dict: &Bound<'_, PyDict>) -> PyResult> { let Some(item) = json_value(&item)? else { return Ok(None); }; - entries.insert(key.to_str()?.to_owned(), item); + entries.insert(escaped_text(key)?, item); } Ok(Some(Value::Object(entries))) } @@ -53,7 +62,7 @@ fn json_value(value: &Bound<'_, PyAny>) -> PyResult> { return Ok(Number::from_f64(value.extract::()?).map(Value::Number)); } if let Ok(text) = value.downcast::() { - return Ok(Some(Value::String(text.to_str()?.to_owned()))); + return Ok(Some(Value::String(escaped_text(text)?))); } if value.is_instance_of::() || value.is_instance_of::() { let mut items: Vec = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index 7c574a89c4..179ad65510 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -6,7 +6,6 @@ use std::sync::Arc; use std::time::{Duration, Instant, SystemTime}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; -use pyo3::types::PyStringMethods; use sqlbuild_cli::dispatch::models::CommandFacts; use sqlbuild_recall::records::main::format_timestamp::format_timestamp; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; @@ -34,6 +33,7 @@ use crate::bindings::_helpers::recall::constants::{ MINIMUM_HEARTBEAT_SECONDS, NOTICE_INTERVAL_SECONDS, QUARANTINE_START_WAIT_MS, STDOUT_STREAM_INDEX, }; +use crate::bindings::_helpers::recall::events::escaped_text; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::rules::config_json::dumped_json; @@ -54,7 +54,7 @@ fn text_value(value: &Bound<'_, PyAny>) -> PyResult> { if value.is_none() { return Ok(None); } - Ok(Some(value.str()?.to_string_lossy().into_owned())) + escaped_text(&value.str()?).map(Some) } fn list_attribute(args: &Bound<'_, PyAny>, name: &str) -> PyResult> { @@ -64,7 +64,7 @@ fn list_attribute(args: &Bound<'_, PyAny>, name: &str) -> PyResult> return Ok(items); } for item in value.try_iter()? { - items.push(item?.str()?.to_string_lossy().into_owned()); + items.push(escaped_text(&item?.str()?)?); } Ok(items) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs index d0caa1b737..7373814272 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs @@ -2,13 +2,14 @@ use std::sync::Arc; +use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; -use pyo3::types::{PyString, PyStringMethods}; +use pyo3::types::PyString; use sqlbuild_recall::writer::models::RecallSession; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::recall::events::event_object; +use crate::bindings::_helpers::recall::events::{escaped_text, event_object}; /// Publishes lifecycle events and SQL artifacts of the active session; delivery is best effort. #[pyclass(module = "sqlbuild._native", frozen)] @@ -18,22 +19,54 @@ pub(crate) struct RecallLifecycleSink { #[pymethods] impl RecallLifecycleSink { + /// Record `event`; an event Recall cannot encode is skipped, never raised into the command. fn __call__(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { + recording(py, self.publish(py, event)) + } + + /// Record one SQL artifact; text Recall cannot encode is skipped, never raised into the command. + #[pyo3(signature = (*, sql, statement_id=None, resource_id=None))] + fn record_sql( + &self, + py: Python<'_>, + sql: &Bound<'_, PyString>, + statement_id: Option<&Bound<'_, PyString>>, + resource_id: Option<&Bound<'_, PyString>>, + ) -> PyResult<()> { + let texts = (|| -> PyResult<(String, Option, Option)> { + Ok(( + escaped_text(sql)?, + statement_id.map(escaped_text).transpose()?, + resource_id.map(escaped_text).transpose()?, + )) + })(); + recording( + py, + texts.map(|(sql, statement_id, resource_id)| { + self.session + .publish_sql(&sql, statement_id.as_deref(), resource_id.as_deref()); + }), + ) + } +} + +impl RecallLifecycleSink { + fn publish(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { if let Some(object) = event_object(event)? { self.session.publish_event_object(object); return Ok(()); } let encoded = host::entry(py, ExecutorHost::LifecycleEventToJson)?.call1((event,))?; - let text = encoded - .downcast::()? - .to_string_lossy() - .into_owned(); + let text = escaped_text(encoded.downcast::()?)?; drop(self.session.publish_event(&text)); Ok(()) } +} - #[pyo3(signature = (*, sql, statement_id=None, resource_id=None))] - fn record_sql(&self, sql: &str, statement_id: Option<&str>, resource_id: Option<&str>) { - self.session.publish_sql(sql, statement_id, resource_id); +/// `outcome` with ordinary exceptions dropped, so recording never changes what the command raises; interrupts still propagate. +fn recording(py: Python<'_>, outcome: PyResult<()>) -> PyResult<()> { + match outcome { + Err(error) if error.is_instance_of::(py) => Ok(()), + other => other, } } diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 53a79a8232..c6d2807663 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -147,3 +147,11 @@ class InvalidOutputLimitTestCase: description: str value: str expected_message: bytes + + +@dataclass(frozen=True) +class UndecodableArgumentTestCase: + description: str + selector: str + expected_error: bytes + expected_recorded: list[list[object]] diff --git a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py index bb4af78561..9677a44ee9 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py @@ -27,6 +27,7 @@ RepeatedBuildTestCase, TextOutputTestCase, TruncationNoteTestCase, + UndecodableArgumentTestCase, ) from tests.integration.src.sqlbuild.cli.recall.helpers import ( EMITTED_STDERR, @@ -823,3 +824,35 @@ def test_given_an_invalid_output_limit_when_running_then_the_command_fails_with_ assert completed.returncode != 0 assert test_case.expected_message in completed.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + UndecodableArgumentTestCase( + description="an undecodable selector is recorded escaped; the command fails as before", + selector="orders\udcff", + expected_error=b"TypeError: str is not valid UTF-8: surrogates not allowed", + expected_recorded=[['["orders\\\\xff"]', 1]], + ), + ], + ids=lambda case: case.description, +) +def test_given_an_undecodable_argument_when_recorded_then_it_is_escaped_and_the_command_unchanged( + test_case: UndecodableArgumentTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + + completed: subprocess.CompletedProcess[bytes] = run( + project_dir, "compile", "--json", "--select", test_case.selector + ) + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT i.selection, json_extract(e.payload, '$.selection_count') " + "FROM invocations i JOIN events e USING (invocation_id) " + "WHERE e.event_type = 'invocation_started'", + ) + + assert completed.returncode == 1 + assert completed.stderr.rstrip().endswith(test_case.expected_error) + assert recorded == test_case.expected_recorded From 9d474648523da737341730c777ac21ad0453fcc6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:55:12 +0100 Subject: [PATCH 18/19] fix: join the Recall delivery thread when the session closes The delivery thread signalled completion before closing its store and exiting, so it could outlive the command for a moment. It now closes the store first and close() joins it once it has finished; a thread still busy at the shutdown deadline stays detached. Helpers that list child processes through /proc skip threads that exit during the walk. --- .../src/writer/_helpers/delivery.rs | 1 + .../src/writer/_helpers/session.rs | 10 +++++++++ .../src/writer/main/start_recall_session.rs | 21 +++++++++++++------ crates/sqlbuild-recall/src/writer/models.rs | 5 ++++- .../src/sqlbuild/cli/commands/main/helpers.py | 6 +++++- .../rule_engine/_helpers/host/helpers.py | 6 +++++- 6 files changed, 40 insertions(+), 9 deletions(-) diff --git a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs index 8a358137f9..50bd700728 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs @@ -311,6 +311,7 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { }; drop(maintain_sqlite_store(&store, &request)); } + drop(store); let mut state = locked(&shared.state); state.finished = true; shared.wake.notify_all(); diff --git a/crates/sqlbuild-recall/src/writer/_helpers/session.rs b/crates/sqlbuild-recall/src/writer/_helpers/session.rs index 1fdfcde0b5..f656bb31cc 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/session.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/session.rs @@ -238,6 +238,16 @@ impl RecallSession { pub fn close(&self) -> DeliverySummary { self.begin_close(); while !self.wait_closed(self.shutdown_timeout) {} + if locked(&self.shared.state).finished { + let delivery = self + .delivery + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + if let Some(delivery) = delivery { + drop(delivery.join()); + } + } self.summary() } diff --git a/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs b/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs index ea1cb1f2fb..e388056c3c 100644 --- a/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs +++ b/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs @@ -28,16 +28,25 @@ pub fn start_recall_session(config: SessionConfig) -> RecallSession { shared: Arc::clone(&shared), shutdown_timeout: config.shutdown_timeout, min_severity: severity_rank(&config.diagnostics_min_severity), + delivery: Mutex::new(None), }; let spawned = std::thread::Builder::new() .name("sqlbuild-recall".to_owned()) .spawn(move || deliver(&shared, &config)); - if let Err(error) = spawned { - let mut state = locked(&session.shared.state); - state.degraded = Some(format!("cannot start the Recall delivery thread: {error}")); - state.rejecting = true; - state.drop_all(); - state.finished = true; + match spawned { + Ok(handle) => { + *session + .delivery + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(handle); + } + Err(error) => { + let mut state = locked(&session.shared.state); + state.degraded = Some(format!("cannot start the Recall delivery thread: {error}")); + state.rejecting = true; + state.drop_all(); + state.finished = true; + } } session } diff --git a/crates/sqlbuild-recall/src/writer/models.rs b/crates/sqlbuild-recall/src/writer/models.rs index f298403e67..4b77260ae5 100644 --- a/crates/sqlbuild-recall/src/writer/models.rs +++ b/crates/sqlbuild-recall/src/writer/models.rs @@ -1,7 +1,8 @@ //! Writer configuration, delivery streams, priorities and the delivery summary. use std::path::PathBuf; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; use std::time::Duration; use crate::records::models::{DeliveryCounts, InvocationContext, RowEnvelope}; @@ -129,4 +130,6 @@ pub struct RecallSession { pub(crate) shared: Arc, pub(crate) shutdown_timeout: Duration, pub(crate) min_severity: usize, + /// The delivery thread, joined at close once it has finished so no thread outlives the session. + pub(crate) delivery: Mutex>>, } diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 5a2b39a3e8..df194d7a58 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -899,7 +899,11 @@ def child_processes() -> frozenset[int]: children: set[int] = set() for task in (Path("/proc") / str(os.getpid()) / "task").iterdir(): - children.update(map(int, (task / "children").read_text().split())) + try: + listed: str = (task / "children").read_text() + except FileNotFoundError: + continue # the thread exited after the task list was read + children.update(map(int, listed.split())) return frozenset(children) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py index 6b9654ef5c..8fe68caed4 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py @@ -143,7 +143,11 @@ def child_processes() -> frozenset[int]: children: set[int] = set() for task in (Path("/proc") / str(os.getpid()) / "task").iterdir(): - children.update(map(int, (task / "children").read_text().split())) + try: + listed: str = (task / "children").read_text() + except FileNotFoundError: + continue # the thread exited after the task list was read + children.update(map(int, listed.split())) return frozenset(children) From 4c5e09e0e9a824b3245973b1a700897cbf1c16b5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:26:48 +0100 Subject: [PATCH 19/19] chore: record the Recall crate at the workspace version in Cargo.lock --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index ac2d7236c6..f92915519b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1336,7 +1336,7 @@ dependencies = [ [[package]] name = "sqlbuild-recall" -version = "0.160.1" +version = "0.160.2" dependencies = [ "blake3", "rusqlite",