diff --git a/Cargo.lock b/Cargo.lock index 2b797f0b6d..f92915519b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -303,6 +303,18 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.5.0" @@ -328,6 +340,12 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "generic-array" version = "0.14.7" @@ -357,7 +375,7 @@ checksum = "823645bc6404ae2915707777061a47d3a031a9ee0bff51b34ec973df3d8d2990" dependencies = [ "compact_str", "get-size-derive2", - "hashbrown", + "hashbrown 0.17.1", "ordermap", "smallvec", "thin-vec", @@ -407,12 +425,30 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + [[package]] name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "heck" version = "0.5.0" @@ -426,7 +462,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.17.1", ] [[package]] @@ -496,6 +532,17 @@ dependencies = [ "cc", ] +[[package]] +name = "libsqlite3-sys" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -655,6 +702,12 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + [[package]] name = "polyglot-sql" version = "0.13.9" @@ -982,6 +1035,20 @@ dependencies = [ "get-size2", ] +[[package]] +name = "rusqlite" +version = "0.37.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -1257,6 +1324,7 @@ dependencies = [ "sqlbuild-discovery", "sqlbuild-model-config", "sqlbuild-pysource", + "sqlbuild-recall", "sqlbuild-refactor", "sqlbuild-render", "sqlbuild-rules", @@ -1266,6 +1334,20 @@ dependencies = [ "stacker", ] +[[package]] +name = "sqlbuild-recall" +version = "0.160.2" +dependencies = [ + "blake3", + "rusqlite", + "rustix", + "serde_json", + "sha2", + "sqlbuild-config", + "sqlbuild-core", + "tempfile", +] + [[package]] name = "sqlbuild-refactor" version = "0.160.2" @@ -1580,6 +1662,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index 78a797517c..7e109473f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/sqlbuild-cache", "crates/sqlbuild-sqltext", "crates/sqlbuild-config", + "crates/sqlbuild-recall", "crates/sqlbuild-discovery", "crates/sqlbuild-scopes", "crates/sqlbuild-model-config", @@ -32,6 +33,7 @@ order = [ "sqlbuild-cache", "sqlbuild-sqltext", "sqlbuild-config", + "sqlbuild-recall", "sqlbuild-model-config", "sqlbuild-discovery", "sqlbuild-render", @@ -86,6 +88,8 @@ ruff_text_size = "=0.0.4" # Unix only: `sqb` shares the compute log prune lock with Python's `filelock` through `flock`. # Fast deterministic hasher for hot in-memory maps whose iteration order never reaches output. rustc-hash = "2" +# Recall's local store: SQLite compiled in, so every build has the same engine version. +rusqlite = { version = "0.37", features = ["bundled"] } rustix = { version = "1.1.4", default-features = false, features = ["fs", "std"] } saphyr-parser = "0.1" regex = "1" @@ -106,6 +110,7 @@ sqlbuild-refactor = { path = "crates/sqlbuild-refactor" } sqlbuild-rules = { path = "crates/sqlbuild-rules" } sqlbuild-model-config = { path = "crates/sqlbuild-model-config" } sqlbuild-pysource = { path = "crates/sqlbuild-pysource" } +sqlbuild-recall = { path = "crates/sqlbuild-recall" } sqlbuild-scopes = { path = "crates/sqlbuild-scopes" } sqlbuild-sqltext = { path = "crates/sqlbuild-sqltext" } sqlbuild-state = { path = "crates/sqlbuild-state" } diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index f0af9a7345..8f0360fb38 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -6722,6 +6722,466 @@ } ] } + }, + { + "name": "recall", + "parser": { + "prog": "sqb recall", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "parsers", + "dest": "recall_command", + "nargs": "A...", + "required": true, + "group": 0, + "parsers": [ + { + "name": "query", + "parser": { + "prog": "sqb recall query", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "mutex_groups": [ + false + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "dest": "recall_sql", + "metavar": "sql", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--backend" + ], + "dest": "recall_backend", + "default": "logs", + "choices": [ + "logs", + "warehouse" + ], + "help": "logs reads the Recall store; warehouse queries the target's connection", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--format" + ], + "dest": "recall_format", + "default": "long", + "choices": [ + "long", + "table", + "json", + "csv" + ], + "mutex_group": 0, + "group": 1 + }, + { + "kind": "store_const", + "options": [ + "--json" + ], + "dest": "recall_format", + "nargs": 0, + "const": "json", + "help": "shorthand for --format json", + "mutex_group": 0, + "group": 1 + }, + { + "kind": "store", + "options": [ + "--limit" + ], + "dest": "recall_limit", + "default": 100, + "type": "int", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--no-limit" + ], + "dest": "recall_no_limit", + "nargs": 0, + "const": true, + "default": false, + "group": 1 + }, + { + "kind": "store", + "options": [ + "--max-cell-bytes" + ], + "dest": "recall_max_cell_bytes", + "type": "int", + "help": "longest cell in text formats, in bytes (default 4096; 0 for no limit)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--target" + ], + "dest": "target", + "help": "target whose Recall store to read (default: the default target)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--project-dir", + "--sqb-project-dir" + ], + "dest": "project_dir", + "default": { + "suppress": true + }, + "help": "SQLBuild project directory (default: the current directory)", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--no-color" + ], + "dest": "no_color", + "nargs": 0, + "const": true, + "default": { + "suppress": true + }, + "help": "disable colored output", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--debug" + ], + "dest": "debug", + "nargs": 0, + "const": true, + "default": { + "suppress": true + }, + "help": "enable verbose output and mirror internal diagnostics to stderr", + "group": 1 + } + ] + } + }, + { + "name": "schema", + "parser": { + "prog": "sqb recall schema", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "mutex_groups": [ + false + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--backend" + ], + "dest": "recall_backend", + "default": "logs", + "choices": [ + "logs", + "warehouse" + ], + "help": "logs reads the Recall store; warehouse queries the target's connection", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--state" + ], + "dest": "recall_state", + "nargs": 0, + "const": true, + "default": false, + "help": "with --backend warehouse, list SQLBuild state tables (the default)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--relation" + ], + "dest": "recall_relation", + "help": "with --backend warehouse, describe one relation", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--format" + ], + "dest": "recall_format", + "default": "long", + "choices": [ + "long", + "table", + "json", + "csv" + ], + "mutex_group": 0, + "group": 1 + }, + { + "kind": "store_const", + "options": [ + "--json" + ], + "dest": "recall_format", + "nargs": 0, + "const": "json", + "help": "shorthand for --format json", + "mutex_group": 0, + "group": 1 + }, + { + "kind": "store", + "options": [ + "--target" + ], + "dest": "target", + "help": "target whose Recall store to read (default: the default target)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--project-dir", + "--sqb-project-dir" + ], + "dest": "project_dir", + "default": { + "suppress": true + }, + "help": "SQLBuild project directory (default: the current directory)", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--no-color" + ], + "dest": "no_color", + "nargs": 0, + "const": true, + "default": { + "suppress": true + }, + "help": "disable colored output", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--debug" + ], + "dest": "debug", + "nargs": 0, + "const": true, + "default": { + "suppress": true + }, + "help": "enable verbose output and mirror internal diagnostics to stderr", + "group": 1 + } + ] + } + }, + { + "name": "output", + "parser": { + "prog": "sqb recall output", + "groups": [ + { + "title": "positional arguments" + }, + { + "title": "options" + } + ], + "actions": [ + { + "kind": "help", + "options": [ + "-h", + "--help" + ], + "dest": "help", + "nargs": 0, + "default": { + "suppress": true + }, + "help": "show this help message and exit", + "group": 1 + }, + { + "kind": "store", + "dest": "recall_invocation", + "nargs": "?", + "default": "last", + "metavar": "invocation", + "help": "invocation ID, unique prefix, or last (default)", + "group": 0 + }, + { + "kind": "store", + "options": [ + "--stream" + ], + "dest": "recall_stream", + "default": "combined", + "choices": [ + "stdout", + "stderr", + "combined" + ], + "group": 1 + }, + { + "kind": "store", + "options": [ + "--tail" + ], + "dest": "recall_tail", + "type": "int", + "help": "print only the last N lines", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--target" + ], + "dest": "target", + "help": "target whose Recall store to read (default: the default target)", + "group": 1 + }, + { + "kind": "store", + "options": [ + "--project-dir", + "--sqb-project-dir" + ], + "dest": "project_dir", + "default": { + "suppress": true + }, + "help": "SQLBuild project directory (default: the current directory)", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--no-color" + ], + "dest": "no_color", + "nargs": 0, + "const": true, + "default": { + "suppress": true + }, + "help": "disable colored output", + "group": 1 + }, + { + "kind": "store_true", + "options": [ + "--debug" + ], + "dest": "debug", + "nargs": 0, + "const": true, + "default": { + "suppress": true + }, + "help": "enable verbose output and mirror internal diagnostics to stderr", + "group": 1 + } + ] + } + } + ] + } + ] + } } ], "choices_help": [ @@ -6739,6 +7199,11 @@ "dest": "mv", "metavar": "mv", "help": "move a model file, optionally renaming it, and update references" + }, + { + "dest": "recall", + "metavar": "recall", + "help": "query the recorded history of past invocations" } ] } diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 2a653e1255..e343239742 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -63,7 +63,7 @@ pub(crate) const INT_CONVERTER: &str = "int"; pub(crate) const PATH_CONVERTER: &str = "Path"; /// Every `CliNamespace` attribute with a default, in declaration order. -pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 162] = [ +pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 174] = [ ("command", NamespaceDefault::None), ("project_dir", NamespaceDefault::None), ("dbt_project_dir", NamespaceDefault::None), @@ -212,6 +212,18 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 162] = [ ("rules_command", NamespaceDefault::None), ("rules_rule_selector", NamespaceDefault::None), ("rules_skills_check", NamespaceDefault::Bool(false)), + ("recall_command", NamespaceDefault::None), + ("recall_sql", NamespaceDefault::None), + ("recall_backend", NamespaceDefault::Text("logs")), + ("recall_format", NamespaceDefault::Text("long")), + ("recall_limit", NamespaceDefault::Int(100)), + ("recall_no_limit", NamespaceDefault::Bool(false)), + ("recall_max_cell_bytes", NamespaceDefault::None), + ("recall_state", NamespaceDefault::Bool(false)), + ("recall_relation", NamespaceDefault::None), + ("recall_invocation", NamespaceDefault::Text("last")), + ("recall_stream", NamespaceDefault::Text("combined")), + ("recall_tail", NamespaceDefault::None), ("format_check", NamespaceDefault::Bool(false)), ("format_diff", NamespaceDefault::Bool(false)), ("format_fixtures_only", NamespaceDefault::Bool(false)), diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 16ecc2f8af..e1240666b5 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -10,6 +10,8 @@ pub(crate) const SKILL_EXEMPT_COMMANDS: [&str; 3] = ["init", "playground", "skil pub(crate) const DBT_COMMAND_NAME: &str = "dbt"; pub(crate) const DBT_INIT_SUBCOMMAND: &str = "init"; pub(crate) const DIFF_COMMAND_NAME: &str = "diff"; +/// `sqb recall` reads history and is never recorded itself. +pub(crate) const RECALL_COMMAND_NAME: &str = "recall"; const PROJECT_DIR: Field = field("project_dir", Source::ProjectDir); const SELECT: Field = field("select", Source::Select); @@ -418,6 +420,31 @@ pub(crate) const QUERY_ROUTE: Route = routed( }, ); +const RECALL_ARGUMENTS: [Field; 13] = [ + PROJECT_DIR, + SELECTED_TARGET, + same("recall_command"), + attr("sql", "recall_sql"), + attr("backend", "recall_backend"), + attr("output_format", "recall_format"), + attr("limit", "recall_limit"), + attr("no_limit", "recall_no_limit"), + attr("max_cell_bytes", "recall_max_cell_bytes"), + attr("relation", "recall_relation"), + attr("invocation", "recall_invocation"), + attr("stream", "recall_stream"), + attr("tail", "recall_tail"), +]; +pub(crate) const RECALL_ROUTE: Route = routed( + "run_recall", + "run_recall_command", + &RECALL_ARGUMENTS, + CallShape { + slot_positional: 0, + implementation_keywords: true, + }, +); + const COST_FIELDS: [Field; 11] = [ PROJECT_DIR, attr("selector", "cost_selector"), diff --git a/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs b/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs index 251ef7a369..2a8444ef27 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/command_dispatch.rs @@ -6,8 +6,8 @@ use crate::dispatch::_helpers::subcommands::{ use crate::dispatch::constants::{ AUDIT_ROUTE, BUILD_ROUTE, CHECK_ROUTE, CLONE_ROUTE, COMPILE_ROUTE, CONTRACT_ROUTE, COST_ROUTE, DAG_ROUTE, DEBUG_ROUTE, DIFF_ROUTE, FORMAT_ROUTE, FRESHNESS_ROUTE, INIT_ROUTE, JANITOR_ROUTE, - LINEAGE_ROUTE, LOAD_ROUTE, PLAN_ROUTE, PLAYGROUND_ROUTE, QUERY_ROUTE, REFACTOR_ROUTE, - RULES_ROUTE, SCOPE_ROUTE, SEED_ROUTE, SKILLS_ROUTE, TEST_ROUTE, + LINEAGE_ROUTE, LOAD_ROUTE, PLAN_ROUTE, PLAYGROUND_ROUTE, QUERY_ROUTE, RECALL_COMMAND_NAME, + RECALL_ROUTE, REFACTOR_ROUTE, RULES_ROUTE, SCOPE_ROUTE, SEED_ROUTE, SKILLS_ROUTE, TEST_ROUTE, }; use crate::dispatch::main::unavailable_handler::unavailable_handler; use crate::dispatch::models::{CommandFacts, Dispatch, DispatchFailure, Route}; @@ -54,6 +54,7 @@ pub fn command_dispatch(facts: &CommandFacts) -> Dispatch { ), "diff" => run(&DIFF_ROUTE), "query" => run(&QUERY_ROUTE), + RECALL_COMMAND_NAME => run(&RECALL_ROUTE), "cost" => run(&COST_ROUTE), "debug" => run(&DEBUG_ROUTE), "janitor" => run(&JANITOR_ROUTE), diff --git a/crates/sqlbuild-cli/src/dispatch/main/invocation_policy.rs b/crates/sqlbuild-cli/src/dispatch/main/invocation_policy.rs index 3396d875c2..e01102eb49 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/invocation_policy.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/invocation_policy.rs @@ -2,11 +2,11 @@ use crate::dispatch::constants::{ DBT_COMMAND_NAME, DBT_INIT_SUBCOMMAND, DIFF_COMMAND_NAME, PROJECT_CREATING_COMMANDS, - SKILL_EXEMPT_COMMANDS, + RECALL_COMMAND_NAME, SKILL_EXEMPT_COMMANDS, }; use crate::dispatch::models::{CommandFacts, InvocationPolicy}; -/// Whether the command creates a project, is observed, and checks skill freshness. +/// Whether the command creates a project, is observed, checks skill freshness and is recorded; only observed commands are recorded, so every recorded invocation has terminal events. #[must_use] pub fn invocation_policy(facts: &CommandFacts) -> InvocationPolicy { let command: &str = facts.command.as_deref().unwrap_or_default(); @@ -14,9 +14,11 @@ pub fn invocation_policy(facts: &CommandFacts) -> InvocationPolicy { || (command == DBT_COMMAND_NAME && facts.subcommand.as_deref() == Some(DBT_INIT_SUBCOMMAND)); let raw_query_diff: bool = command == DIFF_COMMAND_NAME && facts.raw_query_diff; + let observed: bool = !creates_project && !raw_query_diff; InvocationPolicy { creates_project, - observed: !creates_project && !raw_query_diff, + observed, maintains_skills: !SKILL_EXEMPT_COMMANDS.contains(&command) && !raw_query_diff, + recalled: observed && !command.is_empty() && command != RECALL_COMMAND_NAME, } } diff --git a/crates/sqlbuild-cli/src/dispatch/models.rs b/crates/sqlbuild-cli/src/dispatch/models.rs index 9e22a922ce..ecca6454b4 100644 --- a/crates/sqlbuild-cli/src/dispatch/models.rs +++ b/crates/sqlbuild-cli/src/dispatch/models.rs @@ -119,6 +119,8 @@ pub struct InvocationPolicy { pub observed: bool, /// Whether installed SQLBuild skills are checked for freshness first. pub maintains_skills: bool, + /// Whether Recall records the invocation; `sqb recall` never records itself. + pub recalled: bool, } /// What a failed command reports: its fallback error code and the debug log message. diff --git a/crates/sqlbuild-cli/src/dispatch/tests/test_invocation_policy.rs b/crates/sqlbuild-cli/src/dispatch/tests/test_invocation_policy.rs index ce61f40996..fe47e5f633 100644 --- a/crates/sqlbuild-cli/src/dispatch/tests/test_invocation_policy.rs +++ b/crates/sqlbuild-cli/src/dispatch/tests/test_invocation_policy.rs @@ -13,6 +13,7 @@ fn given_parsed_command_when_choosing_services_then_matches_python_entry_rules() creates_project: false, observed: true, maintains_skills: true, + recalled: true, }, }, InvocationPolicyTestCase { @@ -22,6 +23,7 @@ fn given_parsed_command_when_choosing_services_then_matches_python_entry_rules() creates_project: true, observed: false, maintains_skills: true, + recalled: false, }, }, InvocationPolicyTestCase { @@ -31,10 +33,11 @@ fn given_parsed_command_when_choosing_services_then_matches_python_entry_rules() creates_project: true, observed: false, maintains_skills: false, + recalled: false, }, }, InvocationPolicyTestCase { - description: "a raw query diff is neither observed nor checks skills", + description: "a raw query diff is not observed, so it is not recorded either", facts: CommandFacts { raw_query_diff: true, ..facts("diff", None) @@ -43,6 +46,17 @@ fn given_parsed_command_when_choosing_services_then_matches_python_entry_rules() creates_project: false, observed: false, maintains_skills: false, + recalled: false, + }, + }, + InvocationPolicyTestCase { + description: "recall reads history and is never recorded itself", + facts: facts("recall", Some("query")), + expected_policy: InvocationPolicy { + creates_project: false, + observed: true, + maintains_skills: true, + recalled: false, }, }, ]; diff --git a/crates/sqlbuild-config/src/command_connection/constants.rs b/crates/sqlbuild-config/src/command_connection/constants.rs index 6c70b9781a..ab3bddc173 100644 --- a/crates/sqlbuild-config/src/command_connection/constants.rs +++ b/crates/sqlbuild-config/src/command_connection/constants.rs @@ -25,7 +25,7 @@ pub(crate) const SETTING_SNIPPET_INDENT: &str = " "; /// Two quotes around at least one character. pub(crate) const MIN_QUOTED_IDENTIFIER_LENGTH: usize = 3; /// The warehouse group each CLI command selects a target's default warehouse from. -pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 29] = [ +pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 30] = [ ("audit", Some("build")), ("build", Some("build")), ("check", Some("build")), @@ -40,6 +40,7 @@ pub const COMMAND_WAREHOUSE_GROUPS: [(&str, Option<&str>); 29] = [ ("diff", Some("query")), ("freshness", Some("query")), ("query", Some("query")), + ("recall", Some("query")), ("janitor", Some("query")), ("scenario", Some("query")), ("test", Some("query")), diff --git a/crates/sqlbuild-config/src/lib.rs b/crates/sqlbuild-config/src/lib.rs index da69b29e36..66fb1a8c59 100644 --- a/crates/sqlbuild-config/src/lib.rs +++ b/crates/sqlbuild-config/src/lib.rs @@ -10,6 +10,7 @@ pub mod effective; pub mod errors; pub mod models; pub mod project; +pub mod recall; pub mod toml; pub mod workspace; pub mod yaml; diff --git a/crates/sqlbuild-config/src/project/_helpers/documents.rs b/crates/sqlbuild-config/src/project/_helpers/documents.rs index 80ff474882..5690123c50 100644 --- a/crates/sqlbuild-config/src/project/_helpers/documents.rs +++ b/crates/sqlbuild-config/src/project/_helpers/documents.rs @@ -30,6 +30,8 @@ use crate::project::constants::{ use crate::project::errors::{ConfigFailure, FailureClass, LoadFailure, MessagePiece}; use crate::project::models::{Built, ConfigFile, ConfigPaths}; use crate::project::types::{Entries, Fields, Loaded}; +use crate::recall::constants::RECALL_SECTION; +use crate::recall::main::recall_section::recall_section; use crate::toml::main::load_toml::load_toml; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_core::constants::SETTINGS_SECTION; @@ -270,6 +272,10 @@ fn project_fields(payload: &Entries) -> Loaded { "event_exporters was replaced by sinks.lifecycle", )); } + if let Some(recall) = get(payload, RECALL_SECTION) { + recall_section(recall, RECALL_SECTION) + .map_err(|message| ConfigFailure::at(PROJECT_FILE, message))?; + } let mut fields = identity(payload)?; fields.extend(sections(payload)?); fields.push(("targets", targets(payload)?)); @@ -331,6 +337,10 @@ fn local_settings(payload: Option<&ConfigValue>) -> Loaded { } fn local_fields(payload: &Entries) -> Loaded { + if let Some(recall) = get(payload, RECALL_SECTION) { + recall_section(recall, RECALL_SECTION) + .map_err(|message| ConfigFailure::at(LOCAL_FILE, message))?; + } let mut fields: Fields = vec![ ("target", text_or_none(optional_str(payload, "target")?)), ("adapter", text_or_none(optional_str(payload, "adapter")?)), diff --git a/crates/sqlbuild-config/src/project/_helpers/targets.rs b/crates/sqlbuild-config/src/project/_helpers/targets.rs index 63c3c214b4..2ffa133ab5 100644 --- a/crates/sqlbuild-config/src/project/_helpers/targets.rs +++ b/crates/sqlbuild-config/src/project/_helpers/targets.rs @@ -21,6 +21,8 @@ use crate::project::constants::{ use crate::project::errors::ConfigFailure; use crate::project::models::{Built, ConfigFile}; use crate::project::types::{Entries, Fields, Loaded}; +use crate::recall::constants::RECALL_SECTION; +use crate::recall::main::recall_section::recall_section; /// Policy fields of a target, with the enum each projects to. const TARGET_POLICIES: [(&str, &str); 3] = [ @@ -293,7 +295,12 @@ fn validate_target_keys(mapping: &Entries, target: TargetPlace<'_>) -> Result<() label: &label, }; reject_removed_virtual_keys(mapping, &["state", "changes_only"], place)?; - validate_allowed_keys(mapping, &TARGET_KEYS, place) + validate_allowed_keys(mapping, &TARGET_KEYS, place)?; + if let Some(recall) = get(mapping, RECALL_SECTION) { + recall_section(recall, &format!("{label}.{RECALL_SECTION}")) + .map_err(|message| ConfigFailure::at(target.file, message))?; + } + Ok(()) } fn clone_mapping<'a>(mapping: &'a Entries, target: TargetPlace<'_>) -> Loaded<&'a Entries> { diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index d225cbba22..faa5439e1b 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -171,7 +171,7 @@ pub(crate) const MODEL_HEADER_KEYS: [&str; 73] = [ "valid_from_column", "valid_to_column", ]; -pub(crate) const TARGET_KEYS: [&str; 18] = [ +pub(crate) const TARGET_KEYS: [&str; 19] = [ "connection", "vars", "database", @@ -190,6 +190,7 @@ pub(crate) const TARGET_KEYS: [&str; 18] = [ "execution_limits", "diff", "warehouses", + "recall", ]; pub(crate) const CLONE_KEYS: [&str; 2] = ["allow_as_clone_origin", "allow_as_clone_destination"]; pub(crate) const EXECUTION_LIMIT_KEYS: [&str; 5] = [ diff --git a/crates/sqlbuild-config/src/recall/_helpers/mod.rs b/crates/sqlbuild-config/src/recall/_helpers/mod.rs new file mode 100644 index 0000000000..4b9235503b --- /dev/null +++ b/crates/sqlbuild-config/src/recall/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod values; diff --git a/crates/sqlbuild-config/src/recall/_helpers/values.rs b/crates/sqlbuild-config/src/recall/_helpers/values.rs new file mode 100644 index 0000000000..07c5c7f842 --- /dev/null +++ b/crates/sqlbuild-config/src/recall/_helpers/values.rs @@ -0,0 +1,60 @@ +//! Recall setting value parsing: durations, retention days and strings. + +use crate::models::ConfigValue; +use crate::recall::constants::{ + DECIMAL_RADIX, HOURS_PER_DAY, MAX_RETENTION_DAYS, MAX_SHUTDOWN_TIMEOUT_MS, MILLIS_PER_SECOND, +}; + +/// The unsigned decimal before `unit`, `None` when it is not one or overflows. +fn amount(text: &str, unit: &str) -> Option { + let digits = text.strip_suffix(unit)?; + if digits.is_empty() { + return None; + } + digits.bytes().try_fold(0_u64, |total, byte| { + let digit = char::from(byte).to_digit(DECIMAL_RADIX)?; + total + .checked_mul(u64::from(DECIMAL_RADIX))? + .checked_add(u64::from(digit)) + }) +} + +/// Whole days of `30d` or `720h`, within the supported range. +pub(crate) fn retention_days(text: &str) -> Option { + let days = amount(text, "d").or_else(|| { + amount(text, "h") + .filter(|hours| hours % HOURS_PER_DAY == 0) + .map(|hours| hours / HOURS_PER_DAY) + })?; + if days == 0 || days > MAX_RETENTION_DAYS { + return None; + } + Some(u32::try_from(days).unwrap_or(u32::MAX)) +} + +/// Milliseconds of `3s` or `500ms`, within the supported range. +pub(crate) fn timeout_ms(text: &str) -> Option { + let millis = amount(text, "ms") + .or_else(|| amount(text, "s").and_then(|seconds| seconds.checked_mul(MILLIS_PER_SECOND)))?; + (millis <= MAX_SHUTDOWN_TIMEOUT_MS).then_some(millis) +} + +/// A non-negative TOML integer of bytes; `None` for anything else. +pub(crate) fn output_bytes(value: &ConfigValue) -> Option { + match value { + ConfigValue::Integer(bytes) if *bytes >= 0 => Some(bytes.unsigned_abs()), + _ => None, + } +} + +/// A trimmed non-empty string, or the message for `label.key`. +pub(crate) fn non_empty<'a>( + value: &'a ConfigValue, + label: &str, + key: &str, +) -> Result<&'a str, String> { + match value { + ConfigValue::String(text) if !text.trim().is_empty() => Ok(text.trim()), + _ => Err(format!("{label}.{key} must be a non-empty string")), + } +} diff --git a/crates/sqlbuild-config/src/recall/constants.rs b/crates/sqlbuild-config/src/recall/constants.rs new file mode 100644 index 0000000000..3d61cf1666 --- /dev/null +++ b/crates/sqlbuild-config/src/recall/constants.rs @@ -0,0 +1,21 @@ +//! Recall setting keys and value vocabularies. + +pub const RECALL_SECTION: &str = "recall"; +/// Keys `[recall]` and `targets..recall` accept, sorted. +pub const RECALL_KEYS: [&str; 6] = [ + "connection", + "diagnostics_min_severity", + "enabled", + "max_output_bytes", + "retention", + "shutdown_timeout", +]; +/// The diagnostic severities a Recall store can keep from, least severe first. +pub const RECALL_SEVERITIES: [&str; 6] = ["trace", "debug", "info", "warning", "error", "critical"]; +/// The longest shutdown flush budget, in milliseconds. +pub const MAX_SHUTDOWN_TIMEOUT_MS: u64 = 600_000; +/// The longest retention, in days. +pub const MAX_RETENTION_DAYS: u64 = 3_650; +pub(crate) const DECIMAL_RADIX: u32 = 10; +pub(crate) const HOURS_PER_DAY: u64 = 24; +pub(crate) const MILLIS_PER_SECOND: u64 = 1_000; diff --git a/crates/sqlbuild-config/src/recall/main/mod.rs b/crates/sqlbuild-config/src/recall/main/mod.rs new file mode 100644 index 0000000000..619cdce30e --- /dev/null +++ b/crates/sqlbuild-config/src/recall/main/mod.rs @@ -0,0 +1 @@ +pub mod recall_section; diff --git a/crates/sqlbuild-config/src/recall/main/recall_section.rs b/crates/sqlbuild-config/src/recall/main/recall_section.rs new file mode 100644 index 0000000000..536f3d4ac9 --- /dev/null +++ b/crates/sqlbuild-config/src/recall/main/recall_section.rs @@ -0,0 +1,69 @@ +//! `recall_section`: validate one Recall settings table. + +use crate::models::ConfigValue; +use crate::recall::_helpers::values::{non_empty, output_bytes, retention_days, timeout_ms}; +use crate::recall::constants::{MAX_RETENTION_DAYS, RECALL_KEYS, RECALL_SEVERITIES}; +use crate::recall::models::RecallSection; + +/// The settings of `value`, or the message naming its first problem at `label`. +pub fn recall_section(value: &ConfigValue, label: &str) -> Result { + let Some(entries) = value.as_map() else { + return Err(format!("{label} must be a table")); + }; + let mut section = RecallSection::default(); + for (key, value) in entries { + let ConfigValue::String(key) = key else { + return Err(format!("{label} keys must be strings")); + }; + match key.as_str() { + "enabled" => match value { + ConfigValue::Bool(flag) => section.enabled = Some(*flag), + _ => return Err(format!("{label}.enabled must be true or false")), + }, + "connection" => section.connection = Some(non_empty(value, label, key)?.to_owned()), + "retention" => { + let text = non_empty(value, label, key)?; + section.retention_days = Some(retention_days(text).ok_or_else(|| { + format!( + "{label}.retention must be a whole number of days such as '30d', \ + at most {MAX_RETENTION_DAYS}d" + ) + })?); + } + "shutdown_timeout" => { + let text = non_empty(value, label, key)?; + section.shutdown_timeout_ms = Some(timeout_ms(text).ok_or_else(|| { + format!( + "{label}.shutdown_timeout must be a duration such as '3s' or '500ms', \ + at most 600s" + ) + })?); + } + "max_output_bytes" => { + section.max_output_bytes = Some(output_bytes(value).ok_or_else(|| { + format!( + "{label}.max_output_bytes must be a whole number of bytes, such as \ + 2097152, or 0 to record all output" + ) + })?); + } + "diagnostics_min_severity" => { + let text = non_empty(value, label, key)?; + if !RECALL_SEVERITIES.contains(&text) { + return Err(format!( + "{label}.diagnostics_min_severity must be one of: {}", + RECALL_SEVERITIES.join(", ") + )); + } + section.diagnostics_min_severity = Some(text.to_owned()); + } + other => { + return Err(format!( + "{label} contains unknown key(s): {other}. Allowed keys: {}", + RECALL_KEYS.join(", ") + )); + } + } + } + Ok(section) +} diff --git a/crates/sqlbuild-config/src/recall/mod.rs b/crates/sqlbuild-config/src/recall/mod.rs new file mode 100644 index 0000000000..97389af3ff --- /dev/null +++ b/crates/sqlbuild-config/src/recall/mod.rs @@ -0,0 +1,9 @@ +//! The `[recall]` section and `targets..recall` tables. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-config/src/recall/models.rs b/crates/sqlbuild-config/src/recall/models.rs new file mode 100644 index 0000000000..6c9e735663 --- /dev/null +++ b/crates/sqlbuild-config/src/recall/models.rs @@ -0,0 +1,13 @@ +//! One validated Recall settings table; absent keys inherit. + +/// The settings one `[recall]` or `targets..recall` table sets. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RecallSection { + pub enabled: Option, + pub connection: Option, + pub retention_days: Option, + pub shutdown_timeout_ms: Option, + pub diagnostics_min_severity: Option, + /// Bytes of each output stream to record; `Some(0)` records everything. + pub max_output_bytes: Option, +} diff --git a/crates/sqlbuild-config/src/recall/tests.rs b/crates/sqlbuild-config/src/recall/tests.rs new file mode 100644 index 0000000000..403c449344 --- /dev/null +++ b/crates/sqlbuild-config/src/recall/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_recall_section.rs"] +mod recall_section; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-config/src/recall/tests/helpers.rs b/crates/sqlbuild-config/src/recall/tests/helpers.rs new file mode 100644 index 0000000000..67459d0856 --- /dev/null +++ b/crates/sqlbuild-config/src/recall/tests/helpers.rs @@ -0,0 +1,16 @@ +use crate::models::ConfigValue; + +/// A configuration table holding `entries` in order. +pub(crate) fn table(entries: &[(&str, ConfigValue)]) -> ConfigValue { + ConfigValue::Map( + entries + .iter() + .map(|(key, value)| (ConfigValue::String((*key).to_owned()), value.clone())) + .collect(), + ) +} + +/// A configuration string. +pub(crate) fn text(value: &str) -> ConfigValue { + ConfigValue::String(value.to_owned()) +} diff --git a/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs b/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs new file mode 100644 index 0000000000..b8c144918d --- /dev/null +++ b/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs @@ -0,0 +1,122 @@ +use crate::models::ConfigValue; +use crate::recall::main::recall_section::recall_section; +use crate::recall::models::RecallSection; +use crate::recall::tests::helpers::{table, text}; +use crate::recall::tests::test_types::{InvalidSectionTestCase, ValidSectionTestCase}; + +#[test] +fn given_valid_sections_when_validated_then_settings_are_typed() { + let test_cases = [ + ValidSectionTestCase { + description: "every key is typed and trimmed", + value: table(&[ + ("enabled", ConfigValue::Bool(false)), + ("connection", text(" history ")), + ("retention", text("720h")), + ("shutdown_timeout", text("1500ms")), + ("diagnostics_min_severity", text("debug")), + ("max_output_bytes", ConfigValue::Integer(4_194_304)), + ]), + expected_section: RecallSection { + enabled: Some(false), + connection: Some("history".to_owned()), + retention_days: Some(30), + shutdown_timeout_ms: Some(1500), + diagnostics_min_severity: Some("debug".to_owned()), + max_output_bytes: Some(4_194_304), + }, + }, + ValidSectionTestCase { + description: "zero output bytes means record everything", + value: table(&[("max_output_bytes", ConfigValue::Integer(0))]), + expected_section: RecallSection { + max_output_bytes: Some(0), + ..RecallSection::default() + }, + }, + ValidSectionTestCase { + description: "an empty table leaves every setting unset", + value: table(&[]), + expected_section: RecallSection::default(), + }, + ]; + + for test_case in test_cases { + assert_eq!( + recall_section(&test_case.value, "recall"), + Ok(test_case.expected_section), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_invalid_sections_when_validated_then_the_message_names_the_key() { + let test_cases = [ + InvalidSectionTestCase { + description: "output bytes must not be negative", + value: table(&[("max_output_bytes", ConfigValue::Integer(-1))]), + label: "recall", + expected_message_prefix: "recall.max_output_bytes must be", + }, + InvalidSectionTestCase { + description: "output bytes must be an integer, not a size string", + value: table(&[("max_output_bytes", text("2MiB"))]), + label: "recall", + expected_message_prefix: "recall.max_output_bytes must be", + }, + InvalidSectionTestCase { + description: "unknown keys are listed", + value: table(&[("mode", text("x"))]), + label: "recall", + expected_message_prefix: "recall contains unknown key(s): mode", + }, + InvalidSectionTestCase { + description: "retention needs a known unit", + value: table(&[("retention", text("30x"))]), + label: "recall", + expected_message_prefix: "recall.retention must be", + }, + InvalidSectionTestCase { + description: "retention hours must be whole days", + value: table(&[("retention", text("25h"))]), + label: "recall", + expected_message_prefix: "recall.retention must be", + }, + InvalidSectionTestCase { + description: "retention must be positive", + value: table(&[("retention", text("0d"))]), + label: "recall", + expected_message_prefix: "recall.retention must be", + }, + InvalidSectionTestCase { + description: "shutdown timeout is bounded", + value: table(&[("shutdown_timeout", text("601s"))]), + label: "recall", + expected_message_prefix: "recall.shutdown_timeout must be", + }, + InvalidSectionTestCase { + description: "severity must be known", + value: table(&[("diagnostics_min_severity", text("loud"))]), + label: "recall", + expected_message_prefix: "recall.diagnostics_min_severity must be one of", + }, + InvalidSectionTestCase { + description: "the section must be a table", + value: text("x"), + label: "targets.dev.recall", + expected_message_prefix: "targets.dev.recall must be a table", + }, + ]; + + for test_case in test_cases { + let message: String = recall_section(&test_case.value, test_case.label) + .expect_err("the settings are invalid"); + assert!( + message.starts_with(test_case.expected_message_prefix), + "{}: {message}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/recall/tests/test_types.rs b/crates/sqlbuild-config/src/recall/tests/test_types.rs new file mode 100644 index 0000000000..e083d04520 --- /dev/null +++ b/crates/sqlbuild-config/src/recall/tests/test_types.rs @@ -0,0 +1,15 @@ +use crate::models::ConfigValue; +use crate::recall::models::RecallSection; + +pub(crate) struct ValidSectionTestCase { + pub(crate) description: &'static str, + pub(crate) value: ConfigValue, + pub(crate) expected_section: RecallSection, +} + +pub(crate) struct InvalidSectionTestCase { + pub(crate) description: &'static str, + pub(crate) value: ConfigValue, + pub(crate) label: &'static str, + pub(crate) expected_message_prefix: &'static str, +} diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index cb337572fc..fd5d643514 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -79,7 +79,7 @@ pub(crate) const FORBIDDEN_STATEMENT_PAYLOAD_FIELDS: [&str; 8] = [ "sql", "statement_sql", ]; -pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 51] = [ +pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 55] = [ "adapter", "attached_column_name", "attached_target_kind", @@ -115,8 +115,11 @@ pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 51] = [ "operation_kind", "operation_name", "outcome", + "plan_action", + "plan_reason_code", "planned_cursor_end_exclusive", "planned_cursor_start", + "purpose", "query_id", "resource_kind", "resource_name", @@ -128,11 +131,12 @@ pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 51] = [ "severity", "sql_digest", "statement_kind", + "target", "target_database", "target_name", "target_schema", ]; -pub(crate) const NONNEGATIVE_INTEGER_PAYLOAD_FIELDS: [&str; 25] = [ +pub(crate) const NONNEGATIVE_INTEGER_PAYLOAD_FIELDS: [&str; 26] = [ "affected_rows", "attempt_number", "batch_count", @@ -152,6 +156,7 @@ pub(crate) const NONNEGATIVE_INTEGER_PAYLOAD_FIELDS: [&str; 25] = [ "row_count", "sample_count", "selected_count", + "selection_count", "signal_number", "skipped_count", "succeeded_count", @@ -628,7 +633,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ ]; /// The schema version 2 catalogue in declaration order. pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ - ("invocation_started", &[], &[], &["command"], false), + ( + "invocation_started", + &[], + &[], + &["adapter", "command", "selection_count", "target"], + false, + ), ( "invocation_completed", &[], @@ -699,7 +710,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "resource_attempt_started", &["resource_attempt_id", "resource_id", "run_id"], &[], - &["attempt_number", "resource_kind", "resource_name"], + &[ + "attempt_number", + "plan_action", + "plan_reason_code", + "resource_kind", + "resource_name", + ], false, ), ( @@ -709,6 +726,8 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ &[ "attempt_number", "duration_ms", + "plan_action", + "plan_reason_code", "resource_kind", "resource_name", ], @@ -723,6 +742,8 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "duration_ms", "error_code", "error_type", + "plan_action", + "plan_reason_code", "resource_kind", "resource_name", ], @@ -740,6 +761,8 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ &[ "attempt_number", "duration_ms", + "plan_action", + "plan_reason_code", "resource_kind", "resource_name", "skip_code", @@ -897,6 +920,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "intent", "job_id", "metadata", + "purpose", "query_id", "row_count", "sql_digest", @@ -915,6 +939,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "intent", "job_id", "metadata", + "purpose", "query_id", "sql_digest", "statement_kind", @@ -933,6 +958,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "intent", "job_id", "metadata", + "purpose", "query_id", "row_count", "sql_digest", @@ -952,6 +978,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "intent", "job_id", "metadata", + "purpose", "query_id", "row_count", "sql_digest", @@ -973,6 +1000,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "intent", "job_id", "metadata", + "purpose", "query_id", "row_count", "sql_digest", diff --git a/crates/sqlbuild-core/src/lifecycle_events/tests/test_validate.rs b/crates/sqlbuild-core/src/lifecycle_events/tests/test_validate.rs index 8d98a1c185..6a841b4bbe 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/tests/test_validate.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/tests/test_validate.rs @@ -16,7 +16,7 @@ fn given_lifecycle_event_when_validating_then_first_contract_violation_is_report description: "unexpected payload lists allowed fields", event: event("invocation_started", vec![("x", int(1))]), expected_result: Err( - "payload field(s) not allowed for known event 'invocation_started': x; allowed fields: command", + "payload field(s) not allowed for known event 'invocation_started': x; allowed fields: adapter, command, selection_count, target", ), }, ValidateTestCase { diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index 8741c66d01..a1d6a32b0a 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -33,6 +33,7 @@ sqlbuild-cursor.workspace = true sqlbuild-discovery.workspace = true sqlbuild-model-config.workspace = true sqlbuild-pysource.workspace = true +sqlbuild-recall.workspace = true sqlbuild-refactor.workspace = true sqlbuild-render.workspace = true sqlbuild-rules.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index da8cc80927..cdc5fde694 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -27,6 +27,7 @@ use crate::bindings::_helpers::plan_command::plan::run_plan_command; use crate::bindings::_helpers::plan_command::rules::run_rules_command; use crate::bindings::_helpers::query_command::dag::run_dag_command; use crate::bindings::_helpers::query_command::query::run_query_command; +use crate::bindings::_helpers::recall::command::run_recall_command; use crate::bindings::_helpers::records::construct::{new_record_with, record_class}; use crate::bindings::_helpers::refactor::refactor_command::run_refactor_command; use crate::bindings::_helpers::scenario_command::capture_command::run_scenario_capture_command; @@ -394,6 +395,7 @@ fn run_implementation<'py>( "run_plan_command" => run_plan_command(&request()?)?, "run_playground_command" => run_playground_command(py, &request()?)?, "run_query_command" => run_query_command(py, options())?, + "run_recall_command" => run_recall_command(py, options())?, "run_refactor_command" => run_refactor_command(py, &request()?)?, "run_rules_command" => run_rules_command(&request()?)?, "run_scenario_capture_command" => run_scenario_capture_command(py, &request()?)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index d4500c5832..1d6ec0a116 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -27,6 +27,10 @@ use crate::bindings::_helpers::native_host::facade::FacadeHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::output_capture::records::invocation_context; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +use crate::bindings::_helpers::recall::session::{ + InvocationRecall, finish_invocation_recall, start_invocation_recall, +}; +use crate::bindings::_helpers::recall::sinks::RecallLifecycleSink; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::record_class; use crate::bindings::_helpers::records::fields::field as native_field; @@ -58,6 +62,7 @@ struct Invocation<'py> { handlers: Option>, use_color: bool, facts: CommandFacts, + recall: InvocationRecall, } impl<'py> Invocation<'py> { @@ -392,6 +397,34 @@ fn lifecycle_event<'py>( ) } +/// Subscribe the invocation's Recall session to every lifecycle fact and SQL artifact. +fn subscribe_recall(dispatcher: &Bound<'_, PyAny>, recall: &InvocationRecall) -> PyResult<()> { + let Some(session) = recall.session.clone() else { + return Ok(()); + }; + let sink = Bound::new(dispatcher.py(), RecallLifecycleSink { session })?; + dispatcher.call_method1("subscribe_history", (sink,))?; + Ok(()) +} + +/// `target`, `adapter` and `selection_count` of the started invocation, when known. +fn add_invocation_context( + invocation: &Invocation<'_>, + payload: &Bound<'_, PyDict>, +) -> PyResult<()> { + if let Some(target) = &invocation.recall.target { + payload.set_item("target", target)?; + } + if let Some(adapter) = &invocation.recall.adapter { + payload.set_item("adapter", adapter)?; + } + let select: Bound<'_, PyAny> = invocation.args.getattr("select")?; + if !select.is_none() { + payload.set_item("selection_count", select.len()?)?; + } + Ok(()) +} + /// Dispatch once while publishing canonical invocation lifecycle facts. fn observed_dispatch<'py>( invocation: &Invocation<'py>, @@ -421,8 +454,10 @@ fn observed_dispatch<'py>( dispatcher.call_method1("publish_lifecycle", (event,))?; Ok(()) }; + subscribe_recall(dispatcher, &invocation.recall)?; let payload: Bound<'py, PyDict> = PyDict::new(py); payload.set_item("command", &command)?; + add_invocation_context(invocation, &payload)?; publish(lifecycle_event(py, "invocation_started", payload)?)?; match dispatch_command(&invocation.args, invocation.handlers.clone()) { Ok(exit_code) => { @@ -510,6 +545,16 @@ fn parsed_invocation<'py>( } let facts: CommandFacts = command_facts(&args)?; let policy: InvocationPolicy = invocation_policy(&facts); + let recall: InvocationRecall = if policy.recalled { + start_invocation_recall(py, &args, identity, &facts)? + } else { + InvocationRecall::default() + }; + let recall_sink: Bound<'py, PyAny> = match recall.session.clone() { + Some(session) => Bound::new(py, RecallLifecycleSink { session })?.into_any(), + None => py.None().into_bound(py), + }; + let closure_recall: InvocationRecall = recall.clone(); let retained: (Py, Option>) = (args.clone().unbind(), handlers.map(Bound::unbind)); let operation: Bound<'py, PyCFunction> = PyCFunction::new_closure( @@ -528,11 +573,12 @@ fn parsed_invocation<'py>( .map(|handlers| handlers.bind(py).clone()), use_color, facts: facts.clone(), + recall: closure_recall.clone(), }; Ok(handled_dispatch(&invocation)?.unbind()) }, )?; - call_with( + let dispatched: PyResult> = call_with( &host::entry(py, FacadeHost::DispatchWithComputeLogs)?, &[ ("args", args), @@ -544,8 +590,17 @@ fn parsed_invocation<'py>( .to_owned() .into_any(), ), + ("recall", recall_sink), ], - ) + ); + let exit_status: Option = match &dispatched { + Ok(code) if code.is_instance_of::() => Some(code.extract().unwrap_or(1)), + Ok(_) | Err(_) => None, + }; + let finished: PyResult<()> = finish_invocation_recall(py, &recall, exit_status); + let exit_code: Bound<'py, PyAny> = dispatched?; + finished?; + Ok(exit_code) } /// `argv` without a leading launch record option, after offering the record it names. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 3875d8810a..7f26b84889 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -11,7 +11,7 @@ use crate::bindings::_helpers::{ contract_command, cost_command, cost_report, cost_telemetry, dbt_command, dbt_events, dbt_interop, debug_command, diff_report, event_exporting, execution_history, format_command, freshness, init_command, janitor, load, output_capture, plan_command, presentation_text, - query_command, result_output, scenario_command, seed_command, sql_test_command, + query_command, recall, result_output, scenario_command, seed_command, sql_test_command, workspace_commands, }; @@ -50,6 +50,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { build_command::registration::register(module)?; build_progress::registration::register(module)?; query_command::registration::register(module)?; + recall::registration::register(module)?; scenario_command::registration::register(module)?; contract_command::registration::register(module)?; init_command::registration::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs b/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs index 0a7f1c73ac..02bc5c5556 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs @@ -4,7 +4,7 @@ use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io::{Seek, Write}; use std::path::PathBuf; -use std::sync::{Mutex, MutexGuard}; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock}; use pyo3::IntoPyObjectExt; use pyo3::exceptions::PyRuntimeError; @@ -27,6 +27,7 @@ use sqlbuild_cli::compute_logs::main::read_stream_chunk::read_stream_chunk; use sqlbuild_cli::compute_logs::main::stream_byte_counts::stream_byte_counts; use sqlbuild_cli::compute_logs::main::validate_capture_id::validate_capture_id; use sqlbuild_cli::compute_logs::models::CaptureDocument; +use sqlbuild_recall::writer::models::RecallSession; use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, @@ -37,6 +38,8 @@ use crate::bindings::_helpers::compute_logs::conversion::{ }; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::recall::session::{forward_diagnostic, forward_output}; +use crate::bindings::_helpers::recall::sinks::RecallLifecycleSink; const DIAGNOSTICS_STREAM: usize = 2; @@ -53,6 +56,8 @@ pub(crate) struct LocalFilesystemComputeLogStorage { root: PathBuf, retention_count: Option, state: Mutex, + /// The invocation's Recall session, fed every byte and diagnostic the capture accepts. + recall: OnceLock>, } /// The stdout, stderr and diagnostics byte counts of open streams, as `tell()` reports them. @@ -109,16 +114,24 @@ impl LocalFilesystemComputeLogStorage { stream: usize, data: &[u8], ) -> PyResult<()> { - let written: Option> = { - let mut state = self.locked()?; - state - .writers - .get_mut(invocation_id) - .map(|writers| writers[stream].write_all(data)) + let mut state = self.locked()?; + let Some(writers) = state.writers.get_mut(invocation_id) else { + return Err(not_active(py)); }; - match written { - Some(result) => Ok(result?), - None => Err(not_active(py)), + writers[stream].write_all(data)?; + self.forward_to_recall(invocation_id, stream, data); + Ok(()) + } + + /// Hand bytes the capture just accepted to Recall while its lock is held, so both keep one order. + fn forward_to_recall(&self, invocation_id: &str, stream: usize, data: &[u8]) { + let session = self.recall.get().map(Arc::as_ref); + if stream != DIAGNOSTICS_STREAM { + forward_output(session, invocation_id, (stream, data)); + return; + } + if let Ok(line) = std::str::from_utf8(data) { + forward_diagnostic(session, invocation_id, line.trim_end_matches('\n')); } } @@ -164,9 +177,15 @@ impl LocalFilesystemComputeLogStorage { root, retention_count, state: Mutex::new(StorageState::default()), + recall: OnceLock::new(), }) } + /// Feed every accepted byte and diagnostic to the invocation's Recall session. + fn attach_recall(&self, recall: PyRef<'_, RecallLifecycleSink>) { + drop(self.recall.set(Arc::clone(&recall.session))); + } + /// Return the resolved configured log root. #[getter] pub(crate) fn root<'py>(&self, py: Python<'py>) -> PyResult> { @@ -230,11 +249,8 @@ impl LocalFilesystemComputeLogStorage { } let encoded: Bound<'_, PyAny> = host::entry(py, ExecutorHost::DiagnosticLogToJson)?.call1((record,))?; - let mut line: Vec = encoded - .downcast::()? - .to_str()? - .as_bytes() - .to_vec(); + let text: &str = encoded.downcast::()?.to_str()?; + let mut line: Vec = text.as_bytes().to_vec(); line.push(b'\n'); self.ensure_open(py)?; let invocation_id = capture_id(invocation_id)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index d9d2208ac8..0db8c1a73f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -79,6 +79,7 @@ pub(crate) mod presentation_text; pub(crate) mod project_assembly; pub(crate) mod python_values; pub(crate) mod query_command; +pub(crate) mod recall; pub(crate) mod record_behaviours; pub(crate) mod records; pub(crate) mod refactor; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs new file mode 100644 index 0000000000..c7f01ac6fd --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs @@ -0,0 +1,327 @@ +//! `sqb recall query | schema | output`. + +use std::path::PathBuf; +use std::time::Instant; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; +use pyo3::types::{PyDict, PyDictMethods, PyStringMethods}; +use sqlbuild_recall::records::models::OutputStream; +use sqlbuild_recall::settings::main::resolve_recall_settings::resolve_recall_settings; +use sqlbuild_recall::settings::models::{RecallSettings, SettingsRequest, StoreLocation}; +use sqlbuild_recall::store::errors::{StoreError, StoreErrorKind}; +use sqlbuild_recall::store::main::describe_store::describe_store; +use sqlbuild_recall::store::main::open_sqlite_store::open_sqlite_store; +use sqlbuild_recall::store::main::query_store::query_store; +use sqlbuild_recall::store::main::read_output_transcript::read_output_transcript; +use sqlbuild_recall::store::main::resolve_invocation::resolve_invocation; +use sqlbuild_recall::store::models::{OpenMode, SqliteStore}; + +use crate::bindings::_helpers::compile_driver::python_calls::KeywordArguments; +use crate::bindings::_helpers::query_command::query::run_query_command; +use crate::bindings::_helpers::recall::console::{write_stderr, write_stdout_bytes}; +use crate::bindings::_helpers::recall::constants::{ + DEFAULT_MAX_CELL_BYTES, DEFAULT_QUERY_LIMIT, JSON_FORMAT, LAST_INVOCATION, LONG_FORMAT, + RECALL_AMBIGUOUS_CODE, RECALL_DISABLED_CODE, RECALL_NOT_FOUND_CODE, RECALL_OPTIONS, + RECALL_QUERY_CODE, RECALL_SETTINGS_CODE, RECALL_STORE_CODE, WAREHOUSE_BACKEND, +}; +use crate::bindings::_helpers::recall::rendering::{QueryContext, render_result, render_schema}; +use crate::bindings::_helpers::records::construct::new_error; +use crate::bindings::_helpers::records::ids::ErrorId; + +fn user_error(py: Python<'_>, message: &str, code: &str) -> PyErr { + new_error(py, ErrorId::CliUser, message.to_owned(), code) +} + +fn store_failure(py: Python<'_>, error: &StoreError) -> PyErr { + let code = match error.kind { + StoreErrorKind::NotFound => RECALL_NOT_FOUND_CODE, + StoreErrorKind::Ambiguous => RECALL_AMBIGUOUS_CODE, + StoreErrorKind::Query => RECALL_QUERY_CODE, + _ => RECALL_STORE_CODE, + }; + user_error(py, &error.message, code) +} + +/// The options of one `sqb recall` invocation, as the dispatch route passes them. +struct RecallRequest { + command: Option, + project_dir: Option, + selected_target: Option, + sql: Option, + backend: Option, + format: String, + limit: Option, + no_limit: bool, + max_cell_bytes: Option, + relation: Option, + invocation: Option, + stream: Option, + tail: Option, +} + +fn optional_text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + Ok(Some(value.str()?.to_string_lossy().into_owned())) +} + +fn recall_request(py: Python<'_>, options: Option>) -> PyResult { + let arguments: KeywordArguments<'_> = + KeywordArguments::read(py, "run_recall_command", options, (&RECALL_OPTIONS, &[]))?; + let text = |name: &str| optional_text(&arguments.value(name)?); + let number = |name: &str| -> PyResult> { arguments.value(name)?.extract() }; + Ok(RecallRequest { + command: text("recall_command")?, + project_dir: text("project_dir")?.map(PathBuf::from), + selected_target: text("selected_target")?, + sql: text("sql")?, + backend: text("backend")?, + format: text("output_format")?.unwrap_or_else(|| LONG_FORMAT.to_owned()), + limit: number("limit")?, + no_limit: arguments.value("no_limit")?.is_truthy()?, + max_cell_bytes: number("max_cell_bytes")?, + relation: text("relation")?, + invocation: text("invocation")?, + stream: text("stream")?, + tail: number("tail")?, + }) +} + +/// The project directory, target and settings, or the coded failure. +fn settings(py: Python<'_>, recall: &RecallRequest) -> PyResult<(PathBuf, RecallSettings)> { + let raw: PathBuf = match &recall.project_dir { + Some(directory) => directory.clone(), + None => std::env::current_dir()?, + }; + let project_dir = std::fs::canonicalize(&raw).unwrap_or(raw); + let request = SettingsRequest { + project_dir: project_dir.clone(), + selected_target: recall.selected_target.clone(), + }; + let settings = resolve_recall_settings(&request) + .map_err(|error| user_error(py, &error.message, RECALL_SETTINGS_CODE))?; + if !settings.enabled { + let target = settings + .target + .as_deref() + .map_or_else(String::new, |target| format!(" for target `{target}`")); + return Err(user_error( + py, + &format!("Recall is disabled{target}; set enabled = true in its recall settings"), + RECALL_DISABLED_CODE, + )); + } + Ok((project_dir, settings)) +} + +fn open_store(py: Python<'_>, settings: &RecallSettings) -> PyResult<(SqliteStore, String)> { + match &settings.store { + StoreLocation::Sqlite { path } => { + let store = open_sqlite_store(path, OpenMode::ReadOnly).map_err(|error| { + if error.kind == StoreErrorKind::NotFound { + user_error( + py, + &format!( + "no Recall history is recorded yet at {}; run a command first", + path.display() + ), + RECALL_STORE_CODE, + ) + } else { + store_failure(py, &error) + } + })?; + Ok((store, path.display().to_string())) + } + StoreLocation::Unsupported { + connection, + adapter, + } => Err(user_error( + py, + &format!( + "connection `{connection}` uses adapter `{adapter}`, which this version cannot \ + read Recall history from" + ), + RECALL_STORE_CODE, + )), + } +} + +/// `sqb recall query --backend warehouse` and `schema --backend warehouse` run through `sqb query`. +fn warehouse_query( + py: Python<'_>, + recall: &RecallRequest, + project_dir: &std::path::Path, + (sql, limit): (&str, Option), +) -> PyResult { + let options = PyDict::new(py); + options.set_item("project_dir", project_dir)?; + options.set_item("sql", sql)?; + options.set_item("selected_target", recall.selected_target.as_deref())?; + options.set_item("output_format", &recall.format)?; + options.set_item("limit", limit)?; + run_query_command(py, Some(options)) +} + +fn limit(recall: &RecallRequest) -> Option { + if recall.no_limit { + return None; + } + recall.limit.or(Some(DEFAULT_QUERY_LIMIT)) +} + +fn run_query(py: Python<'_>, recall: &RecallRequest) -> PyResult { + let (project_dir, settings) = settings(py, recall)?; + let sql: String = recall.sql.clone().unwrap_or_default(); + if sql.trim().is_empty() { + return Err(user_error( + py, + "recall query SQL must not be empty", + RECALL_QUERY_CODE, + )); + } + let limit = limit(recall); + if limit.is_some_and(|limit| limit < 0) { + return Err(user_error( + py, + "--limit must not be negative", + RECALL_QUERY_CODE, + )); + } + if recall.backend.as_deref() == Some(WAREHOUSE_BACKEND) { + return warehouse_query(py, recall, &project_dir, (&sql, limit)); + } + let (store, path) = open_store(py, &settings)?; + let started = Instant::now(); + let limit = limit.map(|limit| usize::try_from(limit).unwrap_or(usize::MAX)); + let result = query_store(&store, &sql, limit).map_err(|error| store_failure(py, &error))?; + let format: &str = &recall.format; + let max_bytes: Option = match recall.max_cell_bytes { + Some(bytes) if bytes <= 0 => None, + Some(bytes) => Some(usize::try_from(bytes).unwrap_or(usize::MAX)), + None if format == JSON_FORMAT => None, + None => Some(DEFAULT_MAX_CELL_BYTES), + }; + let rendered = render_result( + &result, + format, + max_bytes, + &QueryContext { + target: settings.target.as_deref(), + store_path: &path, + limit, + elapsed_ms: started.elapsed().as_secs_f64() * 1000.0, + }, + ); + write_stdout_bytes(py, rendered.as_bytes())?; + Ok(0) +} + +fn run_schema(py: Python<'_>, recall: &RecallRequest) -> PyResult { + let (project_dir, settings) = settings(py, recall)?; + if recall.backend.as_deref() == Some(WAREHOUSE_BACKEND) { + let sql = match &recall.relation { + Some(relation) => format!( + "SELECT table_schema, table_name, column_name, data_type, is_nullable \ + FROM information_schema.columns WHERE table_name = '{}' \ + ORDER BY table_schema, ordinal_position", + relation.replace('\'', "''") + ), + None => "SELECT table_schema, table_name, table_type FROM information_schema.tables \ + WHERE table_name LIKE '\\_sqlbuild\\_%' ESCAPE '\\' \ + ORDER BY table_schema, table_name" + .to_owned(), + }; + return warehouse_query(py, recall, &project_dir, (&sql, None)); + } + let (store, path) = open_store(py, &settings)?; + let relations = describe_store(&store).map_err(|error| store_failure(py, &error))?; + let json_output: bool = recall.format == JSON_FORMAT; + write_stdout_bytes( + py, + render_schema(&relations, json_output, (&path, settings.retention_days)).as_bytes(), + )?; + Ok(0) +} + +fn tail_lines(data: &[u8], lines: usize) -> &[u8] { + let body = data.strip_suffix(b"\n").unwrap_or(data); + let start = body + .iter() + .enumerate() + .rev() + .filter(|(_, byte)| **byte == b'\n') + .nth(lines.saturating_sub(1)) + .map_or(0, |(index, _)| index + 1); + if lines == 0 { + &data[data.len()..] + } else { + &data[start..] + } +} + +fn run_output(py: Python<'_>, recall: &RecallRequest) -> PyResult { + let (_, settings) = settings(py, recall)?; + let (store, _) = open_store(py, &settings)?; + let selector: &str = recall.invocation.as_deref().unwrap_or(LAST_INVOCATION); + let invocation_id = + resolve_invocation(&store, selector).map_err(|error| store_failure(py, &error))?; + let stream = recall.stream.as_deref().and_then(OutputStream::from_name); + let transcript = read_output_transcript(&store, &invocation_id, stream) + .map_err(|error| store_failure(py, &error))?; + let data: &[u8] = match recall.tail { + Some(lines) => tail_lines(&transcript.data, usize::try_from(lines).unwrap_or(0)), + None => &transcript.data, + }; + write_stdout_bytes(py, data)?; + let mut problems: Vec = Vec::new(); + if transcript.missing_sequences > 0 { + problems.push(format!( + "{} chunk(s) missing ({})", + transcript.missing_sequences, + transcript.gap_reason.as_deref().unwrap_or("unexplained") + )); + } + if transcript.reported_drops > 0 { + problems.push(format!("{} chunk(s) dropped", transcript.reported_drops)); + } + if transcript.filtered > 0 { + problems.push(format!("{} chunk(s) filtered", transcript.filtered)); + } + if !transcript.flush_complete { + problems.push("delivery did not report a complete flush".to_owned()); + } + match transcript.status.as_deref() { + Some("completed" | "failed") | None => {} + Some(status) => problems.push(format!("the invocation is {status}")), + } + if !problems.is_empty() { + write_stderr( + py, + &format!( + "note: output of {invocation_id} may be incomplete: {}\n", + problems.join("; ") + ), + )?; + } + Ok(0) +} + +/// Run one `sqb recall` subcommand and return its exit code. +pub(crate) fn run_recall_command( + py: Python<'_>, + options: Option>, +) -> PyResult { + let recall: RecallRequest = recall_request(py, options)?; + match recall.command.as_deref() { + Some("query") => run_query(py, &recall), + Some("schema") => run_schema(py, &recall), + Some("output") => run_output(py, &recall), + _ => Err(user_error( + py, + "sqb recall requires a subcommand: query, schema or output", + RECALL_QUERY_CODE, + )), + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs new file mode 100644 index 0000000000..349d0053fa --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs @@ -0,0 +1,22 @@ +//! Recall's writes to the process streams, through Python so captures and tees see them. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::types::PyBytes; + +/// Write `data` unchanged to stdout's binary buffer after flushing pending text. +pub(crate) fn write_stdout_bytes(py: Python<'_>, data: &[u8]) -> PyResult<()> { + let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; + stdout.call_method0("flush")?; + let target: Bound<'_, PyAny> = stdout.getattr("buffer").unwrap_or(stdout); + target.call_method1("write", (PyBytes::new(py, data),))?; + target.call_method0("flush")?; + Ok(()) +} + +/// Write `text` to stderr and flush it. +pub(crate) fn write_stderr(py: Python<'_>, text: &str) -> PyResult<()> { + let stderr: Bound<'_, PyAny> = py.import("sys")?.getattr("stderr")?; + stderr.call_method1("write", (text,))?; + stderr.call_method0("flush")?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs new file mode 100644 index 0000000000..ff0f8711f2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -0,0 +1,91 @@ +//! Recall binding names, environment variables and error codes. + +/// Seconds between heartbeats; tests shorten it to observe abandoned invocations quickly. +pub(crate) const HEARTBEAT_ENVIRONMENT: &str = "SQLBUILD_RECALL_HEARTBEAT_SECONDS"; +/// Marker files under `target/` limiting store notices to one per interval. +pub(crate) const DDL_NOTICE_PREFIX: &str = ".recall-notice-"; +pub(crate) const NOTICE_INTERVAL_SECONDS: u64 = 24 * 60 * 60; +/// Hex digits of the hostname digest recorded as the host token. +pub(crate) const HOST_TOKEN_LENGTH: usize = 16; +/// The shortest heartbeat interval the test environment variable may request. +pub(crate) const MINIMUM_HEARTBEAT_SECONDS: f64 = 0.01; +/// The compute-log stream index of stdout; every other output index is stderr. +pub(crate) const STDOUT_STREAM_INDEX: usize = 0; +pub(crate) const RECALL_DISABLED_CODE: &str = "C970"; +pub(crate) const RECALL_STORE_CODE: &str = "C971"; +pub(crate) const RECALL_NOT_FOUND_CODE: &str = "C972"; +pub(crate) const RECALL_AMBIGUOUS_CODE: &str = "C973"; +pub(crate) const RECALL_QUERY_CODE: &str = "C974"; +pub(crate) const RECALL_SETTINGS_CODE: &str = "C975"; +pub(crate) const WAREHOUSE_BACKEND: &str = "warehouse"; +/// Default cell width in text formats, in bytes. +pub(crate) const DEFAULT_MAX_CELL_BYTES: usize = 4096; +pub(crate) const LONG_FORMAT: &str = "long"; +pub(crate) const TABLE_FORMAT: &str = "table"; +pub(crate) const CSV_FORMAT: &str = "csv"; +pub(crate) const JSON_FORMAT: &str = "json"; +/// The standard base64 alphabet binary JSON cells are encoded with. +pub(crate) const BASE64_ALPHABET: &[u8; 64] = + b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +/// Join keys `sqb recall schema` points out. +pub(crate) const JOIN_KEYS: [&str; 5] = [ + "invocation_id", + "statement_id", + "sql_digest", + "run_id", + "resource_id", +]; +/// Rows `sqb recall query` returns without `--limit` or `--no-limit`. +pub(crate) const DEFAULT_QUERY_LIMIT: i64 = 100; +/// The invocation selector naming the most recently started invocation. +pub(crate) const LAST_INVOCATION: &str = "last"; +/// The keyword arguments the `sqb recall` dispatch route passes. +pub(crate) const RECALL_OPTIONS: [&str; 13] = [ + "project_dir", + "selected_target", + "recall_command", + "sql", + "backend", + "output_format", + "limit", + "no_limit", + "max_cell_bytes", + "relation", + "invocation", + "stream", + "tail", +]; +/// Required text envelope fields of a known lifecycle event. +pub(crate) const EVENT_TEXT_FIELDS: [&str; 5] = [ + "event_id", + "event_type", + "producer", + "producer_version", + "invocation_id", +]; +/// Optional correlation fields of a known lifecycle event. +pub(crate) const EVENT_OPTIONAL_TEXT_FIELDS: [&str; 5] = [ + "run_id", + "resource_id", + "resource_attempt_id", + "operation_id", + "statement_id", +]; +/// The first lifecycle schema version carrying invocation sequence and external context. +pub(crate) const INVOCATION_METADATA_SCHEMA_VERSION: i64 = 2; +/// `datetime` attributes of an ISO-8601 timestamp, most significant first. +pub(crate) const TIMESTAMP_PARTS: [&str; 7] = [ + "year", + "month", + "day", + "hour", + "minute", + "second", + "microsecond", +]; +/// `str(datetime.UTC)`. +pub(crate) const UTC_ZONE_NAME: &str = "UTC"; +/// How often closing checks for an interrupt while delivery finishes. +pub(crate) const CLOSE_POLL_MS: u64 = 50; +/// Longest wait at startup for another process moving a corrupt store aside; the writer thread retries within the shutdown budget. +pub(crate) const QUARANTINE_START_WAIT_MS: u64 = 200; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs new file mode 100644 index 0000000000..fb3cfd60fb --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/events.rs @@ -0,0 +1,135 @@ +//! Lifecycle events as the JSON object `lifecycle_event_to_json` encodes, built natively. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use pyo3::types::{ + PyBool, PyBoolMethods, PyDict, PyDictMethods, PyFloat, PyInt, PyList, PyMapping, PyString, + PyStringMethods, PyTuple, +}; +use serde_json::{Map, Number, Value}; + +use crate::bindings::_helpers::discovery::snapshot::escaped; +use crate::bindings::_helpers::native_host::executor::ExecutorHost; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::recall::constants::{ + EVENT_OPTIONAL_TEXT_FIELDS, EVENT_TEXT_FIELDS, INVOCATION_METADATA_SCHEMA_VERSION, + TIMESTAMP_PARTS, UTC_ZONE_NAME, +}; +use crate::bindings::_helpers::records::fields::field as native_field; + +/// The text of `text`; lone surrogates, which UTF-8 cannot hold, are kept as `\\xNN` or `\\uXXXX` escapes. +pub(crate) fn escaped_text(text: &Bound<'_, PyString>) -> PyResult { + match text.to_str() { + Ok(valid) => Ok(valid.to_owned()), + Err(_) => escaped(text), + } +} + +fn text(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + escaped_text(&value.str()?).map(Some) +} + +fn json_items(dict: &Bound<'_, PyDict>) -> PyResult> { + let mut entries: Map = Map::new(); + for (key, item) in dict.iter() { + let Ok(key) = key.downcast::() else { + return Ok(None); + }; + let Some(item) = json_value(&item)? else { + return Ok(None); + }; + entries.insert(escaped_text(key)?, item); + } + Ok(Some(Value::Object(entries))) +} + +/// A lifecycle JSON value as `lifecycle_event_to_json` encodes it; `None` for anything else. +fn json_value(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(Some(Value::Null)); + } + if let Ok(flag) = value.downcast::() { + return Ok(Some(Value::Bool(flag.is_true()))); + } + if value.is_instance_of::() { + return Ok(value + .extract::() + .map_or(None, |number| Some(number.into()))); + } + if value.is_instance_of::() { + return Ok(Number::from_f64(value.extract::()?).map(Value::Number)); + } + if let Ok(text) = value.downcast::() { + return Ok(Some(Value::String(escaped_text(text)?))); + } + if value.is_instance_of::() || value.is_instance_of::() { + let mut items: Vec = Vec::new(); + for item in value.try_iter()? { + let Some(item) = json_value(&item?)? else { + return Ok(None); + }; + items.push(item); + } + return Ok(Some(Value::Array(items))); + } + if let Ok(dict) = value.downcast::() { + return json_items(dict); + } + if let Ok(mapping) = value.downcast::() { + let dict: Bound<'_, PyDict> = PyDict::new(value.py()); + dict.update(mapping)?; + return json_items(&dict); + } + Ok(None) +} + +/// `occurred_at` as microsecond ISO-8601 with `Z`, when it is a UTC datetime. +fn utc_timestamp(value: &Bound<'_, PyAny>) -> PyResult> { + let zone: Bound<'_, PyAny> = native_field(value, "tzinfo")?; + if text(&zone)?.as_deref() != Some(UTC_ZONE_NAME) { + return Ok(None); + } + let mut parts: [i64; 7] = [0; 7]; + for (part, name) in parts.iter_mut().zip(TIMESTAMP_PARTS) { + *part = native_field(value, name)?.extract()?; + } + let [year, month, day, hour, minute, second, microsecond] = parts; + Ok(Some(format!( + "{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}.{microsecond:06}Z" + ))) +} + +/// The canonical JSON object of a known lifecycle event; `None` when only Python can encode it. +pub(crate) fn event_object(event: &Bound<'_, PyAny>) -> PyResult>> { + if !event.is_instance(&host::entry(event.py(), ExecutorHost::LifecycleEvent)?)? { + return Ok(None); + } + let mut object: Map = Map::new(); + for name in EVENT_TEXT_FIELDS { + object.insert(name.to_owned(), text(&native_field(event, name)?)?.into()); + } + for name in EVENT_OPTIONAL_TEXT_FIELDS { + object.insert(name.to_owned(), text(&native_field(event, name)?)?.into()); + } + let schema_version: i64 = native_field(event, "schema_version")?.extract()?; + object.insert("schema_version".to_owned(), schema_version.into()); + let Some(occurred_at) = utc_timestamp(&native_field(event, "occurred_at")?)? else { + return Ok(None); + }; + object.insert("occurred_at".to_owned(), occurred_at.into()); + let Some(payload) = json_value(&native_field(event, "payload")?)? else { + return Ok(None); + }; + object.insert("payload".to_owned(), payload); + if schema_version >= INVOCATION_METADATA_SCHEMA_VERSION { + let sequence: Option = native_field(event, "invocation_sequence")?.extract()?; + object.insert("invocation_sequence".to_owned(), sequence.into()); + let Some(context) = json_value(&native_field(event, "external_context")?)? else { + return Ok(None); + }; + object.insert("external_context".to_owned(), context); + } + Ok(Some(object)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs new file mode 100644 index 0000000000..e05fcbcf73 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod command; +pub(crate) mod console; +pub(crate) mod constants; +pub(crate) mod events; +pub(crate) mod registration; +pub(crate) mod rendering; +pub(crate) mod session; +pub(crate) mod sinks; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs new file mode 100644 index 0000000000..6e7568ce35 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/registration.rs @@ -0,0 +1,9 @@ +//! Register the native Recall classes. + +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; + +use crate::bindings::_helpers::recall::sinks::RecallLifecycleSink; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs new file mode 100644 index 0000000000..40b51110f2 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs @@ -0,0 +1,254 @@ +//! `sqb recall query` and `schema` output: text cells, CSV and the JSON document. + +use serde_json::{Map, Value, json}; +use sqlbuild_core::query_command::main::render_query_csv::render_query_csv; +use sqlbuild_core::query_command::main::render_query_long::render_query_long; +use sqlbuild_core::query_command::main::render_query_table::render_query_table; +use sqlbuild_core::query_command::models::QueryDisplay; +use sqlbuild_recall::records::models::RecallValue; +use sqlbuild_recall::store::models::{DescribedColumn, DescribedRelation, QueryResult}; + +use crate::bindings::_helpers::recall::constants::{ + BASE64_ALPHABET, CSV_FORMAT, JOIN_KEYS, JSON_FORMAT, TABLE_FORMAT, +}; + +fn base64(data: &[u8]) -> String { + let mut encoded = String::with_capacity(data.len().div_ceil(3) * 4); + for chunk in data.chunks(3) { + let bytes = [ + chunk[0], + *chunk.get(1).unwrap_or(&0), + *chunk.get(2).unwrap_or(&0), + ]; + let triple = u32::from(bytes[0]) << 16 | u32::from(bytes[1]) << 8 | u32::from(bytes[2]); + for index in 0..4 { + if index <= chunk.len() { + encoded.push(char::from( + BASE64_ALPHABET[(triple >> (18 - 6 * index) & 63) as usize], + )); + } else { + encoded.push('='); + } + } + } + encoded +} + +fn truncated(text: &str, max_bytes: Option) -> String { + let Some(limit) = max_bytes.filter(|limit| text.len() > *limit) else { + return text.to_owned(); + }; + let mut end = limit; + while !text.is_char_boundary(end) { + end -= 1; + } + format!("{}…[+{} bytes]", &text[..end], text.len() - end) +} + +/// A cell as text formats show it; `null` is the text for SQL NULL. +pub(crate) fn display_cell(value: &RecallValue, null: &str, max_bytes: Option) -> String { + match value { + RecallValue::Null => null.to_owned(), + RecallValue::Integer(number) => number.to_string(), + RecallValue::Real(number) => format!("{number:?}"), + RecallValue::Text(text) => truncated(text, max_bytes), + RecallValue::Blob(data) => match std::str::from_utf8(data) { + Ok(text) => truncated(text, max_bytes), + Err(_) => format!("<{} bytes of binary data>", data.len()), + }, + } +} + +fn display_row(row: &[RecallValue], null: &str, max_bytes: Option) -> Vec { + let mut cells: Vec = Vec::with_capacity(row.len()); + for value in row { + cells.push(display_cell(value, null, max_bytes)); + } + cells +} + +fn column_json(column: &DescribedColumn) -> Value { + json!({ + "name": column.name, + "type": column.declared_type, + "nullable": column.nullable, + }) +} + +fn json_cell(value: &RecallValue) -> Value { + match value { + RecallValue::Null => Value::Null, + RecallValue::Integer(number) => json!(number), + RecallValue::Real(number) => json!(number), + RecallValue::Text(text) => json!(text), + RecallValue::Blob(data) => match std::str::from_utf8(data) { + Ok(text) => json!(text), + Err(_) => json!({ "base64": base64(data) }), + }, + } +} + +/// What the JSON document reports besides rows. +pub(crate) struct QueryContext<'a> { + pub(crate) target: Option<&'a str>, + pub(crate) store_path: &'a str, + pub(crate) limit: Option, + pub(crate) elapsed_ms: f64, +} + +/// The rendered result in `format`: `long`, `table`, `csv` or `json`. +pub(crate) fn render_result( + result: &QueryResult, + format: &str, + max_bytes: Option, + context: &QueryContext<'_>, +) -> String { + if format == JSON_FORMAT { + let rows: Vec = result + .rows + .iter() + .map(|row| Value::Array(row.iter().map(json_cell).collect())) + .collect(); + let document = json!({ + "backend": "logs", + "target": context.target, + "store": {"adapter": "sqlite", "database": context.store_path}, + "columns": result.columns, + "rows": rows, + "row_count": result.rows.len(), + "truncated": result.truncated, + "limit": context.limit, + "statistics": { + "elapsed_ms": context.elapsed_ms, + "rows_read": Value::Null, + "bytes_read": Value::Null, + "query_id": Value::Null, + }, + }); + return format!("{document}\n"); + } + let csv: bool = format == CSV_FORMAT; + let null: &str = if csv { "" } else { "NULL" }; + let cells: Vec> = result + .rows + .iter() + .map(|row| display_row(row, null, max_bytes)) + .collect(); + if csv { + return render_query_csv(&result.columns, &cells); + } + let display = QueryDisplay { + columns: result.columns.clone(), + rows: cells, + truncated: result.truncated, + limit: context + .limit + .map(|limit| i64::try_from(limit).unwrap_or(i64::MAX)), + }; + if format == TABLE_FORMAT { + render_query_table(&display) + } else { + render_query_long(&display) + } +} + +fn relation_note(name: &str) -> &'static str { + match name { + "invocations" => { + "one row per invocation; status is completed, failed, running, abandoned or unknown" + } + "statements" => "one row per statement; outcome no_terminal means no terminal fact arrived", + "audit_results" => "one row per evaluated audit", + "output_invocations" => { + "output completeness per invocation: gaps, drops, filtering, truncation, flush" + } + "output_tail" => "the last 200 output chunks of each invocation as text", + "output_transcripts" => "exact stdout, stderr and combined bytes per invocation", + "events" => "every lifecycle fact", + "diagnostics" => "structured diagnostics at or above the configured severity", + "artifacts" => "captured SQL and other artifacts, by reference", + "conflicts" => "record keys reused with different content; the first row is kept", + _ => "", + } +} + +/// The schema document: relations with columns, join keys and time semantics. +pub(crate) fn render_schema( + relations: &[DescribedRelation], + json_output: bool, + (store_path, retention_days): (&str, u32), +) -> String { + let filter_hint = + "filter invocation_started_at to bound scans; all rows of an invocation expire together"; + if json_output { + let items: Vec = relations + .iter() + .map(|relation| { + let mut item = Map::new(); + item.insert("name".to_owned(), json!(relation.name)); + item.insert("kind".to_owned(), json!(relation.kind)); + item.insert( + "description".to_owned(), + json!(relation_note(&relation.name)), + ); + item.insert( + "join_keys".to_owned(), + json!( + relation + .columns + .iter() + .filter(|column| JOIN_KEYS.contains(&column.name.as_str())) + .map(|column| column.name.clone()) + .collect::>() + ), + ); + item.insert( + "columns".to_owned(), + Value::Array(relation.columns.iter().map(column_json).collect()), + ); + Value::Object(item) + }) + .collect(); + let document = json!({ + "backend": "logs", + "dialect": "sqlite", + "store": {"adapter": "sqlite", "database": store_path}, + "retention_days": retention_days, + "time_semantics": "timestamps are ISO-8601 UTC text with microseconds", + "bounds": filter_hint, + "relations": items, + }); + return format!("{document}\n"); + } + let mut lines: Vec = vec![ + format!("Recall store {store_path} (SQLite dialect, retention {retention_days} days)"), + "Timestamps are ISO-8601 UTC text with microseconds; ".to_owned() + filter_hint + ".", + ]; + for relation in relations { + lines.push(String::new()); + let note = relation_note(&relation.name); + lines.push(if note.is_empty() { + format!("{} ({})", relation.name, relation.kind) + } else { + format!("{} ({}): {note}", relation.name, relation.kind) + }); + for column in &relation.columns { + let key = if JOIN_KEYS.contains(&column.name.as_str()) { + " join key" + } else { + "" + }; + let nullability = if column.nullable { "" } else { " not null" }; + lines.push(format!( + " {} {}{nullability}{key}", + column.name, + if column.declared_type.is_empty() { + "ANY" + } else { + &column.declared_type + } + )); + } + } + format!("{}\n", lines.join("\n")) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs new file mode 100644 index 0000000000..179ad65510 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -0,0 +1,454 @@ +//! The invocation's Recall session: started before the command runs and closed after it returns. + +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime}; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_cli::dispatch::models::CommandFacts; +use sqlbuild_recall::records::main::format_timestamp::format_timestamp; +use sqlbuild_recall::records::main::sha256_hex::sha256_hex; +use sqlbuild_recall::records::models::{InvocationContext, OutputStream, RowEnvelope}; +use sqlbuild_recall::schema::main::ddl_script::ddl_script; +use sqlbuild_recall::schema::models::{DdlOptions, Dialect}; +use sqlbuild_recall::settings::main::resolve_recall_settings::resolve_recall_settings; +use sqlbuild_recall::settings::models::{RecallSettings, SettingsRequest, StoreLocation}; +use sqlbuild_recall::store::main::observe_store_file::observe_store_file; +use sqlbuild_recall::store::main::quarantine_store::quarantine_store; +use sqlbuild_recall::store::main::store_location_problem::store_location_problem; +use sqlbuild_recall::store::models::QuarantineOutcome; +use sqlbuild_recall::writer::constants::DEFAULT_HEARTBEAT_INTERVAL_MS; +use sqlbuild_recall::writer::main::start_recall_session::start_recall_session; +use sqlbuild_recall::writer::models::{ + DeliverySummary, RecallSession, SessionConfig, SetupOutcome, +}; + +use crate::bindings::_helpers::native_host::executor::ExecutorHost; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::output_capture::records::invocation_context; +use crate::bindings::_helpers::recall::console::write_stderr; +use crate::bindings::_helpers::recall::constants::{ + CLOSE_POLL_MS, DDL_NOTICE_PREFIX, HEARTBEAT_ENVIRONMENT, HOST_TOKEN_LENGTH, + MINIMUM_HEARTBEAT_SECONDS, NOTICE_INTERVAL_SECONDS, QUARANTINE_START_WAIT_MS, + STDOUT_STREAM_INDEX, +}; +use crate::bindings::_helpers::recall::events::escaped_text; +use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::rules::config_json::dumped_json; + +/// What the invocation records: its session, when enabled, and the resolved target and adapter. +#[derive(Clone, Debug, Default)] +pub(crate) struct InvocationRecall { + pub(crate) session: Option>, + pub(crate) target: Option, + pub(crate) adapter: Option, + /// The command name, for the closing status line. + pub(crate) command: String, + /// The resolved settings and project directory, for notices at the end of the run. + pub(crate) settings: Option<(RecallSettings, PathBuf)>, +} + +/// The text of an optional attribute value. +fn text_value(value: &Bound<'_, PyAny>) -> PyResult> { + if value.is_none() { + return Ok(None); + } + escaped_text(&value.str()?).map(Some) +} + +fn list_attribute(args: &Bound<'_, PyAny>, name: &str) -> PyResult> { + let value: Bound<'_, PyAny> = args.getattr(name)?; + let mut items: Vec = Vec::new(); + if value.is_none() { + return Ok(items); + } + for item in value.try_iter()? { + items.push(escaped_text(&item?.str()?)?); + } + Ok(items) +} + +fn environment(name: &str) -> Option { + std::env::var_os(name).map(|value| value.to_string_lossy().into_owned()) +} + +/// `path`'s text when it exists; Recall metadata is best effort, so unreadable means unknown. +fn file_text(path: &Path) -> Option { + match std::fs::read_to_string(path) { + Ok(text) => Some(text), + Err(error) if error.kind() == ErrorKind::NotFound => None, + Err(_unreadable) => None, + } +} + +fn project_dir(args: &Bound<'_, PyAny>) -> PyResult { + let raw: PathBuf = match text_value(&args.getattr("project_dir")?)? { + Some(directory) => PathBuf::from(directory), + None => std::env::current_dir()?, + }; + Ok(std::fs::canonicalize(&raw).unwrap_or(raw)) +} + +/// The commit `HEAD` names, read from the repository files without running git. +fn git_revision(project_dir: &Path) -> Option { + let git: PathBuf = project_dir + .ancestors() + .map(|directory| directory.join(".git")) + .find(|candidate| candidate.is_dir())?; + let head: String = file_text(&git.join("HEAD"))?; + let Some(reference) = head.trim().strip_prefix("ref: ") else { + return Some(head.trim().to_owned()); + }; + if let Some(revision) = file_text(&git.join(reference)) { + return Some(revision.trim().to_owned()); + } + file_text(&git.join("packed-refs"))? + .lines() + .filter_map(|line| line.split_once(' ')) + .find(|(_, name)| *name == reference) + .map(|(revision, _)| revision.to_owned()) +} + +fn host_token() -> Option { + let name: String = file_text(Path::new("/etc/hostname")) + .or_else(|| environment("COMPUTERNAME")) + .or_else(|| environment("HOSTNAME"))?; + Some(sha256_hex(name.trim().as_bytes())[..HOST_TOKEN_LENGTH].to_owned()) +} + +/// The orchestrator context as JSON; an invalid environment context records none. +fn external_context(py: Python<'_>) -> PyResult> { + let context: Bound<'_, PyAny> = match invocation_context(py, None) { + Ok(context) => context, + Err(error) + if error.is_instance(py, &host::entry(py, ExecutorHost::OutputCaptureInputError)?) => + { + return Ok(None); + } + Err(error) => return Err(error), + }; + if !context.is_truthy()? { + return Ok(None); + } + Ok(dumped_json(&context)?.map(|encoded| encoded.to_string())) +} + +fn heartbeat_interval() -> Duration { + let default = Duration::from_millis(DEFAULT_HEARTBEAT_INTERVAL_MS); + let Some(value) = environment(HEARTBEAT_ENVIRONMENT) else { + return default; + }; + match value.parse::() { + Ok(seconds) if seconds.is_finite() && seconds >= MINIMUM_HEARTBEAT_SECONDS => { + Duration::from_secs_f64(seconds) + } + Ok(_) | Err(_) => default, + } +} + +fn invocation_facts( + py: Python<'_>, + (args, facts): (&Bound<'_, PyAny>, &CommandFacts), + (settings, project_dir): (&RecallSettings, &Path), +) -> PyResult { + let fingerprint_source = format!( + "{}\u{0}{}\u{0}{}", + settings.project_adapter.as_deref().unwrap_or_default(), + settings.destination_database.as_deref().unwrap_or_default(), + settings.destination_schema.as_deref().unwrap_or_default() + ); + Ok(InvocationContext { + command: facts.command.clone(), + subcommand: facts.subcommand.clone(), + selection: list_attribute(args, "select")?, + exclusion: list_attribute(args, "exclude")?, + project_name: settings.project_name.clone(), + project_dir_hash: Some(sha256_hex(project_dir.to_string_lossy().as_bytes())), + target: settings.target.clone(), + adapter: settings.project_adapter.clone(), + destination_database: settings.destination_database.clone(), + destination_schema: settings.destination_schema.clone(), + destination_fingerprint: Some(sha256_hex(fingerprint_source.as_bytes())), + sqlbuild_version: Some(env!("CARGO_PKG_VERSION").to_owned()), + git_revision: git_revision(project_dir), + git_dirty: None, + host_token: host_token(), + pid: Some(std::process::id()), + external_context: external_context(py)?, + recall_filter_active: false, + argv_digest: None, + }) +} + +/// Whether a notice for `store` is due, recording it best effort; a missing marker is due. +fn notice_due(project_dir: &Path, store: &str) -> bool { + let directory: PathBuf = project_dir.join("target"); + let marker: PathBuf = directory.join(format!( + "{DDL_NOTICE_PREFIX}{}", + &sha256_hex(store.as_bytes())[..HOST_TOKEN_LENGTH] + )); + let modified: std::io::Result = + std::fs::metadata(&marker).and_then(|metadata| metadata.modified()); + if let Ok(modified) = modified + && SystemTime::now() + .duration_since(modified) + .is_ok_and(|age| age < Duration::from_secs(NOTICE_INTERVAL_SECONDS)) + { + return false; + } + drop( + std::fs::create_dir_all(&directory) + .and_then(|()| std::fs::write(&marker, format_timestamp(SystemTime::now()))), + ); + true +} + +fn target_phrase(settings: &RecallSettings) -> String { + settings + .target + .as_deref() + .map_or_else(String::new, |target| format!(" for target `{target}`")) +} + +/// The DDL notice for an unwritable store, when one is due for it. +fn permission_notice( + (settings, project_dir): (&RecallSettings, &Path), + reason: &str, +) -> Option { + let StoreLocation::Sqlite { path } = &settings.store else { + return None; + }; + if !notice_due(project_dir, &path.to_string_lossy()) { + return None; + } + let script = ddl_script( + Dialect::Sqlite, + &DdlOptions { + database: None, + retention_days: settings.retention_days, + cluster: None, + }, + ); + Some(format!( + "Recall history is not being written{}: {reason}. An administrator can create the \ + store at {} with:\n\n{script}\n", + target_phrase(settings), + path.display() + )) +} + +fn unsupported_notice( + (settings, project_dir): (&RecallSettings, &Path), + (connection, adapter): (&str, &str), +) -> Option { + notice_due(project_dir, connection).then(|| { + format!( + "Recall history is not being written{}: connection `{connection}` uses adapter \ + `{adapter}`, which this version cannot store Recall history in.\n", + target_phrase(settings) + ) + }) +} + +/// Write `text` to stderr; a notice that cannot be written never changes the command's outcome. +fn notify(py: Python<'_>, text: Option) { + if let Some(text) = text { + drop(write_stderr(py, &text)); + } +} + +/// Move a store whose file is not a database aside before the command prints anything. +fn quarantined_notice(path: &Path) -> Option { + let Ok(Some(observed)) = observe_store_file(path) else { + return None; + }; + if !observed.header_foreign { + return None; + } + let give_up: Instant = Instant::now() + Duration::from_millis(QUARANTINE_START_WAIT_MS); + match quarantine_store(path, &observed, (give_up, SystemTime::now())) { + Ok(QuarantineOutcome::Moved(moved)) => Some(format!( + "Recall: {} was not a valid database; it was moved to {} and a fresh store was \ + started.\n", + path.display(), + moved.display() + )), + Ok(QuarantineOutcome::Kept | QuarantineOutcome::LockBusy) => None, + Err(error) => Some(format!("Recall history is not being written: {error}.\n")), + } +} + +fn started_session( + py: Python<'_>, + (args, identity, facts): (&Bound<'_, PyAny>, &Bound<'_, PyAny>, &CommandFacts), + (settings, project_dir): (&RecallSettings, &Path), +) -> PyResult>> { + let store_path: PathBuf = match &settings.store { + StoreLocation::Sqlite { path } => path.clone(), + StoreLocation::Unsupported { + connection, + adapter, + } => { + notify( + py, + unsupported_notice((settings, project_dir), (connection, adapter)), + ); + return Ok(None); + } + }; + notify(py, quarantined_notice(&store_path)); + if let Some(reason) = store_location_problem(&store_path) { + notify(py, permission_notice((settings, project_dir), &reason)); + return Ok(None); + } + let Some(invocation_id) = text_value(&native_field(identity, "invocation_id")?)? else { + return Ok(None); + }; + let context = invocation_facts(py, (args, facts), (settings, project_dir))?; + let session = start_recall_session(SessionConfig { + store_path, + maintenance_owner: format!( + "{}:{}", + context.host_token.as_deref().unwrap_or("host"), + std::process::id() + ), + envelope: RowEnvelope { + invocation_id, + invocation_started_at: format_timestamp(SystemTime::now()), + producer_version: env!("CARGO_PKG_VERSION").to_owned(), + }, + context, + retention_days: settings.retention_days, + shutdown_timeout: Duration::from_millis(settings.shutdown_timeout_ms), + diagnostics_min_severity: settings.diagnostics_min_severity.clone(), + max_output_bytes: settings.max_output_bytes, + heartbeat_interval: heartbeat_interval(), + setup_budget: Duration::ZERO, + maintenance: true, + }); + Ok(Some(Arc::new(session))) +} + +/// Begin recording when Recall is enabled, without waiting on the store; invalid settings record nothing and the command reports them. +pub(crate) fn start_invocation_recall( + py: Python<'_>, + args: &Bound<'_, PyAny>, + identity: &Bound<'_, PyAny>, + facts: &CommandFacts, +) -> PyResult { + let project_dir: PathBuf = project_dir(args)?; + let request = SettingsRequest { + project_dir: project_dir.clone(), + selected_target: text_value(&args.getattr("target")?)?, + }; + let Ok(settings) = resolve_recall_settings(&request) else { + return Ok(InvocationRecall::default()); + }; + let session: Option> = if settings.enabled && settings.project_name.is_some() + { + started_session(py, (args, identity, facts), (&settings, &project_dir))? + } else { + None + }; + Ok(InvocationRecall { + session, + target: settings.target.clone(), + adapter: settings.project_adapter.clone(), + command: facts.command.clone().unwrap_or_default(), + settings: Some((settings, project_dir)), + }) +} + +/// The closing sentence after a Recall notice, so the command's own outcome stays the last word. +fn final_status(command: &str, exit_code: Option) -> String { + match exit_code { + Some(0) => { + format!("sqb {command} completed successfully; only its Recall history is affected.\n") + } + Some(code) => format!("sqb {command} failed with exit code {code}.\n"), + None => format!("sqb {command} failed.\n"), + } +} + +/// What Recall must tell the user about this run once delivery has finished. +fn closing_notices(recall: &InvocationRecall, session: &RecallSession) -> Vec { + let mut notices: Vec = session + .notices() + .into_iter() + .map(|notice| format!("Recall: {notice}.\n")) + .collect(); + if let (Some(SetupOutcome::PermissionDenied { reason }), Some((settings, project_dir))) = + (session.wait_for_setup(Duration::ZERO), &recall.settings) + { + notices.extend(permission_notice((settings, project_dir), &reason)); + return notices; + } + let summary: DeliverySummary = session.summary(); + let total = summary.total(); + if total.dropped > 0 || total.failed > 0 { + let reason = summary + .degraded_reason + .as_deref() + .map_or_else(String::new, |reason| format!(": {reason}")); + notices.push(format!( + "warning: Recall history for this run is incomplete ({} dropped, {} failed){reason}\n", + total.dropped, total.failed + )); + } + notices +} + +/// Close the session within its shutdown budget; a further interrupt stops waiting at once. Notices end with the command's own outcome. +pub(crate) fn finish_invocation_recall( + py: Python<'_>, + recall: &InvocationRecall, + exit_code: Option, +) -> PyResult<()> { + let Some(session) = &recall.session else { + return Ok(()); + }; + session.begin_close(); + while !py.detach(|| session.wait_closed(Duration::from_millis(CLOSE_POLL_MS))) { + if let Err(interrupt) = py.check_signals() { + session.abandon(); + return Err(interrupt); + } + } + let notices: Vec = closing_notices(recall, session); + if !notices.is_empty() { + notify( + py, + Some(notices.concat() + &final_status(&recall.command, exit_code)), + ); + } + Ok(()) +} + +/// Forward bytes a process stream accepted to `session` when it records `invocation_id`. +pub(crate) fn forward_output( + session: Option<&RecallSession>, + invocation_id: &str, + (stream, data): (usize, &[u8]), +) { + let Some(session) = session.filter(|session| session.invocation_id() == invocation_id) else { + return; + }; + let stream = if stream == STDOUT_STREAM_INDEX { + OutputStream::Stdout + } else { + OutputStream::Stderr + }; + session.append_output(stream, data); +} + +/// Forward one diagnostic's JSON to `session` for `invocation_id`; unparsable ones are skipped. +pub(crate) fn forward_diagnostic( + session: Option<&RecallSession>, + invocation_id: &str, + diagnostic_json: &str, +) { + if let Some(session) = session.filter(|session| session.invocation_id() == invocation_id) { + drop(session.publish_diagnostic(diagnostic_json)); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs new file mode 100644 index 0000000000..7373814272 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs @@ -0,0 +1,72 @@ +//! The native subscriber that publishes every lifecycle fact and artifact into Recall. + +use std::sync::Arc; + +use pyo3::exceptions::PyException; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; +use pyo3::types::PyString; +use sqlbuild_recall::writer::models::RecallSession; + +use crate::bindings::_helpers::native_host::executor::ExecutorHost; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::recall::events::{escaped_text, event_object}; + +/// Publishes lifecycle events and SQL artifacts of the active session; delivery is best effort. +#[pyclass(module = "sqlbuild._native", frozen)] +pub(crate) struct RecallLifecycleSink { + pub(crate) session: Arc, +} + +#[pymethods] +impl RecallLifecycleSink { + /// Record `event`; an event Recall cannot encode is skipped, never raised into the command. + fn __call__(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { + recording(py, self.publish(py, event)) + } + + /// Record one SQL artifact; text Recall cannot encode is skipped, never raised into the command. + #[pyo3(signature = (*, sql, statement_id=None, resource_id=None))] + fn record_sql( + &self, + py: Python<'_>, + sql: &Bound<'_, PyString>, + statement_id: Option<&Bound<'_, PyString>>, + resource_id: Option<&Bound<'_, PyString>>, + ) -> PyResult<()> { + let texts = (|| -> PyResult<(String, Option, Option)> { + Ok(( + escaped_text(sql)?, + statement_id.map(escaped_text).transpose()?, + resource_id.map(escaped_text).transpose()?, + )) + })(); + recording( + py, + texts.map(|(sql, statement_id, resource_id)| { + self.session + .publish_sql(&sql, statement_id.as_deref(), resource_id.as_deref()); + }), + ) + } +} + +impl RecallLifecycleSink { + fn publish(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { + if let Some(object) = event_object(event)? { + self.session.publish_event_object(object); + return Ok(()); + } + let encoded = host::entry(py, ExecutorHost::LifecycleEventToJson)?.call1((event,))?; + let text = escaped_text(encoded.downcast::()?)?; + drop(self.session.publish_event(&text)); + Ok(()) + } +} + +/// `outcome` with ordinary exceptions dropped, so recording never changes what the command raises; interrupts still propagate. +fn recording(py: Python<'_>, outcome: PyResult<()>) -> PyResult<()> { + match outcome { + Err(error) if error.is_instance_of::(py) => Ok(()), + other => other, + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 9fe49eb2dd..39f8e07f68 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -941,6 +941,7 @@ const CLI_ENTRYPOINT_HANDLERS: RecordSpec = RecordSpec { FieldSpec::required("run_scope"), FieldSpec::new("run_contract", FieldDefault::None), FieldSpec::new("run_refactor", FieldDefault::None), + FieldSpec::new("run_recall", FieldDefault::None), ], }; const COMMAND_WAREHOUSE_SCOPE: RecordSpec = RecordSpec { @@ -1531,6 +1532,7 @@ const CLI_COMMAND: EnumSpec = EnumSpec { ("CONTRACT", "contract"), ("RENAME", "rename"), ("MV", "mv"), + ("RECALL", "recall"), ], }; const PLAYGROUND_TEMPLATE: EnumSpec = EnumSpec { diff --git a/crates/sqlbuild-recall/Cargo.toml b/crates/sqlbuild-recall/Cargo.toml new file mode 100644 index 0000000000..5738b05067 --- /dev/null +++ b/crates/sqlbuild-recall/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "sqlbuild-recall" +version.workspace = true +description = "Recall: invocation history records, their SQLite and ClickHouse schema, settings and the local store." +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true + +[dependencies] +blake3.workspace = true +rusqlite.workspace = true +serde_json.workspace = true +sha2.workspace = true +sqlbuild-config.workspace = true +sqlbuild-core.workspace = true + +[target.'cfg(unix)'.dependencies] +rustix.workspace = true + +[dev-dependencies] +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/sqlbuild-recall/src/lib.rs b/crates/sqlbuild-recall/src/lib.rs new file mode 100644 index 0000000000..53e5776303 --- /dev/null +++ b/crates/sqlbuild-recall/src/lib.rs @@ -0,0 +1,9 @@ +//! Recall: the records of every invocation, their SQLite and ClickHouse schema and views, retention, the `[recall]` settings, the local SQLite store and its delivery thread. + +#![forbid(unsafe_code)] + +pub mod records; +pub mod schema; +pub mod settings; +pub mod store; +pub mod writer; diff --git a/crates/sqlbuild-recall/src/records/_helpers/events.rs b/crates/sqlbuild-recall/src/records/_helpers/events.rs new file mode 100644 index 0000000000..904f592754 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/_helpers/events.rs @@ -0,0 +1,113 @@ +//! The envelope and promoted payload columns of a lifecycle event row. + +use serde_json::{Map, Value}; +use sqlbuild_core::event_export::main::lifecycle_export_policy::lifecycle_export_policy; +use sqlbuild_core::lifecycle_events::main::catalog::catalog; + +use crate::records::_helpers::rows::{optional_text, text}; +use crate::records::constants::{ + CORRELATION_FIELDS, PROMOTED_INTEGER_FIELDS, PROMOTED_REAL_FIELDS, PROMOTED_TEXT_FIELDS, +}; +use crate::records::errors::RecordError; +use crate::records::models::RecallValue; + +/// One named column value. +pub(crate) type Column = (&'static str, RecallValue); + +fn required_text<'a>(event: &'a Map, field: &str) -> Result<&'a str, RecordError> { + event + .get(field) + .and_then(Value::as_str) + .ok_or_else(|| RecordError::new(format!("lifecycle event {field} must be a string"))) +} + +fn is_terminal(event_type: &str, schema_version: i64) -> bool { + catalog(schema_version) + .into_iter() + .flatten() + .any(|definition| definition.event_type == event_type && definition.terminal) +} + +/// The event JSON object. +pub(crate) fn event_object(event_json: &str) -> Result, RecordError> { + match serde_json::from_str(event_json) { + Ok(Value::Object(event)) => Ok(event), + Ok(_) => Err(RecordError::new("lifecycle event must be a JSON object")), + Err(error) => Err(RecordError::new(format!( + "lifecycle event is not JSON: {error}" + ))), + } +} + +/// The export severity of an event's type; `None` without a text `event_type`. +pub(crate) fn severity_of(event: &Map) -> Option<&'static str> { + let event_type: &str = event.get("event_type").and_then(Value::as_str)?; + let schema_version: i64 = event + .get("schema_version") + .and_then(Value::as_i64) + .unwrap_or_default(); + Some( + lifecycle_export_policy(event_type, is_terminal(event_type, schema_version)) + .severity + .value(), + ) +} + +/// Identity, type, export policy and correlation columns. +pub(crate) fn envelope_columns(event: &Map) -> Result, RecordError> { + let event_type: &str = required_text(event, "event_type")?; + let schema_version: i64 = event + .get("schema_version") + .and_then(Value::as_i64) + .unwrap_or_default(); + let policy = lifecycle_export_policy(event_type, is_terminal(event_type, schema_version)); + let mut columns: Vec = vec![ + ("event_id", text(required_text(event, "event_id")?)), + ( + "invocation_sequence", + event + .get("invocation_sequence") + .and_then(Value::as_i64) + .map_or(RecallValue::Null, RecallValue::Integer), + ), + ("occurred_at", text(required_text(event, "occurred_at")?)), + ("event_type", text(event_type)), + ("event_kind", text(policy.kind)), + ("severity", text(policy.severity.value())), + ("producer", text(required_text(event, "producer")?)), + ("event_schema_version", RecallValue::Integer(schema_version)), + ]; + columns.extend(CORRELATION_FIELDS.iter().map(|field| { + ( + *field, + optional_text(event.get(*field).and_then(Value::as_str)), + ) + })); + Ok(columns) +} + +/// Payload fields promoted to columns, then the payload itself. +pub(crate) fn payload_columns(payload: &Map) -> Vec { + let mut columns: Vec = PROMOTED_TEXT_FIELDS + .iter() + .map(|field| { + ( + *field, + optional_text(payload.get(*field).and_then(Value::as_str)), + ) + }) + .collect(); + columns.extend(PROMOTED_INTEGER_FIELDS.iter().map(|field| { + let value = payload.get(*field).and_then(Value::as_i64); + ( + *field, + value.map_or(RecallValue::Null, RecallValue::Integer), + ) + })); + columns.extend(PROMOTED_REAL_FIELDS.iter().map(|field| { + let value = payload.get(*field).and_then(Value::as_f64); + (*field, value.map_or(RecallValue::Null, RecallValue::Real)) + })); + columns.push(("payload", text(Value::Object(payload.clone()).to_string()))); + columns +} diff --git a/crates/sqlbuild-recall/src/records/_helpers/ids.rs b/crates/sqlbuild-recall/src/records/_helpers/ids.rs new file mode 100644 index 0000000000..97ef2a2302 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/_helpers/ids.rs @@ -0,0 +1,57 @@ +//! Deterministic record IDs and content digests. + +use crate::records::constants::{ + CONTENT_HASH_BUFFER_BYTES, DIAGNOSTIC_RECORD_ID_INFIX, OUTPUT_RECORD_ID_INFIX, +}; +use crate::records::models::RecallValue; +use crate::schema::constants::UNHASHED_COLUMNS; + +/// `:command-output:`, the command-output sink record ID scheme. +pub(crate) fn output_record_id(invocation_id: &str, sequence: u64) -> String { + format!("{invocation_id}:{OUTPUT_RECORD_ID_INFIX}:{sequence}") +} + +/// `:diagnostic:`. +pub(crate) fn diagnostic_record_id(invocation_id: &str, sequence: u64) -> String { + format!("{invocation_id}:{DIAGNOSTIC_RECORD_ID_INFIX}:{sequence}") +} + +fn framed(mut buffer: Vec, tag: u8, bytes: &[u8]) -> Vec { + buffer.push(tag); + buffer.extend_from_slice(&(bytes.len() as u64).to_le_bytes()); + buffer.extend_from_slice(bytes); + buffer +} + +fn valued(mut buffer: Vec, value: &RecallValue) -> Vec { + match value { + RecallValue::Null => buffer.push(b'0'), + RecallValue::Integer(number) => { + buffer.push(b'i'); + buffer.extend_from_slice(&number.to_le_bytes()); + } + RecallValue::Real(number) => { + buffer.push(b'r'); + buffer.extend_from_slice(&number.to_bits().to_le_bytes()); + } + RecallValue::Text(text) => return framed(buffer, b't', text.as_bytes()), + RecallValue::Blob(data) => return framed(buffer, b'b', data), + } + buffer +} + +/// Hex BLAKE3 over every non-NULL named value except `ingested_at` and `content_hash`, by column name; a NULL column hashes like an absent one. +pub(crate) fn content_hash(values: &[(&str, RecallValue)]) -> String { + let mut sorted: Vec<&(&str, RecallValue)> = values + .iter() + .filter(|(name, value)| { + !UNHASHED_COLUMNS.contains(name) && !matches!(value, RecallValue::Null) + }) + .collect(); + sorted.sort_by(|left, right| left.0.cmp(right.0)); + let encoded: Vec = sorted.into_iter().fold( + Vec::with_capacity(CONTENT_HASH_BUFFER_BYTES), + |buffer, (name, value)| valued(framed(buffer, b'n', name.as_bytes()), value), + ); + blake3::hash(&encoded).to_hex().to_string() +} diff --git a/crates/sqlbuild-recall/src/records/_helpers/mod.rs b/crates/sqlbuild-recall/src/records/_helpers/mod.rs new file mode 100644 index 0000000000..eaf86fcc66 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod events; +pub(crate) mod ids; +pub(crate) mod rows; +pub(crate) mod time; diff --git a/crates/sqlbuild-recall/src/records/_helpers/rows.rs b/crates/sqlbuild-recall/src/records/_helpers/rows.rs new file mode 100644 index 0000000000..5ad2768b37 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/_helpers/rows.rs @@ -0,0 +1,54 @@ +//! Assemble a row: the dataset's values, the common columns and the content digest. + +use crate::records::_helpers::ids::content_hash; +use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; +use crate::schema::constants::{ARTIFACT_CONTENT_COLUMNS, RECALL_SCHEMA_VERSION}; +use crate::schema::models::Table; + +pub(crate) fn text(value: impl Into) -> RecallValue { + RecallValue::Text(value.into()) +} + +pub(crate) fn optional_text(value: Option<&str>) -> RecallValue { + value.map_or(RecallValue::Null, |value| { + RecallValue::Text(value.to_owned()) + }) +} + +pub(crate) fn integer(value: u64) -> RecallValue { + RecallValue::Integer(i64::try_from(value).unwrap_or(i64::MAX)) +} + +pub(crate) fn boolean(value: bool) -> RecallValue { + RecallValue::Integer(i64::from(value)) +} + +/// `values` plus the envelope columns and `content_hash`; an artifact's digest covers only its content, since every invocation that sees it writes the same row. +pub(crate) fn finish_row( + table: Table, + envelope: &RowEnvelope, + mut values: Vec<(&'static str, RecallValue)>, +) -> RecallRow { + values.push(("invocation_id", text(&envelope.invocation_id))); + values.push(( + "invocation_started_at", + text(&envelope.invocation_started_at), + )); + values.push(( + "schema_version", + RecallValue::Integer(RECALL_SCHEMA_VERSION), + )); + values.push(("producer_version", text(&envelope.producer_version))); + let digest: String = match table { + Table::Artifacts => content_hash( + &values + .iter() + .filter(|(name, _)| ARTIFACT_CONTENT_COLUMNS.contains(name)) + .cloned() + .collect::>(), + ), + _ => content_hash(&values), + }; + values.push(("content_hash", RecallValue::Text(digest))); + RecallRow { table, values } +} diff --git a/crates/sqlbuild-recall/src/records/_helpers/time.rs b/crates/sqlbuild-recall/src/records/_helpers/time.rs new file mode 100644 index 0000000000..d32586c82e --- /dev/null +++ b/crates/sqlbuild-recall/src/records/_helpers/time.rs @@ -0,0 +1,101 @@ +//! UTC timestamps as Recall stores them: `YYYY-MM-DDTHH:MM:SS.ffffffZ`. + +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::records::constants::{ + DECIMAL_RADIX, FEBRUARY, MARCH_BASED_JANUARY_INDEX, MICROS_PER_SECOND, SECONDS_PER_DAY, +}; + +/// Days since 1970-01-01 of a proleptic Gregorian date. +fn days_from_civil(year: i64, month: i64, day: i64) -> i64 { + let year = if month <= FEBRUARY { year - 1 } else { year }; + let era = year.div_euclid(400); + let year_of_era = year - era * 400; + let month_index = (month + 9) % 12; + let day_of_year = (153 * month_index + 2) / 5 + day - 1; + let day_of_era = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year; + era * 146_097 + day_of_era - 719_468 +} + +/// The date of a day count since 1970-01-01. +fn civil_from_days(days: i64) -> (i64, i64, i64) { + let shifted = days + 719_468; + let era = shifted.div_euclid(146_097); + let day_of_era = shifted - era * 146_097; + let year_of_era = + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = day_of_year - (153 * month_index + 2) / 5 + 1; + let month = if month_index < MARCH_BASED_JANUARY_INDEX { + month_index + 3 + } else { + month_index - 9 + }; + let year = year_of_era + era * 400 + i64::from(month <= FEBRUARY); + (year, month, day) +} + +/// Microseconds since the epoch as a stored timestamp. +pub(crate) fn format_micros(micros: i64) -> String { + let seconds = micros.div_euclid(MICROS_PER_SECOND); + let fraction = micros.rem_euclid(MICROS_PER_SECOND); + let days = seconds.div_euclid(SECONDS_PER_DAY); + let second_of_day = seconds.rem_euclid(SECONDS_PER_DAY); + let (year, month, day) = civil_from_days(days); + format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}.{fraction:06}Z", + second_of_day / 3600, + second_of_day % 3600 / 60, + second_of_day % 60 + ) +} + +/// Microseconds since the epoch of a system time. +pub(crate) fn system_micros(time: SystemTime) -> i64 { + match time.duration_since(UNIX_EPOCH) { + Ok(elapsed) => i64::try_from(elapsed.as_micros()).unwrap_or(i64::MAX), + Err(before) => -i64::try_from(before.duration().as_micros()).unwrap_or(i64::MAX), + } +} + +fn digits(text: &str) -> Option { + if text.is_empty() { + return None; + } + text.bytes().try_fold(0_i64, |total, byte| { + let digit = char::from(byte).to_digit(DECIMAL_RADIX)?; + total + .checked_mul(i64::from(DECIMAL_RADIX))? + .checked_add(i64::from(digit)) + }) +} + +/// Microseconds since the epoch of an ISO-8601 timestamp in UTC (`Z` or `+00:00`) or naive UTC. +pub(crate) fn parse_micros(text: &str) -> Option { + let text = text + .strip_suffix('Z') + .or_else(|| text.strip_suffix("+00:00")) + .unwrap_or(text); + let (date, time) = text.split_once(['T', ' '])?; + let mut date_parts = date.split('-'); + let year = digits(date_parts.next()?)?; + let month = digits(date_parts.next()?)?; + let day = digits(date_parts.next()?)?; + let (clock, fraction) = time.split_once('.').unwrap_or((time, "")); + let mut clock_parts = clock.split(':'); + let hour = digits(clock_parts.next()?)?; + let minute = digits(clock_parts.next()?)?; + let second = digits(clock_parts.next()?)?; + if date_parts.next().is_some() || clock_parts.next().is_some() { + return None; + } + let micros = if fraction.is_empty() { + 0 + } else { + let padded: String = fraction.chars().chain("000000".chars()).take(6).collect(); + digits(&padded)? + }; + let days = days_from_civil(year, month, day); + Some((days * SECONDS_PER_DAY + hour * 3600 + minute * 60 + second) * MICROS_PER_SECOND + micros) +} diff --git a/crates/sqlbuild-recall/src/records/constants.rs b/crates/sqlbuild-recall/src/records/constants.rs new file mode 100644 index 0000000000..733d77ae5f --- /dev/null +++ b/crates/sqlbuild-recall/src/records/constants.rs @@ -0,0 +1,56 @@ +//! Record vocabularies and limits. + +/// The largest output chunk, in bytes. +pub const MAX_OUTPUT_CHUNK_BYTES: usize = 64 * 1024; +/// The largest external context, in bytes of JSON. +pub const MAX_EXTERNAL_CONTEXT_BYTES: usize = 16 * 1024; +/// The output record ID infix, shared with command-output sink records. +pub const OUTPUT_RECORD_ID_INFIX: &str = "command-output"; +pub const DIAGNOSTIC_RECORD_ID_INFIX: &str = "diagnostic"; +/// Artifact digests are content digests with this prefix, as statement `sql_digest` without it. +pub const ARTIFACT_DIGEST_PREFIX: &str = "sha256:"; +pub const SQL_ARTIFACT_KIND: &str = "sql"; +pub const SQL_MEDIA_TYPE: &str = "text/x-sql"; +pub const EXECUTED_SQL_ROLE: &str = "executed_sql"; +/// Delivery stream names, one `recall_delivery` row each. +pub const DELIVERY_STREAMS: [&str; 5] = + ["invocation", "output", "events", "diagnostics", "artifacts"]; +/// Diagnostic severities, least severe first. +pub const SEVERITY_ORDER: [&str; 6] = ["trace", "debug", "info", "warning", "error", "critical"]; +/// Event payload fields promoted to their own `recall_events` columns. +pub const PROMOTED_TEXT_FIELDS: [&str; 11] = [ + "command", + "resource_kind", + "resource_name", + "error_code", + "error_type", + "query_id", + "sql_digest", + "intent", + "statement_kind", + "purpose", + "outcome", +]; +pub const PROMOTED_INTEGER_FIELDS: [&str; 3] = ["exit_code", "row_count", "affected_rows"]; +pub const PROMOTED_REAL_FIELDS: [&str; 1] = ["duration_ms"]; +pub const CORRELATION_FIELDS: [&str; 5] = [ + "run_id", + "resource_id", + "resource_attempt_id", + "operation_id", + "statement_id", +]; +pub(crate) const MICROS_PER_SECOND: i64 = 1_000_000; +pub(crate) const SECONDS_PER_DAY: i64 = 86_400; +pub(crate) const DECIMAL_RADIX: u32 = 10; +pub(crate) const FEBRUARY: i64 = 2; +/// In a year counted from March, January is month index 10. +pub(crate) const MARCH_BASED_JANUARY_INDEX: i64 = 10; +/// The output column counting chunks dropped just before this one. +pub const DROPPED_BEFORE_COLUMN: &str = "dropped_before"; +/// Initial capacity of the buffer a row's content hash is computed over. +pub(crate) const CONTENT_HASH_BUFFER_BYTES: usize = 1_024; +/// The output column holding the bytes a truncation gap replaces. +pub const OMITTED_BYTES_COLUMN: &str = "omitted_bytes"; +/// The output column holding the sequences a truncation gap accounts for. +pub const OMITTED_SEQUENCES_COLUMN: &str = "omitted_sequences"; diff --git a/crates/sqlbuild-recall/src/records/errors.rs b/crates/sqlbuild-recall/src/records/errors.rs new file mode 100644 index 0000000000..e0f42c5b60 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/errors.rs @@ -0,0 +1,23 @@ +//! Why an input cannot become a Recall record. + +/// A rejected record input, with a message naming the problem. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RecordError { + pub message: String, +} + +impl RecordError { + pub(crate) fn new(message: impl Into) -> Self { + Self { + message: message.into(), + } + } +} + +impl std::fmt::Display for RecordError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.message) + } +} + +impl std::error::Error for RecordError {} diff --git a/crates/sqlbuild-recall/src/records/main/artifact_rows.rs b/crates/sqlbuild-recall/src/records/main/artifact_rows.rs new file mode 100644 index 0000000000..22379043b5 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/artifact_rows.rs @@ -0,0 +1,42 @@ +//! `artifact_rows`: the content row of one artifact and the reference row naming it. + +use crate::records::_helpers::rows::{finish_row, integer, optional_text, text}; +use crate::records::models::{ArtifactInput, RecallRow, RecallValue, RowEnvelope}; +use crate::schema::models::Table; + +/// `[recall_artifacts row, recall_artifact_refs row]`, written in one batch so retention never sees the content without its reference. +#[must_use] +pub fn artifact_rows(envelope: &RowEnvelope, artifact: &ArtifactInput) -> [RecallRow; 2] { + [ + finish_row( + Table::Artifacts, + envelope, + vec![ + ("digest", text(&artifact.digest)), + ("kind", text(&artifact.kind)), + ("media_type", text(&artifact.media_type)), + ("size", integer(artifact.content.len() as u64)), + ("content", RecallValue::Blob(artifact.content.clone())), + ("seen_at", text(&artifact.seen_at)), + ], + ), + finish_row( + Table::ArtifactRefs, + envelope, + vec![ + ("digest", text(&artifact.digest)), + ("ref_id", text(&artifact.ref_id)), + ( + "statement_id", + optional_text(artifact.statement_id.as_deref()), + ), + ( + "resource_id", + optional_text(artifact.resource_id.as_deref()), + ), + ("role", text(&artifact.role)), + ("referenced_at", text(&artifact.seen_at)), + ], + ), + ] +} diff --git a/crates/sqlbuild-recall/src/records/main/conflict_row.rs b/crates/sqlbuild-recall/src/records/main/conflict_row.rs new file mode 100644 index 0000000000..89265416b2 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/conflict_row.rs @@ -0,0 +1,26 @@ +//! `conflict_row`: evidence that a record key was reused with different content. + +use crate::records::_helpers::rows::{finish_row, text}; +use crate::records::models::{RecallRow, RowEnvelope}; +use crate::schema::models::Table; + +/// A `recall_conflicts` row; the first stored row is kept and the rejected digest recorded. +#[must_use] +pub fn conflict_row( + envelope: &RowEnvelope, + (dataset, record_key): (&str, &str), + (existing_sha256, rejected_sha256): (&str, &str), + detected_at: &str, +) -> RecallRow { + finish_row( + Table::Conflicts, + envelope, + vec![ + ("dataset", text(dataset)), + ("record_key", text(record_key)), + ("existing_sha256", text(existing_sha256)), + ("rejected_sha256", text(rejected_sha256)), + ("detected_at", text(detected_at)), + ], + ) +} diff --git a/crates/sqlbuild-recall/src/records/main/delivery_row.rs b/crates/sqlbuild-recall/src/records/main/delivery_row.rs new file mode 100644 index 0000000000..950768aa11 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/delivery_row.rs @@ -0,0 +1,30 @@ +//! `delivery_row`: the final accounting of one stream, written last. + +use crate::records::_helpers::rows::{boolean, finish_row, integer, text}; +use crate::records::models::{DeliveryCounts, RecallRow, RowEnvelope}; +use crate::schema::models::Table; + +/// The delivery row of `stream`. +#[must_use] +pub fn delivery_row( + envelope: &RowEnvelope, + stream: &str, + (counts, flush_complete): (&DeliveryCounts, bool), + written_at: &str, +) -> RecallRow { + finish_row( + Table::Delivery, + envelope, + vec![ + ("stream", text(stream)), + ("accepted", integer(counts.accepted)), + ("delivered", integer(counts.delivered)), + ("filtered", integer(counts.filtered)), + ("dropped", integer(counts.dropped)), + ("failed", integer(counts.failed)), + ("conflicts", integer(counts.conflicts)), + ("flush_complete", boolean(flush_complete)), + ("written_at", text(written_at)), + ], + ) +} diff --git a/crates/sqlbuild-recall/src/records/main/diagnostic_input_from_json.rs b/crates/sqlbuild-recall/src/records/main/diagnostic_input_from_json.rs new file mode 100644 index 0000000000..8d6ba82341 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/diagnostic_input_from_json.rs @@ -0,0 +1,51 @@ +//! `diagnostic_input_from_json`: a diagnostic from `diagnostic_log_to_json` text. + +use serde_json::{Map, Value}; + +use crate::records::errors::RecordError; +use crate::records::models::{Correlations, DiagnosticInput}; + +fn text(object: &Map, field: &str) -> Result { + object + .get(field) + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| RecordError::new(format!("diagnostic {field} must be a string"))) +} + +fn optional(object: &Map, field: &str) -> Option { + object.get(field).and_then(Value::as_str).map(str::to_owned) +} + +/// The diagnostic encoded in `json`, numbered `sequence` within its invocation. +pub fn diagnostic_input_from_json( + json: &str, + sequence: u64, +) -> Result { + let parsed: Value = serde_json::from_str(json) + .map_err(|error| RecordError::new(format!("diagnostic is not JSON: {error}")))?; + let Value::Object(object) = parsed else { + return Err(RecordError::new("diagnostic must be a JSON object")); + }; + let fields: String = object + .get("fields") + .filter(|value| value.is_object()) + .map_or_else(|| "{}".to_owned(), Value::to_string); + Ok(DiagnosticInput { + sequence, + occurred_at: text(&object, "occurred_at")?, + severity: text(&object, "severity")?, + logger: text(&object, "logger")?, + source: text(&object, "source")?, + message: text(&object, "message")?, + fields, + log_stream_id: optional(&object, "log_stream_id"), + correlations: Correlations { + run_id: optional(&object, "run_id"), + resource_id: optional(&object, "resource_id"), + resource_attempt_id: optional(&object, "resource_attempt_id"), + operation_id: optional(&object, "operation_id"), + statement_id: optional(&object, "statement_id"), + }, + }) +} diff --git a/crates/sqlbuild-recall/src/records/main/diagnostic_row.rs b/crates/sqlbuild-recall/src/records/main/diagnostic_row.rs new file mode 100644 index 0000000000..c839d2d5a3 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/diagnostic_row.rs @@ -0,0 +1,53 @@ +//! `diagnostic_row`: the `recall_diagnostics` row of one diagnostic. + +use crate::records::_helpers::ids::diagnostic_record_id; +use crate::records::_helpers::rows::{finish_row, integer, optional_text, text}; +use crate::records::models::{DiagnosticInput, RecallRow, RowEnvelope}; +use crate::schema::models::Table; + +/// The row of `diagnostic`, keyed by invocation and sequence. +#[must_use] +pub fn diagnostic_row(envelope: &RowEnvelope, diagnostic: &DiagnosticInput) -> RecallRow { + let correlations = &diagnostic.correlations; + finish_row( + Table::Diagnostics, + envelope, + vec![ + ( + "record_id", + text(diagnostic_record_id( + &envelope.invocation_id, + diagnostic.sequence, + )), + ), + ("sequence", integer(diagnostic.sequence)), + ("occurred_at", text(&diagnostic.occurred_at)), + ("severity", text(&diagnostic.severity)), + ("logger", text(&diagnostic.logger)), + ("source", text(&diagnostic.source)), + ("message", text(&diagnostic.message)), + ("fields", text(&diagnostic.fields)), + ( + "log_stream_id", + optional_text(diagnostic.log_stream_id.as_deref()), + ), + ("run_id", optional_text(correlations.run_id.as_deref())), + ( + "resource_id", + optional_text(correlations.resource_id.as_deref()), + ), + ( + "resource_attempt_id", + optional_text(correlations.resource_attempt_id.as_deref()), + ), + ( + "operation_id", + optional_text(correlations.operation_id.as_deref()), + ), + ( + "statement_id", + optional_text(correlations.statement_id.as_deref()), + ), + ], + ) +} diff --git a/crates/sqlbuild-recall/src/records/main/event_object_row.rs b/crates/sqlbuild-recall/src/records/main/event_object_row.rs new file mode 100644 index 0000000000..c67acb614b --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/event_object_row.rs @@ -0,0 +1,24 @@ +//! `event_object_row`: the `recall_events` row of one lifecycle event already decoded to JSON. + +use serde_json::{Map, Value}; + +use crate::records::_helpers::events::{envelope_columns, payload_columns}; +use crate::records::_helpers::rows::finish_row; +use crate::records::errors::RecordError; +use crate::records::models::{RecallRow, RowEnvelope}; +use crate::schema::models::Table; + +/// The row of one event given as the object `lifecycle_event_to_json` would encode. +pub fn event_object_row( + envelope: &RowEnvelope, + event: &Map, +) -> Result { + let mut columns = envelope_columns(event)?; + let empty: Map = Map::new(); + let payload = event + .get("payload") + .and_then(Value::as_object) + .unwrap_or(&empty); + columns.extend(payload_columns(payload)); + Ok(finish_row(Table::Events, envelope, columns)) +} diff --git a/crates/sqlbuild-recall/src/records/main/event_severity.rs b/crates/sqlbuild-recall/src/records/main/event_severity.rs new file mode 100644 index 0000000000..2f80356364 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/event_severity.rs @@ -0,0 +1,11 @@ +//! `event_severity`: the export severity of a lifecycle event, without building its row. + +use serde_json::{Map, Value}; + +use crate::records::_helpers::events::severity_of; + +/// The severity the event's row will carry; `None` when the event has no text `event_type`. +#[must_use] +pub fn event_severity(event: &Map) -> Option<&'static str> { + severity_of(event) +} diff --git a/crates/sqlbuild-recall/src/records/main/format_timestamp.rs b/crates/sqlbuild-recall/src/records/main/format_timestamp.rs new file mode 100644 index 0000000000..2b3a8b5012 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/format_timestamp.rs @@ -0,0 +1,11 @@ +//! `format_timestamp`: a system time as Recall stores it. + +use std::time::SystemTime; + +use crate::records::_helpers::time::{format_micros, system_micros}; + +/// `YYYY-MM-DDTHH:MM:SS.ffffffZ` in UTC. +#[must_use] +pub fn format_timestamp(time: SystemTime) -> String { + format_micros(system_micros(time)) +} diff --git a/crates/sqlbuild-recall/src/records/main/heartbeat_row.rs b/crates/sqlbuild-recall/src/records/main/heartbeat_row.rs new file mode 100644 index 0000000000..340c09365f --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/heartbeat_row.rs @@ -0,0 +1,27 @@ +//! `heartbeat_row`: one liveness fact of a running invocation. + +use crate::records::_helpers::rows::{finish_row, integer, optional_text, text}; +use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; +use crate::schema::models::Table; + +/// A heartbeat at `beat_at` that declares the invocation abandoned after `stale_at`. +#[must_use] +pub fn heartbeat_row( + envelope: &RowEnvelope, + (beat_at, stale_at): (&str, &str), + (host_token, pid): (Option<&str>, Option), +) -> RecallRow { + finish_row( + Table::Heartbeats, + envelope, + vec![ + ("beat_at", text(beat_at)), + ("stale_at", text(stale_at)), + ("host_token", optional_text(host_token)), + ( + "pid", + pid.map_or(RecallValue::Null, |pid| integer(u64::from(pid))), + ), + ], + ) +} diff --git a/crates/sqlbuild-recall/src/records/main/invocation_row.rs b/crates/sqlbuild-recall/src/records/main/invocation_row.rs new file mode 100644 index 0000000000..53c678fd43 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/invocation_row.rs @@ -0,0 +1,74 @@ +//! `invocation_row`: the `recall_invocations` context row. + +use crate::records::_helpers::rows::{boolean, finish_row, integer, optional_text, text}; +use crate::records::models::{InvocationContext, RecallRow, RecallValue, RowEnvelope}; +use crate::schema::models::Table; + +fn json_list(items: &[String]) -> RecallValue { + text(serde_json::Value::from(items.to_vec()).to_string()) +} + +/// The context row of one invocation. +#[must_use] +pub fn invocation_row(envelope: &RowEnvelope, context: &InvocationContext) -> RecallRow { + finish_row( + Table::Invocations, + envelope, + vec![ + ("command", optional_text(context.command.as_deref())), + ("subcommand", optional_text(context.subcommand.as_deref())), + ("selection", json_list(&context.selection)), + ("exclusion", json_list(&context.exclusion)), + ( + "project_name", + optional_text(context.project_name.as_deref()), + ), + ( + "project_dir_hash", + optional_text(context.project_dir_hash.as_deref()), + ), + ("target", optional_text(context.target.as_deref())), + ("adapter", optional_text(context.adapter.as_deref())), + ( + "destination_database", + optional_text(context.destination_database.as_deref()), + ), + ( + "destination_schema", + optional_text(context.destination_schema.as_deref()), + ), + ( + "destination_fingerprint", + optional_text(context.destination_fingerprint.as_deref()), + ), + ( + "sqlbuild_version", + optional_text(context.sqlbuild_version.as_deref()), + ), + ( + "git_revision", + optional_text(context.git_revision.as_deref()), + ), + ( + "git_dirty", + context.git_dirty.map_or(RecallValue::Null, boolean), + ), + ("host_token", optional_text(context.host_token.as_deref())), + ( + "pid", + context + .pid + .map_or(RecallValue::Null, |pid| integer(u64::from(pid))), + ), + ( + "external_context", + optional_text(context.external_context.as_deref()), + ), + ( + "recall_filter_active", + boolean(context.recall_filter_active), + ), + ("argv_digest", optional_text(context.argv_digest.as_deref())), + ], + ) +} diff --git a/crates/sqlbuild-recall/src/records/main/meta_row.rs b/crates/sqlbuild-recall/src/records/main/meta_row.rs new file mode 100644 index 0000000000..ecf40303f5 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/meta_row.rs @@ -0,0 +1,24 @@ +//! `meta_row`: one insert-only store metadata fact. + +use crate::records::_helpers::rows::{finish_row, text}; +use crate::records::models::{RecallRow, RowEnvelope}; +use crate::schema::models::Table; + +/// A `recall_meta` row. +#[must_use] +pub fn meta_row( + envelope: &RowEnvelope, + (component, key, value): (&str, &str, &str), + recorded_at: &str, +) -> RecallRow { + finish_row( + Table::Meta, + envelope, + vec![ + ("component", text(component)), + ("key", text(key)), + ("value", text(value)), + ("recorded_at", text(recorded_at)), + ], + ) +} diff --git a/crates/sqlbuild-recall/src/records/main/mod.rs b/crates/sqlbuild-recall/src/records/main/mod.rs new file mode 100644 index 0000000000..2e466594b9 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/mod.rs @@ -0,0 +1,15 @@ +pub mod artifact_rows; +pub mod conflict_row; +pub mod delivery_row; +pub mod diagnostic_input_from_json; +pub mod diagnostic_row; +pub mod event_object_row; +pub mod event_severity; +pub mod format_timestamp; +pub mod heartbeat_row; +pub mod invocation_row; +pub mod meta_row; +pub mod output_row; +pub mod parse_event; +pub mod parse_timestamp; +pub mod sha256_hex; diff --git a/crates/sqlbuild-recall/src/records/main/output_row.rs b/crates/sqlbuild-recall/src/records/main/output_row.rs new file mode 100644 index 0000000000..de74c9a480 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/output_row.rs @@ -0,0 +1,44 @@ +//! `output_row`: the `recall_output` row of one chunk. + +use crate::records::_helpers::ids::output_record_id; +use crate::records::_helpers::rows::{finish_row, integer, text}; +use crate::records::constants::{ + DROPPED_BEFORE_COLUMN, OMITTED_BYTES_COLUMN, OMITTED_SEQUENCES_COLUMN, +}; +use crate::records::models::{OutputChunk, RecallRow, RecallValue, RowEnvelope}; +use crate::schema::models::Table; + +/// The row of `chunk`, keyed by invocation and sequence. +#[must_use] +pub fn output_row(envelope: &RowEnvelope, chunk: &OutputChunk) -> RecallRow { + finish_row( + Table::Output, + envelope, + vec![ + ( + "record_id", + text(output_record_id(&envelope.invocation_id, chunk.sequence)), + ), + ("sequence", integer(chunk.sequence)), + ("stream", text(chunk.stream.as_str())), + ("occurred_at", text(&chunk.occurred_at)), + ("data", RecallValue::Blob(chunk.data.clone())), + ("byte_offset", integer(chunk.byte_offset)), + ("byte_count", integer(chunk.data.len() as u64)), + ("priority", text("output")), + (DROPPED_BEFORE_COLUMN, integer(chunk.dropped_before)), + ( + OMITTED_BYTES_COLUMN, + chunk + .omitted + .map_or(RecallValue::Null, |omitted| integer(omitted.bytes)), + ), + ( + OMITTED_SEQUENCES_COLUMN, + chunk + .omitted + .map_or(RecallValue::Null, |omitted| integer(omitted.sequences)), + ), + ], + ) +} diff --git a/crates/sqlbuild-recall/src/records/main/parse_event.rs b/crates/sqlbuild-recall/src/records/main/parse_event.rs new file mode 100644 index 0000000000..97c4043d18 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/parse_event.rs @@ -0,0 +1,11 @@ +//! `parse_event`: a lifecycle event's canonical JSON as its object. + +use serde_json::{Map, Value}; + +use crate::records::_helpers::events::event_object; +use crate::records::errors::RecordError; + +/// The JSON object of one lifecycle event; an error unless `event_json` is a JSON object. +pub fn parse_event(event_json: &str) -> Result, RecordError> { + event_object(event_json) +} diff --git a/crates/sqlbuild-recall/src/records/main/parse_timestamp.rs b/crates/sqlbuild-recall/src/records/main/parse_timestamp.rs new file mode 100644 index 0000000000..c5a2470f0d --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/parse_timestamp.rs @@ -0,0 +1,17 @@ +//! `parse_timestamp`: a stored or ISO-8601 UTC timestamp as a system time. + +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use crate::records::_helpers::time::parse_micros; + +/// The time of `text`, or `None` when it is not a UTC ISO-8601 timestamp. +#[must_use] +pub fn parse_timestamp(text: &str) -> Option { + let micros = parse_micros(text)?; + let magnitude = Duration::from_micros(micros.unsigned_abs()); + if micros >= 0 { + UNIX_EPOCH.checked_add(magnitude) + } else { + UNIX_EPOCH.checked_sub(magnitude) + } +} diff --git a/crates/sqlbuild-recall/src/records/main/sha256_hex.rs b/crates/sqlbuild-recall/src/records/main/sha256_hex.rs new file mode 100644 index 0000000000..f9fe2b38fb --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/sha256_hex.rs @@ -0,0 +1,14 @@ +//! `sha256_hex`: the lowercase hexadecimal SHA-256 of bytes. + +use sha2::{Digest, Sha256}; + +/// Lowercase hex SHA-256 of `data`. +#[must_use] +pub fn sha256_hex(data: &[u8]) -> String { + let digest = Sha256::digest(data); + let mut text = String::with_capacity(64); + for byte in digest { + text.push_str(&format!("{byte:02x}")); + } + text +} diff --git a/crates/sqlbuild-recall/src/records/mod.rs b/crates/sqlbuild-recall/src/records/mod.rs new file mode 100644 index 0000000000..bfc1b59309 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/mod.rs @@ -0,0 +1,10 @@ +//! Recall records: deterministic IDs, content digests and the rows each dataset writes. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-recall/src/records/models.rs b/crates/sqlbuild-recall/src/records/models.rs new file mode 100644 index 0000000000..40146f65be --- /dev/null +++ b/crates/sqlbuild-recall/src/records/models.rs @@ -0,0 +1,175 @@ +//! Recall record inputs and the rows they become. + +use crate::schema::models::Table; + +/// One stored value. +#[derive(Clone, Debug, PartialEq)] +pub enum RecallValue { + Null, + Integer(i64), + Real(f64), + Text(String), + Blob(Vec), +} + +/// One row bound for a Recall table: named values, without `ingested_at`, which the store sets. +#[derive(Clone, Debug, PartialEq)] +pub struct RecallRow { + pub table: Table, + pub values: Vec<(&'static str, RecallValue)>, +} + +impl RecallRow { + /// The value of `column`, if the row has it. + #[must_use] + pub fn value(&self, column: &str) -> Option<&RecallValue> { + self.values + .iter() + .find(|(name, _)| *name == column) + .map(|(_, value)| value) + } + + /// The row's content digest. + #[must_use] + pub fn content_hash(&self) -> Option<&str> { + match self.value("content_hash") { + Some(RecallValue::Text(text)) => Some(text), + _ => None, + } + } +} + +/// What every row of one invocation carries. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RowEnvelope { + pub invocation_id: String, + pub invocation_started_at: String, + pub producer_version: String, +} + +/// A process output stream. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum OutputStream { + Stdout, + Stderr, +} + +impl OutputStream { + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::Stdout => "stdout", + Self::Stderr => "stderr", + } + } + + /// The stream named `name`. + #[must_use] + pub fn from_name(name: &str) -> Option { + match name { + "stdout" => Some(Self::Stdout), + "stderr" => Some(Self::Stderr), + _ => None, + } + } +} + +/// One exact-byte output chunk. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OutputChunk { + /// Per invocation, from 0, shared by both streams. + pub sequence: u64, + pub stream: OutputStream, + pub occurred_at: String, + pub data: Vec, + /// Bytes of the same stream before this chunk. + pub byte_offset: u64, + /// Output records the queue dropped just before this one. + pub dropped_before: u64, + /// On the empty chunk standing for output past the stream's limit: the bytes and sequences it replaces. + pub omitted: Option, +} + +/// Output of one stream that was not recorded because it exceeded `max_output_bytes`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct OmittedOutput { + pub bytes: u64, + /// Sequences this gap accounts for, its own included. + pub sequences: u64, +} + +/// The context of one invocation, written once when it starts. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct InvocationContext { + pub command: Option, + pub subcommand: Option, + pub selection: Vec, + pub exclusion: Vec, + pub project_name: Option, + pub project_dir_hash: Option, + pub target: Option, + pub adapter: Option, + pub destination_database: Option, + pub destination_schema: Option, + pub destination_fingerprint: Option, + pub sqlbuild_version: Option, + pub git_revision: Option, + pub git_dirty: Option, + pub host_token: Option, + pub pid: Option, + /// JSON object text, at most 16 KiB. + pub external_context: Option, + pub recall_filter_active: bool, + pub argv_digest: Option, +} + +/// The correlation identifiers of a fact. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Correlations { + pub run_id: Option, + pub resource_id: Option, + pub resource_attempt_id: Option, + pub operation_id: Option, + pub statement_id: Option, +} + +/// One structured diagnostic. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DiagnosticInput { + pub sequence: u64, + pub occurred_at: String, + pub severity: String, + pub logger: String, + pub source: String, + pub message: String, + /// JSON object text. + pub fields: String, + pub log_stream_id: Option, + pub correlations: Correlations, +} + +/// One captured artifact and the reference that names it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ArtifactInput { + /// `sha256:` of `content`. + pub digest: String, + pub kind: String, + pub media_type: String, + pub content: Vec, + pub seen_at: String, + pub ref_id: String, + pub role: String, + pub statement_id: Option, + pub resource_id: Option, +} + +/// Delivery accounting of one stream. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DeliveryCounts { + pub accepted: u64, + pub delivered: u64, + pub filtered: u64, + pub dropped: u64, + pub failed: u64, + pub conflicts: u64, +} diff --git a/crates/sqlbuild-recall/src/records/tests.rs b/crates/sqlbuild-recall/src/records/tests.rs new file mode 100644 index 0000000000..9e55cb0928 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_records.rs"] +mod records; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-recall/src/records/tests/helpers.rs b/crates/sqlbuild-recall/src/records/tests/helpers.rs new file mode 100644 index 0000000000..e4166567da --- /dev/null +++ b/crates/sqlbuild-recall/src/records/tests/helpers.rs @@ -0,0 +1,47 @@ +use crate::records::errors::RecordError; +use crate::records::main::event_object_row::event_object_row; +use crate::records::main::parse_event::parse_event; +use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; + +/// The envelope every record test row shares. +pub(crate) fn envelope() -> RowEnvelope { + RowEnvelope { + invocation_id: "inv1".to_owned(), + invocation_started_at: "2026-10-19T00:00:00.000000Z".to_owned(), + producer_version: "0.0.0".to_owned(), + } +} + +/// `columns` of `row` paired with their values, when present. +pub(crate) fn row_columns( + row: &RecallRow, + columns: &[(&'static str, RecallValue)], +) -> Vec<(&'static str, Option)> { + columns + .iter() + .map(|(name, _)| (*name, row.value(name).cloned())) + .collect() +} + +/// `expected` in the shape `row_columns` returns. +pub(crate) fn present( + expected: Vec<(&'static str, RecallValue)>, +) -> Vec<(&'static str, Option)> { + expected + .into_iter() + .map(|(name, value)| (name, Some(value))) + .collect() +} + +/// A text value. +pub(crate) fn text(value: &str) -> RecallValue { + RecallValue::Text(value.to_owned()) +} + +/// The events row of one lifecycle event's canonical JSON, as the delivery thread builds it. +pub(crate) fn event_row( + envelope: &RowEnvelope, + event_json: &str, +) -> Result { + parse_event(event_json).and_then(|event| event_object_row(envelope, &event)) +} diff --git a/crates/sqlbuild-recall/src/records/tests/test_records.rs b/crates/sqlbuild-recall/src/records/tests/test_records.rs new file mode 100644 index 0000000000..0b8f7fb19c --- /dev/null +++ b/crates/sqlbuild-recall/src/records/tests/test_records.rs @@ -0,0 +1,276 @@ +use std::time::{Duration, UNIX_EPOCH}; + +use crate::records::_helpers::ids::content_hash; +use crate::records::_helpers::ids::diagnostic_record_id; +use crate::records::_helpers::ids::output_record_id; +use crate::records::main::diagnostic_input_from_json::diagnostic_input_from_json; +use crate::records::main::format_timestamp::format_timestamp; +use crate::records::main::output_row::output_row; +use crate::records::main::parse_timestamp::parse_timestamp; +use crate::records::models::{OutputChunk, OutputStream, RecallValue}; +use crate::records::tests::helpers::{envelope, event_row, present, row_columns, text}; +use crate::records::tests::test_types::{ + ContentHashTestCase, DiagnosticTestCase, ParseTimestampTestCase, RecordIdTestCase, + RejectedInputTestCase, RowColumnsTestCase, TimestampTestCase, +}; + +#[test] +fn given_sequences_when_ids_are_built_then_they_follow_the_record_id_scheme() { + let test_cases = [ + RecordIdTestCase { + description: "output chunks are keyed by invocation and sequence", + record_id: output_record_id("abc", 7), + expected_record_id: "abc:command-output:7", + }, + RecordIdTestCase { + description: "diagnostics are keyed by invocation and sequence", + record_id: diagnostic_record_id("abc", 0), + expected_record_id: "abc:diagnostic:0", + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_case.record_id, test_case.expected_record_id, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_times_when_formatted_and_parsed_then_they_round_trip() { + let test_cases = [ + TimestampTestCase { + description: "the epoch", + micros: 0, + expected_text: "1970-01-01T00:00:00.000000Z", + }, + TimestampTestCase { + description: "a leap day with microseconds", + micros: 951_782_400_123_456, + expected_text: "2000-02-29T00:00:00.123456Z", + }, + TimestampTestCase { + description: "one microsecond past midnight", + micros: 1_792_281_600_000_001, + expected_text: "2026-10-18T00:00:00.000001Z", + }, + ]; + + for test_case in test_cases { + let time = UNIX_EPOCH + Duration::from_micros(test_case.micros); + assert_eq!( + ( + format_timestamp(time), + parse_timestamp(test_case.expected_text) + ), + (test_case.expected_text.to_owned(), Some(time)), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_timestamp_text_when_parsed_then_offsets_are_accepted_and_garbage_rejected() { + let test_cases = [ + ParseTimestampTestCase { + description: "a zero offset equals Z", + text: "2026-10-18T00:00:00+00:00", + expected_micros: Some(1_792_281_600_000_000), + }, + ParseTimestampTestCase { + description: "missing fractions are zero", + text: "2026-10-18T00:00:00Z", + expected_micros: Some(1_792_281_600_000_000), + }, + ParseTimestampTestCase { + description: "words are not timestamps", + text: "yesterday", + expected_micros: None, + }, + ]; + + for test_case in test_cases { + assert_eq!( + parse_timestamp(test_case.text), + test_case + .expected_micros + .map(|micros| UNIX_EPOCH + Duration::from_micros(micros)), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_equal_and_different_content_when_hashed_then_only_content_matters() { + let test_cases = [ + ContentHashTestCase { + description: "column order and ingestion time do not matter", + left: vec![("a", text("x")), ("b", RecallValue::Integer(1))], + right: vec![ + ("b", RecallValue::Integer(1)), + ("a", text("x")), + ("ingested_at", text("later")), + ], + expected_equal: true, + }, + ContentHashTestCase { + description: "value types matter", + left: vec![("a", text("x")), ("b", RecallValue::Integer(1))], + right: vec![ + ("a", RecallValue::Blob(b"x".to_vec())), + ("b", RecallValue::Integer(1)), + ], + expected_equal: false, + }, + ]; + + for test_case in test_cases { + assert_eq!( + content_hash(&test_case.left) == content_hash(&test_case.right), + test_case.expected_equal, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_records_when_rowed_then_columns_are_promoted() { + let lifecycle = r#"{"event_id":"e1","event_type":"statement_failed","schema_version":2, + "producer":"sqlbuild","producer_version":"1","occurred_at":"2026-10-19T00:00:02.000000Z", + "invocation_id":"inv1","invocation_sequence":4,"statement_id":"s1", + "payload":{"error_type":"ValueError","duration_ms":1.5,"sql_digest":"ab","purpose":"audit"}}"#; + let test_cases = [ + RowColumnsTestCase { + description: "output chunks keep bytes, counts and record ids", + row: output_row( + &envelope(), + &OutputChunk { + sequence: 3, + stream: OutputStream::Stderr, + occurred_at: "2026-10-19T00:00:01.000000Z".to_owned(), + data: vec![0, 255, 10], + byte_offset: 12, + dropped_before: 1, + omitted: None, + }, + ), + expected_columns: vec![ + ("data", RecallValue::Blob(vec![0, 255, 10])), + ("byte_count", RecallValue::Integer(3)), + ("byte_offset", RecallValue::Integer(12)), + ("record_id", text("inv1:command-output:3")), + ], + }, + RowColumnsTestCase { + description: "lifecycle events promote kind, severity and payload dimensions", + row: event_row(&envelope(), lifecycle).expect("the fixture is a valid lifecycle event"), + expected_columns: vec![ + ("event_kind", text("statement")), + ("severity", text("error")), + ("statement_id", text("s1")), + ("duration_ms", RecallValue::Real(1.5)), + ("purpose", text("audit")), + ("invocation_sequence", RecallValue::Integer(4)), + ], + }, + ]; + + for test_case in test_cases { + assert_eq!( + row_columns(&test_case.row, &test_case.expected_columns), + present(test_case.expected_columns), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_malformed_lifecycle_json_when_rowed_then_it_is_rejected() { + let test_cases = [ + RejectedInputTestCase { + description: "an array is not an event", + json: "[]", + expected_rejected: true, + }, + RejectedInputTestCase { + description: "an event needs its identity fields", + json: r#"{"event_type":"x"}"#, + expected_rejected: true, + }, + ]; + + for test_case in test_cases { + assert_eq!( + event_row(&envelope(), test_case.json).is_err(), + test_case.expected_rejected, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_diagnostic_json_when_parsed_then_sequence_fields_and_correlations_are_kept() { + let test_cases = [ + DiagnosticTestCase { + description: "fields and run correlation survive", + json: r#"{"occurred_at":"2026-10-19T00:00:00.000000Z","severity":"warning", + "logger":"sqlbuild.x","source":"python","message":"careful","fields":{"k":1}, + "run_id":"r1"}"#, + sequence: 5, + expected_fields: r#"{"k":1}"#, + expected_run_id: Some("r1"), + }, + DiagnosticTestCase { + description: "absent fields become an empty object", + json: r#"{"occurred_at":"2026-10-19T00:00:00.000000Z","severity":"info", + "logger":"sqlbuild.x","source":"python","message":"hello"}"#, + sequence: 0, + expected_fields: "{}", + expected_run_id: None, + }, + ]; + + for test_case in test_cases { + let diagnostic = diagnostic_input_from_json(test_case.json, test_case.sequence) + .expect("the fixture is a valid diagnostic"); + assert_eq!( + ( + diagnostic.sequence, + diagnostic.fields.as_str(), + diagnostic.correlations.run_id.as_deref() + ), + ( + test_case.sequence, + test_case.expected_fields, + test_case.expected_run_id + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_incomplete_diagnostic_json_when_parsed_then_it_is_rejected() { + let test_cases = [RejectedInputTestCase { + description: "a diagnostic needs a message and logger", + json: r#"{"severity":"info"}"#, + expected_rejected: true, + }]; + + for test_case in test_cases { + assert_eq!( + diagnostic_input_from_json(test_case.json, 0).is_err(), + test_case.expected_rejected, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/records/tests/test_types.rs b/crates/sqlbuild-recall/src/records/tests/test_types.rs new file mode 100644 index 0000000000..86272de086 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/tests/test_types.rs @@ -0,0 +1,46 @@ +use crate::records::models::{RecallRow, RecallValue}; + +pub(crate) struct RecordIdTestCase { + pub(crate) description: &'static str, + pub(crate) record_id: String, + pub(crate) expected_record_id: &'static str, +} + +pub(crate) struct TimestampTestCase { + pub(crate) description: &'static str, + pub(crate) micros: u64, + pub(crate) expected_text: &'static str, +} + +pub(crate) struct ParseTimestampTestCase { + pub(crate) description: &'static str, + pub(crate) text: &'static str, + pub(crate) expected_micros: Option, +} + +pub(crate) struct ContentHashTestCase { + pub(crate) description: &'static str, + pub(crate) left: Vec<(&'static str, RecallValue)>, + pub(crate) right: Vec<(&'static str, RecallValue)>, + pub(crate) expected_equal: bool, +} + +pub(crate) struct RowColumnsTestCase { + pub(crate) description: &'static str, + pub(crate) row: RecallRow, + pub(crate) expected_columns: Vec<(&'static str, RecallValue)>, +} + +pub(crate) struct RejectedInputTestCase { + pub(crate) description: &'static str, + pub(crate) json: &'static str, + pub(crate) expected_rejected: bool, +} + +pub(crate) struct DiagnosticTestCase { + pub(crate) description: &'static str, + pub(crate) json: &'static str, + pub(crate) sequence: u64, + pub(crate) expected_fields: &'static str, + pub(crate) expected_run_id: Option<&'static str>, +} diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs new file mode 100644 index 0000000000..d84e3e1713 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -0,0 +1,360 @@ +//! The bodies of the supported views in ClickHouse: rows deduplicated by key, within retention. + +use crate::schema::_helpers::rendering::clickhouse_table; +use crate::schema::constants::OUTPUT_TAIL_RECORDS; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::{DdlOptions, Table}; + +/// `table`'s rows within retention, first ingested copy per key. +fn deduplicated(table: Table, options: &DdlOptions) -> String { + let spec = table_spec(table); + format!( + "(SELECT * FROM {name} WHERE {expiry} >= now64(6) - INTERVAL {days} DAY \ + ORDER BY ingested_at ASC LIMIT 1 BY {key})", + name = clickhouse_table(spec.name, options), + expiry = spec.expiry_column, + days = options.retention_days, + key = spec.key.join(", "), + ) +} + +fn invocations(options: &DdlOptions) -> String { + let context = deduplicated(Table::Invocations, options); + let events = deduplicated(Table::Events, options); + let beats = deduplicated(Table::Heartbeats, options); + format!( + "WITH +ids AS ( + SELECT invocation_id, min(invocation_started_at) AS invocation_started_at + FROM ( + SELECT invocation_id, invocation_started_at FROM {context} + UNION ALL SELECT invocation_id, invocation_started_at FROM {beats} + UNION ALL SELECT invocation_id, invocation_started_at FROM {events} + WHERE event_type IN ('invocation_started', 'invocation_completed', 'invocation_failed') + ) + GROUP BY invocation_id +), +started AS ( + SELECT invocation_id, min(occurred_at) AS started_at, any(command) AS command + FROM {events} WHERE event_type = 'invocation_started' GROUP BY invocation_id +), +terminals AS ( + SELECT invocation_id, + argMax(event_type, tuple(invocation_sequence, occurred_at)) AS event_type, + argMax(occurred_at, tuple(invocation_sequence, occurred_at)) AS occurred_at, + argMax(exit_code, tuple(invocation_sequence, occurred_at)) AS exit_code, + argMax(duration_ms, tuple(invocation_sequence, occurred_at)) AS duration_ms, + argMax(error_type, tuple(invocation_sequence, occurred_at)) AS error_type, + argMax(error_code, tuple(invocation_sequence, occurred_at)) AS error_code + FROM {events} + WHERE event_type IN ('invocation_completed', 'invocation_failed') + GROUP BY invocation_id +), +beats AS ( + SELECT invocation_id, max(beat_at) AS last_heartbeat_at, max(stale_at) AS stale_at + FROM {beats} GROUP BY invocation_id +) +SELECT + ids.invocation_id AS invocation_id, + ids.invocation_started_at AS invocation_started_at, + coalesce(started.started_at, ids.invocation_started_at) AS started_at, + multiIf( + terminals.event_type = 'invocation_completed', 'completed', + terminals.event_type = 'invocation_failed', 'failed', + beats.stale_at IS NULL, 'unknown', + beats.stale_at >= now64(6), 'running', + 'abandoned' + ) AS status, + if(terminals.event_type IS NOT NULL, terminals.occurred_at, + if(beats.stale_at < now64(6), beats.last_heartbeat_at, NULL)) AS ended_at, + if(terminals.event_type IS NOT NULL, terminals.duration_ms, + if(beats.stale_at < now64(6), + toFloat64(dateDiff('millisecond', started_at, beats.last_heartbeat_at)), NULL)) + AS duration_ms, + terminals.exit_code AS exit_code, + terminals.error_type AS error_type, + terminals.error_code AS error_code, + coalesce(context.command, started.command) AS command, + context.subcommand AS subcommand, + context.target AS target, + context.adapter AS adapter, + context.project_name AS project_name, + context.selection AS selection, + context.exclusion AS exclusion, + context.sqlbuild_version AS sqlbuild_version, + context.git_revision AS git_revision, + context.git_dirty AS git_dirty, + context.host_token AS host_token, + context.pid AS pid, + context.external_context AS external_context, + context.recall_filter_active AS recall_filter_active, + beats.last_heartbeat_at AS last_heartbeat_at +FROM ids +LEFT JOIN {context} AS context ON context.invocation_id = ids.invocation_id +LEFT JOIN started ON started.invocation_id = ids.invocation_id +LEFT JOIN terminals ON terminals.invocation_id = ids.invocation_id +LEFT JOIN beats ON beats.invocation_id = ids.invocation_id +SETTINGS join_use_nulls = 1" + ) +} + +fn statements(options: &DdlOptions) -> String { + let events = deduplicated(Table::Events, options); + let artifacts = clickhouse_table(table_spec(Table::Artifacts).name, options); + format!( + "WITH +facts AS ( + SELECT * FROM {events} WHERE statement_id IS NOT NULL AND startsWith(event_type, 'statement_') +), +firsts AS ( + SELECT statement_id, + argMin(invocation_id, tuple(invocation_sequence, occurred_at)) AS invocation_id, + argMin(run_id, tuple(invocation_sequence, occurred_at)) AS run_id, + argMin(resource_id, tuple(invocation_sequence, occurred_at)) AS resource_id, + argMin(resource_attempt_id, tuple(invocation_sequence, occurred_at)) AS resource_attempt_id, + argMin(operation_id, tuple(invocation_sequence, occurred_at)) AS operation_id, + argMin(intent, tuple(invocation_sequence, occurred_at)) AS intent, + argMin(statement_kind, tuple(invocation_sequence, occurred_at)) AS statement_kind, + argMin(purpose, tuple(invocation_sequence, occurred_at)) AS purpose, + argMin(sql_digest, tuple(invocation_sequence, occurred_at)) AS sql_digest, + minIf(occurred_at, event_type = 'statement_started') AS started_at, + minIf(occurred_at, event_type = 'statement_submitted') AS submitted_at, + max(query_id) AS any_query_id + FROM facts GROUP BY statement_id +), +terminals AS ( + SELECT statement_id, + argMax(event_type, tuple(invocation_sequence, occurred_at)) AS event_type, + argMax(occurred_at, tuple(invocation_sequence, occurred_at)) AS ended_at, + argMax(query_id, tuple(invocation_sequence, occurred_at)) AS query_id, + argMax(error_type, tuple(invocation_sequence, occurred_at)) AS error_type, + argMax(error_code, tuple(invocation_sequence, occurred_at)) AS error_code, + argMax(duration_ms, tuple(invocation_sequence, occurred_at)) AS duration_ms, + argMax(row_count, tuple(invocation_sequence, occurred_at)) AS row_count, + argMax(affected_rows, tuple(invocation_sequence, occurred_at)) AS affected_rows + FROM facts + WHERE event_type IN ('statement_completed', 'statement_failed') + GROUP BY statement_id +), +resources AS ( + SELECT invocation_id, resource_id, min(resource_name) AS resource_name, + min(resource_kind) AS resource_kind + FROM {events} + WHERE resource_id IS NOT NULL AND resource_name IS NOT NULL + GROUP BY invocation_id, resource_id +) +SELECT + firsts.invocation_id AS invocation_id, + firsts.statement_id AS statement_id, + firsts.run_id AS run_id, + firsts.resource_id AS resource_id, + resources.resource_name AS resource_name, + resources.resource_kind AS resource_kind, + firsts.resource_attempt_id AS resource_attempt_id, + firsts.operation_id AS operation_id, + firsts.intent AS intent, + firsts.statement_kind AS statement_kind, + firsts.purpose AS purpose, + coalesce(terminals.query_id, firsts.any_query_id) AS query_id, + firsts.started_at AS started_at, + firsts.submitted_at AS submitted_at, + terminals.ended_at AS ended_at, + multiIf( + terminals.event_type = 'statement_completed', 'completed', + terminals.event_type = 'statement_failed', 'failed', + 'no_terminal' + ) AS outcome, + terminals.error_type AS error_type, + terminals.error_code AS error_code, + terminals.duration_ms AS duration_ms, + terminals.row_count AS row_count, + terminals.affected_rows AS affected_rows, + firsts.sql_digest AS sql_digest, + concat('sha256:', firsts.sql_digest) IN (SELECT digest FROM {artifacts}) AS sql_available +FROM firsts +LEFT JOIN terminals ON terminals.statement_id = firsts.statement_id +LEFT JOIN resources + ON resources.invocation_id = firsts.invocation_id + AND resources.resource_id = firsts.resource_id +SETTINGS join_use_nulls = 1" + ) +} + +fn audit_results(options: &DdlOptions) -> String { + let events = deduplicated(Table::Events, options); + format!( + "SELECT + invocation_id, + invocation_started_at, + event_id, + occurred_at, + run_id, + resource_id, + resource_attempt_id, + JSONExtract(payload, 'audit_name', 'Nullable(String)') AS audit_name, + JSONExtract(payload, 'audit_definition_name', 'Nullable(String)') AS audit_definition_name, + if(JSONHas(payload, 'target_name'), concat( + if(JSONHas(payload, 'target_database'), concat(JSONExtractString(payload, 'target_database'), '.'), ''), + if(JSONHas(payload, 'target_schema'), concat(JSONExtractString(payload, 'target_schema'), '.'), ''), + JSONExtractString(payload, 'target_name')), NULL) AS target_relation, + JSONExtract(payload, 'attached_target_kind', 'Nullable(String)') AS attached_target_kind, + JSONExtract(payload, 'attached_target_name', 'Nullable(String)') AS attached_target_name, + JSONExtract(payload, 'attached_column_name', 'Nullable(String)') AS column, + outcome, + JSONExtract(payload, 'severity', 'Nullable(String)') AS severity, + JSONExtract(payload, 'violation_count', 'Nullable(Int64)') AS violation_count, + JSONExtract(payload, 'measured_value', 'Nullable(Float64)') AS measured_value, + JSONExtract(payload, 'evaluation_mode', 'Nullable(String)') AS evaluation_mode, + JSONExtract(payload, 'execution_error', 'Nullable(String)') AS execution_error, + sql_digest, + JSONExtract(payload, 'sample_digest', 'Nullable(String)') AS sample_digest +FROM {events} +WHERE event_type = 'audit_completed'" + ) +} + +fn output_invocations(options: &DdlOptions) -> String { + let output = deduplicated(Table::Output, options); + let delivery = deduplicated(Table::Delivery, options); + let context = deduplicated(Table::Invocations, options); + format!( + "WITH +received AS ( + SELECT invocation_id, + min(invocation_started_at) AS invocation_started_at, + count() AS chunks, + sum(byte_count) AS bytes, + sumIf(byte_count, stream = 'stdout') AS stdout_bytes, + sumIf(byte_count, stream = 'stderr') AS stderr_bytes, + max(sequence) AS max_sequence, + sum(dropped_before) AS queue_drops, + sum(coalesce(omitted_sequences, 1)) AS spans, + sum(coalesce(omitted_bytes, 0)) AS truncated_bytes + FROM {output} GROUP BY invocation_id +), +delivery AS (SELECT * FROM {delivery} WHERE stream = 'output'), +ids AS ( + SELECT DISTINCT invocation_id, invocation_started_at + FROM ( + SELECT invocation_id, invocation_started_at FROM received + UNION ALL SELECT invocation_id, invocation_started_at FROM delivery + ) +), +counted AS ( + SELECT + ids.invocation_id AS invocation_id, + ids.invocation_started_at AS invocation_started_at, + coalesce(received.chunks, 0) AS chunks, + coalesce(received.bytes, 0) AS bytes, + coalesce(received.stdout_bytes, 0) AS stdout_bytes, + coalesce(received.stderr_bytes, 0) AS stderr_bytes, + coalesce(received.truncated_bytes, 0) AS truncated_bytes, + received.max_sequence AS max_sequence, + greatest( + coalesce(toInt64(received.max_sequence) + 1 - toInt64(received.spans), 0), + coalesce(toInt64(delivery.accepted), 0) - coalesce(toInt64(received.chunks), 0) + ) AS missing_sequences, + greatest(coalesce(received.queue_drops, 0), coalesce(delivery.dropped, 0)) AS reported_drops, + coalesce(delivery.filtered, 0) AS filtered, + coalesce(delivery.failed, 0) AS failed, + coalesce(context.recall_filter_active, 0) AS filter_active, + coalesce(delivery.flush_complete, 0) AS flush_complete, + delivery.invocation_id IS NOT NULL AS delivery_reported, + coalesce(delivery.delivered, 0) AS delivered + FROM ids + LEFT JOIN received ON received.invocation_id = ids.invocation_id + LEFT JOIN delivery ON delivery.invocation_id = ids.invocation_id + LEFT JOIN {context} AS context ON context.invocation_id = ids.invocation_id + SETTINGS join_use_nulls = 1 +) +SELECT + invocation_id, invocation_started_at, chunks, bytes, stdout_bytes, stderr_bytes, + truncated_bytes, max_sequence, missing_sequences, reported_drops, filtered, failed, + filter_active, flush_complete, + multiIf( + missing_sequences = 0 AND truncated_bytes > 0, 'truncated', + missing_sequences = 0, NULL, + filtered >= missing_sequences, 'filtered', + filtered + reported_drops + failed >= missing_sequences, 'dropped', + 'unexplained' + ) AS gap_reason, + toUInt8(delivery_reported AND flush_complete = 1 AND missing_sequences = 0 + AND reported_drops = 0 AND filtered = 0 AND failed = 0 AND truncated_bytes = 0 + AND chunks >= delivered) + AS complete +FROM counted" + ) +} + +fn output_tail(options: &DdlOptions) -> String { + let output = deduplicated(Table::Output, options); + format!( + "SELECT invocation_id, invocation_started_at, sequence, stream, occurred_at, byte_count, + toValidUTF8(data) AS text +FROM {output} +ORDER BY invocation_id, sequence DESC +LIMIT {OUTPUT_TAIL_RECORDS} BY invocation_id" + ) +} + +fn output_transcripts(options: &DdlOptions) -> String { + let output = deduplicated(Table::Output, options); + format!( + "SELECT + invocation_id, + min(invocation_started_at) AS invocation_started_at, + arrayStringConcat(arrayMap(chunk -> chunk.2, arraySort(chunk -> chunk.1, + groupArrayIf(tuple(sequence, data), stream = 'stdout')))) AS stdout, + arrayStringConcat(arrayMap(chunk -> chunk.2, arraySort(chunk -> chunk.1, + groupArrayIf(tuple(sequence, data), stream = 'stderr')))) AS stderr, + arrayStringConcat(arrayMap(chunk -> chunk.2, arraySort(chunk -> chunk.1, + groupArray(tuple(sequence, data))))) AS combined +FROM {output} +GROUP BY invocation_id" + ) +} + +fn passthrough(table: Table, options: &DdlOptions) -> String { + format!("SELECT * FROM {}", deduplicated(table, options)) +} + +fn artifacts(options: &DdlOptions) -> String { + let refs = deduplicated(Table::ArtifactRefs, options); + let stored = deduplicated(Table::Artifacts, options); + format!( + "SELECT + refs.invocation_id AS invocation_id, + refs.invocation_started_at AS invocation_started_at, + refs.digest AS digest, + refs.ref_id AS ref_id, + refs.role AS role, + refs.statement_id AS statement_id, + refs.resource_id AS resource_id, + artifact.kind AS kind, + artifact.media_type AS media_type, + artifact.size AS size, + artifact.digest IS NOT NULL AS available, + if(artifact.digest IS NULL, 'expired_or_not_captured', NULL) AS unavailable_reason, + if(startsWith(artifact.media_type, 'text/'), toValidUTF8(artifact.content), NULL) AS text, + artifact.content AS content +FROM {refs} AS refs +LEFT JOIN {stored} AS artifact ON artifact.digest = refs.digest +SETTINGS join_use_nulls = 1" + ) +} + +/// Each supported view's name and ClickHouse body, in creation order. +pub(crate) fn clickhouse_view_bodies(options: &DdlOptions) -> Vec<(&'static str, String)> { + vec![ + ("invocations", invocations(options)), + ("statements", statements(options)), + ("audit_results", audit_results(options)), + ("output_invocations", output_invocations(options)), + ("output_tail", output_tail(options)), + ("output_transcripts", output_transcripts(options)), + ("events", passthrough(Table::Events, options)), + ("diagnostics", passthrough(Table::Diagnostics, options)), + ("artifacts", artifacts(options)), + ("conflicts", passthrough(Table::Conflicts, options)), + ] +} diff --git a/crates/sqlbuild-recall/src/schema/_helpers/mod.rs b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs new file mode 100644 index 0000000000..8e71b76aae --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/_helpers/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod clickhouse_views; +pub(crate) mod rendering; +pub(crate) mod sqlite_views; diff --git a/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs b/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs new file mode 100644 index 0000000000..658aef267d --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/_helpers/rendering.rs @@ -0,0 +1,88 @@ +//! Column, table and identifier rendering shared by the DDL statements. + +use crate::schema::constants::COMMON_COLUMNS; +use crate::schema::models::{ColumnSpec, ColumnType, DdlOptions, TableSpec}; + +/// The table's own columns followed by the common columns. +pub(crate) fn all_columns(spec: &TableSpec) -> Vec { + spec.columns + .iter() + .chain(COMMON_COLUMNS.iter()) + .copied() + .collect() +} + +pub(crate) fn sqlite_type(column_type: ColumnType) -> &'static str { + match column_type { + ColumnType::Text | ColumnType::Timestamp | ColumnType::Json => "TEXT", + ColumnType::Integer | ColumnType::Boolean => "INTEGER", + ColumnType::Real => "REAL", + ColumnType::Blob => "BLOB", + } +} + +pub(crate) fn clickhouse_type(column: &ColumnSpec) -> String { + let base: &str = match column.column_type { + ColumnType::Text | ColumnType::Json | ColumnType::Blob => "String", + ColumnType::Integer => "Int64", + ColumnType::Real => "Float64", + ColumnType::Boolean => "UInt8", + ColumnType::Timestamp => "DateTime64(6, 'UTC')", + }; + if column.nullable { + format!("Nullable({base})") + } else { + base.to_owned() + } +} + +/// `"name"` for SQLite. +pub(crate) fn sqlite_identifier(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) +} + +/// `` `name` `` for ClickHouse. +pub(crate) fn clickhouse_identifier(name: &str) -> String { + format!("`{}`", name.replace('\\', "\\\\").replace('`', "\\`")) +} + +/// A ClickHouse table or view name, qualified by the configured database. +pub(crate) fn clickhouse_table(name: &str, options: &DdlOptions) -> String { + match &options.database { + Some(database) => format!( + "{}.{}", + clickhouse_identifier(database), + clickhouse_identifier(name) + ), + None => clickhouse_identifier(name), + } +} + +/// ` ON CLUSTER `c`` when a cluster is configured. +pub(crate) fn on_cluster(options: &DdlOptions) -> String { + options + .cluster + .as_deref() + .map(|cluster| format!(" ON CLUSTER {}", clickhouse_identifier(cluster))) + .unwrap_or_default() +} + +/// The TTL expression of a table: expiry column plus retention, artifacts outliving references. +pub(crate) fn clickhouse_ttl(spec: &TableSpec, options: &DdlOptions) -> String { + let days: u32 = options.retention_days.saturating_add(spec.ttl_extra_days); + format!("{} + INTERVAL {days} DAY", spec.expiry_column) +} + +/// The engine clause, replicated with one shard and every replica on a cluster. +pub(crate) fn clickhouse_engine(spec: &TableSpec, options: &DdlOptions) -> String { + let version: &str = spec.clickhouse_version.unwrap_or_default(); + if options.cluster.is_some() { + let separator: &str = if version.is_empty() { "" } else { ", " }; + format!( + "Replicated{}('/clickhouse/tables/{{uuid}}/{{shard}}', '{{replica}}'{separator}{version})", + spec.clickhouse_engine + ) + } else { + format!("{}({version})", spec.clickhouse_engine) + } +} diff --git a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs new file mode 100644 index 0000000000..bc7a1922d2 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs @@ -0,0 +1,369 @@ +//! The bodies of the supported views in SQLite. + +use crate::schema::constants::OUTPUT_TAIL_RECORDS; + +/// The current time as stored timestamps spell it. +const NOW: &str = "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')"; + +const STATEMENTS: &str = "WITH facts AS ( + SELECT * + FROM recall_events + WHERE statement_id IS NOT NULL AND event_type LIKE 'statement!_%' ESCAPE '!' +), +firsts AS ( + SELECT invocation_id, statement_id, run_id, resource_id, resource_attempt_id, operation_id, + intent, statement_kind, purpose, sql_digest, + ROW_NUMBER() OVER ( + PARTITION BY statement_id ORDER BY invocation_sequence, occurred_at, event_id + ) AS position + FROM facts +), +started AS ( + SELECT statement_id, MIN(occurred_at) AS started_at + FROM facts WHERE event_type = 'statement_started' GROUP BY statement_id +), +submitted AS ( + SELECT statement_id, MIN(occurred_at) AS submitted_at + FROM facts WHERE event_type = 'statement_submitted' GROUP BY statement_id +), +query_ids AS ( + SELECT statement_id, MAX(query_id) AS query_id FROM facts GROUP BY statement_id +), +terminals AS ( + SELECT statement_id, event_type, occurred_at, query_id, error_type, error_code, duration_ms, + row_count, affected_rows, + ROW_NUMBER() OVER ( + PARTITION BY statement_id + ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC + ) AS position + FROM facts + WHERE event_type IN ('statement_completed', 'statement_failed') +), +resources AS ( + SELECT invocation_id, resource_id, MIN(resource_name) AS resource_name, + MIN(resource_kind) AS resource_kind + FROM recall_events + WHERE resource_id IS NOT NULL AND resource_name IS NOT NULL + GROUP BY invocation_id, resource_id +) +SELECT + firsts.invocation_id, + firsts.statement_id, + firsts.run_id, + firsts.resource_id, + resources.resource_name, + resources.resource_kind, + firsts.resource_attempt_id, + firsts.operation_id, + firsts.intent, + firsts.statement_kind, + firsts.purpose, + COALESCE(terminals.query_id, query_ids.query_id) AS query_id, + started.started_at, + submitted.submitted_at, + terminals.occurred_at AS ended_at, + CASE terminals.event_type + WHEN 'statement_completed' THEN 'completed' + WHEN 'statement_failed' THEN 'failed' + ELSE 'no_terminal' + END AS outcome, + terminals.error_type, + terminals.error_code, + terminals.duration_ms, + terminals.row_count, + terminals.affected_rows, + firsts.sql_digest, + EXISTS ( + SELECT 1 FROM recall_artifacts AS artifact + WHERE artifact.digest = 'sha256:' || firsts.sql_digest + ) AS sql_available +FROM firsts +LEFT JOIN started ON started.statement_id = firsts.statement_id +LEFT JOIN submitted ON submitted.statement_id = firsts.statement_id +LEFT JOIN query_ids ON query_ids.statement_id = firsts.statement_id +LEFT JOIN terminals + ON terminals.statement_id = firsts.statement_id AND terminals.position = 1 +LEFT JOIN resources + ON resources.invocation_id = firsts.invocation_id + AND resources.resource_id = firsts.resource_id +WHERE firsts.position = 1"; + +const AUDIT_RESULTS: &str = "SELECT + invocation_id, + invocation_started_at, + event_id, + occurred_at, + run_id, + resource_id, + resource_attempt_id, + json_extract(payload, '$.audit_name') AS audit_name, + json_extract(payload, '$.audit_definition_name') AS audit_definition_name, + CASE WHEN json_extract(payload, '$.target_name') IS NULL THEN NULL ELSE + COALESCE(json_extract(payload, '$.target_database') || '.', '') + || COALESCE(json_extract(payload, '$.target_schema') || '.', '') + || json_extract(payload, '$.target_name') + END AS target_relation, + json_extract(payload, '$.attached_target_kind') AS attached_target_kind, + json_extract(payload, '$.attached_target_name') AS attached_target_name, + json_extract(payload, '$.attached_column_name') AS \"column\", + outcome, + json_extract(payload, '$.severity') AS severity, + json_extract(payload, '$.violation_count') AS violation_count, + json_extract(payload, '$.measured_value') AS measured_value, + json_extract(payload, '$.evaluation_mode') AS evaluation_mode, + json_extract(payload, '$.execution_error') AS execution_error, + sql_digest, + json_extract(payload, '$.sample_digest') AS sample_digest +FROM recall_events +WHERE event_type = 'audit_completed'"; + +const OUTPUT_INVOCATIONS: &str = "WITH received AS ( + SELECT invocation_id, + MIN(invocation_started_at) AS invocation_started_at, + COUNT(*) AS chunks, + SUM(byte_count) AS bytes, + SUM(CASE WHEN stream = 'stdout' THEN byte_count ELSE 0 END) AS stdout_bytes, + SUM(CASE WHEN stream = 'stderr' THEN byte_count ELSE 0 END) AS stderr_bytes, + MAX(sequence) AS max_sequence, + SUM(dropped_before) AS queue_drops, + SUM(COALESCE(omitted_sequences, 1)) AS spans, + SUM(COALESCE(omitted_bytes, 0)) AS truncated_bytes + FROM recall_output + GROUP BY invocation_id +), +delivery AS ( + SELECT * FROM recall_delivery WHERE stream = 'output' +), +ids AS ( + SELECT invocation_id, invocation_started_at FROM received + UNION SELECT invocation_id, invocation_started_at FROM delivery +), +counted AS ( + SELECT + ids.invocation_id, + ids.invocation_started_at, + COALESCE(received.chunks, 0) AS chunks, + COALESCE(received.bytes, 0) AS bytes, + COALESCE(received.stdout_bytes, 0) AS stdout_bytes, + COALESCE(received.stderr_bytes, 0) AS stderr_bytes, + COALESCE(received.truncated_bytes, 0) AS truncated_bytes, + received.max_sequence, + MAX( + COALESCE(received.max_sequence + 1 - received.spans, 0), + COALESCE(delivery.accepted, 0) - COALESCE(received.chunks, 0) + ) AS missing_sequences, + MAX(COALESCE(received.queue_drops, 0), COALESCE(delivery.dropped, 0)) AS reported_drops, + COALESCE(delivery.filtered, 0) AS filtered, + COALESCE(delivery.failed, 0) AS failed, + COALESCE(context.recall_filter_active, 0) AS filter_active, + COALESCE(delivery.flush_complete, 0) AS flush_complete, + delivery.invocation_id IS NOT NULL AS delivery_reported, + COALESCE(delivery.delivered, 0) AS delivered + FROM ids + LEFT JOIN received ON received.invocation_id = ids.invocation_id + LEFT JOIN delivery ON delivery.invocation_id = ids.invocation_id + LEFT JOIN recall_invocations AS context ON context.invocation_id = ids.invocation_id +) +SELECT + invocation_id, + invocation_started_at, + chunks, + bytes, + stdout_bytes, + stderr_bytes, + truncated_bytes, + max_sequence, + missing_sequences, + reported_drops, + filtered, + failed, + filter_active, + flush_complete, + CASE + WHEN missing_sequences = 0 AND truncated_bytes > 0 THEN 'truncated' + WHEN missing_sequences = 0 THEN NULL + WHEN filtered >= missing_sequences THEN 'filtered' + WHEN filtered + reported_drops + failed >= missing_sequences THEN 'dropped' + ELSE 'unexplained' + END AS gap_reason, + CASE + WHEN delivery_reported AND flush_complete = 1 AND missing_sequences = 0 + AND reported_drops = 0 AND filtered = 0 AND failed = 0 AND truncated_bytes = 0 + AND chunks >= delivered + THEN 1 ELSE 0 + END AS complete +FROM counted"; + +const OUTPUT_TRANSCRIPTS: &str = "SELECT + invocation_id, + MIN(invocation_started_at) AS invocation_started_at, + CAST(COALESCE( + group_concat(CASE WHEN stream = 'stdout' THEN data END, x'' ORDER BY sequence), x'' + ) AS BLOB) AS stdout, + CAST(COALESCE( + group_concat(CASE WHEN stream = 'stderr' THEN data END, x'' ORDER BY sequence), x'' + ) AS BLOB) AS stderr, + CAST(COALESCE(group_concat(data, x'' ORDER BY sequence), x'') AS BLOB) AS combined +FROM recall_output +GROUP BY invocation_id"; + +const EVENTS: &str = "SELECT * FROM recall_events"; + +const DIAGNOSTICS: &str = "SELECT * FROM recall_diagnostics"; + +const ARTIFACTS: &str = "SELECT + refs.invocation_id, + refs.invocation_started_at, + refs.digest, + refs.ref_id, + refs.role, + refs.statement_id, + refs.resource_id, + artifact.kind, + artifact.media_type, + artifact.size, + artifact.digest IS NOT NULL AS available, + CASE WHEN artifact.digest IS NULL THEN 'expired_or_not_captured' END AS unavailable_reason, + CASE WHEN artifact.media_type LIKE 'text/%' THEN CAST(artifact.content AS TEXT) END AS text, + artifact.content +FROM recall_artifact_refs AS refs +LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest"; + +const CONFLICTS: &str = "SELECT * FROM recall_conflicts"; + +fn invocations() -> String { + format!( + "WITH ids AS ( + SELECT invocation_id, MIN(invocation_started_at) AS invocation_started_at + FROM ( + SELECT invocation_id, invocation_started_at FROM recall_invocations + UNION ALL SELECT invocation_id, invocation_started_at FROM recall_heartbeats + UNION ALL SELECT invocation_id, invocation_started_at FROM recall_events + WHERE event_type IN ('invocation_started', 'invocation_completed', 'invocation_failed') + ) + GROUP BY invocation_id +), +started AS ( + SELECT invocation_id, MIN(occurred_at) AS started_at, MIN(command) AS command + FROM recall_events + WHERE event_type = 'invocation_started' + GROUP BY invocation_id +), +terminals AS ( + SELECT invocation_id, event_type, occurred_at, exit_code, duration_ms, error_type, error_code, + ROW_NUMBER() OVER ( + PARTITION BY invocation_id + ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC + ) AS position + FROM recall_events + WHERE event_type IN ('invocation_completed', 'invocation_failed') +), +beats AS ( + SELECT invocation_id, MAX(beat_at) AS last_heartbeat_at, MAX(stale_at) AS stale_at + FROM recall_heartbeats + GROUP BY invocation_id +), +projected AS ( + SELECT + ids.invocation_id, + ids.invocation_started_at, + COALESCE(started.started_at, ids.invocation_started_at) AS started_at, + CASE + WHEN terminals.event_type = 'invocation_completed' THEN 'completed' + WHEN terminals.event_type = 'invocation_failed' THEN 'failed' + WHEN beats.stale_at IS NULL THEN 'unknown' + WHEN beats.stale_at >= {NOW} THEN 'running' + ELSE 'abandoned' + END AS status, + terminals.event_type AS terminal_event, + terminals.occurred_at AS terminal_at, + terminals.duration_ms AS terminal_duration_ms, + terminals.exit_code, + terminals.error_type, + terminals.error_code, + COALESCE(context.command, started.command) AS command, + context.subcommand, + context.target, + context.adapter, + context.project_name, + context.selection, + context.exclusion, + context.sqlbuild_version, + context.git_revision, + context.git_dirty, + context.host_token, + context.pid, + context.external_context, + context.recall_filter_active, + beats.last_heartbeat_at + FROM ids + LEFT JOIN recall_invocations AS context ON context.invocation_id = ids.invocation_id + LEFT JOIN started ON started.invocation_id = ids.invocation_id + LEFT JOIN terminals + ON terminals.invocation_id = ids.invocation_id AND terminals.position = 1 + LEFT JOIN beats ON beats.invocation_id = ids.invocation_id +) +SELECT + invocation_id, + invocation_started_at, + started_at, + status, + CASE + WHEN terminal_event IS NOT NULL THEN terminal_at + WHEN status = 'abandoned' THEN last_heartbeat_at + END AS ended_at, + CASE + WHEN terminal_event IS NOT NULL THEN terminal_duration_ms + WHEN status = 'abandoned' THEN ROUND( + (unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3 + ) + END AS duration_ms, + exit_code, + error_type, + error_code, + command, + subcommand, + target, + adapter, + project_name, + selection, + exclusion, + sqlbuild_version, + git_revision, + git_dirty, + host_token, + pid, + external_context, + recall_filter_active, + last_heartbeat_at +FROM projected" + ) +} + +fn output_tail() -> String { + format!( + "SELECT invocation_id, invocation_started_at, sequence, stream, occurred_at, byte_count, + CAST(data AS TEXT) AS text +FROM ( + SELECT *, ROW_NUMBER() OVER (PARTITION BY invocation_id ORDER BY sequence DESC) AS position + FROM recall_output +) +WHERE position <= {OUTPUT_TAIL_RECORDS}" + ) +} + +/// Each supported view's name and SQLite body, in creation order. +pub(crate) fn sqlite_view_bodies() -> Vec<(&'static str, String)> { + vec![ + ("invocations", invocations()), + ("statements", STATEMENTS.to_owned()), + ("audit_results", AUDIT_RESULTS.to_owned()), + ("output_invocations", OUTPUT_INVOCATIONS.to_owned()), + ("output_tail", output_tail()), + ("output_transcripts", OUTPUT_TRANSCRIPTS.to_owned()), + ("events", EVENTS.to_owned()), + ("diagnostics", DIAGNOSTICS.to_owned()), + ("artifacts", ARTIFACTS.to_owned()), + ("conflicts", CONFLICTS.to_owned()), + ] +} diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs new file mode 100644 index 0000000000..0da7cf7f80 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -0,0 +1,344 @@ +//! Recall table definitions, view names and schema versions. + +use crate::schema::models::{ColumnSpec, ColumnType, Table, TableSpec}; + +/// The Recall schema version this build writes. Raise it when a column or view is added. +pub const RECALL_SCHEMA_VERSION: i64 = 1; +/// The oldest SQLite library the store's views run on (ordered `group_concat`). +pub const MINIMUM_SQLITE_VERSION: (u32, u32) = (3, 44); +/// The `recall_meta` component of schema rows. +pub const META_SCHEMA_COMPONENT: &str = "schema"; +pub const META_SCHEMA_VERSION_KEY: &str = "schema_version"; +/// The `recall_meta` component of maintenance claims. +pub const META_MAINTENANCE_COMPONENT: &str = "maintenance"; +pub const META_MAINTENANCE_CLAIM_KEY: &str = "claim"; + +/// Columns every Recall row carries, after the table's own columns. +pub const COMMON_COLUMNS: [ColumnSpec; 6] = [ + ColumnSpec::required("invocation_id", ColumnType::Text), + ColumnSpec::required("invocation_started_at", ColumnType::Timestamp), + ColumnSpec::required("schema_version", ColumnType::Integer), + ColumnSpec::required("producer_version", ColumnType::Text), + ColumnSpec::required("ingested_at", ColumnType::Timestamp), + ColumnSpec::required("content_hash", ColumnType::Text), +]; +/// The artifact columns its content digest covers: what it is, not who saw it when. +pub const ARTIFACT_CONTENT_COLUMNS: [&str; 5] = ["digest", "kind", "media_type", "size", "content"]; +/// Common columns left out of a row's content digest. +pub const UNHASHED_COLUMNS: [&str; 2] = ["ingested_at", "content_hash"]; + +const CORRELATIONS: [ColumnSpec; 5] = [ + ColumnSpec::optional("run_id", ColumnType::Text), + ColumnSpec::optional("resource_id", ColumnType::Text), + ColumnSpec::optional("resource_attempt_id", ColumnType::Text), + ColumnSpec::optional("operation_id", ColumnType::Text), + ColumnSpec::optional("statement_id", ColumnType::Text), +]; + +const INVOCATION_COLUMNS: [ColumnSpec; 19] = [ + ColumnSpec::optional("command", ColumnType::Text), + ColumnSpec::optional("subcommand", ColumnType::Text), + ColumnSpec::optional("selection", ColumnType::Json), + ColumnSpec::optional("exclusion", ColumnType::Json), + ColumnSpec::optional("project_name", ColumnType::Text), + ColumnSpec::optional("project_dir_hash", ColumnType::Text), + ColumnSpec::optional("target", ColumnType::Text), + ColumnSpec::optional("adapter", ColumnType::Text), + ColumnSpec::optional("destination_database", ColumnType::Text), + ColumnSpec::optional("destination_schema", ColumnType::Text), + ColumnSpec::optional("destination_fingerprint", ColumnType::Text), + ColumnSpec::optional("sqlbuild_version", ColumnType::Text), + ColumnSpec::optional("git_revision", ColumnType::Text), + ColumnSpec::optional("git_dirty", ColumnType::Boolean), + ColumnSpec::optional("host_token", ColumnType::Text), + ColumnSpec::optional("pid", ColumnType::Integer), + ColumnSpec::optional("external_context", ColumnType::Json), + ColumnSpec::optional("recall_filter_active", ColumnType::Boolean), + ColumnSpec::optional("argv_digest", ColumnType::Text), +]; + +const OUTPUT_COLUMNS: [ColumnSpec; 11] = [ + ColumnSpec::required("record_id", ColumnType::Text), + ColumnSpec::required("sequence", ColumnType::Integer), + ColumnSpec::required("stream", ColumnType::Text), + ColumnSpec::required("occurred_at", ColumnType::Timestamp), + ColumnSpec::required("data", ColumnType::Blob), + ColumnSpec::required("byte_offset", ColumnType::Integer), + ColumnSpec::required("byte_count", ColumnType::Integer), + ColumnSpec::optional("priority", ColumnType::Text), + ColumnSpec::required("dropped_before", ColumnType::Integer), + ColumnSpec::optional("omitted_bytes", ColumnType::Integer), + ColumnSpec::optional("omitted_sequences", ColumnType::Integer), +]; + +const EVENT_COLUMNS: [ColumnSpec; 29] = [ + ColumnSpec::required("event_id", ColumnType::Text), + ColumnSpec::optional("invocation_sequence", ColumnType::Integer), + ColumnSpec::required("occurred_at", ColumnType::Timestamp), + ColumnSpec::required("event_type", ColumnType::Text), + ColumnSpec::required("event_kind", ColumnType::Text), + ColumnSpec::required("severity", ColumnType::Text), + ColumnSpec::required("producer", ColumnType::Text), + ColumnSpec::required("event_schema_version", ColumnType::Integer), + CORRELATIONS[0], + CORRELATIONS[1], + CORRELATIONS[2], + CORRELATIONS[3], + CORRELATIONS[4], + ColumnSpec::optional("command", ColumnType::Text), + ColumnSpec::optional("resource_kind", ColumnType::Text), + ColumnSpec::optional("resource_name", ColumnType::Text), + ColumnSpec::optional("error_code", ColumnType::Text), + ColumnSpec::optional("error_type", ColumnType::Text), + ColumnSpec::optional("duration_ms", ColumnType::Real), + ColumnSpec::optional("exit_code", ColumnType::Integer), + ColumnSpec::optional("query_id", ColumnType::Text), + ColumnSpec::optional("sql_digest", ColumnType::Text), + ColumnSpec::optional("intent", ColumnType::Text), + ColumnSpec::optional("statement_kind", ColumnType::Text), + ColumnSpec::optional("purpose", ColumnType::Text), + ColumnSpec::optional("outcome", ColumnType::Text), + ColumnSpec::optional("row_count", ColumnType::Integer), + ColumnSpec::optional("affected_rows", ColumnType::Integer), + ColumnSpec::required("payload", ColumnType::Json), +]; + +const DIAGNOSTIC_COLUMNS: [ColumnSpec; 14] = [ + ColumnSpec::required("record_id", ColumnType::Text), + ColumnSpec::required("sequence", ColumnType::Integer), + ColumnSpec::required("occurred_at", ColumnType::Timestamp), + ColumnSpec::required("severity", ColumnType::Text), + ColumnSpec::required("logger", ColumnType::Text), + ColumnSpec::required("source", ColumnType::Text), + ColumnSpec::required("message", ColumnType::Text), + ColumnSpec::required("fields", ColumnType::Json), + ColumnSpec::optional("log_stream_id", ColumnType::Text), + CORRELATIONS[0], + CORRELATIONS[1], + CORRELATIONS[2], + CORRELATIONS[3], + CORRELATIONS[4], +]; + +const ARTIFACT_COLUMNS: [ColumnSpec; 6] = [ + ColumnSpec::required("digest", ColumnType::Text), + ColumnSpec::required("kind", ColumnType::Text), + ColumnSpec::required("media_type", ColumnType::Text), + ColumnSpec::required("size", ColumnType::Integer), + ColumnSpec::required("content", ColumnType::Blob), + ColumnSpec::required("seen_at", ColumnType::Timestamp), +]; + +const ARTIFACT_REF_COLUMNS: [ColumnSpec; 6] = [ + ColumnSpec::required("digest", ColumnType::Text), + ColumnSpec::required("ref_id", ColumnType::Text), + ColumnSpec::optional("statement_id", ColumnType::Text), + ColumnSpec::optional("resource_id", ColumnType::Text), + ColumnSpec::required("role", ColumnType::Text), + ColumnSpec::required("referenced_at", ColumnType::Timestamp), +]; + +const HEARTBEAT_COLUMNS: [ColumnSpec; 4] = [ + ColumnSpec::required("beat_at", ColumnType::Timestamp), + ColumnSpec::required("stale_at", ColumnType::Timestamp), + ColumnSpec::optional("host_token", ColumnType::Text), + ColumnSpec::optional("pid", ColumnType::Integer), +]; + +const DELIVERY_COLUMNS: [ColumnSpec; 9] = [ + ColumnSpec::required("stream", ColumnType::Text), + ColumnSpec::required("accepted", ColumnType::Integer), + ColumnSpec::required("delivered", ColumnType::Integer), + ColumnSpec::required("filtered", ColumnType::Integer), + ColumnSpec::required("dropped", ColumnType::Integer), + ColumnSpec::required("failed", ColumnType::Integer), + ColumnSpec::required("conflicts", ColumnType::Integer), + ColumnSpec::required("flush_complete", ColumnType::Boolean), + ColumnSpec::required("written_at", ColumnType::Timestamp), +]; + +const META_COLUMNS: [ColumnSpec; 4] = [ + ColumnSpec::required("component", ColumnType::Text), + ColumnSpec::required("key", ColumnType::Text), + ColumnSpec::required("value", ColumnType::Text), + ColumnSpec::required("recorded_at", ColumnType::Timestamp), +]; + +const CONFLICT_COLUMNS: [ColumnSpec; 5] = [ + ColumnSpec::required("dataset", ColumnType::Text), + ColumnSpec::required("record_key", ColumnType::Text), + ColumnSpec::required("existing_sha256", ColumnType::Text), + ColumnSpec::required("rejected_sha256", ColumnType::Text), + ColumnSpec::required("detected_at", ColumnType::Timestamp), +]; + +pub(crate) const INVOCATIONS_TABLE: TableSpec = TableSpec { + name: "recall_invocations", + clickhouse_version: None, + columns: &INVOCATION_COLUMNS, + key: &["invocation_id"], + sqlite_indexes: &[("started", &["invocation_started_at"])], + clickhouse_engine: "ReplacingMergeTree", + clickhouse_order_by: &["invocation_started_at", "invocation_id"], + ttl_extra_days: 0, + expiry_column: "invocation_started_at", +}; +pub(crate) const OUTPUT_TABLE: TableSpec = TableSpec { + name: "recall_output", + clickhouse_version: None, + columns: &OUTPUT_COLUMNS, + key: &["invocation_id", "sequence"], + sqlite_indexes: &[("expiry", &["invocation_started_at"])], + clickhouse_engine: "ReplacingMergeTree", + clickhouse_order_by: &["invocation_id", "sequence"], + ttl_extra_days: 0, + expiry_column: "invocation_started_at", +}; +pub(crate) const EVENTS_TABLE: TableSpec = TableSpec { + name: "recall_events", + clickhouse_version: None, + columns: &EVENT_COLUMNS, + key: &["event_id"], + sqlite_indexes: &[ + ("invocation", &["invocation_id", "invocation_sequence"]), + ("type", &["event_type", "occurred_at"]), + ("statement", &["statement_id"]), + ("expiry", &["invocation_started_at"]), + ], + clickhouse_engine: "ReplacingMergeTree", + clickhouse_order_by: &["invocation_id", "event_id"], + ttl_extra_days: 0, + expiry_column: "invocation_started_at", +}; +pub(crate) const DIAGNOSTICS_TABLE: TableSpec = TableSpec { + name: "recall_diagnostics", + clickhouse_version: None, + columns: &DIAGNOSTIC_COLUMNS, + key: &["invocation_id", "sequence"], + sqlite_indexes: &[ + ("severity", &["severity"]), + ("expiry", &["invocation_started_at"]), + ], + clickhouse_engine: "ReplacingMergeTree", + clickhouse_order_by: &["invocation_id", "sequence"], + ttl_extra_days: 0, + expiry_column: "invocation_started_at", +}; +pub(crate) const ARTIFACTS_TABLE: TableSpec = TableSpec { + name: "recall_artifacts", + clickhouse_version: Some("seen_at"), + columns: &ARTIFACT_COLUMNS, + key: &["digest"], + sqlite_indexes: &[("seen", &["seen_at"])], + clickhouse_engine: "ReplacingMergeTree", + clickhouse_order_by: &["digest"], + ttl_extra_days: ARTIFACT_REFRESH_DAYS, + expiry_column: "seen_at", +}; +pub(crate) const ARTIFACT_REFS_TABLE: TableSpec = TableSpec { + name: "recall_artifact_refs", + clickhouse_version: None, + columns: &ARTIFACT_REF_COLUMNS, + key: &["invocation_id", "digest", "ref_id"], + sqlite_indexes: &[("digest", &["digest"])], + clickhouse_engine: "MergeTree", + clickhouse_order_by: &["invocation_id", "digest", "ref_id"], + ttl_extra_days: 0, + expiry_column: "invocation_started_at", +}; +pub(crate) const HEARTBEATS_TABLE: TableSpec = TableSpec { + name: "recall_heartbeats", + clickhouse_version: None, + columns: &HEARTBEAT_COLUMNS, + key: &["invocation_id", "beat_at"], + sqlite_indexes: &[], + clickhouse_engine: "MergeTree", + clickhouse_order_by: &["invocation_id", "beat_at"], + ttl_extra_days: 0, + expiry_column: "invocation_started_at", +}; +pub(crate) const DELIVERY_TABLE: TableSpec = TableSpec { + name: "recall_delivery", + clickhouse_version: None, + columns: &DELIVERY_COLUMNS, + key: &["invocation_id", "stream"], + sqlite_indexes: &[], + clickhouse_engine: "MergeTree", + clickhouse_order_by: &["invocation_id", "stream"], + ttl_extra_days: 0, + expiry_column: "invocation_started_at", +}; +pub(crate) const META_TABLE: TableSpec = TableSpec { + name: "recall_meta", + clickhouse_version: None, + columns: &META_COLUMNS, + key: &["component", "key", "recorded_at"], + sqlite_indexes: &[], + clickhouse_engine: "MergeTree", + clickhouse_order_by: &["component", "key", "recorded_at"], + ttl_extra_days: 0, + expiry_column: "recorded_at", +}; +pub(crate) const CONFLICTS_TABLE: TableSpec = TableSpec { + name: "recall_conflicts", + clickhouse_version: None, + columns: &CONFLICT_COLUMNS, + key: &["dataset", "record_key", "rejected_sha256"], + sqlite_indexes: &[("invocation", &["invocation_id"])], + clickhouse_engine: "MergeTree", + clickhouse_order_by: &["dataset", "record_key", "rejected_sha256"], + ttl_extra_days: 0, + expiry_column: "invocation_started_at", +}; + +/// Every table, in creation order. +pub const ALL_TABLES: [Table; 10] = [ + Table::Meta, + Table::Invocations, + Table::Output, + Table::Events, + Table::Diagnostics, + Table::Artifacts, + Table::ArtifactRefs, + Table::Heartbeats, + Table::Delivery, + Table::Conflicts, +]; +/// Tables whose rows belong to one invocation and expire with it. +pub const INVOCATION_TABLES: [Table; 8] = [ + Table::Invocations, + Table::Output, + Table::Events, + Table::Diagnostics, + Table::ArtifactRefs, + Table::Heartbeats, + Table::Delivery, + Table::Conflicts, +]; +/// The supported views, in creation order. +pub const VIEW_NAMES: [&str; 10] = [ + "invocations", + "statements", + "audit_results", + "output_invocations", + "output_tail", + "output_transcripts", + "events", + "diagnostics", + "artifacts", + "conflicts", +]; +/// The number of records `output_tail` keeps per invocation. +pub const OUTPUT_TAIL_RECORDS: u32 = 200; +/// Invocations deleted per maintenance transaction. +pub const RETENTION_BATCH_INVOCATIONS: u32 = 200; +/// ClickHouse re-inserts an artifact whose newest copy is older than this. +pub const ARTIFACT_REFRESH_DAYS: u32 = 7; +/// Tables whose expiry index finds candidate invocations for retention. +pub(crate) const RETENTION_CANDIDATE_TABLES: [Table; 5] = [ + Table::Invocations, + Table::Heartbeats, + Table::Events, + Table::Output, + Table::Diagnostics, +]; diff --git a/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs b/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs new file mode 100644 index 0000000000..cea0815488 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/add_column_statement.rs @@ -0,0 +1,37 @@ +//! `add_column_statement`: the additive upgrade of one table by one column. + +use crate::schema::_helpers::rendering::{ + clickhouse_identifier, clickhouse_table, clickhouse_type, on_cluster, sqlite_identifier, + sqlite_type, +}; +use crate::schema::models::{ColumnSpec, DdlOptions, Dialect, TableSpec}; + +/// `ALTER TABLE … ADD COLUMN`; added columns are always nullable so older rows stay valid. +#[must_use] +pub fn add_column_statement( + dialect: Dialect, + options: &DdlOptions, + (spec, column): (&TableSpec, &ColumnSpec), +) -> String { + match dialect { + Dialect::Sqlite => format!( + "ALTER TABLE {} ADD COLUMN {} {}", + sqlite_identifier(spec.name), + sqlite_identifier(column.name), + sqlite_type(column.column_type) + ), + Dialect::ClickHouse => { + let nullable: ColumnSpec = ColumnSpec { + nullable: true, + ..*column + }; + format!( + "ALTER TABLE {}{} ADD COLUMN IF NOT EXISTS {} {}", + clickhouse_table(spec.name, options), + on_cluster(options), + clickhouse_identifier(column.name), + clickhouse_type(&nullable) + ) + } + } +} diff --git a/crates/sqlbuild-recall/src/schema/main/ddl_script.rs b/crates/sqlbuild-recall/src/schema/main/ddl_script.rs new file mode 100644 index 0000000000..42ea6a014c --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/ddl_script.rs @@ -0,0 +1,19 @@ +//! `ddl_script`: the complete ordered script an administrator runs to create a Recall store. + +use crate::schema::main::table_statements::table_statements; +use crate::schema::main::view_statements::view_statements; +use crate::schema::models::{DdlOptions, Dialect}; + +/// Every table, index and view statement, each terminated by `;`. +#[must_use] +pub fn ddl_script(dialect: Dialect, options: &DdlOptions) -> String { + let mut statements: Vec = table_statements(dialect, options); + statements.extend( + view_statements(dialect, options) + .into_iter() + .map(|(_, statement)| statement), + ); + let mut script: String = statements.join(";\n\n"); + script.push_str(";\n"); + script +} diff --git a/crates/sqlbuild-recall/src/schema/main/mod.rs b/crates/sqlbuild-recall/src/schema/main/mod.rs new file mode 100644 index 0000000000..288b8f6614 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/mod.rs @@ -0,0 +1,7 @@ +pub mod add_column_statement; +pub mod ddl_script; +pub mod sqlite_retention_sql; +pub mod table_columns; +pub mod table_spec; +pub mod table_statements; +pub mod view_statements; diff --git a/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs b/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs new file mode 100644 index 0000000000..1a44c7b2e4 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/sqlite_retention_sql.rs @@ -0,0 +1,59 @@ +//! `sqlite_retention_sql`: the deletes SQLite maintenance runs, all rows of an invocation together. + +use crate::schema::_helpers::rendering::sqlite_identifier; +use crate::schema::constants::{ + INVOCATION_TABLES, META_MAINTENANCE_COMPONENT, RETENTION_CANDIDATE_TABLES, +}; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::SqliteRetentionSql; + +fn expired_invocations() -> String { + let candidates: Vec = RETENTION_CANDIDATE_TABLES + .iter() + .map(|table| { + format!( + "SELECT invocation_id FROM {} WHERE invocation_started_at < ?1", + sqlite_identifier(table_spec(*table).name) + ) + }) + .collect(); + format!( + "SELECT DISTINCT invocation_id FROM (\n {}\n)\n\ + WHERE invocation_id NOT IN (\n\ + \x20 SELECT beat.invocation_id FROM recall_heartbeats AS beat\n\ + \x20 WHERE (beat.stale_at >= ?2 OR beat.beat_at >= ?1) AND NOT EXISTS (\n\ + \x20 SELECT 1 FROM recall_events AS terminal\n\ + \x20 WHERE terminal.invocation_id = beat.invocation_id\n\ + \x20 AND terminal.event_type IN ('invocation_completed', 'invocation_failed')\n\ + \x20 )\n\ + )\nLIMIT ?3", + candidates.join("\n UNION ALL ") + ) +} + +/// The maintenance statements; running invocations, and unfinished ones that beat within retention, such as a suspended run, are never deleted. +#[must_use] +pub fn sqlite_retention_sql() -> SqliteRetentionSql { + SqliteRetentionSql { + expired_invocations: expired_invocations(), + delete_invocations: INVOCATION_TABLES + .iter() + .map(|table| { + ( + *table, + format!( + "DELETE FROM {} WHERE invocation_id IN (SELECT value FROM json_each(?1))", + sqlite_identifier(table_spec(*table).name) + ), + ) + }) + .collect(), + delete_orphan_artifacts: "DELETE FROM recall_artifacts WHERE seen_at < ?1 AND NOT EXISTS (\ + SELECT 1 FROM recall_artifact_refs AS ref WHERE ref.digest = recall_artifacts.digest)" + .to_owned(), + delete_old_claims: format!( + "DELETE FROM recall_meta WHERE component = '{META_MAINTENANCE_COMPONENT}' \ + AND recorded_at < ?1" + ), + } +} diff --git a/crates/sqlbuild-recall/src/schema/main/table_columns.rs b/crates/sqlbuild-recall/src/schema/main/table_columns.rs new file mode 100644 index 0000000000..87d842c064 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/table_columns.rs @@ -0,0 +1,11 @@ +//! `table_columns`: every column of one table, its own then the common columns. + +use crate::schema::_helpers::rendering::all_columns; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::{ColumnSpec, Table}; + +/// The table's columns in DDL order. +#[must_use] +pub fn table_columns(table: Table) -> Vec { + all_columns(table_spec(table)) +} diff --git a/crates/sqlbuild-recall/src/schema/main/table_spec.rs b/crates/sqlbuild-recall/src/schema/main/table_spec.rs new file mode 100644 index 0000000000..5fed770f82 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/table_spec.rs @@ -0,0 +1,24 @@ +//! `table_spec`: the definition of one Recall table. + +use crate::schema::constants::{ + ARTIFACT_REFS_TABLE, ARTIFACTS_TABLE, CONFLICTS_TABLE, DELIVERY_TABLE, DIAGNOSTICS_TABLE, + EVENTS_TABLE, HEARTBEATS_TABLE, INVOCATIONS_TABLE, META_TABLE, OUTPUT_TABLE, +}; +use crate::schema::models::{Table, TableSpec}; + +/// The physical definition of `table`. +#[must_use] +pub fn table_spec(table: Table) -> &'static TableSpec { + match table { + Table::Invocations => &INVOCATIONS_TABLE, + Table::Output => &OUTPUT_TABLE, + Table::Events => &EVENTS_TABLE, + Table::Diagnostics => &DIAGNOSTICS_TABLE, + Table::Artifacts => &ARTIFACTS_TABLE, + Table::ArtifactRefs => &ARTIFACT_REFS_TABLE, + Table::Heartbeats => &HEARTBEATS_TABLE, + Table::Delivery => &DELIVERY_TABLE, + Table::Meta => &META_TABLE, + Table::Conflicts => &CONFLICTS_TABLE, + } +} diff --git a/crates/sqlbuild-recall/src/schema/main/table_statements.rs b/crates/sqlbuild-recall/src/schema/main/table_statements.rs new file mode 100644 index 0000000000..621f8e1f12 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/table_statements.rs @@ -0,0 +1,84 @@ +//! `table_statements`: idempotent `CREATE TABLE` and index statements of every Recall table. + +use crate::schema::_helpers::rendering::{ + all_columns, clickhouse_engine, clickhouse_identifier, clickhouse_table, clickhouse_ttl, + clickhouse_type, on_cluster, sqlite_identifier, sqlite_type, +}; +use crate::schema::constants::ALL_TABLES; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::{DdlOptions, Dialect, TableSpec}; + +fn sqlite_table(spec: &TableSpec) -> Vec { + let mut definitions: Vec = all_columns(spec) + .iter() + .map(|column| { + let not_null: &str = if column.nullable { "" } else { " NOT NULL" }; + format!( + " {} {}{not_null}", + sqlite_identifier(column.name), + sqlite_type(column.column_type) + ) + }) + .collect(); + let key: Vec = spec + .key + .iter() + .map(|name| sqlite_identifier(name)) + .collect(); + definitions.push(format!(" UNIQUE ({})", key.join(", "))); + let mut statements: Vec = vec![format!( + "CREATE TABLE IF NOT EXISTS {} (\n{}\n)", + sqlite_identifier(spec.name), + definitions.join(",\n") + )]; + for (suffix, columns) in spec.sqlite_indexes { + let columns: Vec = columns.iter().map(|name| sqlite_identifier(name)).collect(); + statements.push(format!( + "CREATE INDEX IF NOT EXISTS {} ON {} ({})", + sqlite_identifier(&format!("{}_{suffix}", spec.name)), + sqlite_identifier(spec.name), + columns.join(", ") + )); + } + statements +} + +fn clickhouse_table_statement(spec: &TableSpec, options: &DdlOptions) -> String { + let columns: Vec = all_columns(spec) + .iter() + .map(|column| { + format!( + " {} {}", + clickhouse_identifier(column.name), + clickhouse_type(column) + ) + }) + .collect(); + format!( + "CREATE TABLE IF NOT EXISTS {name}{cluster} (\n{columns}\n)\nENGINE = {engine}\n\ + PARTITION BY toDate({expiry})\nORDER BY ({order})\nTTL {ttl}\n\ + SETTINGS ttl_only_drop_parts = 1, non_replicated_deduplication_window = 1000", + name = clickhouse_table(spec.name, options), + cluster = on_cluster(options), + columns = columns.join(",\n"), + engine = clickhouse_engine(spec, options), + expiry = spec.expiry_column, + order = spec.clickhouse_order_by.join(", "), + ttl = clickhouse_ttl(spec, options), + ) +} + +/// Every table's creation statements, in dependency order; safe to rerun. +#[must_use] +pub fn table_statements(dialect: Dialect, options: &DdlOptions) -> Vec { + ALL_TABLES + .iter() + .flat_map(|table| { + let spec: &TableSpec = table_spec(*table); + match dialect { + Dialect::Sqlite => sqlite_table(spec), + Dialect::ClickHouse => vec![clickhouse_table_statement(spec, options)], + } + }) + .collect() +} diff --git a/crates/sqlbuild-recall/src/schema/main/view_statements.rs b/crates/sqlbuild-recall/src/schema/main/view_statements.rs new file mode 100644 index 0000000000..ce34504a3c --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/main/view_statements.rs @@ -0,0 +1,38 @@ +//! `view_statements`: the statements that create the supported views. + +use crate::schema::_helpers::clickhouse_views::clickhouse_view_bodies; +use crate::schema::_helpers::rendering::{clickhouse_table, on_cluster, sqlite_identifier}; +use crate::schema::_helpers::sqlite_views::sqlite_view_bodies; +use crate::schema::models::{DdlOptions, Dialect}; + +/// Each view's name and creation statement. SQLite views are created only when missing; the store drops them first when the schema version rises. ClickHouse views are replaced. +#[must_use] +pub fn view_statements(dialect: Dialect, options: &DdlOptions) -> Vec<(&'static str, String)> { + match dialect { + Dialect::Sqlite => sqlite_view_bodies() + .into_iter() + .map(|(name, body)| { + ( + name, + format!( + "CREATE VIEW IF NOT EXISTS {} AS\n{body}", + sqlite_identifier(name) + ), + ) + }) + .collect(), + Dialect::ClickHouse => clickhouse_view_bodies(options) + .into_iter() + .map(|(name, body)| { + ( + name, + format!( + "CREATE OR REPLACE VIEW {}{} AS\n{body}", + clickhouse_table(name, options), + on_cluster(options) + ), + ) + }) + .collect(), + } +} diff --git a/crates/sqlbuild-recall/src/schema/mod.rs b/crates/sqlbuild-recall/src/schema/mod.rs new file mode 100644 index 0000000000..55009b62ab --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/mod.rs @@ -0,0 +1,9 @@ +//! Recall's physical schema: tables, supported views, DDL and retention SQL for each dialect. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-recall/src/schema/models.rs b/crates/sqlbuild-recall/src/schema/models.rs new file mode 100644 index 0000000000..526db74cf9 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/models.rs @@ -0,0 +1,111 @@ +//! The physical shape of Recall tables and the options their DDL is rendered with. + +/// The SQL dialect a Recall store speaks. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Dialect { + Sqlite, + ClickHouse, +} + +/// The logical type of one Recall column. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ColumnType { + Text, + Integer, + Real, + Blob, + /// ISO-8601 UTC text with microseconds in SQLite, `DateTime64(6, 'UTC')` in ClickHouse. + Timestamp, + /// JSON text. + Json, + /// 0 or 1. + Boolean, +} + +/// One column: its name, type and whether it may be null. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ColumnSpec { + pub name: &'static str, + pub column_type: ColumnType, + pub nullable: bool, +} + +impl ColumnSpec { + /// A column that is never null. + #[must_use] + pub const fn required(name: &'static str, column_type: ColumnType) -> Self { + Self { + name, + column_type, + nullable: false, + } + } + + /// A column that may be null. + #[must_use] + pub const fn optional(name: &'static str, column_type: ColumnType) -> Self { + Self { + name, + column_type, + nullable: true, + } + } +} + +/// One Recall table. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TableSpec { + pub name: &'static str, + /// Columns specific to this table; every table also has the common columns. + pub columns: &'static [ColumnSpec], + /// The deterministic identity of a row; a second row with this key is a duplicate. + pub key: &'static [&'static str], + /// SQLite secondary indexes: name suffix and columns. + pub sqlite_indexes: &'static [(&'static str, &'static [&'static str])], + /// ClickHouse engine without arguments: `ReplacingMergeTree` or `MergeTree`. + pub clickhouse_engine: &'static str, + /// ClickHouse `ORDER BY` columns. + pub clickhouse_order_by: &'static [&'static str], + /// Days ClickHouse keeps rows beyond retention; artifacts outlive their references. + pub ttl_extra_days: u32, + /// The `ReplacingMergeTree` version column: the newest copy survives merges. + pub clickhouse_version: Option<&'static str>, + /// The timestamp column partitions and TTL derive from. + pub expiry_column: &'static str, +} + +/// The tables Recall writes. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum Table { + Invocations, + Output, + Events, + Diagnostics, + Artifacts, + ArtifactRefs, + Heartbeats, + Delivery, + Meta, + Conflicts, +} + +/// What DDL is rendered with: database (ClickHouse), retention and cluster. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DdlOptions { + pub database: Option, + pub retention_days: u32, + pub cluster: Option, +} + +/// The statements SQLite maintenance runs, with their positional parameters. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SqliteRetentionSql { + /// `?1` cutoff, `?2` now, `?3` limit: expired invocation IDs that are not running. + pub expired_invocations: String, + /// Per table, `?1` a JSON array of invocation IDs to delete. + pub delete_invocations: Vec<(Table, String)>, + /// `?1` cutoff: artifacts no remaining reference names. + pub delete_orphan_artifacts: String, + /// `?1` cutoff: maintenance claims older than retention. + pub delete_old_claims: String, +} diff --git a/crates/sqlbuild-recall/src/schema/tests.rs b/crates/sqlbuild-recall/src/schema/tests.rs new file mode 100644 index 0000000000..e448cf632c --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_ddl.rs"] +mod ddl; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-recall/src/schema/tests/helpers.rs b/crates/sqlbuild-recall/src/schema/tests/helpers.rs new file mode 100644 index 0000000000..2c8fd299d6 --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/tests/helpers.rs @@ -0,0 +1,68 @@ +use crate::schema::constants::VIEW_NAMES; +use crate::schema::main::ddl_script::ddl_script; +use crate::schema::main::sqlite_retention_sql::sqlite_retention_sql; +use crate::schema::main::table_statements::table_statements; +use crate::schema::models::{DdlOptions, Dialect}; + +/// Options rendering into `observability` with 14-day retention, optionally on `cluster`. +pub(crate) fn options(cluster: Option<&str>) -> DdlOptions { + DdlOptions { + database: Some("observability".to_owned()), + retention_days: 14, + cluster: cluster.map(str::to_owned), + } +} + +/// The ClickHouse `CREATE TABLE` statement for `table`. +pub(crate) fn clickhouse_table(table: &str, cluster: Option<&str>) -> String { + let quoted = format!("`{table}`"); + table_statements(Dialect::ClickHouse, &options(cluster)) + .into_iter() + .find(|statement| statement.contains("ed)) + .unwrap_or_default() +} + +/// The `fragments` that `statement` contains, in order. +pub(crate) fn contained<'a>(statement: &str, fragments: &[&'a str]) -> Vec<&'a str> { + fragments + .iter() + .copied() + .filter(|fragment| statement.contains(fragment)) + .collect() +} + +/// Schema objects after running the SQLite script `runs` times, and the statements that fail. +pub(crate) fn sqlite_schema_after(runs: usize) -> (usize, Vec) { + let connection = rusqlite::Connection::open_in_memory().expect("an in-memory database opens"); + let script = ddl_script(Dialect::Sqlite, &options(None)); + let mut failures: Vec = Vec::new(); + for run in 0..runs { + let outcome: rusqlite::Result<()> = connection.execute_batch(&script); + failures.extend(outcome.err().map(|error| format!("run {run}: {error}"))); + } + let retention = sqlite_retention_sql(); + let mut statements: Vec = VIEW_NAMES + .iter() + .map(|view| format!("SELECT * FROM \"{view}\"")) + .collect(); + statements.push(retention.expired_invocations.clone()); + statements.extend( + retention + .delete_invocations + .iter() + .map(|(_, delete)| delete.clone()), + ); + statements.push(retention.delete_orphan_artifacts.clone()); + for statement in statements { + let prepared: rusqlite::Result> = connection.prepare(&statement); + failures.extend(prepared.err().map(|error| format!("{statement}: {error}"))); + } + let count: i64 = connection + .query_row( + "SELECT COUNT(*) FROM sqlite_schema WHERE type IN ('table', 'view')", + [], + |row| row.get(0), + ) + .expect("the count query runs"); + (usize::try_from(count).unwrap_or_default(), failures) +} diff --git a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs new file mode 100644 index 0000000000..b59d10139a --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs @@ -0,0 +1,169 @@ +use crate::schema::constants::{ALL_TABLES, VIEW_NAMES}; +use crate::schema::main::add_column_statement::add_column_statement; +use crate::schema::main::table_spec::table_spec; +use crate::schema::main::table_statements::table_statements; +use crate::schema::main::view_statements::view_statements; +use crate::schema::models::{ColumnSpec, ColumnType, Dialect, Table}; +use crate::schema::tests::helpers::{clickhouse_table, contained, options, sqlite_schema_after}; +use crate::schema::tests::test_types::{ + AddColumnTestCase, FragmentTestCase, SqliteScriptTestCase, StatementSetTestCase, +}; + +const NEW_COLUMN: ColumnSpec = ColumnSpec { + name: "new_field", + column_type: ColumnType::Integer, + nullable: false, +}; + +#[test] +fn given_sqlite_script_when_run_then_every_table_view_and_retention_statement_prepares() { + let test_cases = [ + SqliteScriptTestCase { + description: "a fresh database gets every object", + runs: 1, + expected_objects: ALL_TABLES.len() + VIEW_NAMES.len(), + expected_failures: Vec::new(), + }, + SqliteScriptTestCase { + description: "rerunning the script is idempotent", + runs: 2, + expected_objects: ALL_TABLES.len() + VIEW_NAMES.len(), + expected_failures: Vec::new(), + }, + ]; + + for test_case in test_cases { + assert_eq!( + sqlite_schema_after(test_case.runs), + (test_case.expected_objects, test_case.expected_failures), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_clickhouse_tables_when_rendered_then_engines_ttl_and_partitions_follow_the_table() { + let test_cases = [ + FragmentTestCase { + description: "output is a replacing table partitioned and expired by start time", + statement: clickhouse_table("recall_output", None), + expected_fragments: &[ + "CREATE TABLE IF NOT EXISTS `observability`.`recall_output` (", + "`data` String", + "`priority` Nullable(String)", + "ENGINE = ReplacingMergeTree()", + "PARTITION BY toDate(invocation_started_at)", + "ORDER BY (invocation_id, sequence)", + "TTL invocation_started_at + INTERVAL 14 DAY", + ], + }, + FragmentTestCase { + description: "artifacts keep the latest sighting for an extra week", + statement: clickhouse_table("recall_artifacts", None), + expected_fragments: &[ + "ENGINE = ReplacingMergeTree(seen_at)", + "TTL seen_at + INTERVAL 21 DAY", + ], + }, + FragmentTestCase { + description: "heartbeats are append-only", + statement: clickhouse_table("recall_heartbeats", None), + expected_fragments: &["ENGINE = MergeTree()"], + }, + FragmentTestCase { + description: "clustered replacing tables are replicated", + statement: clickhouse_table("recall_output", Some("analytics")), + expected_fragments: &[ + " ON CLUSTER `analytics` (", + "ReplicatedReplacingMergeTree('/clickhouse/tables/{uuid}/{shard}', '{replica}')", + ], + }, + ]; + + for test_case in test_cases { + assert_eq!( + contained(&test_case.statement, test_case.expected_fragments), + test_case.expected_fragments.to_vec(), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_clickhouse_statement_sets_when_rendered_then_every_statement_has_the_shared_shape() { + let views = |cluster: Option<&str>| -> Vec { + view_statements(Dialect::ClickHouse, &options(cluster)) + .into_iter() + .map(|(_, statement)| statement) + .collect() + }; + let test_cases = [ + StatementSetTestCase { + description: "every clustered table is created on the cluster", + statements: table_statements(Dialect::ClickHouse, &options(Some("analytics"))), + required_fragments: &[" ON CLUSTER `analytics` ("], + expected_count: ALL_TABLES.len(), + }, + StatementSetTestCase { + description: "clustered views are replaced on the cluster", + statements: views(Some("analytics")), + required_fragments: &["CREATE OR REPLACE VIEW", "ON CLUSTER `analytics`"], + expected_count: VIEW_NAMES.len(), + }, + StatementSetTestCase { + description: "views deduplicate rows and filter by retention", + statements: views(None), + required_fragments: &["now64(6) - INTERVAL 14 DAY", "LIMIT 1 BY"], + expected_count: VIEW_NAMES.len(), + }, + ]; + + for test_case in test_cases { + let shaped: usize = test_case + .statements + .iter() + .filter(|statement| { + contained(statement, test_case.required_fragments).len() + == test_case.required_fragments.len() + }) + .count(); + assert_eq!( + (test_case.statements.len(), shaped), + (test_case.expected_count, test_case.expected_count), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_new_column_when_upgraded_then_the_statement_is_additive_and_nullable() { + let spec = table_spec(Table::Events); + let test_cases = [ + AddColumnTestCase { + description: "SQLite columns are nullable by default", + statement: add_column_statement(Dialect::Sqlite, &options(None), (spec, &NEW_COLUMN)), + expected_statement: "ALTER TABLE \"recall_events\" ADD COLUMN \"new_field\" INTEGER", + }, + AddColumnTestCase { + description: "ClickHouse columns are added idempotently as Nullable", + statement: add_column_statement( + Dialect::ClickHouse, + &options(None), + (spec, &NEW_COLUMN), + ), + expected_statement: "ALTER TABLE `observability`.`recall_events` ADD COLUMN IF NOT \ + EXISTS `new_field` Nullable(Int64)", + }, + ]; + + for test_case in test_cases { + assert_eq!( + test_case.statement, test_case.expected_statement, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/schema/tests/test_types.rs b/crates/sqlbuild-recall/src/schema/tests/test_types.rs new file mode 100644 index 0000000000..e064d5472a --- /dev/null +++ b/crates/sqlbuild-recall/src/schema/tests/test_types.rs @@ -0,0 +1,25 @@ +pub(crate) struct SqliteScriptTestCase { + pub(crate) description: &'static str, + pub(crate) runs: usize, + pub(crate) expected_objects: usize, + pub(crate) expected_failures: Vec, +} + +pub(crate) struct FragmentTestCase { + pub(crate) description: &'static str, + pub(crate) statement: String, + pub(crate) expected_fragments: &'static [&'static str], +} + +pub(crate) struct StatementSetTestCase { + pub(crate) description: &'static str, + pub(crate) statements: Vec, + pub(crate) required_fragments: &'static [&'static str], + pub(crate) expected_count: usize, +} + +pub(crate) struct AddColumnTestCase { + pub(crate) description: &'static str, + pub(crate) statement: String, + pub(crate) expected_statement: &'static str, +} diff --git a/crates/sqlbuild-recall/src/settings/_helpers/documents.rs b/crates/sqlbuild-recall/src/settings/_helpers/documents.rs new file mode 100644 index 0000000000..1fb6e4e52d --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/_helpers/documents.rs @@ -0,0 +1,73 @@ +//! Reading the project and local configuration documents for Recall. + +use std::path::Path; + +use sqlbuild_config::models::ConfigValue; +use sqlbuild_config::toml::main::load_toml::load_toml; + +use crate::settings::errors::SettingsError; + +/// The root table of a TOML file, `None` when it does not exist. +pub(crate) fn read_document(path: &Path) -> Result, SettingsError> { + let text = match std::fs::read_to_string(path) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(SettingsError::new(format!( + "cannot read {}: {error}", + path.display() + ))); + } + }; + load_toml(&text).map(Some).map_err(|error| { + SettingsError::new(format!( + "{} is not valid TOML: {}", + path.display(), + error.message + )) + }) +} + +/// The value at a dotted path of tables. +pub(crate) fn lookup<'a>( + document: Option<&'a ConfigValue>, + path: &[&str], +) -> Option<&'a ConfigValue> { + let mut value: &ConfigValue = document?; + for key in path { + value = value.get(key)?; + } + Some(value) +} + +pub(crate) fn string_at(document: Option<&ConfigValue>, path: &[&str]) -> Option { + match lookup(document, path)? { + ConfigValue::String(text) if !text.trim().is_empty() => Some(text.trim().to_owned()), + _ => None, + } +} + +/// Replace `${env:NAME}` with the environment variable, failing when it is unset. +pub(crate) fn expand_environment(text: &str, label: &str) -> Result { + let mut expanded = String::with_capacity(text.len()); + let mut rest = text; + while let Some(start) = rest.find("${env:") { + expanded.push_str(&rest[..start]); + let after = &rest[start + "${env:".len()..]; + let Some(end) = after.find('}') else { + return Err(SettingsError::new(format!( + "{label} has an unterminated ${{env:...}} reference" + ))); + }; + let name = &after[..end]; + let value = std::env::var(name).map_err(|_| { + SettingsError::new(format!( + "{label} references unset environment variable {name}" + )) + })?; + expanded.push_str(&value); + rest = &after[end + 1..]; + } + expanded.push_str(rest); + Ok(expanded) +} diff --git a/crates/sqlbuild-recall/src/settings/_helpers/layers.rs b/crates/sqlbuild-recall/src/settings/_helpers/layers.rs new file mode 100644 index 0000000000..5ea24d617e --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/_helpers/layers.rs @@ -0,0 +1,118 @@ +//! Layering Recall settings tables and locating the store connection. + +use std::path::{Path, PathBuf}; + +use sqlbuild_config::models::ConfigValue; +use sqlbuild_config::recall::main::recall_section::recall_section; +use sqlbuild_config::recall::models::RecallSection; + +use crate::settings::_helpers::documents::{expand_environment, lookup}; +use crate::settings::constants::{ + CONNECTION_ADAPTER_KEY, DEFAULT_STORE_DIRECTORY, DEFAULT_STORE_FILE_NAME, SQLITE_ADAPTER, + SQLITE_PATH_KEY, +}; +use crate::settings::errors::SettingsError; +use crate::settings::models::StoreLocation; + +pub(crate) fn section( + document: Option<&ConfigValue>, + path: &[&str], + label: &str, +) -> Result { + match lookup(document, path) { + None => Ok(RecallSection::default()), + Some(value) => recall_section(value, label).map_err(SettingsError::new), + } +} + +pub(crate) fn overlay(base: RecallSection, layer: RecallSection) -> RecallSection { + RecallSection { + enabled: layer.enabled.or(base.enabled), + connection: layer.connection.or(base.connection), + retention_days: layer.retention_days.or(base.retention_days), + shutdown_timeout_ms: layer.shutdown_timeout_ms.or(base.shutdown_timeout_ms), + diagnostics_min_severity: layer + .diagnostics_min_severity + .or(base.diagnostics_min_severity), + max_output_bytes: layer.max_output_bytes.or(base.max_output_bytes), + } +} + +/// The named connection's entries, local keys overriding project keys. +fn connection_entries( + documents: [Option<&ConfigValue>; 2], + name: &str, +) -> Option> { + let mut merged: Option> = None; + for document in documents { + let Some(entries) = lookup(document, &["connections", name]).and_then(ConfigValue::as_map) + else { + continue; + }; + let target = merged.get_or_insert_with(Vec::new); + for (key, value) in entries { + let ConfigValue::String(key) = key else { + continue; + }; + target.retain(|(existing, _)| existing != key); + target.push((key.clone(), value.clone())); + } + } + merged +} + +pub(crate) fn store_location( + project_dir: &Path, + documents: [Option<&ConfigValue>; 2], + (connection, project_adapter): (Option<&str>, Option<&str>), +) -> Result { + let Some(name) = connection else { + return Ok(StoreLocation::Sqlite { + path: project_dir + .join(DEFAULT_STORE_DIRECTORY) + .join(DEFAULT_STORE_FILE_NAME), + }); + }; + let entries = connection_entries(documents, name).ok_or_else(|| { + SettingsError::new(format!( + "recall.connection names unknown connection '{name}'; define [connections.{name}]" + )) + })?; + let value = |key: &str| { + entries + .iter() + .find(|(candidate, _)| candidate == key) + .map(|(_, value)| value) + }; + let adapter: String = match value(CONNECTION_ADAPTER_KEY) { + Some(ConfigValue::String(adapter)) => adapter.trim().to_owned(), + Some(_) => { + return Err(SettingsError::new(format!( + "connections.{name}.adapter must be a string" + ))); + } + None => project_adapter.unwrap_or_default().to_owned(), + }; + if adapter != SQLITE_ADAPTER { + return Ok(StoreLocation::Unsupported { + connection: name.to_owned(), + adapter, + }); + } + let Some(ConfigValue::String(path)) = value(SQLITE_PATH_KEY) else { + return Err(SettingsError::new(format!( + "connections.{name}.path must name the SQLite file Recall writes" + ))); + }; + let path: PathBuf = PathBuf::from(expand_environment( + path.trim(), + &format!("connections.{name}.path"), + )?); + Ok(StoreLocation::Sqlite { + path: if path.is_absolute() { + path + } else { + project_dir.join(path) + }, + }) +} diff --git a/crates/sqlbuild-recall/src/settings/_helpers/mod.rs b/crates/sqlbuild-recall/src/settings/_helpers/mod.rs new file mode 100644 index 0000000000..3e9da610d7 --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod documents; +pub(crate) mod layers; diff --git a/crates/sqlbuild-recall/src/settings/constants.rs b/crates/sqlbuild-recall/src/settings/constants.rs new file mode 100644 index 0000000000..76d7d584e6 --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/constants.rs @@ -0,0 +1,16 @@ +//! Built-in Recall defaults. + +/// The local store, relative to the project directory. +pub const DEFAULT_STORE_DIRECTORY: &str = "logs"; +pub const DEFAULT_STORE_FILE_NAME: &str = "recall.sqlite3"; +pub const DEFAULT_RETENTION_DAYS: u32 = 30; +pub const DEFAULT_SHUTDOWN_TIMEOUT_MS: u64 = 3_000; +pub const DEFAULT_DIAGNOSTICS_MIN_SEVERITY: &str = "info"; +/// The connection key naming a connection's adapter; never passed to `connect()`. +pub const CONNECTION_ADAPTER_KEY: &str = "adapter"; +pub const SQLITE_ADAPTER: &str = "sqlite"; +pub const SQLITE_PATH_KEY: &str = "path"; +pub const PROJECT_FILE_NAME: &str = "sqlbuild_project.toml"; +pub const LOCAL_FILE_NAME: &str = "sqlbuild_local.toml"; +/// Bytes of each output stream the local SQLite store keeps per invocation: the first and last MiB. +pub const DEFAULT_SQLITE_MAX_OUTPUT_BYTES: u64 = 2 * 1024 * 1024; diff --git a/crates/sqlbuild-recall/src/settings/errors.rs b/crates/sqlbuild-recall/src/settings/errors.rs new file mode 100644 index 0000000000..fe4baa26dd --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/errors.rs @@ -0,0 +1,23 @@ +//! Why Recall settings cannot be resolved. + +/// A settings problem, with a message naming the setting. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SettingsError { + pub message: String, +} + +impl SettingsError { + pub(crate) fn new(message: impl Into) -> Self { + Self { + message: message.into(), + } + } +} + +impl std::fmt::Display for SettingsError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.message) + } +} + +impl std::error::Error for SettingsError {} diff --git a/crates/sqlbuild-recall/src/settings/main/mod.rs b/crates/sqlbuild-recall/src/settings/main/mod.rs new file mode 100644 index 0000000000..189f053d5e --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/main/mod.rs @@ -0,0 +1 @@ +pub mod resolve_recall_settings; diff --git a/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs b/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs new file mode 100644 index 0000000000..a85925af60 --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs @@ -0,0 +1,74 @@ +//! `resolve_recall_settings`: the effective Recall settings of one invocation. + +use sqlbuild_config::models::ConfigValue; +use sqlbuild_config::recall::constants::RECALL_SECTION; + +use crate::settings::_helpers::documents::{read_document, string_at}; +use crate::settings::_helpers::layers::{overlay, section, store_location}; +use crate::settings::constants::{ + DEFAULT_DIAGNOSTICS_MIN_SEVERITY, DEFAULT_RETENTION_DAYS, DEFAULT_SHUTDOWN_TIMEOUT_MS, + LOCAL_FILE_NAME, PROJECT_FILE_NAME, +}; +use crate::settings::errors::SettingsError; +use crate::settings::models::{RecallSettings, SettingsRequest}; + +/// The effective settings: `targets..recall` over `[recall]` over the built-in local store, local configuration over project configuration at each level. +pub fn resolve_recall_settings(request: &SettingsRequest) -> Result { + let project = read_document(&request.project_dir.join(PROJECT_FILE_NAME))?; + let local = read_document(&request.project_dir.join(LOCAL_FILE_NAME))?; + let documents: [Option<&ConfigValue>; 2] = [project.as_ref(), local.as_ref()]; + let target: Option = request + .selected_target + .clone() + .or_else(|| string_at(local.as_ref(), &["target"])) + .or_else(|| string_at(project.as_ref(), &["default_target"])); + let mut effective = section(project.as_ref(), &[RECALL_SECTION], RECALL_SECTION)?; + effective = overlay( + effective, + section(local.as_ref(), &[RECALL_SECTION], RECALL_SECTION)?, + ); + if let Some(target) = target.as_deref() { + let label = format!("targets.{target}.recall"); + for document in documents { + effective = overlay( + effective, + section(document, &["targets", target, RECALL_SECTION], &label)?, + ); + } + } + let project_adapter: Option = string_at(local.as_ref(), &["adapter"]) + .or_else(|| string_at(project.as_ref(), &["adapter"])); + let target_field = |key: &str| -> Option { + let target = target.as_deref()?; + string_at(local.as_ref(), &["targets", target, key]) + .or_else(|| string_at(project.as_ref(), &["targets", target, key])) + }; + let store = store_location( + &request.project_dir, + documents, + (effective.connection.as_deref(), project_adapter.as_deref()), + )?; + let max_output_bytes: Option = match effective.max_output_bytes { + Some(0) => None, + Some(bytes) => Some(bytes), + None => store.default_max_output_bytes(), + }; + Ok(RecallSettings { + enabled: effective.enabled.unwrap_or(true), + target: target.clone(), + store, + connection: effective.connection, + retention_days: effective.retention_days.unwrap_or(DEFAULT_RETENTION_DAYS), + shutdown_timeout_ms: effective + .shutdown_timeout_ms + .unwrap_or(DEFAULT_SHUTDOWN_TIMEOUT_MS), + diagnostics_min_severity: effective + .diagnostics_min_severity + .unwrap_or_else(|| DEFAULT_DIAGNOSTICS_MIN_SEVERITY.to_owned()), + max_output_bytes, + project_name: string_at(project.as_ref(), &["name"]), + project_adapter, + destination_database: target_field("database"), + destination_schema: target_field("schema"), + }) +} diff --git a/crates/sqlbuild-recall/src/settings/mod.rs b/crates/sqlbuild-recall/src/settings/mod.rs new file mode 100644 index 0000000000..69fa1e7523 --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/mod.rs @@ -0,0 +1,10 @@ +//! Effective Recall settings: `[recall]`, `targets..recall` and the store connection. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-recall/src/settings/models.rs b/crates/sqlbuild-recall/src/settings/models.rs new file mode 100644 index 0000000000..cf9ffa698c --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/models.rs @@ -0,0 +1,52 @@ +//! Resolved Recall settings for one invocation. + +use std::path::PathBuf; + +use crate::settings::constants::DEFAULT_SQLITE_MAX_OUTPUT_BYTES; + +/// Where Recall writes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StoreLocation { + /// A SQLite file, written natively. + Sqlite { path: PathBuf }, + /// A connection whose adapter this build cannot write Recall rows to yet. + Unsupported { connection: String, adapter: String }, +} + +impl StoreLocation { + /// The per-stream output limit when none is configured: bounded for the local file, which every command writes; `None` records everything. + #[must_use] + pub fn default_max_output_bytes(&self) -> Option { + match self { + Self::Sqlite { .. } => Some(DEFAULT_SQLITE_MAX_OUTPUT_BYTES), + Self::Unsupported { .. } => None, + } + } +} + +/// The effective settings: target, store and policies. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RecallSettings { + pub enabled: bool, + /// The effective target; `None` for commands without one. + pub target: Option, + pub store: StoreLocation, + /// The configured connection name, `None` for the built-in local store. + pub connection: Option, + pub retention_days: u32, + pub shutdown_timeout_ms: u64, + pub diagnostics_min_severity: String, + /// Bytes of each output stream recorded, half from its start and half from its end; `None` records everything. + pub max_output_bytes: Option, + pub project_name: Option, + pub project_adapter: Option, + pub destination_database: Option, + pub destination_schema: Option, +} + +/// What selects the settings: the project directory and the `--target` option. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SettingsRequest { + pub project_dir: PathBuf, + pub selected_target: Option, +} diff --git a/crates/sqlbuild-recall/src/settings/tests.rs b/crates/sqlbuild-recall/src/settings/tests.rs new file mode 100644 index 0000000000..5cf4652f06 --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_resolve.rs"] +mod resolve; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-recall/src/settings/tests/helpers.rs b/crates/sqlbuild-recall/src/settings/tests/helpers.rs new file mode 100644 index 0000000000..39ef48f7e4 --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/tests/helpers.rs @@ -0,0 +1,43 @@ +use std::path::PathBuf; + +use crate::settings::main::resolve_recall_settings::resolve_recall_settings; +use crate::settings::models::{RecallSettings, SettingsRequest, StoreLocation}; + +/// The placeholder project directory resolved paths are reported under. +pub(crate) const PROJECT: &str = ""; + +/// Settings resolved for a project holding `files`, debug-printed with its directory as `PROJECT`. +pub(crate) fn resolved(files: &[(&str, &str)], target: Option<&str>) -> Result { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + for (name, text) in files { + std::fs::write(directory.path().join(name), text).expect("test files can be written"); + } + let request = SettingsRequest { + project_dir: directory.path().to_path_buf(), + selected_target: target.map(str::to_owned), + }; + let project: String = directory.path().to_string_lossy().into_owned(); + resolve_recall_settings(&request) + .map(|settings| format!("{settings:?}").replace(&project, PROJECT)) + .map_err(|error| error.message) +} + +/// The default settings for project `shop` on the duckdb adapter. +pub(crate) fn defaults() -> RecallSettings { + RecallSettings { + enabled: true, + target: None, + store: StoreLocation::Sqlite { + path: PathBuf::from(PROJECT).join("logs").join("recall.sqlite3"), + }, + connection: None, + retention_days: 30, + shutdown_timeout_ms: 3000, + diagnostics_min_severity: "info".to_owned(), + max_output_bytes: Some(2 * 1024 * 1024), + project_name: Some("shop".to_owned()), + project_adapter: Some("duckdb".to_owned()), + destination_database: None, + destination_schema: None, + } +} diff --git a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs new file mode 100644 index 0000000000..cd7a225b7b --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs @@ -0,0 +1,142 @@ +use crate::settings::models::{RecallSettings, StoreLocation}; +use crate::settings::tests::helpers::{defaults, resolved}; +use crate::settings::tests::test_types::{ResolveErrorTestCase, ResolveTestCase}; + +const DEFAULT_PROJECT: &[(&str, &str)] = &[( + "sqlbuild_project.toml", + "name = \"shop\"\nadapter = \"duckdb\"\ndefault_target = \"dev\"\n[targets.dev]\nschema = \"analytics\"\n", +)]; + +const LAYERED_PROJECT: &[(&str, &str)] = &[ + ( + "sqlbuild_project.toml", + "name = \"shop\"\nadapter = \"duckdb\"\n\ + [recall]\nretention = \"10d\"\nconnection = \"history\"\n\ + [connections.history]\nadapter = \"sqlite\"\npath = \"state/history.sqlite3\"\n\ + [connections.central]\nadapter = \"clickhouse\"\nhost = \"localhost\"\n\ + [targets.dev]\n[targets.dev.recall]\nenabled = false\n\ + [targets.prod]\n[targets.prod.recall]\nconnection = \"central\"\n", + ), + ( + "sqlbuild_local.toml", + "[recall]\nshutdown_timeout = \"1s\"\n[connections.history]\npath = \"/tmp/opencode/elsewhere.sqlite3\"\n", + ), +]; + +const OUTPUT_LIMIT_PROJECT: &[(&str, &str)] = &[( + "sqlbuild_project.toml", + "name = \"shop\"\nadapter = \"duckdb\"\n[recall]\nmax_output_bytes = 0\n\ + [targets.dev]\n[targets.dev.recall]\nmax_output_bytes = 8388608\n[targets.prod]\n", +)]; + +#[test] +fn given_project_layers_when_resolved_then_target_over_local_over_project_over_defaults() { + let test_cases = [ + ResolveTestCase { + description: "without a recall section local SQLite is on by default", + files: DEFAULT_PROJECT, + target: None, + expected_settings: RecallSettings { + target: Some("dev".to_owned()), + destination_schema: Some("analytics".to_owned()), + ..defaults() + }, + }, + ResolveTestCase { + description: "the target disables recall and the local file moves the store", + files: LAYERED_PROJECT, + target: Some("dev"), + expected_settings: RecallSettings { + enabled: false, + target: Some("dev".to_owned()), + store: StoreLocation::Sqlite { + path: "/tmp/opencode/elsewhere.sqlite3".into(), + }, + connection: Some("history".to_owned()), + retention_days: 10, + shutdown_timeout_ms: 1000, + ..defaults() + }, + }, + ResolveTestCase { + description: "a non-SQLite connection is reported as unsupported", + files: LAYERED_PROJECT, + target: Some("prod"), + expected_settings: RecallSettings { + target: Some("prod".to_owned()), + store: StoreLocation::Unsupported { + connection: "central".to_owned(), + adapter: "clickhouse".to_owned(), + }, + connection: Some("central".to_owned()), + retention_days: 10, + shutdown_timeout_ms: 1000, + max_output_bytes: None, + ..defaults() + }, + }, + ResolveTestCase { + description: "a target raises the output limit over the project's unlimited one", + files: OUTPUT_LIMIT_PROJECT, + target: Some("dev"), + expected_settings: RecallSettings { + target: Some("dev".to_owned()), + max_output_bytes: Some(8_388_608), + ..defaults() + }, + }, + ResolveTestCase { + description: "zero records all output", + files: OUTPUT_LIMIT_PROJECT, + target: Some("prod"), + expected_settings: RecallSettings { + target: Some("prod".to_owned()), + max_output_bytes: None, + ..defaults() + }, + }, + ]; + + for test_case in test_cases { + assert_eq!( + resolved(test_case.files, test_case.target), + Ok(format!("{:?}", test_case.expected_settings)), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_invalid_settings_when_resolved_then_the_error_names_the_setting() { + let test_cases = [ + ResolveErrorTestCase { + description: "unknown connections are named", + files: &[( + "sqlbuild_project.toml", + "name = \"x\"\nadapter = \"duckdb\"\n[recall]\nconnection = \"missing\"\n", + )], + target: None, + expected_message_fragment: "unknown connection 'missing'", + }, + ResolveErrorTestCase { + description: "target settings are named with their path", + files: &[( + "sqlbuild_project.toml", + "name = \"x\"\nadapter = \"duckdb\"\n[targets.dev.recall]\nretention = \"soon\"\n", + )], + target: Some("dev"), + expected_message_fragment: "targets.dev.recall.retention", + }, + ]; + + for test_case in test_cases { + let message: String = + resolved(test_case.files, test_case.target).expect_err("the settings are invalid"); + assert!( + message.contains(test_case.expected_message_fragment), + "{}: {message}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/settings/tests/test_types.rs b/crates/sqlbuild-recall/src/settings/tests/test_types.rs new file mode 100644 index 0000000000..d038a10486 --- /dev/null +++ b/crates/sqlbuild-recall/src/settings/tests/test_types.rs @@ -0,0 +1,15 @@ +use crate::settings::models::RecallSettings; + +pub(crate) struct ResolveTestCase { + pub(crate) description: &'static str, + pub(crate) files: &'static [(&'static str, &'static str)], + pub(crate) target: Option<&'static str>, + pub(crate) expected_settings: RecallSettings, +} + +pub(crate) struct ResolveErrorTestCase { + pub(crate) description: &'static str, + pub(crate) files: &'static [(&'static str, &'static str)], + pub(crate) target: Option<&'static str>, + pub(crate) expected_message_fragment: &'static str, +} diff --git a/crates/sqlbuild-recall/src/store/_helpers/identity.rs b/crates/sqlbuild-recall/src/store/_helpers/identity.rs new file mode 100644 index 0000000000..d2ed39c27b --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/identity.rs @@ -0,0 +1,39 @@ +//! File identity and SQLite header checks for moving a corrupt store aside. + +use std::fs::{File, Metadata}; +use std::io::Read; + +use crate::store::constants::SQLITE_HEADER; +use crate::store::models::StoreFileIdentity; + +#[cfg(unix)] +pub(crate) fn file_identity(metadata: &Metadata) -> StoreFileIdentity { + use std::os::unix::fs::MetadataExt; + StoreFileIdentity { + volume: metadata.dev(), + file: metadata.ino(), + } +} + +#[cfg(not(unix))] +pub(crate) fn file_identity(metadata: &Metadata) -> StoreFileIdentity { + let modified: u64 = metadata.modified().map_or(0, |time| { + time.duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| { + u64::try_from(elapsed.as_nanos()).unwrap_or(u64::MAX) + }) + }); + StoreFileIdentity { + volume: metadata.len(), + file: modified, + } +} + +/// Whether `file` has content that does not start with the SQLite header. +pub(crate) fn header_is_foreign(file: &File) -> bool { + let mut header: Vec = Vec::with_capacity(SQLITE_HEADER.len()); + let read = file + .take(SQLITE_HEADER.len() as u64) + .read_to_end(&mut header); + read.is_ok_and(|length| length > 0) && header.as_slice() != SQLITE_HEADER.as_slice() +} diff --git a/crates/sqlbuild-recall/src/store/_helpers/mod.rs b/crates/sqlbuild-recall/src/store/_helpers/mod.rs new file mode 100644 index 0000000000..c1f9469a7f --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod identity; +pub(crate) mod quarantine; +pub(crate) mod rows; +pub(crate) mod schema; +pub(crate) mod sqlite; +pub(crate) mod truncation; diff --git a/crates/sqlbuild-recall/src/store/_helpers/quarantine.rs b/crates/sqlbuild-recall/src/store/_helpers/quarantine.rs new file mode 100644 index 0000000000..bd84f0bbe4 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/quarantine.rs @@ -0,0 +1,72 @@ +//! The quarantine lock and the checks that keep a corrupt store from being moved aside twice. + +use std::fs::{File, OpenOptions}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use crate::store::constants::{QUARANTINE_LOCK_RETRY_MS, QUARANTINE_LOCK_SUFFIX}; +use crate::store::errors::{StoreError, StoreErrorKind}; +use crate::store::main::observe_store_file::observe_store_file; +use crate::store::models::StoreObservation; + +pub(crate) fn moved_name(path: &Path, suffix: &str) -> PathBuf { + let mut name = path.as_os_str().to_owned(); + name.push(suffix); + PathBuf::from(name) +} + +pub(crate) fn corrupt(path: &Path, doing: &str, error: &std::io::Error) -> StoreError { + StoreError::new( + StoreErrorKind::Corrupt, + format!( + "cannot {doing} for the corrupt store {}: {error}", + path.display() + ), + ) +} + +/// One non-blocking attempt at the exclusive `flock`; `Ok(false)` while another process holds it. +#[cfg(unix)] +fn try_lock(file: &File) -> std::io::Result { + match rustix::fs::flock(file, rustix::fs::FlockOperation::NonBlockingLockExclusive) { + Ok(()) => Ok(true), + Err(rustix::io::Errno::WOULDBLOCK) => Ok(false), + Err(error) => Err(std::io::Error::from(error)), + } +} + +/// Without `flock`, only the identity re-check guards the move. +#[cfg(not(unix))] +fn try_lock(_file: &File) -> std::io::Result { + Ok(true) +} + +/// The exclusive lock on `.lock`, waiting until `give_up`; `None` when another process holds it that long. +pub(crate) fn quarantine_lock(path: &Path, give_up: Instant) -> Result, StoreError> { + let lock_path: PathBuf = moved_name(path, QUARANTINE_LOCK_SUFFIX); + let file: File = OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(&lock_path) + .map_err(|error| corrupt(path, "open the quarantine lock", &error))?; + loop { + match try_lock(&file) { + Ok(true) => return Ok(Some(file)), + Ok(false) if Instant::now() < give_up => { + std::thread::sleep(Duration::from_millis(QUARANTINE_LOCK_RETRY_MS)); + } + Ok(false) => return Ok(None), + Err(error) => return Err(corrupt(path, "take the quarantine lock", &error)), + } + } +} + +/// Whether the file now at `path` is still the one observed corrupt. +pub(crate) fn still_observed(path: &Path, observed: &StoreObservation) -> bool { + matches!( + observe_store_file(path), + Ok(Some(current)) if current.identity == observed.identity + && (current.header_foreign || !observed.header_foreign) + ) +} diff --git a/crates/sqlbuild-recall/src/store/_helpers/rows.rs b/crates/sqlbuild-recall/src/store/_helpers/rows.rs new file mode 100644 index 0000000000..91ee58ea45 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/rows.rs @@ -0,0 +1,138 @@ +//! Idempotent row writes: insert once, compare content on a duplicate key, record conflicts. + +use rusqlite::{Connection, OptionalExtension}; + +use crate::records::main::conflict_row::conflict_row; +use crate::records::models::{RecallRow, RecallValue, RowEnvelope}; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::Table; +use crate::store::_helpers::sqlite::{SqlValue, checked}; +use crate::store::errors::StoreError; +use crate::store::models::{ConflictEvidence, InsertOutcome}; + +fn key_text(value: Option<&RecallValue>) -> String { + match value { + Some(RecallValue::Text(text)) => text.clone(), + Some(RecallValue::Integer(number)) => number.to_string(), + Some(RecallValue::Real(number)) => number.to_string(), + Some(RecallValue::Blob(_)) => "".to_owned(), + Some(RecallValue::Null) | None => String::new(), + } +} + +fn text_of(row: &RecallRow, column: &str) -> String { + key_text(row.value(column)) +} + +/// Insert one row; `Some(evidence)` when its key holds different content. +fn write_row( + connection: &Connection, + row: &RecallRow, + ingested_at: &str, +) -> Result, StoreError> { + let spec = table_spec(row.table); + let ingested: RecallValue = RecallValue::Text(ingested_at.to_owned()); + let mut names: Vec<&str> = row.values.iter().map(|(name, _)| *name).collect(); + names.push("ingested_at"); + let placeholders: Vec = (1..=names.len()).map(|index| format!("?{index}")).collect(); + let sql = format!( + "INSERT INTO \"{}\" ({}) VALUES ({}) ON CONFLICT DO NOTHING", + spec.name, + names.join(", "), + placeholders.join(", ") + ); + let parameters: Vec> = row + .values + .iter() + .map(|(_, value)| SqlValue(value)) + .chain(std::iter::once(SqlValue(&ingested))) + .collect(); + let mut statement = checked(connection.prepare_cached(&sql), "prepare Recall insert")?; + let changed: usize = checked( + statement.execute(rusqlite::params_from_iter(parameters.iter())), + "write Recall rows", + )?; + if changed > 0 { + return Ok(None); + } + let condition: Vec = spec + .key + .iter() + .enumerate() + .map(|(index, name)| format!("{name} = ?{}", index + 1)) + .collect(); + let lookup = format!( + "SELECT content_hash FROM \"{}\" WHERE {}", + spec.name, + condition.join(" AND ") + ); + let null = RecallValue::Null; + let key_values: Vec> = spec + .key + .iter() + .map(|name| SqlValue(row.value(name).unwrap_or(&null))) + .collect(); + let mut statement = checked(connection.prepare_cached(&lookup), "prepare Recall lookup")?; + let existing: Option = checked( + statement + .query_row(rusqlite::params_from_iter(key_values.iter()), |found| { + found.get(0) + }) + .optional(), + "read stored Recall row", + )?; + let rejected: &str = row.content_hash().unwrap_or_default(); + match existing { + Some(existing) if existing != rejected => Ok(Some(ConflictEvidence { + table: row.table, + record_key: spec + .key + .iter() + .map(|name| key_text(row.value(name))) + .collect::>() + .join(":"), + existing_sha256: existing, + rejected_sha256: rejected.to_owned(), + })), + _ => Ok(None), + } +} + +/// Write `rows` on the current transaction; conflicts become `recall_conflicts` rows. +pub(crate) fn write_rows( + connection: &Connection, + rows: &[RecallRow], + ingested_at: &str, +) -> Result { + let mut outcome = InsertOutcome::default(); + for row in rows { + match write_row(connection, row, ingested_at)? { + None => match outcome + .delivered + .iter_mut() + .find(|(seen, _)| *seen == row.table) + { + Some((_, count)) => *count += 1, + None => outcome.delivered.push((row.table, 1)), + }, + Some(evidence) => { + let envelope = RowEnvelope { + invocation_id: text_of(row, "invocation_id"), + invocation_started_at: text_of(row, "invocation_started_at"), + producer_version: text_of(row, "producer_version"), + }; + let conflict = conflict_row( + &envelope, + (table_spec(row.table).name, &evidence.record_key), + (&evidence.existing_sha256, &evidence.rejected_sha256), + ingested_at, + ); + if row.table != Table::Conflicts { + write_row(connection, &conflict, ingested_at)?; + } + outcome.conflicts.push(evidence); + } + } + } + Ok(outcome) +} diff --git a/crates/sqlbuild-recall/src/store/_helpers/schema.rs b/crates/sqlbuild-recall/src/store/_helpers/schema.rs new file mode 100644 index 0000000000..0a95844fcb --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/schema.rs @@ -0,0 +1,80 @@ +//! Schema state checks and additive column upgrades of a SQLite store. + +use std::collections::HashSet; + +use rusqlite::Connection; + +use crate::schema::constants::{ALL_TABLES, RECALL_SCHEMA_VERSION, VIEW_NAMES}; +use crate::schema::main::add_column_statement::add_column_statement; +use crate::schema::main::table_columns::table_columns; +use crate::schema::main::table_spec::table_spec; +use crate::schema::models::{DdlOptions, Dialect}; +use crate::store::_helpers::sqlite::{checked, stored_schema_version}; +use crate::store::errors::StoreError; + +pub(crate) fn sqlite_options() -> DdlOptions { + DdlOptions { + database: None, + retention_days: 0, + cluster: None, + } +} + +/// Whether every table and view exists at this build's schema version, read without a lock. +pub(crate) fn current(connection: &Connection) -> Result { + if stored_schema_version(connection)?.is_none_or(|version| version < RECALL_SCHEMA_VERSION) { + return Ok(false); + } + let present: i64 = checked( + connection.query_row( + "SELECT COUNT(*) FROM sqlite_schema WHERE type IN ('table', 'view') \ + AND name IN (SELECT value FROM json_each(?1))", + [expected_names()], + |row| row.get(0), + ), + "inspect Recall store", + )?; + Ok(present == i64::try_from(ALL_TABLES.len() + VIEW_NAMES.len()).unwrap_or(i64::MAX)) +} + +fn expected_names() -> String { + let names: Vec<&str> = ALL_TABLES + .iter() + .map(|table| table_spec(*table).name) + .chain(VIEW_NAMES) + .collect(); + serde_json::Value::from(names).to_string() +} + +pub(crate) fn add_missing_columns(connection: &Connection) -> Result<(), StoreError> { + let options = sqlite_options(); + for table in ALL_TABLES { + let spec = table_spec(table); + let mut statement = checked( + connection.prepare(&format!( + "SELECT name FROM pragma_table_xinfo('{}')", + spec.name + )), + "inspect Recall table", + )?; + let existing: HashSet = checked( + statement + .query_map([], |row| row.get::<_, String>(0)) + .and_then(Iterator::collect), + "inspect Recall table", + )?; + for column in table_columns(table) { + if !existing.contains(column.name) { + checked( + connection.execute_batch(&add_column_statement( + Dialect::Sqlite, + &options, + (spec, &column), + )), + "add Recall column", + )?; + } + } + } + Ok(()) +} diff --git a/crates/sqlbuild-recall/src/store/_helpers/sqlite.rs b/crates/sqlbuild-recall/src/store/_helpers/sqlite.rs new file mode 100644 index 0000000000..ef93fde778 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/sqlite.rs @@ -0,0 +1,100 @@ +//! rusqlite plumbing: error kinds, value conversion and transactions. + +use rusqlite::types::{ToSqlOutput, Value, ValueRef}; +use rusqlite::{Connection, ErrorCode}; + +use crate::records::models::RecallValue; +use crate::store::errors::{StoreError, StoreErrorKind}; + +/// The store error of a rusqlite failure, with what was being done. +pub(crate) fn store_error(error: &rusqlite::Error, doing: &str) -> StoreError { + let kind = match error.sqlite_error_code() { + Some(ErrorCode::ReadOnly | ErrorCode::CannotOpen | ErrorCode::PermissionDenied) => { + StoreErrorKind::Permission + } + Some(ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) => StoreErrorKind::Busy, + Some(ErrorCode::NotADatabase | ErrorCode::DatabaseCorrupt) => StoreErrorKind::Corrupt, + _ => StoreErrorKind::Other, + }; + StoreError::new(kind, format!("{doing}: {error}")) +} + +/// A closure's `rusqlite::Result` as a store result. +pub(crate) fn checked(result: rusqlite::Result, doing: &str) -> Result { + result.map_err(|error| store_error(&error, doing)) +} + +pub(crate) fn to_sql(value: &RecallValue) -> ToSqlOutput<'_> { + match value { + RecallValue::Null => ToSqlOutput::Owned(Value::Null), + RecallValue::Integer(number) => ToSqlOutput::Owned(Value::Integer(*number)), + RecallValue::Real(number) => ToSqlOutput::Owned(Value::Real(*number)), + RecallValue::Text(text) => ToSqlOutput::Borrowed(ValueRef::Text(text.as_bytes())), + RecallValue::Blob(data) => ToSqlOutput::Borrowed(ValueRef::Blob(data)), + } +} + +pub(crate) fn from_sql(value: ValueRef<'_>) -> RecallValue { + match value { + ValueRef::Null => RecallValue::Null, + ValueRef::Integer(number) => RecallValue::Integer(number), + ValueRef::Real(number) => RecallValue::Real(number), + ValueRef::Text(bytes) => RecallValue::Text(String::from_utf8_lossy(bytes).into_owned()), + ValueRef::Blob(bytes) => RecallValue::Blob(bytes.to_vec()), + } +} + +/// Run `work` inside `BEGIN IMMEDIATE … COMMIT`, rolling back when it fails. +pub(crate) fn immediate( + connection: &Connection, + doing: &str, + work: impl FnOnce(&Connection) -> Result, +) -> Result { + checked(connection.execute_batch("BEGIN IMMEDIATE"), doing)?; + match work(connection) { + Ok(value) => { + if let Err(error) = connection.execute_batch("COMMIT") { + drop(connection.execute_batch("ROLLBACK")); + return Err(store_error(&error, doing)); + } + Ok(value) + } + Err(error) => { + drop(connection.execute_batch("ROLLBACK")); + Err(error) + } + } +} + +/// The highest schema version recorded in `recall_meta`, `None` before the first setup. +pub(crate) fn stored_schema_version(connection: &Connection) -> Result, StoreError> { + let exists: bool = checked( + connection.query_row( + "SELECT EXISTS (SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = 'recall_meta')", + [], + |row| row.get(0), + ), + "inspect Recall store", + )?; + if !exists { + return Ok(None); + } + checked( + connection.query_row( + "SELECT MAX(CAST(value AS INTEGER)) FROM recall_meta \ + WHERE component = 'schema' AND key = 'schema_version'", + [], + |row| row.get(0), + ), + "read Recall schema version", + ) +} + +/// A borrowed Recall value bound as a statement parameter. +pub(crate) struct SqlValue<'a>(pub(crate) &'a RecallValue); + +impl rusqlite::ToSql for SqlValue<'_> { + fn to_sql(&self) -> rusqlite::Result> { + Ok(to_sql(self.0)) + } +} diff --git a/crates/sqlbuild-recall/src/store/_helpers/truncation.rs b/crates/sqlbuild-recall/src/store/_helpers/truncation.rs new file mode 100644 index 0000000000..678b9cde3d --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/truncation.rs @@ -0,0 +1,89 @@ +//! The note `sqb recall output` prints where output past `max_output_bytes` was left out. + +use crate::store::constants::{BINARY_UNIT_STEP, BINARY_UNITS, DECIMAL_UNIT_STEP, DECIMAL_UNITS}; +use crate::store::models::StoredChunk; + +/// `23.6 MB`: decimal units with one decimal, whole bytes below a kilobyte. +fn decimal_size(bytes: u64) -> String { + let mut value = bytes as f64; + let mut unit: Option<&str> = None; + for candidate in DECIMAL_UNITS { + if value < DECIMAL_UNIT_STEP { + break; + } + value /= DECIMAL_UNIT_STEP; + unit = Some(candidate); + } + unit.map_or_else( + || format!("{bytes} bytes"), + |unit| format!("{value:.1} {unit}"), + ) +} + +/// `1 MiB`: the largest binary unit dividing `bytes` exactly. +fn binary_size(bytes: u64) -> String { + let mut value = bytes; + let mut unit: Option<&str> = None; + for candidate in BINARY_UNITS { + if value == 0 || !value.is_multiple_of(BINARY_UNIT_STEP) { + break; + } + value /= BINARY_UNIT_STEP; + unit = Some(candidate); + } + unit.map_or_else( + || format!("{bytes} bytes"), + |unit| format!("{value} {unit}"), + ) +} + +/// One line naming the bytes of `stream` left out between the kept `head` and `tail`. +pub(crate) fn truncation_note(stream: &str, omitted: u64, (head, tail): (u64, u64)) -> String { + let kept: String = if head == tail { + format!("kept the first and last {}", binary_size(head)) + } else { + format!( + "kept the first {} and the last {}", + binary_size(head), + binary_size(tail) + ) + }; + format!( + "\u{2026} {} of {stream} not recorded ({kept}). Raise [recall] max_output_bytes to keep \ + more.\n", + decimal_size(omitted) + ) +} + +/// Bytes of `stream` in `chunks`. +fn stream_bytes(chunks: &[StoredChunk], stream: &str) -> u64 { + chunks + .iter() + .filter(|chunk| chunk.stream == stream) + .map(|chunk| chunk.data.len() as u64) + .sum() +} + +/// The chunks' bytes in order, with a note line where output was left out. +pub(crate) fn with_truncation_notes(chunks: &[StoredChunk]) -> Vec { + let mut data: Vec = Vec::with_capacity(chunks.iter().map(|chunk| chunk.data.len()).sum()); + for (position, chunk) in chunks.iter().enumerate() { + data.extend_from_slice(&chunk.data); + let omitted: u64 = chunk + .omitted_bytes + .filter(|bytes| *bytes > 0) + .map_or(0, i64::unsigned_abs); + if omitted == 0 { + continue; + } + if data.last().is_some_and(|byte| *byte != b'\n') { + data.push(b'\n'); + } + let kept = ( + stream_bytes(&chunks[..position], &chunk.stream), + stream_bytes(&chunks[position + 1..], &chunk.stream), + ); + data.extend_from_slice(truncation_note(&chunk.stream, omitted, kept).as_bytes()); + } + data +} diff --git a/crates/sqlbuild-recall/src/store/constants.rs b/crates/sqlbuild-recall/src/store/constants.rs new file mode 100644 index 0000000000..f7ee4fb4ae --- /dev/null +++ b/crates/sqlbuild-recall/src/store/constants.rs @@ -0,0 +1,32 @@ +//! SQLite store pragmas and maintenance budgets. + +pub const BUSY_TIMEOUT_MS: u64 = 5_000; +/// `journal_size_limit`: the WAL is truncated to this size after checkpoints. +pub const JOURNAL_SIZE_LIMIT_BYTES: u64 = 64 * 1024 * 1024; +/// Maintenance runs at most once per this many seconds. +pub const MAINTENANCE_INTERVAL_SECONDS: u64 = 24 * 60 * 60; +/// Total time one maintenance run may spend deleting. +pub const MAINTENANCE_BUDGET_MS: u64 = 2_000; +/// Pages released by one `incremental_vacuum`. +pub const INCREMENTAL_VACUUM_PAGES: u32 = 2_048; +pub const META_MAINTENANCE_OVERDUE_KEY: &str = "overdue"; +/// The selector naming the most recently started invocation. +pub const LAST_INVOCATION_SELECTOR: &str = "last"; +/// The first bytes of every SQLite database file. +pub const SQLITE_HEADER: &[u8; 16] = b"SQLite format 3\0"; +/// What a store moved aside for corruption is renamed to, before its UTC timestamp. +pub const CORRUPT_SUFFIX: &str = ".corrupt-"; +/// SQLite's companion files, moved along with a corrupt store. +pub const COMPANION_SUFFIXES: [&str; 2] = ["-wal", "-shm"]; +/// Appended to the store path to name the lock file that serialises moving a corrupt store aside. +pub const QUARANTINE_LOCK_SUFFIX: &str = ".lock"; +/// Pause between attempts to take the quarantine lock. +pub const QUARANTINE_LOCK_RETRY_MS: u64 = 10; +/// The step between decimal size units. +pub const DECIMAL_UNIT_STEP: f64 = 1000.0; +/// The step between binary size units. +pub const BINARY_UNIT_STEP: u64 = 1024; +/// Decimal size units above bytes, for omitted output. +pub const DECIMAL_UNITS: [&str; 4] = ["kB", "MB", "GB", "TB"]; +/// Binary size units above bytes, for kept output. +pub const BINARY_UNITS: [&str; 4] = ["KiB", "MiB", "GiB", "TiB"]; diff --git a/crates/sqlbuild-recall/src/store/errors.rs b/crates/sqlbuild-recall/src/store/errors.rs new file mode 100644 index 0000000000..a1422e93b2 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/errors.rs @@ -0,0 +1,45 @@ +//! Why a Recall store operation failed. + +/// What kind of failure a store reported. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StoreErrorKind { + /// The file or directory cannot be created or written. + Permission, + /// Another writer holds the lock past the busy timeout; retrying may succeed. + Busy, + /// The file is not a SQLite database, or its pages are damaged. + Corrupt, + /// The SQLite library is older than the store needs. + Version, + /// A query was rejected: not read-only, or invalid SQL. + Query, + /// No invocation matches. + NotFound, + /// More than one invocation matches. + Ambiguous, + Other, +} + +/// A store failure with its kind and message. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StoreError { + pub kind: StoreErrorKind, + pub message: String, +} + +impl StoreError { + pub(crate) fn new(kind: StoreErrorKind, message: impl Into) -> Self { + Self { + kind, + message: message.into(), + } + } +} + +impl std::fmt::Display for StoreError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(&self.message) + } +} + +impl std::error::Error for StoreError {} diff --git a/crates/sqlbuild-recall/src/store/main/checkpoint_sqlite_store.rs b/crates/sqlbuild-recall/src/store/main/checkpoint_sqlite_store.rs new file mode 100644 index 0000000000..d4541ba3bb --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/checkpoint_sqlite_store.rs @@ -0,0 +1,15 @@ +//! `checkpoint_sqlite_store`: copy the WAL into the store without waiting on anyone. + +use crate::store::_helpers::sqlite::checked; +use crate::store::errors::StoreError; +use crate::store::models::SqliteStore; + +/// A passive checkpoint: writers never checkpoint as they commit or close, so each session runs one when it starts, beside the command rather than after it. +pub fn checkpoint_sqlite_store(store: &SqliteStore) -> Result<(), StoreError> { + checked( + store + .connection + .query_row("PRAGMA wal_checkpoint(PASSIVE)", [], |_| Ok(())), + "checkpoint Recall WAL", + ) +} diff --git a/crates/sqlbuild-recall/src/store/main/describe_store.rs b/crates/sqlbuild-recall/src/store/main/describe_store.rs new file mode 100644 index 0000000000..a23de3603a --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/describe_store.rs @@ -0,0 +1,49 @@ +//! `describe_store`: the store's views and tables with their columns. + +use crate::store::_helpers::sqlite::checked; +use crate::store::errors::StoreError; +use crate::store::models::{DescribedColumn, DescribedRelation, SqliteStore}; + +/// Views first, then `recall_*` tables, each sorted by name. +pub fn describe_store(store: &SqliteStore) -> Result, StoreError> { + let mut statement = checked( + store.connection.prepare( + "SELECT name, type FROM sqlite_schema WHERE type IN ('table', 'view') \ + AND name NOT LIKE 'sqlite!_%' ESCAPE '!' ORDER BY type = 'table', name", + ), + "describe Recall store", + )?; + let relations: Vec<(String, String)> = checked( + statement + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .and_then(Iterator::collect), + "describe Recall store", + )?; + let mut described = Vec::with_capacity(relations.len()); + for (name, kind) in relations { + let mut columns = checked( + store + .connection + .prepare("SELECT name, type, \"notnull\" FROM pragma_table_xinfo(?1)"), + "describe Recall relation", + )?; + let columns: Vec = checked( + columns + .query_map([&name], |row| { + Ok(DescribedColumn { + name: row.get(0)?, + declared_type: row.get(1)?, + nullable: row.get::<_, i64>(2)? == 0, + }) + }) + .and_then(Iterator::collect), + "describe Recall relation", + )?; + described.push(DescribedRelation { + name, + kind, + columns, + }); + } + Ok(described) +} diff --git a/crates/sqlbuild-recall/src/store/main/ensure_sqlite_schema.rs b/crates/sqlbuild-recall/src/store/main/ensure_sqlite_schema.rs new file mode 100644 index 0000000000..01ad4d8695 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/ensure_sqlite_schema.rs @@ -0,0 +1,57 @@ +//! `ensure_sqlite_schema`: create and additively upgrade the store's tables and views. + +use crate::records::main::meta_row::meta_row; +use crate::records::models::RowEnvelope; +use crate::schema::constants::{ + META_SCHEMA_COMPONENT, META_SCHEMA_VERSION_KEY, RECALL_SCHEMA_VERSION, +}; +use crate::schema::main::table_statements::table_statements; +use crate::schema::main::view_statements::view_statements; +use crate::schema::models::Dialect; +use crate::store::_helpers::rows::write_rows; +use crate::store::_helpers::schema::{add_missing_columns, current, sqlite_options}; +use crate::store::_helpers::sqlite::{checked, immediate, stored_schema_version}; +use crate::store::errors::StoreError; +use crate::store::models::SqliteStore; + +/// Create missing tables, indexes and views, add missing columns and record the schema version, in one `BEGIN IMMEDIATE` transaction. A store written by a newer SQLBuild keeps its views. +pub fn ensure_sqlite_schema( + store: &SqliteStore, + envelope: &RowEnvelope, + recorded_at: &str, +) -> Result<(), StoreError> { + if current(&store.connection)? { + return Ok(()); + } + let options = sqlite_options(); + immediate(&store.connection, "create Recall tables", |connection| { + for statement in table_statements(Dialect::Sqlite, &options) { + checked(connection.execute_batch(&statement), "create Recall table")?; + } + add_missing_columns(connection)?; + let stored: Option = stored_schema_version(connection)?; + let upgrade: bool = stored.is_none_or(|version| version < RECALL_SCHEMA_VERSION); + for (name, statement) in view_statements(Dialect::Sqlite, &options) { + if upgrade { + checked( + connection.execute_batch(&format!("DROP VIEW IF EXISTS \"{name}\"")), + "replace Recall view", + )?; + } + checked(connection.execute_batch(&statement), "create Recall view")?; + } + if upgrade { + let version = RECALL_SCHEMA_VERSION.to_string(); + write_rows( + connection, + &[meta_row( + envelope, + (META_SCHEMA_COMPONENT, META_SCHEMA_VERSION_KEY, &version), + recorded_at, + )], + recorded_at, + )?; + } + Ok(()) + }) +} diff --git a/crates/sqlbuild-recall/src/store/main/insert_rows.rs b/crates/sqlbuild-recall/src/store/main/insert_rows.rs new file mode 100644 index 0000000000..aaec9c77e8 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/insert_rows.rs @@ -0,0 +1,18 @@ +//! `insert_rows`: write one batch atomically and idempotently. + +use crate::records::models::RecallRow; +use crate::store::_helpers::rows::write_rows; +use crate::store::_helpers::sqlite::immediate; +use crate::store::errors::StoreError; +use crate::store::models::{InsertOutcome, SqliteStore}; + +/// Write `rows` in one transaction. A row whose key exists with equal content is a no-op; one with different content keeps the stored row and is reported and recorded as a conflict. +pub fn insert_rows( + store: &SqliteStore, + rows: &[RecallRow], + ingested_at: &str, +) -> Result { + immediate(&store.connection, "write Recall rows", |connection| { + write_rows(connection, rows, ingested_at) + }) +} diff --git a/crates/sqlbuild-recall/src/store/main/maintain_sqlite_store.rs b/crates/sqlbuild-recall/src/store/main/maintain_sqlite_store.rs new file mode 100644 index 0000000000..0961d2a5cc --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/maintain_sqlite_store.rs @@ -0,0 +1,174 @@ +//! `maintain_sqlite_store`: claimed, budgeted retention deletes and space reclamation. + +use std::time::{Duration, Instant, SystemTime}; + +use rusqlite::Connection; + +use crate::records::main::format_timestamp::format_timestamp; +use crate::records::main::meta_row::meta_row; +use crate::records::main::parse_timestamp::parse_timestamp; +use crate::schema::constants::{ + META_MAINTENANCE_CLAIM_KEY, META_MAINTENANCE_COMPONENT, RETENTION_BATCH_INVOCATIONS, +}; +use crate::schema::main::sqlite_retention_sql::sqlite_retention_sql; +use crate::store::_helpers::rows::write_rows; +use crate::store::_helpers::sqlite::{checked, immediate}; +use crate::store::constants::{ + INCREMENTAL_VACUUM_PAGES, MAINTENANCE_INTERVAL_SECONDS, META_MAINTENANCE_OVERDUE_KEY, +}; +use crate::store::errors::StoreError; +use crate::store::models::{MaintenanceOutcome, MaintenanceRequest, SqliteStore}; + +fn latest(connection: &Connection, key: &str) -> Result, StoreError> { + checked( + connection.query_row( + "SELECT MAX(recorded_at) FROM recall_meta WHERE component = ?1 AND key = ?2", + [META_MAINTENANCE_COMPONENT, key], + |row| row.get(0), + ), + "read Recall maintenance state", + ) +} + +/// Whether this process should maintain now: no recent claim, or the last run left work. +fn due(connection: &Connection, now: SystemTime) -> Result { + let Some(claim) = latest(connection, META_MAINTENANCE_CLAIM_KEY)? else { + return Ok(true); + }; + if latest(connection, META_MAINTENANCE_OVERDUE_KEY)?.is_some_and(|overdue| overdue >= claim) { + return Ok(true); + } + let claimed: SystemTime = parse_timestamp(&claim).unwrap_or(SystemTime::UNIX_EPOCH); + Ok(now + .duration_since(claimed) + .is_ok_and(|elapsed| elapsed >= Duration::from_secs(MAINTENANCE_INTERVAL_SECONDS))) +} + +/// Delete invocations started before the retention cutoff, at most once per interval across processes, in short batches within the budget, then reclaim pages and checkpoint the WAL. +pub fn maintain_sqlite_store( + store: &SqliteStore, + request: &MaintenanceRequest, +) -> Result { + let started = Instant::now(); + let now_text: String = format_timestamp(request.now); + let claimed: bool = immediate( + &store.connection, + "claim Recall maintenance", + |connection| { + if !due(connection, request.now)? { + return Ok(false); + } + write_rows( + connection, + &[meta_row( + &request.envelope, + ( + META_MAINTENANCE_COMPONENT, + META_MAINTENANCE_CLAIM_KEY, + &request.owner, + ), + &now_text, + )], + &now_text, + )?; + Ok(true) + }, + )?; + if !claimed { + return Ok(MaintenanceOutcome::default()); + } + let retention = Duration::from_secs(u64::from(request.retention_days) * 86_400); + let cutoff: String = format_timestamp( + request + .now + .checked_sub(retention) + .unwrap_or(SystemTime::UNIX_EPOCH), + ); + let sql = sqlite_retention_sql(); + let mut outcome = MaintenanceOutcome { + ran: true, + ..MaintenanceOutcome::default() + }; + loop { + if started.elapsed() >= request.budget { + break; + } + let deleted: usize = + immediate(&store.connection, "apply Recall retention", |connection| { + let mut statement = checked( + connection.prepare_cached(&sql.expired_invocations), + "find expired Recall invocations", + )?; + let ids: Vec = checked( + statement + .query_map( + rusqlite::params![cutoff, now_text, RETENTION_BATCH_INVOCATIONS], + |row| row.get(0), + ) + .and_then(Iterator::collect), + "find expired Recall invocations", + )?; + if ids.is_empty() { + return Ok(0); + } + let encoded: String = serde_json::Value::from(ids.clone()).to_string(); + for (_, delete) in &sql.delete_invocations { + checked( + connection.execute(delete, [&encoded]), + "delete expired Recall rows", + )?; + } + Ok(ids.len()) + })?; + if deleted == 0 { + outcome.complete = true; + break; + } + outcome.deleted_invocations += deleted as u64; + } + outcome.deleted_artifacts = checked( + store + .connection + .execute(&sql.delete_orphan_artifacts, [&cutoff]), + "delete unreferenced Recall artifacts", + )? as u64; + checked( + store.connection.execute(&sql.delete_old_claims, [&cutoff]), + "delete old Recall maintenance claims", + )?; + if !outcome.complete { + immediate( + &store.connection, + "record overdue maintenance", + |connection| { + write_rows( + connection, + &[meta_row( + &request.envelope, + ( + META_MAINTENANCE_COMPONENT, + META_MAINTENANCE_OVERDUE_KEY, + &request.owner, + ), + &now_text, + )], + &now_text, + ) + .map(drop) + }, + )?; + } + checked( + store.connection.execute_batch(&format!( + "PRAGMA incremental_vacuum({INCREMENTAL_VACUUM_PAGES})" + )), + "reclaim Recall pages", + )?; + checked( + store + .connection + .query_row("PRAGMA wal_checkpoint(PASSIVE)", [], |_| Ok(())), + "checkpoint Recall WAL", + )?; + Ok(outcome) +} diff --git a/crates/sqlbuild-recall/src/store/main/mod.rs b/crates/sqlbuild-recall/src/store/main/mod.rs new file mode 100644 index 0000000000..584ad7dfc5 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/mod.rs @@ -0,0 +1,12 @@ +pub mod checkpoint_sqlite_store; +pub mod describe_store; +pub mod ensure_sqlite_schema; +pub mod insert_rows; +pub mod maintain_sqlite_store; +pub mod observe_store_file; +pub mod open_sqlite_store; +pub mod quarantine_store; +pub mod query_store; +pub mod read_output_transcript; +pub mod resolve_invocation; +pub mod store_location_problem; diff --git a/crates/sqlbuild-recall/src/store/main/observe_store_file.rs b/crates/sqlbuild-recall/src/store/main/observe_store_file.rs new file mode 100644 index 0000000000..01fb7282d3 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/observe_store_file.rs @@ -0,0 +1,21 @@ +//! `observe_store_file`: which file a store path names and whether it is a SQLite database. + +use std::io::ErrorKind; +use std::path::Path; + +use crate::store::_helpers::identity::{file_identity, header_is_foreign}; +use crate::store::models::StoreObservation; + +/// The identity and header verdict of the file at `path`, read from one open handle; `None` when there is no file. +pub fn observe_store_file(path: &Path) -> std::io::Result> { + let file = match std::fs::File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + }; + let identity = file_identity(&file.metadata()?); + Ok(Some(StoreObservation { + identity, + header_foreign: header_is_foreign(&file), + })) +} diff --git a/crates/sqlbuild-recall/src/store/main/open_sqlite_store.rs b/crates/sqlbuild-recall/src/store/main/open_sqlite_store.rs new file mode 100644 index 0000000000..9208bf66cd --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/open_sqlite_store.rs @@ -0,0 +1,130 @@ +//! `open_sqlite_store`: open, or create, a SQLite Recall store with its pragmas. + +use std::path::Path; +use std::time::Duration; + +use rusqlite::config::DbConfig; +use rusqlite::{Connection, OpenFlags}; + +use crate::schema::constants::MINIMUM_SQLITE_VERSION; +use crate::store::_helpers::sqlite::{checked, store_error}; +use crate::store::constants::{BUSY_TIMEOUT_MS, JOURNAL_SIZE_LIMIT_BYTES}; +use crate::store::errors::{StoreError, StoreErrorKind}; +use crate::store::models::{OpenMode, SqliteStore}; + +fn check_version() -> Result<(), StoreError> { + let version: i32 = rusqlite::version_number(); + let (major, minor) = MINIMUM_SQLITE_VERSION; + let minimum = i32::try_from(major * 1_000_000 + minor * 1_000).unwrap_or(i32::MAX); + if version < minimum { + return Err(StoreError::new( + StoreErrorKind::Version, + format!( + "Recall needs SQLite {major}.{minor} or newer; this build has {}", + rusqlite::version() + ), + )); + } + Ok(()) +} + +fn create_parent(path: &Path) -> Result<(), StoreError> { + let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + else { + return Ok(()); + }; + std::fs::create_dir_all(parent).map_err(|error| { + let kind = if error.kind() == std::io::ErrorKind::PermissionDenied + || error.kind() == std::io::ErrorKind::ReadOnlyFilesystem + { + StoreErrorKind::Permission + } else { + StoreErrorKind::Other + }; + StoreError::new( + kind, + format!("cannot create directory {}: {error}", parent.display()), + ) + }) +} + +/// The open store at `path`. Writers create the file and its directory and switch it to WAL; a new file gets incremental auto-vacuum before any table exists. +pub fn open_sqlite_store(path: &Path, mode: OpenMode) -> Result { + check_version()?; + let connection: Connection = match mode { + OpenMode::ReadWrite => { + create_parent(path)?; + checked( + Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_WRITE + | OpenFlags::SQLITE_OPEN_CREATE + | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ), + &format!("cannot open {}", path.display()), + )? + } + OpenMode::ReadOnly => { + if !path.exists() { + return Err(StoreError::new( + StoreErrorKind::NotFound, + format!("no Recall store at {}", path.display()), + )); + } + checked( + Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ), + &format!("cannot open {}", path.display()), + )? + } + }; + checked( + connection.busy_timeout(Duration::from_millis(BUSY_TIMEOUT_MS)), + "set busy timeout", + )?; + if mode == OpenMode::ReadWrite { + let page_count: i64 = checked( + connection.query_row("PRAGMA page_count", [], |row| row.get(0)), + &format!("cannot read {}", path.display()), + )?; + if page_count == 0 { + checked( + connection.execute_batch("PRAGMA auto_vacuum = INCREMENTAL"), + "enable incremental vacuum", + )?; + } + let journal: String = connection + .query_row("PRAGMA journal_mode = WAL", [], |row| row.get(0)) + .map_err(|error| store_error(&error, &format!("cannot write {}", path.display())))?; + if !journal.eq_ignore_ascii_case("wal") { + return Err(StoreError::new( + StoreErrorKind::Permission, + format!("cannot switch {} to WAL mode", path.display()), + )); + } + checked( + connection.execute_batch(&format!( + "PRAGMA synchronous = NORMAL; PRAGMA journal_size_limit = {JOURNAL_SIZE_LIMIT_BYTES}; \ + PRAGMA wal_autocheckpoint = 0;" + )), + "set store pragmas", + )?; + checked( + connection.set_db_config(DbConfig::SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, true), + "keep the WAL at close", + )?; + } else { + checked( + connection.execute_batch("PRAGMA query_only = ON"), + "open store read-only", + )?; + } + Ok(SqliteStore { + connection, + path: path.to_path_buf(), + }) +} diff --git a/crates/sqlbuild-recall/src/store/main/quarantine_store.rs b/crates/sqlbuild-recall/src/store/main/quarantine_store.rs new file mode 100644 index 0000000000..4567fc059e --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/quarantine_store.rs @@ -0,0 +1,38 @@ +//! `quarantine_store`: move a corrupt store aside, once, so a fresh one can start. + +use std::path::{Path, PathBuf}; +use std::time::{Instant, SystemTime}; + +use crate::records::main::format_timestamp::format_timestamp; +use crate::store::_helpers::quarantine::{corrupt, moved_name, quarantine_lock, still_observed}; +use crate::store::constants::{COMPANION_SUFFIXES, CORRUPT_SUFFIX}; +use crate::store::errors::StoreError; +use crate::store::models::{QuarantineOutcome, StoreObservation}; + +/// Under the store's quarantine lock, rename the observed corrupt file and its companions to `.corrupt-`. When another process already replaced it, keep the store that is there. +pub fn quarantine_store( + path: &Path, + observed: &StoreObservation, + (give_up, now): (Instant, SystemTime), +) -> Result { + let Some(lock) = quarantine_lock(path, give_up)? else { + return Ok(QuarantineOutcome::LockBusy); + }; + if !still_observed(path, observed) { + drop(lock); + return Ok(QuarantineOutcome::Kept); + } + let stamp: String = format_timestamp(now).replace(':', ""); + let target: PathBuf = moved_name(path, &format!("{CORRUPT_SUFFIX}{stamp}")); + for companion in COMPANION_SUFFIXES { + let source: PathBuf = moved_name(path, companion); + if source.exists() { + drop(std::fs::rename(&source, moved_name(&target, companion))); + } + } + let moved = std::fs::rename(path, &target) + .map(|()| QuarantineOutcome::Moved(target)) + .map_err(|error| corrupt(path, "move the file aside", &error)); + drop(lock); + moved +} diff --git a/crates/sqlbuild-recall/src/store/main/query_store.rs b/crates/sqlbuild-recall/src/store/main/query_store.rs new file mode 100644 index 0000000000..26461a3ac5 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/query_store.rs @@ -0,0 +1,54 @@ +//! `query_store`: run one read-only query against a store. + +use crate::store::_helpers::sqlite::{checked, from_sql}; +use crate::store::errors::{StoreError, StoreErrorKind}; +use crate::store::models::{QueryResult, SqliteStore}; + +/// The columns and at most `limit` rows of `sql`; statements that write are rejected. +pub fn query_store( + store: &SqliteStore, + sql: &str, + limit: Option, +) -> Result { + let mut statement = store.connection.prepare(sql).map_err(|error| { + StoreError::new( + StoreErrorKind::Query, + format!("invalid Recall query: {error}"), + ) + })?; + if !statement.readonly() { + return Err(StoreError::new( + StoreErrorKind::Query, + "Recall queries must be read-only SELECT statements", + )); + } + let columns: Vec = statement + .column_names() + .into_iter() + .map(str::to_owned) + .collect(); + let width: usize = columns.len(); + let mut rows = checked(statement.query([]), "run Recall query")?; + let mut result = QueryResult { + columns, + rows: Vec::new(), + truncated: false, + }; + while let Some(row) = rows.next().map_err(|error| { + StoreError::new( + StoreErrorKind::Query, + format!("Recall query failed: {error}"), + ) + })? { + if limit.is_some_and(|limit| result.rows.len() >= limit) { + result.truncated = true; + break; + } + let mut values = Vec::with_capacity(width); + for index in 0..width { + values.push(from_sql(checked(row.get_ref(index), "read Recall row")?)); + } + result.rows.push(values); + } + Ok(result) +} diff --git a/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs b/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs new file mode 100644 index 0000000000..c21864bb78 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs @@ -0,0 +1,85 @@ +//! `read_output_transcript`: one invocation's exact output bytes and their completeness. + +use rusqlite::OptionalExtension; + +use crate::records::models::OutputStream; +use crate::store::_helpers::sqlite::checked; +use crate::store::_helpers::truncation::with_truncation_notes; +use crate::store::errors::StoreError; +use crate::store::models::{OutputTranscript, SqliteStore, StoredChunk}; + +/// The bytes of `stream`, or both streams interleaved by sequence when `None`. +pub fn read_output_transcript( + store: &SqliteStore, + invocation_id: &str, + stream: Option, +) -> Result { + let mut statement = checked( + store.connection.prepare( + "SELECT stream, data, omitted_bytes FROM recall_output WHERE invocation_id = ?1 \ + AND (?2 IS NULL OR stream = ?2) ORDER BY sequence", + ), + "read Recall output", + )?; + let chunks: Vec = checked( + statement + .query_map( + rusqlite::params![invocation_id, stream.map(OutputStream::as_str)], + |row| { + Ok(StoredChunk { + stream: row.get(0)?, + data: row.get(1)?, + omitted_bytes: row.get(2)?, + }) + }, + ) + .and_then(Iterator::collect), + "read Recall output", + )?; + let mut transcript = OutputTranscript { + invocation_id: invocation_id.to_owned(), + data: with_truncation_notes(&chunks), + ..OutputTranscript::default() + }; + transcript.status = checked( + store + .connection + .query_row( + "SELECT status FROM invocations WHERE invocation_id = ?1", + [invocation_id], + |row| row.get(0), + ) + .optional(), + "read Recall invocation", + )?; + let completeness: Option<(i64, i64, i64, bool, bool, Option)> = checked( + store + .connection + .query_row( + "SELECT missing_sequences, reported_drops, filtered, flush_complete, complete, \ + gap_reason FROM output_invocations WHERE invocation_id = ?1", + [invocation_id], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + )) + }, + ) + .optional(), + "read Recall output completeness", + )?; + if let Some((missing, drops, filtered, flushed, complete, reason)) = completeness { + transcript.missing_sequences = missing; + transcript.reported_drops = drops; + transcript.filtered = filtered; + transcript.flush_complete = flushed; + transcript.complete = complete; + transcript.gap_reason = reason; + } + Ok(transcript) +} diff --git a/crates/sqlbuild-recall/src/store/main/resolve_invocation.rs b/crates/sqlbuild-recall/src/store/main/resolve_invocation.rs new file mode 100644 index 0000000000..71245acf0b --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/resolve_invocation.rs @@ -0,0 +1,62 @@ +//! `resolve_invocation`: the invocation an ID, unique prefix or `last` names. + +use crate::store::_helpers::sqlite::checked; +use crate::store::constants::LAST_INVOCATION_SELECTOR as LAST; +use crate::store::errors::{StoreError, StoreErrorKind}; +use crate::store::models::SqliteStore; + +/// The full invocation ID of `selector`; `last` is the most recently started invocation. +pub fn resolve_invocation(store: &SqliteStore, selector: &str) -> Result { + let candidates: Vec = if selector == LAST { + let mut statement = checked( + store.connection.prepare( + "SELECT invocation_id FROM invocations \ + ORDER BY invocation_started_at DESC, invocation_id DESC LIMIT 1", + ), + "find the last invocation", + )?; + checked( + statement + .query_map([], |row| row.get(0)) + .and_then(Iterator::collect), + "find the last invocation", + )? + } else { + let mut statement = checked( + store.connection.prepare( + "SELECT invocation_id FROM invocations \ + WHERE substr(invocation_id, 1, length(?1)) = ?1 \ + ORDER BY invocation_id LIMIT 3", + ), + "find the invocation", + )?; + checked( + statement + .query_map([selector], |row| row.get(0)) + .and_then(Iterator::collect), + "find the invocation", + )? + }; + if let Some(exact) = candidates.iter().find(|candidate| *candidate == selector) { + return Ok(exact.clone()); + } + match candidates.as_slice() { + [] => Err(StoreError::new( + StoreErrorKind::NotFound, + if selector == LAST { + "no invocations are recorded yet".to_owned() + } else { + format!("no recorded invocation matches '{selector}'") + }, + )), + [only] => Ok(only.clone()), + several => Err(StoreError::new( + StoreErrorKind::Ambiguous, + format!( + "'{selector}' matches several invocations: {}{}", + several[..2].join(", "), + if several.len() > 2 { ", …" } else { "" } + ), + )), + } +} diff --git a/crates/sqlbuild-recall/src/store/main/store_location_problem.rs b/crates/sqlbuild-recall/src/store/main/store_location_problem.rs new file mode 100644 index 0000000000..cde286eb90 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/main/store_location_problem.rs @@ -0,0 +1,20 @@ +//! `store_location_problem`: why a store could not be written, judged from file permissions alone. + +use std::path::Path; + +/// Why `path` cannot be created or written, from permission bits only, so it never waits on a lock. +#[must_use] +pub fn store_location_problem(path: &Path) -> Option { + let directory: &Path = path + .ancestors() + .skip(1) + .find(|ancestor| ancestor.is_dir())?; + let directory_readonly: bool = + std::fs::metadata(directory).is_ok_and(|metadata| metadata.permissions().readonly()); + if directory_readonly { + return Some(format!("{} is not writable", directory.display())); + } + let file_readonly: bool = + std::fs::metadata(path).is_ok_and(|metadata| metadata.permissions().readonly()); + file_readonly.then(|| format!("{} is read-only", path.display())) +} diff --git a/crates/sqlbuild-recall/src/store/mod.rs b/crates/sqlbuild-recall/src/store/mod.rs new file mode 100644 index 0000000000..0126341c03 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/mod.rs @@ -0,0 +1,10 @@ +//! The native SQLite Recall store: automatic DDL, idempotent inserts, maintenance and reads. + +pub(crate) mod _helpers; +pub mod constants; +pub mod errors; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-recall/src/store/models.rs b/crates/sqlbuild-recall/src/store/models.rs new file mode 100644 index 0000000000..8440f7aee7 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/models.rs @@ -0,0 +1,134 @@ +//! An open SQLite Recall store and the outcomes of its operations. + +use std::path::PathBuf; +use std::time::{Duration, SystemTime}; + +use crate::records::models::{RecallValue, RowEnvelope}; +use crate::schema::models::Table; + +/// One open store file. Only one thread uses a store. +#[derive(Debug)] +pub struct SqliteStore { + pub(crate) connection: rusqlite::Connection, + pub path: PathBuf, +} + +/// Whether a store is opened for writing, creating it when missing, or only for reading. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OpenMode { + ReadWrite, + ReadOnly, +} + +/// Which file a path named when it was observed: device and inode on Unix, else size and modification time. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StoreFileIdentity { + pub(crate) volume: u64, + pub(crate) file: u64, +} + +/// One look at the file at a store path. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StoreObservation { + pub identity: StoreFileIdentity, + /// The file has content and does not start with the SQLite header. + pub header_foreign: bool, +} + +/// What moving a corrupt store aside did. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum QuarantineOutcome { + /// The observed file and its companions were renamed to this path. + Moved(PathBuf), + /// The path no longer holds the observed corrupt file, so the store there is used as it is. + Kept, + /// Another process held the quarantine lock past the wait; nothing was moved. + LockBusy, +} + +/// A row whose key was already stored with different content; the first row is kept. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConflictEvidence { + pub table: Table, + pub record_key: String, + pub existing_sha256: String, + pub rejected_sha256: String, +} + +/// What one insert batch did, per table. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct InsertOutcome { + /// Rows written or already present with equal content, by table. + pub delivered: Vec<(Table, u64)>, + pub conflicts: Vec, +} + +/// What a maintenance run is allowed to do. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MaintenanceRequest { + pub now: SystemTime, + pub retention_days: u32, + pub owner: String, + pub budget: Duration, + pub envelope: RowEnvelope, +} + +/// What a maintenance run did. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MaintenanceOutcome { + /// False when another process maintained the store within the interval. + pub ran: bool, + pub deleted_invocations: u64, + pub deleted_artifacts: u64, + /// False when the budget ran out with expired invocations left; the next run is overdue. + pub complete: bool, +} + +/// The rows of a read-only query. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct QueryResult { + pub columns: Vec, + pub rows: Vec>, + /// More rows existed beyond the limit. + pub truncated: bool, +} + +/// One column of a table or view, as the store describes it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DescribedColumn { + pub name: String, + pub declared_type: String, + pub nullable: bool, +} + +/// One table or view of the store. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DescribedRelation { + pub name: String, + /// `table` or `view`. + pub kind: String, + pub columns: Vec, +} + +/// One invocation's stored output and how complete it is. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct OutputTranscript { + pub invocation_id: String, + /// The recorded bytes in order, with a note line where output past `max_output_bytes` was left out. + pub data: Vec, + pub status: Option, + pub missing_sequences: i64, + pub reported_drops: i64, + pub filtered: i64, + pub flush_complete: bool, + pub complete: bool, + pub gap_reason: Option, +} + +/// One stored output row, as the transcript needs it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct StoredChunk { + pub(crate) stream: String, + pub(crate) data: Vec, + pub(crate) omitted_bytes: Option, +} diff --git a/crates/sqlbuild-recall/src/store/tests.rs b/crates/sqlbuild-recall/src/store/tests.rs new file mode 100644 index 0000000000..5d10c71849 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/tests.rs @@ -0,0 +1,10 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_maintenance.rs"] +mod maintenance; +#[path = "tests/test_store.rs"] +mod store; +#[path = "tests/test_types.rs"] +mod test_types; +#[path = "tests/test_views.rs"] +mod views; diff --git a/crates/sqlbuild-recall/src/store/tests/helpers.rs b/crates/sqlbuild-recall/src/store/tests/helpers.rs new file mode 100644 index 0000000000..24ee0c53c8 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/tests/helpers.rs @@ -0,0 +1,317 @@ +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use tempfile::TempDir; + +use crate::records::main::event_object_row::event_object_row; +use crate::records::main::format_timestamp::format_timestamp; +use crate::records::main::output_row::output_row; +use crate::records::main::parse_event::parse_event; +use crate::records::main::parse_timestamp::parse_timestamp; +use crate::records::models::{ + ArtifactInput, DeliveryCounts, OutputChunk, OutputStream, RecallRow, RecallValue, RowEnvelope, +}; +use crate::store::errors::{StoreError, StoreErrorKind}; +use crate::store::main::ensure_sqlite_schema::ensure_sqlite_schema; +use crate::store::main::insert_rows::insert_rows; +use crate::store::main::maintain_sqlite_store::maintain_sqlite_store; +use crate::store::main::observe_store_file::observe_store_file; +use crate::store::main::open_sqlite_store::open_sqlite_store; +use crate::store::main::quarantine_store::quarantine_store; +use crate::store::main::query_store::query_store; +use crate::store::models::{ + MaintenanceOutcome, MaintenanceRequest, OpenMode, QuarantineOutcome, SqliteStore, + StoreObservation, +}; +use crate::store::tests::test_types::QuarantineRace; + +pub(crate) const NOW: &str = "2026-10-19T12:00:00.000000Z"; + +/// The envelope of invocation `invocation_id` started at `started_at`. +pub(crate) fn envelope(invocation_id: &str, started_at: &str) -> RowEnvelope { + RowEnvelope { + invocation_id: invocation_id.to_owned(), + invocation_started_at: started_at.to_owned(), + producer_version: "test".to_owned(), + } +} + +/// A store with the current schema in a fresh temporary project. +pub(crate) fn new_store() -> (TempDir, SqliteStore) { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let store = open_sqlite_store( + &directory.path().join("logs/recall.sqlite3"), + OpenMode::ReadWrite, + ) + .expect("the store opens"); + ensure_sqlite_schema(&store, &envelope("setup", NOW), NOW).expect("the schema can be ensured"); + (directory, store) +} + +/// Insert `rows` at `NOW`. +pub(crate) fn write(store: &SqliteStore, rows: &[RecallRow]) { + insert_rows(store, rows, NOW).expect("rows can be inserted"); +} + +/// One output chunk row. +pub(crate) fn chunk( + envelope: &RowEnvelope, + (sequence, stream, data): (u64, OutputStream, &[u8]), +) -> RecallRow { + output_row( + envelope, + &OutputChunk { + sequence, + stream, + occurred_at: NOW.to_owned(), + data: data.to_vec(), + byte_offset: 0, + dropped_before: 0, + omitted: None, + }, + ) +} + +/// One `x` stdout chunk per sequence. +pub(crate) fn chunks(envelope: &RowEnvelope, sequences: &[u64]) -> Vec { + sequences + .iter() + .map(|sequence| chunk(envelope, (*sequence, OutputStream::Stdout, b"x"))) + .collect() +} + +/// One lifecycle event row; `correlations` is spliced in as extra JSON members. +pub(crate) fn event( + envelope: &RowEnvelope, + (event_id, event_type, sequence): (&str, &str, i64), + correlations: &str, + payload: &str, +) -> RecallRow { + let json = format!( + r#"{{"event_id":"{event_id}","event_type":"{event_type}","schema_version":2, + "producer":"sqlbuild","producer_version":"1","occurred_at":"{NOW}", + "invocation_id":"{}","invocation_sequence":{sequence}{correlations},"payload":{payload}}}"#, + envelope.invocation_id + ); + parse_event(&json) + .and_then(|event| event_object_row(envelope, &event)) + .expect("the fixture is a valid lifecycle event") +} + +/// The result rows of `sql`. +pub(crate) fn rows(store: &SqliteStore, sql: &str) -> Vec> { + query_store(store, sql, None) + .expect("the query is valid") + .rows +} + +/// The rows `sql` projects after writing each batch in order to a fresh store. +pub(crate) fn projected(batches: &[Vec], sql: &str) -> Vec> { + let (_directory, store) = new_store(); + for batch in batches { + write(&store, batch); + } + rows(&store, sql) +} + +/// A text value. +pub(crate) fn text(value: &str) -> RecallValue { + RecallValue::Text(value.to_owned()) +} + +/// An integer value. +pub(crate) fn integer(value: i64) -> RecallValue { + RecallValue::Integer(value) +} + +fn moved(base: SystemTime, seconds: i64) -> String { + let base_seconds: i64 = base.duration_since(UNIX_EPOCH).map_or(0, |elapsed| { + i64::try_from(elapsed.as_secs()).unwrap_or(i64::MAX) + }); + let target: u64 = u64::try_from(base_seconds.saturating_add(seconds)).unwrap_or_default(); + format_timestamp(UNIX_EPOCH + Duration::from_secs(target)) +} + +/// `NOW` moved by `seconds`. +pub(crate) fn shifted(seconds: i64) -> String { + moved( + parse_timestamp(NOW).expect("the timestamp is valid"), + seconds, + ) +} + +/// The real clock moved by `seconds_from_now`. +pub(crate) fn wall_clock(seconds_from_now: i64) -> String { + moved(SystemTime::now(), seconds_from_now) +} + +/// Delivery counts with `filtered`, `dropped` and `delivered` rows. +pub(crate) fn delivered(filtered: u64, dropped: u64, delivered: u64) -> DeliveryCounts { + DeliveryCounts { + accepted: delivered + filtered + dropped, + delivered, + filtered, + dropped, + ..DeliveryCounts::default() + } +} + +/// A SQL artifact `digest` referenced by itself and seen at `seen_at`. +pub(crate) fn artifact(digest: &str, seen_at: &str) -> ArtifactInput { + ArtifactInput { + digest: digest.to_owned(), + kind: "sql".to_owned(), + media_type: "text/x-sql".to_owned(), + content: b"select 1".to_vec(), + seen_at: seen_at.to_owned(), + ref_id: digest.to_owned(), + role: "executed_sql".to_owned(), + statement_id: None, + resource_id: None, + } +} + +/// Outcomes of maintenance runs at `(offset seconds, budget seconds)` and the rows `sql` keeps. +pub(crate) fn maintained( + batch: &[RecallRow], + runs: &[(i64, u64)], + sql: &str, +) -> (Vec, Vec>) { + let (_directory, store) = new_store(); + write(&store, batch); + let outcomes: Vec = runs + .iter() + .map(|(offset, budget)| { + let now = shifted(*offset); + let request = MaintenanceRequest { + now: parse_timestamp(&now).expect("the timestamp is valid"), + retention_days: 30, + owner: "host:1".to_owned(), + budget: Duration::from_secs(*budget), + envelope: envelope("maintainer", &now), + }; + maintain_sqlite_store(&store, &request).expect("maintenance succeeds") + }) + .collect(); + (outcomes, rows(&store, sql)) +} + +/// `result` with the error reduced to its kind. +pub(crate) fn error_kind(result: Result) -> Result { + result.map_err(|error| error.kind) +} + +/// A store whose recorded schema version is newer, reopened and ensured by this version. +pub(crate) fn newer_schema_store() -> (TempDir, SqliteStore) { + let (directory, store) = new_store(); + write( + &store, + &[crate::records::main::meta_row::meta_row( + &envelope("newer", NOW), + ("schema", "schema_version", "99"), + NOW, + )], + ); + store + .connection + .execute_batch("DROP VIEW conflicts; CREATE VIEW conflicts AS SELECT 'newer' AS marker") + .expect("the fixture SQL runs"); + drop(store); + let path = directory.path().join("logs/recall.sqlite3"); + let reopened = open_sqlite_store(&path, OpenMode::ReadWrite).expect("the store opens"); + ensure_sqlite_schema(&reopened, &envelope("older", NOW), NOW) + .expect("the schema can be ensured"); + drop(reopened); + let read_only = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); + (directory, read_only) +} + +/// The error kind of opening `relative_path` read-write under a read-only directory. +#[cfg(unix)] +pub(crate) fn open_under_locked_directory(relative_path: &str) -> Result<(), StoreErrorKind> { + use std::os::unix::fs::PermissionsExt; + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let locked = directory.path().join("locked"); + std::fs::create_dir(&locked).expect("the test directory can be created"); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o555)) + .expect("test directory permissions can change"); + let opened = open_sqlite_store(&locked.join(relative_path), OpenMode::ReadWrite); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)) + .expect("test directory permissions can change"); + error_kind(opened.map(drop)) +} + +#[cfg(unix)] +fn corrupt_copies(directory: &std::path::Path) -> usize { + std::fs::read_dir(directory) + .expect("the directory lists") + .filter_map(Result::ok) + .filter(|entry| { + let name = entry.file_name().to_string_lossy().into_owned(); + name.contains(".corrupt-") && !name.ends_with("-wal") && !name.ends_with("-shm") + }) + .count() +} + +#[cfg(unix)] +/// Nobody else acts on the corrupt file. +pub(crate) fn nobody_else( + _path: &std::path::Path, + _observed: &StoreObservation, + _lock: &std::fs::File, +) { +} + +#[cfg(unix)] +/// Another process moves the corrupt file aside and starts a fresh store. +pub(crate) fn already_replaced( + path: &std::path::Path, + observed: &StoreObservation, + _lock: &std::fs::File, +) { + let give_up = std::time::Instant::now() + Duration::from_millis(100); + drop( + quarantine_store(path, observed, (give_up, SystemTime::now())) + .expect("the first move succeeds"), + ); + let fresh = open_sqlite_store(path, OpenMode::ReadWrite).expect("a fresh store opens"); + ensure_sqlite_schema(&fresh, &envelope("fresh", NOW), NOW).expect("the fresh store is set up"); +} + +#[cfg(unix)] +/// Another process holds the quarantine lock. +pub(crate) fn lock_held( + _path: &std::path::Path, + _observed: &StoreObservation, + lock: &std::fs::File, +) { + rustix::fs::flock(lock, rustix::fs::FlockOperation::LockExclusive).expect("the lock is taken"); +} + +#[cfg(unix)] +/// Observe a corrupt store, let `race` happen, then move it aside with the now possibly stale observation: whether it moved, how many copies were moved aside, and whether a SQLite store is at the path. +pub(crate) fn quarantine_after(race: QuarantineRace) -> (bool, usize, bool) { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("recall.sqlite3"); + std::fs::write(&path, b"this is not a database").expect("test files can be written"); + std::fs::write(directory.path().join("recall.sqlite3-wal"), b"stale") + .expect("test files can be written"); + let observed = observe_store_file(&path) + .expect("the file can be read") + .expect("the corrupt file is observed"); + let lock = std::fs::File::create(directory.path().join("recall.sqlite3.lock")) + .expect("the lock file can be created"); + race(&path, &observed, &lock); + let give_up = std::time::Instant::now() + Duration::from_millis(100); + let outcome = quarantine_store(&path, &observed, (give_up, SystemTime::now())) + .expect("the move does not fail"); + let live_store: bool = matches!( + observe_store_file(&path), + Ok(Some(current)) if !current.header_foreign + ) && open_sqlite_store(&path, OpenMode::ReadOnly).is_ok(); + drop(lock); + ( + matches!(outcome, QuarantineOutcome::Moved(_)), + corrupt_copies(directory.path()), + live_store, + ) +} diff --git a/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs b/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs new file mode 100644 index 0000000000..c94249656a --- /dev/null +++ b/crates/sqlbuild-recall/src/store/tests/test_maintenance.rs @@ -0,0 +1,92 @@ +use crate::records::main::artifact_rows::artifact_rows; +use crate::records::main::heartbeat_row::heartbeat_row; +use crate::records::main::invocation_row::invocation_row; +use crate::records::models::{InvocationContext, OutputStream, RecallRow}; +use crate::store::models::MaintenanceOutcome; +use crate::store::tests::helpers::{artifact, chunk, envelope, integer, maintained, shifted, text}; +use crate::store::tests::test_types::MaintenanceTestCase; + +const FORTY_DAYS: i64 = -40 * 86_400; + +#[test] +fn given_expired_recent_and_running_invocations_when_maintained_then_only_expired_rows_go() { + let old = envelope("old", &shifted(FORTY_DAYS)); + let recent = envelope("recent", &shifted(-86_400)); + let old_running = envelope("old_running", &shifted(FORTY_DAYS)); + let mut mixed: Vec = [&old, &recent, &old_running] + .into_iter() + .flat_map(|env| { + [ + invocation_row(env, &InvocationContext::default()), + chunk(env, (0, OutputStream::Stdout, b"x")), + ] + }) + .collect(); + mixed.push(heartbeat_row( + &old_running, + (&shifted(-10), &shifted(80)), + (None, None), + )); + mixed.extend(artifact_rows( + &old, + &artifact("sha256:old", &shifted(FORTY_DAYS)), + )); + mixed.extend(artifact_rows( + &recent, + &artifact("sha256:recent", &shifted(FORTY_DAYS)), + )); + let ran = + |deleted_invocations: u64, deleted_artifacts: u64, complete: bool| MaintenanceOutcome { + ran: true, + deleted_invocations, + deleted_artifacts, + complete, + }; + let test_cases = [ + MaintenanceTestCase { + description: "expired idle invocations go while recent and running ones stay", + batch: mixed.clone(), + runs: &[(0, 5)], + sql: "SELECT invocation_id FROM recall_output ORDER BY invocation_id", + expected_outcomes: vec![ran(1, 1, true)], + expected_rows: vec![vec![text("old_running")], vec![text("recent")]], + }, + MaintenanceTestCase { + description: "artifacts referenced only by expired invocations go", + batch: mixed.clone(), + runs: &[(0, 5)], + sql: "SELECT digest FROM recall_artifacts", + expected_outcomes: vec![ran(1, 1, true)], + expected_rows: vec![vec![text("sha256:recent")]], + }, + MaintenanceTestCase { + description: "a claim stops another run within a day, and an unfinished run that beat within retention is kept", + batch: mixed, + runs: &[(0, 5), (3600, 5), (90_000, 5)], + sql: "SELECT COUNT(*) FROM recall_invocations", + expected_outcomes: vec![ + ran(1, 1, true), + MaintenanceOutcome::default(), + ran(0, 0, true), + ], + expected_rows: vec![vec![integer(2)]], + }, + MaintenanceTestCase { + description: "a starved run leaves the next run overdue", + batch: vec![invocation_row(&old, &InvocationContext::default())], + runs: &[(0, 0), (60, 5)], + sql: "SELECT COUNT(*) FROM recall_invocations", + expected_outcomes: vec![ran(0, 0, false), ran(1, 0, true)], + expected_rows: vec![vec![integer(0)]], + }, + ]; + + for test_case in test_cases { + assert_eq!( + maintained(&test_case.batch, test_case.runs, test_case.sql), + (test_case.expected_outcomes, test_case.expected_rows), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/store/tests/test_store.rs b/crates/sqlbuild-recall/src/store/tests/test_store.rs new file mode 100644 index 0000000000..27f7320f06 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/tests/test_store.rs @@ -0,0 +1,275 @@ +use crate::records::main::invocation_row::invocation_row; +use crate::records::models::{InvocationContext, OutputStream, RecallValue}; +use crate::store::_helpers::truncation::truncation_note; +use crate::store::errors::StoreErrorKind; +use crate::store::main::describe_store::describe_store; +use crate::store::main::query_store::query_store; +use crate::store::main::read_output_transcript::read_output_transcript; +use crate::store::main::resolve_invocation::resolve_invocation; +use crate::store::tests::helpers::{ + NOW, chunk, envelope, error_kind, new_store, newer_schema_store, rows, shifted, write, +}; +#[cfg(unix)] +use crate::store::tests::helpers::{ + already_replaced, lock_held, nobody_else, open_under_locked_directory, quarantine_after, +}; +#[cfg(unix)] +use crate::store::tests::test_types::{LockedDirectoryTestCase, QuarantineTestCase}; +use crate::store::tests::test_types::{ + NewerSchemaTestCase, QueryTestCase, ResolveInvocationTestCase, TranscriptTestCase, + TruncationNoteTestCase, +}; + +#[test] +fn given_ids_when_resolved_then_prefixes_last_and_ambiguity_are_exact() { + let (_directory, store) = new_store(); + write( + &store, + &[ + invocation_row( + &envelope("abc123", &shifted(-20)), + &InvocationContext::default(), + ), + invocation_row( + &envelope("abd456", &shifted(-10)), + &InvocationContext::default(), + ), + ], + ); + let test_cases = [ + ResolveInvocationTestCase { + description: "a unique prefix resolves", + selector: "abc", + expected_invocation: Ok("abc123"), + }, + ResolveInvocationTestCase { + description: "last is the most recently started invocation", + selector: "last", + expected_invocation: Ok("abd456"), + }, + ResolveInvocationTestCase { + description: "a shared prefix is ambiguous", + selector: "ab", + expected_invocation: Err(StoreErrorKind::Ambiguous), + }, + ResolveInvocationTestCase { + description: "an unknown prefix is not found", + selector: "zz", + expected_invocation: Err(StoreErrorKind::NotFound), + }, + ]; + + for test_case in test_cases { + assert_eq!( + error_kind(resolve_invocation(&store, test_case.selector)), + test_case.expected_invocation.map(str::to_owned), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_queries_when_run_then_writes_are_rejected_and_truncation_reported() { + let (_directory, store) = new_store(); + let env = envelope("q", NOW); + for sequence in 0..3 { + write( + &store, + &[chunk(&env, (sequence, OutputStream::Stdout, b"line\n"))], + ); + } + let test_cases = [ + QueryTestCase { + description: "a limit truncates and is reported", + sql: "SELECT sequence FROM recall_output ORDER BY sequence", + limit: Some(2), + expected_shape: Ok((vec!["sequence".to_owned()], 2, true)), + }, + QueryTestCase { + description: "without a limit every row returns", + sql: "SELECT sequence FROM recall_output ORDER BY sequence", + limit: None, + expected_shape: Ok((vec!["sequence".to_owned()], 3, false)), + }, + QueryTestCase { + description: "writes are rejected", + sql: "DELETE FROM recall_output", + limit: None, + expected_shape: Err(StoreErrorKind::Query), + }, + QueryTestCase { + description: "invalid SQL is a query error", + sql: "SELEC nope", + limit: None, + expected_shape: Err(StoreErrorKind::Query), + }, + ]; + + for test_case in test_cases { + let shape = error_kind(query_store(&store, test_case.sql, test_case.limit)) + .map(|result| (result.columns, result.rows.len(), result.truncated)); + assert_eq!(shape, test_case.expected_shape, "{}", test_case.description); + } +} + +#[test] +fn given_output_without_delivery_facts_when_read_then_bytes_return_and_completeness_is_unproven() { + let (_directory, store) = new_store(); + let env = envelope("q", NOW); + for sequence in 0..3 { + write( + &store, + &[chunk(&env, (sequence, OutputStream::Stdout, b"line\n"))], + ); + } + let test_cases = [TranscriptTestCase { + description: "chunks concatenate in sequence order", + invocation_id: "q", + expected_transcript: Ok((b"line\nline\nline\n".to_vec(), false)), + }]; + + for test_case in test_cases { + let transcript = error_kind(read_output_transcript( + &store, + test_case.invocation_id, + None, + )) + .map(|transcript| (transcript.data, transcript.complete)); + assert_eq!( + transcript, test_case.expected_transcript, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_newer_schema_version_when_ensured_then_views_are_kept_and_read_only_opens_work() { + let (_directory, store) = newer_schema_store(); + let relations = describe_store(&store).expect("the store can be described"); + let data_nullable: Option = relations + .iter() + .find(|relation| relation.name == "recall_output") + .and_then(|relation| relation.columns.iter().find(|column| column.name == "data")) + .map(|column| column.nullable); + let test_cases = [ + NewerSchemaTestCase { + description: "a newer view definition is not replaced", + sql: "SELECT marker FROM conflicts", + expected_rows: vec![vec![RecallValue::Text("newer".to_owned())]], + expected_first_kind: "view", + expected_data_nullable: Some(false), + }, + NewerSchemaTestCase { + description: "the read-only store answers view queries", + sql: "SELECT COUNT(*) FROM invocations", + expected_rows: vec![vec![RecallValue::Integer(0)]], + expected_first_kind: "view", + expected_data_nullable: Some(false), + }, + ]; + + for test_case in test_cases { + assert_eq!( + ( + rows(&store, test_case.sql), + relations[0].kind.as_str(), + data_nullable + ), + ( + test_case.expected_rows, + test_case.expected_first_kind, + test_case.expected_data_nullable + ), + "{}", + test_case.description + ); + } +} + +#[cfg(unix)] +#[test] +fn given_a_read_only_directory_when_opened_then_the_error_is_a_permission_failure() { + let test_cases = [ + LockedDirectoryTestCase { + description: "a missing logs directory cannot be created", + relative_path: "logs/recall.sqlite3", + expected_kind: Err(StoreErrorKind::Permission), + }, + LockedDirectoryTestCase { + description: "the database file cannot be created", + relative_path: "recall.sqlite3", + expected_kind: Err(StoreErrorKind::Permission), + }, + ]; + + for test_case in test_cases { + assert_eq!( + open_under_locked_directory(test_case.relative_path), + test_case.expected_kind, + "{}", + test_case.description + ); + } +} + +#[cfg(unix)] +#[test] +fn given_a_stale_observation_when_quarantining_then_only_the_observed_file_is_moved() { + let test_cases = [ + QuarantineTestCase { + description: "the observed corrupt file is moved aside", + race: nobody_else, + expected_outcome: (true, 1, false), + }, + QuarantineTestCase { + description: "a store another process already replaced is kept", + race: already_replaced, + expected_outcome: (false, 1, true), + }, + QuarantineTestCase { + description: "a held quarantine lock moves nothing", + race: lock_held, + expected_outcome: (false, 0, false), + }, + ]; + + for test_case in test_cases { + assert_eq!( + quarantine_after(test_case.race), + test_case.expected_outcome, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_truncated_output_when_noted_then_sizes_read_naturally() { + let test_cases = [ + TruncationNoteTestCase { + description: "the default limit keeps equal halves", + omitted: 23_600_000, + kept: (1_048_576, 1_048_576), + expected_note: "\u{2026} 23.6 MB of stdout not recorded (kept the first and last \ + 1 MiB). Raise [recall] max_output_bytes to keep more.\n", + }, + TruncationNoteTestCase { + description: "an odd limit keeps one more byte at the start", + omitted: 999, + kept: (3, 2), + expected_note: "\u{2026} 999 bytes of stdout not recorded (kept the first 3 bytes \ + and the last 2 bytes). Raise [recall] max_output_bytes to keep more.\n", + }, + ]; + + for test_case in test_cases { + assert_eq!( + truncation_note("stdout", test_case.omitted, test_case.kept), + test_case.expected_note, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/store/tests/test_types.rs b/crates/sqlbuild-recall/src/store/tests/test_types.rs new file mode 100644 index 0000000000..dffbb076a4 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/tests/test_types.rs @@ -0,0 +1,69 @@ +use crate::records::models::{RecallRow, RecallValue}; +use crate::store::errors::StoreErrorKind; +use crate::store::models::{MaintenanceOutcome, StoreObservation}; + +pub(crate) struct ProjectionTestCase { + pub(crate) description: &'static str, + pub(crate) batches: Vec>, + pub(crate) sql: &'static str, + pub(crate) expected_rows: Vec>, +} + +pub(crate) struct ResolveInvocationTestCase { + pub(crate) description: &'static str, + pub(crate) selector: &'static str, + pub(crate) expected_invocation: Result<&'static str, StoreErrorKind>, +} + +pub(crate) struct QueryTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) limit: Option, + pub(crate) expected_shape: Result<(Vec, usize, bool), StoreErrorKind>, +} + +pub(crate) struct TranscriptTestCase { + pub(crate) description: &'static str, + pub(crate) invocation_id: &'static str, + pub(crate) expected_transcript: Result<(Vec, bool), StoreErrorKind>, +} + +pub(crate) struct NewerSchemaTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + pub(crate) expected_rows: Vec>, + pub(crate) expected_first_kind: &'static str, + pub(crate) expected_data_nullable: Option, +} + +pub(crate) struct LockedDirectoryTestCase { + pub(crate) description: &'static str, + pub(crate) relative_path: &'static str, + pub(crate) expected_kind: Result<(), StoreErrorKind>, +} + +pub(crate) struct MaintenanceTestCase { + pub(crate) description: &'static str, + pub(crate) batch: Vec, + pub(crate) runs: &'static [(i64, u64)], + pub(crate) sql: &'static str, + pub(crate) expected_outcomes: Vec, + pub(crate) expected_rows: Vec>, +} + +/// What another process does between observing a corrupt store and moving it aside. +pub(crate) type QuarantineRace = fn(&std::path::Path, &StoreObservation, &std::fs::File); + +pub(crate) struct QuarantineTestCase { + pub(crate) description: &'static str, + pub(crate) race: QuarantineRace, + /// Whether the move was made, copies moved aside, and whether a SQLite store is at the path. + pub(crate) expected_outcome: (bool, usize, bool), +} + +pub(crate) struct TruncationNoteTestCase { + pub(crate) description: &'static str, + pub(crate) omitted: u64, + pub(crate) kept: (u64, u64), + pub(crate) expected_note: &'static str, +} diff --git a/crates/sqlbuild-recall/src/store/tests/test_views.rs b/crates/sqlbuild-recall/src/store/tests/test_views.rs new file mode 100644 index 0000000000..6875dd5ab2 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/tests/test_views.rs @@ -0,0 +1,462 @@ +use crate::records::main::artifact_rows::artifact_rows; +use crate::records::main::delivery_row::delivery_row; +use crate::records::main::heartbeat_row::heartbeat_row; +use crate::records::main::invocation_row::invocation_row; +use crate::records::main::sha256_hex::sha256_hex; +use crate::records::models::{ArtifactInput, InvocationContext, OutputStream, RecallValue}; +use crate::store::tests::helpers::{ + NOW, artifact, chunk, chunks, delivered, envelope, event, integer, projected, text, wall_clock, +}; +use crate::store::tests::test_types::ProjectionTestCase; + +const BINARY_PIECES: [(OutputStream, &[u8]); 4] = [ + (OutputStream::Stdout, b"line one\n\x00\xff"), + (OutputStream::Stderr, b"warn \xe2\x9c\x93\n"), + (OutputStream::Stdout, b"\xc3\x28 tail"), + (OutputStream::Stderr, b"\x00end"), +]; + +const STATEMENT_SQL: &str = "select 1"; + +#[test] +fn given_output_with_and_without_gaps_when_projected_then_completeness_is_exact() { + let complete = envelope("complete", NOW); + let gap = envelope("gap", NOW); + let filtered = envelope("filtered", NOW); + let undelivered = envelope("no_delivery", NOW); + let mut output = chunks(&complete, &[0, 1, 2]); + output.extend(chunks(&gap, &[0, 2])); + output.extend(chunks(&filtered, &[0, 2])); + output.extend(chunks(&undelivered, &[0, 1, 2])); + let deliveries = vec![ + delivery_row(&complete, "output", (&delivered(0, 0, 3), true), NOW), + delivery_row(&gap, "output", (&delivered(0, 0, 2), true), NOW), + delivery_row(&filtered, "output", (&delivered(1, 0, 2), true), NOW), + ]; + let test_cases = [ProjectionTestCase { + description: "only gap-free delivered output is complete and gaps are explained", + batches: vec![output, deliveries], + sql: "SELECT invocation_id, chunks, missing_sequences, filtered, gap_reason, complete \ + FROM output_invocations ORDER BY invocation_id", + expected_rows: vec![ + vec![ + text("complete"), + integer(3), + integer(0), + integer(0), + RecallValue::Null, + integer(1), + ], + vec![ + text("filtered"), + integer(2), + integer(1), + integer(1), + text("filtered"), + integer(0), + ], + vec![ + text("gap"), + integer(2), + integer(1), + integer(0), + text("unexplained"), + integer(0), + ], + vec![ + text("no_delivery"), + integer(3), + integer(0), + integer(0), + RecallValue::Null, + integer(0), + ], + ], + }]; + + for test_case in test_cases { + assert_eq!( + projected(&test_case.batches, test_case.sql), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_binary_interleaved_output_when_reconstructed_then_transcripts_are_byte_exact() { + let env = envelope("binary", NOW); + let mut reversed: Vec> = (0_u64..) + .zip(BINARY_PIECES) + .map(|(sequence, (stream, data))| vec![chunk(&env, (sequence, stream, data))]) + .collect(); + reversed.reverse(); + let stdout: Vec = [BINARY_PIECES[0].1, BINARY_PIECES[2].1].concat(); + let stderr: Vec = [BINARY_PIECES[1].1, BINARY_PIECES[3].1].concat(); + let combined: Vec = BINARY_PIECES + .iter() + .flat_map(|(_, data)| data.iter().copied()) + .collect(); + let test_cases = [ + ProjectionTestCase { + description: "out-of-order writes reconstruct each stream and the combined order", + batches: reversed.clone(), + sql: "SELECT stdout, stderr, combined FROM output_transcripts \ + WHERE invocation_id = 'binary'", + expected_rows: vec![vec![ + RecallValue::Blob(stdout), + RecallValue::Blob(stderr), + RecallValue::Blob(combined), + ]], + }, + ProjectionTestCase { + description: "the tail lists every chunk", + batches: reversed, + sql: "SELECT COUNT(*) FROM output_tail WHERE invocation_id = 'binary'", + expected_rows: vec![vec![integer(4)]], + }, + ]; + + for test_case in test_cases { + assert_eq!( + projected(&test_case.batches, test_case.sql), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_duplicates_and_conflicts_when_written_then_first_row_is_kept_and_conflicts_reported() { + let env = envelope("dup", NOW); + let original = chunk(&env, (0, OutputStream::Stdout, b"first")); + let conflicting = chunk(&env, (0, OutputStream::Stdout, b"second")); + let writes = vec![ + vec![original.clone()], + vec![original.clone()], + vec![conflicting.clone()], + ]; + let test_cases = [ + ProjectionTestCase { + description: "the first row wins", + batches: writes.clone(), + sql: "SELECT data FROM recall_output", + expected_rows: vec![vec![RecallValue::Blob(b"first".to_vec())]], + }, + ProjectionTestCase { + description: "only the differing rewrite is a conflict", + batches: writes, + sql: "SELECT invocation_id, dataset, record_key FROM conflicts", + expected_rows: vec![vec![text("dup"), text("recall_output"), text("dup:0")]], + }, + ProjectionTestCase { + description: "identical rewrites are not conflicts", + batches: vec![vec![original.clone()], vec![original]], + sql: "SELECT COUNT(*) FROM conflicts", + expected_rows: vec![vec![integer(0)]], + }, + ]; + + for test_case in test_cases { + assert_eq!( + projected(&test_case.batches, test_case.sql), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_terminals_and_heartbeats_when_projected_then_status_never_claims_false_success() { + let done = envelope("done", NOW); + let failed = envelope("failed_late", NOW); + let running = envelope("running", NOW); + let abandoned = envelope("abandoned", "2026-10-01T12:00:00.000000Z"); + let unknown = envelope("unknown", NOW); + let facts = vec![ + event( + &done, + ("d0", "invocation_started", 0), + "", + r#"{"command":"build"}"#, + ), + event( + &done, + ("d1", "invocation_completed", 9), + "", + r#"{"command":"build","duration_ms":12.5,"exit_code":0}"#, + ), + event( + &failed, + ("f1", "invocation_completed", 3), + "", + r#"{"command":"build","exit_code":0}"#, + ), + event( + &failed, + ("f2", "invocation_failed", 4), + "", + r#"{"command":"build","exit_code":1}"#, + ), + event( + &unknown, + ("u0", "invocation_started", 0), + "", + r#"{"command":"run"}"#, + ), + invocation_row( + &running, + &InvocationContext { + command: Some("build".to_owned()), + target: Some("dev".to_owned()), + ..InvocationContext::default() + }, + ), + heartbeat_row( + &running, + (&wall_clock(-1), &wall_clock(90)), + (None, Some(1)), + ), + heartbeat_row( + &abandoned, + ("2026-10-01T12:00:30.000000Z", "2026-10-01T12:02:00.000000Z"), + (None, None), + ), + heartbeat_row( + &abandoned, + ("2026-10-01T12:00:00.000000Z", "2026-10-01T12:01:30.000000Z"), + (None, None), + ), + ]; + let test_cases = [ProjectionTestCase { + description: "failures outrank completions and stale heartbeats are abandoned", + batches: vec![facts], + sql: "SELECT invocation_id, status, ended_at, duration_ms, exit_code, command, target \ + FROM invocations ORDER BY invocation_id", + expected_rows: vec![ + vec![ + text("abandoned"), + text("abandoned"), + text("2026-10-01T12:00:30.000000Z"), + RecallValue::Real(30000.0), + RecallValue::Null, + RecallValue::Null, + RecallValue::Null, + ], + vec![ + text("done"), + text("completed"), + text(NOW), + RecallValue::Real(12.5), + integer(0), + text("build"), + RecallValue::Null, + ], + vec![ + text("failed_late"), + text("failed"), + text(NOW), + RecallValue::Null, + integer(1), + RecallValue::Null, + RecallValue::Null, + ], + vec![ + text("running"), + text("running"), + RecallValue::Null, + RecallValue::Null, + RecallValue::Null, + text("build"), + text("dev"), + ], + vec![ + text("unknown"), + text("unknown"), + RecallValue::Null, + RecallValue::Null, + RecallValue::Null, + text("run"), + RecallValue::Null, + ], + ], + }]; + + for test_case in test_cases { + assert_eq!( + projected(&test_case.batches, test_case.sql), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_statement_facts_when_projected_then_failures_and_missing_terminals_stay_visible() { + let env = envelope("stmts", NOW); + let digest = sha256_hex(STATEMENT_SQL.as_bytes()); + let payload = format!( + r#"{{"intent":"execute","statement_kind":"SELECT","purpose":"state_write","sql_digest":"{digest}"}}"# + ); + let failed_payload = format!( + r#"{{"intent":"execute","statement_kind":"SELECT","sql_digest":"{digest}","error_type":"DatabaseError","error_code":"42P01","query_id":"q9"}}"# + ); + let mut facts = vec![ + event( + &env, + ("r0", "resource_attempt_started", 0), + r#","run_id":"run","resource_id":"orders","resource_attempt_id":"a1""#, + r#"{"attempt_number":1,"resource_kind":"model","resource_name":"orders"}"#, + ), + event( + &env, + ("s0", "statement_started", 1), + r#","statement_id":"s1","resource_id":"orders""#, + &payload, + ), + event( + &env, + ("s1", "statement_failed", 2), + r#","statement_id":"s1","resource_id":"orders""#, + &failed_payload, + ), + event( + &env, + ("s2", "statement_started", 3), + r#","statement_id":"s2""#, + r#"{"intent":"query","statement_kind":"SELECT"}"#, + ), + ]; + facts.extend(artifact_rows( + &env, + &ArtifactInput { + digest: format!("sha256:{digest}"), + kind: "sql".to_owned(), + media_type: "text/x-sql".to_owned(), + content: STATEMENT_SQL.as_bytes().to_vec(), + seen_at: NOW.to_owned(), + ref_id: "s1".to_owned(), + role: "executed_sql".to_owned(), + statement_id: Some("s1".to_owned()), + resource_id: Some("orders".to_owned()), + }, + )); + let test_cases = [ + ProjectionTestCase { + description: "statements join resources, terminals and their SQL", + batches: vec![facts.clone()], + sql: "SELECT statement_id, resource_name, purpose, outcome, error_code, query_id, \ + sql_available FROM statements ORDER BY statement_id", + expected_rows: vec![ + vec![ + text("s1"), + text("orders"), + text("state_write"), + text("failed"), + text("42P01"), + text("q9"), + integer(1), + ], + vec![ + text("s2"), + RecallValue::Null, + RecallValue::Null, + text("no_terminal"), + RecallValue::Null, + RecallValue::Null, + integer(0), + ], + ], + }, + ProjectionTestCase { + description: "artifacts decode their SQL text", + batches: vec![facts], + sql: "SELECT available, text FROM artifacts", + expected_rows: vec![vec![integer(1), text(STATEMENT_SQL)]], + }, + ]; + + for test_case in test_cases { + assert_eq!( + projected(&test_case.batches, test_case.sql), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_audit_completed_facts_when_projected_then_audit_results_carry_outcome_and_target() { + let env = envelope("audits", NOW); + let test_cases = [ProjectionTestCase { + description: "audit outcomes name the audited relation and column", + batches: vec![vec![event( + &env, + ("a0", "audit_completed", 1), + r#","run_id":"run""#, + r#"{"audit_name":"revenue_positive","audit_definition_name":"positive","evaluation_mode":"violations","outcome":"warn","severity":"warn","run_scope_phase":"model","attachment_kind":"column","target_schema":"analytics","target_name":"daily_revenue","attached_column_name":"total_revenue_cents","violation_count":3,"sql_digest":"ab"}"#, + )]], + sql: "SELECT audit_name, target_relation, \"column\", outcome, violation_count \ + FROM audit_results", + expected_rows: vec![vec![ + text("revenue_positive"), + text("analytics.daily_revenue"), + text("total_revenue_cents"), + text("warn"), + integer(3), + ]], + }]; + + for test_case in test_cases { + assert_eq!( + projected(&test_case.batches, test_case.sql), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_the_same_sql_in_two_invocations_when_written_then_artifacts_are_shared_without_conflicts() +{ + let first = envelope("first", NOW); + let second = envelope("second", &wall_clock(60)); + let seen_twice = || { + let mut rows = artifact_rows(&first, &artifact("sha256:shared", NOW)).to_vec(); + rows.extend(artifact_rows( + &second, + &artifact("sha256:shared", &wall_clock(60)), + )); + vec![rows] + }; + let test_cases = [ + ProjectionTestCase { + description: "a repeated artifact is a duplicate, not a conflict", + batches: seen_twice(), + sql: "SELECT COUNT(*) FROM conflicts", + expected_rows: vec![vec![integer(0)]], + }, + ProjectionTestCase { + description: "one content row serves both invocations' references", + batches: seen_twice(), + sql: "SELECT (SELECT COUNT(*) FROM recall_artifacts), \ + (SELECT COUNT(*) FROM artifacts WHERE available = 1)", + expected_rows: vec![vec![integer(1), integer(2)]], + }, + ]; + + for test_case in test_cases { + assert_eq!( + projected(&test_case.batches, test_case.sql), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs new file mode 100644 index 0000000000..50bd700728 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs @@ -0,0 +1,318 @@ +//! The delivery thread: table setup, batched writes with retries, heartbeats, the final delivery rows and maintenance. It alone opens and uses the store connection. + +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; +use std::time::{Duration, Instant, SystemTime}; + +use crate::records::main::delivery_row::delivery_row; +use crate::records::main::format_timestamp::format_timestamp; +use crate::records::main::heartbeat_row::heartbeat_row; +use crate::records::models::RecallRow; +use crate::schema::models::Table; +use crate::store::constants::{BUSY_TIMEOUT_MS, MAINTENANCE_BUDGET_MS}; +use crate::store::errors::{StoreError, StoreErrorKind}; +use crate::store::main::checkpoint_sqlite_store::checkpoint_sqlite_store; +use crate::store::main::ensure_sqlite_schema::ensure_sqlite_schema; +use crate::store::main::insert_rows::insert_rows; +use crate::store::main::maintain_sqlite_store::maintain_sqlite_store; +use crate::store::main::observe_store_file::observe_store_file; +use crate::store::main::open_sqlite_store::open_sqlite_store; +use crate::store::main::quarantine_store::quarantine_store; +use crate::store::models::{ + InsertOutcome, MaintenanceRequest, OpenMode, QuarantineOutcome, SqliteStore, StoreObservation, +}; +use crate::writer::_helpers::materialize::{Built, Materializer}; +use crate::writer::_helpers::queue::{Queued, WriterState}; +use crate::writer::constants::{ + BATCH_ROWS, BATCH_WINDOW_MS, CLOCK_CHECK_MS, FINAL_ROWS_BUDGET_MS, HEARTBEAT_STALE_INTERVALS, + MAX_RETRY_BACKOFF_MS, MINIMUM_MAINTENANCE_MS, OUTPUT_COALESCE_MS, RUNNING_RETRY_LIMIT_MS, + SETUP_RETRY_MS, +}; +use crate::writer::models::{DeliveryStream, SessionConfig, SetupOutcome}; + +/// State shared by producers, the session handle and the delivery thread. +#[derive(Debug)] +pub(crate) struct Shared { + pub(crate) state: Mutex, + pub(crate) wake: Condvar, + pub(crate) setup: Mutex>, + pub(crate) setup_done: Condvar, +} + +pub(crate) fn locked(mutex: &Mutex) -> MutexGuard<'_, WriterState> { + mutex + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn publish_setup(shared: &Shared, outcome: SetupOutcome) { + let mut setup = shared + .setup + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *setup = Some(outcome); + shared.setup_done.notify_all(); +} + +/// When busy retries stop: the close deadline once closing, else the running retry limit. +fn retry_limit(shared: &Shared, started: Instant) -> Instant { + locked(&shared.state) + .deadline + .unwrap_or_else(|| started + Duration::from_millis(RUNNING_RETRY_LIMIT_MS)) +} + +/// Each SQLite busy wait ends by `give_up`, so no write outlasts the shutdown budget. +fn bound_busy_wait(store: &SqliteStore, give_up: Instant) { + let remaining: Duration = give_up + .saturating_duration_since(Instant::now()) + .min(Duration::from_millis(BUSY_TIMEOUT_MS)); + drop(store.connection.busy_timeout(remaining)); +} + +fn open_and_setup(shared: &Shared, config: &SessionConfig) -> Result { + let started = Instant::now(); + let mut quarantined: bool = false; + loop { + let observed: Option = + observe_store_file(&config.store_path).unwrap_or(None); + let attempt = + open_sqlite_store(&config.store_path, OpenMode::ReadWrite).and_then(|store| { + bound_busy_wait(&store, retry_limit(shared, started)); + let now = format_timestamp(SystemTime::now()); + ensure_sqlite_schema(&store, &config.envelope, &now).map(|()| store) + }); + match attempt { + Err(error) + if error.kind == StoreErrorKind::Busy + && Instant::now() < retry_limit(shared, started) => + { + std::thread::sleep(Duration::from_millis(SETUP_RETRY_MS)); + } + Err(error) if error.kind == StoreErrorKind::Corrupt && !quarantined => { + quarantined = true; + let Some(observed) = observed else { + return Err(error); + }; + let outcome = quarantine_store( + &config.store_path, + &observed, + (retry_limit(shared, started), SystemTime::now()), + )?; + match outcome { + QuarantineOutcome::Moved(moved) => { + locked(&shared.state).notices.push(format!( + "the Recall store {} was not a valid database; it was moved to {} \ + and a fresh store was started", + config.store_path.display(), + moved.display() + )); + } + QuarantineOutcome::Kept => {} + QuarantineOutcome::LockBusy => return Err(error), + } + } + other => return other, + } + } +} + +/// Write one batch, retrying a busy store with backoff until `give_up`. +fn write_batch( + store: &SqliteStore, + rows: &[RecallRow], + give_up: Instant, +) -> Result { + let mut backoff = Duration::from_millis(10); + loop { + bound_busy_wait(store, give_up); + let now = format_timestamp(SystemTime::now()); + match insert_rows(store, rows, &now) { + Err(error) if error.kind == StoreErrorKind::Busy && Instant::now() < give_up => { + std::thread::sleep(backoff.min(give_up.saturating_duration_since(Instant::now()))); + backoff = (backoff * 2).min(Duration::from_millis(MAX_RETRY_BACKOFF_MS)); + } + other => return other, + } + } +} + +fn heartbeat(config: &SessionConfig, now: SystemTime) -> RecallRow { + let stale = now + config.heartbeat_interval * HEARTBEAT_STALE_INTERVALS; + heartbeat_row( + &config.envelope, + (&format_timestamp(now), &format_timestamp(stale)), + (config.context.host_token.as_deref(), config.context.pid), + ) +} + +pub(crate) fn stream_of(table: Table) -> DeliveryStream { + match table { + Table::Output => DeliveryStream::Output, + Table::Events => DeliveryStream::Events, + Table::Diagnostics => DeliveryStream::Diagnostics, + Table::Artifacts | Table::ArtifactRefs => DeliveryStream::Artifacts, + _ => DeliveryStream::Invocation, + } +} + +/// Wait until a batch is due: 500 rows, the batch window, closing, or a timed duty. +fn next_batch(shared: &Shared, next_heartbeat: SystemTime) -> (Vec, bool) { + let mut state = locked(&shared.state); + let mut first_seen: Option = None; + loop { + let now = Instant::now(); + if state + .output_due(Duration::from_millis(OUTPUT_COALESCE_MS)) + .is_some_and(|due| due <= now) + || (state.closing && state.has_pending_output()) + { + state.flush_output(); + } + if state.queued() > 0 && first_seen.is_none() { + first_seen = Some(now); + } + let window_over = first_seen + .is_some_and(|seen| now.duration_since(seen) >= Duration::from_millis(BATCH_WINDOW_MS)); + if state.queued() >= BATCH_ROWS || (state.queued() > 0 && (window_over || state.closing)) { + return (state.take_batch(BATCH_ROWS), state.closing); + } + if state.closing && state.queued() == 0 && !state.has_pending_output() { + return (Vec::new(), true); + } + let until_heartbeat: Duration = next_heartbeat + .duration_since(SystemTime::now()) + .unwrap_or(Duration::ZERO); + if until_heartbeat.is_zero() { + return (Vec::new(), state.closing); + } + let mut wait_until = now + until_heartbeat.min(Duration::from_millis(CLOCK_CHECK_MS)); + if let Some(due) = state.output_due(Duration::from_millis(OUTPUT_COALESCE_MS)) { + wait_until = wait_until.min(due); + } + if let Some(seen) = first_seen { + wait_until = wait_until.min(seen + Duration::from_millis(BATCH_WINDOW_MS)); + } + let timeout = wait_until + .saturating_duration_since(now) + .max(Duration::from_millis(1)); + state = shared + .wake + .wait_timeout(state, timeout) + .unwrap_or_else(std::sync::PoisonError::into_inner) + .0; + } +} + +fn degrade_and_drain(shared: &Shared, reason: String) { + let mut state = locked(&shared.state); + state.degraded.get_or_insert(reason); + state.rejecting = true; + state.drop_all(); + state.finished = true; + shared.wake.notify_all(); +} + +fn final_rows(config: &SessionConfig, state: &WriterState) -> Vec { + let now = format_timestamp(SystemTime::now()); + let complete = state.degraded.is_none(); + DeliveryStream::ALL + .iter() + .map(|stream| { + let counts = state.counts[stream.index()]; + let flushed = complete && counts.dropped == 0 && counts.failed == 0; + delivery_row(&config.envelope, stream.as_str(), (&counts, flushed), &now) + }) + .collect() +} + +/// The delivery thread body. +pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { + let store = match open_and_setup(shared, config) { + Ok(store) => { + publish_setup(shared, SetupOutcome::Ready); + store + } + Err(error) => { + let outcome = if error.kind == StoreErrorKind::Permission { + SetupOutcome::PermissionDenied { + reason: error.message.clone(), + } + } else { + SetupOutcome::Failed { + reason: error.message.clone(), + } + }; + publish_setup(shared, outcome); + degrade_and_drain(shared, error.message); + return; + } + }; + drop(checkpoint_sqlite_store(&store)); + let mut next_heartbeat: SystemTime = SystemTime::now(); + let mut materializer = Materializer::new(config.envelope.clone()); + loop { + let (batch, closing) = next_batch(shared, next_heartbeat); + let built: Built = materializer.build(batch); + if !built.settled.is_empty() || !built.unbuildable.is_empty() { + let mut state = locked(&shared.state); + state.account(&built.settled, Ok(&InsertOutcome::default())); + state.account_unbuildable(&built.unbuildable); + } + let mut rows: Vec = built.rows; + let now: SystemTime = SystemTime::now(); + if now >= next_heartbeat && !closing { + rows.push(heartbeat(config, now)); + next_heartbeat = now + config.heartbeat_interval; + } + if !rows.is_empty() { + let give_up = { + let state = locked(&shared.state); + state.deadline.unwrap_or_else(|| { + Instant::now() + Duration::from_millis(RUNNING_RETRY_LIMIT_MS) + }) + }; + let written = write_batch(&store, &rows, give_up); + let mut state = locked(&shared.state); + match &written { + Ok(outcome) => state.account(&built.written, Ok(outcome)), + Err(error) => state.account(&built.written, Err(&error.message)), + } + } + let mut state = locked(&shared.state); + let expired = state + .deadline + .is_some_and(|deadline| Instant::now() >= deadline); + if expired { + state.flush_output(); + state.drop_all(); + } + if (closing || expired) && state.queued() == 0 && !state.has_pending_output() { + state.rejecting = true; + let rows = final_rows(config, &state); + drop(state); + let give_up = (Instant::now() + Duration::from_millis(FINAL_ROWS_BUDGET_MS)) + .min(retry_limit(shared, Instant::now())); + if let Err(error) = write_batch(&store, &rows, give_up) { + locked(&shared.state).degraded.get_or_insert(error.message); + } + break; + } + } + let remaining: Duration = retry_limit(shared, Instant::now()) + .saturating_duration_since(Instant::now()) + .min(Duration::from_millis(MAINTENANCE_BUDGET_MS)); + if config.maintenance && remaining >= Duration::from_millis(MINIMUM_MAINTENANCE_MS) { + bound_busy_wait(&store, Instant::now() + remaining); + let request = MaintenanceRequest { + now: SystemTime::now(), + retention_days: config.retention_days, + owner: config.maintenance_owner.clone(), + budget: remaining, + envelope: config.envelope.clone(), + }; + drop(maintain_sqlite_store(&store, &request)); + } + drop(store); + let mut state = locked(&shared.state); + state.finished = true; + shared.wake.notify_all(); +} diff --git a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs new file mode 100644 index 0000000000..7038867b34 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs @@ -0,0 +1,93 @@ +//! Rows of queued input, built on the delivery thread so producers only enqueue. + +use std::collections::HashSet; + +use crate::records::constants::{ + ARTIFACT_DIGEST_PREFIX, EXECUTED_SQL_ROLE, SQL_ARTIFACT_KIND, SQL_MEDIA_TYPE, +}; +use crate::records::main::artifact_rows::artifact_rows; +use crate::records::main::event_object_row::event_object_row; +use crate::records::main::format_timestamp::format_timestamp; +use crate::records::main::sha256_hex::sha256_hex; +use crate::records::models::{ArtifactInput, RecallRow, RowEnvelope}; +use crate::writer::_helpers::queue::{Payload, Queued, SqlInput}; +use crate::writer::models::DeliveryStream; + +/// A batch ready to write: its rows, the streams of the records they came from, and records already settled. +#[derive(Debug, Default)] +pub(crate) struct Built { + pub(crate) rows: Vec, + /// One entry per record the rows were built from, for delivery accounting. + pub(crate) written: Vec, + /// Records that are already stored, such as SQL this invocation recorded before. + pub(crate) settled: Vec, + /// Records whose rows could not be built. + pub(crate) unbuildable: Vec, +} + +/// Builds rows for one session; remembers which SQL it has stored. +#[derive(Debug)] +pub(crate) struct Materializer { + envelope: RowEnvelope, + artifacts_seen: HashSet, +} + +impl Materializer { + pub(crate) fn new(envelope: RowEnvelope) -> Self { + Self { + envelope, + artifacts_seen: HashSet::new(), + } + } + + fn sql_rows(&mut self, input: SqlInput) -> Option> { + let digest = format!( + "{ARTIFACT_DIGEST_PREFIX}{}", + sha256_hex(input.sql.as_bytes()) + ); + if !self.artifacts_seen.insert(digest.clone()) { + return None; + } + let artifact = ArtifactInput { + ref_id: input.statement_id.clone().unwrap_or_else(|| digest.clone()), + digest, + kind: SQL_ARTIFACT_KIND.to_owned(), + media_type: SQL_MEDIA_TYPE.to_owned(), + content: input.sql.into_bytes(), + seen_at: format_timestamp(input.seen_at), + role: EXECUTED_SQL_ROLE.to_owned(), + statement_id: input.statement_id, + resource_id: input.resource_id, + }; + Some(artifact_rows(&self.envelope, &artifact).to_vec()) + } + + /// The rows of `batch`, with each record's stream sorted by how it is accounted. + pub(crate) fn build(&mut self, batch: Vec) -> Built { + let mut built = Built::default(); + for record in batch { + let stream: DeliveryStream = record.stream; + match record.payload { + Payload::Rows(rows) => { + built.rows.extend(rows); + built.written.push(stream); + } + Payload::Event(event) => match event_object_row(&self.envelope, &event) { + Ok(row) => { + built.rows.push(row); + built.written.push(stream); + } + Err(_) => built.unbuildable.push(stream), + }, + Payload::Sql(input) => match self.sql_rows(input) { + Some(rows) => { + built.rows.extend(rows); + built.written.push(stream); + } + None => built.settled.push(stream), + }, + } + } + built + } +} diff --git a/crates/sqlbuild-recall/src/writer/_helpers/mod.rs b/crates/sqlbuild-recall/src/writer/_helpers/mod.rs new file mode 100644 index 0000000000..2115f0b0b2 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod delivery; +pub(crate) mod materialize; +pub(crate) mod queue; +pub(crate) mod session; +pub(crate) mod tail; diff --git a/crates/sqlbuild-recall/src/writer/_helpers/queue.rs b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs new file mode 100644 index 0000000000..ed710a05bf --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/_helpers/queue.rs @@ -0,0 +1,457 @@ +//! The bounded priority queue and the output coalescer, guarded together by one mutex. + +use std::collections::VecDeque; +use std::time::{Duration, Instant, SystemTime}; + +use serde_json::{Map, Value}; + +use crate::records::constants::{DROPPED_BEFORE_COLUMN, MAX_OUTPUT_CHUNK_BYTES}; +use crate::records::main::format_timestamp::format_timestamp; +use crate::records::main::output_row::output_row; +use crate::records::models::{ + DeliveryCounts, OutputChunk, OutputStream, RecallRow, RecallValue, RowEnvelope, +}; +use crate::store::models::InsertOutcome; +use crate::writer::_helpers::delivery::stream_of; +use crate::writer::_helpers::tail::{TailRing, TailSegment}; +use crate::writer::constants::{QUEUE_BYTE_CAPACITY, QUEUE_RECORD_CAPACITY}; +use crate::writer::models::{DeliveryStream, Priority}; + +/// SQL a statement executed, hashed and turned into rows on the delivery thread. +#[derive(Debug)] +pub(crate) struct SqlInput { + pub(crate) sql: String, + pub(crate) statement_id: Option, + pub(crate) resource_id: Option, + pub(crate) seen_at: SystemTime, +} + +/// What a queued record holds: finished rows, or input the delivery thread turns into rows. +#[derive(Debug)] +pub(crate) enum Payload { + Rows(Vec), + Event(Map), + Sql(SqlInput), +} + +/// One queued record: what is written together and where it is accounted. +#[derive(Debug)] +pub(crate) struct Queued { + pub(crate) payload: Payload, + pub(crate) stream: DeliveryStream, + pub(crate) bytes: usize, +} + +impl Queued { + /// The finished rows; empty while the delivery thread has yet to build them. + pub(crate) fn rows(&self) -> &[RecallRow] { + match &self.payload { + Payload::Rows(rows) => rows, + Payload::Event(_) | Payload::Sql(_) => &[], + } + } + + fn row_count(&self) -> usize { + match &self.payload { + Payload::Rows(rows) => rows.len(), + Payload::Event(_) => 1, + Payload::Sql(_) => 2, + } + } +} + +/// Output accepted but not yet chunked: contiguous writes to one stream. +#[derive(Debug)] +struct Pending { + stream: OutputStream, + data: Vec, + since: Instant, + occurred_at: String, +} + +/// Everything producers and the delivery thread share. +#[derive(Debug)] +pub(crate) struct WriterState { + pub(crate) envelope: RowEnvelope, + buckets: [VecDeque; Priority::COUNT], + records: usize, + bytes: usize, + pub(crate) counts: [DeliveryCounts; 5], + pending: Option, + next_sequence: u64, + offsets: [u64; 2], + /// Bytes of each stream recorded from its start, pending or emitted. + captured: [usize; 2], + /// Bytes of each stream recorded from its start; `None` records everything. + head_limit: Option, + /// The end of each stream past its head limit, kept until the session closes. + tails: [TailRing; 2], + /// The stream whose newest tail segment holds the latest sequence, so it may grow. + tail_open: Option, + unreported_output_drops: u64, + pub(crate) diagnostic_sequence: u64, + pub(crate) closing: bool, + pub(crate) deadline: Option, + pub(crate) finished: bool, + pub(crate) rejecting: bool, + pub(crate) degraded: Option, + /// Store notices the user should see, such as a corrupt store moved aside. + pub(crate) notices: Vec, + capacity: (usize, usize), +} + +fn row_bytes(rows: &[RecallRow]) -> usize { + rows.iter() + .flat_map(|row| row.values.iter()) + .map(|(name, value)| { + name.len() + + match value { + RecallValue::Text(text) => text.len(), + RecallValue::Blob(data) => data.len(), + _ => 8, + } + }) + .sum() +} + +impl WriterState { + pub(crate) fn new(envelope: RowEnvelope) -> Self { + Self::with_capacity(envelope, (QUEUE_RECORD_CAPACITY, QUEUE_BYTE_CAPACITY)) + } + + pub(crate) fn with_capacity(envelope: RowEnvelope, capacity: (usize, usize)) -> Self { + Self { + envelope, + buckets: Default::default(), + records: 0, + bytes: 0, + counts: [DeliveryCounts::default(); 5], + pending: None, + next_sequence: 0, + offsets: [0; 2], + captured: [0; 2], + head_limit: None, + tails: Default::default(), + tail_open: None, + unreported_output_drops: 0, + diagnostic_sequence: 0, + closing: false, + deadline: None, + finished: false, + rejecting: false, + degraded: None, + notices: Vec::new(), + capacity, + } + } + + /// The same state recording at most `limit` bytes of each output stream: the first half and the last half; `None` records everything. + pub(crate) fn with_output_limit(mut self, limit: Option) -> Self { + let Some(limit) = limit.map(|limit| usize::try_from(limit).unwrap_or(usize::MAX)) else { + self.head_limit = None; + return self; + }; + let tail: usize = limit / 2; + self.head_limit = Some(limit - tail); + self.tails = [TailRing::new(tail), TailRing::new(tail)]; + self + } + + pub(crate) fn counts_mut(&mut self, stream: DeliveryStream) -> &mut DeliveryCounts { + &mut self.counts[stream.index()] + } + + /// Count `record` as dropped; a dropped output chunk hands its own earlier drops to the next chunk, so each drop is reported exactly once. + fn drop_record(&mut self, record: &Queued) { + self.counts_mut(record.stream).dropped += 1; + if record.stream == DeliveryStream::Output { + let carried: u64 = record + .rows() + .iter() + .map(|row| match row.value(DROPPED_BEFORE_COLUMN) { + Some(RecallValue::Integer(count)) => u64::try_from(*count).unwrap_or_default(), + _ => 0, + }) + .sum(); + self.unreported_output_drops += 1 + carried; + } + } + + fn full(&self, bytes: usize) -> bool { + self.records >= self.capacity.0 || self.bytes + bytes > self.capacity.1 + } + + /// Queue `rows`, displacing lower-priority records when full; counts acceptance and drops. + pub(crate) fn push( + &mut self, + rows: Vec, + stream: DeliveryStream, + priority: Priority, + ) { + let bytes: usize = row_bytes(&rows); + self.push_record( + Queued { + payload: Payload::Rows(rows), + stream, + bytes, + }, + priority, + ); + } + + /// Queue one record of any payload, displacing lower-priority records when full. + pub(crate) fn push_record(&mut self, record: Queued, priority: Priority) { + self.counts_mut(record.stream).accepted += 1; + if self.rejecting { + self.drop_record(&record); + return; + } + while self.full(record.bytes) { + let Some(lowest) = self + .buckets + .iter() + .position(|bucket| !bucket.is_empty()) + .filter(|lowest| *lowest < priority as usize) + else { + self.drop_record(&record); + return; + }; + if let Some(displaced) = self.buckets[lowest].pop_front() { + self.records -= 1; + self.bytes -= displaced.bytes; + self.drop_record(&displaced); + } + } + self.records += 1; + self.bytes += record.bytes; + self.buckets[priority as usize].push_back(record); + } + + pub(crate) fn queued(&self) -> usize { + self.records + } + + /// Up to `limit` rows, highest priority first. + pub(crate) fn take_batch(&mut self, limit: usize) -> Vec { + let mut batch: Vec = Vec::new(); + let mut rows = 0; + for bucket in self.buckets.iter_mut().rev() { + while rows < limit { + let Some(record) = bucket.pop_front() else { + break; + }; + rows += record.row_count(); + self.records -= 1; + self.bytes -= record.bytes; + batch.push(record); + } + } + batch + } + + /// Drop everything queued, as when the store is unusable or the flush budget is spent. + pub(crate) fn drop_all(&mut self) { + let records: Vec = self.buckets.iter_mut().flat_map(std::mem::take).collect(); + for record in &records { + self.drop_record(record); + } + self.records = 0; + self.bytes = 0; + } + + fn push_chunk(&mut self, mut chunk: OutputChunk) { + chunk.dropped_before = std::mem::take(&mut self.unreported_output_drops); + let row = output_row(&self.envelope, &chunk); + self.push(vec![row], DeliveryStream::Output, Priority::Output); + } + + fn emit(&mut self, stream: OutputStream, data: Vec, occurred_at: &str) { + let index = usize::from(stream == OutputStream::Stderr); + let chunk = OutputChunk { + sequence: self.next_sequence, + stream, + occurred_at: occurred_at.to_owned(), + byte_offset: self.offsets[index], + dropped_before: 0, + data, + omitted: None, + }; + self.next_sequence += 1; + self.tail_open = None; + self.offsets[index] += chunk.data.len() as u64; + self.push_chunk(chunk); + } + + /// Accept `data` written to `stream`: a stream switch flushes the other stream first, so the shared sequence preserves interleaving; full chunks are emitted at once. Past the head limit, bytes go to the stream's tail ring. + pub(crate) fn append_output(&mut self, stream: OutputStream, data: &[u8], now: SystemTime) { + if data.is_empty() { + return; + } + if self + .pending + .as_ref() + .is_some_and(|pending| pending.stream != stream) + { + self.flush_output(); + } + let index = usize::from(stream == OutputStream::Stderr); + let Some(head_limit) = self.head_limit else { + self.record_output(stream, data, now); + return; + }; + let room: usize = head_limit.saturating_sub(self.captured[index]); + let (head, rest) = data.split_at(room.min(data.len())); + self.captured[index] += head.len(); + self.record_output(stream, head, now); + if !rest.is_empty() { + self.tail_output(stream, rest, now); + } + if self.closing { + self.flush_tails(); + } + } + + /// Keep the newest bytes of `stream` past its head limit in its ring, each run under a sequence reserved now so the global order holds; what the ring cannot hold is counted as omitted. + fn tail_output(&mut self, stream: OutputStream, data: &[u8], now: SystemTime) { + let index = usize::from(stream == OutputStream::Stderr); + let occurred_at: String = format_timestamp(now); + if !self.tails[index].active() { + self.flush_output(); + self.tails[index].activate( + self.next_sequence, + self.offsets[index], + occurred_at.clone(), + ); + self.next_sequence += 1; + self.tail_open = None; + } + let capacity: usize = self.tails[index].capacity(); + let skipped: usize = data.len().saturating_sub(capacity); + let mut rest: &[u8] = &data[skipped..]; + if skipped > 0 { + self.tails[index].evict_to(0); + self.tails[index].omit(skipped); + self.offsets[index] += skipped as u64; + self.tail_open = None; + } + self.tails[index].evict_to(capacity - rest.len()); + while !rest.is_empty() { + let room: usize = if self.tail_open == Some(stream) { + self.tails[index].room(MAX_OUTPUT_CHUNK_BYTES) + } else { + 0 + }; + if room == 0 { + self.tails[index].open(TailSegment::new( + self.next_sequence, + self.offsets[index], + occurred_at.clone(), + )); + self.next_sequence += 1; + self.tail_open = Some(stream); + continue; + } + let (taken, after) = rest.split_at(room.min(rest.len())); + self.tails[index].extend(taken); + self.offsets[index] += taken.len() as u64; + rest = after; + } + } + + /// Queue each stream's gap and kept tail, as when the session closes. + pub(crate) fn flush_tails(&mut self) { + self.flush_output(); + for stream in [OutputStream::Stdout, OutputStream::Stderr] { + let index = usize::from(stream == OutputStream::Stderr); + let chunks: Vec = self.tails[index].take(stream); + for chunk in chunks { + self.push_chunk(chunk); + } + } + self.tail_open = None; + } + + /// Chunk `data` without copying it more than once: fill the pending chunk, emit full chunks straight from `data`, keep the remainder pending. + fn record_output(&mut self, stream: OutputStream, data: &[u8], now: SystemTime) { + if data.is_empty() { + return; + } + let mut rest: &[u8] = data; + if let Some(pending) = self.pending.as_mut() { + let room: usize = MAX_OUTPUT_CHUNK_BYTES.saturating_sub(pending.data.len()); + let (head, tail) = rest.split_at(room.min(rest.len())); + pending.data.extend_from_slice(head); + rest = tail; + if pending.data.len() >= MAX_OUTPUT_CHUNK_BYTES { + self.flush_output(); + } + } + if rest.is_empty() { + return; + } + let occurred_at: String = format_timestamp(now); + let (full, remainder) = rest.as_chunks::(); + for chunk in full { + self.emit(stream, chunk.to_vec(), &occurred_at); + } + if !remainder.is_empty() { + self.pending = Some(Pending { + stream, + data: remainder.to_vec(), + since: Instant::now(), + occurred_at, + }); + } + } + + /// Emit pending output as one chunk. + pub(crate) fn flush_output(&mut self) { + if let Some(pending) = self.pending.take() { + self.emit(pending.stream, pending.data, &pending.occurred_at); + } + } + + /// When pending output must be flushed: `None` with nothing pending. + pub(crate) fn output_due(&self, window: Duration) -> Option { + self.pending.as_ref().map(|pending| pending.since + window) + } + + /// Whether any output is pending. + pub(crate) fn has_pending_output(&self) -> bool { + self.pending.is_some() + } + + /// Count records the delivery thread could not turn into rows as failed. + pub(crate) fn account_unbuildable(&mut self, streams: &[DeliveryStream]) { + for stream in streams { + self.counts_mut(*stream).failed += 1; + } + } + + /// Count a written batch as delivered, less conflicts, or a failed one as failed. + pub(crate) fn account( + &mut self, + batch: &[DeliveryStream], + outcome: Result<&InsertOutcome, &str>, + ) { + match outcome { + Ok(outcome) => { + for stream in batch { + self.counts_mut(*stream).delivered += 1; + } + for conflict in &outcome.conflicts { + let stream = stream_of(conflict.table); + let counts = self.counts_mut(stream); + counts.conflicts += 1; + counts.delivered = counts.delivered.saturating_sub(1); + } + } + Err(reason) => { + for stream in batch { + self.counts_mut(*stream).failed += 1; + } + self.degraded.get_or_insert_with(|| reason.to_owned()); + } + } + } +} diff --git a/crates/sqlbuild-recall/src/writer/_helpers/session.rs b/crates/sqlbuild-recall/src/writer/_helpers/session.rs new file mode 100644 index 0000000000..f656bb31cc --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/_helpers/session.rs @@ -0,0 +1,259 @@ +//! The session handle: producers queue records; `close` flushes within the shutdown budget. + +use std::time::{Duration, Instant, SystemTime}; + +use serde_json::{Map, Value}; + +use crate::records::constants::SEVERITY_ORDER; +use crate::records::errors::RecordError; +use crate::records::main::diagnostic_input_from_json::diagnostic_input_from_json; +use crate::records::main::diagnostic_row::diagnostic_row; +use crate::records::main::event_severity::event_severity; +use crate::records::main::parse_event::parse_event; +use crate::records::models::OutputStream; +use crate::writer::_helpers::delivery::locked; +use crate::writer::_helpers::queue::{Payload, Queued, SqlInput}; +use crate::writer::constants::{JSON_SCALAR_BYTES, MAX_ARTIFACT_BYTES}; +use crate::writer::models::{ + DeliveryStream, DeliverySummary, Priority, RecallSession, SetupOutcome, +}; + +pub(crate) fn severity_rank(severity: &str) -> usize { + SEVERITY_ORDER + .iter() + .position(|candidate| *candidate == severity) + .unwrap_or(0) +} + +/// Roughly how many bytes `event` holds, for the queue's byte capacity. +fn json_bytes(event: &Map) -> usize { + event + .iter() + .map(|(key, value)| key.len() + value_bytes(value)) + .sum() +} + +fn value_bytes(value: &Value) -> usize { + match value { + Value::String(text) => text.len(), + Value::Array(items) => items.iter().map(value_bytes).sum(), + Value::Object(entries) => json_bytes(entries), + Value::Null | Value::Bool(_) | Value::Number(_) => JSON_SCALAR_BYTES, + } +} + +fn event_priority(severity: &str) -> Priority { + match severity { + "error" => Priority::FailedTerminal, + "info" => Priority::Terminal, + _ => Priority::Lifecycle, + } +} + +impl RecallSession { + /// Wait up to `budget` for table setup; `None` while it is still running. + pub fn wait_for_setup(&self, budget: Duration) -> Option { + let deadline = Instant::now() + budget; + let mut setup = self + .shared + .setup + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + while setup.is_none() { + let now = Instant::now(); + if now >= deadline { + return None; + } + setup = self + .shared + .setup_done + .wait_timeout(setup, deadline - now) + .unwrap_or_else(std::sync::PoisonError::into_inner) + .0; + } + setup.clone() + } + + /// The invocation this session records. + #[must_use] + pub fn invocation_id(&self) -> String { + locked(&self.shared.state).envelope.invocation_id.clone() + } + + /// Accept bytes a process stream accepted. + pub fn append_output(&self, stream: OutputStream, data: &[u8]) { + let mut state = locked(&self.shared.state); + state.append_output(stream, data, SystemTime::now()); + self.shared.wake.notify_one(); + } + + /// Queue one lifecycle event from its canonical JSON. + pub fn publish_event(&self, event_json: &str) -> Result<(), RecordError> { + self.publish_event_object(parse_event(event_json)?); + Ok(()) + } + + /// Queue one lifecycle event already decoded to its canonical JSON object; its row is built on the delivery thread. + pub fn publish_event_object(&self, event: Map) { + let priority: Priority = event_severity(&event).map_or(Priority::Lifecycle, event_priority); + let bytes: usize = json_bytes(&event); + let record = Queued { + payload: Payload::Event(event), + stream: DeliveryStream::Events, + bytes, + }; + let mut state = locked(&self.shared.state); + state.push_record(record, priority); + self.shared.wake.notify_one(); + } + + /// Queue one diagnostic from its canonical JSON, when at or above the minimum severity. + pub fn publish_diagnostic(&self, diagnostic_json: &str) -> Result<(), RecordError> { + let mut state = locked(&self.shared.state); + let sequence = state.diagnostic_sequence; + let diagnostic = diagnostic_input_from_json(diagnostic_json, sequence)?; + let rank = severity_rank(&diagnostic.severity); + if rank < self.min_severity { + return Ok(()); + } + state.diagnostic_sequence += 1; + let priority = if rank >= severity_rank("warning") { + Priority::Diagnostic + } else { + Priority::DebugDiagnostic + }; + let row = diagnostic_row(&state.envelope, &diagnostic); + state.push(vec![row], DeliveryStream::Diagnostics, priority); + self.shared.wake.notify_one(); + Ok(()) + } + + /// Queue the SQL a statement executed, once per digest per invocation. + pub fn publish_sql(&self, sql: &str, statement_id: Option<&str>, resource_id: Option<&str>) { + if sql.len() > MAX_ARTIFACT_BYTES { + let mut state = locked(&self.shared.state); + let counts = state.counts_mut(DeliveryStream::Artifacts); + counts.accepted += 1; + counts.dropped += 1; + return; + } + let record = Queued { + bytes: sql.len(), + payload: Payload::Sql(SqlInput { + sql: sql.to_owned(), + statement_id: statement_id.map(str::to_owned), + resource_id: resource_id.map(str::to_owned), + seen_at: SystemTime::now(), + }), + stream: DeliveryStream::Artifacts, + }; + let mut state = locked(&self.shared.state); + state.push_record(record, Priority::Artifact); + self.shared.wake.notify_one(); + } + + /// Count records a replacement sink filtered out of `stream`. + pub fn record_filtered(&self, stream: DeliveryStream, count: u64) { + let mut state = locked(&self.shared.state); + let counts = state.counts_mut(stream); + counts.accepted += count; + counts.filtered += count; + } + + /// Count records a replacement sink failed on in `stream`. + pub fn record_failed(&self, stream: DeliveryStream, count: u64) { + let mut state = locked(&self.shared.state); + let counts = state.counts_mut(stream); + counts.accepted += count; + counts.failed += count; + } + + /// The accounting so far. + #[must_use] + pub fn summary(&self) -> DeliverySummary { + let state = locked(&self.shared.state); + DeliverySummary { + streams: DeliveryStream::ALL + .iter() + .map(|stream| (*stream, state.counts[stream.index()])) + .collect(), + flush_complete: state.finished && state.queued() == 0 && state.degraded.is_none(), + degraded_reason: state.degraded.clone(), + } + } + + /// Start closing: flush pending output and give delivery the shutdown budget from now. + pub fn begin_close(&self) { + let mut state = locked(&self.shared.state); + if !state.closing { + state.closing = true; + state.deadline = Some(Instant::now() + self.shutdown_timeout); + state.flush_tails(); + } + self.shared.wake.notify_all(); + } + + /// Wait up to `slice` for delivery to finish; past the deadline, drop what is left and report done. + pub fn wait_closed(&self, slice: Duration) -> bool { + let mut state = locked(&self.shared.state); + let Some(deadline) = state.deadline else { + return state.finished; + }; + let until: Instant = (Instant::now() + slice).min(deadline); + while !state.finished { + let now = Instant::now(); + if now >= deadline { + state.rejecting = true; + state.drop_all(); + state.degraded.get_or_insert_with(|| { + "Recall delivery did not finish within the shutdown timeout".to_owned() + }); + return true; + } + if now >= until { + return false; + } + state = self + .shared + .wake + .wait_timeout(state, until - now) + .unwrap_or_else(std::sync::PoisonError::into_inner) + .0; + } + true + } + + /// Stop waiting for delivery at once, as on a second interrupt; what is queued is dropped. + pub fn abandon(&self) { + let mut state = locked(&self.shared.state); + state.rejecting = true; + state.drop_all(); + state + .degraded + .get_or_insert_with(|| "Recall delivery was interrupted".to_owned()); + self.shared.wake.notify_all(); + } + + /// Flush within the shutdown budget, a hard cap on the delay, and return the accounting; records still queued at the deadline are counted as dropped. + pub fn close(&self) -> DeliverySummary { + self.begin_close(); + while !self.wait_closed(self.shutdown_timeout) {} + if locked(&self.shared.state).finished { + let delivery = self + .delivery + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + if let Some(delivery) = delivery { + drop(delivery.join()); + } + } + self.summary() + } + + /// Store notices the user should see, such as a corrupt store moved aside. + #[must_use] + pub fn notices(&self) -> Vec { + locked(&self.shared.state).notices.clone() + } +} diff --git a/crates/sqlbuild-recall/src/writer/_helpers/tail.rs b/crates/sqlbuild-recall/src/writer/_helpers/tail.rs new file mode 100644 index 0000000000..b4fe8b5571 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/_helpers/tail.rs @@ -0,0 +1,139 @@ +//! The bounded ring holding the end of one output stream past its head limit until the session closes. + +use std::collections::VecDeque; + +use crate::records::models::{OmittedOutput, OutputChunk, OutputStream}; + +/// Bytes kept from the end of a stream, with the sequence reserved when they were written. +#[derive(Debug)] +pub(crate) struct TailSegment { + pub(crate) sequence: u64, + pub(crate) byte_offset: u64, + pub(crate) occurred_at: String, + pub(crate) data: Vec, + /// Bytes at the front already evicted. + start: usize, +} + +impl TailSegment { + pub(crate) fn new(sequence: u64, byte_offset: u64, occurred_at: String) -> Self { + Self { + sequence, + byte_offset, + occurred_at, + data: Vec::new(), + start: 0, + } + } + + fn kept(&self) -> usize { + self.data.len() - self.start + } +} + +/// One stream's tail: at most `capacity` bytes, the reserved gap sequence and what was left out. +#[derive(Debug, Default)] +pub(crate) struct TailRing { + capacity: usize, + segments: VecDeque, + bytes: usize, + /// The gap's sequence, stream offset and time, reserved when the stream passed its head limit. + marker: Option<(u64, u64, String)>, + omitted_bytes: u64, + evicted_sequences: u64, +} + +impl TailRing { + pub(crate) fn new(capacity: usize) -> Self { + Self { + capacity, + ..Self::default() + } + } + + pub(crate) fn capacity(&self) -> usize { + self.capacity + } + + /// Whether the stream has passed its head limit. + pub(crate) fn active(&self) -> bool { + self.marker.is_some() + } + + /// Reserve the gap's position, right after the head. + pub(crate) fn activate(&mut self, sequence: u64, byte_offset: u64, occurred_at: String) { + self.marker = Some((sequence, byte_offset, occurred_at)); + } + + /// Leave out `bytes` that never entered the ring. + pub(crate) fn omit(&mut self, bytes: usize) { + self.omitted_bytes += bytes as u64; + } + + /// Evict from the front until at most `limit` bytes remain. + pub(crate) fn evict_to(&mut self, limit: usize) { + while self.bytes > limit { + let Some(front) = self.segments.front_mut() else { + break; + }; + let removed: usize = (self.bytes - limit).min(front.kept()); + front.start += removed; + front.byte_offset += removed as u64; + self.bytes -= removed; + self.omitted_bytes += removed as u64; + if front.kept() == 0 { + drop(self.segments.pop_front()); + self.evicted_sequences += 1; + } + } + } + + /// Room left in the newest segment for contiguous bytes, below `chunk_bytes`. + pub(crate) fn room(&self, chunk_bytes: usize) -> usize { + self.segments + .back() + .map_or(0, |segment| chunk_bytes.saturating_sub(segment.data.len())) + } + + pub(crate) fn open(&mut self, segment: TailSegment) { + self.segments.push_back(segment); + } + + /// Append to the newest segment; the caller checked `room`. + pub(crate) fn extend(&mut self, data: &[u8]) { + if let Some(segment) = self.segments.back_mut() { + segment.data.extend_from_slice(data); + self.bytes += data.len(); + } + } + + /// The gap chunk and the kept tail chunks of `stream`, leaving the ring empty and inactive. + pub(crate) fn take(&mut self, stream: OutputStream) -> Vec { + let Some((sequence, byte_offset, occurred_at)) = self.marker.take() else { + return Vec::new(); + }; + let mut chunks: Vec = vec![OutputChunk { + sequence, + stream, + occurred_at, + data: Vec::new(), + byte_offset, + dropped_before: 0, + omitted: Some(OmittedOutput { + bytes: std::mem::take(&mut self.omitted_bytes), + sequences: std::mem::take(&mut self.evicted_sequences) + 1, + }), + }]; + chunks.extend(self.segments.drain(..).map(|segment| OutputChunk { + sequence: segment.sequence, + stream, + occurred_at: segment.occurred_at, + data: segment.data[segment.start..].to_vec(), + byte_offset: segment.byte_offset, + dropped_before: 0, + omitted: None, + })); + self.bytes = 0; + chunks + } +} diff --git a/crates/sqlbuild-recall/src/writer/constants.rs b/crates/sqlbuild-recall/src/writer/constants.rs new file mode 100644 index 0000000000..f38530229d --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/constants.rs @@ -0,0 +1,29 @@ +//! Queue bounds, batching and timing of the Recall writer. + +pub const QUEUE_RECORD_CAPACITY: usize = 10_000; +pub const QUEUE_BYTE_CAPACITY: usize = 32 * 1024 * 1024; +pub const BATCH_ROWS: usize = 500; +pub const BATCH_WINDOW_MS: u64 = 250; +/// Contiguous writes to one stream coalesce into one chunk for at most this long. +pub const OUTPUT_COALESCE_MS: u64 = 250; +pub const DEFAULT_HEARTBEAT_INTERVAL_MS: u64 = 30_000; +/// An invocation is abandoned once its last heartbeat is this many intervals old. +pub const HEARTBEAT_STALE_INTERVALS: u32 = 3; +/// Table setup may delay the command by at most this long. +pub const SETUP_BUDGET_MS: u64 = 2_000; +/// Longest single backoff between retries of a busy write. +pub const MAX_RETRY_BACKOFF_MS: u64 = 500; +/// A batch that stays busy this long while the command runs is counted as failed. +pub const RUNNING_RETRY_LIMIT_MS: u64 = 30_000; +/// Pause between setup attempts while another writer holds the store. +pub const SETUP_RETRY_MS: u64 = 50; +/// Longest wait between wall-clock checks, so a resumed process beats promptly. +pub const CLOCK_CHECK_MS: u64 = 1_000; +/// Budget for the final delivery rows, inside the shutdown budget. +pub const FINAL_ROWS_BUDGET_MS: u64 = 500; +/// Maintenance is skipped when less than this remains of the shutdown budget. +pub const MINIMUM_MAINTENANCE_MS: u64 = 200; +/// SQL longer than this is counted as dropped without hashing or copying it. +pub const MAX_ARTIFACT_BYTES: usize = 4 * 1024 * 1024; +/// What a JSON scalar counts toward the queue's byte capacity. +pub const JSON_SCALAR_BYTES: usize = 8; diff --git a/crates/sqlbuild-recall/src/writer/main/mod.rs b/crates/sqlbuild-recall/src/writer/main/mod.rs new file mode 100644 index 0000000000..777f9535f8 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/main/mod.rs @@ -0,0 +1 @@ +pub mod start_recall_session; diff --git a/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs b/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs new file mode 100644 index 0000000000..e388056c3c --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/main/start_recall_session.rs @@ -0,0 +1,52 @@ +//! `start_recall_session`: begin recording one invocation. + +use std::sync::{Arc, Condvar, Mutex}; + +use crate::records::main::invocation_row::invocation_row; +use crate::writer::_helpers::delivery::{Shared, deliver, locked}; +use crate::writer::_helpers::queue::WriterState; +use crate::writer::_helpers::session::severity_rank; +use crate::writer::models::{DeliveryStream, Priority, RecallSession, SessionConfig}; + +/// Start the delivery thread and queue the invocation's context row first. +#[must_use] +pub fn start_recall_session(config: SessionConfig) -> RecallSession { + let mut state = + WriterState::new(config.envelope.clone()).with_output_limit(config.max_output_bytes); + state.push( + vec![invocation_row(&config.envelope, &config.context)], + DeliveryStream::Invocation, + Priority::Context, + ); + let shared = Arc::new(Shared { + state: Mutex::new(state), + wake: Condvar::new(), + setup: Mutex::new(None), + setup_done: Condvar::new(), + }); + let session = RecallSession { + shared: Arc::clone(&shared), + shutdown_timeout: config.shutdown_timeout, + min_severity: severity_rank(&config.diagnostics_min_severity), + delivery: Mutex::new(None), + }; + let spawned = std::thread::Builder::new() + .name("sqlbuild-recall".to_owned()) + .spawn(move || deliver(&shared, &config)); + match spawned { + Ok(handle) => { + *session + .delivery + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(handle); + } + Err(error) => { + let mut state = locked(&session.shared.state); + state.degraded = Some(format!("cannot start the Recall delivery thread: {error}")); + state.rejecting = true; + state.drop_all(); + state.finished = true; + } + } + session +} diff --git a/crates/sqlbuild-recall/src/writer/mod.rs b/crates/sqlbuild-recall/src/writer/mod.rs new file mode 100644 index 0000000000..9a334c477b --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/mod.rs @@ -0,0 +1,9 @@ +//! The Recall writer: a bounded priority queue fed by producers and drained by one delivery thread that alone owns the store connection. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-recall/src/writer/models.rs b/crates/sqlbuild-recall/src/writer/models.rs new file mode 100644 index 0000000000..4b77260ae5 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/models.rs @@ -0,0 +1,135 @@ +//! Writer configuration, delivery streams, priorities and the delivery summary. + +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; +use std::time::Duration; + +use crate::records::models::{DeliveryCounts, InvocationContext, RowEnvelope}; +use crate::writer::_helpers::delivery::Shared; + +/// The datasets whose delivery is accounted separately. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum DeliveryStream { + Invocation, + Output, + Events, + Diagnostics, + Artifacts, +} + +impl DeliveryStream { + pub const ALL: [Self; 5] = [ + Self::Invocation, + Self::Output, + Self::Events, + Self::Diagnostics, + Self::Artifacts, + ]; + + #[must_use] + pub fn as_str(self) -> &'static str { + match self { + Self::Invocation => "invocation", + Self::Output => "output", + Self::Events => "events", + Self::Diagnostics => "diagnostics", + Self::Artifacts => "artifacts", + } + } + + #[must_use] + pub fn index(self) -> usize { + match self { + Self::Invocation => 0, + Self::Output => 1, + Self::Events => 2, + Self::Diagnostics => 3, + Self::Artifacts => 4, + } + } +} + +/// Queue priority, lowest first; a full queue displaces the oldest record of the lowest priority below the arriving one. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Priority { + DebugDiagnostic = 0, + Artifact = 1, + Output = 2, + Diagnostic = 3, + Lifecycle = 4, + Terminal = 5, + FailedTerminal = 6, + Context = 7, +} + +impl Priority { + pub const COUNT: usize = 8; +} + +/// What one writer session needs. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SessionConfig { + pub store_path: PathBuf, + pub envelope: RowEnvelope, + pub context: InvocationContext, + pub retention_days: u32, + pub shutdown_timeout: Duration, + pub diagnostics_min_severity: String, + /// Bytes of each output stream recorded, the first and last half; `None` records everything. + pub max_output_bytes: Option, + pub heartbeat_interval: Duration, + pub setup_budget: Duration, + /// Whether this session runs retention maintenance after delivery. + pub maintenance: bool, + pub maintenance_owner: String, +} + +/// How table setup went. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SetupOutcome { + Ready, + /// The store cannot be created or written for lack of permission. + PermissionDenied { + reason: String, + }, + Failed { + reason: String, + }, +} + +/// The delivery accounting of a closed session. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DeliverySummary { + pub streams: Vec<(DeliveryStream, DeliveryCounts)>, + pub flush_complete: bool, + pub degraded_reason: Option, +} + +impl DeliverySummary { + /// Every stream's counts added together. + #[must_use] + pub fn total(&self) -> DeliveryCounts { + self.streams + .iter() + .fold(DeliveryCounts::default(), |mut total, (_, counts)| { + total.accepted += counts.accepted; + total.delivered += counts.delivered; + total.filtered += counts.filtered; + total.dropped += counts.dropped; + total.failed += counts.failed; + total.conflicts += counts.conflicts; + total + }) + } +} + +/// A running Recall session. Producers only touch the queue; the delivery thread writes. +#[derive(Debug)] +pub struct RecallSession { + pub(crate) shared: Arc, + pub(crate) shutdown_timeout: Duration, + pub(crate) min_severity: usize, + /// The delivery thread, joined at close once it has finished so no thread outlives the session. + pub(crate) delivery: Mutex>>, +} diff --git a/crates/sqlbuild-recall/src/writer/tests.rs b/crates/sqlbuild-recall/src/writer/tests.rs new file mode 100644 index 0000000000..81c66254ff --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_queue.rs"] +mod queue; +#[path = "tests/test_session.rs"] +mod session; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs new file mode 100644 index 0000000000..eddddbd71d --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -0,0 +1,323 @@ +use std::path::PathBuf; +use std::time::{Duration, SystemTime}; + +use crate::records::main::invocation_row::invocation_row; +use crate::records::models::{ + DeliveryCounts, InvocationContext, OutputStream, RecallValue, RowEnvelope, +}; +use crate::store::main::open_sqlite_store::open_sqlite_store; +use crate::store::main::query_store::query_store; +use crate::store::main::read_output_transcript::read_output_transcript; +use crate::store::models::OpenMode; +use crate::writer::_helpers::queue::WriterState; +use crate::writer::main::start_recall_session::start_recall_session; +use crate::writer::models::{ + DeliveryStream, DeliverySummary, Priority, SessionConfig, SetupOutcome, +}; +use crate::writer::tests::test_types::RecordedSession; + +/// The envelope of invocation `invocation_id`. +pub(crate) fn envelope(invocation_id: &str) -> RowEnvelope { + RowEnvelope { + invocation_id: invocation_id.to_owned(), + invocation_started_at: "2026-10-19T00:00:00.000000Z".to_owned(), + producer_version: "test".to_owned(), + } +} + +/// Dropped counts per stream and the streams taken after pushing `pushes` into `capacity`. +pub(crate) fn pushed( + capacity: usize, + pushes: &[(DeliveryStream, Priority)], +) -> (Vec<(DeliveryStream, u64)>, Vec) { + let mut state = WriterState::with_capacity(envelope("inv"), (capacity, usize::MAX)); + for (stream, priority) in pushes { + state.push( + vec![invocation_row( + &envelope("inv"), + &InvocationContext::default(), + )], + *stream, + *priority, + ); + } + let mut streams: Vec = pushes.iter().map(|(stream, _)| *stream).collect(); + streams.dedup(); + let dropped: Vec<(DeliveryStream, u64)> = streams + .iter() + .map(|stream| (*stream, state.counts[stream.index()].dropped)) + .collect(); + let taken: Vec = state + .take_batch(usize::MAX) + .iter() + .map(|record| record.stream) + .collect(); + (dropped, taken) +} + +/// `(sequence, stream, byte_offset)` of each output chunk after `writes` and a flush. +pub(crate) fn chunked(writes: &[(OutputStream, Vec)]) -> Vec<[Option; 3]> { + let mut state = WriterState::new(envelope("inv")); + for (stream, data) in writes { + state.append_output(*stream, data, SystemTime::now()); + } + state.flush_output(); + state + .take_batch(usize::MAX) + .iter() + .flat_map(|record| record.rows().iter()) + .map(|row| ["sequence", "stream", "byte_offset"].map(|name| row.value(name).cloned())) + .collect() +} + +/// A session configuration writing invocation `invocation_id` to `path`. +pub(crate) fn config(path: PathBuf, invocation_id: &str) -> SessionConfig { + SessionConfig { + store_path: path, + envelope: envelope(invocation_id), + context: InvocationContext { + command: Some("build".to_owned()), + pid: Some(42), + ..InvocationContext::default() + }, + retention_days: 30, + shutdown_timeout: Duration::from_secs(5), + diagnostics_min_severity: "info".to_owned(), + max_output_bytes: None, + heartbeat_interval: Duration::from_secs(30), + setup_budget: Duration::from_secs(2), + maintenance: true, + maintenance_owner: "test:42".to_owned(), + } +} + +/// Run a session that writes `output`, one completed event, two diagnostics and one SQL twice. +pub(crate) fn recorded_session(output: &[(OutputStream, Vec)]) -> RecordedSession { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("logs/recall.sqlite3"); + let session = start_recall_session(config(path.clone(), "inv-1")); + let setup = session.wait_for_setup(Duration::from_secs(10)); + for (stream, data) in output { + session.append_output(*stream, data); + } + session + .publish_event( + r#"{"event_id":"e1","event_type":"invocation_completed","schema_version":2, + "producer":"sqlbuild","producer_version":"1","occurred_at":"2026-10-19T00:00:01.000000Z", + "invocation_id":"inv-1","invocation_sequence":1,"payload":{"command":"build","exit_code":0,"duration_ms":3.0}}"#, + ) + .expect("the event is valid"); + for severity in ["debug", "warning"] { + session + .publish_diagnostic(&format!( + r#"{{"occurred_at":"2026-10-19T00:00:00.500000Z","severity":"{severity}","logger":"l","source":"python","message":"m"}}"# + )) + .expect("the diagnostic is valid"); + } + session.publish_sql("select 1", Some("s1"), None); + session.publish_sql("select 1", Some("s2"), None); + let summary = session.close(); + let total = summary.total(); + let output_fully_delivered: bool = summary + .streams + .iter() + .find(|(stream, _)| *stream == DeliveryStream::Output) + .is_some_and(|(_, counts)| counts.delivered == counts.accepted); + let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); + let transcript = + read_output_transcript(&store, "inv-1", None).expect("the transcript can be read"); + let stdout = read_output_transcript(&store, "inv-1", Some(OutputStream::Stdout)) + .expect("the transcript can be read"); + let facts = query_store( + &store, + "SELECT (SELECT status FROM invocations), (SELECT COUNT(*) FROM recall_diagnostics), \ + (SELECT COUNT(*) FROM recall_artifacts), (SELECT COUNT(*) FROM recall_delivery), (SELECT command FROM recall_invocations)", + None, + ) + .expect("the query is valid"); + RecordedSession { + setup, + lost: (total.dropped, total.failed), + output_fully_delivered, + transcript: (transcript.data, transcript.complete), + stdout: stdout.data, + facts: facts.rows, + } +} + +/// Whether a store under a read-only directory reports permission at setup and degrades. +#[cfg(unix)] +pub(crate) fn unwritable_session() -> (bool, bool) { + use std::os::unix::fs::PermissionsExt; + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o555)) + .expect("test directory permissions can change"); + let session = start_recall_session(config( + directory.path().join("logs/recall.sqlite3"), + "inv-2", + )); + let setup = session.wait_for_setup(Duration::from_secs(10)); + session.append_output(OutputStream::Stdout, b"still printed\n"); + let summary = session.close(); + std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o755)) + .expect("test directory permissions can change"); + ( + matches!(setup, Some(SetupOutcome::PermissionDenied { .. })), + summary.degraded_reason.is_some(), + ) +} + +/// Seconds `close` took and whether delivery degraded, while another connection holds the write lock. +pub(crate) fn close_under_held_lock(shutdown: Duration) -> (f64, bool) { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("logs/recall.sqlite3"); + let warm = start_recall_session(config(path.clone(), "warm")); + drop(warm.close()); + let holder = rusqlite::Connection::open(&path).expect("the store opens"); + holder + .execute_batch("BEGIN EXCLUSIVE") + .expect("the write lock is taken"); + let session = start_recall_session(SessionConfig { + shutdown_timeout: shutdown, + ..config(path, "held") + }); + session.append_output(OutputStream::Stdout, b"waiting\n"); + let started = std::time::Instant::now(); + let summary = session.close(); + let elapsed = started.elapsed().as_secs_f64(); + drop(holder); + (elapsed, summary.degraded_reason.is_some()) +} + +/// The notices and resulting store state after starting on a file that is not a database. +pub(crate) fn session_on_corrupt_store() -> (usize, bool, Vec>) { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("recall.sqlite3"); + std::fs::write(&path, b"this is not a database").expect("test files can be written"); + let session = start_recall_session(config(path.clone(), "fresh")); + session.append_output(OutputStream::Stdout, b"kept\n"); + let summary = session.close(); + let moved: bool = std::fs::read_dir(directory.path()) + .expect("the directory lists") + .filter_map(Result::ok) + .any(|entry| entry.file_name().to_string_lossy().contains(".corrupt-")); + let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); + let rows = query_store(&store, "SELECT status FROM invocations", None) + .expect("the query is valid") + .rows; + drop(summary); + (session.notices().len(), moved, rows) +} + +fn stream_counts(summary: &DeliverySummary, stream: DeliveryStream) -> DeliveryCounts { + summary + .streams + .iter() + .find(|(candidate, _)| *candidate == stream) + .map(|(_, counts)| *counts) + .unwrap_or_default() +} + +/// Artifact counts and stored content rows after publishing the same SQL of `length` bytes `times` times. +pub(crate) fn artifact_counts( + length: usize, + times: usize, +) -> (u64, u64, u64, Vec>) { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("recall.sqlite3"); + let session = start_recall_session(config(path.clone(), "sql")); + let sql: String = "x".repeat(length); + for index in 0..times { + session.publish_sql(&sql, Some(&format!("s{index}")), None); + } + let counts = stream_counts(&session.close(), DeliveryStream::Artifacts); + let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); + let stored = query_store(&store, "SELECT COUNT(*) FROM recall_artifacts", None) + .expect("the query is valid") + .rows; + (counts.accepted, counts.delivered, counts.dropped, stored) +} + +/// Event counts after publishing `events`; rows are built on the delivery thread. +pub(crate) fn event_counts(events: &[&str]) -> (u64, u64, u64) { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let session = start_recall_session(config(directory.path().join("recall.sqlite3"), "events")); + let parsed: Vec> = events + .iter() + .map(|event| serde_json::from_str(event).expect("the fixture is a JSON object")) + .collect(); + parsed + .into_iter() + .for_each(|event| session.publish_event_object(event)); + let counts = stream_counts(&session.close(), DeliveryStream::Events); + (counts.accepted, counts.delivered, counts.failed) +} + +/// Output chunks dropped by a queue of `capacity` records, as counted and as each delivered chunk reports them. +pub(crate) fn output_drops(capacity: usize, chunks: usize) -> (u64, Vec>) { + let mut state = WriterState::with_capacity(envelope("inv"), (capacity, usize::MAX)); + for index in 0..chunks { + let stream = [OutputStream::Stdout, OutputStream::Stderr][index % 2]; + state.append_output(stream, b"x", SystemTime::now()); + } + let mut taken: Vec = state.take_batch(usize::MAX); + state.flush_output(); + taken.extend(state.take_batch(usize::MAX)); + let reported: Vec> = taken + .iter() + .flat_map(|record| record.rows().iter()) + .map(|row| row.value("dropped_before").cloned()) + .collect(); + ( + state.counts[DeliveryStream::Output.index()].dropped, + reported, + ) +} + +/// Every output row after `writes` and close, by sequence: sequence, stream, byte offset, data and omitted bytes and sequences. +pub(crate) fn capped( + limit: Option, + writes: &[(OutputStream, &[u8])], +) -> Vec<[Option; 6]> { + let mut state = WriterState::new(envelope("inv")).with_output_limit(limit); + for (stream, data) in writes { + state.append_output(*stream, data, SystemTime::now()); + } + state.flush_tails(); + let mut rows: Vec<[Option; 6]> = state + .take_batch(usize::MAX) + .iter() + .flat_map(|record| record.rows().iter()) + .map(|row| { + [ + "sequence", + "stream", + "byte_offset", + "data", + "omitted_bytes", + "omitted_sequences", + ] + .map(|name| row.value(name).cloned()) + }) + .collect(); + rows.sort_by_key(|row| format!("{:020?}", row[0])); + rows +} + +/// A row as `capped` reports it; `omitted` is set on a truncation gap. +pub(crate) fn chunk_facts( + (sequence, stream, offset): (i64, &str, i64), + data: &[u8], + omitted: Option<(i64, i64)>, +) -> [Option; 6] { + [ + Some(RecallValue::Integer(sequence)), + Some(RecallValue::Text(stream.to_owned())), + Some(RecallValue::Integer(offset)), + Some(RecallValue::Blob(data.to_vec())), + Some(omitted.map_or(RecallValue::Null, |(bytes, _)| RecallValue::Integer(bytes))), + Some(omitted.map_or(RecallValue::Null, |(_, sequences)| { + RecallValue::Integer(sequences) + })), + ] +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_queue.rs b/crates/sqlbuild-recall/src/writer/tests/test_queue.rs new file mode 100644 index 0000000000..5b74ba43bb --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/tests/test_queue.rs @@ -0,0 +1,184 @@ +use crate::records::models::{OutputStream, RecallValue}; +use crate::writer::models::{DeliveryStream, Priority}; +use crate::writer::tests::helpers::{capped, chunk_facts, chunked, output_drops, pushed}; +use crate::writer::tests::test_types::{ + CaptureLimitTestCase, ChunkingTestCase, DisplacementTestCase, OutputDropTestCase, +}; + +#[test] +fn given_a_full_queue_when_higher_priority_arrives_then_the_oldest_lowest_is_displaced() { + let test_cases = [ + DisplacementTestCase { + description: "debug diagnostics give way first, artifacts survive", + capacity: 2, + pushes: &[ + (DeliveryStream::Diagnostics, Priority::DebugDiagnostic), + (DeliveryStream::Artifacts, Priority::Artifact), + (DeliveryStream::Events, Priority::FailedTerminal), + (DeliveryStream::Diagnostics, Priority::DebugDiagnostic), + ], + expected_dropped: vec![ + (DeliveryStream::Diagnostics, 2), + (DeliveryStream::Artifacts, 0), + (DeliveryStream::Events, 0), + (DeliveryStream::Diagnostics, 2), + ], + expected_taken: vec![DeliveryStream::Events, DeliveryStream::Artifacts], + }, + DisplacementTestCase { + description: "a queue with room keeps everything in priority order", + capacity: 4, + pushes: &[ + (DeliveryStream::Diagnostics, Priority::DebugDiagnostic), + (DeliveryStream::Events, Priority::FailedTerminal), + ], + expected_dropped: vec![ + (DeliveryStream::Diagnostics, 0), + (DeliveryStream::Events, 0), + ], + expected_taken: vec![DeliveryStream::Events, DeliveryStream::Diagnostics], + }, + ]; + + for test_case in test_cases { + assert_eq!( + pushed(test_case.capacity, test_case.pushes), + (test_case.expected_dropped, test_case.expected_taken), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_interleaved_and_large_writes_when_chunked_then_sequences_and_offsets_are_exact() { + let chunk = |sequence: i64, stream: &str, offset: i64| { + [ + Some(RecallValue::Integer(sequence)), + Some(RecallValue::Text(stream.to_owned())), + Some(RecallValue::Integer(offset)), + ] + }; + let test_cases = [ + ChunkingTestCase { + description: "adjacent writes coalesce, stream switches and 64KiB split chunks", + writes: vec![ + (OutputStream::Stdout, b"one ".to_vec()), + (OutputStream::Stdout, b"two\n".to_vec()), + (OutputStream::Stderr, b"err\n".to_vec()), + (OutputStream::Stdout, vec![b'a'; 64 * 1024 + 10]), + ], + expected_chunks: vec![ + chunk(0, "stdout", 0), + chunk(1, "stderr", 0), + chunk(2, "stdout", 8), + chunk(3, "stdout", 65_544), + ], + }, + ChunkingTestCase { + description: "one small write is one chunk", + writes: vec![(OutputStream::Stderr, b"only\n".to_vec())], + expected_chunks: vec![chunk(0, "stderr", 0)], + }, + ]; + + for test_case in test_cases { + assert_eq!( + chunked(&test_case.writes), + test_case.expected_chunks, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_full_queue_when_output_is_dropped_then_each_drop_is_counted_once() { + let test_cases = [OutputDropTestCase { + description: "two chunks dropped by a full queue are counted once and reported once", + capacity: 1, + chunks: 4, + expected_drops: ( + 2, + vec![Some(RecallValue::Integer(0)), Some(RecallValue::Integer(2))], + ), + }]; + + for test_case in test_cases { + assert_eq!( + output_drops(test_case.capacity, test_case.chunks), + test_case.expected_drops, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_output_past_the_limit_when_closed_then_head_and_tail_are_exact_around_one_gap() { + let test_cases = [ + CaptureLimitTestCase { + description: "output within the first half is one chunk", + limit: Some(8), + writes: vec![(OutputStream::Stdout, b"01"), (OutputStream::Stdout, b"23")], + expected_rows: vec![chunk_facts((0, "stdout", 0), b"0123", None)], + }, + CaptureLimitTestCase { + description: "output within the limit is kept whole around an empty gap", + limit: Some(8), + writes: vec![ + (OutputStream::Stdout, b"0123"), + (OutputStream::Stdout, b"45"), + ], + expected_rows: vec![ + chunk_facts((0, "stdout", 0), b"0123", None), + chunk_facts((1, "stdout", 4), b"", Some((0, 1))), + chunk_facts((2, "stdout", 4), b"45", None), + ], + }, + CaptureLimitTestCase { + description: "the first and last halves are kept around a gap counting the middle", + limit: Some(8), + writes: vec![(OutputStream::Stdout, b"0123456789")], + expected_rows: vec![ + chunk_facts((0, "stdout", 0), b"0123", None), + chunk_facts((1, "stdout", 4), b"", Some((2, 1))), + chunk_facts((2, "stdout", 6), b"6789", None), + ], + }, + CaptureLimitTestCase { + description: "interleaved stderr keeps its place around the kept stdout tail", + limit: Some(8), + writes: vec![ + (OutputStream::Stdout, b"0123"), + (OutputStream::Stdout, b"ab"), + (OutputStream::Stderr, b"E1"), + (OutputStream::Stdout, b"cdef"), + (OutputStream::Stdout, b"gh"), + (OutputStream::Stderr, b"E2"), + ], + expected_rows: vec![ + chunk_facts((0, "stdout", 0), b"0123", None), + chunk_facts((1, "stdout", 4), b"", Some((4, 2))), + chunk_facts((3, "stderr", 0), b"E1", None), + chunk_facts((4, "stdout", 8), b"efgh", None), + chunk_facts((5, "stderr", 2), b"E2", None), + ], + }, + CaptureLimitTestCase { + description: "no limit records everything", + limit: None, + writes: vec![(OutputStream::Stdout, b"0123456789")], + expected_rows: vec![chunk_facts((0, "stdout", 0), b"0123456789", None)], + }, + ]; + + for test_case in test_cases { + assert_eq!( + capped(test_case.limit, &test_case.writes), + test_case.expected_rows, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_session.rs b/crates/sqlbuild-recall/src/writer/tests/test_session.rs new file mode 100644 index 0000000000..4bb2d460aa --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/tests/test_session.rs @@ -0,0 +1,187 @@ +use std::time::Duration; + +use crate::records::models::{OutputStream, RecallValue}; +use crate::writer::models::SetupOutcome; + +#[cfg(unix)] +use crate::writer::tests::helpers::unwritable_session; +use crate::writer::tests::helpers::{ + artifact_counts, close_under_held_lock, event_counts, recorded_session, + session_on_corrupt_store, +}; +#[cfg(unix)] +use crate::writer::tests::test_types::UnwritableTestCase; +use crate::writer::tests::test_types::{ + ArtifactSizeTestCase, CorruptStoreTestCase, EventBuildTestCase, HeldLockTestCase, + RecordedSession, SessionTestCase, +}; + +const CLOSE_SLACK_SECONDS: f64 = 0.3; + +#[test] +fn given_a_session_when_closed_then_every_record_and_the_delivery_rows_are_written() { + let large: Vec = (0..200_000_u32) + .map(|value| u8::try_from(value % 251).unwrap_or_default()) + .collect(); + let test_cases = [SessionTestCase { + description: "binary output, events, filtered diagnostics and deduplicated SQL land", + output: vec![ + (OutputStream::Stdout, b"start\n".to_vec()), + (OutputStream::Stderr, b"\xff\x00warn\n".to_vec()), + (OutputStream::Stdout, large.clone()), + ], + expected_session: RecordedSession { + setup: Some(SetupOutcome::Ready), + lost: (0, 0), + output_fully_delivered: true, + transcript: ( + [b"start\n".as_slice(), b"\xff\x00warn\n", &large].concat(), + true, + ), + stdout: [b"start\n".as_slice(), &large].concat(), + facts: vec![vec![ + RecallValue::Text("completed".to_owned()), + RecallValue::Integer(1), + RecallValue::Integer(1), + RecallValue::Integer(5), + RecallValue::Text("build".to_owned()), + ]], + }, + }]; + + for test_case in test_cases { + assert_eq!( + recorded_session(&test_case.output), + test_case.expected_session, + "{}", + test_case.description + ); + } +} + +#[cfg(unix)] +#[test] +fn given_an_unwritable_store_when_started_then_setup_reports_permission_and_close_is_quick() { + let test_cases = [UnwritableTestCase { + description: "permission is reported at setup and the summary is degraded", + expected_permission_and_degraded: (true, true), + }]; + + for test_case in test_cases { + assert_eq!( + unwritable_session(), + test_case.expected_permission_and_degraded, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_held_write_lock_when_closed_then_the_shutdown_budget_caps_the_delay() { + let test_cases = [HeldLockTestCase { + description: "a locked store costs at most the shutdown budget at exit", + shutdown_ms: 700, + expected_degraded: true, + }]; + + for test_case in test_cases { + let (elapsed, degraded) = + close_under_held_lock(Duration::from_millis(test_case.shutdown_ms)); + let budget: f64 = Duration::from_millis(test_case.shutdown_ms).as_secs_f64(); + assert!( + elapsed <= budget + CLOSE_SLACK_SECONDS, + "{}: {elapsed:.3}s", + test_case.description + ); + assert_eq!( + degraded, test_case.expected_degraded, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_file_that_is_not_a_database_when_started_then_it_is_moved_aside_once() { + let test_cases = [CorruptStoreTestCase { + description: "the corrupt file is kept aside and a fresh store records the run", + expected_notices: 1, + expected_moved: true, + expected_rows: vec![vec![RecallValue::Text("unknown".to_owned())]], + }]; + + for test_case in test_cases { + assert_eq!( + session_on_corrupt_store(), + ( + test_case.expected_notices, + test_case.expected_moved, + test_case.expected_rows + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_sql_of_different_sizes_when_published_then_oversized_sql_is_dropped_unhashed() { + let test_cases = [ + ArtifactSizeTestCase { + description: "ordinary SQL is delivered with its reference", + length: 64, + times: 1, + expected_counts: (1, 1, 0, vec![vec![RecallValue::Integer(1)]]), + }, + ArtifactSizeTestCase { + description: "SQL seen twice is stored once and both records count as delivered", + length: 64, + times: 2, + expected_counts: (2, 2, 0, vec![vec![RecallValue::Integer(1)]]), + }, + ArtifactSizeTestCase { + description: "SQL above the artifact limit is counted as dropped", + length: 4 * 1024 * 1024 + 1, + times: 1, + expected_counts: (1, 0, 1, vec![vec![RecallValue::Integer(0)]]), + }, + ]; + + for test_case in test_cases { + assert_eq!( + artifact_counts(test_case.length, test_case.times), + test_case.expected_counts, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_events_when_built_on_the_delivery_thread_then_unbuildable_ones_count_as_failed() { + let valid: &str = r#"{"event_id":"e1","event_type":"invocation_started","schema_version":2, + "producer":"sqlbuild","producer_version":"1","occurred_at":"2026-10-19T00:00:00.000000Z", + "invocation_id":"events","invocation_sequence":1,"external_context":{},"payload":{}}"#; + let test_cases = [ + EventBuildTestCase { + description: "a complete event is delivered", + events: vec![valid], + expected_counts: (1, 1, 0), + }, + EventBuildTestCase { + description: "an event without an ID is accepted, then counted as failed", + events: vec![valid, r#"{"event_type":"invocation_started"}"#], + expected_counts: (2, 1, 1), + }, + ]; + + for test_case in test_cases { + assert_eq!( + event_counts(&test_case.events), + test_case.expected_counts, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs new file mode 100644 index 0000000000..3aaa2c7048 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -0,0 +1,78 @@ +use crate::records::models::{OutputStream, RecallValue}; +use crate::writer::models::{DeliveryStream, Priority, SetupOutcome}; + +/// What one recorded session left in the store. +#[derive(Debug, PartialEq)] +pub(crate) struct RecordedSession { + pub(crate) setup: Option, + pub(crate) lost: (u64, u64), + pub(crate) output_fully_delivered: bool, + pub(crate) transcript: (Vec, bool), + pub(crate) stdout: Vec, + pub(crate) facts: Vec>, +} + +pub(crate) struct DisplacementTestCase { + pub(crate) description: &'static str, + pub(crate) capacity: usize, + pub(crate) pushes: &'static [(DeliveryStream, Priority)], + pub(crate) expected_dropped: Vec<(DeliveryStream, u64)>, + pub(crate) expected_taken: Vec, +} + +pub(crate) struct ChunkingTestCase { + pub(crate) description: &'static str, + pub(crate) writes: Vec<(OutputStream, Vec)>, + pub(crate) expected_chunks: Vec<[Option; 3]>, +} + +pub(crate) struct SessionTestCase { + pub(crate) description: &'static str, + pub(crate) output: Vec<(OutputStream, Vec)>, + pub(crate) expected_session: RecordedSession, +} + +pub(crate) struct UnwritableTestCase { + pub(crate) description: &'static str, + pub(crate) expected_permission_and_degraded: (bool, bool), +} + +pub(crate) struct HeldLockTestCase { + pub(crate) description: &'static str, + pub(crate) shutdown_ms: u64, + pub(crate) expected_degraded: bool, +} + +pub(crate) struct CorruptStoreTestCase { + pub(crate) description: &'static str, + pub(crate) expected_notices: usize, + pub(crate) expected_moved: bool, + pub(crate) expected_rows: Vec>, +} + +pub(crate) struct ArtifactSizeTestCase { + pub(crate) description: &'static str, + pub(crate) length: usize, + pub(crate) times: usize, + pub(crate) expected_counts: (u64, u64, u64, Vec>), +} + +pub(crate) struct EventBuildTestCase { + pub(crate) description: &'static str, + pub(crate) events: Vec<&'static str>, + pub(crate) expected_counts: (u64, u64, u64), +} + +pub(crate) struct OutputDropTestCase { + pub(crate) description: &'static str, + pub(crate) capacity: usize, + pub(crate) chunks: usize, + pub(crate) expected_drops: (u64, Vec>), +} + +pub(crate) struct CaptureLimitTestCase { + pub(crate) description: &'static str, + pub(crate) limit: Option, + pub(crate) writes: Vec<(OutputStream, &'static [u8])>, + pub(crate) expected_rows: Vec<[Option; 6]>, +} diff --git a/fensu.toml b/fensu.toml index c4802e31b1..8a3dfb12ec 100644 --- a/fensu.toml +++ b/fensu.toml @@ -472,6 +472,7 @@ crate_names = [ "sqlbuild-model-config", "sqlbuild-pysource", "sqlbuild-python", + "sqlbuild-recall", "sqlbuild-refactor", "sqlbuild-render", "sqlbuild-rules", @@ -554,7 +555,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=19", - "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=26", + "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=28", "crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=26", @@ -959,6 +960,9 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs=10", + "crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/auditing.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/compiled.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/cursors.rs=1", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index f007592421..c659d00a80 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -76,6 +76,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in ## Advanced Concepts - [Execution Observability](concepts/observability.md) (`concepts/observability`) - Choose authoritative lifecycle facts, readable logs, or command-output records. +- [Recall](concepts/observability/recall.md) (`concepts/observability/recall`) - The local history of every command: what ran, what it executed, and what it printed. - [Typed Sinks](concepts/observability/sinks.md) (`concepts/observability/sinks`) - Export lifecycle facts and command output through project-owned providers. - [Execution history](concepts/observability/execution-history.md) (`concepts/observability/execution-history`) - Store and query lifecycle events and run history from Python, with SQLite locally or PostgreSQL when deployed. - [Overview](concepts/declaration-scopes.md) (`concepts/declaration-scopes`) - Limit enums, constants, macros, audits, schemas, and named hooks to the parts of a project that use them. @@ -130,6 +131,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [lineage](cli/lineage.md) (`cli/lineage`) - Explore model and column-level dependency graphs from the command line. - [sqb dag](cli/dag.md) (`cli/dag`) - Generate the static DAG artifact for Dagster and other integrations. - [sqb query](cli/query.md) (`cli/query`) - Run ad hoc SQL queries against the project database. +- [sqb recall](cli/recall.md) (`cli/recall`) - Query the recorded history of past invocations. - [debug](cli/debug.md) (`cli/debug`) - Validate project configuration and test the warehouse connection. - [sqb janitor](cli/janitor.md) (`cli/janitor`) - Archive and then delete stale warehouse relations. - [sqb clean](cli/clean.md) (`cli/clean`) - Remove compiled artifacts from the target directory. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md new file mode 100644 index 0000000000..0abc6ee443 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md @@ -0,0 +1,125 @@ + + +# sqb recall + +> Query the recorded history of past invocations. + +Online: https://sqlbuild.com/docs/cli/recall/ + +## Contents + +- Usage +- sqb recall query +- sqb recall schema +- sqb recall output +- Errors + +Read the local [Recall](../concepts/observability/recall.md) store: past invocations, the +statements they executed, and what they printed. `sqb recall` never records itself. + +## Usage + +```bash +sqb recall query "SELECT command, status, duration_ms FROM invocations ORDER BY invocation_started_at DESC" +sqb recall schema +sqb recall output last +``` + +## sqb recall query + +Run one read-only SQLite `SELECT` against the Recall views. Statements that write, and multiple +statements, are rejected. + +| Flag | Description | +|------|-------------| +| `sql` | The query (positional argument) | +| `--format` | `long` (default), `table`, `json`, or `csv` | +| `--json` | Shorthand for `--format json` | +| `--limit` | Maximum rows to return (default: 100) | +| `--no-limit` | Disable the row limit | +| `--max-cell-bytes` | Longest cell in text formats, in bytes (default: 4096; `0` for no limit) | +| `--target` | Target whose store to read (default: the default target) | +| `--backend` | `logs` (default) reads the Recall store; `warehouse` runs the query on the target's connection | + +The latest invocation in a project named `orders_shop`: + +```bash +sqb recall query --format table \ + "SELECT command, status, exit_code, duration_ms FROM invocations + ORDER BY invocation_started_at DESC LIMIT 1" +``` + +``` +command | status | exit_code | duration_ms +------- | --------- | --------- | ----------- +build | completed | 0 | 752.0 + +1 row +``` + +The slowest statements of past builds, with their SQL: + +```bash +sqb recall query --format table --max-cell-bytes 60 \ + "SELECT s.resource_name, round(s.duration_ms, 1) AS ms, a.text AS sql + FROM statements s JOIN artifacts a USING (invocation_id, statement_id) + WHERE s.statement_kind = 'CREATE' ORDER BY s.duration_ms DESC LIMIT 2" +``` + +``` +resource_name | ms | sql +------------- | --- | ------------------------------------------------------------------------ +orders | 3.6 | CREATE OR REPLACE TABLE dev.orders__staging AS SELECT 1 …[+74 bytes] +NULL | 3.2 | CREATE SCHEMA IF NOT EXISTS dev + +2 rows +``` + +Failed invocations of the last week: + +```bash +sqb recall query \ + "SELECT invocation_id, command, error_code FROM invocations + WHERE status = 'failed' AND invocation_started_at >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-7 days')" +``` + +Timestamps are ISO-8601 UTC text, so they compare correctly as strings. Filter on +`invocation_started_at` to bound a scan. + +## sqb recall schema + +List every Recall view with its columns, types, and join keys. Accepts `--format`, `--json`, and +`--target`. With `--backend warehouse`, it lists SQLBuild's state tables in the target's warehouse +instead, or describes one with `--relation `. + +## sqb recall output + +Print what an invocation wrote, byte for byte. + +```bash +sqb recall output last # the latest invocation, both streams in order +sqb recall output 7607b5b6 --stream stderr # an invocation ID or unique prefix +sqb recall output last --tail 20 # only the last 20 lines +``` + +| Flag | Description | +|------|-------------| +| `invocation` | Invocation ID, unique prefix, or `last` (default) | +| `--stream` | `combined` (default), `stdout`, or `stderr` | +| `--tail` | Print only the last N lines | +| `--target` | Target whose store to read | + +Where output past [`max_output_bytes`](../concepts/observability/recall.md#output-limit) was left +out, a note line marks the gap in place. Output that Recall could not record completely for any +other reason is followed by a note on stderr naming the gap. + +## Errors + +| Code | Meaning | +|------|---------| +| `C970` | Recall is disabled for this project or target | +| `C971` | The store cannot be read, or no history is recorded yet | +| `C972` | No invocation matches | +| `C973` | The invocation prefix matches more than one invocation | +| `C974` | The query is invalid or not read-only | +| `C975` | The `[recall]` settings are invalid | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability.md index 36bb9f197d..1129254621 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability.md @@ -14,6 +14,7 @@ whether work started, completed, failed, skipped, or retried. | `LifecycleEvent` | Immutable invocation, run, resource-attempt, operation, statement, retry, and audit facts | Canonical lifecycle evidence once durably stored | | `DiagnosticLog` | Structured framework diagnostics | Explanatory only | | `stdout.log` / `stderr.log` | Exact host-local process output | Troubleshooting transcript | +| [Recall](observability/recall.md) | Local queryable history of invocations, statements, SQL, and output | Troubleshooting history; best effort | | `CommandOutputRecord` | Bounded, ANSI-free remote stdout/stderr chunks | Potentially sensitive and lossy troubleshooting data | | Final JSON output | End-of-command aggregate | Result projection, not event history | @@ -83,4 +84,5 @@ A start without a terminal fact is unknown or presumed lost. Consumers must not or failure. Sink delivery is bounded and best effort; destination failure does not change successful warehouse work, and successful delivery does not prove the command succeeded. -See [Typed Sinks](observability/sinks.md) to export lifecycle and command-output records. +See [Recall](observability/recall.md) for the local history `sqb recall` queries, and +[Typed Sinks](observability/sinks.md) to export lifecycle and command-output records. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md new file mode 100644 index 0000000000..eb4a8a3705 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -0,0 +1,145 @@ + + +# Recall + +> The local history of every command: what ran, what it executed, and what it printed. + +Online: https://sqlbuild.com/docs/concepts/observability/recall/ + +## Contents + +- What is recorded +- Settings +- Output limit +- Retention +- When the store cannot be written +- Querying history +- Relationship to compute logs + +Recall records each `sqb` invocation in a local SQLite store so you can ask what happened after the +fact: which command ran, how it ended, which statements it executed and how long they took, and +exactly what it printed. Recall is on by default and needs no setup. + +## What is recorded + +For every recorded invocation, Recall keeps: + +- the invocation itself: command, arguments, target, adapter, status, exit code, and timing; +- its [lifecycle events](../observability.md): runs, resource attempts, statements, + retries, and audit results; +- the SQL text of every executed statement, stored once per distinct statement and shared across + invocations; +- its stdout and stderr, byte for byte, in the order they were written, up to + [`max_output_bytes`](#output-limit) of each stream; +- framework diagnostics at or above `diagnostics_min_severity`. + +Writes happen on a background thread and never fail the command. A command that cannot record all +of its history still succeeds, prints a warning followed by its own final status, and +`output_invocations` reports the gap. An invocation whose process died without finishing has +status `abandoned`. + +Every command is recorded except `sqb recall` itself, commands that create a project (such as +`sqb init`), and raw-query `sqb diff`. A `sqb compile` answered from the launcher's warm cache is not +recorded either. + +The store is `logs/recall.sqlite3` in the project directory. Add `logs/` to `.gitignore`; the store +can contain sensitive SQL and output. + +## Settings + +Configure Recall in `[recall]` in `sqlbuild_project.toml`. A `[targets..recall]` table +overrides individual keys for one target. + +```toml +[recall] +retention = "30d" # days of history to keep; also accepts hours, such as "720h" +shutdown_timeout = "3s" # longest delay Recall may add when a command exits +diagnostics_min_severity = "info" +max_output_bytes = 2097152 # bytes of each output stream to keep; 0 keeps everything + +[targets.ci.recall] +enabled = false # do not record runs against the ci target +``` + +| Key | Default | Description | +|-----|---------|-------------| +| `enabled` | `true` | Set to `false` to stop recording | +| `connection` | (none) | A `sqlite` connection whose `path` names the store; defaults to `logs/recall.sqlite3` | +| `retention` | `"30d"` | How long invocations are kept, in days (`d`) or hours (`h`), up to 3650 days | +| `shutdown_timeout` | `"3s"` | The cap on time spent flushing history at exit, in `s` or `ms`, up to 10 minutes | +| `diagnostics_min_severity` | `"info"` | `trace`, `debug`, `info`, `warning`, `error`, or `critical` | +| `max_output_bytes` | `2097152` for the local store | Bytes of stdout and of stderr kept per invocation, an integer; `0` keeps everything. See [Output limit](#output-limit) | + +To turn Recall off for the whole project: + +```toml +[recall] +enabled = false +``` + +Recall never delays the start of a command. At exit it waits at most `shutdown_timeout` for queued +history to reach the store; anything left is counted as dropped. A second Ctrl-C stops waiting at +once. + +## Output limit + +Recall keeps at most `max_output_bytes` of each output stream per invocation: the first half and +the last half, byte for byte. The local SQLite store defaults to 2 MiB, so a command that prints +more keeps its first and last MiB of stdout and of stderr. Set `max_output_bytes = 0` to keep +everything, or a larger number to keep more; a `[targets..recall]` table can set it for one +target. The command itself prints nothing about the limit. + +The middle that was left out is one gap: `output_invocations` reports `gap_reason = 'truncated'`, +the left-out byte count in `truncated_bytes`, and `complete = 0`. `sqb recall output` prints a +note in its place: + +``` +… 23.6 MB of stdout not recorded (kept the first and last 1 MiB). Raise [recall] max_output_bytes to keep more. +``` + +The end of a stream is held in memory until the command exits, so a process that is killed keeps +only the first half. + +## Retention + +Commands occasionally prune invocations older than `retention`, with all of their events, output, +and diagnostics. SQL text is removed once no remaining invocation references it. Pruning runs in +the background within the exit budget and never removes an invocation that is still running. + +## When the store cannot be written + +If the store location is not writable, the command still runs and prints a notice on stderr with +the SQL an administrator can use to create the store; the notice comes before the command's own +output whenever the problem is visible from file permissions. The notice +repeats at most once a day. If the store file is not a SQLite database, Recall moves it aside to +`recall.sqlite3.corrupt-`, starts a fresh store, and says so once. Commands that +find the same corrupt file coordinate through `recall.sqlite3.lock`, so only one of them moves it. + +A `connection` that does not use the `sqlite` adapter is not supported yet; Recall prints a notice +and records nothing. + +## Querying history + +Use [`sqb recall`](../../cli/recall.md) to read the store. Queries are read-only SQLite `SELECT` +statements against these views: + +| View | One row per | +|------|-------------| +| `invocations` | Invocation, with status, exit code, and timing | +| `statements` | Executed statement, with resource, kind, purpose, duration, and row counts | +| `audit_results` | Evaluated audit | +| `events` | Lifecycle event | +| `diagnostics` | Retained diagnostic | +| `artifacts` | Reference from an invocation to captured SQL, with its text | +| `output_invocations` | Invocation's output, with completeness | +| `output_transcripts` | Invocation's stdout, stderr, and combined output | +| `output_tail` | Recent output chunk | +| `conflicts` | Record written twice with different content | + +Run `sqb recall schema` for every column. + +## Relationship to compute logs + +Compute logs (`logs///stdout.log`, `stderr.log`, and `diagnostics.jsonl`) are +still written alongside Recall. Recall stores the same output bytes, captured at the same point, so +`sqb recall output` matches the compute log files. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 1e057364aa..f163e88da3 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3323,6 +3323,14 @@ def invocation_context_from_environment( *, environment: Mapping[str, str] | None = None ) -> Mapping[str, object]: ... +class RecallLifecycleSink: + """The invocation's Recall session as lifecycle and SQL artifact subscribers.""" + + def __call__(self, event: LifecycleEvent) -> None: ... + def record_sql( + self, *, sql: str, statement_id: str | None = None, resource_id: str | None = None + ) -> None: ... + class LocalFilesystemComputeLogStorage: """Ordered fsync capture storage subject to underlying filesystem and hardware guarantees.""" @@ -3333,6 +3341,7 @@ class LocalFilesystemComputeLogStorage: root: Path | None = None, retention_count: int | None = 20, ) -> None: ... + def attach_recall(self, recall: RecallLifecycleSink) -> None: ... @property def root(self) -> Path: ... def start_capture(self, metadata: CaptureMetadata) -> None: ... @@ -3646,6 +3655,18 @@ class CliNamespace: rules_command: str | None rules_rule_selector: str | None rules_skills_check: bool + recall_command: str | None + recall_sql: str | None + recall_backend: str + recall_format: str + recall_limit: int + recall_no_limit: bool + recall_max_cell_bytes: int | None + recall_state: bool + recall_relation: str | None + recall_invocation: str + recall_stream: str + recall_tail: int | None format_check: bool format_diff: bool format_fixtures_only: bool @@ -8725,6 +8746,7 @@ class CliEntrypointHandlers(NativeRecord): run_scope: ScopeCommandHandler run_contract: Callable[..., int] | None run_refactor: Callable[..., int] | None + run_recall: Callable[..., int] | None def __init__( self, run_compile: Callable[[CompileCommandRequest], int], @@ -8759,6 +8781,7 @@ class CliEntrypointHandlers(NativeRecord): run_scope: ScopeCommandHandler, run_contract: Callable[..., int] | None = None, run_refactor: Callable[..., int] | None = None, + run_recall: Callable[..., int] | None = None, ) -> None: ... class CommandWarehouseScope(NativeRecord): @@ -9558,6 +9581,7 @@ class CliCommand(StrEnum): CONTRACT = "contract" RENAME = "rename" MV = "mv" + RECALL = "recall" class ScopeCommandHandler(Protocol): def __call__(self, *, request: ScopeCommandRequest) -> int: ... diff --git a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py index 926b51d22c..c6f6242d35 100644 --- a/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py +++ b/src/sqlbuild/cli/commands/main/entrypoint/_dispatch_with_compute_logs.py @@ -10,6 +10,7 @@ COMPUTE_LOG_FORMAT_VERSION, CliNamespace, LocalFilesystemComputeLogStorage, + RecallLifecycleSink, supports_color, ) from sqlbuild.cli.commands._helpers.entry.compute_log_diagnostics import ( @@ -31,6 +32,7 @@ def dispatch_with_compute_logs( identity: ExecutionIdentity, operation: Callable[[], int], creates_project: bool, + recall: RecallLifecycleSink | None = None, ) -> int: """Run one composed dispatch while retaining exact process streams best-effort.""" @@ -56,6 +58,8 @@ def dispatch_with_compute_logs( storage: LocalFilesystemComputeLogStorage | None = None try: storage = LocalFilesystemComputeLogStorage(project_dir=project_dir) + if recall is not None: + storage.attach_recall(recall) storage.start_capture(metadata) except Exception as error: failures: list[tuple[Exception, str]] = [(error, "capture_open")] diff --git a/src/sqlbuild/executor/auditing/main/_execute.py b/src/sqlbuild/executor/auditing/main/_execute.py index 716360750f..97be843670 100644 --- a/src/sqlbuild/executor/auditing/main/_execute.py +++ b/src/sqlbuild/executor/auditing/main/_execute.py @@ -9,6 +9,8 @@ from sqlbuild.compiler.auditing.types import AuditRunScope from sqlbuild.executor.auditing._helpers.execution import execute_audit_impl from sqlbuild.executor.auditing.models import AuditExecutionResult +from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle +from sqlbuild.runtime.observability.constants import AUDIT_PURPOSE def execute_audit( @@ -25,14 +27,15 @@ def execute_audit( ) -> AuditExecutionResult: """Execute and evaluate one audit, returning quality failure as result data.""" - return execute_audit_impl( - audit=audit, - adapter=adapter, - connection=connection, - model_locations=model_locations, - seed_locations=seed_locations, - source_map=source_map, - relation_overrides=relation_overrides, - run_scope_phase=run_scope_phase, - quality_scope=quality_scope, - ) + with StatementLifecycle.purpose_scope(AUDIT_PURPOSE): + return execute_audit_impl( + audit=audit, + adapter=adapter, + connection=connection, + model_locations=model_locations, + seed_locations=seed_locations, + source_map=source_map, + relation_overrides=relation_overrides, + run_scope_phase=run_scope_phase, + quality_scope=quality_scope, + ) diff --git a/src/sqlbuild/executor/build/_helpers/scheduler.py b/src/sqlbuild/executor/build/_helpers/scheduler.py index fcf165264f..18f57fbb91 100644 --- a/src/sqlbuild/executor/build/_helpers/scheduler.py +++ b/src/sqlbuild/executor/build/_helpers/scheduler.py @@ -47,6 +47,8 @@ from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.executor.testing.models import SqlTestExecutionResult from sqlbuild.runtime.contracts.types import ExecutionResourceKind +from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle +from sqlbuild.runtime.observability.constants import MODEL_BUILD_PURPOSE type _BuildWorkerResult = ( ModelExecutionResult @@ -91,19 +93,20 @@ def _dispatch_model( ) -> ModelExecutionResult: """Route a model to the correct executor, then report a stalled cursor window as a warning.""" - return with_waiting_on_empty_inputs_warning( - _dispatch_model_entry( - context=context, - promotion_mode=promotion_mode, - snapshots=snapshots, - allow_snapshot_schema_change=allow_snapshot_schema_change, - custom_materializations=custom_materializations, - target=target, - effective_vars=effective_vars, - warehouse_relations=warehouse_relations, - on_progress=on_progress, + with StatementLifecycle.purpose_scope(MODEL_BUILD_PURPOSE): + return with_waiting_on_empty_inputs_warning( + _dispatch_model_entry( + context=context, + promotion_mode=promotion_mode, + snapshots=snapshots, + allow_snapshot_schema_change=allow_snapshot_schema_change, + custom_materializations=custom_materializations, + target=target, + effective_vars=effective_vars, + warehouse_relations=warehouse_relations, + on_progress=on_progress, + ) ) - ) def _dispatch_model_entry( diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 2c9515e434..5ebd3dae3b 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -764,11 +764,14 @@ def _execute_node( | LoadExecutionResult ): resource_kind: str = self._canonical_resource_kind(key=key) + planned: ModelPlanEntry | None = self._indexes.model_entries_by_key.get(key) with ResourceAttemptLifecycle( resource_id=self._canonical_resource_id(key=key), resource_kind=resource_kind, resource_name=key.name, run_id=self._run_id, + plan_action=None if planned is None else str(planned.action), + plan_reason_code=None if planned is None else str(planned.reason), ) as lifecycle: with CostContext.scope( run_id=self._run_id, diff --git a/src/sqlbuild/runtime/observability/classes/event_dispatcher.py b/src/sqlbuild/runtime/observability/classes/event_dispatcher.py index 4bffd0c4ff..f4bbc28bf6 100644 --- a/src/sqlbuild/runtime/observability/classes/event_dispatcher.py +++ b/src/sqlbuild/runtime/observability/classes/event_dispatcher.py @@ -24,9 +24,11 @@ OpaqueLifecycleEvent, ) from sqlbuild.runtime.observability.types import ( + ArtifactSubscriber, DiagnosticRegistration, DiagnosticSubscriber, HealthCallback, + HistorySubscriber, KnownLifecycleSubscriber, LifecycleRegistration, OpaqueLifecycleSubscriber, @@ -55,6 +57,7 @@ def __init__(self, *, health_callback: HealthCallback | None = None) -> None: | tuple[Literal["diagnostic"], DiagnosticLog, bool] ] = deque() self._publishing = False + self._artifact_subscribers: tuple[ArtifactSubscriber, ...] = () @overload def subscribe_lifecycle( @@ -89,6 +92,35 @@ def subscribe_diagnostics(self, subscriber: DiagnosticSubscriber) -> Unsubscribe self._diagnostics = (*self._diagnostics, registration) return partial(self._unsubscribe_diagnostics, token=token) + def subscribe_artifacts(self, subscriber: ArtifactSubscriber) -> Unsubscribe: + """Register a subscriber for executed SQL text; only history stores subscribe.""" + + with self._lock: + self._artifact_subscribers = (*self._artifact_subscribers, subscriber) + return partial(self._unsubscribe_artifacts, subscriber=subscriber) + + def subscribe_history(self, subscriber: HistorySubscriber) -> Unsubscribe: + """Register a history store for every known lifecycle fact and executed SQL text.""" + + lifecycle: Unsubscribe = self.subscribe_lifecycle( + subscriber=subscriber, accepts_opaque=False + ) + artifacts: Unsubscribe = self.subscribe_artifacts(subscriber.record_sql) + return partial(self._unsubscribe_history, unsubscribers=(lifecycle, artifacts)) + + def publish_sql_artifact( + self, *, sql: str, statement_id: str | None, resource_id: str | None + ) -> None: + """Hand executed SQL text to artifact subscribers; it never enters lifecycle payloads.""" + + with self._lock: + subscribers: tuple[ArtifactSubscriber, ...] = self._artifact_subscribers + for subscriber in subscribers: + try: + subscriber(sql=sql, statement_id=statement_id, resource_id=resource_id) + except Exception as error: + self._report_failure(channel="lifecycle", subscriber=subscriber, error=error) + def publish_lifecycle(self, event: LifecycleEvent | OpaqueLifecycleEvent) -> None: """Synchronously publish a lifecycle fact to the current ordered snapshot.""" @@ -184,6 +216,18 @@ def _unsubscribe_lifecycle(self, *, token: object) -> None: registration for registration in self._lifecycle if registration[0] is not token ) + def _unsubscribe_history(self, *, unsubscribers: tuple[Unsubscribe, ...]) -> None: + for unsubscribe in unsubscribers: + unsubscribe() + + def _unsubscribe_artifacts(self, *, subscriber: ArtifactSubscriber) -> None: + with self._lock: + self._artifact_subscribers = tuple( + registered + for registered in self._artifact_subscribers + if registered is not subscriber + ) + def _unsubscribe_diagnostics(self, *, token: object) -> None: with self._lock: self._diagnostics = tuple( @@ -194,7 +238,10 @@ def _report_failure( self, *, channel: Literal["lifecycle", "diagnostic"], - subscriber: KnownLifecycleSubscriber | OpaqueLifecycleSubscriber | DiagnosticSubscriber, + subscriber: KnownLifecycleSubscriber + | OpaqueLifecycleSubscriber + | DiagnosticSubscriber + | ArtifactSubscriber, error: BaseException, ) -> None: callback: HealthCallback | None = self._health_callback diff --git a/src/sqlbuild/runtime/observability/classes/resource_attempt_lifecycle.py b/src/sqlbuild/runtime/observability/classes/resource_attempt_lifecycle.py index 9b0bf9aa84..4d7c7b37e9 100644 --- a/src/sqlbuild/runtime/observability/classes/resource_attempt_lifecycle.py +++ b/src/sqlbuild/runtime/observability/classes/resource_attempt_lifecycle.py @@ -41,8 +41,12 @@ def __init__( attempt_number: int = 1, resource_attempt_id: str | None = None, run_id: str | None = None, + plan_action: str | None = None, + plan_reason_code: str | None = None, ) -> None: self._resource_id: str = resource_id + self._plan_action: str | None = plan_action + self._plan_reason_code: str | None = plan_reason_code self._resource_kind: str = resource_kind self._resource_name: str = resource_name self._attempt_number: int = attempt_number @@ -140,11 +144,16 @@ def __exit__( self._stack.close() def _base_payload(self) -> dict[str, JSONValue]: - return { + payload: dict[str, JSONValue] = { "resource_kind": self._resource_kind, "resource_name": self._resource_name, "attempt_number": self._attempt_number, } + if self._plan_action is not None: + payload["plan_action"] = self._plan_action + if self._plan_reason_code is not None: + payload["plan_reason_code"] = self._plan_reason_code + return payload def _terminal_payload(self) -> dict[str, JSONValue]: payload: dict[str, JSONValue] = self._base_payload() diff --git a/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py b/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py index 33ec8eff2f..85d0901508 100644 --- a/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py +++ b/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py @@ -43,6 +43,7 @@ _STATEMENT_LIFECYCLE_OWNER: ContextVar[StatementLifecycle | None] = ContextVar( "sqlbuild_statement_lifecycle_owner", default=None ) +_STATEMENT_PURPOSE: ContextVar[str | None] = ContextVar("sqlbuild_statement_purpose", default=None) type _ExecutionOwnerKey = tuple[int, int | None] @@ -84,6 +85,7 @@ def __init__( ) self._pending_failure: tuple[BaseException, str | None, str | None] | None = None self._statement_facts: tuple[str, str] | None = None + self._purpose: str | None = _STATEMENT_PURPOSE.get() self._item_deadline: ItemDeadline | None = None self._terminal_query_id: str | None = None @@ -95,6 +97,17 @@ def listener_scope(listener: StatementListener) -> Iterator[None]: with statement_listener_scope(listener): yield + @staticmethod + @contextmanager + def purpose_scope(purpose: str) -> Iterator[None]: + """Label statements run inside the block with why SQLBuild runs them, such as audit.""" + + token: Token[str | None] = _STATEMENT_PURPOSE.set(purpose) + try: + yield + finally: + _STATEMENT_PURPOSE.reset(token) + @staticmethod def discard_failed_attempt() -> None: """Forget a failed attempt deferred to the active statement before an adapter retry.""" @@ -124,6 +137,9 @@ def __enter__(self) -> StatementLifecycle: self._started_monotonic = self._clock() self._execution_owner_key = execution_owner_key self._publish(event_type="statement_started", payload=self._base_payload()) + dispatcher.publish_sql_artifact( + sql=self._sql, statement_id=self._statement_id, resource_id=identity.resource_id + ) self._item_deadline = current_item_deadline() if self._item_deadline is not None: self._item_deadline.statement_started() @@ -303,6 +319,8 @@ def _base_payload(self) -> dict[str, JSONValue]: } if self._batch_size is not None: payload["batch_size"] = self._batch_size + if self._purpose is not None: + payload["purpose"] = self._purpose return payload def _terminal_payload(self) -> dict[str, JSONValue]: diff --git a/src/sqlbuild/runtime/observability/constants.py b/src/sqlbuild/runtime/observability/constants.py index 47ecb4b533..df9f0d299d 100644 --- a/src/sqlbuild/runtime/observability/constants.py +++ b/src/sqlbuild/runtime/observability/constants.py @@ -8,6 +8,8 @@ RENAME_OPERATION_STRATEGY: str = "rename" RETRY_SCHEDULED_EVENT: str = "retry_scheduled" STATEMENT_HEARTBEAT_THRESHOLD_SECONDS: float = 30.0 +MODEL_BUILD_PURPOSE: str = "model_build" +AUDIT_PURPOSE: str = "audit" LIFECYCLE_ENVELOPE_FIELDS_V1: frozenset[str] = frozenset( { "event_id", diff --git a/src/sqlbuild/runtime/observability/types.py b/src/sqlbuild/runtime/observability/types.py index 94cb1e76e9..f2f8f45ac9 100644 --- a/src/sqlbuild/runtime/observability/types.py +++ b/src/sqlbuild/runtime/observability/types.py @@ -18,6 +18,20 @@ type OpaqueLifecycleSubscriber = Callable[[LifecycleEvent | OpaqueLifecycleEvent], None] type DiagnosticSubscriber = Callable[[DiagnosticLog], None] type HealthCallback = Callable[[DispatchFailure], None] + + +class ArtifactSubscriber(Protocol): + def __call__(self, *, sql: str, statement_id: str | None, resource_id: str | None) -> None: ... + + +class HistorySubscriber(Protocol): + def __call__(self, event: "LifecycleEvent") -> None: ... + + def record_sql( + self, *, sql: str, statement_id: str | None, resource_id: str | None + ) -> None: ... + + type LifecycleRegistration = tuple[ object, KnownLifecycleSubscriber | OpaqueLifecycleSubscriber, bool ] diff --git a/tests/goldens/compiler/failure/target-unknown-key.json b/tests/goldens/compiler/failure/target-unknown-key.json index 238270d0ba..127cffc49b 100644 --- a/tests/goldens/compiler/failure/target-unknown-key.json +++ b/tests/goldens/compiler/failure/target-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: clone, compile_cache, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", + "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: clone, compile_cache, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", "help": "did you mean 'warehouses'?" } ] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 5a2b39a3e8..df194d7a58 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -899,7 +899,11 @@ def child_processes() -> frozenset[int]: children: set[int] = set() for task in (Path("/proc") / str(os.getpid()) / "task").iterdir(): - children.update(map(int, (task / "children").read_text().split())) + try: + listed: str = (task / "children").read_text() + except FileNotFoundError: + continue # the thread exited after the task list was read + children.update(map(int, listed.split())) return frozenset(children) diff --git a/tests/integration/src/sqlbuild/cli/recall/__init__.py b/tests/integration/src/sqlbuild/cli/recall/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py new file mode 100644 index 0000000000..c6d2807663 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -0,0 +1,157 @@ +from dataclasses import dataclass + + +@dataclass(frozen=True) +class RecallOutputTestCase: + description: str + args: tuple[str, ...] + expected_stream: str + + +@dataclass(frozen=True) +class CombinedOutputTestCase: + description: str + expected_order: tuple[bytes, ...] + + +@dataclass(frozen=True) +class RecordedFactsTestCase: + description: str + sql: str + expected_rows: list[list[object]] + + +@dataclass(frozen=True) +class JsonQueryTestCase: + description: str + args: tuple[str, ...] + expected_row_count: int + expected_truncated: bool + expected_columns: list[str] + + +@dataclass(frozen=True) +class TextOutputTestCase: + description: str + args: tuple[str, ...] + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class RecallRejectionTestCase: + description: str + project_suffix: str + args: tuple[str, ...] + expected_code: str + + +@dataclass(frozen=True) +class DisabledRecallTestCase: + description: str + project_suffix: str + expected_store_exists: bool + + +@dataclass(frozen=True) +class ReadOnlyStoreTestCase: + description: str + expected_fragments: tuple[str, ...] + expected_repeat_notice: bool + + +@dataclass(frozen=True) +class AbandonedRunTestCase: + description: str + heartbeat_seconds: str + expected_status: str + expected_output_complete: int + + +@dataclass(frozen=True) +class ConcurrentBuildsTestCase: + description: str + targets: tuple[str, ...] + expected_status: str + + +@dataclass(frozen=True) +class RepeatedBuildTestCase: + description: str + builds: int + sql: str + expected_rows: list[list[object]] + + +@dataclass(frozen=True) +class RecordedCommandTestCase: + description: str + args: tuple[str, ...] + expected_recorded: list[list[object]] + + +@dataclass(frozen=True) +class HeldLockTestCase: + description: str + shutdown_seconds: float + builds_before_lock: int + expected_exit_code: int + + +@dataclass(frozen=True) +class ConcurrentWritersTestCase: + description: str + lines_per_stream: int + expected_order: list[int] + + +@dataclass(frozen=True) +class CorruptStoreTestCase: + description: str + expected_notice_prefix: str + expected_recorded: list[list[object]] + + +@dataclass(frozen=True) +class FailingStderrTestCase: + description: str + expected_exit_code: int + + +@dataclass(frozen=True) +class CorruptStoreRaceTestCase: + description: str + targets: tuple[str, ...] + expected_exit_codes: list[int] + expected_moved: int + expected_recorded: list[list[object]] + + +@dataclass(frozen=True) +class CaptureLimitTestCase: + description: str + extra_toml: str + stdout_bytes: int + expected_kept: tuple[int, int] + expected_gap: list[object] + + +@dataclass(frozen=True) +class TruncationNoteTestCase: + description: str + stdout_bytes: int + expected_kept: int + + +@dataclass(frozen=True) +class InvalidOutputLimitTestCase: + description: str + value: str + expected_message: bytes + + +@dataclass(frozen=True) +class UndecodableArgumentTestCase: + description: str + selector: str + expected_error: bytes + expected_recorded: list[list[object]] diff --git a/tests/integration/src/sqlbuild/cli/recall/conftest.py b/tests/integration/src/sqlbuild/cli/recall/conftest.py new file mode 100644 index 0000000000..fef97213ac --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/recall/conftest.py @@ -0,0 +1,16 @@ +"""Built synthetic projects for local Recall tests.""" + +import subprocess +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.recall.helpers import run, write_project + + +@pytest.fixture(name="built_project") +def fixture_built_project(tmp_path: Path) -> tuple[Path, subprocess.CompletedProcess[bytes]]: + project_dir: Path = write_project(tmp_path / "orders_shop") + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + assert built.returncode == 0, built.stderr.decode(errors="replace") + return project_dir, built diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py new file mode 100644 index 0000000000..423fdcdf46 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -0,0 +1,329 @@ +"""Synthetic DuckDB projects and process helpers for local Recall tests.""" + +import json +import os +import re +import signal +import stat +import subprocess +import sys +import time +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +PYTHON_CLI: Path = Path(sys.executable).with_name("sqlbuild-python") +EMITTED_STDOUT: bytes = b"binary \x00\xff\xfe chunk\n" +EMITTED_STDERR: bytes = "warning \u2713 from the task\n".encode() +EMITTED_TAIL: bytes = b"after the warning\n" +MEBIBYTE: int = 1024 * 1024 +EMPTY_RESULT: bytes = b'{"rows": []}' +NUMBERED_LINE: re.Pattern[bytes] = re.compile(rb"^(\d+) std(?:out|err)$", re.MULTILINE) +PROJECT_TOML: str = ( + 'name = "orders_shop"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connections.local]\n" + 'database = "warehouse.duckdb"\n\n' + "[targets.dev]\n" + 'connection = "local"\n' + 'schema = "dev"\n' +) +ORDERS_MODEL: str = ( + 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + "SELECT 1 AS order_id, CAST('2026-04-01' AS DATE) AS ordered_on\n" +) +EMIT_TASK: str = ( + '"""A task that writes exact bytes to both streams."""\n\n' + "import itertools\n" + "import os\n" + "import sys\n" + "import threading\n" + "import time\n\n" + "from sqlbuild.tasks import task\n\n\n" + "@task()\n" + "def emit_bytes(ctx):\n" + ' """Write binary stdout, UTF-8 stderr, then more stdout."""\n\n' + " sys.stdout.flush()\n" + f" sys.stdout.buffer.write({EMITTED_STDOUT!r})\n" + " sys.stdout.buffer.flush()\n" + f" sys.stderr.write({EMITTED_STDERR.decode()!r})\n" + " sys.stderr.flush()\n" + f" sys.stdout.write({EMITTED_TAIL.decode()!r})\n" + " sys.stdout.flush()\n" + ' time.sleep(float(os.environ.get("ORDERS_TASK_SLEEP_SECONDS", "0")))\n' + ' _write_concurrently(int(os.environ.get("ORDERS_TASK_THREAD_LINES", "0")))\n' + ' sys.stdout.write("y" * int(os.environ.get("ORDERS_TASK_STDOUT_BYTES", "0")))\n' + " sys.stdout.flush()\n" + " return ctx.result()\n\n\n" + "def _write_concurrently(lines):\n" + ' """Two threads write numbered lines; the number gives the true global order."""\n\n' + " lock = threading.Lock()\n" + " counter = itertools.count()\n\n" + " def writer(name, stream):\n" + " for _ in range(lines):\n" + " with lock:\n" + ' stream.write(f"{next(counter)} {name}\\n")\n' + " stream.flush()\n\n" + " threads = [\n" + ' threading.Thread(target=writer, args=("stdout", sys.stdout)),\n' + ' threading.Thread(target=writer, args=("stderr", sys.stderr)),\n' + " ]\n" + " for thread in threads:\n" + " thread.start()\n" + " for thread in threads:\n" + " thread.join()\n" +) + + +def write_project(project_dir: Path, *, extra_toml: str = "", targets: int = 1) -> Path: + """Write a DuckDB project with one model and one byte-emitting task.""" + + toml: str = PROJECT_TOML + extra_toml + for index in range(1, targets): + toml += ( + f"\n[connections.local_{index}]\n" + f'database = "warehouse_{index}.duckdb"\n\n' + f"[targets.dev_{index}]\n" + f'connection = "local_{index}"\n' + 'schema = "dev"\n' + ) + files: dict[str, str] = { + "sqlbuild_project.toml": toml, + "models/orders.sql": ORDERS_MODEL, + "python/tasks/emit.py": EMIT_TASK, + } + for relative, contents in files.items(): + path: Path = project_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + return project_dir + + +def environment(**overrides: str) -> dict[str, str]: + """The process environment for one CLI run.""" + + return {**os.environ, "NO_COLOR": "1", **overrides} + + +def run(project_dir: Path, *args: str, **env: str) -> subprocess.CompletedProcess[bytes]: + """Run the Python CLI once in the project and capture exact streams.""" + + return subprocess.run( + [str(PYTHON_CLI), *args], + cwd=project_dir, + capture_output=True, + check=False, + env=environment(**env), + timeout=600, + ) + + +def start(project_dir: Path, *args: str, **env: str) -> subprocess.Popen[bytes]: + """Start the Python CLI in its own process group.""" + + return subprocess.Popen( + [str(PYTHON_CLI), *args], + cwd=project_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environment(**env), + start_new_session=True, + ) + + +def recall_rows(project_dir: Path, sql: str) -> list[list[object]]: + """The rows `sqb recall query --json` returns.""" + + completed: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "query", "--json", "--no-limit", sql + ) + assert completed.returncode == 0, completed.stderr.decode(errors="replace") + return json.loads(completed.stdout)["rows"] + + +def wait_for(predicate_sql: str, project_dir: Path, *, timeout: float) -> list[list[object]]: + """Poll the store until `predicate_sql` returns a row; a store not yet created has none.""" + + deadline: float = time.monotonic() + timeout + rows: list[list[object]] = [] + while time.monotonic() < deadline and not rows: + time.sleep(0.2) + completed: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "query", "--json", predicate_sql + ) + rows = json.loads(completed.stdout or EMPTY_RESULT)["rows"] + return rows + + +def build_with_read_only_logs( + project_dir: Path, +) -> tuple[subprocess.CompletedProcess[bytes], subprocess.CompletedProcess[bytes]]: + """Two builds while the project's `logs` directory cannot be written.""" + + logs: Path = project_dir / "logs" + logs.mkdir() + logs.chmod(stat.S_IRUSR | stat.S_IXUSR) + try: + return run(project_dir, "build"), run(project_dir, "build") + finally: + logs.chmod(stat.S_IRWXU) + + +def killed_build(project_dir: Path, *, heartbeat_seconds: str) -> tuple[str, list[list[object]]]: + """The running invocation a SIGKILLed build left, and the store's rows for it afterwards.""" + + process: subprocess.Popen[bytes] = start( + project_dir, + "build", + SQLBUILD_RECALL_HEARTBEAT_SECONDS=heartbeat_seconds, + ORDERS_TASK_SLEEP_SECONDS="60", + ) + running: list[list[object]] = wait_for( + "SELECT invocation_id FROM invocations WHERE status = 'running' " + "AND invocation_id IN (SELECT invocation_id FROM recall_output)", + project_dir, + timeout=120, + ) + os.killpg(process.pid, signal.SIGKILL) + _ = process.wait(timeout=30) + time.sleep(5 * float(heartbeat_seconds)) + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT i.invocation_id, i.status, i.ended_at IS NOT NULL, o.complete " + "FROM invocations i JOIN output_invocations o USING (invocation_id)", + ) + return str(running[0][0]), recorded + + +def concurrent_builds(project_dir: Path, targets: tuple[str, ...]) -> tuple[list[int], int, int]: + """Exit codes of concurrent builds, reads made meanwhile and unexpected read failures.""" + + _ = run(project_dir, "recall", "query", "SELECT 1") + builds: list[subprocess.Popen[bytes]] = [ + start(project_dir, "build", "--target", target) for target in targets + ] + reads: int = 0 + unexpected_failures: int = 0 + while any(build.poll() is None for build in builds): + reader: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "query", "--json", "SELECT COUNT(*) FROM invocations" + ) + reads += 1 + unexpected_failures += int(reader.returncode != 0 and b"[C971]" not in reader.stderr) + return [build.wait(timeout=600) for build in builds], reads, unexpected_failures + + +def timed_build(project_dir: Path) -> tuple[subprocess.CompletedProcess[bytes], float, float]: + """A build with the seconds until its first stderr byte and until it exited.""" + + started: float = time.monotonic() + process: subprocess.Popen[bytes] = start(project_dir, "build") + assert process.stderr is not None + first_byte: bytes = process.stderr.read(1) + first_output: float = time.monotonic() - started + stdout, stderr = process.communicate(timeout=600) + return ( + subprocess.CompletedProcess(process.args, process.returncode, stdout, first_byte + stderr), + first_output, + time.monotonic() - started, + ) + + +@contextmanager +def held_store_lock(project_dir: Path) -> Iterator[None]: + """Another process holds the store's write lock while the block runs, creating it if absent.""" + + store: Path = project_dir / "logs" / "recall.sqlite3" + store.parent.mkdir(parents=True, exist_ok=True) + holder: subprocess.Popen[bytes] = subprocess.Popen( + [ + sys.executable, + "-c", + "import sqlite3, sys, time\n" + "connection = sqlite3.connect(sys.argv[1], isolation_level=None)\n" + "connection.execute('BEGIN EXCLUSIVE')\n" + "print('locked', flush=True)\n" + "time.sleep(600)\n", + str(store), + ], + stdout=subprocess.PIPE, + ) + try: + assert holder.stdout is not None + _ = holder.stdout.readline() + yield + finally: + holder.kill() + _ = holder.wait(timeout=30) + + +def build_while_store_locked( + project_dir: Path, +) -> tuple[subprocess.CompletedProcess[bytes], float, float]: + """A timed build while another process holds the store's write lock.""" + + with held_store_lock(project_dir): + return timed_build(project_dir) + + +def compute_log_streams(project_dir: Path, invocation_id: str) -> tuple[bytes, bytes]: + """The stdout and stderr bytes compute logs captured for one invocation.""" + + capture: Path = next((project_dir / "logs").glob(f"*/{invocation_id}")) + return (capture / "stdout.log").read_bytes(), (capture / "stderr.log").read_bytes() + + +def write_corrupt_store(project_dir: Path) -> None: + """Replace the Recall store with a file that is not a database.""" + + store: Path = project_dir / "logs" / "recall.sqlite3" + store.parent.mkdir(parents=True, exist_ok=True) + _ = store.write_bytes(b"this file is not a database\n") + + +FAILING_RECALL_STDERR: str = ( + "import sys\n\n\n" + "class _RejectsRecallNotices:\n" + " def __init__(self, stream):\n" + " self._stream = stream\n\n" + " def write(self, text):\n" + ' if "Recall" in text:\n' + ' raise OSError("stderr rejected the Recall notice")\n' + " return self._stream.write(text)\n\n" + " def __getattr__(self, name):\n" + " return getattr(self._stream, name)\n\n\n" + "sys.stderr = _RejectsRecallNotices(sys.stderr)\n" +) + + +def build_with_failing_recall_notices(project_dir: Path, site_dir: Path) -> int: + """The exit code of a build whose Recall notices fail to write, while the store is locked.""" + + site_dir.mkdir(parents=True, exist_ok=True) + _ = (site_dir / "sitecustomize.py").write_text(FAILING_RECALL_STDERR, encoding="utf-8") + with held_store_lock(project_dir): + return run(project_dir, "build", PYTHONPATH=str(site_dir)).returncode + + +def prepared_project(parent: Path, extra_toml: str, *, builds: int) -> Path: + """A project under `parent`, built `builds` times first.""" + + project_dir: Path = write_project(parent / "orders_shop", extra_toml=extra_toml) + _ = [run(project_dir, "build") for _ in range(builds)] + return project_dir + + +def numbered_lines(output: bytes) -> list[int]: + """The leading numbers of the numbered lines in `output`, in order.""" + + return [int(number) for number in NUMBERED_LINE.findall(output)] + + +def simultaneous_builds(project_dir: Path, targets: tuple[str, ...]) -> list[int]: + """Exit codes of builds of `targets`, each with its own warehouse, started together.""" + + started: list[subprocess.Popen[bytes]] = [ + start(project_dir, "build", "--target", target) for target in targets + ] + return [process.wait(timeout=600) for process in started] diff --git a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py new file mode 100644 index 0000000000..9677a44ee9 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py @@ -0,0 +1,858 @@ +import json +import os +import subprocess +from pathlib import Path + +import jsonschema +import pytest + +from tests.integration.src.sqlbuild.cli.recall._test_types import ( + AbandonedRunTestCase, + CaptureLimitTestCase, + CombinedOutputTestCase, + ConcurrentBuildsTestCase, + ConcurrentWritersTestCase, + CorruptStoreRaceTestCase, + CorruptStoreTestCase, + DisabledRecallTestCase, + FailingStderrTestCase, + HeldLockTestCase, + InvalidOutputLimitTestCase, + JsonQueryTestCase, + ReadOnlyStoreTestCase, + RecallOutputTestCase, + RecallRejectionTestCase, + RecordedCommandTestCase, + RecordedFactsTestCase, + RepeatedBuildTestCase, + TextOutputTestCase, + TruncationNoteTestCase, + UndecodableArgumentTestCase, +) +from tests.integration.src.sqlbuild.cli.recall.helpers import ( + EMITTED_STDERR, + EMITTED_STDOUT, + EMITTED_TAIL, + MEBIBYTE, + build_while_store_locked, + build_with_failing_recall_notices, + build_with_read_only_logs, + compute_log_streams, + concurrent_builds, + killed_build, + numbered_lines, + prepared_project, + recall_rows, + run, + simultaneous_builds, + timed_build, + write_corrupt_store, + write_project, +) + +RECALL_QUERY_SCHEMA: dict[str, object] = { + "type": "object", + "required": [ + "backend", + "target", + "store", + "columns", + "rows", + "row_count", + "truncated", + "limit", + "statistics", + ], + "additionalProperties": False, + "properties": { + "backend": {"const": "logs"}, + "target": {"type": ["string", "null"]}, + "store": { + "type": "object", + "required": ["adapter", "database"], + "properties": {"adapter": {"const": "sqlite"}, "database": {"type": "string"}}, + }, + "columns": {"type": "array", "items": {"type": "string"}}, + "rows": {"type": "array", "items": {"type": "array"}}, + "row_count": {"type": "integer", "minimum": 0}, + "truncated": {"type": "boolean"}, + "limit": {"type": ["integer", "null"]}, + "statistics": { + "type": "object", + "required": ["elapsed_ms", "rows_read", "bytes_read", "query_id"], + "properties": { + "elapsed_ms": {"type": "number"}, + "rows_read": {"type": "null"}, + "bytes_read": {"type": "null"}, + "query_id": {"type": "null"}, + }, + }, + }, +} +DISABLED_TARGET: str = "\n[targets.dev.recall]\nenabled = false\n" +POSIX_PERMISSIONS: bool = os.name == "posix" and os.geteuid() != 0 + + +@pytest.mark.parametrize( + "test_case", + [ + RecallOutputTestCase( + description="stdout", args=("--stream", "stdout"), expected_stream="stdout" + ), + RecallOutputTestCase( + description="stderr", args=("--stream", "stderr"), expected_stream="stderr" + ), + ], + ids=lambda case: case.description, +) +def test_given_a_build_with_binary_output_when_recalled_then_each_stream_is_byte_exact( + test_case: RecallOutputTestCase, + built_project: tuple[Path, subprocess.CompletedProcess[bytes]], +) -> None: + project_dir, built = built_project + + recalled: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "output", "last", *test_case.args + ) + + assert recalled.returncode == 0, recalled.stderr + assert recalled.stdout == getattr(built, test_case.expected_stream) + assert recalled.stderr == b"" + + +@pytest.mark.parametrize( + "test_case", + [ + CombinedOutputTestCase( + description="stdout, then the stderr warning, then more stdout", + expected_order=(EMITTED_STDOUT, EMITTED_STDERR, EMITTED_TAIL), + ), + ], + ids=lambda case: case.description, +) +def test_given_interleaved_writes_when_recalled_combined_then_order_is_preserved( + test_case: CombinedOutputTestCase, + built_project: tuple[Path, subprocess.CompletedProcess[bytes]], +) -> None: + project_dir, built = built_project + + combined: bytes = run(project_dir, "recall", "output", "last").stdout + transcript: list[list[object]] = recall_rows( + project_dir, + "SELECT hex(stdout), hex(stderr) FROM output_transcripts t " + "JOIN output_invocations o USING (invocation_id) WHERE o.complete = 1", + ) + + positions: list[int] = [combined.index(piece) for piece in test_case.expected_order] + assert positions == sorted(positions) + assert len(combined) == len(built.stdout) + len(built.stderr) + assert transcript == [[built.stdout.hex().upper(), built.stderr.hex().upper()]] + + +@pytest.mark.parametrize( + "test_case", + [ + RecordedFactsTestCase( + description="the invocation carries its command, target and adapter", + sql="SELECT status, command, target, adapter, exit_code FROM invocations", + expected_rows=[["completed", "build", "dev", "duckdb", 0]], + ), + RecordedFactsTestCase( + description="every statement completed and kept its SQL", + sql="SELECT COUNT(*) > 0, SUM(outcome = 'completed') = COUNT(*), " + "SUM(sql_available) = COUNT(*) FROM statements", + expected_rows=[[1, 1, 1]], + ), + RecordedFactsTestCase( + description="natively encoded events keep payloads, promotions and UTC timestamps", + sql="SELECT COUNT(*) > 0, " + "SUM(payload = '{}' OR statement_kind IS NULL OR intent IS NULL), " + "SUM(occurred_at NOT GLOB " + "'[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9].[0-9][0-9][0-9][0-9][0-9][0-9]Z'), " + "SUM(invocation_sequence IS NULL) " + "FROM events WHERE event_type LIKE 'statement_%'", + expected_rows=[[1, 0, 0, 0]], + ), + RecordedFactsTestCase( + description="the started event names target and adapter", + sql="SELECT json_extract(payload, '$.target'), json_extract(payload, '$.adapter') " + "FROM events WHERE event_type = 'invocation_started'", + expected_rows=[["dev", "duckdb"]], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_build_when_recalled_then_invocation_statements_and_events_are_queryable( + test_case: RecordedFactsTestCase, + built_project: tuple[Path, subprocess.CompletedProcess[bytes]], +) -> None: + project_dir, _ = built_project + + rows: list[list[object]] = recall_rows(project_dir, test_case.sql) + + assert rows == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + JsonQueryTestCase( + description="a limited query reports truncation", + args=( + "--limit", + "1", + "SELECT invocation_id, data FROM recall_output ORDER BY sequence", + ), + expected_row_count=1, + expected_truncated=True, + expected_columns=["invocation_id", "data"], + ), + JsonQueryTestCase( + description="an unlimited query is complete", + args=("--no-limit", "SELECT COUNT(*) AS total FROM invocations"), + expected_row_count=1, + expected_truncated=False, + expected_columns=["total"], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_recall_query_when_rendered_as_json_then_it_matches_the_schema( + test_case: JsonQueryTestCase, + built_project: tuple[Path, subprocess.CompletedProcess[bytes]], +) -> None: + project_dir, _ = built_project + + queried: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "query", "--json", *test_case.args + ) + + document: dict[str, object] = json.loads(queried.stdout) + jsonschema.validate(document, RECALL_QUERY_SCHEMA) + assert document["row_count"] == test_case.expected_row_count + assert document["truncated"] is test_case.expected_truncated + assert document["columns"] == test_case.expected_columns + + +@pytest.mark.parametrize( + "test_case", + [ + TextOutputTestCase( + description="long cells are truncated with the remaining byte count", + args=( + "query", + "--max-cell-bytes", + "8", + "SELECT 'first line' || char(10) || 'second line' AS note", + ), + expected_fragments=("note | first li…[+14 bytes]", "1 row"), + ), + TextOutputTestCase( + description="the schema lists views and join keys", + args=("schema",), + expected_fragments=( + "output_transcripts (view)", + "invocation_id TEXT not null join key", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_recall_text_output_when_rendered_then_cells_and_schema_are_readable( + test_case: TextOutputTestCase, + built_project: tuple[Path, subprocess.CompletedProcess[bytes]], +) -> None: + project_dir, _ = built_project + + rendered: subprocess.CompletedProcess[bytes] = run(project_dir, "recall", *test_case.args) + + stdout: str = rendered.stdout.decode() + assert rendered.returncode == 0 + assert [fragment in stdout for fragment in test_case.expected_fragments] == [ + True for _ in test_case.expected_fragments + ] + + +@pytest.mark.parametrize( + "test_case", + [ + RecallRejectionTestCase( + description="disabled target", + project_suffix=DISABLED_TARGET, + args=("recall", "query", "SELECT 1"), + expected_code="C970", + ), + RecallRejectionTestCase( + description="unknown invocation", + project_suffix="", + args=("recall", "output", "no-such-invocation"), + expected_code="C972", + ), + RecallRejectionTestCase( + description="write statement", + project_suffix="", + args=("recall", "query", "DELETE FROM recall_output"), + expected_code="C974", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_recall_requests_when_run_then_coded_errors_are_reported( + test_case: RecallRejectionTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop", extra_toml=test_case.project_suffix) + _ = run(project_dir, "build") + + rejected: subprocess.CompletedProcess[bytes] = run(project_dir, *test_case.args) + + assert rejected.returncode == 1 + assert f"[{test_case.expected_code}]" in rejected.stderr.decode() + + +@pytest.mark.parametrize( + "test_case", + [ + DisabledRecallTestCase( + description="a disabled target writes no store", + project_suffix=DISABLED_TARGET, + expected_store_exists=False, + ), + DisabledRecallTestCase( + description="the default writes the local store", + project_suffix="", + expected_store_exists=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_recall_settings_when_building_then_the_local_store_follows_them( + test_case: DisabledRecallTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop", extra_toml=test_case.project_suffix) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + assert built.returncode == 0 + assert (project_dir / "logs" / "recall.sqlite3").exists() is test_case.expected_store_exists + + +@pytest.mark.skipif(not POSIX_PERMISSIONS, reason="needs POSIX permissions as a non-root user") +@pytest.mark.parametrize( + "test_case", + [ + ReadOnlyStoreTestCase( + description="the notice carries the DDL once and the build still succeeds", + expected_fragments=( + "Recall history is not being written for target `dev`", + 'CREATE TABLE IF NOT EXISTS "recall_output"', + 'CREATE VIEW IF NOT EXISTS "invocations"', + ), + expected_repeat_notice=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_read_only_logs_directory_when_building_then_ddl_is_printed_and_exit_is_zero( + test_case: ReadOnlyStoreTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + + built, repeated = build_with_read_only_logs(project_dir) + + stderr: str = built.stderr.decode(errors="replace") + assert (built.returncode, repeated.returncode) == (0, 0) + assert [fragment in stderr for fragment in test_case.expected_fragments] == [ + True for _ in test_case.expected_fragments + ] + assert "Completed successfully" in built.stdout.decode(errors="replace") + assert ( + "Recall history is not being written" in repeated.stderr.decode(errors="replace") + ) is test_case.expected_repeat_notice + + +@pytest.mark.parametrize( + "test_case", + [ + AbandonedRunTestCase( + description="a SIGKILLed build is abandoned with incomplete output", + heartbeat_seconds="0.3", + expected_status="abandoned", + expected_output_complete=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_killed_build_when_heartbeats_stop_then_the_invocation_is_abandoned( + test_case: AbandonedRunTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + + running_id, recorded = killed_build(project_dir, heartbeat_seconds=test_case.heartbeat_seconds) + + assert recorded == [ + [running_id, test_case.expected_status, 1, test_case.expected_output_complete] + ] + + +@pytest.mark.parametrize( + "test_case", + [ + ConcurrentBuildsTestCase( + description="four builds and a polling reader share one store", + targets=("dev", "dev_1", "dev_2", "dev_3"), + expected_status="completed", + ), + ], + ids=lambda case: case.description, +) +def test_given_four_concurrent_builds_and_a_reader_when_run_then_every_run_is_recorded( + test_case: ConcurrentBuildsTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop", targets=len(test_case.targets)) + + exit_codes, reads, unexpected_failures = concurrent_builds(project_dir, test_case.targets) + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT i.target, i.status, o.complete FROM invocations i " + "JOIN output_invocations o USING (invocation_id) ORDER BY i.target", + ) + + assert exit_codes == [0 for _ in test_case.targets] + assert reads > 0 + assert unexpected_failures == 0 + assert recorded == [ + [target, test_case.expected_status, 1] for target in sorted(test_case.targets) + ] + + +@pytest.mark.parametrize( + "test_case", + [ + RepeatedBuildTestCase( + description="repeated SQL is shared, never a conflict", + builds=2, + sql="SELECT (SELECT COUNT(*) FROM conflicts), " + "(SELECT SUM(conflicts) FROM recall_delivery)", + expected_rows=[[0, 0]], + ), + RepeatedBuildTestCase( + description="every stream of every build delivers exactly what it accepted", + builds=2, + sql="SELECT COUNT(DISTINCT invocation_id), " + "SUM(accepted != delivered + filtered + dropped + failed), " + "SUM(dropped + failed), MIN(flush_complete) FROM recall_delivery", + expected_rows=[[2, 0, 0, 1]], + ), + ], + ids=lambda case: case.description, +) +def test_given_identical_builds_when_recorded_then_artifacts_and_delivery_counts_stay_exact( + test_case: RepeatedBuildTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + + exit_codes: list[int] = [run(project_dir, "build").returncode for _ in range(test_case.builds)] + rows: list[list[object]] = recall_rows(project_dir, test_case.sql) + + assert exit_codes == [0 for _ in range(test_case.builds)] + assert rows == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + [ + RecordedCommandTestCase( + description="compile", + args=("compile",), + expected_recorded=[["compile", "completed"]], + ), + RecordedCommandTestCase( + description="build", + args=("build",), + expected_recorded=[["build", "completed"]], + ), + RecordedCommandTestCase( + description="plan", + args=("plan",), + expected_recorded=[["plan", "completed"]], + ), + RecordedCommandTestCase( + description="dag", + args=("dag",), + expected_recorded=[["dag", "completed"]], + ), + RecordedCommandTestCase( + description="check", + args=("check",), + expected_recorded=[["check", "completed"]], + ), + RecordedCommandTestCase( + description="test", + args=("test",), + expected_recorded=[["test", "completed"]], + ), + RecordedCommandTestCase( + description="lineage orders", + args=("lineage", "orders"), + expected_recorded=[["lineage", "completed"]], + ), + RecordedCommandTestCase( + description="debug", + args=("debug",), + expected_recorded=[["debug", "completed"]], + ), + RecordedCommandTestCase( + description="query SELECT 1 AS one", + args=("query", "SELECT 1 AS one"), + expected_recorded=[["query", "completed"]], + ), + RecordedCommandTestCase( + description="a raw-query diff is not observed, so it is not recorded", + args=( + "diff", + "--unkeyed", + "--left-query", + "SELECT 1 AS one", + "--right-query", + "SELECT 1 AS one", + ), + expected_recorded=[], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_successful_command_when_recorded_then_it_ends_completed( + test_case: RecordedCommandTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + _ = run(project_dir, "build") + + completed: subprocess.CompletedProcess[bytes] = run(project_dir, *test_case.args) + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT command, status FROM invocations " + f"WHERE command = '{test_case.args[0]}' ORDER BY invocation_started_at DESC LIMIT 1", + ) + + assert completed.returncode == 0, completed.stderr.decode(errors="replace") + assert recorded == test_case.expected_recorded + + +@pytest.mark.parametrize( + "test_case", + [ + HeldLockTestCase( + description="a store locked by another process costs at most the budget", + shutdown_seconds=1.0, + builds_before_lock=1, + expected_exit_code=0, + ), + HeldLockTestCase( + description="a store another process holds before setup never delays the start", + shutdown_seconds=1.0, + builds_before_lock=0, + expected_exit_code=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_locked_store_when_building_then_start_and_exit_each_add_at_most_the_budget( + test_case: HeldLockTestCase, tmp_path: Path +) -> None: + budget: str = f'\n[recall]\nshutdown_timeout = "{int(test_case.shutdown_seconds * 1000)}ms"\n' + baseline_dir: Path = prepared_project(tmp_path / "baseline", budget, builds=1) + project_dir: Path = prepared_project( + tmp_path / "locked", budget, builds=test_case.builds_before_lock + ) + _, baseline_first, baseline_total = timed_build(baseline_dir) + + locked, locked_first, locked_total = build_while_store_locked(project_dir) + + slack: float = 0.75 + assert locked.returncode == test_case.expected_exit_code + assert locked_first <= baseline_first + slack + assert locked_total <= baseline_total + test_case.shutdown_seconds + slack + assert ( + locked.stderr.decode(errors="replace") + .rstrip() + .endswith("completed successfully; only its Recall history is affected.") + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ConcurrentWritersTestCase( + description="two threads writing both streams keep one global order", + lines_per_stream=400, + expected_order=list(range(800)), + ), + ], + ids=lambda case: case.description, +) +def test_given_concurrent_stream_writers_when_recalled_then_order_matches_compute_logs( + test_case: ConcurrentWritersTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + + built: subprocess.CompletedProcess[bytes] = run( + project_dir, "build", ORDERS_TASK_THREAD_LINES=str(test_case.lines_per_stream) + ) + [[invocation_id, stdout_hex, stderr_hex, combined_hex]] = recall_rows( + project_dir, + "SELECT invocation_id, hex(stdout), hex(stderr), hex(combined) FROM output_transcripts", + ) + numbered: list[int] = numbered_lines(bytes.fromhex(str(combined_hex))) + + assert built.returncode == 0, built.stderr.decode(errors="replace") + assert (bytes.fromhex(str(stdout_hex)), bytes.fromhex(str(stderr_hex))) == ( + compute_log_streams(project_dir, str(invocation_id)) + ) + assert numbered == test_case.expected_order + + +@pytest.mark.parametrize( + "test_case", + [ + CorruptStoreTestCase( + description="a store that is not a database is moved aside before any output", + expected_notice_prefix="Recall: ", + expected_recorded=[["build", "completed"]], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_corrupt_store_when_building_then_it_is_moved_aside_and_recording_restarts( + test_case: CorruptStoreTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + write_corrupt_store(project_dir) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + again: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + recorded: list[list[object]] = recall_rows( + project_dir, "SELECT command, status FROM invocations ORDER BY invocation_started_at" + ) + moved: list[Path] = list((project_dir / "logs").glob("recall.sqlite3.corrupt-*")) + first_line: str = built.stderr.decode(errors="replace").splitlines()[0] + + assert (built.returncode, again.returncode) == (0, 0) + assert first_line.startswith(test_case.expected_notice_prefix) + assert str(moved[0].name) in first_line + assert len(moved) == 1 + assert b".corrupt-" not in again.stderr + assert recorded == [*test_case.expected_recorded, *test_case.expected_recorded] + + +@pytest.mark.parametrize( + "test_case", + [ + FailingStderrTestCase( + description="a Recall notice that cannot be written keeps the exit code", + expected_exit_code=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_failing_stderr_when_recall_reports_then_the_exit_code_is_unchanged( + test_case: FailingStderrTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project( + tmp_path / "orders_shop", extra_toml='\n[recall]\nshutdown_timeout = "300ms"\n' + ) + _ = run(project_dir, "build") + + exit_code: int = build_with_failing_recall_notices(project_dir, tmp_path / "site") + + assert exit_code == test_case.expected_exit_code + + +@pytest.mark.parametrize( + "test_case", + [ + CorruptStoreRaceTestCase( + description="builds starting together on a corrupt store move it aside exactly once", + targets=("dev", "dev_1"), + expected_exit_codes=[0, 0], + expected_moved=1, + expected_recorded=[["build", "completed", 2]], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_corrupt_store_when_builds_race_then_one_moves_it_and_both_are_recorded( + test_case: CorruptStoreRaceTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop", targets=len(test_case.targets)) + _ = run(project_dir, "compile") + write_corrupt_store(project_dir) + + exit_codes: list[int] = simultaneous_builds(project_dir, test_case.targets) + moved: list[Path] = list((project_dir / "logs").glob("recall.sqlite3.corrupt-*[0-9Z]")) + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT command, status, COUNT(*) FROM invocations WHERE command = 'build' " + "GROUP BY command, status", + ) + + assert exit_codes == test_case.expected_exit_codes + assert len(moved) == test_case.expected_moved + assert recorded == test_case.expected_recorded + + +@pytest.mark.parametrize( + "test_case", + [ + CaptureLimitTestCase( + description="under the default limit stdout is byte-exact", + extra_toml="", + stdout_bytes=1536 * 1024, + expected_kept=(MEBIBYTE * 1024, 0), + expected_gap=[None, 1], + ), + CaptureLimitTestCase( + description="past the default limit the first and last MiB are kept exactly", + extra_toml="", + stdout_bytes=3 * MEBIBYTE, + expected_kept=(MEBIBYTE, MEBIBYTE), + expected_gap=["truncated", 0], + ), + CaptureLimitTestCase( + description="a larger target limit keeps everything", + extra_toml="\n[targets.dev.recall]\nmax_output_bytes = 8388608\n", + stdout_bytes=3 * MEBIBYTE, + expected_kept=(MEBIBYTE * 1024, 0), + expected_gap=[None, 1], + ), + CaptureLimitTestCase( + description="zero records all output", + extra_toml="\n[recall]\nmax_output_bytes = 0\n", + stdout_bytes=3 * MEBIBYTE, + expected_kept=(MEBIBYTE * 1024, 0), + expected_gap=[None, 1], + ), + ], + ids=lambda case: case.description, +) +def test_given_an_output_limit_when_recorded_then_kept_bytes_are_exact_around_one_gap( + test_case: CaptureLimitTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop", extra_toml=test_case.extra_toml) + + built: subprocess.CompletedProcess[bytes] = run( + project_dir, "build", ORDERS_TASK_STDOUT_BYTES=str(test_case.stdout_bytes) + ) + [[stdout_hex, gap_reason, complete, truncated, warning_first]] = recall_rows( + project_dir, + "SELECT hex(t.stdout), o.gap_reason, o.complete, o.truncated_bytes, " + "instr(CAST(t.combined AS TEXT), 'from the task') < instr(t.combined, 'yyyy') " + "FROM output_invocations o JOIN output_transcripts t USING (invocation_id)", + ) + head, tail = test_case.expected_kept + expected: bytes = built.stdout[:head] + built.stdout[len(built.stdout) - tail :] + + assert built.returncode == 0, built.stderr.decode(errors="replace") + assert bytes.fromhex(str(stdout_hex)) == expected + assert [gap_reason, complete] == test_case.expected_gap + assert truncated == len(built.stdout) - len(expected) + assert warning_first == 1 + assert b"max_output_bytes" not in built.stdout + built.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + TruncationNoteTestCase( + description="the note sits where stdout was left out", + stdout_bytes=3 * MEBIBYTE, + expected_kept=MEBIBYTE, + ), + ], + ids=lambda case: case.description, +) +def test_given_truncated_output_when_shown_then_a_note_marks_the_gap( + test_case: TruncationNoteTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + built: subprocess.CompletedProcess[bytes] = run( + project_dir, "build", ORDERS_TASK_STDOUT_BYTES=str(test_case.stdout_bytes) + ) + + shown: subprocess.CompletedProcess[bytes] = run( + project_dir, "recall", "output", "last", "--stream", "stdout" + ) + omitted: int = len(built.stdout) - 2 * test_case.expected_kept + note: bytes = ( + f"\u2026 {omitted / 1_000_000:.1f} MB of stdout not recorded (kept the first and last " + "1 MiB). Raise [recall] max_output_bytes to keep more.\n" + ).encode() + + assert shown.returncode == 0, shown.stderr.decode(errors="replace") + assert shown.stdout == ( + built.stdout[: test_case.expected_kept] + + b"\n" + + note + + built.stdout[len(built.stdout) - test_case.expected_kept :] + ) + assert shown.stderr == b"" + + +@pytest.mark.parametrize( + "test_case", + [ + InvalidOutputLimitTestCase( + description="a negative limit is a configuration error", + value="-1", + expected_message=b"error[D001]", + ), + InvalidOutputLimitTestCase( + description="a size string is a configuration error", + value='"2MiB"', + expected_message=b"recall.max_output_bytes must be a whole number of bytes", + ), + ], + ids=lambda case: case.description, +) +def test_given_an_invalid_output_limit_when_running_then_the_command_fails_with_its_code( + test_case: InvalidOutputLimitTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project( + tmp_path / "orders_shop", extra_toml=f"\n[recall]\nmax_output_bytes = {test_case.value}\n" + ) + + completed: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + + assert completed.returncode != 0 + assert test_case.expected_message in completed.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + UndecodableArgumentTestCase( + description="an undecodable selector is recorded escaped; the command fails as before", + selector="orders\udcff", + expected_error=b"TypeError: str is not valid UTF-8: surrogates not allowed", + expected_recorded=[['["orders\\\\xff"]', 1]], + ), + ], + ids=lambda case: case.description, +) +def test_given_an_undecodable_argument_when_recorded_then_it_is_escaped_and_the_command_unchanged( + test_case: UndecodableArgumentTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + + completed: subprocess.CompletedProcess[bytes] = run( + project_dir, "compile", "--json", "--select", test_case.selector + ) + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT i.selection, json_extract(e.payload, '$.selection_count') " + "FROM invocations i JOIN events e USING (invocation_id) " + "WHERE e.event_type = 'invocation_started'", + ) + + assert completed.returncode == 1 + assert completed.stderr.rstrip().endswith(test_case.expected_error) + assert recorded == test_case.expected_recorded diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py index 6b9654ef5c..8fe68caed4 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/helpers.py @@ -143,7 +143,11 @@ def child_processes() -> frozenset[int]: children: set[int] = set() for task in (Path("/proc") / str(os.getpid()) / "task").iterdir(): - children.update(map(int, (task / "children").read_text().split())) + try: + listed: str = (task / "children").read_text() + except FileNotFoundError: + continue # the thread exited after the task list was read + children.update(map(int, listed.split())) return frozenset(children) diff --git a/website/src/content/docs/docs/cli/recall.mdx b/website/src/content/docs/docs/cli/recall.mdx new file mode 100644 index 0000000000..3879c74d27 --- /dev/null +++ b/website/src/content/docs/docs/cli/recall.mdx @@ -0,0 +1,116 @@ +--- +title: "sqb recall" +description: "Query the recorded history of past invocations." +sidebar: + label: "recall" +--- + +Read the local [Recall](/docs/concepts/observability/recall) store: past invocations, the +statements they executed, and what they printed. `sqb recall` never records itself. + +## Usage + +```bash +sqb recall query "SELECT command, status, duration_ms FROM invocations ORDER BY invocation_started_at DESC" +sqb recall schema +sqb recall output last +``` + +## sqb recall query + +Run one read-only SQLite `SELECT` against the Recall views. Statements that write, and multiple +statements, are rejected. + +| Flag | Description | +|------|-------------| +| `sql` | The query (positional argument) | +| `--format` | `long` (default), `table`, `json`, or `csv` | +| `--json` | Shorthand for `--format json` | +| `--limit` | Maximum rows to return (default: 100) | +| `--no-limit` | Disable the row limit | +| `--max-cell-bytes` | Longest cell in text formats, in bytes (default: 4096; `0` for no limit) | +| `--target` | Target whose store to read (default: the default target) | +| `--backend` | `logs` (default) reads the Recall store; `warehouse` runs the query on the target's connection | + +The latest invocation in a project named `orders_shop`: + +```bash +sqb recall query --format table \ + "SELECT command, status, exit_code, duration_ms FROM invocations + ORDER BY invocation_started_at DESC LIMIT 1" +``` + +``` +command | status | exit_code | duration_ms +------- | --------- | --------- | ----------- +build | completed | 0 | 752.0 + +1 row +``` + +The slowest statements of past builds, with their SQL: + +```bash +sqb recall query --format table --max-cell-bytes 60 \ + "SELECT s.resource_name, round(s.duration_ms, 1) AS ms, a.text AS sql + FROM statements s JOIN artifacts a USING (invocation_id, statement_id) + WHERE s.statement_kind = 'CREATE' ORDER BY s.duration_ms DESC LIMIT 2" +``` + +``` +resource_name | ms | sql +------------- | --- | ------------------------------------------------------------------------ +orders | 3.6 | CREATE OR REPLACE TABLE dev.orders__staging AS SELECT 1 …[+74 bytes] +NULL | 3.2 | CREATE SCHEMA IF NOT EXISTS dev + +2 rows +``` + +Failed invocations of the last week: + +```bash +sqb recall query \ + "SELECT invocation_id, command, error_code FROM invocations + WHERE status = 'failed' AND invocation_started_at >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-7 days')" +``` + +Timestamps are ISO-8601 UTC text, so they compare correctly as strings. Filter on +`invocation_started_at` to bound a scan. + +## sqb recall schema + +List every Recall view with its columns, types, and join keys. Accepts `--format`, `--json`, and +`--target`. With `--backend warehouse`, it lists SQLBuild's state tables in the target's warehouse +instead, or describes one with `--relation `. + +## sqb recall output + +Print what an invocation wrote, byte for byte. + +```bash +sqb recall output last # the latest invocation, both streams in order +sqb recall output 7607b5b6 --stream stderr # an invocation ID or unique prefix +sqb recall output last --tail 20 # only the last 20 lines +``` + +| Flag | Description | +|------|-------------| +| `invocation` | Invocation ID, unique prefix, or `last` (default) | +| `--stream` | `combined` (default), `stdout`, or `stderr` | +| `--tail` | Print only the last N lines | +| `--target` | Target whose store to read | + +Where output past [`max_output_bytes`](/docs/concepts/observability/recall#output-limit) was left +out, a note line marks the gap in place. Output that Recall could not record completely for any +other reason is followed by a note on stderr naming the gap. + +## Errors + +| Code | Meaning | +|------|---------| +| `C970` | Recall is disabled for this project or target | +| `C971` | The store cannot be read, or no history is recorded yet | +| `C972` | No invocation matches | +| `C973` | The invocation prefix matches more than one invocation | +| `C974` | The query is invalid or not read-only | +| `C975` | The `[recall]` settings are invalid | diff --git a/website/src/content/docs/docs/concepts/observability.mdx b/website/src/content/docs/docs/concepts/observability.mdx index d747362233..7819885ff9 100644 --- a/website/src/content/docs/docs/concepts/observability.mdx +++ b/website/src/content/docs/docs/concepts/observability.mdx @@ -11,6 +11,7 @@ whether work started, completed, failed, skipped, or retried. | `LifecycleEvent` | Immutable invocation, run, resource-attempt, operation, statement, retry, and audit facts | Canonical lifecycle evidence once durably stored | | `DiagnosticLog` | Structured framework diagnostics | Explanatory only | | `stdout.log` / `stderr.log` | Exact host-local process output | Troubleshooting transcript | +| [Recall](/docs/concepts/observability/recall) | Local queryable history of invocations, statements, SQL, and output | Troubleshooting history; best effort | | `CommandOutputRecord` | Bounded, ANSI-free remote stdout/stderr chunks | Potentially sensitive and lossy troubleshooting data | | Final JSON output | End-of-command aggregate | Result projection, not event history | @@ -80,4 +81,5 @@ A start without a terminal fact is unknown or presumed lost. Consumers must not or failure. Sink delivery is bounded and best effort; destination failure does not change successful warehouse work, and successful delivery does not prove the command succeeded. -See [Typed Sinks](/docs/concepts/observability/sinks) to export lifecycle and command-output records. +See [Recall](/docs/concepts/observability/recall) for the local history `sqb recall` queries, and +[Typed Sinks](/docs/concepts/observability/sinks) to export lifecycle and command-output records. diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx new file mode 100644 index 0000000000..959319b3b8 --- /dev/null +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -0,0 +1,132 @@ +--- +title: "Recall" +description: "The local history of every command: what ran, what it executed, and what it printed." +--- + +Recall records each `sqb` invocation in a local SQLite store so you can ask what happened after the +fact: which command ran, how it ended, which statements it executed and how long they took, and +exactly what it printed. Recall is on by default and needs no setup. + +## What is recorded + +For every recorded invocation, Recall keeps: + +- the invocation itself: command, arguments, target, adapter, status, exit code, and timing; +- its [lifecycle events](/docs/concepts/observability): runs, resource attempts, statements, + retries, and audit results; +- the SQL text of every executed statement, stored once per distinct statement and shared across + invocations; +- its stdout and stderr, byte for byte, in the order they were written, up to + [`max_output_bytes`](#output-limit) of each stream; +- framework diagnostics at or above `diagnostics_min_severity`. + +Writes happen on a background thread and never fail the command. A command that cannot record all +of its history still succeeds, prints a warning followed by its own final status, and +`output_invocations` reports the gap. An invocation whose process died without finishing has +status `abandoned`. + +Every command is recorded except `sqb recall` itself, commands that create a project (such as +`sqb init`), and raw-query `sqb diff`. A `sqb compile` answered from the launcher's warm cache is not +recorded either. + +The store is `logs/recall.sqlite3` in the project directory. Add `logs/` to `.gitignore`; the store +can contain sensitive SQL and output. + +## Settings + +Configure Recall in `[recall]` in `sqlbuild_project.toml`. A `[targets..recall]` table +overrides individual keys for one target. + +```toml +[recall] +retention = "30d" # days of history to keep; also accepts hours, such as "720h" +shutdown_timeout = "3s" # longest delay Recall may add when a command exits +diagnostics_min_severity = "info" +max_output_bytes = 2097152 # bytes of each output stream to keep; 0 keeps everything + +[targets.ci.recall] +enabled = false # do not record runs against the ci target +``` + +| Key | Default | Description | +|-----|---------|-------------| +| `enabled` | `true` | Set to `false` to stop recording | +| `connection` | (none) | A `sqlite` connection whose `path` names the store; defaults to `logs/recall.sqlite3` | +| `retention` | `"30d"` | How long invocations are kept, in days (`d`) or hours (`h`), up to 3650 days | +| `shutdown_timeout` | `"3s"` | The cap on time spent flushing history at exit, in `s` or `ms`, up to 10 minutes | +| `diagnostics_min_severity` | `"info"` | `trace`, `debug`, `info`, `warning`, `error`, or `critical` | +| `max_output_bytes` | `2097152` for the local store | Bytes of stdout and of stderr kept per invocation, an integer; `0` keeps everything. See [Output limit](#output-limit) | + +To turn Recall off for the whole project: + +```toml +[recall] +enabled = false +``` + +Recall never delays the start of a command. At exit it waits at most `shutdown_timeout` for queued +history to reach the store; anything left is counted as dropped. A second Ctrl-C stops waiting at +once. + +## Output limit + +Recall keeps at most `max_output_bytes` of each output stream per invocation: the first half and +the last half, byte for byte. The local SQLite store defaults to 2 MiB, so a command that prints +more keeps its first and last MiB of stdout and of stderr. Set `max_output_bytes = 0` to keep +everything, or a larger number to keep more; a `[targets..recall]` table can set it for one +target. The command itself prints nothing about the limit. + +The middle that was left out is one gap: `output_invocations` reports `gap_reason = 'truncated'`, +the left-out byte count in `truncated_bytes`, and `complete = 0`. `sqb recall output` prints a +note in its place: + +``` +… 23.6 MB of stdout not recorded (kept the first and last 1 MiB). Raise [recall] max_output_bytes to keep more. +``` + +The end of a stream is held in memory until the command exits, so a process that is killed keeps +only the first half. + +## Retention + +Commands occasionally prune invocations older than `retention`, with all of their events, output, +and diagnostics. SQL text is removed once no remaining invocation references it. Pruning runs in +the background within the exit budget and never removes an invocation that is still running. + +## When the store cannot be written + +If the store location is not writable, the command still runs and prints a notice on stderr with +the SQL an administrator can use to create the store; the notice comes before the command's own +output whenever the problem is visible from file permissions. The notice +repeats at most once a day. If the store file is not a SQLite database, Recall moves it aside to +`recall.sqlite3.corrupt-`, starts a fresh store, and says so once. Commands that +find the same corrupt file coordinate through `recall.sqlite3.lock`, so only one of them moves it. + +A `connection` that does not use the `sqlite` adapter is not supported yet; Recall prints a notice +and records nothing. + +## Querying history + +Use [`sqb recall`](/docs/cli/recall) to read the store. Queries are read-only SQLite `SELECT` +statements against these views: + +| View | One row per | +|------|-------------| +| `invocations` | Invocation, with status, exit code, and timing | +| `statements` | Executed statement, with resource, kind, purpose, duration, and row counts | +| `audit_results` | Evaluated audit | +| `events` | Lifecycle event | +| `diagnostics` | Retained diagnostic | +| `artifacts` | Reference from an invocation to captured SQL, with its text | +| `output_invocations` | Invocation's output, with completeness | +| `output_transcripts` | Invocation's stdout, stderr, and combined output | +| `output_tail` | Recent output chunk | +| `conflicts` | Record written twice with different content | + +Run `sqb recall schema` for every column. + +## Relationship to compute logs + +Compute logs (`logs///stdout.log`, `stderr.log`, and `diagnostics.jsonl`) are +still written alongside Recall. Recall stores the same output bytes, captured at the same point, so +`sqb recall output` matches the compute log files. diff --git a/website/src/sidebar.json b/website/src/sidebar.json index 05f688d4f2..770ed3304d 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -136,6 +136,7 @@ "collapsed": true, "items": [ "docs/concepts/observability", + "docs/concepts/observability/recall", "docs/concepts/observability/sinks", "docs/concepts/observability/execution-history" ] @@ -216,6 +217,7 @@ "docs/cli/lineage", "docs/cli/dag", "docs/cli/query", + "docs/cli/recall", "docs/cli/debug", "docs/cli/janitor", "docs/cli/clean",