refactor(native): move discovery, scopes and the model config build into plain-Rust crates

crates/sqlbuild-discovery/Cargo.toml

@@ -9,6 +9,7 @@ rust-version.workspace = true
 
 [dependencies]
 rayon.workspace = true
+sqlbuild-cache.workspace = true
 sqlbuild-config.workspace = true
 sqlbuild-core.workspace = true
 sqlbuild-model-config.workspace = true

crates/sqlbuild-discovery/src/_helpers/mod.rs

@@ -4,5 +4,6 @@ pub(crate) mod locations;
 pub(crate) mod pool;
 pub(crate) mod reading;
 pub(crate) mod scoped_paths;
+pub(crate) mod sorted_keys;
 pub(crate) mod statement_headers;
 pub(crate) mod suggestions;

crates/sqlbuild-python/src/bindings/_helpers/discovery/sorted_keys.rs → crates/sqlbuild-discovery/src/_helpers/sorted_keys.rs

@@ -1,4 +1,4 @@
-//! Header key sets as the sorted lists native discovery options take.
+//! Header key sets as the sorted lists the discovery options take.
 
 /// A header key set as owned keys, sorted by code point as Python sorts them.
 pub(crate) fn sorted_keys(keys: &[&str]) -> Vec<String> {

crates/sqlbuild-discovery/src/constants.rs

@@ -47,3 +47,124 @@ pub const AUDIT_ROLE_DIRECTORY: &str = "audits";
 pub const LOCAL_AUDIT_ROLE_DIRECTORY: &str = "_audits";
 /// The singular audit directory below an audit role.
 pub const SINGULAR_AUDIT_DIRECTORY: &str = "singular";
+/// Header keys a `MODEL` statement accepts, sorted by code point.
+pub const SQL_MODEL_HEADER_KEYS: [&str; 73] = [
+    "alias",
+    "append_cursor_inclusive",
+    "audit_factories",
+    "audits",
+    "batch_concurrency",
+    "batch_size",
+    "check_columns",
+    "columns",
+    "config",
+    "constants",
+    "contract",
+    "cursor",
+    "cursor_end",
+    "cursor_future_action",
+    "cursor_future_max_distance",
+    "cursor_grain",
+    "cursor_inputs",
+    "cursor_start",
+    "cursor_start_max_action",
+    "cursor_start_max_ahead",
+    "cursor_type",
+    "cursor_watermark_mode",
+    "database",
+    "description",
+    "dynamic_columns",
+    "enabled",
+    "engine",
+    "enums",
+    "full_refresh",
+    "historical_input",
+    "incremental_mode",
+    "incremental_strategy",
+    "initial_valid_from",
+    "invalidate_hard_deletes",
+    "lookback",
+    "materialized",
+    "max_duration",
+    "max_microbatches",
+    "merge_exclude_columns",
+    "microbatch_limit",
+    "microbatch_strategy",
+    "migrate_force",
+    "migrate_from",
+    "model_schema",
+    "observed_at",
+    "old_name_view",
+    "on_schema_change",
+    "order_by",
+    "partition_by",
+    "placeholders",
+    "post_hooks",
+    "pre_hooks",
+    "primary_key",
+    "replay_on_change",
+    "row_diff_exclude_columns",
+    "row_diff_sample_rows",
+    "row_diff_sample_seed",
+    "row_diff_tolerances",
+    "schema",
+    "snapshot_full_refresh",
+    "snapshot_schema_change",
+    "snapshot_strategy",
+    "sql_analysis",
+    "sql_validation",
+    "table_settings",
+    "table_type",
+    "tags",
+    "time_travel_retention",
+    "unaccounted_partition_policy",
+    "unique_key",
+    "updated_at",
+    "valid_from_column",
+    "valid_to_column",
+];
+/// `MODEL` header keys that were removed and fail with a dedicated message.
+pub const REMOVED_SQL_MODEL_HEADER_KEYS: [&str; 1] = ["run_despite_unchanged"];
+/// Header keys a `FUNCTION` statement accepts.
+pub const SQL_FUNCTION_HEADER_KEYS: [&str; 6] = [
+    "arguments",
+    "database",
+    "description",
+    "returns",
+    "schema",
+    "tags",
+];
+/// Header keys an `AUDIT` statement accepts.
+pub const SQL_AUDIT_HEADER_KEYS: [&str; 11] = [
+    "always_run",
+    "evaluation",
+    "minimum_samples",
+    "name",
+    "run_scope",
+    "sample_count",
+    "sample_unit",
+    "severity",
+    "sql_analysis",
+    "thresholds",
+    "value",
+];
+/// Header keys a `HOOK` statement accepts.
+pub const SQL_HOOK_HEADER_KEYS: [&str; 1] = ["description"];
+/// Header keys a `TEST` statement accepts.
+pub const SQL_TEST_HEADER_KEYS: [&str; 8] = [
+    "cases",
+    "cursor_end",
+    "cursor_start",
+    "mode",
+    "name",
+    "parameters",
+    "sql_analysis",
+    "timeout",
+];
+/// Header keys a `SCENARIO` statement accepts.
+pub const SQL_SCENARIO_HEADER_KEYS: [&str; 4] =
+    ["description", "expected_columns", "tags", "timeout"];
+/// `SCENARIO (expected_columns ...)` values: compare listed columns, or require every column.
+pub const SCENARIO_EXPECTED_COLUMNS_KEY: &str = "expected_columns";
+pub const SCENARIO_EXPECTED_COLUMNS_VALUES: [&str; 2] = ["listed", "all"];
+pub const SCENARIO_EXPECTED_COLUMNS_ALL: &str = "all";

crates/sqlbuild-discovery/src/declaration_files/main/declaration_file_options.rs

@@ -0,0 +1,15 @@
+//! The options declaration discovery parses headers with.
+
+use crate::_helpers::sorted_keys::sorted_keys;
+use crate::constants::{SQL_AUDIT_HEADER_KEYS, SQL_FUNCTION_HEADER_KEYS, SQL_HOOK_HEADER_KEYS};
+use crate::declaration_files::models::DeclarationFileOptions;
+
+/// The header keys `FUNCTION`, `AUDIT` and `HOOK` statements accept.
+#[must_use]
+pub fn declaration_file_options() -> DeclarationFileOptions {
+    DeclarationFileOptions {
+        function_keys: sorted_keys(&SQL_FUNCTION_HEADER_KEYS),
+        audit_keys: sorted_keys(&SQL_AUDIT_HEADER_KEYS),
+        hook_keys: sorted_keys(&SQL_HOOK_HEADER_KEYS),
+    }
+}

crates/sqlbuild-discovery/src/declaration_files/main/mod.rs

@@ -1,4 +1,5 @@
 pub(crate) mod bare_number_failure;
+pub mod declaration_file_options;
 pub mod discover_declaration_collection;
 pub mod parse_declaration_text;
 pub mod python_function_files;

crates/sqlbuild-discovery/src/header_changes/_helpers/mod.rs

@@ -0,0 +1,2 @@
+pub(crate) mod references;
+pub(crate) mod value_encoding;

crates/sqlbuild-discovery/src/header_changes/_helpers/references.rs

@@ -0,0 +1,39 @@
+//! Whether an edited query still references only what the stored model read.
+
+use sqlbuild_sqltext::sql_references::main::extract_sql_references::extract_sql_references;
+use sqlbuild_sqltext::sql_references::models::ReferenceExtraction;
+use sqlbuild_sqltext::sql_scan::models::LexicalSyntax;
+
+use crate::header_changes::constants::{
+    LINE_COMMENT, MODEL_KIND, REF_CALL, SEED_CALL, SEED_KIND, SOURCE_CALL, SOURCE_KIND,
+};
+
+/// Whether every model, source and seed `query_sql` names was among the stored `upstream`.
+pub(crate) fn references_read(query_sql: &str, upstream: &[(String, String)]) -> bool {
+    let syntax: LexicalSyntax = LexicalSyntax {
+        line_comment_prefixes: vec![LINE_COMMENT.to_owned()],
+        ..LexicalSyntax::default()
+    };
+    let ReferenceExtraction::Extracted(scan) = extract_sql_references(query_sql, &syntax) else {
+        return false;
+    };
+    scan.references.iter().all(|reference| {
+        let kind: Option<&str> = match reference.kind {
+            REF_CALL => Some(MODEL_KIND),
+            SOURCE_CALL => Some(SOURCE_KIND),
+            SEED_CALL => Some(SEED_KIND),
+            _ => None,
+        };
+        match (kind, &reference.package) {
+            (Some(kind), None) => stored_upstream(upstream, kind, &reference.name),
+            _ => true,
+        }
+    })
+}
+
+/// Whether `upstream` holds the `kind` resource `name`.
+fn stored_upstream(upstream: &[(String, String)], kind: &str, name: &str) -> bool {
+    upstream
+        .iter()
+        .any(|(stored_kind, stored_name)| stored_kind == kind && stored_name == name)
+}

crates/sqlbuild-discovery/src/header_changes/_helpers/value_encoding.rs

@@ -0,0 +1,79 @@
+//! Header values as short digests: an unambiguous byte encoding, hashed and truncated.
+
+use sqlbuild_cache::digest::main::bytes_digest::bytes_digest;
+use sqlbuild_sqltext::compiler::models::AuthoredValue;
+
+use crate::header_changes::constants::{HEX, KEPT_DIGEST_BYTES};
+
+/// An unambiguous encoding of header values: a tag byte, then length-prefixed parts.
+#[derive(Default)]
+pub(crate) struct ValueEncoding {
+    bytes: Vec<u8>,
+}
+
+impl ValueEncoding {
+    fn text(&mut self, tag: u8, text: &str) {
+        self.bytes.push(tag);
+        self.bytes
+            .extend_from_slice(&(text.len() as u64).to_le_bytes());
+        self.bytes.extend_from_slice(text.as_bytes());
+    }
+
+    fn list(&mut self, tag: u8, items: &[AuthoredValue]) {
+        self.bytes.push(tag);
+        self.bytes
+            .extend_from_slice(&(items.len() as u64).to_le_bytes());
+        for item in items {
+            self.value(item);
+        }
+    }
+
+    fn pairs(&mut self, tag: u8, items: &[(String, AuthoredValue)]) {
+        self.bytes.push(tag);
+        self.bytes
+            .extend_from_slice(&(items.len() as u64).to_le_bytes());
+        for (key, item) in items {
+            self.text(b'k', key);
+            self.value(item);
+        }
+    }
+
+    pub(crate) fn value(&mut self, value: &AuthoredValue) {
+        match value {
+            AuthoredValue::Null => self.bytes.push(b'n'),
+            AuthoredValue::Boolean(flag) => self.bytes.extend_from_slice(&[b'b', u8::from(*flag)]),
+            AuthoredValue::BareWord(word) => self.text(b'w', word),
+            AuthoredValue::String(string) => self.text(b's', string),
+            AuthoredValue::List(items) => self.list(b'l', items),
+            AuthoredValue::Map(items) => self.pairs(b'm', items),
+            AuthoredValue::Set(items) => self.list(b'e', items),
+            AuthoredValue::Tuple(items) => self.list(b't', items),
+            AuthoredValue::TypedConstant(items) => self.pairs(b'c', items),
+            AuthoredValue::InlineSqlHook(sql) => self.text(b'i', sql),
+            AuthoredValue::NamedSqlHook(name, items) => {
+                self.text(b'h', name);
+                self.pairs(b'a', items);
+            }
+            AuthoredValue::PythonHook(name, items) => {
+                self.text(b'p', name);
+                self.pairs(b'a', items);
+            }
+        }
+    }
+}
+
+pub(crate) fn short_hex(digest: &[u8]) -> String {
+    let mut hex: String = String::with_capacity(KEPT_DIGEST_BYTES * 2);
+    for byte in &digest[..KEPT_DIGEST_BYTES] {
+        hex.push(char::from(HEX[usize::from(byte >> 4)]));
+        hex.push(char::from(HEX[usize::from(byte & 0x0f)]));
+    }
+    hex
+}
+
+/// The short digest of one header value.
+pub(crate) fn value_digest(value: &AuthoredValue) -> String {
+    let mut encoding: ValueEncoding = ValueEncoding::default();
+    encoding.value(value);
+    short_hex(&bytes_digest(&encoding.bytes))
+}

crates/sqlbuild-discovery/src/header_changes/constants.rs

@@ -0,0 +1,27 @@
+//! The header keys and values a delta compile can recompile a model for.
+
+/// Header keys a delta compile recompiles the model for; any other changed key falls back.
+pub(crate) const PATCHED_KEYS: [&str; 6] = [
+    "description",
+    "columns",
+    "dynamic_columns",
+    "model_schema",
+    "audits",
+    "contract",
+];
+/// `materialized` values a delta compile handles; others name materializations it cannot.
+pub(crate) const PATCHED_MATERIALIZATIONS: [&str; 2] = ["view", "table"];
+pub(crate) const MATERIALIZED_KEY: &str = "materialized";
+/// Plain `--` comments, which every adapter's SQL reads as comments.
+pub(crate) const LINE_COMMENT: &str = "--";
+/// Reference call kinds and the upstream kinds they resolve to.
+pub(crate) const REF_CALL: &str = "ref";
+pub(crate) const SOURCE_CALL: &str = "source";
+pub(crate) const SEED_CALL: &str = "seed";
+pub(crate) const MODEL_KIND: &str = "model";
+pub(crate) const SOURCE_KIND: &str = "source";
+pub(crate) const SEED_KIND: &str = "seed";
+
+/// Bytes of a digest kept per header value; a collision would need 2^64 work to find.
+pub(crate) const KEPT_DIGEST_BYTES: usize = 16;
+pub(crate) const HEX: &[u8; 16] = b"0123456789abcdef";

crates/sqlbuild-discovery/src/header_changes/main/custom_rule_reads.rs

@@ -0,0 +1,36 @@
+//! Whether a custom rule reads what changed in a model header.
+
+use std::collections::BTreeSet;
+
+/// Whether a custom rule reads a fact of this model that the changed header keys decide.
+#[must_use]
+pub fn custom_rule_reads(
+    fact_digests: &[(Vec<String>, String)],
+    model: (&str, &str),
+    changed: &BTreeSet<String>,
+) -> Option<String> {
+    let (path, name) = model;
+    for (key, _) in fact_digests {
+        let Some((fact, arguments)) = key.split_first() else {
+            continue;
+        };
+        if !arguments
+            .iter()
+            .any(|argument| argument == path || argument == name)
+        {
+            continue;
+        }
+        let reads: &[&str] = match fact.as_str() {
+            "contracts.enforced" | "contract" => &["contract"],
+            "contracts.grain" | "grain" => &["contract", "columns"],
+            "columns.declared" | "columns.names" => &["columns", "dynamic_columns"],
+            "materialized" => &["materialized"],
+            "tests.for_model" | "audits.for_model" => &["audits", "columns"],
+            _ => &[],
+        };
+        if let Some(key) = reads.iter().find(|key| changed.contains(**key)) {
+            return Some(format!("a custom rule reads {name}'s {key}"));
+        }
+    }
+    None
+}