refactor: keep cache, reporting and SQLite history native

crates/sqlbuild-cache/src/artifacts/constants.rs

@@ -1,5 +1,10 @@
 /// The mode new artifacts are created with before the process umask, as Python's `os.open`.
 pub const NEW_FILE_MODE: u32 = 0o666;
+/// The existing SQL-test cache snapshot format, shared by native readers and writers.
+pub(crate) const SQL_TEST_CACHE_VERSION: u64 = 5;
+pub(crate) const SQL_TEST_CACHE_FILE: &str = "sql-test-artifacts.json";
+pub(crate) const MAX_SQL_TEST_CACHE_BYTES: u64 = 10_000_000;
+pub(crate) const MAX_SQL_TEST_CACHE_RECORDS: usize = 100_000;
 /// The extension every compiled SQL test artifact carries.
 pub(crate) const SQL_EXTENSION: &str = "sql";
 #[cfg(unix)]

crates/sqlbuild-cache/src/artifacts/main/mod.rs

@@ -1,8 +1,9 @@
 pub mod new_sql_test_record;
 pub mod publish_staged;
+pub mod read_sql_test_cache;
 pub mod remove_stale;
 pub mod remove_stale_unless_unchanged;
 pub mod reusable_sql_test_record;
 pub mod staged_files;
-pub mod stored_sql_test_record;
 pub mod write_artifacts;
+pub mod write_sql_test_cache;

crates/sqlbuild-cache/src/artifacts/main/read_sql_test_cache.rs

@@ -0,0 +1,66 @@
+//! Read the persisted SQL-test artifact records without an interpreter.
+
+use std::collections::HashMap;
+use std::path::Path;
+
+use serde_json::Value;
+
+use crate::artifacts::_helpers::sql_test_records::{
+    is_safe_relative_sql_path, normalized_relative_path,
+};
+use crate::artifacts::constants::{
+    MAX_SQL_TEST_CACHE_BYTES, MAX_SQL_TEST_CACHE_RECORDS, SQL_TEST_CACHE_FILE,
+    SQL_TEST_CACHE_VERSION,
+};
+use crate::artifacts::models::SqlTestArtifactRecord;
+
+/// Valid records in the existing snapshot; an unreadable or malformed cache is a miss.
+#[must_use]
+pub fn read_sql_test_cache(directory: &Path) -> HashMap<String, SqlTestArtifactRecord> {
+    let path = directory.join(SQL_TEST_CACHE_FILE);
+    let Ok(metadata) = std::fs::metadata(&path) else {
+        return HashMap::new();
+    };
+    if !metadata.is_file() || metadata.len() > MAX_SQL_TEST_CACHE_BYTES {
+        return HashMap::new();
+    }
+    let Ok(bytes) = std::fs::read(&path) else {
+        return HashMap::new();
+    };
+    let Ok(payload) = serde_json::from_slice::<Value>(&bytes) else {
+        return HashMap::new();
+    };
+    if payload.get("version").and_then(Value::as_f64) != Some(SQL_TEST_CACHE_VERSION as f64) {
+        return HashMap::new();
+    }
+    let Some(rows) = payload.get("records").and_then(Value::as_object) else {
+        return HashMap::new();
+    };
+    if rows.len() > MAX_SQL_TEST_CACHE_RECORDS {
+        return HashMap::new();
+    }
+    let mut records: HashMap<String, SqlTestArtifactRecord> = HashMap::new();
+    for (key, value) in rows {
+        if let Some(record) = record(value) {
+            records.insert(key.clone(), record);
+        }
+    }
+    records
+}
+
+fn record(value: &Value) -> Option<SqlTestArtifactRecord> {
+    let relative_path = normalized_relative_path(Path::new(value.get("relative_path")?.as_str()?));
+    if !is_safe_relative_sql_path(&relative_path) {
+        return None;
+    }
+    let Ok(mtime_ns) = i64::try_from(value.get("mtime_ns")?.as_u64()?) else {
+        return None;
+    };
+    Some(SqlTestArtifactRecord {
+        identity: value.get("identity")?.as_str()?.to_owned(),
+        relative_path,
+        size: value.get("size")?.as_u64()?,
+        mtime_ns,
+        content_sha256: value.get("content_sha256")?.as_str()?.to_owned(),
+    })
+}

crates/sqlbuild-cache/src/artifacts/main/write_sql_test_cache.rs

@@ -0,0 +1,64 @@
+//! Publish the existing SQL-test cache snapshot directly from native records.
+
+use std::collections::BTreeMap;
+use std::path::Path;
+
+use sqlbuild_core::json::main::dumps::dumps;
+use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions};
+
+use crate::artifacts::constants::{SQL_TEST_CACHE_FILE, SQL_TEST_CACHE_VERSION};
+use crate::artifacts::models::SqlTestArtifactRecord;
+
+/// Atomically replace the snapshot with Python-compatible JSON; write failures are cache misses.
+pub fn write_sql_test_cache(directory: &Path, records: &[(String, SqlTestArtifactRecord)]) {
+    let rows: BTreeMap<String, JsonValue> = records
+        .iter()
+        .map(|(key, record)| {
+            let fields = JsonValue::Object(vec![
+                (
+                    "identity".to_owned(),
+                    JsonValue::String(record.identity.clone()),
+                ),
+                (
+                    "relative_path".to_owned(),
+                    JsonValue::String(record.relative_posix()),
+                ),
+                ("size".to_owned(), JsonValue::Integer(record.size.into())),
+                (
+                    "mtime_ns".to_owned(),
+                    JsonValue::Integer(record.mtime_ns.into()),
+                ),
+                (
+                    "content_sha256".to_owned(),
+                    JsonValue::String(record.content_sha256.clone()),
+                ),
+            ]);
+            (key.clone(), fields)
+        })
+        .collect();
+    let payload = JsonValue::Object(vec![
+        (
+            "version".to_owned(),
+            JsonValue::Integer(SQL_TEST_CACHE_VERSION.into()),
+        ),
+        (
+            "records".to_owned(),
+            JsonValue::Object(rows.into_iter().collect()),
+        ),
+    ]);
+    let Ok(encoded) = dumps(
+        &payload,
+        &JsonDialect::Stdlib(StdlibJsonOptions {
+            sort_keys: true,
+            ..StdlibJsonOptions::new(None).with_separators(",", ":")
+        }),
+    ) else {
+        return;
+    };
+    let temporary = directory.join(format!(".{SQL_TEST_CACHE_FILE}.{}.tmp", std::process::id()));
+    let _ = (|| -> std::io::Result<()> {
+        std::fs::create_dir_all(directory)?;
+        std::fs::write(&temporary, encoded.as_bytes())?;
+        std::fs::rename(&temporary, directory.join(SQL_TEST_CACHE_FILE))
+    })();
+}

crates/sqlbuild-cache/src/artifacts/models.rs

@@ -59,17 +59,6 @@ impl SqlTestRecordMiss {
     }
 }
 
-/// The fields of one cache file record, already checked to be strings and non-negative integers.
-#[derive(Clone, Debug, PartialEq, Eq)]
-pub struct StoredSqlTestRecord {
-    pub identity: String,
-    /// The `relative_path` text as stored.
-    pub relative_path: String,
-    pub size: u64,
-    pub mtime_ns: i64,
-    pub content_sha256: String,
-}
-
 impl SqlTestArtifactRecord {
     /// The relative path's `as_posix()` text, as the cache file stores it.
     #[must_use]

crates/sqlbuild-cache/src/artifacts/tests/test_reusable_record.rs

@@ -1,8 +1,12 @@
 use crate::artifacts::main::new_sql_test_record::new_sql_test_record;
+use crate::artifacts::main::read_sql_test_cache::read_sql_test_cache;
 use crate::artifacts::main::reusable_sql_test_record::reusable_sql_test_record;
-use crate::artifacts::models::SqlTestRecordMiss;
+use crate::artifacts::main::write_sql_test_cache::write_sql_test_cache;
+use crate::artifacts::models::{SqlTestArtifactRecord, SqlTestRecordMiss};
 use crate::artifacts::tests::helpers::{bump_mtime, place};
-use crate::artifacts::tests::test_types::ReusableRecordTestCase;
+use crate::artifacts::tests::test_types::{
+    ReusableRecordTestCase, SqlTestSnapshotReadTestCase, SqlTestSnapshotWriteTestCase,
+};
 
 const IDENTITY: &str = "identity-1";
 
@@ -66,3 +70,124 @@ fn given_recorded_artifact_when_checking_reuse_then_reports_why_it_missed() {
         );
     }
 }
+
+#[test]
+fn given_native_records_when_persisting_then_preserves_python_snapshot_bytes_and_reuse() {
+    let test_cases = [SqlTestSnapshotWriteTestCase {
+        description: "sorted Unicode records, exact large integers and last duplicate wins",
+        expected_text: concat!(
+            r#"{"records":{"a":{"content_sha256":"abc","identity":"caf\u00e9 \ud83d\ude00","mtime_ns":9223372036854775807,"relative_path":"nested/orders.sql","size":18446744073709551615},"#,
+            r#""z":{"content_sha256":"abc","identity":"caf\u00e9 \ud83d\ude00","mtime_ns":9223372036854775807,"relative_path":"nested/orders.sql","size":7}},"version":5}"#,
+        ),
+        expected_record_count: 2,
+    }];
+    for test_case in test_cases {
+        let root = tempfile::tempdir().expect("cache folder");
+        let record = SqlTestArtifactRecord {
+            identity: "café 😀".to_owned(),
+            relative_path: "nested/orders.sql".into(),
+            size: u64::MAX,
+            mtime_ns: i64::MAX,
+            content_sha256: "abc".to_owned(),
+        };
+        let records = vec![
+            ("z".to_owned(), record.clone()),
+            ("a".to_owned(), record.clone()),
+            (
+                "z".to_owned(),
+                SqlTestArtifactRecord {
+                    size: 7,
+                    ..record.clone()
+                },
+            ),
+        ];
+
+        write_sql_test_cache(root.path(), &records);
+
+        let path = root.path().join("sql-test-artifacts.json");
+        let text = std::fs::read_to_string(&path).expect("written snapshot");
+        assert_eq!(text, test_case.expected_text, "{}", test_case.description);
+        let loaded = read_sql_test_cache(root.path());
+        assert_eq!(
+            loaded.len(),
+            test_case.expected_record_count,
+            "{}",
+            test_case.description
+        );
+        assert_eq!(loaded["a"], record, "{}", test_case.description);
+        assert_eq!(loaded["z"].size, 7, "{}", test_case.description);
+
+        std::fs::remove_file(&path).expect("remove snapshot");
+        std::fs::create_dir(&path).expect("blocked destination");
+        write_sql_test_cache(root.path(), &records);
+        assert!(path.is_dir(), "{}", test_case.description);
+    }
+}
+
+#[test]
+fn given_mixed_or_broken_snapshots_when_reading_then_keeps_only_safe_well_typed_records() {
+    let test_cases = [
+        SqlTestSnapshotReadTestCase {
+            description: "empty snapshot",
+            contents: "",
+            expected_records: &[],
+        },
+        SqlTestSnapshotReadTestCase {
+            description: "truncated JSON",
+            contents: "{",
+            expected_records: &[],
+        },
+        SqlTestSnapshotReadTestCase {
+            description: "non-object root",
+            contents: "[]",
+            expected_records: &[],
+        },
+        SqlTestSnapshotReadTestCase {
+            description: "wrong version",
+            contents: r#"{"version":6,"records":{}}"#,
+            expected_records: &[],
+        },
+        SqlTestSnapshotReadTestCase {
+            description: "non-object records",
+            contents: r#"{"version":5,"records":[]}"#,
+            expected_records: &[],
+        },
+        SqlTestSnapshotReadTestCase {
+            description: "valid record survives bad types, ranges and paths; numeric version is compatible",
+            contents: r#"{"version":5.0,"records":{
+                "valid":{"identity":"orders","relative_path":"nested/orders.sql","size":11,"mtime_ns":123,"content_sha256":"abc"},
+                "bool":{"identity":"orders","relative_path":"orders.sql","size":true,"mtime_ns":123,"content_sha256":"abc"},
+                "float":{"identity":"orders","relative_path":"orders.sql","size":11.0,"mtime_ns":123,"content_sha256":"abc"},
+                "negative":{"identity":"orders","relative_path":"orders.sql","size":-1,"mtime_ns":123,"content_sha256":"abc"},
+                "overflow":{"identity":"orders","relative_path":"orders.sql","size":11,"mtime_ns":9223372036854775808,"content_sha256":"abc"},
+                "type":{"identity":17,"relative_path":"orders.sql","size":11,"mtime_ns":123,"content_sha256":"abc"},
+                "escape":{"identity":"orders","relative_path":"../outside.sql","size":11,"mtime_ns":123,"content_sha256":"abc"},
+                "extension":{"identity":"orders","relative_path":"orders.txt","size":11,"mtime_ns":123,"content_sha256":"abc"}
+            }}"#,
+            expected_records: &[("orders", 11)],
+        },
+    ];
+    for test_case in test_cases {
+        let root = tempfile::tempdir().expect("cache folder");
+        assert!(
+            read_sql_test_cache(root.path()).is_empty(),
+            "{}",
+            test_case.description
+        );
+        std::fs::write(
+            root.path().join("sql-test-artifacts.json"),
+            test_case.contents,
+        )
+        .expect("snapshot");
+        let loaded = read_sql_test_cache(root.path());
+        let actual: Vec<(&str, u64)> = loaded
+            .values()
+            .map(|record| (record.identity.as_str(), record.size))
+            .collect();
+        assert_eq!(
+            actual, test_case.expected_records,
+            "{}",
+            test_case.description
+        );
+    }
+}

crates/sqlbuild-cache/src/artifacts/tests/test_types.rs

@@ -57,3 +57,17 @@ pub(crate) struct ReusableRecordTestCase {
     pub(crate) identity: &'static str,
     pub(crate) expected_outcome: Result<(), crate::artifacts::models::SqlTestRecordMiss>,
 }
+
+/// A cache snapshot's externally visible encoding and recovered records.
+pub(crate) struct SqlTestSnapshotWriteTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) expected_text: &'static str,
+    pub(crate) expected_record_count: usize,
+}
+
+/// A disposable on-disk snapshot, including absent and malformed files.
+pub(crate) struct SqlTestSnapshotReadTestCase {
+    pub(crate) description: &'static str,
+    pub(crate) contents: &'static str,
+    pub(crate) expected_records: &'static [(&'static str, u64)],
+}