perf: cut edit-loop compile overhead in the reuse store, hashing and walks

crates/sqlbuild-cache/src/artifacts/_helpers/files.rs

@@ -141,23 +141,50 @@ pub(crate) fn remove_stale(
     compiled_dir: &Path,
     managed: &HashSet<PathBuf>,
 ) -> Result<usize, ArtifactError> {
+    remove_stale_listing(compiled_dir, managed).map(|(removed, _)| removed)
+}
+
+/// `remove_stale`, with every directory left after the walk, or `None` when one was not read.
+pub(crate) fn remove_stale_listing(
+    compiled_dir: &Path,
+    managed: &HashSet<PathBuf>,
+) -> Result<(usize, Option<Vec<PathBuf>>), ArtifactError> {
     if !compiled_dir.is_dir() {
-        return Ok(0);
+        return Ok((0, None));
     }
-    let (_, removed) = remove_stale_under(compiled_dir, managed)?;
+    let cleaned: Cleaned = remove_stale_under(compiled_dir, managed)?;
     let _ = std::fs::remove_dir(compiled_dir);
-    Ok(removed)
+    let listed: Option<Vec<PathBuf>> = cleaned.complete.then_some(cleaned.directories);
+    Ok((cleaned.removed, listed))
 }
 
-/// Clean one directory bottom-up as `os.walk(topdown=False)`; returns (gone, files removed).
+/// What cleaning one directory did, and whether every directory below it could be read.
+#[derive(Debug)]
+struct Cleaned {
+    gone: bool,
+    removed: usize,
+    directories: Vec<PathBuf>,
+    complete: bool,
+}
+
+/// Clean one directory bottom-up as `os.walk(topdown=False)`.
 fn remove_stale_under(
     directory: &Path,
     managed: &HashSet<PathBuf>,
-) -> Result<(bool, usize), ArtifactError> {
+) -> Result<Cleaned, ArtifactError> {
     let entries = match std::fs::read_dir(directory) {
         Ok(entries) => entries,
-        Err(_) => return Ok((false, 0)),
+        Err(_) => {
+            return Ok(Cleaned {
+                gone: false,
+                removed: 0,
+                directories: Vec::new(),
+                complete: false,
+            });
+        }
     };
+    let mut directories: Vec<PathBuf> = vec![directory.to_path_buf()];
+    let mut complete: bool = true;
     let mut removed: usize = 0;
     let mut kept_file: bool = false;
     let mut subdirectories_removed: bool = true;
@@ -176,9 +203,11 @@ fn remove_stale_under(
             subdirectories_removed = false;
             continue;
         }
-        let (gone, below) = remove_stale_under(&subdirectory, managed)?;
-        removed += below;
-        subdirectories_removed &= gone;
+        let below: Cleaned = remove_stale_under(&subdirectory, managed)?;
+        removed += below.removed;
+        subdirectories_removed &= below.gone;
+        complete &= below.complete;
+        directories.extend(below.directories);
     }
     for file in files {
         if managed.contains(&file) {
@@ -189,7 +218,15 @@ fn remove_stale_under(
         }
     }
     let gone: bool = !kept_file && subdirectories_removed && removed_directory(directory);
-    Ok((gone, removed))
+    if gone {
+        directories.clear();
+    }
+    Ok(Cleaned {
+        gone,
+        removed,
+        directories,
+        complete,
+    })
 }
 
 fn removed_directory(directory: &Path) -> bool {

crates/sqlbuild-cache/src/artifacts/_helpers/mod.rs

@@ -1,2 +1,3 @@
 pub(crate) mod files;
 pub(crate) mod sql_test_records;
+pub(crate) mod tree_stamp;

crates/sqlbuild-cache/src/artifacts/_helpers/tree_stamp.rs

@@ -0,0 +1,162 @@
+//! The stamp of a compiled tree that held exactly its managed files when last cleaned.
+
+use std::collections::HashSet;
+use std::path::{Path, PathBuf};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use crate::digest::main::content_digest::content_digest;
+use crate::digest::types::ContentDigest;
+use crate::store::main::open_native_store::open_native_store;
+use crate::store::models::{NativeStore, RecordReader, RecordWriter};
+
+const STORE_KIND: &str = "compiled-tree";
+const STORE_ENVIRONMENT: &str = "compiled-tree-1";
+
+/// One folder's identity: its path, inode, and modification and change times.
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct FolderStamp {
+    path: String,
+    inode: u64,
+    mtime_ns: i64,
+    ctime_ns: i64,
+}
+
+fn entry_key() -> ContentDigest {
+    content_digest(&["tree"])
+}
+
+/// The digest of the compiled folder and its managed file set, in any order.
+pub(crate) fn managed_digest(compiled_dir: &Path, managed: &HashSet<PathBuf>) -> ContentDigest {
+    let mut paths: Vec<String> = managed
+        .iter()
+        .map(|path| path.to_string_lossy().into_owned())
+        .collect();
+    paths.sort_unstable();
+    let mut hasher: blake3::Hasher = blake3::Hasher::new();
+    for part in std::iter::once(compiled_dir.to_string_lossy().into_owned()).chain(paths) {
+        let _ = hasher.update(&(part.len() as u64).to_le_bytes());
+        let _ = hasher.update(part.as_bytes());
+    }
+    *hasher.finalize().as_bytes()
+}
+
+#[cfg(unix)]
+fn folder_stamp(path: &Path) -> Option<FolderStamp> {
+    use std::os::unix::fs::MetadataExt;
+
+    let metadata: std::fs::Metadata = match std::fs::symlink_metadata(path) {
+        Ok(metadata) => metadata,
+        Err(_absent) => return None,
+    };
+    if !metadata.is_dir() {
+        return None;
+    }
+    Some(FolderStamp {
+        path: path.to_string_lossy().into_owned(),
+        inode: metadata.ino(),
+        mtime_ns: metadata.mtime() * 1_000_000_000 + metadata.mtime_nsec(),
+        ctime_ns: metadata.ctime() * 1_000_000_000 + metadata.ctime_nsec(),
+    })
+}
+
+#[cfg(not(unix))]
+fn folder_stamp(_path: &Path) -> Option<FolderStamp> {
+    None
+}
+
+/// Whether the tree recorded at `stamp_path` for `digest` is unchanged on disk.
+pub(crate) fn tree_unchanged(stamp_path: &Path, digest: &ContentDigest) -> bool {
+    let Ok(mut store) = open_native_store(stamp_path, STORE_KIND, STORE_ENVIRONMENT) else {
+        return false;
+    };
+    let Some(bytes) = store.get(&entry_key()) else {
+        return false;
+    };
+    let Ok((recorded, folders)) = decode(bytes) else {
+        return false;
+    };
+    recorded == *digest
+        && !folders.is_empty()
+        && folders
+            .iter()
+            .all(|folder| folder_stamp(Path::new(&folder.path)).as_ref() == Some(folder))
+}
+
+/// Remove the recorded stamp, before the tree is changed.
+pub(crate) fn forget_tree(stamp_path: &Path) {
+    let _ = std::fs::remove_file(stamp_path);
+}
+
+/// The current time in nanoseconds since the epoch, as file stamps record it.
+pub(crate) fn now_ns() -> i64 {
+    SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map_or(0, |elapsed| {
+            i64::try_from(elapsed.as_nanos()).unwrap_or(i64::MAX)
+        })
+}
+
+/// Record `folders` as the tree `digest` names when each stamps and changed before `trusted_ns`.
+pub(crate) fn record_tree(
+    stamp_path: &Path,
+    digest: &ContentDigest,
+    folders: &[PathBuf],
+    trusted_ns: i64,
+) {
+    let Some(stamps) = folders
+        .iter()
+        .map(|path| folder_stamp(path))
+        .collect::<Option<Vec<FolderStamp>>>()
+    else {
+        return;
+    };
+    let settled: bool = !stamps.is_empty()
+        && stamps
+            .iter()
+            .all(|stamp| stamp.mtime_ns.max(stamp.ctime_ns) < trusted_ns);
+    if !settled {
+        return;
+    }
+    let mut store: NativeStore = NativeStore {
+        kind: STORE_KIND.to_owned(),
+        environment: STORE_ENVIRONMENT.to_owned(),
+        ..NativeStore::default()
+    };
+    store.put(entry_key(), encode(digest, &stamps));
+    let _ = store.save(stamp_path, &[]);
+}
+
+fn encode(digest: &ContentDigest, folders: &[FolderStamp]) -> Vec<u8> {
+    let mut writer: RecordWriter = RecordWriter::default();
+    writer.put_bytes(digest);
+    writer.put_u64(folders.len() as u64);
+    for folder in folders {
+        writer.put_str(&folder.path);
+        writer.put_u64(folder.inode);
+        writer.put_u64(folder.mtime_ns as u64);
+        writer.put_u64(folder.ctime_ns as u64);
+    }
+    writer.into_bytes()
+}
+
+fn decode(
+    bytes: &[u8],
+) -> Result<(ContentDigest, Vec<FolderStamp>), crate::store::errors::StoreDecodeError> {
+    let mut reader: RecordReader<'_> = RecordReader::new(bytes);
+    let digest: ContentDigest = reader
+        .bytes()?
+        .try_into()
+        .map_err(|_| crate::store::errors::StoreDecodeError)?;
+    let count: u64 = reader.u64()?;
+    let mut folders: Vec<FolderStamp> = Vec::new();
+    for _ in 0..count {
+        folders.push(FolderStamp {
+            path: reader.string()?,
+            inode: reader.u64()?,
+            mtime_ns: reader.u64()? as i64,
+            ctime_ns: reader.u64()? as i64,
+        });
+    }
+    reader.finish()?;
+    Ok((digest, folders))
+}

crates/sqlbuild-cache/src/artifacts/main/mod.rs

@@ -1,6 +1,7 @@
 pub mod new_sql_test_record;
 pub mod publish_staged;
 pub mod remove_stale;
+pub mod remove_stale_unless_unchanged;
 pub mod reusable_sql_test_record;
 pub mod staged_files;
 pub mod stored_sql_test_record;

crates/sqlbuild-cache/src/artifacts/main/remove_stale_unless_unchanged.rs

@@ -0,0 +1,29 @@
+use std::collections::HashSet;
+use std::path::{Path, PathBuf};
+
+use crate::artifacts::_helpers::files::remove_stale_listing;
+use crate::artifacts::_helpers::tree_stamp::{
+    forget_tree, managed_digest, now_ns, record_tree, tree_unchanged,
+};
+use crate::artifacts::errors::ArtifactError;
+use crate::digest::types::ContentDigest;
+
+/// `remove_stale`, skipped while the tree stamped at `stamp_path` still holds exactly `managed`.
+pub fn remove_stale_unless_unchanged(
+    compiled_dir: &Path,
+    managed: &HashSet<PathBuf>,
+    stamp_path: &Path,
+    racy_window_ns: i64,
+) -> Result<usize, ArtifactError> {
+    let digest: ContentDigest = managed_digest(compiled_dir, managed);
+    if tree_unchanged(stamp_path, &digest) {
+        return Ok(0);
+    }
+    forget_tree(stamp_path);
+    let trusted_ns: i64 = now_ns().saturating_sub(racy_window_ns);
+    let (removed, folders) = remove_stale_listing(compiled_dir, managed)?;
+    if let Some(folders) = folders {
+        record_tree(stamp_path, &digest, &folders, trusted_ns);
+    }
+    Ok(removed)
+}

crates/sqlbuild-cache/src/artifacts/tests.rs

@@ -1,5 +1,8 @@
 #[path = "tests/helpers.rs"]
 mod helpers;
+#[cfg(unix)]
+#[path = "tests/test_remove_stale.rs"]
+mod remove_stale;
 #[path = "tests/test_types.rs"]
 mod test_types;
 #[path = "tests/test_write_artifacts.rs"]

crates/sqlbuild-cache/src/artifacts/tests/helpers.rs

@@ -17,3 +17,33 @@ pub(crate) fn place(path: &Path, contents: &[u8]) {
     std::fs::create_dir_all(path.parent().expect("artifact parent")).expect("artifact directory");
     std::fs::write(path, contents).expect("existing artifact");
 }
+
+/// Every file below `root` as a sorted relative path.
+pub(crate) fn tree_files(root: &Path) -> Vec<String> {
+    let mut files: Vec<String> = walkdir::WalkDir::new(root)
+        .into_iter()
+        .flatten()
+        .filter(|entry| entry.file_type().is_file())
+        .filter_map(|entry| {
+            entry
+                .path()
+                .strip_prefix(root)
+                .ok()
+                .map(|relative| relative.to_string_lossy().into_owned())
+        })
+        .collect();
+    files.sort();
+    files
+}
+
+/// A compiled folder that is a plain directory.
+pub(crate) fn plain_compiled_root(target: &Path) {
+    std::fs::create_dir_all(target.join("compiled")).expect("compiled folder");
+}
+
+/// A compiled folder that is a link to a directory elsewhere in the target folder.
+#[cfg(unix)]
+pub(crate) fn linked_compiled_root(target: &Path) {
+    std::fs::create_dir_all(target.join("elsewhere")).expect("linked folder");
+    std::os::unix::fs::symlink(target.join("elsewhere"), target.join("compiled")).expect("link");
+}