From 64e1e8c08f8200b89041653221650a93763e83bd Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 16 Nov 2026 14:21:58 -0800 Subject: [PATCH 1/3] ci(e2e): key the tmachine disk cache on the setup layer hash Add `tmachine layer-key `, which prints the hash tmachine names the environment's setup layer after, or `none` when the environment has no setup playbooks, without starting a VM. setup() now derives its layer name from the same function. Key the e2e-suites disk cache on that value instead of a hand-written hashFiles list, so a new setup input changes the key without a workflow edit. Skip the cache for environments without a setup layer. Closes #5627 Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 30 +++++--- TESTING.md | 10 +-- tests/tmachine/src/lib.rs | 5 +- tests/tmachine/src/main.rs | 12 ++- tests/tmachine/src/qemu/ansible_hash.rs | 5 +- tests/tmachine/src/qemu/mod.rs | 2 +- tests/tmachine/src/qemu/setup.rs | 99 +++++++++++++++++++++++-- 7 files changed, 135 insertions(+), 28 deletions(-) diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 032a5a4186..93f01b0870 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -147,24 +147,32 @@ jobs: with: cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} + # The hash tmachine names the environment's setup layer after, or `none` + # when the environment has no setup layer. It reads the base image, which + # the run step needs anyway, so this mostly moves that download earlier. + - name: Compute the tmachine setup layer key + id: layer-key + env: + ENVIRONMENT: ${{ matrix.environment }} + run: | + set -euo pipefail + nix build --no-link .#tmachine || nix build --no-link .#tmachine + key=$(nix run .#tmachine -- layer-key "${ENVIRONMENT}") + echo "key=${key}" >> "$GITHUB_OUTPUT" + - name: Cache tmachine disks + if: steps.layer-key.outputs.key != 'none' uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: # Only the setup layers, which depend on the environment alone. The # install layers under disks/install hold this commit's build (and # on k3s, the gateway's credentials), so no later run could use them. path: ~/.cache/tmachine/disks/*.qcow2 - # tmachine keys its setup layers on a hash of the Ansible tree, the - # environments' playbook lists and setup inputs (config.nix, and - # files such as the Keycloak realm file) and the cache version in - # setup.rs. actions/cache doesn't save after an exact key hit, so a - # key that misses one of those would rebuild the changed layers on - # every run. Environments share no layers, and - # only the first job to save a key gets its disks in, so each - # environment needs its own key; its installers share it. No - # restore-keys fallback: restoring an older entry would carry its - # stale disks into the new one, which then only grows. - key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/layer.rs', 'tests/tmachine/src/qemu/ansible_hash.rs', 'scripts/keycloak-realm.json') }} + # Keyed on the layer's own hash, so the cache misses exactly when + # tmachine would rebuild the layer. The environment's installers + # share the key. No restore-keys fallback: restoring an older entry + # would carry its stale disks into the new one, which then only grows. + key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ steps.layer-key.outputs.key }} - name: Download integration inputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/TESTING.md b/TESTING.md index 58bb638831..93e410cf86 100644 --- a/TESTING.md +++ b/TESTING.md @@ -305,11 +305,11 @@ service. setup playbook starts the service (with a restart policy, since setup is cached as a disk) and writes the variables as `KEY=VALUE` lines to `/etc/openshell-test/env.d/.env`. Pass files the playbook needs - through the environment's setup `inputs`, and add them to the tmachine disk - cache key in `e2e-suites.yml`. The runner rejects names without the - `OPENSHELL_TEST_` prefix, and the tests get the variables in the guest, so - values never appear in a runner command. Values are taken literally, without - quoting. + through the environment's setup `inputs`; they are part of the setup layer's + hash, which CI's disk cache keys on too. The runner rejects names without + the `OPENSHELL_TEST_` prefix, and the tests get the variables in the guest, + so values never appear in a runner command. Values are taken literally, + without quoting. - **Fixture.** Put the service's fixture in a support crate under `tests/support/`, named `-support`, so any suite can depend on it, or reuse the existing one. It defines the variable names as constants and a diff --git a/tests/tmachine/src/lib.rs b/tests/tmachine/src/lib.rs index 5376c367af..662e6d562f 100644 --- a/tests/tmachine/src/lib.rs +++ b/tests/tmachine/src/lib.rs @@ -4,7 +4,8 @@ //! Isolated QEMU test machines provisioned with Ansible. //! //! The `tmachine` binary drives one-shot `setup`, `install`, and `test` -//! commands. Callers that need to act on a running machine use [`Session`]. +//! commands, and prints the key of an environment's setup layer with +//! `layer-key`. Callers that need to act on a running machine use [`Session`]. //! A binary that starts machines should run its work in [`until_signal`], so //! a `SIGTERM` does not leave QEMU running. @@ -12,4 +13,4 @@ pub mod ansible; pub mod config; mod qemu; -pub use qemu::{Session, install, setup, ssh_port, test, until_signal}; +pub use qemu::{Session, install, setup, setup_layer_key, ssh_port, test, until_signal}; diff --git a/tests/tmachine/src/main.rs b/tests/tmachine/src/main.rs index d38325cd36..5cb644c17e 100644 --- a/tests/tmachine/src/main.rs +++ b/tests/tmachine/src/main.rs @@ -6,7 +6,7 @@ use std::path::PathBuf; use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; use tmachine::config::{Config, Environment, Installer, Machine}; -use tmachine::{install, setup, test}; +use tmachine::{install, setup, setup_layer_key, test}; #[derive(Parser)] #[command(name = "tmachine")] @@ -23,6 +23,11 @@ enum Command { Setup { environment: String, }, + /// Print the key of the environment's setup layer, or `none` when it has + /// no setup playbooks, without starting a VM. + LayerKey { + environment: String, + }, Install { environment: String, installer: String, @@ -51,6 +56,11 @@ async fn run(cli: Cli) -> Result<()> { let (machine, environment) = find_environment(&config, &environment)?; setup(&machine, &environment).await?; } + Command::LayerKey { environment } => { + let (machine, environment) = find_environment(&config, &environment)?; + let key = setup_layer_key(&machine, &environment)?; + println!("{}", key.as_deref().unwrap_or("none")); + } Command::Install { environment, installer, diff --git a/tests/tmachine/src/qemu/ansible_hash.rs b/tests/tmachine/src/qemu/ansible_hash.rs index 3426225898..ddfc7054cc 100644 --- a/tests/tmachine/src/qemu/ansible_hash.rs +++ b/tests/tmachine/src/qemu/ansible_hash.rs @@ -12,7 +12,10 @@ use super::layer::hash_file; pub(super) fn hash_sources(hasher: &mut Hasher) -> Result<()> { let requirements = crate::ansible::requirements_path(); - let root = requirements.parent().unwrap(); + hash_sources_in(hasher, requirements.parent().unwrap()) +} + +pub(super) fn hash_sources_in(hasher: &mut Hasher, root: &Path) -> Result<()> { hash_tree(hasher, root, root) } diff --git a/tests/tmachine/src/qemu/mod.rs b/tests/tmachine/src/qemu/mod.rs index d1b7c119fa..d0f611b896 100644 --- a/tests/tmachine/src/qemu/mod.rs +++ b/tests/tmachine/src/qemu/mod.rs @@ -14,6 +14,6 @@ mod vm; pub use cleanup::until_signal; pub use install::install; pub use session::Session; -pub use setup::setup; +pub use setup::{setup, setup_layer_key}; pub use test::test; pub use vm::ssh_port; diff --git a/tests/tmachine/src/qemu/setup.rs b/tests/tmachine/src/qemu/setup.rs index 83c846fee9..d4b9a9ca91 100644 --- a/tests/tmachine/src/qemu/setup.rs +++ b/tests/tmachine/src/qemu/setup.rs @@ -1,24 +1,23 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use anyhow::{Context, Result}; use blake3::Hasher; use crate::config::{Environment, Machine}; -use super::ansible_hash::hash_sources; +use super::ansible_hash::hash_sources_in; use super::layer::{SETUP_DISKS, cached_layer, hash_file, hash_files, hash_inputs}; const SETUP_CACHE_VERSION: &[u8] = b"tmachine-disk-blake3-v1"; pub async fn setup(machine: &Machine, environment: &Environment) -> Result { - if environment.setup.playbooks.is_empty() { + let Some(hash) = setup_layer_key(machine, environment)? else { return Ok(machine.base_image.clone()); - } + }; - let hash = setup_hash(machine, environment)?; cached_layer( SETUP_DISKS, &machine.base_image, @@ -30,13 +29,23 @@ pub async fn setup(machine: &Machine, environment: &Environment) -> Result Result { +/// The hash that names the environment's setup layer (`disks/.qcow2`), +/// or `None` when it has no setup playbooks and so runs on the base image. +pub fn setup_layer_key(machine: &Machine, environment: &Environment) -> Result> { + if environment.setup.playbooks.is_empty() { + return Ok(None); + } + let requirements = crate::ansible::requirements_path(); + setup_hash(machine, environment, requirements.parent().unwrap()).map(Some) +} + +fn setup_hash(machine: &Machine, environment: &Environment, ansible: &Path) -> Result { let mut hasher = Hasher::new(); hasher.update(SETUP_CACHE_VERSION); hash_file(&mut hasher, &machine.base_image) .with_context(|| format!("failed to hash base image for machine {:?}", machine.name))?; hasher.update(&[u8::from(environment.setup.use_galaxy)]); - hash_sources(&mut hasher).context("failed to hash Ansible sources")?; + hash_sources_in(&mut hasher, ansible).context("failed to hash Ansible sources")?; hash_files(&mut hasher, &environment.setup.playbooks) .context("failed to hash setup playbooks")?; // Skipped when empty so environments without inputs keep their cache keys. @@ -45,3 +54,79 @@ fn setup_hash(machine: &Machine, environment: &Environment) -> Result { } Ok(hasher.finalize().to_hex().to_string()) } + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::fs; + + use super::*; + use crate::config::Setup; + + fn environment(playbooks: Vec, inputs: BTreeMap) -> Environment { + Environment { + name: "test".into(), + machine: "test".into(), + setup: Setup { + use_galaxy: false, + playbooks, + inputs, + }, + } + } + + #[test] + fn an_environment_without_setup_playbooks_has_no_layer() { + let machine = Machine { + name: "test".into(), + base_image: "/nonexistent/base.qcow2".into(), + }; + let key = setup_layer_key(&machine, &environment(Vec::new(), BTreeMap::new())).unwrap(); + assert_eq!(key, None); + } + + #[test] + fn the_setup_hash_follows_every_setup_input() { + let dir = tempfile::tempdir().unwrap(); + let ansible = dir.path().join("ansible"); + fs::create_dir(&ansible).unwrap(); + fs::write(ansible.join("requirements.yaml"), "[]").unwrap(); + let base_image = dir.path().join("base.qcow2"); + fs::write(&base_image, "base").unwrap(); + let playbook = dir.path().join("setup.yaml"); + fs::write(&playbook, "- hosts: all").unwrap(); + let input = dir.path().join("realm.json"); + fs::write(&input, "{}").unwrap(); + + let machine = Machine { + name: "test".into(), + base_image: base_image.clone(), + }; + let environment = environment( + vec![playbook.clone()], + BTreeMap::from([("realm".into(), input.clone())]), + ); + let hash = || setup_hash(&machine, &environment, &ansible).unwrap(); + + let original = hash(); + assert_eq!(original.len(), 64); + assert_eq!(hash(), original); + + let mut seen = vec![original]; + for (file, contents) in [ + (&input, "{\"realm\": 1}"), + (&playbook, "- hosts: guest"), + (&ansible.join("requirements.yaml"), "[role]"), + (&base_image, "rebuilt base"), + ] { + fs::write(file, contents).unwrap(); + let changed = hash(); + assert!( + !seen.contains(&changed), + "{} did not change the hash", + file.display() + ); + seen.push(changed); + } + } +} From 4a53efdc8747d23039960ef9f49c89e25ef28303 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 16 Nov 2026 22:56:40 -0800 Subject: [PATCH 2/3] ci(integration): key the tmachine disk cache on the setup layer hash The integration runner cached all of ~/.cache/tmachine, install layers included, under one fixed key per OS and arch, and skipped ubuntu-k3s. Cache only the setup layers and key them on `tmachine layer-key`, as e2e-suites.yml does. Signed-off-by: Matthew Grossman --- .github/workflows/integration-runner.yml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml index f84811bdd9..0c39f71a98 100644 --- a/.github/workflows/integration-runner.yml +++ b/.github/workflows/integration-runner.yml @@ -64,12 +64,26 @@ jobs: - uses: ./.github/actions/setup-nix + # The hash tmachine names the environment's setup layer after, or `none` + # when the environment has no setup layer. See e2e-suites.yml. + - name: Compute the tmachine setup layer key + id: layer-key + env: + ENVIRONMENT: ${{ matrix.environment }} + run: | + set -euo pipefail + nix build --no-link .#tmachine || nix build --no-link .#tmachine + key=$(nix run .#tmachine -- layer-key "${ENVIRONMENT}") + echo "key=${key}" >> "$GITHUB_OUTPUT" + + # Only the setup layers: install layers under disks/install hold this + # commit's build, so no later run could use them. - name: Cache tmachine disks - if: matrix.environment != 'ubuntu-k3s' + if: steps.layer-key.outputs.key != 'none' uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: - path: ~/.cache/tmachine - key: tmachine-${{ runner.os }}-${{ runner.arch }} + path: ~/.cache/tmachine/disks/*.qcow2 + key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ steps.layer-key.outputs.key }} - name: Download integration inputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 From e553a96c2d50435b7347a1e073571491f0568138 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 16:13:27 -0800 Subject: [PATCH 3/3] refactor(tmachine): simplify the layer-key command Print the setup-layer hash for every environment instead of `none` for environments without setup playbooks, and drop the workflow checks for it. Expose the existing hash function directly, and drop the tests and the Ansible hashing split that only they needed. Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 7 +- .github/workflows/integration-runner.yml | 5 +- tests/tmachine/src/main.rs | 7 +- tests/tmachine/src/qemu/ansible_hash.rs | 5 +- tests/tmachine/src/qemu/setup.rs | 100 ++--------------------- 5 files changed, 17 insertions(+), 107 deletions(-) diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 93f01b0870..446d29c3be 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -147,9 +147,9 @@ jobs: with: cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} - # The hash tmachine names the environment's setup layer after, or `none` - # when the environment has no setup layer. It reads the base image, which - # the run step needs anyway, so this mostly moves that download earlier. + # The hash tmachine names the environment's setup layer after. It reads + # the base image, which the run step needs anyway, so this mostly moves + # that download earlier. - name: Compute the tmachine setup layer key id: layer-key env: @@ -161,7 +161,6 @@ jobs: echo "key=${key}" >> "$GITHUB_OUTPUT" - name: Cache tmachine disks - if: steps.layer-key.outputs.key != 'none' uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: # Only the setup layers, which depend on the environment alone. The diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml index 0c39f71a98..84df0faa00 100644 --- a/.github/workflows/integration-runner.yml +++ b/.github/workflows/integration-runner.yml @@ -64,8 +64,8 @@ jobs: - uses: ./.github/actions/setup-nix - # The hash tmachine names the environment's setup layer after, or `none` - # when the environment has no setup layer. See e2e-suites.yml. + # The hash tmachine names the environment's setup layer after. See + # e2e-suites.yml. - name: Compute the tmachine setup layer key id: layer-key env: @@ -79,7 +79,6 @@ jobs: # Only the setup layers: install layers under disks/install hold this # commit's build, so no later run could use them. - name: Cache tmachine disks - if: steps.layer-key.outputs.key != 'none' uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ~/.cache/tmachine/disks/*.qcow2 diff --git a/tests/tmachine/src/main.rs b/tests/tmachine/src/main.rs index 5cb644c17e..5ab7d2c260 100644 --- a/tests/tmachine/src/main.rs +++ b/tests/tmachine/src/main.rs @@ -23,8 +23,8 @@ enum Command { Setup { environment: String, }, - /// Print the key of the environment's setup layer, or `none` when it has - /// no setup playbooks, without starting a VM. + /// Print the hash that names the environment's setup layer, without + /// starting a VM. LayerKey { environment: String, }, @@ -58,8 +58,7 @@ async fn run(cli: Cli) -> Result<()> { } Command::LayerKey { environment } => { let (machine, environment) = find_environment(&config, &environment)?; - let key = setup_layer_key(&machine, &environment)?; - println!("{}", key.as_deref().unwrap_or("none")); + println!("{}", setup_layer_key(&machine, &environment)?); } Command::Install { environment, diff --git a/tests/tmachine/src/qemu/ansible_hash.rs b/tests/tmachine/src/qemu/ansible_hash.rs index ddfc7054cc..3426225898 100644 --- a/tests/tmachine/src/qemu/ansible_hash.rs +++ b/tests/tmachine/src/qemu/ansible_hash.rs @@ -12,10 +12,7 @@ use super::layer::hash_file; pub(super) fn hash_sources(hasher: &mut Hasher) -> Result<()> { let requirements = crate::ansible::requirements_path(); - hash_sources_in(hasher, requirements.parent().unwrap()) -} - -pub(super) fn hash_sources_in(hasher: &mut Hasher, root: &Path) -> Result<()> { + let root = requirements.parent().unwrap(); hash_tree(hasher, root, root) } diff --git a/tests/tmachine/src/qemu/setup.rs b/tests/tmachine/src/qemu/setup.rs index d4b9a9ca91..66fbc765ee 100644 --- a/tests/tmachine/src/qemu/setup.rs +++ b/tests/tmachine/src/qemu/setup.rs @@ -1,23 +1,24 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use anyhow::{Context, Result}; use blake3::Hasher; use crate::config::{Environment, Machine}; -use super::ansible_hash::hash_sources_in; +use super::ansible_hash::hash_sources; use super::layer::{SETUP_DISKS, cached_layer, hash_file, hash_files, hash_inputs}; const SETUP_CACHE_VERSION: &[u8] = b"tmachine-disk-blake3-v1"; pub async fn setup(machine: &Machine, environment: &Environment) -> Result { - let Some(hash) = setup_layer_key(machine, environment)? else { + if environment.setup.playbooks.is_empty() { return Ok(machine.base_image.clone()); - }; + } + let hash = setup_layer_key(machine, environment)?; cached_layer( SETUP_DISKS, &machine.base_image, @@ -29,23 +30,14 @@ pub async fn setup(machine: &Machine, environment: &Environment) -> Result.qcow2`), -/// or `None` when it has no setup playbooks and so runs on the base image. -pub fn setup_layer_key(machine: &Machine, environment: &Environment) -> Result> { - if environment.setup.playbooks.is_empty() { - return Ok(None); - } - let requirements = crate::ansible::requirements_path(); - setup_hash(machine, environment, requirements.parent().unwrap()).map(Some) -} - -fn setup_hash(machine: &Machine, environment: &Environment, ansible: &Path) -> Result { +/// The hash that names the environment's setup layer (`disks/.qcow2`). +pub fn setup_layer_key(machine: &Machine, environment: &Environment) -> Result { let mut hasher = Hasher::new(); hasher.update(SETUP_CACHE_VERSION); hash_file(&mut hasher, &machine.base_image) .with_context(|| format!("failed to hash base image for machine {:?}", machine.name))?; hasher.update(&[u8::from(environment.setup.use_galaxy)]); - hash_sources_in(&mut hasher, ansible).context("failed to hash Ansible sources")?; + hash_sources(&mut hasher).context("failed to hash Ansible sources")?; hash_files(&mut hasher, &environment.setup.playbooks) .context("failed to hash setup playbooks")?; // Skipped when empty so environments without inputs keep their cache keys. @@ -54,79 +46,3 @@ fn setup_hash(machine: &Machine, environment: &Environment, ansible: &Path) -> R } Ok(hasher.finalize().to_hex().to_string()) } - -#[cfg(test)] -mod tests { - use std::collections::BTreeMap; - use std::fs; - - use super::*; - use crate::config::Setup; - - fn environment(playbooks: Vec, inputs: BTreeMap) -> Environment { - Environment { - name: "test".into(), - machine: "test".into(), - setup: Setup { - use_galaxy: false, - playbooks, - inputs, - }, - } - } - - #[test] - fn an_environment_without_setup_playbooks_has_no_layer() { - let machine = Machine { - name: "test".into(), - base_image: "/nonexistent/base.qcow2".into(), - }; - let key = setup_layer_key(&machine, &environment(Vec::new(), BTreeMap::new())).unwrap(); - assert_eq!(key, None); - } - - #[test] - fn the_setup_hash_follows_every_setup_input() { - let dir = tempfile::tempdir().unwrap(); - let ansible = dir.path().join("ansible"); - fs::create_dir(&ansible).unwrap(); - fs::write(ansible.join("requirements.yaml"), "[]").unwrap(); - let base_image = dir.path().join("base.qcow2"); - fs::write(&base_image, "base").unwrap(); - let playbook = dir.path().join("setup.yaml"); - fs::write(&playbook, "- hosts: all").unwrap(); - let input = dir.path().join("realm.json"); - fs::write(&input, "{}").unwrap(); - - let machine = Machine { - name: "test".into(), - base_image: base_image.clone(), - }; - let environment = environment( - vec![playbook.clone()], - BTreeMap::from([("realm".into(), input.clone())]), - ); - let hash = || setup_hash(&machine, &environment, &ansible).unwrap(); - - let original = hash(); - assert_eq!(original.len(), 64); - assert_eq!(hash(), original); - - let mut seen = vec![original]; - for (file, contents) in [ - (&input, "{\"realm\": 1}"), - (&playbook, "- hosts: guest"), - (&ansible.join("requirements.yaml"), "[role]"), - (&base_image, "rebuilt base"), - ] { - fs::write(file, contents).unwrap(); - let changed = hash(); - assert!( - !seen.contains(&changed), - "{} did not change the hash", - file.display() - ); - seen.push(changed); - } - } -}