diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 032a5a4186..446d29c3be 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -147,6 +147,19 @@ jobs: with: cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} + # The hash tmachine names the environment's setup layer after. It reads + # the base image, which the run step needs anyway, so this mostly moves + # that download earlier. + - name: Compute the tmachine setup layer key + id: layer-key + env: + ENVIRONMENT: ${{ matrix.environment }} + run: | + set -euo pipefail + nix build --no-link .#tmachine || nix build --no-link .#tmachine + key=$(nix run .#tmachine -- layer-key "${ENVIRONMENT}") + echo "key=${key}" >> "$GITHUB_OUTPUT" + - name: Cache tmachine disks uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: @@ -154,17 +167,11 @@ jobs: # install layers under disks/install hold this commit's build (and # on k3s, the gateway's credentials), so no later run could use them. path: ~/.cache/tmachine/disks/*.qcow2 - # tmachine keys its setup layers on a hash of the Ansible tree, the - # environments' playbook lists and setup inputs (config.nix, and - # files such as the Keycloak realm file) and the cache version in - # setup.rs. actions/cache doesn't save after an exact key hit, so a - # key that misses one of those would rebuild the changed layers on - # every run. Environments share no layers, and - # only the first job to save a key gets its disks in, so each - # environment needs its own key; its installers share it. No - # restore-keys fallback: restoring an older entry would carry its - # stale disks into the new one, which then only grows. - key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/layer.rs', 'tests/tmachine/src/qemu/ansible_hash.rs', 'scripts/keycloak-realm.json') }} + # Keyed on the layer's own hash, so the cache misses exactly when + # tmachine would rebuild the layer. The environment's installers + # share the key. No restore-keys fallback: restoring an older entry + # would carry its stale disks into the new one, which then only grows. + key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ steps.layer-key.outputs.key }} - name: Download integration inputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/.github/workflows/integration-runner.yml b/.github/workflows/integration-runner.yml index f84811bdd9..84df0faa00 100644 --- a/.github/workflows/integration-runner.yml +++ b/.github/workflows/integration-runner.yml @@ -64,12 +64,25 @@ jobs: - uses: ./.github/actions/setup-nix + # The hash tmachine names the environment's setup layer after. See + # e2e-suites.yml. + - name: Compute the tmachine setup layer key + id: layer-key + env: + ENVIRONMENT: ${{ matrix.environment }} + run: | + set -euo pipefail + nix build --no-link .#tmachine || nix build --no-link .#tmachine + key=$(nix run .#tmachine -- layer-key "${ENVIRONMENT}") + echo "key=${key}" >> "$GITHUB_OUTPUT" + + # Only the setup layers: install layers under disks/install hold this + # commit's build, so no later run could use them. - name: Cache tmachine disks - if: matrix.environment != 'ubuntu-k3s' uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: - path: ~/.cache/tmachine - key: tmachine-${{ runner.os }}-${{ runner.arch }} + path: ~/.cache/tmachine/disks/*.qcow2 + key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ steps.layer-key.outputs.key }} - name: Download integration inputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/TESTING.md b/TESTING.md index 58bb638831..93e410cf86 100644 --- a/TESTING.md +++ b/TESTING.md @@ -305,11 +305,11 @@ service. setup playbook starts the service (with a restart policy, since setup is cached as a disk) and writes the variables as `KEY=VALUE` lines to `/etc/openshell-test/env.d/.env`. Pass files the playbook needs - through the environment's setup `inputs`, and add them to the tmachine disk - cache key in `e2e-suites.yml`. The runner rejects names without the - `OPENSHELL_TEST_` prefix, and the tests get the variables in the guest, so - values never appear in a runner command. Values are taken literally, without - quoting. + through the environment's setup `inputs`; they are part of the setup layer's + hash, which CI's disk cache keys on too. The runner rejects names without + the `OPENSHELL_TEST_` prefix, and the tests get the variables in the guest, + so values never appear in a runner command. Values are taken literally, + without quoting. - **Fixture.** Put the service's fixture in a support crate under `tests/support/`, named `-support`, so any suite can depend on it, or reuse the existing one. It defines the variable names as constants and a diff --git a/tests/tmachine/src/lib.rs b/tests/tmachine/src/lib.rs index 5376c367af..662e6d562f 100644 --- a/tests/tmachine/src/lib.rs +++ b/tests/tmachine/src/lib.rs @@ -4,7 +4,8 @@ //! Isolated QEMU test machines provisioned with Ansible. //! //! The `tmachine` binary drives one-shot `setup`, `install`, and `test` -//! commands. Callers that need to act on a running machine use [`Session`]. +//! commands, and prints the key of an environment's setup layer with +//! `layer-key`. Callers that need to act on a running machine use [`Session`]. //! A binary that starts machines should run its work in [`until_signal`], so //! a `SIGTERM` does not leave QEMU running. @@ -12,4 +13,4 @@ pub mod ansible; pub mod config; mod qemu; -pub use qemu::{Session, install, setup, ssh_port, test, until_signal}; +pub use qemu::{Session, install, setup, setup_layer_key, ssh_port, test, until_signal}; diff --git a/tests/tmachine/src/main.rs b/tests/tmachine/src/main.rs index d38325cd36..5ab7d2c260 100644 --- a/tests/tmachine/src/main.rs +++ b/tests/tmachine/src/main.rs @@ -6,7 +6,7 @@ use std::path::PathBuf; use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; use tmachine::config::{Config, Environment, Installer, Machine}; -use tmachine::{install, setup, test}; +use tmachine::{install, setup, setup_layer_key, test}; #[derive(Parser)] #[command(name = "tmachine")] @@ -23,6 +23,11 @@ enum Command { Setup { environment: String, }, + /// Print the hash that names the environment's setup layer, without + /// starting a VM. + LayerKey { + environment: String, + }, Install { environment: String, installer: String, @@ -51,6 +56,10 @@ async fn run(cli: Cli) -> Result<()> { let (machine, environment) = find_environment(&config, &environment)?; setup(&machine, &environment).await?; } + Command::LayerKey { environment } => { + let (machine, environment) = find_environment(&config, &environment)?; + println!("{}", setup_layer_key(&machine, &environment)?); + } Command::Install { environment, installer, diff --git a/tests/tmachine/src/qemu/mod.rs b/tests/tmachine/src/qemu/mod.rs index d1b7c119fa..d0f611b896 100644 --- a/tests/tmachine/src/qemu/mod.rs +++ b/tests/tmachine/src/qemu/mod.rs @@ -14,6 +14,6 @@ mod vm; pub use cleanup::until_signal; pub use install::install; pub use session::Session; -pub use setup::setup; +pub use setup::{setup, setup_layer_key}; pub use test::test; pub use vm::ssh_port; diff --git a/tests/tmachine/src/qemu/setup.rs b/tests/tmachine/src/qemu/setup.rs index 83c846fee9..66fbc765ee 100644 --- a/tests/tmachine/src/qemu/setup.rs +++ b/tests/tmachine/src/qemu/setup.rs @@ -18,7 +18,7 @@ pub async fn setup(machine: &Machine, environment: &Environment) -> Result Result Result { +/// The hash that names the environment's setup layer (`disks/.qcow2`). +pub fn setup_layer_key(machine: &Machine, environment: &Environment) -> Result { let mut hasher = Hasher::new(); hasher.update(SETUP_CACHE_VERSION); hash_file(&mut hasher, &machine.base_image)