Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 18 additions & 11 deletions .github/workflows/e2e-suites.yml
Original file line number Diff line number Diff line change
Expand Up @@ -147,24 +147,31 @@ jobs:
with:
cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}

# The hash tmachine names the environment's setup layer after. It reads
# the base image, which the run step needs anyway, so this mostly moves
# that download earlier.
- name: Compute the tmachine setup layer key
id: layer-key
env:
ENVIRONMENT: ${{ matrix.environment }}
run: |
set -euo pipefail
nix build --no-link .#tmachine || nix build --no-link .#tmachine
key=$(nix run .#tmachine -- layer-key "${ENVIRONMENT}")
echo "key=${key}" >> "$GITHUB_OUTPUT"

- name: Cache tmachine disks
uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
with:
# Only the setup layers, which depend on the environment alone. The
# install layers under disks/install hold this commit's build (and
# on k3s, the gateway's credentials), so no later run could use them.
path: ~/.cache/tmachine/disks/*.qcow2
# tmachine keys its setup layers on a hash of the Ansible tree, the
# environments' playbook lists and setup inputs (config.nix, and
# files such as the Keycloak realm file) and the cache version in
# setup.rs. actions/cache doesn't save after an exact key hit, so a
# key that misses one of those would rebuild the changed layers on
# every run. Environments share no layers, and
# only the first job to save a key gets its disks in, so each
# environment needs its own key; its installers share it. No
# restore-keys fallback: restoring an older entry would carry its
# stale disks into the new one, which then only grows.
key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/layer.rs', 'tests/tmachine/src/qemu/ansible_hash.rs', 'scripts/keycloak-realm.json') }}
# Keyed on the layer's own hash, so the cache misses exactly when
# tmachine would rebuild the layer. The environment's installers
# share the key. No restore-keys fallback: restoring an older entry
# would carry its stale disks into the new one, which then only grows.
key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ steps.layer-key.outputs.key }}

- name: Download integration inputs
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
Expand Down
19 changes: 16 additions & 3 deletions .github/workflows/integration-runner.yml
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,25 @@ jobs:

- uses: ./.github/actions/setup-nix

# The hash tmachine names the environment's setup layer after. See
# e2e-suites.yml.
- name: Compute the tmachine setup layer key
id: layer-key
env:
ENVIRONMENT: ${{ matrix.environment }}
run: |
set -euo pipefail
nix build --no-link .#tmachine || nix build --no-link .#tmachine
key=$(nix run .#tmachine -- layer-key "${ENVIRONMENT}")
echo "key=${key}" >> "$GITHUB_OUTPUT"

# Only the setup layers: install layers under disks/install hold this
# commit's build, so no later run could use them.
- name: Cache tmachine disks
if: matrix.environment != 'ubuntu-k3s'
uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
with:
path: ~/.cache/tmachine
key: tmachine-${{ runner.os }}-${{ runner.arch }}
path: ~/.cache/tmachine/disks/*.qcow2
key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ steps.layer-key.outputs.key }}

- name: Download integration inputs
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
Expand Down
10 changes: 5 additions & 5 deletions TESTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -305,11 +305,11 @@ service.
setup playbook starts the service (with a restart policy, since setup is
cached as a disk) and writes the variables as `KEY=VALUE` lines to
`/etc/openshell-test/env.d/<service>.env`. Pass files the playbook needs
through the environment's setup `inputs`, and add them to the tmachine disk
cache key in `e2e-suites.yml`. The runner rejects names without the
`OPENSHELL_TEST_` prefix, and the tests get the variables in the guest, so
values never appear in a runner command. Values are taken literally, without
quoting.
through the environment's setup `inputs`; they are part of the setup layer's
hash, which CI's disk cache keys on too. The runner rejects names without
the `OPENSHELL_TEST_` prefix, and the tests get the variables in the guest,
so values never appear in a runner command. Values are taken literally,
without quoting.
- **Fixture.** Put the service's fixture in a support crate under
`tests/support/<service>`, named `<service>-support`, so any suite can
depend on it, or reuse the existing one. It defines the variable names as constants and a
Expand Down
5 changes: 3 additions & 2 deletions tests/tmachine/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,13 @@
//! Isolated QEMU test machines provisioned with Ansible.
//!
//! The `tmachine` binary drives one-shot `setup`, `install`, and `test`
//! commands. Callers that need to act on a running machine use [`Session`].
//! commands, and prints the key of an environment's setup layer with
//! `layer-key`. Callers that need to act on a running machine use [`Session`].
//! A binary that starts machines should run its work in [`until_signal`], so
//! a `SIGTERM` does not leave QEMU running.

pub mod ansible;
pub mod config;
mod qemu;

pub use qemu::{Session, install, setup, ssh_port, test, until_signal};
pub use qemu::{Session, install, setup, setup_layer_key, ssh_port, test, until_signal};
11 changes: 10 additions & 1 deletion tests/tmachine/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use std::path::PathBuf;
use anyhow::{Context, Result};
use clap::{Parser, Subcommand};
use tmachine::config::{Config, Environment, Installer, Machine};
use tmachine::{install, setup, test};
use tmachine::{install, setup, setup_layer_key, test};

#[derive(Parser)]
#[command(name = "tmachine")]
Expand All @@ -23,6 +23,11 @@ enum Command {
Setup {
environment: String,
},
/// Print the hash that names the environment's setup layer, without
/// starting a VM.
LayerKey {
environment: String,
},
Install {
environment: String,
installer: String,
Expand Down Expand Up @@ -51,6 +56,10 @@ async fn run(cli: Cli) -> Result<()> {
let (machine, environment) = find_environment(&config, &environment)?;
setup(&machine, &environment).await?;
}
Command::LayerKey { environment } => {
let (machine, environment) = find_environment(&config, &environment)?;
println!("{}", setup_layer_key(&machine, &environment)?);
}
Command::Install {
environment,
installer,
Expand Down
2 changes: 1 addition & 1 deletion tests/tmachine/src/qemu/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,6 @@ mod vm;
pub use cleanup::until_signal;
pub use install::install;
pub use session::Session;
pub use setup::setup;
pub use setup::{setup, setup_layer_key};
pub use test::test;
pub use vm::ssh_port;
5 changes: 3 additions & 2 deletions tests/tmachine/src/qemu/setup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ pub async fn setup(machine: &Machine, environment: &Environment) -> Result<PathB
return Ok(machine.base_image.clone());
}

let hash = setup_hash(machine, environment)?;
let hash = setup_layer_key(machine, environment)?;
cached_layer(
SETUP_DISKS,
&machine.base_image,
Expand All @@ -30,7 +30,8 @@ pub async fn setup(machine: &Machine, environment: &Environment) -> Result<PathB
.await
}

fn setup_hash(machine: &Machine, environment: &Environment) -> Result<String> {
/// The hash that names the environment's setup layer (`disks/<hash>.qcow2`).
pub fn setup_layer_key(machine: &Machine, environment: &Environment) -> Result<String> {
let mut hasher = Hasher::new();
hasher.update(SETUP_CACHE_VERSION);
hash_file(&mut hasher, &machine.base_image)
Expand Down
Loading