Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions .github/workflows/e2e-suites.yml
Original file line number Diff line number Diff line change
Expand Up @@ -147,24 +147,24 @@ jobs:
with:
cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}

# integration-runner.yml skips the disk cache for ubuntu-k3s and doesn't say
# why, so the k3s environment skips it here too until that is understood.
- name: Cache tmachine disks
if: matrix.environment != 'ubuntu-k3s'
uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
with:
path: ~/.cache/tmachine
# tmachine keys its disk layers on a hash of the Ansible tree, the
# Only the setup layers, which depend on the environment alone. The
# install layers under disks/install hold this commit's build (and
# on k3s, the gateway's credentials), so no later run could use them.
path: ~/.cache/tmachine/disks/*.qcow2
# tmachine keys its setup layers on a hash of the Ansible tree, the
# environments' playbook lists and setup inputs (config.nix, and
# files such as the Keycloak realm file) and the cache versions in
# setup.rs and install.rs. actions/cache doesn't save after an
# exact key hit, so a key that misses one of those would rebuild the
# changed layers on every run. Environments share no layers, and
# files such as the Keycloak realm file) and the cache version in
# setup.rs. actions/cache doesn't save after an exact key hit, so a
# key that misses one of those would rebuild the changed layers on
# every run. Environments share no layers, and
# only the first job to save a key gets its disks in, so each
# environment and installer needs its own key. No restore-keys
# fallback: restoring an older entry would carry its stale disks
# into the new one, which then only grows.
key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ matrix.installer }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/install.rs', 'scripts/keycloak-realm.json') }}
# environment needs its own key; its installers share it. No
# restore-keys fallback: restoring an older entry would carry its
# stale disks into the new one, which then only grows.
key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/layer.rs', 'tests/tmachine/src/qemu/ansible_hash.rs', 'scripts/keycloak-realm.json') }}

- name: Download integration inputs
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
Expand Down
28 changes: 28 additions & 0 deletions tests/tmachine/ansible/playbooks/k3s.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -71,3 +71,31 @@
creates: /usr/local/bin/helm
environment:
DESIRED_VERSION: "v{{ helm_version }}"

# This layer is cached across runs, so it must not carry cluster
# credentials: every disk built on it would share one CA, token and admin
# kubeconfig. Stop K3s and delete the state it created on first start,
# keeping the containerd image store (agent/containerd) and the K3s data
# directory. The next start generates fresh credentials.
- name: Stop K3s and its containers
ansible.builtin.command:
argv: [/usr/local/bin/k3s-killall.sh]
changed_when: true

- name: Remove the K3s credentials and cluster state
ansible.builtin.shell: |
set -eu
cd /var/lib/rancher/k3s
rm -rf server/tls server/cred server/db server/token server/agent-token server/node-token
find agent -maxdepth 1 -type f \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' \) -delete
rm -rf /etc/rancher/node/password /etc/rancher/k3s/k3s.yaml /var/lib/kubelet/pods /var/lib/cni
changed_when: true

- name: Check no K3s credentials remain
ansible.builtin.shell: |
find /var/lib/rancher/k3s/server /var/lib/rancher/k3s/agent /etc/rancher/k3s /etc/rancher/node \
-maxdepth 1 \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' -o -name token -o -name password \
-o -name 'k3s.yaml' -o -name '*.db' -o -name tls -o -name cred \) -print
register: k3s_leftover_credentials
changed_when: false
failed_when: k3s_leftover_credentials.stdout | length > 0
3 changes: 2 additions & 1 deletion tests/tmachine/src/qemu/install.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use blake3::Hasher;
use crate::config::{Environment, Installer, Machine};

use super::ansible_hash::hash_sources;
use super::layer::{cached_layer, hash_file, hash_files, hash_inputs};
use super::layer::{INSTALL_DISKS, cached_layer, hash_file, hash_files, hash_inputs};
use super::setup::setup;

const INSTALL_CACHE_VERSION: &[u8] = b"tmachine-install-blake3-v1";
Expand All @@ -26,6 +26,7 @@ pub async fn install(

let hash = install_hash(&setup_disk, installer)?;
cached_layer(
INSTALL_DISKS,
&setup_disk,
&hash,
installer.use_galaxy,
Expand Down
10 changes: 9 additions & 1 deletion tests/tmachine/src/qemu/layer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,23 @@ use directories::ProjectDirs;
use super::img::QemuImage;
use super::vm::{LAYER_POWER_OFF_TIMEOUT, QemuVm};

/// Where setup layers are kept, under tmachine's cache directory. They depend
/// only on the environment, so CI caches the disks here across commits.
pub(super) const SETUP_DISKS: &str = "disks";
/// Where install layers are kept. They hold the build under test, so CI leaves
/// them out of its cache.
pub(super) const INSTALL_DISKS: &str = "disks/install";

pub(super) async fn cached_layer(
disks: &str,
base_image: &Path,
hash: &str,
use_galaxy: bool,
playbooks: &[PathBuf],
inputs: &BTreeMap<String, PathBuf>,
) -> Result<PathBuf> {
let project_dirs = ProjectDirs::from("com", "nvidia", "tmachine").unwrap();
let disks_dir = project_dirs.cache_dir().join("disks");
let disks_dir = project_dirs.cache_dir().join(disks);
std::fs::create_dir_all(&disks_dir).unwrap();

let disk = disks_dir.join(format!("{hash}.qcow2"));
Expand Down
14 changes: 8 additions & 6 deletions tests/tmachine/src/qemu/session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(600);
/// A machine booted from a throwaway overlay of the installed disk.
///
/// The VM stops when the session is dropped, so a caller that fails or
/// panics cannot leak it. Call [`Session::down`] for a clean shutdown.
/// panics cannot leak it. Call [`Session::down`] to stop it and wait until
/// QEMU has exited.
pub struct Session {
// Declared before `_dir` so the VM stops before its disk is removed.
vm: QemuVm,
Expand Down Expand Up @@ -173,13 +174,14 @@ impl Session {
.context("open SSH shell in tmachine VM")
}

/// Power the machine off and wait for QEMU to exit.
/// Stop the machine and wait for QEMU to exit.
///
/// This never panics and never fails: teardown has no useful recovery,
/// so a guest that does not power off cleanly is killed and the problem
/// is logged to stderr.
/// QEMU is killed rather than the guest powered off: the overlay is
/// thrown away, so a clean shutdown would only cost time. Copy out
/// anything the guest holds first. This never panics and never fails.
pub async fn down(self) {
self.vm.power_off_or_kill().await;
let Self { mut vm, .. } = self;
vm.kill().await;
}
}

Expand Down
3 changes: 2 additions & 1 deletion tests/tmachine/src/qemu/setup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use blake3::Hasher;
use crate::config::{Environment, Machine};

use super::ansible_hash::hash_sources;
use super::layer::{cached_layer, hash_file, hash_files, hash_inputs};
use super::layer::{SETUP_DISKS, cached_layer, hash_file, hash_files, hash_inputs};

const SETUP_CACHE_VERSION: &[u8] = b"tmachine-disk-blake3-v1";

Expand All @@ -20,6 +20,7 @@ pub async fn setup(machine: &Machine, environment: &Environment) -> Result<PathB

let hash = setup_hash(machine, environment)?;
cached_layer(
SETUP_DISKS,
&machine.base_image,
&hash,
environment.setup.use_galaxy,
Expand Down
34 changes: 2 additions & 32 deletions tests/tmachine/src/qemu/vm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,18 +25,10 @@ use super::img::QemuImage;
/// `TMACHINE_SSH_PORT` says otherwise.
const DEFAULT_SSH_PORT: u16 = 2222;

/// How long the QMP power-off request has to be acknowledged. A guest or QEMU
/// that stops answering must not hang teardown.
const QMP_TIMEOUT: Duration = Duration::from_secs(10);

/// How long a powered-off guest has to exit at teardown before QEMU is
/// killed.
const POWER_OFF_TIMEOUT: Duration = Duration::from_secs(120);

/// How long a guest being saved as a cached layer has to power off. A heavily
/// provisioned guest (Docker, k3s) can take minutes to stop its services, and
/// a layer that is killed instead is not cached, so this is much longer than
/// [`POWER_OFF_TIMEOUT`]. It still bounds a guest that never stops.
/// a layer that is killed instead is not cached. It still bounds a guest that
/// never stops.
pub(super) const LAYER_POWER_OFF_TIMEOUT: Duration = Duration::from_secs(600);

/// The host port forwarded to the guest's SSH server.
Expand Down Expand Up @@ -170,28 +162,6 @@ impl QemuVm {
}
}
}

/// Power the guest off and wait for QEMU to exit. Never panics: a guest
/// that can't be powered off cleanly is killed, and the problem is
/// logged. Use this when tearing down, where there is nothing left to do
/// about a failure.
pub(super) async fn power_off_or_kill(mut self) {
let shutdown = match tokio::time::timeout(QMP_TIMEOUT, self.shutdown()).await {
Ok(result) => result,
Err(_) => Err(anyhow::anyhow!("QMP did not answer within {QMP_TIMEOUT:?}")),
};
match shutdown {
Ok(()) => {
if let Err(error) = self.wait(POWER_OFF_TIMEOUT).await {
eprintln!("tmachine: {error:#}");
}
}
Err(error) => {
eprintln!("tmachine: could not power off the VM cleanly, killing it: {error:#}");
self.kill().await;
}
}
}
}

fn command(
Expand Down
Loading