diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index ec6cc2e54b..032a5a4186 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -147,24 +147,24 @@ jobs: with: cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} - # integration-runner.yml skips the disk cache for ubuntu-k3s and doesn't say - # why, so the k3s environment skips it here too until that is understood. - name: Cache tmachine disks - if: matrix.environment != 'ubuntu-k3s' uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: - path: ~/.cache/tmachine - # tmachine keys its disk layers on a hash of the Ansible tree, the + # Only the setup layers, which depend on the environment alone. The + # install layers under disks/install hold this commit's build (and + # on k3s, the gateway's credentials), so no later run could use them. + path: ~/.cache/tmachine/disks/*.qcow2 + # tmachine keys its setup layers on a hash of the Ansible tree, the # environments' playbook lists and setup inputs (config.nix, and - # files such as the Keycloak realm file) and the cache versions in - # setup.rs and install.rs. actions/cache doesn't save after an - # exact key hit, so a key that misses one of those would rebuild the - # changed layers on every run. Environments share no layers, and + # files such as the Keycloak realm file) and the cache version in + # setup.rs. actions/cache doesn't save after an exact key hit, so a + # key that misses one of those would rebuild the changed layers on + # every run. Environments share no layers, and # only the first job to save a key gets its disks in, so each - # environment and installer needs its own key. No restore-keys - # fallback: restoring an older entry would carry its stale disks - # into the new one, which then only grows. - key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ matrix.installer }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/install.rs', 'scripts/keycloak-realm.json') }} + # environment needs its own key; its installers share it. No + # restore-keys fallback: restoring an older entry would carry its + # stale disks into the new one, which then only grows. + key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/layer.rs', 'tests/tmachine/src/qemu/ansible_hash.rs', 'scripts/keycloak-realm.json') }} - name: Download integration inputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/tests/tmachine/ansible/playbooks/k3s.yaml b/tests/tmachine/ansible/playbooks/k3s.yaml index d27c6c487b..99c9f667ff 100644 --- a/tests/tmachine/ansible/playbooks/k3s.yaml +++ b/tests/tmachine/ansible/playbooks/k3s.yaml @@ -71,3 +71,31 @@ creates: /usr/local/bin/helm environment: DESIRED_VERSION: "v{{ helm_version }}" + + # This layer is cached across runs, so it must not carry cluster + # credentials: every disk built on it would share one CA, token and admin + # kubeconfig. Stop K3s and delete the state it created on first start, + # keeping the containerd image store (agent/containerd) and the K3s data + # directory. The next start generates fresh credentials. + - name: Stop K3s and its containers + ansible.builtin.command: + argv: [/usr/local/bin/k3s-killall.sh] + changed_when: true + + - name: Remove the K3s credentials and cluster state + ansible.builtin.shell: | + set -eu + cd /var/lib/rancher/k3s + rm -rf server/tls server/cred server/db server/token server/agent-token server/node-token + find agent -maxdepth 1 -type f \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' \) -delete + rm -rf /etc/rancher/node/password /etc/rancher/k3s/k3s.yaml /var/lib/kubelet/pods /var/lib/cni + changed_when: true + + - name: Check no K3s credentials remain + ansible.builtin.shell: | + find /var/lib/rancher/k3s/server /var/lib/rancher/k3s/agent /etc/rancher/k3s /etc/rancher/node \ + -maxdepth 1 \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' -o -name token -o -name password \ + -o -name 'k3s.yaml' -o -name '*.db' -o -name tls -o -name cred \) -print + register: k3s_leftover_credentials + changed_when: false + failed_when: k3s_leftover_credentials.stdout | length > 0 diff --git a/tests/tmachine/src/qemu/install.rs b/tests/tmachine/src/qemu/install.rs index e1e5910520..25b3d6abf0 100644 --- a/tests/tmachine/src/qemu/install.rs +++ b/tests/tmachine/src/qemu/install.rs @@ -9,7 +9,7 @@ use blake3::Hasher; use crate::config::{Environment, Installer, Machine}; use super::ansible_hash::hash_sources; -use super::layer::{cached_layer, hash_file, hash_files, hash_inputs}; +use super::layer::{INSTALL_DISKS, cached_layer, hash_file, hash_files, hash_inputs}; use super::setup::setup; const INSTALL_CACHE_VERSION: &[u8] = b"tmachine-install-blake3-v1"; @@ -26,6 +26,7 @@ pub async fn install( let hash = install_hash(&setup_disk, installer)?; cached_layer( + INSTALL_DISKS, &setup_disk, &hash, installer.use_galaxy, diff --git a/tests/tmachine/src/qemu/layer.rs b/tests/tmachine/src/qemu/layer.rs index e6ec03650d..7cc9b09aa1 100644 --- a/tests/tmachine/src/qemu/layer.rs +++ b/tests/tmachine/src/qemu/layer.rs @@ -12,7 +12,15 @@ use directories::ProjectDirs; use super::img::QemuImage; use super::vm::{LAYER_POWER_OFF_TIMEOUT, QemuVm}; +/// Where setup layers are kept, under tmachine's cache directory. They depend +/// only on the environment, so CI caches the disks here across commits. +pub(super) const SETUP_DISKS: &str = "disks"; +/// Where install layers are kept. They hold the build under test, so CI leaves +/// them out of its cache. +pub(super) const INSTALL_DISKS: &str = "disks/install"; + pub(super) async fn cached_layer( + disks: &str, base_image: &Path, hash: &str, use_galaxy: bool, @@ -20,7 +28,7 @@ pub(super) async fn cached_layer( inputs: &BTreeMap, ) -> Result { let project_dirs = ProjectDirs::from("com", "nvidia", "tmachine").unwrap(); - let disks_dir = project_dirs.cache_dir().join("disks"); + let disks_dir = project_dirs.cache_dir().join(disks); std::fs::create_dir_all(&disks_dir).unwrap(); let disk = disks_dir.join(format!("{hash}.qcow2")); diff --git a/tests/tmachine/src/qemu/session.rs b/tests/tmachine/src/qemu/session.rs index c3682d5958..576ece9dc6 100644 --- a/tests/tmachine/src/qemu/session.rs +++ b/tests/tmachine/src/qemu/session.rs @@ -60,7 +60,8 @@ pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(600); /// A machine booted from a throwaway overlay of the installed disk. /// /// The VM stops when the session is dropped, so a caller that fails or -/// panics cannot leak it. Call [`Session::down`] for a clean shutdown. +/// panics cannot leak it. Call [`Session::down`] to stop it and wait until +/// QEMU has exited. pub struct Session { // Declared before `_dir` so the VM stops before its disk is removed. vm: QemuVm, @@ -173,13 +174,14 @@ impl Session { .context("open SSH shell in tmachine VM") } - /// Power the machine off and wait for QEMU to exit. + /// Stop the machine and wait for QEMU to exit. /// - /// This never panics and never fails: teardown has no useful recovery, - /// so a guest that does not power off cleanly is killed and the problem - /// is logged to stderr. + /// QEMU is killed rather than the guest powered off: the overlay is + /// thrown away, so a clean shutdown would only cost time. Copy out + /// anything the guest holds first. This never panics and never fails. pub async fn down(self) { - self.vm.power_off_or_kill().await; + let Self { mut vm, .. } = self; + vm.kill().await; } } diff --git a/tests/tmachine/src/qemu/setup.rs b/tests/tmachine/src/qemu/setup.rs index 185d6c290b..83c846fee9 100644 --- a/tests/tmachine/src/qemu/setup.rs +++ b/tests/tmachine/src/qemu/setup.rs @@ -9,7 +9,7 @@ use blake3::Hasher; use crate::config::{Environment, Machine}; use super::ansible_hash::hash_sources; -use super::layer::{cached_layer, hash_file, hash_files, hash_inputs}; +use super::layer::{SETUP_DISKS, cached_layer, hash_file, hash_files, hash_inputs}; const SETUP_CACHE_VERSION: &[u8] = b"tmachine-disk-blake3-v1"; @@ -20,6 +20,7 @@ pub async fn setup(machine: &Machine, environment: &Environment) -> Result result, - Err(_) => Err(anyhow::anyhow!("QMP did not answer within {QMP_TIMEOUT:?}")), - }; - match shutdown { - Ok(()) => { - if let Err(error) = self.wait(POWER_OFF_TIMEOUT).await { - eprintln!("tmachine: {error:#}"); - } - } - Err(error) => { - eprintln!("tmachine: could not power off the VM cleanly, killing it: {error:#}"); - self.kill().await; - } - } - } } fn command(