From d52deca9162150e702f88e491c29b893a08f67ee Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 16 Nov 2026 09:11:56 -0800 Subject: [PATCH 1/3] ci(e2e): cut fixed setup and teardown costs of the k3s conformance rows Cache the tmachine setup layers for ubuntu-k3s too, and leave install layers out of the disk cache: they hold the build under test and, on k3s, the gateway's credentials. Install layers now live under disks/install so the cache can take only the setup layers, keyed per environment. Kill QEMU at session teardown instead of powering the guest off, since the session overlay is thrown away. Layer builds still power off cleanly. Pull the HA PostgreSQL fixture image in the k3s setup layer, and run the prebuilt test program with a 180-second per-test limit instead of the 900-second default. Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 26 ++++++++--------- TESTING.md | 6 ++-- tests/runner/src/execute.rs | 16 +++++++++-- tests/tmachine/ansible/playbooks/k3s.yaml | 17 ++++++++++++ tests/tmachine/config.nix | 9 ++++-- tests/tmachine/src/qemu/install.rs | 3 +- tests/tmachine/src/qemu/layer.rs | 10 ++++++- tests/tmachine/src/qemu/session.rs | 14 ++++++---- tests/tmachine/src/qemu/setup.rs | 3 +- tests/tmachine/src/qemu/vm.rs | 34 ++--------------------- 10 files changed, 77 insertions(+), 61 deletions(-) diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index ec6cc2e54b..8c72512e5a 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -147,24 +147,24 @@ jobs: with: cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }} - # integration-runner.yml skips the disk cache for ubuntu-k3s and doesn't say - # why, so the k3s environment skips it here too until that is understood. - name: Cache tmachine disks - if: matrix.environment != 'ubuntu-k3s' uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: - path: ~/.cache/tmachine - # tmachine keys its disk layers on a hash of the Ansible tree, the + # Only the setup layers, which depend on the environment alone. The + # install layers under disks/install hold this commit's build (and + # on k3s, the gateway's credentials), so no later run could use them. + path: ~/.cache/tmachine/disks/*.qcow2 + # tmachine keys its setup layers on a hash of the Ansible tree, the # environments' playbook lists and setup inputs (config.nix, and - # files such as the Keycloak realm file) and the cache versions in - # setup.rs and install.rs. actions/cache doesn't save after an - # exact key hit, so a key that misses one of those would rebuild the - # changed layers on every run. Environments share no layers, and + # files such as the Keycloak realm file and the PostgreSQL fixture) + # and the cache version in setup.rs. actions/cache doesn't save after + # an exact key hit, so a key that misses one of those would rebuild + # the changed layers on every run. Environments share no layers, and # only the first job to save a key gets its disks in, so each - # environment and installer needs its own key. No restore-keys - # fallback: restoring an older entry would carry its stale disks - # into the new one, which then only grows. - key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ matrix.installer }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/install.rs', 'scripts/keycloak-realm.json') }} + # environment needs its own key; its installers share it. No + # restore-keys fallback: restoring an older entry would carry its + # stale disks into the new one, which then only grows. + key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/layer.rs', 'scripts/keycloak-realm.json', 'e2e/kubernetes/postgres-fixture.yaml') }} - name: Download integration inputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/TESTING.md b/TESTING.md index 58bb638831..9a36097f4d 100644 --- a/TESTING.md +++ b/TESTING.md @@ -254,8 +254,10 @@ combined with `--package`. With it, `--filter` (repeatable) takes parts of test names, as the program does, such as `--filter policy_advisor::`, rather than a nextest filterset. The runner runs the program's `ready` and `run` commands in the guest and still writes `junit.xml` and `nextest.log` -(the program's output) to the results directory. Branch E2E builds -`openshell-conformance` once and passes it to every conformance row this way. +(the program's output) to the results directory. It stops a test that runs +longer than two minutes under nextest, or three minutes under a program. +Branch E2E builds `openshell-conformance` once and passes it to every +conformance row this way. `openshell-test-runner build --family --out ` builds a family's test bundle for the guest without booting a VM, and `run --bundle ` runs diff --git a/tests/runner/src/execute.rs b/tests/runner/src/execute.rs index 2f477134f9..4b9486fb08 100644 --- a/tests/runner/src/execute.rs +++ b/tests/runner/src/execute.rs @@ -32,9 +32,12 @@ const NEXTEST_LOG: &str = "nextest.log"; // disk can take minutes to report healthy. const READY_TIMEOUT_SECS: u64 = 300; /// How long the nextest run may take in total. Individual tests are bounded -/// by the nextest profile's slow-timeout; this catches a run that stops -/// making progress some other way. +/// by the nextest profile's slow-timeout or [`PROGRAM_TEST_TIMEOUT_SECS`]; +/// this catches a run that stops making progress some other way. const RUN_TIMEOUT: Duration = Duration::from_mins(45); +/// How long a test program gives each test before stopping it. A test that +/// stalls fails in minutes instead of at the program's 15-minute default. +const PROGRAM_TEST_TIMEOUT_SECS: &str = "180"; /// Slack beyond the readiness check's own timeout for SSH and process start. const READY_SLACK: Duration = Duration::from_mins(1); /// nextest's exit code when a test failed, as opposed to nextest itself @@ -342,7 +345,14 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { // The program runs its tests one at a time, and writes the report // where `collect_results` looks for it. Kind::Program => { - tests.extend([program.as_str(), "run", "--junit", &junit]); + tests.extend([ + program.as_str(), + "run", + "--timeout", + PROGRAM_TEST_TIMEOUT_SECS, + "--junit", + &junit, + ]); // `run` takes name filters as arguments, after `--` so that one // starting with a dash isn't read as an option. if !run.filters.is_empty() { diff --git a/tests/tmachine/ansible/playbooks/k3s.yaml b/tests/tmachine/ansible/playbooks/k3s.yaml index d27c6c487b..91c346841d 100644 --- a/tests/tmachine/ansible/playbooks/k3s.yaml +++ b/tests/tmachine/ansible/playbooks/k3s.yaml @@ -57,6 +57,23 @@ retries: 12 delay: 5 + # The k3s-ha installer starts PostgreSQL from this fixture. Its image is + # pinned by digest and pulled IfNotPresent, so pulling it here keeps it in + # the cached setup layer instead of pulling it on every install. + - name: Pull the PostgreSQL fixture image + ansible.builtin.command: + argv: [/usr/local/bin/k3s, crictl, pull, "{{ postgres_image }}"] + vars: + postgres_image: >- + {{ (lookup('ansible.builtin.file', openshell_postgres_fixture) + | ansible.builtin.from_yaml_all + | selectattr('kind', 'equalto', 'Deployment') | first) + .spec.template.spec.containers[0].image }} + register: postgres_pull + until: postgres_pull.rc == 0 + retries: 3 + delay: 5 + - name: Download Helm installer ansible.builtin.get_url: url: https://raw.githubusercontent.com/helm/helm/06468084e85c244c712834933d25ea232a4c2093/scripts/get-helm-4 diff --git a/tests/tmachine/config.nix b/tests/tmachine/config.nix index 7255dc1d85..4b76e3d0b4 100644 --- a/tests/tmachine/config.nix +++ b/tests/tmachine/config.nix @@ -25,6 +25,7 @@ let qemu = qemuPkgs.qemu.override { hostCpuOnly = true; }; # Every installer loads the sandbox workload image tests run python3 in. e2ePythonImage = "../../artifacts/images/openshell-e2e-python-dev.tar"; + postgresFixture = "../../e2e/kubernetes/postgres-fixture.yaml"; k3sInputs = { agent_sandbox_version = "0.5.0"; openshell_cli_binary = "../../artifacts/binaries/${muslTarget}/openshell"; @@ -66,6 +67,8 @@ let "ansible/playbooks/test-tools.yaml" "ansible/playbooks/k3s.yaml" ]; + # k3s.yaml pulls the image of the k3s-ha installer's PostgreSQL fixture. + inputs.openshell_postgres_fixture = postgresFixture; }; fedora-podman-rootful = fedoraPodman "rootful"; fedora-podman-rootless = fedoraPodman "rootless"; @@ -91,7 +94,8 @@ let inherit (parent) use_galaxy; playbooks = parent.playbooks ++ setup.playbooks; inputs = - (if parent ? driver then { environment_driver = parent.driver; } else { }) + (parent.inputs or { }) + // (if parent ? driver then { environment_driver = parent.driver; } else { }) // (setup.inputs or { }); }; }; @@ -153,6 +157,7 @@ let inherit (base) machine; setup = { inherit (base) use_galaxy playbooks; + inputs = base.inputs or { }; }; }) bases ++ featureEnvironments @@ -170,7 +175,7 @@ let use_galaxy = false; playbooks = [ "ansible/playbooks/openshell-k3s-ha.yaml" ]; inputs = k3sInputs // { - openshell_postgres_fixture = "../../e2e/kubernetes/postgres-fixture.yaml"; + openshell_postgres_fixture = postgresFixture; }; } { diff --git a/tests/tmachine/src/qemu/install.rs b/tests/tmachine/src/qemu/install.rs index e1e5910520..25b3d6abf0 100644 --- a/tests/tmachine/src/qemu/install.rs +++ b/tests/tmachine/src/qemu/install.rs @@ -9,7 +9,7 @@ use blake3::Hasher; use crate::config::{Environment, Installer, Machine}; use super::ansible_hash::hash_sources; -use super::layer::{cached_layer, hash_file, hash_files, hash_inputs}; +use super::layer::{INSTALL_DISKS, cached_layer, hash_file, hash_files, hash_inputs}; use super::setup::setup; const INSTALL_CACHE_VERSION: &[u8] = b"tmachine-install-blake3-v1"; @@ -26,6 +26,7 @@ pub async fn install( let hash = install_hash(&setup_disk, installer)?; cached_layer( + INSTALL_DISKS, &setup_disk, &hash, installer.use_galaxy, diff --git a/tests/tmachine/src/qemu/layer.rs b/tests/tmachine/src/qemu/layer.rs index e6ec03650d..7cc9b09aa1 100644 --- a/tests/tmachine/src/qemu/layer.rs +++ b/tests/tmachine/src/qemu/layer.rs @@ -12,7 +12,15 @@ use directories::ProjectDirs; use super::img::QemuImage; use super::vm::{LAYER_POWER_OFF_TIMEOUT, QemuVm}; +/// Where setup layers are kept, under tmachine's cache directory. They depend +/// only on the environment, so CI caches the disks here across commits. +pub(super) const SETUP_DISKS: &str = "disks"; +/// Where install layers are kept. They hold the build under test, so CI leaves +/// them out of its cache. +pub(super) const INSTALL_DISKS: &str = "disks/install"; + pub(super) async fn cached_layer( + disks: &str, base_image: &Path, hash: &str, use_galaxy: bool, @@ -20,7 +28,7 @@ pub(super) async fn cached_layer( inputs: &BTreeMap, ) -> Result { let project_dirs = ProjectDirs::from("com", "nvidia", "tmachine").unwrap(); - let disks_dir = project_dirs.cache_dir().join("disks"); + let disks_dir = project_dirs.cache_dir().join(disks); std::fs::create_dir_all(&disks_dir).unwrap(); let disk = disks_dir.join(format!("{hash}.qcow2")); diff --git a/tests/tmachine/src/qemu/session.rs b/tests/tmachine/src/qemu/session.rs index c3682d5958..576ece9dc6 100644 --- a/tests/tmachine/src/qemu/session.rs +++ b/tests/tmachine/src/qemu/session.rs @@ -60,7 +60,8 @@ pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(600); /// A machine booted from a throwaway overlay of the installed disk. /// /// The VM stops when the session is dropped, so a caller that fails or -/// panics cannot leak it. Call [`Session::down`] for a clean shutdown. +/// panics cannot leak it. Call [`Session::down`] to stop it and wait until +/// QEMU has exited. pub struct Session { // Declared before `_dir` so the VM stops before its disk is removed. vm: QemuVm, @@ -173,13 +174,14 @@ impl Session { .context("open SSH shell in tmachine VM") } - /// Power the machine off and wait for QEMU to exit. + /// Stop the machine and wait for QEMU to exit. /// - /// This never panics and never fails: teardown has no useful recovery, - /// so a guest that does not power off cleanly is killed and the problem - /// is logged to stderr. + /// QEMU is killed rather than the guest powered off: the overlay is + /// thrown away, so a clean shutdown would only cost time. Copy out + /// anything the guest holds first. This never panics and never fails. pub async fn down(self) { - self.vm.power_off_or_kill().await; + let Self { mut vm, .. } = self; + vm.kill().await; } } diff --git a/tests/tmachine/src/qemu/setup.rs b/tests/tmachine/src/qemu/setup.rs index 185d6c290b..83c846fee9 100644 --- a/tests/tmachine/src/qemu/setup.rs +++ b/tests/tmachine/src/qemu/setup.rs @@ -9,7 +9,7 @@ use blake3::Hasher; use crate::config::{Environment, Machine}; use super::ansible_hash::hash_sources; -use super::layer::{cached_layer, hash_file, hash_files, hash_inputs}; +use super::layer::{SETUP_DISKS, cached_layer, hash_file, hash_files, hash_inputs}; const SETUP_CACHE_VERSION: &[u8] = b"tmachine-disk-blake3-v1"; @@ -20,6 +20,7 @@ pub async fn setup(machine: &Machine, environment: &Environment) -> Result result, - Err(_) => Err(anyhow::anyhow!("QMP did not answer within {QMP_TIMEOUT:?}")), - }; - match shutdown { - Ok(()) => { - if let Err(error) = self.wait(POWER_OFF_TIMEOUT).await { - eprintln!("tmachine: {error:#}"); - } - } - Err(error) => { - eprintln!("tmachine: could not power off the VM cleanly, killing it: {error:#}"); - self.kill().await; - } - } - } } fn command( From 9c481adcbdabb52aa74cdb7381abe3b23c165f28 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 16 Nov 2026 14:59:22 -0800 Subject: [PATCH 2/3] ci(e2e): drop the test timeout and PostgreSQL pre-pull from the k3s fixed-cost change Leave per-test timeouts to the runner work that owns them, and stop pulling the PostgreSQL fixture image in the k3s setup layer, which saved no measurable time and tied the tmachine cache key to e2e/. Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 10 +++++----- TESTING.md | 6 ++---- tests/runner/src/execute.rs | 16 +++------------- tests/tmachine/ansible/playbooks/k3s.yaml | 17 ----------------- tests/tmachine/config.nix | 9 ++------- 5 files changed, 12 insertions(+), 46 deletions(-) diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 8c72512e5a..f4dee9ab32 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -156,15 +156,15 @@ jobs: path: ~/.cache/tmachine/disks/*.qcow2 # tmachine keys its setup layers on a hash of the Ansible tree, the # environments' playbook lists and setup inputs (config.nix, and - # files such as the Keycloak realm file and the PostgreSQL fixture) - # and the cache version in setup.rs. actions/cache doesn't save after - # an exact key hit, so a key that misses one of those would rebuild - # the changed layers on every run. Environments share no layers, and + # files such as the Keycloak realm file) and the cache version in + # setup.rs. actions/cache doesn't save after an exact key hit, so a + # key that misses one of those would rebuild the changed layers on + # every run. Environments share no layers, and # only the first job to save a key gets its disks in, so each # environment needs its own key; its installers share it. No # restore-keys fallback: restoring an older entry would carry its # stale disks into the new one, which then only grows. - key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/layer.rs', 'scripts/keycloak-realm.json', 'e2e/kubernetes/postgres-fixture.yaml') }} + key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/layer.rs', 'scripts/keycloak-realm.json') }} - name: Download integration inputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/TESTING.md b/TESTING.md index 9a36097f4d..58bb638831 100644 --- a/TESTING.md +++ b/TESTING.md @@ -254,10 +254,8 @@ combined with `--package`. With it, `--filter` (repeatable) takes parts of test names, as the program does, such as `--filter policy_advisor::`, rather than a nextest filterset. The runner runs the program's `ready` and `run` commands in the guest and still writes `junit.xml` and `nextest.log` -(the program's output) to the results directory. It stops a test that runs -longer than two minutes under nextest, or three minutes under a program. -Branch E2E builds `openshell-conformance` once and passes it to every -conformance row this way. +(the program's output) to the results directory. Branch E2E builds +`openshell-conformance` once and passes it to every conformance row this way. `openshell-test-runner build --family --out ` builds a family's test bundle for the guest without booting a VM, and `run --bundle ` runs diff --git a/tests/runner/src/execute.rs b/tests/runner/src/execute.rs index 4b9486fb08..2f477134f9 100644 --- a/tests/runner/src/execute.rs +++ b/tests/runner/src/execute.rs @@ -32,12 +32,9 @@ const NEXTEST_LOG: &str = "nextest.log"; // disk can take minutes to report healthy. const READY_TIMEOUT_SECS: u64 = 300; /// How long the nextest run may take in total. Individual tests are bounded -/// by the nextest profile's slow-timeout or [`PROGRAM_TEST_TIMEOUT_SECS`]; -/// this catches a run that stops making progress some other way. +/// by the nextest profile's slow-timeout; this catches a run that stops +/// making progress some other way. const RUN_TIMEOUT: Duration = Duration::from_mins(45); -/// How long a test program gives each test before stopping it. A test that -/// stalls fails in minutes instead of at the program's 15-minute default. -const PROGRAM_TEST_TIMEOUT_SECS: &str = "180"; /// Slack beyond the readiness check's own timeout for SSH and process start. const READY_SLACK: Duration = Duration::from_mins(1); /// nextest's exit code when a test failed, as opposed to nextest itself @@ -345,14 +342,7 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { // The program runs its tests one at a time, and writes the report // where `collect_results` looks for it. Kind::Program => { - tests.extend([ - program.as_str(), - "run", - "--timeout", - PROGRAM_TEST_TIMEOUT_SECS, - "--junit", - &junit, - ]); + tests.extend([program.as_str(), "run", "--junit", &junit]); // `run` takes name filters as arguments, after `--` so that one // starting with a dash isn't read as an option. if !run.filters.is_empty() { diff --git a/tests/tmachine/ansible/playbooks/k3s.yaml b/tests/tmachine/ansible/playbooks/k3s.yaml index 91c346841d..d27c6c487b 100644 --- a/tests/tmachine/ansible/playbooks/k3s.yaml +++ b/tests/tmachine/ansible/playbooks/k3s.yaml @@ -57,23 +57,6 @@ retries: 12 delay: 5 - # The k3s-ha installer starts PostgreSQL from this fixture. Its image is - # pinned by digest and pulled IfNotPresent, so pulling it here keeps it in - # the cached setup layer instead of pulling it on every install. - - name: Pull the PostgreSQL fixture image - ansible.builtin.command: - argv: [/usr/local/bin/k3s, crictl, pull, "{{ postgres_image }}"] - vars: - postgres_image: >- - {{ (lookup('ansible.builtin.file', openshell_postgres_fixture) - | ansible.builtin.from_yaml_all - | selectattr('kind', 'equalto', 'Deployment') | first) - .spec.template.spec.containers[0].image }} - register: postgres_pull - until: postgres_pull.rc == 0 - retries: 3 - delay: 5 - - name: Download Helm installer ansible.builtin.get_url: url: https://raw.githubusercontent.com/helm/helm/06468084e85c244c712834933d25ea232a4c2093/scripts/get-helm-4 diff --git a/tests/tmachine/config.nix b/tests/tmachine/config.nix index 4b76e3d0b4..7255dc1d85 100644 --- a/tests/tmachine/config.nix +++ b/tests/tmachine/config.nix @@ -25,7 +25,6 @@ let qemu = qemuPkgs.qemu.override { hostCpuOnly = true; }; # Every installer loads the sandbox workload image tests run python3 in. e2ePythonImage = "../../artifacts/images/openshell-e2e-python-dev.tar"; - postgresFixture = "../../e2e/kubernetes/postgres-fixture.yaml"; k3sInputs = { agent_sandbox_version = "0.5.0"; openshell_cli_binary = "../../artifacts/binaries/${muslTarget}/openshell"; @@ -67,8 +66,6 @@ let "ansible/playbooks/test-tools.yaml" "ansible/playbooks/k3s.yaml" ]; - # k3s.yaml pulls the image of the k3s-ha installer's PostgreSQL fixture. - inputs.openshell_postgres_fixture = postgresFixture; }; fedora-podman-rootful = fedoraPodman "rootful"; fedora-podman-rootless = fedoraPodman "rootless"; @@ -94,8 +91,7 @@ let inherit (parent) use_galaxy; playbooks = parent.playbooks ++ setup.playbooks; inputs = - (parent.inputs or { }) - // (if parent ? driver then { environment_driver = parent.driver; } else { }) + (if parent ? driver then { environment_driver = parent.driver; } else { }) // (setup.inputs or { }); }; }; @@ -157,7 +153,6 @@ let inherit (base) machine; setup = { inherit (base) use_galaxy playbooks; - inputs = base.inputs or { }; }; }) bases ++ featureEnvironments @@ -175,7 +170,7 @@ let use_galaxy = false; playbooks = [ "ansible/playbooks/openshell-k3s-ha.yaml" ]; inputs = k3sInputs // { - openshell_postgres_fixture = postgresFixture; + openshell_postgres_fixture = "../../e2e/kubernetes/postgres-fixture.yaml"; }; } { From d8d1a0681f5da4af71081bf6e9d72a83d621148a Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Mon, 16 Nov 2026 20:59:23 -0800 Subject: [PATCH 3/3] ci(e2e): wipe k3s credentials from the cached setup layer The k3s setup layer boots k3s once, so it holds the cluster CA, token, node password and admin kubeconfig. Stop k3s and delete that state before the layer is saved, so each run's cluster generates its own credentials, and fail the layer build if any credential file remains. Also key the disk cache on ansible_hash.rs, which feeds the setup layer hash. Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 2 +- tests/tmachine/ansible/playbooks/k3s.yaml | 28 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index f4dee9ab32..032a5a4186 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -164,7 +164,7 @@ jobs: # environment needs its own key; its installers share it. No # restore-keys fallback: restoring an older entry would carry its # stale disks into the new one, which then only grows. - key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/layer.rs', 'scripts/keycloak-realm.json') }} + key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ matrix.environment }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/config.nix', 'tests/tmachine/images.nix', 'tests/tmachine/src/qemu/setup.rs', 'tests/tmachine/src/qemu/layer.rs', 'tests/tmachine/src/qemu/ansible_hash.rs', 'scripts/keycloak-realm.json') }} - name: Download integration inputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/tests/tmachine/ansible/playbooks/k3s.yaml b/tests/tmachine/ansible/playbooks/k3s.yaml index d27c6c487b..99c9f667ff 100644 --- a/tests/tmachine/ansible/playbooks/k3s.yaml +++ b/tests/tmachine/ansible/playbooks/k3s.yaml @@ -71,3 +71,31 @@ creates: /usr/local/bin/helm environment: DESIRED_VERSION: "v{{ helm_version }}" + + # This layer is cached across runs, so it must not carry cluster + # credentials: every disk built on it would share one CA, token and admin + # kubeconfig. Stop K3s and delete the state it created on first start, + # keeping the containerd image store (agent/containerd) and the K3s data + # directory. The next start generates fresh credentials. + - name: Stop K3s and its containers + ansible.builtin.command: + argv: [/usr/local/bin/k3s-killall.sh] + changed_when: true + + - name: Remove the K3s credentials and cluster state + ansible.builtin.shell: | + set -eu + cd /var/lib/rancher/k3s + rm -rf server/tls server/cred server/db server/token server/agent-token server/node-token + find agent -maxdepth 1 -type f \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' \) -delete + rm -rf /etc/rancher/node/password /etc/rancher/k3s/k3s.yaml /var/lib/kubelet/pods /var/lib/cni + changed_when: true + + - name: Check no K3s credentials remain + ansible.builtin.shell: | + find /var/lib/rancher/k3s/server /var/lib/rancher/k3s/agent /etc/rancher/k3s /etc/rancher/node \ + -maxdepth 1 \( -name '*.crt' -o -name '*.key' -o -name '*.kubeconfig' -o -name token -o -name password \ + -o -name 'k3s.yaml' -o -name '*.db' -o -name tls -o -name cred \) -print + register: k3s_leftover_credentials + changed_when: false + failed_when: k3s_leftover_credentials.stdout | length > 0