diff --git a/tests/runner/src/bundle.rs b/tests/runner/src/bundle.rs index 4f953f6a8e..8257a5f108 100644 --- a/tests/runner/src/bundle.rs +++ b/tests/runner/src/bundle.rs @@ -32,12 +32,17 @@ pub const PROFILE: &str = "openshell"; pub const READY_BINARY: &str = "testkit-ready"; /// Nextest flags a test as slow after this period and terminates it after -/// [`SLOW_TERMINATE_AFTER`] periods, so a hung test fails after two minutes -/// instead of holding the lane until the job times out. +/// [`SLOW_TERMINATE_AFTER`] periods. const SLOW_PERIOD_SECS: u64 = 30; const SLOW_TERMINATE_AFTER: u64 = 4; -/// How long a terminated test has between SIGTERM and SIGKILL. -const GRACE_PERIOD_SECS: u64 = 10; + +/// How long one test may run before the runner stops it, on either kind of +/// bundle, so a hung test fails after two minutes instead of holding the lane +/// until the job times out. +pub const TEST_TIMEOUT_SECS: u64 = SLOW_PERIOD_SECS * SLOW_TERMINATE_AFTER; +/// How long a stopped test has between SIGTERM and SIGKILL, on either kind of +/// bundle. +pub const TEST_GRACE_SECS: u64 = 10; /// The nextest configuration written into a nextest bundle. fn nextest_config() -> String { @@ -45,7 +50,7 @@ fn nextest_config() -> String { r#"# Written by the OpenShell test runner. [profile.openshell] fail-fast = false -slow-timeout = {{ period = "{SLOW_PERIOD_SECS}s", terminate-after = {SLOW_TERMINATE_AFTER}, grace-period = "{GRACE_PERIOD_SECS}s" }} +slow-timeout = {{ period = "{SLOW_PERIOD_SECS}s", terminate-after = {SLOW_TERMINATE_AFTER}, grace-period = "{TEST_GRACE_SECS}s" }} [profile.openshell.junit] path = "junit.xml" @@ -354,11 +359,12 @@ mod tests { use super::*; #[test] - fn nextest_profile_stops_a_hung_test_after_two_minutes() { + fn nextest_profile_stops_a_test_at_the_shared_timeout() { let config = nextest_config(); assert!(config.contains( r#"slow-timeout = { period = "30s", terminate-after = 4, grace-period = "10s" }"# )); + assert_eq!(TEST_TIMEOUT_SECS, 120); } #[test] diff --git a/tests/runner/src/collect-k3s.sh b/tests/runner/src/collect-k3s.sh index 3b8408a2a5..bdaf774e57 100755 --- a/tests/runner/src/collect-k3s.sh +++ b/tests/runner/src/collect-k3s.sh @@ -28,6 +28,27 @@ collect_log() { printf '\ncommand status: %s\n' "${PIPESTATUS[0]}" } +# Gateway logs cover the whole run, so a failing test's window is included. +# The runner records when the run started; without that, use a fixed window. +# The tail is larger than for other logs because a gateway logs many lines. +RUN_STARTED_FILE=/var/tmp/openshell-run-started +log_since=--since=30m +if run_started=$(cat "$RUN_STARTED_FILE" 2>/dev/null) && + [[ $run_started =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]; then + log_since=--since-time=$run_started +fi + +collect_gateway_log() { + printf '\n### %s (%s)\n' "$1" "$log_since" + shift + timeout 20s "$@" 2>&1 | { + head -c 65536 + printf '\n### Remaining log tail\n' + tail -c 524288 + } + printf '\ncommand status: %s\n' "${PIPESTATUS[0]}" +} + { date --utc --iso-8601=seconds collect 'K3s service state' systemctl show k3s.service \ @@ -74,8 +95,8 @@ collect_log() { mapfile -t gateway_pods < <(timeout 10s /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \ --selector app.kubernetes.io/name=openshell,app.kubernetes.io/instance=openshell --output name 2>/dev/null) for pod in "${gateway_pods[@]}"; do - collect "Gateway current logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --tail=300 --timestamps - collect "Gateway previous logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --previous --tail=300 --timestamps + collect_gateway_log "Gateway current logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway "$log_since" --timestamps + collect_gateway_log "Gateway previous logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --previous "$log_since" --timestamps done collect 'Memory' free -m collect 'Disk' df -h / /var/lib/rancher/k3s diff --git a/tests/runner/src/execute.rs b/tests/runner/src/execute.rs index 4ac41d743b..d7e64c53cd 100644 --- a/tests/runner/src/execute.rs +++ b/tests/runner/src/execute.rs @@ -20,7 +20,9 @@ use anyhow::{Context, Result, ensure}; use tmachine::Session; use tmachine::config::Config; -use crate::bundle::{Bundle, Kind, PROFILE, PROGRAM_BINARY, READY_BINARY}; +use crate::bundle::{ + Bundle, Kind, PROFILE, PROGRAM_BINARY, READY_BINARY, TEST_GRACE_SECS, TEST_TIMEOUT_SECS, +}; use crate::test_env; const GUEST_DIR: &str = "/var/tmp/openshell-tests"; @@ -32,8 +34,8 @@ const NEXTEST_LOG: &str = "nextest.log"; // disk can take minutes to report healthy. const READY_TIMEOUT_SECS: u64 = 300; /// How long the test run may take in total. Individual tests are bounded by -/// the nextest profile's slow-timeout, or by a program's own `run --timeout` -/// default; this catches a run that stops making progress some other way. +/// [`TEST_TIMEOUT_SECS`]; this catches a run that stops making progress some +/// other way. const RUN_TIMEOUT: Duration = Duration::from_mins(45); /// Slack beyond the readiness check's own timeout for SSH and process start. const READY_SLACK: Duration = Duration::from_mins(1); @@ -42,6 +44,9 @@ const READY_SLACK: Duration = Duration::from_mins(1); const NEXTEST_TEST_FAILED: i32 = 100; /// nextest's exit code when `--no-tests fail` found no tests to run. const NEXTEST_NO_TESTS_RUN: i32 = 4; +/// The variable that tells a test its per-test limit; testkit reads it +/// (`openshell_testkit::TIMEOUT_ENV`). +const TEST_TIMEOUT_ENV: &str = "OPENSHELL_TEST_TIMEOUT_SECS"; /// A testkit program's exit code when a test failed. It exits 2 when tests /// could not run. const PROGRAM_TEST_FAILED: i32 = 1; @@ -308,7 +313,12 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { let archive = format!("{guest}/tests.tar.zst"); let extract = format!("{guest}/extract"); let config = format!("{guest}/nextest.toml"); - let mut tests = vec!["env"]; + let timeout = TEST_TIMEOUT_SECS.to_string(); + let grace = TEST_GRACE_SECS.to_string(); + // Each test is told its own limit, so a stuck step fails with its own + // error before the runner stops the test. + let budget = format!("{TEST_TIMEOUT_ENV}={TEST_TIMEOUT_SECS}"); + let mut tests = vec!["env", budget.as_str()]; tests.extend(env.iter().map(String::as_str)); match kind { Kind::Nextest => { @@ -342,7 +352,16 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { // The program runs its tests one at a time, and writes the report // where `collect_results` looks for it. Kind::Program => { - tests.extend([program.as_str(), "run", "--junit", &junit]); + tests.extend([ + program.as_str(), + "run", + "--timeout", + &timeout, + "--grace", + &grace, + "--junit", + &junit, + ]); // `run` takes name filters as arguments, after `--` so that one // starting with a dash isn't read as an option. if !run.filters.is_empty() { @@ -357,7 +376,7 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result { // can be credentials, never appear in the command (or its errors). let log = format!("{guest}/{NEXTEST_LOG}"); let script = format!( - r#"exec >"$0" 2>&1; {}; exec "$@""#, + r#"exec >"$0" 2>&1; rm -f {RUN_STARTED_FILE}; date -u +%Y-%m-%dT%H:%M:%SZ >{RUN_STARTED_FILE} || true; {}; exec "$@""#, test_env::export_script(test_env::ENV_DIR) ); let mut command = vec!["sh", "-c", &script, &log]; @@ -427,6 +446,9 @@ const WAIT_K3S_CONTROLLER: &str = include_str!("wait-k3s-controller.sh"); /// The script retries for up to ten minutes; this catches one that stalls. const WAIT_K3S_CONTROLLER_TIMEOUT: Duration = Duration::from_mins(12); +/// Where the run records when it started, in RFC 3339 UTC. The k3s diagnostics +/// read it (see `collect-k3s.sh`) to collect the gateway's logs from then on. +const RUN_STARTED_FILE: &str = "/var/tmp/openshell-run-started"; /// Where the guest keeps the script's output until it is copied out. const COLLECT_K3S_OUTPUT: &str = "/var/tmp/openshell-k3s-diagnostics.txt"; /// How long the guest lets the script run before stopping it. The script @@ -536,6 +558,11 @@ mod tests { assert!(matches!(status(2), Status::Error(_))); } + #[test] + fn k3s_diagnostics_read_the_file_the_run_writes() { + assert!(COLLECT_K3S.contains(RUN_STARTED_FILE)); + } + #[test] fn k3s_diagnostics_keep_everything_the_old_playbook_collected() { for wanted in [ @@ -562,6 +589,7 @@ mod tests { "free -m", "df -h", "[REDACTED]", + "--since-time", ] { assert!(COLLECT_K3S.contains(wanted), "missing {wanted:?}"); } diff --git a/tests/suites/conformance/src/lifecycle.rs b/tests/suites/conformance/src/lifecycle.rs index 71854d2b52..d0cae8cf8e 100644 --- a/tests/suites/conformance/src/lifecycle.rs +++ b/tests/suites/conformance/src/lifecycle.rs @@ -34,8 +34,8 @@ async fn long_running_exec_delete(cx: &mut Context) -> testkit::Result { /// Longer than the supervisor's 30s startup recovery deadline. const PAST_RECOVERY_DEADLINE: Duration = Duration::from_secs(31); -/// The exec's own timeout is the runner's two-minute kill, so fail sooner, -/// with a message that says which step hung. +/// The exec's own timeout is longer than the runner's per-test limit, so fail +/// sooner, with a message that says which step hung. const PAST_RECOVERY_EXEC_DEADLINE: Duration = Duration::from_mins(1); /// An exec admitted before the startup recovery deadline expires keeps diff --git a/tests/suites/conformance/src/policy_advisor.rs b/tests/suites/conformance/src/policy_advisor.rs index 82ff54cc09..81b841a685 100644 --- a/tests/suites/conformance/src/policy_advisor.rs +++ b/tests/suites/conformance/src/policy_advisor.rs @@ -9,7 +9,7 @@ //! Lezar. Requests to `policy.local` run inside the sandbox; settings and drafts //! go through the gateway's gRPC API. -use std::time::{Duration, Instant}; +use std::time::Duration; use serde_json::Value; use testkit::sdk::raw::proto; @@ -31,32 +31,36 @@ async fn policy_local_accepts_proposals(cx: &mut Context) -> testkit::Result { // The current-policy route is local; proposal lookup also needs the // supervisor's session with the gateway, so wait for both. - let started = Instant::now(); let readiness = format!("/v1/proposals/test-{}-readiness", cx.run_id()); - loop { - let ready = async { - let current = policy_local(&sandbox, "GET", "/v1/policy/current", "", 200).await?; - testkit::ensure!( - current["format"] == "yaml" - && current["policy_yaml"] - .as_str() - .is_some_and(|yaml| yaml.contains("version: 1")), - "policy.local returned an invalid current policy: {current}" - ); - let lookup = policy_local(&sandbox, "GET", &readiness, "", 404).await?; - testkit::ensure!( - lookup["error"] == "chunk_not_found", - "policy.local proposal lookup returned {lookup}" - ); - Ok(()) - } - .await; - match ready { - Ok(()) => break, - Err(error) if started.elapsed() >= READY_TIMEOUT => return Err(error), - Err(_) => tokio::time::sleep(POLL_INTERVAL).await, - } - } + testkit::poll_until( + "policy.local to be ready", + READY_TIMEOUT, + POLL_INTERVAL, + async || { + let ready = async { + let current = policy_local(&sandbox, "GET", "/v1/policy/current", "", 200).await?; + testkit::ensure!( + current["format"] == "yaml" + && current["policy_yaml"] + .as_str() + .is_some_and(|yaml| yaml.contains("version: 1")), + "policy.local returned an invalid current policy: {current}" + ); + let lookup = policy_local(&sandbox, "GET", &readiness, "", 404).await?; + testkit::ensure!( + lookup["error"] == "chunk_not_found", + "policy.local proposal lookup returned {lookup}" + ); + Ok(()) + } + .await; + Ok(match ready { + Ok(()) => testkit::Poll::Ready(()), + Err(error) => testkit::Poll::Pending(error.to_string()), + }) + }, + ) + .await?; let denials = policy_local(&sandbox, "GET", "/v1/denials?last=1", "", 200).await?; testkit::ensure!( @@ -360,22 +364,21 @@ async fn expect_draft( binary: &str, probe_stderr: &str, ) -> testkit::Result { - let started = Instant::now(); - let chunks = loop { - match sandbox.draft_policy().await { - Ok(draft) if !draft.chunks.is_empty() => break draft.chunks, - Ok(_) | Err(_) if started.elapsed() < PROPOSAL_TIMEOUT => { - tokio::time::sleep(POLL_INTERVAL).await; - } - Ok(_) => { - return Err(Error::msg(format!( - "no draft rule for {}:{} after {PROPOSAL_TIMEOUT:?}; probe stderr: {probe_stderr}", - endpoint.0, endpoint.1 - ))); - } - Err(error) => return Err(error), - } - }; + let chunks = testkit::poll_until( + &format!("a draft rule for {}:{}", endpoint.0, endpoint.1), + PROPOSAL_TIMEOUT, + POLL_INTERVAL, + async || { + Ok(match sandbox.draft_policy().await { + Ok(draft) if !draft.chunks.is_empty() => testkit::Poll::Ready(draft.chunks), + Ok(_) => { + testkit::Poll::Pending(format!("no draft rule; probe stderr: {probe_stderr}")) + } + Err(error) => testkit::Poll::Pending(error.to_string()), + }) + }, + ) + .await?; testkit::ensure!( chunks.len() == 1, diff --git a/tests/support/keycloak/src/lib.rs b/tests/support/keycloak/src/lib.rs index 468cf29278..543ecb042a 100644 --- a/tests/support/keycloak/src/lib.rs +++ b/tests/support/keycloak/src/lib.rs @@ -55,13 +55,12 @@ pub const PASSWORD_ENV: &str = "OPENSHELL_TEST_KEYCLOAK_PASSWORD"; // The wait runs inside the test, so it must leave room for the test body // within the runner's 2-minute per-test budget (`slow-timeout` in // tests/runner/src/bundle.rs). Otherwise a slow Keycloak gets the test killed -// instead of failing with a readiness error. `poll_until` checks the deadline -// only between attempts, so the wait can last this plus one request -// (`CURL_MAX_TIME`) and one interval: 87 seconds, leaving 33 for the test. +// instead of failing with a readiness error. `poll_until` bounds the attempt in +// flight as well as the retry interval, so the wait lasts this and no more, +// leaving 45 seconds for the test. pub const READY_TIMEOUT: Duration = Duration::from_secs(75); -/// How long one request to Keycloak may take, in seconds. `poll_until` checks -/// its deadline only between attempts, so a request to a Keycloak that -/// accepts connections but stalls must give up on its own. +/// How long one request to Keycloak may take, in seconds, so a Keycloak that +/// accepts connections but stalls does not hold one attempt for the whole wait. const CURL_MAX_TIME: &str = "10"; /// How to reach the environment's Keycloak realm, and a user in it. diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 3ce6f58013..3ee41a8742 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -134,6 +134,14 @@ plaintext and mTLS gateways. `OPENSHELL_WORKSPACE` selects the workspace (default `openshell` on PATH). The CLI always targets the same gateway and workspace as the SDK. Each test logs the gateway it used. +`OPENSHELL_TEST_TIMEOUT_SECS` is the time one test may take, as the runner +enforces it. The runner sets it. When set, `cx.cli`, `cx.exec` and the wait +helpers give up about five seconds before that limit, with their own error, +instead of leaving the test to be stopped. When unset, as under plain +`cargo test`, each step keeps its own timeout. A CLI command that runs longer +than ten seconds, exits non-zero, or is still running after 30 seconds is +reported on stderr; fast successful commands print nothing. + Wait for the gateway, then run a suite. Run tests that share a gateway one at a time: diff --git a/tests/testkit/testkit/Cargo.toml b/tests/testkit/testkit/Cargo.toml index abfa8cf97b..7d5cda4851 100644 --- a/tests/testkit/testkit/Cargo.toml +++ b/tests/testkit/testkit/Cargo.toml @@ -30,7 +30,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yml = { package = "noyalib", version = "0.0.53", default-features = false, features = ["std", "compat-serde-yaml"] } tempfile = "3" -tokio = { version = "1.43", features = ["macros", "process", "rt", "time"] } +tokio = { version = "1.43", features = ["io-util", "macros", "process", "rt", "time"] } tonic = "0.14" [target.'cfg(unix)'.dependencies] diff --git a/tests/testkit/testkit/src/budget.rs b/tests/testkit/testkit/src/budget.rs new file mode 100644 index 0000000000..ba29abf071 --- /dev/null +++ b/tests/testkit/testkit/src/budget.rs @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The time one test may take, so a stuck step fails with its own error +//! before whoever runs the tests stops the test with a generic timeout. +//! +//! The runner exports the limit it enforces in [`TIMEOUT_ENV`]. Without it, +//! as when a test runs under plain `cargo test`, nothing is capped and each +//! step keeps its own timeout. + +use std::sync::OnceLock; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +/// The per-test limit the runner enforces, in seconds. +pub const TIMEOUT_ENV: &str = "OPENSHELL_TEST_TIMEOUT_SECS"; + +/// How long before the limit a step gives up, so its error is reported and +/// the test cleans up before the runner stops it. The clock starts at +/// [`start`], not when the runner spawns the test, so the real margin is a +/// little less. +const MARGIN: Duration = Duration::from_secs(5); + +/// The least a step is given, so it still reports a timeout of its own. +const FLOOR: Duration = Duration::from_secs(1); + +/// Set once the test body is over: cleanup needs its own timeouts, because it +/// is most needed when the budget ran out. +static RELEASED: AtomicBool = AtomicBool::new(false); + +static DEADLINE: OnceLock> = OnceLock::new(); + +/// Start the clock for the current test. Call once, when the test starts. +pub(crate) fn start() { + DEADLINE.get_or_init(|| deadline_from(std::env::var(TIMEOUT_ENV).ok().as_deref())); +} + +/// Stop capping, for cleanup. +pub(crate) fn release() { + RELEASED.store(true, Ordering::Relaxed); +} + +fn deadline_from(value: Option<&str>) -> Option { + let secs: u64 = value?.trim().parse().ok().filter(|secs| *secs > 0)?; + Instant::now().checked_add(Duration::from_secs(secs).saturating_sub(MARGIN)) +} + +/// `timeout`, or the time left in the test's budget if that is less. +pub(crate) fn cap(timeout: Duration) -> Duration { + if RELEASED.load(Ordering::Relaxed) { + return timeout; + } + cap_at(timeout, DEADLINE.get().copied().flatten()) +} + +fn cap_at(timeout: Duration, deadline: Option) -> Duration { + match deadline { + Some(deadline) => timeout.min( + deadline + .saturating_duration_since(Instant::now()) + .max(FLOOR), + ), + None => timeout, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn no_limit_leaves_the_timeout_alone() { + assert_eq!(deadline_from(None), None); + assert_eq!(deadline_from(Some("soon")), None); + assert_eq!(deadline_from(Some("0")), None); + assert_eq!( + cap_at(Duration::from_secs(300), None), + Duration::from_secs(300) + ); + } + + #[test] + fn a_step_gets_no_more_than_the_time_left() { + let deadline = deadline_from(Some("120")).unwrap(); + let capped = cap_at(Duration::from_secs(300), Some(deadline)); + assert!(capped <= Duration::from_secs(115), "{capped:?}"); + assert!(capped > Duration::from_secs(100), "{capped:?}"); + // A short step keeps its own timeout. + assert_eq!( + cap_at(Duration::from_secs(10), Some(deadline)), + Duration::from_secs(10) + ); + } + + #[test] + fn an_exhausted_budget_still_leaves_a_floor() { + let passed = Instant::now(); + assert_eq!(cap_at(Duration::from_secs(300), Some(passed)), FLOOR); + } +} diff --git a/tests/testkit/testkit/src/cli.rs b/tests/testkit/testkit/src/cli.rs index ad597482db..3ced2d9164 100644 --- a/tests/testkit/testkit/src/cli.rs +++ b/tests/testkit/testkit/src/cli.rs @@ -11,7 +11,10 @@ use std::ffi::OsStr; use std::path::{Path, PathBuf}; use std::process::Stdio; -use std::time::Duration; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use tokio::io::AsyncReadExt; use serde::de::DeserializeOwned; @@ -21,6 +24,14 @@ use crate::output::Outcome; /// How long one CLI invocation may take. pub const CLI_TIMEOUT: Duration = Duration::from_mins(5); +/// A command that takes longer than this is reported when it finishes. +const SLOW_COMMAND: Duration = Duration::from_secs(10); +/// How often a command that is still running is reported. +const STILL_RUNNING_EVERY: Duration = Duration::from_secs(30); +/// How much of a timed-out command's output the error includes. +const TIMEOUT_OUTPUT_BYTES: usize = 2048; +/// How long to wait for a stopped command's output to close. +const CAPTURE_CLOSE: Duration = Duration::from_secs(2); /// The CLI binary to run. Defaults to `openshell` on PATH. pub(crate) const BIN_ENV: &str = "OPENSHELL_BIN"; @@ -95,6 +106,21 @@ impl Cli { env: &[(&str, &str)], args: I, ) -> Result + where + I: IntoIterator, + S: AsRef, + { + self.run_limited(crate::budget::cap(CLI_TIMEOUT), dir, env, args) + .await + } + + async fn run_limited( + &self, + limit: Duration, + dir: Option<&Path>, + env: &[(&str, &str)], + args: I, + ) -> Result where I: IntoIterator, S: AsRef, @@ -133,17 +159,75 @@ impl Cli { process.current_dir(dir); } - let output = tokio::time::timeout(CLI_TIMEOUT, process.output()) - .await - .map_err(|_| Error::msg(format!("timed out after {CLI_TIMEOUT:?}"))) - .context(format!("run {command}"))? + let started = Instant::now(); + let mut child = process + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .context(format!("run {command}"))?; + let stdout = Capture::start(child.stdout.take()); + let stderr = Capture::start(child.stderr.take()); + + // Say so while a command is still running, so a hang shows in the + // test's output instead of only in nextest's generic timeout. + let mut next_note = STILL_RUNNING_EVERY; + let status = loop { + let elapsed = started.elapsed(); + if elapsed >= limit { + break None; + } + let wait = next_note.min(limit).saturating_sub(elapsed); + tokio::select! { + status = child.wait() => break Some(status), + () = tokio::time::sleep(wait) => {} + } + if started.elapsed() >= next_note && started.elapsed() < limit { + eprintln!( + "still running after {}s: {command}", + started.elapsed().as_secs() + ); + next_note += STILL_RUNNING_EVERY; + } + }; + + let Some(status) = status else { + // Stop it, and report what it printed so far. + let _ = child.kill().await; + let (stdout, stderr) = (stdout.finish().await, stderr.finish().await); + let budget = if limit < CLI_TIMEOUT { + " (the test's time budget)" + } else { + "" + }; + return Err(Error::msg(format!( + "timed out after {limit:?}{budget}; stdout: {}; stderr: {}", + tail(&stdout), + tail(&stderr) + )) + .context(format!("run {command}"))); + }; + let status = status + .map_err(|error| Error::from_error(&error)) .context(format!("run {command}"))?; + let stdout = stdout.finish().await; + let stderr = stderr.finish().await; + + // A signal-terminated CLI has no code; report it as a failure. + let exit_code = status.code().unwrap_or(-1); + let took = started.elapsed(); + if exit_code != 0 { + eprintln!( + "`{command}` exited with {exit_code} after {}s", + took.as_secs() + ); + } else if took >= SLOW_COMMAND { + eprintln!("`{command}` took {}s", took.as_secs()); + } Ok(CliOutput { command, - // A signal-terminated CLI has no code; report it as a failure. - exit_code: output.status.code().unwrap_or(-1), - stdout: String::from_utf8_lossy(&output.stdout).into_owned(), - stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + exit_code, + stdout: String::from_utf8_lossy(&stdout).into_owned(), + stderr: String::from_utf8_lossy(&stderr).into_owned(), }) } } @@ -199,3 +283,119 @@ impl CliOutput { .context(format!("parse JSON output of `{}`", self.command)) } } + +/// A child's output, collected as it arrives so that what a command printed +/// before it was stopped is not lost. +struct Capture { + buffer: Arc>>, + task: tokio::task::JoinHandle<()>, +} + +impl Capture { + fn start(stream: Option) -> Self + where + R: tokio::io::AsyncRead + Unpin + Send + 'static, + { + let buffer = Arc::new(Mutex::new(Vec::new())); + let sink = Arc::clone(&buffer); + let task = tokio::spawn(async move { + let Some(mut stream) = stream else { return }; + let mut chunk = [0u8; 8192]; + while let Ok(n) = stream.read(&mut chunk).await { + if n == 0 { + break; + } + sink.lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .extend_from_slice(&chunk[..n]); + } + }); + Self { buffer, task } + } + + /// Wait for the stream to close, up to [`CAPTURE_CLOSE`], and return what + /// was read. A grandchild that keeps the pipe open does not hold this up. + async fn finish(self) -> Vec { + let Self { buffer, mut task } = self; + if tokio::time::timeout(CAPTURE_CLOSE, &mut task) + .await + .is_err() + { + task.abort(); + } + std::mem::take( + &mut *buffer + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner), + ) + } +} + +/// The end of `output`, trimmed, for an error message. +fn tail(output: &[u8]) -> String { + let start = output.len().saturating_sub(TIMEOUT_OUTPUT_BYTES); + let text = String::from_utf8_lossy(&output[start..]); + let text = text.trim(); + if text.is_empty() { + "(none)".to_string() + } else if start > 0 { + format!("...{text}") + } else { + text.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sh() -> Cli { + Cli { + binary: PathBuf::from("sh"), + gateway: Gateway::from_endpoint( + "http://127.0.0.1:1".to_string(), + "default".to_string(), + ), + } + } + + #[tokio::test] + async fn a_command_past_its_limit_fails_with_its_name_and_output() { + let error = sh() + .run_limited( + Duration::from_millis(500), + None, + &[], + ["-c", "echo partial; echo oops >&2; sleep 30"], + ) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("run sh -c echo partial"), "{error}"); + assert!(error.contains("timed out after"), "{error}"); + assert!(error.contains("stdout: partial"), "{error}"); + assert!(error.contains("stderr: oops"), "{error}"); + } + + #[tokio::test] + async fn a_fast_command_returns_its_output() { + let output = sh() + .run_limited( + Duration::from_secs(30), + None, + &[], + ["-c", "echo hi; exit 3"], + ) + .await + .unwrap(); + assert_eq!(output.exit_code, 3); + assert_eq!(output.stdout.trim(), "hi"); + } + + #[test] + fn tail_keeps_the_end_of_long_output() { + let long = "x".repeat(TIMEOUT_OUTPUT_BYTES + 10); + assert!(tail(long.as_bytes()).starts_with("...")); + assert_eq!(tail(b" "), "(none)"); + } +} diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index 47221f68ec..1113fb6dc4 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -209,6 +209,7 @@ impl Context { /// created and has not deleted. Returns the cleanup failures; they are /// reported separately from the test result. pub(crate) async fn cleanup(&self) -> Vec { + crate::budget::release(); self.handles.cleanup.run(CLEANUP_TIMEOUT).await } } @@ -491,7 +492,7 @@ impl Sandbox { let snapshot = self .handles .client - .wait_ready(&self.name, READY_TIMEOUT) + .wait_ready(&self.name, crate::budget::cap(READY_TIMEOUT)) .await .context(format!("wait for sandbox {} to be ready", self.name))?; crate::ensure!( @@ -632,7 +633,7 @@ impl Sandbox { let command: Vec = command.into_iter().map(Into::into).collect(); let description = format!("exec {command:?} in sandbox {}", self.name); let options = ExecOptions { - timeout: Some(EXEC_TIMEOUT), + timeout: Some(crate::budget::cap(EXEC_TIMEOUT)), ..ExecOptions::default() }; let result = self @@ -744,7 +745,7 @@ impl Sandbox { pub async fn wait_absent(&self, timeout: Duration) -> Result { self.handles .client - .wait_deleted(&self.name, timeout, Some(&self.id)) + .wait_deleted(&self.name, crate::budget::cap(timeout), Some(&self.id)) .await .context(format!("wait for sandbox {} to be removed", self.name))?; self.expect_absent().await @@ -759,7 +760,11 @@ impl Sandbox { .context(format!("delete sandbox {}", self.name))?; self.handles .client - .wait_deleted(&self.name, DELETE_TIMEOUT, Some(&self.id)) + .wait_deleted( + &self.name, + crate::budget::cap(DELETE_TIMEOUT), + Some(&self.id), + ) .await .context(format!("wait for sandbox {} to be deleted", self.name))?; self.handles diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 86f28e2599..869babae5a 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -26,6 +26,7 @@ //! [`main`] is a test program that lists, runs, and reports its tests without //! Cargo (see [`program`]). +mod budget; mod cleanup; mod cli; mod cli_sandbox; @@ -43,6 +44,7 @@ mod registry; mod run; mod yaml_file; +pub use budget::TIMEOUT_ENV; pub use cleanup::CLEANUP_TIMEOUT; pub use cli::{CLI_TIMEOUT, Cli, CliOutput}; pub use cli_sandbox::CliSandboxBuilder; diff --git a/tests/testkit/testkit/src/poll.rs b/tests/testkit/testkit/src/poll.rs index f65592be88..d4ae41a20c 100644 --- a/tests/testkit/testkit/src/poll.rs +++ b/tests/testkit/testkit/src/poll.rs @@ -29,24 +29,94 @@ pub async fn with_deadline( /// Observe until the condition holds or `timeout` elapses. /// -/// An error from `observe` fails immediately. On timeout, the error reports -/// the last observation. +/// An error from `observe` fails immediately. The deadline also covers an +/// observation in flight, so one that hangs, such as a request that never +/// answers, fails with the last completed observation instead of outliving +/// the wait. On timeout, the error reports that observation. pub async fn poll_until( what: &str, timeout: Duration, interval: Duration, mut observe: impl AsyncFnMut() -> Result>, ) -> Result { + let timeout = crate::budget::cap(timeout); let started = Instant::now(); + let mut last = String::from("none completed"); loop { - match observe().await? { - Poll::Ready(value) => return Ok(value), - Poll::Pending(observation) if started.elapsed() >= timeout => { - return Err(Error::msg(format!( - "timed out after {timeout:?} waiting for {what}; last observation: {observation}" - ))); - } - Poll::Pending(_) => tokio::time::sleep(interval).await, + let left = timeout.saturating_sub(started.elapsed()); + let timed_out = || { + Error::msg(format!( + "timed out after {timeout:?} waiting for {what}; last observation: {last}" + )) + }; + if left.is_zero() { + return Err(timed_out()); } + match tokio::time::timeout(left, observe()).await { + Err(_) => return Err(timed_out()), + Ok(observed) => match observed? { + Poll::Ready(value) => return Ok(value), + Poll::Pending(observation) => last = observation, + }, + } + let left = timeout.saturating_sub(started.elapsed()); + tokio::time::sleep(interval.min(left)).await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn a_hung_observation_times_out_with_the_last_completed_one() { + let mut calls = 0; + let started = Instant::now(); + let error = poll_until( + "the thing", + Duration::from_millis(300), + Duration::from_millis(10), + async || { + calls += 1; + if calls > 1 { + std::future::pending::<()>().await; + } + Ok(Poll::<()>::Pending("not yet".into())) + }, + ) + .await + .unwrap_err(); + assert!(started.elapsed() < Duration::from_secs(5)); + let message = error.to_string(); + assert!(message.contains("waiting for the thing"), "{message}"); + assert!(message.contains("last observation: not yet"), "{message}"); + } + + #[tokio::test] + async fn the_retry_sleep_does_not_pass_the_deadline() { + let started = Instant::now(); + let error = poll_until( + "the thing", + Duration::from_millis(200), + Duration::from_secs(30), + async || Ok(Poll::<()>::Pending("not yet".into())), + ) + .await + .unwrap_err(); + assert!(started.elapsed() < Duration::from_secs(5)); + assert!(error.to_string().contains("last observation: not yet")); + } + + #[tokio::test] + async fn a_ready_observation_returns_its_value() { + let value = poll_until( + "the thing", + Duration::from_secs(1), + Duration::from_millis(10), + async || Ok(Poll::Ready(7)), + ) + .await + .unwrap(); + assert_eq!(value, 7); } } diff --git a/tests/testkit/testkit/src/run.rs b/tests/testkit/testkit/src/run.rs index 701fc6e6ec..7ed1f1d87d 100644 --- a/tests/testkit/testkit/src/run.rs +++ b/tests/testkit/testkit/src/run.rs @@ -33,6 +33,7 @@ where F: AsyncFnOnce(&mut Context) -> Result, { let name = info.name; + crate::budget::start(); eprintln!("[{name}] {}", info.description); let gateway = Gateway::from_env().unwrap_or_else(|error| panic!("[{name}] {error}")); @@ -54,6 +55,13 @@ where F: AsyncFnOnce(&mut Context) -> Result, { let outcome = AssertUnwindSafe(body(cx)).catch_unwind().await; + // Say what failed before cleanup, which can run long enough for the + // runner to stop the test before the final failure is reported. + match &outcome { + Ok(Err(error)) => eprintln!("[{name}] {error}"), + Err(_) => eprintln!("[{name}] the test body panicked"), + Ok(Ok(())) => {} + } let cleanup_failures = cx.cleanup().await; for failure in &cleanup_failures { eprintln!("[{name}] cleanup failed: {failure}");