Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions tests/runner/src/bundle.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,20 +32,25 @@ pub const PROFILE: &str = "openshell";
pub const READY_BINARY: &str = "testkit-ready";

/// Nextest flags a test as slow after this period and terminates it after
/// [`SLOW_TERMINATE_AFTER`] periods, so a hung test fails after two minutes
/// instead of holding the lane until the job times out.
/// [`SLOW_TERMINATE_AFTER`] periods.
const SLOW_PERIOD_SECS: u64 = 30;
const SLOW_TERMINATE_AFTER: u64 = 4;
/// How long a terminated test has between SIGTERM and SIGKILL.
const GRACE_PERIOD_SECS: u64 = 10;

/// How long one test may run before the runner stops it, on either kind of
/// bundle, so a hung test fails after two minutes instead of holding the lane
/// until the job times out.
pub const TEST_TIMEOUT_SECS: u64 = SLOW_PERIOD_SECS * SLOW_TERMINATE_AFTER;
/// How long a stopped test has between SIGTERM and SIGKILL, on either kind of
/// bundle.
pub const TEST_GRACE_SECS: u64 = 10;

/// The nextest configuration written into a nextest bundle.
fn nextest_config() -> String {
format!(
r#"# Written by the OpenShell test runner.
[profile.openshell]
fail-fast = false
slow-timeout = {{ period = "{SLOW_PERIOD_SECS}s", terminate-after = {SLOW_TERMINATE_AFTER}, grace-period = "{GRACE_PERIOD_SECS}s" }}
slow-timeout = {{ period = "{SLOW_PERIOD_SECS}s", terminate-after = {SLOW_TERMINATE_AFTER}, grace-period = "{TEST_GRACE_SECS}s" }}

[profile.openshell.junit]
path = "junit.xml"
Expand Down Expand Up @@ -354,11 +359,12 @@ mod tests {
use super::*;

#[test]
fn nextest_profile_stops_a_hung_test_after_two_minutes() {
fn nextest_profile_stops_a_test_at_the_shared_timeout() {
let config = nextest_config();
assert!(config.contains(
r#"slow-timeout = { period = "30s", terminate-after = 4, grace-period = "10s" }"#
));
assert_eq!(TEST_TIMEOUT_SECS, 120);
}

#[test]
Expand Down
25 changes: 23 additions & 2 deletions tests/runner/src/collect-k3s.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,27 @@ collect_log() {
printf '\ncommand status: %s\n' "${PIPESTATUS[0]}"
}

# Gateway logs cover the whole run, so a failing test's window is included.
# The runner records when the run started; without that, use a fixed window.
# The tail is larger than for other logs because a gateway logs many lines.
RUN_STARTED_FILE=/var/tmp/openshell-run-started
log_since=--since=30m
if run_started=$(cat "$RUN_STARTED_FILE" 2>/dev/null) &&
[[ $run_started =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ ]]; then
log_since=--since-time=$run_started
fi

collect_gateway_log() {
printf '\n### %s (%s)\n' "$1" "$log_since"
shift
timeout 20s "$@" 2>&1 | {
head -c 65536
printf '\n### Remaining log tail\n'
tail -c 524288
}
printf '\ncommand status: %s\n' "${PIPESTATUS[0]}"
}

{
date --utc --iso-8601=seconds
collect 'K3s service state' systemctl show k3s.service \
Expand Down Expand Up @@ -74,8 +95,8 @@ collect_log() {
mapfile -t gateway_pods < <(timeout 10s /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell get pods \
--selector app.kubernetes.io/name=openshell,app.kubernetes.io/instance=openshell --output name 2>/dev/null)
for pod in "${gateway_pods[@]}"; do
collect "Gateway current logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --tail=300 --timestamps
collect "Gateway previous logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --previous --tail=300 --timestamps
collect_gateway_log "Gateway current logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway "$log_since" --timestamps
collect_gateway_log "Gateway previous logs ($pod)" /usr/local/bin/k3s kubectl --request-timeout=5s -n openshell logs "$pod" -c openshell-gateway --previous "$log_since" --timestamps
done
collect 'Memory' free -m
collect 'Disk' df -h / /var/lib/rancher/k3s
Expand Down
40 changes: 34 additions & 6 deletions tests/runner/src/execute.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ use anyhow::{Context, Result, ensure};
use tmachine::Session;
use tmachine::config::Config;

use crate::bundle::{Bundle, Kind, PROFILE, PROGRAM_BINARY, READY_BINARY};
use crate::bundle::{
Bundle, Kind, PROFILE, PROGRAM_BINARY, READY_BINARY, TEST_GRACE_SECS, TEST_TIMEOUT_SECS,
};
use crate::test_env;

const GUEST_DIR: &str = "/var/tmp/openshell-tests";
Expand All @@ -32,8 +34,8 @@ const NEXTEST_LOG: &str = "nextest.log";
// disk can take minutes to report healthy.
const READY_TIMEOUT_SECS: u64 = 300;
/// How long the test run may take in total. Individual tests are bounded by
/// the nextest profile's slow-timeout, or by a program's own `run --timeout`
/// default; this catches a run that stops making progress some other way.
/// [`TEST_TIMEOUT_SECS`]; this catches a run that stops making progress some
/// other way.
const RUN_TIMEOUT: Duration = Duration::from_mins(45);
/// Slack beyond the readiness check's own timeout for SSH and process start.
const READY_SLACK: Duration = Duration::from_mins(1);
Expand All @@ -42,6 +44,9 @@ const READY_SLACK: Duration = Duration::from_mins(1);
const NEXTEST_TEST_FAILED: i32 = 100;
/// nextest's exit code when `--no-tests fail` found no tests to run.
const NEXTEST_NO_TESTS_RUN: i32 = 4;
/// The variable that tells a test its per-test limit; testkit reads it
/// (`openshell_testkit::TIMEOUT_ENV`).
const TEST_TIMEOUT_ENV: &str = "OPENSHELL_TEST_TIMEOUT_SECS";
/// A testkit program's exit code when a test failed. It exits 2 when tests
/// could not run.
const PROGRAM_TEST_FAILED: i32 = 1;
Expand Down Expand Up @@ -308,7 +313,12 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result<Status> {
let archive = format!("{guest}/tests.tar.zst");
let extract = format!("{guest}/extract");
let config = format!("{guest}/nextest.toml");
let mut tests = vec!["env"];
let timeout = TEST_TIMEOUT_SECS.to_string();
let grace = TEST_GRACE_SECS.to_string();
// Each test is told its own limit, so a stuck step fails with its own
// error before the runner stops the test.
let budget = format!("{TEST_TIMEOUT_ENV}={TEST_TIMEOUT_SECS}");
let mut tests = vec!["env", budget.as_str()];
tests.extend(env.iter().map(String::as_str));
match kind {
Kind::Nextest => {
Expand Down Expand Up @@ -342,7 +352,16 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result<Status> {
// The program runs its tests one at a time, and writes the report
// where `collect_results` looks for it.
Kind::Program => {
tests.extend([program.as_str(), "run", "--junit", &junit]);
tests.extend([
program.as_str(),
"run",
"--timeout",
&timeout,
"--grace",
&grace,
"--junit",
&junit,
]);
// `run` takes name filters as arguments, after `--` so that one
// starting with a dash isn't read as an option.
if !run.filters.is_empty() {
Expand All @@ -357,7 +376,7 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result<Status> {
// can be credentials, never appear in the command (or its errors).
let log = format!("{guest}/{NEXTEST_LOG}");
let script = format!(
r#"exec >"$0" 2>&1; {}; exec "$@""#,
r#"exec >"$0" 2>&1; rm -f {RUN_STARTED_FILE}; date -u +%Y-%m-%dT%H:%M:%SZ >{RUN_STARTED_FILE} || true; {}; exec "$@""#,
test_env::export_script(test_env::ENV_DIR)
);
let mut command = vec!["sh", "-c", &script, &log];
Expand Down Expand Up @@ -427,6 +446,9 @@ const WAIT_K3S_CONTROLLER: &str = include_str!("wait-k3s-controller.sh");
/// The script retries for up to ten minutes; this catches one that stalls.
const WAIT_K3S_CONTROLLER_TIMEOUT: Duration = Duration::from_mins(12);

/// Where the run records when it started, in RFC 3339 UTC. The k3s diagnostics
/// read it (see `collect-k3s.sh`) to collect the gateway's logs from then on.
const RUN_STARTED_FILE: &str = "/var/tmp/openshell-run-started";
/// Where the guest keeps the script's output until it is copied out.
const COLLECT_K3S_OUTPUT: &str = "/var/tmp/openshell-k3s-diagnostics.txt";
/// How long the guest lets the script run before stopping it. The script
Expand Down Expand Up @@ -536,6 +558,11 @@ mod tests {
assert!(matches!(status(2), Status::Error(_)));
}

#[test]
fn k3s_diagnostics_read_the_file_the_run_writes() {
assert!(COLLECT_K3S.contains(RUN_STARTED_FILE));
}

#[test]
fn k3s_diagnostics_keep_everything_the_old_playbook_collected() {
for wanted in [
Expand All @@ -562,6 +589,7 @@ mod tests {
"free -m",
"df -h",
"[REDACTED]",
"--since-time",
] {
assert!(COLLECT_K3S.contains(wanted), "missing {wanted:?}");
}
Expand Down
4 changes: 2 additions & 2 deletions tests/suites/conformance/src/lifecycle.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ async fn long_running_exec_delete(cx: &mut Context) -> testkit::Result {

/// Longer than the supervisor's 30s startup recovery deadline.
const PAST_RECOVERY_DEADLINE: Duration = Duration::from_secs(31);
/// The exec's own timeout is the runner's two-minute kill, so fail sooner,
/// with a message that says which step hung.
/// The exec's own timeout is longer than the runner's per-test limit, so fail
/// sooner, with a message that says which step hung.
const PAST_RECOVERY_EXEC_DEADLINE: Duration = Duration::from_mins(1);

/// An exec admitted before the startup recovery deadline expires keeps
Expand Down
87 changes: 45 additions & 42 deletions tests/suites/conformance/src/policy_advisor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
//! Lezar. Requests to `policy.local` run inside the sandbox; settings and drafts
//! go through the gateway's gRPC API.

use std::time::{Duration, Instant};
use std::time::Duration;

use serde_json::Value;
use testkit::sdk::raw::proto;
Expand All @@ -31,32 +31,36 @@ async fn policy_local_accepts_proposals(cx: &mut Context) -> testkit::Result {

// The current-policy route is local; proposal lookup also needs the
// supervisor's session with the gateway, so wait for both.
let started = Instant::now();
let readiness = format!("/v1/proposals/test-{}-readiness", cx.run_id());
loop {
let ready = async {
let current = policy_local(&sandbox, "GET", "/v1/policy/current", "", 200).await?;
testkit::ensure!(
current["format"] == "yaml"
&& current["policy_yaml"]
.as_str()
.is_some_and(|yaml| yaml.contains("version: 1")),
"policy.local returned an invalid current policy: {current}"
);
let lookup = policy_local(&sandbox, "GET", &readiness, "", 404).await?;
testkit::ensure!(
lookup["error"] == "chunk_not_found",
"policy.local proposal lookup returned {lookup}"
);
Ok(())
}
.await;
match ready {
Ok(()) => break,
Err(error) if started.elapsed() >= READY_TIMEOUT => return Err(error),
Err(_) => tokio::time::sleep(POLL_INTERVAL).await,
}
}
testkit::poll_until(
"policy.local to be ready",
READY_TIMEOUT,
POLL_INTERVAL,
async || {
let ready = async {
let current = policy_local(&sandbox, "GET", "/v1/policy/current", "", 200).await?;
testkit::ensure!(
current["format"] == "yaml"
&& current["policy_yaml"]
.as_str()
.is_some_and(|yaml| yaml.contains("version: 1")),
"policy.local returned an invalid current policy: {current}"
);
let lookup = policy_local(&sandbox, "GET", &readiness, "", 404).await?;
testkit::ensure!(
lookup["error"] == "chunk_not_found",
"policy.local proposal lookup returned {lookup}"
);
Ok(())
}
.await;
Ok(match ready {
Ok(()) => testkit::Poll::Ready(()),
Err(error) => testkit::Poll::Pending(error.to_string()),
})
},
)
.await?;

let denials = policy_local(&sandbox, "GET", "/v1/denials?last=1", "", 200).await?;
testkit::ensure!(
Expand Down Expand Up @@ -360,22 +364,21 @@ async fn expect_draft(
binary: &str,
probe_stderr: &str,
) -> testkit::Result {
let started = Instant::now();
let chunks = loop {
match sandbox.draft_policy().await {
Ok(draft) if !draft.chunks.is_empty() => break draft.chunks,
Ok(_) | Err(_) if started.elapsed() < PROPOSAL_TIMEOUT => {
tokio::time::sleep(POLL_INTERVAL).await;
}
Ok(_) => {
return Err(Error::msg(format!(
"no draft rule for {}:{} after {PROPOSAL_TIMEOUT:?}; probe stderr: {probe_stderr}",
endpoint.0, endpoint.1
)));
}
Err(error) => return Err(error),
}
};
let chunks = testkit::poll_until(
&format!("a draft rule for {}:{}", endpoint.0, endpoint.1),
PROPOSAL_TIMEOUT,
POLL_INTERVAL,
async || {
Ok(match sandbox.draft_policy().await {
Ok(draft) if !draft.chunks.is_empty() => testkit::Poll::Ready(draft.chunks),
Ok(_) => {
testkit::Poll::Pending(format!("no draft rule; probe stderr: {probe_stderr}"))
}
Err(error) => testkit::Poll::Pending(error.to_string()),
})
},
)
.await?;

testkit::ensure!(
chunks.len() == 1,
Expand Down
11 changes: 5 additions & 6 deletions tests/support/keycloak/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,12 @@ pub const PASSWORD_ENV: &str = "OPENSHELL_TEST_KEYCLOAK_PASSWORD";
// The wait runs inside the test, so it must leave room for the test body
// within the runner's 2-minute per-test budget (`slow-timeout` in
// tests/runner/src/bundle.rs). Otherwise a slow Keycloak gets the test killed
// instead of failing with a readiness error. `poll_until` checks the deadline
// only between attempts, so the wait can last this plus one request
// (`CURL_MAX_TIME`) and one interval: 87 seconds, leaving 33 for the test.
// instead of failing with a readiness error. `poll_until` bounds the attempt in
// flight as well as the retry interval, so the wait lasts this and no more,
// leaving 45 seconds for the test.
pub const READY_TIMEOUT: Duration = Duration::from_secs(75);
/// How long one request to Keycloak may take, in seconds. `poll_until` checks
/// its deadline only between attempts, so a request to a Keycloak that
/// accepts connections but stalls must give up on its own.
/// How long one request to Keycloak may take, in seconds, so a Keycloak that
/// accepts connections but stalls does not hold one attempt for the whole wait.
const CURL_MAX_TIME: &str = "10";

/// How to reach the environment's Keycloak realm, and a user in it.
Expand Down
8 changes: 8 additions & 0 deletions tests/testkit/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,14 @@ plaintext and mTLS gateways. `OPENSHELL_WORKSPACE` selects the workspace
(default `openshell` on PATH). The CLI always targets the same gateway and
workspace as the SDK. Each test logs the gateway it used.

`OPENSHELL_TEST_TIMEOUT_SECS` is the time one test may take, as the runner
enforces it. The runner sets it. When set, `cx.cli`, `cx.exec` and the wait
helpers give up about five seconds before that limit, with their own error,
instead of leaving the test to be stopped. When unset, as under plain
`cargo test`, each step keeps its own timeout. A CLI command that runs longer
than ten seconds, exits non-zero, or is still running after 30 seconds is
reported on stderr; fast successful commands print nothing.

Wait for the gateway, then run a suite. Run tests that share a gateway one at
a time:

Expand Down
2 changes: 1 addition & 1 deletion tests/testkit/testkit/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ serde = { version = "1", features = ["derive"] }
serde_json = "1"
serde_yml = { package = "noyalib", version = "0.0.53", default-features = false, features = ["std", "compat-serde-yaml"] }
tempfile = "3"
tokio = { version = "1.43", features = ["macros", "process", "rt", "time"] }
tokio = { version = "1.43", features = ["io-util", "macros", "process", "rt", "time"] }
tonic = "0.14"

[target.'cfg(unix)'.dependencies]
Expand Down
Loading
Loading