diff --git a/Cargo.lock b/Cargo.lock index ef2c75d4c1..228a539d9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2265,20 +2265,30 @@ dependencies = [ name = "osbench" version = "0.1.0" dependencies = [ + "getrandom 0.3.4", "libc", "miniz_oxide", "osbench-lan", "osbench-rules", + "rustls", "syn 2.0.117", "toyos", "toyos-abi", "toyos-crypto", "toyos-trace", + "ureq", ] +[[package]] +name = "osbench-compare" +version = "0.1.0" + [[package]] name = "osbench-lan" version = "0.1.0" +dependencies = [ + "libc", +] [[package]] name = "osbench-rules" diff --git a/Cargo.toml b/Cargo.toml index 9627dc1ca0..2ae66bd8a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -146,6 +146,7 @@ members = [ "tests/guest/kernelprobe", "tests/guest/metalprobe", "tests/guest/osbench", + "tests/guest/osbench/compare", "tests/guest/osbench/lan", "tests/guest/osbench/rules", "tests/guest/pciholder", diff --git a/issues/netstack-sends-tcp-at-a-third-of-what-the-t14s-wired-link-carries.md b/issues/netstack-sends-tcp-at-a-third-of-what-the-t14s-wired-link-carries.md new file mode 100644 index 0000000000..64f29c85d3 --- /dev/null +++ b/issues/netstack-sends-tcp-at-a-third-of-what-the-t14s-wired-link-carries.md @@ -0,0 +1,48 @@ +--- +status: open +kind: defect +opened: 2026-10-24 +--- + +# netstack sends TCP at a third of what the T14's wired link carries + +On the T14's I219, a ToyOS TCP sender moves 41 MB/s where the same +program under the bench's Linux moves 113 MB/s on the same path and the +same peer. Received TCP is at parity on that path, so the link, the peer +and the access point are not the limit. + +**Measured** on 2026-10-24 with `osbench lan` at `d99ef5a35`. The path is +the T14's I219 at 1000 Mb/s, through the access point, to the AMD bench's +Linux `osbench-peer` on its Wi-Fi. Each cell is one run's p50 of five 3 s +rounds: + +| | ToyOS (3 runs) | Linux (4 runs) | +|---|---|---| +| `lan.wired.tcp.upload`, MB/s | 41, 48, 31 | 113, 113, 114, 113 | +| `lan.wired.tcp.download`, MB/s | 112, 112, 109 | 112, 110, 109, 112 | + +- No ToyOS round reached a Linux one: ToyOS's rounds ran 31 to 59 MB/s and + Linux's 112 to 114. The exact rank test over the runs gives p = 1/35. +- The same runs' HTTPS upload to the public internet took 1.40 times + Linux's time. + +Evidence: `ToyOSOrg/evidence` branch `pr-1065`, `r3/`. + +Where the time goes is not measured. These are candidates to measure first, +not findings: + +- every sent byte's copies through a kernel pipe + (`issues/netstack-moves-every-byte-through-two-kernel-pipes.md`); +- the I219 transmit path's `TXDCTL` and `TIDV` setup + (`issues/the-intel-driver-writes-txdctl-and-tidv-against-the-82574-datasheets-newer-revision.md`); +- the send window and the ACK clocking. + +The clock does not separate the two systems. Each upload row's own +readings, `mhz` just after it and `mhz_settled` before it, were about +1.6 GHz under ToyOS (settled 1.6, 1.8 or 3.8 GHz). Under Linux they were +1.5 GHz (settled 1.5 or 4.2 GHz), and Linux still moved 113 MB/s. At most, +the clock matters through what ToyOS spends per byte sent. + +**Exit**: `osbench_lan` on the T14 reads `lan.wired.tcp.upload` within +Linux's run-to-run range on the same path, or the cause is named and owned +by its own issue. diff --git a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md index b1ecac0749..fb51fb9df2 100644 --- a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md +++ b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md @@ -24,11 +24,24 @@ its groups: - an idle minute's package energy: `energy.idle_60s.package`; - the machine's PCI, USB, NVMe and cpuidle identity: `identity`, with no serial number or network identifier; -- the wired LAN's TCP round trip and throughput, and Wi-Fi's under Linux: - `lan.wired.*`, `lan.wifi.*`, against `osbench-peer` on another machine; +- TCP connect, round trip and throughput between the two benches: + `lan.wired.*`, `lan.wifi.*`, the bench under test against the + `osbench-peer` the other bench's Linux runs, each way. ToyOS and Linux on + one path are the comparison, and Linux's runs on it its ceiling and its + noise; `osbench-compare` reads the runs, interleaved, four of each at least + for its exact test to resolve a difference. `lan` and `internet` run only + by name, each its own metal row, since they need another party; - a short CPU crunch, one thread then all, that stops at TjMax less 15 C where the package's temperature can be read: `crunch.*`. +`internet.*`, HTTPS to Cloudflare's speed test, is exploratory evidence of a +path neither system owns, run only when a pass names its row: no automated +test depends on the public internet. + +The first paired session (`ToyOSOrg/evidence` branch `pr-1065`, `r3/`) put +ToyOS's T14 LAN download and 64 B round trip at Linux's and its upload at +a third of it (`issues/netstack-sends-tcp-at-a-third-of-what-the-t14s-wired-link-carries.md`). + The cryptographic rows are not osbench's. AES-GCM, ChaCha20-Poly1305, SHA-256 and the rest were measured under Linux on both benches against ring, OpenSSL and AWS-LC by the crypto oracle probe, and that table is the @@ -51,7 +64,9 @@ comparison until it can: 1.7-1.9 GHz, which `issues/the-t14s-thermal-modes-do-not-move-its-package-power-limits.md` would explain: ToyOS refuses the firmware's PL1 and PL2 writes. -- **Wi-Fi**: `issues/there-is-no-wifi.md`. +- **Wi-Fi**: `issues/there-is-no-wifi.md`. The T14's `lan.wifi.*` and every + AMD bench `lan.*` and `internet.*` row, whose one network card is its + AX200, are measured under Linux alone. - **USB, NVMe and cpuidle identity**: their owners serve no program that reads them. - **Loopback** (`net.*`): netstack refuses 127/8 as a destination. diff --git a/issues/toyos-crypto-is-in-the-tree-on-par-with-ring-and-secure.md b/issues/toyos-crypto-is-in-the-tree-on-par-with-ring-and-secure.md index 3a5bcce652..7a54fdf633 100644 --- a/issues/toyos-crypto-is-in-the-tree-on-par-with-ring-and-secure.md +++ b/issues/toyos-crypto-is-in-the-tree-on-par-with-ring-and-secure.md @@ -64,7 +64,7 @@ review has been done, and passing vectors does not show it is safer than ring. | `system/sshserver` | `toyos-ssh`, since the SSH track's stage 2 | none directly; it reaches the crate through `toyos-ssh` | | Image and package signing: `lib/update` (the loader, `/system/bin/update`, `build/signing.rs`, `build/root`, #892's repository keys) | the crate's Ed25519 and SHA-2, since #1019; signing and a seed's public key behind `sign`, on the host | stage 6's bench readings | | Every SHA-256: the build, `pkg`, `lib/swap`, `system/update`, `lib/update` and the loader | the crate, since #1016: SHA-NI on x86-64, the UEFI loader's in a block that saves the XMM registers it uses; the Armv8 SHA instructions where the target promises them. No slower than `toyos-sha2-hw`, which it replaces, on either bench: on the T14 4.6 times as fast at 64 B and 9 and 6 percent faster at 16 KiB and 1 MiB, on the AMD bench 2.8 times at 64 B, 2 percent at 16 KiB and equal at 1 MiB (#1016's evidence) | nothing | -| ToyOS's own HTTPS clients. On the host: the build's `ureq` (`build/release.rs`) and doom's build script (`apps/doom/build.rs`). In a guest, none today: `https_tls13` and `pkg install ` are stages 3 and 5 of `issues/the-internet-clients-work-unchanged.md` | the host's two: the crate's rustls provider through ureq's `rustls-no-provider`, since #1020, its key exchange on X25519 and X25519MLKEM768 alone while the T14 finding stands. In a guest, none | the NIST groups back once the T14 finding is resolved | +| ToyOS's own HTTPS clients. On the host: the build's `ureq` (`build/release.rs`) and doom's build script (`apps/doom/build.rs`). In a guest, `osbench internet` (`tests/guest/osbench/src/internet.rs`), on ToyOS and Linux alike; `https_tls13` and `pkg install ` are stages 3 and 5 of `issues/the-internet-clients-work-unchanged.md` | all three: the crate's rustls provider through ureq's `rustls-no-provider`, since #1020, its key exchange on X25519 and X25519MLKEM768 alone while the T14 finding stands. Each copies the provider's random source and that group filter | the NIST groups back once the T14 finding is resolved, in all three copies, or once in the crate if it comes to own the policy | | Wi-Fi's `wlan` (`issues/there-is-no-wifi.md` P6, W2) | nothing yet | PBKDF2-HMAC-SHA1, the 802.11i PRF and RFC 3394 unwrap, which the crate has. AES-CMAC-128, SAE over P-256 by hash-to-element, a public constant-time comparison and a public wiping secret, which it lacks | | Encrypted DATA storage (`issues/storage-is-layers-and-a-role-is-a-filesystem.md`): the bcachefs engine's `crypt` unlock and its key holder | nothing yet | scrypt (RFC 7914) at bcachefs's default and smallest costs, which the crate has; ChaCha20 at a chosen block counter and raw Poly1305 for the format's own AEAD composition, which it lacks publicly | | `issues/programs-hold-aes-keys-as-key-locker-handles.md` | blocked on a session-long AES key | a TLS client on the crate is that consumer: AES behind a Key Locker handle. SSH holds no AES key | diff --git a/issues/toyos-uses-what-modern-hardware-offers-for-speed.md b/issues/toyos-uses-what-modern-hardware-offers-for-speed.md index 26e2b7c673..2ffebd594f 100644 --- a/issues/toyos-uses-what-modern-hardware-offers-for-speed.md +++ b/issues/toyos-uses-what-modern-hardware-offers-for-speed.md @@ -70,6 +70,14 @@ is nothing to enable. `issues/idle-cpus-enter-the-c-states-cst-names.md`'s to answer; that the idle state reaches HWP's clock choice is inference. Next: a T14 scout whose idle CPUs wait in a deep C-state, read over the same windows. + #1065's network session on the T14 (`ToyOSOrg/evidence` branch + `pr-1065`, `r3/`) shows the same ceiling in `cpu.clock`, whose 200 probes + ramp from idle. Its maximum was 3789 MHz under ToyOS in every group that + reached its top, against 4188 to 4190 MHz in every Linux group. That + covers three ToyOS and four Linux runs of `osbench lan` and + `osbench internet`. The probes' p50, which reads idle and ramp policy + (`issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md`), + is no reading of this ceiling. - **Global-line contention** is bounded at 2 to 3 % of C0 on the T14 and not attributed; next, the HITM count attributed to its lines on both benches (`issues/per-cpu-scheduler-and-pmm-state-shares-lines-every-cpu-writes.md`). diff --git a/tests/guest/osbench/Cargo.toml b/tests/guest/osbench/Cargo.toml index 9475d4f121..aa9c83d8b7 100644 --- a/tests/guest/osbench/Cargo.toml +++ b/tests/guest/osbench/Cargo.toml @@ -8,8 +8,13 @@ license = "MIT OR Apache-2.0" [dependencies] # The work and crunch groups' compression, pure Rust and the same code on both systems. miniz_oxide = "0.9" -# The crunch group's hashing: ToyOS's own SHA-256, which its ssh server runs. -toyos-crypto = { path = "../../../lib/crypto" } +# The crunch group's hashing: ToyOS's own SHA-256, which its ssh server runs; +# and the internet group's TLS provider, with its types and its entropy. +toyos-crypto = { path = "../../../lib/crypto", features = ["std", "rustls-provider"] } +rustls = { version = "0.23.45", default-features = false } +getrandom = "0.3" +# The internet group's HTTPS, with webpki's roots, on toyos-crypto's provider. +ureq = { version = "3", default-features = false, features = ["rustls-no-provider", "rustls-webpki-roots"] } # The crunch group's compile-like task: parsing Rust, with no compiler to load. syn = { version = "2", default-features = false, features = ["full", "parsing"] } # The LAN group's two ends, and the rules readings are judged by. diff --git a/tests/guest/osbench/compare/Cargo.toml b/tests/guest/osbench/compare/Cargo.toml new file mode 100644 index 0000000000..9d49e834ee --- /dev/null +++ b/tests/guest/osbench/compare/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "osbench-compare" +description = "Compares osbench runs of ToyOS and Linux on one machine: each benchmark's per-run medians, each system's run-to-run range, their ratio and an exact rank test." +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +publish = false + +# A package of its own because osbench builds for ToyOS and Linux alone, and +# its runs are compared wherever they are collected. + +[lib] +doctest = false + +[[bin]] +name = "osbench-compare" +path = "src/main.rs" diff --git a/tests/guest/osbench/compare/src/lib.rs b/tests/guest/osbench/compare/src/lib.rs new file mode 100644 index 0000000000..58c197eebe --- /dev/null +++ b/tests/guest/osbench/compare/src/lib.rs @@ -0,0 +1,499 @@ +//! osbench runs of ToyOS and Linux on one machine, compared. +//! +//! A run is one file of osbench's lines, a Linux run's stdout or a ToyOS +//! boot's log, from whatever prefix each line carries. A benchmark's value in +//! a run is its p50. Per machine and benchmark, each system's runs give the +//! median of their values and their run-to-run range, `(max - min) / median`: +//! Linux's range on a path is that path's noise floor. ToyOS's median over +//! Linux's is the ratio, and the exact two-sided Mann-Whitney test over the +//! two sets of runs says whether the difference is resolved at all: with four +//! runs each, complete separation gives p = 2/70. Each p is one row's, +//! uncorrected for the table's other rows. Better or worse is judged only +//! for a unit whose better direction is known ([`better`]); for any other +//! the verdict says which way ToyOS differs. + +use std::collections::BTreeMap; +use std::fmt::Write as _; + +/// The test level a difference is resolved at. +pub const RESOLVED: f64 = 0.05; +/// The most runs the exact test enumerates the splits of. +pub const MAX_RUNS: usize = 20; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Os { + Linux, + Toyos, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Outcome { + Measured { unit: String, p50: u64 }, + /// Unmeasured, unsupported or refused, with the run's reason. + Not { kind: String, why: String }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Run { + pub os: Os, + pub conds: BTreeMap, + pub benches: BTreeMap, +} + +impl Run { + /// The machine a run is of: its CPU model. + pub fn machine(&self) -> Option<&str> { + self.conds.get("cpu_model").map(String::as_str) + } +} + +/// One run's lines. A benchmark a run reports in more than one group, as +/// every group's `cpu.clock`, is named `@`, and the `k`th of +/// one name in a group, as `fair.flat`'s jobs, `#k`. A run with no +/// osbench line, two systems' lines or one group started twice is refused. +pub fn parse(text: &str) -> Result { + let mut os = None; + let mut conds = BTreeMap::new(); + let mut group = String::new(); + let mut started = std::collections::BTreeSet::new(); + let mut read: Vec<(String, String, Outcome)> = Vec::new(); + for line in text.lines() { + let Some(at) = line.find("osbench v=2 os=") else { continue }; + let mut words = line[at..].split_whitespace().skip(2); + let line_os = match words.next() { + Some("os=linux") => Os::Linux, + Some("os=toyos") => Os::Toyos, + other => return Err(format!("a line of no system osbench knows: {other:?}")), + }; + if *os.get_or_insert(line_os) != line_os { + return Err("a run holds two systems' lines".into()); + } + let first = words.next().unwrap_or_default(); + if first == "cond" { + if let Some((key, value)) = words.next().and_then(|w| w.split_once('=')) { + conds.entry(key.to_string()).or_insert_with(|| value.to_string()); + } + continue; + } + if let Some(name) = first.strip_prefix("group=") { + if words.next() == Some("start") { + if started.contains(name) { + return Err(format!("group {name} starts twice: a file holds one run")); + } + started.insert(name.to_string()); + } + group = name.to_string(); + continue; + } + let Some(bench) = first.strip_prefix("bench=") else { continue }; + let fields: BTreeMap<&str, &str> = words.filter_map(|w| w.split_once('=')).collect(); + let outcome = if let Some(unit) = fields.get("unit") { + let p50 = fields.get("p50").and_then(|v| v.parse().ok()).ok_or_else(|| format!("{bench}: no p50"))?; + Outcome::Measured { unit: unit.to_string(), p50 } + } else if let Some((kind, why)) = + ["unmeasured", "unsupported", "refused"].iter().find_map(|k| fields.get(k).map(|why| (*k, *why))) + { + Outcome::Not { kind: kind.to_string(), why: why.to_string() } + } else { + return Err(format!("{bench}: neither a distribution nor a reason")); + }; + read.push((group.clone(), bench.to_string(), outcome)); + } + let mut benches = BTreeMap::new(); + for (i, (group, bench, outcome)) in read.iter().enumerate() { + let same = |(g, b, _): &&(String, String, Outcome)| b == bench && (g == group || group.is_empty()); + let mut name = bench.clone(); + if read.iter().any(|(g, b, _)| b == bench && g != group) { + name = format!("{name}@{group}"); + } + if read.iter().filter(same).count() > 1 { + name = format!("{name}#{}", read[..=i].iter().filter(same).count()); + } + benches.insert(name, outcome.clone()); + } + Ok(Run { os: os.ok_or("no osbench line")?, conds, benches }) +} + +pub fn median(values: &[u64]) -> f64 { + let mut v = values.to_vec(); + v.sort_unstable(); + let n = v.len(); + assert!(n > 0, "a median of no values"); + if n % 2 == 1 { + v[n / 2] as f64 + } else { + f64::midpoint(v[n / 2 - 1] as f64, v[n / 2] as f64) + } +} + +/// `(max - min) / median`, the spread of a set of runs; `None` about a +/// median of 0. +pub fn range(values: &[u64]) -> Option { + let (min, max) = (values.iter().min().expect("values"), values.iter().max().expect("values")); + let median = median(values); + (median > 0.0).then(|| (max - min) as f64 / median) +} + +/// The exact two-sided Mann-Whitney test of `a` against `b`: `p`, and +/// `least`, the smallest p these pooled values could give whichever runs +/// were whose. Ties count half a win. Over every split of the pooled values +/// into sets of the two sizes, `p` is the share of splits whose |2U - nm| is +/// at least the observed one: the distance from the centre, not twice the +/// one-sided p. Since the splits hold the observed ties, a tie can make a +/// split's mirror image less extreme. Then fewer splits reach the farthest +/// distance than the 2 of C(n + m, n) that untied values give, and `least` +/// is lower. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Exact { + pub p: f64, + pub least: f64, +} + +/// `None` past [`MAX_RUNS`] values or with an empty side. +pub fn mann_whitney(a: &[u64], b: &[u64]) -> Option { + let pooled: Vec = a.iter().chain(b).copied().collect(); + let total = pooled.len(); + if a.is_empty() || b.is_empty() || total > MAX_RUNS { + return None; + } + // Twice U: two per pair the first set wins, one per tie. + let doubled_u = |first: &[u64], second: &[u64]| -> i64 { + let mut u = 0; + for x in first { + for y in second { + u += match x.cmp(y) { + std::cmp::Ordering::Greater => 2, + std::cmp::Ordering::Equal => 1, + std::cmp::Ordering::Less => 0, + }; + } + } + u + }; + let centre = (a.len() * b.len()) as i64; + let seen = (doubled_u(a, b) - centre).abs(); + let mut distances = Vec::new(); + let (mut first, mut second) = (Vec::with_capacity(total), Vec::with_capacity(total)); + for mask in 0u32..(1 << total) { + if mask.count_ones() as usize != a.len() { + continue; + } + first.clear(); + second.clear(); + for (i, v) in pooled.iter().enumerate() { + if mask & (1 << i) != 0 { first.push(*v) } else { second.push(*v) } + } + distances.push((doubled_u(&first, &second) - centre).abs()); + } + let share = |at_least: i64| distances.iter().filter(|&&d| d >= at_least).count() as f64 / distances.len() as f64; + let farthest = *distances.iter().max().expect("a split"); + Some(Exact { p: share(seen), least: share(farthest) }) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Better { + Lower, + Higher, +} + +/// Which way `unit` is better, for the units whose direction is known: a +/// time and a rate. A clock, a power or a share is no better higher or lower. +pub fn better(unit: &str) -> Option { + match unit { + "ns" => Some(Better::Lower), + "MB/s" => Some(Better::Higher), + _ => None, + } +} + +/// The condition every run of a machine must agree on for a benchmark's +/// runs to be compared, and how many of its leading words must agree: the +/// peer's link kind and driver, and the internet link's kind, not what +/// varies by the run. +const PATHS: [(&str, &str, usize); 2] = [("lan.", "lan.peer_link", 2), ("internet.", "internet.link", 1)]; + +fn path(said: &str, words: usize) -> String { + said.split(['_', ' ']).take(words).collect::>().join(" ") +} + +/// Every machine's comparison, as Markdown: one section per machine, one row +/// per benchmark either system reported. +pub fn compare(runs: &[Run]) -> Result { + let mut machines: BTreeMap<&str, Vec<&Run>> = BTreeMap::new(); + for run in runs { + machines.entry(run.machine().ok_or("a run that names no cpu_model")?).or_default().push(run); + } + let mut out = String::new(); + for (machine, runs) in machines { + let count = |os| runs.iter().filter(|r| r.os == os).count(); + let paths: Vec<(&str, &str, std::collections::BTreeSet)> = PATHS + .iter() + .map(|&(prefix, cond, words)| { + (prefix, cond, runs.iter().filter_map(|r| r.conds.get(cond)).map(|said| path(said, words)).collect()) + }) + .collect(); + writeln!(out, "## {machine}\n").unwrap(); + write!(out, "Runs: Linux {}, ToyOS {}.", count(Os::Linux), count(Os::Toyos)).unwrap(); + for (_, cond, seen) in &paths { + let seen = if seen.is_empty() { "none".to_string() } else { seen.iter().cloned().collect::>().join(", ") }; + write!(out, " {cond}: {seen}.").unwrap(); + } + writeln!(out, "\n").unwrap(); + writeln!(out, "| bench | unit | Linux p50s | Linux median | Linux range | ToyOS p50s | ToyOS median | ToyOS range | ToyOS/Linux | p | verdict |").unwrap(); + writeln!(out, "|---|---|---|---|---|---|---|---|---|---|---|").unwrap(); + let names: std::collections::BTreeSet<&String> = runs.iter().flat_map(|r| r.benches.keys()).collect(); + for name in names { + let split = paths.iter().find(|(prefix, _, seen)| name.starts_with(prefix) && seen.len() > 1); + let row = match split { + Some((_, cond, _)) => Err(format!("the runs disagree on {cond}")), + None => Ok(compare_one(&runs, name)), + }; + match row { + Ok(row) => writeln!(out, "| {name} | {row} |").unwrap(), + Err(why) => writeln!(out, "| {name} | | | | | | | | | | {why} |").unwrap(), + } + } + writeln!(out, "\nEach p is its row's alone, uncorrected for the table's other rows.\n").unwrap(); + } + Ok(out) +} + +/// One benchmark's cells after its name. +fn compare_one(runs: &[&Run], name: &str) -> String { + let mut unit = None; + let mut values: BTreeMap> = BTreeMap::new(); + let mut not: BTreeMap> = BTreeMap::new(); + for run in runs { + match run.benches.get(name) { + Some(Outcome::Measured { unit: u, p50 }) => { + if unit.get_or_insert(u.as_str()) != u { + return "| | | | | | | | | units differ between runs".into(); + } + values.entry(run.os).or_default().push(*p50); + } + Some(Outcome::Not { kind, why }) => not.entry(run.os).or_default().push(format!("{kind}: {why}")), + None => not.entry(run.os).or_default().push("absent".into()), + } + } + let side = |os: Os| -> [String; 3] { + match values.get(&os) { + Some(v) => [ + v.iter().map(u64::to_string).collect::>().join(", "), + format!("{:.0}", median(v)), + range(v).map_or_else(|| "-".into(), |r| format!("{:.1}%", r * 100.0)), + ], + None => [String::new(), "-".into(), "-".into()], + } + }; + let [lp, lm, lr] = side(Os::Linux); + let [tp, tm, tr] = side(Os::Toyos); + let (ratio, p, verdict) = match (values.get(&Os::Linux), values.get(&Os::Toyos)) { + (Some(l), Some(t)) => { + let ratio = median(t) / median(l); + let exact = mann_whitney(t, l); + let verdict = match exact { + None => "too many runs for the exact test".to_string(), + Some(Exact { p, .. }) if p <= RESOLVED => { + let higher = median(t) > median(l); + match better(unit.unwrap_or_default()) { + Some(way) => format!("ToyOS {}", if higher == (way == Better::Higher) { "better" } else { "worse" }), + None => format!("ToyOS {}", if higher { "higher" } else { "lower" }), + } + } + Some(Exact { least, .. }) if least > RESOLVED => format!("too few runs to resolve (least p {least:.3})"), + Some(_) => "not resolved".to_string(), + }; + let p = exact.map(|e| e.p); + (format!("{ratio:.2}"), p.map_or_else(|| "-".into(), |p| format!("{p:.3}")), verdict) + } + _ => ("-".into(), "-".into(), String::new()), + }; + let mut verdict = verdict; + for (os, reasons) in ¬ { + let mut distinct = reasons.clone(); + distinct.sort(); + distinct.dedup(); + let who = if *os == Os::Linux { "Linux" } else { "ToyOS" }; + let sep = if verdict.is_empty() { "" } else { "; " }; + verdict = format!("{verdict}{sep}{who} {} run(s) {}", reasons.len(), distinct.join(" / ")); + } + format!("{} | {lp} | {lm} | {lr} | {tp} | {tm} | {tr} | {ratio} | {p} | {verdict}", unit.unwrap_or_default()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn run(os: &str, lines: &[&str]) -> Run { + let mut text = format!("osbench v=2 os={os} cond cpu_model=Bench_CPU\n"); + for line in lines { + text.push_str(&format!("[12.5 cpu3 osbench] osbench v=2 os={os} {line}\n")); + } + parse(&text).unwrap() + } + + #[test] + fn a_run_reads_each_benchmarks_p50_or_its_reason() { + let r = run( + "toyos", + &[ + "bench=lan.wired.tcp.rtt.64 unit=ns n=2000 min=1 p50=812 p90=900 p99=990 p999=- max=2000 mean=850 mhz=4000", + "bench=lan.wifi.tcp.upload unsupported=ToyOS_has_no_Wi-Fi_driver", + "cond lan.peer_link=wireless_driver=iwlwifi_signal_dbm=-52", + "group=lan done secs=40", + ], + ); + assert_eq!(r.os, Os::Toyos); + assert_eq!(r.machine(), Some("Bench_CPU")); + assert_eq!(r.benches["lan.wired.tcp.rtt.64"], Outcome::Measured { unit: "ns".into(), p50: 812 }); + assert_eq!( + r.benches["lan.wifi.tcp.upload"], + Outcome::Not { kind: "unsupported".into(), why: "ToyOS_has_no_Wi-Fi_driver".into() } + ); + assert_eq!(r.conds["lan.peer_link"], "wireless_driver=iwlwifi_signal_dbm=-52"); + } + + #[test] + fn repeated_names_are_told_apart_and_a_file_of_two_runs_is_refused() { + let twice = "osbench v=2 os=linux group=lan start\n".repeat(2); + assert!(parse(&twice).unwrap_err().contains("twice")); + let groups = ["sched", "fair"].map(|g| format!("osbench v=2 os=linux group={g} start\nosbench v=2 os=linux bench=cpu.clock unit=MHz n=1 min=1 p50=4000 max=1 mean=1\n")).concat(); + let run = parse(&groups).unwrap(); + assert_eq!(run.benches.keys().collect::>(), ["cpu.clock@fair", "cpu.clock@sched"]); + let jobs = "osbench v=2 os=linux bench=fair.flat unit=permille n=5 min=1 p50=63 max=1 mean=1 job=build\n".repeat(2); + assert_eq!(parse(&jobs).unwrap().benches.keys().collect::>(), ["fair.flat#1", "fair.flat#2"]); + let mixed = "osbench v=2 os=linux cond cpus=8\nosbench v=2 os=toyos cond cpus=8\n"; + assert!(parse(mixed).unwrap_err().contains("two systems")); + assert!(parse("nothing here\n").is_err()); + } + + #[test] + fn the_exact_test_matches_its_tables() { + // Complete separation has two splits as extreme as itself, of C(n+m, n). + let p = |a: &[u64], b: &[u64]| mann_whitney(a, b).unwrap().p; + assert!((p(&[1, 2, 3, 4], &[5, 6, 7, 8]) - 2.0 / 70.0).abs() < 1e-12); + assert!((p(&[8, 7, 6, 5], &[1, 2, 3, 4]) - 2.0 / 70.0).abs() < 1e-12); + assert!((p(&[1, 2, 3], &[4, 5, 6]) - 0.1).abs() < 1e-12); + assert!((p(&[1, 2, 3, 4, 5], &[6, 7, 8, 9, 10]) - 2.0 / 252.0).abs() < 1e-12); + // Interleaved sets are as central as a split can be. + assert_eq!(p(&[1, 3, 5, 7], &[2, 4, 6, 8]), p(&[2, 4, 6, 8], &[1, 3, 5, 7])); + assert!(p(&[1, 4, 5, 8], &[2, 3, 6, 7]) > 0.9); + // Every value tied is no evidence of a difference. + assert_eq!(p(&[5, 5, 5], &[5, 5, 5]), 1.0); + // One tie across the sets: U = 0.5 of 16 has 4 of the 70 splits at least as far out. + assert!((p(&[1, 2, 3, 5], &[5, 6, 7, 8]) - 4.0 / 70.0).abs() < 1e-12); + // A tie is half a win: [2, 2] against [1, 2] is U = 3 of 4, and every + // split is as far from the centre's 2, where ties counted whole would + // put half of them nearer. + assert_eq!(p(&[2, 2], &[1, 2]), 1.0); + assert_eq!(mann_whitney(&[], &[1]), None); + assert_eq!(mann_whitney(&[1; 11], &[2; 10]), None); + } + + #[test] + fn median_and_range_of_a_set_of_runs() { + assert_eq!(median(&[3, 1, 2]), 2.0); + assert_eq!(median(&[4, 1, 3, 2]), 2.5); + assert!((range(&[90, 100, 110, 100]).unwrap() - 0.2).abs() < 1e-12); + assert_eq!(range(&[7]), Some(0.0)); + assert_eq!(range(&[0, 0, 3]), None, "no spread relative to a median of 0"); + } + + #[test] + fn the_least_p_is_the_farthest_splits_share() { + let least = |a: &[u64], b: &[u64]| mann_whitney(a, b).unwrap().least; + // Distinct values: complete separation either way, 2 of C(n + m, n). + assert!((least(&[1, 2, 3, 4], &[5, 6, 7, 8]) - 2.0 / 70.0).abs() < 1e-12); + assert!((least(&[9, 1, 5, 3], &[2, 8, 4, 6]) - 2.0 / 70.0).abs() < 1e-12); + assert!((least(&[1, 2, 3, 4], &[5, 6, 7]) - 2.0 / 35.0).abs() < 1e-12); + assert_eq!(least(&[1], &[2]), 1.0); + assert_eq!(least(&[1], &[2, 3, 4]), 0.5); + // Three tied runs of four: only the three lowest values set apart are + // that far out, so one split of 35, the session's T14 upload. + let upload = mann_whitney(&[41, 48, 31], &[113, 113, 114, 113]).unwrap(); + assert!((upload.p - 1.0 / 35.0).abs() < 1e-12 && (upload.least - 1.0 / 35.0).abs() < 1e-12, "{upload:?}"); + } + + fn rtt(os: &str, p50: u64) -> Run { + run(os, &[&format!("bench=lan.wired.tcp.rtt.64 unit=ns n=2000 min=1 p50={p50} max=9 mean=5")]) + } + + #[test] + fn separated_runs_resolve_a_difference_and_overlapping_ones_do_not() { + let mut runs: Vec = [100, 104, 98, 102].iter().map(|v| rtt("linux", *v)).collect(); + runs.extend([150, 149, 155, 153].iter().map(|v| rtt("toyos", *v))); + let table = compare(&runs).unwrap(); + let row = table.lines().find(|l| l.starts_with("| lan.wired.tcp.rtt.64")).unwrap(); + assert!(row.contains("| 101 | 5.9% |"), "{row}"); + assert!(row.contains("| 1.50 | 0.029 | ToyOS worse |"), "{row}"); + + let mut runs: Vec = [100, 104, 98, 101].iter().map(|v| rtt("linux", *v)).collect(); + runs.extend([99, 103, 97, 102].iter().map(|v| rtt("toyos", *v))); + let table = compare(&runs).unwrap(); + assert!(table.contains("| not resolved |"), "{table}"); + } + + #[test] + fn counts_that_cannot_reach_the_level_say_so_and_not_that_nothing_differs() { + let mut runs: Vec = [100, 104, 98, 102].iter().map(|v| rtt("linux", *v)).collect(); + runs.extend([150, 149, 155].iter().map(|v| rtt("toyos", *v))); + let table = compare(&runs).unwrap(); + assert!(table.contains("| 0.057 | too few runs to resolve (least p 0.057) |"), "{table}"); + // Ties can let the same counts resolve: the test decides, not the count. + let mbps = |os: &str, v: u64| run(os, &[&format!("bench=lan.wired.tcp.upload unit=MB/s n=5 min=1 p50={v} max=9 mean=5")]); + let mut runs: Vec = [113, 113, 114, 113].iter().map(|v| mbps("linux", *v)).collect(); + runs.extend([41, 48, 31].iter().map(|v| mbps("toyos", *v))); + assert!(compare(&runs).unwrap().contains("| 0.029 | ToyOS worse |")); + } + + #[test] + fn a_unit_of_no_known_direction_is_told_only_which_way_it_differs() { + let power = |os: &str, v: u64| run(os, &[&format!("bench=energy.idle_60s.package unit=mW n=1 min=1 p50={v} max=9 mean=5")]); + let mut runs: Vec = [900, 910, 920, 930].iter().map(|v| power("linux", *v)).collect(); + runs.extend([1500, 1510, 1520, 1530].iter().map(|v| power("toyos", *v))); + let table = compare(&runs).unwrap(); + assert!(table.contains("| ToyOS higher |"), "{table}"); + assert!(!table.contains("better") && !table.contains("worse"), "{table}"); + } + + #[test] + fn a_throughputs_higher_value_is_the_better_one() { + let mbps = |os: &str, v: u64| run(os, &[&format!("bench=lan.wired.tcp.upload unit=MB/s n=5 min=1 p50={v} max=9 mean=5")]); + let mut runs: Vec = [40, 41, 42, 43].iter().map(|v| mbps("linux", *v)).collect(); + runs.extend([50, 51, 52, 53].iter().map(|v| mbps("toyos", *v))); + assert!(compare(&runs).unwrap().contains("| ToyOS better |")); + } + + #[test] + fn a_system_that_cannot_measure_says_so_beside_the_others_figures() { + let mut runs: Vec = [10, 11].iter().map(|v| run("linux", &[&format!("bench=lan.wifi.tcp.connect unit=ns n=50 min=1 p50={v} max=9 mean=5")])).collect(); + runs.push(run("toyos", &["bench=lan.wifi.tcp.connect unsupported=no_Wi-Fi"])); + let table = compare(&runs).unwrap(); + assert!(table.contains("| ToyOS 1 run(s) unsupported: no_Wi-Fi |"), "{table}"); + } + + #[test] + fn lan_runs_whose_peers_were_on_different_links_are_not_compared() { + let peer = |os: &str, said: &str, v: u64| { + run(os, &[&format!("cond lan.peer_link={said}"), &format!("bench=lan.wired.tcp.rtt.64 unit=ns n=1 min=1 p50={v} max=1 mean=1")]) + }; + let runs = [ + peer("linux", "wireless_driver=iwlwifi_signal_dbm=-52", 10), + peer("toyos", "wireless_driver=iwlwifi_signal_dbm=-60", 12), + ]; + assert!(!compare(&runs).unwrap().contains("different links"), "a signal is no other link"); + let runs = [peer("linux", "wireless_driver=iwlwifi_signal_dbm=-52", 10), peer("toyos", "wired_driver=e1000e_speed_mbps=1000", 12)]; + assert!(compare(&runs).unwrap().contains("| the runs disagree on lan.peer_link |")); + } + + #[test] + fn internet_runs_that_left_by_different_kinds_of_link_are_not_compared() { + let left = |os: &str, said: &str| { + run(os, &[&format!("cond internet.link={said}"), "bench=internet.https.rtt unit=ns n=1 min=1 p50=9 max=1 mean=1"]) + }; + let same = [left("linux", "wired_driver=e1000e_speed_mbps=1000"), left("toyos", "wired_netstack's_one_Ethernet_card")]; + assert!(!compare(&same).unwrap().contains("disagree"), "both wired, by any driver"); + let split = [left("linux", "wireless_driver=iwlwifi_signal_dbm=-50"), left("toyos", "wired_netstack's_one_Ethernet_card")]; + assert!(compare(&split).unwrap().contains("| the runs disagree on internet.link |")); + } +} diff --git a/tests/guest/osbench/compare/src/main.rs b/tests/guest/osbench/compare/src/main.rs new file mode 100644 index 0000000000..4281998c0d --- /dev/null +++ b/tests/guest/osbench/compare/src/main.rs @@ -0,0 +1,25 @@ +//! `osbench-compare ...`: each file one osbench run of ToyOS or Linux, +//! compared per machine as Markdown on stdout (`osbench_compare`). + +fn main() { + let files: Vec = std::env::args().skip(1).collect(); + if files.is_empty() { + fail("usage: osbench-compare ..."); + } + let runs: Vec = files + .iter() + .map(|file| { + let text = std::fs::read_to_string(file).unwrap_or_else(|e| fail(&format!("{file}: {e}"))); + osbench_compare::parse(&text).unwrap_or_else(|why| fail(&format!("{file}: {why}"))) + }) + .collect(); + match osbench_compare::compare(&runs) { + Ok(table) => print!("{table}"), + Err(why) => fail(&why), + } +} + +fn fail(why: &str) -> ! { + eprintln!("osbench-compare: {why}"); + std::process::exit(1); +} diff --git a/tests/guest/osbench/lan/Cargo.toml b/tests/guest/osbench/lan/Cargo.toml index 9cabd28663..2edf593f12 100644 --- a/tests/guest/osbench/lan/Cargo.toml +++ b/tests/guest/osbench/lan/Cargo.toml @@ -1,13 +1,13 @@ [package] name = "osbench-lan" -description = "osbench's LAN measurement, both ends: a bench finds its peer by broadcast and times TCP to it; the peer, `osbench-peer`, echoes, sinks and sources for it on any host." +description = "osbench's LAN measurement, both ends: a bench finds its peer by broadcast and times TCP to it; the peer, `osbench-peer`, on the other bench's Linux, echoes, sinks and sources for it." version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" publish = false # A package of its own because osbench builds for ToyOS and Linux alone, and -# the peer runs on whatever machine stands on the other end of the link. +# a host tests this part of it. [lib] doctest = false @@ -15,3 +15,7 @@ doctest = false [[bin]] name = "osbench-peer" path = "src/main.rs" + +# The interface holding an address, which `link` describes from sysfs. +[target.'cfg(target_os = "linux")'.dependencies] +libc = "0.2" diff --git a/tests/guest/osbench/lan/src/lib.rs b/tests/guest/osbench/lan/src/lib.rs index d1d7f832e1..944740362e 100644 --- a/tests/guest/osbench/lan/src/lib.rs +++ b/tests/guest/osbench/lan/src/lib.rs @@ -1,9 +1,10 @@ //! A bench's link to a peer, timed from the bench. //! //! **The bench finds the peer and opens every connection**: it broadcasts -//! [`HELLO`] to [`PORT`] and takes the source of the first [`ANSWER`] as its +//! [`HELLO`] to [`PORT`] and takes the source of the first answer as its //! peer, so neither side is configured with an address and the bench accepts -//! nothing. The peer only answers; every figure is the bench's clock. +//! nothing. The answer says which of its links the peer answered over +//! ([`link`]). The peer only answers; every figure is the bench's clock. //! //! Each TCP connection to the peer's [`PORT`] opens with one [`Request`]: //! echo fixed-size messages, sink everything to end-of-file and answer the @@ -15,12 +16,16 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; +pub mod link; + /// The peer's UDP port for discovery and its TCP port for requests. pub const PORT: u16 = 47_321; -/// What a bench broadcasts, and what the peer answers it with. -pub const HELLO: &[u8] = b"osbench-lan/1 hello"; -pub const ANSWER: &[u8] = b"osbench-lan/1 peer"; +/// What a bench broadcasts, and what the peer's answer starts with: the rest +/// is its link's description, at most [`MAX_SAID`] bytes of printable ASCII. +pub const HELLO: &[u8] = b"osbench-lan/2 hello"; +pub const ANSWER: &[u8] = b"osbench-lan/2 peer "; +pub const MAX_SAID: usize = 160; /// How long either end waits on one read or write before it gives the /// connection up. @@ -94,16 +99,31 @@ impl Request { } } -/// The bench's end: broadcast [`HELLO`] to `to` once a second until an -/// [`ANSWER`] comes or `patience` has passed, and answer where it came from. -/// `Ok(None)` is no answer; an error is a socket the system would not give. -/// Sends refused while the link has no address yet are tried again. -pub fn discover(to: SocketAddr, patience: Duration) -> io::Result> { +/// A peer that answered: where from, and what it said of its link. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Found { + pub ip: IpAddr, + pub link: String, +} + +/// The description an answer carries, or `None` for a datagram that is no +/// answer. +pub fn answered(datagram: &[u8]) -> Option<&str> { + let said = datagram.strip_prefix(ANSWER)?; + let printable = said.iter().all(|b| b.is_ascii_graphic() || *b == b' '); + (said.len() <= MAX_SAID && printable).then(|| std::str::from_utf8(said).expect("ASCII")) +} + +/// The bench's end: broadcast [`HELLO`] to `to` once a second until an answer +/// comes or `patience` has passed. `Ok(None)` is no answer; an error is a +/// socket the system would not give. Sends refused while the link has no +/// address yet are tried again. +pub fn discover(to: SocketAddr, patience: Duration) -> io::Result> { let socket = UdpSocket::bind((Ipv4Addr::UNSPECIFIED, 0))?; socket.set_broadcast(true)?; socket.set_read_timeout(Some(Duration::from_secs(1)))?; let start = Instant::now(); - let mut buf = [0u8; 64]; + let mut buf = [0u8; ANSWER.len() + MAX_SAID + 1]; while start.elapsed() < patience { if socket.send_to(HELLO, to).is_err() { std::thread::sleep(Duration::from_secs(1)); @@ -112,8 +132,11 @@ pub fn discover(to: SocketAddr, patience: Duration) -> io::Result let asked = Instant::now(); while asked.elapsed() < Duration::from_secs(1) { match socket.recv_from(&mut buf) { - Ok((n, from)) if &buf[..n] == ANSWER => return Ok(Some(from.ip())), - Ok(_) => {} + Ok((n, from)) => { + if let Some(said) = answered(&buf[..n]) { + return Ok(Some(Found { ip: from.ip(), link: said.to_string() })); + } + } Err(e) if matches!(e.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => break, Err(e) => return Err(e), } @@ -225,13 +248,17 @@ impl Echo { /// The most connections a peer serves at once. pub const MAX_SESSIONS: usize = 8; -/// The peer's end of discovery: answer every [`HELLO`] on `socket`, for good. +/// The peer's end of discovery: answer every [`HELLO`] on `socket` with the +/// link it answers over, for good. pub fn answer(socket: &UdpSocket) -> io::Result<()> { let mut buf = [0u8; 64]; loop { let (n, from) = socket.recv_from(&mut buf)?; if &buf[..n] == HELLO { - socket.send_to(ANSWER, from)?; + let said = link::toward(from.ip()).map_or_else(|why| format!("unread: {why}"), |(_, said)| said); + let mut reply = ANSWER.to_vec(); + reply.extend(said.bytes().filter(|b| b.is_ascii_graphic() || *b == b' ').take(MAX_SAID)); + socket.send_to(&reply, from)?; } } } @@ -346,8 +373,9 @@ mod tests { #[test] fn a_bench_finds_its_peer_and_measures_each_request() { let (discovery, requests) = peer(); - let found = discover(discovery, Duration::from_secs(5)).unwrap(); - assert_eq!(found, Some(IpAddr::V4(Ipv4Addr::LOCALHOST))); + let found = discover(discovery, Duration::from_secs(5)).unwrap().expect("the peer answers"); + assert_eq!(found.ip, IpAddr::V4(Ipv4Addr::LOCALHOST)); + assert!(!found.link.is_empty(), "the answer says something of the peer's link"); let peer = Peer(requests); peer.connect().unwrap(); @@ -361,6 +389,21 @@ mod tests { assert!(down.bytes >= CHUNK as u64, "{down:?}"); } + #[test] + fn an_answer_carries_the_peers_link_and_nothing_else_is_one() { + assert_eq!(answered(b"osbench-lan/2 peer wired driver=e1000e speed_mbps=1000"), Some("wired driver=e1000e speed_mbps=1000")); + assert_eq!(answered(b"osbench-lan/2 peer "), Some("")); + assert_eq!(answered(b"osbench-lan/1 peer"), None); + assert_eq!(answered(b"osbench-lan/2 hello"), None); + assert_eq!(answered(b"osbench-lan/2 peer wired\n"), None); + assert_eq!(answered("osbench-lan/2 peer รค".as_bytes()), None); + let mut long = ANSWER.to_vec(); + long.extend([b'x'; MAX_SAID]); + assert!(answered(&long).is_some()); + long.push(b'x'); + assert_eq!(answered(&long), None); + } + #[test] fn no_answer_is_none_once_patience_runs_out() { let silent = UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); diff --git a/tests/guest/osbench/lan/src/link.rs b/tests/guest/osbench/lan/src/link.rs new file mode 100644 index 0000000000..3f9d091595 --- /dev/null +++ b/tests/guest/osbench/lan/src/link.rs @@ -0,0 +1,125 @@ +//! Which of this machine's links one of its own addresses is on, described +//! as Linux's sysfs and `/proc/net/wireless` give it. Every other system +//! answers that it reads none. A description names no interface or address. + +use std::net::{IpAddr, Ipv4Addr, UdpSocket}; + +/// The links a result can be named for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Link { + Wired, + Wifi, +} + +impl Link { + pub fn name(self) -> &'static str { + match self { + Link::Wired => "wired", + Link::Wifi => "wifi", + } + } +} + +/// The link a datagram to `peer` would leave by. Connecting a UDP socket +/// chooses its route and sends nothing. +pub fn toward(peer: IpAddr) -> Result<(Link, String), String> { + let probe = UdpSocket::bind((Ipv4Addr::UNSPECIFIED, 0)).map_err(|e| format!("no socket: {e}"))?; + probe.connect((peer, 9)).map_err(|e| format!("no route: {e}"))?; + of(probe.local_addr().map_err(|e| format!("no local address: {e}"))?.ip()) +} + +/// The link holding `local`, and its description: kind, driver, and the +/// wired link's speed or the wireless one's signal. +#[cfg(target_os = "linux")] +pub fn of(local: IpAddr) -> Result<(Link, String), String> { + let name = linux::interface_holding(local)?; + let at = std::path::Path::new("/sys/class/net").join(&name); + let Ok(driver) = std::fs::read_link(at.join("device/driver")) else { + return Err("the address is on an interface no device drives".into()); + }; + let driver = driver.file_name().map_or_else(|| "unnamed".into(), |n| n.to_string_lossy().into_owned()); + if at.join("wireless").exists() || at.join("phy80211").exists() { + let signal = std::fs::read_to_string("/proc/net/wireless") + .ok() + .and_then(|table| signal_dbm(&table, &name)) + .map_or_else(|| "unread".into(), |dbm| dbm.to_string()); + Ok((Link::Wifi, format!("wireless driver={driver} signal_dbm={signal}"))) + } else { + let speed = std::fs::read_to_string(at.join("speed")).map_or_else(|_| "unread".into(), |s| s.trim().to_string()); + Ok((Link::Wired, format!("wired driver={driver} speed_mbps={speed}"))) + } +} + +#[cfg(not(target_os = "linux"))] +pub fn of(_local: IpAddr) -> Result<(Link, String), String> { + Err("this system's links are not read".into()) +} + +/// `interface`'s signal level in dBm from a `/proc/net/wireless` table, whose +/// rows are `: ...`, each number +/// perhaps ending in `.`. +pub fn signal_dbm(table: &str, interface: &str) -> Option { + table.lines().find_map(|line| { + let (name, fields) = line.trim_start().split_once(':')?; + if name != interface { + return None; + } + fields.split_whitespace().nth(2)?.trim_end_matches('.').parse().ok() + }) +} + +#[cfg(target_os = "linux")] +mod linux { + use std::net::IpAddr; + + /// The name of the interface holding `local`. + pub fn interface_holding(local: IpAddr) -> Result { + let IpAddr::V4(local) = local else { return Err("the address is IPv6".into()) }; + let mut list: *mut libc::ifaddrs = std::ptr::null_mut(); + // SAFETY: `list` is a valid out-pointer; the list is freed below. + if unsafe { libc::getifaddrs(&mut list) } != 0 { + return Err(format!("getifaddrs: {}", std::io::Error::last_os_error())); + } + let mut found = None; + let mut at = list; + while !at.is_null() { + // SAFETY: a node of the list getifaddrs made, alive until it is freed. + let ifa = unsafe { &*at }; + // SAFETY: a non-null `ifa_addr` of family AF_INET is a sockaddr_in. + let v4 = (!ifa.ifa_addr.is_null() && unsafe { (*ifa.ifa_addr).sa_family } == libc::AF_INET as libc::sa_family_t) + .then(|| unsafe { (*ifa.ifa_addr.cast::()).sin_addr.s_addr }); + if v4 == Some(u32::from(local).to_be()) { + // SAFETY: an interface's name is a NUL-terminated string. + found = Some(unsafe { std::ffi::CStr::from_ptr(ifa.ifa_name) }.to_string_lossy().into_owned()); + break; + } + at = ifa.ifa_next; + } + // SAFETY: the list getifaddrs made, read no more. + unsafe { libc::freeifaddrs(list) }; + found.ok_or_else(|| "no interface holds the address".into()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const TABLE: &str = "Inter-| sta-| Quality | Discarded packets | Missed | WE + face | tus | link level noise | nwid crypt frag retry misc | beacon | 22 +wlp1s0: 0000 58. -52. -256 0 0 0 0 0 0 + wlan1: 0000 70 -40 -256 0 0 0 0 0 0 +"; + + #[test] + fn a_wireless_interfaces_signal_is_its_level_column() { + assert_eq!(signal_dbm(TABLE, "wlp1s0"), Some(-52)); + assert_eq!(signal_dbm(TABLE, "wlan1"), Some(-40)); + } + + #[test] + fn an_interface_the_table_does_not_list_has_no_signal() { + assert_eq!(signal_dbm(TABLE, "wlp1"), None); + assert_eq!(signal_dbm("wlp1s0: 0000 58.", "wlp1s0"), None); + } +} diff --git a/tests/guest/osbench/lan/src/main.rs b/tests/guest/osbench/lan/src/main.rs index 80f6ba4823..c8d580b231 100644 --- a/tests/guest/osbench/lan/src/main.rs +++ b/tests/guest/osbench/lan/src/main.rs @@ -1,5 +1,5 @@ -//! `osbench-peer`: the far end of a bench's `osbench lan`, on any host on the -//! bench's link. It answers discovery and serves requests on +//! `osbench-peer`: the far end of a bench's `osbench lan`, the fixed reference +//! on the other bench's Linux. It answers discovery and serves requests on //! [`osbench_lan::PORT`] until it is stopped, and names no address. use std::net::{Ipv4Addr, TcpListener, UdpSocket}; diff --git a/tests/guest/osbench/src/internet.rs b/tests/guest/osbench/src/internet.rs new file mode 100644 index 0000000000..f365d66d98 --- /dev/null +++ b/tests/guest/osbench/src/internet.rs @@ -0,0 +1,139 @@ +//! Exploratory rows over the public internet, run only by name and never by +//! `all` or a judged boot, since no automated test depends on the public +//! internet: evidence of a path neither system owns, not a regression test. +//! +//! The one endpoint is Cloudflare's speed test at [`HOST`], over HTTPS: +//! `__down?bytes=N` answers N bytes and `__up` takes a POST's body, the +//! endpoints Cloudflare's own measurement engine drives +//! (). It refuses a download of +//! 100 MB. The round trip asks the same host's `cdn-cgi/trace`, which its edge +//! answers without the speed test's own processing. Answers name the client's +//! address and place, and nothing here keeps or prints them. TLS is rustls on +//! toyos-crypto's provider on both systems, so the cryptography is the same +//! code. + +use std::io::{ErrorKind, Read}; +use std::net::{Ipv4Addr, SocketAddr, TcpStream, ToSocketAddrs, UdpSocket}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crate::measure::each; +use crate::report::{condition, refuse, unsupported}; + +const HOST: &str = "speed.cloudflare.com"; +const USER_AGENT: &str = "toyos-build (https://github.com/ToyOSOrg/ToyOS)"; +/// One download's bytes and one upload's, each timed whole: a slow uplink's +/// rate needs more resolution than whole MB/s. +const DOWN: u64 = 10_000_000; +const UP: usize = 10_000_000; +/// Timed transfers each way after one unread. The endpoint answers 429 past +/// a download volume it does not publish, so a session of runs stays well +/// under the volume that first drew one. +const ROUNDS: usize = 3; +/// The most a short answer is read to. +const ANSWER_MAX: u64 = 64 * 1024; +/// How long one connection or request may take before it is given up. +const PATIENCE: Duration = Duration::from_secs(30); +/// How long the address may take to arrive, as `lan`'s peer may. +const LEASE: Duration = Duration::from_secs(60); + +const BENCHES: [&str; 4] = ["internet.tcp.connect", "internet.https.rtt", "internet.https.download", "internet.https.upload"]; + +pub fn suite() { + condition("internet.endpoint", HOST); + if let Err(e) = UdpSocket::bind((Ipv4Addr::UNSPECIFIED, 0)) { + if cfg!(target_os = "toyos") && e.kind() == ErrorKind::NotConnected { + return BENCHES.iter().for_each(|b| unsupported(b, format!("no network stack answers: {e}"))); + } + return refuse("internet", format!("no socket: {e}")); + } + let addr = match resolve() { + Ok(addr) => addr, + Err(why) => return refuse("internet", why), + }; + match TcpStream::connect_timeout(&addr, PATIENCE).and_then(|s| s.local_addr()) { + Ok(local) => match crate::sys::link_of(local.ip()) { + Ok((_, said)) => condition("internet.link", said), + Err(why) => return refuse("internet", why), + }, + Err(e) => return refuse("internet", format!("{HOST} took no connection: {e}")), + } + let [connect, rtt, download, upload] = BENCHES; + + each(connect, 2, 20, &[], |_| TcpStream::connect_timeout(&addr, PATIENCE).map(drop).map_err(|e| format!("connect: {e}"))); + + let agent = agent(); + let url = |path: &str| format!("https://{HOST}/{path}"); + let get = |path: &str, most: u64| -> Result { + let mut answer = agent.get(url(path)).call().map_err(|e| format!("GET: {e}"))?; + drain(answer.body_mut(), most) + }; + each(rtt, 5, 50, &[("path", "cdn-cgi/trace".into())], |_| get("cdn-cgi/trace", ANSWER_MAX).map(drop)); + let down = format!("__down?bytes={DOWN}"); + each(download, 1, ROUNDS, &[("bytes", DOWN.to_string())], |_| match get(&down, DOWN)? { + DOWN => Ok(()), + n => Err(format!("the download ended at {n} of {DOWN} bytes")), + }); + let sent = vec![0x5au8; UP]; + each(upload, 1, ROUNDS, &[("bytes", UP.to_string())], |_| { + let mut answer = agent.post(url("__up")).send(&sent[..]).map_err(|e| format!("POST: {e}"))?; + drain(answer.body_mut(), ANSWER_MAX).map(drop) + }); +} + +/// An answer's body read to its end, unkept, so its connection serves the +/// next request: its length, refused past `most`. +fn drain(body: &mut ureq::Body, most: u64) -> Result { + match std::io::copy(&mut body.as_reader().take(most + 1), &mut std::io::sink()) { + Ok(n) if n <= most => Ok(n), + Ok(_) => Err(format!("an answer ran past {most} bytes")), + Err(e) => Err(format!("an answer's body: {e}")), + } +} + +/// [`HOST`]'s first IPv4 address. A ToyOS netstack answers `NotConnected` +/// until its lease lands and tells no one when it does, so a lookup it +/// answers so is asked again each second for up to [`LEASE`]. +fn resolve() -> Result { + let start = Instant::now(); + loop { + match (HOST, 443).to_socket_addrs() { + Ok(mut addrs) => return addrs.find(SocketAddr::is_ipv4).ok_or_else(|| format!("{HOST} has no IPv4 address")), + Err(e) if e.kind() == ErrorKind::NotConnected && start.elapsed() < LEASE => std::thread::sleep(Duration::from_secs(1)), + Err(e) => return Err(format!("{HOST} did not resolve: {e}")), + } + } +} + +/// One agent, so every request after the first rides a connection it keeps. +fn agent() -> ureq::Agent { + let tls = ureq::tls::TlsConfig::builder().unversioned_rustls_crypto_provider(provider()).build(); + ureq::Agent::config_builder() + .user_agent(USER_AGENT) + .timeout_global(Some(PATIENCE)) + .tls_config(tls) + .build() + .new_agent() +} + +/// The system's random bytes, which the provider's key exchanges draw. +#[derive(Debug)] +struct SystemRandom; + +impl rustls::crypto::SecureRandom for SystemRandom { + fn fill(&self, bytes: &mut [u8]) -> Result<(), rustls::crypto::GetRandomFailed> { + getrandom::fill(bytes).map_err(|_| rustls::crypto::GetRandomFailed) + } +} + +static TLS: toyos_crypto::rustls_provider::Backend = toyos_crypto::rustls_provider::Backend::new(&SystemRandom); + +/// toyos-crypto's provider, exchanging keys on X25519 alone, hybrid or not, +/// as the build's own requests do: no P-256 or P-384 secret meets a server's +/// input while the T14 measures those curves leaking +/// (`issues/toyos-crypto-is-in-the-tree-on-par-with-ring-and-secure.md`). +fn provider() -> Arc { + let mut provider = TLS.provider(); + provider.kx_groups.retain(|group| matches!(group.name(), rustls::NamedGroup::X25519MLKEM768 | rustls::NamedGroup::X25519)); + Arc::new(provider) +} diff --git a/tests/guest/osbench/src/lan.rs b/tests/guest/osbench/src/lan.rs index e0bbc1c8f6..7554a6ec74 100644 --- a/tests/guest/osbench/src/lan.rs +++ b/tests/guest/osbench/src/lan.rs @@ -1,40 +1,28 @@ -//! The machine's link to a peer on its network, through std's `net` on both -//! systems (`osbench-lan`): this machine broadcasts for an `osbench-peer`, -//! then times TCP to it. Every connection is this machine's own, so a system -//! that accepts none from its network is measured all the same. +//! The machine's link to the other bench, through std's `net` on both +//! systems (`osbench-lan`): this machine broadcasts for the `osbench-peer` +//! that bench's Linux runs as the fixed reference, then times TCP to it. +//! Every connection is this machine's own, so a system that accepts none +//! from its network is measured all the same. //! //! Each result is named for the link that carried it, `lan.wired.*` or //! `lan.wifi.*`, and the link not measured says why: a Linux run measures //! whichever link reaches the peer, and ToyOS drives Ethernet alone -//! (`issues/there-is-no-wifi.md`). No address is printed. +//! (`issues/there-is-no-wifi.md`). `lan.link` and `lan.peer_link` describe +//! both ends' links; no address is printed. use std::net::{Ipv4Addr, SocketAddr}; use std::time::{Duration, Instant}; +use osbench_lan::link::Link; use osbench_lan::Peer; use crate::report::{condition, refuse, result, unmeasured, unsupported}; /// How long a peer is asked for, which a boot's lease lands inside too. const DISCOVERY: Duration = Duration::from_secs(60); -/// Each transfer's span. -const SPAN: Duration = Duration::from_secs(5); - -/// The links a result can be named for. -#[derive(Clone, Copy, PartialEq, Eq)] -pub enum Link { - Wired, - Wifi, -} - -impl Link { - fn name(self) -> &'static str { - match self { - Link::Wired => "wired", - Link::Wifi => "wifi", - } - } -} +/// Each transfer's span, and the rounds timed after one unread. +const SPAN: Duration = Duration::from_secs(3); +const ROUNDS: usize = 5; fn benches(link: Link) -> [String; 4] { ["tcp.connect", "tcp.rtt.64", "tcp.upload", "tcp.download"].map(|b| format!("lan.{}.{b}", link.name())) @@ -48,8 +36,8 @@ pub fn suite() { } let broadcast = SocketAddr::from((Ipv4Addr::BROADCAST, osbench_lan::PORT)); let started = Instant::now(); - let ip = match osbench_lan::discover(broadcast, DISCOVERY) { - Ok(Some(ip)) => ip, + let found = match osbench_lan::discover(broadcast, DISCOVERY) { + Ok(Some(found)) => found, Ok(None) => return refuse("lan", format!("no osbench-peer answered on the link within {DISCOVERY:?}")), Err(e) if cfg!(target_os = "toyos") && e.kind() == std::io::ErrorKind::NotConnected => { return benches(Link::Wired).iter().for_each(|b| unsupported(b, format!("no network stack answers: {e}"))); @@ -57,7 +45,8 @@ pub fn suite() { Err(e) => return refuse("lan", format!("no socket to look for a peer with: {e}")), }; condition("lan.peer_found_after_ms", started.elapsed().as_millis()); - let peer = Peer(SocketAddr::new(ip, osbench_lan::PORT)); + condition("lan.peer_link", &found.link); + let peer = Peer(SocketAddr::new(found.ip, osbench_lan::PORT)); let link = match peer.local() { Ok(local) => crate::sys::link_of(local.ip()), Err(e) => return refuse("lan", format!("the peer answered and took no connection: {e}")), @@ -89,7 +78,7 @@ fn measure(peer: &Peer, [connect, rtt, upload, download]: &[String; 4]) { let rounds = || -> Result, String> { run(peer, SPAN).map_err(|e| e.to_string())?; crate::clock::settle(); - (0..3) + (0..ROUNDS) .map(|_| { let moved = run(peer, SPAN).map_err(|e| e.to_string())?; Ok(moved.bytes * 1000 / moved.ns) diff --git a/tests/guest/osbench/src/main.rs b/tests/guest/osbench/src/main.rs index b6ba242e2b..d9e64686c2 100644 --- a/tests/guest/osbench/src/main.rs +++ b/tests/guest/osbench/src/main.rs @@ -1,9 +1,10 @@ //! `osbench`: the same system workloads, built from one source for ToyOS //! and for Linux, so one machine's two systems can be compared. //! -//! `osbench ` runs a group, or `all` of them in [`GROUPS`]' order. A -//! ToyOS job list passes no arguments, so there the group is the name the -//! program was started under, `osbench_`. Every other command is one +//! `osbench ` runs a group, or `all` of them in [`GROUPS`]' order; the +//! groups in [`BY_NAME`] need another party and run only by name. A ToyOS job +//! list passes no arguments, so there the group is the name the program was +//! started under, `osbench_`. Every other command is one //! this program gives its own children. //! //! Every result is one line of `key=value` words (`report`): a distribution @@ -30,8 +31,7 @@ //! That nightly is the build's stage-0 compiler (`build/sysroot.rs`'s //! `STAGE0_CARGO`, which this follows), the upstream one nearest ToyOS's own. //! On Linux `nvme` and `energy` read what only root may (the raw namespace, -//! the package's energy counter) and say so unprivileged; `lan` needs an -//! `osbench-peer` (`lan/`) running on another machine on the link. +//! the package's energy counter) and say so unprivileged. //! With the `toyos` profile every guest binary is built with, osbench's own //! code is generated alike for both; std's platform layer and the C library //! beneath it (musl's on Linux) are each system's own. @@ -44,6 +44,7 @@ mod energy; mod fair; mod files; mod identity; +mod internet; mod ipc; mod lan; mod measure; @@ -141,12 +142,12 @@ fn files() { } } -/// Every group, in the order `all` and the ToyOS job list run them: `boot` -/// first, since it reads how long the boot took to reach it; `energy` while -/// the machine is still quiet; `crunch`, which heats it, after every other -/// measurement; and `persist` last, since it writes the volume a ToyOS -/// boot's log is on. -const GROUPS: [(&str, fn()); 18] = [ +/// Every group the machine alone runs, in the order `all` and the ToyOS job +/// list run them: `boot` first, since it reads how long the boot took to +/// reach it; `energy` while the machine is still quiet; `crunch`, which heats +/// it, after every other measurement; and `persist` last, since it writes the +/// volume a ToyOS boot's log is on. +const GROUPS: [(&str, fn()); 17] = [ ("boot", boot::suite), ("identity", identity::suite), ("energy", energy::suite), @@ -162,11 +163,14 @@ const GROUPS: [(&str, fn()); 18] = [ ("work", work::suite), ("tail", tail::suite), ("nvme", nvme::suite), - ("lan", lan::suite), ("crunch", crunch::suite), ("persist", storage::suite), ]; +/// The groups that need another party: `lan` the `osbench-peer` on the other +/// bench, `internet` the public internet. +const BY_NAME: [(&str, fn()); 2] = [("lan", lan::suite), ("internet", internet::suite)]; + /// A group between its first and last lines, after the machine's conditions /// and the clock its CPU starts at. fn group(name: &str, run: fn()) { @@ -194,10 +198,10 @@ fn main() { "fair-helper" => return fair::helper(rest), "ipc-serve" => return ipc::serve(), "cc" => return work::cc(rest), - named => match GROUPS.iter().find(|(name, _)| *name == named) { + named => match GROUPS.iter().chain(&BY_NAME).find(|(name, _)| *name == named) { Some((name, run)) => group(name, *run), None => { - let names: Vec<&str> = GROUPS.iter().map(|(name, _)| *name).collect(); + let names: Vec<&str> = GROUPS.iter().chain(&BY_NAME).map(|(name, _)| *name).collect(); eprintln!("osbench: no command {named:?}; the groups are {} and all", names.join(", ")); std::process::exit(2); } diff --git a/tests/guest/osbench/src/sys/linux.rs b/tests/guest/osbench/src/sys/linux.rs index c2da1440b9..0e8372a662 100644 --- a/tests/guest/osbench/src/sys/linux.rs +++ b/tests/guest/osbench/src/sys/linux.rs @@ -583,49 +583,7 @@ fn backing_disk(mut device: std::path::PathBuf) -> (std::path::PathBuf, bool) { (device, mapped) } -/// The interface holding `local`, off `getifaddrs`, and whether it is a -/// wireless one, as sysfs says: its kind, and its driver and speed. The -/// interface's name and addresses are never printed. -pub fn link_of(local: std::net::IpAddr) -> Result<(crate::lan::Link, String), String> { - let name = interface_holding(local)?; - let at = std::path::Path::new("/sys/class/net").join(&name); - let wireless = at.join("wireless").exists() || at.join("phy80211").exists(); - let driver = fs::read_link(at.join("device/driver")) - .ok() - .and_then(|p| p.file_name().map(|n| n.to_string_lossy().into_owned())) - .unwrap_or_else(|| "none".into()); - let speed = read(&at.join("speed").to_string_lossy()).unwrap_or_else(|| "unknown".into()); - let link = if wireless { crate::lan::Link::Wifi } else { crate::lan::Link::Wired }; - let kind = if wireless { "wireless" } else { "wired" }; - Ok((link, format!("{kind} driver={driver} speed_mbps={speed}"))) -} - -fn interface_holding(local: std::net::IpAddr) -> Result { - let std::net::IpAddr::V4(local) = local else { return Err("the connection left from an IPv6 address".into()) }; - let mut list: *mut libc::ifaddrs = std::ptr::null_mut(); - // SAFETY: `list` is a valid out-pointer; the list is freed below. - if unsafe { libc::getifaddrs(&mut list) } != 0 { - return Err(format!("getifaddrs: {}", std::io::Error::last_os_error())); - } - let mut found = None; - let mut at = list; - while !at.is_null() { - // SAFETY: a node of the list getifaddrs made, alive until it is freed. - let ifa = unsafe { &*at }; - // SAFETY: a non-null `ifa_addr` of family AF_INET is a sockaddr_in. - let v4 = (!ifa.ifa_addr.is_null() && unsafe { (*ifa.ifa_addr).sa_family } == libc::AF_INET as libc::sa_family_t) - .then(|| unsafe { (*ifa.ifa_addr.cast::()).sin_addr.s_addr }); - if v4 == Some(u32::from(local).to_be()) { - // SAFETY: an interface's name is a NUL-terminated string. - found = Some(unsafe { std::ffi::CStr::from_ptr(ifa.ifa_name) }.to_string_lossy().into_owned()); - break; - } - at = ifa.ifa_next; - } - // SAFETY: the list getifaddrs made, read no more. - unsafe { libc::freeifaddrs(list) }; - found.ok_or_else(|| "no interface holds the address the connection left from".into()) -} +pub use osbench_lan::link::of as link_of; /// The package's temperature off hwmon: `coretemp`'s `Package id 0`, or /// `k10temp`'s `Tdie` where it has one and its `Tctl` otherwise. diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index b2fb54efcd..d4a50d3324 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -364,8 +364,8 @@ pub fn persistent_dir() -> Option<(String, String)> { } /// What carries a connection on ToyOS: netstack drives one card, Ethernet. -pub fn link_of(_local: std::net::IpAddr) -> Result<(crate::lan::Link, String), String> { - Ok((crate::lan::Link::Wired, "netstack's one Ethernet card".into())) +pub fn link_of(_local: std::net::IpAddr) -> Result<(osbench_lan::link::Link, String), String> { + Ok((osbench_lan::link::Link::Wired, "wired netstack's one Ethernet card".into())) } /// ToyOS reads no temperature: the kernel counts none diff --git a/tests/images/benchcase.toml b/tests/images/benchcase.toml index 8c8464922d..f810120f75 100644 --- a/tests/images/benchcase.toml +++ b/tests/images/benchcase.toml @@ -2,8 +2,9 @@ # on the bench's own Linux (`tests/guest/osbench`). # # Only what a job list needs to run and to reach the log on the boot stick, the -# ACPI server, and netstack for `osbench lan`: no compositor or sound, so little -# but the measurement asks for a CPU. DATA is memory here, the stick having no DATA +# ACPI server, and netstack for `osbench lan` and `osbench internet`, each its +# own metal row (`tests/toyos.rs`): no compositor or sound, so little but the +# measurement asks for a CPU. DATA is memory here, the stick having no DATA # partition, and the log volume is the one persistent volume a job can write # (`osbench persist`). # @@ -31,21 +32,21 @@ devices = ["acpi"] receives = ["power"] # netstack on the T14's wired I219, the only card a bench has that ToyOS drives: -# `osbench lan` reaches its peer through it. On the AMD bench, which has none, -# netstack leaves and `lan` says so. +# `osbench lan` reaches its peer and `osbench internet` the internet through it. +# On the AMD bench, which has none, netstack leaves and both say so. [programs.netstack] service = true serves = ["netstack"] devices = ["pci:02:00:00/8086:15fc"] # The order `osbench all` runs the groups in. The bound is the metal row's, its -# 480 s on the 60 s every list has (`tests/toyos.rs`), restated for a boot of +# 370 s on the 60 s every list has (`tests/toyos.rs`), restated for a boot of # this config built without the row; a derived metal config states its own. [programs.test-runner] receives = ["power", "netstack"] syscap = ["jobs", "counters", "trace", "roster", "inventory", "dup"] args = [ - "--bound-ms=540000", + "--bound-ms=430000", "osbench_boot", "osbench_identity", "osbench_energy", @@ -61,7 +62,6 @@ args = [ "osbench_work", "osbench_tail", "osbench_nvme", - "osbench_lan", "osbench_crunch", "osbench_persist", "reboot", @@ -88,6 +88,7 @@ args = [ "bin/osbench_tail" = "/system/bin/osbench" "bin/osbench_nvme" = "/system/bin/osbench" "bin/osbench_lan" = "/system/bin/osbench" +"bin/osbench_internet" = "/system/bin/osbench" "bin/osbench_crunch" = "/system/bin/osbench" "bin/osbench_persist" = "/system/bin/osbench" "bin/reboot" = "/system/bin/toybox" diff --git a/tests/toyos.rs b/tests/toyos.rs index d5afe06f45..ae94baa7f2 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -1093,7 +1093,22 @@ const METAL: &[(&str, metal::Metal)] = &[ // holds only that every group ran to its end. The T14 and the AMD bench. // One boot runs the whole suite, so three readbacks are three runs. "osbench", - metal::Metal { arms: BENCHCASE, judge: |b| osbench_ran(b[0]) }, + metal::Metal { arms: BENCHCASE, judge: |b| osbench_ran(b[0], &OSBENCH_GROUPS) }, + ), + ( + // `osbench lan` against the `osbench-peer` the other bench's Linux runs, + // compared with the same program under this bench's Linux on the same + // path. A pass names it: the other bench has to be running the peer. + // The T14 over its I219; the AMD bench has no card ToyOS drives. + "osbench_lan", + metal::Metal { arms: OSBENCH_LAN, judge: |b| osbench_ran(b[0], &["osbench_lan"]) }, + ), + ( + // `osbench internet`: exploratory evidence over the public internet, + // which no automated test depends on, so only a pass that names it runs + // it. The T14; the AMD bench has no card ToyOS drives. + "osbench_internet", + metal::Metal { arms: OSBENCH_INTERNET, judge: |b| osbench_ran(b[0], &["osbench_internet"]) }, ), // ---- tests/images/jobcase.toml, each image armed to end its boot its own way ---- ( @@ -1587,7 +1602,7 @@ const TESTCASES: &[metal::Arm] = &[metal::once( /// `osbench`'s groups, each a job named by its symlink, in `osbench all`'s /// order: `boot` first, which reads how long the boot took to reach it, and /// `persist` last, which writes the log volume. -const OSBENCH_GROUPS: [&str; 18] = [ +const OSBENCH_GROUPS: [&str; 17] = [ "osbench_boot", "osbench_identity", "osbench_energy", @@ -1603,7 +1618,6 @@ const OSBENCH_GROUPS: [&str; 18] = [ "osbench_work", "osbench_tail", "osbench_nvme", - "osbench_lan", "osbench_crunch", "osbench_persist", ]; @@ -1612,21 +1626,43 @@ const OSBENCH_GROUPS: [&str; 18] = [ /// 100 s of wakes and spinners. Six boots of `e5573b7fe` ended their last job /// at 175 to 191 s on five, and at 244 s on the T14's first, where one /// served 64 MiB write stalled and `files` took 80 s against 13. The groups -/// since add at most 160 s by their own spans: `lan`'s 60 s looking for its -/// peer and 50 s measuring, `crunch`'s one-thread 16 s (ToyOS reads no -/// temperature, so its all-core phase does not run), and under 30 s of memcpy -/// sizes, pipe sizes and 1 ms timers. So a list bound of 540 s leaves that -/// stall 136 s. +/// since add at most 50 s by their own spans: `crunch`'s one-thread 16 s +/// (ToyOS reads no temperature, so its all-core phase does not run), and +/// under 30 s of memcpy sizes, pipe sizes and 1 ms timers. So a list bound of +/// 430 s leaves that stall 136 s. const BENCHCASE: &[metal::Arm] = &[metal::Arm { - adds_ms: 480_000, + adds_ms: 370_000, ..metal::once("benchcase", "tests/images/benchcase.toml", &[], &OSBENCH_GROUPS) }]; -/// Every group exited 0, which `osbench` does only once each of its +/// The two groups that need another party share one boot when one pass names +/// both. Each adds its own worst span; the 60 s every list has is the +/// boot's margin, not either group's. +/// +/// `lan` looks for its peer for up to 60 s, inside which the boot's lease +/// lands, then measures for under 60 s over a Wi-Fi hop: 2,155 round trips +/// at up to 10 ms and twelve 3 s transfers with their warm-ups. +const OSBENCH_LAN: &[metal::Arm] = &[metal::Arm { + adds_ms: 120_000, + named_only: true, + ..metal::once("netbenchcase", "tests/images/benchcase.toml", &[], &["osbench_lan"]) +}]; +/// `internet` waits up to 60 s for the lease when it runs first, then +/// measures four 10 MB downloads, four 10 MB uploads and 77 short exchanges: +/// under 45 s at 3 MB/s down and 1.5 MB/s up. Those rates are an assumed +/// floor, far under the T14's first ToyOS readings, and a slower link ends +/// the list at the bound. +const OSBENCH_INTERNET: &[metal::Arm] = &[metal::Arm { + adds_ms: 105_000, + named_only: true, + ..metal::once("netbenchcase", "tests/images/benchcase.toml", &[], &["osbench_internet"]) +}]; + +/// Every group's job exited 0, which `osbench` does only once each of its /// benchmarks measured, and said it was done. -fn osbench_ran(boot: &metal::Readback) -> Result<(), String> { +fn osbench_ran(boot: &metal::Readback, jobs: &[&str]) -> Result<(), String> { in_acpi_mode_before_any_job(boot)?; - for job in OSBENCH_GROUPS { + for &job in jobs { boot.job_passed(job)?; let group = job.strip_prefix("osbench_").expect("a group's job"); boot.log().must_say(&format!("group={group} done"))?;