From c3b124b3fb2d717b1b920e7b118d142fd7b44ff3 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 24 Oct 2026 12:21:09 +0200 Subject: [PATCH 1/5] osbench: the benches are each other's LAN peer, and the internet is an explicit evidence run The development Mac moves and is on Wi-Fi, so it is no peer and no part of a measurement. `osbench lan` now runs on one bench against the `osbench-peer` the other bench's Linux runs as a fixed reference, each way; ToyOS against Linux on one path is the comparison, and Linux's own runs give the path's ceiling and noise. - lan: the peer's answer says which of its links it answered over (kind, driver, and speed or Wi-Fi signal), refused past 160 printable bytes; the bench records it as `lan.peer_link`. Link reading moves from osbench's Linux layer into `osbench-lan::link`, so the peer uses it too, and an address on an interface with no device is refused rather than called wired. Transfers are five timed 3 s rounds after one unread. - `lan` and the new `internet` leave `all` and the full suite's boot: they need another party, so each runs only by name, and the full suite no longer needs the other bench. Its list bound drops by `lan`'s 110 s. - internet: TCP connect, an HTTPS round trip on a kept connection, 25 MB downloads and 10 MB uploads against Cloudflare's speed test (speed.cloudflare.com, the endpoints Cloudflare's own engine drives), over ureq and rustls on toyos-crypto's provider on both systems. Answers' headers and bodies name the client's address; nothing reads or prints them. A ToyOS netstack answers NotConnected until its lease lands and announces none, so the lookup is asked again each second for up to 60 s. - Metal rows `osbench_lan` and `osbench_internet`, named-only, on one `netbenchcase` boot of benchcase.toml when a pass names both. - osbench-compare: reads run files (a Linux run's stdout or a ToyOS boot's log) and prints per machine and benchmark each system's per-run p50s, median, run-to-run range, ToyOS/Linux and the exact two-sided Mann-Whitney p over the runs, with ties as half wins. LAN rows whose runs' peers were on different links are not compared. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 10 + Cargo.toml | 1 + ...ompares-toyos-with-linux-on-one-machine.md | 17 +- tests/guest/osbench/Cargo.toml | 9 +- tests/guest/osbench/compare/Cargo.toml | 17 + tests/guest/osbench/compare/src/lib.rs | 406 ++++++++++++++++++ tests/guest/osbench/compare/src/main.rs | 25 ++ tests/guest/osbench/lan/Cargo.toml | 8 +- tests/guest/osbench/lan/src/lib.rs | 77 +++- tests/guest/osbench/lan/src/link.rs | 125 ++++++ tests/guest/osbench/lan/src/main.rs | 4 +- tests/guest/osbench/src/internet.rs | 136 ++++++ tests/guest/osbench/src/lan.rs | 43 +- tests/guest/osbench/src/main.rs | 32 +- tests/guest/osbench/src/sys/linux.rs | 44 +- tests/guest/osbench/src/sys/toyos.rs | 4 +- tests/images/benchcase.toml | 15 +- tests/toyos.rs | 54 ++- 18 files changed, 896 insertions(+), 131 deletions(-) create mode 100644 tests/guest/osbench/compare/Cargo.toml create mode 100644 tests/guest/osbench/compare/src/lib.rs create mode 100644 tests/guest/osbench/compare/src/main.rs create mode 100644 tests/guest/osbench/lan/src/link.rs create mode 100644 tests/guest/osbench/src/internet.rs diff --git a/Cargo.lock b/Cargo.lock index ef2c75d4c1..228a539d9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2265,20 +2265,30 @@ dependencies = [ name = "osbench" version = "0.1.0" dependencies = [ + "getrandom 0.3.4", "libc", "miniz_oxide", "osbench-lan", "osbench-rules", + "rustls", "syn 2.0.117", "toyos", "toyos-abi", "toyos-crypto", "toyos-trace", + "ureq", ] +[[package]] +name = "osbench-compare" +version = "0.1.0" + [[package]] name = "osbench-lan" version = "0.1.0" +dependencies = [ + "libc", +] [[package]] name = "osbench-rules" diff --git a/Cargo.toml b/Cargo.toml index 9627dc1ca0..2ae66bd8a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -146,6 +146,7 @@ members = [ "tests/guest/kernelprobe", "tests/guest/metalprobe", "tests/guest/osbench", + "tests/guest/osbench/compare", "tests/guest/osbench/lan", "tests/guest/osbench/rules", "tests/guest/pciholder", diff --git a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md index b1ecac0749..679ec5a201 100644 --- a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md +++ b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md @@ -24,11 +24,20 @@ its groups: - an idle minute's package energy: `energy.idle_60s.package`; - the machine's PCI, USB, NVMe and cpuidle identity: `identity`, with no serial number or network identifier; -- the wired LAN's TCP round trip and throughput, and Wi-Fi's under Linux: - `lan.wired.*`, `lan.wifi.*`, against `osbench-peer` on another machine; +- TCP connect, round trip and throughput between the two benches: + `lan.wired.*`, `lan.wifi.*`, the bench under test against the + `osbench-peer` the other bench's Linux runs, each way. ToyOS and Linux on + one path are the comparison, and Linux's runs on it its ceiling and its + noise; `osbench-compare` reads the runs, interleaved, four of each at least + for its exact test to resolve a difference. `lan` and `internet` run only + by name, each its own metal row, since they need another party; - a short CPU crunch, one thread then all, that stops at TjMax less 15 C where the package's temperature can be read: `crunch.*`. +`internet.*`, HTTPS to Cloudflare's speed test, is exploratory evidence of a +path neither system owns, run only when a pass names its row: no automated +test depends on the public internet. + The cryptographic rows are not osbench's. AES-GCM, ChaCha20-Poly1305, SHA-256 and the rest were measured under Linux on both benches against ring, OpenSSL and AWS-LC by the crypto oracle probe, and that table is the @@ -51,7 +60,9 @@ comparison until it can: 1.7-1.9 GHz, which `issues/the-t14s-thermal-modes-do-not-move-its-package-power-limits.md` would explain: ToyOS refuses the firmware's PL1 and PL2 writes. -- **Wi-Fi**: `issues/there-is-no-wifi.md`. +- **Wi-Fi**: `issues/there-is-no-wifi.md`. The T14's `lan.wifi.*` and every + AMD bench `lan.*` and `internet.*` row, whose one network card is its + AX200, are measured under Linux alone. - **USB, NVMe and cpuidle identity**: their owners serve no program that reads them. - **Loopback** (`net.*`): netstack refuses 127/8 as a destination. diff --git a/tests/guest/osbench/Cargo.toml b/tests/guest/osbench/Cargo.toml index 9475d4f121..aa9c83d8b7 100644 --- a/tests/guest/osbench/Cargo.toml +++ b/tests/guest/osbench/Cargo.toml @@ -8,8 +8,13 @@ license = "MIT OR Apache-2.0" [dependencies] # The work and crunch groups' compression, pure Rust and the same code on both systems. miniz_oxide = "0.9" -# The crunch group's hashing: ToyOS's own SHA-256, which its ssh server runs. -toyos-crypto = { path = "../../../lib/crypto" } +# The crunch group's hashing: ToyOS's own SHA-256, which its ssh server runs; +# and the internet group's TLS provider, with its types and its entropy. +toyos-crypto = { path = "../../../lib/crypto", features = ["std", "rustls-provider"] } +rustls = { version = "0.23.45", default-features = false } +getrandom = "0.3" +# The internet group's HTTPS, with webpki's roots, on toyos-crypto's provider. +ureq = { version = "3", default-features = false, features = ["rustls-no-provider", "rustls-webpki-roots"] } # The crunch group's compile-like task: parsing Rust, with no compiler to load. syn = { version = "2", default-features = false, features = ["full", "parsing"] } # The LAN group's two ends, and the rules readings are judged by. diff --git a/tests/guest/osbench/compare/Cargo.toml b/tests/guest/osbench/compare/Cargo.toml new file mode 100644 index 0000000000..9d49e834ee --- /dev/null +++ b/tests/guest/osbench/compare/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "osbench-compare" +description = "Compares osbench runs of ToyOS and Linux on one machine: each benchmark's per-run medians, each system's run-to-run range, their ratio and an exact rank test." +version = "0.1.0" +edition = "2021" +license = "MIT OR Apache-2.0" +publish = false + +# A package of its own because osbench builds for ToyOS and Linux alone, and +# its runs are compared wherever they are collected. + +[lib] +doctest = false + +[[bin]] +name = "osbench-compare" +path = "src/main.rs" diff --git a/tests/guest/osbench/compare/src/lib.rs b/tests/guest/osbench/compare/src/lib.rs new file mode 100644 index 0000000000..cae0ace2f9 --- /dev/null +++ b/tests/guest/osbench/compare/src/lib.rs @@ -0,0 +1,406 @@ +//! osbench runs of ToyOS and Linux on one machine, compared. +//! +//! A run is one file of osbench's lines, a Linux run's stdout or a ToyOS +//! boot's log, from whatever prefix each line carries. A benchmark's value in +//! a run is its p50. Per machine and benchmark, each system's runs give the +//! median of their values and their run-to-run range, `(max - min) / median`: +//! Linux's range on a path is that path's noise floor. ToyOS's median over +//! Linux's is the ratio, and the exact two-sided Mann-Whitney test over the +//! two sets of runs says whether the difference is resolved at all: with four +//! runs each, complete separation gives p = 2/70. + +use std::collections::BTreeMap; +use std::fmt::Write as _; + +/// The test level a difference is resolved at. +pub const RESOLVED: f64 = 0.05; +/// The most runs the exact test enumerates the splits of. +pub const MAX_RUNS: usize = 20; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Os { + Linux, + Toyos, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Outcome { + Measured { unit: String, p50: u64 }, + /// Unmeasured, unsupported or refused, with the run's reason. + Not { kind: String, why: String }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Run { + pub os: Os, + pub conds: BTreeMap, + pub benches: BTreeMap, +} + +impl Run { + /// The machine a run is of: its CPU model. + pub fn machine(&self) -> Option<&str> { + self.conds.get("cpu_model").map(String::as_str) + } +} + +/// One run's lines. A benchmark a run reports in more than one group, as +/// every group's `cpu.clock`, is named `@`, and the `k`th of +/// one name in a group, as `fair.flat`'s jobs, `#k`. A run with no +/// osbench line, two systems' lines or one group started twice is refused. +pub fn parse(text: &str) -> Result { + let mut os = None; + let mut conds = BTreeMap::new(); + let mut group = String::new(); + let mut started = std::collections::BTreeSet::new(); + let mut read: Vec<(String, String, Outcome)> = Vec::new(); + for line in text.lines() { + let Some(at) = line.find("osbench v=2 os=") else { continue }; + let mut words = line[at..].split_whitespace().skip(2); + let line_os = match words.next() { + Some("os=linux") => Os::Linux, + Some("os=toyos") => Os::Toyos, + other => return Err(format!("a line of no system osbench knows: {other:?}")), + }; + if *os.get_or_insert(line_os) != line_os { + return Err("a run holds two systems' lines".into()); + } + let first = words.next().unwrap_or_default(); + if first == "cond" { + if let Some((key, value)) = words.next().and_then(|w| w.split_once('=')) { + conds.entry(key.to_string()).or_insert_with(|| value.to_string()); + } + continue; + } + if let Some(name) = first.strip_prefix("group=") { + if words.next() == Some("start") { + if started.contains(name) { + return Err(format!("group {name} starts twice: a file holds one run")); + } + started.insert(name.to_string()); + } + group = name.to_string(); + continue; + } + let Some(bench) = first.strip_prefix("bench=") else { continue }; + let fields: BTreeMap<&str, &str> = words.filter_map(|w| w.split_once('=')).collect(); + let outcome = if let Some(unit) = fields.get("unit") { + let p50 = fields.get("p50").and_then(|v| v.parse().ok()).ok_or_else(|| format!("{bench}: no p50"))?; + Outcome::Measured { unit: unit.to_string(), p50 } + } else if let Some((kind, why)) = + ["unmeasured", "unsupported", "refused"].iter().find_map(|k| fields.get(k).map(|why| (*k, *why))) + { + Outcome::Not { kind: kind.to_string(), why: why.to_string() } + } else { + return Err(format!("{bench}: neither a distribution nor a reason")); + }; + read.push((group.clone(), bench.to_string(), outcome)); + } + let mut benches = BTreeMap::new(); + for (i, (group, bench, outcome)) in read.iter().enumerate() { + let same = |(g, b, _): &&(String, String, Outcome)| b == bench && (g == group || group.is_empty()); + let mut name = bench.clone(); + if read.iter().any(|(g, b, _)| b == bench && g != group) { + name = format!("{name}@{group}"); + } + if read.iter().filter(same).count() > 1 { + name = format!("{name}#{}", read[..=i].iter().filter(same).count()); + } + benches.insert(name, outcome.clone()); + } + Ok(Run { os: os.ok_or("no osbench line")?, conds, benches }) +} + +pub fn median(values: &[u64]) -> f64 { + let mut v = values.to_vec(); + v.sort_unstable(); + let n = v.len(); + assert!(n > 0, "a median of no values"); + if n % 2 == 1 { + v[n / 2] as f64 + } else { + f64::midpoint(v[n / 2 - 1] as f64, v[n / 2] as f64) + } +} + +/// `(max - min) / median`, the spread of a set of runs. +pub fn range(values: &[u64]) -> f64 { + let (min, max) = (values.iter().min().expect("values"), values.iter().max().expect("values")); + (max - min) as f64 / median(values) +} + +/// The exact two-sided Mann-Whitney p of `a` against `b`, ties counted half, +/// by every split of the pooled values into sets of their sizes; `None` past +/// [`MAX_RUNS`] values or with an empty side. +pub fn mann_whitney(a: &[u64], b: &[u64]) -> Option { + let pooled: Vec = a.iter().chain(b).copied().collect(); + let total = pooled.len(); + if a.is_empty() || b.is_empty() || total > MAX_RUNS { + return None; + } + // Twice U: two per pair the first set wins, one per tie. + let doubled_u = |first: &[u64], second: &[u64]| -> i64 { + let mut u = 0; + for x in first { + for y in second { + u += match x.cmp(y) { + std::cmp::Ordering::Greater => 2, + std::cmp::Ordering::Equal => 1, + std::cmp::Ordering::Less => 0, + }; + } + } + u + }; + let centre = (a.len() * b.len()) as i64; + let seen = (doubled_u(a, b) - centre).abs(); + let (mut splits, mut extreme) = (0u64, 0u64); + let (mut first, mut second) = (Vec::with_capacity(total), Vec::with_capacity(total)); + for mask in 0u32..(1 << total) { + if mask.count_ones() as usize != a.len() { + continue; + } + first.clear(); + second.clear(); + for (i, v) in pooled.iter().enumerate() { + if mask & (1 << i) != 0 { first.push(*v) } else { second.push(*v) } + } + splits += 1; + if (doubled_u(&first, &second) - centre).abs() >= seen { + extreme += 1; + } + } + Some(extreme as f64 / splits as f64) +} + +/// Whether a lower value of `unit` is the better one. +fn lower_is_better(unit: &str) -> bool { + unit == "ns" +} + +/// A link description's kind and driver, without what varies by the run. +fn path(said: &str) -> String { + said.split(['_', ' ']).take(2).collect::>().join(" ") +} + +/// Every machine's comparison, as Markdown: one section per machine, one row +/// per benchmark either system reported. +pub fn compare(runs: &[Run]) -> Result { + let mut machines: BTreeMap<&str, Vec<&Run>> = BTreeMap::new(); + for run in runs { + machines.entry(run.machine().ok_or("a run that names no cpu_model")?).or_default().push(run); + } + let mut out = String::new(); + for (machine, runs) in machines { + let count = |os| runs.iter().filter(|r| r.os == os).count(); + let peers: std::collections::BTreeSet = + runs.iter().filter_map(|r| r.conds.get("lan.peer_link")).map(|said| path(said)).collect(); + writeln!(out, "## {machine}\n").unwrap(); + writeln!(out, "Runs: Linux {}, ToyOS {}. LAN peer: {}.\n", count(Os::Linux), count(Os::Toyos), { + if peers.is_empty() { "none".to_string() } else { peers.iter().cloned().collect::>().join(", ") } + }) + .unwrap(); + writeln!(out, "| bench | unit | Linux p50s | Linux median | Linux range | ToyOS p50s | ToyOS median | ToyOS range | ToyOS/Linux | p | verdict |").unwrap(); + writeln!(out, "|---|---|---|---|---|---|---|---|---|---|---|").unwrap(); + let names: std::collections::BTreeSet<&String> = runs.iter().flat_map(|r| r.benches.keys()).collect(); + for name in names { + let row = if name.starts_with("lan.") && peers.len() > 1 { + Err("the runs' peers were on different links".to_string()) + } else { + Ok(compare_one(&runs, name)) + }; + match row { + Ok(row) => writeln!(out, "| {name} | {row} |").unwrap(), + Err(why) => writeln!(out, "| {name} | | | | | | | | | | {why} |").unwrap(), + } + } + writeln!(out).unwrap(); + } + Ok(out) +} + +/// One benchmark's cells after its name. +fn compare_one(runs: &[&Run], name: &str) -> String { + let mut unit = None; + let mut values: BTreeMap> = BTreeMap::new(); + let mut not: BTreeMap> = BTreeMap::new(); + for run in runs { + match run.benches.get(name) { + Some(Outcome::Measured { unit: u, p50 }) => { + if unit.get_or_insert(u.as_str()) != u { + return "| | | | | | | | | units differ between runs".into(); + } + values.entry(run.os).or_default().push(*p50); + } + Some(Outcome::Not { kind, why }) => not.entry(run.os).or_default().push(format!("{kind}: {why}")), + None => not.entry(run.os).or_default().push("absent".into()), + } + } + let side = |os: Os| -> [String; 3] { + match values.get(&os) { + Some(v) => [ + v.iter().map(u64::to_string).collect::>().join(", "), + format!("{:.0}", median(v)), + format!("{:.1}%", range(v) * 100.0), + ], + None => [String::new(), "-".into(), "-".into()], + } + }; + let [lp, lm, lr] = side(Os::Linux); + let [tp, tm, tr] = side(Os::Toyos); + let (ratio, p, verdict) = match (values.get(&Os::Linux), values.get(&Os::Toyos)) { + (Some(l), Some(t)) => { + let ratio = median(t) / median(l); + let p = mann_whitney(t, l); + let verdict = match p { + None => "too many runs for the exact test".to_string(), + Some(_) if l.len() < 2 || t.len() < 2 => "too few runs".to_string(), + Some(p) if p <= RESOLVED => { + let better = (ratio < 1.0) == lower_is_better(unit.unwrap_or_default()); + format!("ToyOS {}", if better { "better" } else { "worse" }) + } + Some(_) => "not resolved".to_string(), + }; + (format!("{ratio:.2}"), p.map_or_else(|| "-".into(), |p| format!("{p:.3}")), verdict) + } + _ => ("-".into(), "-".into(), String::new()), + }; + let mut verdict = verdict; + for (os, reasons) in ¬ { + let mut distinct = reasons.clone(); + distinct.sort(); + distinct.dedup(); + let who = if *os == Os::Linux { "Linux" } else { "ToyOS" }; + let sep = if verdict.is_empty() { "" } else { "; " }; + verdict = format!("{verdict}{sep}{who} {} run(s) {}", reasons.len(), distinct.join(" / ")); + } + format!("{} | {lp} | {lm} | {lr} | {tp} | {tm} | {tr} | {ratio} | {p} | {verdict}", unit.unwrap_or_default()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn run(os: &str, lines: &[&str]) -> Run { + let mut text = format!("osbench v=2 os={os} cond cpu_model=Bench_CPU\n"); + for line in lines { + text.push_str(&format!("[12.5 cpu3 osbench] osbench v=2 os={os} {line}\n")); + } + parse(&text).unwrap() + } + + #[test] + fn a_run_reads_each_benchmarks_p50_or_its_reason() { + let r = run( + "toyos", + &[ + "bench=lan.wired.tcp.rtt.64 unit=ns n=2000 min=1 p50=812 p90=900 p99=990 p999=- max=2000 mean=850 mhz=4000", + "bench=lan.wifi.tcp.upload unsupported=ToyOS_has_no_Wi-Fi_driver", + "cond lan.peer_link=wireless_driver=iwlwifi_signal_dbm=-52", + "group=lan done secs=40", + ], + ); + assert_eq!(r.os, Os::Toyos); + assert_eq!(r.machine(), Some("Bench_CPU")); + assert_eq!(r.benches["lan.wired.tcp.rtt.64"], Outcome::Measured { unit: "ns".into(), p50: 812 }); + assert_eq!( + r.benches["lan.wifi.tcp.upload"], + Outcome::Not { kind: "unsupported".into(), why: "ToyOS_has_no_Wi-Fi_driver".into() } + ); + assert_eq!(r.conds["lan.peer_link"], "wireless_driver=iwlwifi_signal_dbm=-52"); + } + + #[test] + fn repeated_names_are_told_apart_and_a_file_of_two_runs_is_refused() { + let twice = "osbench v=2 os=linux group=lan start\n".repeat(2); + assert!(parse(&twice).unwrap_err().contains("twice")); + let groups = ["sched", "fair"].map(|g| format!("osbench v=2 os=linux group={g} start\nosbench v=2 os=linux bench=cpu.clock unit=MHz n=1 min=1 p50=4000 max=1 mean=1\n")).concat(); + let run = parse(&groups).unwrap(); + assert_eq!(run.benches.keys().collect::>(), ["cpu.clock@fair", "cpu.clock@sched"]); + let jobs = "osbench v=2 os=linux bench=fair.flat unit=permille n=5 min=1 p50=63 max=1 mean=1 job=build\n".repeat(2); + assert_eq!(parse(&jobs).unwrap().benches.keys().collect::>(), ["fair.flat#1", "fair.flat#2"]); + let mixed = "osbench v=2 os=linux cond cpus=8\nosbench v=2 os=toyos cond cpus=8\n"; + assert!(parse(mixed).unwrap_err().contains("two systems")); + assert!(parse("nothing here\n").is_err()); + } + + #[test] + fn the_exact_test_matches_its_tables() { + // Complete separation has two splits as extreme as itself, of C(n+m, n). + let p = |a: &[u64], b: &[u64]| mann_whitney(a, b).unwrap(); + assert!((p(&[1, 2, 3, 4], &[5, 6, 7, 8]) - 2.0 / 70.0).abs() < 1e-12); + assert!((p(&[8, 7, 6, 5], &[1, 2, 3, 4]) - 2.0 / 70.0).abs() < 1e-12); + assert!((p(&[1, 2, 3], &[4, 5, 6]) - 0.1).abs() < 1e-12); + assert!((p(&[1, 2, 3, 4, 5], &[6, 7, 8, 9, 10]) - 2.0 / 252.0).abs() < 1e-12); + // Interleaved sets are as central as a split can be. + assert_eq!(p(&[1, 3, 5, 7], &[2, 4, 6, 8]), p(&[2, 4, 6, 8], &[1, 3, 5, 7])); + assert!(p(&[1, 4, 5, 8], &[2, 3, 6, 7]) > 0.9); + // Every value tied is no evidence of a difference. + assert_eq!(p(&[5, 5, 5], &[5, 5, 5]), 1.0); + // One tie across the sets: U = 0.5 of 16 has 4 of the 70 splits at least as far out. + assert!((p(&[1, 2, 3, 5], &[5, 6, 7, 8]) - 4.0 / 70.0).abs() < 1e-12); + // A tie is half a win: [2, 2] against [1, 2] is U = 3 of 4, and every + // split is as far from the centre's 2, where ties counted whole would + // put half of them nearer. + assert_eq!(p(&[2, 2], &[1, 2]), 1.0); + assert_eq!(mann_whitney(&[], &[1]), None); + assert_eq!(mann_whitney(&[1; 11], &[2; 10]), None); + } + + #[test] + fn median_and_range_of_a_set_of_runs() { + assert_eq!(median(&[3, 1, 2]), 2.0); + assert_eq!(median(&[4, 1, 3, 2]), 2.5); + assert!((range(&[90, 100, 110, 100]) - 0.2).abs() < 1e-12); + assert_eq!(range(&[7]), 0.0); + } + + fn rtt(os: &str, p50: u64) -> Run { + run(os, &[&format!("bench=lan.wired.tcp.rtt.64 unit=ns n=2000 min=1 p50={p50} max=9 mean=5")]) + } + + #[test] + fn separated_runs_resolve_a_difference_and_overlapping_ones_do_not() { + let mut runs: Vec = [100, 104, 98, 102].iter().map(|v| rtt("linux", *v)).collect(); + runs.extend([150, 149, 155, 153].iter().map(|v| rtt("toyos", *v))); + let table = compare(&runs).unwrap(); + let row = table.lines().find(|l| l.starts_with("| lan.wired.tcp.rtt.64")).unwrap(); + assert!(row.contains("| 101 | 5.9% |"), "{row}"); + assert!(row.contains("| 1.50 | 0.029 | ToyOS worse |"), "{row}"); + + let mut runs: Vec = [100, 104, 98, 101].iter().map(|v| rtt("linux", *v)).collect(); + runs.extend([99, 103, 97, 102].iter().map(|v| rtt("toyos", *v))); + let table = compare(&runs).unwrap(); + assert!(table.contains("| not resolved |"), "{table}"); + } + + #[test] + fn a_throughputs_higher_value_is_the_better_one() { + let mbps = |os: &str, v: u64| run(os, &[&format!("bench=lan.wired.tcp.upload unit=MB/s n=5 min=1 p50={v} max=9 mean=5")]); + let mut runs: Vec = [40, 41, 42, 43].iter().map(|v| mbps("linux", *v)).collect(); + runs.extend([50, 51, 52, 53].iter().map(|v| mbps("toyos", *v))); + assert!(compare(&runs).unwrap().contains("| ToyOS better |")); + } + + #[test] + fn a_system_that_cannot_measure_says_so_beside_the_others_figures() { + let mut runs: Vec = [10, 11].iter().map(|v| run("linux", &[&format!("bench=lan.wifi.tcp.connect unit=ns n=50 min=1 p50={v} max=9 mean=5")])).collect(); + runs.push(run("toyos", &["bench=lan.wifi.tcp.connect unsupported=no_Wi-Fi"])); + let table = compare(&runs).unwrap(); + assert!(table.contains("| ToyOS 1 run(s) unsupported: no_Wi-Fi |"), "{table}"); + } + + #[test] + fn lan_runs_whose_peers_were_on_different_links_are_not_compared() { + let peer = |os: &str, said: &str, v: u64| { + run(os, &[&format!("cond lan.peer_link={said}"), &format!("bench=lan.wired.tcp.rtt.64 unit=ns n=1 min=1 p50={v} max=1 mean=1")]) + }; + let runs = [ + peer("linux", "wireless_driver=iwlwifi_signal_dbm=-52", 10), + peer("toyos", "wireless_driver=iwlwifi_signal_dbm=-60", 12), + ]; + assert!(!compare(&runs).unwrap().contains("different links"), "a signal is no other link"); + let runs = [peer("linux", "wireless_driver=iwlwifi_signal_dbm=-52", 10), peer("toyos", "wired_driver=e1000e_speed_mbps=1000", 12)]; + assert!(compare(&runs).unwrap().contains("| the runs' peers were on different links |")); + } +} diff --git a/tests/guest/osbench/compare/src/main.rs b/tests/guest/osbench/compare/src/main.rs new file mode 100644 index 0000000000..4281998c0d --- /dev/null +++ b/tests/guest/osbench/compare/src/main.rs @@ -0,0 +1,25 @@ +//! `osbench-compare ...`: each file one osbench run of ToyOS or Linux, +//! compared per machine as Markdown on stdout (`osbench_compare`). + +fn main() { + let files: Vec = std::env::args().skip(1).collect(); + if files.is_empty() { + fail("usage: osbench-compare ..."); + } + let runs: Vec = files + .iter() + .map(|file| { + let text = std::fs::read_to_string(file).unwrap_or_else(|e| fail(&format!("{file}: {e}"))); + osbench_compare::parse(&text).unwrap_or_else(|why| fail(&format!("{file}: {why}"))) + }) + .collect(); + match osbench_compare::compare(&runs) { + Ok(table) => print!("{table}"), + Err(why) => fail(&why), + } +} + +fn fail(why: &str) -> ! { + eprintln!("osbench-compare: {why}"); + std::process::exit(1); +} diff --git a/tests/guest/osbench/lan/Cargo.toml b/tests/guest/osbench/lan/Cargo.toml index 9cabd28663..2edf593f12 100644 --- a/tests/guest/osbench/lan/Cargo.toml +++ b/tests/guest/osbench/lan/Cargo.toml @@ -1,13 +1,13 @@ [package] name = "osbench-lan" -description = "osbench's LAN measurement, both ends: a bench finds its peer by broadcast and times TCP to it; the peer, `osbench-peer`, echoes, sinks and sources for it on any host." +description = "osbench's LAN measurement, both ends: a bench finds its peer by broadcast and times TCP to it; the peer, `osbench-peer`, on the other bench's Linux, echoes, sinks and sources for it." version = "0.1.0" edition = "2021" license = "MIT OR Apache-2.0" publish = false # A package of its own because osbench builds for ToyOS and Linux alone, and -# the peer runs on whatever machine stands on the other end of the link. +# a host tests this part of it. [lib] doctest = false @@ -15,3 +15,7 @@ doctest = false [[bin]] name = "osbench-peer" path = "src/main.rs" + +# The interface holding an address, which `link` describes from sysfs. +[target.'cfg(target_os = "linux")'.dependencies] +libc = "0.2" diff --git a/tests/guest/osbench/lan/src/lib.rs b/tests/guest/osbench/lan/src/lib.rs index d1d7f832e1..944740362e 100644 --- a/tests/guest/osbench/lan/src/lib.rs +++ b/tests/guest/osbench/lan/src/lib.rs @@ -1,9 +1,10 @@ //! A bench's link to a peer, timed from the bench. //! //! **The bench finds the peer and opens every connection**: it broadcasts -//! [`HELLO`] to [`PORT`] and takes the source of the first [`ANSWER`] as its +//! [`HELLO`] to [`PORT`] and takes the source of the first answer as its //! peer, so neither side is configured with an address and the bench accepts -//! nothing. The peer only answers; every figure is the bench's clock. +//! nothing. The answer says which of its links the peer answered over +//! ([`link`]). The peer only answers; every figure is the bench's clock. //! //! Each TCP connection to the peer's [`PORT`] opens with one [`Request`]: //! echo fixed-size messages, sink everything to end-of-file and answer the @@ -15,12 +16,16 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; +pub mod link; + /// The peer's UDP port for discovery and its TCP port for requests. pub const PORT: u16 = 47_321; -/// What a bench broadcasts, and what the peer answers it with. -pub const HELLO: &[u8] = b"osbench-lan/1 hello"; -pub const ANSWER: &[u8] = b"osbench-lan/1 peer"; +/// What a bench broadcasts, and what the peer's answer starts with: the rest +/// is its link's description, at most [`MAX_SAID`] bytes of printable ASCII. +pub const HELLO: &[u8] = b"osbench-lan/2 hello"; +pub const ANSWER: &[u8] = b"osbench-lan/2 peer "; +pub const MAX_SAID: usize = 160; /// How long either end waits on one read or write before it gives the /// connection up. @@ -94,16 +99,31 @@ impl Request { } } -/// The bench's end: broadcast [`HELLO`] to `to` once a second until an -/// [`ANSWER`] comes or `patience` has passed, and answer where it came from. -/// `Ok(None)` is no answer; an error is a socket the system would not give. -/// Sends refused while the link has no address yet are tried again. -pub fn discover(to: SocketAddr, patience: Duration) -> io::Result> { +/// A peer that answered: where from, and what it said of its link. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Found { + pub ip: IpAddr, + pub link: String, +} + +/// The description an answer carries, or `None` for a datagram that is no +/// answer. +pub fn answered(datagram: &[u8]) -> Option<&str> { + let said = datagram.strip_prefix(ANSWER)?; + let printable = said.iter().all(|b| b.is_ascii_graphic() || *b == b' '); + (said.len() <= MAX_SAID && printable).then(|| std::str::from_utf8(said).expect("ASCII")) +} + +/// The bench's end: broadcast [`HELLO`] to `to` once a second until an answer +/// comes or `patience` has passed. `Ok(None)` is no answer; an error is a +/// socket the system would not give. Sends refused while the link has no +/// address yet are tried again. +pub fn discover(to: SocketAddr, patience: Duration) -> io::Result> { let socket = UdpSocket::bind((Ipv4Addr::UNSPECIFIED, 0))?; socket.set_broadcast(true)?; socket.set_read_timeout(Some(Duration::from_secs(1)))?; let start = Instant::now(); - let mut buf = [0u8; 64]; + let mut buf = [0u8; ANSWER.len() + MAX_SAID + 1]; while start.elapsed() < patience { if socket.send_to(HELLO, to).is_err() { std::thread::sleep(Duration::from_secs(1)); @@ -112,8 +132,11 @@ pub fn discover(to: SocketAddr, patience: Duration) -> io::Result let asked = Instant::now(); while asked.elapsed() < Duration::from_secs(1) { match socket.recv_from(&mut buf) { - Ok((n, from)) if &buf[..n] == ANSWER => return Ok(Some(from.ip())), - Ok(_) => {} + Ok((n, from)) => { + if let Some(said) = answered(&buf[..n]) { + return Ok(Some(Found { ip: from.ip(), link: said.to_string() })); + } + } Err(e) if matches!(e.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut) => break, Err(e) => return Err(e), } @@ -225,13 +248,17 @@ impl Echo { /// The most connections a peer serves at once. pub const MAX_SESSIONS: usize = 8; -/// The peer's end of discovery: answer every [`HELLO`] on `socket`, for good. +/// The peer's end of discovery: answer every [`HELLO`] on `socket` with the +/// link it answers over, for good. pub fn answer(socket: &UdpSocket) -> io::Result<()> { let mut buf = [0u8; 64]; loop { let (n, from) = socket.recv_from(&mut buf)?; if &buf[..n] == HELLO { - socket.send_to(ANSWER, from)?; + let said = link::toward(from.ip()).map_or_else(|why| format!("unread: {why}"), |(_, said)| said); + let mut reply = ANSWER.to_vec(); + reply.extend(said.bytes().filter(|b| b.is_ascii_graphic() || *b == b' ').take(MAX_SAID)); + socket.send_to(&reply, from)?; } } } @@ -346,8 +373,9 @@ mod tests { #[test] fn a_bench_finds_its_peer_and_measures_each_request() { let (discovery, requests) = peer(); - let found = discover(discovery, Duration::from_secs(5)).unwrap(); - assert_eq!(found, Some(IpAddr::V4(Ipv4Addr::LOCALHOST))); + let found = discover(discovery, Duration::from_secs(5)).unwrap().expect("the peer answers"); + assert_eq!(found.ip, IpAddr::V4(Ipv4Addr::LOCALHOST)); + assert!(!found.link.is_empty(), "the answer says something of the peer's link"); let peer = Peer(requests); peer.connect().unwrap(); @@ -361,6 +389,21 @@ mod tests { assert!(down.bytes >= CHUNK as u64, "{down:?}"); } + #[test] + fn an_answer_carries_the_peers_link_and_nothing_else_is_one() { + assert_eq!(answered(b"osbench-lan/2 peer wired driver=e1000e speed_mbps=1000"), Some("wired driver=e1000e speed_mbps=1000")); + assert_eq!(answered(b"osbench-lan/2 peer "), Some("")); + assert_eq!(answered(b"osbench-lan/1 peer"), None); + assert_eq!(answered(b"osbench-lan/2 hello"), None); + assert_eq!(answered(b"osbench-lan/2 peer wired\n"), None); + assert_eq!(answered("osbench-lan/2 peer รค".as_bytes()), None); + let mut long = ANSWER.to_vec(); + long.extend([b'x'; MAX_SAID]); + assert!(answered(&long).is_some()); + long.push(b'x'); + assert_eq!(answered(&long), None); + } + #[test] fn no_answer_is_none_once_patience_runs_out() { let silent = UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); diff --git a/tests/guest/osbench/lan/src/link.rs b/tests/guest/osbench/lan/src/link.rs new file mode 100644 index 0000000000..3f9d091595 --- /dev/null +++ b/tests/guest/osbench/lan/src/link.rs @@ -0,0 +1,125 @@ +//! Which of this machine's links one of its own addresses is on, described +//! as Linux's sysfs and `/proc/net/wireless` give it. Every other system +//! answers that it reads none. A description names no interface or address. + +use std::net::{IpAddr, Ipv4Addr, UdpSocket}; + +/// The links a result can be named for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Link { + Wired, + Wifi, +} + +impl Link { + pub fn name(self) -> &'static str { + match self { + Link::Wired => "wired", + Link::Wifi => "wifi", + } + } +} + +/// The link a datagram to `peer` would leave by. Connecting a UDP socket +/// chooses its route and sends nothing. +pub fn toward(peer: IpAddr) -> Result<(Link, String), String> { + let probe = UdpSocket::bind((Ipv4Addr::UNSPECIFIED, 0)).map_err(|e| format!("no socket: {e}"))?; + probe.connect((peer, 9)).map_err(|e| format!("no route: {e}"))?; + of(probe.local_addr().map_err(|e| format!("no local address: {e}"))?.ip()) +} + +/// The link holding `local`, and its description: kind, driver, and the +/// wired link's speed or the wireless one's signal. +#[cfg(target_os = "linux")] +pub fn of(local: IpAddr) -> Result<(Link, String), String> { + let name = linux::interface_holding(local)?; + let at = std::path::Path::new("/sys/class/net").join(&name); + let Ok(driver) = std::fs::read_link(at.join("device/driver")) else { + return Err("the address is on an interface no device drives".into()); + }; + let driver = driver.file_name().map_or_else(|| "unnamed".into(), |n| n.to_string_lossy().into_owned()); + if at.join("wireless").exists() || at.join("phy80211").exists() { + let signal = std::fs::read_to_string("/proc/net/wireless") + .ok() + .and_then(|table| signal_dbm(&table, &name)) + .map_or_else(|| "unread".into(), |dbm| dbm.to_string()); + Ok((Link::Wifi, format!("wireless driver={driver} signal_dbm={signal}"))) + } else { + let speed = std::fs::read_to_string(at.join("speed")).map_or_else(|_| "unread".into(), |s| s.trim().to_string()); + Ok((Link::Wired, format!("wired driver={driver} speed_mbps={speed}"))) + } +} + +#[cfg(not(target_os = "linux"))] +pub fn of(_local: IpAddr) -> Result<(Link, String), String> { + Err("this system's links are not read".into()) +} + +/// `interface`'s signal level in dBm from a `/proc/net/wireless` table, whose +/// rows are `: ...`, each number +/// perhaps ending in `.`. +pub fn signal_dbm(table: &str, interface: &str) -> Option { + table.lines().find_map(|line| { + let (name, fields) = line.trim_start().split_once(':')?; + if name != interface { + return None; + } + fields.split_whitespace().nth(2)?.trim_end_matches('.').parse().ok() + }) +} + +#[cfg(target_os = "linux")] +mod linux { + use std::net::IpAddr; + + /// The name of the interface holding `local`. + pub fn interface_holding(local: IpAddr) -> Result { + let IpAddr::V4(local) = local else { return Err("the address is IPv6".into()) }; + let mut list: *mut libc::ifaddrs = std::ptr::null_mut(); + // SAFETY: `list` is a valid out-pointer; the list is freed below. + if unsafe { libc::getifaddrs(&mut list) } != 0 { + return Err(format!("getifaddrs: {}", std::io::Error::last_os_error())); + } + let mut found = None; + let mut at = list; + while !at.is_null() { + // SAFETY: a node of the list getifaddrs made, alive until it is freed. + let ifa = unsafe { &*at }; + // SAFETY: a non-null `ifa_addr` of family AF_INET is a sockaddr_in. + let v4 = (!ifa.ifa_addr.is_null() && unsafe { (*ifa.ifa_addr).sa_family } == libc::AF_INET as libc::sa_family_t) + .then(|| unsafe { (*ifa.ifa_addr.cast::()).sin_addr.s_addr }); + if v4 == Some(u32::from(local).to_be()) { + // SAFETY: an interface's name is a NUL-terminated string. + found = Some(unsafe { std::ffi::CStr::from_ptr(ifa.ifa_name) }.to_string_lossy().into_owned()); + break; + } + at = ifa.ifa_next; + } + // SAFETY: the list getifaddrs made, read no more. + unsafe { libc::freeifaddrs(list) }; + found.ok_or_else(|| "no interface holds the address".into()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const TABLE: &str = "Inter-| sta-| Quality | Discarded packets | Missed | WE + face | tus | link level noise | nwid crypt frag retry misc | beacon | 22 +wlp1s0: 0000 58. -52. -256 0 0 0 0 0 0 + wlan1: 0000 70 -40 -256 0 0 0 0 0 0 +"; + + #[test] + fn a_wireless_interfaces_signal_is_its_level_column() { + assert_eq!(signal_dbm(TABLE, "wlp1s0"), Some(-52)); + assert_eq!(signal_dbm(TABLE, "wlan1"), Some(-40)); + } + + #[test] + fn an_interface_the_table_does_not_list_has_no_signal() { + assert_eq!(signal_dbm(TABLE, "wlp1"), None); + assert_eq!(signal_dbm("wlp1s0: 0000 58.", "wlp1s0"), None); + } +} diff --git a/tests/guest/osbench/lan/src/main.rs b/tests/guest/osbench/lan/src/main.rs index 80f6ba4823..c8d580b231 100644 --- a/tests/guest/osbench/lan/src/main.rs +++ b/tests/guest/osbench/lan/src/main.rs @@ -1,5 +1,5 @@ -//! `osbench-peer`: the far end of a bench's `osbench lan`, on any host on the -//! bench's link. It answers discovery and serves requests on +//! `osbench-peer`: the far end of a bench's `osbench lan`, the fixed reference +//! on the other bench's Linux. It answers discovery and serves requests on //! [`osbench_lan::PORT`] until it is stopped, and names no address. use std::net::{Ipv4Addr, TcpListener, UdpSocket}; diff --git a/tests/guest/osbench/src/internet.rs b/tests/guest/osbench/src/internet.rs new file mode 100644 index 0000000000..ce73f3e0ad --- /dev/null +++ b/tests/guest/osbench/src/internet.rs @@ -0,0 +1,136 @@ +//! Exploratory rows over the public internet, run only by name and never by +//! `all` or a judged boot, since no automated test depends on the public +//! internet: evidence of a path neither system owns, not a regression test. +//! +//! The one endpoint is Cloudflare's speed test at [`HOST`], over HTTPS: +//! `__down?bytes=N` answers N bytes and `__up` takes a POST's body, the +//! endpoints Cloudflare's own measurement engine drives +//! (). It refuses a download of +//! 100 MB. The round trip asks the same host's `cdn-cgi/trace`, which its edge +//! answers without the speed test's own processing. Answers name the client's +//! address and place, and nothing here keeps or prints them. TLS is rustls on +//! toyos-crypto's provider on both systems, so the cryptography is the same +//! code. + +use std::io::{ErrorKind, Read}; +use std::net::{Ipv4Addr, SocketAddr, TcpStream, ToSocketAddrs, UdpSocket}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use crate::measure::each; +use crate::report::{condition, refuse, unsupported}; + +const HOST: &str = "speed.cloudflare.com"; +const USER_AGENT: &str = "toyos-build (https://github.com/ToyOSOrg/ToyOS)"; +/// One download's bytes and one upload's, each timed whole: a slow uplink's +/// rate needs more resolution than whole MB/s. +const DOWN: u64 = 25_000_000; +const UP: usize = 10_000_000; +const ROUNDS: usize = 5; +/// The most a short answer is read to. +const ANSWER_MAX: u64 = 64 * 1024; +/// How long one connection or request may take before it is given up. +const PATIENCE: Duration = Duration::from_secs(30); +/// How long the address may take to arrive, as `lan`'s peer may. +const LEASE: Duration = Duration::from_secs(60); + +const BENCHES: [&str; 4] = ["internet.tcp.connect", "internet.https.rtt", "internet.https.download", "internet.https.upload"]; + +pub fn suite() { + condition("internet.endpoint", HOST); + if let Err(e) = UdpSocket::bind((Ipv4Addr::UNSPECIFIED, 0)) { + if cfg!(target_os = "toyos") && e.kind() == ErrorKind::NotConnected { + return BENCHES.iter().for_each(|b| unsupported(b, format!("no network stack answers: {e}"))); + } + return refuse("internet", format!("no socket: {e}")); + } + let addr = match resolve() { + Ok(addr) => addr, + Err(why) => return refuse("internet", why), + }; + match TcpStream::connect_timeout(&addr, PATIENCE).and_then(|s| s.local_addr()) { + Ok(local) => match crate::sys::link_of(local.ip()) { + Ok((_, said)) => condition("internet.link", said), + Err(why) => return refuse("internet", why), + }, + Err(e) => return refuse("internet", format!("{HOST} took no connection: {e}")), + } + let [connect, rtt, download, upload] = BENCHES; + + each(connect, 2, 20, &[], |_| TcpStream::connect_timeout(&addr, PATIENCE).map(drop).map_err(|e| format!("connect: {e}"))); + + let agent = agent(); + let url = |path: &str| format!("https://{HOST}/{path}"); + let get = |path: &str, most: u64| -> Result { + let mut answer = agent.get(url(path)).call().map_err(|e| format!("GET: {e}"))?; + drain(answer.body_mut(), most) + }; + each(rtt, 5, 50, &[("path", "cdn-cgi/trace".into())], |_| get("cdn-cgi/trace", ANSWER_MAX).map(drop)); + let down = format!("__down?bytes={DOWN}"); + each(download, 1, ROUNDS, &[("bytes", DOWN.to_string())], |_| match get(&down, DOWN)? { + DOWN => Ok(()), + n => Err(format!("the download ended at {n} of {DOWN} bytes")), + }); + let sent = vec![0x5au8; UP]; + each(upload, 1, ROUNDS, &[("bytes", UP.to_string())], |_| { + let mut answer = agent.post(url("__up")).send(&sent[..]).map_err(|e| format!("POST: {e}"))?; + drain(answer.body_mut(), ANSWER_MAX).map(drop) + }); +} + +/// An answer's body read to its end, unkept, so its connection serves the +/// next request: its length, refused past `most`. +fn drain(body: &mut ureq::Body, most: u64) -> Result { + match std::io::copy(&mut body.as_reader().take(most + 1), &mut std::io::sink()) { + Ok(n) if n <= most => Ok(n), + Ok(_) => Err(format!("an answer ran past {most} bytes")), + Err(e) => Err(format!("an answer's body: {e}")), + } +} + +/// [`HOST`]'s first IPv4 address. A ToyOS netstack answers `NotConnected` +/// until its lease lands and tells no one when it does, so a lookup it +/// answers so is asked again each second for up to [`LEASE`]. +fn resolve() -> Result { + let start = Instant::now(); + loop { + match (HOST, 443).to_socket_addrs() { + Ok(mut addrs) => return addrs.find(SocketAddr::is_ipv4).ok_or_else(|| format!("{HOST} has no IPv4 address")), + Err(e) if e.kind() == ErrorKind::NotConnected && start.elapsed() < LEASE => std::thread::sleep(Duration::from_secs(1)), + Err(e) => return Err(format!("{HOST} did not resolve: {e}")), + } + } +} + +/// One agent, so every request after the first rides a connection it keeps. +fn agent() -> ureq::Agent { + let tls = ureq::tls::TlsConfig::builder().unversioned_rustls_crypto_provider(provider()).build(); + ureq::Agent::config_builder() + .user_agent(USER_AGENT) + .timeout_global(Some(PATIENCE)) + .tls_config(tls) + .build() + .new_agent() +} + +/// The system's random bytes, which the provider's key exchanges draw. +#[derive(Debug)] +struct SystemRandom; + +impl rustls::crypto::SecureRandom for SystemRandom { + fn fill(&self, bytes: &mut [u8]) -> Result<(), rustls::crypto::GetRandomFailed> { + getrandom::fill(bytes).map_err(|_| rustls::crypto::GetRandomFailed) + } +} + +static TLS: toyos_crypto::rustls_provider::Backend = toyos_crypto::rustls_provider::Backend::new(&SystemRandom); + +/// toyos-crypto's provider, exchanging keys on X25519 alone, hybrid or not, +/// as the build's own requests do: no P-256 or P-384 secret meets a server's +/// input while the T14 measures those curves leaking +/// (`issues/toyos-crypto-is-in-the-tree-on-par-with-ring-and-secure.md`). +fn provider() -> Arc { + let mut provider = TLS.provider(); + provider.kx_groups.retain(|group| matches!(group.name(), rustls::NamedGroup::X25519MLKEM768 | rustls::NamedGroup::X25519)); + Arc::new(provider) +} diff --git a/tests/guest/osbench/src/lan.rs b/tests/guest/osbench/src/lan.rs index e0bbc1c8f6..7554a6ec74 100644 --- a/tests/guest/osbench/src/lan.rs +++ b/tests/guest/osbench/src/lan.rs @@ -1,40 +1,28 @@ -//! The machine's link to a peer on its network, through std's `net` on both -//! systems (`osbench-lan`): this machine broadcasts for an `osbench-peer`, -//! then times TCP to it. Every connection is this machine's own, so a system -//! that accepts none from its network is measured all the same. +//! The machine's link to the other bench, through std's `net` on both +//! systems (`osbench-lan`): this machine broadcasts for the `osbench-peer` +//! that bench's Linux runs as the fixed reference, then times TCP to it. +//! Every connection is this machine's own, so a system that accepts none +//! from its network is measured all the same. //! //! Each result is named for the link that carried it, `lan.wired.*` or //! `lan.wifi.*`, and the link not measured says why: a Linux run measures //! whichever link reaches the peer, and ToyOS drives Ethernet alone -//! (`issues/there-is-no-wifi.md`). No address is printed. +//! (`issues/there-is-no-wifi.md`). `lan.link` and `lan.peer_link` describe +//! both ends' links; no address is printed. use std::net::{Ipv4Addr, SocketAddr}; use std::time::{Duration, Instant}; +use osbench_lan::link::Link; use osbench_lan::Peer; use crate::report::{condition, refuse, result, unmeasured, unsupported}; /// How long a peer is asked for, which a boot's lease lands inside too. const DISCOVERY: Duration = Duration::from_secs(60); -/// Each transfer's span. -const SPAN: Duration = Duration::from_secs(5); - -/// The links a result can be named for. -#[derive(Clone, Copy, PartialEq, Eq)] -pub enum Link { - Wired, - Wifi, -} - -impl Link { - fn name(self) -> &'static str { - match self { - Link::Wired => "wired", - Link::Wifi => "wifi", - } - } -} +/// Each transfer's span, and the rounds timed after one unread. +const SPAN: Duration = Duration::from_secs(3); +const ROUNDS: usize = 5; fn benches(link: Link) -> [String; 4] { ["tcp.connect", "tcp.rtt.64", "tcp.upload", "tcp.download"].map(|b| format!("lan.{}.{b}", link.name())) @@ -48,8 +36,8 @@ pub fn suite() { } let broadcast = SocketAddr::from((Ipv4Addr::BROADCAST, osbench_lan::PORT)); let started = Instant::now(); - let ip = match osbench_lan::discover(broadcast, DISCOVERY) { - Ok(Some(ip)) => ip, + let found = match osbench_lan::discover(broadcast, DISCOVERY) { + Ok(Some(found)) => found, Ok(None) => return refuse("lan", format!("no osbench-peer answered on the link within {DISCOVERY:?}")), Err(e) if cfg!(target_os = "toyos") && e.kind() == std::io::ErrorKind::NotConnected => { return benches(Link::Wired).iter().for_each(|b| unsupported(b, format!("no network stack answers: {e}"))); @@ -57,7 +45,8 @@ pub fn suite() { Err(e) => return refuse("lan", format!("no socket to look for a peer with: {e}")), }; condition("lan.peer_found_after_ms", started.elapsed().as_millis()); - let peer = Peer(SocketAddr::new(ip, osbench_lan::PORT)); + condition("lan.peer_link", &found.link); + let peer = Peer(SocketAddr::new(found.ip, osbench_lan::PORT)); let link = match peer.local() { Ok(local) => crate::sys::link_of(local.ip()), Err(e) => return refuse("lan", format!("the peer answered and took no connection: {e}")), @@ -89,7 +78,7 @@ fn measure(peer: &Peer, [connect, rtt, upload, download]: &[String; 4]) { let rounds = || -> Result, String> { run(peer, SPAN).map_err(|e| e.to_string())?; crate::clock::settle(); - (0..3) + (0..ROUNDS) .map(|_| { let moved = run(peer, SPAN).map_err(|e| e.to_string())?; Ok(moved.bytes * 1000 / moved.ns) diff --git a/tests/guest/osbench/src/main.rs b/tests/guest/osbench/src/main.rs index b6ba242e2b..d9e64686c2 100644 --- a/tests/guest/osbench/src/main.rs +++ b/tests/guest/osbench/src/main.rs @@ -1,9 +1,10 @@ //! `osbench`: the same system workloads, built from one source for ToyOS //! and for Linux, so one machine's two systems can be compared. //! -//! `osbench ` runs a group, or `all` of them in [`GROUPS`]' order. A -//! ToyOS job list passes no arguments, so there the group is the name the -//! program was started under, `osbench_`. Every other command is one +//! `osbench ` runs a group, or `all` of them in [`GROUPS`]' order; the +//! groups in [`BY_NAME`] need another party and run only by name. A ToyOS job +//! list passes no arguments, so there the group is the name the program was +//! started under, `osbench_`. Every other command is one //! this program gives its own children. //! //! Every result is one line of `key=value` words (`report`): a distribution @@ -30,8 +31,7 @@ //! That nightly is the build's stage-0 compiler (`build/sysroot.rs`'s //! `STAGE0_CARGO`, which this follows), the upstream one nearest ToyOS's own. //! On Linux `nvme` and `energy` read what only root may (the raw namespace, -//! the package's energy counter) and say so unprivileged; `lan` needs an -//! `osbench-peer` (`lan/`) running on another machine on the link. +//! the package's energy counter) and say so unprivileged. //! With the `toyos` profile every guest binary is built with, osbench's own //! code is generated alike for both; std's platform layer and the C library //! beneath it (musl's on Linux) are each system's own. @@ -44,6 +44,7 @@ mod energy; mod fair; mod files; mod identity; +mod internet; mod ipc; mod lan; mod measure; @@ -141,12 +142,12 @@ fn files() { } } -/// Every group, in the order `all` and the ToyOS job list run them: `boot` -/// first, since it reads how long the boot took to reach it; `energy` while -/// the machine is still quiet; `crunch`, which heats it, after every other -/// measurement; and `persist` last, since it writes the volume a ToyOS -/// boot's log is on. -const GROUPS: [(&str, fn()); 18] = [ +/// Every group the machine alone runs, in the order `all` and the ToyOS job +/// list run them: `boot` first, since it reads how long the boot took to +/// reach it; `energy` while the machine is still quiet; `crunch`, which heats +/// it, after every other measurement; and `persist` last, since it writes the +/// volume a ToyOS boot's log is on. +const GROUPS: [(&str, fn()); 17] = [ ("boot", boot::suite), ("identity", identity::suite), ("energy", energy::suite), @@ -162,11 +163,14 @@ const GROUPS: [(&str, fn()); 18] = [ ("work", work::suite), ("tail", tail::suite), ("nvme", nvme::suite), - ("lan", lan::suite), ("crunch", crunch::suite), ("persist", storage::suite), ]; +/// The groups that need another party: `lan` the `osbench-peer` on the other +/// bench, `internet` the public internet. +const BY_NAME: [(&str, fn()); 2] = [("lan", lan::suite), ("internet", internet::suite)]; + /// A group between its first and last lines, after the machine's conditions /// and the clock its CPU starts at. fn group(name: &str, run: fn()) { @@ -194,10 +198,10 @@ fn main() { "fair-helper" => return fair::helper(rest), "ipc-serve" => return ipc::serve(), "cc" => return work::cc(rest), - named => match GROUPS.iter().find(|(name, _)| *name == named) { + named => match GROUPS.iter().chain(&BY_NAME).find(|(name, _)| *name == named) { Some((name, run)) => group(name, *run), None => { - let names: Vec<&str> = GROUPS.iter().map(|(name, _)| *name).collect(); + let names: Vec<&str> = GROUPS.iter().chain(&BY_NAME).map(|(name, _)| *name).collect(); eprintln!("osbench: no command {named:?}; the groups are {} and all", names.join(", ")); std::process::exit(2); } diff --git a/tests/guest/osbench/src/sys/linux.rs b/tests/guest/osbench/src/sys/linux.rs index c2da1440b9..0e8372a662 100644 --- a/tests/guest/osbench/src/sys/linux.rs +++ b/tests/guest/osbench/src/sys/linux.rs @@ -583,49 +583,7 @@ fn backing_disk(mut device: std::path::PathBuf) -> (std::path::PathBuf, bool) { (device, mapped) } -/// The interface holding `local`, off `getifaddrs`, and whether it is a -/// wireless one, as sysfs says: its kind, and its driver and speed. The -/// interface's name and addresses are never printed. -pub fn link_of(local: std::net::IpAddr) -> Result<(crate::lan::Link, String), String> { - let name = interface_holding(local)?; - let at = std::path::Path::new("/sys/class/net").join(&name); - let wireless = at.join("wireless").exists() || at.join("phy80211").exists(); - let driver = fs::read_link(at.join("device/driver")) - .ok() - .and_then(|p| p.file_name().map(|n| n.to_string_lossy().into_owned())) - .unwrap_or_else(|| "none".into()); - let speed = read(&at.join("speed").to_string_lossy()).unwrap_or_else(|| "unknown".into()); - let link = if wireless { crate::lan::Link::Wifi } else { crate::lan::Link::Wired }; - let kind = if wireless { "wireless" } else { "wired" }; - Ok((link, format!("{kind} driver={driver} speed_mbps={speed}"))) -} - -fn interface_holding(local: std::net::IpAddr) -> Result { - let std::net::IpAddr::V4(local) = local else { return Err("the connection left from an IPv6 address".into()) }; - let mut list: *mut libc::ifaddrs = std::ptr::null_mut(); - // SAFETY: `list` is a valid out-pointer; the list is freed below. - if unsafe { libc::getifaddrs(&mut list) } != 0 { - return Err(format!("getifaddrs: {}", std::io::Error::last_os_error())); - } - let mut found = None; - let mut at = list; - while !at.is_null() { - // SAFETY: a node of the list getifaddrs made, alive until it is freed. - let ifa = unsafe { &*at }; - // SAFETY: a non-null `ifa_addr` of family AF_INET is a sockaddr_in. - let v4 = (!ifa.ifa_addr.is_null() && unsafe { (*ifa.ifa_addr).sa_family } == libc::AF_INET as libc::sa_family_t) - .then(|| unsafe { (*ifa.ifa_addr.cast::()).sin_addr.s_addr }); - if v4 == Some(u32::from(local).to_be()) { - // SAFETY: an interface's name is a NUL-terminated string. - found = Some(unsafe { std::ffi::CStr::from_ptr(ifa.ifa_name) }.to_string_lossy().into_owned()); - break; - } - at = ifa.ifa_next; - } - // SAFETY: the list getifaddrs made, read no more. - unsafe { libc::freeifaddrs(list) }; - found.ok_or_else(|| "no interface holds the address the connection left from".into()) -} +pub use osbench_lan::link::of as link_of; /// The package's temperature off hwmon: `coretemp`'s `Package id 0`, or /// `k10temp`'s `Tdie` where it has one and its `Tctl` otherwise. diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index b2fb54efcd..972bd13451 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -364,8 +364,8 @@ pub fn persistent_dir() -> Option<(String, String)> { } /// What carries a connection on ToyOS: netstack drives one card, Ethernet. -pub fn link_of(_local: std::net::IpAddr) -> Result<(crate::lan::Link, String), String> { - Ok((crate::lan::Link::Wired, "netstack's one Ethernet card".into())) +pub fn link_of(_local: std::net::IpAddr) -> Result<(osbench_lan::link::Link, String), String> { + Ok((osbench_lan::link::Link::Wired, "netstack's one Ethernet card".into())) } /// ToyOS reads no temperature: the kernel counts none diff --git a/tests/images/benchcase.toml b/tests/images/benchcase.toml index 8c8464922d..f810120f75 100644 --- a/tests/images/benchcase.toml +++ b/tests/images/benchcase.toml @@ -2,8 +2,9 @@ # on the bench's own Linux (`tests/guest/osbench`). # # Only what a job list needs to run and to reach the log on the boot stick, the -# ACPI server, and netstack for `osbench lan`: no compositor or sound, so little -# but the measurement asks for a CPU. DATA is memory here, the stick having no DATA +# ACPI server, and netstack for `osbench lan` and `osbench internet`, each its +# own metal row (`tests/toyos.rs`): no compositor or sound, so little but the +# measurement asks for a CPU. DATA is memory here, the stick having no DATA # partition, and the log volume is the one persistent volume a job can write # (`osbench persist`). # @@ -31,21 +32,21 @@ devices = ["acpi"] receives = ["power"] # netstack on the T14's wired I219, the only card a bench has that ToyOS drives: -# `osbench lan` reaches its peer through it. On the AMD bench, which has none, -# netstack leaves and `lan` says so. +# `osbench lan` reaches its peer and `osbench internet` the internet through it. +# On the AMD bench, which has none, netstack leaves and both say so. [programs.netstack] service = true serves = ["netstack"] devices = ["pci:02:00:00/8086:15fc"] # The order `osbench all` runs the groups in. The bound is the metal row's, its -# 480 s on the 60 s every list has (`tests/toyos.rs`), restated for a boot of +# 370 s on the 60 s every list has (`tests/toyos.rs`), restated for a boot of # this config built without the row; a derived metal config states its own. [programs.test-runner] receives = ["power", "netstack"] syscap = ["jobs", "counters", "trace", "roster", "inventory", "dup"] args = [ - "--bound-ms=540000", + "--bound-ms=430000", "osbench_boot", "osbench_identity", "osbench_energy", @@ -61,7 +62,6 @@ args = [ "osbench_work", "osbench_tail", "osbench_nvme", - "osbench_lan", "osbench_crunch", "osbench_persist", "reboot", @@ -88,6 +88,7 @@ args = [ "bin/osbench_tail" = "/system/bin/osbench" "bin/osbench_nvme" = "/system/bin/osbench" "bin/osbench_lan" = "/system/bin/osbench" +"bin/osbench_internet" = "/system/bin/osbench" "bin/osbench_crunch" = "/system/bin/osbench" "bin/osbench_persist" = "/system/bin/osbench" "bin/reboot" = "/system/bin/toybox" diff --git a/tests/toyos.rs b/tests/toyos.rs index d5afe06f45..49aa92f297 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -1093,7 +1093,22 @@ const METAL: &[(&str, metal::Metal)] = &[ // holds only that every group ran to its end. The T14 and the AMD bench. // One boot runs the whole suite, so three readbacks are three runs. "osbench", - metal::Metal { arms: BENCHCASE, judge: |b| osbench_ran(b[0]) }, + metal::Metal { arms: BENCHCASE, judge: |b| osbench_ran(b[0], &OSBENCH_GROUPS) }, + ), + ( + // `osbench lan` against the `osbench-peer` the other bench's Linux runs, + // compared with the same program under this bench's Linux on the same + // path. A pass names it: the other bench has to be running the peer. + // The T14 over its I219; the AMD bench has no card ToyOS drives. + "osbench_lan", + metal::Metal { arms: OSBENCH_LAN, judge: |b| osbench_ran(b[0], &["osbench_lan"]) }, + ), + ( + // `osbench internet`: exploratory evidence over the public internet, + // which no automated test depends on, so only a pass that names it runs + // it. The T14; the AMD bench has no card ToyOS drives. + "osbench_internet", + metal::Metal { arms: OSBENCH_INTERNET, judge: |b| osbench_ran(b[0], &["osbench_internet"]) }, ), // ---- tests/images/jobcase.toml, each image armed to end its boot its own way ---- ( @@ -1587,7 +1602,7 @@ const TESTCASES: &[metal::Arm] = &[metal::once( /// `osbench`'s groups, each a job named by its symlink, in `osbench all`'s /// order: `boot` first, which reads how long the boot took to reach it, and /// `persist` last, which writes the log volume. -const OSBENCH_GROUPS: [&str; 18] = [ +const OSBENCH_GROUPS: [&str; 17] = [ "osbench_boot", "osbench_identity", "osbench_energy", @@ -1603,7 +1618,6 @@ const OSBENCH_GROUPS: [&str; 18] = [ "osbench_work", "osbench_tail", "osbench_nvme", - "osbench_lan", "osbench_crunch", "osbench_persist", ]; @@ -1612,21 +1626,37 @@ const OSBENCH_GROUPS: [&str; 18] = [ /// 100 s of wakes and spinners. Six boots of `e5573b7fe` ended their last job /// at 175 to 191 s on five, and at 244 s on the T14's first, where one /// served 64 MiB write stalled and `files` took 80 s against 13. The groups -/// since add at most 160 s by their own spans: `lan`'s 60 s looking for its -/// peer and 50 s measuring, `crunch`'s one-thread 16 s (ToyOS reads no -/// temperature, so its all-core phase does not run), and under 30 s of memcpy -/// sizes, pipe sizes and 1 ms timers. So a list bound of 540 s leaves that -/// stall 136 s. +/// since add at most 50 s by their own spans: `crunch`'s one-thread 16 s +/// (ToyOS reads no temperature, so its all-core phase does not run), and +/// under 30 s of memcpy sizes, pipe sizes and 1 ms timers. So a list bound of +/// 430 s leaves that stall 136 s. const BENCHCASE: &[metal::Arm] = &[metal::Arm { - adds_ms: 480_000, + adds_ms: 370_000, ..metal::once("benchcase", "tests/images/benchcase.toml", &[], &OSBENCH_GROUPS) }]; -/// Every group exited 0, which `osbench` does only once each of its +/// The two groups that need another party share one boot when one pass names +/// both. `lan` looks for its peer up to 60 s, inside which the boot's lease +/// lands, then measures for under 60 s over a Wi-Fi hop: 2,155 round trips +/// at up to 10 ms and twelve 3 s transfers, so 150 s leaves it 30 s. +/// `internet` measures for under 95 s down to 3 MB/s and up to 1.5 MB/s: six +/// 25 MB downloads, six 10 MB uploads and 77 short exchanges. +const OSBENCH_LAN: &[metal::Arm] = &[metal::Arm { + adds_ms: 90_000, + named_only: true, + ..metal::once("netbenchcase", "tests/images/benchcase.toml", &[], &["osbench_lan"]) +}]; +const OSBENCH_INTERNET: &[metal::Arm] = &[metal::Arm { + adds_ms: 60_000, + named_only: true, + ..metal::once("netbenchcase", "tests/images/benchcase.toml", &[], &["osbench_internet"]) +}]; + +/// Every group's job exited 0, which `osbench` does only once each of its /// benchmarks measured, and said it was done. -fn osbench_ran(boot: &metal::Readback) -> Result<(), String> { +fn osbench_ran(boot: &metal::Readback, jobs: &[&str]) -> Result<(), String> { in_acpi_mode_before_any_job(boot)?; - for job in OSBENCH_GROUPS { + for &job in jobs { boot.job_passed(job)?; let group = job.strip_prefix("osbench_").expect("a group's job"); boot.log().must_say(&format!("group={group} done"))?; From d99ef5a3535fe01167a72ff8bd70670374fff97a Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 24 Oct 2026 13:20:18 +0200 Subject: [PATCH 2/5] review r1: osbench-compare judges only known directions, says when runs are too few, and the net rows' bounds add up - A verdict says better or worse only for `ns` (lower) and `MB/s` (higher); any other unit, a power or a clock, is told which way ToyOS differs. Counts whose complete separation cannot reach 0.05, 4 runs against 3 among them, say "too few runs to resolve (least p ...)" rather than "not resolved". A median of 0 has no range. Each table says its p is per row and uncorrected. - `internet.*` rows are compared only across runs that left by one kind of link, as `lan.*` rows are across one kind of peer link. ToyOS's link description now starts with its kind, `wired`. - The network rows' bounds: each arm adds its own worst span and the list's 60 s stays the boot's margin. `lan` 120 s (60 s discovery, under 60 s measuring); `internet` 155 s (60 s for a lease, under 95 s at an assumed 3 MB/s down and 1.5 MB/s up, which no metal reading confirms). The shared boot's bound is 335 s. - The crypto issue's consumer table names osbench's guest HTTPS client and the third copy of the provider's random source and group filter. Co-Authored-By: Claude Opus 5.5 --- ...in-the-tree-on-par-with-ring-and-secure.md | 2 +- tests/guest/osbench/compare/src/lib.rs | 135 ++++++++++++++---- tests/guest/osbench/src/sys/toyos.rs | 2 +- tests/toyos.rs | 18 ++- 4 files changed, 121 insertions(+), 36 deletions(-) diff --git a/issues/toyos-crypto-is-in-the-tree-on-par-with-ring-and-secure.md b/issues/toyos-crypto-is-in-the-tree-on-par-with-ring-and-secure.md index 3a5bcce652..7a54fdf633 100644 --- a/issues/toyos-crypto-is-in-the-tree-on-par-with-ring-and-secure.md +++ b/issues/toyos-crypto-is-in-the-tree-on-par-with-ring-and-secure.md @@ -64,7 +64,7 @@ review has been done, and passing vectors does not show it is safer than ring. | `system/sshserver` | `toyos-ssh`, since the SSH track's stage 2 | none directly; it reaches the crate through `toyos-ssh` | | Image and package signing: `lib/update` (the loader, `/system/bin/update`, `build/signing.rs`, `build/root`, #892's repository keys) | the crate's Ed25519 and SHA-2, since #1019; signing and a seed's public key behind `sign`, on the host | stage 6's bench readings | | Every SHA-256: the build, `pkg`, `lib/swap`, `system/update`, `lib/update` and the loader | the crate, since #1016: SHA-NI on x86-64, the UEFI loader's in a block that saves the XMM registers it uses; the Armv8 SHA instructions where the target promises them. No slower than `toyos-sha2-hw`, which it replaces, on either bench: on the T14 4.6 times as fast at 64 B and 9 and 6 percent faster at 16 KiB and 1 MiB, on the AMD bench 2.8 times at 64 B, 2 percent at 16 KiB and equal at 1 MiB (#1016's evidence) | nothing | -| ToyOS's own HTTPS clients. On the host: the build's `ureq` (`build/release.rs`) and doom's build script (`apps/doom/build.rs`). In a guest, none today: `https_tls13` and `pkg install ` are stages 3 and 5 of `issues/the-internet-clients-work-unchanged.md` | the host's two: the crate's rustls provider through ureq's `rustls-no-provider`, since #1020, its key exchange on X25519 and X25519MLKEM768 alone while the T14 finding stands. In a guest, none | the NIST groups back once the T14 finding is resolved | +| ToyOS's own HTTPS clients. On the host: the build's `ureq` (`build/release.rs`) and doom's build script (`apps/doom/build.rs`). In a guest, `osbench internet` (`tests/guest/osbench/src/internet.rs`), on ToyOS and Linux alike; `https_tls13` and `pkg install ` are stages 3 and 5 of `issues/the-internet-clients-work-unchanged.md` | all three: the crate's rustls provider through ureq's `rustls-no-provider`, since #1020, its key exchange on X25519 and X25519MLKEM768 alone while the T14 finding stands. Each copies the provider's random source and that group filter | the NIST groups back once the T14 finding is resolved, in all three copies, or once in the crate if it comes to own the policy | | Wi-Fi's `wlan` (`issues/there-is-no-wifi.md` P6, W2) | nothing yet | PBKDF2-HMAC-SHA1, the 802.11i PRF and RFC 3394 unwrap, which the crate has. AES-CMAC-128, SAE over P-256 by hash-to-element, a public constant-time comparison and a public wiping secret, which it lacks | | Encrypted DATA storage (`issues/storage-is-layers-and-a-role-is-a-filesystem.md`): the bcachefs engine's `crypt` unlock and its key holder | nothing yet | scrypt (RFC 7914) at bcachefs's default and smallest costs, which the crate has; ChaCha20 at a chosen block counter and raw Poly1305 for the format's own AEAD composition, which it lacks publicly | | `issues/programs-hold-aes-keys-as-key-locker-handles.md` | blocked on a session-long AES key | a TLS client on the crate is that consumer: AES behind a Key Locker handle. SSH holds no AES key | diff --git a/tests/guest/osbench/compare/src/lib.rs b/tests/guest/osbench/compare/src/lib.rs index cae0ace2f9..3bb307f24e 100644 --- a/tests/guest/osbench/compare/src/lib.rs +++ b/tests/guest/osbench/compare/src/lib.rs @@ -7,7 +7,10 @@ //! Linux's range on a path is that path's noise floor. ToyOS's median over //! Linux's is the ratio, and the exact two-sided Mann-Whitney test over the //! two sets of runs says whether the difference is resolved at all: with four -//! runs each, complete separation gives p = 2/70. +//! runs each, complete separation gives p = 2/70. Each p is one row's, +//! uncorrected for the table's other rows. Better or worse is judged only +//! for a unit whose better direction is known ([`better`]); for any other +//! the verdict says which way ToyOS differs. use std::collections::BTreeMap; use std::fmt::Write as _; @@ -123,10 +126,19 @@ pub fn median(values: &[u64]) -> f64 { } } -/// `(max - min) / median`, the spread of a set of runs. -pub fn range(values: &[u64]) -> f64 { +/// `(max - min) / median`, the spread of a set of runs; `None` about a +/// median of 0. +pub fn range(values: &[u64]) -> Option { let (min, max) = (values.iter().min().expect("values"), values.iter().max().expect("values")); - (max - min) as f64 / median(values) + let median = median(values); + (median > 0.0).then(|| (max - min) as f64 / median) +} + +/// The least two-sided p `n` runs against `m` can reach: complete +/// separation, two of the C(n + m, n) splits. +pub fn least_p(n: usize, m: usize) -> f64 { + let splits = (1..=n).fold(1.0, |c, k| c * (m + k) as f64 / k as f64); + (2.0 / splits).min(1.0) } /// The exact two-sided Mann-Whitney p of `a` against `b`, ties counted half, @@ -173,14 +185,30 @@ pub fn mann_whitney(a: &[u64], b: &[u64]) -> Option { Some(extreme as f64 / splits as f64) } -/// Whether a lower value of `unit` is the better one. -fn lower_is_better(unit: &str) -> bool { - unit == "ns" +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Better { + Lower, + Higher, +} + +/// Which way `unit` is better, for the units whose direction is known: a +/// time and a rate. A clock, a power or a share is no better higher or lower. +pub fn better(unit: &str) -> Option { + match unit { + "ns" => Some(Better::Lower), + "MB/s" => Some(Better::Higher), + _ => None, + } } -/// A link description's kind and driver, without what varies by the run. -fn path(said: &str) -> String { - said.split(['_', ' ']).take(2).collect::>().join(" ") +/// The condition every run of a machine must agree on for a benchmark's +/// runs to be compared, and how many of its leading words must agree: the +/// peer's link kind and driver, and the internet link's kind, not what +/// varies by the run. +const PATHS: [(&str, &str, usize); 2] = [("lan.", "lan.peer_link", 2), ("internet.", "internet.link", 1)]; + +fn path(said: &str, words: usize) -> String { + said.split(['_', ' ']).take(words).collect::>().join(" ") } /// Every machine's comparison, as Markdown: one section per machine, one row @@ -193,28 +221,34 @@ pub fn compare(runs: &[Run]) -> Result { let mut out = String::new(); for (machine, runs) in machines { let count = |os| runs.iter().filter(|r| r.os == os).count(); - let peers: std::collections::BTreeSet = - runs.iter().filter_map(|r| r.conds.get("lan.peer_link")).map(|said| path(said)).collect(); + let paths: Vec<(&str, &str, std::collections::BTreeSet)> = PATHS + .iter() + .map(|&(prefix, cond, words)| { + (prefix, cond, runs.iter().filter_map(|r| r.conds.get(cond)).map(|said| path(said, words)).collect()) + }) + .collect(); writeln!(out, "## {machine}\n").unwrap(); - writeln!(out, "Runs: Linux {}, ToyOS {}. LAN peer: {}.\n", count(Os::Linux), count(Os::Toyos), { - if peers.is_empty() { "none".to_string() } else { peers.iter().cloned().collect::>().join(", ") } - }) - .unwrap(); + write!(out, "Runs: Linux {}, ToyOS {}.", count(Os::Linux), count(Os::Toyos)).unwrap(); + for (_, cond, seen) in &paths { + let seen = if seen.is_empty() { "none".to_string() } else { seen.iter().cloned().collect::>().join(", ") }; + write!(out, " {cond}: {seen}.").unwrap(); + } + writeln!(out, "\n").unwrap(); writeln!(out, "| bench | unit | Linux p50s | Linux median | Linux range | ToyOS p50s | ToyOS median | ToyOS range | ToyOS/Linux | p | verdict |").unwrap(); writeln!(out, "|---|---|---|---|---|---|---|---|---|---|---|").unwrap(); let names: std::collections::BTreeSet<&String> = runs.iter().flat_map(|r| r.benches.keys()).collect(); for name in names { - let row = if name.starts_with("lan.") && peers.len() > 1 { - Err("the runs' peers were on different links".to_string()) - } else { - Ok(compare_one(&runs, name)) + let split = paths.iter().find(|(prefix, _, seen)| name.starts_with(prefix) && seen.len() > 1); + let row = match split { + Some((_, cond, _)) => Err(format!("the runs disagree on {cond}")), + None => Ok(compare_one(&runs, name)), }; match row { Ok(row) => writeln!(out, "| {name} | {row} |").unwrap(), Err(why) => writeln!(out, "| {name} | | | | | | | | | | {why} |").unwrap(), } } - writeln!(out).unwrap(); + writeln!(out, "\nEach p is its row's alone, uncorrected for the table's other rows.\n").unwrap(); } Ok(out) } @@ -241,7 +275,7 @@ fn compare_one(runs: &[&Run], name: &str) -> String { Some(v) => [ v.iter().map(u64::to_string).collect::>().join(", "), format!("{:.0}", median(v)), - format!("{:.1}%", range(v) * 100.0), + range(v).map_or_else(|| "-".into(), |r| format!("{:.1}%", r * 100.0)), ], None => [String::new(), "-".into(), "-".into()], } @@ -252,12 +286,16 @@ fn compare_one(runs: &[&Run], name: &str) -> String { (Some(l), Some(t)) => { let ratio = median(t) / median(l); let p = mann_whitney(t, l); + let least = least_p(t.len(), l.len()); let verdict = match p { None => "too many runs for the exact test".to_string(), - Some(_) if l.len() < 2 || t.len() < 2 => "too few runs".to_string(), + Some(_) if least > RESOLVED => format!("too few runs to resolve (least p {least:.3})"), Some(p) if p <= RESOLVED => { - let better = (ratio < 1.0) == lower_is_better(unit.unwrap_or_default()); - format!("ToyOS {}", if better { "better" } else { "worse" }) + let higher = median(t) > median(l); + match better(unit.unwrap_or_default()) { + Some(way) => format!("ToyOS {}", if higher == (way == Better::Higher) { "better" } else { "worse" }), + None => format!("ToyOS {}", if higher { "higher" } else { "lower" }), + } } Some(_) => "not resolved".to_string(), }; @@ -351,8 +389,20 @@ mod tests { fn median_and_range_of_a_set_of_runs() { assert_eq!(median(&[3, 1, 2]), 2.0); assert_eq!(median(&[4, 1, 3, 2]), 2.5); - assert!((range(&[90, 100, 110, 100]) - 0.2).abs() < 1e-12); - assert_eq!(range(&[7]), 0.0); + assert!((range(&[90, 100, 110, 100]).unwrap() - 0.2).abs() < 1e-12); + assert_eq!(range(&[7]), Some(0.0)); + assert_eq!(range(&[0, 0, 3]), None, "no spread relative to a median of 0"); + } + + #[test] + fn the_least_p_is_complete_separations() { + assert!((least_p(4, 4) - 2.0 / 70.0).abs() < 1e-12); + assert!((least_p(4, 3) - 2.0 / 35.0).abs() < 1e-12); + assert!((least_p(3, 4) - least_p(4, 3)).abs() < 1e-12); + assert_eq!(least_p(1, 1), 1.0); + assert_eq!(least_p(1, 3), 0.5); + let p = mann_whitney(&[1, 2, 3, 4], &[5, 6, 7]).unwrap(); + assert!((p - least_p(4, 3)).abs() < 1e-12); } fn rtt(os: &str, p50: u64) -> Run { @@ -374,6 +424,24 @@ mod tests { assert!(table.contains("| not resolved |"), "{table}"); } + #[test] + fn counts_that_cannot_reach_the_level_say_so_and_not_that_nothing_differs() { + let mut runs: Vec = [100, 104, 98, 102].iter().map(|v| rtt("linux", *v)).collect(); + runs.extend([150, 149, 155].iter().map(|v| rtt("toyos", *v))); + let table = compare(&runs).unwrap(); + assert!(table.contains("| 0.057 | too few runs to resolve (least p 0.057) |"), "{table}"); + } + + #[test] + fn a_unit_of_no_known_direction_is_told_only_which_way_it_differs() { + let power = |os: &str, v: u64| run(os, &[&format!("bench=energy.idle_60s.package unit=mW n=1 min=1 p50={v} max=9 mean=5")]); + let mut runs: Vec = [900, 910, 920, 930].iter().map(|v| power("linux", *v)).collect(); + runs.extend([1500, 1510, 1520, 1530].iter().map(|v| power("toyos", *v))); + let table = compare(&runs).unwrap(); + assert!(table.contains("| ToyOS higher |"), "{table}"); + assert!(!table.contains("better") && !table.contains("worse"), "{table}"); + } + #[test] fn a_throughputs_higher_value_is_the_better_one() { let mbps = |os: &str, v: u64| run(os, &[&format!("bench=lan.wired.tcp.upload unit=MB/s n=5 min=1 p50={v} max=9 mean=5")]); @@ -401,6 +469,17 @@ mod tests { ]; assert!(!compare(&runs).unwrap().contains("different links"), "a signal is no other link"); let runs = [peer("linux", "wireless_driver=iwlwifi_signal_dbm=-52", 10), peer("toyos", "wired_driver=e1000e_speed_mbps=1000", 12)]; - assert!(compare(&runs).unwrap().contains("| the runs' peers were on different links |")); + assert!(compare(&runs).unwrap().contains("| the runs disagree on lan.peer_link |")); + } + + #[test] + fn internet_runs_that_left_by_different_kinds_of_link_are_not_compared() { + let left = |os: &str, said: &str| { + run(os, &[&format!("cond internet.link={said}"), "bench=internet.https.rtt unit=ns n=1 min=1 p50=9 max=1 mean=1"]) + }; + let same = [left("linux", "wired_driver=e1000e_speed_mbps=1000"), left("toyos", "wired_netstack's_one_Ethernet_card")]; + assert!(!compare(&same).unwrap().contains("disagree"), "both wired, by any driver"); + let split = [left("linux", "wireless_driver=iwlwifi_signal_dbm=-50"), left("toyos", "wired_netstack's_one_Ethernet_card")]; + assert!(compare(&split).unwrap().contains("| the runs disagree on internet.link |")); } } diff --git a/tests/guest/osbench/src/sys/toyos.rs b/tests/guest/osbench/src/sys/toyos.rs index 972bd13451..d4a50d3324 100644 --- a/tests/guest/osbench/src/sys/toyos.rs +++ b/tests/guest/osbench/src/sys/toyos.rs @@ -365,7 +365,7 @@ pub fn persistent_dir() -> Option<(String, String)> { /// What carries a connection on ToyOS: netstack drives one card, Ethernet. pub fn link_of(_local: std::net::IpAddr) -> Result<(osbench_lan::link::Link, String), String> { - Ok((osbench_lan::link::Link::Wired, "netstack's one Ethernet card".into())) + Ok((osbench_lan::link::Link::Wired, "wired netstack's one Ethernet card".into())) } /// ToyOS reads no temperature: the kernel counts none diff --git a/tests/toyos.rs b/tests/toyos.rs index 49aa92f297..47478cacc9 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -1636,18 +1636,24 @@ const BENCHCASE: &[metal::Arm] = &[metal::Arm { }]; /// The two groups that need another party share one boot when one pass names -/// both. `lan` looks for its peer up to 60 s, inside which the boot's lease +/// both. Each adds its own worst span; the 60 s every list has is the +/// boot's margin, not either group's. +/// +/// `lan` looks for its peer for up to 60 s, inside which the boot's lease /// lands, then measures for under 60 s over a Wi-Fi hop: 2,155 round trips -/// at up to 10 ms and twelve 3 s transfers, so 150 s leaves it 30 s. -/// `internet` measures for under 95 s down to 3 MB/s and up to 1.5 MB/s: six -/// 25 MB downloads, six 10 MB uploads and 77 short exchanges. +/// at up to 10 ms and twelve 3 s transfers with their warm-ups. const OSBENCH_LAN: &[metal::Arm] = &[metal::Arm { - adds_ms: 90_000, + adds_ms: 120_000, named_only: true, ..metal::once("netbenchcase", "tests/images/benchcase.toml", &[], &["osbench_lan"]) }]; +/// `internet` waits up to 60 s for the lease when it runs first, then +/// measures six 25 MB downloads, six 10 MB uploads and 77 short exchanges: +/// under 95 s at 3 MB/s down and 1.5 MB/s up. Those rates are an +/// assumption, not a reading: no ToyOS TLS transfer has been timed on metal +/// by this group, and a slower one ends the list at the bound. const OSBENCH_INTERNET: &[metal::Arm] = &[metal::Arm { - adds_ms: 60_000, + adds_ms: 155_000, named_only: true, ..metal::once("netbenchcase", "tests/images/benchcase.toml", &[], &["osbench_internet"]) }]; From 548bef4bf9e392c9de593abf0b5692d4ef453f34 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 24 Oct 2026 14:14:01 +0200 Subject: [PATCH 3/5] osbench: the exact test's least p is the observed ties' own, internet downloads stay under the endpoint's limit, and the first paired session's findings - osbench-compare: the least attainable p is read from the same exact distribution as p, the farthest splits' share, rather than assumed to be 2/C(n+m,n). A difference whose p reaches 0.05 is resolved whatever the counts. The session's T14 upload, three ToyOS runs against four Linux runs three of which tie, has p = least = 1/35, and the old count-based gate called it "too few runs". - internet: the downloads are four of 10 MB rather than six of 25 MB. Every Linux download in the session drew Cloudflare's 429 after four runs of the larger volume had passed. Its bound goes to 105 s. - issues: netstack's TCP send on the T14's I219 runs at 0.36x Linux's on the same path while receive is at parity (new defect); the session's cpu.clock readings join the idle-clock issue; the comparison issue points at the session. Co-Authored-By: Claude Opus 5.5 --- ...ird-of-what-the-t14s-wired-link-carries.md | 47 +++++++++++++ ...ompares-toyos-with-linux-on-one-machine.md | 4 ++ ...eaving-idle-and-not-their-settled-clock.md | 12 ++++ tests/guest/osbench/compare/src/lib.rs | 69 +++++++++++-------- tests/guest/osbench/src/internet.rs | 7 +- tests/toyos.rs | 10 +-- 6 files changed, 113 insertions(+), 36 deletions(-) create mode 100644 issues/netstack-sends-tcp-at-a-third-of-what-the-t14s-wired-link-carries.md diff --git a/issues/netstack-sends-tcp-at-a-third-of-what-the-t14s-wired-link-carries.md b/issues/netstack-sends-tcp-at-a-third-of-what-the-t14s-wired-link-carries.md new file mode 100644 index 0000000000..b019606aa3 --- /dev/null +++ b/issues/netstack-sends-tcp-at-a-third-of-what-the-t14s-wired-link-carries.md @@ -0,0 +1,47 @@ +--- +status: open +kind: defect +opened: 2026-10-24 +--- + +# netstack sends TCP at a third of what the T14's wired link carries + +On the T14's I219, a ToyOS TCP sender moves 41 MB/s where the same +program under the bench's Linux moves 113 MB/s on the same path and the +same peer. Received TCP is at parity on that path, so the link, the peer +and the access point are not the limit. + +**Measured** on 2026-10-24 with `osbench lan` at `d99ef5a35`. The path is +the T14's I219 at 1000 Mb/s, through the access point, to the AMD bench's +Linux `osbench-peer` on its Wi-Fi. Each cell is one run's p50 of five 3 s +rounds: + +| | ToyOS (3 runs) | Linux (4 runs) | +|---|---|---| +| `lan.wired.tcp.upload`, MB/s | 41, 48, 31 | 113, 113, 114, 113 | +| `lan.wired.tcp.download`, MB/s | 112, 112, 109 | 112, 110, 109, 112 | + +- No ToyOS round reached a Linux one: ToyOS's rounds ran 31 to 59 MB/s and + Linux's 112 to 114. The exact rank test over the runs gives p = 1/35. +- ToyOS's rounds cluster near 31 and 58 MB/s, an observation from three + runs that suggests a cap rather than noise. +- The same runs' HTTPS upload to the public internet took 1.40 times + Linux's time. + +Evidence: `ToyOSOrg/evidence` branch `pr-1065`, `r3/`. + +Where the time goes is not measured. These are candidates to measure first, +not findings: + +- every sent byte's copies through a kernel pipe + (`issues/netstack-moves-every-byte-through-two-kernel-pipes.md`); +- the I219 transmit path's `TXDCTL` and `TIDV` setup + (`issues/the-intel-driver-writes-txdctl-and-tidv-against-the-82574-datasheets-newer-revision.md`); +- the send window and the ACK clocking. + +The sending CPU's clock read 1.6 to 3.8 GHz around these runs. That is +not excluded either. + +**Exit**: `osbench_lan` on the T14 reads `lan.wired.tcp.upload` within +Linux's run-to-run range on the same path, or the cause is named and owned +by its own issue. diff --git a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md index 679ec5a201..fb51fb9df2 100644 --- a/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md +++ b/issues/no-published-run-compares-toyos-with-linux-on-one-machine.md @@ -38,6 +38,10 @@ its groups: path neither system owns, run only when a pass names its row: no automated test depends on the public internet. +The first paired session (`ToyOSOrg/evidence` branch `pr-1065`, `r3/`) put +ToyOS's T14 LAN download and 64 B round trip at Linux's and its upload at +a third of it (`issues/netstack-sends-tcp-at-a-third-of-what-the-t14s-wired-link-carries.md`). + The cryptographic rows are not osbench's. AES-GCM, ChaCha20-Poly1305, SHA-256 and the rest were measured under Linux on both benches against ring, OpenSSL and AWS-LC by the crypto oracle probe, and that table is the diff --git a/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md b/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md index 1bcde5520b..83e5abda81 100644 --- a/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md +++ b/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md @@ -32,6 +32,18 @@ measuring CPU until three probes of a dependent `add` chain agree within 1%, and every result line prints that clock as `mhz_settled` and the clock just after as `mhz`; each group also prints `cpu.clock`, 200 probes from idle. +The network runs show it again. On the T14 at `d99ef5a35`, each group's +`cpu.clock` p50 in MHz was: + +| | ToyOS (3 runs) | Linux (4 runs) | +|---|---|---| +| before `lan` | 2791, 2791, 1495 | 4186, 4184, 4185, 4186 | +| before `internet` | 3189, 1595, 2990 | 3790, 4186, 3788, 4186 | + +Both groups wait mostly on the network. In those runs ToyOS's settled clock +also ranged from 1.6 to 3.8 GHz across rows +(`ToyOSOrg/evidence` branch `pr-1065`, `r3/`). + **Exit**: a T14 run on each system shows `mhz_settled` near the all-CPU rate per CPU within the run-to-run spread, or names the benchmark whose CPU did not settle. diff --git a/tests/guest/osbench/compare/src/lib.rs b/tests/guest/osbench/compare/src/lib.rs index 3bb307f24e..d5b03ce871 100644 --- a/tests/guest/osbench/compare/src/lib.rs +++ b/tests/guest/osbench/compare/src/lib.rs @@ -134,17 +134,20 @@ pub fn range(values: &[u64]) -> Option { (median > 0.0).then(|| (max - min) as f64 / median) } -/// The least two-sided p `n` runs against `m` can reach: complete -/// separation, two of the C(n + m, n) splits. -pub fn least_p(n: usize, m: usize) -> f64 { - let splits = (1..=n).fold(1.0, |c, k| c * (m + k) as f64 / k as f64); - (2.0 / splits).min(1.0) +/// The exact two-sided Mann-Whitney test of `a` against `b`: `p`, and +/// `least`, the smallest p these pooled values could give whichever runs +/// were whose. Ties count half a win, and the distribution is every split of +/// the pooled values into sets of the two sizes, so it holds the observed +/// ties: with none, `least` is 2/C(n + m, n); runs tied apart from the other +/// system's can lower it. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Exact { + pub p: f64, + pub least: f64, } -/// The exact two-sided Mann-Whitney p of `a` against `b`, ties counted half, -/// by every split of the pooled values into sets of their sizes; `None` past -/// [`MAX_RUNS`] values or with an empty side. -pub fn mann_whitney(a: &[u64], b: &[u64]) -> Option { +/// `None` past [`MAX_RUNS`] values or with an empty side. +pub fn mann_whitney(a: &[u64], b: &[u64]) -> Option { let pooled: Vec = a.iter().chain(b).copied().collect(); let total = pooled.len(); if a.is_empty() || b.is_empty() || total > MAX_RUNS { @@ -166,7 +169,7 @@ pub fn mann_whitney(a: &[u64], b: &[u64]) -> Option { }; let centre = (a.len() * b.len()) as i64; let seen = (doubled_u(a, b) - centre).abs(); - let (mut splits, mut extreme) = (0u64, 0u64); + let mut distances = Vec::new(); let (mut first, mut second) = (Vec::with_capacity(total), Vec::with_capacity(total)); for mask in 0u32..(1 << total) { if mask.count_ones() as usize != a.len() { @@ -177,12 +180,11 @@ pub fn mann_whitney(a: &[u64], b: &[u64]) -> Option { for (i, v) in pooled.iter().enumerate() { if mask & (1 << i) != 0 { first.push(*v) } else { second.push(*v) } } - splits += 1; - if (doubled_u(&first, &second) - centre).abs() >= seen { - extreme += 1; - } + distances.push((doubled_u(&first, &second) - centre).abs()); } - Some(extreme as f64 / splits as f64) + let share = |at_least: i64| distances.iter().filter(|&&d| d >= at_least).count() as f64 / distances.len() as f64; + let farthest = *distances.iter().max().expect("a split"); + Some(Exact { p: share(seen), least: share(farthest) }) } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -285,20 +287,20 @@ fn compare_one(runs: &[&Run], name: &str) -> String { let (ratio, p, verdict) = match (values.get(&Os::Linux), values.get(&Os::Toyos)) { (Some(l), Some(t)) => { let ratio = median(t) / median(l); - let p = mann_whitney(t, l); - let least = least_p(t.len(), l.len()); - let verdict = match p { + let exact = mann_whitney(t, l); + let verdict = match exact { None => "too many runs for the exact test".to_string(), - Some(_) if least > RESOLVED => format!("too few runs to resolve (least p {least:.3})"), - Some(p) if p <= RESOLVED => { + Some(Exact { p, .. }) if p <= RESOLVED => { let higher = median(t) > median(l); match better(unit.unwrap_or_default()) { Some(way) => format!("ToyOS {}", if higher == (way == Better::Higher) { "better" } else { "worse" }), None => format!("ToyOS {}", if higher { "higher" } else { "lower" }), } } + Some(Exact { least, .. }) if least > RESOLVED => format!("too few runs to resolve (least p {least:.3})"), Some(_) => "not resolved".to_string(), }; + let p = exact.map(|e| e.p); (format!("{ratio:.2}"), p.map_or_else(|| "-".into(), |p| format!("{p:.3}")), verdict) } _ => ("-".into(), "-".into(), String::new()), @@ -365,7 +367,7 @@ mod tests { #[test] fn the_exact_test_matches_its_tables() { // Complete separation has two splits as extreme as itself, of C(n+m, n). - let p = |a: &[u64], b: &[u64]| mann_whitney(a, b).unwrap(); + let p = |a: &[u64], b: &[u64]| mann_whitney(a, b).unwrap().p; assert!((p(&[1, 2, 3, 4], &[5, 6, 7, 8]) - 2.0 / 70.0).abs() < 1e-12); assert!((p(&[8, 7, 6, 5], &[1, 2, 3, 4]) - 2.0 / 70.0).abs() < 1e-12); assert!((p(&[1, 2, 3], &[4, 5, 6]) - 0.1).abs() < 1e-12); @@ -395,14 +397,18 @@ mod tests { } #[test] - fn the_least_p_is_complete_separations() { - assert!((least_p(4, 4) - 2.0 / 70.0).abs() < 1e-12); - assert!((least_p(4, 3) - 2.0 / 35.0).abs() < 1e-12); - assert!((least_p(3, 4) - least_p(4, 3)).abs() < 1e-12); - assert_eq!(least_p(1, 1), 1.0); - assert_eq!(least_p(1, 3), 0.5); - let p = mann_whitney(&[1, 2, 3, 4], &[5, 6, 7]).unwrap(); - assert!((p - least_p(4, 3)).abs() < 1e-12); + fn the_least_p_is_the_farthest_splits_share() { + let least = |a: &[u64], b: &[u64]| mann_whitney(a, b).unwrap().least; + // Distinct values: complete separation either way, 2 of C(n + m, n). + assert!((least(&[1, 2, 3, 4], &[5, 6, 7, 8]) - 2.0 / 70.0).abs() < 1e-12); + assert!((least(&[9, 1, 5, 3], &[2, 8, 4, 6]) - 2.0 / 70.0).abs() < 1e-12); + assert!((least(&[1, 2, 3, 4], &[5, 6, 7]) - 2.0 / 35.0).abs() < 1e-12); + assert_eq!(least(&[1], &[2]), 1.0); + assert_eq!(least(&[1], &[2, 3, 4]), 0.5); + // Three tied runs of four: only the three lowest values set apart are + // that far out, so one split of 35, the session's T14 upload. + let upload = mann_whitney(&[41, 48, 31], &[113, 113, 114, 113]).unwrap(); + assert!((upload.p - 1.0 / 35.0).abs() < 1e-12 && (upload.least - 1.0 / 35.0).abs() < 1e-12, "{upload:?}"); } fn rtt(os: &str, p50: u64) -> Run { @@ -430,6 +436,11 @@ mod tests { runs.extend([150, 149, 155].iter().map(|v| rtt("toyos", *v))); let table = compare(&runs).unwrap(); assert!(table.contains("| 0.057 | too few runs to resolve (least p 0.057) |"), "{table}"); + // Ties can let the same counts resolve: the test decides, not the count. + let mbps = |os: &str, v: u64| run(os, &[&format!("bench=lan.wired.tcp.upload unit=MB/s n=5 min=1 p50={v} max=9 mean=5")]); + let mut runs: Vec = [113, 113, 114, 113].iter().map(|v| mbps("linux", *v)).collect(); + runs.extend([41, 48, 31].iter().map(|v| mbps("toyos", *v))); + assert!(compare(&runs).unwrap().contains("| 0.029 | ToyOS worse |")); } #[test] diff --git a/tests/guest/osbench/src/internet.rs b/tests/guest/osbench/src/internet.rs index ce73f3e0ad..f365d66d98 100644 --- a/tests/guest/osbench/src/internet.rs +++ b/tests/guest/osbench/src/internet.rs @@ -24,9 +24,12 @@ const HOST: &str = "speed.cloudflare.com"; const USER_AGENT: &str = "toyos-build (https://github.com/ToyOSOrg/ToyOS)"; /// One download's bytes and one upload's, each timed whole: a slow uplink's /// rate needs more resolution than whole MB/s. -const DOWN: u64 = 25_000_000; +const DOWN: u64 = 10_000_000; const UP: usize = 10_000_000; -const ROUNDS: usize = 5; +/// Timed transfers each way after one unread. The endpoint answers 429 past +/// a download volume it does not publish, so a session of runs stays well +/// under the volume that first drew one. +const ROUNDS: usize = 3; /// The most a short answer is read to. const ANSWER_MAX: u64 = 64 * 1024; /// How long one connection or request may take before it is given up. diff --git a/tests/toyos.rs b/tests/toyos.rs index 47478cacc9..ae94baa7f2 100644 --- a/tests/toyos.rs +++ b/tests/toyos.rs @@ -1648,12 +1648,12 @@ const OSBENCH_LAN: &[metal::Arm] = &[metal::Arm { ..metal::once("netbenchcase", "tests/images/benchcase.toml", &[], &["osbench_lan"]) }]; /// `internet` waits up to 60 s for the lease when it runs first, then -/// measures six 25 MB downloads, six 10 MB uploads and 77 short exchanges: -/// under 95 s at 3 MB/s down and 1.5 MB/s up. Those rates are an -/// assumption, not a reading: no ToyOS TLS transfer has been timed on metal -/// by this group, and a slower one ends the list at the bound. +/// measures four 10 MB downloads, four 10 MB uploads and 77 short exchanges: +/// under 45 s at 3 MB/s down and 1.5 MB/s up. Those rates are an assumed +/// floor, far under the T14's first ToyOS readings, and a slower link ends +/// the list at the bound. const OSBENCH_INTERNET: &[metal::Arm] = &[metal::Arm { - adds_ms: 155_000, + adds_ms: 105_000, named_only: true, ..metal::once("netbenchcase", "tests/images/benchcase.toml", &[], &["osbench_internet"]) }]; From ec8b41de1a522b93693beae098b2f7999ca24d27 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 24 Oct 2026 14:23:26 +0200 Subject: [PATCH 4/5] issues: the session's cpu.clock readings join the T14's clock-ceiling record The network session's cpu.clock readings were ToyOS 1.5-3.2 GHz against Linux 3.8-4.2 GHz per group. They are the same unexplained clock as the 3,788 MHz single-thread ceiling and the 2.67 GHz download windows, so they belong with that record in toyos-uses-what-modern-hardware-offers-for-speed. They leave the idle-clock issue, whose subject is the gauge and not the clock. The upload defect points at the record. Co-Authored-By: Claude Opus 5.5 --- ...at-a-third-of-what-the-t14s-wired-link-carries.md | 3 ++- ...-cpus-leaving-idle-and-not-their-settled-clock.md | 12 ------------ ...yos-uses-what-modern-hardware-offers-for-speed.md | 5 +++++ 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/issues/netstack-sends-tcp-at-a-third-of-what-the-t14s-wired-link-carries.md b/issues/netstack-sends-tcp-at-a-third-of-what-the-t14s-wired-link-carries.md index b019606aa3..9d3b8f6ee5 100644 --- a/issues/netstack-sends-tcp-at-a-third-of-what-the-t14s-wired-link-carries.md +++ b/issues/netstack-sends-tcp-at-a-third-of-what-the-t14s-wired-link-carries.md @@ -40,7 +40,8 @@ not findings: - the send window and the ACK clocking. The sending CPU's clock read 1.6 to 3.8 GHz around these runs. That is -not excluded either. +not excluded either: it is the T14's clock ceiling in +`issues/toyos-uses-what-modern-hardware-offers-for-speed.md`. **Exit**: `osbench_lan` on the T14 reads `lan.wired.tcp.upload` within Linux's run-to-run range on the same path, or the cause is named and owned diff --git a/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md b/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md index 83e5abda81..1bcde5520b 100644 --- a/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md +++ b/issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md @@ -32,18 +32,6 @@ measuring CPU until three probes of a dependent `add` chain agree within 1%, and every result line prints that clock as `mhz_settled` and the clock just after as `mhz`; each group also prints `cpu.clock`, 200 probes from idle. -The network runs show it again. On the T14 at `d99ef5a35`, each group's -`cpu.clock` p50 in MHz was: - -| | ToyOS (3 runs) | Linux (4 runs) | -|---|---|---| -| before `lan` | 2791, 2791, 1495 | 4186, 4184, 4185, 4186 | -| before `internet` | 3189, 1595, 2990 | 3790, 4186, 3788, 4186 | - -Both groups wait mostly on the network. In those runs ToyOS's settled clock -also ranged from 1.6 to 3.8 GHz across rows -(`ToyOSOrg/evidence` branch `pr-1065`, `r3/`). - **Exit**: a T14 run on each system shows `mhz_settled` near the all-CPU rate per CPU within the run-to-run spread, or names the benchmark whose CPU did not settle. diff --git a/issues/toyos-uses-what-modern-hardware-offers-for-speed.md b/issues/toyos-uses-what-modern-hardware-offers-for-speed.md index 26e2b7c673..536f7cb72b 100644 --- a/issues/toyos-uses-what-modern-hardware-offers-for-speed.md +++ b/issues/toyos-uses-what-modern-hardware-offers-for-speed.md @@ -70,6 +70,11 @@ is nothing to enable. `issues/idle-cpus-enter-the-c-states-cst-names.md`'s to answer; that the idle state reaches HWP's clock choice is inference. Next: a T14 scout whose idle CPUs wait in a deep C-state, read over the same windows. + #1065's network session on the T14 (`ToyOSOrg/evidence` branch + `pr-1065`, `r3/`) read the same: each group's `cpu.clock` p50 was 1.5 to + 3.2 GHz under ToyOS against 3.8 to 4.2 GHz under Linux. That covers three + ToyOS runs and four Linux runs of `osbench lan` and `osbench internet`, both + of which mostly wait on the network. - **Global-line contention** is bounded at 2 to 3 % of C0 on the T14 and not attributed; next, the HITM count attributed to its lines on both benches (`issues/per-cpu-scheduler-and-pmm-state-shares-lines-every-cpu-writes.md`). From 48c9d05f93c1928a98d37d96a5f9db86c7ebbb61 Mon Sep 17 00:00:00 2001 From: japabu Date: Sat, 24 Oct 2026 14:30:11 +0200 Subject: [PATCH 5/5] review r2: the session records say what the readings show, and the exact test names its two-sided rule - netstack upload defect: the "cluster near 31 and 58 MB/s" claim goes; the runs' min, p50 and max do not show it. The clock paragraph now gives both systems' readings: Linux sent 113 MB/s at the same ~1.5 GHz readings, so the clock does not explain the gap. - The T14's clock-ceiling record compares cpu.clock's maximum: 3789 MHz under ToyOS against 4188-4190 under Linux. It no longer compares the probes' idle-ramp p50. - osbench-compare's doc names its two-sided p, the share of splits whose |2U - nm| is at least the observed one, and says how ties lower the least p. Co-Authored-By: Claude Opus 5.5 --- ...-at-a-third-of-what-the-t14s-wired-link-carries.md | 10 +++++----- ...oyos-uses-what-modern-hardware-offers-for-speed.md | 11 +++++++---- tests/guest/osbench/compare/src/lib.rs | 11 +++++++---- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/issues/netstack-sends-tcp-at-a-third-of-what-the-t14s-wired-link-carries.md b/issues/netstack-sends-tcp-at-a-third-of-what-the-t14s-wired-link-carries.md index 9d3b8f6ee5..64f29c85d3 100644 --- a/issues/netstack-sends-tcp-at-a-third-of-what-the-t14s-wired-link-carries.md +++ b/issues/netstack-sends-tcp-at-a-third-of-what-the-t14s-wired-link-carries.md @@ -23,8 +23,6 @@ rounds: - No ToyOS round reached a Linux one: ToyOS's rounds ran 31 to 59 MB/s and Linux's 112 to 114. The exact rank test over the runs gives p = 1/35. -- ToyOS's rounds cluster near 31 and 58 MB/s, an observation from three - runs that suggests a cap rather than noise. - The same runs' HTTPS upload to the public internet took 1.40 times Linux's time. @@ -39,9 +37,11 @@ not findings: (`issues/the-intel-driver-writes-txdctl-and-tidv-against-the-82574-datasheets-newer-revision.md`); - the send window and the ACK clocking. -The sending CPU's clock read 1.6 to 3.8 GHz around these runs. That is -not excluded either: it is the T14's clock ceiling in -`issues/toyos-uses-what-modern-hardware-offers-for-speed.md`. +The clock does not separate the two systems. Each upload row's own +readings, `mhz` just after it and `mhz_settled` before it, were about +1.6 GHz under ToyOS (settled 1.6, 1.8 or 3.8 GHz). Under Linux they were +1.5 GHz (settled 1.5 or 4.2 GHz), and Linux still moved 113 MB/s. At most, +the clock matters through what ToyOS spends per byte sent. **Exit**: `osbench_lan` on the T14 reads `lan.wired.tcp.upload` within Linux's run-to-run range on the same path, or the cause is named and owned diff --git a/issues/toyos-uses-what-modern-hardware-offers-for-speed.md b/issues/toyos-uses-what-modern-hardware-offers-for-speed.md index 536f7cb72b..2ffebd594f 100644 --- a/issues/toyos-uses-what-modern-hardware-offers-for-speed.md +++ b/issues/toyos-uses-what-modern-hardware-offers-for-speed.md @@ -71,10 +71,13 @@ is nothing to enable. idle state reaches HWP's clock choice is inference. Next: a T14 scout whose idle CPUs wait in a deep C-state, read over the same windows. #1065's network session on the T14 (`ToyOSOrg/evidence` branch - `pr-1065`, `r3/`) read the same: each group's `cpu.clock` p50 was 1.5 to - 3.2 GHz under ToyOS against 3.8 to 4.2 GHz under Linux. That covers three - ToyOS runs and four Linux runs of `osbench lan` and `osbench internet`, both - of which mostly wait on the network. + `pr-1065`, `r3/`) shows the same ceiling in `cpu.clock`, whose 200 probes + ramp from idle. Its maximum was 3789 MHz under ToyOS in every group that + reached its top, against 4188 to 4190 MHz in every Linux group. That + covers three ToyOS and four Linux runs of `osbench lan` and + `osbench internet`. The probes' p50, which reads idle and ramp policy + (`issues/osbench-times-cpus-leaving-idle-and-not-their-settled-clock.md`), + is no reading of this ceiling. - **Global-line contention** is bounded at 2 to 3 % of C0 on the T14 and not attributed; next, the HITM count attributed to its lines on both benches (`issues/per-cpu-scheduler-and-pmm-state-shares-lines-every-cpu-writes.md`). diff --git a/tests/guest/osbench/compare/src/lib.rs b/tests/guest/osbench/compare/src/lib.rs index d5b03ce871..58c197eebe 100644 --- a/tests/guest/osbench/compare/src/lib.rs +++ b/tests/guest/osbench/compare/src/lib.rs @@ -136,10 +136,13 @@ pub fn range(values: &[u64]) -> Option { /// The exact two-sided Mann-Whitney test of `a` against `b`: `p`, and /// `least`, the smallest p these pooled values could give whichever runs -/// were whose. Ties count half a win, and the distribution is every split of -/// the pooled values into sets of the two sizes, so it holds the observed -/// ties: with none, `least` is 2/C(n + m, n); runs tied apart from the other -/// system's can lower it. +/// were whose. Ties count half a win. Over every split of the pooled values +/// into sets of the two sizes, `p` is the share of splits whose |2U - nm| is +/// at least the observed one: the distance from the centre, not twice the +/// one-sided p. Since the splits hold the observed ties, a tie can make a +/// split's mirror image less extreme. Then fewer splits reach the farthest +/// distance than the 2 of C(n + m, n) that untied values give, and `least` +/// is lower. #[derive(Clone, Copy, Debug, PartialEq)] pub struct Exact { pub p: f64,