diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a2f45a..9db2b94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,8 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + # Student code runs in the operating system's sandbox — Landlock and seccomp here, Seatbelt + # on macOS — so the suite, its sandbox probes included, runs on both. check: runs-on: ubuntu-latest steps: @@ -29,13 +31,27 @@ jobs: - run: cargo clippy --workspace --all-targets -- -D warnings - run: cargo test --workspace - # Releases ship macOS and Windows binaries; build them on every change, not first at tag time. + test-macos: + runs-on: macos-latest + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + # The runner's own python3 may be older than the 3.10 the harness needs. + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: cargo test --workspace + + # Releases ship Windows binaries too; build them on every change, not first at tag time. build: name: build (${{ matrix.os }}) strategy: fail-fast: false matrix: - os: [macos-latest, windows-latest] + os: [windows-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v7 diff --git a/CLAUDE.md b/CLAUDE.md index 98d7f3f..cd1b511 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,8 +82,9 @@ revisions) → Display/DB/HTML/Canvas. `rescore` adds a revision from saved evid ``` core/src/models/ Data models, TOML spec parsing, grading policies core/src/discovery Student file discovery + archive extraction -core/src/runner/ PythonExecutor (subprocess), orchestrator, sandbox (setrlimit), - parametrize, expander, oracle, linter +core/src/runner/ PythonExecutor (teacher + student processes), harness/*.py, + orchestrator, sandbox (Seatbelt / Landlock + seccomp, setrlimit), + records (framed), parametrize, expander, oracle, linter core/src/checker/ Checker trait + Rhai + Python checkers core/src/record Versioned evidence, score revisions, rescore reuse checks cli/src/db/ SQLite: students, sessions (= revisions), results, similarity @@ -93,11 +94,11 @@ cli/src/tui/ ratatui terminal UI: students/sessions/similarity tabs ## Key Design Decisions -**No pytest.** ScriptMark is its own test engine. Student code runs in subprocess via embedded Python helper script (`python.rs::HELPER_SCRIPT`). The helper: syntax-checks with py_compile, suppresses input()/print() during module load, calls target function, returns JSON on stdout. +**No pytest.** ScriptMark is its own test engine. A unit (a case or a scenario) runs as two Python processes from embedded scripts (`runner/harness/`): the teacher side (`teacher.py`: teacher modules, function checkers, the plan, per-request deadlines, the records) and the student side (`student.py`: the student module only, one call per request, under its timer and import guard). They share `common.py` (framing, the record and transfer value encodings, `call()`). ScriptMark joins the two with a pair of pipes and reads length-prefixed JSON records from the teacher side; values cross by value (shared-class instances rebuilt from the receiver's copy, other objects as snapshots). -**Sandboxed execution.** Student subprocess runs with: (1) `env_clear()` — no access to host env vars, (2) import allowlist — only safe stdlib + teacher-specified `allowed_imports`, (3) `setrlimit` — kernel-enforced CPU/file size/fd/process limits, (4) `tokio::join!` pipe reading + explicit `child.kill()` on timeout. Python interpreter resolved to absolute path at construction time so `env_clear()` doesn't change which interpreter runs. +**Sandboxed execution.** Each side runs confined by default (`runner/sandbox.rs`): macOS Seatbelt via `sandbox-exec` with a generated profile, Linux Landlock + seccomp applied in `pre_exec`. The student side reads its interpreter and its own directory only; the teacher side also reads the bundle and a read-only source copy. Fail closed: no sandbox, no grading, unless `--no-sandbox` (recorded as `isolation: unisolated`). Both also get `env_clear()`, `setrlimit` (RLIMIT_AS only as a backstop), and an absolute interpreter path; the student side is watched against `memory_mb` beyond what it takes idle with every module the submission imports loaded (spec, case or scenario; undeclared, estimated by `runner/memory.rs` from `space` — O(n^2) by default — and the test data's size, 16 MiB–1 GiB; `harness/idle.py` measures the idle share once per set of imports, never running the submission). The import allowlist is a teaching rule, not a boundary. Sandbox tests need a real sandbox: run `cargo test` outside any other Seatbelt sandbox (Claude Code's included), and on Linux from a filesystem with stable inodes (not a Docker Desktop bind mount). -**TOML test specs** have three sections: `[vars]` (constants injected as Python globals), `[[setup]]` (function calls storing results for `$ref`), `[[cases]]` (scored test cases). Cases support `expect`, `expect_error`, checkers (`check = "sorted"` / `{ rhai = "..." }` / `{ python = "verifiers/check.py" }`), and `[cases.parametrize]` with oracle. `[meta]` supports `allowed_imports` for teacher-controlled package access. +**TOML test specs** have three sections: `[vars]` (constants injected as Python globals), `[[setup]]` (function calls storing results for `$ref`), `[[cases]]` (scored test cases). Cases support `expect`, `expect_error`, checkers (`check = "sorted"` / `{ rhai = "..." }` / `{ python = "verifiers/check.py" }` / `{ function = "..." }`, which may take the reserved `stdout` and `source`), and `[cases.parametrize]` with oracle. `[meta]` supports `allowed_imports` (a teaching rule) and `shared` (modules whose classes cross between the two sides by value). Prepare refuses a path into the bundle handed to student code: `data_files` stages it instead. **Grading** uses declared per-item points and aggregation in `assignment.toml`. The default grade is the raw score scaled to the configured total; curves are opt-in. @@ -109,8 +110,10 @@ Zero and withheld grades stay distinct. See the notes' teacher contract for poli ## Critical Files -- `src/core/src/runner/python.rs` — PythonExecutor and embedded harness. Starts the isolated Python subprocess and checks its output. -- `src/core/src/runner/sandbox.rs` — SandboxConfig + setrlimit application. Platform-conditional: RLIMIT_AS skipped on macOS, RLIMIT_* constants differ between Linux (c_uint) and macOS (c_int). +- `src/core/src/runner/python.rs` — PythonExecutor: stages a unit, starts and joins both sides, reads the records, watches memory, probes the interpreter's read roots. +- `src/core/src/runner/harness/{common,teacher,student}.py` — the two sides of a unit and what they share; `idle.py` is the student that measures what a submission's imports take, and `imports.py` reads those imports in a process of its own. +- `src/core/src/runner/sandbox.rs` — `Isolation`, `Confinement`, the Seatbelt profile and Landlock/seccomp rules, plus setrlimit. Platform-conditional: RLIMIT_AS skipped on macOS, RLIMIT_* constants differ between Linux (c_uint) and macOS (c_int). +- `src/core/src/runner/records.rs` — framed record reader with plan-derived bounds. - `src/core/src/runner/orchestrator.rs` — Runs prepared bundles per student, tokio parallel. - `src/core/src/models/spec.rs` — TOML test specification types. - `src/core/src/discovery.rs` — File discovery and archive extraction with size/count limits. diff --git a/Cargo.toml b/Cargo.toml index 4efc021..4a457c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ clap = { version = "4", features = ["derive", "env"] } # Scripting rhai = "1" +shlex = "2" # Display comfy-table = "7" @@ -53,6 +54,8 @@ calamine = "0.36" # System libc = "0.2" +landlock = "0.4" +seccompiler = "0.5" # Random rand = "0.9" diff --git a/README.md b/README.md index 4fd1601..4fe653d 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,16 @@ notes/ Private project documentation submodule ``` Run Cargo commands from the repository root. The CLI depends on `scriptmark-core`; -the core has no CLI, network, database or terminal UI dependencies. Python 3 is still -required to execute student submissions, teacher checkers and reference solutions. +the core has no CLI, network, database or terminal UI dependencies. Python 3.10 or newer +is still required to execute student submissions, teacher checkers and reference +solutions. ScriptMark is distributed as a Rust CLI; it does not expose a Python package/API. +Student code and teacher code run in separate processes, each in the operating system's +sandbox: Seatbelt on macOS, Landlock and seccomp on Linux (5.13 or later, Landlock +enabled). Grading refuses where no sandbox can be applied; `--no-sandbox` grades without +one — on Windows, for now — and the results are marked unisolated. + ## Documentation and checkout Public documentation belongs in `docs/`. Private project documentation lives in @@ -387,13 +393,13 @@ See [the scoring contract](https://github.com/Acture/obsidian-vault/blob/project | `sorted` / `set_eq` / `contains` | Collection checks | | Rhai expression | `check = { rhai = "result != () && result > 0" }` | | Python script | `check = { python = "verifiers/check.py" }` | -| Teacher function | `check = { function = "is_valid" }`, run on the live value | +| Teacher function | `check = { function = "is_valid" }`, run on the returned value | | Output and files | `expected_stdout = "..."`, `expect_files = { "out.txt" = "..." }` | ## Features - **Custom test engine** -- subprocess execution, no pytest dependency -- **Isolated units** -- a process and directory per case, env isolation, import allowlist, setrlimit, timeout with kill; contains accidents, not a security sandbox ([details](https://github.com/Acture/obsidian-vault/blob/project/scriptmark/docs/test-bundles.md#what-grading-does-not-defend-against)) +- **Sandboxed units** -- every case runs as a teacher process and a student process, each in the OS sandbox: the student's reads only its own directory, with no network, no new programs and no signals to others; the teacher's reads the bundle but not other students' work or your environment. Resource limits, a memory watch and per-call deadlines contain accidents; built for introductory courses, not against a student attacking the grader ([details](https://github.com/Acture/obsidian-vault/blob/project/scriptmark/docs/test-bundles.md#what-grading-does-not-defend-against)) - **Parallel** -- tokio orchestrator, grades 80+ students in seconds - **Parametrize + oracle** -- random inputs with teacher reference implementations - **Per-item scoring** -- declared points and aggregation; zero and withheld kept apart in every export diff --git a/notes b/notes index 785624f..a7e30b6 160000 --- a/notes +++ b/notes @@ -1 +1 @@ -Subproject commit 785624fda41ce4229b2ad8cedd6c354853af5200 +Subproject commit a7e30b64dd7b259b1224fc0c4f35414a7524eb14 diff --git a/src/cli/src/main.rs b/src/cli/src/main.rs index fac41d5..63bdb17 100644 --- a/src/cli/src/main.rs +++ b/src/cli/src/main.rs @@ -20,6 +20,7 @@ use scriptmark_core::runner::generation::SeedSource; use scriptmark_core::runner::orchestrator::{self, RunOptions}; use scriptmark_core::runner::prepare::prepare; use scriptmark_core::runner::python::PythonExecutor; +use scriptmark_core::runner::sandbox::{self, Isolation}; use scriptmark_core::spec_loader::load_specs_from_dir; #[derive(Parser)] @@ -80,6 +81,27 @@ struct FrozenArgs { fresh: bool, } +#[derive(clap::Args)] +struct IsolationArgs { + /// Run student code without the operating system's sandbox. Each unit still runs as a + /// teacher side and a student side under resource limits, but student code can then + /// read and write whatever your user can — other students' files, the test bundle, your + /// home directory — and reach the network. The grading record marks the results + /// unisolated. For platforms ScriptMark cannot sandbox yet. + #[arg(long)] + no_sandbox: bool, +} + +impl IsolationArgs { + fn isolation(&self) -> Isolation { + if self.no_sandbox { + Isolation::Unisolated + } else { + Isolation::Sandboxed + } + } +} + #[derive(Parser)] struct GradeArgs { /// Directories containing student submissions @@ -133,6 +155,9 @@ struct GradeArgs { #[command(flatten)] frozen: FrozenArgs, + #[command(flatten)] + isolation: IsolationArgs, + /// Replace the grading record at --output even when it holds rescored revisions, /// discarding them. Long form only: there is no -f. #[arg(long)] @@ -183,6 +208,9 @@ struct RunArgs { #[command(flatten)] frozen: FrozenArgs, + #[command(flatten)] + isolation: IsolationArgs, + /// Replace the grading record at --output even when it holds rescored revisions, /// discarding them. Long form only: there is no -f. #[arg(long)] @@ -650,9 +678,17 @@ fn scored(view: &View, path: &Path) -> Result { /// How a summary names what it shows. fn shown(path: &Path, view: &View) -> String { - match view.revision { + let title = match view.revision { Some(n) => format!("{} (revision {n} of {})", path.display(), view.of), None => format!("{} (unscored)", path.display()), + }; + if view.sandboxed() { + title + } else { + format!( + "{title} — UNISOLATED: {}", + record::isolation_label(view.isolation) + ) } } @@ -670,14 +706,33 @@ async fn run_bundles( output: &Path, options: &FrozenArgs, ) -> Result<(Vec, Frozen, String)> { + if run_options.isolation.is_sandboxed() { + // Fail closed, before anything runs: a unit whose sandbox cannot be applied is + // never graded. + sandbox::available().map_err(anyhow::Error::msg).context( + "refusing to grade: student code cannot be sandboxed here (pass --no-sandbox to grade without isolation; the results will say so)", + )?; + } else { + eprintln!( + "warning: --no-sandbox: student code runs without a sandbox and can reach everything your user can; the results are marked unisolated" + ); + } let generation = match &options.replay { Some(path) => Generation::Replay(Frozen::load(path)?), None => Generation::fresh(), }; - let executor = Arc::new(PythonExecutor::with_python_cmd(&run_options.python)); - let bundles = prepare(specs, &generation, executor.clone(), timeout) - .await - .context("refusing to grade: the test bundle is not ready")?; + let executor = Arc::new( + PythonExecutor::with_python_cmd(&run_options.python).with_isolation(run_options.isolation), + ); + // Preparing runs teacher code in units of its own process groups too: Ctrl-C ends them. + let bundles = tokio::select! { + prepared = prepare(specs, &generation, executor.clone(), timeout) => prepared + .context("refusing to grade: the test bundle is not ready")?, + _ = tokio::signal::ctrl_c() => { + scriptmark_core::runner::python::kill_all_units(); + anyhow::bail!("interrupted while preparing: every running unit was stopped") + } + }; println!( "Prepared {} test bundle(s): {} unit(s) per student", bundles.len(), @@ -877,6 +932,7 @@ async fn execute( let spec_versions = record::spec_versions(&specs)?; let versions = record::submission_versions(&input.students)?; let matching = run_options.matching.clone(); + let isolation = run_options.isolation; let (mut reports, frozen, python) = run_bundles( &input.students, specs, @@ -891,6 +947,7 @@ async fn execute( spec_versions, timeout, python, + isolation, &frozen, (!frozen.is_empty()).then(|| frozen::beside(output)), ); @@ -940,6 +997,7 @@ async fn cmd_grade(args: GradeArgs) -> Result<()> { concurrency: args .concurrency .map(|n| usize::try_from(n).unwrap_or(usize::MAX)), + isolation: args.isolation.isolation(), }, args.timeout, &args.output, @@ -1053,6 +1111,7 @@ async fn cmd_run(args: RunArgs) -> Result<()> { concurrency: args .concurrency .map(|n| usize::try_from(n).unwrap_or(usize::MAX)), + isolation: args.isolation.isolation(), }, args.timeout, &args.output, @@ -1412,6 +1471,13 @@ async fn cmd_grades_push(args: GradesPushArgs) -> Result<()> { } } let view = record.view(args.revision)?; + if !view.sandboxed() { + eprintln!( + "warning: {} is {}: student code was not isolated when these grades were made", + args.results.display(), + record::isolation_label(view.isolation) + ); + } let revision = scored(&view, &args.results)?; // Only graded students are pushed — a real 0 included, a withheld grade never. The @@ -1527,6 +1593,7 @@ fn cmd_similarity(args: SimilarityArgs) -> Result<()> { fn cmd_report(args: ReportArgs) -> Result<()> { let (_, view) = load_view(&args.results, args.revision.revision)?; + let (sandboxed, isolation) = (view.sandboxed(), view.isolation); let reports = view.reports; let similarity = if let Some(sim_dir) = &args.similarity_dir { @@ -1553,7 +1620,16 @@ fn cmd_report(args: ReportArgs) -> Result<()> { None }; - report::generate_html_report(&reports, similarity.as_deref(), &args.title, &args.output)?; + let title = if sandboxed { + args.title.clone() + } else { + format!( + "{} — UNISOLATED: {}", + args.title, + record::isolation_label(isolation) + ) + }; + report::generate_html_report(&reports, similarity.as_deref(), &title, &args.output)?; println!("HTML report generated: {}", args.output.display()); Ok(()) diff --git a/src/cli/tests/record.rs b/src/cli/tests/record.rs index 23a77bd..b65bca8 100644 --- a/src/cli/tests/record.rs +++ b/src/cli/tests/record.rs @@ -6,6 +6,7 @@ use std::path::Path; use std::process::{Command, Output}; use scriptmark_core::record::Record; +use scriptmark_core::runner::sandbox::Isolation; const SPEC: &str = r#" [meta] @@ -877,3 +878,42 @@ fn a_canvas_record_rescores_from_its_bundle_and_refuses_another_attempt_or_excus std::fs::write(&payload_path, payload.to_string()).unwrap(); refused(rescore(dir, &[]), "2024002 is excused now"); } + +/// OSS-10: a record says whether student code ran in its sandbox, and a summary of one +/// that did not says so. +#[test] +fn a_run_without_the_sandbox_is_recorded_as_unisolated() { + let dir = bench(); + for (flags, isolated) in [(&[][..], true), (&["--no-sandbox"][..], false)] { + let mut args = vec![ + "grade", + "submissions", + "-t", + "tests", + "-o", + "out/result.json", + "--force", + ]; + args.extend(flags); + let output = scriptmark(dir.path(), &args); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let text = std::fs::read_to_string(dir.path().join("out/result.json")).unwrap(); + let record = Record::load(&dir.path().join("out/result.json")).unwrap(); + let expected = if isolated { + Isolation::Sandboxed + } else { + Isolation::Unisolated + }; + assert_eq!(record.evidence.bundle.isolation, Some(expected)); + assert!(text.contains("\"isolation\""), "{text}"); + let summary = scriptmark(dir.path(), &["summarize", "out/result.json"]); + assert_eq!( + String::from_utf8_lossy(&summary.stdout).contains("UNISOLATED"), + !isolated + ); + } +} diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index b8586ea..387f309 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -37,4 +37,9 @@ rars = "0.9.4" bzip2-rs = "0.1.2" lzma-rs = "0.3.0" tempfile = "3" +shlex = { workspace = true } sha2 = "0.10" + +[target.'cfg(target_os = "linux")'.dependencies] +landlock = { workspace = true } +seccompiler = { workspace = true } diff --git a/src/core/src/checker/python_checker.rs b/src/core/src/checker/python_checker.rs index bfbc2e4..587fade 100644 --- a/src/core/src/checker/python_checker.rs +++ b/src/core/src/checker/python_checker.rs @@ -1,5 +1,5 @@ use std::io::{Read, Write}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::time::Duration; @@ -13,10 +13,16 @@ use super::{CheckError, CheckInput, CheckOutput, Checker}; /// /// The script's existence is checked when the spec loads, so a script that then times out, /// crashes or prints no verdict did so on this student's value: the answer is rejected. +/// +/// It runs with a cleared environment and a private directory for its working directory, +/// `HOME` and `TMPDIR`; confined, in the teacher's sandbox, it reads only its interpreter +/// and the places it is given — the test bundle — and writes only that directory. pub struct PythonChecker { pub script_path: PathBuf, pub python_cmd: String, pub timeout_secs: u64, + /// What it may read besides its interpreter, when it runs in the teacher's sandbox. + pub confined: Option>, } impl PythonChecker { @@ -25,6 +31,7 @@ impl PythonChecker { script_path: script_path.into(), python_cmd: "python3".to_string(), timeout_secs: 10, + confined: None, } } @@ -32,6 +39,41 @@ impl PythonChecker { self.python_cmd = cmd.into(); self } + + /// Run in the teacher's sandbox, reading `read` besides the interpreter. + pub fn confined(mut self, read: Vec) -> Self { + self.confined = Some(read); + self + } + + fn command(&self, scratch: &Path) -> Result { + let mut cmd = match &self.confined { + Some(read) => { + let confinement = crate::runner::python::teacher_confinement( + &self.python_cmd, + read.clone(), + vec![scratch.to_path_buf()], + false, + )?; + crate::runner::sandbox::confined(Path::new(&self.python_cmd), &confinement)? + } + None => Command::new(&self.python_cmd), + }; + cmd.env_clear() + .env("PATH", "/usr/bin:/usr/local/bin:/opt/homebrew/bin") + .env("HOME", scratch) + .env("TMPDIR", scratch) + .current_dir(scratch) + // No user site-packages and no bytecode beside the script; the script's own + // directory stays on sys.path, for the helpers beside it. + .args(["-s", "-B"]) + .arg(&self.script_path); + #[cfg(windows)] + if let Some(root) = std::env::var_os("SYSTEMROOT") { + cmd.env("SYSTEMROOT", root); + } + Ok(cmd) + } } impl Checker for PythonChecker { @@ -40,61 +82,39 @@ impl Checker for PythonChecker { let input_json = serde_json::to_string(input) .map_err(|e| CheckError::environment(format!("could not encode checker input: {e}")))?; - let mut child = Command::new(&self.python_cmd) - .arg(&self.script_path) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() + let scratch = tempfile::Builder::new() + .prefix("scriptmark-check-") + .tempdir() + .and_then(|dir| std::fs::canonicalize(dir.path()).map(|path| (dir, path))) .map_err(|e| { - CheckError::environment(format!("could not start checker '{script}': {e}")) + CheckError::environment(format!("could not stage checker '{script}': {e}")) })?; - - // Write and read on their own threads: a checker that ignores its input, or prints - // more than a pipe holds, must not wedge the grader until the timeout. - if let Some(mut stdin) = child.stdin.take() { - std::thread::spawn(move || { - // A checker that exits without reading its input closes the pipe; its output decides. - let _ = stdin.write_all(input_json.as_bytes()); - }); - } - let drain = |pipe: Option>| { - std::thread::spawn(move || { - let mut buf = String::new(); - if let Some(mut pipe) = pipe { - let _ = pipe.read_to_string(&mut buf); - } - buf - }) - }; - let stdout = drain( - child - .stdout - .take() - .map(|p| Box::new(p) as Box), - ); - let stderr = drain( - child - .stderr - .take() - .map(|p| Box::new(p) as Box), - ); - - let status = child - .wait_timeout(Duration::from_secs(self.timeout_secs)) - .map_err(|e| CheckError::environment(format!("lost checker '{script}': {e}")))?; - let Some(status) = status else { - let _ = child.kill(); - let _ = child.wait(); + let cmd = self.command(&scratch.1).map_err(|e| { + CheckError::environment(format!("cannot sandbox checker '{script}': {e}")) + })?; + let ran = run_with_timeout( + cmd, + Some(input_json), + Duration::from_secs(self.timeout_secs), + ) + .map_err(|e| { + CheckError::environment(format!( + "could not start checker '{script}': {}", + crate::runner::sandbox::spawn_error(&e) + )) + })?; + let Some(Ran { + status, + stdout, + stderr, + }) = ran + else { return Err(CheckError::student(format!( "checker '{script}' timed out after {}s", self.timeout_secs ))); }; - - let stdout = stdout.join().unwrap_or_default(); if !status.success() && stdout.trim().is_empty() { - let stderr = stderr.join().unwrap_or_default(); return Err(CheckError::student(format!( "checker '{script}' exited with {status}: {}", stderr.trim() @@ -109,6 +129,87 @@ impl Checker for PythonChecker { } } +/// What a finished tool left: how it ended, and what it printed. +pub(crate) struct Ran { + pub status: std::process::ExitStatus, + pub stdout: String, + pub stderr: String, +} + +/// Run `cmd` with `stdin` written to it, killing it at `timeout`: `None` then. Its input is +/// written and its output read on their own threads, so a tool that ignores its input, or +/// prints more than a pipe holds, cannot wedge the grader until the timeout. +pub(crate) fn run_with_timeout( + mut cmd: Command, + stdin: Option, + timeout: Duration, +) -> std::io::Result> { + #[cfg(unix)] + crate::runner::sandbox::limit_tool(&mut cmd, timeout.as_secs()); + let mut child = cmd + .stdin(if stdin.is_some() { + Stdio::piped() + } else { + Stdio::null() + }) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + if let (Some(text), Some(mut pipe)) = (stdin, child.stdin.take()) { + std::thread::spawn(move || { + // A tool that exits without reading its input closes the pipe; its output decides. + let _ = pipe.write_all(text.as_bytes()); + }); + } + let drain = |pipe: Option>| { + std::thread::spawn(move || { + let mut buf = String::new(); + if let Some(mut pipe) = pipe { + let _ = pipe.read_to_string(&mut buf); + } + buf + }) + }; + let stdout = drain( + child + .stdout + .take() + .map(|p| Box::new(p) as Box), + ); + let stderr = drain( + child + .stderr + .take() + .map(|p| Box::new(p) as Box), + ); + // Its own process group: whatever it started goes with it, so nothing left behind + // holds its pipes open. + let group = child.id(); + let end_group = || { + #[cfg(unix)] + if let Ok(group) = i32::try_from(group) { + // SAFETY: killpg only sends a signal; the group may already be gone. + unsafe { + libc::killpg(group, libc::SIGKILL); + } + } + #[cfg(not(unix))] + let _ = group; + }; + let Some(status) = child.wait_timeout(timeout)? else { + end_group(); + let _ = child.kill(); + let _ = child.wait(); + return Ok(None); + }; + end_group(); + Ok(Some(Ran { + status, + stdout: stdout.join().unwrap_or_default(), + stderr: stderr.join().unwrap_or_default(), + })) +} + // wait_timeout is not in std — implement using a thread trait WaitTimeout { fn wait_timeout( diff --git a/src/core/src/export.rs b/src/core/src/export.rs index 61fc448..04d4d1f 100644 --- a/src/core/src/export.rs +++ b/src/core/src/export.rs @@ -472,6 +472,10 @@ pub fn about(record: &Record, n: u32) -> Result { } rows.extend([ ("run_by".into(), Cell::text(&evidence.scriptmark)), + ( + "isolation".into(), + Cell::text(crate::record::isolation_label(evidence.bundle.isolation)), + ), ("scored_by".into(), Cell::text(&revision.scriptmark)), ( "derived_items".into(), diff --git a/src/core/src/models/result.rs b/src/core/src/models/result.rs index 8bedded..57cd9b4 100644 --- a/src/core/src/models/result.rs +++ b/src/core/src/models/result.rs @@ -53,6 +53,8 @@ pub enum Cause { Timeout, /// The process died, or was killed at its deadline, during this call. Killed, + /// The student side went past its memory limit during this call, and was ended. + Memory, /// Not run: an earlier call in the unit hung or killed the process. NotRun, /// Not run: a value it needs was never produced. diff --git a/src/core/src/models/spec.rs b/src/core/src/models/spec.rs index ea8af13..9dea244 100644 --- a/src/core/src/models/spec.rs +++ b/src/core/src/models/spec.rs @@ -495,6 +495,16 @@ pub struct TestCase { #[serde(default)] pub timeout: Option, + /// The most memory the case's student process may hold beyond what it takes idle, in + /// MiB. Not on a scenario step: its steps share their scenario's process. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory_mb: Option, + + /// The space the algorithm may use, as `O(n)` and the like: the memory is then estimated + /// from it and the size of the test data. Not beside `memory_mb`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub space: Option, + /// Generate concrete cases from this one. #[serde(default)] pub parametrize: Option, @@ -568,11 +578,30 @@ pub struct TestMeta { #[serde(default)] pub data_files: Vec, + /// Teacher modules whose classes cross between the two sides of a unit by value. Each + /// is copied beside the student's file, where student code imports it without an + /// `allowed_imports` entry, and the teacher side imports its own copy from the bundle: + /// an instance one side sends is rebuilt from the other side's copy of its class. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub shared: Vec, + + /// The most memory a unit's student process may hold beyond what it takes idle, in MiB, + /// where a case or scenario does not say. Where nothing does, it is estimated from + /// `space`, or `DEFAULT_SPACE`, and the size of the unit's test data. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory_mb: Option, + + /// The space the algorithm may use, as `O(n)` and the like: the memory is then estimated + /// from it and the size of the test data. Not beside `memory_mb`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub space: Option, + /// Refused: every case now has its own process. #[serde(default)] pub copy_refs: Option, - /// Extra packages allowed in student code (beyond safe stdlib). + /// Extra packages allowed in student code (beyond safe stdlib). An entry is a module + /// name, and allows its top-level package: `numpy`, `matplotlib.pyplot`. #[serde(default)] pub allowed_imports: Vec, } @@ -642,6 +671,16 @@ pub struct Scenario { #[serde(default)] pub timeout: Option, + /// The most memory the scenario's student process may hold beyond what it takes idle, + /// in MiB. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory_mb: Option, + + /// The space the algorithm may use, as `O(n)` and the like: the memory is then estimated + /// from it and the size of the test data. Not beside `memory_mb`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub space: Option, + /// Runs once, at the start of the scenario, after the top-level setup. #[serde(default)] pub setup: Vec, diff --git a/src/core/src/record.rs b/src/core/src/record.rs index 3af8fdc..de0a45d 100644 --- a/src/core/src/record.rs +++ b/src/core/src/record.rs @@ -23,6 +23,7 @@ use crate::models::{ }; use crate::runner::answers::{digest, fingerprint}; use crate::runner::frozen::{Frozen, scratch}; +use crate::runner::sandbox::Isolation; /// The layout of the file. A newer one is refused by number, never half-read. pub const FORMAT: u32 = 1; @@ -121,6 +122,20 @@ pub struct BundleVersion { /// The frozen inputs and answers the batch used, when it had any. #[serde(default, skip_serializing_if = "Option::is_none")] pub frozen: Option, + /// Whether student and teacher code ran in their sandboxes. Absent from a record made + /// before ScriptMark sandboxed them, whose bytes and digest it keeps: nothing was + /// isolated then. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub isolation: Option, +} + +/// How a record's evidence was gathered, in words. +pub fn isolation_label(isolation: Option) -> &'static str { + match isolation { + Some(Isolation::Sandboxed) => "sandboxed", + Some(Isolation::Unisolated) => "unisolated (--no-sandbox)", + None => "unisolated (graded before ScriptMark sandboxed student code)", + } } /// One test spec as it ran. @@ -187,6 +202,15 @@ pub struct View { pub reports: Vec, /// The items the revision scored, in declaration order; empty when unscored. pub items: Vec, + /// How the evidence was gathered: an unisolated run is shown as one. + pub isolation: Option, +} + +impl View { + /// Student code ran in its sandbox. + pub fn sandboxed(&self) -> bool { + self.isolation == Some(Isolation::Sandboxed) + } } #[derive(Debug, thiserror::Error)] @@ -377,6 +401,7 @@ impl Record { of: self.revisions.len(), reports, items: revision.map(|r| r.policy.items.clone()).unwrap_or_default(), + isolation: self.evidence.bundle.isolation, }) } @@ -607,6 +632,7 @@ fn sources(spec: &TestSpec) -> Result, String> { .map(|p| spec.dir.join(p)) .collect(); let mut python: BTreeSet = spec.meta.imports.iter().map(PathBuf::from).collect(); + python.extend(spec.meta.shared.iter().map(|p| spec.dir.join(p))); for case in spec .cases .iter() @@ -659,6 +685,7 @@ pub fn bundle_version( mut specs: Vec, timeout: u64, python: String, + isolation: Isolation, frozen: &Frozen, frozen_path: Option, ) -> BundleVersion { @@ -679,6 +706,7 @@ pub fn bundle_version( digest: digest(&serde_json::to_vec(&(timeout, semantics)).expect("plain JSON")), timeout, python, + isolation: Some(isolation), frozen: (!frozen.is_empty()).then(|| FrozenLink { path: frozen_path, digest: digest(frozen.to_json().as_bytes()), @@ -934,7 +962,14 @@ mod tests { }, attempt_policy: AttemptPolicy::default(), matching: matching::Config::default(), - bundle: bundle_version(Vec::new(), 10, "python3".into(), &frozen, None), + bundle: bundle_version( + Vec::new(), + 10, + "python3".into(), + Isolation::Sandboxed, + &frozen, + None, + ), students, } } diff --git a/src/core/src/runner/answers.rs b/src/core/src/runner/answers.rs index 9ec564c..93e658c 100644 --- a/src/core/src/runner/answers.rs +++ b/src/core/src/runner/answers.rs @@ -70,6 +70,12 @@ impl Contract { pub fn of(spec: &TestSpec, executor: &E, timeout: u64) -> Result { let mut sources: BTreeMap = BTreeMap::new(); let mut paths: BTreeSet = spec.meta.imports.iter().cloned().collect(); + paths.extend( + spec.meta + .shared + .iter() + .map(|p| spec.dir.join(p).to_string_lossy().into_owned()), + ); paths.extend( spec.meta .data_files @@ -98,11 +104,14 @@ impl Contract { runtime: executor.identity()?, sources, protocol: digest( - concat!( + [ include_str!("oracle.rs"), - include_str!("harness.py"), - include_str!("../checker/rhai_checker.rs") - ) + include_str!("../checker/rhai_checker.rs"), + ] + .iter() + .chain(&crate::runner::python::PROTOCOL) + .copied() + .collect::() .as_bytes(), ), }) diff --git a/src/core/src/runner/executor.rs b/src/core/src/runner/executor.rs index 330fc84..0ae3e2a 100644 --- a/src/core/src/runner/executor.rs +++ b/src/core/src/runner/executor.rs @@ -54,6 +54,17 @@ pub struct UnitPlan { pub vars: Arc>, /// `(source, relative destination)` pairs staged into the working directory. pub data_files: Vec<(PathBuf, String)>, + /// The test bundle's directory: what teacher code may read, besides its own modules. + pub bundle: PathBuf, + /// Shared modules, as `(source in the bundle, relative destination)`: staged beside the + /// student's file, and imported by the teacher side from the bundle. + pub shared: Vec<(PathBuf, String)>, + /// The most memory the student side may hold beyond what it takes idle with the + /// submission's imports loaded, in MiB, before it is ended. + pub memory_mb: u64, + /// How `memory_mb` was estimated, where the spec does not declare it: + /// `O(n^2) over 1000 elements`. + pub memory_basis: Option, pub allowed_imports: Vec, pub load_timeout: u64, pub setup: Vec, @@ -61,22 +72,31 @@ pub struct UnitPlan { } impl UnitPlan { - /// Seconds the unit may run in total: every call's own timeout, plus slack. The + /// Seconds the unit may run in total: every request the teacher side makes with the + /// grace it allows each, plus the time both sides have to start. The teacher side gives + /// up on a request well before this; it is the backstop for the teacher side itself. The /// kernel's CPU limit sits just above it as a backstop for a single busy core. pub fn deadline_secs(&self) -> u64 { + let grace = |timeout: u64| timeout.saturating_add(GRACE_SECS); let calls = match &self.script { - Some(script) => script.timeout, + Some(script) => grace(script.timeout), None => self .setup .iter() .chain(&self.steps) - .map(CallPlan::budget) - .fold(self.load_timeout, u64::saturating_add), + .map(|c| c.budget(grace)) + .fold(grace(self.load_timeout), u64::saturating_add), }; - calls.saturating_add(2) + calls.saturating_add(2 * STARTUP_SECS) } } +/// Seconds the teacher side waits past a request's own timeout before it gives up on the +/// student side: the student's own timer has fired by then, and its answer is late. +pub const GRACE_SECS: u64 = 2; +/// Seconds each side has to start: an interpreter under load, and the teacher modules. +pub const STARTUP_SECS: u64 = 10; + /// A script run: the unit's only call. #[derive(Debug, Clone, Serialize)] pub struct ScriptRun { @@ -101,9 +121,9 @@ pub struct CallPlan { } impl CallPlan { - fn budget(&self) -> u64 { - self.timeout - .saturating_add(self.check.as_ref().map_or(0, |_| self.timeout)) + /// The call's time, and its checker's, each with `grace` applied. + fn budget(&self, grace: impl Fn(u64) -> u64) -> u64 { + grace(self.timeout).saturating_add(self.check.as_ref().map_or(0, |_| grace(self.timeout))) } } @@ -129,6 +149,9 @@ pub struct Export { /// The parameters, when the export is callable and introspectable. #[serde(default)] pub params: Option>, + /// A path into the test bundle: teacher code may open it, student code cannot. + #[serde(default)] + pub bundle_path: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -245,7 +268,7 @@ pub enum CheckObservation { Error(ErrorInfo), } -/// How the unit's process ended. +/// How one side's process ended. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum Exit { @@ -253,26 +276,52 @@ pub enum Exit { Signal(i32), /// Killed by the grader at the unit's deadline. Deadline, + /// Killed by the grader for holding more than the unit's `memory_mb` beyond what the + /// student side takes idle with the submission's imports loaded (`idle_mb`); `basis` + /// says how an undeclared limit was estimated. + Memory { + limit_mb: u64, + idle_mb: u64, + basis: Option, + }, + /// Still running when the teacher side finished, and ended by the grader then. + Stopped, /// Never started. Spawn(String), } +impl Exit { + /// Ended by itself, without an error. + pub fn is_clean(&self) -> bool { + *self == Exit::Code(0) + } +} + /// Everything a unit reported, in plan order. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct UnitObservation { - /// Teacher modules imported and the harness running. + /// Teacher modules imported, and the student side started and ready. pub ready: bool, pub load: Option, pub setup: Vec, pub steps: Vec, pub checks: BTreeMap, pub fatal: Option, + /// The teacher side gave up on the student side: nothing after this call ran. + pub stopped: Option, pub done: bool, + /// How the student side ended. pub exit: Exit, - /// The record stream broke: the unit's records cannot be trusted past this point. + /// How the teacher side ended. + pub teacher_exit: Exit, + /// The teacher side's record stream broke: its records cannot be trusted past here. pub protocol_error: Option, - /// The tail of stderr, for diagnosing a crash. + /// The tail of each side's stderr, for diagnosing a crash. pub stderr: String, + /// Each side ran in its own sandbox. Without one, student code can reach the teacher + /// side — end it, or write into its records — so what breaks there may be the + /// student's doing. + pub sandboxed: bool, } impl UnitObservation { @@ -285,24 +334,50 @@ impl UnitObservation { steps: Vec::new(), checks: BTreeMap::new(), fatal: None, + stopped: None, done: false, + teacher_exit: exit.clone(), exit, protocol_error: None, stderr: String::new(), + sandboxed: false, } } } -/// How the record stream broke. +/// Where and why the teacher side stopped asking the student side anything. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Stopped { + pub phase: Phase, + pub index: usize, + pub reason: StopReason, + #[serde(default)] + pub message: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum StopReason { + /// A request ran past its timeout and grace: the student's timer did not stop it. + Deadline, + /// The student side went away mid-request. + Exited, + /// The student side sent more than the request allows. + Flooded, + /// The student side sent something no request allows. + Garbled, +} + +/// How the teacher side's record stream broke. Student code cannot reach it, so each is +/// the harness's or the teacher code's doing. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ProtocolError { - /// A framed record did not parse — the harness wrote something it should not have. + /// A framed record did not parse. Unreadable(String), - /// A record repeated or came out of order — somebody else wrote it. + /// A record repeated or came out of order. Tampered(String), - /// More reached the record channel than the harness ever writes: something other than - /// the harness wrote to it. + /// A record longer than any this unit can produce, or more records than it has. Flooded(String), } diff --git a/src/core/src/runner/harness.py b/src/core/src/runner/harness.py deleted file mode 100644 index 4b4e223..0000000 --- a/src/core/src/runner/harness.py +++ /dev/null @@ -1,723 +0,0 @@ -"""ScriptMark's Python harness: runs one execution unit and reports what happened. - -It reports observations, never verdicts about who is at fault: the grader derives -those from which call was running, what kind of code it runs, and how the process -ended. Every piece of student code — loading the module, finding a function, reading -an attribute, serialising a value — runs inside `call()`, under its timer and import -guard, so a failure there is that call's outcome and never a harness crash. - -The nonce framing protects the records from accidental output only; a student -determined to reach harness state from inside the same process could, which is why -nothing here decides anything. -""" - -import sys - -# The unit's working directory holds the student's file, and `python -c` puts it first -# on sys.path: a submission named json.py must not become the harness's json. -sys.path[:] = [p for p in sys.path if p] - -import builtins # noqa: E402 -import importlib.machinery # noqa: E402 -import importlib.util # noqa: E402 -import inspect # noqa: E402 -import io # noqa: E402 -import json # noqa: E402 -import math # noqa: E402 -import os # noqa: E402 -import runpy # noqa: E402 -import signal # noqa: E402 -import time # noqa: E402 -from typing import TYPE_CHECKING # noqa: E402 - -if TYPE_CHECKING: - from matching import FunctionRules, MatchDecision, function_decision - -with open(sys.argv[1], encoding="utf-8") as _fh: - PAYLOAD = json.load(_fh) -os.unlink(sys.argv[1]) - -_PREFIX = f"\n@@scriptmark:{PAYLOAD['nonce']}@@ " -# The record channel: a descriptor of its own where the grader provides one (fd 3 on unix), -# which no student output setting can reach; otherwise a private copy of stdout, where the -# nonce framing keeps accidental output out. `replace`: a lone surrogate a student printed or -# returned must not crash it. -_CHANNEL_FD = PAYLOAD.get("channel", 1) -_channel = os.fdopen( - _CHANNEL_FD if _CHANNEL_FD != 1 else os.dup(1), "w", encoding="utf-8", errors="replace" -) -# Between calls, stdout is a sink: what a student's leftover thread prints is not evidence. -_quiet = open(os.devnull, "w", encoding="utf-8") -sys.stdout = _quiet -_real_stdin = sys.stdin -STUDENT = PAYLOAD.get("subject", "student") == "student" -# What one call's record can carry; python.rs sizes the record channel from these. -STDOUT_LIMIT = 64 * 1024 -FILE_LIMIT = 1024 * 1024 -VALUE_LIMIT = 4 * 1024 * 1024 -MESSAGE_LIMIT = 64 * 1024 -DEPTH_LIMIT = 100 - - -def emit(record): - _channel.write(_PREFIX + json.dumps(record, ensure_ascii=False) + "\n") - _channel.flush() - - -def finish(): - """Stop now: threads a student left running must not hold the unit open.""" - emit({"kind": "done"}) - _channel.close() - os._exit(0) - - -# --- The import guard ----------------------------------------------------------------- -# A policy check, not a security boundary: importlib.import_module bypasses it. It looks -# at where the import statement runs, so an allowlisted stdlib module importing its own -# internals is never mistaken for the student importing them. -ALLOWED = frozenset( - { - # core - "builtins", "sys", "io", "abc", "enum", "types", "typing", "dataclasses", - "collections", "functools", "itertools", "operator", "copy", "copyreg", - # math/science - "math", "cmath", "decimal", "fractions", "numbers", "random", "statistics", - # strings/text - "string", "re", "difflib", "textwrap", "unicodedata", "codecs", "encodings", - # data formats - "json", "csv", "struct", "base64", "binascii", "hashlib", "hmac", - # time - "time", "datetime", "calendar", "zoneinfo", - # data structures - "array", "heapq", "bisect", "queue", "graphlib", - # functional - "contextlib", "contextvars", - # introspection - "inspect", "dis", "token", "tokenize", "ast", "symtable", - # error handling - "traceback", "warnings", "errno", "faulthandler", - # internals needed by Python itself - "importlib", "_thread", "__future__", "_collections_abc", "_abc", - "_functools", "_operator", "_io", "_json", "_string", "_struct", - "_sre", "sre_compile", "sre_constants", "sre_parse", "_locale", - "_frozen_importlib", "_frozen_importlib_external", "_imp", - "_warnings", "_weakref", "_weakrefset", "_threading_local", - "_codecs", "_signal", "_stat", "_opcode", "_typing", - "genericpath", "posixpath", "ntpath", "stat", "keyword", - "reprlib", "pprint", "linecache", - # testing (students may use unittest/doctest) - "unittest", "doctest", - # filesystem (pathlib is safe — open() is a builtin anyway) - "pathlib", "fnmatch", "glob", "tempfile", - # misc safe - "secrets", "uuid", "ipaddress", "colorsys", "timeit", "py_compile", "compileall", - "gzip", "bz2", "lzma", "zipfile", "tarfile", "xml", "html", "email", - "logging", "argparse", "getopt", "configparser", "tomllib", "threading", "concurrent", - } -) | frozenset(PAYLOAD.get("allowed_imports", [])) -STUDENT_MARK = "__scriptmark_student__" -_original_import = builtins.__import__ - - -def _guarded_import(name, globals=None, locals=None, fromlist=(), level=0): - # `__import__('os')` called directly passes no globals: look at the caller instead. - scope = globals if globals is not None else sys._getframe(1).f_globals - if scope.get(STUDENT_MARK) and level == 0: - if name.split(".")[0] not in ALLOWED: - raise ImportError(f"Module '{name}' is not allowed in student code") - return _original_import(name, globals, locals, fromlist, level) - - -# --- Timers --------------------------------------------------------------------------- -class CallTimeout(BaseException): - """Raised by SIGALRM. A BaseException, so `except Exception` cannot swallow it.""" - - -_fired = [False] - - -def _on_alarm(signum, frame): - _fired[0] = True - raise CallTimeout() - - -_HAS_TIMER = hasattr(signal, "setitimer") -if _HAS_TIMER: - signal.signal(signal.SIGALRM, _on_alarm) - - -class _CappedBytes(io.BytesIO): - """Keeps at most STDOUT_LIMIT bytes and remembers whether it dropped any.""" - - truncated = False - - def write(self, b): - room = STDOUT_LIMIT - self.tell() - if len(b) > room: - self.truncated = True - if room > 0: - super().write(bytes(b[:room])) - return len(b) - - def writelines(self, lines): - # The C method would write past the limit without calling `write`. - for line in lines: - self.write(line) - - def close(self): - """A student's wrapper around `sys.stdout.buffer` closes it when it is collected: - what was written must survive that.""" - - -def capture(): - """A stdout that behaves like a real one — `buffer`, `reconfigure`, `print(flush=True)` — - and the bytes behind it, kept apart so a detached or closed wrapper cannot lose them.""" - raw = _CappedBytes() - return io.TextIOWrapper(raw, encoding="utf-8", errors="backslashreplace", write_through=True), raw - - -def captured(stream, raw): - try: - stream.flush() - except (ValueError, OSError): # detached or closed by the student - pass - # `io.BytesIO.write(buffer, b)` still reaches past `write`: cut here too. - data = raw.getvalue() - return data[:STDOUT_LIMIT].decode("utf-8", "replace"), raw.truncated or len(data) > STDOUT_LIMIT - - -# --- The one value serialiser ---------------------------------------------------------- -class Unserialisable(Exception): - pass - - -def to_json(value, depth=0, seen=None): - """What the grader sees of a Python value. Total: it never raises for a value's sake.""" - if isinstance(value, bool) or value is None or isinstance(value, str): - return value - if isinstance(value, int): - if -(2**63) <= value < 2**64: - return int(value) - # Past what the grader's JSON can hold exactly: tag it, exactly and with its sign, so - # it is never mistaken for a number and a record always parses. Decimal where Python - # will print it, hexadecimal past its 4300-digit limit. - try: - return {"$bigint": int.__repr__(value)} - except ValueError: - return {"$bigint": hex(value)} - if isinstance(value, float): - return value if math.isfinite(value) else repr(value) - if depth >= DEPTH_LIMIT: - return "" - seen = seen or set() - if id(value) in seen: - return "" - if isinstance(value, (list, tuple, set, frozenset, dict)): - seen = seen | {id(value)} - if isinstance(value, (list, tuple)): - return [to_json(v, depth + 1, seen) for v in value] - if isinstance(value, (set, frozenset)): - items = [to_json(v, depth + 1, seen) for v in value] - try: - return sorted(items) - except TypeError: # mixed types: any stable order will do - return sorted(items, key=lambda v: json.dumps(v, sort_keys=True)) - if isinstance(value, dict): - out = {} - for k, v in value.items(): - key = k if isinstance(k, str) else text_of(k) - if key in out: - raise Unserialisable(f"two keys both read as '{key}'") - out[key] = to_json(v, depth + 1, seen) - return out - return text_of(value) - - -def text_of(value): - """`str(value)`, or the type's name when the value's own `__str__` fails.""" - try: - text = str(value) - except BaseException: # never under a live timer: this is harness bookkeeping - return f"<{type(value).__name__}>" - return text if isinstance(text, str) else f"<{type(value).__name__}>" - - -def clip(message): - """A message kept to MESSAGE_LIMIT characters, saying how long it was.""" - if len(message) <= MESSAGE_LIMIT: - return message - return f"{message[:MESSAGE_LIMIT]}… (cut from {len(message)} characters)" - - -def error_of(exc): - info = { - "type": type(exc).__name__, - "types": [cls.__name__ for cls in type(exc).__mro__], - "message": clip(text_of(exc)), - } - if isinstance(exc, SystemExit): - # The exit status CPython would report: `exit()`, `exit(0)` and `exit(False)` succeed; - # any other code, `exit(0.0)` included, fails. - try: - code = exc.code - info["clean"] = code is None or (type(code) in (int, bool) and int(code) == 0) - except BaseException: - info["clean"] = False - return info - - -def stdin_stream(text): - return io.TextIOWrapper(io.BytesIO((text or "").encode("utf-8")), encoding="utf-8") - - -class Missing(Exception): - """The call's target does not exist.""" - - -class MatchingRequired(Exception): - def __init__(self, decision: MatchDecision) -> None: - self.decision: MatchDecision = decision - super().__init__(f"function matching needs review: {decision}") - - -class Unresolved(Exception): - """A `$ref` or `object` whose value was never produced in this unit.""" - - -def call(fn, timeout, stdin=None, guarded=True, serialise=True): - """Run one call — its lookup, its body and its serialisation — under its timer and guard. - - Returns (outcome, stdout, truncated, elapsed_ms, live_value).""" - out, raw = capture() - sys.stdout = out - sys.stdin = stdin_stream(stdin) if stdin is not False else _real_stdin - if guarded: - builtins.__import__ = _guarded_import - _fired[0] = False - start = time.perf_counter() - value = None - try: - if _HAS_TIMER and timeout: - signal.setitimer(signal.ITIMER_REAL, timeout, 0.05) - try: - value = fn() - if sys.stdout is not out: - # The student replaced stdout with their own wrapper: flush it in their scope. - sys.stdout.flush() - outcome = {"returned": {"value": wire(value) if serialise else None, "type": type(value).__name__}} - finally: - if _HAS_TIMER: - signal.setitimer(signal.ITIMER_REAL, 0) - except CallTimeout: - outcome = {"timeout": {}} - except Missing as exc: - outcome = {"missing": {"message": str(exc)}} - except MatchingRequired as exc: - outcome = {"matching": {"message": str(exc), "decision": exc.decision}} - except Unresolved as exc: - outcome = {"unresolved": {"name": str(exc)}} - except Unserialisable as exc: - outcome = {"unserialisable": {"type": "Unserialisable", "message": str(exc)}} - except BaseException as exc: # SystemExit and KeyboardInterrupt are the code's own too - outcome = {"raised": error_of(exc)} - finally: - builtins.__import__ = _original_import - sys.stdout = _quiet - sys.stdin = _real_stdin - if _fired[0]: - # The timer went off even if the code swallowed it with a bare `except:`. - outcome = {"timeout": {}} - elapsed = int((time.perf_counter() - start) * 1000) - stdout, truncated = captured(out, raw) - return outcome, stdout, truncated, elapsed, value - - -def utf8_size(text): - """The bytes `text` takes on the channel, counted a slice at a time rather than through - a second full copy of a value that may already fill most of the memory limit.""" - step = 1 << 20 - return sum(len(text[i : i + step].encode("utf-8", "replace")) for i in range(0, len(text), step)) - - -def wire(value): - """The value as a record carries it. One too large to report is replaced by its size: - the call still returned, and a function checker still judges the live value.""" - value = to_json(value) - size = utf8_size(json.dumps(value, ensure_ascii=False)) - if size > VALUE_LIMIT: - return {"$too_large": f"{size} bytes of JSON"} - return value - - -# --- Teacher modules ------------------------------------------------------------------- -def _removed_checker(*args, **kwargs): - raise RuntimeError( - "@checker no longer binds a checker to a function: put " - 'check = { function = "" } on each case it should judge ' - "(https://github.com/Acture/obsidian-vault/blob/project/scriptmark/" - "docs/test-bundles.md#migrating-an-older-spec)" - ) - - -def exports_of(module): - """`__all__` when present; otherwise public names, minus modules and minus classes and - functions the module merely imported from elsewhere.""" - names = getattr(module, "__all__", None) - if names is None: - names = [] - for name in dir(module): - if name.startswith("_"): - continue - value = getattr(module, name) - if inspect.ismodule(value): - continue - if (inspect.isclass(value) or inspect.isroutine(value)) and getattr( - value, "__module__", None - ) != module.__name__: - continue - names.append(name) - return {name: getattr(module, name) for name in names} - - -def load_teacher_modules(): - """Returns (exports, duplicates): a name two modules export differently is refused.""" - exports, owner, duplicates = {}, {}, {} - builtins.checker = _removed_checker - try: - for index, path in enumerate(PAYLOAD.get("imports", [])): - directory = os.path.dirname(path) - if directory not in sys.path: - # After the stdlib: a helper beside the module must not shadow it. - sys.path.append(directory) - spec = importlib.util.spec_from_file_location(f"teacher_mod_{index}", path) - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - for name, value in exports_of(module).items(): - if name in exports and not _same(exports[name], value): - duplicates[name] = [owner[name], path] - exports[name] = value - owner.setdefault(name, path) - finally: - del builtins.checker - return exports, duplicates - - -def _same(a, b): - """One object, or two equal ones: two modules that both define `TOL = 1e-6` agree.""" - if a is b: - return True - try: - return bool(a == b) - except BaseException: - return False - - -def params_of(fn): - """Each parameter's name, kind, and whether it has a default.""" - try: - parameters = inspect.signature(fn).parameters.values() - except (TypeError, ValueError): - return None - return [{"name": p.name, "kind": p.kind.name.lower(), "default": p.default is not p.empty} for p in parameters] - - -# --- Calls ----------------------------------------------------------------------------- -context = {} - - -def resolve(value): - if isinstance(value, str) and value.startswith("$"): - if value.startswith("$$"): - return value[1:] - if value[1:] not in context: - raise Unresolved(value[1:]) - return context[value[1:]] - if isinstance(value, list): - return [resolve(v) for v in value] - if isinstance(value, dict): - return {k: resolve(v) for k, v in value.items()} - return value - - -def matched_function( - module: object, name: str, evidence: list[MatchDecision | None] -) -> tuple[object, str]: - policy: FunctionRules = PAYLOAD.get("functions", {"aliases": {}, "overrides": {}}) - # Read attributes only once: dynamic exports/properties belong to the guarded call. - exports: dict[str, object] = dict(vars(module)) - absent: object = object() - for candidate in { - name, - *policy.get("aliases", {}).get(name, []), - *([policy["overrides"][name]] if name in policy.get("overrides", {}) else []), - }: - if candidate not in exports: - value: object = getattr(module, candidate, absent) - if value is not absent: - exports[candidate] = value - decision: MatchDecision = function_decision( - [candidate for candidate, obj in exports.items() if callable(obj)], name, policy - ) - evidence[0] = decision - selected: str | None = decision["selected"] - if selected is not None: - return exports[selected], selected - if ( - decision["state"] != "missing" - or name in policy.get("overrides", {}) - or name in policy.get("aliases", {}) - ): - raise MatchingRequired(decision) - raise Missing(f"function '{name}' not found") - - -def live(object_id): - if object_id not in context: - raise Unresolved(object_id) - return context[object_id] - - -def member(obj, object_id, name, what): - """`getattr(obj, name)`, read once. A name that does not exist is Missing; a property - that raises — AttributeError included — is the student's exception.""" - try: - return getattr(obj, name) - except AttributeError: - try: - inspect.getattr_static(obj, name) - except AttributeError: - if not hasattr(type(obj), "__getattr__"): - raise Missing(f"'{object_id}' has no {what} '{name}'") from None - raise - - -def invoke( - plan: dict[str, object], - student: object, - resolved: list[str], - evidence: list[MatchDecision | None], -) -> object: - """The call a plan names. Runs inside `call()`: the lookup is student code too.""" - kind, spec = next(iter(plan["target"].items())) - args = resolve(plan.get("args", [])) - if kind == "function": - if PAYLOAD.get("lookup") == "exact": - if not hasattr(student, spec["name"]): - raise Missing(f"function '{spec['name']}' not found") - fn, resolved[0] = getattr(student, spec["name"]), spec["name"] - else: - fn, resolved[0] = matched_function(student, spec["name"], evidence) - return fn(*args) - if kind == "method": - fn = member(live(spec["object"]), spec["object"], spec["name"], "method") - if not callable(fn): - raise Missing(f"'{spec['object']}.{spec['name']}' is not a method") - return fn(*args) - if kind == "attribute": - return member(live(spec["object"]), spec["object"], spec["name"], "attribute") - fn = context.get(spec["name"]) - if fn is None or not callable(fn): - raise Missing(f"teacher function '{spec['name']}' not found") - return fn(*args) - - -def verdict_of(returned): - if isinstance(returned, bool): - return {"verdict": {"pass": returned, "message": ""}} - if ( - isinstance(returned, tuple) - and len(returned) == 2 - and isinstance(returned[0], bool) - and isinstance(returned[1], (str, type(None))) - ): - return {"verdict": {"pass": returned[0], "message": clip(returned[1] or "")}} - return { - "error": { - "type": "CheckerContract", - "types": ["CheckerContract"], - "message": f"checker returned {type(returned).__name__}; expected bool or (bool, str)", - } - } - - -def run_check(check, value, stdout, timeout): - fn = context.get(check["function"]) - deps = {} - for param in (params_of(fn) or [])[2:]: - name = param["name"] - if "var" in param["kind"]: - continue - if name == "stdout": - deps[name] = stdout - elif name in context: - deps[name] = context[name] - elif not param["default"]: - # prepare saw this name in scope: it is an id whose producing call failed. - return {"unresolved": {"name": name}} - outcome, _, _, _, returned = call( - lambda: fn(value, check.get("expected"), **deps), timeout, guarded=False, serialise=False - ) - if "returned" in outcome: - return verdict_of(returned) - if "timeout" in outcome: - return {"error": {"type": "Timeout", "types": ["Timeout"], "message": f"checker timed out after {timeout}s"}} - raised = outcome.get("raised") or {"type": "Error", "types": [], "message": str(outcome)} - if "AssertionError" in raised["types"]: - return {"rejected": {"message": raised["message"]}} - return {"error": raised} - - -def observe_files(paths: list[str]) -> dict[str, str | None]: - seen: dict[str, str | None] = {} - for path in paths: - try: - with open(path, encoding="utf-8") as fh: - content: str = fh.read(FILE_LIMIT + 1) - seen[path] = content if len(content) <= FILE_LIMIT else None - except ( - OSError, - ValueError, - ): # absent, oversized or invalid UTF-8 is no text expectation - seen[path] = None - return seen - - -def run_calls(phase, calls, student): - """Run setup calls or steps in order. Returns False when a setup call failed.""" - setup = phase == "setup" - for index, plan in enumerate(calls): - requested = next(iter(plan["target"].values()))["name"] - resolved = [requested] - evidence: list[MatchDecision | None] = [None] - outcome, stdout, truncated, elapsed, value = call( - lambda: invoke(plan, student, resolved, evidence), - plan["timeout"], - plan.get("stdin"), - guarded=STUDENT and "teacher" not in plan["target"], - serialise=not setup, # setup values are never judged, only passed on - ) - record = { - "kind": "call", - "phase": phase, - "index": index, - "target": {"requested": requested, "resolved": resolved[0], "matching": evidence[0]}, - "outcome": outcome, - "stdout": stdout, - "stdout_truncated": truncated, - "elapsed_ms": elapsed, - } - if plan.get("files"): - record["files"] = observe_files(plan["files"]) - emit(record) - returned = "returned" in outcome - if returned and plan.get("id"): - context[plan["id"]] = value - if returned and plan.get("check"): - verdict = run_check(plan["check"], value, stdout, plan["timeout"]) - emit({"kind": "check", "index": index, **verdict}) - if setup and not returned: - return False - return True - - -def compiles(path): - """Compile the student's file in memory — writing no __pycache__ beside it — and record a - syntax error as the load call's outcome.""" - try: - with open(path, "rb") as fh: - compile(fh.read(), path, "exec") - return True - except (SyntaxError, ValueError, RecursionError, MemoryError) as exc: - emit({"kind": "call", "phase": "load", "index": 0, - "outcome": {"raised": {"type": "SyntaxError", "types": ["SyntaxError"], "message": text_of(exc)}}, - "stdout": "", "stdout_truncated": False, "elapsed_ms": 0}) - return False - - -def load_student(): - path = PAYLOAD["student"] - if not compiles(path): - return None - module_holder = [] - - def load(): - # An explicit loader: `Lab5.PY` is Python too, whatever its extension. - loader = importlib.machinery.SourceFileLoader("student_mod", path) - spec = importlib.util.spec_from_loader("student_mod", loader) - module = importlib.util.module_from_spec(spec) - if STUDENT: - setattr(module, STUDENT_MARK, True) - for key, val in PAYLOAD.get("vars", {}).items(): - setattr(module, key, val) - sys.modules["student_mod"] = module - loader.exec_module(module) - module_holder.append(module) - - original_input = builtins.input - builtins.input = lambda *a, **k: "0" - try: - outcome, stdout, truncated, elapsed, _ = call( - load, PAYLOAD["load_timeout"], guarded=STUDENT, serialise=False - ) - finally: - builtins.input = original_input - emit({"kind": "call", "phase": "load", "index": 0, "outcome": outcome, - "stdout": stdout, "stdout_truncated": truncated, "elapsed_ms": elapsed}) - return module_holder[0] if "returned" in outcome else None - - -def run_script(): - script, path = PAYLOAD["script"], PAYLOAD["student"] - if not compiles(path): - return - init = {STUDENT_MARK: True, **PAYLOAD.get("vars", {})} - sys.argv = [path] # as `python student.py` would see it, not the harness's payload - outcome, stdout, truncated, elapsed, _ = call( - lambda: runpy.run_path(path, init_globals=init, run_name="__main__"), - script["timeout"], - stdin=False, - serialise=False, - ) - raised = outcome.get("raised") - if raised and raised["type"] == "SystemExit" and raised.get("clean"): - outcome = {"returned": {"value": None, "type": "NoneType"}} - record = {"kind": "call", "phase": "step", "index": 0, "outcome": outcome, - "stdout": stdout, "stdout_truncated": truncated, "elapsed_ms": elapsed} - if script.get("files"): - record["files"] = observe_files(script["files"]) - emit(record) - - -def main(): - try: - exports, duplicates = load_teacher_modules() - except BaseException as exc: - emit({"kind": "fatal", "stage": "teacher_import", "error": error_of(exc)}) - finish() - # Student code may import helpers staged beside it: after the stdlib and the teacher - # modules' directories, so a submission never stands in for what either imports. - sys.path.append(os.getcwd()) - if PAYLOAD["mode"] == "inspect": - emit({"kind": "inspect", "duplicates": duplicates, "exports": { - name: {"callable": callable(value), "params": params_of(value) if callable(value) else None} - for name, value in exports.items() - }}) - finish() - context.update(exports) - context.update(PAYLOAD.get("vars", {})) - emit({"kind": "ready"}) - if PAYLOAD.get("script"): - run_script() - finish() - student = load_student() - if student is not None and run_calls("setup", PAYLOAD.get("setup", []), student): - run_calls("step", PAYLOAD.get("steps", []), student) - finish() - - -try: - main() -except BaseException as exc: # a harness bug, not anybody's code under test - emit({"kind": "fatal", "stage": "harness", "error": error_of(exc)}) - finish() diff --git a/src/core/src/runner/harness/common.py b/src/core/src/runner/harness/common.py new file mode 100644 index 0000000..2fceaf6 --- /dev/null +++ b/src/core/src/runner/harness/common.py @@ -0,0 +1,732 @@ +"""What both sides of a unit share: framing, the two value encodings, and one call under +its timer. + +A unit runs as two processes. The teacher side (`teacher.py`) holds the teacher's modules +and checkers, drives the plan and writes the records ScriptMark reads; the student side +(`student.py`) holds only the student's module and runs the calls the teacher side asks +for. They talk over one pair of pipes, a frame at a time: a 4-byte big-endian length, then +that many bytes of JSON. Every frame a side reads is checked against the largest frame +that side could legitimately be sent at that point, before its body is read. + +Two encodings cross the pipes: + +- the *record* encoding (`to_json`, `wire`) is what the grader judges and keeps as + evidence: plain JSON, lossy where JSON is (sets sorted, tuples as lists, objects as + their text); +- the *transfer* encoding (`encode`, `decode`) rebuilds a value on the other side for code + that runs there: a teacher checker judging a student's value, or a student call taking a + teacher's. It keeps tuples, sets, exact numbers, shared references and cycles. An + instance of a class from a shared module is rebuilt from the receiving side's own copy + of the class, without `__init__`; any other object arrives as a `Snapshot` of its + attributes and public properties, read on the side that owns it. +""" + +import sys + +# The unit's working directory holds the student's file, and `python -c` puts it first on +# sys.path: a submission named json.py must not become the harness's json. +sys.path[:] = [p for p in sys.path if p] + +import base64 # noqa: E402 +import builtins # noqa: E402 +import datetime # noqa: E402 +import decimal # noqa: E402 +import fractions # noqa: E402 +import inspect # noqa: E402 +import io # noqa: E402 +import json # noqa: E402 +import math # noqa: E402 +import os # noqa: E402 +import signal # noqa: E402 +import struct # noqa: E402 +import time # noqa: E402 + +HEADER = struct.Struct(">I") +# The record limits, set from the grader's own numbers before anything runs (`configure`). +STDOUT_LIMIT = 64 * 1024 +FILE_LIMIT = 1024 * 1024 +VALUE_LIMIT = 4 * 1024 * 1024 +MESSAGE_LIMIT = 64 * 1024 +TRANSFER_LIMIT = 32 * 1024 * 1024 +DEPTH_LIMIT = 100 + + +def configure(limits): + global STDOUT_LIMIT, FILE_LIMIT, VALUE_LIMIT, MESSAGE_LIMIT, TRANSFER_LIMIT, DEPTH_LIMIT + STDOUT_LIMIT = limits["stdout"] + FILE_LIMIT = limits["file"] + VALUE_LIMIT = limits["value"] + MESSAGE_LIMIT = limits["message"] + TRANSFER_LIMIT = limits["transfer"] + DEPTH_LIMIT = limits["depth"] + + +def channel(read_fd, write_fd): + """Move this process's pipe to the other side off descriptors 0 and 1, which then read + nothing and write nowhere: no print, `sys.__stdout__` or `os.write(1, …)` reaches it.""" + read_fd, write_fd = os.dup(read_fd), os.dup(write_fd) + null = os.open(os.devnull, os.O_RDWR) + os.dup2(null, 0) + os.dup2(null, 1) + os.close(null) + return read_fd, write_fd + + +# --- Framing ---------------------------------------------------------------------------- +class Closed(Exception): + """The other side closed its end, or went away mid-frame.""" + + +class Oversized(Exception): + """A frame announced more bytes than may arrive at this point.""" + + +class Garbled(Exception): + """A frame that is not what this point of the protocol expects.""" + + +def frame(obj): + # `replace`: a lone surrogate a student printed or returned must not stop the frame. + body = json.dumps(obj, ensure_ascii=False, separators=(",", ":")).encode("utf-8", "replace") + return HEADER.pack(len(body)) + body + + +def write_all(fd, data): + view = memoryview(data) + while view: + view = view[os.write(fd, view) :] + + +def send(fd, obj): + write_all(fd, frame(obj)) + + +def read_exact(fd, n): + chunks, got = [], 0 + while got < n: + chunk = os.read(fd, min(n - got, 1 << 20)) + if not chunk: + raise Closed() + chunks.append(chunk) + got += len(chunk) + return b"".join(chunks) + + +def receive(fd, bound): + """One frame, refused before its body is read when it announces more than `bound`.""" + (size,) = HEADER.unpack(read_exact(fd, HEADER.size)) + if size > bound: + raise Oversized(f"a frame of {size} bytes where at most {bound} may arrive") + return parse(read_exact(fd, size)) + + +def parse(body): + try: + return json.loads(body.decode("utf-8")) + except (ValueError, RecursionError) as exc: # bad UTF-8, bad JSON, nested past the parser + raise Garbled(f"not a frame: {text_of(exc)}") from None + + +# --- Reporting ---------------------------------------------------------------------------- +def text_of(value): + """`str(value)`, or the type's name when the value's own `__str__` fails.""" + try: + text = str(value) + except BaseException: # never under a live timer: this is harness bookkeeping + return f"<{type(value).__name__}>" + return text if isinstance(text, str) else f"<{type(value).__name__}>" + + +def clip(message): + """A message kept to MESSAGE_LIMIT characters, saying how long it was.""" + if len(message) <= MESSAGE_LIMIT: + return message + return f"{message[:MESSAGE_LIMIT]}… (cut from {len(message)} characters)" + + +def error_of(exc): + info = { + "type": type(exc).__name__, + "types": [cls.__name__ for cls in type(exc).__mro__], + "message": clip(text_of(exc)), + } + if isinstance(exc, SystemExit): + # The exit status CPython would report: `exit()`, `exit(0)` and `exit(False)` succeed; + # any other code, `exit(0.0)` included, fails. + try: + code = exc.code + info["clean"] = code is None or (type(code) in (int, bool) and int(code) == 0) + except BaseException: + info["clean"] = False + return info + + +def problem(kind, message): + return {"type": kind, "types": [kind], "message": clip(message)} + + +# --- Timers --------------------------------------------------------------------------- +class CallTimeout(BaseException): + """Raised by SIGALRM. A BaseException, so `except Exception` cannot swallow it.""" + + +_fired = [False] + + +def _on_alarm(signum, frame): + _fired[0] = True + raise CallTimeout() + + +_HAS_TIMER = hasattr(signal, "setitimer") +if _HAS_TIMER: + signal.signal(signal.SIGALRM, _on_alarm) + + +class _CappedBytes(io.BytesIO): + """Keeps at most STDOUT_LIMIT bytes and remembers whether it dropped any.""" + + truncated = False + + def write(self, b): + room = STDOUT_LIMIT - self.tell() + if len(b) > room: + self.truncated = True + if room > 0: + super().write(bytes(b[:room])) + return len(b) + + def writelines(self, lines): + # The C method would write past the limit without calling `write`. + for line in lines: + self.write(line) + + def close(self): + """A student's wrapper around `sys.stdout.buffer` closes it when it is collected: + what was written must survive that.""" + + +def capture(): + """A stdout that behaves like a real one — `buffer`, `reconfigure`, `print(flush=True)` — + and the bytes behind it, kept apart so a detached or closed wrapper cannot lose them.""" + raw = _CappedBytes() + return io.TextIOWrapper(raw, encoding="utf-8", errors="backslashreplace", write_through=True), raw + + +def captured(stream, raw): + try: + stream.flush() + except (ValueError, OSError): # detached or closed by the student + pass + # `io.BytesIO.write(buffer, b)` still reaches past `write`: cut here too. + data = raw.getvalue() + return data[:STDOUT_LIMIT].decode("utf-8", "replace"), raw.truncated or len(data) > STDOUT_LIMIT + + +def stdin_stream(text): + return io.TextIOWrapper(io.BytesIO((text or "").encode("utf-8")), encoding="utf-8") + + +# Between calls, stdout is a sink: what a leftover thread prints is not evidence. +_quiet = open(os.devnull, "w", encoding="utf-8") +_real_stdin = sys.stdin +sys.stdout = _quiet + + +class Missing(Exception): + """The call's target does not exist.""" + + +class Unresolved(Exception): + """A `$ref` or `object` whose value was never produced in this unit.""" + + +class Unserialisable(Exception): + pass + + +class MatchingRequired(Exception): + """Function lookup needs a teacher's decision: never the student's outcome.""" + + def __init__(self, decision): + self.decision = decision + super().__init__(f"function matching needs review: {decision}") + + +class Call: + """One call's outcome: what `call()` observed, and the live value it returned.""" + + def __init__(self, outcome, stdout, truncated, elapsed, value, extra): + self.outcome = outcome + self.stdout = stdout + self.truncated = truncated + self.elapsed = elapsed + self.value = value + self.extra = extra + + def record(self): + return { + "outcome": self.outcome, + "stdout": self.stdout, + "stdout_truncated": self.truncated, + "elapsed_ms": self.elapsed, + } + + +def call(fn, timeout, stdin=None, guard=None, serialise=True, then=None): + """Run one call — its lookup, its body, its serialisation and `then(value)` — under its + timer and, for student code, the import `guard`. + + `then` runs inside the timed region too: it may run the code that owns the value (a + property read for a snapshot), and what it raises is that call's outcome.""" + out, raw = capture() + sys.stdout = out + sys.stdin = stdin_stream(stdin) if stdin is not False else _real_stdin + original_import = builtins.__import__ + if guard is not None: + builtins.__import__ = guard + _fired[0] = False + start = time.perf_counter() + value, extra = None, None + try: + if _HAS_TIMER and timeout: + signal.setitimer(signal.ITIMER_REAL, timeout, 0.05) + try: + value = fn() + if sys.stdout is not out: + # The student replaced stdout with their own wrapper: flush it in their scope. + sys.stdout.flush() + outcome = {"returned": {"value": wire(value) if serialise else None, "type": type(value).__name__}} + if then is not None: + extra = then(value) + finally: + if _HAS_TIMER: + signal.setitimer(signal.ITIMER_REAL, 0) + except CallTimeout: + outcome = {"timeout": {}} + except Missing as exc: + outcome = {"missing": {"message": str(exc)}} + except MatchingRequired as exc: + outcome = {"matching": {"message": str(exc), "decision": exc.decision}} + except Unresolved as exc: + outcome = {"unresolved": {"name": str(exc)}} + except Unserialisable as exc: + outcome = {"unserialisable": {"type": "Unserialisable", "message": str(exc)}} + except BaseException as exc: # SystemExit and KeyboardInterrupt are the code's own too + outcome = {"raised": error_of(exc)} + finally: + builtins.__import__ = original_import + sys.stdout = _quiet + sys.stdin = _real_stdin + if _fired[0]: + # The timer went off even if the code swallowed it with a bare `except:`. + outcome = {"timeout": {}} + elapsed = int((time.perf_counter() - start) * 1000) + stdout, truncated = captured(out, raw) + return Call(outcome, stdout, truncated, elapsed, value, extra if "returned" in outcome else None) + + +# --- The record encoding -------------------------------------------------------------------- +def to_json(value, depth=0, seen=None): + """What the grader sees of a Python value. Total: it never raises for a value's sake.""" + if isinstance(value, bool) or value is None or isinstance(value, str): + return value + if isinstance(value, int): + if -(2**63) <= value < 2**64: + return int(value) + # Past what the grader's JSON can hold exactly: tag it, exactly and with its sign, so + # it is never mistaken for a number and a record always parses. Decimal where Python + # will print it, hexadecimal past its 4300-digit limit. + try: + return {"$bigint": int.__repr__(value)} + except ValueError: + return {"$bigint": hex(value)} + if isinstance(value, float): + return value if math.isfinite(value) else repr(value) + if depth >= DEPTH_LIMIT: + return "" + seen = seen or set() + if id(value) in seen: + return "" + if isinstance(value, (list, tuple, set, frozenset, dict)): + seen = seen | {id(value)} + if isinstance(value, (list, tuple)): + return [to_json(v, depth + 1, seen) for v in value] + if isinstance(value, (set, frozenset)): + items = [to_json(v, depth + 1, seen) for v in value] + try: + return sorted(items) + except TypeError: # mixed types: any stable order will do + return sorted(items, key=lambda v: json.dumps(v, sort_keys=True)) + if isinstance(value, dict): + out = {} + for k, v in value.items(): + key = k if isinstance(k, str) else text_of(k) + if key in out: + raise Unserialisable(f"two keys both read as '{key}'") + out[key] = to_json(v, depth + 1, seen) + return out + return text_of(value) + + +def utf8_size(text): + """The bytes `text` takes on the channel, counted a slice at a time rather than through + a second full copy of a value that may already fill most of the memory limit.""" + step = 1 << 20 + return sum(len(text[i : i + step].encode("utf-8", "replace")) for i in range(0, len(text), step)) + + +def json_size(tree): + return utf8_size(json.dumps(tree, ensure_ascii=False, separators=(",", ":"))) + + +def wire(value): + """The value as a record carries it. One too large to report is replaced by its size: + the call still returned, and a function checker still judges it through the transfer + encoding.""" + value = to_json(value) + size = utf8_size(json.dumps(value, ensure_ascii=False)) + if size > VALUE_LIMIT: + return {"$too_large": f"{size} bytes of JSON"} + return value + + +# --- The transfer encoding -------------------------------------------------------------------- +class Snapshot: + """An object whose class the receiving side does not have: its instance attributes and + public properties, as read on the side that owns it. `__scriptmark_class__` names the + class it was taken from.""" + + def __init__(self, cls, attrs): + object.__setattr__(self, "__scriptmark_class__", cls) + for name, value in attrs.items(): + object.__setattr__(self, name, value) + + def __repr__(self): + fields = ", ".join( + f"{k}={v!r}" for k, v in vars(self).items() if k != "__scriptmark_class__" + ) + return f"<{self.__scriptmark_class__} snapshot: {fields}>" + + +# Classes whose instances are rebuilt from the receiving side's own copy, by `module.qualname`. +SHARED = {} + + +def share(module): + """Register every class a shared module defines.""" + for value in vars(module).values(): + if inspect.isclass(value) and getattr(value, "__module__", None) == module.__name__: + SHARED[f"{module.__name__}.{value.__qualname__}"] = value + + +def _shared_name(cls): + """The shared class `cls` is, or derives from: a student's subclass travels as its + shared base.""" + for base in cls.__mro__: + name = f"{getattr(base, '__module__', '')}.{getattr(base, '__qualname__', '')}" + if SHARED.get(name) is base: + return name + return None + + +def _state(value): + """An instance's own attributes: its `__dict__`, and the slots it has filled.""" + state = dict(vars(value)) if hasattr(value, "__dict__") else {} + for cls in type(value).__mro__: + slots = cls.__dict__.get("__slots__", ()) + for slot in [slots] if isinstance(slots, str) else slots: + if slot in ("__dict__", "__weakref__") or slot in state: + continue + try: + state[slot] = getattr(value, slot) + except AttributeError: # declared, never set + pass + return state + + +def _properties(value): + """The public properties `value`'s class defines, read now: this runs the owner's code.""" + read = {} + for cls in type(value).__mro__: + if cls is object: + continue + for name, attr in cls.__dict__.items(): + if isinstance(attr, property) and not name.startswith("_") and name not in read: + read[name] = getattr(value, name) + return read + + +_ATOMS = (decimal.Decimal, fractions.Fraction, complex, datetime.date, datetime.time, datetime.timedelta, bytes, bytearray) + + +class _Node: + __slots__ = ("kind", "body", "refs", "number", "building") + + def __init__(self, kind): + self.kind = kind + self.body = None + self.refs = 1 + self.number = None + self.building = False + + +def encode(value): + """The transfer encoding of `value`. Runs the owner's code (properties, `__iter__` of a + container subclass) and raises what it raises; call it under the owner's timer.""" + nodes = {} + keep = [] # every object visited stays alive, so no id is reused while encoding + root = _graph(value, 0, nodes, keep) + numbers = [0] + return _emit(root, numbers) + + +def _graph(value, depth, nodes, keep): + """Pass one: the value graph, each container and object visited once, with how often it + is referred to — so pass two numbers only what is actually shared.""" + if value is None or isinstance(value, (bool, str)): + return value + if isinstance(value, int): + return value if -(2**63) <= value < 2**64 else ("int", int(value)) + if isinstance(value, float): + return value if math.isfinite(value) else ("float", repr(value)) + if isinstance(value, _ATOMS): + return ("atom", value) + if depth >= DEPTH_LIMIT: + return ("text", "", type(value).__name__) + node = nodes.get(id(value)) + if node is not None: + node.refs += 1 + return node + keep.append(value) + shared = _shared_name(type(value)) + if shared is not None: + node = nodes[id(value)] = _Node("shared") + node.body = (shared, {k: _graph(v, depth + 1, nodes, keep) for k, v in _state(value).items()}) + return node + if isinstance(value, (list, tuple, set, frozenset)): + kind = "list" if isinstance(value, list) else "tuple" if isinstance(value, tuple) else "set" if isinstance(value, set) else "frozenset" + node = nodes[id(value)] = _Node(kind) + node.body = [_graph(v, depth + 1, nodes, keep) for v in value] + return node + if isinstance(value, dict): + node = nodes[id(value)] = _Node("dict") + node.body = [(_graph(k, depth + 1, nodes, keep), _graph(v, depth + 1, nodes, keep)) for k, v in value.items()] + return node + if inspect.isroutine(value) or inspect.isclass(value) or inspect.ismodule(value): + return ("text", text_of(value), type(value).__name__) + module = getattr(type(value), "__module__", "") or "" + if module.split(".")[0] == "numpy" and callable(getattr(value, "tolist", None)): + return _graph(value.tolist(), depth, nodes, keep) + if hasattr(value, "__dict__") or any("__slots__" in cls.__dict__ for cls in type(value).__mro__[:-1]): + node = nodes[id(value)] = _Node("object") + attrs = {k: _graph(v, depth + 1, nodes, keep) for k, v in _state(value).items()} + props = {k: _graph(v, depth + 1, nodes, keep) for k, v in _properties(value).items()} + node.body = (type(value).__qualname__, attrs, props) + return node + return ("text", text_of(value), type(value).__name__) + + +def _atom(value): + if isinstance(value, decimal.Decimal): + return {"$": "decimal", "v": str(value)} + if isinstance(value, fractions.Fraction): + return {"$": "fraction", "v": [_emit_int(value.numerator), _emit_int(value.denominator)]} + if isinstance(value, complex): + return {"$": "complex", "v": [repr(value.real), repr(value.imag)]} + if isinstance(value, datetime.datetime): + return {"$": "datetime", "v": value.isoformat()} + if isinstance(value, datetime.date): + return {"$": "date", "v": value.isoformat()} + if isinstance(value, datetime.time): + return {"$": "time", "v": value.isoformat()} + if isinstance(value, datetime.timedelta): + return {"$": "timedelta", "v": [value.days, value.seconds, value.microseconds]} + return {"$": "bytes" if isinstance(value, bytes) else "bytearray", "v": base64.b64encode(bytes(value)).decode("ascii")} + + +def _emit_int(value): + return value if -(2**63) <= value < 2**64 else {"$": "int", "v": hex(value)} + + +def _emit(item, numbers): + """Pass two: JSON, plain wherever plain JSON says the same thing.""" + if not isinstance(item, (_Node, tuple)): + return item + if isinstance(item, tuple): + kind = item[0] + if kind == "int": + return {"$": "int", "v": hex(item[1])} + if kind == "float": + return {"$": "float", "v": item[1]} + if kind == "atom": + return _atom(item[1]) + return {"$": "text", "v": item[1], "type": item[2]} + if item.number is not None: + return {"$": "ref", "#": item.number} + kind = item.kind + if kind in ("tuple", "frozenset"): + # Made from their items, so they are numbered once those are: a way back into one + # while its items are written cannot be rebuilt, and reads as a cycle, as the + # record encoding says. + if item.building: + return {"$": "text", "v": "", "type": kind} + item.building = True + items = [_emit(v, numbers) for v in item.body] + item.building = False + tagged = {} + if item.refs > 1: + item.number = numbers[0] + numbers[0] += 1 + tagged["#"] = item.number + return {"$": kind, **tagged, "v": items} + tagged = {} + if item.refs > 1: + item.number = numbers[0] + numbers[0] += 1 + tagged["#"] = item.number + if kind == "list": + items = [_emit(v, numbers) for v in item.body] + return items if not tagged else {"$": "list", **tagged, "v": items} + if kind == "set": + return {"$": kind, **tagged, "v": [_emit(v, numbers) for v in item.body]} + if kind == "dict": + pairs = [(_emit(k, numbers), _emit(v, numbers)) for k, v in item.body] + if not tagged and all(isinstance(k, str) and k != "$" for k, _ in pairs): + return dict(pairs) + return {"$": "dict", **tagged, "v": [[k, v] for k, v in pairs]} + if kind == "shared": + name, state = item.body + return {"$": "shared", **tagged, "class": name, "state": {k: _emit(v, numbers) for k, v in state.items()}} + cls, attrs, props = item.body + return { + "$": "object", + **tagged, + "class": cls, + "attrs": {k: _emit(v, numbers) for k, v in attrs.items()}, + "properties": {k: _emit(v, numbers) for k, v in props.items()}, + } + + +def decode(tree, resolve=None): + """Rebuild a transfer-encoded value. `resolve(name)` answers `{"$": "id"}` references to + values that live on this side. Raises Garbled for anything the encoding cannot mean.""" + return _decode(tree, {}, resolve, 0) + + +def _decode(tree, made, resolve, depth): + if depth > DEPTH_LIMIT + 2: + raise Garbled(f"a value nested deeper than {DEPTH_LIMIT}") + if tree is None or isinstance(tree, (bool, int, float, str)): + return tree + if isinstance(tree, list): + return [_decode(v, made, resolve, depth + 1) for v in tree] + if not isinstance(tree, dict): + raise Garbled(f"unexpected {type(tree).__name__}") + kind = tree.get("$") + if kind is None: + return {k: _decode(v, made, resolve, depth + 1) for k, v in tree.items()} + try: + return _tagged(kind, tree, made, resolve, depth) + except Garbled: + raise + except RecursionError: + raise Garbled("a value nested too deeply") from None + except Exception as exc: # a tag whose body does not fit it + raise Garbled(f"a '{kind}' value that cannot be rebuilt: {text_of(exc)}") from None + + +def _number(tree, made, obj): + if "#" in tree: + number = tree["#"] + if not isinstance(number, int) or number in made: + raise Garbled("a value numbered twice") + made[number] = obj + return obj + + +def _tagged(kind, tree, made, resolve, depth): + body = tree.get("v") + again = lambda v: _decode(v, made, resolve, depth + 1) # noqa: E731 + if kind == "ref": + if tree.get("#") not in made: + raise Garbled("a reference to nothing before it") + return made[tree["#"]] + if kind == "id": + if resolve is None: + raise Garbled("a reference to a value this side does not hold") + return resolve(tree["name"]) + if kind == "int": + # Its length is the frame's, already bounded; hexadecimal parses in linear time. + if not isinstance(body, str): + raise Garbled("an integer that is not text") + return int(body, 16) + if kind == "float": + return float(body) + if kind == "decimal": + return decimal.Decimal(body) + if kind == "fraction": + return fractions.Fraction(again(body[0]), again(body[1])) + if kind == "complex": + return complex(float(body[0]), float(body[1])) + if kind == "datetime": + return datetime.datetime.fromisoformat(body) + if kind == "date": + return datetime.date.fromisoformat(body) + if kind == "time": + return datetime.time.fromisoformat(body) + if kind == "timedelta": + return datetime.timedelta(days=body[0], seconds=body[1], microseconds=body[2]) + if kind in ("bytes", "bytearray"): + data = base64.b64decode(body, validate=True) + return data if kind == "bytes" else bytearray(data) + if kind == "text": + return str(body) + if kind == "list": + out = _number(tree, made, []) + out.extend(again(v) for v in body) + return out + if kind == "set": + out = _number(tree, made, set()) + out.update(again(v) for v in body) + return out + if kind in ("tuple", "frozenset"): + # Built from their items, so a cycle back through one cannot be rebuilt. + number = tree.get("#") + if number is not None and number in made: + raise Garbled("a value numbered twice") + items = [again(v) for v in body] + return _number(tree, made, tuple(items) if kind == "tuple" else frozenset(items)) + if kind == "dict": + out = _number(tree, made, {}) + for key, value in body: + out[again(key)] = again(value) + return out + if kind == "shared": + cls = SHARED.get(tree["class"]) + if cls is None: + raise Garbled(f"no shared class '{tree['class']}' on this side") + try: + obj = cls.__new__(cls) + except TypeError: # a __new__ that wants arguments: rebuilt without it, as without __init__ + obj = object.__new__(cls) + obj = _number(tree, made, obj) + for name, value in tree["state"].items(): + object.__setattr__(obj, name, again(value)) + return obj + if kind == "object": + obj = _number(tree, made, Snapshot(str(tree["class"]), {})) + for part in ("attrs", "properties"): + for name, value in tree[part].items(): + object.__setattr__(obj, name, again(value)) + return obj + raise Garbled(f"an unknown kind of value '{kind}'") + + +def transfer(value): + """The transfer encoding of `value`, or a marker saying how large it would have been.""" + tree = encode(value) + size = json_size(tree) + if size > TRANSFER_LIMIT: + return {"$": "too_large", "size": size} + return tree diff --git a/src/core/src/runner/harness/idle.py b/src/core/src/runner/harness/idle.py new file mode 100644 index 0000000..82ece93 --- /dev/null +++ b/src/core/src/runner/harness/idle.py @@ -0,0 +1,15 @@ +# The student ScriptMark measures a unit's idle memory with: it imports the modules the +# submission imports — read from its source beforehand, in another process — and waits +# while the grader samples it. `memory_mb` counts beyond what this holds, so no import is +# the student's. The grader samples rather than asking: on Linux `ru_maxrss` also counts +# the grader itself, whose memory the process held between fork and exec. +import time + + +def idle(imports): + for name, names in imports: + try: + __import__(name, fromlist=names) + except (Exception, SystemExit): # refused, missing, broken or exiting: skipped + pass + time.sleep(0.05) diff --git a/src/core/src/runner/harness/imports.py b/src/core/src/runner/harness/imports.py new file mode 100644 index 0000000..2633610 --- /dev/null +++ b/src/core/src/runner/harness/imports.py @@ -0,0 +1,71 @@ +# Reads a submission's source on stdin and prints, as JSON, each module an import in it +# names, wherever in the file, with what `from … import` takes from it. Nothing in the +# source runs. It reads tokens as they come and keeps only those of import statements, so +# what it holds does not grow with the source, whatever the source holds. +import json +import sys +import tokenize + +_STARTS = {tokenize.NEWLINE, tokenize.INDENT, tokenize.DEDENT} +_SKIPPED = {tokenize.NL, tokenize.COMMENT, tokenize.ENCODING} + + +def _module(words): + """`import a.b as c, d` or `from a.b import (x as y, z)`: the module, and what it takes.""" + if words[0] == "import": + modules, name, renamed = [], [], False + for word in words[1:] + [","]: + if word == ",": + if name: + modules.append(["".join(name), []]) + name, renamed = [], False + elif word == "as": + renamed = True + elif not renamed: + name.append(word) + return modules + if "import" not in words or words[1] in (".", "..."): # relative: the submission's own + return [] + at = words.index("import") + names, renamed = [], False + for word in words[at + 1 :]: + if word in ("(", ")", "*"): + continue + if word == ",": + renamed = False + elif word == "as": + renamed = True + elif not renamed: + names.append(word) + return [["".join(words[1:at]), names]] + + +def imports(readline): + found = [] + start, nesting, statement = True, 0, None + try: + for token in tokenize.tokenize(readline): + kind, text = token.type, token.string + if kind in _SKIPPED: + continue + ends = kind in _STARTS or kind == tokenize.ENDMARKER or (kind == tokenize.OP and text == ";" and not nesting) + if ends: + if statement: + found.extend(_module(statement)) + start, statement = True, None + continue + if kind == tokenize.OP: + nesting += text in "([{" + nesting -= text in ")]}" + if statement is not None: + statement.append(text) + elif start and kind == tokenize.NAME and text in ("import", "from"): + statement = [text] + # A statement after a colon on the same line: `if x: import y`. + start = kind == tokenize.OP and text == ":" and not nesting + except (tokenize.TokenError, SyntaxError, ValueError): # what was read so far stands + pass + return found + + +print(json.dumps(imports(sys.stdin.buffer.readline))) diff --git a/src/core/src/runner/harness/student.py b/src/core/src/runner/harness/student.py new file mode 100644 index 0000000..8f7ff40 --- /dev/null +++ b/src/core/src/runner/harness/student.py @@ -0,0 +1,363 @@ +"""The student side of a unit: the student's module, and the calls the teacher side asks for. + +Nothing here decides anything. It answers one request at a time: what a call did, what it +printed, which files it left, and — when the teacher side needs it — the value it made, in +the transfer encoding. Every piece of student code — loading the module, finding a +function, reading an attribute, serialising a value, reading a property for a snapshot — +runs inside `call()`, under its timer and import guard, so a failure there is that call's +outcome and never a harness crash. + +What this process sends is the student's own evidence, and the teacher side treats it as +such: a student who writes to the pipe directly can only misreport their own calls. +""" + +import importlib.machinery # noqa: E402 +import importlib.util # noqa: E402 +import runpy # noqa: E402 +from typing import TYPE_CHECKING # noqa: E402 + +if TYPE_CHECKING: + from matching import FunctionRules, MatchDecision, function_decision + +_IN, _OUT = channel(0, 1) +# What the teacher side may send in one frame: its own values, never the student's. +REQUEST_LIMIT = 1 << 30 +INIT = receive(_IN, REQUEST_LIMIT) +configure(INIT["limits"]) +REFERENCE = INIT["role"] == "reference" + + +def reply(message): + send(_OUT, message) + + +# --- The import guard ----------------------------------------------------------------- +# A teaching rule, not a security boundary: the operating system's sandbox is that. It looks +# at where the import statement runs, so an allowlisted stdlib module importing its own +# internals is never mistaken for the student importing them. +ALLOWED = frozenset( + { + # core + "builtins", "sys", "io", "abc", "enum", "types", "typing", "dataclasses", + "collections", "functools", "itertools", "operator", "copy", "copyreg", + # math/science + "math", "cmath", "decimal", "fractions", "numbers", "random", "statistics", + # strings/text + "string", "re", "difflib", "textwrap", "unicodedata", "codecs", "encodings", + # data formats + "json", "csv", "struct", "base64", "binascii", "hashlib", "hmac", + # time + "time", "datetime", "calendar", "zoneinfo", + # data structures + "array", "heapq", "bisect", "queue", "graphlib", + # functional + "contextlib", "contextvars", + # introspection + "inspect", "dis", "token", "tokenize", "ast", "symtable", + # error handling + "traceback", "warnings", "errno", "faulthandler", + # internals needed by Python itself + "importlib", "_thread", "__future__", "_collections_abc", "_abc", + "_functools", "_operator", "_io", "_json", "_string", "_struct", + "_sre", "sre_compile", "sre_constants", "sre_parse", "_locale", + "_frozen_importlib", "_frozen_importlib_external", "_imp", + "_warnings", "_weakref", "_weakrefset", "_threading_local", + "_codecs", "_signal", "_stat", "_opcode", "_typing", + "genericpath", "posixpath", "ntpath", "stat", "keyword", + "reprlib", "pprint", "linecache", + # testing (students may use unittest/doctest) + "unittest", "doctest", + # filesystem (pathlib is safe — open() is a builtin anyway) + "pathlib", "fnmatch", "glob", "tempfile", + # misc safe + "secrets", "uuid", "ipaddress", "colorsys", "timeit", "py_compile", "compileall", + "gzip", "bz2", "lzma", "zipfile", "tarfile", "xml", "html", "email", + "logging", "argparse", "getopt", "configparser", "tomllib", "threading", "concurrent", + } +) | frozenset(name.split(".")[0] for name in INIT.get("allowed_imports", [])) +STUDENT_MARK = "__scriptmark_student__" +_original_import = builtins.__import__ + + +def _guarded_import(name, globals=None, locals=None, fromlist=(), level=0): + # `__import__('os')` called directly passes no globals: look at the caller instead. + scope = globals if globals is not None else sys._getframe(1).f_globals + if scope.get(STUDENT_MARK) and level == 0: + if name.split(".")[0] not in ALLOWED: + raise ImportError(f"Module '{name}' is not allowed in student code") + return _original_import(name, globals, locals, fromlist, level) + + +GUARD = None if REFERENCE else _guarded_import + + +# --- Values held here ----------------------------------------------------------------- +# What the student's calls made, by id. Values the teacher side holds arrive as copies. +context = dict(INIT.get("vars", {})) + + +def live(name): + if name not in context: + raise Unresolved(name) + return context[name] + + +def received(tree): + """A value the teacher side sent, rebuilt here: an id names a value made on this side.""" + return decode(tree, live) + + +def matched_function( + module: object, name: str, evidence: list[MatchDecision | None] +) -> tuple[object, str]: + policy: FunctionRules = INIT.get("functions", {"aliases": {}, "overrides": {}}) + # Read attributes only once: dynamic exports/properties belong to the guarded call. + exports: dict[str, object] = dict(vars(module)) + absent: object = object() + for candidate in { + name, + *policy.get("aliases", {}).get(name, []), + *([policy["overrides"][name]] if name in policy.get("overrides", {}) else []), + }: + if candidate not in exports: + value: object = getattr(module, candidate, absent) + if value is not absent: + exports[candidate] = value + decision: MatchDecision = function_decision( + [candidate for candidate, obj in exports.items() if callable(obj)], name, policy + ) + evidence[0] = decision + selected: str | None = decision["selected"] + if selected is not None: + return exports[selected], selected + if ( + decision["state"] != "missing" + or name in policy.get("overrides", {}) + or name in policy.get("aliases", {}) + ): + raise MatchingRequired(decision) + raise Missing(f"function '{name}' not found") + + +def member(obj, object_id, name, what): + """`getattr(obj, name)`, read once. A name that does not exist is Missing; a property + that raises — AttributeError included — is the student's exception.""" + try: + return getattr(obj, name) + except AttributeError: + try: + inspect.getattr_static(obj, name) + except AttributeError: + if not hasattr(type(obj), "__getattr__"): + raise Missing(f"'{object_id}' has no {what} '{name}'") from None + raise + + +def observe_files(paths: list[str]) -> dict[str, str | None]: + seen: dict[str, str | None] = {} + for path in paths: + try: + with open(path, encoding="utf-8") as fh: + content: str = fh.read(FILE_LIMIT + 1) + seen[path] = content if len(content) <= FILE_LIMIT else None + except (OSError, ValueError): # absent, oversized or invalid UTF-8 is no text expectation + seen[path] = None + return seen + + +# --- Requests ------------------------------------------------------------------------- +module = [None] + + +def compiles(path): + """Compile the file in memory — writing no __pycache__ beside it — and answer with a + syntax error as the load call's outcome.""" + try: + with open(path, "rb") as fh: + compile(fh.read(), path, "exec") + return True + except (SyntaxError, ValueError, RecursionError, MemoryError) as exc: + reply({ + "compiles": False, + "outcome": {"raised": {"type": "SyntaxError", "types": ["SyntaxError"], "message": text_of(exc)}}, + "stdout": "", + "stdout_truncated": False, + "elapsed_ms": 0, + }) + return False + + +def load(request): + path = INIT["subject"] + if not compiles(path): + return + + def run(): + # An explicit loader: `Lab5.PY` is Python too, whatever its extension. + loader = importlib.machinery.SourceFileLoader("student_mod", path) + spec = importlib.util.spec_from_loader("student_mod", loader) + made = importlib.util.module_from_spec(spec) + if not REFERENCE: + setattr(made, STUDENT_MARK, True) + for key, val in INIT.get("vars", {}).items(): + setattr(made, key, val) + sys.modules["student_mod"] = made + loader.exec_module(made) + module[0] = made + + original_input = builtins.input + builtins.input = lambda *a, **k: "0" + try: + done = call(run, request["timeout"], guard=GUARD, serialise=False) + finally: + builtins.input = original_input + reply(done.record()) + + +def feed_stdin(text): + """Descriptor 0 becomes a file holding `text`, as `python script.py < input` would see + it: `input()`, `sys.stdin.buffer` and `os.read(0, …)` all read the same bytes. A file in + the unit's own temporary directory, not a pipe and a thread to fill it: the process + limit counts threads, and a busy machine may have none to spare.""" + import tempfile + + with tempfile.TemporaryFile() as held: + held.write(text.encode("utf-8")) + held.flush() + held.seek(0) + os.dup2(held.fileno(), 0) + + +def script(request): + path = INIT["subject"] + if not compiles(path): + return + init = {STUDENT_MARK: True, **INIT.get("vars", {})} + sys.argv = [path] # as `python student.py` would see it, not the harness's + feed_stdin(request.get("stdin") or "") + done = call( + lambda: runpy.run_path(path, init_globals=init, run_name="__main__"), + request["timeout"], + stdin=False, + guard=GUARD, + serialise=False, + ) + record = done.record() + raised = done.outcome.get("raised") + if raised and raised["type"] == "SystemExit" and raised.get("clean"): + record["outcome"] = {"returned": {"value": None, "type": "NoneType"}} + if request.get("files"): + record["files"] = observe_files(request["files"]) + reply(record) + + +def invoke(request, resolved, evidence): + """The call a request names. Runs inside `call()`: the lookup is student code too.""" + kind, spec = next(iter(request["target"].items())) + args = [received(a) for a in request.get("args", [])] + if kind == "function": + student = module[0] + if REFERENCE: + if not hasattr(student, spec["name"]): + raise Missing(f"function '{spec['name']}' not found") + fn, resolved[0] = getattr(student, spec["name"]), spec["name"] + else: + fn, resolved[0] = matched_function(student, spec["name"], evidence) + return fn(*args) + obj = received(request["object"]) if "object" in request else live(spec["object"]) + if kind == "method": + fn = member(obj, spec["object"], spec["name"], "method") + if not callable(fn): + raise Missing(f"'{spec['object']}.{spec['name']}' is not a method") + return fn(*args) + return member(obj, spec["object"], spec["name"], "attribute") + + +def run_call(request): + requested = next(iter(request["target"].values()))["name"] + resolved = [requested] + evidence: list[MatchDecision | None] = [None] + done = call( + lambda: invoke(request, resolved, evidence), + request["timeout"], + request.get("stdin"), + guard=GUARD, + serialise=request.get("serialise", True), + then=transfer if request.get("transfer") else None, + ) + if "returned" in done.outcome and request.get("id"): + context[request["id"]] = done.value + record = done.record() + record["target"] = {"requested": requested, "resolved": resolved[0], "matching": evidence[0]} + if request.get("files"): + record["files"] = observe_files(request["files"]) + if done.extra is not None and done.extra != done.outcome["returned"]["value"]: + # Sent only when the record's value would not rebuild the same thing. + record["transfer"] = done.extra + reply(record) + + +def fetch(request): + """A value made here, for teacher code to take by value: read now, under the timer, + since a snapshot runs the student's properties.""" + if request["name"] not in context: + return reply({"unresolved": request["name"]}) + value = context[request["name"]] + done = call(lambda: transfer(value), request["timeout"], guard=GUARD, serialise=False) + if "returned" in done.outcome: + return reply({"value": done.value}) + reply({"outcome": done.outcome}) + + +def canary_holds(): + """The sandbox's own check: the grader left a file outside everything this side may read. + Reading it means the sandbox is not there.""" + path = INIT.get("canary") + if not path: + return True + try: + with open(path, "rb"): + return False + except OSError: + return True + + +HANDLERS = {"load": load, "script": script, "call": run_call, "fetch": fetch} + + +def serve(): + if not canary_holds(): + reply({"fatal": problem("Sandbox", "the student sandbox did not hold: its canary was readable")}) + os._exit(0) + for path in INIT.get("shared", []): + # The student's own copy of each shared module, importable as a module of its own. + name = os.path.splitext(os.path.basename(path))[0] + spec = importlib.util.spec_from_file_location(name, path) + shared = importlib.util.module_from_spec(spec) + sys.modules[name] = shared + spec.loader.exec_module(shared) + share(shared) + # Student code may import helpers staged beside it: after the stdlib, so a submission + # never stands in for what the harness imports. + sys.path.append(os.getcwd()) + reply({"ready": True}) + while True: + try: + request = receive(_IN, REQUEST_LIMIT) + except (Closed, OSError): + break + op = request.get("op") + if op == "exit": + break + HANDLERS[op](request) + # Threads a student left running must not hold the unit open. + os._exit(0) + + +try: + serve() +except BaseException as exc: # a harness bug, not the student's code + try: + reply({"fatal": error_of(exc)}) + finally: + os._exit(70) diff --git a/src/core/src/runner/harness/teacher.py b/src/core/src/runner/harness/teacher.py new file mode 100644 index 0000000..07d9a97 --- /dev/null +++ b/src/core/src/runner/harness/teacher.py @@ -0,0 +1,803 @@ +"""The teacher side of a unit: teacher modules and checkers, the plan, and the records. + +It drives the student side one request at a time, sets a deadline on each, and reports +observations, never verdicts about who is at fault: the grader derives those from which +call was running, what kind of code it runs, and how each side ended. What the student +side answers is the student's evidence; it is checked against the largest answer that +request allows before it is read, and it never reaches a record unchecked. +""" + +import importlib.util # noqa: E402 +import select # noqa: E402 + +with open(sys.argv[1], encoding="utf-8") as _fh: + PAYLOAD = json.load(_fh) +os.unlink(sys.argv[1]) +configure(PAYLOAD["limits"]) + +_RECORDS = PAYLOAD["records"] +if _RECORDS == 2: + # Windows has no third pipe: records take stderr, so the teacher's own stderr — a + # module's warning, a checker's print — goes nowhere rather than into them. + _RECORDS = os.dup(2) + _null = os.open(os.devnull, os.O_WRONLY) + os.dup2(_null, 2) + os.close(_null) +# Seconds a request may run past its own timeout before the student side is given up on, +# and how long the student side has to start. +GRACE = PAYLOAD["grace"] +STARTUP = PAYLOAD["startup"] +# The most a frame from the student side carries besides its value, stdout and files: +# the call's target and matching evidence, and the keys around them. +SLACK = 1 << 20 + + +def emit(record): + write_all(_RECORDS, frame(record)) + + +def emit_bounded(record, bound, student): + """Emit a record built from a student's answer, if it fits what a record of its call + may hold: an answer padded past that is the student's, not a record the grader + cannot read.""" + data = frame(record) + if len(data) - HEADER.size > bound: + raise student.garbled(f"an answer that makes a record of {len(data)} bytes, where at most {bound} may") + write_all(_RECORDS, data) + + +def finish(student=None): + """Stop now: threads teacher code left running must not hold the unit open.""" + if student is not None: + student.close() + emit({"kind": "done"}) + os._exit(0) + + +# --- The student side ----------------------------------------------------------------- +class Stop(Exception): + """The student side cannot be asked anything more: it went away, ran past a deadline, + or answered with something no request allows.""" + + def __init__(self, reason, message): + super().__init__(message) + self.reason = reason + self.message = message + + +class Student: + """The pipe to the student side. One request is pending at a time: its answer is read + before its deadline, and its length checked against the request's bound before its body + is read. Reading needs no thread: the kernel's process limit counts threads, and a + busy machine may have none to spare.""" + + def __init__(self, read_fd, write_fd): + self.read_fd, self.write_fd = read_fd, write_fd + self.stopped = None + self.windows = os.name == "nt" + if self.windows: + # Pipes cannot be polled there: a reader thread waits instead, and the process + # limit that makes threads scarce is a unix one. + import queue + import threading + + self.answers = queue.Queue() + threading.Thread(target=self._pump, daemon=True).start() + + def _pump(self): + try: + while True: + (size,) = HEADER.unpack(read_exact(self.read_fd, HEADER.size)) + self.answers.put(size) + self.answers.put(read_exact(self.read_fd, size) if size <= self.bound else None) + if size > self.bound: + return + except (Closed, OSError): + self.answers.put(Closed()) + + def _exactly(self, n, deadline): + """`n` bytes from the student side, before `deadline`.""" + chunks, got = [], 0 + while got < n: + left = deadline - time.monotonic() + if left <= 0: + raise TimeoutError() + if not select.select([self.read_fd], [], [], left)[0]: + continue + chunk = os.read(self.read_fd, min(n - got, 1 << 20)) + if not chunk: + raise Closed() + chunks.append(chunk) + got += len(chunk) + return b"".join(chunks) + + def _answer(self, bound, timeout): + deadline = time.monotonic() + timeout + if self.windows: + import queue + + try: + size = self.answers.get(timeout=timeout) + body = None if isinstance(size, Closed) or size > bound else self.answers.get(timeout=max(deadline - time.monotonic(), 0.001)) + except queue.Empty: + raise TimeoutError() from None + if isinstance(size, Closed) or isinstance(body, Closed): + raise Closed() + else: + (size,) = HEADER.unpack(self._exactly(HEADER.size, deadline)) + body = None if size > bound else self._exactly(size, deadline) + if body is None: + raise Stop("flooded", f"the student side sent {size} bytes where at most {bound} may arrive") + return body + + def ask(self, request, bound, timeout): + if self.stopped is not None: + raise self.stopped + self.bound = bound + try: + send(self.write_fd, request) + except OSError: + self.stopped = Stop("exited", "the student side ended") + raise self.stopped from None + try: + answer = self._answer(bound, timeout) + except TimeoutError: + self.stopped = Stop("deadline", f"the student side did not answer within {timeout:g}s") + raise self.stopped from None + except (Closed, OSError): + self.stopped = Stop("exited", "the student side ended") + raise self.stopped from None + except Stop as stop: + self.stopped = stop + raise + try: + parsed = parse(answer) + if not isinstance(parsed, dict): + raise Garbled("an answer that is not an object") + return parsed + except Garbled as exc: + self.stopped = Stop("garbled", str(exc)) + raise self.stopped from None + + def garbled(self, message): + self.stopped = Stop("garbled", message) + return self.stopped + + def close(self): + if self.stopped is None: + try: + send(self.write_fd, {"op": "exit"}) + except OSError: + pass + + +def call_bound(files=0, value=True, transfer=False): + """The largest answer a call can legitimately make: its value, its stdout and an error + message, each of which JSON escaping can grow sixfold, and each file it observes.""" + bound = SLACK + 6 * STDOUT_LIMIT + 6 * MESSAGE_LIMIT + files * (6 * FILE_LIMIT + 4096) + if value: + bound += VALUE_LIMIT + if transfer: + bound += TRANSFER_LIMIT + return bound + + +# --- Checking what the student side says ------------------------------------------------- +# The outcomes each kind of request can have. The student side resolves nothing by name — this +# side does, before asking — and only a fuzzy function lookup can need a teacher's decision: +# anything else claimed is not an answer to the request. +RAN = {"returned", "raised", "timeout"} +LOOKED_UP = RAN | {"missing", "unserialisable"} +MATCHED = LOOKED_UP | {"matching"} + + +def _str(value, what): + if not isinstance(value, str): + raise Garbled(f"{what} is not text") + return value + + +def _error(info, what): + if not isinstance(info, dict): + raise Garbled(f"{what} is not an error") + error = { + "type": _str(info.get("type"), what), + "types": [_str(t, what) for t in info.get("types", [])] if isinstance(info.get("types", []), list) else None, + "message": _str(info.get("message", ""), what), + } + if error["types"] is None: + raise Garbled(f"{what} has no list of types") + if "clean" in info: + if not isinstance(info["clean"], bool): + raise Garbled(f"{what} says neither clean nor not") + error["clean"] = info["clean"] + return error + + +def _decision(decision): + if decision is None: + return None + if not isinstance(decision, dict) or not isinstance(decision.get("candidates"), list): + raise Garbled("matching evidence that is not a decision") + selected = decision.get("selected") + if selected is not None: + _str(selected, "a selected function") + return { + "state": _str(decision.get("state"), "a matching state"), + "selected": selected, + "candidates": [ + { + "value": _str(c.get("value"), "a candidate"), + "rules": [_str(r, "a rule") for r in c.get("rules", [])], + } + for c in decision["candidates"] + if isinstance(c, dict) and isinstance(c.get("rules", []), list) + ], + } + + +def _outcome(outcome, allowed): + if not isinstance(outcome, dict) or len(outcome) != 1: + raise Garbled("an outcome that is not exactly one thing") + ((kind, body),) = outcome.items() + if kind not in allowed or not isinstance(body, dict): + raise Garbled(f"an outcome '{kind}' this request cannot have") + if kind == "returned": + return {"returned": {"value": body.get("value"), "type": _str(body.get("type"), "a type name")}} + if kind in ("raised", "unserialisable"): + return {kind: _error(body, "an error")} + if kind == "timeout": + return {"timeout": {}} + if kind == "missing": + return {"missing": {"message": _str(body.get("message"), "a message")}} + decision = _decision(body.get("decision")) + if decision is None or decision["selected"] is not None: + raise Garbled("a lookup that needs a decision but settled on a function") + return {"matching": {"message": _str(body.get("message"), "a message"), "decision": decision}} + + +def checked(answer, files, allowed): + """The call record an answer describes, rebuilt from only what a record may hold.""" + if "fatal" in answer: + raise Garbled(f"the student side failed: {text_of(answer['fatal'])[:MESSAGE_LIMIT]}") + stdout = _str(answer.get("stdout", ""), "stdout") + truncated = answer.get("stdout_truncated", False) + elapsed = answer.get("elapsed_ms", 0) + if not isinstance(truncated, bool) or not isinstance(elapsed, int) or isinstance(elapsed, bool): + raise Garbled("a call record with malformed fields") + record = { + "outcome": _outcome(answer.get("outcome"), allowed), + "stdout": stdout, + "stdout_truncated": truncated, + "elapsed_ms": max(elapsed, 0), + } + target = answer.get("target") + if target is not None: + if not isinstance(target, dict): + raise Garbled("a target that is not an object") + record["target"] = { + "requested": _str(target.get("requested"), "a target"), + "resolved": _str(target.get("resolved"), "a target"), + "matching": _decision(target.get("matching")), + } + if files: + seen = answer.get("files") + if not isinstance(seen, dict) or set(seen) != set(files): + raise Garbled("observed files other than the ones asked for") + record["files"] = { + path: None if content is None else _str(content, "a file's content") + for path, content in seen.items() + } + return record + + +# --- Teacher modules ------------------------------------------------------------------- +def _removed_checker(*args, **kwargs): + raise RuntimeError( + "@checker no longer binds a checker to a function: put " + 'check = { function = "" } on each case it should judge ' + "(https://github.com/Acture/obsidian-vault/blob/project/scriptmark/" + "docs/test-bundles.md#migrating-an-older-spec)" + ) + + +def exports_of(module): + """`__all__` when present; otherwise public names, minus modules and minus classes and + functions the module merely imported from elsewhere.""" + names = getattr(module, "__all__", None) + if names is None: + names = [] + for name in dir(module): + if name.startswith("_"): + continue + value = getattr(module, name) + if inspect.ismodule(value): + continue + if (inspect.isclass(value) or inspect.isroutine(value)) and getattr( + value, "__module__", None + ) != module.__name__: + continue + names.append(name) + return {name: getattr(module, name) for name in names} + + +def load_shared(): + """The teacher's own copy of each shared module, importable under its name, so teacher + modules that import it share its classes with what the student side sends.""" + for path in PAYLOAD.get("shared", []): + name = os.path.splitext(os.path.basename(path))[0] + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + share(module) + + +def load_teacher_modules(): + """Returns (exports, duplicates): a name two modules export differently is refused.""" + exports, owner, duplicates = {}, {}, {} + builtins.checker = _removed_checker + try: + load_shared() + for index, path in enumerate(PAYLOAD.get("imports", [])): + directory = os.path.dirname(path) + if directory not in sys.path: + # After the stdlib: a helper beside the module must not shadow it. + sys.path.append(directory) + spec = importlib.util.spec_from_file_location(f"teacher_mod_{index}", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + for name, value in exports_of(module).items(): + if name in exports and not _same(exports[name], value): + duplicates[name] = [owner[name], path] + exports[name] = value + owner.setdefault(name, path) + finally: + del builtins.checker + return exports, duplicates + + +def into_bundle(value, depth=0): + """Whether `value` is, or holds, an absolute path into the test bundle, which only + teacher code can open: student code gets data through `data_files`, by a path relative + to it.""" + bundle = PAYLOAD.get("bundle") + if not bundle or depth > DEPTH_LIMIT: + return False + if isinstance(value, (list, tuple, set, frozenset)): + return any(into_bundle(v, depth + 1) for v in value) + if isinstance(value, dict): + return any(into_bundle(v, depth + 1) for v in [*value.keys(), *value.values()]) + if not isinstance(value, (str, os.PathLike)): + return False + try: + path = os.fspath(value) + if not isinstance(path, str) or not os.path.isabs(path): + return False + path, bundle = os.path.realpath(path), os.path.realpath(bundle) + except (TypeError, ValueError, OSError): + return False + return path == bundle or path.startswith(bundle.rstrip(os.sep) + os.sep) + + +def _same(a, b): + """One object, or two equal ones: two modules that both define `TOL = 1e-6` agree.""" + if a is b: + return True + try: + return bool(a == b) + except BaseException: + return False + + +def params_of(fn): + """Each parameter's name, kind, and whether it has a default.""" + try: + parameters = inspect.signature(fn).parameters.values() + except (TypeError, ValueError): + return None + return [{"name": p.name, "kind": p.kind.name.lower(), "default": p.default is not p.empty} for p in parameters] + + +# --- Values ------------------------------------------------------------------------------ +# Values held on this side: teacher exports, `[vars]`, and what teacher calls made, by name. +context = {} +# Names whose values live on the student side: what the student's calls made. +held = set() +SOURCE = None + + +def bind(name, value=None, student=False): + if student: + context.pop(name, None) + held.add(name) + else: + held.discard(name) + context[name] = value + + +def outgoing(value): + """A plan argument as the student side receives it: a `$name` the student side holds + is named, one this side holds is copied, and `$$` escapes a literal `$`.""" + if isinstance(value, str) and value.startswith("$"): + if value.startswith("$$"): + return value[1:] + name = value[1:] + if name in held: + return {"$": "id", "name": name} + if name in context: + return encode(context[name]) + raise Unresolved(name) + if isinstance(value, list): + return [outgoing(v) for v in value] + if isinstance(value, dict): + pairs = [(k, outgoing(v)) for k, v in value.items()] + if "$" in value: + return {"$": "dict", "v": [[k, v] for k, v in pairs]} + return dict(pairs) + return value + + +def fetch(student, name, timeout): + """A value the student side holds, taken by value now. Returns the value, or the + outcome that kept it from arriving.""" + answer = student.ask({"op": "fetch", "name": name, "timeout": timeout}, call_bound(value=False, transfer=True), timeout + GRACE) + try: + if "value" in answer: + tree = answer["value"] + if isinstance(tree, dict) and tree.get("$") == "too_large": + return None, {"error": problem("TooLarge", f"'{name}' is {tree.get('size')} bytes of JSON, more than a checker can be given")} + return decode(tree), None + outcome = _outcome(answer.get("outcome"), RAN) + except Garbled as exc: + raise student.garbled(str(exc)) from None + if "timeout" in outcome: + return None, {"error": problem("Timeout", f"reading '{name}' for the checker took longer than {timeout}s")} + if "raised" in outcome: + return None, {"error": outcome["raised"]} + return None, {"error": problem("Unreadable", f"'{name}' could not be read for the checker")} + + +def local(value, student, timeout): + """A plan argument for teacher code: student-side values are fetched by value.""" + if isinstance(value, str) and value.startswith("$"): + if value.startswith("$$"): + return value[1:] + name = value[1:] + if name in context: + return context[name] + if name in held: + got, failed = fetch(student, name, timeout) + if failed is not None: + raise Unresolved(name) + return got + raise Unresolved(name) + if isinstance(value, list): + return [local(v, student, timeout) for v in value] + if isinstance(value, dict): + return {k: local(v, student, timeout) for k, v in value.items()} + return value + + +# --- Checks ------------------------------------------------------------------------------ +def verdict_of(returned): + if isinstance(returned, bool): + return {"verdict": {"pass": returned, "message": ""}} + if ( + isinstance(returned, tuple) + and len(returned) == 2 + and isinstance(returned[0], bool) + and isinstance(returned[1], (str, type(None))) + ): + return {"verdict": {"pass": returned[0], "message": clip(returned[1] or "")}} + return { + "error": { + "type": "CheckerContract", + "types": ["CheckerContract"], + "message": f"checker returned {type(returned).__name__}; expected bool or (bool, str)", + } + } + + +def run_check(check, value, stdout, timeout, student): + fn = context.get(check["function"]) + deps = {} + for param in (params_of(fn) or [])[2:]: + name = param["name"] + if "var" in param["kind"]: + continue + if name == "stdout": + deps[name] = stdout + elif name == "source": + deps[name] = SOURCE + elif name in context: + deps[name] = context[name] + elif name in held: + got, failed = fetch(student, name, timeout) + if failed is not None: + return failed + deps[name] = got + elif not param["default"]: + # prepare saw this name in scope: it is an id whose producing call failed. + return {"unresolved": {"name": name}} + done = call(lambda: fn(value, check.get("expected"), **deps), timeout, serialise=False) + if "returned" in done.outcome: + return verdict_of(done.value) + if "timeout" in done.outcome: + return {"error": problem("Timeout", f"checker timed out after {timeout}s")} + raised = done.outcome.get("raised") or {"type": "Error", "types": [], "message": str(done.outcome)} + if "AssertionError" in raised["types"]: + return {"rejected": {"message": raised["message"]}} + return {"error": raised} + + +def judged_value(answer, record, student): + """The value a checker judges: rebuilt from the transfer encoding when the student side + sent one, otherwise from the record, which then says the same thing.""" + tree = answer["transfer"] if "transfer" in answer else record["outcome"]["returned"]["value"] + if isinstance(tree, dict) and tree.get("$") == "too_large": + return None, {"error": problem("TooLarge", f"the value is {tree.get('size')} bytes of JSON, more than a checker can be given")} + try: + return decode(tree), None + except Garbled as exc: + raise student.garbled(str(exc)) from None + + +# --- Calls ------------------------------------------------------------------------------- +def stopped(phase, index, stop): + emit({"kind": "stopped", "phase": phase, "index": index, "reason": stop.reason, "message": clip(stop.message)}) + + +def teacher_call(plan, phase, index, student): + name = plan["target"]["teacher"]["name"] + fn = context.get(name) + try: + # Student-side arguments are read before the teacher's call and its timer starts. + args = local(plan.get("args", []), student, plan["timeout"]) + except Unresolved as exc: + args = exc + except Garbled as exc: + stopped(phase, index, student.garbled(str(exc))) + return None + except Stop as stop: + stopped(phase, index, stop) + return None + + def run(): + if isinstance(args, Unresolved): + raise args + if fn is None or not callable(fn): + raise Missing(f"teacher function '{name}' not found") + return fn(*args) + + done = call(run, plan["timeout"], plan.get("stdin"), serialise=phase != "setup") + record = {"kind": "call", "phase": phase, "index": index, **done.record()} + record["target"] = {"requested": name, "resolved": name, "matching": None} + emit(record) + if "returned" in done.outcome and plan.get("id"): + bind(plan["id"], done.value) + return "returned" in done.outcome + + +def student_call(plan, phase, index, student): + kind, spec = next(iter(plan["target"].items())) + files = plan.get("files") or [] + check = plan.get("check") if phase == "step" else None + request = { + "op": "call", + "target": plan["target"], + "timeout": plan["timeout"], + "stdin": plan.get("stdin"), + "id": plan.get("id"), + "files": files, + "serialise": phase != "setup", + "transfer": check is not None, + } + try: + request["args"] = outgoing(plan.get("args", [])) + if kind in ("method", "attribute") and spec["object"] not in held: + if spec["object"] not in context: + raise Unresolved(spec["object"]) + request["object"] = encode(context[spec["object"]]) + except Unresolved as exc: + emit({ + "kind": "call", "phase": phase, "index": index, + "target": {"requested": spec["name"], "resolved": spec["name"], "matching": None}, + "outcome": {"unresolved": {"name": str(exc)}}, + "stdout": "", "stdout_truncated": False, "elapsed_ms": 0, + }) + return False + if kind != "function": + allowed = LOOKED_UP + elif PAYLOAD["role"] == "student": + allowed = MATCHED + else: + allowed = LOOKED_UP + try: + answer = student.ask(request, call_bound(len(files), transfer=check is not None), plan["timeout"] + GRACE) + try: + record = checked(answer, files, allowed) + except Garbled as exc: + raise student.garbled(str(exc)) from None + returned = "returned" in record["outcome"] + value, failed = (None, None) + if returned and check is not None: + value, failed = judged_value(answer, record, student) + except Stop as stop: + if stop.reason == "deadline": + # The call's own timer should have ended it: the call ran past its deadline. + emit({ + "kind": "call", "phase": phase, "index": index, + "target": {"requested": spec["name"], "resolved": spec["name"], "matching": None}, + "outcome": {"timeout": {}}, "stdout": "", "stdout_truncated": False, + "elapsed_ms": int((plan["timeout"] + GRACE) * 1000), + }) + stopped(phase, index, stop) + return None + try: + emit_bounded({"kind": "call", "phase": phase, "index": index, **record}, call_bound(len(files)), student) + except Stop as stop: + stopped(phase, index, stop) + return None + if returned and plan.get("id"): + bind(plan["id"], student=True) + if returned and check is not None: + try: + try: + verdict = failed or run_check(check, value, record["stdout"], plan["timeout"], student) + except Garbled as exc: + raise student.garbled(str(exc)) from None + except Stop as stop: + emit({"kind": "check", "index": index, "error": problem("StudentEnded", f"the student side stopped while the checker read its values: {stop.message}")}) + stopped(phase, index, stop) + return None + emit({"kind": "check", "index": index, **verdict}) + return returned + + +def run_calls(phase, calls, student): + """Run setup calls or steps in order. Returns False when a setup call failed, and None + when the student side was stopped.""" + for index, plan in enumerate(calls): + if "teacher" in plan["target"]: + returned = teacher_call(plan, phase, index, student) + else: + returned = student_call(plan, phase, index, student) + if returned is None: + return None + if phase == "setup" and not returned: + return False + return True + + +def run_load(student): + try: + answer = student.ask({"op": "load", "timeout": PAYLOAD["load_timeout"]}, call_bound(value=False), PAYLOAD["load_timeout"] + GRACE) + record = checked(answer, [], RAN) + except Garbled as exc: + stopped("load", 0, student.garbled(str(exc))) + return False + except Stop as stop: + if stop.reason == "deadline": + emit({"kind": "call", "phase": "load", "index": 0, "outcome": {"timeout": {}}, + "stdout": "", "stdout_truncated": False, "elapsed_ms": int((PAYLOAD["load_timeout"] + GRACE) * 1000)}) + stopped("load", 0, stop) + return False + try: + emit_bounded({"kind": "call", "phase": "load", "index": 0, **record}, call_bound(value=False), student) + except Stop as stop: + stopped("load", 0, stop) + return False + return "returned" in record["outcome"] + + +def run_script(student): + script = PAYLOAD["script"] + files = script.get("files") or [] + try: + answer = student.ask( + {"op": "script", "timeout": script["timeout"], "stdin": script.get("stdin"), "files": files}, + call_bound(len(files), value=False), + script["timeout"] + GRACE, + ) + compile_error = answer.get("compiles") is False + record = checked(answer, [] if compile_error else files, RAN) + except Garbled as exc: + stopped("step", 0, student.garbled(str(exc))) + return + except Stop as stop: + if stop.reason == "deadline": + emit({"kind": "call", "phase": "step", "index": 0, "outcome": {"timeout": {}}, + "stdout": "", "stdout_truncated": False, "elapsed_ms": int((script["timeout"] + GRACE) * 1000)}) + stopped("step", 0, stop) + return + # A script that does not compile never ran: that is its load failing. + phase = "load" if compile_error else "step" + try: + emit_bounded({"kind": "call", "phase": phase, "index": 0, **record}, call_bound(len(files), value=False), student) + except Stop as stop: + stopped("step", 0, stop) + + +def start_student(): + """Introduce the unit to the student side and wait for it to say it is ready.""" + student = Student(*channel(0, 1)) + init = { + "limits": PAYLOAD["limits"], + "role": PAYLOAD["role"], + "subject": PAYLOAD["subject"], + "vars": PAYLOAD.get("vars", {}), + "allowed_imports": PAYLOAD.get("allowed_imports", []) + + [os.path.splitext(os.path.basename(p))[0] for p in PAYLOAD.get("student_shared", [])], + "functions": PAYLOAD.get("functions", {"aliases": {}, "overrides": {}}), + "shared": PAYLOAD.get("student_shared", []), + "canary": PAYLOAD.get("student_canary"), + } + try: + answer = student.ask(init, SLACK + 6 * MESSAGE_LIMIT, STARTUP) + except Stop as stop: + emit({"kind": "fatal", "stage": "student", "error": problem("StudentStart", f"the student side did not start: {stop.message}")}) + finish() + if answer.get("ready") is not True: + fatal = answer.get("fatal") + error = fatal if isinstance(fatal, dict) else {} + stage = "sandbox" if error.get("type") == "Sandbox" else "student" + emit({"kind": "fatal", "stage": stage, "error": problem(str(error.get("type", "StudentStart")), str(error.get("message", "the student side did not start")))}) + finish() + return student + + +def canary_holds(): + path = PAYLOAD.get("canary") + if not path: + return True + try: + with open(path, "rb"): + return False + except OSError: + return True + + +def main(): + global SOURCE + if not canary_holds(): + emit({"kind": "fatal", "stage": "sandbox", "error": problem("Sandbox", "the teacher sandbox did not hold: its canary was readable")}) + finish() + try: + exports, duplicates = load_teacher_modules() + except BaseException as exc: + emit({"kind": "fatal", "stage": "teacher_import", "error": error_of(exc)}) + finish() + if PAYLOAD["mode"] == "inspect": + emit({"kind": "inspect", "duplicates": duplicates, "exports": { + name: { + "callable": callable(value), + "params": params_of(value) if callable(value) else None, + "bundle_path": not callable(value) and into_bundle(value), + } + for name, value in exports.items() + }}) + finish() + context.update(exports) + context.update(PAYLOAD.get("vars", {})) + if PAYLOAD.get("source"): + with open(PAYLOAD["source"], encoding="utf-8", errors="replace") as fh: + SOURCE = fh.read() + student = start_student() + emit({"kind": "ready"}) + if PAYLOAD.get("script"): + run_script(student) + finish(student) + if run_load(student): + setup = run_calls("setup", PAYLOAD.get("setup", []), student) + if setup: + run_calls("step", PAYLOAD.get("steps", []), student) + finish(student) + + +try: + main() +except BaseException as exc: # a harness bug, not anybody's code under test + emit({"kind": "fatal", "stage": "harness", "error": error_of(exc)}) + finish() diff --git a/src/core/src/runner/judge.rs b/src/core/src/runner/judge.rs index 0b23342..2535ed6 100644 --- a/src/core/src/runner/judge.rs +++ b/src/core/src/runner/judge.rs @@ -1,8 +1,9 @@ //! Turning what a unit did into results. //! -//! The harness only observes. Here every non-pass gets its owner, derived from three -//! things: which call the plan says was running, what kind of code that call runs, and -//! how the process ended. Nothing a student can print decides it. +//! The harness only observes. Here every non-pass gets its owner, derived from four +//! things: which call the plan says was running, what kind of code that call runs, whether +//! the teacher side gave up on the student side there, and how each side ended. Nothing a +//! student can print decides it. use serde_json::{Value, json}; @@ -14,8 +15,10 @@ use crate::models::{ CaseInput, CaseResult, Cause, Check, FailureDetail, Fault, Target, TestCase, TestStatus, }; use crate::runner::executor::{ - CallObservation, CheckObservation, Exit, Outcome, ProtocolError, UnitObservation, UnitPlan, + CallObservation, CheckObservation, Exit, Outcome, Phase, ProtocolError, StopReason, Stopped, + UnitObservation, UnitPlan, }; +use crate::runner::sandbox::Isolation; /// A scored call: the name its result is reported under, and what it expects. #[derive(Debug, Clone, Copy)] @@ -34,12 +37,26 @@ struct Blanket { } /// Judge one unit. `scored[i]` is `plan.steps[i]` — or, in script mode, the script run. +/// How the teacher's Python checker scripts run. +#[derive(Debug, Clone, Copy)] +struct Scripts<'a> { + python: &'a str, + isolation: Isolation, + bundle: &'a std::path::Path, +} + pub fn judge( plan: &UnitPlan, scored: &[Scored], obs: &UnitObservation, python_cmd: &str, + isolation: Isolation, ) -> Vec { + let scripts = Scripts { + python: python_cmd, + isolation, + bundle: &plan.bundle, + }; let target = |index: usize| plan.steps.get(index).map(|c| target_name(&c.target)); if let Some(blanket) = unit_failure(plan, obs) { return scored @@ -50,6 +67,8 @@ pub fn judge( } // The harness broke after student code ran: what it recorded before that still stands. let broken = stream_break(obs); + // The teacher side gave up on the student side at a step: nothing after it ran. + let stop = obs.stopped.as_ref().filter(|s| s.phase == Phase::Step); let mut results = Vec::with_capacity(scored.len()); // Once a call hangs or the process dies, nothing after it ran — on whoever's account. @@ -63,13 +82,30 @@ pub fn judge( match obs.steps.get(index) { Some(record) => { let check = obs.checks.get(&index); + if let (Some(stop), Some(CheckObservation::Error(e))) = (stop, check) + && stop.index == index + && e.type_name == "StudentEnded" + { + // The student side went while the checker read its values: how it + // went decides, not the checker. + let blanket = + student_stop(obs, stop, "the reading of its values for the checker"); + results.push(blanket_result(s, &blanket, target(index))); + stopped = Some(not_run(&blanket, s.name)); + continue; + } if check.is_none() && call.is_some_and(|c| c.check.is_some()) && matches!(record.outcome, Outcome::Returned { .. }) { // The student's call finished; the teacher's checker never did. let blanket = broken.clone().unwrap_or_else(|| { - in_flight(&obs.exit, Fault::Teacher, Cause::Checker, "its checker") + in_flight( + &obs.teacher_exit, + Fault::Teacher, + Cause::Checker, + "its checker", + ) }); results.push(blanket_result(s, &blanket, target(index))); stopped = Some(broken.clone().unwrap_or_else(|| not_run(&blanket, s.name))); @@ -85,23 +121,35 @@ pub fn judge( record, check, plan.script.is_some(), - python_cmd, + scripts, )); } None => { - let blanket = match &broken { - Some(broken) => broken.clone(), - None if obs.done => Blanket { + let blanket = match (&broken, stop) { + (Some(broken), _) => broken.clone(), + (None, Some(stop)) if stop.index < index => { + // It stopped at a call that reported: its record says how. + let at = scored.get(stop.index).map_or("?", |s| s.name); + not_run(&student_stop(obs, stop, "this call"), at) + } + (None, Some(stop)) => student_stop(obs, stop, "this call"), + (None, None) if obs.done => Blanket { status: TestStatus::Error, fault: Fault::Environment, cause: Cause::Harness, message: "the harness finished without running this call".into(), }, - None => in_flight(&obs.exit, Fault::Student, Cause::Killed, "this call"), + (None, None) => unfinished( + obs, + call.map_or(Fault::Student, |c| owner_of(&c.target)), + Cause::Killed, + "this call", + ), }; results.push(blanket_result(s, &blanket, target(index))); stopped = Some(match &broken { Some(broken) => broken.clone(), + None if blanket.cause == Cause::NotRun => blanket.clone(), None => not_run(&blanket, s.name), }); } @@ -120,26 +168,44 @@ fn unit_failure(plan: &UnitPlan, obs: &UnitObservation) -> Option { message, }) }; - if let Exit::Spawn(message) = &obs.exit { + for exit in [&obs.teacher_exit, &obs.exit] { + if let Exit::Spawn(message) = exit { + return blanket( + TestStatus::Error, + Fault::Environment, + Cause::Spawn, + message.clone(), + ); + } + } + if let Some(ProtocolError::Tampered(problem)) = &obs.protocol_error { + let (fault, cause) = teacher_side_broken(obs); + return blanket( + TestStatus::Error, + fault, + cause, + format!("the unit's records cannot be trusted: {problem}"), + ); + } + if let Some(fatal) = obs.fatal.as_ref().filter(|f| f.stage == "sandbox") { return blanket( TestStatus::Error, Fault::Environment, Cause::Spawn, - message.clone(), + format!("refused to grade: {}", fatal.error.message), ); } - if let Some(ProtocolError::Tampered(problem)) = &obs.protocol_error { - // After `ready`, student code has had the chance to write; before it, nobody's has. - let fault = if obs.ready { - Fault::Student - } else { - Fault::Environment - }; + if let Some(fatal) = obs.fatal.as_ref().filter(|f| f.stage == "student") { return blanket( TestStatus::Error, - fault, - Cause::Protocol, - format!("the unit's records cannot be trusted: {problem}"), + Fault::Environment, + Cause::Harness, + format!( + "{}: {} ({})", + fatal.error.type_name, + fatal.error.message, + stderr_hint(obs) + ), ); } if let Some(fatal) = obs.fatal.as_ref().filter(|f| f.stage == "teacher_import") { @@ -172,7 +238,12 @@ fn unit_failure(plan: &UnitPlan, obs: &UnitObservation) -> Option { TestStatus::Error, fault, cause, - format!("{} {} ({})", death(&obs.exit).1, what, stderr_hint(obs)), + format!( + "{} {} ({})", + death(&obs.teacher_exit).1, + what, + stderr_hint(obs) + ), ); } if plan.script.is_some() { @@ -191,12 +262,11 @@ fn unit_failure(plan: &UnitPlan, obs: &UnitObservation) -> Option { match &obs.load { None if stream_break(obs).is_some() => return stream_break(obs), None => { - return Some(in_flight( - &obs.exit, - Fault::Student, - Cause::Killed, - "the student module while it loaded", - )); + let what = "the student module while it loaded"; + return Some(match &obs.stopped { + Some(stop) => student_stop(obs, stop, what), + None => unfinished(obs, Fault::Student, Cause::Killed, what), + }); } Some(load) => match &load.outcome { Outcome::Returned { .. } => {} @@ -252,6 +322,17 @@ fn unit_failure(plan: &UnitPlan, obs: &UnitObservation) -> Option { message.clone(), ); } + // Every name is resolved before the call: a setup call that produced one + // failing stops the loop first, so what is unresolved here is a student's + // value that could not be read — whoever's call needed it. + outcome @ Outcome::Unresolved { .. } => { + return blanket( + TestStatus::Error, + Fault::Student, + Cause::Dependency, + format!("not run: {label} {}", describe(outcome, call.timeout)), + ); + } outcome => { return blanket( TestStatus::Error, @@ -265,7 +346,21 @@ fn unit_failure(plan: &UnitPlan, obs: &UnitObservation) -> Option { if let Some(broken) = stream_break(obs) { return Some(broken); } - let killed = in_flight(&obs.exit, owner, Cause::Setup, &label); + let killed = match &obs.stopped { + // How the student side stopped says more than "setup": a timeout stays + // a timeout, a memory kill a memory kill. + Some(stop) => { + let stopped = student_stop(obs, stop, &label); + return Some(Blanket { + message: format!("not run: {}", stopped.message), + ..stopped + }); + } + None => unfinished(obs, owner, Cause::Setup, &label), + }; + if killed.fault != owner { + return Some(killed); + } return blanket( TestStatus::Error, owner, @@ -278,7 +373,7 @@ fn unit_failure(plan: &UnitPlan, obs: &UnitObservation) -> Option { None } -/// The verdict for the call in flight when the process stopped. +/// The verdict for the call in flight when a process stopped. fn in_flight(exit: &Exit, fault: Fault, cause: Cause, what: &str) -> Blanket { let (status, how) = death(exit); Blanket { @@ -289,11 +384,70 @@ fn in_flight(exit: &Exit, fault: Fault, cause: Cause, what: &str) -> Blanket { TestStatus::Error }, fault, - cause, + cause: if matches!(exit, Exit::Memory { .. }) { + Cause::Memory + } else { + cause + }, message: format!("{how} {what}"), } } +/// The verdict for the call during which the teacher side gave up on the student side. +fn student_stop(obs: &UnitObservation, stop: &Stopped, what: &str) -> Blanket { + match stop.reason { + StopReason::Exited => in_flight(&obs.exit, Fault::Student, Cause::Killed, what), + StopReason::Deadline => Blanket { + status: TestStatus::Timeout, + fault: Fault::Student, + cause: Cause::Killed, + message: format!("the student side stopped answering during {what}"), + }, + StopReason::Flooded | StopReason::Garbled => Blanket { + status: TestStatus::Error, + fault: Fault::Student, + cause: Cause::Protocol, + message: format!( + "the student side's answer could not be read during {what}: {}", + stop.message + ), + }, + } +} + +/// The verdict for a call whose record never came, when the teacher side did not say it +/// gave up on the student side: the teacher side itself ended. Teacher code in flight is +/// the teacher's; a student call in flight was waiting on the teacher side, which student +/// code cannot reach. +fn unfinished(obs: &UnitObservation, owner: Fault, cause: Cause, what: &str) -> Blanket { + if owner == Fault::Teacher { + return in_flight(&obs.teacher_exit, Fault::Teacher, cause, what); + } + let (fault, cause) = teacher_side_broken(obs); + let (_, how) = death(&obs.teacher_exit); + Blanket { + status: TestStatus::Error, + fault, + cause, + message: format!( + "the teacher side ended: {how} {what} ({})", + stderr_hint(obs) + ), + } +} + +/// Whose doing it is when the teacher side breaks — ends, or writes records it could not +/// have meant. In its sandbox, student code cannot reach it: the harness's. Unsandboxed, +/// once student code has run, it can — signal it, or write into its records — and it is +/// the student's, as it was when both shared one process. +fn teacher_side_broken(obs: &UnitObservation) -> (Fault, Cause) { + if obs.sandboxed || !obs.ready { + (Fault::Environment, Cause::Harness) + } else { + (Fault::Student, Cause::Protocol) + } +} + /// The verdict for scored calls after one that stopped the unit. fn not_run(culprit: &Blanket, after: &str) -> Blanket { Blanket { @@ -311,6 +465,27 @@ fn death(exit: &Exit) -> (TestStatus, String) { TestStatus::Timeout, "the unit's deadline passed during".into(), ), + Exit::Memory { + limit_mb, + idle_mb, + basis, + } => { + let how = match basis { + Some(basis) => format!("estimated from {basis}"), + None => "`memory_mb`".into(), + }; + ( + TestStatus::Error, + format!( + "the {limit_mb} MiB memory limit ({how}, on top of the {idle_mb} MiB Python \ + and the submission's imports take) was exceeded during" + ), + ) + } + Exit::Stopped => ( + TestStatus::Error, + "the process was ended with its unit during".into(), + ), #[cfg(unix)] Exit::Signal(signal) if *signal == libc::SIGXCPU => ( TestStatus::Timeout, @@ -334,7 +509,7 @@ fn death(exit: &Exit) -> (TestStatus, String) { fn stderr_hint(obs: &UnitObservation) -> String { let tail = obs.stderr.trim(); if tail.is_empty() { - format!("{:?}", obs.exit) + format!("{:?}", obs.teacher_exit) } else { tail.lines().last().unwrap_or(tail).to_string() } @@ -362,17 +537,18 @@ fn describe(outcome: &Outcome, timeout: u64) -> String { } } -/// The record stream stopped meaning anything part-way: the harness crashed, wrote a -/// record it could not have meant, or somebody flooded the channel. What was recorded -/// before that stands; nothing after it can be known. +/// The record stream stopped meaning anything part-way: the harness crashed, or the +/// teacher side wrote a record it could not have meant. What was recorded before that +/// stands; nothing after it can be known. fn stream_break(obs: &UnitObservation) -> Option { + let broken = teacher_side_broken(obs); let (fault, cause, message) = match (&obs.fatal, &obs.protocol_error) { (_, Some(ProtocolError::Flooded(problem))) => ( - Fault::Student, - Cause::Protocol, - format!("the unit's output could not be read past this point: {problem}"), + broken.0, + broken.1, + format!("the unit's records could not be read past this point: {problem}"), ), - (Some(fatal), _) if fatal.stage != "teacher_import" => ( + (Some(fatal), _) if fatal.stage == "harness" => ( Fault::Environment, Cause::Harness, format!( @@ -381,8 +557,8 @@ fn stream_break(obs: &UnitObservation) -> Option { ), ), (_, Some(ProtocolError::Unreadable(problem))) => ( - Fault::Environment, - Cause::Harness, + broken.0, + broken.1, format!("the harness wrote a record it could not have meant: {problem}"), ), _ => return None, @@ -472,7 +648,7 @@ fn judge_call( record: &CallObservation, check: Option<&CheckObservation>, script: bool, - python_cmd: &str, + scripts: Scripts, ) -> CaseResult { let case = s.case; let mut verdict = Verdict { @@ -554,7 +730,7 @@ fn judge_call( case.expected_stdout.clone().map(Value::String), record, check, - python_cmd, + scripts, ) { return failed; } @@ -597,15 +773,9 @@ fn judge_call( } else { (value.clone(), case.expect.clone()) }; - if let Some(failed) = check_value( - &mut verdict, - case, - &value, - expected, - record, - check, - python_cmd, - ) { + if let Some(failed) = + check_value(&mut verdict, case, &value, expected, record, check, scripts) + { return failed; } } @@ -662,7 +832,7 @@ fn check_value( expected: Option, record: &CallObservation, check: Option<&CheckObservation>, - python_cmd: &str, + scripts: Scripts, ) -> Option { let take = |verdict: &mut Verdict| { std::mem::replace( @@ -735,7 +905,17 @@ fn check_value( } Check::Rhai(expr) => Box::new(RhaiChecker::new(expr)), Check::Python(script) => { - Box::new(PythonChecker::new(script).with_python_cmd(python_cmd)) + let checker = PythonChecker::new(script).with_python_cmd(scripts.python); + Box::new(if scripts.isolation.is_sandboxed() { + let beside = std::path::Path::new(script).parent().map(Into::into); + checker.confined( + std::iter::once(scripts.bundle.to_path_buf()) + .chain(beside) + .collect(), + ) + } else { + checker + }) } Check::Function(_) => unreachable!("handled above"), }; @@ -795,6 +975,10 @@ mod tests { imports: Vec::new(), vars: Arc::new(BTreeMap::new()), data_files: Vec::new(), + bundle: std::path::PathBuf::new(), + shared: Vec::new(), + memory_mb: 64, + memory_basis: None, allowed_imports: Vec::new(), load_timeout: 10, setup: Vec::new(), @@ -854,6 +1038,7 @@ mod tests { steps, done: true, exit: Exit::Code(0), + sandboxed: true, ..UnitObservation::not_started(Exit::Code(0)) } } @@ -863,7 +1048,14 @@ mod tests { } fn one(case: &TestCase, plan: &UnitPlan, obs: &UnitObservation) -> CaseResult { - judge(plan, &[Scored { name: "c", case }], obs, "python3").remove(0) + judge( + plan, + &[Scored { name: "c", case }], + obs, + "python3", + Isolation::Sandboxed, + ) + .remove(0) } fn verdict(r: &CaseResult) -> (TestStatus, Option, Option) { @@ -1099,10 +1291,16 @@ mod tests { case: &c, }, ]; + // The teacher side saw the student side go during 'b', and said so. let mut obs = finished(vec![record(0, returned(json!(1)))]); - obs.done = false; + obs.stopped = Some(Stopped { + phase: Phase::Step, + index: 1, + reason: StopReason::Exited, + message: "the student side ended".into(), + }); obs.exit = Exit::Signal(libc::SIGXCPU); - let r = judge(&p, &scored, &obs, "python3"); + let r = judge(&p, &scored, &obs, "python3", Isolation::Sandboxed); assert_eq!(r[0].status, TestStatus::Passed); assert_eq!( verdict(&r[1]), @@ -1118,11 +1316,198 @@ mod tests { ); obs.exit = Exit::Signal(libc::SIGSEGV); - let r = judge(&p, &scored, &obs, "python3"); + let r = judge(&p, &scored, &obs, "python3", Isolation::Sandboxed); assert_eq!( verdict(&r[1]), (TestStatus::Error, Some(Fault::Student), Some(Cause::Killed)) ); + + obs.exit = Exit::Memory { + limit_mb: 64, + idle_mb: 24, + basis: None, + }; + let r = judge(&p, &scored, &obs, "python3", Isolation::Sandboxed); + assert_eq!( + verdict(&r[1]), + (TestStatus::Error, Some(Fault::Student), Some(Cause::Memory)) + ); + assert_eq!( + verdict(&r[2]), + (TestStatus::Error, Some(Fault::Student), Some(Cause::NotRun)) + ); + + // A step that answered too much, or nonsense, is the student's. + obs.stopped.as_mut().unwrap().reason = StopReason::Flooded; + let r = judge(&p, &scored, &obs, "python3", Isolation::Sandboxed); + assert_eq!( + verdict(&r[1]), + ( + TestStatus::Error, + Some(Fault::Student), + Some(Cause::Protocol) + ) + ); + + // Given up on at its deadline: the teacher side reported the call as timed out, and + // the rest as never run. + let mut late = finished(vec![ + record(0, returned(json!(1))), + record(1, Outcome::Timeout {}), + ]); + late.stopped = Some(Stopped { + phase: Phase::Step, + index: 1, + reason: StopReason::Deadline, + message: String::new(), + }); + late.exit = Exit::Stopped; + let r = judge(&p, &scored, &late, "python3", Isolation::Sandboxed); + assert_eq!( + verdict(&r[1]), + ( + TestStatus::Timeout, + Some(Fault::Student), + Some(Cause::Timeout) + ) + ); + assert_eq!( + verdict(&r[2]), + (TestStatus::Error, Some(Fault::Student), Some(Cause::NotRun)) + ); + assert!( + r[2].failure.as_ref().unwrap().message.contains("'b'"), + "{:?}", + r[2].failure + ); + + // Without a word from the teacher side, the teacher side itself went: in its sandbox + // the student cannot reach it... + let mut lost = finished(vec![record(0, returned(json!(1)))]); + lost.done = false; + lost.teacher_exit = Exit::Signal(libc::SIGKILL); + let r = judge(&p, &scored, &lost, "python3", Isolation::Sandboxed); + assert_eq!( + verdict(&r[1]), + ( + TestStatus::Error, + Some(Fault::Environment), + Some(Cause::Harness) + ) + ); + // ...but unsandboxed, student code could have signalled it. + lost.sandboxed = false; + let r = judge(&p, &scored, &lost, "python3", Isolation::Sandboxed); + assert_eq!( + verdict(&r[1]), + ( + TestStatus::Error, + Some(Fault::Student), + Some(Cause::Protocol) + ) + ); + assert_eq!( + verdict(&r[2]), + (TestStatus::Error, Some(Fault::Student), Some(Cause::NotRun)) + ); + } + + /// A student value that cannot be read for a teacher setup call is the student's; a + /// student side that stops during setup keeps how it stopped; and one that goes while a + /// checker reads its values is judged by how it went, not as a failed checker. + #[test] + fn test_what_the_student_side_does_outside_a_scored_call_is_still_the_students() { + let c = case("expect = 1"); + let mut p = plan(1); + let mut student_setup = call(Target::Function { + name: "make".into(), + }); + student_setup.id = Some("obj".into()); + let mut teacher_setup = call(Target::Teacher { + name: "wrap".into(), + }); + teacher_setup.id = Some("wrapped".into()); + p.setup = vec![student_setup, teacher_setup]; + + let mut unreadable = finished(vec![]); + unreadable.setup = vec![ + record(0, returned(Value::Null)), + record(1, Outcome::Unresolved { name: "obj".into() }), + ]; + assert_eq!( + verdict(&one(&c, &p, &unreadable)), + ( + TestStatus::Error, + Some(Fault::Student), + Some(Cause::Dependency) + ) + ); + + let mut hog = finished(vec![]); + hog.setup = Vec::new(); + hog.stopped = Some(Stopped { + phase: Phase::Setup, + index: 0, + reason: StopReason::Exited, + message: String::new(), + }); + hog.exit = Exit::Memory { + limit_mb: 64, + idle_mb: 24, + basis: None, + }; + let r = one(&c, &p, &hog); + assert_eq!( + verdict(&r), + (TestStatus::Error, Some(Fault::Student), Some(Cause::Memory)) + ); + assert!(r.failure.unwrap().message.starts_with("not run:")); + + let mut p = plan(2); + p.steps[0].check = Some(InProcessCheck { + function: "chk".into(), + expected: None, + }); + let c = case("expect = 1\ncheck = { function = \"chk\" }"); + let scored = [ + Scored { + name: "a", + case: &c, + }, + Scored { + name: "b", + case: &c, + }, + ]; + let mut gone = finished(vec![record(0, returned(json!(1)))]); + gone.checks.insert( + 0, + CheckObservation::Error(ErrorInfo { + type_name: "StudentEnded".into(), + types: vec!["StudentEnded".into()], + message: "the student side ended".into(), + }), + ); + gone.stopped = Some(Stopped { + phase: Phase::Step, + index: 0, + reason: StopReason::Exited, + message: String::new(), + }); + gone.exit = Exit::Memory { + limit_mb: 64, + idle_mb: 24, + basis: None, + }; + let r = judge(&p, &scored, &gone, "python3", Isolation::Sandboxed); + assert_eq!( + verdict(&r[0]), + (TestStatus::Error, Some(Fault::Student), Some(Cause::Memory)) + ); + assert_eq!( + verdict(&r[1]), + (TestStatus::Error, Some(Fault::Student), Some(Cause::NotRun)) + ); } #[test] @@ -1175,8 +1560,19 @@ mod tests { (TestStatus::Error, Some(Fault::Student), Some(Cause::Load)) ); + // Only the teacher side writes records: sandboxed, student code cannot have broken + // them; unsandboxed, it could have written into them. let mut tampered = finished(vec![record(0, returned(json!(1)))]); tampered.protocol_error = Some(ProtocolError::Tampered("step record 0 repeated".into())); + assert_eq!( + verdict(&one(&c, &p, &tampered)), + ( + TestStatus::Error, + Some(Fault::Environment), + Some(Cause::Harness) + ) + ); + tampered.sandboxed = false; assert_eq!( verdict(&one(&c, &p, &tampered)), ( @@ -1185,6 +1581,40 @@ mod tests { Some(Cause::Protocol) ) ); + + // A sandbox that did not hold grades nobody. + let mut leaky = UnitObservation::not_started(Exit::Code(0)); + leaky.fatal = Some(Fatal { + stage: "sandbox".into(), + error: ErrorInfo { + type_name: "Sandbox".into(), + types: vec![], + message: "the student sandbox did not hold".into(), + }, + }); + assert_eq!( + verdict(&one(&c, &p, &leaky)), + ( + TestStatus::Error, + Some(Fault::Environment), + Some(Cause::Spawn) + ) + ); + + // A student module that never finished loading because its side went away. + let mut gone = finished(vec![]); + gone.load = None; + gone.stopped = Some(Stopped { + phase: Phase::Load, + index: 0, + reason: StopReason::Exited, + message: String::new(), + }); + gone.exit = Exit::Signal(libc::SIGKILL); + assert_eq!( + verdict(&one(&c, &p, &gone)), + (TestStatus::Error, Some(Fault::Student), Some(Cause::Killed)) + ); } #[test] @@ -1276,7 +1706,7 @@ mod tests { let mut obs = finished(vec![record(0, returned(json!(1)))]); obs.done = false; broke(&mut obs); - let r = judge(&p, &scored, &obs, "python3"); + let r = judge(&p, &scored, &obs, "python3", Isolation::Sandboxed); assert_eq!(r[0].status, TestStatus::Passed, "an earlier pass stands"); assert_eq!( verdict(&r[1]), @@ -1435,15 +1865,15 @@ mod tests { ]; let mut obs = finished(vec![record(0, returned(json!(1)))]); obs.done = false; - obs.protocol_error = Some(ProtocolError::Flooded("64 MiB".into())); - let r = judge(&p, &scored, &obs, "python3"); + obs.protocol_error = Some(ProtocolError::Flooded("a record of 64 MiB".into())); + let r = judge(&p, &scored, &obs, "python3", Isolation::Sandboxed); assert_eq!(r[0].status, TestStatus::Passed); assert_eq!( verdict(&r[1]), ( TestStatus::Error, - Some(Fault::Student), - Some(Cause::Protocol) + Some(Fault::Environment), + Some(Cause::Harness) ) ); } diff --git a/src/core/src/runner/linter.rs b/src/core/src/runner/linter.rs index d59ba5d..7e283ee 100644 --- a/src/core/src/runner/linter.rs +++ b/src/core/src/runner/linter.rs @@ -1,31 +1,128 @@ -use std::path::Path; -use std::process::Command; +//! Style scores from a teacher's lint tool. +//! +//! The tool runs on a read-only copy of the student's file, in a private directory, with a +//! cleared environment and a deadline. Sandboxed, it runs in the teacher's sandbox: it reads +//! the system's programs, its own installation, the test bundle and that copy, writes only +//! its directory, and reaches no network. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Duration; use crate::models::LintConfig; +use crate::runner::sandbox::{Confinement, Isolation}; + +/// How long a lint tool may run on one file. +pub const LINT_TIMEOUT: Duration = Duration::from_secs(120); -/// Run a lint command on a student file and return its style score, 0 to 100. +/// Run a lint command on a copy of `file_path` and return its style score, 0 to 100. /// -/// Errs when the tool did not run properly — it could not start, or exited with a code -/// outside `ok_exit_codes`. A tool that failed has found nothing, and reading that as a -/// clean file would hand out full marks for a broken grader. -pub fn run_lint(config: &LintConfig, file_path: &Path) -> Result { - // Split the template before filling in the path, so a path with a space in it stays one - // argument. - let file = file_path.display().to_string(); - let parts: Vec = config - .command - .split_whitespace() - .map(|part| part.replace("{file}", &file)) - .collect(); - let command = parts.join(" "); - let Some((program, args)) = parts.split_first() else { +/// Errs when the tool did not run properly — it could not start, ran past its deadline, or +/// exited with a code outside `ok_exit_codes`. A tool that failed has found nothing, and +/// reading that as a clean file would hand out full marks for a broken grader. +pub fn run_lint( + config: &LintConfig, + file_path: &Path, + bundle: &Path, + isolation: Isolation, + timeout: Duration, +) -> Result { + // Split as a shell would, quotes included, before filling in the path: a path with a + // space in it stays one argument. + let parts = split(&config.command)?; + let Some((program, _)) = parts.split_first() else { return Err("the lint command is empty".into()); }; + let program = locate(program)?; + + let scratch = tempfile::Builder::new() + .prefix("scriptmark-lint-") + .tempdir() + .and_then(|dir| std::fs::canonicalize(dir.path()).map(|path| (dir, path))) + .map_err(|e| format!("could not stage the file to lint: {e}"))?; + let (_keep, base) = scratch; + let work = base.join("work"); + let name = file_path + .file_name() + .ok_or_else(|| format!("'{}' names no file", file_path.display()))?; + let copy = base + .join("submission") + .join(safe_name(&name.to_string_lossy())); + std::fs::create_dir_all(&work) + .and_then(|_| std::fs::create_dir_all(copy.parent().expect("a parent"))) + .and_then(|_| std::fs::copy(file_path, ©)) + .and_then(|_| { + let mut readonly = std::fs::metadata(©)?.permissions(); + readonly.set_readonly(true); + std::fs::set_permissions(©, readonly) + }) + .map_err(|e| format!("could not stage the file to lint: {e}"))?; - let out = Command::new(program) - .args(args) - .output() - .map_err(|e| format!("could not run `{program}`: {e}"))?; + let file = copy.display().to_string(); + // `{bundle}` names the test bundle, for a configuration file kept in it. + let bundle_dir = bundle.display().to_string(); + let args: Vec = parts[1..] + .iter() + .map(|part| { + part.replace("{file}", &file) + .replace("{bundle}", &bundle_dir) + }) + .collect(); + let command = std::iter::once(program.display().to_string()) + .chain(args.iter().cloned()) + .collect::>() + .join(" "); + + let mut cmd = if isolation.is_sandboxed() { + let confinement = Confinement { + read: crate::runner::sandbox::covering( + crate::runner::sandbox::system_programs() + .into_iter() + .chain(tool_reads(&program)) + .chain([bundle.to_path_buf(), base.join("submission")]) + .filter_map(|p| std::fs::canonicalize(p).ok()) + .collect(), + ), + write: vec![work.clone()], + exec: Vec::new(), + spawn: true, + no_ffi: false, + }; + crate::runner::sandbox::confined(&program, &confinement) + .map_err(|e| format!("cannot sandbox `{command}`: {e}"))? + } else { + Command::new(&program) + }; + // The tool's own directory, then the system's: what a wrapper script it runs may need. + let path = std::env::join_paths( + program.parent().map(Path::to_path_buf).into_iter().chain( + PROGRAM_DIRS + .iter() + .map(PathBuf::from) + .filter(|d| d.is_dir()), + ), + ) + .unwrap_or_default(); + cmd.args(&args) + .env_clear() + .env("PATH", path) + .env("HOME", &work) + .env("TMPDIR", &work) + .current_dir(&work) + .stdin(Stdio::null()); + #[cfg(windows)] + if let Some(root) = std::env::var_os("SYSTEMROOT") { + cmd.env("SYSTEMROOT", root); + } + + let out = crate::checker::python_checker::run_with_timeout(cmd, None, timeout) + .map_err(|e| { + format!( + "could not run `{command}`: {}", + crate::runner::sandbox::spawn_error(&e) + ) + })? + .ok_or_else(|| format!("`{command}` ran longer than {}s", timeout.as_secs()))?; if !out .status .code() @@ -34,18 +131,161 @@ pub fn run_lint(config: &LintConfig, file_path: &Path) -> Result { return Err(format!( "`{command}` exited with {}: {}", out.status, - String::from_utf8_lossy(&out.stderr).trim() + out.stderr.trim() )); } - let stdout = String::from_utf8_lossy(&out.stdout); - let warnings = match serde_json::from_str::>(&stdout) { + let warnings = match serde_json::from_str::>(&out.stdout) { Ok(findings) => findings.len(), - Err(_) => stdout.lines().filter(|l| !l.trim().is_empty()).count(), + Err(_) => out.stdout.lines().filter(|l| !l.trim().is_empty()).count(), }; Ok(style_score(warnings, config.max_warnings)) } +/// The student's file name with anything a shell would read as syntax replaced: the name +/// reaches the tool's arguments, and a teacher's command may hand it to `sh -c`. Letters, +/// digits, `.`, `_` and `-` stay, so a linter judging module names judges the same one. +fn safe_name(name: &str) -> String { + let safe: String = name + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-') { + c + } else { + '_' + } + }) + .collect(); + // Never an option, and never `.` or `..`. + if safe.starts_with(['-', '.']) { + format!("_{safe}") + } else { + safe + } +} + +/// Where a tool finds the system's programs, in the order a shell would look. +#[cfg(unix)] +const PROGRAM_DIRS: &[&str] = &[ + "/usr/local/bin", + "/opt/homebrew/bin", + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", +]; +#[cfg(not(unix))] +const PROGRAM_DIRS: &[&str] = &[]; + +/// A command split as a shell would, quotes included. On Windows, where a path is written +/// with backslashes a POSIX shell would read as escapes, on whitespace. +fn split(command: &str) -> Result, String> { + if cfg!(windows) { + return Ok(command.split_whitespace().map(String::from).collect()); + } + shlex::split(command) + .ok_or_else(|| format!("the lint command has an unclosed quote: {command}")) +} + +/// The program a command names: a path as given, or the first match on the grader's PATH +/// (with Windows' executable extensions there). +fn locate(program: &str) -> Result { + let given = Path::new(program); + if given.components().count() > 1 { + return std::path::absolute(given).map_err(|e| format!("cannot find `{program}`: {e}")); + } + let extensions: Vec = if cfg!(windows) { + std::iter::once(String::new()) + .chain( + std::env::var("PATHEXT") + .unwrap_or_else(|_| ".EXE;.CMD;.BAT".into()) + .split(';') + .map(String::from), + ) + .collect() + } else { + vec![String::new()] + }; + std::env::var_os("PATH") + .iter() + .flat_map(std::env::split_paths) + .flat_map(|dir| { + extensions + .iter() + .map(move |ext| dir.join(format!("{program}{ext}"))) + }) + .find(|candidate| candidate.is_file()) + .ok_or_else(|| format!("could not run `{program}`: not found on PATH")) +} + +/// What a tool reads to run: its installation — the directory above its `bin`, or its own — +/// and, for a script, its interpreter's, a Python interpreter's prefixes and packages +/// included. +fn tool_reads(program: &Path) -> Vec { + let Ok(real) = std::fs::canonicalize(program) else { + return Vec::new(); + }; + let mut reads = vec![program.to_path_buf(), installation(&real)]; + let interpreter = std::fs::read(&real).ok().and_then(|bytes| { + let line = bytes + .split(|b| *b == b'\n') + .next()? + .strip_prefix(b"#!")? + .to_vec(); + let line = String::from_utf8(line).ok()?; + let mut words = line.split_whitespace(); + let first = words.next()?; + // `#!/usr/bin/env python3` names its interpreter by the word after `env`. + match Path::new(first).file_name()?.to_str()? { + "env" => locate(words.next()?).ok(), + _ => Some(PathBuf::from(first)), + } + }); + if let Some(interpreter) = interpreter { + let is_python = interpreter + .file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.starts_with("python")); + if is_python + && let Ok(confinement) = crate::runner::python::teacher_confinement( + &interpreter.to_string_lossy(), + Vec::new(), + Vec::new(), + true, + ) { + reads.extend(confinement.read); + } + if let Ok(real) = std::fs::canonicalize(&interpreter) { + reads.push(installation(&real)); + } + reads.push(interpreter); + } + reads +} + +/// What a program's installation is: the whole of it when it is self-contained — a virtual +/// environment, as `uv tool` and `pipx` make, or a Homebrew keg — and otherwise only the +/// program itself. A directory it merely sits in, such as `~/bin` or `/usr/local/bin`, may +/// hold anything. +fn installation(real: &Path) -> PathBuf { + let dir = real.parent().unwrap_or(real); + let above = dir.parent().filter(|above| { + let in_bin = matches!( + dir.file_name().and_then(|n| n.to_str()), + Some("bin" | "sbin") + ); + let venv = above.join("pyvenv.cfg").is_file(); + // …/Cellar///bin/ + let keg = above + .parent() + .and_then(Path::parent) + .and_then(Path::file_name) + .is_some_and(|n| n == "Cellar"); + in_bin && (venv || keg) + }); + above.unwrap_or(real).to_path_buf() +} + /// `max(0, 1 - warnings / max_warnings) * 100`; with `max_warnings = 0`, any warning is 0. fn style_score(warnings: usize, max_warnings: usize) -> f64 { if max_warnings == 0 { @@ -73,6 +313,80 @@ mod tests { (dir, file) } + /// Unconfined, so these run inside any sandbox the test itself is in; the sandboxed + /// runs are exercised end to end in the integration tests. + fn lint(config: &LintConfig, file: &Path) -> Result { + run_lint( + config, + file, + file.parent().unwrap(), + Isolation::Unisolated, + LINT_TIMEOUT, + ) + } + + #[test] + fn test_a_tool_is_read_whole_only_when_its_installation_is_its_own() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + for made in ["venv/bin", "Cellar/ruff/0.6/bin", "home/bin"] { + std::fs::create_dir_all(root.join(made)).unwrap(); + } + std::fs::write(root.join("venv/pyvenv.cfg"), "home = /usr/bin\n").unwrap(); + assert_eq!(installation(&root.join("venv/bin/ruff")), root.join("venv")); + assert_eq!( + installation(&root.join("Cellar/ruff/0.6/bin/ruff")), + root.join("Cellar/ruff/0.6") + ); + // `~/bin` holds whatever its owner put there: only the tool itself. + assert_eq!( + installation(&root.join("home/bin/ruff")), + root.join("home/bin/ruff") + ); + assert_eq!(installation(Path::new("/bin/sh")), PathBuf::from("/bin/sh")); + } + + #[test] + fn test_a_students_file_name_never_reaches_the_tool_as_syntax() { + assert_eq!(safe_name("Lab5_1.py"), "Lab5_1.py"); + assert_eq!(safe_name("x'; rm -rf ~ #.py"), "x___rm_-rf____.py"); + assert_eq!(safe_name("-rf.py"), "_-rf.py"); + assert_eq!(safe_name("..py"), "_..py"); + assert_eq!(safe_name("实验一.py"), "___.py"); + // The tool sees the safe name, whatever the submission was called: one line. + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("a;echo injected;b.py"); + std::fs::write(&file, "x = 1\n").unwrap(); + let probe = config("sh -c 'echo {file}' sh", 10); + assert_eq!(lint(&probe, &file), Ok(90.0)); + } + + #[test] + fn test_a_tool_past_its_deadline_has_failed_and_its_children_go_with_it() { + let (_dir, file) = student_file(); + let config = config("sh -c 'sleep 30 & sleep 30'", 10); + let started = std::time::Instant::now(); + let late = run_lint( + &config, + &file, + file.parent().unwrap(), + Isolation::Unisolated, + Duration::from_secs(1), + ); + assert!(late.unwrap_err().contains("ran longer than 1s")); + assert!( + started.elapsed() < Duration::from_secs(10), + "a child holding the pipes was ended with its group" + ); + } + + #[test] + fn test_bundle_names_the_test_bundle() { + let (_dir, file) = student_file(); + let probe = config("sh -c 'test -d \"$1\" && echo found' sh {bundle}", 10); + assert_eq!(lint(&probe, &file), Ok(90.0)); + } + #[test] fn test_style_score() { assert!((style_score(3, 10) - 70.0).abs() < 1e-9); @@ -85,38 +399,61 @@ mod tests { #[test] fn test_run_lint_counts_output_lines() { let (_dir, file) = student_file(); - assert_eq!(run_lint(&config("echo warning1", 10), &file), Ok(90.0)); - assert_eq!(run_lint(&config("true", 0), &file), Ok(100.0)); + assert_eq!(lint(&config("echo warning1", 10), &file), Ok(90.0)); + assert_eq!(lint(&config("true", 0), &file), Ok(100.0)); } #[test] fn test_a_path_with_a_space_stays_one_argument() { let dir = tempfile::tempdir().unwrap(); - let file = dir.path().join("Zhang San_1").join("lab.py"); + let file = dir.path().join("Zhang San_1").join("lab one.py"); std::fs::create_dir_all(file.parent().unwrap()).unwrap(); std::fs::write(&file, "x = 1\n").unwrap(); // `test -f` fails, exit 1 outside [0], unless it gets the path whole. - let lint = LintConfig { + let lint_config = LintConfig { ok_exit_codes: vec![0], ..config("test -f {file}", 10) }; - assert_eq!(run_lint(&lint, &file), Ok(100.0)); + assert_eq!(lint(&lint_config, &file), Ok(100.0)); + } + + #[test] + fn test_quoted_arguments_are_split_as_a_shell_would() { + let (_dir, file) = student_file(); + // Two arguments to `sh -c`, the script and $0: one warning line. + let quoted = config("sh -c 'echo \"one warning\"' sh", 10); + assert_eq!(lint("ed, &file), Ok(90.0)); + assert!( + lint(&config("echo 'unclosed", 10), &file) + .unwrap_err() + .contains("unclosed quote") + ); + } + + #[test] + fn test_the_tool_sees_a_read_only_copy_and_no_environment() { + let (_dir, file) = student_file(); + let probe = config( + "sh -c 'test ! -w \"$1\" && test -z \"$CANVAS_TOKEN\" && echo seen' sh {file}", + 10, + ); + assert_eq!(lint(&probe, &file), Ok(90.0)); } #[test] fn test_a_linter_reporting_findings_with_exit_1_is_scored() { let (_dir, file) = student_file(); - let sh = config("sh -c echo${IFS}w;exit${IFS}1", 10); - assert_eq!(run_lint(&sh, &file), Ok(90.0)); + let sh = config("sh -c 'echo w; exit 1'", 10); + assert_eq!(lint(&sh, &file), Ok(90.0)); } #[test] fn test_a_linter_that_fails_is_an_error_not_100() { let (_dir, file) = student_file(); - let crashed = run_lint(&config("sh -c exit${IFS}2", 10), &file); + let crashed = lint(&config("sh -c 'exit 2'", 10), &file); assert!(crashed.unwrap_err().contains("exited with")); - let missing = run_lint(&config("/nonexistent/ruff {file}", 10), &file); + let missing = lint(&config("/nonexistent/ruff {file}", 10), &file); assert!(missing.unwrap_err().contains("could not run")); - assert!(run_lint(&config(" ", 10), &file).is_err()); + assert!(lint(&config(" ", 10), &file).is_err()); } } diff --git a/src/core/src/runner/memory.rs b/src/core/src/runner/memory.rs new file mode 100644 index 0000000..d7e2b59 --- /dev/null +++ b/src/core/src/runner/memory.rs @@ -0,0 +1,290 @@ +//! What a unit's student side may hold beyond what it takes idle, when the spec does not +//! say in MiB: estimated from the space the teacher allows the algorithm (`space`, O(n²) +//! unless declared) and the size of the unit's test data. + +use std::collections::BTreeMap; +use std::fmt; +use std::path::Path; + +use serde_json::Value; + +use crate::models::{TestCase, TestSpec}; +use crate::runner::executor::UnitPlan; + +/// How much memory an algorithm may use, as the size of its data grows. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Space { + Constant, + Logarithmic, + Linear, + Linearithmic, + Quadratic, +} + +/// What a unit may hold where nothing is declared: lenient, as an exercise's data is small. +pub const DEFAULT_SPACE: Space = Space::Quadratic; + +/// The least an estimate gives, in MiB: what a small exercise holds, above the noise of +/// loading a module. +pub const ESTIMATE_FLOOR_MB: u64 = 16; + +/// The most an estimate gives, in MiB, and what a unit whose data cannot be sized before +/// grading gets: past it, a teacher declares `memory_mb`. +pub const ESTIMATE_CAP_MB: u64 = 1024; + +/// Bytes per step of the growth: about 64 bytes a Python value takes, eight times over +/// for the input held, copies of it, the result and ScriptMark's encodings of it. +const BYTES_PER_STEP: f64 = 512.0; + +impl Space { + /// `O(1)`, `O(log n)`, `O(n)`, `O(n log n)` or `O(n^2)` (also `O(n²)`, `O(n**2)`). + pub fn parse(text: &str) -> Option { + let squeezed: String = text + .chars() + .filter(|c| !c.is_whitespace()) + .collect::() + .to_lowercase(); + let inner = squeezed.strip_prefix("o(")?.strip_suffix(')')?; + Some(match inner { + "1" => Space::Constant, + "logn" => Space::Logarithmic, + "n" => Space::Linear, + "nlogn" => Space::Linearithmic, + "n^2" | "n²" | "n**2" | "n*n" => Space::Quadratic, + _ => return None, + }) + } + + fn grow(self, n: f64) -> f64 { + let log = n.max(2.0).log2(); + match self { + Space::Constant => 1.0, + Space::Logarithmic => log, + Space::Linear => n, + Space::Linearithmic => n * log, + Space::Quadratic => n * n, + } + } +} + +impl fmt::Display for Space { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Space::Constant => "O(1)", + Space::Logarithmic => "O(log n)", + Space::Linear => "O(n)", + Space::Linearithmic => "O(n log n)", + Space::Quadratic => "O(n^2)", + }) + } +} + +/// The MiB `space` allows over data of `size` elements, and how it was reached. Data that +/// cannot be sized before grading gets the cap. +pub fn estimate(space: Space, size: Option) -> (u64, String) { + let Some(n) = size else { + return ( + ESTIMATE_CAP_MB, + format!("{space}, the data's size unknown before grading"), + ); + }; + let mb = (BYTES_PER_STEP * space.grow(n as f64) / f64::from(1 << 20)).ceil(); + let mb = mb.clamp(ESTIMATE_FLOOR_MB as f64, ESTIMATE_CAP_MB as f64) as u64; + let elements = if n == 1 { "element" } else { "elements" }; + (mb, format!("{space} over {n} {elements}")) +} + +/// What a unit may hold, in MiB, and how an undeclared limit was estimated: the narrowest +/// declaration first — `memory_mb` as written, or `space` estimated over `size` — and +/// `DEFAULT_SPACE` where none is. +pub fn resolve( + declared: [(Option, Option<&str>); 2], + size: Option, +) -> (u64, Option) { + let space = declared + .into_iter() + .find_map( + |(memory_mb, space)| match (memory_mb, space.and_then(Space::parse)) { + (Some(mb), _) => Some(Err(mb)), + (None, Some(space)) => Some(Ok(space)), + (None, None) => None, + }, + ) + .unwrap_or(Ok(DEFAULT_SPACE)); + match space { + Err(mb) => (mb, None), + Ok(space) => { + let (mb, basis) = estimate(space, size); + (mb, Some(basis)) + } + } +} + +/// The size of a unit's test data, in elements: its calls' arguments and stdin, what its +/// `expected` cases expect, and the bytes of the data files staged for it. None where an +/// argument holds a setup's result, which is not known before grading. +pub fn unit_size<'a>( + spec: &TestSpec, + plan: &UnitPlan, + expected: impl IntoIterator, +) -> Option { + let text = |s: &Option| s.as_ref().map_or(0, |s| s.chars().count() as u64); + let mut total = spec + .meta + .data_files + .iter() + .map(|rel| bytes_under(&spec.dir.join(rel))) + .fold(0u64, u64::saturating_add); + total = total.saturating_add(plan.script.as_ref().map_or(0, |s| text(&s.stdin))); + for call in plan.setup.iter().chain(&plan.steps) { + total = total + .saturating_add(args_size(&call.args, &spec.vars)?) + .saturating_add(text(&call.stdin)); + } + for case in expected { + if let Some(expect) = &case.expect { + total = total.saturating_add(value_size(expect, &spec.vars)?); + } + total = total.saturating_add(text(&case.expected_stdout)); + } + Some(total) +} + +fn bytes_under(path: &Path) -> u64 { + match std::fs::metadata(path) { + Ok(meta) if meta.is_dir() => std::fs::read_dir(path).map_or(0, |entries| { + entries + .filter_map(Result::ok) + .map(|entry| bytes_under(&entry.path())) + .fold(0, u64::saturating_add) + }), + Ok(meta) => meta.len(), + Err(_) => 0, + } +} + +/// The size of a call's arguments, in elements: a container counts its elements, a string +/// its characters, and an integer argument its value — `fib(30)` and `primes(10**6)` are +/// sized by their number. `$name` reads the var `name`; a reference to a setup result +/// cannot be sized before grading: None. +pub fn args_size(args: &[Value], vars: &BTreeMap) -> Option { + args.iter().try_fold(0u64, |total, arg| { + Some(total.saturating_add(size(arg, true, Some(vars))?)) + }) +} + +/// The size of an expected value: as an argument's, but an integer is one element. +pub fn value_size(value: &Value, vars: &BTreeMap) -> Option { + size(value, false, Some(vars)) +} + +/// `vars` resolves `$name`; a var's own value, None, holds text where it says `$…`. +fn size(value: &Value, top: bool, vars: Option<&BTreeMap>) -> Option { + Some(match value { + // Resolved once, as the teacher side does: a var's own `$…` strings are text. + Value::String(s) if vars.is_some() && s.starts_with('$') && !s.starts_with("$$") => { + return size(vars?.get(&s[1..])?, top, None); + } + Value::String(s) => s.chars().count() as u64, + Value::Number(number) if top => number + .as_i64() + .map(i64::unsigned_abs) + .or_else(|| number.as_u64()) + .unwrap_or(1), + Value::Array(items) => items.iter().try_fold(0u64, |total, item| { + Some(total.saturating_add(size(item, false, vars)?.max(1))) + })?, + Value::Object(map) => map.values().try_fold(0u64, |total, item| { + Some(total.saturating_add(size(item, false, vars)?.saturating_add(1))) + })?, + _ => 1, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_space_is_written_as_big_o() { + for (text, space) in [ + ("O(1)", Space::Constant), + ("O(log n)", Space::Logarithmic), + ("o(n)", Space::Linear), + ("O(n log n)", Space::Linearithmic), + ("O(n^2)", Space::Quadratic), + ("O(n²)", Space::Quadratic), + ("O( n**2 )", Space::Quadratic), + ] { + assert_eq!(Space::parse(text), Some(space), "{text}"); + } + for text in ["n", "O(n^3)", "O(2^n)", "O()", "linear"] { + assert_eq!(Space::parse(text), None, "{text}"); + } + } + + #[test] + fn test_an_estimate_grows_with_the_data_between_floor_and_cap() { + assert_eq!(estimate(Space::Quadratic, Some(10)).0, ESTIMATE_FLOOR_MB); + assert_eq!( + estimate(Space::Quadratic, Some(1000)), + (489, "O(n^2) over 1000 elements".to_string()) + ); + assert_eq!(estimate(Space::Linear, Some(1_000_000)).0, 489); + assert_eq!(estimate(Space::Quadratic, Some(100_000)).0, ESTIMATE_CAP_MB); + assert_eq!( + estimate(Space::Constant, Some(u64::MAX)).0, + ESTIMATE_FLOOR_MB + ); + assert_eq!(estimate(Space::Linear, None).0, ESTIMATE_CAP_MB); + assert_eq!(estimate(Space::Linear, Some(1)).1, "O(n) over 1 element"); + } + + #[test] + fn test_the_narrowest_declaration_applies() { + let n = Some(1000); + assert_eq!( + resolve([(None, None), (None, None)], n), + (489, Some("O(n^2) over 1000 elements".into())) + ); + assert_eq!( + resolve([(Some(32), None), (None, Some("O(1)"))], n), + (32, None) + ); + assert_eq!( + resolve([(None, Some("O(1)")), (Some(512), None)], n).0, + ESTIMATE_FLOOR_MB + ); + assert_eq!(resolve([(None, None), (Some(512), None)], n), (512, None)); + assert_eq!( + resolve([(None, None), (None, Some("O(n)"))], None).0, + ESTIMATE_CAP_MB + ); + } + + #[test] + fn test_data_is_sized_by_elements_characters_and_integer_arguments() { + let vars = BTreeMap::from([ + ("N".to_string(), json!(30)), + ("XS".to_string(), json!([1, 2])), + ]); + assert_eq!(args_size(&[json!([3, 1, 5, 2])], &vars), Some(4)); + assert_eq!(args_size(&[json!(3), json!(5)], &vars), Some(8)); + assert_eq!(args_size(&[json!(1_000_000)], &vars), Some(1_000_000)); + assert_eq!(args_size(&[json!(-7), json!(2.5)], &vars), Some(8)); + assert_eq!(args_size(&[json!("hello")], &vars), Some(5)); + assert_eq!( + args_size(&[json!({"a": [1, 2], "b": "xy"})], &vars), + Some(6) + ); + assert_eq!(args_size(&[json!("$N"), json!("$XS")], &vars), Some(32)); + assert_eq!(args_size(&[json!("$$literal")], &vars), Some(9)); + assert_eq!(args_size(&[json!("$data")], &vars), None); + // A var that names itself is text, not a loop. + let looped = BTreeMap::from([("A".to_string(), json!(["$A", "$A"]))]); + assert_eq!(args_size(&[json!("$A")], &looped), Some(4)); + assert_eq!(value_size(&json!(8_000_000), &vars), Some(1)); + assert_eq!(value_size(&json!([[1, 2], []]), &vars), Some(3)); + } +} diff --git a/src/core/src/runner/mod.rs b/src/core/src/runner/mod.rs index 098f8f2..422a366 100644 --- a/src/core/src/runner/mod.rs +++ b/src/core/src/runner/mod.rs @@ -5,6 +5,7 @@ pub mod generation; pub mod generator; pub mod judge; pub mod linter; +pub mod memory; pub mod oracle; pub mod orchestrator; pub mod prepare; diff --git a/src/core/src/runner/oracle.rs b/src/core/src/runner/oracle.rs index 601296f..273123f 100644 --- a/src/core/src/runner/oracle.rs +++ b/src/core/src/runner/oracle.rs @@ -36,7 +36,17 @@ pub async fn resolve_oracle( .iter() .map(|rel| (spec.dir.join(rel), rel.clone())) .collect(), + bundle: spec.dir.clone(), + shared: spec + .meta + .shared + .iter() + .map(|rel| (spec.dir.join(rel), rel.clone())) + .collect(), allowed_imports: spec.meta.allowed_imports.clone(), + // From the case's declarations and arguments, below: its answer is what this finds. + memory_mb: 0, + memory_basis: None, load_timeout: timeout_secs, setup: Vec::new(), steps: vec![CallPlan { @@ -49,6 +59,14 @@ pub async fn resolve_oracle( check: None, }], }; + let mut plan = plan; + (plan.memory_mb, plan.memory_basis) = crate::runner::memory::resolve( + [ + (case.memory_mb, case.space.as_deref()), + (spec.meta.memory_mb, spec.meta.space.as_deref()), + ], + crate::runner::memory::unit_size(spec, &plan, []), + ); let obs = executor.run(&plan).await; if !obs.ready || !obs.done diff --git a/src/core/src/runner/orchestrator.rs b/src/core/src/runner/orchestrator.rs index 9850d02..80455e6 100644 --- a/src/core/src/runner/orchestrator.rs +++ b/src/core/src/runner/orchestrator.rs @@ -21,6 +21,9 @@ pub struct RunOptions { pub concurrency: Option, /// The interpreter that runs teacher Python checker scripts. pub python: String, + /// Whether teacher checker scripts and lint tools run in the teacher's sandbox, as the + /// units do. + pub isolation: crate::runner::sandbox::Isolation, } impl Default for RunOptions { @@ -29,10 +32,18 @@ impl Default for RunOptions { matching: crate::matching::Config::default(), concurrency: None, python: "python3".into(), + isolation: crate::runner::sandbox::Isolation::default(), } } } +/// How teacher scripts and tools run outside a unit: checker scripts and lint. +#[derive(Debug, Clone)] +struct Tools { + python: Arc, + isolation: crate::runner::sandbox::Isolation, +} + /// Run every prepared bundle against every student. /// /// Takes `&[StudentSubmission]` rather than the whole `AssignmentInput` so that the roster, @@ -59,7 +70,10 @@ pub async fn run_all( }) .clamp(1, Semaphore::MAX_PERMITS); let semaphore = Arc::new(Semaphore::new(concurrency)); - let python: Arc = options.python.as_str().into(); + let tools = Tools { + python: options.python.as_str().into(), + isolation: options.isolation, + }; let matching = Arc::new(options.matching.clone()); let mut handles = Vec::new(); @@ -77,11 +91,11 @@ pub async fn run_all( .map(|bundle| crate::matching::item_match(&matching, student, &bundle.spec)) .collect(); let matching = matching.clone(); - let (bundles, executor, semaphore, python) = ( + let (bundles, executor, semaphore, tools) = ( bundles.clone(), executor.clone(), semaphore.clone(), - python.clone(), + tools.clone(), ); let handle = tokio::spawn(async move { @@ -95,7 +109,7 @@ pub async fn run_all( bundles, executor, semaphore, - python, + tools, matching, ) .await; @@ -135,7 +149,7 @@ async fn run_student( bundles: Arc<[Bundle]>, executor: Arc, semaphore: Arc, - python: Arc, + tools: Tools, matching: Arc, ) { let mut slots: Vec>>> = @@ -191,11 +205,11 @@ async fn run_student( let functions = matching.functions(&report.student_id, &bundle.spec.meta.name); for u in 0..bundle.units.len() { let functions = functions.clone(); - let (bundles, executor, semaphore, python, path) = ( + let (bundles, executor, semaphore, tools, path) = ( bundles.clone(), executor.clone(), semaphore.clone(), - python.clone(), + tools.clone(), path.clone(), ); let handle = tasks.spawn(async move { @@ -223,7 +237,13 @@ async fn run_student( } } ( - judge(&plan, &unit.scored(), &observation, &python), + judge( + &plan, + &unit.scored(), + &observation, + &tools.python, + tools.isolation, + ), functions, ) }) @@ -285,7 +305,7 @@ async fn run_student( } } } - report.lint = lint(&bundles, &graded_files, &semaphore).await; + report.lint = lint(&bundles, &graded_files, &semaphore, tools.isolation).await; report.test_results = bundles .iter() .zip(slots) @@ -299,30 +319,67 @@ async fn run_student( report.backend_name = Some(executor.language().to_string()); } -/// Lint the file of the first spec that asks for it — that item's file, not whichever the -/// student happened to hand in first. +/// Lint every file a spec asks to — each item's own file, with its own `[lint]` — and score +/// the student by their mean: an item whose file is missing counts 0. A file two items +/// share under the same command is linted once. async fn lint( bundles: &[Bundle], graded_files: &[Option], semaphore: &Semaphore, + isolation: crate::runner::sandbox::Isolation, ) -> Option { - let (b, config) = bundles - .iter() - .enumerate() - .find_map(|(b, bundle)| bundle.spec.lint.clone().map(|c| (b, c)))?; - let Some(file) = graded_files[b].clone() else { + let mut asked: Vec<( + (Option, String), + crate::models::LintConfig, + std::path::PathBuf, + )> = Vec::new(); + for (bundle, file) in bundles.iter().zip(graded_files) { + if let Some(config) = &bundle.spec.lint { + let key = (file.clone(), config.command.clone()); + if !asked.iter().any(|(k, ..)| *k == key) { + asked.push((key, config.clone(), bundle.spec.dir.clone())); + } + } + } + if asked.is_empty() { + return None; + } + if asked.iter().all(|((file, _), ..)| file.is_none()) { return Some(LintOutcome::NoFile); - }; - let _permit = semaphore.acquire().await.expect("semaphore closed"); - let outcome = tokio::task::spawn_blocking(move || { - crate::runner::linter::run_lint(&config, std::path::Path::new(&file)) - }) - .await - .map_err(|e| format!("linting failed: {e}")) - .and_then(|scored| scored); - Some(match outcome { - Ok(score) => LintOutcome::Scored { score }, - Err(message) => LintOutcome::Failed { message }, + } + let mut scores = Vec::new(); + let mut failures = Vec::new(); + for ((file, _), config, bundle) in asked { + let Some(file) = file else { + scores.push(0.0); + continue; + }; + let _permit = semaphore.acquire().await.expect("semaphore closed"); + let outcome = tokio::task::spawn_blocking(move || { + crate::runner::linter::run_lint( + &config, + std::path::Path::new(&file), + &bundle, + isolation, + crate::runner::linter::LINT_TIMEOUT, + ) + }) + .await + .map_err(|e| format!("linting failed: {e}")) + .and_then(|scored| scored); + match outcome { + Ok(score) => scores.push(score), + Err(message) => failures.push(message), + } + } + Some(if failures.is_empty() { + LintOutcome::Scored { + score: scores.iter().sum::() / scores.len() as f64, + } + } else { + LintOutcome::Failed { + message: failures.join("; "), + } }) } diff --git a/src/core/src/runner/prepare.rs b/src/core/src/runner/prepare.rs index c42c78c..1ca0cb2 100644 --- a/src/core/src/runner/prepare.rs +++ b/src/core/src/runner/prepare.rs @@ -25,8 +25,9 @@ use crate::runner::executor::{ use crate::runner::frozen::{Generation, replay_entry}; use crate::runner::generation::{DrawSeed, Generated, generate, seed_for}; use crate::runner::judge::Scored; +use crate::runner::memory; use crate::runner::oracle::{resolve_oracle, validate_references}; -use crate::spec_loader::{MAX_TIMEOUT_SECS, RESERVED, refs, validate, validate_expanded}; +use crate::spec_loader::{MAX_TIMEOUT_SECS, refs, reserved, validate, validate_expanded}; /// A prepared test bundle: one spec, ready to run against any student. #[derive(Debug, Serialize)] @@ -340,6 +341,7 @@ async fn prepare_one( } problems.extend(check_names(&spec, &teacher)); + problems.extend(check_bundle_paths(&spec, &teacher)); problems.extend(dry_run_rhai(&spec)); if !problems.is_empty() { return Err(fail(problems)); @@ -367,10 +369,12 @@ fn check_names(spec: &TestSpec, teacher: &TeacherRuntime) -> Vec { ) }) .collect(); - if teacher.exports.contains_key(RESERVED) { - problems.push(format!( - "a teacher module exports '{RESERVED}', a name checkers reserve for the call's output" - )); + for name in teacher.exports.keys() { + if let Some(meaning) = reserved(name) { + problems.push(format!( + "a teacher module exports '{name}', a name checkers reserve for {meaning}" + )); + } } let exports: BTreeSet<&str> = teacher.exports.keys().map(String::as_str).collect(); let mut top: BTreeSet = BTreeSet::new(); @@ -422,6 +426,93 @@ fn check_names(spec: &TestSpec, teacher: &TeacherRuntime) -> Vec { problems } +/// Student code runs in a sandbox that cannot read the test bundle: a path into it, handed to +/// student code, names a file the student can never open. Such a file belongs in +/// `data_files`, which stages it beside the student's file, under a relative path. +fn check_bundle_paths(spec: &TestSpec, teacher: &TeacherRuntime) -> Vec { + let Ok(bundle) = std::fs::canonicalize(&spec.dir) else { + return Vec::new(); + }; + let fix = "list it in [meta] data_files and pass its path relative to the student's directory"; + let mut problems = Vec::new(); + let mut literals = |at: &str, value: &Value| { + for path in bundle_paths(value, &bundle) { + problems.push(format!( + "{at}: '{path}' is a path into the test bundle, which student code cannot read: {fix}" + )); + } + }; + for (name, value) in &spec.vars { + literals(&format!("[vars] '{name}'"), value); + } + let student_setup = |steps: &[SetupStep]| -> Vec<(String, Vec)> { + steps + .iter() + .filter(|s| s.teacher.is_none()) + .map(|s| (format!("setup '{}'", s.id), s.args.clone())) + .collect() + }; + let mut calls: Vec<(String, Vec)> = student_setup(&spec.setup); + calls.extend( + spec.cases + .iter() + .map(|c| (format!("case '{}'", c.name), c.args.clone())), + ); + for scenario in &spec.scenarios { + calls.extend(student_setup(&scenario.setup)); + calls.extend(scenario.steps.iter().map(|s| { + ( + format!("scenario '{}' step '{}'", scenario.name, s.name), + s.args.clone(), + ) + })); + } + for (at, args) in &calls { + for arg in args { + literals(at, arg); + } + } + for (at, args) in calls { + for name in refs(&args) { + if teacher.exports.get(&name).is_some_and(|e| e.bundle_path) { + problems.push(format!( + "{at}: '${name}' is a path into the test bundle, which student code cannot read: {fix}" + )); + } + } + } + problems +} + +/// `path` with its links resolved as far as it exists: a file the student would write +/// still resolves through its directory. +fn resolved(path: &std::path::Path) -> PathBuf { + if let Ok(real) = std::fs::canonicalize(path) { + return real; + } + match (path.parent(), path.file_name()) { + (Some(parent), Some(name)) => resolved(parent).join(name), + _ => path.to_path_buf(), + } +} + +/// Every string in `value` that is an absolute path into `bundle`. +fn bundle_paths(value: &Value, bundle: &std::path::Path) -> Vec { + match value { + Value::String(text) => { + let path = std::path::Path::new(text); + if path.is_absolute() && resolved(path).starts_with(bundle) { + vec![text.clone()] + } else { + Vec::new() + } + } + Value::Array(items) => items.iter().flat_map(|v| bundle_paths(v, bundle)).collect(), + Value::Object(map) => map.values().flat_map(|v| bundle_paths(v, bundle)).collect(), + _ => Vec::new(), + } +} + fn unknown_refs(args: &[Value], scope: &BTreeSet, at: &str) -> Vec { refs(args) .into_iter() @@ -471,7 +562,7 @@ fn check_call( for param in params.iter().skip(2) { // *args, **kwargs and defaulted parameters are filled only when named. let filled = param.is_variadic() - || param.default || param.name == RESERVED + || param.default || reserved(¶m.name).is_some() || scope.contains(¶m.name); if !filled { problems.push(format!( @@ -555,7 +646,17 @@ fn plan_units(spec: &TestSpec, timeout_secs: u64) -> Vec { .iter() .map(|rel| (spec.dir.join(rel), rel.clone())) .collect(), + bundle: spec.dir.clone(), + shared: spec + .meta + .shared + .iter() + .map(|rel| (spec.dir.join(rel), rel.clone())) + .collect(), allowed_imports: spec.meta.allowed_imports.clone(), + // Each unit's, from its own declarations and data: below. + memory_mb: 0, + memory_basis: None, load_timeout: timeout_secs, setup: Vec::new(), steps: Vec::new(), @@ -567,6 +668,14 @@ fn plan_units(spec: &TestSpec, timeout_secs: u64) -> Vec { .map(|s| setup_call(s, timeout_secs)) .collect(); + let meta = (spec.meta.memory_mb, spec.meta.space.as_deref()); + let held = + |mut plan: UnitPlan, narrow: (Option, Option<&str>), scored: &[(String, TestCase)]| { + let size = memory::unit_size(spec, &plan, scored.iter().map(|(_, c)| c)); + (plan.memory_mb, plan.memory_basis) = memory::resolve([narrow, meta], size); + plan + }; + let cases = spec.cases.iter().map(|case| { let plan = if case.script { UnitPlan { @@ -584,32 +693,39 @@ fn plan_units(spec: &TestSpec, timeout_secs: u64) -> Vec { ..base.clone() } }; + let scored = vec![(case.name.clone(), case.clone())]; Unit { - plan, - scored: vec![(case.name.clone(), case.clone())], + plan: held(plan, (case.memory_mb, case.space.as_deref()), &scored), + scored, } }); let scenarios = spec.scenarios.iter().map(|scenario| { let timeout = scenario.timeout.unwrap_or(timeout_secs); - Unit { - plan: UnitPlan { - setup: top_setup - .iter() - .cloned() - .chain(scenario.setup.iter().map(|s| setup_call(s, timeout))) - .collect(), - steps: scenario - .steps - .iter() - .map(|s| call_of(s, default_function, timeout)) - .collect(), - ..base.clone() - }, - scored: scenario + let scored: Vec<(String, TestCase)> = scenario + .steps + .iter() + .map(|s| (scenario.step_name(s), s.clone())) + .collect(); + let plan = UnitPlan { + setup: top_setup + .iter() + .cloned() + .chain(scenario.setup.iter().map(|s| setup_call(s, timeout))) + .collect(), + steps: scenario .steps .iter() - .map(|s| (scenario.step_name(s), s.clone())) + .map(|s| call_of(s, default_function, timeout)) .collect(), + ..base.clone() + }; + Unit { + plan: held( + plan, + (scenario.memory_mb, scenario.space.as_deref()), + &scored, + ), + scored, } }); cases.chain(scenarios).collect() diff --git a/src/core/src/runner/python.rs b/src/core/src/runner/python.rs index ed65421..c625317 100644 --- a/src/core/src/runner/python.rs +++ b/src/core/src/runner/python.rs @@ -1,21 +1,39 @@ +//! The Python backend. A unit runs as two processes joined by a pair of pipes: the teacher +//! side (`harness/teacher.py`) holds the teacher's modules and checkers, drives the plan +//! and writes the records; the student side (`harness/student.py`) holds only the +//! student's module. ScriptMark starts both, connects them to each other, and reads the +//! teacher side's records; it never relays what passes between them. + use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, LazyLock, Mutex}; +use std::time::Duration; -use tokio::process::Command; +use serde::Serialize; +use tokio::process::{Child, Command}; use crate::models::TestSpec; use crate::runner::executor::{ - Executor, Exit, ProtocolError, Subject, TeacherRuntime, UnitObservation, UnitPlan, + Executor, Exit, GRACE_SECS, STARTUP_SECS, Subject, TeacherRuntime, UnitObservation, UnitPlan, }; +use crate::runner::records::{Bounds, Reader, Records}; #[cfg(unix)] use crate::runner::sandbox::apply_sandbox; +use crate::runner::sandbox::{Confinement, Isolation, covering}; /// Resolve a python command to an absolute path so env_clear() doesn't change /// which interpreter runs. Falls back to the original string if resolution fails. fn resolve_python_path(cmd: &str) -> String { - if cmd.starts_with('/') { + let path = Path::new(cmd); + if path.is_absolute() { return cmd.to_string(); } + // A relative path, `.venv/bin/python`, is the grader's: each side starts elsewhere. + if path.components().count() > 1 { + return std::path::absolute(path) + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_else(|_| cmd.to_string()); + } if let Ok(output) = std::process::Command::new("which").arg(cmd).output() && output.status.success() && let Ok(path) = String::from_utf8(output.stdout) @@ -31,6 +49,8 @@ fn resolve_python_path(cmd: &str) -> String { /// Executor for Python student code. pub struct PythonExecutor { python_cmd: String, + /// The most memory the student side may hold before it is ended, in MiB. + isolation: Isolation, } impl PythonExecutor { @@ -41,61 +61,275 @@ impl PythonExecutor { pub fn with_python_cmd(python_cmd: impl Into) -> Self { Self { python_cmd: resolve_python_path(&python_cmd.into()), + isolation: Isolation::default(), } } + /// Whether each side runs in its own sandbox. Sandboxed is the default; a unit whose + /// sandbox cannot be applied is not graded. + pub fn with_isolation(mut self, isolation: Isolation) -> Self { + self.isolation = isolation; + self + } + + pub fn isolation(&self) -> Isolation { + self.isolation + } + pub fn python_cmd(&self) -> &str { &self.python_cmd } } -/// The one harness every unit runs under. See `harness.py`. -const HARNESS: &str = include_str!("harness.py"); +/// What each interpreter needs to run, found once per interpreter, the first time +/// anything it runs is confined. +static INTERPRETERS: LazyLock< + Mutex>>, +> = LazyLock::new(Default::default); + +fn interpreter(python_cmd: &str) -> Result { + INTERPRETERS + .lock() + .expect("interpreter cache poisoned") + .entry(python_cmd.to_string()) + .or_insert_with(|| Interpreter::probe(python_cmd)) + .clone() +} + +/// A confinement for teacher code `python_cmd` runs outside a unit — a Python checker +/// script, or a lint tool written in Python: the interpreter, `read`, and `write`. +pub fn teacher_confinement( + python_cmd: &str, + read: Vec, + write: Vec, + spawn: bool, +) -> Result { + let interpreter = interpreter(python_cmd)?; + Ok(Confinement { + read: covering( + interpreter + .read + .into_iter() + .chain(canonical(read)) + .collect(), + ), + write, + exec: interpreter.exec, + spawn, + no_ffi: !spawn, + }) +} + +/// What one Python interpreter reads and runs: its prefixes, its standard library and +/// packages, and the shared libraries they load — not the rest of the machine. +#[derive(Debug, Clone)] +struct Interpreter { + read: Vec, + exec: Vec, +} + +/// The oldest Python the harness runs on. +const MINIMUM_PYTHON: (u32, u32) = (3, 10); + +/// Asks the interpreter where it lives, unconfined and once: its prefixes, and where its +/// standard library and installed packages are — not every `sys.path` entry, which a +/// package's `.pth` file can point anywhere, a home directory included. +const PROBE: &str = "import json, os, sys, sysconfig +places = sysconfig.get_paths() +paths = [sys.prefix, sys.base_prefix, sys.exec_prefix, sys.base_exec_prefix, *(places[k] for k in ('stdlib', 'platstdlib', 'purelib', 'platlib'))] +print(json.dumps({ + 'version': list(sys.version_info[:2]), + 'executable': os.path.realpath(sys.executable), + 'base': os.path.realpath(sys.base_prefix), + 'paths': [os.path.realpath(p) for p in paths if p and os.path.exists(p)], +}))"; + +impl Interpreter { + fn probe(python_cmd: &str) -> Result { + #[derive(serde::Deserialize)] + struct Found { + version: (u32, u32), + executable: PathBuf, + base: PathBuf, + paths: Vec, + } + let output = std::process::Command::new(python_cmd) + .env_clear() + .args(["-I", "-c", PROBE]) + .output() + .map_err(|e| format!("could not ask {python_cmd} where it lives: {e}"))?; + let found: Found = serde_json::from_slice(&output.stdout).map_err(|e| { + format!( + "{python_cmd} did not say where it lives ({e}): {}", + String::from_utf8_lossy(&output.stderr).trim() + ) + })?; + // The harness uses `X | None` and `match`-era typing: older interpreters die starting it. + if found.version < MINIMUM_PYTHON { + return Err(format!( + "{python_cmd} is Python {}.{}; ScriptMark needs Python {}.{} or newer", + found.version.0, found.version.1, MINIMUM_PYTHON.0, MINIMUM_PYTHON.1 + )); + } + let command = PathBuf::from(python_cmd); + let mut read = found.paths; + read.extend(system_libraries(&found.base)); + // The command itself may be a link into the interpreter's prefix, as a venv's is. + read.push(command.clone()); + let mut exec = vec![command]; + #[cfg(target_os = "macos")] + // A framework build's `python3` starts `Python.app` inside its prefix. + exec.push(found.base); + #[cfg(not(target_os = "macos"))] + { + exec.extend(crate::runner::sandbox::loader_of(&found.executable)); + let _ = found.base; + } + exec.push(found.executable); + Ok(Self { + read: covering(read), + exec, + }) + } +} + +/// Shared libraries an interpreter loads from outside its prefix. +#[cfg(target_os = "macos")] +fn system_libraries(base: &Path) -> Vec { + // A Homebrew keg links against other kegs through `opt`: those, and only those — not + // the prefix's `etc` or `var`, which hold service configuration and data. + let text = base.to_string_lossy(); + match text.find("/Cellar/") { + Some(at) => { + let prefix = Path::new(&text[..at]); + vec![prefix.join("Cellar"), prefix.join("opt")] + } + None => Vec::new(), + } +} + +#[cfg(not(target_os = "macos"))] +fn system_libraries(_base: &Path) -> Vec { + [ + "/usr/lib", + "/usr/lib64", + "/lib", + "/lib64", + "/usr/local/lib", + "/etc/ld.so.cache", + "/etc/localtime", + "/usr/share/zoneinfo", + ] + .iter() + .filter_map(|p| std::fs::canonicalize(p).ok()) + .collect() +} + +const COMMON: &str = include_str!("harness/common.py"); +const TEACHER: &str = include_str!("harness/teacher.py"); +const STUDENT: &str = include_str!("harness/student.py"); +const MATCHING: &str = include_str!("../matching.py"); + +/// Every line either side runs, for the frozen contract: a change to any of it can change +/// what a reference implementation is observed to do. +pub(crate) const PROTOCOL: [&str; 4] = [COMMON, TEACHER, STUDENT, MATCHING]; + +/// `python -c` puts the working directory first on `sys.path`: it goes before any import. +const PRELUDE: &str = "import sys; sys.path[:] = [p for p in sys.path if p]\n"; +static TEACHER_SCRIPT: LazyLock = LazyLock::new(|| format!("{PRELUDE}{COMMON}\n{TEACHER}")); +static STUDENT_SCRIPT: LazyLock = + LazyLock::new(|| format!("{PRELUDE}{MATCHING}\n{COMMON}\n{STUDENT}")); + +/// What one call's record may carry, which both sides are given rather than assume. +#[derive(Debug, Clone, Copy, Serialize)] +struct Limits { + stdout: usize, + file: usize, + value: usize, + message: usize, + /// The most a value handed to the other side's code may take, as JSON. + transfer: usize, + depth: usize, +} + +const LIMITS: Limits = Limits { + stdout: 64 << 10, + file: 1 << 20, + value: 4 << 20, + message: 64 << 10, + transfer: 32 << 20, + depth: 100, +}; + +/// The most a record carries besides its value, stdout, message and files: its target, +/// the matching evidence, and the keys around them. +const SLACK: usize = 1 << 20; /// The tail of stderr kept for diagnosing a crash. const STDERR_TAIL: usize = 4096; -/// The descriptor the harness writes records to. On unix it is a pipe of its own, so -/// nothing a student does to stdout — rewrapping it, writing to `sys.__stdout__`, printing -/// from a thread — can reach the records. Elsewhere it is stdout, framed by the nonce. +/// How long the student side has to finish by itself once the teacher side has: it was +/// told to, and only a hung or stopped student side takes longer. +const LINGER: Duration = Duration::from_millis(500); + +/// The descriptor the teacher side writes records to. On unix it is a pipe of its own; +/// elsewhere stderr, since stdin and stdout are its pipe to the student side. #[cfg(unix)] const RECORD_FD: i32 = 3; #[cfg(not(unix))] -const RECORD_FD: i32 = 1; - -/// What a unit's records may total before the channel counts as flooded, which is also -/// what the grader buffers for it at most: this floor, plus the most each call carries. -const BASE_CAP: usize = 64 << 20; -/// The most one call's records carry, measured: a 4 MiB value, plus 64 KiB each of -/// stdout and checker message, which JSON escaping can grow sixfold (`harness.py`'s -/// `VALUE_LIMIT`, `STDOUT_LIMIT` and `MESSAGE_LIMIT`). -const CALL_BOUND: usize = 6 << 20; -/// The most one observed file adds: `FILE_LIMIT` characters, escaped. -const FILE_BOUND: usize = 7 << 20; - -/// The record channel's cap for `plan`: a correct submission never reaches it, however -/// many calls the unit makes. -fn record_cap(plan: &UnitPlan) -> usize { - let files = |n: usize| n.saturating_mul(FILE_BOUND); - let script = plan.script.as_ref().map_or(0, |s| files(s.files.len())); - plan.setup +const RECORD_FD: i32 = 2; + +/// What the teacher side's record stream may hold for `plan`: its largest record — a call +/// with its value, stdout, an error message and every file it observes, each of which JSON +/// escaping can grow sixfold — and one record per call and check, plus the frame around +/// them. +fn record_bounds(plan: &UnitPlan) -> Bounds { + let files = plan + .setup .iter() .chain(&plan.steps) - .map(|c| CALL_BOUND.saturating_add(files(c.files.len()))) - .fold(BASE_CAP.saturating_add(script), usize::saturating_add) + .map(|c| c.files.len()) + .chain(plan.script.as_ref().map(|s| s.files.len())) + .max() + .unwrap_or(0); + let per_file = 6 * LIMITS.file + 4096; + Bounds { + frame: (SLACK + LIMITS.value + 6 * (LIMITS.stdout + LIMITS.message)) + .saturating_add(files.saturating_mul(per_file)), + // ready, load, done, and a fatal or a stop, besides the calls and checks. + count: 5 + plan.setup.len() + 2 * plan.steps.len().max(1), + } } -/// A unit's private directory: the payload beside a working directory the student runs in. -struct Staged { - /// Removed on drop. - root: tempfile::TempDir, +/// What an inspection may report: every export and its parameters. +const INSPECT_BOUNDS: Bounds = Bounds { + frame: 64 << 20, + count: 4, +}; + +/// One side's private directories. +struct Side { work: PathBuf, - /// The unit's own `HOME` and `TMPDIR`, so nothing one unit leaves there reaches another. + /// Its own `HOME` and `TMPDIR`, so nothing one unit leaves there reaches another. home: PathBuf, tmp: PathBuf, +} + +/// A unit's private directory: one working directory per side, the teacher side's +/// payload, and a read-only copy of the subject's source. +struct Staged { + /// Removed on drop. + root: tempfile::TempDir, + teacher: Side, + student: Side, payload: PathBuf, - /// The subject file's copy inside `work`. - subject: PathBuf, + /// The subject file's copy in the student side's working directory. + subject: Option, + /// The subject's source, for teacher code to read. + source: Option, + /// A file outside everything either side may read: a side that can open it is not in + /// its sandbox, and refuses to go on. + canary: PathBuf, } /// Copy `src` (a file or a directory) to `dest`, creating parents. @@ -115,171 +349,446 @@ fn copy_into(src: &Path, dest: &Path) -> std::io::Result<()> { } } +fn side(root: &Path, name: &str) -> std::io::Result { + let side = Side { + work: root.join(name), + home: root.join(format!("{name}-home")), + tmp: root.join(format!("{name}-tmp")), + }; + for dir in [&side.work, &side.home, &side.tmp] { + std::fs::create_dir(dir)?; + } + Ok(side) +} + fn stage( subject: Option<&Path>, data_files: &[(PathBuf, String)], - payload: impl FnOnce(Option<&Path>) -> serde_json::Value, + shared: &[(PathBuf, String)], + payload: impl FnOnce(&Staged) -> serde_json::Value, ) -> std::io::Result { let root = tempfile::Builder::new() .prefix("scriptmark-unit-") .tempdir()?; - let work = root.path().join("work"); - let home = root.path().join("home"); - let tmp = root.path().join("tmp"); - for dir in [&work, &home, &tmp] { - std::fs::create_dir(dir)?; - } + // The sandbox sees resolved paths: `/var/folders/…` is `/private/var/folders/…` on macOS. + let base = std::fs::canonicalize(root.path())?; + let teacher = side(&base, "teacher")?; + let student = side(&base, "student")?; + let canary = base.join("canary"); + std::fs::write(&canary, "a sandboxed side must not be able to read this\n")?; + // Teacher code that opens a data file by its relative path finds it, as student code does. for (src, rel) in data_files { - copy_into(src, &work.join(rel))?; + copy_into(src, &student.work.join(rel))?; + copy_into(src, &teacher.work.join(rel))?; + } + // The student side's own copy of each shared module; the teacher side's is the bundle's. + for (src, rel) in shared { + copy_into(src, &student.work.join(rel))?; } - let copied = match subject { + let (copied, source) = match subject { Some(file) => { let name = file .file_name() .unwrap_or(std::ffi::OsStr::new("subject.py")); - let dest = work.join(name); + let dest = student.work.join(name); std::fs::copy(file, &dest)?; - Some(dest) + let sources = base.join("source"); + std::fs::create_dir(&sources)?; + let source = sources.join(name); + std::fs::copy(file, &source)?; + let mut readonly = std::fs::metadata(&source)?.permissions(); + readonly.set_readonly(true); + std::fs::set_permissions(&source, readonly)?; + (Some(dest), Some(source)) } - None => None, + None => (None, None), }; - let payload_path = root.path().join("payload.json"); - std::fs::write(&payload_path, payload(copied.as_deref()).to_string())?; - Ok(Staged { - work, - home, - tmp, - payload: payload_path, - subject: copied.unwrap_or_default(), + let staged = Staged { + // The teacher side's argument: it reads the payload and removes it. + payload: teacher.work.join("payload.json"), + teacher, + student, + subject: copied, + source, + canary, root, - }) -} - -fn nonce() -> String { - use rand::Rng; - let bytes: [u8; 16] = rand::rng().random(); - bytes.iter().map(|b| format!("{b:02x}")).collect() + }; + std::fs::write(&staged.payload, payload(&staged).to_string())?; + Ok(staged) } -/// What a finished harness process left behind. +/// What a finished unit left behind. struct Finished { - stdout: String, - /// The process wrote more than its cap to the record channel. - stdout_overflow: bool, - stderr: String, + records: Records, exit: Exit, + teacher_exit: Exit, + stderr: String, + /// The most the student side was seen holding, in bytes. + peak: u64, +} + +impl Finished { + fn spawn_failed(message: String) -> Self { + Self { + records: Records::default(), + exit: Exit::Spawn(message.clone()), + teacher_exit: Exit::Spawn(message), + stderr: String::new(), + peak: 0, + } + } +} + +/// What the student side may hold: `limit_mb` beyond `idle`, the bytes it takes idle, +/// sampled `every` so often; `basis` says how an undeclared limit was estimated. +#[derive(Debug, Clone)] +struct Memory { + limit_mb: u64, + idle: u64, + every: Duration, + basis: Option, +} + +/// How often a unit's student side is sampled. +const SAMPLE_EVERY: Duration = Duration::from_millis(50); + +impl Memory { + fn watched(&self) -> u64 { + self.idle + .saturating_add(self.limit_mb.saturating_mul(1 << 20)) + } + + fn total_mb(&self) -> u64 { + self.watched().div_ceil(1 << 20) + } + + fn exceeded(&self) -> Exit { + Exit::Memory { + limit_mb: self.limit_mb, + idle_mb: self.idle.div_ceil(1 << 20), + basis: self.basis.clone(), + } + } +} + +/// Values worked out once per key, by a digest of everything they depend on. A failure is +/// not kept: the next caller works it out again. +#[cfg(any(target_os = "macos", target_os = "linux"))] +type Once = LazyLock>>>>; + +/// The modules each submission imports, by its source. +#[cfg(any(target_os = "macos", target_os = "linux"))] +static SOURCE_IMPORTS: Once = LazyLock::new(Default::default); + +/// What the student side takes idle, by what it imports and what the teacher gives it. +#[cfg(any(target_os = "macos", target_os = "linux"))] +static IDLE_MEMORY: Once = LazyLock::new(Default::default); + +/// Modules, each with what `from … import` takes from it. +#[cfg(any(target_os = "macos", target_os = "linux"))] +type Imports = Vec<(String, Vec)>; + +/// The student whose idle memory is measured (`harness/idle.py`). +#[cfg(any(target_os = "macos", target_os = "linux"))] +const IDLE: &str = include_str!("harness/idle.py"); + +/// Reads a source's imports without running it (`harness/imports.py`). +#[cfg(any(target_os = "macos", target_os = "linux"))] +const IMPORTS: &str = include_str!("harness/imports.py"); + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn digest(given: &serde_json::Value) -> String { + use sha2::{Digest, Sha256}; + format!("{:x}", Sha256::digest(given.to_string())) +} + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn once(cache: &Once, key: String) -> Arc> { + cache + .lock() + .expect("once cache poisoned") + .entry(key) + .or_default() + .clone() } impl PythonExecutor { - /// Run the harness on a staged unit, killing it at `deadline_secs`. - async fn run_harness( + /// A command for one side: confined when sandboxed, with a cleared environment, the + /// side's own `HOME` and `TMPDIR`, and the kernel's limits. + fn command( &self, - staged: &Staged, + script: &str, + side: &Side, deadline_secs: u64, - stdin: Option, - cap: usize, - ) -> Finished { - use tokio::io::AsyncWriteExt; - - let mut cmd = Command::new(&self.python_cmd); + memory_mb: u64, + confinement: Option<&Confinement>, + ) -> Result { + // `memory_mb`: all the side may hold, interpreter included. + let mut cmd = match confinement { + Some(c) => Command::from( + crate::runner::sandbox::confined(Path::new(&self.python_cmd), c) + .map_err(|e| format!("cannot sandbox this unit: {e}"))?, + ), + None => Command::new(&self.python_cmd), + }; cmd.env_clear() .env("PATH", "/usr/bin:/usr/local/bin:/opt/homebrew/bin") - .env("HOME", &staged.home) - .env("TMPDIR", &staged.tmp) + .env("HOME", &side.home) + .env("TMPDIR", &side.tmp) // Isolated: no user site-packages, whose `.pth` files run before the harness, no - // PYTHON* variables, and the unit's directory, which holds the student's file, - // off sys.path. What those variables used to set is spelled as flags. - .args(["-I", "-B", "-X", "utf8"]) - .arg("-c") - .arg(format!( - "import sys; sys.path[:] = [p for p in sys.path if p]\n{}\n{}", - include_str!("../matching.py"), - HARNESS - )) - .arg(&staged.payload) - .current_dir(&staged.work) - .stdin(if stdin.is_some() { - std::process::Stdio::piped() - } else { - std::process::Stdio::null() - }) - .stderr(std::process::Stdio::piped()) + // PYTHON* variables, and the working directory off sys.path. What those variables + // used to set is spelled as flags. + .args(["-I", "-B", "-X", "utf8", "-c", script]) + .current_dir(&side.work) .kill_on_drop(true); - let spawn_failed = |message: String| Finished { - stdout: String::new(), - stdout_overflow: false, - stderr: String::new(), - exit: Exit::Spawn(message), - }; + #[cfg(windows)] + if let Some(root) = std::env::var_os("SYSTEMROOT") { + // Python cannot seed its hash randomisation on Windows without it. + cmd.env("SYSTEMROOT", root); + } #[cfg(unix)] - let records = { + { // Its own process group, so anything it spawns dies with it. cmd.process_group(0); - apply_sandbox(&mut cmd, &sandbox_for(deadline_secs)); - // The student's stdout goes nowhere: what they print during a call is captured - // in-process, and the records have a pipe of their own. - cmd.stdout(std::process::Stdio::null()); - match record_pipe(&mut cmd) { + apply_sandbox( + &mut cmd, + &sandbox_for(deadline_secs, memory_mb, confinement.is_some()), + ); + } + #[cfg(not(unix))] + let _ = (deadline_secs, memory_mb); + Ok(cmd) + } + + /// Each side's confinement, when sandboxed: the interpreter, its own directories, and + /// what `teacher` and `student` add. Neither side makes native calls: on macOS they + /// would read the grader's environment. + fn confinements( + &self, + staged: &Staged, + teacher: Vec, + student: Vec, + ) -> Result, String> { + // Asked even unsandboxed: an interpreter too old for the harness is refused here, + // with its version, rather than by both sides failing to start. + let interpreter = interpreter(&self.python_cmd)?; + if !self.isolation.is_sandboxed() { + return Ok(None); + } + let confine = |side: &Side, extra: Vec| Confinement { + read: covering(interpreter.read.iter().cloned().chain(extra).collect()), + write: vec![side.work.clone(), side.home.clone(), side.tmp.clone()], + exec: interpreter.exec.clone(), + spawn: false, + no_ffi: true, + }; + let mut teacher = teacher; + teacher.extend( + staged + .source + .as_ref() + .and_then(|s| s.parent()) + .map(Path::to_path_buf), + ); + Ok(Some([ + confine(&staged.teacher, canonical(teacher)), + confine(&staged.student, canonical(student)), + ])) + } + + /// Run a staged unit — the teacher side, and the student side when `with_student` — + /// end both at `deadline_secs`, and the student side past `memory`. + #[allow(clippy::too_many_arguments)] + async fn run_unit( + &self, + staged: &Staged, + deadline_secs: u64, + memory: Memory, + bounds: Bounds, + with_student: bool, + confinements: Option<[Confinement; 2]>, + ) -> Finished { + let [teacher_confinement, student_confinement] = match confinements { + Some([teacher, student]) => [Some(teacher), Some(student)], + None => [None, None], + }; + let mut teacher_cmd = match self.command( + &TEACHER_SCRIPT, + &staged.teacher, + deadline_secs, + memory.total_mb(), + teacher_confinement.as_ref(), + ) { + Ok(cmd) => cmd, + Err(e) => return Finished::spawn_failed(e), + }; + teacher_cmd.arg(&staged.payload); + let mut student = None; + if with_student { + // teacher → student, and student → teacher. + let pipes = std::io::pipe().and_then(|a| Ok((a, std::io::pipe()?))); + let ((to_student, from_teacher), (to_teacher, from_student)) = match pipes { + Ok(pipes) => pipes, + Err(e) => { + return Finished::spawn_failed(format!("could not join the two sides: {e}")); + } + }; + let mut cmd = match self.command( + &STUDENT_SCRIPT, + &staged.student, + deadline_secs, + memory.total_mb(), + student_confinement.as_ref(), + ) { + Ok(cmd) => cmd, + Err(e) => return Finished::spawn_failed(e), + }; + cmd.stdin(std::process::Stdio::from(to_student)) + .stdout(std::process::Stdio::from(from_student)) + .stderr(std::process::Stdio::piped()); + // The command holds the parent's copies of the student's ends: they go with it. + match cmd.spawn() { + Ok(child) => student = Some(child), + Err(e) => { + return Finished::spawn_failed(format!( + "could not start {}: {}", + self.python_cmd, + crate::runner::sandbox::spawn_error(&e) + )); + } + } + drop(cmd); + teacher_cmd + .stdin(std::process::Stdio::from(to_teacher)) + .stdout(std::process::Stdio::from(from_teacher)); + } else { + teacher_cmd + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()); + } + #[cfg(unix)] + let records = { + teacher_cmd.stderr(std::process::Stdio::piped()); + match record_pipe(&mut teacher_cmd) { Ok(pipe) => pipe, - Err(e) => return spawn_failed(format!("could not open the record channel: {e}")), + Err(e) => { + return Finished::spawn_failed(format!( + "could not open the record channel: {e}" + )); + } } }; #[cfg(not(unix))] - cmd.stdout(std::process::Stdio::piped()); + teacher_cmd.stderr(std::process::Stdio::piped()); - let mut child = match cmd.spawn() { + let mut teacher = match teacher_cmd.spawn() { Ok(child) => child, - Err(e) => return spawn_failed(format!("could not start {}: {e}", self.python_cmd)), + Err(e) => { + return Finished::spawn_failed(format!( + "could not start {}: {}", + self.python_cmd, + crate::runner::sandbox::spawn_error(&e) + )); + } }; - // The parent's copy of the write end goes, so the channel ends when the unit does. + // The parent's copies of every pipe end go, so each pipe ends when its side does. + drop(teacher_cmd); #[cfg(unix)] let records = { let (reader, writer) = records; drop(writer); match tokio::net::unix::pipe::Receiver::from_owned_fd(reader.into()) { Ok(receiver) => receiver, - Err(e) => return spawn_failed(format!("could not read the record channel: {e}")), + Err(e) => { + return Finished::spawn_failed(format!( + "could not read the record channel: {e}" + )); + } } }; #[cfg(not(unix))] - let records = child.stdout.take(); + let records = teacher.stderr.take(); + // Killed however this future ends: finished, timed out, or dropped by a caller. - let _group = GroupGuard::new(child.id()); + let _teacher_group = GroupGuard::new(teacher.id()); + let student_group = GroupGuard::new(student.as_ref().and_then(Child::id)); - if let (Some(text), Some(mut pipe)) = (stdin, child.stdin.take()) { - tokio::spawn(async move { - // A script that never reads its stdin closes the pipe; that is not an error. - let _ = pipe.write_all(text.as_bytes()).await; - }); - } - let stdout = Arc::new(Mutex::new(Sink::head(cap))); - let stderr = Arc::new(Mutex::new(Sink::tail(STDERR_TAIL))); + let reader = Arc::new(Mutex::new(Reader::new(bounds))); + let teacher_tail = Arc::new(Mutex::new(Sink::tail(STDERR_TAIL))); + let student_tail = Arc::new(Mutex::new(Sink::tail(STDERR_TAIL))); let mut pumps = tokio::task::JoinSet::new(); #[cfg(unix)] - pumps.spawn(pump(records, stdout.clone())); + pumps.spawn(pump_records(records, reader.clone())); #[cfg(not(unix))] if let Some(pipe) = records { - pumps.spawn(pump(pipe, stdout.clone())); + pumps.spawn(pump_records(pipe, reader.clone())); } - if let Some(pipe) = child.stderr.take() { - pumps.spawn(pump(pipe, stderr.clone())); + #[cfg(unix)] + if let Some(pipe) = teacher.stderr.take() { + pumps.spawn(pump(pipe, teacher_tail.clone())); + } + let over_memory = Arc::new(AtomicBool::new(false)); + let peak = Arc::new(AtomicU64::new(0)); + let mut watch = None; + if let Some(child) = student.as_mut() { + if let Some(pipe) = child.stderr.take() { + pumps.spawn(pump(pipe, student_tail.clone())); + } + if let Some(pid) = child.id() { + let (over, held) = (over_memory.clone(), peak.clone()); + let group = student_group.0; + watch = Some(tokio::spawn(watch_memory( + pid, + group, + memory.watched(), + memory.every, + over, + held, + ))); + } } - let exit = - match tokio::time::timeout(std::time::Duration::from_secs(deadline_secs), child.wait()) - .await - { - Ok(Ok(status)) => exit_of(status), - Ok(Err(e)) => Exit::Spawn(format!("lost the process: {e}")), - Err(_) => Exit::Deadline, - }; - // Whatever is left of the group — the harness at its deadline, or a process a student - // left behind holding the pipes — goes now. - _group.kill(); - let _ = child.kill().await; + let deadline = Duration::from_secs(deadline_secs); + let teacher_exit = match tokio::time::timeout(deadline, teacher.wait()).await { + Ok(Ok(status)) => exit_of(status), + Ok(Err(e)) => Exit::Spawn(format!("lost the teacher side: {e}")), + Err(_) => Exit::Deadline, + }; + let exit = match student.as_mut() { + None => Exit::Code(0), + Some(child) => { + let linger = if teacher_exit == Exit::Deadline { + Duration::ZERO + } else { + LINGER + }; + let ended = tokio::time::timeout(linger, child.wait()).await; + let over = || over_memory.load(Ordering::SeqCst); + match ended { + Ok(Ok(_)) if over() => memory.exceeded(), + Ok(Ok(status)) => exit_of(status), + Ok(Err(e)) => Exit::Spawn(format!("lost the student side: {e}")), + Err(_) if over() => memory.exceeded(), + Err(_) if teacher_exit == Exit::Deadline => Exit::Deadline, + Err(_) => Exit::Stopped, + } + } + }; + // Its exit is settled: the watch has nothing left to end, and is not waited for. + if let Some(watch) = watch { + watch.abort(); + } + // Whatever is left of either side — a hung student side, or a process either left + // behind holding a pipe — goes now. + drop(_teacher_group); + drop(student_group); + let _ = teacher.kill().await; + if let Some(child) = student.as_mut() { + let _ = child.kill().await; + } // What was read stays read: a pipe held open past this only loses what comes after. - let drain = tokio::time::sleep(std::time::Duration::from_secs(1)); + let drain = tokio::time::sleep(Duration::from_secs(1)); tokio::pin!(drain); loop { tokio::select! { @@ -290,20 +799,28 @@ impl PythonExecutor { } } } - let (stdout, stdout_overflow) = { - let sink = stdout.lock().expect("stdout sink poisoned"); - ( - String::from_utf8_lossy(&sink.bytes).into_owned(), - sink.overflow, - ) + let records = std::mem::replace( + &mut *reader.lock().expect("record reader poisoned"), + Reader::new(bounds), + ) + .finish(); + let tail = |sink: &Arc>| { + String::from_utf8_lossy(&sink.lock().expect("stderr sink poisoned").bytes) + .trim() + .to_string() + }; + let (teacher_err, student_err) = (tail(&teacher_tail), tail(&student_tail)); + let stderr = match (teacher_err.is_empty(), student_err.is_empty()) { + (_, true) => teacher_err, + (true, false) => format!("student side: {student_err}"), + (false, false) => format!("{teacher_err}\nstudent side: {student_err}"), }; - let stderr = String::from_utf8_lossy(&stderr.lock().expect("stderr sink poisoned").bytes) - .into_owned(); Finished { - stdout, - stdout_overflow, - stderr, + records, exit, + teacher_exit, + stderr, + peak: peak.load(Ordering::SeqCst), } } } @@ -334,12 +851,109 @@ fn record_pipe(cmd: &mut Command) -> std::io::Result<(std::io::PipeReader, std:: Ok((reader, writer)) } -/// The sandbox for a unit: its CPU limit sits above its deadline, a backstop for one busy -/// core rather than a second, shorter timeout. -fn sandbox_for(deadline_secs: u64) -> crate::runner::sandbox::SandboxConfig { +/// Read frames as they arrive. Past a frame the stream may not hold, keep reading and +/// drop what comes, so a teacher side still writing can finish. +async fn pump_records(mut pipe: R, reader: Arc>) { + use tokio::io::AsyncReadExt; + let mut header = [0u8; 4]; + loop { + if pipe.read_exact(&mut header).await.is_err() { + break; + } + let size = u32::from_be_bytes(header) as usize; + if !reader.lock().expect("record reader poisoned").admits(size) { + let mut sink = tokio::io::sink(); + let _ = tokio::io::copy(&mut pipe, &mut sink).await; + break; + } + let mut body = vec![0u8; size]; + // A frame cut off by a kill is the end of the stream, not a forgery. + if pipe.read_exact(&mut body).await.is_err() { + break; + } + reader.lock().expect("record reader poisoned").frame(&body); + } +} + +/// End the student side once it holds more than `limit` bytes: the operating system's +/// sandbox cannot bound its memory everywhere, so the grader watches it. +async fn watch_memory( + pid: u32, + group: Option, + limit: u64, + every: Duration, + over: Arc, + peak: Arc, +) { + let mut tick = tokio::time::interval(every); + while let Some(held) = resident_bytes(pid) { + peak.fetch_max(held, Ordering::SeqCst); + if held > limit { + over.store(true, Ordering::SeqCst); + if let Some(group) = group { + kill_group(group); + } + return; + } + tick.tick().await; + } +} + +/// The memory `pid` holds, or None once it is gone. +#[cfg(target_os = "macos")] +fn resident_bytes(pid: u32) -> Option { + let mut info: libc::proc_taskinfo = unsafe { std::mem::zeroed() }; + let size = std::mem::size_of::() as libc::c_int; + // SAFETY: `info` is a properly sized, writable proc_taskinfo. + let written = unsafe { + libc::proc_pidinfo( + i32::try_from(pid).ok()?, + libc::PROC_PIDTASKINFO, + 0, + (&mut info as *mut libc::proc_taskinfo).cast(), + size, + ) + }; + (written == size).then_some(info.pti_resident_size) +} + +#[cfg(target_os = "linux")] +fn resident_bytes(pid: u32) -> Option { + let statm = std::fs::read_to_string(format!("/proc/{pid}/statm")).ok()?; + let pages: u64 = statm.split_whitespace().nth(1)?.parse().ok()?; + // SAFETY: sysconf only reads a constant. + let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + Some(pages.saturating_mul(u64::try_from(page).ok()?)) +} + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +fn resident_bytes(_pid: u32) -> Option { + None +} + +/// The limits for a unit: its CPU limit sits above its deadline, a backstop for one busy +/// core rather than a second, shorter timeout. Its address space (Linux) sits far above +/// the memory the grader's watch holds it to: a backstop for an allocation faster than the +/// watch, which a library reserving address space it never uses (OpenBLAS) must not hit. +/// Confined, it cannot start processes, so the per-user process limit — which counts +/// threads on Linux — only stops a thread bomb, and leaves a busy machine's own threads +/// room. +#[cfg(unix)] +fn sandbox_for( + deadline_secs: u64, + memory_mb: u64, + confined: bool, +) -> crate::runner::sandbox::SandboxConfig { + let defaults = crate::runner::sandbox::SandboxConfig::default(); crate::runner::sandbox::SandboxConfig { cpu_secs: deadline_secs.saturating_add(1), - ..Default::default() + mem_mb: memory_mb.saturating_mul(4).max(defaults.mem_mb), + nproc: if confined { + Some(crate::runner::sandbox::CONFINED_NPROC) + } else { + defaults.nproc + }, + ..defaults } } @@ -355,7 +969,7 @@ pub fn kill_all_units() { } } -/// A unit's process group: registered while it lives, killed when the guard drops. +/// A side's process group: registered while it lives, killed when the guard drops. struct GroupGuard(Option); impl GroupGuard { @@ -369,12 +983,6 @@ impl GroupGuard { } Self(group) } - - fn kill(&self) { - if let Some(group) = self.0 { - kill_group(group); - } - } } impl Drop for GroupGuard { @@ -400,43 +1008,25 @@ fn kill_group(group: i32) { #[cfg(not(unix))] fn kill_group(_group: i32) {} -/// A pipe's contents, bounded: the first `cap` bytes, or only the last. +/// A pipe's contents, bounded: the last `cap` bytes. struct Sink { bytes: Vec, cap: usize, - keep_tail: bool, - overflow: bool, } impl Sink { - fn head(cap: usize) -> Self { + fn tail(cap: usize) -> Self { Self { bytes: Vec::new(), cap, - keep_tail: false, - overflow: false, - } - } - - fn tail(cap: usize) -> Self { - Self { - keep_tail: true, - ..Self::head(cap) } } fn push(&mut self, chunk: &[u8]) { - if self.keep_tail { - self.bytes.extend_from_slice(chunk); - if self.bytes.len() > self.cap { - let excess = self.bytes.len() - self.cap; - self.bytes.drain(..excess); - } - } else { - let room = self.cap.saturating_sub(self.bytes.len()); - self.bytes - .extend_from_slice(&chunk[..room.min(chunk.len())]); - self.overflow |= chunk.len() > room; + self.bytes.extend_from_slice(chunk); + if self.bytes.len() > self.cap { + let excess = self.bytes.len() - self.cap; + self.bytes.drain(..excess); } } } @@ -471,6 +1061,46 @@ async fn discard(staged: Staged) { let _ = tokio::task::spawn_blocking(move || drop(staged.root)).await; } +/// The teacher side's payload, common to every unit: with each side's canary when sandboxed. +fn base_payload(mode: &str, staged: &Staged, sandboxed: bool) -> serde_json::Value { + let canary = sandboxed.then_some(&staged.canary); + serde_json::json!({ + "mode": mode, + "records": RECORD_FD, + "limits": LIMITS, + "grace": GRACE_SECS, + "startup": STARTUP_SECS, + "canary": canary, + "student_canary": canary, + }) +} + +/// Each path that exists, resolved. +fn canonical(paths: Vec) -> Vec { + paths + .into_iter() + .filter_map(|p| std::fs::canonicalize(p).ok()) + .collect() +} + +/// What teacher code reads: the bundle, and the directories its modules import beside. +fn teacher_reads(bundle: &Path, imports: &[String]) -> Vec { + std::iter::once(bundle.to_path_buf()) + .chain( + imports + .iter() + .filter_map(|i| Path::new(i).parent().map(Path::to_path_buf)), + ) + .collect() +} + +fn extend(mut base: serde_json::Value, more: serde_json::Value) -> serde_json::Value { + if let (Some(base), serde_json::Value::Object(more)) = (base.as_object_mut(), more) { + base.extend(more); + } + base +} + impl Executor for PythonExecutor { fn language(&self) -> &str { "python" @@ -487,32 +1117,70 @@ impl Executor for PythonExecutor { } async fn inspect(&self, spec: &TestSpec, timeout_secs: u64) -> Result { - if spec.meta.imports.is_empty() { + if spec.meta.imports.is_empty() && spec.meta.shared.is_empty() { return Ok(TeacherRuntime::default()); } - let nonce = nonce(); let data = spec .meta .data_files .iter() .map(|rel| (spec.dir.join(rel), rel.clone())) .collect::>(); - let payload = serde_json::json!({ - "nonce": nonce, - "channel": RECORD_FD, - "mode": "inspect", - "imports": spec.meta.imports, - }); - let staged = tokio::task::spawn_blocking(move || stage(None, &data, |_| payload)) - .await - .map_err(|e| e.to_string())? - .map_err(|e| format!("could not stage the teacher modules: {e}"))?; - let finished = self - .run_harness(&staged, timeout_secs + 2, None, BASE_CAP) - .await; + let imports = spec.meta.imports.clone(); + let shared: Vec<(PathBuf, String)> = spec + .meta + .shared + .iter() + .map(|rel| (spec.dir.join(rel), rel.clone())) + .collect(); + let teacher_shared: Vec = shared.iter().map(|(src, _)| src.clone()).collect(); + let bundle = spec.dir.clone(); + let sandboxed = self.isolation.is_sandboxed(); + let staged = tokio::task::spawn_blocking(move || { + stage(None, &data, &shared, |staged| { + extend( + base_payload("inspect", staged, sandboxed), + serde_json::json!({ + "imports": imports, + "shared": teacher_shared, + "bundle": bundle, + }), + ) + }) + }) + .await + .map_err(|e| e.to_string())? + .map_err(|e| format!("could not stage the teacher modules: {e}"))?; + let deadline = timeout_secs.saturating_add(STARTUP_SECS); + let finished = match self.confinements( + &staged, + teacher_reads(&spec.dir, &spec.meta.imports), + Vec::new(), + ) { + Ok(confinements) => { + self.run_unit( + &staged, + deadline, + Memory { + limit_mb: spec + .meta + .memory_mb + .unwrap_or(crate::runner::memory::ESTIMATE_CAP_MB), + idle: 0, + every: SAMPLE_EVERY, + basis: None, + }, + INSPECT_BOUNDS, + false, + confinements, + ) + .await + } + Err(e) => Finished::spawn_failed(e), + }; discard(staged).await; - let records = crate::runner::records::parse(&finished.stdout, &nonce); + let records = finished.records; if let Some(fatal) = records.fatal { return Err(format!( "teacher module failed to import: {}: {}", @@ -521,7 +1189,7 @@ impl Executor for PythonExecutor { } match records.inspect { Some(runtime) => Ok(runtime), - None => Err(match finished.exit { + None => Err(match finished.teacher_exit { Exit::Deadline => { format!("importing the teacher modules took longer than {timeout_secs}s") } @@ -535,81 +1203,305 @@ impl Executor for PythonExecutor { } async fn run(&self, plan: &UnitPlan) -> UnitObservation { - let nonce = nonce(); + let idle = match self.idle_memory(plan).await { + Ok(bytes) => bytes, + Err(e) => { + let mut obs = UnitObservation::not_started(Exit::Spawn(e)); + obs.sandboxed = self.isolation.is_sandboxed(); + return obs; + } + }; + self.observe( + plan, + Memory { + limit_mb: plan.memory_mb, + idle, + every: SAMPLE_EVERY, + basis: plan.memory_basis.clone(), + }, + ) + .await + .0 + } +} + +impl PythonExecutor { + /// What the student side of `plan` takes idle, in bytes: Python, the harness, the + /// teacher's values and modules, and every module the submission imports — the most a + /// unit whose student is ScriptMark's own was seen holding, sampled every 2 ms. + /// `memory_mb` counts beyond it. Once per set of imports and what the teacher gives. + #[cfg(any(target_os = "macos", target_os = "linux"))] + async fn idle_memory(&self, plan: &UnitPlan) -> Result { + let source = tokio::fs::read(&plan.file) + .await + .map_err(|e| format!("could not read {}: {e}", plan.file.display()))?; + let sandboxed = self.isolation.is_sandboxed(); + let read = once( + &SOURCE_IMPORTS, + digest(&serde_json::json!({ + "python": self.python_cmd, + "sandboxed": sandboxed, + "source": String::from_utf8_lossy(&source), + })), + ); + let imports = read + .get_or_try_init(|| async { + let reader = Self { + python_cmd: self.python_cmd.clone(), + isolation: self.isolation, + }; + tokio::task::spawn_blocking(move || reader.source_imports(&source)) + .await + .map_err(|e| format!("reading the submission's imports panicked: {e}"))? + }) + .await? + .clone(); + let measured = once( + &IDLE_MEMORY, + digest(&serde_json::json!({ + "python": self.python_cmd, + "sandboxed": sandboxed, + "subject": plan.subject, + "allowed_imports": plan.allowed_imports, + "vars": &*plan.vars, + "shared": plan.shared, + "data_files": plan.data_files, + "imports": imports, + })), + ); + measured + .get_or_try_init(|| async { + match self.measure_idle_memory(plan, &imports).await { + // An import that keeps the idle unit from finishing — one that exits, say — + // is the submission's doing: it is then measured without its imports, and + // they count. + Err(_) if !imports.is_empty() => self.measure_idle_memory(plan, &[]).await, + measured => measured, + } + }) + .await + .copied() + } + + /// Nothing to count it by where the student side's memory is not watched. + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + async fn idle_memory(&self, _: &UnitPlan) -> Result { + Ok(0) + } + + /// The modules `source` imports, wherever in it: read from its tokens in a process of its + /// own, confined and not watched, so nothing the source holds is part of what a unit + /// takes idle. Nothing in the source runs. A reader that cannot finish — the source is + /// too strange, or too long to read in time — finds no imports, and they count: only a + /// reader that cannot start is an error. + #[cfg(any(target_os = "macos", target_os = "linux"))] + fn source_imports(&self, source: &[u8]) -> Result { + let (_dir, scratch) = tempfile::Builder::new() + .prefix("scriptmark-imports-") + .tempdir() + .and_then(|dir| std::fs::canonicalize(dir.path()).map(|path| (dir, path))) + .map_err(|e| format!("could not stage the import reader: {e}"))?; + let mut cmd = if self.isolation.is_sandboxed() { + let confinement = + teacher_confinement(&self.python_cmd, Vec::new(), vec![scratch.clone()], false)?; + crate::runner::sandbox::confined(Path::new(&self.python_cmd), &confinement)? + } else { + std::process::Command::new(&self.python_cmd) + }; + cmd.env_clear() + .env("HOME", &scratch) + .env("TMPDIR", &scratch) + .current_dir(&scratch) + .args(["-I", "-B", "-c", IMPORTS]); + let ran = crate::checker::python_checker::run_with_timeout( + cmd, + Some(String::from_utf8_lossy(source).into_owned()), + Duration::from_secs(STARTUP_SECS), + ) + .map_err(|e| { + format!( + "could not start the import reader: {}", + crate::runner::sandbox::spawn_error(&e) + ) + })?; + let Some(found) = ran.and_then(|ran| { + serde_json::from_str::)>>(ran.stdout.trim()).ok() + }) else { + return Ok(Vec::new()); + }; + let mut imports: Imports = found + .into_iter() + .filter(|(name, names)| { + crate::spec_loader::is_module_name(name) + && names.iter().all(|n| crate::spec_loader::is_module_name(n)) + }) + .collect(); + imports.sort(); + imports.dedup(); + Ok(imports) + } + + #[cfg(any(target_os = "macos", target_os = "linux"))] + async fn measure_idle_memory( + &self, + given: &UnitPlan, + imports: &[(String, Vec)], + ) -> Result { + let dir = tempfile::tempdir().map_err(|e| format!("could not stage the idle unit: {e}"))?; + let file = dir.path().join("idle.py"); + std::fs::write(&file, IDLE).map_err(|e| format!("could not stage the idle unit: {e}"))?; + let plan = UnitPlan { + subject: given.subject, + functions: crate::matching::Functions::default(), + file, + script: None, + imports: Vec::new(), + vars: given.vars.clone(), + data_files: given.data_files.clone(), + bundle: given.bundle.clone(), + shared: given.shared.clone(), + memory_mb: *crate::spec_loader::MEMORY_MB.end(), + memory_basis: None, + allowed_imports: given.allowed_imports.clone(), + load_timeout: given.load_timeout.max(STARTUP_SECS), + setup: Vec::new(), + steps: vec![crate::runner::executor::CallPlan { + target: crate::models::Target::Function { + name: "idle".into(), + }, + args: vec![serde_json::json!(imports)], + stdin: None, + timeout: STARTUP_SECS, + id: None, + files: Vec::new(), + check: None, + }], + }; + let memory = Memory { + limit_mb: plan.memory_mb, + idle: 0, + every: Duration::from_millis(2), + basis: None, + }; + let (obs, peak) = self.observe(&plan, memory).await; + if let Exit::Spawn(e) = &obs.exit { + return Err(e.clone()); + } + let idled = obs.done + && obs.steps.first().is_some_and(|s| { + matches!(s.outcome, crate::runner::executor::Outcome::Returned { .. }) + }); + if !idled || peak == 0 { + return Err(format!( + "could not measure what Python takes idle ({:?}): {}", + obs.exit, + obs.stderr.trim() + )); + } + Ok(peak) + } + + /// Run `plan` with the student side held to `memory`: what it reported, and the most + /// the student side was seen holding. + async fn observe(&self, plan: &UnitPlan, memory: Memory) -> (UnitObservation, u64) { let data = plan.data_files.clone(); + let shared = plan.shared.clone(); let file = plan.file.clone(); let payload = { - let nonce = nonce.clone(); let plan = plan.clone(); - move |subject: Option<&Path>| { - serde_json::json!({ - "nonce": nonce, - "channel": RECORD_FD, - "mode": "unit", - "student": subject, - "script": plan.script.as_ref().map(|s| serde_json::json!({ - "timeout": s.timeout, - "files": s.files, - })), - "imports": plan.imports, - "vars": &*plan.vars, - "allowed_imports": plan.allowed_imports, - "load_timeout": plan.load_timeout, - "subject": plan.subject, - "functions": plan.functions, - "lookup": match plan.subject { - Subject::Student => "fuzzy", - Subject::Reference => "exact", - }, - "setup": plan.setup, - "steps": plan.steps, - }) + let sandboxed = self.isolation.is_sandboxed(); + move |staged: &Staged| { + extend( + base_payload("unit", staged, sandboxed), + serde_json::json!({ + "role": match plan.subject { + Subject::Student => "student", + Subject::Reference => "reference", + }, + "subject": staged + .subject + .as_deref() + .and_then(Path::file_name) + .map(|n| n.to_string_lossy().into_owned()), + "source": staged.source, + "script": plan.script.as_ref().map(|s| serde_json::json!({ + "timeout": s.timeout, + "stdin": s.stdin, + "files": s.files, + })), + "imports": plan.imports, + "shared": plan.shared.iter().map(|(src, _)| src).collect::>(), + "student_shared": plan.shared.iter().map(|(_, rel)| rel).collect::>(), + "vars": &*plan.vars, + "allowed_imports": plan.allowed_imports, + "load_timeout": plan.load_timeout, + "functions": plan.functions, + "setup": plan.setup, + "steps": plan.steps, + }), + ) } }; - let staged = match tokio::task::spawn_blocking(move || stage(Some(&file), &data, payload)) - .await - { - Ok(Ok(staged)) => staged, - Ok(Err(e)) => { - return UnitObservation::not_started(Exit::Spawn(format!( - "could not stage the unit: {e}" - ))); - } - Err(e) => { - return UnitObservation::not_started(Exit::Spawn(format!("staging panicked: {e}"))); + let staged = + match tokio::task::spawn_blocking(move || stage(Some(&file), &data, &shared, payload)) + .await + { + Ok(Ok(staged)) => staged, + Ok(Err(e)) => { + let exit = Exit::Spawn(format!("could not stage the unit: {e}")); + return (UnitObservation::not_started(exit), 0); + } + Err(e) => { + let exit = Exit::Spawn(format!("staging panicked: {e}")); + return (UnitObservation::not_started(exit), 0); + } + }; + debug_assert!(staged.subject.as_ref().is_some_and(|s| s.is_file())); + + // A reference implementation is teacher code: it may read the bundle, as teacher + // code beside it may. + let student_reads = if plan.subject == Subject::Reference { + vec![plan.bundle.clone()] + } else { + Vec::new() + }; + let finished = match self.confinements( + &staged, + teacher_reads(&plan.bundle, &plan.imports), + student_reads, + ) { + Ok(confinements) => { + self.run_unit( + &staged, + plan.deadline_secs(), + memory, + record_bounds(plan), + true, + confinements, + ) + .await } + Err(e) => Finished::spawn_failed(e), }; - debug_assert!(staged.subject.is_file()); - - let stdin = plan - .script - .as_ref() - .map(|s| s.stdin.clone().unwrap_or_default()); - let cap = record_cap(plan); - let finished = self - .run_harness(&staged, plan.deadline_secs(), stdin, cap) - .await; discard(staged).await; - let mut records = crate::runner::records::parse(&finished.stdout, &nonce); - if finished.stdout_overflow && records.protocol_error.is_none() { - records.protocol_error = Some(ProtocolError::Flooded(format!( - "more than {} MiB was written to the record channel", - cap >> 20 - ))); - } - UnitObservation { + let records = finished.records; + let obs = UnitObservation { ready: records.ready, load: records.load, setup: records.setup, steps: records.steps, checks: records.checks, fatal: records.fatal, + stopped: records.stopped, done: records.done, exit: finished.exit, + teacher_exit: finished.teacher_exit, protocol_error: records.protocol_error, stderr: finished.stderr, - } + sandboxed: self.isolation.is_sandboxed(), + }; + (obs, finished.peak) } } @@ -624,20 +1516,49 @@ mod tests { use super::*; use crate::runner::executor::CallPlan; + /// The reader finds every import by its tokens — wherever it is, nothing it does not + /// spell out — and runs none of the source. + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn test_imports_are_read_from_the_source_without_running_it() { + let source = "import os, sys as system\nfrom x.y import (p as q,\n r)\nfrom . import mine\n\ + if False: import padded\ndef f():\n import inner; s = 'import fake'\n # import no\n\ + open('ran', 'w')\nPAD = [1,\n 2]\nimport last\n"; + let reader = PythonExecutor::new().with_isolation(Isolation::Unisolated); + let found = reader.source_imports(source.as_bytes()).unwrap(); + let names: Vec<&str> = found.iter().map(|(m, _)| m.as_str()).collect(); + assert_eq!(names, ["inner", "last", "os", "padded", "sys", "x.y"]); + assert!(found.contains(&("x.y".into(), vec!["p".into(), "r".into()]))); + let garbled = reader + .source_imports(b"import ok\nx = '''never ends") + .unwrap(); + assert_eq!(garbled, [("ok".to_string(), Vec::::new())]); + } + /// The CPU limit is derived from the unit's own deadline — never the fixed 30 s it used /// to be, which a scenario of several long steps would outrun and be killed by. #[test] fn test_the_cpu_limit_sits_above_any_units_deadline() { for deadline in [3, 45, 4 * 10 + 2, 86_400 * 4] { - assert!(sandbox_for(deadline).cpu_secs > deadline); + assert!(sandbox_for(deadline, 256, true).cpu_secs > deadline); } - assert_eq!(sandbox_for(u64::MAX).cpu_secs, u64::MAX, "saturates"); + assert_eq!( + sandbox_for(u64::MAX, 256, true).cpu_secs, + u64::MAX, + "saturates" + ); + assert_eq!( + sandbox_for(3, 256, true).nproc, + Some(crate::runner::sandbox::CONFINED_NPROC) + ); + assert_eq!(sandbox_for(3, 256, false).nproc, Some(64)); } - /// The cap grows with every call and observed file, so a long scenario of large but - /// legal values is never taken for a flood. + /// Each record is bounded by the largest a call can make, and the stream by the number + /// of records the plan can produce: a long scenario of large but legal values is never + /// taken for a flood, and nothing is buffered on a guess. #[test] - fn test_the_record_cap_grows_with_the_units_calls_and_files() { + fn test_record_bounds_follow_the_plan() { let step = |files: usize| CallPlan { target: crate::models::Target::Function { name: "f".into() }, args: Vec::new(), @@ -655,15 +1576,22 @@ mod tests { imports: Vec::new(), vars: Default::default(), data_files: Vec::new(), + bundle: PathBuf::new(), + shared: Vec::new(), + memory_mb: 64, + memory_basis: None, allowed_imports: Vec::new(), load_timeout: 1, setup: Vec::new(), steps, }; - assert_eq!(record_cap(&plan(Vec::new())), BASE_CAP); - let wide = record_cap(&plan(vec![step(1); 20])); + let plain = record_bounds(&plan(vec![step(0); 20])); + let filed = record_bounds(&plan(vec![step(2); 20])); // Measured: one call with a 4 MiB value, 64 KiB of escaped stdout, a failing check's // 64 KiB message and one 1 Mi-character file came to 10.75 MiB. - assert!(wide >= BASE_CAP + 20 * (43 << 18), "{wide}"); + assert!(filed.frame >= 43 << 18, "{}", filed.frame); + assert!(filed.frame > plain.frame); + assert_eq!(plain.count, filed.count); + assert!(plain.count >= 2 * 20 + 3); } } diff --git a/src/core/src/runner/records.rs b/src/core/src/runner/records.rs index 7d75dd9..a4e64d5 100644 --- a/src/core/src/runner/records.rs +++ b/src/core/src/runner/records.rs @@ -1,15 +1,17 @@ -//! Reading the harness's record stream. +//! Reading the teacher side's record stream. //! -//! Each record is one line, `@@scriptmark:@@ {json}`, and may follow whatever a -//! student wrote without a newline, so the prefix is looked for anywhere in a line. -//! Everything else on stdout is somebody else's output and is ignored. +//! Each record is a frame: a 4-byte big-endian length, then that many bytes of JSON. A +//! frame longer than the unit's largest legal record, or more records than its plan can +//! produce, ends the stream there; what arrived before it stands. Only the teacher side +//! writes here: the student side's answers reach a record after the teacher side has +//! checked them. use std::collections::BTreeMap; use serde::Deserialize; use crate::runner::executor::{ - CallObservation, CheckObservation, Export, Fatal, Phase, ProtocolError, TeacherRuntime, + CallObservation, CheckObservation, Export, Fatal, Phase, ProtocolError, Stopped, TeacherRuntime, }; #[derive(Deserialize)] @@ -26,6 +28,7 @@ enum Record { #[serde(flatten)] check: CheckObservation, }, + Stopped(Stopped), Fatal(Fatal), Inspect { exports: BTreeMap, @@ -35,7 +38,7 @@ enum Record { Done, } -/// The records of one run, in the order the harness wrote them. +/// The records of one run, in the order the teacher side wrote them. #[derive(Debug, Default)] pub struct Records { pub ready: bool, @@ -43,38 +46,80 @@ pub struct Records { pub setup: Vec, pub steps: Vec, pub checks: BTreeMap, + pub stopped: Option, pub fatal: Option, pub inspect: Option, pub done: bool, - /// The first record that repeated, arrived out of order, or did not parse. + /// The first record that was too long, repeated, arrived out of order, or did not parse. pub protocol_error: Option, } -pub fn parse(stdout: &str, nonce: &str) -> Records { - let prefix = format!("@@scriptmark:{nonce}@@ "); - let mut records = Records::default(); - let lines: Vec<&str> = stdout.split('\n').collect(); - for (i, line) in lines.iter().enumerate() { - let Some(at) = line.find(&prefix) else { - continue; - }; - if records.protocol_error.is_some() { - break; +/// What a stream may hold: the longest frame, and how many frames. +#[derive(Debug, Clone, Copy)] +pub struct Bounds { + pub frame: usize, + pub count: usize, +} + +/// Reads frames as they arrive. +#[derive(Debug)] +pub struct Reader { + bounds: Bounds, + seen: usize, + records: Records, +} + +impl Reader { + pub fn new(bounds: Bounds) -> Self { + Self { + bounds, + seen: 0, + records: Records::default(), + } + } + + /// Whether a frame announcing `size` bytes may be read. One that may not breaks the + /// stream, and nothing after it is read. + pub fn admits(&mut self, size: usize) -> bool { + if self.records.protocol_error.is_some() { + return false; + } + if size > self.bounds.frame { + self.records.protocol_error = Some(ProtocolError::Flooded(format!( + "a record of {size} bytes, longer than any this unit produces ({} bytes)", + self.bounds.frame + ))); + return false; } - let body = &line[at + prefix.len()..]; - match serde_json::from_str::(body) { - Ok(record) => records.accept(record), - // A record cut off by a kill is the end of the stream, not a forgery. - Err(_) if i + 1 == lines.len() => {} + true + } + + pub fn frame(&mut self, body: &[u8]) { + if self.records.protocol_error.is_some() { + return; + } + self.seen += 1; + if self.seen > self.bounds.count { + self.records.protocol_error = Some(ProtocolError::Flooded(format!( + "more than the {} records this unit can produce", + self.bounds.count + ))); + return; + } + match serde_json::from_slice::(body) { + Ok(record) => self.records.accept(record), Err(e) => { - records.protocol_error = Some(ProtocolError::Unreadable(format!( + self.records.protocol_error = Some(ProtocolError::Unreadable(format!( "unreadable record ({e}): {}", - body.chars().take(200).collect::() + String::from_utf8_lossy(&body[..body.len().min(200)]) ))) } } } - records + + pub fn finish(self) -> Records { + self.records + } } impl Records { @@ -86,6 +131,9 @@ impl Records { if self.done { return self.tampered("a record arrived after 'done'"); } + if self.stopped.is_some() && !matches!(record, Record::Done | Record::Fatal(_)) { + return self.tampered("a record arrived after the student side was stopped"); + } match record { Record::Ready if self.ready => self.tampered("'ready' repeated"), Record::Ready => self.ready = true, @@ -120,11 +168,12 @@ impl Records { self.checks.insert(index, check); } } + Record::Stopped(stopped) => self.stopped = Some(stopped), // A teacher module is imported before 'ready'; a fatal claiming otherwise, or a - // second fatal, did not come from the harness. + // second fatal, is not the harness's. Record::Fatal(_) if self.fatal.is_some() => self.tampered("a second 'fatal'"), - Record::Fatal(fatal) if self.ready && fatal.stage == "teacher_import" => { - self.tampered("a teacher_import 'fatal' after 'ready'") + Record::Fatal(fatal) if self.ready && fatal.stage != "harness" => { + self.tampered(format!("a {} 'fatal' after 'ready'", fatal.stage)) } Record::Fatal(fatal) => self.fatal = Some(fatal), Record::Inspect { @@ -144,26 +193,34 @@ impl Records { #[cfg(test)] mod tests { use super::*; - use crate::runner::executor::Outcome; + use crate::runner::executor::{Outcome, StopReason}; - const N: &str = "abc123"; + const WIDE: Bounds = Bounds { + frame: 1 << 20, + count: 100, + }; - fn line(json: &str) -> String { - format!("\n@@scriptmark:{N}@@ {json}\n") + fn read(frames: &[&str], bounds: Bounds) -> Records { + let mut reader = Reader::new(bounds); + for frame in frames { + if reader.admits(frame.len()) { + reader.frame(frame.as_bytes()); + } + } + reader.finish() } #[test] - fn test_records_are_read_between_noise() { - let stdout = format!( - "student noise\n{}{}more noise{}{}", - line(r#"{"kind":"ready"}"#), - line( - r#"{"kind":"call","phase":"step","index":0,"target":{"requested":"f","resolved":"f"},"outcome":{"returned":{"value":3,"type":"int"}},"stdout":"hi\n","stdout_truncated":false,"elapsed_ms":1}"# - ), - line(r#"{"kind":"check","index":0,"verdict":{"pass":true,"message":""}}"#), - line(r#"{"kind":"done"}"#), + fn test_records_are_read_in_order() { + let r = read( + &[ + r#"{"kind":"ready"}"#, + r#"{"kind":"call","phase":"step","index":0,"target":{"requested":"f","resolved":"f"},"outcome":{"returned":{"value":3,"type":"int"}},"stdout":"hi\n","stdout_truncated":false,"elapsed_ms":1}"#, + r#"{"kind":"check","index":0,"verdict":{"pass":true,"message":""}}"#, + r#"{"kind":"done"}"#, + ], + WIDE, ); - let r = parse(&stdout, N); assert!(r.ready && r.done && r.protocol_error.is_none()); assert_eq!(r.steps.len(), 1); assert_eq!(r.steps[0].stdout, "hi\n"); @@ -180,22 +237,10 @@ mod tests { ); } - #[test] - fn test_a_record_glued_to_a_partial_student_line_still_counts() { - let stdout = format!("x@@scriptmark:{N}@@ {{\"kind\":\"ready\"}}\n"); - assert!(parse(&stdout, N).ready); - } - - #[test] - fn test_a_forged_record_with_the_wrong_nonce_is_ignored() { - let stdout = "@@scriptmark:guess@@ {\"kind\":\"done\"}\n"; - assert!(!parse(stdout, N).done); - } - #[test] fn test_a_repeated_record_is_a_protocol_error() { let call = r#"{"kind":"call","phase":"step","index":0,"outcome":{"timeout":{}}}"#; - let r = parse(&format!("{}{}", line(call), line(call)), N); + let r = read(&[call, call], WIDE); assert_eq!(r.steps.len(), 1); assert!(matches!( r.protocol_error, @@ -204,54 +249,79 @@ mod tests { } #[test] - fn test_a_stream_cut_short_keeps_what_arrived() { - let stdout = format!( - "{}{}@@scriptmark:{N}@@ {{\"kind\":\"ca", - line(r#"{"kind":"ready"}"#), - line( - r#"{"kind":"call","phase":"step","index":0,"outcome":{"raised":{"type":"KeyError","types":["KeyError","LookupError"],"message":"'k'"}}}"# - ), - ); - let r = parse(&stdout, N); - assert_eq!(r.steps.len(), 1); - assert!(!r.done); - assert!( - r.protocol_error.is_none(), - "a record cut off by a kill is the end of the stream" + fn test_a_garbled_record_mid_stream_is_unreadable() { + let r = read( + &[r#"{"kind":"ready"}"#, "{not json", r#"{"kind":"done"}"#], + WIDE, ); + assert!(r.ready && !r.done); + assert!(matches!( + r.protocol_error, + Some(ProtocolError::Unreadable(_)) + )); } #[test] - fn test_a_garbled_record_mid_stream_is_unreadable_not_tampering() { - let stdout = format!( - "{}@@scriptmark:{N}@@ {{not json\n{}", - line(r#"{"kind":"ready"}"#), - line(r#"{"kind":"done"}"#), + fn test_a_record_past_its_bound_ends_the_stream_and_earlier_records_stand() { + let call = r#"{"kind":"call","phase":"step","index":0,"outcome":{"timeout":{}}}"#; + let long = format!( + r#"{{"kind":"call","phase":"step","index":1,"outcome":{{"missing":{{"message":"{}"}}}}}}"#, + "x".repeat(200) + ); + let tight = Bounds { + frame: 150, + count: 100, + }; + let r = read( + &[r#"{"kind":"ready"}"#, call, &long, r#"{"kind":"done"}"#], + tight, + ); + assert_eq!(r.steps.len(), 1, "the record before the long one stands"); + assert!(!r.done); + assert!(matches!(r.protocol_error, Some(ProtocolError::Flooded(_)))); + + let few = Bounds { + frame: 1 << 20, + count: 2, + }; + let r = read(&[r#"{"kind":"ready"}"#, call, r#"{"kind":"done"}"#], few); + assert!( + matches!(r.protocol_error, Some(ProtocolError::Flooded(m)) if m.contains("2 records")) ); - assert!(matches!( - parse(&stdout, N).protocol_error, - Some(ProtocolError::Unreadable(_)) - )); } #[test] fn test_a_fault_claimed_after_ready_is_tampering() { let fatal = r#"{"kind":"fatal","stage":"teacher_import","error":{"type":"E","message":""}}"#; - let r = parse( - &format!("{}{}", line(r#"{"kind":"ready"}"#), line(fatal)), - N, - ); + let r = read(&[r#"{"kind":"ready"}"#, fatal], WIDE); assert!(r.fatal.is_none()); assert!(matches!(r.protocol_error, Some(ProtocolError::Tampered(_)))); - let after_done = format!( - "{}{}", - line(r#"{"kind":"done"}"#), - line(r#"{"kind":"ready"}"#) + let r = read(&[r#"{"kind":"done"}"#, r#"{"kind":"ready"}"#], WIDE); + assert!(matches!(r.protocol_error, Some(ProtocolError::Tampered(_)))); + let harness = r#"{"kind":"fatal","stage":"harness","error":{"type":"E","message":""}}"#; + assert!( + read(&[r#"{"kind":"ready"}"#, harness], WIDE) + .fatal + .is_some() ); - assert!(matches!( - parse(&after_done, N).protocol_error, - Some(ProtocolError::Tampered(_)) - )); + } + + #[test] + fn test_nothing_but_done_follows_a_stop() { + let stopped = + r#"{"kind":"stopped","phase":"step","index":0,"reason":"deadline","message":"late"}"#; + let r = read( + &[r#"{"kind":"ready"}"#, stopped, r#"{"kind":"done"}"#], + WIDE, + ); + assert_eq!( + r.stopped.as_ref().map(|s| (s.phase, s.reason)), + Some((Phase::Step, StopReason::Deadline)) + ); + assert!(r.done && r.protocol_error.is_none()); + let after = r#"{"kind":"call","phase":"step","index":0,"outcome":{"timeout":{}}}"#; + let r = read(&[r#"{"kind":"ready"}"#, stopped, after], WIDE); + assert!(matches!(r.protocol_error, Some(ProtocolError::Tampered(_)))); } } diff --git a/src/core/src/runner/sandbox.rs b/src/core/src/runner/sandbox.rs index 1e46ef4..74a1f6c 100644 --- a/src/core/src/runner/sandbox.rs +++ b/src/core/src/runner/sandbox.rs @@ -1,3 +1,16 @@ +//! What a unit's processes may do, and the operating system's means of holding them to it. +//! +//! Every process a unit starts runs under the kernel's resource limits (`setrlimit`). +//! Sandboxed, each also runs confined to what its side needs: on macOS by Seatbelt, under +//! `sandbox-exec` with a profile written here; on Linux by Landlock (files, TCP, signals, +//! abstract sockets) and seccomp (other sockets, connecting, starting processes, tracing), +//! both applied between fork and exec. A confinement that cannot be applied fails the +//! spawn: nothing runs unconfined by accident. Windows has no confinement yet. + +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + /// Resource limits for student code execution. /// /// Applied via `setrlimit` in a `pre_exec` hook (between fork and exec), @@ -6,25 +19,33 @@ pub struct SandboxConfig { /// CPU time limit in seconds (RLIMIT_CPU). Default: 30. pub cpu_secs: u64, - /// Virtual memory limit in MB (RLIMIT_AS). Default: 512. + /// Address-space limit in MB (RLIMIT_AS, Linux only): a backstop far above the memory a + /// unit is held to by the grader's watch of its resident memory. Default: 2048. pub mem_mb: u64, /// Max file write size in MB (RLIMIT_FSIZE). Default: 10. pub fsize_mb: u64, /// Max open file descriptors (RLIMIT_NOFILE). Default: 64. pub nofile: u64, /// Max processes for this UID (RLIMIT_NPROC). Default: 64. - /// Note: on macOS this is per-UID, not per-process. - pub nproc: u64, + /// It counts every process of the user — and on Linux every thread — so a low cap + /// refuses even a thread on a busy machine. A sandboxed unit cannot start processes at + /// all, so its cap only stops a runaway of threads, and sits far higher + /// (`CONFINED_NPROC`); the harness itself starts none. + pub nproc: Option, } +/// The per-user process cap of a confined unit: room for a busy desktop's own threads, and +/// a stop to a student's thread bomb. +pub const CONFINED_NPROC: u64 = 4096; + impl Default for SandboxConfig { fn default() -> Self { Self { cpu_secs: 30, - mem_mb: 512, + mem_mb: 2048, fsize_mb: 10, nofile: 64, - nproc: 64, + nproc: Some(64), } } } @@ -56,7 +77,9 @@ pub fn apply_sandbox(cmd: &mut tokio::process::Command, config: &SandboxConfig) // A crash must not write the process's memory, teacher values included, to disk. let _ = set_rlimit(libc::RLIMIT_CORE, 0); let _ = set_rlimit(libc::RLIMIT_NOFILE, nofile); - let _ = set_rlimit(libc::RLIMIT_NPROC, nproc); + if let Some(nproc) = nproc { + let _ = set_rlimit(libc::RLIMIT_NPROC, nproc); + } // RLIMIT_AS: skip on macOS where it's unreliable #[cfg(not(target_os = "macos"))] let _ = set_rlimit(libc::RLIMIT_AS, mem); @@ -65,6 +88,26 @@ pub fn apply_sandbox(cmd: &mut tokio::process::Command, config: &SandboxConfig) } } +/// Limits for a teacher's tool — a checker script or a lint tool — run outside a unit: its +/// own process group, so it ends with everything it started, a CPU limit a second past +/// its deadline, and no file larger than `SandboxConfig`'s or any core dump. +#[cfg(unix)] +pub fn limit_tool(cmd: &mut std::process::Command, deadline_secs: u64) { + use std::os::unix::process::CommandExt; + let cpu = deadline_secs.saturating_add(1); + let fsize = SandboxConfig::default().fsize_mb * 1024 * 1024; + cmd.process_group(0); + // SAFETY: setrlimit is async-signal-safe; this runs between fork and exec. + unsafe { + cmd.pre_exec(move || { + let _ = set_rlimits(libc::RLIMIT_CPU, cpu, cpu.saturating_add(1)); + let _ = set_rlimit(libc::RLIMIT_FSIZE, fsize); + let _ = set_rlimit(libc::RLIMIT_CORE, 0); + Ok(()) + }); + } +} + #[cfg(target_os = "macos")] type RlimitResource = libc::c_int; #[cfg(target_os = "linux")] @@ -88,6 +131,597 @@ fn set_rlimits(resource: RlimitResource, soft: u64, hard: u64) -> std::io::Resul Ok(()) } +/// Whether a run's units are confined by the operating system. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Isolation { + /// Each side in its own sandbox; a unit whose sandbox cannot be applied is not graded. + #[default] + Sandboxed, + /// `--no-sandbox`: resource limits only. Student code can reach whatever the grader's + /// user can, and the results say so. + Unisolated, +} + +impl Isolation { + pub fn is_sandboxed(self) -> bool { + self == Isolation::Sandboxed + } +} + +/// What one confined process may touch. Paths must be canonical: the sandbox sees the paths +/// the kernel resolves, never the symlinks that lead to them. +#[derive(Debug, Clone, Default)] +pub struct Confinement { + /// Directories and files it may read. + pub read: Vec, + /// Directories it may read and write. + pub write: Vec, + /// What it may execute in place of itself: the interpreter, which re-executes itself on + /// some platforms, and its loader. + pub exec: Vec, + /// It may start other processes, running any program it may read: lint tools. Never + /// the student side. + pub spawn: bool, + /// Native calls (`ctypes`, `cffi`) are refused. On macOS they are how a process reads + /// another same-user process's arguments and environment, which Seatbelt does not + /// mediate. + pub no_ffi: bool, +} + +/// Whether this machine can confine a unit, and if not, why — in words a teacher can act on. +pub fn available() -> Result<(), String> { + platform::available() +} + +/// The errno a confined command fails to start with when its sandbox could not be applied +/// between fork and exec, where nothing may allocate to say more. +#[cfg(target_os = "linux")] +const SANDBOX_REFUSED: i32 = libc::ENOTSUP; + +/// Why a command could not start, in words: a sandbox that could not be applied says so. +pub fn spawn_error(e: &std::io::Error) -> String { + #[cfg(target_os = "linux")] + if e.raw_os_error() == Some(SANDBOX_REFUSED) { + return format!("its sandbox could not be applied ({e})"); + } + e.to_string() +} + +/// A command that runs `program` held to `confinement`. Its arguments are the caller's to add. +pub fn confined( + program: &Path, + confinement: &Confinement, +) -> Result { + platform::command(program, confinement) +} + +/// Where the operating system keeps the programs a teacher's tool may run, and what they +/// read: never a home directory. +pub fn system_programs() -> Vec { + #[cfg(target_os = "macos")] + let places = [ + "/usr", + "/bin", + "/sbin", + "/System", + "/Library/Frameworks", + "/private/etc", + "/private/var/db/timezone", + ] + .as_slice(); + #[cfg(not(target_os = "macos"))] + let places = ["/usr", "/bin", "/sbin", "/lib", "/lib64", "/etc"].as_slice(); + places + .iter() + .filter_map(|p| std::fs::canonicalize(p).ok()) + .collect() +} + +/// The program the kernel loads to start `executable`: its ELF interpreter, which a +/// confined process must be allowed to run too. +#[cfg(not(target_os = "macos"))] +pub fn loader_of(executable: &Path) -> Option { + use std::io::{Read, Seek, SeekFrom}; + const PT_INTERP: u32 = 3; + let mut file = std::fs::File::open(executable).ok()?; + let mut header = [0u8; 64]; + file.read_exact(&mut header).ok()?; + // 64-bit, little-endian ELF: what ScriptMark's Linux builds run on. + if &header[..4] != b"\x7fELF" || header[4] != 2 || header[5] != 1 { + return None; + } + let u16_at = |at: usize| u16::from_le_bytes([header[at], header[at + 1]]) as u64; + let phoff = u64::from_le_bytes(header[0x20..0x28].try_into().ok()?); + let (size, count) = (u16_at(0x36), u16_at(0x38)); + for index in 0..count { + let mut entry = [0u8; 56]; + file.seek(SeekFrom::Start(phoff + index * size)).ok()?; + file.read_exact(&mut entry).ok()?; + if u32::from_le_bytes(entry[..4].try_into().ok()?) != PT_INTERP { + continue; + } + let offset = u64::from_le_bytes(entry[8..16].try_into().ok()?); + let length = u64::from_le_bytes(entry[32..40].try_into().ok()?).min(4096) as usize; + let mut path = vec![0u8; length]; + file.seek(SeekFrom::Start(offset)).ok()?; + file.read_exact(&mut path).ok()?; + let path = std::str::from_utf8(path.split(|b| *b == 0).next()?).ok()?; + return std::fs::canonicalize(path).ok(); + } + None +} + +/// The shortest list covering `paths`: one inside another adds nothing. +pub fn covering(mut paths: Vec) -> Vec { + paths.sort(); + paths.dedup(); + let mut kept: Vec = Vec::new(); + for path in paths { + if !kept.iter().any(|k| path.starts_with(k)) { + kept.push(path); + } + } + kept +} + +#[cfg(target_os = "macos")] +mod platform { + use super::*; + + const SANDBOX_EXEC: &str = "/usr/bin/sandbox-exec"; + + pub fn available() -> Result<(), String> { + let probe = std::process::Command::new(SANDBOX_EXEC) + .args(["-p", "(version 1)(allow default)", "/usr/bin/true"]) + .output() + .map_err(|e| format!("{SANDBOX_EXEC} cannot run: {e}"))?; + if probe.status.success() { + return Ok(()); + } + Err(format!( + "{SANDBOX_EXEC} could not apply a sandbox ({}); a process already inside another sandbox cannot start one", + String::from_utf8_lossy(&probe.stderr).trim() + )) + } + + pub fn command(program: &Path, c: &Confinement) -> Result { + let mut cmd = std::process::Command::new(SANDBOX_EXEC); + cmd.arg("-p").arg(profile(c)?).arg(program); + Ok(cmd) + } + + /// An SBPL string literal. + fn quote(path: &Path) -> Result { + let text = path + .to_str() + .ok_or_else(|| format!("'{}' is not valid UTF-8", path.display()))?; + Ok(format!( + "\"{}\"", + text.replace('\\', "\\\\").replace('"', "\\\"") + )) + } + + fn subpaths(paths: &[PathBuf]) -> Result { + paths + .iter() + .map(|p| quote(p).map(|q| format!("(subpath {q})"))) + .collect::, _>>() + .map(|all| all.join(" ")) + } + + /// The Seatbelt profile for `c`. Everything is denied but what `system.sb` grants every + /// process (the dyld cache, system frameworks, `/dev/null`, `/dev/urandom`), and: + pub(super) fn profile(c: &Confinement) -> Result { + let mut rules = vec![ + "(version 1)".to_string(), + "(deny default)".to_string(), + "(import \"system.sb\")".to_string(), + ]; + let readable = covering(c.read.iter().chain(&c.write).cloned().collect()); + rules.push(format!("(allow file-read* {})", subpaths(&readable)?)); + // Resolving a path, and finding the working directory, stats every directory above it. + let ancestors: Vec = readable + .iter() + .flat_map(|p| p.ancestors().skip(1).map(Path::to_path_buf)) + .collect::>() + .into_iter() + .collect(); + if !ancestors.is_empty() { + let literals = ancestors + .iter() + .map(|p| quote(p).map(|q| format!("(literal {q})"))) + .collect::, _>>()?; + rules.push(format!("(allow file-read-metadata {})", literals.join(" "))); + } + if !c.write.is_empty() { + rules.push(format!("(allow file-write* {})", subpaths(&c.write)?)); + } + if !c.exec.is_empty() { + rules.push(format!("(allow process-exec {})", subpaths(&c.exec)?)); + } + rules.push("(allow signal (target self))".into()); + if c.spawn { + rules.push("(allow process-fork)".into()); + rules.push("(allow process-exec)".into()); + rules.push("(allow signal (target children))".into()); + } + if c.no_ffi { + rules.push( + r#"(deny file-read* (regex #"/_ctypes[^/]*\.so$") (regex #"/_cffi_backend[^/]*\.so$"))"# + .into(), + ); + // Nor by another name: no hard link to the library, and no native code loaded + // from anywhere it can write. + rules.push("(deny file-link)".into()); + if !c.write.is_empty() { + rules.push(format!( + "(deny file-map-executable {})", + subpaths(&c.write)? + )); + } + } + Ok(rules.join("\n")) + } +} + +#[cfg(target_os = "linux")] +mod platform { + use super::*; + use landlock::{ + ABI, Access, AccessFs, AccessNet, CompatLevel, Compatible, PathBeneath, PathFd, Ruleset, + RulesetAttr, RulesetCreatedAttr, RulesetStatus, Scope, + }; + use seccompiler::{ + BpfProgram, SeccompAction, SeccompCmpArgLen, SeccompCmpOp, SeccompCondition, SeccompFilter, + SeccompRule, TargetArch, + }; + use std::collections::BTreeMap; + + /// The Landlock ABI the running kernel offers: 0 when it has none. + fn kernel_abi() -> i64 { + const LANDLOCK_CREATE_RULESET_VERSION: libc::c_uint = 1; + // SAFETY: with this flag the call only reports a number. + unsafe { + libc::syscall( + libc::SYS_landlock_create_ruleset, + std::ptr::null::(), + 0usize, + LANDLOCK_CREATE_RULESET_VERSION, + ) + } + } + + fn abi() -> Result { + Ok(match kernel_abi() { + n if n <= 0 => { + return Err( + "this kernel has no Landlock (Linux 5.13 or later, with Landlock enabled at boot, is needed)" + .into(), + ); + } + 1 => ABI::V1, + 2 => ABI::V2, + 3 => ABI::V3, + 4 => ABI::V4, + 5 => ABI::V5, + 6 => ABI::V6, + 7 => ABI::V7, + 8 => ABI::V8, + _ => ABI::V9, + }) + } + + pub fn available() -> Result<(), String> { + abi().map(|_| ()) + } + + pub fn command(program: &Path, c: &Confinement) -> Result { + use std::os::unix::process::CommandExt; + let abi = abi()?; + let ruleset = landlock_ruleset(abi, c).map_err(|e| format!("Landlock: {e}"))?; + // Signals and abstract sockets are Landlock's from ABI 6; before it, seccomp's. + let filters = filters(c, abi < ABI::V6, abi < ABI::V3)?; + let mut cmd = std::process::Command::new(program); + let mut ruleset = Some(ruleset); + // SAFETY: between fork and exec this only makes the syscalls that apply a prepared + // ruleset and prepared filters, and fails with a constant errno: nothing allocates, + // as another thread of the grader may have held the allocator's lock at the fork. + unsafe { + cmd.pre_exec(move || { + let refused = || std::io::Error::from_raw_os_error(SANDBOX_REFUSED); + if let Some(ruleset) = ruleset.take() { + match ruleset.restrict_self() { + Ok(status) if status.ruleset == RulesetStatus::FullyEnforced => {} + _ => return Err(refused()), + } + } + for filter in &filters { + seccompiler::apply_filter(filter).map_err(|_| refused())?; + } + Ok(()) + }); + } + Ok(cmd) + } + + fn landlock_ruleset( + abi: ABI, + c: &Confinement, + ) -> Result { + let all = AccessFs::from_all(abi); + let read = AccessFs::ReadFile | AccessFs::ReadDir; + // Writing never makes devices, pipes or sockets in the filesystem, and never runs + // what it wrote. + let write = all + & !(AccessFs::Execute + | AccessFs::MakeChar + | AccessFs::MakeBlock + | AccessFs::MakeFifo + | AccessFs::MakeSock); + let readable = if c.spawn { + read | AccessFs::Execute + } else { + read + }; + let mut ruleset = Ruleset::default() + .set_compatibility(CompatLevel::HardRequirement) + .handle_access(all)?; + if abi >= ABI::V4 { + ruleset = ruleset.handle_access(AccessNet::from_all(abi))?; + } + if abi >= ABI::V6 { + ruleset = ruleset.scope(Scope::from_all(abi))?; + } + let rule = |path: &PathBuf, access: landlock::BitFlags| { + // A path that does not exist grants nothing. + PathFd::new(path) + .ok() + .map(|fd| PathBeneath::new(fd, access & AccessFs::from_file(abi))) + }; + let dir_rule = |path: &PathBuf, access: landlock::BitFlags| { + PathFd::new(path).ok().map(|fd| { + let access = if path.is_dir() { + access + } else { + access & AccessFs::from_file(abi) + }; + PathBeneath::new(fd, access) + }) + }; + let devices = [ + ("/dev/null", read | AccessFs::WriteFile), + ("/dev/zero", read), + ("/dev/urandom", read), + ("/dev/random", read), + ] + .into_iter() + .filter_map(|(path, access)| rule(&PathBuf::from(path), access)); + // `no_new_privs` is set with the ruleset: a setuid program cannot shed it. + ruleset + .create()? + .add_rules( + c.read + .iter() + .filter_map(|p| dir_rule(p, readable)) + .map(Ok::<_, landlock::RulesetError>), + )? + .add_rules( + c.write + .iter() + .filter_map(|p| dir_rule(p, write)) + .map(Ok::<_, landlock::RulesetError>), + )? + .add_rules( + c.exec + .iter() + .filter_map(|p| rule(p, read | AccessFs::Execute)) + .map(Ok::<_, landlock::RulesetError>), + )? + .add_rules(devices.map(Ok::<_, landlock::RulesetError>)) + } + + fn arch() -> Result { + if cfg!(target_arch = "x86_64") { + Ok(TargetArch::x86_64) + } else if cfg!(target_arch = "aarch64") { + Ok(TargetArch::aarch64) + } else if cfg!(target_arch = "riscv64") { + Ok(TargetArch::riscv64) + } else { + Err("seccomp filters are not built for this architecture".into()) + } + } + + fn condition(arg: u8, op: SeccompCmpOp, value: u64) -> Result { + SeccompCondition::new(arg, SeccompCmpArgLen::Qword, op, value) + .and_then(|c| SeccompRule::new(vec![c])) + .map_err(|e| e.to_string()) + } + + /// The seccomp filters for `c`. Landlock governs paths; these govern what it cannot: + /// sockets other than a pair the process makes for itself, connecting or binding + /// anything, sending to an address, starting processes, tracing, changing another + /// process's limits or scheduling, changing any file's mode, owner or extended + /// attributes, and the kernel's other ways out. `signals` adds the kill family, when + /// Landlock cannot scope signals; `truncate` adds truncating by path, before Landlock + /// governs it (ABI 3). + fn filters(c: &Confinement, signals: bool, truncate: bool) -> Result, String> { + let mut denied: BTreeMap> = BTreeMap::new(); + for call in [ + libc::SYS_connect, + libc::SYS_bind, + libc::SYS_listen, + libc::SYS_accept, + libc::SYS_accept4, + libc::SYS_sendmsg, + libc::SYS_sendmmsg, + libc::SYS_ptrace, + libc::SYS_process_vm_readv, + libc::SYS_process_vm_writev, + libc::SYS_mount, + libc::SYS_umount2, + libc::SYS_pivot_root, + libc::SYS_chroot, + libc::SYS_unshare, + libc::SYS_setns, + libc::SYS_bpf, + libc::SYS_perf_event_open, + libc::SYS_userfaultfd, + libc::SYS_keyctl, + libc::SYS_add_key, + libc::SYS_request_key, + libc::SYS_io_uring_setup, + libc::SYS_io_uring_enter, + libc::SYS_io_uring_register, + libc::SYS_kexec_load, + libc::SYS_init_module, + libc::SYS_finit_module, + libc::SYS_delete_module, + libc::SYS_pidfd_getfd, + libc::SYS_open_by_handle_at, + libc::SYS_name_to_handle_at, + ] { + denied.insert(call, Vec::new()); + } + // Only the local pairs `socketpair` makes, and `socket(AF_UNIX)` for asyncio's. + denied.insert( + libc::SYS_socket, + vec![condition(0, SeccompCmpOp::Ne, libc::AF_UNIX as u64)?], + ); + denied.insert(libc::SYS_sendto, vec![condition(4, SeccompCmpOp::Ne, 0)?]); + if !c.spawn { + // A thread shares its process; anything else is a new process. + denied.insert( + libc::SYS_clone, + vec![condition( + 0, + SeccompCmpOp::MaskedEq(libc::CLONE_THREAD as u64), + 0, + )?], + ); + #[cfg(target_arch = "x86_64")] + for call in [libc::SYS_fork, libc::SYS_vfork] { + denied.insert(call, Vec::new()); + } + } + // Its own limits, priority, scheduling and memory stay its own — pid 0, or the + // calling process by kind and id — and another process's are not. Rules for one call + // refuse it when any of them matches. + const PRIO_PROCESS: u64 = 0; + const IOPRIO_WHO_PROCESS: u64 = 1; + for (call, rules) in [ + (libc::SYS_prlimit64, vec![(0, 0)]), + (libc::SYS_setpriority, vec![(0, PRIO_PROCESS), (1, 0)]), + (libc::SYS_ioprio_set, vec![(0, IOPRIO_WHO_PROCESS), (1, 0)]), + (libc::SYS_sched_setaffinity, vec![(0, 0)]), + (libc::SYS_sched_setscheduler, vec![(0, 0)]), + (libc::SYS_sched_setparam, vec![(0, 0)]), + (libc::SYS_sched_setattr, vec![(0, 0)]), + (libc::SYS_migrate_pages, vec![(0, 0)]), + (libc::SYS_move_pages, vec![(0, 0)]), + ] { + let rules = rules + .into_iter() + .map(|(arg, own)| condition(arg, SeccompCmpOp::Ne, own)) + .collect::, _>>()?; + denied.insert(call, rules); + } + // Landlock has no right for mode, owner or extended attributes: a student could + // otherwise lock another file of the user's away, even one it cannot read. + const SYS_FCHMODAT2: i64 = 452; + for call in [ + libc::SYS_fchmod, + libc::SYS_fchmodat, + SYS_FCHMODAT2, + libc::SYS_fchown, + libc::SYS_fchownat, + libc::SYS_setxattr, + libc::SYS_lsetxattr, + libc::SYS_fsetxattr, + libc::SYS_removexattr, + libc::SYS_lremovexattr, + libc::SYS_fremovexattr, + ] { + denied.insert(call, Vec::new()); + } + #[cfg(target_arch = "x86_64")] + for call in [libc::SYS_chmod, libc::SYS_chown, libc::SYS_lchown] { + denied.insert(call, Vec::new()); + } + if truncate { + denied.insert(libc::SYS_truncate, Vec::new()); + } + if signals { + for call in [ + libc::SYS_kill, + libc::SYS_tkill, + libc::SYS_tgkill, + libc::SYS_rt_sigqueueinfo, + libc::SYS_rt_tgsigqueueinfo, + libc::SYS_pidfd_send_signal, + ] { + denied.insert(call, Vec::new()); + } + } + let arch = arch()?; + let build = |rules, errno: i32| -> Result { + SeccompFilter::new( + rules, + SeccompAction::Allow, + SeccompAction::Errno(errno as u32), + arch, + ) + .and_then(BpfProgram::try_from) + .map_err(|e| e.to_string()) + }; + // `clone3` takes its flags by pointer, where a filter cannot look: refuse it as + // unknown, and the C library falls back to `clone`. + let clone3 = BTreeMap::from([(libc::SYS_clone3, Vec::new())]); + let filters = vec![ + build(denied, libc::EPERM)?, + build(clone3, libc::ENOSYS)?, + unknown_calls(), + ]; + Ok(filters) + } + + /// A list of what to refuse lets through whatever the kernel adds after it was written + /// — `setxattrat` and `removexattrat` came in 6.13. Every syscall numbered from the + /// first this list does not know is refused as unknown (`ENOSYS`), which the C library + /// and Python treat as an older kernel. They share one numbering on every architecture; + /// x32's numbering on x86-64, which starts at `0x40000000`, is refused with them. + fn unknown_calls() -> BpfProgram { + use seccompiler::sock_filter; + const FIRST_UNKNOWN: u32 = 463; // setxattrat + const SECCOMP_RET_ALLOW: u32 = 0x7fff_0000; + const SECCOMP_RET_ERRNO: u32 = 0x0005_0000; + const LOAD_NR: u16 = 0x20; // BPF_LD | BPF_W | BPF_ABS, at seccomp_data.nr + const JUMP_GE: u16 = 0x35; // BPF_JMP | BPF_JGE | BPF_K + const RETURN: u16 = 0x06; // BPF_RET | BPF_K + let op = |code, jt, jf, k| sock_filter { code, jt, jf, k }; + vec![ + op(LOAD_NR, 0, 0, 0), + op(JUMP_GE, 0, 1, FIRST_UNKNOWN), + op(RETURN, 0, 0, SECCOMP_RET_ERRNO | libc::ENOSYS as u32), + op(RETURN, 0, 0, SECCOMP_RET_ALLOW), + ] + } +} + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +mod platform { + use super::*; + + pub fn available() -> Result<(), String> { + Err("ScriptMark cannot sandbox student code on this platform yet".into()) + } + + pub fn command(_program: &Path, _c: &Confinement) -> Result { + Err(available().unwrap_err()) + } +} + #[cfg(test)] mod tests { use super::*; @@ -96,10 +730,10 @@ mod tests { fn test_default_config() { let config = SandboxConfig::default(); assert_eq!(config.cpu_secs, 30); - assert_eq!(config.mem_mb, 512); + assert_eq!(config.mem_mb, 2048); assert_eq!(config.fsize_mb, 10); assert_eq!(config.nofile, 64); - assert_eq!(config.nproc, 64); + assert_eq!(config.nproc, Some(64)); } #[cfg(unix)] @@ -113,4 +747,39 @@ mod tests { assert!(output.status.success()); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "hello"); } + + #[test] + fn test_covering_keeps_only_the_outermost_paths() { + let paths = ["/a/b", "/a", "/c", "/a/b/c", "/ab"] + .map(PathBuf::from) + .to_vec(); + assert_eq!( + covering(paths), + ["/a", "/ab", "/c"].map(PathBuf::from).to_vec() + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn test_a_profile_quotes_paths_and_grants_only_what_it_lists() { + let c = Confinement { + read: vec![PathBuf::from("/opt/py")], + write: vec![PathBuf::from("/tmp/unit \"x\"/work")], + exec: vec![PathBuf::from("/opt/py/bin")], + spawn: false, + no_ffi: true, + }; + let profile = platform::profile(&c).unwrap(); + assert!(profile.contains("(deny default)")); + assert!( + profile.contains(r#"(subpath "/tmp/unit \"x\"/work")"#), + "{profile}" + ); + assert!( + profile.contains(r#"(literal "/tmp/unit \"x\"")"#), + "{profile}" + ); + assert!(profile.contains("_ctypes")); + assert!(!profile.contains("process-fork")); + } } diff --git a/src/core/src/spec_loader.rs b/src/core/src/spec_loader.rs index d7d3609..cc11e5c 100644 --- a/src/core/src/spec_loader.rs +++ b/src/core/src/spec_loader.rs @@ -6,6 +6,7 @@ use serde_json::Value; use crate::checker::rhai_checker; use crate::runner::generation::{MAX_COUNT, MAX_GENERATED, concrete_name, origins}; use crate::runner::generator::Rule; +use crate::runner::memory::Space; use crate::models::{ AssignmentConfig, Check, CourseConfig, Scenario, SetupStep, Target, TestCase, TestSpec, @@ -115,12 +116,27 @@ fn resolve_paths(spec: &mut TestSpec) { } } -/// The name a function checker's `stdout` parameter always receives: the call's output. -pub const RESERVED: &str = "stdout"; +/// The names a function checker's extra parameters always receive, and what they hold. +pub const RESERVED: [(&str, &str); 2] = [ + ("stdout", "the call's output"), + ("source", "the student's source text"), +]; + +/// What `name` always means to a function checker, if it is reserved. +pub fn reserved(name: &str) -> Option<&'static str> { + RESERVED + .iter() + .find(|(reserved, _)| *reserved == name) + .map(|(_, meaning)| *meaning) +} /// The longest timeout a spec or the command line may ask for. pub const MAX_TIMEOUT_SECS: u64 = 86_400; +/// The memory a spec may give a unit beyond what it takes idle, in MiB: from what a small +/// exercise holds, above the noise of loading a module, to 64 GiB. +pub const MEMORY_MB: std::ops::RangeInclusive = 16..=65_536; + /// What the harness keeps of a call's stdout; an expectation longer than this can never match. pub const STDOUT_LIMIT: usize = 64 * 1024; @@ -217,14 +233,28 @@ impl<'a> Validator<'a> { self.problem("[meta]", format!("data file '{data}' does not exist")); } } - if self.spec.vars.contains_key(RESERVED) { - self.problem( - "[vars]", - format!( - "'{RESERVED}' is reserved: function checkers receive the call's output under it" - ), - ); + for name in self.spec.vars.keys() { + if let Some(meaning) = reserved(name) { + self.problem( + "[vars]", + format!("'{name}' is reserved: function checkers receive {meaning} under it"), + ); + } + } + self.shared(); + for name in &self.spec.meta.allowed_imports { + if !is_module_name(name) { + let message = format!( + "allowed import '{name}' must be a module name, such as 'numpy' or 'matplotlib.pyplot'" + ); + self.problem("[meta]", message); + } } + self.memory( + self.spec.meta.memory_mb, + self.spec.meta.space.as_deref(), + "[meta]", + ); for (name, value) in &self.spec.vars { if contains_null(value) { self.problem( @@ -235,6 +265,70 @@ impl<'a> Validator<'a> { } } + /// Shared modules: Python files in the bundle, each importable under a name of its own + /// on both sides, and neither data nor a teacher module besides. + fn shared(&mut self) { + let meta = &self.spec.meta; + let student = Path::new(&meta.file) + .file_name() + .map(|n| n.to_string_lossy().to_lowercase()); + let mut names = BTreeSet::new(); + for shared in &meta.shared { + let at = format!("shared module '{shared}'"); + if !is_contained(shared) { + self.problem( + "[meta]", + format!("{at} must be a relative path without '..'"), + ); + continue; + } + let path = Path::new(shared); + let module = path.file_stem().map(|s| s.to_string_lossy().into_owned()); + if path.extension().is_none_or(|e| e != "py") + || !module.as_deref().is_some_and(is_identifier) + { + self.problem( + "[meta]", + format!("{at} must be a .py file whose name can be imported"), + ); + } else if !self.spec.dir.join(path).is_file() { + self.problem("[meta]", format!("{at} does not exist")); + } + let name = path.file_name().map(|n| n.to_string_lossy().to_lowercase()); + if name.is_some() && name == student { + self.problem( + "[meta]", + format!("{at} has the student file's name: it would replace the submission"), + ); + } + if let Some(module) = module + && !names.insert(module.clone()) + { + self.problem( + "[meta]", + format!("two shared modules are both imported as '{module}'"), + ); + } + let resolved = self.spec.dir.join(path); + if meta.imports.iter().any(|i| Path::new(i) == resolved) { + self.problem( + "[meta]", + format!("{at} is also a teacher module: list it in shared only"), + ); + } + if meta + .data_files + .iter() + .any(|d| path.starts_with(d) || Path::new(d).starts_with(path)) + { + self.problem( + "[meta]", + format!("{at} is also a data file: list it in shared only"), + ); + } + } + } + fn body(&mut self) { let spec = self.spec; if spec.cases.is_empty() && spec.scenarios.is_empty() { @@ -282,6 +376,33 @@ impl<'a> Validator<'a> { } } + fn memory(&mut self, memory: Option, space: Option<&str>, at: &str) { + if let Some(space) = space { + if memory.is_some() { + self.problem( + at, + "memory_mb and space both say what it may hold: keep one", + ); + } + if Space::parse(space).is_none() { + self.problem( + at, + format!("space '{space}' must be O(1), O(log n), O(n), O(n log n) or O(n^2)"), + ); + } + } + if memory.is_some_and(|m| !MEMORY_MB.contains(&m)) { + self.problem( + at, + format!( + "memory_mb must be between {} and {} MiB", + MEMORY_MB.start(), + MEMORY_MB.end() + ), + ); + } + } + fn unique_name(&mut self, names: &mut BTreeSet, name: String) { if !names.insert(name.clone()) { self.problem( @@ -300,6 +421,7 @@ impl<'a> Validator<'a> { self.problem(&at, "has no steps"); } self.timeout(scenario.timeout, &at); + self.memory(scenario.memory_mb, scenario.space.as_deref(), &at); let mut bound = top.clone(); for step in &scenario.setup { self.setup(step, &mut bound, &format!("{at} setup '{}'", step.id)); @@ -311,11 +433,8 @@ impl<'a> Validator<'a> { if bound.all.contains(id) || self.spec.vars.contains_key(id) { self.problem(&step_at, format!("id '{id}' is already bound")); } - if id == RESERVED { - self.problem( - &step_at, - format!("id '{RESERVED}' is reserved for the call's output"), - ); + if let Some(meaning) = reserved(id) { + self.problem(&step_at, format!("id '{id}' is reserved for {meaning}")); } if let Some(target) = step.target(self.spec.meta.function.as_deref()) { bound.bind(id, &target); @@ -332,11 +451,8 @@ impl<'a> Validator<'a> { if bound.all.contains(&step.id) || self.spec.vars.contains_key(&step.id) { self.problem(at, format!("id '{}' is already bound", step.id)); } - if step.id == RESERVED { - self.problem( - at, - format!("id '{RESERVED}' is reserved for the call's output"), - ); + if let Some(meaning) = reserved(&step.id) { + self.problem(at, format!("id '{}' is reserved for {meaning}", step.id)); } if step.args.iter().any(contains_null) { self.problem( @@ -405,6 +521,14 @@ impl<'a> Validator<'a> { ); } self.timeout(case.timeout, at); + if is_step && (case.memory_mb.is_some() || case.space.is_some()) { + self.problem( + at, + "memory_mb and space belong to the scenario: its steps share one process", + ); + } else { + self.memory(case.memory_mb, case.space.as_deref(), at); + } if case.args.iter().any(contains_null) { self.problem( at, @@ -970,6 +1094,11 @@ fn is_contained(path: &str) -> bool { } /// A name a Rhai oracle can bind: letters, digits and `_`, not starting with a digit. +/// A module Python can import by this name: `numpy`, `matplotlib.pyplot`. +pub fn is_module_name(name: &str) -> bool { + name.split('.').all(is_identifier) +} + fn is_identifier(name: &str) -> bool { let mut chars = name.chars(); chars @@ -1812,6 +1941,153 @@ expect = 150 ); } + #[test] + fn test_shared_modules_are_importable_python_files_of_their_own() { + let dir = tempfile::tempdir().unwrap(); + for file in [ + "common.py", + "other/common.py", + "data/extra.py", + "lab.py", + "notes.txt", + "helpers/t.py", + "my-module.py", + ] { + let path = dir.path().join(file); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, "X = 1\n").unwrap(); + } + let problems = |shared: &str, extra: &str| -> Vec { + let toml = format!( + "[meta]\nname = \"t\"\nfile = \"lab.py\"\nfunction = \"f\"\nlanguage = \"python\"\nshared = [{shared}]\n{extra}\n[[cases]]\nname = \"x\"\nexpect = 1\n" + ); + match load_spec_str(&toml, dir.path()) { + Ok(_) => Vec::new(), + Err(SpecError::Invalid { problems, .. }) => problems, + Err(other) => vec![other.to_string()], + } + }; + assert!(problems("\"common.py\"", "").is_empty()); + for (shared, extra, needle) in [ + ("\"../x.py\"", "", "must be a relative path"), + ( + "\"notes.txt\"", + "", + "must be a .py file whose name can be imported", + ), + ( + "\"my-module.py\"", + "", + "must be a .py file whose name can be imported", + ), + ("\"missing.py\"", "", "does not exist"), + ("\"lab.py\"", "", "the student file's name"), + ( + "\"common.py\", \"other/common.py\"", + "", + "both imported as 'common'", + ), + ( + "\"helpers/t.py\"", + "imports = [\"helpers/t.py\"]", + "also a teacher module", + ), + ( + "\"data/extra.py\"", + "data_files = [\"data\"]", + "also a data file", + ), + ] { + let found = problems(shared, extra); + assert!( + found.iter().any(|p| p.contains(needle)), + "{shared}: expected {needle:?}, got {found:?}" + ); + } + } + + #[test] + fn test_memory_is_declared_per_process_within_bounds() { + refused( + "[[cases]]\nname = \"x\"\nexpect = 1\nmemory_mb = 8\n", + "memory_mb must be between 16 and 65536 MiB", + ); + refused( + "[[scenarios]]\nname = \"s\"\n[[scenarios.steps]]\nname = \"a\"\nexpect = 1\nmemory_mb = 512\n", + "memory_mb and space belong to the scenario", + ); + assert!( + problems( + "[[scenarios]]\nname = \"s\"\nmemory_mb = 512\n[[scenarios.steps]]\nname = \"a\"\nexpect = 1\n" + ) + .is_empty() + ); + // Unset, it is not part of what a spec's digest covers: older records still match. + let spec = load_spec_str( + &format!("{META}[[cases]]\nname = \"x\"\nexpect = 1\n"), + Path::new("."), + ) + .unwrap(); + assert!(!serde_json::to_string(&spec).unwrap().contains("memory_mb")); + } + + #[test] + fn test_space_is_big_o_and_not_beside_memory_mb() { + refused( + "[[cases]]\nname = \"x\"\nexpect = 1\nspace = \"O(n^3)\"\n", + "space 'O(n^3)' must be O(1), O(log n), O(n), O(n log n) or O(n^2)", + ); + refused( + "[[cases]]\nname = \"x\"\nexpect = 1\nspace = \"O(n)\"\nmemory_mb = 64\n", + "memory_mb and space both say what it may hold", + ); + refused( + "[[scenarios]]\nname = \"s\"\n[[scenarios.steps]]\nname = \"a\"\nexpect = 1\nspace = \"O(n)\"\n", + "memory_mb and space belong to the scenario", + ); + assert!( + problems( + "space = \"O(n log n)\"\n[[scenarios]]\nname = \"s\"\nspace = \"O(1)\"\n[[scenarios.steps]]\nname = \"a\"\nexpect = 1\n[[cases]]\nname = \"x\"\nexpect = 1\nmemory_mb = 32\n" + ) + .is_empty() + ); + } + + #[test] + fn test_an_allowed_import_is_a_module_name() { + for bad in [ + "os; import shutil", + "numpy.", + ".numpy", + "matplotlib..pyplot", + "1abc", + "", + ] { + refused( + &format!("allowed_imports = [{bad:?}]\n[[cases]]\nname = \"x\"\nexpect = 1\n"), + "must be a module name", + ); + } + assert!( + problems( + "allowed_imports = [\"numpy\", \"matplotlib.pyplot\", \"_private\"]\n[[cases]]\nname = \"x\"\nexpect = 1\n" + ) + .is_empty() + ); + } + + #[test] + fn test_source_is_reserved_like_stdout() { + refused( + "[vars]\nsource = 1\n[[cases]]\nname = \"x\"\nexpect = 1\n", + "'source' is reserved: function checkers receive the student's source text", + ); + refused( + "[[setup]]\nid = \"source\"\nfunction = \"a\"\n[[cases]]\nname = \"x\"\nexpect = 1\n", + "id 'source' is reserved", + ); + } + #[test] fn test_a_script_case_refuses_a_function_checker() { refused( diff --git a/src/core/tests/harness.rs b/src/core/tests/harness.rs index bb70017..be1fd59 100644 --- a/src/core/tests/harness.rs +++ b/src/core/tests/harness.rs @@ -7,8 +7,8 @@ use std::sync::Arc; use scriptmark_core::models::Target; use scriptmark_core::runner::executor::{ - CallPlan, CheckObservation, Executor, Exit, InProcessCheck, Outcome, ScriptRun, Subject, - UnitObservation, UnitPlan, + CallPlan, CheckObservation, Executor, Exit, InProcessCheck, Outcome, Phase, ScriptRun, + StopReason, Subject, UnitObservation, UnitPlan, }; use scriptmark_core::runner::python::PythonExecutor; use serde_json::{Value, json}; @@ -52,6 +52,10 @@ fn unit(file: &Path) -> UnitPlan { imports: Vec::new(), vars: Arc::new(BTreeMap::new()), data_files: Vec::new(), + bundle: file.parent().unwrap().to_path_buf(), + shared: Vec::new(), + memory_mb: 64, + memory_basis: None, allowed_imports: Vec::new(), load_timeout: 5, setup: Vec::new(), @@ -135,13 +139,16 @@ async fn test_a_timeout_is_recorded_even_through_a_bare_except() { ); } +/// A call that defeats its own timer is given up on by the teacher side, which reports it +/// and everything else: the student side is ended, and nothing after it runs. #[tokio::test] -async fn test_a_unit_that_never_yields_is_killed_at_its_deadline() { +async fn test_a_call_that_never_yields_is_given_up_on_and_nothing_after_it_runs() { let dir = tempfile::tempdir().unwrap(); let student = write( dir.path(), "lab.py", - "def forever():\n while True:\n try:\n while True:\n pass\n except BaseException:\n pass\n\ndef ok():\n return 1\n", + // Never returns to the interpreter, where the timer would fire. + "import itertools\n\ndef forever():\n return sum(itertools.count())\n\ndef ok():\n return 1\n", ); let mut plan = unit(&student); plan.load_timeout = 1; @@ -150,10 +157,25 @@ async fn test_a_unit_that_never_yields_is_killed_at_its_deadline() { let mut ok = function("ok", vec![]); ok.timeout = 1; plan.steps = vec![forever, ok]; + let started = std::time::Instant::now(); let obs = run(&plan).await; - assert_eq!(obs.exit, Exit::Deadline); - assert!(obs.steps.is_empty(), "the hung call never reported"); - assert!(!obs.done); + assert!( + started.elapsed() < std::time::Duration::from_secs(10), + "given up on at its own deadline, not the unit's" + ); + assert!(obs.done && obs.teacher_exit == Exit::Code(0), "{obs:?}"); + assert_eq!(obs.exit, Exit::Stopped); + assert_eq!( + obs.steps.len(), + 1, + "the hung call is reported; the next never ran" + ); + assert!(matches!(obs.steps[0].outcome, Outcome::Timeout {})); + let stopped = obs.stopped.as_ref().unwrap(); + assert_eq!( + (stopped.phase, stopped.index, stopped.reason), + (Phase::Step, 0, StopReason::Deadline) + ); } #[tokio::test] @@ -651,6 +673,8 @@ async fn test_serialisation_edges_found_in_rereview() { ); let mut plan = unit(&student); plan.imports = vec![teacher.to_string_lossy().into_owned()]; + // 1.5 million integers, and the harness's copies of them for the checker. + plan.memory_mb = 512; let mut big = function("big_list", vec![]); big.check = Some(InProcessCheck { function: "length".into(), @@ -842,12 +866,14 @@ async fn test_a_long_scenario_of_large_legal_values_is_not_a_flood() { function: "picky".into(), expected: None, }); - // About 10.75 MiB each on the wire: 75 MiB in all, past a fixed 64 MiB cap. + // About 10.75 MiB each on the wire: 75 MiB in all, past a fixed 64 MiB cap. This is + // about the record channel: the student side's copies of each value get room. plan.steps = vec![step; 7]; + plan.memory_mb = 512; let obs = run(&plan).await; assert!(obs.protocol_error.is_none(), "{:?}", obs.protocol_error); assert!(obs.done, "{obs:?}"); - assert_eq!(obs.steps.len(), 7); + assert_eq!(obs.steps.len(), 7, "{:?} {:?}", obs.exit, obs.stopped); assert_eq!(obs.checks.len(), 7); for step in &obs.steps { // The value itself arrived, at the limit rather than replaced by its size. @@ -930,3 +956,502 @@ async fn test_a_unit_cannot_leave_anything_behind_for_the_next_one() { "no core dumps" ); } + +#[tokio::test] +async fn test_a_student_side_that_dies_mid_call_is_reported_by_the_teacher_side() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "import importlib\n\ndef ok():\n return 1\n\ndef die():\n importlib.import_module('os')._exit(3)\n", + ); + let mut plan = unit(&student); + plan.steps = vec![ + function("ok", vec![]), + function("die", vec![]), + function("ok", vec![]), + ]; + let obs = run(&plan).await; + assert!(obs.done && obs.teacher_exit == Exit::Code(0), "{obs:?}"); + assert_eq!(obs.steps.len(), 1); + assert_eq!( + obs.exit, + Exit::Code(3), + "the student side's own exit is kept" + ); + let stopped = obs.stopped.as_ref().unwrap(); + assert_eq!((stopped.index, stopped.reason), (1, StopReason::Exited)); +} + +#[tokio::test] +async fn test_a_student_who_writes_into_the_pipe_only_breaks_their_own_unit() { + let dir = tempfile::tempdir().unwrap(); + // Every descriptor a student could guess at: a frame announcing 2 GiB, then a forged + // verdict, as the teacher side's records would spell it. + let student = write( + dir.path(), + "lab.py", + "import importlib, struct\nos = importlib.import_module('os')\n\ndef flood():\n forged = b'{\"kind\":\"check\",\"index\":0,\"verdict\":{\"pass\":true,\"message\":\"\"}}'\n for fd in range(3, 64):\n for data in (struct.pack('>I', len(forged)) + forged, struct.pack('>I', 1 << 31)):\n try:\n os.write(fd, data)\n except OSError:\n pass\n return 1\n", + ); + let teacher = write( + dir.path(), + "t.py", + "def never(result, expected):\n return False\n", + ); + let mut plan = unit(&student); + plan.imports = vec![teacher.to_string_lossy().into_owned()]; + let mut flood = function("flood", vec![]); + flood.check = Some(InProcessCheck { + function: "never".into(), + expected: None, + }); + plan.steps = vec![flood, function("flood", vec![])]; + let obs = run(&plan).await; + assert!(obs.protocol_error.is_none(), "{:?}", obs.protocol_error); + assert!( + obs.checks + .values() + .all(|c| !matches!(c, CheckObservation::Verdict { pass: true, .. })), + "no forged verdict reached the records: {:?}", + obs.checks + ); + let stopped = obs + .stopped + .as_ref() + .expect("the unit stops at the call that wrote"); + assert_eq!(stopped.index, 0); + assert!( + matches!(stopped.reason, StopReason::Flooded | StopReason::Garbled), + "{stopped:?}" + ); + assert!(obs.done, "{obs:?}"); +} + +/// What a submission imports, wherever in its file, is loaded when ScriptMark measures what +/// its student side takes idle: no import is the student's. The module here, a teacher's +/// data file the spec allows, holds 32 MiB from the moment it is imported. +#[tokio::test] +async fn test_what_a_submission_imports_counts_as_idle() { + let dir = tempfile::tempdir().unwrap(); + let heavy = write(dir.path(), "heavy.py", "BLOB = b'x' * (32 * 1024 * 1024)\n"); + let mut idle = Vec::new(); + for (name, import) in [("bare.py", ""), ("imports.py", " import heavy\n")] { + let student = write( + dir.path(), + name, + &format!( + "import time\n\ndef hog():\n{import} held = b'x' * (256 * 1024 * 1024)\n time.sleep(5)\n return len(held)\n" + ), + ); + let mut plan = unit(&student); + plan.data_files = vec![(heavy.clone(), "heavy.py".into())]; + plan.allowed_imports = vec!["heavy".into()]; + plan.steps = vec![function("hog", vec![])]; + plan.memory_mb = 16; + match run(&plan).await.exit { + Exit::Memory { idle_mb, .. } => idle.push(idle_mb), + other => panic!("{name}: {other:?}"), + } + } + assert!(idle[1] >= idle[0] + 30, "{idle:?} MiB"); +} + +/// Reading a submission's imports takes memory its source decides — about 140 MiB for the +/// 120,000 constants here — and none of it is part of what its unit takes idle. +#[tokio::test] +async fn test_what_reading_a_submission_takes_is_not_idle() { + let dir = tempfile::tempdir().unwrap(); + let hog = "import time\n\ndef hog():\n held = b'x' * (256 * 1024 * 1024)\n time.sleep(5)\n return len(held)\n"; + let mut idle = Vec::new(); + for (name, extra) in [ + ("plain.py", String::new()), + ("padded.py", format!("PAD = [{}]\n", "1,".repeat(120_000))), + ] { + let student = write(dir.path(), name, &format!("{extra}{hog}")); + let mut plan = unit(&student); + plan.steps = vec![function("hog", vec![])]; + plan.memory_mb = 16; + plan.load_timeout = 30; + match run(&plan).await.exit { + Exit::Memory { idle_mb, .. } => idle.push(idle_mb), + other => panic!("{name}: {other:?}"), + } + } + assert!(idle[1] <= idle[0] + 3, "{idle:?} MiB"); +} + +/// An import that exits when loaded — `unittest.__main__` runs the test runner — cannot +/// stop a submission's idle share from being measured, nor turn its units into the +/// environment's failure: it is measured without its imports. +#[tokio::test] +async fn test_an_import_that_exits_does_not_stop_the_measurement() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "def never():\n import unittest.__main__\n\ndef two():\n return 2\n", + ); + let mut plan = unit(&student); + plan.steps = vec![function("two", vec![])]; + let obs = run(&plan).await; + assert!(obs.done && obs.exit == Exit::Code(0), "{obs:?}"); + assert_eq!(returned(&obs.steps[0].outcome), &json!(2)); +} + +/// An entry names a module: a dotted one allows its whole package, and one that is not +/// installed does not keep the unit from running. +#[tokio::test] +async fn test_a_dotted_allowed_import_allows_its_package() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "import wsgiref.util\n\ndef scheme():\n return wsgiref.util.guess_scheme({})\n", + ); + let mut plan = unit(&student); + plan.allowed_imports = vec!["wsgiref.util".into(), "not_installed_anywhere".into()]; + plan.steps = vec![function("scheme", vec![])]; + let obs = run(&plan).await; + assert!(obs.done, "{obs:?}"); + assert_eq!(returned(&obs.steps[0].outcome), &json!("http")); +} + +#[tokio::test] +async fn test_a_student_side_past_its_memory_limit_is_ended() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "import time\n\ndef hog():\n held = b'x' * (256 * 1024 * 1024)\n time.sleep(5)\n return len(held)\n", + ); + let mut plan = unit(&student); + plan.steps = vec![function("hog", vec![]), function("hog", vec![])]; + plan.memory_mb = 96; + let obs = run(&plan).await; + assert!( + matches!(obs.exit, Exit::Memory { limit_mb: 96, idle_mb, basis: None } if idle_mb > 0), + "{obs:?}" + ); + assert!(obs.steps.is_empty()); + assert_eq!( + obs.stopped.as_ref().map(|s| s.reason), + Some(StopReason::Exited) + ); +} + +#[tokio::test] +async fn test_a_checker_sees_a_students_object_as_a_snapshot_and_its_graph_intact() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "class Node:\n def __init__(self, value):\n self.value = value\n self.next = None\n @property\n def doubled(self):\n return self.value * 2\n @property\n def _hidden(self):\n raise RuntimeError('private properties are not read')\n\nclass Bad:\n @property\n def broken(self):\n return 1 / 0\n\ndef ring():\n a, b = Node(1), Node(2)\n a.next, b.next = b, a\n return a\n\ndef shared():\n leaf = Node(3)\n return [leaf, leaf, (1, 2), {1, 2}, 2 ** 70]\n\ndef bad():\n return Bad()\n", + ); + let teacher = write( + dir.path(), + "t.py", + "def ring(result, expected):\n assert result.next.next is result, 'the cycle survives'\n assert (result.value, result.doubled, result.next.doubled) == (1, 2, 4)\n assert result.__scriptmark_class__ == 'Node'\n return True\n\ndef shared(result, expected):\n leaf, again, pair, both, big = result\n assert leaf is again and leaf.value == 3\n assert pair == (1, 2) and both == {1, 2} and big == 2 ** 70\n return True\n\ndef anything(result, expected):\n return True\n", + ); + let mut plan = unit(&student); + plan.imports = vec![teacher.to_string_lossy().into_owned()]; + let checked = |name: &str, check: &str| { + let mut c = function(name, vec![]); + c.check = Some(InProcessCheck { + function: check.into(), + expected: None, + }); + c + }; + plan.steps = vec![ + checked("ring", "ring"), + checked("shared", "shared"), + checked("bad", "anything"), + ]; + let obs = run(&plan).await; + let pass = CheckObservation::Verdict { + pass: true, + message: String::new(), + }; + assert_eq!(obs.checks[&0], pass, "{obs:?}"); + assert_eq!(obs.checks[&1], pass, "{obs:?}"); + assert!( + matches!(&obs.steps[2].outcome, Outcome::Raised(e) if e.is_a("ZeroDivisionError")), + "a property that raises while the snapshot is taken is the call's: {:?}", + obs.steps[2].outcome + ); + assert!(!obs.checks.contains_key(&2)); +} + +#[tokio::test] +async fn test_teacher_values_cross_by_value_and_student_values_are_read_when_checked() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "def spoil(items):\n items.append('student')\n return len(items)\n\nclass Account:\n def __init__(self):\n self.balance = 0\n def deposit(self, n):\n self.balance += n\n return self.balance\n", + ); + let teacher = write( + dir.path(), + "t.py", + "def make():\n return ['teacher']\n\ndef untouched(result, expected, items):\n return items == ['teacher'], f'the teacher still holds {items}'\n\ndef balance(result, expected, acct):\n return acct.balance == 30, f'balance {acct.balance}'\n", + ); + let mut plan = unit(&student); + plan.imports = vec![teacher.to_string_lossy().into_owned()]; + let mut make = call( + Target::Teacher { + name: "make".into(), + }, + vec![], + ); + make.id = Some("items".into()); + let mut acct = function("Account", vec![]); + acct.id = Some("acct".into()); + plan.setup = vec![make, acct]; + let mut spoil = function("spoil", vec![json!("$items")]); + spoil.check = Some(InProcessCheck { + function: "untouched".into(), + expected: None, + }); + let deposit = |n: i64| { + call( + Target::Method { + object: "acct".into(), + name: "deposit".into(), + }, + vec![json!(n)], + ) + }; + let mut last = deposit(20); + last.check = Some(InProcessCheck { + function: "balance".into(), + expected: None, + }); + plan.steps = vec![spoil, deposit(10), last]; + let obs = run(&plan).await; + let pass = |message: &str| CheckObservation::Verdict { + pass: true, + message: message.into(), + }; + assert_eq!(returned(&obs.steps[0].outcome), &json!(2), "{obs:?}"); + assert_eq!(obs.checks[&0], pass("the teacher still holds ['teacher']")); + assert_eq!(obs.checks[&2], pass("balance 30")); +} + +const COMMON: &str = "class ListNode:\n def __init__(self, value, next=None):\n self.value = value\n self.next = next\n\n def length(self):\n n, node, seen = 0, self, set()\n while node is not None and id(node) not in seen:\n seen.add(id(node))\n n += 1\n node = node.next\n return n\n"; + +/// A shared class crosses by value both ways, rebuilt from the receiving side's own copy: +/// the teacher's checker gets the teacher's class, cycles and all, and what a student does +/// to their copy stays on their side. +#[tokio::test] +async fn test_shared_classes_cross_by_value_into_each_sides_own_copy() { + let dir = tempfile::tempdir().unwrap(); + let common = write(dir.path(), "common.py", COMMON); + let teacher = write( + dir.path(), + "teacher.py", + "from common import ListNode\n\ndef is_ring(result, expected):\n assert type(result) is ListNode, type(result)\n assert result.next.next is result, 'the cycle survives'\n return result.length() == 2\n\ndef make():\n return ListNode(1, ListNode(2))\n\ndef base(result, expected):\n return type(result) is ListNode and result.length() == 1, f'{type(result)} {result.length()}'\n", + ); + let student = write( + dir.path(), + "lab.py", + "from common import ListNode\n\nclass Mine(ListNode):\n def length(self):\n return -1\n\ndef ring():\n a, b = ListNode(1), ListNode(2)\n a.next, b.next = b, a\n return a\n\ndef sub():\n return Mine(9)\n\ndef walk(head):\n return [isinstance(head, ListNode), head.value, head.next.value, head.length()]\n\ndef tamper():\n ListNode.length = lambda self: 42\n return ListNode(5)\n", + ); + let mut plan = unit(&student); + plan.imports = vec![teacher.to_string_lossy().into_owned()]; + plan.shared = vec![(common, "common.py".into())]; + let checked = |name: &str, check: &str| { + let mut c = function(name, vec![]); + c.check = Some(InProcessCheck { + function: check.into(), + expected: None, + }); + c + }; + let mut made = call( + Target::Teacher { + name: "make".into(), + }, + vec![], + ); + made.id = Some("made".into()); + plan.setup = vec![made]; + plan.steps = vec![ + checked("ring", "is_ring"), + checked("sub", "base"), + function("walk", vec![json!("$made")]), + checked("tamper", "base"), + ]; + let obs = run(&plan).await; + let pass = CheckObservation::Verdict { + pass: true, + message: String::new(), + }; + assert_eq!(obs.checks[&0], pass, "{obs:?}"); + assert!( + matches!( + &obs.checks[&1], + CheckObservation::Verdict { pass: true, .. } + ), + "a student subclass arrives as the shared base: {:?}", + obs.checks[&1] + ); + assert_eq!( + returned(&obs.steps[2].outcome), + &json!([true, 1, 2, 2]), + "the teacher's list arrives as the student's own class" + ); + assert!( + matches!( + &obs.checks[&3], + CheckObservation::Verdict { pass: true, .. } + ), + "the student's patch stays on the student's side: {:?}", + obs.checks[&3] + ); +} + +/// A shared module imports without an `allowed_imports` entry, and a student file that +/// shares its name with a teacher module never stands in for it. +#[tokio::test] +async fn test_a_shared_module_is_allowed_and_namespaces_stay_apart() { + let dir = tempfile::tempdir().unwrap(); + let common = write(dir.path(), "bundle/common.py", COMMON); + write(dir.path(), "bundle/helper.py", "WHO = 'teacher'\n"); + let teacher = write( + dir.path(), + "bundle/teacher.py", + "import helper\n\ndef who(result, expected):\n return helper.WHO == 'teacher', helper.WHO\n", + ); + let student = write( + dir.path(), + "submission/helper.py", + "import common\nWHO = 'student'\n\ndef f():\n return common.ListNode(1).length()\n", + ); + let mut plan = unit(&student); + plan.bundle = dir.path().join("bundle"); + plan.imports = vec![teacher.to_string_lossy().into_owned()]; + plan.shared = vec![(common, "common.py".into())]; + let mut f = function("f", vec![]); + f.check = Some(InProcessCheck { + function: "who".into(), + expected: None, + }); + plan.steps = vec![f]; + let obs = run(&plan).await; + assert_eq!(returned(&obs.steps[0].outcome), &json!(1), "{obs:?}"); + assert_eq!( + obs.checks[&0], + CheckObservation::Verdict { + pass: true, + message: "teacher".into() + } + ); +} + +/// A checker may take the student's source, to judge it with `ast`. +#[tokio::test] +async fn test_a_checker_can_read_the_students_source() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "def f(xs):\n total = 0\n for x in xs:\n total += x\n return total\n", + ); + let teacher = write( + dir.path(), + "t.py", + "import ast\n\ndef no_sum(result, expected, source):\n calls = [n.func.id for n in ast.walk(ast.parse(source)) if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)]\n return 'sum' not in calls and result == expected, f'calls {calls}'\n", + ); + let mut plan = unit(&student); + plan.imports = vec![teacher.to_string_lossy().into_owned()]; + let mut f = function("f", vec![json!([1, 2, 3])]); + f.check = Some(InProcessCheck { + function: "no_sum".into(), + expected: Some(json!(6)), + }); + plan.steps = vec![f]; + let obs = run(&plan).await; + assert_eq!( + obs.checks[&0], + CheckObservation::Verdict { + pass: true, + message: "calls []".into() + }, + "{obs:?}" + ); +} + +/// Values the transfer encoding once could not rebuild — a cycle back through a tuple, an +/// integer of millions of digits — still reach the checker. +#[tokio::test] +async fn test_a_tuple_cycle_and_a_huge_integer_reach_the_checker() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "def looped():\n inner = []\n outer = (inner, 1)\n inner.append(outer)\n return outer\n\ndef huge():\n return 2 ** 5_000_000\n", + ); + let teacher = write( + dir.path(), + "t.py", + "def looped(result, expected):\n inner, one = result\n return one == 1 and inner == [''], repr(result)[:80]\n\ndef huge(result, expected):\n return result == 2 ** 5_000_000\n", + ); + let mut plan = unit(&student); + plan.imports = vec![teacher.to_string_lossy().into_owned()]; + plan.steps = ["looped", "huge"] + .iter() + .map(|name| { + let mut c = function(name, vec![]); + c.check = Some(InProcessCheck { + function: name.to_string(), + expected: None, + }); + c + }) + .collect(); + let obs = run(&plan).await; + assert!(obs.stopped.is_none(), "{:?}", obs.stopped); + for index in 0..2 { + assert!( + matches!( + &obs.checks[&index], + CheckObservation::Verdict { pass: true, .. } + ), + "{:?}", + obs.checks[&index] + ); + } +} + +/// A forged answer padded past what a record may hold is the student's broken channel, +/// not a record stream the grader cannot read. +#[tokio::test] +async fn test_a_padded_answer_is_the_students_not_the_harnesss() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "import importlib, json, struct\nos = importlib.import_module('os')\n\ndef pad():\n body = json.dumps({'outcome': {'returned': {'value': 1, 'type': 'int'}}, 'stdout': 'x' * (8 * 1024 * 1024), 'stdout_truncated': False, 'elapsed_ms': 0}).encode()\n for fd in range(3, 64):\n try:\n os.write(fd, struct.pack('>I', len(body)) + body)\n except OSError:\n pass\n return 1\n", + ); + let teacher = write( + dir.path(), + "t.py", + "def any(result, expected):\n return True\n", + ); + let mut plan = unit(&student); + plan.imports = vec![teacher.to_string_lossy().into_owned()]; + let mut pad = function("pad", vec![]); + pad.check = Some(InProcessCheck { + function: "any".into(), + expected: None, + }); + plan.steps = vec![pad]; + let obs = run(&plan).await; + assert!(obs.protocol_error.is_none(), "{:?}", obs.protocol_error); + assert!(obs.done, "{obs:?}"); + let stopped = obs.stopped.as_ref().expect("the student side is stopped"); + assert!( + matches!(stopped.reason, StopReason::Garbled | StopReason::Flooded), + "{stopped:?}" + ); +} diff --git a/src/core/tests/integration.rs b/src/core/tests/integration.rs index 60c18b6..53a704f 100644 --- a/src/core/tests/integration.rs +++ b/src/core/tests/integration.rs @@ -11,6 +11,7 @@ use scriptmark_core::runner::frozen::Generation; use scriptmark_core::runner::orchestrator::{RunOptions, run_all}; use scriptmark_core::runner::prepare::prepare; use scriptmark_core::runner::python::PythonExecutor; +use scriptmark_core::runner::sandbox::Isolation; use scriptmark_core::spec_loader::load_spec_str; /// A scratch directory holding specs, teacher files and one directory per student. @@ -651,8 +652,10 @@ expect = 2 #[tokio::test] async fn test_setup_runs_once_per_unit_and_once_per_scenario() { let bench = Bench::new(); - let unit_log = bench.path().join("unit.log"); - let scenario_log = bench.path().join("scenario.log"); + // Outside the bundle: a path into it is refused before it reaches student code. + let logs = tempfile::tempdir().unwrap(); + let unit_log = logs.path().join("unit.log"); + let scenario_log = logs.path().join("scenario.log"); let spec = bench.spec(&format!( r#" [meta] @@ -701,7 +704,10 @@ expect = "recorded" "lab.py", "def record(path):\n with open(path, 'a') as fh:\n fh.write('ran\\n')\n return 'recorded'\n\ndef echo(x):\n return x\n", ); - let results = grade(vec![spec], &[student]).await; + // The student counts its setup calls in logs shared by every unit, outside them all: + // only an unsandboxed unit can write there. + let unconfined = PythonExecutor::new().with_isolation(Isolation::Unisolated); + let results = grade_with(vec![spec], &[student], unconfined, 5).await; assert_eq!(results[0].total_passed(), 4, "{:#?}", results[0]); let lines = |p: &Path| std::fs::read_to_string(p).unwrap().lines().count(); @@ -1340,8 +1346,10 @@ expect = 2 } } +/// A step that defeats its own timer is given up on: it timed out, and the steps after it +/// never ran, on the student's account. #[tokio::test] -async fn test_a_scenario_killed_at_its_deadline_blames_the_call_that_hung() { +async fn test_a_scenario_given_up_on_at_its_deadline_blames_the_call_that_hung() { let bench = Bench::new(); let spec = bench.spec( r#" @@ -1369,7 +1377,9 @@ expect = 1 let student = bench.student( "alice", "lab.py", - "def ok():\n return 1\n\ndef forever():\n while True:\n try:\n while True:\n pass\n except BaseException:\n pass\n", + // A loop that never returns to the interpreter, so the timer never fires in it. One + // that catches the timer's exception escapes when the timer lands in its handler. + "import itertools\n\ndef ok():\n return 1\n\ndef forever():\n return sum(itertools.count())\n", ); let results = grade_with(vec![spec], &[student], PythonExecutor::new(), 1).await; let r = &results[0]; @@ -1379,7 +1389,7 @@ expect = 1 ( TestStatus::Timeout, Some(Fault::Student), - Some(Cause::Killed) + Some(Cause::Timeout) ) ); assert_eq!( @@ -1401,3 +1411,110 @@ impl TapImports for TestSpec { self } } + +/// OSS-10: student code cannot read the bundle, so a path into it handed to student code +/// is refused before anyone is graded — whether a [vars] entry, an argument, or a teacher +/// module's constant — and `data_files` is the way to give the student a file. +#[tokio::test] +async fn test_a_path_into_the_bundle_is_never_handed_to_student_code() { + let bench = Bench::new(); + let data = bench.write("data/numbers.txt", "1 2 3\n"); + bench.write( + "helpers/paths.py", + "from pathlib import Path\nDATA = Path(__file__).parent.parent / 'data' / 'numbers.txt'\nNAME = 'numbers.txt'\n", + ); + let spec = |body: &str| { + bench.spec(&format!( + "[meta]\nname = \"t\"\nfile = \"lab.py\"\nfunction = \"f\"\nlanguage = \"python\"\nimports = [\"helpers/paths.py\"]\ndata_files = [\"data\"]\n{body}" + )) + }; + let path = data.to_string_lossy().into_owned(); + // A file the student would write, which does not exist yet, through the same links + // (`/var` is `/private/var` on macOS). + let unborn = bench.path().join("out.log").to_string_lossy().into_owned(); + for (body, wanted) in [ + ( + format!("[vars]\nOUT = {unborn:?}\n[[cases]]\nname = \"x\"\nexpect = 1\n"), + "[vars] 'OUT'", + ), + ( + format!("[vars]\nINPUT = {path:?}\n[[cases]]\nname = \"x\"\nexpect = 1\n"), + "[vars] 'INPUT'", + ), + ( + format!("[[cases]]\nname = \"x\"\nargs = [{path:?}]\nexpect = 1\n"), + "case 'x'", + ), + ( + "[[cases]]\nname = \"x\"\nargs = [\"$DATA\"]\nexpect = 1\n".to_string(), + "case 'x': '$DATA' is a path into the test bundle", + ), + ] { + let refused = refusal(spec(&body)).await; + assert!(refused.contains(wanted), "{refused}"); + assert!(refused.contains("data_files"), "{refused}"); + } + + // Staged by data_files and named relatively, the same file is the student's to read. + let spec = spec( + "[vars]\nINPUT = \"data/numbers.txt\"\n[[cases]]\nname = \"x\"\nargs = [\"$NAME\"]\nexpect = 6\n", + ); + let student = bench.student( + "alice", + "lab.py", + "def f(name):\n with open(INPUT) as fh:\n return sum(int(n) for n in fh.read().split())\n", + ); + let results = grade(vec![spec], &[student]).await; + assert_eq!(results[0].total_passed(), 1, "{:#?}", results[0]); +} + +/// A unit's student process is held to `memory_mb` beyond what it takes idle. Undeclared, +/// it is estimated from `space` — O(n^2) unless declared — and the size of the test data, +/// an integer argument counting as its value; the narrowest declaration applies. +#[tokio::test] +async fn test_memory_is_estimated_from_space_and_data_unless_declared() { + let bench = Bench::new(); + let spec = |meta: &str, case: &str| { + bench.spec(&format!( + "[meta]\nname = \"m\"\nfile = \"lab.py\"\nfunction = \"big\"\nlanguage = \"python\"\n{meta}\n[[cases]]\nname = \"big\"\nargs = [8000000]\nexpect = 8000000\n{case}\n" + )) + }; + let student = bench.student( + "alice", + "lab.py", + "def big(n):\n return len(list(range(n)))\n", + ); + let failure = |meta: &str, case: &str| { + let spec = spec(meta, case); + let student = student.clone(); + async move { + let r = grade(vec![spec], &[student]).await; + let c = r[0].test_results[0].cases[0].clone(); + assert_eq!( + (c.status, c.fault, c.cause), + (TestStatus::Error, Some(Fault::Student), Some(Cause::Memory)), + "{c:?}" + ); + c.failure.unwrap().message + } + }; + let estimated = failure("", "space = \"O(1)\"").await; + assert!( + estimated.contains("16 MiB memory limit (estimated from O(1) over 8000001 elements"), + "{estimated}" + ); + let declared = failure("memory_mb = 16", "").await; + assert!( + declared.contains("16 MiB memory limit (`memory_mb`"), + "{declared}" + ); + for (meta, case) in [ + // O(n^2) over eight million elements: the 1 GiB cap. + ("", ""), + ("space = \"O(1)\"", "memory_mb = 1024"), + ("memory_mb = 16", "space = \"O(n)\""), + ] { + let roomy = grade(vec![spec(meta, case)], std::slice::from_ref(&student)).await; + assert_eq!(roomy[0].total_passed(), 1, "{meta}{case}: {:#?}", roomy[0]); + } +} diff --git a/src/core/tests/oracle.rs b/src/core/tests/oracle.rs index e882a84..322ddcf 100644 --- a/src/core/tests/oracle.rs +++ b/src/core/tests/oracle.rs @@ -8,6 +8,7 @@ use scriptmark_core::runner::frozen::{Frozen, Generation}; use scriptmark_core::runner::orchestrator::{RunOptions, run_all}; use scriptmark_core::runner::prepare::{Bundle, prepare}; use scriptmark_core::runner::python::PythonExecutor; +use scriptmark_core::runner::sandbox::Isolation; use scriptmark_core::spec_loader::load_spec_str; use serde_json::{Value, json}; @@ -46,6 +47,15 @@ async fn prepared(spec: TestSpec, generation: &Generation) -> Result .map_err(|e| e.to_string()) } +/// For a reference that writes outside its unit — counting its calls, or changing its own +/// source — which only an unsandboxed reference can. +async fn prepared_unconfined(spec: TestSpec) -> Result, String> { + let executor = PythonExecutor::new().with_isolation(Isolation::Unisolated); + prepare(vec![spec], &Generation::fresh(), Arc::new(executor), 3) + .await + .map_err(|e| e.to_string()) +} + async fn run(bundles: Vec, students: &[StudentSubmission]) -> Vec { run_all( students, @@ -80,9 +90,7 @@ async fn fixed_samples_and_seeded_inputs_share_answers_across_students_and_batch let body: String = format!( "{FIXED}\n[[cases]]\nname = 'generated'\n[cases.parametrize]\nargs = {{ x = 'int(0, 10)' }}\nsamples = [[2], [3]]\n[cases.parametrize.random]\ncount = 4\nseed = 42\n[cases.oracle]\nreference = 'ref.py'\nfunction = 'answer'\n" ); - let bundles: Vec = prepared(bench.spec(&body), &Generation::fresh()) - .await - .unwrap(); + let bundles: Vec = prepared_unconfined(bench.spec(&body)).await.unwrap(); let frozen: Frozen = Frozen::of(&bundles); let path: PathBuf = bench.path().join("batch.cases.json"); frozen.write(&path).unwrap(); @@ -323,9 +331,7 @@ async fn sources_cannot_change_while_answers_are_being_prepared() { let bench: Bench = Bench::new(); let source: PathBuf = bench.path().join("ref.py"); bench.write("ref.py", &format!("def answer(x):\n with open({source:?}, 'a') as f:\n f.write('\\n# changed\\n')\n return x\n")); - let error: String = prepared(bench.spec(FIXED), &Generation::fresh()) - .await - .unwrap_err(); + let error: String = prepared_unconfined(bench.spec(FIXED)).await.unwrap_err(); assert!( error.contains("sources changed during preparation"), "{error}" diff --git a/src/core/tests/sandbox.rs b/src/core/tests/sandbox.rs new file mode 100644 index 0000000..8939e2b --- /dev/null +++ b/src/core/tests/sandbox.rs @@ -0,0 +1,452 @@ +//! The sandboxes, probed: what each side of a unit can reach. Every probe runs twice, the +//! second time without a sandbox, so a denial is the sandbox's doing and not a probe that +//! could never have worked. +#![cfg(any(target_os = "macos", target_os = "linux"))] + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use scriptmark_core::models::Target; +use scriptmark_core::runner::executor::{ + CallPlan, Executor, Outcome, Subject, UnitObservation, UnitPlan, +}; +use scriptmark_core::runner::python::PythonExecutor; +use scriptmark_core::runner::sandbox::Isolation; +use serde_json::{Value, json}; + +fn write(dir: &Path, name: &str, content: &str) -> PathBuf { + let path = dir.join(name); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, content).unwrap(); + path +} + +fn call(target: Target, args: Vec) -> CallPlan { + CallPlan { + target, + args, + stdin: None, + timeout: 5, + id: None, + files: Vec::new(), + check: None, + } +} + +fn function(name: &str, args: Vec) -> CallPlan { + call( + Target::Function { + name: name.to_string(), + }, + args, + ) +} + +fn teacher(name: &str, args: Vec) -> CallPlan { + call( + Target::Teacher { + name: name.to_string(), + }, + args, + ) +} + +/// Every probe answers with what happened, rather than raising: `ok …` or the error's type. +const PROBES: &str = r#"import importlib +os = importlib.import_module('os') +socket = importlib.import_module('socket') +subprocess = importlib.import_module('subprocess') + +def attempt(fn): + try: + return 'ok ' + str(fn())[:40] + except BaseException as exc: + return type(exc).__name__ + +def read(path): + return attempt(lambda: open(path).read()) + +def listing(path): + return attempt(lambda: sorted(os.listdir(path))) + +def write(path): + return attempt(lambda: open(path, 'w').write('x')) + +def connect(port): + return attempt(lambda: socket.create_connection(('127.0.0.1', port), timeout=3).close()) + +def spawn(): + return attempt(lambda: subprocess.run(['/bin/echo', 'escaped'], capture_output=True).stdout) + +def run_program(): + # In place of this process: no fork, so no process limit stands in for the sandbox. + return attempt(lambda: os.execv('/bin/echo', ['/bin/echo', 'escaped'])) + +def link_native(): + # The native-call module under a name of the student's own: its directory can be + # listed, though the module itself cannot be read. + sysconfig = importlib.import_module('sysconfig') + where = os.path.join(sysconfig.get_paths()['platstdlib'], 'lib-dynload') + name = next(n for n in os.listdir(where) if n.startswith('_ctypes')) + return attempt(lambda: os.link(os.path.join(where, name), 'mine.so')) + +def signal_grader(): + return attempt(lambda: os.kill(os.getppid(), 0)) + +def native(): + return attempt(lambda: importlib.import_module('ctypes').CDLL(None)) + +def environ_of_grader(): + return attempt(lambda: open(f'/proc/{os.getppid()}/environ', 'rb').read()) + +def env(): + return attempt(lambda: sorted(os.environ)) + +def chmod(path): + stat = importlib.import_module('stat') + def change(): + mode = os.stat(path).st_mode + os.chmod(path, stat.S_IMODE(mode)) # the mode it already has: nothing to undo + return attempt(change) + +def unix_socket(path): + return attempt(lambda: socket.socket(socket.AF_UNIX).connect(path)) + +def limits_of_grader(): + resource = importlib.import_module('resource') + return attempt(lambda: resource.prlimit(os.getppid(), resource.RLIMIT_CPU)) +"#; + +struct Bench { + _dir: tempfile::TempDir, + root: PathBuf, + bundle: PathBuf, + answers: PathBuf, + other: PathBuf, + student: PathBuf, + teacher: PathBuf, + /// A port the grader listens on, which no unit may connect to. + port: u16, + _listener: std::net::TcpListener, + /// A Unix socket the grader listens on, likewise. + socket: PathBuf, + _unix: std::os::unix::net::UnixListener, +} + +impl Bench { + fn new() -> Self { + let dir = tempfile::tempdir().unwrap(); + let root = std::fs::canonicalize(dir.path()).unwrap(); + let bundle = root.join("tests"); + let answers = write(&bundle, "answers.txt", "the teacher's answers\n"); + let teacher = write(&bundle, "helpers/probes.py", PROBES); + let other = write(&root, "submissions/bob/lab.py", "SECRET = 'bob'\n"); + let student = write(&root, "submissions/alice/lab.py", PROBES); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + // Short: a socket path has a length limit, and a temporary directory can be deep. + let unique = dir + .path() + .file_name() + .unwrap() + .to_string_lossy() + .into_owned(); + let socket = std::env::temp_dir().join(format!("sm{unique}.sock")); + let _ = std::fs::remove_file(&socket); + let unix = std::os::unix::net::UnixListener::bind(&socket).unwrap(); + Self { + socket, + _unix: unix, + port: listener.local_addr().unwrap().port(), + _listener: listener, + _dir: dir, + root, + bundle, + answers, + other, + student, + teacher, + } + } + + fn plan(&self, steps: Vec) -> UnitPlan { + UnitPlan { + subject: Subject::Student, + functions: scriptmark_core::matching::Functions::default(), + file: self.student.clone(), + script: None, + imports: vec![self.teacher.to_string_lossy().into_owned()], + vars: Arc::new(BTreeMap::new()), + data_files: Vec::new(), + bundle: self.bundle.clone(), + shared: Vec::new(), + memory_mb: 64, + memory_basis: None, + allowed_imports: Vec::new(), + load_timeout: 5, + setup: Vec::new(), + steps, + } + } +} + +impl Drop for Bench { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.socket); + } +} + +async fn both(plan: &UnitPlan) -> (UnitObservation, UnitObservation) { + let sandboxed = PythonExecutor::new().run(plan).await; + let open = PythonExecutor::new() + .with_isolation(Isolation::Unisolated) + .run(plan) + .await; + (sandboxed, open) +} + +fn said(obs: &UnitObservation, index: usize) -> String { + match &obs.steps[index].outcome { + Outcome::Returned { value, .. } => value.as_str().unwrap_or_default().to_string(), + other => panic!("step {index}: {other:?}\n{obs:?}"), + } +} + +/// What each probe said with the sandbox and without it, by name. +fn compare( + names: &[&str], + sandboxed: &UnitObservation, + open: &UnitObservation, +) -> Vec<(String, String, String)> { + assert!(sandboxed.done && sandboxed.sandboxed, "{sandboxed:?}"); + assert!(open.done && !open.sandboxed, "{open:?}"); + names + .iter() + .enumerate() + .map(|(i, name)| (name.to_string(), said(sandboxed, i), said(open, i))) + .collect() +} + +fn denied(probes: &[(String, String, String)], name: &str) { + let (_, sandboxed, open) = probes.iter().find(|(n, ..)| n == name).unwrap(); + if name == "spawn" { + // The process limit is per user on macOS, where a busy machine already holds more + // than a unit may start: fork can fail for that before the sandbox is asked. + // `run program` probes the sandbox itself. + assert!(!sandboxed.starts_with("ok"), "{name}: {sandboxed}"); + return; + } + if name == "run program" || name == "native by another name" { + // Unsandboxed, the first replaces the process and never answers, and the second + // depends on where the interpreter lives: no control to compare. + assert_eq!(sandboxed, "PermissionError", "{name}: {sandboxed}"); + return; + } + assert!( + open.starts_with("ok"), + "{name}: the probe must work unsandboxed, said {open}" + ); + assert!( + !sandboxed.starts_with("ok"), + "{name}: the sandbox let it through: {sandboxed}" + ); +} + +fn allowed(probes: &[(String, String, String)], name: &str) { + let (_, sandboxed, _) = probes.iter().find(|(n, ..)| n == name).unwrap(); + assert!( + sandboxed.starts_with("ok"), + "{name}: the sandbox refused it: {sandboxed}" + ); +} + +#[tokio::test] +async fn test_the_student_side_reaches_only_its_own_directory() { + let bench = Bench::new(); + let home = std::env::var("HOME").unwrap(); + let path = |p: &Path| json!(p.to_string_lossy()); + let mut probes = vec![ + ("bundle", function("read", vec![path(&bench.answers)])), + ( + "bundle listing", + function("listing", vec![path(&bench.bundle)]), + ), + ( + "teacher module", + function("read", vec![path(&bench.teacher)]), + ), + ("other student", function("read", vec![path(&bench.other)])), + ( + "submissions", + function("listing", vec![path(&bench.root.join("submissions"))]), + ), + ("home", function("listing", vec![json!(home)])), + ( + "write beside the bundle", + function("write", vec![path(&bench.bundle.join("planted.py"))]), + ), + ("network", function("connect", vec![json!(bench.port)])), + ("spawn", function("spawn", vec![])), + ("signal", function("signal_grader", vec![])), + ("own directory", function("write", vec![json!("mine.txt")])), + ( + "mode of a bundle file", + function("chmod", vec![path(&bench.answers)]), + ), + ( + "unix socket", + function("unix_socket", vec![path(&bench.socket)]), + ), + ]; + let mut sandboxed_only = vec![("run program", function("run_program", vec![]))]; + if cfg!(target_os = "macos") { + sandboxed_only.push(("native by another name", function("link_native", vec![]))); + } + if cfg!(target_os = "macos") { + // Seatbelt does not mediate reading another process's arguments and environment; + // refusing native calls is what keeps the grader's out of reach. + probes.push(("native calls", function("native", vec![]))); + } + if cfg!(target_os = "linux") { + probes.push(( + "grader's environment", + function("environ_of_grader", vec![]), + )); + } + let names: Vec<&str> = probes.iter().map(|(n, _)| *n).collect(); + let plan = bench.plan(probes.into_iter().map(|(_, c)| c).collect()); + let (sandboxed, open) = both(&plan).await; + let results = compare(&names, &sandboxed, &open); + for name in names.iter().filter(|n| **n != "own directory") { + denied(&results, name); + } + allowed(&results, "own directory"); + + for (name, probe) in sandboxed_only { + let obs = PythonExecutor::new().run(&bench.plan(vec![probe])).await; + denied(&[(name.to_string(), said(&obs, 0), String::new())], name); + } +} + +#[tokio::test] +async fn test_the_teacher_side_reads_the_bundle_and_the_source_and_nothing_else() { + let bench = Bench::new(); + let home = std::env::var("HOME").unwrap(); + let path = |p: &Path| json!(p.to_string_lossy()); + let probes = vec![ + ("bundle", teacher("read", vec![path(&bench.answers)])), + ("other student", teacher("read", vec![path(&bench.other)])), + ( + "submissions", + teacher("listing", vec![path(&bench.root.join("submissions"))]), + ), + ("home", teacher("listing", vec![json!(home)])), + ( + "write the bundle", + teacher("write", vec![path(&bench.bundle.join("planted.py"))]), + ), + ("network", teacher("connect", vec![json!(bench.port)])), + ("spawn", teacher("spawn", vec![])), + ("signal", teacher("signal_grader", vec![])), + ]; + let exec = teacher("run_program", vec![]); + let names: Vec<&str> = probes.iter().map(|(n, _)| *n).collect(); + let plan = bench.plan(probes.into_iter().map(|(_, c)| c).collect()); + let (sandboxed, open) = both(&plan).await; + let results = compare(&names, &sandboxed, &open); + allowed(&results, "bundle"); + for name in names.iter().filter(|n| **n != "bundle") { + denied(&results, name); + } + let obs = PythonExecutor::new().run(&bench.plan(vec![exec])).await; + denied( + &[("run program".into(), said(&obs, 0), String::new())], + "run program", + ); +} + +#[tokio::test] +async fn test_neither_side_sees_the_graders_environment() { + let bench = Bench::new(); + let plan = bench.plan(vec![function("env", vec![]), teacher("env", vec![])]); + let obs = PythonExecutor::new().run(&plan).await; + for index in 0..2 { + let seen = said(&obs, index); + assert!( + !seen.contains("CANVAS") && !seen.contains("CARGO"), + "step {index} saw {seen}" + ); + } +} + +/// A lint tool runs in the teacher's sandbox: it reads the submission's copy and the bundle, +/// and not another student's work. +#[test] +fn test_a_lint_tool_reads_the_bundle_and_the_copy_and_nothing_else() { + use scriptmark_core::models::LintConfig; + use scriptmark_core::runner::linter::run_lint; + let bench = Bench::new(); + let reading = |path: &Path| LintConfig { + // One finding line, if both reads succeed; a failed read exits 1, outside [0]. + command: format!( + "sh -c 'cat \"$1\" > /dev/null && cat \"$2\" > /dev/null && echo finding' sh {{file}} {}", + path.display() + ), + max_warnings: 10, + ok_exit_codes: vec![0], + }; + let lint = |config: &LintConfig, isolation| { + run_lint( + config, + &bench.student, + &bench.bundle, + isolation, + scriptmark_core::runner::linter::LINT_TIMEOUT, + ) + }; + assert_eq!( + lint(&reading(&bench.answers), Isolation::Sandboxed), + Ok(90.0) + ); + assert_eq!( + lint(&reading(&bench.other), Isolation::Unisolated), + Ok(90.0) + ); + let refused = lint(&reading(&bench.other), Isolation::Sandboxed).unwrap_err(); + assert!(refused.contains("exited with"), "{refused}"); +} + +/// A Python checker script runs in the teacher's sandbox too, with a cleared environment. +#[test] +fn test_a_python_checker_script_reads_the_bundle_and_nothing_else() { + use scriptmark_core::checker::python_checker::PythonChecker; + use scriptmark_core::checker::{CheckInput, Checker}; + let bench = Bench::new(); + let script = write( + &bench.bundle, + "verify.py", + &format!( + "import json, os, sys\njson.load(sys.stdin)\ndef can(path):\n try:\n open(path).read()\n return 'yes'\n except OSError:\n return 'no'\nbundle, other = can({:?}), can({:?})\nprint(json.dumps({{'pass': True, 'message': f'bundle={{bundle}} other={{other}} env={{sorted(os.environ)}}'}}))\n", + bench.answers, bench.other + ), + ); + let input = CheckInput { + result: json!(1), + expected: json!(1), + context: json!({}), + }; + let message = |checker: PythonChecker| checker.check(&input).unwrap().message; + let python = PythonExecutor::new().python_cmd().to_string(); + let open = message(PythonChecker::new(&script).with_python_cmd(&python)); + let confined = message( + PythonChecker::new(&script) + .with_python_cmd(&python) + .confined(vec![bench.bundle.clone()]), + ); + assert!(open.starts_with("bundle=yes other=yes"), "{open}"); + assert!(confined.starts_with("bundle=yes other=no"), "{confined}"); + assert!( + !confined.contains("CARGO"), + "the environment is cleared: {confined}" + ); +}