From 99f837cc2e298473402472a9e29b3b5b60198261 Mon Sep 17 00:00:00 2001 From: Acture Date: Thu, 15 Oct 2026 17:03:54 +0800 Subject: [PATCH 01/20] feat(runner): run each unit as a teacher side and a student side (OSS-10) A unit was one Python process holding the harness, the teacher's modules and checkers, and the student's module. It is now two processes joined by a pair of pipes: the teacher side holds the teacher's code, drives the plan, sets a deadline on every request and writes the records; the student side holds only the student's module and answers one request at a time. ScriptMark starts both, connects them, and reads only the teacher side's records. - Records and answers are length-prefixed JSON frames, each checked against the largest frame legal at that point before its body is read. The record stream's bounds come from the plan, so a long scenario of large legal values is never taken for a flood; a frame past its bound ends the stream there. - Values cross by value. Plain data is JSON as before; a transfer encoding keeps tuples, sets, exact numbers, shared references and cycles, rebuilds instances of shared classes from the receiving side's copy, and turns any other object into a snapshot of its attributes and public properties, read under the owner's timer. - A request that runs past its timeout and grace is given up on: the call is reported as timed out and the rest as not run, on the student's account. A student side that exits, floods or garbles its pipe is reported the same way. - ScriptMark watches the student side's memory and ends it past its limit, reported as the student's resource overrun (Cause::Memory). - Records are no longer reachable from student code, so a broken record stream is the harness's, never the student's. --- src/core/src/models/result.rs | 2 + src/core/src/runner/answers.rs | 11 +- src/core/src/runner/executor.rs | 84 ++- src/core/src/runner/harness.py | 723 ------------------------- src/core/src/runner/harness/common.py | 717 ++++++++++++++++++++++++ src/core/src/runner/harness/student.py | 369 +++++++++++++ src/core/src/runner/harness/teacher.py | 687 +++++++++++++++++++++++ src/core/src/runner/judge.rs | 304 +++++++++-- src/core/src/runner/python.rs | 711 ++++++++++++++++-------- src/core/src/runner/records.rs | 250 ++++++--- src/core/tests/harness.rs | 216 +++++++- src/core/tests/integration.rs | 6 +- 12 files changed, 2956 insertions(+), 1124 deletions(-) delete mode 100644 src/core/src/runner/harness.py create mode 100644 src/core/src/runner/harness/common.py create mode 100644 src/core/src/runner/harness/student.py create mode 100644 src/core/src/runner/harness/teacher.py diff --git a/src/core/src/models/result.rs b/src/core/src/models/result.rs index 8bedded..57cd9b4 100644 --- a/src/core/src/models/result.rs +++ b/src/core/src/models/result.rs @@ -53,6 +53,8 @@ pub enum Cause { Timeout, /// The process died, or was killed at its deadline, during this call. Killed, + /// The student side went past its memory limit during this call, and was ended. + Memory, /// Not run: an earlier call in the unit hung or killed the process. NotRun, /// Not run: a value it needs was never produced. diff --git a/src/core/src/runner/answers.rs b/src/core/src/runner/answers.rs index 9ec564c..dfc892a 100644 --- a/src/core/src/runner/answers.rs +++ b/src/core/src/runner/answers.rs @@ -98,11 +98,14 @@ impl Contract { runtime: executor.identity()?, sources, protocol: digest( - concat!( + [ include_str!("oracle.rs"), - include_str!("harness.py"), - include_str!("../checker/rhai_checker.rs") - ) + include_str!("../checker/rhai_checker.rs"), + ] + .iter() + .chain(&crate::runner::python::PROTOCOL) + .copied() + .collect::() .as_bytes(), ), }) diff --git a/src/core/src/runner/executor.rs b/src/core/src/runner/executor.rs index 330fc84..32b00ea 100644 --- a/src/core/src/runner/executor.rs +++ b/src/core/src/runner/executor.rs @@ -61,22 +61,31 @@ pub struct UnitPlan { } impl UnitPlan { - /// Seconds the unit may run in total: every call's own timeout, plus slack. The + /// Seconds the unit may run in total: every request the teacher side makes with the + /// grace it allows each, plus the time both sides have to start. The teacher side gives + /// up on a request well before this; it is the backstop for the teacher side itself. The /// kernel's CPU limit sits just above it as a backstop for a single busy core. pub fn deadline_secs(&self) -> u64 { + let grace = |timeout: u64| timeout.saturating_add(GRACE_SECS); let calls = match &self.script { - Some(script) => script.timeout, + Some(script) => grace(script.timeout), None => self .setup .iter() .chain(&self.steps) - .map(CallPlan::budget) - .fold(self.load_timeout, u64::saturating_add), + .map(|c| c.budget(grace)) + .fold(grace(self.load_timeout), u64::saturating_add), }; - calls.saturating_add(2) + calls.saturating_add(2 * STARTUP_SECS) } } +/// Seconds the teacher side waits past a request's own timeout before it gives up on the +/// student side: the student's own timer has fired by then, and its answer is late. +pub const GRACE_SECS: u64 = 2; +/// Seconds each side has to start: an interpreter under load, and the teacher modules. +pub const STARTUP_SECS: u64 = 10; + /// A script run: the unit's only call. #[derive(Debug, Clone, Serialize)] pub struct ScriptRun { @@ -101,9 +110,9 @@ pub struct CallPlan { } impl CallPlan { - fn budget(&self) -> u64 { - self.timeout - .saturating_add(self.check.as_ref().map_or(0, |_| self.timeout)) + /// The call's time, and its checker's, each with `grace` applied. + fn budget(&self, grace: impl Fn(u64) -> u64) -> u64 { + grace(self.timeout).saturating_add(self.check.as_ref().map_or(0, |_| grace(self.timeout))) } } @@ -245,7 +254,7 @@ pub enum CheckObservation { Error(ErrorInfo), } -/// How the unit's process ended. +/// How one side's process ended. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum Exit { @@ -253,25 +262,41 @@ pub enum Exit { Signal(i32), /// Killed by the grader at the unit's deadline. Deadline, + /// Killed by the grader for going past the unit's memory limit. + Memory, + /// Still running when the teacher side finished, and ended by the grader then. + Stopped, /// Never started. Spawn(String), } +impl Exit { + /// Ended by itself, without an error. + pub fn is_clean(&self) -> bool { + *self == Exit::Code(0) + } +} + /// Everything a unit reported, in plan order. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct UnitObservation { - /// Teacher modules imported and the harness running. + /// Teacher modules imported, and the student side started and ready. pub ready: bool, pub load: Option, pub setup: Vec, pub steps: Vec, pub checks: BTreeMap, pub fatal: Option, + /// The teacher side gave up on the student side: nothing after this call ran. + pub stopped: Option, pub done: bool, + /// How the student side ended. pub exit: Exit, - /// The record stream broke: the unit's records cannot be trusted past this point. + /// How the teacher side ended. + pub teacher_exit: Exit, + /// The teacher side's record stream broke: its records cannot be trusted past here. pub protocol_error: Option, - /// The tail of stderr, for diagnosing a crash. + /// The tail of each side's stderr, for diagnosing a crash. pub stderr: String, } @@ -285,7 +310,9 @@ impl UnitObservation { steps: Vec::new(), checks: BTreeMap::new(), fatal: None, + stopped: None, done: false, + teacher_exit: exit.clone(), exit, protocol_error: None, stderr: String::new(), @@ -293,16 +320,39 @@ impl UnitObservation { } } -/// How the record stream broke. +/// Where and why the teacher side stopped asking the student side anything. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Stopped { + pub phase: Phase, + pub index: usize, + pub reason: StopReason, + #[serde(default)] + pub message: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum StopReason { + /// A request ran past its timeout and grace: the student's timer did not stop it. + Deadline, + /// The student side went away mid-request. + Exited, + /// The student side sent more than the request allows. + Flooded, + /// The student side sent something no request allows. + Garbled, +} + +/// How the teacher side's record stream broke. Student code cannot reach it, so each is +/// the harness's or the teacher code's doing. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ProtocolError { - /// A framed record did not parse — the harness wrote something it should not have. + /// A framed record did not parse. Unreadable(String), - /// A record repeated or came out of order — somebody else wrote it. + /// A record repeated or came out of order. Tampered(String), - /// More reached the record channel than the harness ever writes: something other than - /// the harness wrote to it. + /// A record longer than any this unit can produce, or more records than it has. Flooded(String), } diff --git a/src/core/src/runner/harness.py b/src/core/src/runner/harness.py deleted file mode 100644 index 4b4e223..0000000 --- a/src/core/src/runner/harness.py +++ /dev/null @@ -1,723 +0,0 @@ -"""ScriptMark's Python harness: runs one execution unit and reports what happened. - -It reports observations, never verdicts about who is at fault: the grader derives -those from which call was running, what kind of code it runs, and how the process -ended. Every piece of student code — loading the module, finding a function, reading -an attribute, serialising a value — runs inside `call()`, under its timer and import -guard, so a failure there is that call's outcome and never a harness crash. - -The nonce framing protects the records from accidental output only; a student -determined to reach harness state from inside the same process could, which is why -nothing here decides anything. -""" - -import sys - -# The unit's working directory holds the student's file, and `python -c` puts it first -# on sys.path: a submission named json.py must not become the harness's json. -sys.path[:] = [p for p in sys.path if p] - -import builtins # noqa: E402 -import importlib.machinery # noqa: E402 -import importlib.util # noqa: E402 -import inspect # noqa: E402 -import io # noqa: E402 -import json # noqa: E402 -import math # noqa: E402 -import os # noqa: E402 -import runpy # noqa: E402 -import signal # noqa: E402 -import time # noqa: E402 -from typing import TYPE_CHECKING # noqa: E402 - -if TYPE_CHECKING: - from matching import FunctionRules, MatchDecision, function_decision - -with open(sys.argv[1], encoding="utf-8") as _fh: - PAYLOAD = json.load(_fh) -os.unlink(sys.argv[1]) - -_PREFIX = f"\n@@scriptmark:{PAYLOAD['nonce']}@@ " -# The record channel: a descriptor of its own where the grader provides one (fd 3 on unix), -# which no student output setting can reach; otherwise a private copy of stdout, where the -# nonce framing keeps accidental output out. `replace`: a lone surrogate a student printed or -# returned must not crash it. -_CHANNEL_FD = PAYLOAD.get("channel", 1) -_channel = os.fdopen( - _CHANNEL_FD if _CHANNEL_FD != 1 else os.dup(1), "w", encoding="utf-8", errors="replace" -) -# Between calls, stdout is a sink: what a student's leftover thread prints is not evidence. -_quiet = open(os.devnull, "w", encoding="utf-8") -sys.stdout = _quiet -_real_stdin = sys.stdin -STUDENT = PAYLOAD.get("subject", "student") == "student" -# What one call's record can carry; python.rs sizes the record channel from these. -STDOUT_LIMIT = 64 * 1024 -FILE_LIMIT = 1024 * 1024 -VALUE_LIMIT = 4 * 1024 * 1024 -MESSAGE_LIMIT = 64 * 1024 -DEPTH_LIMIT = 100 - - -def emit(record): - _channel.write(_PREFIX + json.dumps(record, ensure_ascii=False) + "\n") - _channel.flush() - - -def finish(): - """Stop now: threads a student left running must not hold the unit open.""" - emit({"kind": "done"}) - _channel.close() - os._exit(0) - - -# --- The import guard ----------------------------------------------------------------- -# A policy check, not a security boundary: importlib.import_module bypasses it. It looks -# at where the import statement runs, so an allowlisted stdlib module importing its own -# internals is never mistaken for the student importing them. -ALLOWED = frozenset( - { - # core - "builtins", "sys", "io", "abc", "enum", "types", "typing", "dataclasses", - "collections", "functools", "itertools", "operator", "copy", "copyreg", - # math/science - "math", "cmath", "decimal", "fractions", "numbers", "random", "statistics", - # strings/text - "string", "re", "difflib", "textwrap", "unicodedata", "codecs", "encodings", - # data formats - "json", "csv", "struct", "base64", "binascii", "hashlib", "hmac", - # time - "time", "datetime", "calendar", "zoneinfo", - # data structures - "array", "heapq", "bisect", "queue", "graphlib", - # functional - "contextlib", "contextvars", - # introspection - "inspect", "dis", "token", "tokenize", "ast", "symtable", - # error handling - "traceback", "warnings", "errno", "faulthandler", - # internals needed by Python itself - "importlib", "_thread", "__future__", "_collections_abc", "_abc", - "_functools", "_operator", "_io", "_json", "_string", "_struct", - "_sre", "sre_compile", "sre_constants", "sre_parse", "_locale", - "_frozen_importlib", "_frozen_importlib_external", "_imp", - "_warnings", "_weakref", "_weakrefset", "_threading_local", - "_codecs", "_signal", "_stat", "_opcode", "_typing", - "genericpath", "posixpath", "ntpath", "stat", "keyword", - "reprlib", "pprint", "linecache", - # testing (students may use unittest/doctest) - "unittest", "doctest", - # filesystem (pathlib is safe — open() is a builtin anyway) - "pathlib", "fnmatch", "glob", "tempfile", - # misc safe - "secrets", "uuid", "ipaddress", "colorsys", "timeit", "py_compile", "compileall", - "gzip", "bz2", "lzma", "zipfile", "tarfile", "xml", "html", "email", - "logging", "argparse", "getopt", "configparser", "tomllib", "threading", "concurrent", - } -) | frozenset(PAYLOAD.get("allowed_imports", [])) -STUDENT_MARK = "__scriptmark_student__" -_original_import = builtins.__import__ - - -def _guarded_import(name, globals=None, locals=None, fromlist=(), level=0): - # `__import__('os')` called directly passes no globals: look at the caller instead. - scope = globals if globals is not None else sys._getframe(1).f_globals - if scope.get(STUDENT_MARK) and level == 0: - if name.split(".")[0] not in ALLOWED: - raise ImportError(f"Module '{name}' is not allowed in student code") - return _original_import(name, globals, locals, fromlist, level) - - -# --- Timers --------------------------------------------------------------------------- -class CallTimeout(BaseException): - """Raised by SIGALRM. A BaseException, so `except Exception` cannot swallow it.""" - - -_fired = [False] - - -def _on_alarm(signum, frame): - _fired[0] = True - raise CallTimeout() - - -_HAS_TIMER = hasattr(signal, "setitimer") -if _HAS_TIMER: - signal.signal(signal.SIGALRM, _on_alarm) - - -class _CappedBytes(io.BytesIO): - """Keeps at most STDOUT_LIMIT bytes and remembers whether it dropped any.""" - - truncated = False - - def write(self, b): - room = STDOUT_LIMIT - self.tell() - if len(b) > room: - self.truncated = True - if room > 0: - super().write(bytes(b[:room])) - return len(b) - - def writelines(self, lines): - # The C method would write past the limit without calling `write`. - for line in lines: - self.write(line) - - def close(self): - """A student's wrapper around `sys.stdout.buffer` closes it when it is collected: - what was written must survive that.""" - - -def capture(): - """A stdout that behaves like a real one — `buffer`, `reconfigure`, `print(flush=True)` — - and the bytes behind it, kept apart so a detached or closed wrapper cannot lose them.""" - raw = _CappedBytes() - return io.TextIOWrapper(raw, encoding="utf-8", errors="backslashreplace", write_through=True), raw - - -def captured(stream, raw): - try: - stream.flush() - except (ValueError, OSError): # detached or closed by the student - pass - # `io.BytesIO.write(buffer, b)` still reaches past `write`: cut here too. - data = raw.getvalue() - return data[:STDOUT_LIMIT].decode("utf-8", "replace"), raw.truncated or len(data) > STDOUT_LIMIT - - -# --- The one value serialiser ---------------------------------------------------------- -class Unserialisable(Exception): - pass - - -def to_json(value, depth=0, seen=None): - """What the grader sees of a Python value. Total: it never raises for a value's sake.""" - if isinstance(value, bool) or value is None or isinstance(value, str): - return value - if isinstance(value, int): - if -(2**63) <= value < 2**64: - return int(value) - # Past what the grader's JSON can hold exactly: tag it, exactly and with its sign, so - # it is never mistaken for a number and a record always parses. Decimal where Python - # will print it, hexadecimal past its 4300-digit limit. - try: - return {"$bigint": int.__repr__(value)} - except ValueError: - return {"$bigint": hex(value)} - if isinstance(value, float): - return value if math.isfinite(value) else repr(value) - if depth >= DEPTH_LIMIT: - return "" - seen = seen or set() - if id(value) in seen: - return "" - if isinstance(value, (list, tuple, set, frozenset, dict)): - seen = seen | {id(value)} - if isinstance(value, (list, tuple)): - return [to_json(v, depth + 1, seen) for v in value] - if isinstance(value, (set, frozenset)): - items = [to_json(v, depth + 1, seen) for v in value] - try: - return sorted(items) - except TypeError: # mixed types: any stable order will do - return sorted(items, key=lambda v: json.dumps(v, sort_keys=True)) - if isinstance(value, dict): - out = {} - for k, v in value.items(): - key = k if isinstance(k, str) else text_of(k) - if key in out: - raise Unserialisable(f"two keys both read as '{key}'") - out[key] = to_json(v, depth + 1, seen) - return out - return text_of(value) - - -def text_of(value): - """`str(value)`, or the type's name when the value's own `__str__` fails.""" - try: - text = str(value) - except BaseException: # never under a live timer: this is harness bookkeeping - return f"<{type(value).__name__}>" - return text if isinstance(text, str) else f"<{type(value).__name__}>" - - -def clip(message): - """A message kept to MESSAGE_LIMIT characters, saying how long it was.""" - if len(message) <= MESSAGE_LIMIT: - return message - return f"{message[:MESSAGE_LIMIT]}… (cut from {len(message)} characters)" - - -def error_of(exc): - info = { - "type": type(exc).__name__, - "types": [cls.__name__ for cls in type(exc).__mro__], - "message": clip(text_of(exc)), - } - if isinstance(exc, SystemExit): - # The exit status CPython would report: `exit()`, `exit(0)` and `exit(False)` succeed; - # any other code, `exit(0.0)` included, fails. - try: - code = exc.code - info["clean"] = code is None or (type(code) in (int, bool) and int(code) == 0) - except BaseException: - info["clean"] = False - return info - - -def stdin_stream(text): - return io.TextIOWrapper(io.BytesIO((text or "").encode("utf-8")), encoding="utf-8") - - -class Missing(Exception): - """The call's target does not exist.""" - - -class MatchingRequired(Exception): - def __init__(self, decision: MatchDecision) -> None: - self.decision: MatchDecision = decision - super().__init__(f"function matching needs review: {decision}") - - -class Unresolved(Exception): - """A `$ref` or `object` whose value was never produced in this unit.""" - - -def call(fn, timeout, stdin=None, guarded=True, serialise=True): - """Run one call — its lookup, its body and its serialisation — under its timer and guard. - - Returns (outcome, stdout, truncated, elapsed_ms, live_value).""" - out, raw = capture() - sys.stdout = out - sys.stdin = stdin_stream(stdin) if stdin is not False else _real_stdin - if guarded: - builtins.__import__ = _guarded_import - _fired[0] = False - start = time.perf_counter() - value = None - try: - if _HAS_TIMER and timeout: - signal.setitimer(signal.ITIMER_REAL, timeout, 0.05) - try: - value = fn() - if sys.stdout is not out: - # The student replaced stdout with their own wrapper: flush it in their scope. - sys.stdout.flush() - outcome = {"returned": {"value": wire(value) if serialise else None, "type": type(value).__name__}} - finally: - if _HAS_TIMER: - signal.setitimer(signal.ITIMER_REAL, 0) - except CallTimeout: - outcome = {"timeout": {}} - except Missing as exc: - outcome = {"missing": {"message": str(exc)}} - except MatchingRequired as exc: - outcome = {"matching": {"message": str(exc), "decision": exc.decision}} - except Unresolved as exc: - outcome = {"unresolved": {"name": str(exc)}} - except Unserialisable as exc: - outcome = {"unserialisable": {"type": "Unserialisable", "message": str(exc)}} - except BaseException as exc: # SystemExit and KeyboardInterrupt are the code's own too - outcome = {"raised": error_of(exc)} - finally: - builtins.__import__ = _original_import - sys.stdout = _quiet - sys.stdin = _real_stdin - if _fired[0]: - # The timer went off even if the code swallowed it with a bare `except:`. - outcome = {"timeout": {}} - elapsed = int((time.perf_counter() - start) * 1000) - stdout, truncated = captured(out, raw) - return outcome, stdout, truncated, elapsed, value - - -def utf8_size(text): - """The bytes `text` takes on the channel, counted a slice at a time rather than through - a second full copy of a value that may already fill most of the memory limit.""" - step = 1 << 20 - return sum(len(text[i : i + step].encode("utf-8", "replace")) for i in range(0, len(text), step)) - - -def wire(value): - """The value as a record carries it. One too large to report is replaced by its size: - the call still returned, and a function checker still judges the live value.""" - value = to_json(value) - size = utf8_size(json.dumps(value, ensure_ascii=False)) - if size > VALUE_LIMIT: - return {"$too_large": f"{size} bytes of JSON"} - return value - - -# --- Teacher modules ------------------------------------------------------------------- -def _removed_checker(*args, **kwargs): - raise RuntimeError( - "@checker no longer binds a checker to a function: put " - 'check = { function = "" } on each case it should judge ' - "(https://github.com/Acture/obsidian-vault/blob/project/scriptmark/" - "docs/test-bundles.md#migrating-an-older-spec)" - ) - - -def exports_of(module): - """`__all__` when present; otherwise public names, minus modules and minus classes and - functions the module merely imported from elsewhere.""" - names = getattr(module, "__all__", None) - if names is None: - names = [] - for name in dir(module): - if name.startswith("_"): - continue - value = getattr(module, name) - if inspect.ismodule(value): - continue - if (inspect.isclass(value) or inspect.isroutine(value)) and getattr( - value, "__module__", None - ) != module.__name__: - continue - names.append(name) - return {name: getattr(module, name) for name in names} - - -def load_teacher_modules(): - """Returns (exports, duplicates): a name two modules export differently is refused.""" - exports, owner, duplicates = {}, {}, {} - builtins.checker = _removed_checker - try: - for index, path in enumerate(PAYLOAD.get("imports", [])): - directory = os.path.dirname(path) - if directory not in sys.path: - # After the stdlib: a helper beside the module must not shadow it. - sys.path.append(directory) - spec = importlib.util.spec_from_file_location(f"teacher_mod_{index}", path) - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - for name, value in exports_of(module).items(): - if name in exports and not _same(exports[name], value): - duplicates[name] = [owner[name], path] - exports[name] = value - owner.setdefault(name, path) - finally: - del builtins.checker - return exports, duplicates - - -def _same(a, b): - """One object, or two equal ones: two modules that both define `TOL = 1e-6` agree.""" - if a is b: - return True - try: - return bool(a == b) - except BaseException: - return False - - -def params_of(fn): - """Each parameter's name, kind, and whether it has a default.""" - try: - parameters = inspect.signature(fn).parameters.values() - except (TypeError, ValueError): - return None - return [{"name": p.name, "kind": p.kind.name.lower(), "default": p.default is not p.empty} for p in parameters] - - -# --- Calls ----------------------------------------------------------------------------- -context = {} - - -def resolve(value): - if isinstance(value, str) and value.startswith("$"): - if value.startswith("$$"): - return value[1:] - if value[1:] not in context: - raise Unresolved(value[1:]) - return context[value[1:]] - if isinstance(value, list): - return [resolve(v) for v in value] - if isinstance(value, dict): - return {k: resolve(v) for k, v in value.items()} - return value - - -def matched_function( - module: object, name: str, evidence: list[MatchDecision | None] -) -> tuple[object, str]: - policy: FunctionRules = PAYLOAD.get("functions", {"aliases": {}, "overrides": {}}) - # Read attributes only once: dynamic exports/properties belong to the guarded call. - exports: dict[str, object] = dict(vars(module)) - absent: object = object() - for candidate in { - name, - *policy.get("aliases", {}).get(name, []), - *([policy["overrides"][name]] if name in policy.get("overrides", {}) else []), - }: - if candidate not in exports: - value: object = getattr(module, candidate, absent) - if value is not absent: - exports[candidate] = value - decision: MatchDecision = function_decision( - [candidate for candidate, obj in exports.items() if callable(obj)], name, policy - ) - evidence[0] = decision - selected: str | None = decision["selected"] - if selected is not None: - return exports[selected], selected - if ( - decision["state"] != "missing" - or name in policy.get("overrides", {}) - or name in policy.get("aliases", {}) - ): - raise MatchingRequired(decision) - raise Missing(f"function '{name}' not found") - - -def live(object_id): - if object_id not in context: - raise Unresolved(object_id) - return context[object_id] - - -def member(obj, object_id, name, what): - """`getattr(obj, name)`, read once. A name that does not exist is Missing; a property - that raises — AttributeError included — is the student's exception.""" - try: - return getattr(obj, name) - except AttributeError: - try: - inspect.getattr_static(obj, name) - except AttributeError: - if not hasattr(type(obj), "__getattr__"): - raise Missing(f"'{object_id}' has no {what} '{name}'") from None - raise - - -def invoke( - plan: dict[str, object], - student: object, - resolved: list[str], - evidence: list[MatchDecision | None], -) -> object: - """The call a plan names. Runs inside `call()`: the lookup is student code too.""" - kind, spec = next(iter(plan["target"].items())) - args = resolve(plan.get("args", [])) - if kind == "function": - if PAYLOAD.get("lookup") == "exact": - if not hasattr(student, spec["name"]): - raise Missing(f"function '{spec['name']}' not found") - fn, resolved[0] = getattr(student, spec["name"]), spec["name"] - else: - fn, resolved[0] = matched_function(student, spec["name"], evidence) - return fn(*args) - if kind == "method": - fn = member(live(spec["object"]), spec["object"], spec["name"], "method") - if not callable(fn): - raise Missing(f"'{spec['object']}.{spec['name']}' is not a method") - return fn(*args) - if kind == "attribute": - return member(live(spec["object"]), spec["object"], spec["name"], "attribute") - fn = context.get(spec["name"]) - if fn is None or not callable(fn): - raise Missing(f"teacher function '{spec['name']}' not found") - return fn(*args) - - -def verdict_of(returned): - if isinstance(returned, bool): - return {"verdict": {"pass": returned, "message": ""}} - if ( - isinstance(returned, tuple) - and len(returned) == 2 - and isinstance(returned[0], bool) - and isinstance(returned[1], (str, type(None))) - ): - return {"verdict": {"pass": returned[0], "message": clip(returned[1] or "")}} - return { - "error": { - "type": "CheckerContract", - "types": ["CheckerContract"], - "message": f"checker returned {type(returned).__name__}; expected bool or (bool, str)", - } - } - - -def run_check(check, value, stdout, timeout): - fn = context.get(check["function"]) - deps = {} - for param in (params_of(fn) or [])[2:]: - name = param["name"] - if "var" in param["kind"]: - continue - if name == "stdout": - deps[name] = stdout - elif name in context: - deps[name] = context[name] - elif not param["default"]: - # prepare saw this name in scope: it is an id whose producing call failed. - return {"unresolved": {"name": name}} - outcome, _, _, _, returned = call( - lambda: fn(value, check.get("expected"), **deps), timeout, guarded=False, serialise=False - ) - if "returned" in outcome: - return verdict_of(returned) - if "timeout" in outcome: - return {"error": {"type": "Timeout", "types": ["Timeout"], "message": f"checker timed out after {timeout}s"}} - raised = outcome.get("raised") or {"type": "Error", "types": [], "message": str(outcome)} - if "AssertionError" in raised["types"]: - return {"rejected": {"message": raised["message"]}} - return {"error": raised} - - -def observe_files(paths: list[str]) -> dict[str, str | None]: - seen: dict[str, str | None] = {} - for path in paths: - try: - with open(path, encoding="utf-8") as fh: - content: str = fh.read(FILE_LIMIT + 1) - seen[path] = content if len(content) <= FILE_LIMIT else None - except ( - OSError, - ValueError, - ): # absent, oversized or invalid UTF-8 is no text expectation - seen[path] = None - return seen - - -def run_calls(phase, calls, student): - """Run setup calls or steps in order. Returns False when a setup call failed.""" - setup = phase == "setup" - for index, plan in enumerate(calls): - requested = next(iter(plan["target"].values()))["name"] - resolved = [requested] - evidence: list[MatchDecision | None] = [None] - outcome, stdout, truncated, elapsed, value = call( - lambda: invoke(plan, student, resolved, evidence), - plan["timeout"], - plan.get("stdin"), - guarded=STUDENT and "teacher" not in plan["target"], - serialise=not setup, # setup values are never judged, only passed on - ) - record = { - "kind": "call", - "phase": phase, - "index": index, - "target": {"requested": requested, "resolved": resolved[0], "matching": evidence[0]}, - "outcome": outcome, - "stdout": stdout, - "stdout_truncated": truncated, - "elapsed_ms": elapsed, - } - if plan.get("files"): - record["files"] = observe_files(plan["files"]) - emit(record) - returned = "returned" in outcome - if returned and plan.get("id"): - context[plan["id"]] = value - if returned and plan.get("check"): - verdict = run_check(plan["check"], value, stdout, plan["timeout"]) - emit({"kind": "check", "index": index, **verdict}) - if setup and not returned: - return False - return True - - -def compiles(path): - """Compile the student's file in memory — writing no __pycache__ beside it — and record a - syntax error as the load call's outcome.""" - try: - with open(path, "rb") as fh: - compile(fh.read(), path, "exec") - return True - except (SyntaxError, ValueError, RecursionError, MemoryError) as exc: - emit({"kind": "call", "phase": "load", "index": 0, - "outcome": {"raised": {"type": "SyntaxError", "types": ["SyntaxError"], "message": text_of(exc)}}, - "stdout": "", "stdout_truncated": False, "elapsed_ms": 0}) - return False - - -def load_student(): - path = PAYLOAD["student"] - if not compiles(path): - return None - module_holder = [] - - def load(): - # An explicit loader: `Lab5.PY` is Python too, whatever its extension. - loader = importlib.machinery.SourceFileLoader("student_mod", path) - spec = importlib.util.spec_from_loader("student_mod", loader) - module = importlib.util.module_from_spec(spec) - if STUDENT: - setattr(module, STUDENT_MARK, True) - for key, val in PAYLOAD.get("vars", {}).items(): - setattr(module, key, val) - sys.modules["student_mod"] = module - loader.exec_module(module) - module_holder.append(module) - - original_input = builtins.input - builtins.input = lambda *a, **k: "0" - try: - outcome, stdout, truncated, elapsed, _ = call( - load, PAYLOAD["load_timeout"], guarded=STUDENT, serialise=False - ) - finally: - builtins.input = original_input - emit({"kind": "call", "phase": "load", "index": 0, "outcome": outcome, - "stdout": stdout, "stdout_truncated": truncated, "elapsed_ms": elapsed}) - return module_holder[0] if "returned" in outcome else None - - -def run_script(): - script, path = PAYLOAD["script"], PAYLOAD["student"] - if not compiles(path): - return - init = {STUDENT_MARK: True, **PAYLOAD.get("vars", {})} - sys.argv = [path] # as `python student.py` would see it, not the harness's payload - outcome, stdout, truncated, elapsed, _ = call( - lambda: runpy.run_path(path, init_globals=init, run_name="__main__"), - script["timeout"], - stdin=False, - serialise=False, - ) - raised = outcome.get("raised") - if raised and raised["type"] == "SystemExit" and raised.get("clean"): - outcome = {"returned": {"value": None, "type": "NoneType"}} - record = {"kind": "call", "phase": "step", "index": 0, "outcome": outcome, - "stdout": stdout, "stdout_truncated": truncated, "elapsed_ms": elapsed} - if script.get("files"): - record["files"] = observe_files(script["files"]) - emit(record) - - -def main(): - try: - exports, duplicates = load_teacher_modules() - except BaseException as exc: - emit({"kind": "fatal", "stage": "teacher_import", "error": error_of(exc)}) - finish() - # Student code may import helpers staged beside it: after the stdlib and the teacher - # modules' directories, so a submission never stands in for what either imports. - sys.path.append(os.getcwd()) - if PAYLOAD["mode"] == "inspect": - emit({"kind": "inspect", "duplicates": duplicates, "exports": { - name: {"callable": callable(value), "params": params_of(value) if callable(value) else None} - for name, value in exports.items() - }}) - finish() - context.update(exports) - context.update(PAYLOAD.get("vars", {})) - emit({"kind": "ready"}) - if PAYLOAD.get("script"): - run_script() - finish() - student = load_student() - if student is not None and run_calls("setup", PAYLOAD.get("setup", []), student): - run_calls("step", PAYLOAD.get("steps", []), student) - finish() - - -try: - main() -except BaseException as exc: # a harness bug, not anybody's code under test - emit({"kind": "fatal", "stage": "harness", "error": error_of(exc)}) - finish() diff --git a/src/core/src/runner/harness/common.py b/src/core/src/runner/harness/common.py new file mode 100644 index 0000000..1673911 --- /dev/null +++ b/src/core/src/runner/harness/common.py @@ -0,0 +1,717 @@ +"""What both sides of a unit share: framing, the two value encodings, and one call under +its timer. + +A unit runs as two processes. The teacher side (`teacher.py`) holds the teacher's modules +and checkers, drives the plan and writes the records ScriptMark reads; the student side +(`student.py`) holds only the student's module and runs the calls the teacher side asks +for. They talk over one pair of pipes, a frame at a time: a 4-byte big-endian length, then +that many bytes of JSON. Every frame a side reads is checked against the largest frame +that side could legitimately be sent at that point, before its body is read. + +Two encodings cross the pipes: + +- the *record* encoding (`to_json`, `wire`) is what the grader judges and keeps as + evidence: plain JSON, lossy where JSON is (sets sorted, tuples as lists, objects as + their text); +- the *transfer* encoding (`encode`, `decode`) rebuilds a value on the other side for code + that runs there: a teacher checker judging a student's value, or a student call taking a + teacher's. It keeps tuples, sets, exact numbers, shared references and cycles. An + instance of a class from a shared module is rebuilt from the receiving side's own copy + of the class, without `__init__`; any other object arrives as a `Snapshot` of its + attributes and public properties, read on the side that owns it. +""" + +import sys + +# The unit's working directory holds the student's file, and `python -c` puts it first on +# sys.path: a submission named json.py must not become the harness's json. +sys.path[:] = [p for p in sys.path if p] + +import base64 # noqa: E402 +import builtins # noqa: E402 +import datetime # noqa: E402 +import decimal # noqa: E402 +import fractions # noqa: E402 +import inspect # noqa: E402 +import io # noqa: E402 +import json # noqa: E402 +import math # noqa: E402 +import os # noqa: E402 +import signal # noqa: E402 +import struct # noqa: E402 +import time # noqa: E402 + +HEADER = struct.Struct(">I") +# The record limits, set from the grader's own numbers before anything runs (`configure`). +STDOUT_LIMIT = 64 * 1024 +FILE_LIMIT = 1024 * 1024 +VALUE_LIMIT = 4 * 1024 * 1024 +MESSAGE_LIMIT = 64 * 1024 +TRANSFER_LIMIT = 32 * 1024 * 1024 +DEPTH_LIMIT = 100 +# The longest integer the transfer encoding carries, in hexadecimal digits. +INT_DIGITS = 1 << 20 + + +def configure(limits): + global STDOUT_LIMIT, FILE_LIMIT, VALUE_LIMIT, MESSAGE_LIMIT, TRANSFER_LIMIT, DEPTH_LIMIT + STDOUT_LIMIT = limits["stdout"] + FILE_LIMIT = limits["file"] + VALUE_LIMIT = limits["value"] + MESSAGE_LIMIT = limits["message"] + TRANSFER_LIMIT = limits["transfer"] + DEPTH_LIMIT = limits["depth"] + + +def channel(read_fd, write_fd): + """Move this process's pipe to the other side off descriptors 0 and 1, which then read + nothing and write nowhere: no print, `sys.__stdout__` or `os.write(1, …)` reaches it.""" + read_fd, write_fd = os.dup(read_fd), os.dup(write_fd) + null = os.open(os.devnull, os.O_RDWR) + os.dup2(null, 0) + os.dup2(null, 1) + os.close(null) + return read_fd, write_fd + + +# --- Framing ---------------------------------------------------------------------------- +class Closed(Exception): + """The other side closed its end, or went away mid-frame.""" + + +class Oversized(Exception): + """A frame announced more bytes than may arrive at this point.""" + + +class Garbled(Exception): + """A frame that is not what this point of the protocol expects.""" + + +def frame(obj): + # `replace`: a lone surrogate a student printed or returned must not stop the frame. + body = json.dumps(obj, ensure_ascii=False, separators=(",", ":")).encode("utf-8", "replace") + return HEADER.pack(len(body)) + body + + +def write_all(fd, data): + view = memoryview(data) + while view: + view = view[os.write(fd, view) :] + + +def send(fd, obj): + write_all(fd, frame(obj)) + + +def read_exact(fd, n): + chunks, got = [], 0 + while got < n: + chunk = os.read(fd, min(n - got, 1 << 20)) + if not chunk: + raise Closed() + chunks.append(chunk) + got += len(chunk) + return b"".join(chunks) + + +def receive(fd, bound): + """One frame, refused before its body is read when it announces more than `bound`.""" + (size,) = HEADER.unpack(read_exact(fd, HEADER.size)) + if size > bound: + raise Oversized(f"a frame of {size} bytes where at most {bound} may arrive") + return parse(read_exact(fd, size)) + + +def parse(body): + try: + return json.loads(body.decode("utf-8")) + except (ValueError, RecursionError) as exc: # bad UTF-8, bad JSON, nested past the parser + raise Garbled(f"not a frame: {text_of(exc)}") from None + + +# --- Reporting ---------------------------------------------------------------------------- +def text_of(value): + """`str(value)`, or the type's name when the value's own `__str__` fails.""" + try: + text = str(value) + except BaseException: # never under a live timer: this is harness bookkeeping + return f"<{type(value).__name__}>" + return text if isinstance(text, str) else f"<{type(value).__name__}>" + + +def clip(message): + """A message kept to MESSAGE_LIMIT characters, saying how long it was.""" + if len(message) <= MESSAGE_LIMIT: + return message + return f"{message[:MESSAGE_LIMIT]}… (cut from {len(message)} characters)" + + +def error_of(exc): + info = { + "type": type(exc).__name__, + "types": [cls.__name__ for cls in type(exc).__mro__], + "message": clip(text_of(exc)), + } + if isinstance(exc, SystemExit): + # The exit status CPython would report: `exit()`, `exit(0)` and `exit(False)` succeed; + # any other code, `exit(0.0)` included, fails. + try: + code = exc.code + info["clean"] = code is None or (type(code) in (int, bool) and int(code) == 0) + except BaseException: + info["clean"] = False + return info + + +def problem(kind, message): + return {"type": kind, "types": [kind], "message": clip(message)} + + +# --- Timers --------------------------------------------------------------------------- +class CallTimeout(BaseException): + """Raised by SIGALRM. A BaseException, so `except Exception` cannot swallow it.""" + + +_fired = [False] + + +def _on_alarm(signum, frame): + _fired[0] = True + raise CallTimeout() + + +_HAS_TIMER = hasattr(signal, "setitimer") +if _HAS_TIMER: + signal.signal(signal.SIGALRM, _on_alarm) + + +class _CappedBytes(io.BytesIO): + """Keeps at most STDOUT_LIMIT bytes and remembers whether it dropped any.""" + + truncated = False + + def write(self, b): + room = STDOUT_LIMIT - self.tell() + if len(b) > room: + self.truncated = True + if room > 0: + super().write(bytes(b[:room])) + return len(b) + + def writelines(self, lines): + # The C method would write past the limit without calling `write`. + for line in lines: + self.write(line) + + def close(self): + """A student's wrapper around `sys.stdout.buffer` closes it when it is collected: + what was written must survive that.""" + + +def capture(): + """A stdout that behaves like a real one — `buffer`, `reconfigure`, `print(flush=True)` — + and the bytes behind it, kept apart so a detached or closed wrapper cannot lose them.""" + raw = _CappedBytes() + return io.TextIOWrapper(raw, encoding="utf-8", errors="backslashreplace", write_through=True), raw + + +def captured(stream, raw): + try: + stream.flush() + except (ValueError, OSError): # detached or closed by the student + pass + # `io.BytesIO.write(buffer, b)` still reaches past `write`: cut here too. + data = raw.getvalue() + return data[:STDOUT_LIMIT].decode("utf-8", "replace"), raw.truncated or len(data) > STDOUT_LIMIT + + +def stdin_stream(text): + return io.TextIOWrapper(io.BytesIO((text or "").encode("utf-8")), encoding="utf-8") + + +# Between calls, stdout is a sink: what a leftover thread prints is not evidence. +_quiet = open(os.devnull, "w", encoding="utf-8") +_real_stdin = sys.stdin +sys.stdout = _quiet + + +class Missing(Exception): + """The call's target does not exist.""" + + +class Unresolved(Exception): + """A `$ref` or `object` whose value was never produced in this unit.""" + + +class Unserialisable(Exception): + pass + + +class MatchingRequired(Exception): + """Function lookup needs a teacher's decision: never the student's outcome.""" + + def __init__(self, decision): + self.decision = decision + super().__init__(f"function matching needs review: {decision}") + + +class Call: + """One call's outcome: what `call()` observed, and the live value it returned.""" + + def __init__(self, outcome, stdout, truncated, elapsed, value, extra): + self.outcome = outcome + self.stdout = stdout + self.truncated = truncated + self.elapsed = elapsed + self.value = value + self.extra = extra + + def record(self): + return { + "outcome": self.outcome, + "stdout": self.stdout, + "stdout_truncated": self.truncated, + "elapsed_ms": self.elapsed, + } + + +def call(fn, timeout, stdin=None, guard=None, serialise=True, then=None): + """Run one call — its lookup, its body, its serialisation and `then(value)` — under its + timer and, for student code, the import `guard`. + + `then` runs inside the timed region too: it may run the code that owns the value (a + property read for a snapshot), and what it raises is that call's outcome.""" + out, raw = capture() + sys.stdout = out + sys.stdin = stdin_stream(stdin) if stdin is not False else _real_stdin + original_import = builtins.__import__ + if guard is not None: + builtins.__import__ = guard + _fired[0] = False + start = time.perf_counter() + value, extra = None, None + try: + if _HAS_TIMER and timeout: + signal.setitimer(signal.ITIMER_REAL, timeout, 0.05) + try: + value = fn() + if sys.stdout is not out: + # The student replaced stdout with their own wrapper: flush it in their scope. + sys.stdout.flush() + outcome = {"returned": {"value": wire(value) if serialise else None, "type": type(value).__name__}} + if then is not None: + extra = then(value) + finally: + if _HAS_TIMER: + signal.setitimer(signal.ITIMER_REAL, 0) + except CallTimeout: + outcome = {"timeout": {}} + except Missing as exc: + outcome = {"missing": {"message": str(exc)}} + except MatchingRequired as exc: + outcome = {"matching": {"message": str(exc), "decision": exc.decision}} + except Unresolved as exc: + outcome = {"unresolved": {"name": str(exc)}} + except Unserialisable as exc: + outcome = {"unserialisable": {"type": "Unserialisable", "message": str(exc)}} + except BaseException as exc: # SystemExit and KeyboardInterrupt are the code's own too + outcome = {"raised": error_of(exc)} + finally: + builtins.__import__ = original_import + sys.stdout = _quiet + sys.stdin = _real_stdin + if _fired[0]: + # The timer went off even if the code swallowed it with a bare `except:`. + outcome = {"timeout": {}} + elapsed = int((time.perf_counter() - start) * 1000) + stdout, truncated = captured(out, raw) + return Call(outcome, stdout, truncated, elapsed, value, extra if "returned" in outcome else None) + + +# --- The record encoding -------------------------------------------------------------------- +def to_json(value, depth=0, seen=None): + """What the grader sees of a Python value. Total: it never raises for a value's sake.""" + if isinstance(value, bool) or value is None or isinstance(value, str): + return value + if isinstance(value, int): + if -(2**63) <= value < 2**64: + return int(value) + # Past what the grader's JSON can hold exactly: tag it, exactly and with its sign, so + # it is never mistaken for a number and a record always parses. Decimal where Python + # will print it, hexadecimal past its 4300-digit limit. + try: + return {"$bigint": int.__repr__(value)} + except ValueError: + return {"$bigint": hex(value)} + if isinstance(value, float): + return value if math.isfinite(value) else repr(value) + if depth >= DEPTH_LIMIT: + return "" + seen = seen or set() + if id(value) in seen: + return "" + if isinstance(value, (list, tuple, set, frozenset, dict)): + seen = seen | {id(value)} + if isinstance(value, (list, tuple)): + return [to_json(v, depth + 1, seen) for v in value] + if isinstance(value, (set, frozenset)): + items = [to_json(v, depth + 1, seen) for v in value] + try: + return sorted(items) + except TypeError: # mixed types: any stable order will do + return sorted(items, key=lambda v: json.dumps(v, sort_keys=True)) + if isinstance(value, dict): + out = {} + for k, v in value.items(): + key = k if isinstance(k, str) else text_of(k) + if key in out: + raise Unserialisable(f"two keys both read as '{key}'") + out[key] = to_json(v, depth + 1, seen) + return out + return text_of(value) + + +def utf8_size(text): + """The bytes `text` takes on the channel, counted a slice at a time rather than through + a second full copy of a value that may already fill most of the memory limit.""" + step = 1 << 20 + return sum(len(text[i : i + step].encode("utf-8", "replace")) for i in range(0, len(text), step)) + + +def json_size(tree): + return utf8_size(json.dumps(tree, ensure_ascii=False, separators=(",", ":"))) + + +def wire(value): + """The value as a record carries it. One too large to report is replaced by its size: + the call still returned, and a function checker still judges it through the transfer + encoding.""" + value = to_json(value) + size = utf8_size(json.dumps(value, ensure_ascii=False)) + if size > VALUE_LIMIT: + return {"$too_large": f"{size} bytes of JSON"} + return value + + +# --- The transfer encoding -------------------------------------------------------------------- +class Snapshot: + """An object whose class the receiving side does not have: its instance attributes and + public properties, as read on the side that owns it. `__scriptmark_class__` names the + class it was taken from.""" + + def __init__(self, cls, attrs): + object.__setattr__(self, "__scriptmark_class__", cls) + for name, value in attrs.items(): + object.__setattr__(self, name, value) + + def __repr__(self): + fields = ", ".join( + f"{k}={v!r}" for k, v in vars(self).items() if k != "__scriptmark_class__" + ) + return f"<{self.__scriptmark_class__} snapshot: {fields}>" + + +# Classes whose instances are rebuilt from the receiving side's own copy, by `module.qualname`. +SHARED = {} + + +def share(module): + """Register every class a shared module defines.""" + for value in vars(module).values(): + if inspect.isclass(value) and getattr(value, "__module__", None) == module.__name__: + SHARED[f"{module.__name__}.{value.__qualname__}"] = value + + +def _shared_name(cls): + """The shared class `cls` is, or derives from: a student's subclass travels as its + shared base.""" + for base in cls.__mro__: + name = f"{getattr(base, '__module__', '')}.{getattr(base, '__qualname__', '')}" + if SHARED.get(name) is base: + return name + return None + + +def _state(value): + """An instance's own attributes: its `__dict__`, and the slots it has filled.""" + state = dict(vars(value)) if hasattr(value, "__dict__") else {} + for cls in type(value).__mro__: + slots = cls.__dict__.get("__slots__", ()) + for slot in [slots] if isinstance(slots, str) else slots: + if slot in ("__dict__", "__weakref__") or slot in state: + continue + try: + state[slot] = getattr(value, slot) + except AttributeError: # declared, never set + pass + return state + + +def _properties(value): + """The public properties `value`'s class defines, read now: this runs the owner's code.""" + read = {} + for cls in type(value).__mro__: + if cls is object: + continue + for name, attr in cls.__dict__.items(): + if isinstance(attr, property) and not name.startswith("_") and name not in read: + read[name] = getattr(value, name) + return read + + +_ATOMS = (decimal.Decimal, fractions.Fraction, complex, datetime.date, datetime.time, datetime.timedelta, bytes, bytearray) + + +class _Node: + __slots__ = ("kind", "body", "refs", "number") + + def __init__(self, kind): + self.kind = kind + self.body = None + self.refs = 1 + self.number = None + + +def encode(value): + """The transfer encoding of `value`. Runs the owner's code (properties, `__iter__` of a + container subclass) and raises what it raises; call it under the owner's timer.""" + nodes = {} + keep = [] # every object visited stays alive, so no id is reused while encoding + root = _graph(value, 0, nodes, keep) + numbers = [0] + return _emit(root, numbers) + + +def _graph(value, depth, nodes, keep): + """Pass one: the value graph, each container and object visited once, with how often it + is referred to — so pass two numbers only what is actually shared.""" + if value is None or isinstance(value, (bool, str)): + return value + if isinstance(value, int): + return value if -(2**63) <= value < 2**64 else ("int", int(value)) + if isinstance(value, float): + return value if math.isfinite(value) else ("float", repr(value)) + if isinstance(value, _ATOMS): + return ("atom", value) + if depth >= DEPTH_LIMIT: + return ("text", "", type(value).__name__) + node = nodes.get(id(value)) + if node is not None: + node.refs += 1 + return node + keep.append(value) + shared = _shared_name(type(value)) + if shared is not None: + node = nodes[id(value)] = _Node("shared") + node.body = (shared, {k: _graph(v, depth + 1, nodes, keep) for k, v in _state(value).items()}) + return node + if isinstance(value, (list, tuple, set, frozenset)): + kind = "list" if isinstance(value, list) else "tuple" if isinstance(value, tuple) else "set" if isinstance(value, set) else "frozenset" + node = nodes[id(value)] = _Node(kind) + node.body = [_graph(v, depth + 1, nodes, keep) for v in value] + return node + if isinstance(value, dict): + node = nodes[id(value)] = _Node("dict") + node.body = [(_graph(k, depth + 1, nodes, keep), _graph(v, depth + 1, nodes, keep)) for k, v in value.items()] + return node + if inspect.isroutine(value) or inspect.isclass(value) or inspect.ismodule(value): + return ("text", text_of(value), type(value).__name__) + module = getattr(type(value), "__module__", "") or "" + if module.split(".")[0] == "numpy" and callable(getattr(value, "tolist", None)): + return _graph(value.tolist(), depth, nodes, keep) + if hasattr(value, "__dict__") or any("__slots__" in cls.__dict__ for cls in type(value).__mro__[:-1]): + node = nodes[id(value)] = _Node("object") + attrs = {k: _graph(v, depth + 1, nodes, keep) for k, v in _state(value).items()} + props = {k: _graph(v, depth + 1, nodes, keep) for k, v in _properties(value).items()} + node.body = (type(value).__qualname__, attrs, props) + return node + return ("text", text_of(value), type(value).__name__) + + +def _atom(value): + if isinstance(value, decimal.Decimal): + return {"$": "decimal", "v": str(value)} + if isinstance(value, fractions.Fraction): + return {"$": "fraction", "v": [_emit_int(value.numerator), _emit_int(value.denominator)]} + if isinstance(value, complex): + return {"$": "complex", "v": [repr(value.real), repr(value.imag)]} + if isinstance(value, datetime.datetime): + return {"$": "datetime", "v": value.isoformat()} + if isinstance(value, datetime.date): + return {"$": "date", "v": value.isoformat()} + if isinstance(value, datetime.time): + return {"$": "time", "v": value.isoformat()} + if isinstance(value, datetime.timedelta): + return {"$": "timedelta", "v": [value.days, value.seconds, value.microseconds]} + return {"$": "bytes" if isinstance(value, bytes) else "bytearray", "v": base64.b64encode(bytes(value)).decode("ascii")} + + +def _emit_int(value): + return value if -(2**63) <= value < 2**64 else {"$": "int", "v": hex(value)} + + +def _emit(item, numbers): + """Pass two: JSON, plain wherever plain JSON says the same thing.""" + if not isinstance(item, (_Node, tuple)): + return item + if isinstance(item, tuple): + kind = item[0] + if kind == "int": + return {"$": "int", "v": hex(item[1])} + if kind == "float": + return {"$": "float", "v": item[1]} + if kind == "atom": + return _atom(item[1]) + return {"$": "text", "v": item[1], "type": item[2]} + if item.number is not None: + return {"$": "ref", "#": item.number} + tagged = {} + if item.refs > 1: + item.number = numbers[0] + numbers[0] += 1 + tagged["#"] = item.number + kind = item.kind + if kind == "list": + items = [_emit(v, numbers) for v in item.body] + return items if not tagged else {"$": "list", **tagged, "v": items} + if kind in ("tuple", "set", "frozenset"): + return {"$": kind, **tagged, "v": [_emit(v, numbers) for v in item.body]} + if kind == "dict": + pairs = [(_emit(k, numbers), _emit(v, numbers)) for k, v in item.body] + if not tagged and all(isinstance(k, str) and k != "$" for k, _ in pairs): + return dict(pairs) + return {"$": "dict", **tagged, "v": [[k, v] for k, v in pairs]} + if kind == "shared": + name, state = item.body + return {"$": "shared", **tagged, "class": name, "state": {k: _emit(v, numbers) for k, v in state.items()}} + cls, attrs, props = item.body + return { + "$": "object", + **tagged, + "class": cls, + "attrs": {k: _emit(v, numbers) for k, v in attrs.items()}, + "properties": {k: _emit(v, numbers) for k, v in props.items()}, + } + + +def decode(tree, resolve=None): + """Rebuild a transfer-encoded value. `resolve(name)` answers `{"$": "id"}` references to + values that live on this side. Raises Garbled for anything the encoding cannot mean.""" + return _decode(tree, {}, resolve, 0) + + +def _decode(tree, made, resolve, depth): + if depth > DEPTH_LIMIT + 2: + raise Garbled(f"a value nested deeper than {DEPTH_LIMIT}") + if tree is None or isinstance(tree, (bool, int, float, str)): + return tree + if isinstance(tree, list): + return [_decode(v, made, resolve, depth + 1) for v in tree] + if not isinstance(tree, dict): + raise Garbled(f"unexpected {type(tree).__name__}") + kind = tree.get("$") + if kind is None: + return {k: _decode(v, made, resolve, depth + 1) for k, v in tree.items()} + try: + return _tagged(kind, tree, made, resolve, depth) + except Garbled: + raise + except RecursionError: + raise Garbled("a value nested too deeply") from None + except Exception as exc: # a tag whose body does not fit it + raise Garbled(f"a '{kind}' value that cannot be rebuilt: {text_of(exc)}") from None + + +def _number(tree, made, obj): + if "#" in tree: + number = tree["#"] + if not isinstance(number, int) or number in made: + raise Garbled("a value numbered twice") + made[number] = obj + return obj + + +def _tagged(kind, tree, made, resolve, depth): + body = tree.get("v") + again = lambda v: _decode(v, made, resolve, depth + 1) # noqa: E731 + if kind == "ref": + if tree.get("#") not in made: + raise Garbled("a reference to nothing before it") + return made[tree["#"]] + if kind == "id": + if resolve is None: + raise Garbled("a reference to a value this side does not hold") + return resolve(tree["name"]) + if kind == "int": + if not isinstance(body, str) or len(body) > INT_DIGITS: + raise Garbled("an integer too long to carry") + return int(body, 16) + if kind == "float": + return float(body) + if kind == "decimal": + return decimal.Decimal(body) + if kind == "fraction": + return fractions.Fraction(again(body[0]), again(body[1])) + if kind == "complex": + return complex(float(body[0]), float(body[1])) + if kind == "datetime": + return datetime.datetime.fromisoformat(body) + if kind == "date": + return datetime.date.fromisoformat(body) + if kind == "time": + return datetime.time.fromisoformat(body) + if kind == "timedelta": + return datetime.timedelta(days=body[0], seconds=body[1], microseconds=body[2]) + if kind in ("bytes", "bytearray"): + data = base64.b64decode(body, validate=True) + return data if kind == "bytes" else bytearray(data) + if kind == "text": + return str(body) + if kind == "list": + out = _number(tree, made, []) + out.extend(again(v) for v in body) + return out + if kind == "set": + out = _number(tree, made, set()) + out.update(again(v) for v in body) + return out + if kind in ("tuple", "frozenset"): + # Built from their items, so a cycle back through one cannot be rebuilt. + number = tree.get("#") + if number is not None and number in made: + raise Garbled("a value numbered twice") + items = [again(v) for v in body] + return _number(tree, made, tuple(items) if kind == "tuple" else frozenset(items)) + if kind == "dict": + out = _number(tree, made, {}) + for key, value in body: + out[again(key)] = again(value) + return out + if kind == "shared": + cls = SHARED.get(tree["class"]) + if cls is None: + raise Garbled(f"no shared class '{tree['class']}' on this side") + try: + obj = cls.__new__(cls) + except TypeError: # a __new__ that wants arguments: rebuilt without it, as without __init__ + obj = object.__new__(cls) + obj = _number(tree, made, obj) + for name, value in tree["state"].items(): + object.__setattr__(obj, name, again(value)) + return obj + if kind == "object": + obj = _number(tree, made, Snapshot(str(tree["class"]), {})) + for part in ("attrs", "properties"): + for name, value in tree[part].items(): + object.__setattr__(obj, name, again(value)) + return obj + raise Garbled(f"an unknown kind of value '{kind}'") + + +def transfer(value): + """The transfer encoding of `value`, or a marker saying how large it would have been.""" + tree = encode(value) + size = json_size(tree) + if size > TRANSFER_LIMIT: + return {"$": "too_large", "size": size} + return tree diff --git a/src/core/src/runner/harness/student.py b/src/core/src/runner/harness/student.py new file mode 100644 index 0000000..6f6bc65 --- /dev/null +++ b/src/core/src/runner/harness/student.py @@ -0,0 +1,369 @@ +"""The student side of a unit: the student's module, and the calls the teacher side asks for. + +Nothing here decides anything. It answers one request at a time: what a call did, what it +printed, which files it left, and — when the teacher side needs it — the value it made, in +the transfer encoding. Every piece of student code — loading the module, finding a +function, reading an attribute, serialising a value, reading a property for a snapshot — +runs inside `call()`, under its timer and import guard, so a failure there is that call's +outcome and never a harness crash. + +What this process sends is the student's own evidence, and the teacher side treats it as +such: a student who writes to the pipe directly can only misreport their own calls. +""" + +import importlib.machinery # noqa: E402 +import importlib.util # noqa: E402 +import runpy # noqa: E402 +from typing import TYPE_CHECKING # noqa: E402 + +if TYPE_CHECKING: + from matching import FunctionRules, MatchDecision, function_decision + +_IN, _OUT = channel(0, 1) +# What the teacher side may send in one frame: its own values, never the student's. +REQUEST_LIMIT = 1 << 30 +INIT = receive(_IN, REQUEST_LIMIT) +configure(INIT["limits"]) +REFERENCE = INIT["role"] == "reference" + + +def reply(message): + send(_OUT, message) + + +# --- The import guard ----------------------------------------------------------------- +# A teaching rule, not a security boundary: the operating system's sandbox is that. It looks +# at where the import statement runs, so an allowlisted stdlib module importing its own +# internals is never mistaken for the student importing them. +ALLOWED = frozenset( + { + # core + "builtins", "sys", "io", "abc", "enum", "types", "typing", "dataclasses", + "collections", "functools", "itertools", "operator", "copy", "copyreg", + # math/science + "math", "cmath", "decimal", "fractions", "numbers", "random", "statistics", + # strings/text + "string", "re", "difflib", "textwrap", "unicodedata", "codecs", "encodings", + # data formats + "json", "csv", "struct", "base64", "binascii", "hashlib", "hmac", + # time + "time", "datetime", "calendar", "zoneinfo", + # data structures + "array", "heapq", "bisect", "queue", "graphlib", + # functional + "contextlib", "contextvars", + # introspection + "inspect", "dis", "token", "tokenize", "ast", "symtable", + # error handling + "traceback", "warnings", "errno", "faulthandler", + # internals needed by Python itself + "importlib", "_thread", "__future__", "_collections_abc", "_abc", + "_functools", "_operator", "_io", "_json", "_string", "_struct", + "_sre", "sre_compile", "sre_constants", "sre_parse", "_locale", + "_frozen_importlib", "_frozen_importlib_external", "_imp", + "_warnings", "_weakref", "_weakrefset", "_threading_local", + "_codecs", "_signal", "_stat", "_opcode", "_typing", + "genericpath", "posixpath", "ntpath", "stat", "keyword", + "reprlib", "pprint", "linecache", + # testing (students may use unittest/doctest) + "unittest", "doctest", + # filesystem (pathlib is safe — open() is a builtin anyway) + "pathlib", "fnmatch", "glob", "tempfile", + # misc safe + "secrets", "uuid", "ipaddress", "colorsys", "timeit", "py_compile", "compileall", + "gzip", "bz2", "lzma", "zipfile", "tarfile", "xml", "html", "email", + "logging", "argparse", "getopt", "configparser", "tomllib", "threading", "concurrent", + } +) | frozenset(INIT.get("allowed_imports", [])) +STUDENT_MARK = "__scriptmark_student__" +_original_import = builtins.__import__ + + +def _guarded_import(name, globals=None, locals=None, fromlist=(), level=0): + # `__import__('os')` called directly passes no globals: look at the caller instead. + scope = globals if globals is not None else sys._getframe(1).f_globals + if scope.get(STUDENT_MARK) and level == 0: + if name.split(".")[0] not in ALLOWED: + raise ImportError(f"Module '{name}' is not allowed in student code") + return _original_import(name, globals, locals, fromlist, level) + + +GUARD = None if REFERENCE else _guarded_import + + +# --- Values held here ----------------------------------------------------------------- +# What the student's calls made, by id. Values the teacher side holds arrive as copies. +context = dict(INIT.get("vars", {})) + + +def live(name): + if name not in context: + raise Unresolved(name) + return context[name] + + +def received(tree): + """A value the teacher side sent, rebuilt here: an id names a value made on this side.""" + return decode(tree, live) + + +def matched_function( + module: object, name: str, evidence: list[MatchDecision | None] +) -> tuple[object, str]: + policy: FunctionRules = INIT.get("functions", {"aliases": {}, "overrides": {}}) + # Read attributes only once: dynamic exports/properties belong to the guarded call. + exports: dict[str, object] = dict(vars(module)) + absent: object = object() + for candidate in { + name, + *policy.get("aliases", {}).get(name, []), + *([policy["overrides"][name]] if name in policy.get("overrides", {}) else []), + }: + if candidate not in exports: + value: object = getattr(module, candidate, absent) + if value is not absent: + exports[candidate] = value + decision: MatchDecision = function_decision( + [candidate for candidate, obj in exports.items() if callable(obj)], name, policy + ) + evidence[0] = decision + selected: str | None = decision["selected"] + if selected is not None: + return exports[selected], selected + if ( + decision["state"] != "missing" + or name in policy.get("overrides", {}) + or name in policy.get("aliases", {}) + ): + raise MatchingRequired(decision) + raise Missing(f"function '{name}' not found") + + +def member(obj, object_id, name, what): + """`getattr(obj, name)`, read once. A name that does not exist is Missing; a property + that raises — AttributeError included — is the student's exception.""" + try: + return getattr(obj, name) + except AttributeError: + try: + inspect.getattr_static(obj, name) + except AttributeError: + if not hasattr(type(obj), "__getattr__"): + raise Missing(f"'{object_id}' has no {what} '{name}'") from None + raise + + +def observe_files(paths: list[str]) -> dict[str, str | None]: + seen: dict[str, str | None] = {} + for path in paths: + try: + with open(path, encoding="utf-8") as fh: + content: str = fh.read(FILE_LIMIT + 1) + seen[path] = content if len(content) <= FILE_LIMIT else None + except (OSError, ValueError): # absent, oversized or invalid UTF-8 is no text expectation + seen[path] = None + return seen + + +# --- Requests ------------------------------------------------------------------------- +module = [None] + + +def compiles(path): + """Compile the file in memory — writing no __pycache__ beside it — and answer with a + syntax error as the load call's outcome.""" + try: + with open(path, "rb") as fh: + compile(fh.read(), path, "exec") + return True + except (SyntaxError, ValueError, RecursionError, MemoryError) as exc: + reply({ + "compiles": False, + "outcome": {"raised": {"type": "SyntaxError", "types": ["SyntaxError"], "message": text_of(exc)}}, + "stdout": "", + "stdout_truncated": False, + "elapsed_ms": 0, + }) + return False + + +def load(request): + path = INIT["subject"] + if not compiles(path): + return + + def run(): + # An explicit loader: `Lab5.PY` is Python too, whatever its extension. + loader = importlib.machinery.SourceFileLoader("student_mod", path) + spec = importlib.util.spec_from_loader("student_mod", loader) + made = importlib.util.module_from_spec(spec) + if not REFERENCE: + setattr(made, STUDENT_MARK, True) + for key, val in INIT.get("vars", {}).items(): + setattr(made, key, val) + sys.modules["student_mod"] = made + loader.exec_module(made) + module[0] = made + + original_input = builtins.input + builtins.input = lambda *a, **k: "0" + try: + done = call(run, request["timeout"], guard=GUARD, serialise=False) + finally: + builtins.input = original_input + reply(done.record()) + + +def feed_stdin(text): + """Descriptor 0 becomes a pipe holding `text`, as `python script.py < input` would see + it: `input()`, `sys.stdin.buffer` and `os.read(0, …)` all read the same bytes.""" + import threading + + read_end, write_end = os.pipe() + + def feed(): + try: + write_all(write_end, text.encode("utf-8")) + except OSError: # the script stopped reading + pass + finally: + os.close(write_end) + + threading.Thread(target=feed, daemon=True).start() + os.dup2(read_end, 0) + os.close(read_end) + + +def script(request): + path = INIT["subject"] + if not compiles(path): + return + init = {STUDENT_MARK: True, **INIT.get("vars", {})} + sys.argv = [path] # as `python student.py` would see it, not the harness's + feed_stdin(request.get("stdin") or "") + done = call( + lambda: runpy.run_path(path, init_globals=init, run_name="__main__"), + request["timeout"], + stdin=False, + guard=GUARD, + serialise=False, + ) + record = done.record() + raised = done.outcome.get("raised") + if raised and raised["type"] == "SystemExit" and raised.get("clean"): + record["outcome"] = {"returned": {"value": None, "type": "NoneType"}} + if request.get("files"): + record["files"] = observe_files(request["files"]) + reply(record) + + +def invoke(request, resolved, evidence): + """The call a request names. Runs inside `call()`: the lookup is student code too.""" + kind, spec = next(iter(request["target"].items())) + args = [received(a) for a in request.get("args", [])] + if kind == "function": + student = module[0] + if REFERENCE: + if not hasattr(student, spec["name"]): + raise Missing(f"function '{spec['name']}' not found") + fn, resolved[0] = getattr(student, spec["name"]), spec["name"] + else: + fn, resolved[0] = matched_function(student, spec["name"], evidence) + return fn(*args) + obj = received(request["object"]) if "object" in request else live(spec["object"]) + if kind == "method": + fn = member(obj, spec["object"], spec["name"], "method") + if not callable(fn): + raise Missing(f"'{spec['object']}.{spec['name']}' is not a method") + return fn(*args) + return member(obj, spec["object"], spec["name"], "attribute") + + +def run_call(request): + requested = next(iter(request["target"].values()))["name"] + resolved = [requested] + evidence: list[MatchDecision | None] = [None] + done = call( + lambda: invoke(request, resolved, evidence), + request["timeout"], + request.get("stdin"), + guard=GUARD, + serialise=request.get("serialise", True), + then=transfer if request.get("transfer") else None, + ) + if "returned" in done.outcome and request.get("id"): + context[request["id"]] = done.value + record = done.record() + record["target"] = {"requested": requested, "resolved": resolved[0], "matching": evidence[0]} + if request.get("files"): + record["files"] = observe_files(request["files"]) + if done.extra is not None and done.extra != done.outcome["returned"]["value"]: + # Sent only when the record's value would not rebuild the same thing. + record["transfer"] = done.extra + reply(record) + + +def fetch(request): + """A value made here, for teacher code to take by value: read now, under the timer, + since a snapshot runs the student's properties.""" + if request["name"] not in context: + return reply({"unresolved": request["name"]}) + value = context[request["name"]] + done = call(lambda: transfer(value), request["timeout"], guard=GUARD, serialise=False) + if "returned" in done.outcome: + return reply({"value": done.value}) + reply({"outcome": done.outcome}) + + +def canary_holds(): + """The sandbox's own check: the grader left a file outside everything this side may read. + Reading it means the sandbox is not there.""" + path = INIT.get("canary") + if not path: + return True + try: + with open(path, "rb"): + return False + except OSError: + return True + + +HANDLERS = {"load": load, "script": script, "call": run_call, "fetch": fetch} + + +def serve(): + if not canary_holds(): + reply({"fatal": problem("Sandbox", "the student sandbox did not hold: its canary was readable")}) + os._exit(0) + for path in INIT.get("shared", []): + # The student's own copy of each shared module, importable as a module of its own. + name = os.path.splitext(os.path.basename(path))[0] + spec = importlib.util.spec_from_file_location(name, path) + shared = importlib.util.module_from_spec(spec) + sys.modules[name] = shared + spec.loader.exec_module(shared) + share(shared) + # Student code may import helpers staged beside it: after the stdlib, so a submission + # never stands in for what the harness imports. + sys.path.append(os.getcwd()) + reply({"ready": True}) + while True: + try: + request = receive(_IN, REQUEST_LIMIT) + except (Closed, OSError): + break + op = request.get("op") + if op == "exit": + break + HANDLERS[op](request) + # Threads a student left running must not hold the unit open. + os._exit(0) + + +try: + serve() +except BaseException as exc: # a harness bug, not the student's code + try: + reply({"fatal": error_of(exc)}) + finally: + os._exit(70) diff --git a/src/core/src/runner/harness/teacher.py b/src/core/src/runner/harness/teacher.py new file mode 100644 index 0000000..d69ff26 --- /dev/null +++ b/src/core/src/runner/harness/teacher.py @@ -0,0 +1,687 @@ +"""The teacher side of a unit: teacher modules and checkers, the plan, and the records. + +It drives the student side one request at a time, sets a deadline on each, and reports +observations, never verdicts about who is at fault: the grader derives those from which +call was running, what kind of code it runs, and how each side ended. What the student +side answers is the student's evidence; it is checked against the largest answer that +request allows before it is read, and it never reaches a record unchecked. +""" + +import importlib.util # noqa: E402 +import queue # noqa: E402 +import threading # noqa: E402 + +with open(sys.argv[1], encoding="utf-8") as _fh: + PAYLOAD = json.load(_fh) +os.unlink(sys.argv[1]) +configure(PAYLOAD["limits"]) + +_RECORDS = PAYLOAD["records"] +# Seconds a request may run past its own timeout before the student side is given up on, +# and how long the student side has to start. +GRACE = PAYLOAD["grace"] +STARTUP = PAYLOAD["startup"] +# The most a frame from the student side carries besides its value, stdout and files: +# the call's target and matching evidence, and the keys around them. +SLACK = 1 << 20 + + +def emit(record): + write_all(_RECORDS, frame(record)) + + +def finish(student=None): + """Stop now: threads teacher code left running must not hold the unit open.""" + if student is not None: + student.close() + emit({"kind": "done"}) + os._exit(0) + + +# --- The student side ----------------------------------------------------------------- +class Stop(Exception): + """The student side cannot be asked anything more: it went away, ran past a deadline, + or answered with something no request allows.""" + + def __init__(self, reason, message): + super().__init__(message) + self.reason = reason + self.message = message + + +class Student: + """The pipe to the student side. One request is pending at a time; a reader thread + checks each answer's length against that request's bound before reading it.""" + + def __init__(self, read_fd, write_fd): + self.read_fd, self.write_fd = read_fd, write_fd + self.bound = 0 + self.answers = queue.Queue() + self.stopped = None + threading.Thread(target=self._read, daemon=True).start() + + def _read(self): + try: + while True: + (size,) = HEADER.unpack(read_exact(self.read_fd, HEADER.size)) + bound = self.bound + if size > bound: + self.answers.put(Stop("flooded", f"the student side sent {size} bytes where at most {bound} may arrive")) + return + self.answers.put(read_exact(self.read_fd, size)) + except (Closed, OSError): + self.answers.put(Stop("exited", "the student side ended")) + + def ask(self, request, bound, timeout): + if self.stopped is not None: + raise self.stopped + self.bound = bound + try: + send(self.write_fd, request) + except OSError: + self.stopped = Stop("exited", "the student side ended") + raise self.stopped from None + try: + answer = self.answers.get(timeout=timeout) + except queue.Empty: + self.stopped = Stop("deadline", f"the student side did not answer within {timeout:g}s") + raise self.stopped from None + if isinstance(answer, Stop): + self.stopped = answer + raise answer + try: + parsed = parse(answer) + if not isinstance(parsed, dict): + raise Garbled("an answer that is not an object") + return parsed + except Garbled as exc: + self.stopped = Stop("garbled", str(exc)) + raise self.stopped from None + + def garbled(self, message): + self.stopped = Stop("garbled", message) + return self.stopped + + def close(self): + if self.stopped is None: + try: + send(self.write_fd, {"op": "exit"}) + except OSError: + pass + + +def call_bound(files=0, value=True, transfer=False): + """The largest answer a call can legitimately make: its value, its stdout and an error + message, each of which JSON escaping can grow sixfold, and each file it observes.""" + bound = SLACK + 6 * STDOUT_LIMIT + 6 * MESSAGE_LIMIT + files * (6 * FILE_LIMIT + 4096) + if value: + bound += VALUE_LIMIT + if transfer: + bound += TRANSFER_LIMIT + return bound + + +# --- Checking what the student side says ------------------------------------------------- +OUTCOMES = {"returned", "raised", "timeout", "missing", "matching", "unresolved", "unserialisable"} + + +def _str(value, what): + if not isinstance(value, str): + raise Garbled(f"{what} is not text") + return value + + +def _error(info, what): + if not isinstance(info, dict): + raise Garbled(f"{what} is not an error") + error = { + "type": _str(info.get("type"), what), + "types": [_str(t, what) for t in info.get("types", [])] if isinstance(info.get("types", []), list) else None, + "message": _str(info.get("message", ""), what), + } + if error["types"] is None: + raise Garbled(f"{what} has no list of types") + if "clean" in info: + if not isinstance(info["clean"], bool): + raise Garbled(f"{what} says neither clean nor not") + error["clean"] = info["clean"] + return error + + +def _decision(decision): + if decision is None: + return None + if not isinstance(decision, dict) or not isinstance(decision.get("candidates"), list): + raise Garbled("matching evidence that is not a decision") + selected = decision.get("selected") + if selected is not None: + _str(selected, "a selected function") + return { + "state": _str(decision.get("state"), "a matching state"), + "selected": selected, + "candidates": [ + { + "value": _str(c.get("value"), "a candidate"), + "rules": [_str(r, "a rule") for r in c.get("rules", [])], + } + for c in decision["candidates"] + if isinstance(c, dict) and isinstance(c.get("rules", []), list) + ], + } + + +def _outcome(outcome): + if not isinstance(outcome, dict) or len(outcome) != 1: + raise Garbled("an outcome that is not exactly one thing") + ((kind, body),) = outcome.items() + if kind not in OUTCOMES or not isinstance(body, dict): + raise Garbled(f"an unknown outcome '{kind}'") + if kind == "returned": + return {"returned": {"value": body.get("value"), "type": _str(body.get("type"), "a type name")}} + if kind in ("raised", "unserialisable"): + return {kind: _error(body, "an error")} + if kind == "timeout": + return {"timeout": {}} + if kind == "missing": + return {"missing": {"message": _str(body.get("message"), "a message")}} + if kind == "unresolved": + return {"unresolved": {"name": _str(body.get("name"), "a name")}} + return {"matching": {"message": _str(body.get("message"), "a message"), "decision": _decision(body.get("decision"))}} + + +def checked(answer, files): + """The call record an answer describes, rebuilt from only what a record may hold.""" + if "fatal" in answer: + raise Garbled(f"the student side failed: {text_of(answer['fatal'])[:MESSAGE_LIMIT]}") + stdout = _str(answer.get("stdout", ""), "stdout") + truncated = answer.get("stdout_truncated", False) + elapsed = answer.get("elapsed_ms", 0) + if not isinstance(truncated, bool) or not isinstance(elapsed, int) or isinstance(elapsed, bool): + raise Garbled("a call record with malformed fields") + record = { + "outcome": _outcome(answer.get("outcome")), + "stdout": stdout, + "stdout_truncated": truncated, + "elapsed_ms": max(elapsed, 0), + } + target = answer.get("target") + if target is not None: + if not isinstance(target, dict): + raise Garbled("a target that is not an object") + record["target"] = { + "requested": _str(target.get("requested"), "a target"), + "resolved": _str(target.get("resolved"), "a target"), + "matching": _decision(target.get("matching")), + } + if files: + seen = answer.get("files") + if not isinstance(seen, dict) or set(seen) != set(files): + raise Garbled("observed files other than the ones asked for") + record["files"] = { + path: None if content is None else _str(content, "a file's content") + for path, content in seen.items() + } + return record + + +# --- Teacher modules ------------------------------------------------------------------- +def _removed_checker(*args, **kwargs): + raise RuntimeError( + "@checker no longer binds a checker to a function: put " + 'check = { function = "" } on each case it should judge ' + "(https://github.com/Acture/obsidian-vault/blob/project/scriptmark/" + "docs/test-bundles.md#migrating-an-older-spec)" + ) + + +def exports_of(module): + """`__all__` when present; otherwise public names, minus modules and minus classes and + functions the module merely imported from elsewhere.""" + names = getattr(module, "__all__", None) + if names is None: + names = [] + for name in dir(module): + if name.startswith("_"): + continue + value = getattr(module, name) + if inspect.ismodule(value): + continue + if (inspect.isclass(value) or inspect.isroutine(value)) and getattr( + value, "__module__", None + ) != module.__name__: + continue + names.append(name) + return {name: getattr(module, name) for name in names} + + +def load_shared(): + """The teacher's own copy of each shared module, importable under its name, so teacher + modules that import it share its classes with what the student side sends.""" + for path in PAYLOAD.get("shared", []): + name = os.path.splitext(os.path.basename(path))[0] + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + share(module) + + +def load_teacher_modules(): + """Returns (exports, duplicates): a name two modules export differently is refused.""" + exports, owner, duplicates = {}, {}, {} + builtins.checker = _removed_checker + try: + load_shared() + for index, path in enumerate(PAYLOAD.get("imports", [])): + directory = os.path.dirname(path) + if directory not in sys.path: + # After the stdlib: a helper beside the module must not shadow it. + sys.path.append(directory) + spec = importlib.util.spec_from_file_location(f"teacher_mod_{index}", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + for name, value in exports_of(module).items(): + if name in exports and not _same(exports[name], value): + duplicates[name] = [owner[name], path] + exports[name] = value + owner.setdefault(name, path) + finally: + del builtins.checker + return exports, duplicates + + +def _same(a, b): + """One object, or two equal ones: two modules that both define `TOL = 1e-6` agree.""" + if a is b: + return True + try: + return bool(a == b) + except BaseException: + return False + + +def params_of(fn): + """Each parameter's name, kind, and whether it has a default.""" + try: + parameters = inspect.signature(fn).parameters.values() + except (TypeError, ValueError): + return None + return [{"name": p.name, "kind": p.kind.name.lower(), "default": p.default is not p.empty} for p in parameters] + + +# --- Values ------------------------------------------------------------------------------ +# Values held on this side: teacher exports, `[vars]`, and what teacher calls made, by name. +context = {} +# Names whose values live on the student side: what the student's calls made. +held = set() +SOURCE = None + + +def bind(name, value=None, student=False): + if student: + context.pop(name, None) + held.add(name) + else: + held.discard(name) + context[name] = value + + +def outgoing(value): + """A plan argument as the student side receives it: a `$name` the student side holds + is named, one this side holds is copied, and `$$` escapes a literal `$`.""" + if isinstance(value, str) and value.startswith("$"): + if value.startswith("$$"): + return value[1:] + name = value[1:] + if name in held: + return {"$": "id", "name": name} + if name in context: + return encode(context[name]) + raise Unresolved(name) + if isinstance(value, list): + return [outgoing(v) for v in value] + if isinstance(value, dict): + pairs = [(k, outgoing(v)) for k, v in value.items()] + if "$" in value: + return {"$": "dict", "v": [[k, v] for k, v in pairs]} + return dict(pairs) + return value + + +def fetch(student, name, timeout): + """A value the student side holds, taken by value now. Returns the value, or the + outcome that kept it from arriving.""" + answer = student.ask({"op": "fetch", "name": name, "timeout": timeout}, call_bound(value=False, transfer=True), timeout + GRACE) + if "value" in answer: + tree = answer["value"] + if isinstance(tree, dict) and tree.get("$") == "too_large": + return None, {"error": problem("TooLarge", f"'{name}' is {tree.get('size')} bytes of JSON, more than a checker can be given")} + try: + return decode(tree), None + except Garbled as exc: + raise student.garbled(str(exc)) from None + if "unresolved" in answer: + return None, {"unresolved": {"name": name}} + outcome = _outcome(answer.get("outcome")) + if "timeout" in outcome: + return None, {"error": problem("Timeout", f"reading '{name}' for the checker took longer than {timeout}s")} + if "raised" in outcome: + return None, {"error": outcome["raised"]} + return None, {"error": problem("Unreadable", f"'{name}' could not be read for the checker")} + + +def local(value, student, timeout): + """A plan argument for teacher code: student-side values are fetched by value.""" + if isinstance(value, str) and value.startswith("$"): + if value.startswith("$$"): + return value[1:] + name = value[1:] + if name in context: + return context[name] + if name in held: + got, failed = fetch(student, name, timeout) + if failed is not None: + raise Unresolved(name) + return got + raise Unresolved(name) + if isinstance(value, list): + return [local(v, student, timeout) for v in value] + if isinstance(value, dict): + return {k: local(v, student, timeout) for k, v in value.items()} + return value + + +# --- Checks ------------------------------------------------------------------------------ +def verdict_of(returned): + if isinstance(returned, bool): + return {"verdict": {"pass": returned, "message": ""}} + if ( + isinstance(returned, tuple) + and len(returned) == 2 + and isinstance(returned[0], bool) + and isinstance(returned[1], (str, type(None))) + ): + return {"verdict": {"pass": returned[0], "message": clip(returned[1] or "")}} + return { + "error": { + "type": "CheckerContract", + "types": ["CheckerContract"], + "message": f"checker returned {type(returned).__name__}; expected bool or (bool, str)", + } + } + + +def run_check(check, value, stdout, timeout, student): + fn = context.get(check["function"]) + deps = {} + for param in (params_of(fn) or [])[2:]: + name = param["name"] + if "var" in param["kind"]: + continue + if name == "stdout": + deps[name] = stdout + elif name == "source" and SOURCE is not None and name not in context and name not in held: + deps[name] = SOURCE + elif name in context: + deps[name] = context[name] + elif name in held: + got, failed = fetch(student, name, timeout) + if failed is not None: + return failed + deps[name] = got + elif not param["default"]: + # prepare saw this name in scope: it is an id whose producing call failed. + return {"unresolved": {"name": name}} + done = call(lambda: fn(value, check.get("expected"), **deps), timeout, serialise=False) + if "returned" in done.outcome: + return verdict_of(done.value) + if "timeout" in done.outcome: + return {"error": problem("Timeout", f"checker timed out after {timeout}s")} + raised = done.outcome.get("raised") or {"type": "Error", "types": [], "message": str(done.outcome)} + if "AssertionError" in raised["types"]: + return {"rejected": {"message": raised["message"]}} + return {"error": raised} + + +def judged_value(answer, record, student): + """The value a checker judges: rebuilt from the transfer encoding when the student side + sent one, otherwise from the record, which then says the same thing.""" + tree = answer["transfer"] if "transfer" in answer else record["outcome"]["returned"]["value"] + if isinstance(tree, dict) and tree.get("$") == "too_large": + return None, {"error": problem("TooLarge", f"the value is {tree.get('size')} bytes of JSON, more than a checker can be given")} + try: + return decode(tree), None + except Garbled as exc: + raise student.garbled(str(exc)) from None + + +# --- Calls ------------------------------------------------------------------------------- +def stopped(phase, index, stop): + emit({"kind": "stopped", "phase": phase, "index": index, "reason": stop.reason, "message": clip(stop.message)}) + + +def teacher_call(plan, phase, index, student): + name = plan["target"]["teacher"]["name"] + fn = context.get(name) + try: + # Student-side arguments are read before the teacher's call and its timer starts. + args = local(plan.get("args", []), student, plan["timeout"]) + except Unresolved as exc: + args = exc + except Stop as stop: + stopped(phase, index, stop) + return None + + def run(): + if isinstance(args, Unresolved): + raise args + if fn is None or not callable(fn): + raise Missing(f"teacher function '{name}' not found") + return fn(*args) + + done = call(run, plan["timeout"], plan.get("stdin"), serialise=phase != "setup") + record = {"kind": "call", "phase": phase, "index": index, **done.record()} + record["target"] = {"requested": name, "resolved": name, "matching": None} + emit(record) + if "returned" in done.outcome and plan.get("id"): + bind(plan["id"], done.value) + return "returned" in done.outcome + + +def student_call(plan, phase, index, student): + kind, spec = next(iter(plan["target"].items())) + files = plan.get("files") or [] + check = plan.get("check") if phase == "step" else None + request = { + "op": "call", + "target": plan["target"], + "timeout": plan["timeout"], + "stdin": plan.get("stdin"), + "id": plan.get("id"), + "files": files, + "serialise": phase != "setup", + "transfer": check is not None, + } + try: + request["args"] = outgoing(plan.get("args", [])) + if kind in ("method", "attribute") and spec["object"] not in held: + if spec["object"] not in context: + raise Unresolved(spec["object"]) + request["object"] = encode(context[spec["object"]]) + except Unresolved as exc: + emit({ + "kind": "call", "phase": phase, "index": index, + "target": {"requested": spec["name"], "resolved": spec["name"], "matching": None}, + "outcome": {"unresolved": {"name": str(exc)}}, + "stdout": "", "stdout_truncated": False, "elapsed_ms": 0, + }) + return False + try: + answer = student.ask(request, call_bound(len(files), transfer=check is not None), plan["timeout"] + GRACE) + try: + record = checked(answer, files) + except Garbled as exc: + raise student.garbled(str(exc)) from None + returned = "returned" in record["outcome"] + value, failed = (None, None) + if returned and check is not None: + value, failed = judged_value(answer, record, student) + except Stop as stop: + if stop.reason == "deadline": + # The call's own timer should have ended it: the call ran past its deadline. + emit({ + "kind": "call", "phase": phase, "index": index, + "target": {"requested": spec["name"], "resolved": spec["name"], "matching": None}, + "outcome": {"timeout": {}}, "stdout": "", "stdout_truncated": False, + "elapsed_ms": int((plan["timeout"] + GRACE) * 1000), + }) + stopped(phase, index, stop) + return None + emit({"kind": "call", "phase": phase, "index": index, **record}) + if returned and plan.get("id"): + bind(plan["id"], student=True) + if returned and check is not None: + try: + verdict = failed or run_check(check, value, record["stdout"], plan["timeout"], student) + except Stop as stop: + emit({"kind": "check", "index": index, "error": problem("StudentEnded", f"the student side stopped while the checker read its values: {stop.message}")}) + stopped(phase, index, stop) + return None + emit({"kind": "check", "index": index, **verdict}) + return returned + + +def run_calls(phase, calls, student): + """Run setup calls or steps in order. Returns False when a setup call failed, and None + when the student side was stopped.""" + for index, plan in enumerate(calls): + if "teacher" in plan["target"]: + returned = teacher_call(plan, phase, index, student) + else: + returned = student_call(plan, phase, index, student) + if returned is None: + return None + if phase == "setup" and not returned: + return False + return True + + +def run_load(student): + try: + answer = student.ask({"op": "load", "timeout": PAYLOAD["load_timeout"]}, call_bound(value=False), PAYLOAD["load_timeout"] + GRACE) + record = checked(answer, []) + except Garbled as exc: + stopped("load", 0, student.garbled(str(exc))) + return False + except Stop as stop: + if stop.reason == "deadline": + emit({"kind": "call", "phase": "load", "index": 0, "outcome": {"timeout": {}}, + "stdout": "", "stdout_truncated": False, "elapsed_ms": int((PAYLOAD["load_timeout"] + GRACE) * 1000)}) + stopped("load", 0, stop) + return False + emit({"kind": "call", "phase": "load", "index": 0, **record}) + return "returned" in record["outcome"] + + +def run_script(student): + script = PAYLOAD["script"] + files = script.get("files") or [] + try: + answer = student.ask( + {"op": "script", "timeout": script["timeout"], "stdin": script.get("stdin"), "files": files}, + call_bound(len(files), value=False), + script["timeout"] + GRACE, + ) + compile_error = answer.get("compiles") is False + record = checked(answer, [] if compile_error else files) + except Garbled as exc: + stopped("step", 0, student.garbled(str(exc))) + return + except Stop as stop: + if stop.reason == "deadline": + emit({"kind": "call", "phase": "step", "index": 0, "outcome": {"timeout": {}}, + "stdout": "", "stdout_truncated": False, "elapsed_ms": int((script["timeout"] + GRACE) * 1000)}) + stopped("step", 0, stop) + return + # A script that does not compile never ran: that is its load failing. + emit({"kind": "call", "phase": "load" if compile_error else "step", "index": 0, **record}) + + +def start_student(): + """Introduce the unit to the student side and wait for it to say it is ready.""" + student = Student(*channel(0, 1)) + init = { + "limits": PAYLOAD["limits"], + "role": PAYLOAD["role"], + "subject": PAYLOAD["subject"], + "vars": PAYLOAD.get("vars", {}), + "allowed_imports": PAYLOAD.get("allowed_imports", []) + + [os.path.splitext(os.path.basename(p))[0] for p in PAYLOAD.get("student_shared", [])], + "functions": PAYLOAD.get("functions", {"aliases": {}, "overrides": {}}), + "shared": PAYLOAD.get("student_shared", []), + "canary": PAYLOAD.get("student_canary"), + } + try: + answer = student.ask(init, SLACK + 6 * MESSAGE_LIMIT, STARTUP) + except Stop as stop: + emit({"kind": "fatal", "stage": "student", "error": problem("StudentStart", f"the student side did not start: {stop.message}")}) + finish() + if answer.get("ready") is not True: + fatal = answer.get("fatal") + error = fatal if isinstance(fatal, dict) else {} + stage = "sandbox" if error.get("type") == "Sandbox" else "student" + emit({"kind": "fatal", "stage": stage, "error": problem(str(error.get("type", "StudentStart")), str(error.get("message", "the student side did not start")))}) + finish() + return student + + +def canary_holds(): + path = PAYLOAD.get("canary") + if not path: + return True + try: + with open(path, "rb"): + return False + except OSError: + return True + + +def main(): + global SOURCE + if not canary_holds(): + emit({"kind": "fatal", "stage": "sandbox", "error": problem("Sandbox", "the teacher sandbox did not hold: its canary was readable")}) + finish() + try: + exports, duplicates = load_teacher_modules() + except BaseException as exc: + emit({"kind": "fatal", "stage": "teacher_import", "error": error_of(exc)}) + finish() + if PAYLOAD["mode"] == "inspect": + emit({"kind": "inspect", "duplicates": duplicates, "exports": { + name: {"callable": callable(value), "params": params_of(value) if callable(value) else None} + for name, value in exports.items() + }}) + finish() + context.update(exports) + context.update(PAYLOAD.get("vars", {})) + if PAYLOAD.get("source"): + with open(PAYLOAD["source"], encoding="utf-8", errors="replace") as fh: + SOURCE = fh.read() + student = start_student() + emit({"kind": "ready"}) + if PAYLOAD.get("script"): + run_script(student) + finish(student) + if run_load(student): + setup = run_calls("setup", PAYLOAD.get("setup", []), student) + if setup: + run_calls("step", PAYLOAD.get("steps", []), student) + finish(student) + + +try: + main() +except BaseException as exc: # a harness bug, not anybody's code under test + emit({"kind": "fatal", "stage": "harness", "error": error_of(exc)}) + finish() diff --git a/src/core/src/runner/judge.rs b/src/core/src/runner/judge.rs index 0b23342..253ce1e 100644 --- a/src/core/src/runner/judge.rs +++ b/src/core/src/runner/judge.rs @@ -1,8 +1,9 @@ //! Turning what a unit did into results. //! -//! The harness only observes. Here every non-pass gets its owner, derived from three -//! things: which call the plan says was running, what kind of code that call runs, and -//! how the process ended. Nothing a student can print decides it. +//! The harness only observes. Here every non-pass gets its owner, derived from four +//! things: which call the plan says was running, what kind of code that call runs, whether +//! the teacher side gave up on the student side there, and how each side ended. Nothing a +//! student can print decides it. use serde_json::{Value, json}; @@ -14,7 +15,8 @@ use crate::models::{ CaseInput, CaseResult, Cause, Check, FailureDetail, Fault, Target, TestCase, TestStatus, }; use crate::runner::executor::{ - CallObservation, CheckObservation, Exit, Outcome, ProtocolError, UnitObservation, UnitPlan, + CallObservation, CheckObservation, Exit, Outcome, Phase, ProtocolError, StopReason, Stopped, + UnitObservation, UnitPlan, }; /// A scored call: the name its result is reported under, and what it expects. @@ -50,6 +52,8 @@ pub fn judge( } // The harness broke after student code ran: what it recorded before that still stands. let broken = stream_break(obs); + // The teacher side gave up on the student side at a step: nothing after it ran. + let stop = obs.stopped.as_ref().filter(|s| s.phase == Phase::Step); let mut results = Vec::with_capacity(scored.len()); // Once a call hangs or the process dies, nothing after it ran — on whoever's account. @@ -69,7 +73,12 @@ pub fn judge( { // The student's call finished; the teacher's checker never did. let blanket = broken.clone().unwrap_or_else(|| { - in_flight(&obs.exit, Fault::Teacher, Cause::Checker, "its checker") + in_flight( + &obs.teacher_exit, + Fault::Teacher, + Cause::Checker, + "its checker", + ) }); results.push(blanket_result(s, &blanket, target(index))); stopped = Some(broken.clone().unwrap_or_else(|| not_run(&blanket, s.name))); @@ -89,19 +98,31 @@ pub fn judge( )); } None => { - let blanket = match &broken { - Some(broken) => broken.clone(), - None if obs.done => Blanket { + let blanket = match (&broken, stop) { + (Some(broken), _) => broken.clone(), + (None, Some(stop)) if stop.index < index => { + // It stopped at a call that reported: its record says how. + let at = scored.get(stop.index).map_or("?", |s| s.name); + not_run(&student_stop(obs, stop, "this call"), at) + } + (None, Some(stop)) => student_stop(obs, stop, "this call"), + (None, None) if obs.done => Blanket { status: TestStatus::Error, fault: Fault::Environment, cause: Cause::Harness, message: "the harness finished without running this call".into(), }, - None => in_flight(&obs.exit, Fault::Student, Cause::Killed, "this call"), + (None, None) => unfinished( + obs, + call.map_or(Fault::Student, |c| owner_of(&c.target)), + Cause::Killed, + "this call", + ), }; results.push(blanket_result(s, &blanket, target(index))); stopped = Some(match &broken { Some(broken) => broken.clone(), + None if blanket.cause == Cause::NotRun => blanket.clone(), None => not_run(&blanket, s.name), }); } @@ -120,26 +141,44 @@ fn unit_failure(plan: &UnitPlan, obs: &UnitObservation) -> Option { message, }) }; - if let Exit::Spawn(message) = &obs.exit { + for exit in [&obs.teacher_exit, &obs.exit] { + if let Exit::Spawn(message) = exit { + return blanket( + TestStatus::Error, + Fault::Environment, + Cause::Spawn, + message.clone(), + ); + } + } + if let Some(ProtocolError::Tampered(problem)) = &obs.protocol_error { + // Only the teacher side writes records: student code cannot reach them. + return blanket( + TestStatus::Error, + Fault::Environment, + Cause::Harness, + format!("the unit's records cannot be trusted: {problem}"), + ); + } + if let Some(fatal) = obs.fatal.as_ref().filter(|f| f.stage == "sandbox") { return blanket( TestStatus::Error, Fault::Environment, Cause::Spawn, - message.clone(), + format!("refused to grade: {}", fatal.error.message), ); } - if let Some(ProtocolError::Tampered(problem)) = &obs.protocol_error { - // After `ready`, student code has had the chance to write; before it, nobody's has. - let fault = if obs.ready { - Fault::Student - } else { - Fault::Environment - }; + if let Some(fatal) = obs.fatal.as_ref().filter(|f| f.stage == "student") { return blanket( TestStatus::Error, - fault, - Cause::Protocol, - format!("the unit's records cannot be trusted: {problem}"), + Fault::Environment, + Cause::Harness, + format!( + "{}: {} ({})", + fatal.error.type_name, + fatal.error.message, + stderr_hint(obs) + ), ); } if let Some(fatal) = obs.fatal.as_ref().filter(|f| f.stage == "teacher_import") { @@ -172,7 +211,12 @@ fn unit_failure(plan: &UnitPlan, obs: &UnitObservation) -> Option { TestStatus::Error, fault, cause, - format!("{} {} ({})", death(&obs.exit).1, what, stderr_hint(obs)), + format!( + "{} {} ({})", + death(&obs.teacher_exit).1, + what, + stderr_hint(obs) + ), ); } if plan.script.is_some() { @@ -191,12 +235,11 @@ fn unit_failure(plan: &UnitPlan, obs: &UnitObservation) -> Option { match &obs.load { None if stream_break(obs).is_some() => return stream_break(obs), None => { - return Some(in_flight( - &obs.exit, - Fault::Student, - Cause::Killed, - "the student module while it loaded", - )); + let what = "the student module while it loaded"; + return Some(match &obs.stopped { + Some(stop) => student_stop(obs, stop, what), + None => unfinished(obs, Fault::Student, Cause::Killed, what), + }); } Some(load) => match &load.outcome { Outcome::Returned { .. } => {} @@ -265,7 +308,13 @@ fn unit_failure(plan: &UnitPlan, obs: &UnitObservation) -> Option { if let Some(broken) = stream_break(obs) { return Some(broken); } - let killed = in_flight(&obs.exit, owner, Cause::Setup, &label); + let killed = match &obs.stopped { + Some(stop) => student_stop(obs, stop, &label), + None => unfinished(obs, owner, Cause::Setup, &label), + }; + if killed.fault != owner { + return Some(killed); + } return blanket( TestStatus::Error, owner, @@ -278,7 +327,7 @@ fn unit_failure(plan: &UnitPlan, obs: &UnitObservation) -> Option { None } -/// The verdict for the call in flight when the process stopped. +/// The verdict for the call in flight when a process stopped. fn in_flight(exit: &Exit, fault: Fault, cause: Cause, what: &str) -> Blanket { let (status, how) = death(exit); Blanket { @@ -289,11 +338,57 @@ fn in_flight(exit: &Exit, fault: Fault, cause: Cause, what: &str) -> Blanket { TestStatus::Error }, fault, - cause, + cause: if *exit == Exit::Memory { + Cause::Memory + } else { + cause + }, message: format!("{how} {what}"), } } +/// The verdict for the call during which the teacher side gave up on the student side. +fn student_stop(obs: &UnitObservation, stop: &Stopped, what: &str) -> Blanket { + match stop.reason { + StopReason::Exited => in_flight(&obs.exit, Fault::Student, Cause::Killed, what), + StopReason::Deadline => Blanket { + status: TestStatus::Timeout, + fault: Fault::Student, + cause: Cause::Killed, + message: format!("the student side stopped answering during {what}"), + }, + StopReason::Flooded | StopReason::Garbled => Blanket { + status: TestStatus::Error, + fault: Fault::Student, + cause: Cause::Protocol, + message: format!( + "the student side's answer could not be read during {what}: {}", + stop.message + ), + }, + } +} + +/// The verdict for a call whose record never came, when the teacher side did not say it +/// gave up on the student side: the teacher side itself ended. Teacher code in flight is +/// the teacher's; a student call in flight was waiting on the teacher side, which student +/// code cannot reach. +fn unfinished(obs: &UnitObservation, owner: Fault, cause: Cause, what: &str) -> Blanket { + if owner == Fault::Teacher { + return in_flight(&obs.teacher_exit, Fault::Teacher, cause, what); + } + let (_, how) = death(&obs.teacher_exit); + Blanket { + status: TestStatus::Error, + fault: Fault::Environment, + cause: Cause::Harness, + message: format!( + "the teacher side ended: {how} {what} ({})", + stderr_hint(obs) + ), + } +} + /// The verdict for scored calls after one that stopped the unit. fn not_run(culprit: &Blanket, after: &str) -> Blanket { Blanket { @@ -311,6 +406,14 @@ fn death(exit: &Exit) -> (TestStatus, String) { TestStatus::Timeout, "the unit's deadline passed during".into(), ), + Exit::Memory => ( + TestStatus::Error, + "the memory limit was exceeded during".into(), + ), + Exit::Stopped => ( + TestStatus::Error, + "the process was ended with its unit during".into(), + ), #[cfg(unix)] Exit::Signal(signal) if *signal == libc::SIGXCPU => ( TestStatus::Timeout, @@ -334,7 +437,7 @@ fn death(exit: &Exit) -> (TestStatus, String) { fn stderr_hint(obs: &UnitObservation) -> String { let tail = obs.stderr.trim(); if tail.is_empty() { - format!("{:?}", obs.exit) + format!("{:?}", obs.teacher_exit) } else { tail.lines().last().unwrap_or(tail).to_string() } @@ -362,17 +465,17 @@ fn describe(outcome: &Outcome, timeout: u64) -> String { } } -/// The record stream stopped meaning anything part-way: the harness crashed, wrote a -/// record it could not have meant, or somebody flooded the channel. What was recorded -/// before that stands; nothing after it can be known. +/// The record stream stopped meaning anything part-way: the harness crashed, or the +/// teacher side wrote a record it could not have meant. What was recorded before that +/// stands; nothing after it can be known. fn stream_break(obs: &UnitObservation) -> Option { let (fault, cause, message) = match (&obs.fatal, &obs.protocol_error) { (_, Some(ProtocolError::Flooded(problem))) => ( - Fault::Student, - Cause::Protocol, - format!("the unit's output could not be read past this point: {problem}"), + Fault::Environment, + Cause::Harness, + format!("the unit's records could not be read past this point: {problem}"), ), - (Some(fatal), _) if fatal.stage != "teacher_import" => ( + (Some(fatal), _) if fatal.stage == "harness" => ( Fault::Environment, Cause::Harness, format!( @@ -1099,8 +1202,14 @@ mod tests { case: &c, }, ]; + // The teacher side saw the student side go during 'b', and said so. let mut obs = finished(vec![record(0, returned(json!(1)))]); - obs.done = false; + obs.stopped = Some(Stopped { + phase: Phase::Step, + index: 1, + reason: StopReason::Exited, + message: "the student side ended".into(), + }); obs.exit = Exit::Signal(libc::SIGXCPU); let r = judge(&p, &scored, &obs, "python3"); assert_eq!(r[0].status, TestStatus::Passed); @@ -1123,6 +1232,76 @@ mod tests { verdict(&r[1]), (TestStatus::Error, Some(Fault::Student), Some(Cause::Killed)) ); + + obs.exit = Exit::Memory; + let r = judge(&p, &scored, &obs, "python3"); + assert_eq!( + verdict(&r[1]), + (TestStatus::Error, Some(Fault::Student), Some(Cause::Memory)) + ); + assert_eq!( + verdict(&r[2]), + (TestStatus::Error, Some(Fault::Student), Some(Cause::NotRun)) + ); + + // A step that answered too much, or nonsense, is the student's. + obs.stopped.as_mut().unwrap().reason = StopReason::Flooded; + let r = judge(&p, &scored, &obs, "python3"); + assert_eq!( + verdict(&r[1]), + ( + TestStatus::Error, + Some(Fault::Student), + Some(Cause::Protocol) + ) + ); + + // Given up on at its deadline: the teacher side reported the call as timed out, and + // the rest as never run. + let mut late = finished(vec![ + record(0, returned(json!(1))), + record(1, Outcome::Timeout {}), + ]); + late.stopped = Some(Stopped { + phase: Phase::Step, + index: 1, + reason: StopReason::Deadline, + message: String::new(), + }); + late.exit = Exit::Stopped; + let r = judge(&p, &scored, &late, "python3"); + assert_eq!( + verdict(&r[1]), + ( + TestStatus::Timeout, + Some(Fault::Student), + Some(Cause::Timeout) + ) + ); + assert_eq!( + verdict(&r[2]), + (TestStatus::Error, Some(Fault::Student), Some(Cause::NotRun)) + ); + assert!( + r[2].failure.as_ref().unwrap().message.contains("'b'"), + "{:?}", + r[2].failure + ); + + // Without a word from the teacher side, the teacher side itself went: the student + // cannot reach it. + let mut lost = finished(vec![record(0, returned(json!(1)))]); + lost.done = false; + lost.teacher_exit = Exit::Signal(libc::SIGKILL); + let r = judge(&p, &scored, &lost, "python3"); + assert_eq!( + verdict(&r[1]), + ( + TestStatus::Error, + Some(Fault::Environment), + Some(Cause::Harness) + ) + ); } #[test] @@ -1175,16 +1354,51 @@ mod tests { (TestStatus::Error, Some(Fault::Student), Some(Cause::Load)) ); + // Only the teacher side writes records: student code cannot have broken them. let mut tampered = finished(vec![record(0, returned(json!(1)))]); tampered.protocol_error = Some(ProtocolError::Tampered("step record 0 repeated".into())); assert_eq!( verdict(&one(&c, &p, &tampered)), ( TestStatus::Error, - Some(Fault::Student), - Some(Cause::Protocol) + Some(Fault::Environment), + Some(Cause::Harness) + ) + ); + + // A sandbox that did not hold grades nobody. + let mut leaky = UnitObservation::not_started(Exit::Code(0)); + leaky.fatal = Some(Fatal { + stage: "sandbox".into(), + error: ErrorInfo { + type_name: "Sandbox".into(), + types: vec![], + message: "the student sandbox did not hold".into(), + }, + }); + assert_eq!( + verdict(&one(&c, &p, &leaky)), + ( + TestStatus::Error, + Some(Fault::Environment), + Some(Cause::Spawn) ) ); + + // A student module that never finished loading because its side went away. + let mut gone = finished(vec![]); + gone.load = None; + gone.stopped = Some(Stopped { + phase: Phase::Load, + index: 0, + reason: StopReason::Exited, + message: String::new(), + }); + gone.exit = Exit::Signal(libc::SIGKILL); + assert_eq!( + verdict(&one(&c, &p, &gone)), + (TestStatus::Error, Some(Fault::Student), Some(Cause::Killed)) + ); } #[test] @@ -1435,15 +1649,15 @@ mod tests { ]; let mut obs = finished(vec![record(0, returned(json!(1)))]); obs.done = false; - obs.protocol_error = Some(ProtocolError::Flooded("64 MiB".into())); + obs.protocol_error = Some(ProtocolError::Flooded("a record of 64 MiB".into())); let r = judge(&p, &scored, &obs, "python3"); assert_eq!(r[0].status, TestStatus::Passed); assert_eq!( verdict(&r[1]), ( TestStatus::Error, - Some(Fault::Student), - Some(Cause::Protocol) + Some(Fault::Environment), + Some(Cause::Harness) ) ); } diff --git a/src/core/src/runner/python.rs b/src/core/src/runner/python.rs index ed65421..52f8973 100644 --- a/src/core/src/runner/python.rs +++ b/src/core/src/runner/python.rs @@ -1,12 +1,22 @@ +//! The Python backend. A unit runs as two processes joined by a pair of pipes: the teacher +//! side (`harness/teacher.py`) holds the teacher's modules and checkers, drives the plan +//! and writes the records; the student side (`harness/student.py`) holds only the +//! student's module. ScriptMark starts both, connects them to each other, and reads the +//! teacher side's records; it never relays what passes between them. + use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, LazyLock, Mutex}; +use std::time::Duration; -use tokio::process::Command; +use serde::Serialize; +use tokio::process::{Child, Command}; use crate::models::TestSpec; use crate::runner::executor::{ - Executor, Exit, ProtocolError, Subject, TeacherRuntime, UnitObservation, UnitPlan, + Executor, Exit, GRACE_SECS, STARTUP_SECS, Subject, TeacherRuntime, UnitObservation, UnitPlan, }; +use crate::runner::records::{Bounds, Reader, Records}; #[cfg(unix)] use crate::runner::sandbox::apply_sandbox; @@ -31,6 +41,8 @@ fn resolve_python_path(cmd: &str) -> String { /// Executor for Python student code. pub struct PythonExecutor { python_cmd: String, + /// The most memory the student side may hold before it is ended, in MiB. + memory_mb: u64, } impl PythonExecutor { @@ -41,61 +53,123 @@ impl PythonExecutor { pub fn with_python_cmd(python_cmd: impl Into) -> Self { Self { python_cmd: resolve_python_path(&python_cmd.into()), + memory_mb: crate::runner::sandbox::SandboxConfig::default().mem_mb, } } + /// End the student side once it holds more than `mb` MiB. + pub fn with_memory_limit(mut self, mb: u64) -> Self { + self.memory_mb = mb; + self + } + pub fn python_cmd(&self) -> &str { &self.python_cmd } } -/// The one harness every unit runs under. See `harness.py`. -const HARNESS: &str = include_str!("harness.py"); +const COMMON: &str = include_str!("harness/common.py"); +const TEACHER: &str = include_str!("harness/teacher.py"); +const STUDENT: &str = include_str!("harness/student.py"); +const MATCHING: &str = include_str!("../matching.py"); + +/// Every line either side runs, for the frozen contract: a change to any of it can change +/// what a reference implementation is observed to do. +pub(crate) const PROTOCOL: [&str; 4] = [COMMON, TEACHER, STUDENT, MATCHING]; + +/// `python -c` puts the working directory first on `sys.path`: it goes before any import. +const PRELUDE: &str = "import sys; sys.path[:] = [p for p in sys.path if p]\n"; +static TEACHER_SCRIPT: LazyLock = LazyLock::new(|| format!("{PRELUDE}{COMMON}\n{TEACHER}")); +static STUDENT_SCRIPT: LazyLock = + LazyLock::new(|| format!("{PRELUDE}{MATCHING}\n{COMMON}\n{STUDENT}")); + +/// What one call's record may carry, which both sides are given rather than assume. +#[derive(Debug, Clone, Copy, Serialize)] +struct Limits { + stdout: usize, + file: usize, + value: usize, + message: usize, + /// The most a value handed to the other side's code may take, as JSON. + transfer: usize, + depth: usize, +} + +const LIMITS: Limits = Limits { + stdout: 64 << 10, + file: 1 << 20, + value: 4 << 20, + message: 64 << 10, + transfer: 32 << 20, + depth: 100, +}; + +/// The most a record carries besides its value, stdout, message and files: its target, +/// the matching evidence, and the keys around them. +const SLACK: usize = 1 << 20; /// The tail of stderr kept for diagnosing a crash. const STDERR_TAIL: usize = 4096; -/// The descriptor the harness writes records to. On unix it is a pipe of its own, so -/// nothing a student does to stdout — rewrapping it, writing to `sys.__stdout__`, printing -/// from a thread — can reach the records. Elsewhere it is stdout, framed by the nonce. +/// How long the student side has to finish by itself once the teacher side has: it was +/// told to, and only a hung or stopped student side takes longer. +const LINGER: Duration = Duration::from_millis(500); + +/// The descriptor the teacher side writes records to. On unix it is a pipe of its own; +/// elsewhere stderr, since stdin and stdout are its pipe to the student side. #[cfg(unix)] const RECORD_FD: i32 = 3; #[cfg(not(unix))] -const RECORD_FD: i32 = 1; - -/// What a unit's records may total before the channel counts as flooded, which is also -/// what the grader buffers for it at most: this floor, plus the most each call carries. -const BASE_CAP: usize = 64 << 20; -/// The most one call's records carry, measured: a 4 MiB value, plus 64 KiB each of -/// stdout and checker message, which JSON escaping can grow sixfold (`harness.py`'s -/// `VALUE_LIMIT`, `STDOUT_LIMIT` and `MESSAGE_LIMIT`). -const CALL_BOUND: usize = 6 << 20; -/// The most one observed file adds: `FILE_LIMIT` characters, escaped. -const FILE_BOUND: usize = 7 << 20; - -/// The record channel's cap for `plan`: a correct submission never reaches it, however -/// many calls the unit makes. -fn record_cap(plan: &UnitPlan) -> usize { - let files = |n: usize| n.saturating_mul(FILE_BOUND); - let script = plan.script.as_ref().map_or(0, |s| files(s.files.len())); - plan.setup +const RECORD_FD: i32 = 2; + +/// What the teacher side's record stream may hold for `plan`: its largest record — a call +/// with its value, stdout, an error message and every file it observes, each of which JSON +/// escaping can grow sixfold — and one record per call and check, plus the frame around +/// them. +fn record_bounds(plan: &UnitPlan) -> Bounds { + let files = plan + .setup .iter() .chain(&plan.steps) - .map(|c| CALL_BOUND.saturating_add(files(c.files.len()))) - .fold(BASE_CAP.saturating_add(script), usize::saturating_add) + .map(|c| c.files.len()) + .chain(plan.script.as_ref().map(|s| s.files.len())) + .max() + .unwrap_or(0); + let per_file = 6 * LIMITS.file + 4096; + Bounds { + frame: (SLACK + LIMITS.value + 6 * (LIMITS.stdout + LIMITS.message)) + .saturating_add(files.saturating_mul(per_file)), + // ready, load, done, and a fatal or a stop, besides the calls and checks. + count: 5 + plan.setup.len() + 2 * plan.steps.len().max(1), + } } -/// A unit's private directory: the payload beside a working directory the student runs in. -struct Staged { - /// Removed on drop. - root: tempfile::TempDir, +/// What an inspection may report: every export and its parameters. +const INSPECT_BOUNDS: Bounds = Bounds { + frame: 64 << 20, + count: 4, +}; + +/// One side's private directories. +struct Side { work: PathBuf, - /// The unit's own `HOME` and `TMPDIR`, so nothing one unit leaves there reaches another. + /// Its own `HOME` and `TMPDIR`, so nothing one unit leaves there reaches another. home: PathBuf, tmp: PathBuf, +} + +/// A unit's private directory: one working directory per side, the teacher side's +/// payload, and a read-only copy of the subject's source. +struct Staged { + /// Removed on drop. + root: tempfile::TempDir, + teacher: Side, + student: Side, payload: PathBuf, - /// The subject file's copy inside `work`. - subject: PathBuf, + /// The subject file's copy in the student side's working directory. + subject: Option, + /// The subject's source, for teacher code to read. + source: Option, } /// Copy `src` (a file or a directory) to `dest`, creating parents. @@ -115,171 +189,265 @@ fn copy_into(src: &Path, dest: &Path) -> std::io::Result<()> { } } +fn side(root: &Path, name: &str) -> std::io::Result { + let side = Side { + work: root.join(name), + home: root.join(format!("{name}-home")), + tmp: root.join(format!("{name}-tmp")), + }; + for dir in [&side.work, &side.home, &side.tmp] { + std::fs::create_dir(dir)?; + } + Ok(side) +} + fn stage( subject: Option<&Path>, data_files: &[(PathBuf, String)], - payload: impl FnOnce(Option<&Path>) -> serde_json::Value, + payload: impl FnOnce(&Staged) -> serde_json::Value, ) -> std::io::Result { let root = tempfile::Builder::new() .prefix("scriptmark-unit-") .tempdir()?; - let work = root.path().join("work"); - let home = root.path().join("home"); - let tmp = root.path().join("tmp"); - for dir in [&work, &home, &tmp] { - std::fs::create_dir(dir)?; - } + let teacher = side(root.path(), "teacher")?; + let student = side(root.path(), "student")?; + // Teacher code that opens a data file by its relative path finds it, as student code does. for (src, rel) in data_files { - copy_into(src, &work.join(rel))?; + copy_into(src, &student.work.join(rel))?; + copy_into(src, &teacher.work.join(rel))?; } - let copied = match subject { + let (copied, source) = match subject { Some(file) => { let name = file .file_name() .unwrap_or(std::ffi::OsStr::new("subject.py")); - let dest = work.join(name); + let dest = student.work.join(name); std::fs::copy(file, &dest)?; - Some(dest) + let sources = root.path().join("source"); + std::fs::create_dir(&sources)?; + let source = sources.join(name); + std::fs::copy(file, &source)?; + let mut readonly = std::fs::metadata(&source)?.permissions(); + readonly.set_readonly(true); + std::fs::set_permissions(&source, readonly)?; + (Some(dest), Some(source)) } - None => None, + None => (None, None), }; - let payload_path = root.path().join("payload.json"); - std::fs::write(&payload_path, payload(copied.as_deref()).to_string())?; - Ok(Staged { - work, - home, - tmp, - payload: payload_path, - subject: copied.unwrap_or_default(), + let staged = Staged { + // The teacher side's argument: it reads the payload and removes it. + payload: teacher.work.join("payload.json"), + teacher, + student, + subject: copied, + source, root, - }) -} - -fn nonce() -> String { - use rand::Rng; - let bytes: [u8; 16] = rand::rng().random(); - bytes.iter().map(|b| format!("{b:02x}")).collect() + }; + std::fs::write(&staged.payload, payload(&staged).to_string())?; + Ok(staged) } -/// What a finished harness process left behind. +/// What a finished unit left behind. struct Finished { - stdout: String, - /// The process wrote more than its cap to the record channel. - stdout_overflow: bool, - stderr: String, + records: Records, exit: Exit, + teacher_exit: Exit, + stderr: String, } -impl PythonExecutor { - /// Run the harness on a staged unit, killing it at `deadline_secs`. - async fn run_harness( - &self, - staged: &Staged, - deadline_secs: u64, - stdin: Option, - cap: usize, - ) -> Finished { - use tokio::io::AsyncWriteExt; +impl Finished { + fn spawn_failed(message: String) -> Self { + Self { + records: Records::default(), + exit: Exit::Spawn(message.clone()), + teacher_exit: Exit::Spawn(message), + stderr: String::new(), + } + } +} +impl PythonExecutor { + /// A command for one side: a cleared environment, the side's own `HOME` and `TMPDIR`, + /// and the kernel's limits. + fn command(&self, script: &str, side: &Side, deadline_secs: u64) -> Command { let mut cmd = Command::new(&self.python_cmd); cmd.env_clear() .env("PATH", "/usr/bin:/usr/local/bin:/opt/homebrew/bin") - .env("HOME", &staged.home) - .env("TMPDIR", &staged.tmp) + .env("HOME", &side.home) + .env("TMPDIR", &side.tmp) // Isolated: no user site-packages, whose `.pth` files run before the harness, no - // PYTHON* variables, and the unit's directory, which holds the student's file, - // off sys.path. What those variables used to set is spelled as flags. - .args(["-I", "-B", "-X", "utf8"]) - .arg("-c") - .arg(format!( - "import sys; sys.path[:] = [p for p in sys.path if p]\n{}\n{}", - include_str!("../matching.py"), - HARNESS - )) - .arg(&staged.payload) - .current_dir(&staged.work) - .stdin(if stdin.is_some() { - std::process::Stdio::piped() - } else { - std::process::Stdio::null() - }) - .stderr(std::process::Stdio::piped()) + // PYTHON* variables, and the working directory off sys.path. What those variables + // used to set is spelled as flags. + .args(["-I", "-B", "-X", "utf8", "-c", script]) + .current_dir(&side.work) .kill_on_drop(true); - let spawn_failed = |message: String| Finished { - stdout: String::new(), - stdout_overflow: false, - stderr: String::new(), - exit: Exit::Spawn(message), - }; + #[cfg(windows)] + if let Some(root) = std::env::var_os("SYSTEMROOT") { + // Python cannot seed its hash randomisation on Windows without it. + cmd.env("SYSTEMROOT", root); + } #[cfg(unix)] - let records = { + { // Its own process group, so anything it spawns dies with it. cmd.process_group(0); apply_sandbox(&mut cmd, &sandbox_for(deadline_secs)); - // The student's stdout goes nowhere: what they print during a call is captured - // in-process, and the records have a pipe of their own. - cmd.stdout(std::process::Stdio::null()); - match record_pipe(&mut cmd) { + } + #[cfg(not(unix))] + let _ = deadline_secs; + cmd + } + + /// Run a staged unit — the teacher side, and the student side when `with_student` — + /// and end both at `deadline_secs`. + async fn run_unit( + &self, + staged: &Staged, + deadline_secs: u64, + bounds: Bounds, + with_student: bool, + ) -> Finished { + let mut teacher_cmd = self.command(&TEACHER_SCRIPT, &staged.teacher, deadline_secs); + teacher_cmd.arg(&staged.payload); + let mut student = None; + if with_student { + // teacher → student, and student → teacher. + let pipes = std::io::pipe().and_then(|a| Ok((a, std::io::pipe()?))); + let ((to_student, from_teacher), (to_teacher, from_student)) = match pipes { + Ok(pipes) => pipes, + Err(e) => { + return Finished::spawn_failed(format!("could not join the two sides: {e}")); + } + }; + let mut cmd = self.command(&STUDENT_SCRIPT, &staged.student, deadline_secs); + cmd.stdin(std::process::Stdio::from(to_student)) + .stdout(std::process::Stdio::from(from_student)) + .stderr(std::process::Stdio::piped()); + // The command holds the parent's copies of the student's ends: they go with it. + match cmd.spawn() { + Ok(child) => student = Some(child), + Err(e) => { + return Finished::spawn_failed(format!( + "could not start {}: {e}", + self.python_cmd + )); + } + } + drop(cmd); + teacher_cmd + .stdin(std::process::Stdio::from(to_teacher)) + .stdout(std::process::Stdio::from(from_teacher)); + } else { + teacher_cmd + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()); + } + #[cfg(unix)] + let records = { + teacher_cmd.stderr(std::process::Stdio::piped()); + match record_pipe(&mut teacher_cmd) { Ok(pipe) => pipe, - Err(e) => return spawn_failed(format!("could not open the record channel: {e}")), + Err(e) => { + return Finished::spawn_failed(format!( + "could not open the record channel: {e}" + )); + } } }; #[cfg(not(unix))] - cmd.stdout(std::process::Stdio::piped()); + teacher_cmd.stderr(std::process::Stdio::piped()); - let mut child = match cmd.spawn() { + let mut teacher = match teacher_cmd.spawn() { Ok(child) => child, - Err(e) => return spawn_failed(format!("could not start {}: {e}", self.python_cmd)), + Err(e) => { + return Finished::spawn_failed(format!("could not start {}: {e}", self.python_cmd)); + } }; - // The parent's copy of the write end goes, so the channel ends when the unit does. + // The parent's copies of every pipe end go, so each pipe ends when its side does. + drop(teacher_cmd); #[cfg(unix)] let records = { let (reader, writer) = records; drop(writer); match tokio::net::unix::pipe::Receiver::from_owned_fd(reader.into()) { Ok(receiver) => receiver, - Err(e) => return spawn_failed(format!("could not read the record channel: {e}")), + Err(e) => { + return Finished::spawn_failed(format!( + "could not read the record channel: {e}" + )); + } } }; #[cfg(not(unix))] - let records = child.stdout.take(); + let records = teacher.stderr.take(); + // Killed however this future ends: finished, timed out, or dropped by a caller. - let _group = GroupGuard::new(child.id()); + let _teacher_group = GroupGuard::new(teacher.id()); + let student_group = GroupGuard::new(student.as_ref().and_then(Child::id)); - if let (Some(text), Some(mut pipe)) = (stdin, child.stdin.take()) { - tokio::spawn(async move { - // A script that never reads its stdin closes the pipe; that is not an error. - let _ = pipe.write_all(text.as_bytes()).await; - }); - } - let stdout = Arc::new(Mutex::new(Sink::head(cap))); - let stderr = Arc::new(Mutex::new(Sink::tail(STDERR_TAIL))); + let reader = Arc::new(Mutex::new(Reader::new(bounds))); + let teacher_tail = Arc::new(Mutex::new(Sink::tail(STDERR_TAIL))); + let student_tail = Arc::new(Mutex::new(Sink::tail(STDERR_TAIL))); let mut pumps = tokio::task::JoinSet::new(); #[cfg(unix)] - pumps.spawn(pump(records, stdout.clone())); + pumps.spawn(pump_records(records, reader.clone())); #[cfg(not(unix))] if let Some(pipe) = records { - pumps.spawn(pump(pipe, stdout.clone())); + pumps.spawn(pump_records(pipe, reader.clone())); + } + #[cfg(unix)] + if let Some(pipe) = teacher.stderr.take() { + pumps.spawn(pump(pipe, teacher_tail.clone())); } - if let Some(pipe) = child.stderr.take() { - pumps.spawn(pump(pipe, stderr.clone())); + let over_memory = Arc::new(AtomicBool::new(false)); + if let Some(child) = student.as_mut() { + if let Some(pipe) = child.stderr.take() { + pumps.spawn(pump(pipe, student_tail.clone())); + } + if let Some(pid) = child.id() { + let limit = self.memory_mb.saturating_mul(1 << 20); + let over = over_memory.clone(); + let group = student_group.0; + pumps.spawn(async move { watch_memory(pid, group, limit, over).await }); + } } - let exit = - match tokio::time::timeout(std::time::Duration::from_secs(deadline_secs), child.wait()) - .await - { - Ok(Ok(status)) => exit_of(status), - Ok(Err(e)) => Exit::Spawn(format!("lost the process: {e}")), - Err(_) => Exit::Deadline, - }; - // Whatever is left of the group — the harness at its deadline, or a process a student - // left behind holding the pipes — goes now. - _group.kill(); - let _ = child.kill().await; + let deadline = Duration::from_secs(deadline_secs); + let teacher_exit = match tokio::time::timeout(deadline, teacher.wait()).await { + Ok(Ok(status)) => exit_of(status), + Ok(Err(e)) => Exit::Spawn(format!("lost the teacher side: {e}")), + Err(_) => Exit::Deadline, + }; + let exit = match student.as_mut() { + None => Exit::Code(0), + Some(child) => { + let linger = if teacher_exit == Exit::Deadline { + Duration::ZERO + } else { + LINGER + }; + let ended = tokio::time::timeout(linger, child.wait()).await; + let over = || over_memory.load(Ordering::SeqCst); + match ended { + Ok(Ok(_)) if over() => Exit::Memory, + Ok(Ok(status)) => exit_of(status), + Ok(Err(e)) => Exit::Spawn(format!("lost the student side: {e}")), + Err(_) if over() => Exit::Memory, + Err(_) if teacher_exit == Exit::Deadline => Exit::Deadline, + Err(_) => Exit::Stopped, + } + } + }; + // Whatever is left of either side — a hung student side, or a process either left + // behind holding a pipe — goes now. + drop(_teacher_group); + drop(student_group); + let _ = teacher.kill().await; + if let Some(child) = student.as_mut() { + let _ = child.kill().await; + } // What was read stays read: a pipe held open past this only loses what comes after. - let drain = tokio::time::sleep(std::time::Duration::from_secs(1)); + let drain = tokio::time::sleep(Duration::from_secs(1)); tokio::pin!(drain); loop { tokio::select! { @@ -290,20 +458,27 @@ impl PythonExecutor { } } } - let (stdout, stdout_overflow) = { - let sink = stdout.lock().expect("stdout sink poisoned"); - ( - String::from_utf8_lossy(&sink.bytes).into_owned(), - sink.overflow, - ) + let records = std::mem::replace( + &mut *reader.lock().expect("record reader poisoned"), + Reader::new(bounds), + ) + .finish(); + let tail = |sink: &Arc>| { + String::from_utf8_lossy(&sink.lock().expect("stderr sink poisoned").bytes) + .trim() + .to_string() + }; + let (teacher_err, student_err) = (tail(&teacher_tail), tail(&student_tail)); + let stderr = match (teacher_err.is_empty(), student_err.is_empty()) { + (_, true) => teacher_err, + (true, false) => format!("student side: {student_err}"), + (false, false) => format!("{teacher_err}\nstudent side: {student_err}"), }; - let stderr = String::from_utf8_lossy(&stderr.lock().expect("stderr sink poisoned").bytes) - .into_owned(); Finished { - stdout, - stdout_overflow, - stderr, + records, exit, + teacher_exit, + stderr, } } } @@ -334,6 +509,78 @@ fn record_pipe(cmd: &mut Command) -> std::io::Result<(std::io::PipeReader, std:: Ok((reader, writer)) } +/// Read frames as they arrive. Past a frame the stream may not hold, keep reading and +/// drop what comes, so a teacher side still writing can finish. +async fn pump_records(mut pipe: R, reader: Arc>) { + use tokio::io::AsyncReadExt; + let mut header = [0u8; 4]; + loop { + if pipe.read_exact(&mut header).await.is_err() { + break; + } + let size = u32::from_be_bytes(header) as usize; + if !reader.lock().expect("record reader poisoned").admits(size) { + let mut sink = tokio::io::sink(); + let _ = tokio::io::copy(&mut pipe, &mut sink).await; + break; + } + let mut body = vec![0u8; size]; + // A frame cut off by a kill is the end of the stream, not a forgery. + if pipe.read_exact(&mut body).await.is_err() { + break; + } + reader.lock().expect("record reader poisoned").frame(&body); + } +} + +/// End the student side once it holds more than `limit` bytes: the operating system's +/// sandbox cannot bound its memory everywhere, so the grader watches it. +async fn watch_memory(pid: u32, group: Option, limit: u64, over: Arc) { + let mut tick = tokio::time::interval(Duration::from_millis(50)); + while let Some(held) = resident_bytes(pid) { + if held > limit { + over.store(true, Ordering::SeqCst); + if let Some(group) = group { + kill_group(group); + } + return; + } + tick.tick().await; + } +} + +/// The memory `pid` holds, or None once it is gone. +#[cfg(target_os = "macos")] +fn resident_bytes(pid: u32) -> Option { + let mut info: libc::proc_taskinfo = unsafe { std::mem::zeroed() }; + let size = std::mem::size_of::() as libc::c_int; + // SAFETY: `info` is a properly sized, writable proc_taskinfo. + let written = unsafe { + libc::proc_pidinfo( + i32::try_from(pid).ok()?, + libc::PROC_PIDTASKINFO, + 0, + (&mut info as *mut libc::proc_taskinfo).cast(), + size, + ) + }; + (written == size).then_some(info.pti_resident_size) +} + +#[cfg(target_os = "linux")] +fn resident_bytes(pid: u32) -> Option { + let statm = std::fs::read_to_string(format!("/proc/{pid}/statm")).ok()?; + let pages: u64 = statm.split_whitespace().nth(1)?.parse().ok()?; + // SAFETY: sysconf only reads a constant. + let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + Some(pages.saturating_mul(u64::try_from(page).ok()?)) +} + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +fn resident_bytes(_pid: u32) -> Option { + None +} + /// The sandbox for a unit: its CPU limit sits above its deadline, a backstop for one busy /// core rather than a second, shorter timeout. fn sandbox_for(deadline_secs: u64) -> crate::runner::sandbox::SandboxConfig { @@ -355,7 +602,7 @@ pub fn kill_all_units() { } } -/// A unit's process group: registered while it lives, killed when the guard drops. +/// A side's process group: registered while it lives, killed when the guard drops. struct GroupGuard(Option); impl GroupGuard { @@ -369,12 +616,6 @@ impl GroupGuard { } Self(group) } - - fn kill(&self) { - if let Some(group) = self.0 { - kill_group(group); - } - } } impl Drop for GroupGuard { @@ -400,43 +641,25 @@ fn kill_group(group: i32) { #[cfg(not(unix))] fn kill_group(_group: i32) {} -/// A pipe's contents, bounded: the first `cap` bytes, or only the last. +/// A pipe's contents, bounded: the last `cap` bytes. struct Sink { bytes: Vec, cap: usize, - keep_tail: bool, - overflow: bool, } impl Sink { - fn head(cap: usize) -> Self { + fn tail(cap: usize) -> Self { Self { bytes: Vec::new(), cap, - keep_tail: false, - overflow: false, - } - } - - fn tail(cap: usize) -> Self { - Self { - keep_tail: true, - ..Self::head(cap) } } fn push(&mut self, chunk: &[u8]) { - if self.keep_tail { - self.bytes.extend_from_slice(chunk); - if self.bytes.len() > self.cap { - let excess = self.bytes.len() - self.cap; - self.bytes.drain(..excess); - } - } else { - let room = self.cap.saturating_sub(self.bytes.len()); - self.bytes - .extend_from_slice(&chunk[..room.min(chunk.len())]); - self.overflow |= chunk.len() > room; + self.bytes.extend_from_slice(chunk); + if self.bytes.len() > self.cap { + let excess = self.bytes.len() - self.cap; + self.bytes.drain(..excess); } } } @@ -471,6 +694,24 @@ async fn discard(staged: Staged) { let _ = tokio::task::spawn_blocking(move || drop(staged.root)).await; } +/// The teacher side's payload, common to every unit of a spec. +fn base_payload(mode: &str) -> serde_json::Value { + serde_json::json!({ + "mode": mode, + "records": RECORD_FD, + "limits": LIMITS, + "grace": GRACE_SECS, + "startup": STARTUP_SECS, + }) +} + +fn extend(mut base: serde_json::Value, more: serde_json::Value) -> serde_json::Value { + if let (Some(base), serde_json::Value::Object(more)) = (base.as_object_mut(), more) { + base.extend(more); + } + base +} + impl Executor for PythonExecutor { fn language(&self) -> &str { "python" @@ -490,29 +731,27 @@ impl Executor for PythonExecutor { if spec.meta.imports.is_empty() { return Ok(TeacherRuntime::default()); } - let nonce = nonce(); let data = spec .meta .data_files .iter() .map(|rel| (spec.dir.join(rel), rel.clone())) .collect::>(); - let payload = serde_json::json!({ - "nonce": nonce, - "channel": RECORD_FD, - "mode": "inspect", - "imports": spec.meta.imports, - }); + let payload = extend( + base_payload("inspect"), + serde_json::json!({ "imports": spec.meta.imports }), + ); let staged = tokio::task::spawn_blocking(move || stage(None, &data, |_| payload)) .await .map_err(|e| e.to_string())? .map_err(|e| format!("could not stage the teacher modules: {e}"))?; + let deadline = timeout_secs.saturating_add(STARTUP_SECS); let finished = self - .run_harness(&staged, timeout_secs + 2, None, BASE_CAP) + .run_unit(&staged, deadline, INSPECT_BOUNDS, false) .await; discard(staged).await; - let records = crate::runner::records::parse(&finished.stdout, &nonce); + let records = finished.records; if let Some(fatal) = records.fatal { return Err(format!( "teacher module failed to import: {}: {}", @@ -521,7 +760,7 @@ impl Executor for PythonExecutor { } match records.inspect { Some(runtime) => Ok(runtime), - None => Err(match finished.exit { + None => Err(match finished.teacher_exit { Exit::Deadline => { format!("importing the teacher modules took longer than {timeout_secs}s") } @@ -535,35 +774,38 @@ impl Executor for PythonExecutor { } async fn run(&self, plan: &UnitPlan) -> UnitObservation { - let nonce = nonce(); let data = plan.data_files.clone(); let file = plan.file.clone(); let payload = { - let nonce = nonce.clone(); let plan = plan.clone(); - move |subject: Option<&Path>| { - serde_json::json!({ - "nonce": nonce, - "channel": RECORD_FD, - "mode": "unit", - "student": subject, - "script": plan.script.as_ref().map(|s| serde_json::json!({ - "timeout": s.timeout, - "files": s.files, - })), - "imports": plan.imports, - "vars": &*plan.vars, - "allowed_imports": plan.allowed_imports, - "load_timeout": plan.load_timeout, - "subject": plan.subject, - "functions": plan.functions, - "lookup": match plan.subject { - Subject::Student => "fuzzy", - Subject::Reference => "exact", - }, - "setup": plan.setup, - "steps": plan.steps, - }) + move |staged: &Staged| { + extend( + base_payload("unit"), + serde_json::json!({ + "role": match plan.subject { + Subject::Student => "student", + Subject::Reference => "reference", + }, + "subject": staged + .subject + .as_deref() + .and_then(Path::file_name) + .map(|n| n.to_string_lossy().into_owned()), + "source": staged.source, + "script": plan.script.as_ref().map(|s| serde_json::json!({ + "timeout": s.timeout, + "stdin": s.stdin, + "files": s.files, + })), + "imports": plan.imports, + "vars": &*plan.vars, + "allowed_imports": plan.allowed_imports, + "load_timeout": plan.load_timeout, + "functions": plan.functions, + "setup": plan.setup, + "steps": plan.steps, + }), + ) } }; let staged = match tokio::task::spawn_blocking(move || stage(Some(&file), &data, payload)) @@ -579,25 +821,14 @@ impl Executor for PythonExecutor { return UnitObservation::not_started(Exit::Spawn(format!("staging panicked: {e}"))); } }; - debug_assert!(staged.subject.is_file()); + debug_assert!(staged.subject.as_ref().is_some_and(|s| s.is_file())); - let stdin = plan - .script - .as_ref() - .map(|s| s.stdin.clone().unwrap_or_default()); - let cap = record_cap(plan); let finished = self - .run_harness(&staged, plan.deadline_secs(), stdin, cap) + .run_unit(&staged, plan.deadline_secs(), record_bounds(plan), true) .await; discard(staged).await; - let mut records = crate::runner::records::parse(&finished.stdout, &nonce); - if finished.stdout_overflow && records.protocol_error.is_none() { - records.protocol_error = Some(ProtocolError::Flooded(format!( - "more than {} MiB was written to the record channel", - cap >> 20 - ))); - } + let records = finished.records; UnitObservation { ready: records.ready, load: records.load, @@ -605,8 +836,10 @@ impl Executor for PythonExecutor { steps: records.steps, checks: records.checks, fatal: records.fatal, + stopped: records.stopped, done: records.done, exit: finished.exit, + teacher_exit: finished.teacher_exit, protocol_error: records.protocol_error, stderr: finished.stderr, } @@ -634,10 +867,11 @@ mod tests { assert_eq!(sandbox_for(u64::MAX).cpu_secs, u64::MAX, "saturates"); } - /// The cap grows with every call and observed file, so a long scenario of large but - /// legal values is never taken for a flood. + /// Each record is bounded by the largest a call can make, and the stream by the number + /// of records the plan can produce: a long scenario of large but legal values is never + /// taken for a flood, and nothing is buffered on a guess. #[test] - fn test_the_record_cap_grows_with_the_units_calls_and_files() { + fn test_record_bounds_follow_the_plan() { let step = |files: usize| CallPlan { target: crate::models::Target::Function { name: "f".into() }, args: Vec::new(), @@ -660,10 +894,13 @@ mod tests { setup: Vec::new(), steps, }; - assert_eq!(record_cap(&plan(Vec::new())), BASE_CAP); - let wide = record_cap(&plan(vec![step(1); 20])); + let plain = record_bounds(&plan(vec![step(0); 20])); + let filed = record_bounds(&plan(vec![step(2); 20])); // Measured: one call with a 4 MiB value, 64 KiB of escaped stdout, a failing check's // 64 KiB message and one 1 Mi-character file came to 10.75 MiB. - assert!(wide >= BASE_CAP + 20 * (43 << 18), "{wide}"); + assert!(filed.frame >= 43 << 18, "{}", filed.frame); + assert!(filed.frame > plain.frame); + assert_eq!(plain.count, filed.count); + assert!(plain.count >= 2 * 20 + 3); } } diff --git a/src/core/src/runner/records.rs b/src/core/src/runner/records.rs index 7d75dd9..a4e64d5 100644 --- a/src/core/src/runner/records.rs +++ b/src/core/src/runner/records.rs @@ -1,15 +1,17 @@ -//! Reading the harness's record stream. +//! Reading the teacher side's record stream. //! -//! Each record is one line, `@@scriptmark:@@ {json}`, and may follow whatever a -//! student wrote without a newline, so the prefix is looked for anywhere in a line. -//! Everything else on stdout is somebody else's output and is ignored. +//! Each record is a frame: a 4-byte big-endian length, then that many bytes of JSON. A +//! frame longer than the unit's largest legal record, or more records than its plan can +//! produce, ends the stream there; what arrived before it stands. Only the teacher side +//! writes here: the student side's answers reach a record after the teacher side has +//! checked them. use std::collections::BTreeMap; use serde::Deserialize; use crate::runner::executor::{ - CallObservation, CheckObservation, Export, Fatal, Phase, ProtocolError, TeacherRuntime, + CallObservation, CheckObservation, Export, Fatal, Phase, ProtocolError, Stopped, TeacherRuntime, }; #[derive(Deserialize)] @@ -26,6 +28,7 @@ enum Record { #[serde(flatten)] check: CheckObservation, }, + Stopped(Stopped), Fatal(Fatal), Inspect { exports: BTreeMap, @@ -35,7 +38,7 @@ enum Record { Done, } -/// The records of one run, in the order the harness wrote them. +/// The records of one run, in the order the teacher side wrote them. #[derive(Debug, Default)] pub struct Records { pub ready: bool, @@ -43,38 +46,80 @@ pub struct Records { pub setup: Vec, pub steps: Vec, pub checks: BTreeMap, + pub stopped: Option, pub fatal: Option, pub inspect: Option, pub done: bool, - /// The first record that repeated, arrived out of order, or did not parse. + /// The first record that was too long, repeated, arrived out of order, or did not parse. pub protocol_error: Option, } -pub fn parse(stdout: &str, nonce: &str) -> Records { - let prefix = format!("@@scriptmark:{nonce}@@ "); - let mut records = Records::default(); - let lines: Vec<&str> = stdout.split('\n').collect(); - for (i, line) in lines.iter().enumerate() { - let Some(at) = line.find(&prefix) else { - continue; - }; - if records.protocol_error.is_some() { - break; +/// What a stream may hold: the longest frame, and how many frames. +#[derive(Debug, Clone, Copy)] +pub struct Bounds { + pub frame: usize, + pub count: usize, +} + +/// Reads frames as they arrive. +#[derive(Debug)] +pub struct Reader { + bounds: Bounds, + seen: usize, + records: Records, +} + +impl Reader { + pub fn new(bounds: Bounds) -> Self { + Self { + bounds, + seen: 0, + records: Records::default(), + } + } + + /// Whether a frame announcing `size` bytes may be read. One that may not breaks the + /// stream, and nothing after it is read. + pub fn admits(&mut self, size: usize) -> bool { + if self.records.protocol_error.is_some() { + return false; + } + if size > self.bounds.frame { + self.records.protocol_error = Some(ProtocolError::Flooded(format!( + "a record of {size} bytes, longer than any this unit produces ({} bytes)", + self.bounds.frame + ))); + return false; } - let body = &line[at + prefix.len()..]; - match serde_json::from_str::(body) { - Ok(record) => records.accept(record), - // A record cut off by a kill is the end of the stream, not a forgery. - Err(_) if i + 1 == lines.len() => {} + true + } + + pub fn frame(&mut self, body: &[u8]) { + if self.records.protocol_error.is_some() { + return; + } + self.seen += 1; + if self.seen > self.bounds.count { + self.records.protocol_error = Some(ProtocolError::Flooded(format!( + "more than the {} records this unit can produce", + self.bounds.count + ))); + return; + } + match serde_json::from_slice::(body) { + Ok(record) => self.records.accept(record), Err(e) => { - records.protocol_error = Some(ProtocolError::Unreadable(format!( + self.records.protocol_error = Some(ProtocolError::Unreadable(format!( "unreadable record ({e}): {}", - body.chars().take(200).collect::() + String::from_utf8_lossy(&body[..body.len().min(200)]) ))) } } } - records + + pub fn finish(self) -> Records { + self.records + } } impl Records { @@ -86,6 +131,9 @@ impl Records { if self.done { return self.tampered("a record arrived after 'done'"); } + if self.stopped.is_some() && !matches!(record, Record::Done | Record::Fatal(_)) { + return self.tampered("a record arrived after the student side was stopped"); + } match record { Record::Ready if self.ready => self.tampered("'ready' repeated"), Record::Ready => self.ready = true, @@ -120,11 +168,12 @@ impl Records { self.checks.insert(index, check); } } + Record::Stopped(stopped) => self.stopped = Some(stopped), // A teacher module is imported before 'ready'; a fatal claiming otherwise, or a - // second fatal, did not come from the harness. + // second fatal, is not the harness's. Record::Fatal(_) if self.fatal.is_some() => self.tampered("a second 'fatal'"), - Record::Fatal(fatal) if self.ready && fatal.stage == "teacher_import" => { - self.tampered("a teacher_import 'fatal' after 'ready'") + Record::Fatal(fatal) if self.ready && fatal.stage != "harness" => { + self.tampered(format!("a {} 'fatal' after 'ready'", fatal.stage)) } Record::Fatal(fatal) => self.fatal = Some(fatal), Record::Inspect { @@ -144,26 +193,34 @@ impl Records { #[cfg(test)] mod tests { use super::*; - use crate::runner::executor::Outcome; + use crate::runner::executor::{Outcome, StopReason}; - const N: &str = "abc123"; + const WIDE: Bounds = Bounds { + frame: 1 << 20, + count: 100, + }; - fn line(json: &str) -> String { - format!("\n@@scriptmark:{N}@@ {json}\n") + fn read(frames: &[&str], bounds: Bounds) -> Records { + let mut reader = Reader::new(bounds); + for frame in frames { + if reader.admits(frame.len()) { + reader.frame(frame.as_bytes()); + } + } + reader.finish() } #[test] - fn test_records_are_read_between_noise() { - let stdout = format!( - "student noise\n{}{}more noise{}{}", - line(r#"{"kind":"ready"}"#), - line( - r#"{"kind":"call","phase":"step","index":0,"target":{"requested":"f","resolved":"f"},"outcome":{"returned":{"value":3,"type":"int"}},"stdout":"hi\n","stdout_truncated":false,"elapsed_ms":1}"# - ), - line(r#"{"kind":"check","index":0,"verdict":{"pass":true,"message":""}}"#), - line(r#"{"kind":"done"}"#), + fn test_records_are_read_in_order() { + let r = read( + &[ + r#"{"kind":"ready"}"#, + r#"{"kind":"call","phase":"step","index":0,"target":{"requested":"f","resolved":"f"},"outcome":{"returned":{"value":3,"type":"int"}},"stdout":"hi\n","stdout_truncated":false,"elapsed_ms":1}"#, + r#"{"kind":"check","index":0,"verdict":{"pass":true,"message":""}}"#, + r#"{"kind":"done"}"#, + ], + WIDE, ); - let r = parse(&stdout, N); assert!(r.ready && r.done && r.protocol_error.is_none()); assert_eq!(r.steps.len(), 1); assert_eq!(r.steps[0].stdout, "hi\n"); @@ -180,22 +237,10 @@ mod tests { ); } - #[test] - fn test_a_record_glued_to_a_partial_student_line_still_counts() { - let stdout = format!("x@@scriptmark:{N}@@ {{\"kind\":\"ready\"}}\n"); - assert!(parse(&stdout, N).ready); - } - - #[test] - fn test_a_forged_record_with_the_wrong_nonce_is_ignored() { - let stdout = "@@scriptmark:guess@@ {\"kind\":\"done\"}\n"; - assert!(!parse(stdout, N).done); - } - #[test] fn test_a_repeated_record_is_a_protocol_error() { let call = r#"{"kind":"call","phase":"step","index":0,"outcome":{"timeout":{}}}"#; - let r = parse(&format!("{}{}", line(call), line(call)), N); + let r = read(&[call, call], WIDE); assert_eq!(r.steps.len(), 1); assert!(matches!( r.protocol_error, @@ -204,54 +249,79 @@ mod tests { } #[test] - fn test_a_stream_cut_short_keeps_what_arrived() { - let stdout = format!( - "{}{}@@scriptmark:{N}@@ {{\"kind\":\"ca", - line(r#"{"kind":"ready"}"#), - line( - r#"{"kind":"call","phase":"step","index":0,"outcome":{"raised":{"type":"KeyError","types":["KeyError","LookupError"],"message":"'k'"}}}"# - ), - ); - let r = parse(&stdout, N); - assert_eq!(r.steps.len(), 1); - assert!(!r.done); - assert!( - r.protocol_error.is_none(), - "a record cut off by a kill is the end of the stream" + fn test_a_garbled_record_mid_stream_is_unreadable() { + let r = read( + &[r#"{"kind":"ready"}"#, "{not json", r#"{"kind":"done"}"#], + WIDE, ); + assert!(r.ready && !r.done); + assert!(matches!( + r.protocol_error, + Some(ProtocolError::Unreadable(_)) + )); } #[test] - fn test_a_garbled_record_mid_stream_is_unreadable_not_tampering() { - let stdout = format!( - "{}@@scriptmark:{N}@@ {{not json\n{}", - line(r#"{"kind":"ready"}"#), - line(r#"{"kind":"done"}"#), + fn test_a_record_past_its_bound_ends_the_stream_and_earlier_records_stand() { + let call = r#"{"kind":"call","phase":"step","index":0,"outcome":{"timeout":{}}}"#; + let long = format!( + r#"{{"kind":"call","phase":"step","index":1,"outcome":{{"missing":{{"message":"{}"}}}}}}"#, + "x".repeat(200) + ); + let tight = Bounds { + frame: 150, + count: 100, + }; + let r = read( + &[r#"{"kind":"ready"}"#, call, &long, r#"{"kind":"done"}"#], + tight, + ); + assert_eq!(r.steps.len(), 1, "the record before the long one stands"); + assert!(!r.done); + assert!(matches!(r.protocol_error, Some(ProtocolError::Flooded(_)))); + + let few = Bounds { + frame: 1 << 20, + count: 2, + }; + let r = read(&[r#"{"kind":"ready"}"#, call, r#"{"kind":"done"}"#], few); + assert!( + matches!(r.protocol_error, Some(ProtocolError::Flooded(m)) if m.contains("2 records")) ); - assert!(matches!( - parse(&stdout, N).protocol_error, - Some(ProtocolError::Unreadable(_)) - )); } #[test] fn test_a_fault_claimed_after_ready_is_tampering() { let fatal = r#"{"kind":"fatal","stage":"teacher_import","error":{"type":"E","message":""}}"#; - let r = parse( - &format!("{}{}", line(r#"{"kind":"ready"}"#), line(fatal)), - N, - ); + let r = read(&[r#"{"kind":"ready"}"#, fatal], WIDE); assert!(r.fatal.is_none()); assert!(matches!(r.protocol_error, Some(ProtocolError::Tampered(_)))); - let after_done = format!( - "{}{}", - line(r#"{"kind":"done"}"#), - line(r#"{"kind":"ready"}"#) + let r = read(&[r#"{"kind":"done"}"#, r#"{"kind":"ready"}"#], WIDE); + assert!(matches!(r.protocol_error, Some(ProtocolError::Tampered(_)))); + let harness = r#"{"kind":"fatal","stage":"harness","error":{"type":"E","message":""}}"#; + assert!( + read(&[r#"{"kind":"ready"}"#, harness], WIDE) + .fatal + .is_some() ); - assert!(matches!( - parse(&after_done, N).protocol_error, - Some(ProtocolError::Tampered(_)) - )); + } + + #[test] + fn test_nothing_but_done_follows_a_stop() { + let stopped = + r#"{"kind":"stopped","phase":"step","index":0,"reason":"deadline","message":"late"}"#; + let r = read( + &[r#"{"kind":"ready"}"#, stopped, r#"{"kind":"done"}"#], + WIDE, + ); + assert_eq!( + r.stopped.as_ref().map(|s| (s.phase, s.reason)), + Some((Phase::Step, StopReason::Deadline)) + ); + assert!(r.done && r.protocol_error.is_none()); + let after = r#"{"kind":"call","phase":"step","index":0,"outcome":{"timeout":{}}}"#; + let r = read(&[r#"{"kind":"ready"}"#, stopped, after], WIDE); + assert!(matches!(r.protocol_error, Some(ProtocolError::Tampered(_)))); } } diff --git a/src/core/tests/harness.rs b/src/core/tests/harness.rs index bb70017..0f92027 100644 --- a/src/core/tests/harness.rs +++ b/src/core/tests/harness.rs @@ -7,8 +7,8 @@ use std::sync::Arc; use scriptmark_core::models::Target; use scriptmark_core::runner::executor::{ - CallPlan, CheckObservation, Executor, Exit, InProcessCheck, Outcome, ScriptRun, Subject, - UnitObservation, UnitPlan, + CallPlan, CheckObservation, Executor, Exit, InProcessCheck, Outcome, Phase, ScriptRun, + StopReason, Subject, UnitObservation, UnitPlan, }; use scriptmark_core::runner::python::PythonExecutor; use serde_json::{Value, json}; @@ -135,8 +135,10 @@ async fn test_a_timeout_is_recorded_even_through_a_bare_except() { ); } +/// A call that defeats its own timer is given up on by the teacher side, which reports it +/// and everything else: the student side is ended, and nothing after it runs. #[tokio::test] -async fn test_a_unit_that_never_yields_is_killed_at_its_deadline() { +async fn test_a_call_that_never_yields_is_given_up_on_and_nothing_after_it_runs() { let dir = tempfile::tempdir().unwrap(); let student = write( dir.path(), @@ -150,10 +152,25 @@ async fn test_a_unit_that_never_yields_is_killed_at_its_deadline() { let mut ok = function("ok", vec![]); ok.timeout = 1; plan.steps = vec![forever, ok]; + let started = std::time::Instant::now(); let obs = run(&plan).await; - assert_eq!(obs.exit, Exit::Deadline); - assert!(obs.steps.is_empty(), "the hung call never reported"); - assert!(!obs.done); + assert!( + started.elapsed() < std::time::Duration::from_secs(10), + "given up on at its own deadline, not the unit's" + ); + assert!(obs.done && obs.teacher_exit == Exit::Code(0), "{obs:?}"); + assert_eq!(obs.exit, Exit::Stopped); + assert_eq!( + obs.steps.len(), + 1, + "the hung call is reported; the next never ran" + ); + assert!(matches!(obs.steps[0].outcome, Outcome::Timeout {})); + let stopped = obs.stopped.as_ref().unwrap(); + assert_eq!( + (stopped.phase, stopped.index, stopped.reason), + (Phase::Step, 0, StopReason::Deadline) + ); } #[tokio::test] @@ -930,3 +947,190 @@ async fn test_a_unit_cannot_leave_anything_behind_for_the_next_one() { "no core dumps" ); } + +#[tokio::test] +async fn test_a_student_side_that_dies_mid_call_is_reported_by_the_teacher_side() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "import importlib\n\ndef ok():\n return 1\n\ndef die():\n importlib.import_module('os')._exit(3)\n", + ); + let mut plan = unit(&student); + plan.steps = vec![ + function("ok", vec![]), + function("die", vec![]), + function("ok", vec![]), + ]; + let obs = run(&plan).await; + assert!(obs.done && obs.teacher_exit == Exit::Code(0), "{obs:?}"); + assert_eq!(obs.steps.len(), 1); + assert_eq!( + obs.exit, + Exit::Code(3), + "the student side's own exit is kept" + ); + let stopped = obs.stopped.as_ref().unwrap(); + assert_eq!((stopped.index, stopped.reason), (1, StopReason::Exited)); +} + +#[tokio::test] +async fn test_a_student_who_writes_into_the_pipe_only_breaks_their_own_unit() { + let dir = tempfile::tempdir().unwrap(); + // Every descriptor a student could guess at: a frame announcing 2 GiB, then a forged + // verdict, as the teacher side's records would spell it. + let student = write( + dir.path(), + "lab.py", + "import importlib, struct\nos = importlib.import_module('os')\n\ndef flood():\n forged = b'{\"kind\":\"check\",\"index\":0,\"verdict\":{\"pass\":true,\"message\":\"\"}}'\n for fd in range(3, 64):\n for data in (struct.pack('>I', len(forged)) + forged, struct.pack('>I', 1 << 31)):\n try:\n os.write(fd, data)\n except OSError:\n pass\n return 1\n", + ); + let teacher = write( + dir.path(), + "t.py", + "def never(result, expected):\n return False\n", + ); + let mut plan = unit(&student); + plan.imports = vec![teacher.to_string_lossy().into_owned()]; + let mut flood = function("flood", vec![]); + flood.check = Some(InProcessCheck { + function: "never".into(), + expected: None, + }); + plan.steps = vec![flood, function("flood", vec![])]; + let obs = run(&plan).await; + assert!(obs.protocol_error.is_none(), "{:?}", obs.protocol_error); + assert!( + obs.checks + .values() + .all(|c| !matches!(c, CheckObservation::Verdict { pass: true, .. })), + "no forged verdict reached the records: {:?}", + obs.checks + ); + let stopped = obs + .stopped + .as_ref() + .expect("the unit stops at the call that wrote"); + assert_eq!(stopped.index, 0); + assert!( + matches!(stopped.reason, StopReason::Flooded | StopReason::Garbled), + "{stopped:?}" + ); + assert!(obs.done, "{obs:?}"); +} + +#[tokio::test] +async fn test_a_student_side_past_its_memory_limit_is_ended() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "import time\n\ndef hog():\n held = b'x' * (256 * 1024 * 1024)\n time.sleep(5)\n return len(held)\n", + ); + let mut plan = unit(&student); + plan.steps = vec![function("hog", vec![]), function("hog", vec![])]; + let obs = PythonExecutor::new().with_memory_limit(96).run(&plan).await; + assert_eq!(obs.exit, Exit::Memory, "{obs:?}"); + assert!(obs.steps.is_empty()); + assert_eq!( + obs.stopped.as_ref().map(|s| s.reason), + Some(StopReason::Exited) + ); +} + +#[tokio::test] +async fn test_a_checker_sees_a_students_object_as_a_snapshot_and_its_graph_intact() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "class Node:\n def __init__(self, value):\n self.value = value\n self.next = None\n @property\n def doubled(self):\n return self.value * 2\n @property\n def _hidden(self):\n raise RuntimeError('private properties are not read')\n\nclass Bad:\n @property\n def broken(self):\n return 1 / 0\n\ndef ring():\n a, b = Node(1), Node(2)\n a.next, b.next = b, a\n return a\n\ndef shared():\n leaf = Node(3)\n return [leaf, leaf, (1, 2), {1, 2}, 2 ** 70]\n\ndef bad():\n return Bad()\n", + ); + let teacher = write( + dir.path(), + "t.py", + "def ring(result, expected):\n assert result.next.next is result, 'the cycle survives'\n assert (result.value, result.doubled, result.next.doubled) == (1, 2, 4)\n assert result.__scriptmark_class__ == 'Node'\n return True\n\ndef shared(result, expected):\n leaf, again, pair, both, big = result\n assert leaf is again and leaf.value == 3\n assert pair == (1, 2) and both == {1, 2} and big == 2 ** 70\n return True\n\ndef anything(result, expected):\n return True\n", + ); + let mut plan = unit(&student); + plan.imports = vec![teacher.to_string_lossy().into_owned()]; + let checked = |name: &str, check: &str| { + let mut c = function(name, vec![]); + c.check = Some(InProcessCheck { + function: check.into(), + expected: None, + }); + c + }; + plan.steps = vec![ + checked("ring", "ring"), + checked("shared", "shared"), + checked("bad", "anything"), + ]; + let obs = run(&plan).await; + let pass = CheckObservation::Verdict { + pass: true, + message: String::new(), + }; + assert_eq!(obs.checks[&0], pass, "{obs:?}"); + assert_eq!(obs.checks[&1], pass, "{obs:?}"); + assert!( + matches!(&obs.steps[2].outcome, Outcome::Raised(e) if e.is_a("ZeroDivisionError")), + "a property that raises while the snapshot is taken is the call's: {:?}", + obs.steps[2].outcome + ); + assert!(!obs.checks.contains_key(&2)); +} + +#[tokio::test] +async fn test_teacher_values_cross_by_value_and_student_values_are_read_when_checked() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "def spoil(items):\n items.append('student')\n return len(items)\n\nclass Account:\n def __init__(self):\n self.balance = 0\n def deposit(self, n):\n self.balance += n\n return self.balance\n", + ); + let teacher = write( + dir.path(), + "t.py", + "def make():\n return ['teacher']\n\ndef untouched(result, expected, items):\n return items == ['teacher'], f'the teacher still holds {items}'\n\ndef balance(result, expected, acct):\n return acct.balance == 30, f'balance {acct.balance}'\n", + ); + let mut plan = unit(&student); + plan.imports = vec![teacher.to_string_lossy().into_owned()]; + let mut make = call( + Target::Teacher { + name: "make".into(), + }, + vec![], + ); + make.id = Some("items".into()); + let mut acct = function("Account", vec![]); + acct.id = Some("acct".into()); + plan.setup = vec![make, acct]; + let mut spoil = function("spoil", vec![json!("$items")]); + spoil.check = Some(InProcessCheck { + function: "untouched".into(), + expected: None, + }); + let deposit = |n: i64| { + call( + Target::Method { + object: "acct".into(), + name: "deposit".into(), + }, + vec![json!(n)], + ) + }; + let mut last = deposit(20); + last.check = Some(InProcessCheck { + function: "balance".into(), + expected: None, + }); + plan.steps = vec![spoil, deposit(10), last]; + let obs = run(&plan).await; + let pass = |message: &str| CheckObservation::Verdict { + pass: true, + message: message.into(), + }; + assert_eq!(returned(&obs.steps[0].outcome), &json!(2), "{obs:?}"); + assert_eq!(obs.checks[&0], pass("the teacher still holds ['teacher']")); + assert_eq!(obs.checks[&2], pass("balance 30")); +} diff --git a/src/core/tests/integration.rs b/src/core/tests/integration.rs index 60c18b6..0fd71e0 100644 --- a/src/core/tests/integration.rs +++ b/src/core/tests/integration.rs @@ -1340,8 +1340,10 @@ expect = 2 } } +/// A step that defeats its own timer is given up on: it timed out, and the steps after it +/// never ran, on the student's account. #[tokio::test] -async fn test_a_scenario_killed_at_its_deadline_blames_the_call_that_hung() { +async fn test_a_scenario_given_up_on_at_its_deadline_blames_the_call_that_hung() { let bench = Bench::new(); let spec = bench.spec( r#" @@ -1379,7 +1381,7 @@ expect = 1 ( TestStatus::Timeout, Some(Fault::Student), - Some(Cause::Killed) + Some(Cause::Timeout) ) ); assert_eq!( From 8e2d8f7d27f064456729046428241adce79e8e13 Mon Sep 17 00:00:00 2001 From: Acture Date: Thu, 15 Oct 2026 17:09:50 +0800 Subject: [PATCH 02/20] fix(runner): accept only outcomes a request can have from the student side (OSS-10) The student side's answers are the student's own evidence, but they must not claim more than the request could produce: an `unresolved` outcome (the teacher side resolves every name before asking), a function-matching decision for anything but a fuzzy function lookup, or one that settled on a function. Any such answer, and any fetched value that does not parse, now stops the student side as garbled, on the student's account, instead of escaping as a harness error. --- src/core/src/runner/harness/teacher.py | 62 ++++++++++++++++---------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/src/core/src/runner/harness/teacher.py b/src/core/src/runner/harness/teacher.py index d69ff26..c3c8699 100644 --- a/src/core/src/runner/harness/teacher.py +++ b/src/core/src/runner/harness/teacher.py @@ -122,7 +122,12 @@ def call_bound(files=0, value=True, transfer=False): # --- Checking what the student side says ------------------------------------------------- -OUTCOMES = {"returned", "raised", "timeout", "missing", "matching", "unresolved", "unserialisable"} +# The outcomes each kind of request can have. The student side resolves nothing by name — this +# side does, before asking — and only a fuzzy function lookup can need a teacher's decision: +# anything else claimed is not an answer to the request. +RAN = {"returned", "raised", "timeout"} +LOOKED_UP = RAN | {"missing", "unserialisable"} +MATCHED = LOOKED_UP | {"matching"} def _str(value, what): @@ -170,12 +175,12 @@ def _decision(decision): } -def _outcome(outcome): +def _outcome(outcome, allowed): if not isinstance(outcome, dict) or len(outcome) != 1: raise Garbled("an outcome that is not exactly one thing") ((kind, body),) = outcome.items() - if kind not in OUTCOMES or not isinstance(body, dict): - raise Garbled(f"an unknown outcome '{kind}'") + if kind not in allowed or not isinstance(body, dict): + raise Garbled(f"an outcome '{kind}' this request cannot have") if kind == "returned": return {"returned": {"value": body.get("value"), "type": _str(body.get("type"), "a type name")}} if kind in ("raised", "unserialisable"): @@ -184,12 +189,13 @@ def _outcome(outcome): return {"timeout": {}} if kind == "missing": return {"missing": {"message": _str(body.get("message"), "a message")}} - if kind == "unresolved": - return {"unresolved": {"name": _str(body.get("name"), "a name")}} - return {"matching": {"message": _str(body.get("message"), "a message"), "decision": _decision(body.get("decision"))}} + decision = _decision(body.get("decision")) + if decision is None or decision["selected"] is not None: + raise Garbled("a lookup that needs a decision but settled on a function") + return {"matching": {"message": _str(body.get("message"), "a message"), "decision": decision}} -def checked(answer, files): +def checked(answer, files, allowed): """The call record an answer describes, rebuilt from only what a record may hold.""" if "fatal" in answer: raise Garbled(f"the student side failed: {text_of(answer['fatal'])[:MESSAGE_LIMIT]}") @@ -199,7 +205,7 @@ def checked(answer, files): if not isinstance(truncated, bool) or not isinstance(elapsed, int) or isinstance(elapsed, bool): raise Garbled("a call record with malformed fields") record = { - "outcome": _outcome(answer.get("outcome")), + "outcome": _outcome(answer.get("outcome"), allowed), "stdout": stdout, "stdout_truncated": truncated, "elapsed_ms": max(elapsed, 0), @@ -353,17 +359,15 @@ def fetch(student, name, timeout): """A value the student side holds, taken by value now. Returns the value, or the outcome that kept it from arriving.""" answer = student.ask({"op": "fetch", "name": name, "timeout": timeout}, call_bound(value=False, transfer=True), timeout + GRACE) - if "value" in answer: - tree = answer["value"] - if isinstance(tree, dict) and tree.get("$") == "too_large": - return None, {"error": problem("TooLarge", f"'{name}' is {tree.get('size')} bytes of JSON, more than a checker can be given")} - try: + try: + if "value" in answer: + tree = answer["value"] + if isinstance(tree, dict) and tree.get("$") == "too_large": + return None, {"error": problem("TooLarge", f"'{name}' is {tree.get('size')} bytes of JSON, more than a checker can be given")} return decode(tree), None - except Garbled as exc: - raise student.garbled(str(exc)) from None - if "unresolved" in answer: - return None, {"unresolved": {"name": name}} - outcome = _outcome(answer.get("outcome")) + outcome = _outcome(answer.get("outcome"), RAN) + except Garbled as exc: + raise student.garbled(str(exc)) from None if "timeout" in outcome: return None, {"error": problem("Timeout", f"reading '{name}' for the checker took longer than {timeout}s")} if "raised" in outcome: @@ -469,6 +473,9 @@ def teacher_call(plan, phase, index, student): args = local(plan.get("args", []), student, plan["timeout"]) except Unresolved as exc: args = exc + except Garbled as exc: + stopped(phase, index, student.garbled(str(exc))) + return None except Stop as stop: stopped(phase, index, stop) return None @@ -517,10 +524,16 @@ def student_call(plan, phase, index, student): "stdout": "", "stdout_truncated": False, "elapsed_ms": 0, }) return False + if kind != "function": + allowed = LOOKED_UP + elif PAYLOAD["role"] == "student": + allowed = MATCHED + else: + allowed = LOOKED_UP try: answer = student.ask(request, call_bound(len(files), transfer=check is not None), plan["timeout"] + GRACE) try: - record = checked(answer, files) + record = checked(answer, files, allowed) except Garbled as exc: raise student.garbled(str(exc)) from None returned = "returned" in record["outcome"] @@ -543,7 +556,10 @@ def student_call(plan, phase, index, student): bind(plan["id"], student=True) if returned and check is not None: try: - verdict = failed or run_check(check, value, record["stdout"], plan["timeout"], student) + try: + verdict = failed or run_check(check, value, record["stdout"], plan["timeout"], student) + except Garbled as exc: + raise student.garbled(str(exc)) from None except Stop as stop: emit({"kind": "check", "index": index, "error": problem("StudentEnded", f"the student side stopped while the checker read its values: {stop.message}")}) stopped(phase, index, stop) @@ -570,7 +586,7 @@ def run_calls(phase, calls, student): def run_load(student): try: answer = student.ask({"op": "load", "timeout": PAYLOAD["load_timeout"]}, call_bound(value=False), PAYLOAD["load_timeout"] + GRACE) - record = checked(answer, []) + record = checked(answer, [], RAN) except Garbled as exc: stopped("load", 0, student.garbled(str(exc))) return False @@ -594,7 +610,7 @@ def run_script(student): script["timeout"] + GRACE, ) compile_error = answer.get("compiles") is False - record = checked(answer, [] if compile_error else files) + record = checked(answer, [] if compile_error else files, RAN) except Garbled as exc: stopped("step", 0, student.garbled(str(exc))) return From 9bd1c7c92d42c1029cd508fad95bae96533f4d84 Mon Sep 17 00:00:00 2001 From: Acture Date: Thu, 15 Oct 2026 17:50:12 +0800 Subject: [PATCH 03/20] feat(runner): confine each side of a unit in the operating system's sandbox (OSS-10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sandboxed is now the default, and fails closed: a unit whose sandbox cannot be applied is not graded, and `grade`/`run` refuse up front with a hint when this machine cannot sandbox at all. `--no-sandbox` opts out; the grading record then carries `isolation: unisolated`, and summaries, the HTML report, the export's about sheet and `grades-push` say so. Sandboxed records keep their bytes and digest. - macOS: Seatbelt through `sandbox-exec` with a generated profile. Each side reads only its interpreter (prefix, packages and the Homebrew kegs it links against — not `etc` or `var`), its own directories and, for the teacher side, the bundle and a read-only copy of the subject's source. Native calls are refused: Seatbelt does not mediate KERN_PROCARGS2, through which a same-user process reads another's environment. - Linux: Landlock (files, TCP, and from ABI 6 signals and abstract sockets) and seccomp (other socket families, connect/bind, sending to an address, new processes, tracing, mounts, io_uring, and the kill family below ABI 6), applied between fork and exec. Only the interpreter and its loader may be executed. - Each side checks a canary file outside everything it may read, and refuses to run if it can open it. - The harness no longer needs a thread: RLIMIT_NPROC counts a user's threads on Linux, so a busy machine refused them. Confined sides, which cannot start processes anyway, leave that limit alone. - Without a sandbox, student code can reach the teacher side, so a broken teacher side after student code ran is the student's again. - CI runs the suite, sandbox probes included, on macOS as well as Linux. --- .github/workflows/ci.yml | 16 +- src/cli/src/main.rs | 70 +++- src/cli/tests/record.rs | 34 ++ src/core/Cargo.toml | 4 + src/core/src/export.rs | 8 + src/core/src/record.rs | 23 +- src/core/src/runner/executor.rs | 7 + src/core/src/runner/harness/student.py | 28 +- src/core/src/runner/harness/teacher.py | 80 +++- src/core/src/runner/judge.rs | 65 ++- src/core/src/runner/oracle.rs | 1 + src/core/src/runner/orchestrator.rs | 4 + src/core/src/runner/prepare.rs | 1 + src/core/src/runner/python.rs | 322 +++++++++++++-- src/core/src/runner/sandbox.rs | 534 ++++++++++++++++++++++++- src/core/tests/harness.rs | 1 + src/core/tests/integration.rs | 6 +- src/core/tests/oracle.rs | 18 +- src/core/tests/sandbox.rs | 321 +++++++++++++++ 19 files changed, 1441 insertions(+), 102 deletions(-) create mode 100644 src/core/tests/sandbox.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a2f45a..553bbde 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,8 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + # Student code runs in the operating system's sandbox — Landlock and seccomp here, Seatbelt + # on macOS — so the suite, its sandbox probes included, runs on both. check: runs-on: ubuntu-latest steps: @@ -29,13 +31,23 @@ jobs: - run: cargo clippy --workspace --all-targets -- -D warnings - run: cargo test --workspace - # Releases ship macOS and Windows binaries; build them on every change, not first at tag time. + test-macos: + runs-on: macos-latest + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: cargo test --workspace + + # Releases ship Windows binaries too; build them on every change, not first at tag time. build: name: build (${{ matrix.os }}) strategy: fail-fast: false matrix: - os: [macos-latest, windows-latest] + os: [windows-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v7 diff --git a/src/cli/src/main.rs b/src/cli/src/main.rs index fac41d5..39cdf65 100644 --- a/src/cli/src/main.rs +++ b/src/cli/src/main.rs @@ -20,6 +20,7 @@ use scriptmark_core::runner::generation::SeedSource; use scriptmark_core::runner::orchestrator::{self, RunOptions}; use scriptmark_core::runner::prepare::prepare; use scriptmark_core::runner::python::PythonExecutor; +use scriptmark_core::runner::sandbox::{self, Isolation}; use scriptmark_core::spec_loader::load_specs_from_dir; #[derive(Parser)] @@ -80,6 +81,27 @@ struct FrozenArgs { fresh: bool, } +#[derive(clap::Args)] +struct IsolationArgs { + /// Run student code without the operating system's sandbox. Each unit still runs as a + /// teacher side and a student side under resource limits, but student code can then + /// read and write whatever your user can — other students' files, the test bundle, your + /// home directory — and reach the network. The grading record marks the results + /// unisolated. For platforms ScriptMark cannot sandbox yet. + #[arg(long)] + no_sandbox: bool, +} + +impl IsolationArgs { + fn isolation(&self) -> Isolation { + if self.no_sandbox { + Isolation::Unisolated + } else { + Isolation::Sandboxed + } + } +} + #[derive(Parser)] struct GradeArgs { /// Directories containing student submissions @@ -133,6 +155,9 @@ struct GradeArgs { #[command(flatten)] frozen: FrozenArgs, + #[command(flatten)] + isolation: IsolationArgs, + /// Replace the grading record at --output even when it holds rescored revisions, /// discarding them. Long form only: there is no -f. #[arg(long)] @@ -183,6 +208,9 @@ struct RunArgs { #[command(flatten)] frozen: FrozenArgs, + #[command(flatten)] + isolation: IsolationArgs, + /// Replace the grading record at --output even when it holds rescored revisions, /// discarding them. Long form only: there is no -f. #[arg(long)] @@ -650,9 +678,14 @@ fn scored(view: &View, path: &Path) -> Result { /// How a summary names what it shows. fn shown(path: &Path, view: &View) -> String { - match view.revision { + let title = match view.revision { Some(n) => format!("{} (revision {n} of {})", path.display(), view.of), None => format!("{} (unscored)", path.display()), + }; + if view.isolation.is_sandboxed() { + title + } else { + format!("{title} — UNISOLATED: student code ran without a sandbox") } } @@ -670,11 +703,24 @@ async fn run_bundles( output: &Path, options: &FrozenArgs, ) -> Result<(Vec, Frozen, String)> { + if run_options.isolation.is_sandboxed() { + // Fail closed, before anything runs: a unit whose sandbox cannot be applied is + // never graded. + sandbox::available().map_err(anyhow::Error::msg).context( + "refusing to grade: student code cannot be sandboxed here (pass --no-sandbox to grade without isolation; the results will say so)", + )?; + } else { + eprintln!( + "warning: --no-sandbox: student code runs without a sandbox and can reach everything your user can; the results are marked unisolated" + ); + } let generation = match &options.replay { Some(path) => Generation::Replay(Frozen::load(path)?), None => Generation::fresh(), }; - let executor = Arc::new(PythonExecutor::with_python_cmd(&run_options.python)); + let executor = Arc::new( + PythonExecutor::with_python_cmd(&run_options.python).with_isolation(run_options.isolation), + ); let bundles = prepare(specs, &generation, executor.clone(), timeout) .await .context("refusing to grade: the test bundle is not ready")?; @@ -877,6 +923,7 @@ async fn execute( let spec_versions = record::spec_versions(&specs)?; let versions = record::submission_versions(&input.students)?; let matching = run_options.matching.clone(); + let isolation = run_options.isolation; let (mut reports, frozen, python) = run_bundles( &input.students, specs, @@ -891,6 +938,7 @@ async fn execute( spec_versions, timeout, python, + isolation, &frozen, (!frozen.is_empty()).then(|| frozen::beside(output)), ); @@ -940,6 +988,7 @@ async fn cmd_grade(args: GradeArgs) -> Result<()> { concurrency: args .concurrency .map(|n| usize::try_from(n).unwrap_or(usize::MAX)), + isolation: args.isolation.isolation(), }, args.timeout, &args.output, @@ -1053,6 +1102,7 @@ async fn cmd_run(args: RunArgs) -> Result<()> { concurrency: args .concurrency .map(|n| usize::try_from(n).unwrap_or(usize::MAX)), + isolation: args.isolation.isolation(), }, args.timeout, &args.output, @@ -1412,6 +1462,12 @@ async fn cmd_grades_push(args: GradesPushArgs) -> Result<()> { } } let view = record.view(args.revision)?; + if !view.isolation.is_sandboxed() { + eprintln!( + "warning: {} was graded with --no-sandbox: these grades come from runs where student code was not isolated", + args.results.display() + ); + } let revision = scored(&view, &args.results)?; // Only graded students are pushed — a real 0 included, a withheld grade never. The @@ -1553,7 +1609,15 @@ fn cmd_report(args: ReportArgs) -> Result<()> { None }; - report::generate_html_report(&reports, similarity.as_deref(), &args.title, &args.output)?; + let title = if view.isolation.is_sandboxed() { + args.title.clone() + } else { + format!( + "{} — UNISOLATED: student code ran without a sandbox", + args.title + ) + }; + report::generate_html_report(&reports, similarity.as_deref(), &title, &args.output)?; println!("HTML report generated: {}", args.output.display()); Ok(()) diff --git a/src/cli/tests/record.rs b/src/cli/tests/record.rs index 23a77bd..ee9de25 100644 --- a/src/cli/tests/record.rs +++ b/src/cli/tests/record.rs @@ -877,3 +877,37 @@ fn a_canvas_record_rescores_from_its_bundle_and_refuses_another_attempt_or_excus std::fs::write(&payload_path, payload.to_string()).unwrap(); refused(rescore(dir, &[]), "2024002 is excused now"); } + +/// OSS-10: a run without the sandbox says so in its record, and only then: a sandboxed +/// record's bytes do not mention it. +#[test] +fn a_run_without_the_sandbox_is_recorded_as_unisolated() { + let dir = bench(); + for (flags, isolated) in [(&[][..], true), (&["--no-sandbox"][..], false)] { + let mut args = vec![ + "grade", + "submissions", + "-t", + "tests", + "-o", + "out/result.json", + "--force", + ]; + args.extend(flags); + let output = scriptmark(dir.path(), &args); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let text = std::fs::read_to_string(dir.path().join("out/result.json")).unwrap(); + let record = Record::load(&dir.path().join("out/result.json")).unwrap(); + assert_eq!(record.evidence.bundle.isolation.is_sandboxed(), isolated); + assert_eq!(text.contains("\"isolation\""), !isolated, "{text}"); + let summary = scriptmark(dir.path(), &["summarize", "out/result.json"]); + assert_eq!( + String::from_utf8_lossy(&summary.stdout).contains("UNISOLATED"), + !isolated + ); + } +} diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index b8586ea..04d1e40 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -38,3 +38,7 @@ bzip2-rs = "0.1.2" lzma-rs = "0.3.0" tempfile = "3" sha2 = "0.10" + +[target.'cfg(target_os = "linux")'.dependencies] +landlock = "0.4" +seccompiler = "0.5" diff --git a/src/core/src/export.rs b/src/core/src/export.rs index 61fc448..f9cef29 100644 --- a/src/core/src/export.rs +++ b/src/core/src/export.rs @@ -472,6 +472,14 @@ pub fn about(record: &Record, n: u32) -> Result { } rows.extend([ ("run_by".into(), Cell::text(&evidence.scriptmark)), + ( + "isolation".into(), + Cell::text(if evidence.bundle.isolation.is_sandboxed() { + "sandboxed" + } else { + "unisolated (--no-sandbox)" + }), + ), ("scored_by".into(), Cell::text(&revision.scriptmark)), ( "derived_items".into(), diff --git a/src/core/src/record.rs b/src/core/src/record.rs index 3af8fdc..8ef8b65 100644 --- a/src/core/src/record.rs +++ b/src/core/src/record.rs @@ -23,6 +23,7 @@ use crate::models::{ }; use crate::runner::answers::{digest, fingerprint}; use crate::runner::frozen::{Frozen, scratch}; +use crate::runner::sandbox::Isolation; /// The layout of the file. A newer one is refused by number, never half-read. pub const FORMAT: u32 = 1; @@ -121,6 +122,14 @@ pub struct BundleVersion { /// The frozen inputs and answers the batch used, when it had any. #[serde(default, skip_serializing_if = "Option::is_none")] pub frozen: Option, + /// Whether student and teacher code ran in their sandboxes. Written only when they did + /// not (`--no-sandbox`), so every sandboxed record keeps its bytes and its digest. + #[serde(default, skip_serializing_if = "is_sandboxed")] + pub isolation: Isolation, +} + +fn is_sandboxed(isolation: &Isolation) -> bool { + isolation.is_sandboxed() } /// One test spec as it ran. @@ -187,6 +196,8 @@ pub struct View { pub reports: Vec, /// The items the revision scored, in declaration order; empty when unscored. pub items: Vec, + /// How the evidence was gathered: an unisolated run is shown as one. + pub isolation: Isolation, } #[derive(Debug, thiserror::Error)] @@ -377,6 +388,7 @@ impl Record { of: self.revisions.len(), reports, items: revision.map(|r| r.policy.items.clone()).unwrap_or_default(), + isolation: self.evidence.bundle.isolation, }) } @@ -659,6 +671,7 @@ pub fn bundle_version( mut specs: Vec, timeout: u64, python: String, + isolation: Isolation, frozen: &Frozen, frozen_path: Option, ) -> BundleVersion { @@ -679,6 +692,7 @@ pub fn bundle_version( digest: digest(&serde_json::to_vec(&(timeout, semantics)).expect("plain JSON")), timeout, python, + isolation, frozen: (!frozen.is_empty()).then(|| FrozenLink { path: frozen_path, digest: digest(frozen.to_json().as_bytes()), @@ -934,7 +948,14 @@ mod tests { }, attempt_policy: AttemptPolicy::default(), matching: matching::Config::default(), - bundle: bundle_version(Vec::new(), 10, "python3".into(), &frozen, None), + bundle: bundle_version( + Vec::new(), + 10, + "python3".into(), + Isolation::Sandboxed, + &frozen, + None, + ), students, } } diff --git a/src/core/src/runner/executor.rs b/src/core/src/runner/executor.rs index 32b00ea..7cb4f03 100644 --- a/src/core/src/runner/executor.rs +++ b/src/core/src/runner/executor.rs @@ -54,6 +54,8 @@ pub struct UnitPlan { pub vars: Arc>, /// `(source, relative destination)` pairs staged into the working directory. pub data_files: Vec<(PathBuf, String)>, + /// The test bundle's directory: what teacher code may read, besides its own modules. + pub bundle: PathBuf, pub allowed_imports: Vec, pub load_timeout: u64, pub setup: Vec, @@ -298,6 +300,10 @@ pub struct UnitObservation { pub protocol_error: Option, /// The tail of each side's stderr, for diagnosing a crash. pub stderr: String, + /// Each side ran in its own sandbox. Without one, student code can reach the teacher + /// side — end it, or write into its records — so what breaks there may be the + /// student's doing. + pub sandboxed: bool, } impl UnitObservation { @@ -316,6 +322,7 @@ impl UnitObservation { exit, protocol_error: None, stderr: String::new(), + sandboxed: false, } } } diff --git a/src/core/src/runner/harness/student.py b/src/core/src/runner/harness/student.py index 6f6bc65..077f908 100644 --- a/src/core/src/runner/harness/student.py +++ b/src/core/src/runner/harness/student.py @@ -215,23 +215,17 @@ def run(): def feed_stdin(text): - """Descriptor 0 becomes a pipe holding `text`, as `python script.py < input` would see - it: `input()`, `sys.stdin.buffer` and `os.read(0, …)` all read the same bytes.""" - import threading - - read_end, write_end = os.pipe() - - def feed(): - try: - write_all(write_end, text.encode("utf-8")) - except OSError: # the script stopped reading - pass - finally: - os.close(write_end) - - threading.Thread(target=feed, daemon=True).start() - os.dup2(read_end, 0) - os.close(read_end) + """Descriptor 0 becomes a file holding `text`, as `python script.py < input` would see + it: `input()`, `sys.stdin.buffer` and `os.read(0, …)` all read the same bytes. A file in + the unit's own temporary directory, not a pipe and a thread to fill it: the process + limit counts threads, and a busy machine may have none to spare.""" + import tempfile + + with tempfile.TemporaryFile() as held: + held.write(text.encode("utf-8")) + held.flush() + held.seek(0) + os.dup2(held.fileno(), 0) def script(request): diff --git a/src/core/src/runner/harness/teacher.py b/src/core/src/runner/harness/teacher.py index c3c8699..e5e8d87 100644 --- a/src/core/src/runner/harness/teacher.py +++ b/src/core/src/runner/harness/teacher.py @@ -8,8 +8,7 @@ """ import importlib.util # noqa: E402 -import queue # noqa: E402 -import threading # noqa: E402 +import select # noqa: E402 with open(sys.argv[1], encoding="utf-8") as _fh: PAYLOAD = json.load(_fh) @@ -50,27 +49,69 @@ def __init__(self, reason, message): class Student: - """The pipe to the student side. One request is pending at a time; a reader thread - checks each answer's length against that request's bound before reading it.""" + """The pipe to the student side. One request is pending at a time: its answer is read + before its deadline, and its length checked against the request's bound before its body + is read. Reading needs no thread: the kernel's process limit counts threads, and a + busy machine may have none to spare.""" def __init__(self, read_fd, write_fd): self.read_fd, self.write_fd = read_fd, write_fd - self.bound = 0 - self.answers = queue.Queue() self.stopped = None - threading.Thread(target=self._read, daemon=True).start() + self.windows = os.name == "nt" + if self.windows: + # Pipes cannot be polled there: a reader thread waits instead, and the process + # limit that makes threads scarce is a unix one. + import queue + import threading - def _read(self): + self.answers = queue.Queue() + threading.Thread(target=self._pump, daemon=True).start() + + def _pump(self): try: while True: (size,) = HEADER.unpack(read_exact(self.read_fd, HEADER.size)) - bound = self.bound - if size > bound: - self.answers.put(Stop("flooded", f"the student side sent {size} bytes where at most {bound} may arrive")) + self.answers.put(size) + self.answers.put(read_exact(self.read_fd, size) if size <= self.bound else None) + if size > self.bound: return - self.answers.put(read_exact(self.read_fd, size)) except (Closed, OSError): - self.answers.put(Stop("exited", "the student side ended")) + self.answers.put(Closed()) + + def _exactly(self, n, deadline): + """`n` bytes from the student side, before `deadline`.""" + chunks, got = [], 0 + while got < n: + left = deadline - time.monotonic() + if left <= 0: + raise TimeoutError() + if not select.select([self.read_fd], [], [], left)[0]: + continue + chunk = os.read(self.read_fd, min(n - got, 1 << 20)) + if not chunk: + raise Closed() + chunks.append(chunk) + got += len(chunk) + return b"".join(chunks) + + def _answer(self, bound, timeout): + deadline = time.monotonic() + timeout + if self.windows: + import queue + + try: + size = self.answers.get(timeout=timeout) + body = None if isinstance(size, Closed) or size > bound else self.answers.get(timeout=max(deadline - time.monotonic(), 0.001)) + except queue.Empty: + raise TimeoutError() from None + if isinstance(size, Closed) or isinstance(body, Closed): + raise Closed() + else: + (size,) = HEADER.unpack(self._exactly(HEADER.size, deadline)) + body = None if size > bound else self._exactly(size, deadline) + if body is None: + raise Stop("flooded", f"the student side sent {size} bytes where at most {bound} may arrive") + return body def ask(self, request, bound, timeout): if self.stopped is not None: @@ -82,13 +123,16 @@ def ask(self, request, bound, timeout): self.stopped = Stop("exited", "the student side ended") raise self.stopped from None try: - answer = self.answers.get(timeout=timeout) - except queue.Empty: + answer = self._answer(bound, timeout) + except TimeoutError: self.stopped = Stop("deadline", f"the student side did not answer within {timeout:g}s") raise self.stopped from None - if isinstance(answer, Stop): - self.stopped = answer - raise answer + except (Closed, OSError): + self.stopped = Stop("exited", "the student side ended") + raise self.stopped from None + except Stop as stop: + self.stopped = stop + raise try: parsed = parse(answer) if not isinstance(parsed, dict): diff --git a/src/core/src/runner/judge.rs b/src/core/src/runner/judge.rs index 253ce1e..e9b24df 100644 --- a/src/core/src/runner/judge.rs +++ b/src/core/src/runner/judge.rs @@ -152,11 +152,11 @@ fn unit_failure(plan: &UnitPlan, obs: &UnitObservation) -> Option { } } if let Some(ProtocolError::Tampered(problem)) = &obs.protocol_error { - // Only the teacher side writes records: student code cannot reach them. + let (fault, cause) = teacher_side_broken(obs); return blanket( TestStatus::Error, - Fault::Environment, - Cause::Harness, + fault, + cause, format!("the unit's records cannot be trusted: {problem}"), ); } @@ -377,11 +377,12 @@ fn unfinished(obs: &UnitObservation, owner: Fault, cause: Cause, what: &str) -> if owner == Fault::Teacher { return in_flight(&obs.teacher_exit, Fault::Teacher, cause, what); } + let (fault, cause) = teacher_side_broken(obs); let (_, how) = death(&obs.teacher_exit); Blanket { status: TestStatus::Error, - fault: Fault::Environment, - cause: Cause::Harness, + fault, + cause, message: format!( "the teacher side ended: {how} {what} ({})", stderr_hint(obs) @@ -389,6 +390,18 @@ fn unfinished(obs: &UnitObservation, owner: Fault, cause: Cause, what: &str) -> } } +/// Whose doing it is when the teacher side breaks — ends, or writes records it could not +/// have meant. In its sandbox, student code cannot reach it: the harness's. Unsandboxed, +/// once student code has run, it can — signal it, or write into its records — and it is +/// the student's, as it was when both shared one process. +fn teacher_side_broken(obs: &UnitObservation) -> (Fault, Cause) { + if obs.sandboxed || !obs.ready { + (Fault::Environment, Cause::Harness) + } else { + (Fault::Student, Cause::Protocol) + } +} + /// The verdict for scored calls after one that stopped the unit. fn not_run(culprit: &Blanket, after: &str) -> Blanket { Blanket { @@ -469,10 +482,11 @@ fn describe(outcome: &Outcome, timeout: u64) -> String { /// teacher side wrote a record it could not have meant. What was recorded before that /// stands; nothing after it can be known. fn stream_break(obs: &UnitObservation) -> Option { + let broken = teacher_side_broken(obs); let (fault, cause, message) = match (&obs.fatal, &obs.protocol_error) { (_, Some(ProtocolError::Flooded(problem))) => ( - Fault::Environment, - Cause::Harness, + broken.0, + broken.1, format!("the unit's records could not be read past this point: {problem}"), ), (Some(fatal), _) if fatal.stage == "harness" => ( @@ -484,8 +498,8 @@ fn stream_break(obs: &UnitObservation) -> Option { ), ), (_, Some(ProtocolError::Unreadable(problem))) => ( - Fault::Environment, - Cause::Harness, + broken.0, + broken.1, format!("the harness wrote a record it could not have meant: {problem}"), ), _ => return None, @@ -898,6 +912,7 @@ mod tests { imports: Vec::new(), vars: Arc::new(BTreeMap::new()), data_files: Vec::new(), + bundle: std::path::PathBuf::new(), allowed_imports: Vec::new(), load_timeout: 10, setup: Vec::new(), @@ -957,6 +972,7 @@ mod tests { steps, done: true, exit: Exit::Code(0), + sandboxed: true, ..UnitObservation::not_started(Exit::Code(0)) } } @@ -1288,8 +1304,8 @@ mod tests { r[2].failure ); - // Without a word from the teacher side, the teacher side itself went: the student - // cannot reach it. + // Without a word from the teacher side, the teacher side itself went: in its sandbox + // the student cannot reach it... let mut lost = finished(vec![record(0, returned(json!(1)))]); lost.done = false; lost.teacher_exit = Exit::Signal(libc::SIGKILL); @@ -1302,6 +1318,21 @@ mod tests { Some(Cause::Harness) ) ); + // ...but unsandboxed, student code could have signalled it. + lost.sandboxed = false; + let r = judge(&p, &scored, &lost, "python3"); + assert_eq!( + verdict(&r[1]), + ( + TestStatus::Error, + Some(Fault::Student), + Some(Cause::Protocol) + ) + ); + assert_eq!( + verdict(&r[2]), + (TestStatus::Error, Some(Fault::Student), Some(Cause::NotRun)) + ); } #[test] @@ -1354,7 +1385,8 @@ mod tests { (TestStatus::Error, Some(Fault::Student), Some(Cause::Load)) ); - // Only the teacher side writes records: student code cannot have broken them. + // Only the teacher side writes records: sandboxed, student code cannot have broken + // them; unsandboxed, it could have written into them. let mut tampered = finished(vec![record(0, returned(json!(1)))]); tampered.protocol_error = Some(ProtocolError::Tampered("step record 0 repeated".into())); assert_eq!( @@ -1365,6 +1397,15 @@ mod tests { Some(Cause::Harness) ) ); + tampered.sandboxed = false; + assert_eq!( + verdict(&one(&c, &p, &tampered)), + ( + TestStatus::Error, + Some(Fault::Student), + Some(Cause::Protocol) + ) + ); // A sandbox that did not hold grades nobody. let mut leaky = UnitObservation::not_started(Exit::Code(0)); diff --git a/src/core/src/runner/oracle.rs b/src/core/src/runner/oracle.rs index 601296f..befaf3e 100644 --- a/src/core/src/runner/oracle.rs +++ b/src/core/src/runner/oracle.rs @@ -36,6 +36,7 @@ pub async fn resolve_oracle( .iter() .map(|rel| (spec.dir.join(rel), rel.clone())) .collect(), + bundle: spec.dir.clone(), allowed_imports: spec.meta.allowed_imports.clone(), load_timeout: timeout_secs, setup: Vec::new(), diff --git a/src/core/src/runner/orchestrator.rs b/src/core/src/runner/orchestrator.rs index 9850d02..49f976f 100644 --- a/src/core/src/runner/orchestrator.rs +++ b/src/core/src/runner/orchestrator.rs @@ -21,6 +21,9 @@ pub struct RunOptions { pub concurrency: Option, /// The interpreter that runs teacher Python checker scripts. pub python: String, + /// Whether teacher checker scripts and lint tools run in the teacher's sandbox, as the + /// units do. + pub isolation: crate::runner::sandbox::Isolation, } impl Default for RunOptions { @@ -29,6 +32,7 @@ impl Default for RunOptions { matching: crate::matching::Config::default(), concurrency: None, python: "python3".into(), + isolation: crate::runner::sandbox::Isolation::default(), } } } diff --git a/src/core/src/runner/prepare.rs b/src/core/src/runner/prepare.rs index c42c78c..416364c 100644 --- a/src/core/src/runner/prepare.rs +++ b/src/core/src/runner/prepare.rs @@ -555,6 +555,7 @@ fn plan_units(spec: &TestSpec, timeout_secs: u64) -> Vec { .iter() .map(|rel| (spec.dir.join(rel), rel.clone())) .collect(), + bundle: spec.dir.clone(), allowed_imports: spec.meta.allowed_imports.clone(), load_timeout: timeout_secs, setup: Vec::new(), diff --git a/src/core/src/runner/python.rs b/src/core/src/runner/python.rs index 52f8973..296e1fd 100644 --- a/src/core/src/runner/python.rs +++ b/src/core/src/runner/python.rs @@ -6,7 +6,7 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, LazyLock, Mutex}; +use std::sync::{Arc, LazyLock, Mutex, OnceLock}; use std::time::Duration; use serde::Serialize; @@ -19,6 +19,7 @@ use crate::runner::executor::{ use crate::runner::records::{Bounds, Reader, Records}; #[cfg(unix)] use crate::runner::sandbox::apply_sandbox; +use crate::runner::sandbox::{Confinement, Isolation, covering}; /// Resolve a python command to an absolute path so env_clear() doesn't change /// which interpreter runs. Falls back to the original string if resolution fails. @@ -43,6 +44,9 @@ pub struct PythonExecutor { python_cmd: String, /// The most memory the student side may hold before it is ended, in MiB. memory_mb: u64, + isolation: Isolation, + /// What the interpreter needs to run, found once, the first time a unit is confined. + interpreter: OnceLock>, } impl PythonExecutor { @@ -54,6 +58,8 @@ impl PythonExecutor { Self { python_cmd: resolve_python_path(&python_cmd.into()), memory_mb: crate::runner::sandbox::SandboxConfig::default().mem_mb, + isolation: Isolation::default(), + interpreter: OnceLock::new(), } } @@ -63,9 +69,117 @@ impl PythonExecutor { self } + /// Whether each side runs in its own sandbox. Sandboxed is the default; a unit whose + /// sandbox cannot be applied is not graded. + pub fn with_isolation(mut self, isolation: Isolation) -> Self { + self.isolation = isolation; + self + } + + pub fn isolation(&self) -> Isolation { + self.isolation + } + pub fn python_cmd(&self) -> &str { &self.python_cmd } + + fn interpreter(&self) -> Result<&Interpreter, String> { + self.interpreter + .get_or_init(|| Interpreter::probe(&self.python_cmd)) + .as_ref() + .map_err(Clone::clone) + } +} + +/// What one Python interpreter reads and runs: its prefixes, its standard library and +/// packages, and the shared libraries they load — not the rest of the machine. +#[derive(Debug, Clone)] +struct Interpreter { + read: Vec, + exec: Vec, +} + +/// Asks the interpreter where it lives, unconfined and once. +const PROBE: &str = "import json, os, sys, sysconfig +paths = [sys.prefix, sys.base_prefix, sys.exec_prefix, sys.base_exec_prefix, *sysconfig.get_paths().values(), *sys.path] +print(json.dumps({ + 'executable': os.path.realpath(sys.executable), + 'base': os.path.realpath(sys.base_prefix), + 'paths': [os.path.realpath(p) for p in paths if p and os.path.exists(p)], +}))"; + +impl Interpreter { + fn probe(python_cmd: &str) -> Result { + #[derive(serde::Deserialize)] + struct Found { + executable: PathBuf, + base: PathBuf, + paths: Vec, + } + let output = std::process::Command::new(python_cmd) + .env_clear() + .args(["-I", "-c", PROBE]) + .output() + .map_err(|e| format!("could not ask {python_cmd} where it lives: {e}"))?; + let found: Found = serde_json::from_slice(&output.stdout).map_err(|e| { + format!( + "{python_cmd} did not say where it lives ({e}): {}", + String::from_utf8_lossy(&output.stderr).trim() + ) + })?; + let command = PathBuf::from(python_cmd); + let mut read = found.paths; + read.extend(system_libraries(&found.base)); + // The command itself may be a link into the interpreter's prefix, as a venv's is. + read.push(command.clone()); + let mut exec = vec![command]; + #[cfg(target_os = "macos")] + // A framework build's `python3` starts `Python.app` inside its prefix. + exec.push(found.base); + #[cfg(not(target_os = "macos"))] + { + exec.extend(crate::runner::sandbox::loader_of(&found.executable)); + let _ = found.base; + } + exec.push(found.executable); + Ok(Self { + read: covering(read), + exec, + }) + } +} + +/// Shared libraries an interpreter loads from outside its prefix. +#[cfg(target_os = "macos")] +fn system_libraries(base: &Path) -> Vec { + // A Homebrew keg links against other kegs through `opt`: those, and only those — not + // the prefix's `etc` or `var`, which hold service configuration and data. + let text = base.to_string_lossy(); + match text.find("/Cellar/") { + Some(at) => { + let prefix = Path::new(&text[..at]); + vec![prefix.join("Cellar"), prefix.join("opt")] + } + None => Vec::new(), + } +} + +#[cfg(not(target_os = "macos"))] +fn system_libraries(_base: &Path) -> Vec { + [ + "/usr/lib", + "/usr/lib64", + "/lib", + "/lib64", + "/usr/local/lib", + "/etc/ld.so.cache", + "/etc/localtime", + "/usr/share/zoneinfo", + ] + .iter() + .filter_map(|p| std::fs::canonicalize(p).ok()) + .collect() } const COMMON: &str = include_str!("harness/common.py"); @@ -170,6 +284,9 @@ struct Staged { subject: Option, /// The subject's source, for teacher code to read. source: Option, + /// A file outside everything either side may read: a side that can open it is not in + /// its sandbox, and refuses to go on. + canary: PathBuf, } /// Copy `src` (a file or a directory) to `dest`, creating parents. @@ -209,8 +326,12 @@ fn stage( let root = tempfile::Builder::new() .prefix("scriptmark-unit-") .tempdir()?; - let teacher = side(root.path(), "teacher")?; - let student = side(root.path(), "student")?; + // The sandbox sees resolved paths: `/var/folders/…` is `/private/var/folders/…` on macOS. + let base = std::fs::canonicalize(root.path())?; + let teacher = side(&base, "teacher")?; + let student = side(&base, "student")?; + let canary = base.join("canary"); + std::fs::write(&canary, "a sandboxed side must not be able to read this\n")?; // Teacher code that opens a data file by its relative path finds it, as student code does. for (src, rel) in data_files { copy_into(src, &student.work.join(rel))?; @@ -223,7 +344,7 @@ fn stage( .unwrap_or(std::ffi::OsStr::new("subject.py")); let dest = student.work.join(name); std::fs::copy(file, &dest)?; - let sources = root.path().join("source"); + let sources = base.join("source"); std::fs::create_dir(&sources)?; let source = sources.join(name); std::fs::copy(file, &source)?; @@ -241,6 +362,7 @@ fn stage( student, subject: copied, source, + canary, root, }; std::fs::write(&staged.payload, payload(&staged).to_string())?; @@ -267,10 +389,20 @@ impl Finished { } impl PythonExecutor { - /// A command for one side: a cleared environment, the side's own `HOME` and `TMPDIR`, - /// and the kernel's limits. - fn command(&self, script: &str, side: &Side, deadline_secs: u64) -> Command { - let mut cmd = Command::new(&self.python_cmd); + /// A command for one side: confined when sandboxed, with a cleared environment, the + /// side's own `HOME` and `TMPDIR`, and the kernel's limits. + fn command( + &self, + script: &str, + side: &Side, + deadline_secs: u64, + confinement: Option<&Confinement>, + ) -> Result { + let mut cmd = match confinement { + Some(c) => crate::runner::sandbox::confined(Path::new(&self.python_cmd), c) + .map_err(|e| format!("cannot sandbox this unit: {e}"))?, + None => Command::new(&self.python_cmd), + }; cmd.env_clear() .env("PATH", "/usr/bin:/usr/local/bin:/opt/homebrew/bin") .env("HOME", &side.home) @@ -290,11 +422,45 @@ impl PythonExecutor { { // Its own process group, so anything it spawns dies with it. cmd.process_group(0); - apply_sandbox(&mut cmd, &sandbox_for(deadline_secs)); + apply_sandbox(&mut cmd, &sandbox_for(deadline_secs, confinement.is_some())); } #[cfg(not(unix))] let _ = deadline_secs; - cmd + Ok(cmd) + } + + /// Each side's confinement, when sandboxed: the interpreter, its own directories, and + /// what `teacher` and `student` add. Neither side makes native calls: on macOS they + /// would read the grader's environment. + fn confinements( + &self, + staged: &Staged, + teacher: Vec, + student: Vec, + ) -> Result, String> { + if !self.isolation.is_sandboxed() { + return Ok(None); + } + let interpreter = self.interpreter()?; + let confine = |side: &Side, extra: Vec| Confinement { + read: covering(interpreter.read.iter().cloned().chain(extra).collect()), + write: vec![side.work.clone(), side.home.clone(), side.tmp.clone()], + exec: interpreter.exec.clone(), + spawn: false, + no_ffi: true, + }; + let mut teacher = teacher; + teacher.extend( + staged + .source + .as_ref() + .and_then(|s| s.parent()) + .map(Path::to_path_buf), + ); + Ok(Some([ + confine(&staged.teacher, canonical(teacher)), + confine(&staged.student, canonical(student)), + ])) } /// Run a staged unit — the teacher side, and the student side when `with_student` — @@ -305,8 +471,21 @@ impl PythonExecutor { deadline_secs: u64, bounds: Bounds, with_student: bool, + confinements: Option<[Confinement; 2]>, ) -> Finished { - let mut teacher_cmd = self.command(&TEACHER_SCRIPT, &staged.teacher, deadline_secs); + let [teacher_confinement, student_confinement] = match confinements { + Some([teacher, student]) => [Some(teacher), Some(student)], + None => [None, None], + }; + let mut teacher_cmd = match self.command( + &TEACHER_SCRIPT, + &staged.teacher, + deadline_secs, + teacher_confinement.as_ref(), + ) { + Ok(cmd) => cmd, + Err(e) => return Finished::spawn_failed(e), + }; teacher_cmd.arg(&staged.payload); let mut student = None; if with_student { @@ -318,7 +497,15 @@ impl PythonExecutor { return Finished::spawn_failed(format!("could not join the two sides: {e}")); } }; - let mut cmd = self.command(&STUDENT_SCRIPT, &staged.student, deadline_secs); + let mut cmd = match self.command( + &STUDENT_SCRIPT, + &staged.student, + deadline_secs, + student_confinement.as_ref(), + ) { + Ok(cmd) => cmd, + Err(e) => return Finished::spawn_failed(e), + }; cmd.stdin(std::process::Stdio::from(to_student)) .stdout(std::process::Stdio::from(from_student)) .stderr(std::process::Stdio::piped()); @@ -581,12 +768,16 @@ fn resident_bytes(_pid: u32) -> Option { None } -/// The sandbox for a unit: its CPU limit sits above its deadline, a backstop for one busy -/// core rather than a second, shorter timeout. -fn sandbox_for(deadline_secs: u64) -> crate::runner::sandbox::SandboxConfig { +/// The limits for a unit: its CPU limit sits above its deadline, a backstop for one busy +/// core rather than a second, shorter timeout. Confined, it cannot start processes, so the +/// per-user process limit — which counts threads on Linux — is left alone. +#[cfg(unix)] +fn sandbox_for(deadline_secs: u64, confined: bool) -> crate::runner::sandbox::SandboxConfig { + let defaults = crate::runner::sandbox::SandboxConfig::default(); crate::runner::sandbox::SandboxConfig { cpu_secs: deadline_secs.saturating_add(1), - ..Default::default() + nproc: if confined { None } else { defaults.nproc }, + ..defaults } } @@ -694,17 +885,39 @@ async fn discard(staged: Staged) { let _ = tokio::task::spawn_blocking(move || drop(staged.root)).await; } -/// The teacher side's payload, common to every unit of a spec. -fn base_payload(mode: &str) -> serde_json::Value { +/// The teacher side's payload, common to every unit: with each side's canary when sandboxed. +fn base_payload(mode: &str, staged: &Staged, sandboxed: bool) -> serde_json::Value { + let canary = sandboxed.then_some(&staged.canary); serde_json::json!({ "mode": mode, "records": RECORD_FD, "limits": LIMITS, "grace": GRACE_SECS, "startup": STARTUP_SECS, + "canary": canary, + "student_canary": canary, }) } +/// Each path that exists, resolved. +fn canonical(paths: Vec) -> Vec { + paths + .into_iter() + .filter_map(|p| std::fs::canonicalize(p).ok()) + .collect() +} + +/// What teacher code reads: the bundle, and the directories its modules import beside. +fn teacher_reads(bundle: &Path, imports: &[String]) -> Vec { + std::iter::once(bundle.to_path_buf()) + .chain( + imports + .iter() + .filter_map(|i| Path::new(i).parent().map(Path::to_path_buf)), + ) + .collect() +} + fn extend(mut base: serde_json::Value, more: serde_json::Value) -> serde_json::Value { if let (Some(base), serde_json::Value::Object(more)) = (base.as_object_mut(), more) { base.extend(more); @@ -737,18 +950,31 @@ impl Executor for PythonExecutor { .iter() .map(|rel| (spec.dir.join(rel), rel.clone())) .collect::>(); - let payload = extend( - base_payload("inspect"), - serde_json::json!({ "imports": spec.meta.imports }), - ); - let staged = tokio::task::spawn_blocking(move || stage(None, &data, |_| payload)) - .await - .map_err(|e| e.to_string())? - .map_err(|e| format!("could not stage the teacher modules: {e}"))?; + let imports = spec.meta.imports.clone(); + let sandboxed = self.isolation.is_sandboxed(); + let staged = tokio::task::spawn_blocking(move || { + stage(None, &data, |staged| { + extend( + base_payload("inspect", staged, sandboxed), + serde_json::json!({ "imports": imports }), + ) + }) + }) + .await + .map_err(|e| e.to_string())? + .map_err(|e| format!("could not stage the teacher modules: {e}"))?; let deadline = timeout_secs.saturating_add(STARTUP_SECS); - let finished = self - .run_unit(&staged, deadline, INSPECT_BOUNDS, false) - .await; + let finished = match self.confinements( + &staged, + teacher_reads(&spec.dir, &spec.meta.imports), + Vec::new(), + ) { + Ok(confinements) => { + self.run_unit(&staged, deadline, INSPECT_BOUNDS, false, confinements) + .await + } + Err(e) => Finished::spawn_failed(e), + }; discard(staged).await; let records = finished.records; @@ -778,9 +1004,10 @@ impl Executor for PythonExecutor { let file = plan.file.clone(); let payload = { let plan = plan.clone(); + let sandboxed = self.isolation.is_sandboxed(); move |staged: &Staged| { extend( - base_payload("unit"), + base_payload("unit", staged, sandboxed), serde_json::json!({ "role": match plan.subject { Subject::Student => "student", @@ -823,9 +1050,30 @@ impl Executor for PythonExecutor { }; debug_assert!(staged.subject.as_ref().is_some_and(|s| s.is_file())); - let finished = self - .run_unit(&staged, plan.deadline_secs(), record_bounds(plan), true) - .await; + // A reference implementation is teacher code: it may read the bundle, as teacher + // code beside it may. + let student_reads = if plan.subject == Subject::Reference { + vec![plan.bundle.clone()] + } else { + Vec::new() + }; + let finished = match self.confinements( + &staged, + teacher_reads(&plan.bundle, &plan.imports), + student_reads, + ) { + Ok(confinements) => { + self.run_unit( + &staged, + plan.deadline_secs(), + record_bounds(plan), + true, + confinements, + ) + .await + } + Err(e) => Finished::spawn_failed(e), + }; discard(staged).await; let records = finished.records; @@ -842,6 +1090,7 @@ impl Executor for PythonExecutor { teacher_exit: finished.teacher_exit, protocol_error: records.protocol_error, stderr: finished.stderr, + sandboxed: self.isolation.is_sandboxed(), } } } @@ -862,9 +1111,11 @@ mod tests { #[test] fn test_the_cpu_limit_sits_above_any_units_deadline() { for deadline in [3, 45, 4 * 10 + 2, 86_400 * 4] { - assert!(sandbox_for(deadline).cpu_secs > deadline); + assert!(sandbox_for(deadline, true).cpu_secs > deadline); } - assert_eq!(sandbox_for(u64::MAX).cpu_secs, u64::MAX, "saturates"); + assert_eq!(sandbox_for(u64::MAX, true).cpu_secs, u64::MAX, "saturates"); + assert_eq!(sandbox_for(3, true).nproc, None); + assert!(sandbox_for(3, false).nproc.is_some()); } /// Each record is bounded by the largest a call can make, and the stream by the number @@ -889,6 +1140,7 @@ mod tests { imports: Vec::new(), vars: Default::default(), data_files: Vec::new(), + bundle: PathBuf::new(), allowed_imports: Vec::new(), load_timeout: 1, setup: Vec::new(), diff --git a/src/core/src/runner/sandbox.rs b/src/core/src/runner/sandbox.rs index 1e46ef4..07d213f 100644 --- a/src/core/src/runner/sandbox.rs +++ b/src/core/src/runner/sandbox.rs @@ -1,3 +1,17 @@ +//! What a unit's processes may do, and the operating system's means of holding them to it. +//! +//! Every process a unit starts runs under the kernel's resource limits (`setrlimit`). +//! Sandboxed, each also runs confined to what its side needs: on macOS by Seatbelt, under +//! `sandbox-exec` with a profile written here; on Linux by Landlock (files, TCP, signals, +//! abstract sockets) and seccomp (other sockets, connecting, starting processes, tracing), +//! both applied between fork and exec. A confinement that cannot be applied fails the +//! spawn: nothing runs unconfined by accident. Windows has no confinement yet. + +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use tokio::process::Command; + /// Resource limits for student code execution. /// /// Applied via `setrlimit` in a `pre_exec` hook (between fork and exec), @@ -6,15 +20,18 @@ pub struct SandboxConfig { /// CPU time limit in seconds (RLIMIT_CPU). Default: 30. pub cpu_secs: u64, - /// Virtual memory limit in MB (RLIMIT_AS). Default: 512. + /// Memory limit in MB: RLIMIT_AS on Linux, and the grader's watch on the student side + /// everywhere. Default: 512. pub mem_mb: u64, /// Max file write size in MB (RLIMIT_FSIZE). Default: 10. pub fsize_mb: u64, /// Max open file descriptors (RLIMIT_NOFILE). Default: 64. pub nofile: u64, /// Max processes for this UID (RLIMIT_NPROC). Default: 64. - /// Note: on macOS this is per-UID, not per-process. - pub nproc: u64, + /// It counts every process of the user — and on Linux every thread — so on a busy + /// machine it refuses even a thread. A sandboxed unit cannot start processes at all, + /// and leaves it unset (`None`). + pub nproc: Option, } impl Default for SandboxConfig { @@ -24,7 +41,7 @@ impl Default for SandboxConfig { mem_mb: 512, fsize_mb: 10, nofile: 64, - nproc: 64, + nproc: Some(64), } } } @@ -35,7 +52,7 @@ impl Default for SandboxConfig { /// in the forked child process before exec, so it only affects the /// student Python process. #[cfg(unix)] -pub fn apply_sandbox(cmd: &mut tokio::process::Command, config: &SandboxConfig) { +pub fn apply_sandbox(cmd: &mut Command, config: &SandboxConfig) { let cpu = config.cpu_secs; #[cfg(not(target_os = "macos"))] let mem = config.mem_mb * 1024 * 1024; @@ -56,7 +73,9 @@ pub fn apply_sandbox(cmd: &mut tokio::process::Command, config: &SandboxConfig) // A crash must not write the process's memory, teacher values included, to disk. let _ = set_rlimit(libc::RLIMIT_CORE, 0); let _ = set_rlimit(libc::RLIMIT_NOFILE, nofile); - let _ = set_rlimit(libc::RLIMIT_NPROC, nproc); + if let Some(nproc) = nproc { + let _ = set_rlimit(libc::RLIMIT_NPROC, nproc); + } // RLIMIT_AS: skip on macOS where it's unreliable #[cfg(not(target_os = "macos"))] let _ = set_rlimit(libc::RLIMIT_AS, mem); @@ -88,6 +107,472 @@ fn set_rlimits(resource: RlimitResource, soft: u64, hard: u64) -> std::io::Resul Ok(()) } +/// Whether a run's units are confined by the operating system. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Isolation { + /// Each side in its own sandbox; a unit whose sandbox cannot be applied is not graded. + #[default] + Sandboxed, + /// `--no-sandbox`: resource limits only. Student code can reach whatever the grader's + /// user can, and the results say so. + Unisolated, +} + +impl Isolation { + pub fn is_sandboxed(self) -> bool { + self == Isolation::Sandboxed + } +} + +/// What one confined process may touch. Paths must be canonical: the sandbox sees the paths +/// the kernel resolves, never the symlinks that lead to them. +#[derive(Debug, Clone, Default)] +pub struct Confinement { + /// Directories and files it may read. + pub read: Vec, + /// Directories it may read and write. + pub write: Vec, + /// What it may execute in place of itself: the interpreter, which re-executes itself on + /// some platforms, and its loader. + pub exec: Vec, + /// It may start other processes, running any program it may read: lint tools. Never + /// the student side. + pub spawn: bool, + /// Native calls (`ctypes`, `cffi`) are refused. On macOS they are how a process reads + /// another same-user process's arguments and environment, which Seatbelt does not + /// mediate. + pub no_ffi: bool, +} + +/// Whether this machine can confine a unit, and if not, why — in words a teacher can act on. +pub fn available() -> Result<(), String> { + platform::available() +} + +/// A command that runs `program` held to `confinement`. Its arguments are the caller's to add. +pub fn confined(program: &Path, confinement: &Confinement) -> Result { + platform::command(program, confinement) +} + +/// The program the kernel loads to start `executable`: its ELF interpreter, which a +/// confined process must be allowed to run too. +#[cfg(not(target_os = "macos"))] +pub fn loader_of(executable: &Path) -> Option { + use std::io::{Read, Seek, SeekFrom}; + const PT_INTERP: u32 = 3; + let mut file = std::fs::File::open(executable).ok()?; + let mut header = [0u8; 64]; + file.read_exact(&mut header).ok()?; + // 64-bit, little-endian ELF: what ScriptMark's Linux builds run on. + if &header[..4] != b"\x7fELF" || header[4] != 2 || header[5] != 1 { + return None; + } + let u16_at = |at: usize| u16::from_le_bytes([header[at], header[at + 1]]) as u64; + let phoff = u64::from_le_bytes(header[0x20..0x28].try_into().ok()?); + let (size, count) = (u16_at(0x36), u16_at(0x38)); + for index in 0..count { + let mut entry = [0u8; 56]; + file.seek(SeekFrom::Start(phoff + index * size)).ok()?; + file.read_exact(&mut entry).ok()?; + if u32::from_le_bytes(entry[..4].try_into().ok()?) != PT_INTERP { + continue; + } + let offset = u64::from_le_bytes(entry[8..16].try_into().ok()?); + let length = u64::from_le_bytes(entry[32..40].try_into().ok()?).min(4096) as usize; + let mut path = vec![0u8; length]; + file.seek(SeekFrom::Start(offset)).ok()?; + file.read_exact(&mut path).ok()?; + let path = std::str::from_utf8(path.split(|b| *b == 0).next()?).ok()?; + return std::fs::canonicalize(path).ok(); + } + None +} + +/// The shortest list covering `paths`: one inside another adds nothing. +pub fn covering(mut paths: Vec) -> Vec { + paths.sort(); + paths.dedup(); + let mut kept: Vec = Vec::new(); + for path in paths { + if !kept.iter().any(|k| path.starts_with(k)) { + kept.push(path); + } + } + kept +} + +#[cfg(target_os = "macos")] +mod platform { + use super::*; + + const SANDBOX_EXEC: &str = "/usr/bin/sandbox-exec"; + + pub fn available() -> Result<(), String> { + let probe = std::process::Command::new(SANDBOX_EXEC) + .args(["-p", "(version 1)(allow default)", "/usr/bin/true"]) + .output() + .map_err(|e| format!("{SANDBOX_EXEC} cannot run: {e}"))?; + if probe.status.success() { + return Ok(()); + } + Err(format!( + "{SANDBOX_EXEC} could not apply a sandbox ({}); a process already inside another sandbox cannot start one", + String::from_utf8_lossy(&probe.stderr).trim() + )) + } + + pub fn command(program: &Path, c: &Confinement) -> Result { + let mut cmd = Command::new(SANDBOX_EXEC); + cmd.arg("-p").arg(profile(c)?).arg(program); + Ok(cmd) + } + + /// An SBPL string literal. + fn quote(path: &Path) -> Result { + let text = path + .to_str() + .ok_or_else(|| format!("'{}' is not valid UTF-8", path.display()))?; + Ok(format!( + "\"{}\"", + text.replace('\\', "\\\\").replace('"', "\\\"") + )) + } + + fn subpaths(paths: &[PathBuf]) -> Result { + paths + .iter() + .map(|p| quote(p).map(|q| format!("(subpath {q})"))) + .collect::, _>>() + .map(|all| all.join(" ")) + } + + /// The Seatbelt profile for `c`. Everything is denied but what `system.sb` grants every + /// process (the dyld cache, system frameworks, `/dev/null`, `/dev/urandom`), and: + pub(super) fn profile(c: &Confinement) -> Result { + let mut rules = vec![ + "(version 1)".to_string(), + "(deny default)".to_string(), + "(import \"system.sb\")".to_string(), + ]; + let readable = covering(c.read.iter().chain(&c.write).cloned().collect()); + rules.push(format!("(allow file-read* {})", subpaths(&readable)?)); + // Resolving a path, and finding the working directory, stats every directory above it. + let ancestors: Vec = readable + .iter() + .flat_map(|p| p.ancestors().skip(1).map(Path::to_path_buf)) + .collect::>() + .into_iter() + .collect(); + if !ancestors.is_empty() { + let literals = ancestors + .iter() + .map(|p| quote(p).map(|q| format!("(literal {q})"))) + .collect::, _>>()?; + rules.push(format!("(allow file-read-metadata {})", literals.join(" "))); + } + if !c.write.is_empty() { + rules.push(format!("(allow file-write* {})", subpaths(&c.write)?)); + } + if !c.exec.is_empty() { + rules.push(format!("(allow process-exec {})", subpaths(&c.exec)?)); + } + rules.push("(allow signal (target self))".into()); + if c.spawn { + rules.push("(allow process-fork)".into()); + rules.push("(allow process-exec)".into()); + rules.push("(allow signal (target children))".into()); + } + if c.no_ffi { + rules.push( + r#"(deny file-read* (regex #"/_ctypes[^/]*\.so$") (regex #"/_cffi_backend[^/]*\.so$"))"# + .into(), + ); + } + Ok(rules.join("\n")) + } +} + +#[cfg(target_os = "linux")] +mod platform { + use super::*; + use landlock::{ + ABI, Access, AccessFs, AccessNet, CompatLevel, Compatible, PathBeneath, PathFd, Ruleset, + RulesetAttr, RulesetCreatedAttr, RulesetStatus, Scope, + }; + use seccompiler::{ + BpfProgram, SeccompAction, SeccompCmpArgLen, SeccompCmpOp, SeccompCondition, SeccompFilter, + SeccompRule, TargetArch, + }; + use std::collections::BTreeMap; + + /// The Landlock ABI the running kernel offers: 0 when it has none. + fn kernel_abi() -> i64 { + const LANDLOCK_CREATE_RULESET_VERSION: libc::c_uint = 1; + // SAFETY: with this flag the call only reports a number. + unsafe { + libc::syscall( + libc::SYS_landlock_create_ruleset, + std::ptr::null::(), + 0usize, + LANDLOCK_CREATE_RULESET_VERSION, + ) + } + } + + fn abi() -> Result { + Ok(match kernel_abi() { + n if n <= 0 => { + return Err( + "this kernel has no Landlock (Linux 5.13 or later, with Landlock enabled at boot, is needed)" + .into(), + ); + } + 1 => ABI::V1, + 2 => ABI::V2, + 3 => ABI::V3, + 4 => ABI::V4, + 5 => ABI::V5, + 6 => ABI::V6, + 7 => ABI::V7, + 8 => ABI::V8, + _ => ABI::V9, + }) + } + + pub fn available() -> Result<(), String> { + abi().map(|_| ()) + } + + pub fn command(program: &Path, c: &Confinement) -> Result { + let abi = abi()?; + let ruleset = landlock_ruleset(abi, c).map_err(|e| format!("Landlock: {e}"))?; + // Signals and abstract sockets are Landlock's from ABI 6; before it, seccomp's. + let filters = filters(c, abi < ABI::V6)?; + let mut cmd = Command::new(program); + let mut ruleset = Some(ruleset); + // SAFETY: between fork and exec this only makes the syscalls that apply a prepared + // ruleset and prepared filters; nothing is allocated unless one fails. + unsafe { + cmd.pre_exec(move || { + if let Some(ruleset) = ruleset.take() { + let status = ruleset.restrict_self().map_err(std::io::Error::other)?; + if status.ruleset != RulesetStatus::FullyEnforced { + return Err(std::io::Error::other("Landlock was not fully enforced")); + } + } + for filter in &filters { + seccompiler::apply_filter(filter).map_err(std::io::Error::other)?; + } + Ok(()) + }); + } + Ok(cmd) + } + + fn landlock_ruleset( + abi: ABI, + c: &Confinement, + ) -> Result { + let all = AccessFs::from_all(abi); + let read = AccessFs::ReadFile | AccessFs::ReadDir; + // Writing never makes devices, pipes or sockets in the filesystem, and never runs + // what it wrote. + let write = all + & !(AccessFs::Execute + | AccessFs::MakeChar + | AccessFs::MakeBlock + | AccessFs::MakeFifo + | AccessFs::MakeSock); + let readable = if c.spawn { + read | AccessFs::Execute + } else { + read.into() + }; + let mut ruleset = Ruleset::default() + .set_compatibility(CompatLevel::HardRequirement) + .handle_access(all)?; + if abi >= ABI::V4 { + ruleset = ruleset.handle_access(AccessNet::from_all(abi))?; + } + if abi >= ABI::V6 { + ruleset = ruleset.scope(Scope::from_all(abi))?; + } + let rule = |path: &PathBuf, access: landlock::BitFlags| { + // A path that does not exist grants nothing. + PathFd::new(path) + .ok() + .map(|fd| PathBeneath::new(fd, access & AccessFs::from_file(abi))) + }; + let dir_rule = |path: &PathBuf, access: landlock::BitFlags| { + PathFd::new(path).ok().map(|fd| { + let access = if path.is_dir() { + access + } else { + access & AccessFs::from_file(abi) + }; + PathBeneath::new(fd, access) + }) + }; + let devices = [ + ("/dev/null", read | AccessFs::WriteFile), + ("/dev/zero", read.into()), + ("/dev/urandom", read.into()), + ("/dev/random", read.into()), + ] + .into_iter() + .filter_map(|(path, access)| rule(&PathBuf::from(path), access)); + // `no_new_privs` is set with the ruleset: a setuid program cannot shed it. + ruleset + .create()? + .add_rules( + c.read + .iter() + .filter_map(|p| dir_rule(p, readable)) + .map(Ok::<_, landlock::RulesetError>), + )? + .add_rules( + c.write + .iter() + .filter_map(|p| dir_rule(p, write)) + .map(Ok::<_, landlock::RulesetError>), + )? + .add_rules( + c.exec + .iter() + .filter_map(|p| rule(p, read | AccessFs::Execute)) + .map(Ok::<_, landlock::RulesetError>), + )? + .add_rules(devices.map(Ok::<_, landlock::RulesetError>)) + } + + fn arch() -> Result { + if cfg!(target_arch = "x86_64") { + Ok(TargetArch::x86_64) + } else if cfg!(target_arch = "aarch64") { + Ok(TargetArch::aarch64) + } else if cfg!(target_arch = "riscv64") { + Ok(TargetArch::riscv64) + } else { + Err("seccomp filters are not built for this architecture".into()) + } + } + + fn condition(arg: u8, op: SeccompCmpOp, value: u64) -> Result { + SeccompCondition::new(arg, SeccompCmpArgLen::Qword, op, value) + .and_then(|c| SeccompRule::new(vec![c])) + .map_err(|e| e.to_string()) + } + + /// The seccomp filters for `c`. Landlock governs paths; these govern what it cannot: + /// sockets other than a pair the process makes for itself, connecting or binding + /// anything, sending to an address, starting processes, tracing, and the kernel's other + /// ways out. `signals` adds the kill family, when Landlock cannot scope signals. + fn filters(c: &Confinement, signals: bool) -> Result, String> { + let mut denied: BTreeMap> = BTreeMap::new(); + for call in [ + libc::SYS_connect, + libc::SYS_bind, + libc::SYS_listen, + libc::SYS_accept, + libc::SYS_accept4, + libc::SYS_sendmsg, + libc::SYS_sendmmsg, + libc::SYS_ptrace, + libc::SYS_process_vm_readv, + libc::SYS_process_vm_writev, + libc::SYS_mount, + libc::SYS_umount2, + libc::SYS_pivot_root, + libc::SYS_chroot, + libc::SYS_unshare, + libc::SYS_setns, + libc::SYS_bpf, + libc::SYS_perf_event_open, + libc::SYS_userfaultfd, + libc::SYS_keyctl, + libc::SYS_add_key, + libc::SYS_request_key, + libc::SYS_io_uring_setup, + libc::SYS_io_uring_enter, + libc::SYS_io_uring_register, + libc::SYS_kexec_load, + libc::SYS_init_module, + libc::SYS_finit_module, + libc::SYS_delete_module, + libc::SYS_pidfd_getfd, + libc::SYS_open_by_handle_at, + libc::SYS_name_to_handle_at, + ] { + denied.insert(call, Vec::new()); + } + // Only the local pairs `socketpair` makes, and `socket(AF_UNIX)` for asyncio's. + denied.insert( + libc::SYS_socket, + vec![condition(0, SeccompCmpOp::Ne, libc::AF_UNIX as u64)?], + ); + denied.insert(libc::SYS_sendto, vec![condition(4, SeccompCmpOp::Ne, 0)?]); + if !c.spawn { + // A thread shares its process; anything else is a new process. + denied.insert( + libc::SYS_clone, + vec![condition( + 0, + SeccompCmpOp::MaskedEq(libc::CLONE_THREAD as u64), + 0, + )?], + ); + #[cfg(target_arch = "x86_64")] + for call in [libc::SYS_fork, libc::SYS_vfork] { + denied.insert(call, Vec::new()); + } + } + if signals { + for call in [ + libc::SYS_kill, + libc::SYS_tkill, + libc::SYS_tgkill, + libc::SYS_rt_sigqueueinfo, + libc::SYS_rt_tgsigqueueinfo, + libc::SYS_pidfd_send_signal, + ] { + denied.insert(call, Vec::new()); + } + } + let arch = arch()?; + let build = |rules, errno: i32| -> Result { + SeccompFilter::new( + rules, + SeccompAction::Allow, + SeccompAction::Errno(errno as u32), + arch, + ) + .and_then(BpfProgram::try_from) + .map_err(|e| e.to_string()) + }; + // `clone3` takes its flags by pointer, where a filter cannot look: refuse it as + // unknown, and the C library falls back to `clone`. + let clone3 = BTreeMap::from([(libc::SYS_clone3, Vec::new())]); + Ok(vec![ + build(denied, libc::EPERM)?, + build(clone3, libc::ENOSYS)?, + ]) + } +} + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +mod platform { + use super::*; + + pub fn available() -> Result<(), String> { + Err("ScriptMark cannot sandbox student code on this platform yet".into()) + } + + pub fn command(_program: &Path, _c: &Confinement) -> Result { + Err(available().unwrap_err()) + } +} + #[cfg(test)] mod tests { use super::*; @@ -99,7 +584,7 @@ mod tests { assert_eq!(config.mem_mb, 512); assert_eq!(config.fsize_mb, 10); assert_eq!(config.nofile, 64); - assert_eq!(config.nproc, 64); + assert_eq!(config.nproc, Some(64)); } #[cfg(unix)] @@ -113,4 +598,39 @@ mod tests { assert!(output.status.success()); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "hello"); } + + #[test] + fn test_covering_keeps_only_the_outermost_paths() { + let paths = ["/a/b", "/a", "/c", "/a/b/c", "/ab"] + .map(PathBuf::from) + .to_vec(); + assert_eq!( + covering(paths), + ["/a", "/ab", "/c"].map(PathBuf::from).to_vec() + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn test_a_profile_quotes_paths_and_grants_only_what_it_lists() { + let c = Confinement { + read: vec![PathBuf::from("/opt/py")], + write: vec![PathBuf::from("/tmp/unit \"x\"/work")], + exec: vec![PathBuf::from("/opt/py/bin")], + spawn: false, + no_ffi: true, + }; + let profile = platform::profile(&c).unwrap(); + assert!(profile.contains("(deny default)")); + assert!( + profile.contains(r#"(subpath "/tmp/unit \"x\"/work")"#), + "{profile}" + ); + assert!( + profile.contains(r#"(literal "/tmp/unit \"x\"")"#), + "{profile}" + ); + assert!(profile.contains("_ctypes")); + assert!(!profile.contains("process-fork")); + } } diff --git a/src/core/tests/harness.rs b/src/core/tests/harness.rs index 0f92027..1fb6b69 100644 --- a/src/core/tests/harness.rs +++ b/src/core/tests/harness.rs @@ -52,6 +52,7 @@ fn unit(file: &Path) -> UnitPlan { imports: Vec::new(), vars: Arc::new(BTreeMap::new()), data_files: Vec::new(), + bundle: file.parent().unwrap().to_path_buf(), allowed_imports: Vec::new(), load_timeout: 5, setup: Vec::new(), diff --git a/src/core/tests/integration.rs b/src/core/tests/integration.rs index 0fd71e0..a218c00 100644 --- a/src/core/tests/integration.rs +++ b/src/core/tests/integration.rs @@ -11,6 +11,7 @@ use scriptmark_core::runner::frozen::Generation; use scriptmark_core::runner::orchestrator::{RunOptions, run_all}; use scriptmark_core::runner::prepare::prepare; use scriptmark_core::runner::python::PythonExecutor; +use scriptmark_core::runner::sandbox::Isolation; use scriptmark_core::spec_loader::load_spec_str; /// A scratch directory holding specs, teacher files and one directory per student. @@ -701,7 +702,10 @@ expect = "recorded" "lab.py", "def record(path):\n with open(path, 'a') as fh:\n fh.write('ran\\n')\n return 'recorded'\n\ndef echo(x):\n return x\n", ); - let results = grade(vec![spec], &[student]).await; + // The student counts its setup calls in logs shared by every unit, outside them all: + // only an unsandboxed unit can write there. + let unconfined = PythonExecutor::new().with_isolation(Isolation::Unisolated); + let results = grade_with(vec![spec], &[student], unconfined, 5).await; assert_eq!(results[0].total_passed(), 4, "{:#?}", results[0]); let lines = |p: &Path| std::fs::read_to_string(p).unwrap().lines().count(); diff --git a/src/core/tests/oracle.rs b/src/core/tests/oracle.rs index e882a84..322ddcf 100644 --- a/src/core/tests/oracle.rs +++ b/src/core/tests/oracle.rs @@ -8,6 +8,7 @@ use scriptmark_core::runner::frozen::{Frozen, Generation}; use scriptmark_core::runner::orchestrator::{RunOptions, run_all}; use scriptmark_core::runner::prepare::{Bundle, prepare}; use scriptmark_core::runner::python::PythonExecutor; +use scriptmark_core::runner::sandbox::Isolation; use scriptmark_core::spec_loader::load_spec_str; use serde_json::{Value, json}; @@ -46,6 +47,15 @@ async fn prepared(spec: TestSpec, generation: &Generation) -> Result .map_err(|e| e.to_string()) } +/// For a reference that writes outside its unit — counting its calls, or changing its own +/// source — which only an unsandboxed reference can. +async fn prepared_unconfined(spec: TestSpec) -> Result, String> { + let executor = PythonExecutor::new().with_isolation(Isolation::Unisolated); + prepare(vec![spec], &Generation::fresh(), Arc::new(executor), 3) + .await + .map_err(|e| e.to_string()) +} + async fn run(bundles: Vec, students: &[StudentSubmission]) -> Vec { run_all( students, @@ -80,9 +90,7 @@ async fn fixed_samples_and_seeded_inputs_share_answers_across_students_and_batch let body: String = format!( "{FIXED}\n[[cases]]\nname = 'generated'\n[cases.parametrize]\nargs = {{ x = 'int(0, 10)' }}\nsamples = [[2], [3]]\n[cases.parametrize.random]\ncount = 4\nseed = 42\n[cases.oracle]\nreference = 'ref.py'\nfunction = 'answer'\n" ); - let bundles: Vec = prepared(bench.spec(&body), &Generation::fresh()) - .await - .unwrap(); + let bundles: Vec = prepared_unconfined(bench.spec(&body)).await.unwrap(); let frozen: Frozen = Frozen::of(&bundles); let path: PathBuf = bench.path().join("batch.cases.json"); frozen.write(&path).unwrap(); @@ -323,9 +331,7 @@ async fn sources_cannot_change_while_answers_are_being_prepared() { let bench: Bench = Bench::new(); let source: PathBuf = bench.path().join("ref.py"); bench.write("ref.py", &format!("def answer(x):\n with open({source:?}, 'a') as f:\n f.write('\\n# changed\\n')\n return x\n")); - let error: String = prepared(bench.spec(FIXED), &Generation::fresh()) - .await - .unwrap_err(); + let error: String = prepared_unconfined(bench.spec(FIXED)).await.unwrap_err(); assert!( error.contains("sources changed during preparation"), "{error}" diff --git a/src/core/tests/sandbox.rs b/src/core/tests/sandbox.rs new file mode 100644 index 0000000..efe5f62 --- /dev/null +++ b/src/core/tests/sandbox.rs @@ -0,0 +1,321 @@ +//! The sandboxes, probed: what each side of a unit can reach. Every probe runs twice, the +//! second time without a sandbox, so a denial is the sandbox's doing and not a probe that +//! could never have worked. +#![cfg(any(target_os = "macos", target_os = "linux"))] + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use scriptmark_core::models::Target; +use scriptmark_core::runner::executor::{ + CallPlan, Executor, Outcome, Subject, UnitObservation, UnitPlan, +}; +use scriptmark_core::runner::python::PythonExecutor; +use scriptmark_core::runner::sandbox::Isolation; +use serde_json::{Value, json}; + +fn write(dir: &Path, name: &str, content: &str) -> PathBuf { + let path = dir.join(name); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, content).unwrap(); + path +} + +fn call(target: Target, args: Vec) -> CallPlan { + CallPlan { + target, + args, + stdin: None, + timeout: 5, + id: None, + files: Vec::new(), + check: None, + } +} + +fn function(name: &str, args: Vec) -> CallPlan { + call( + Target::Function { + name: name.to_string(), + }, + args, + ) +} + +fn teacher(name: &str, args: Vec) -> CallPlan { + call( + Target::Teacher { + name: name.to_string(), + }, + args, + ) +} + +/// Every probe answers with what happened, rather than raising: `ok …` or the error's type. +const PROBES: &str = r#"import importlib +os = importlib.import_module('os') +socket = importlib.import_module('socket') +subprocess = importlib.import_module('subprocess') + +def attempt(fn): + try: + return 'ok ' + str(fn())[:40] + except BaseException as exc: + return type(exc).__name__ + +def read(path): + return attempt(lambda: open(path).read()) + +def listing(path): + return attempt(lambda: sorted(os.listdir(path))) + +def write(path): + return attempt(lambda: open(path, 'w').write('x')) + +def connect(port): + return attempt(lambda: socket.create_connection(('127.0.0.1', port), timeout=3).close()) + +def spawn(): + return attempt(lambda: subprocess.run(['/bin/echo', 'escaped'], capture_output=True).stdout) + +def run_program(): + # In place of this process: no fork, so no process limit stands in for the sandbox. + return attempt(lambda: os.execv('/bin/echo', ['/bin/echo', 'escaped'])) + +def signal_grader(): + return attempt(lambda: os.kill(os.getppid(), 0)) + +def native(): + return attempt(lambda: importlib.import_module('ctypes').CDLL(None)) + +def environ_of_grader(): + return attempt(lambda: open(f'/proc/{os.getppid()}/environ', 'rb').read()) + +def env(): + return attempt(lambda: sorted(os.environ)) +"#; + +struct Bench { + _dir: tempfile::TempDir, + root: PathBuf, + bundle: PathBuf, + answers: PathBuf, + other: PathBuf, + student: PathBuf, + teacher: PathBuf, + /// A port the grader listens on, which no unit may connect to. + port: u16, + _listener: std::net::TcpListener, +} + +impl Bench { + fn new() -> Self { + let dir = tempfile::tempdir().unwrap(); + let root = std::fs::canonicalize(dir.path()).unwrap(); + let bundle = root.join("tests"); + let answers = write(&bundle, "answers.txt", "the teacher's answers\n"); + let teacher = write(&bundle, "helpers/probes.py", PROBES); + let other = write(&root, "submissions/bob/lab.py", "SECRET = 'bob'\n"); + let student = write(&root, "submissions/alice/lab.py", PROBES); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + Self { + port: listener.local_addr().unwrap().port(), + _listener: listener, + _dir: dir, + root, + bundle, + answers, + other, + student, + teacher, + } + } + + fn plan(&self, steps: Vec) -> UnitPlan { + UnitPlan { + subject: Subject::Student, + functions: scriptmark_core::matching::Functions::default(), + file: self.student.clone(), + script: None, + imports: vec![self.teacher.to_string_lossy().into_owned()], + vars: Arc::new(BTreeMap::new()), + data_files: Vec::new(), + bundle: self.bundle.clone(), + allowed_imports: Vec::new(), + load_timeout: 5, + setup: Vec::new(), + steps, + } + } +} + +async fn both(plan: &UnitPlan) -> (UnitObservation, UnitObservation) { + let sandboxed = PythonExecutor::new().run(plan).await; + let open = PythonExecutor::new() + .with_isolation(Isolation::Unisolated) + .run(plan) + .await; + (sandboxed, open) +} + +fn said(obs: &UnitObservation, index: usize) -> String { + match &obs.steps[index].outcome { + Outcome::Returned { value, .. } => value.as_str().unwrap_or_default().to_string(), + other => panic!("step {index}: {other:?}\n{obs:?}"), + } +} + +/// What each probe said with the sandbox and without it, by name. +fn compare( + names: &[&str], + sandboxed: &UnitObservation, + open: &UnitObservation, +) -> Vec<(String, String, String)> { + assert!(sandboxed.done && sandboxed.sandboxed, "{sandboxed:?}"); + assert!(open.done && !open.sandboxed, "{open:?}"); + names + .iter() + .enumerate() + .map(|(i, name)| (name.to_string(), said(sandboxed, i), said(open, i))) + .collect() +} + +fn denied(probes: &[(String, String, String)], name: &str) { + let (_, sandboxed, open) = probes.iter().find(|(n, ..)| n == name).unwrap(); + if name == "spawn" { + // The process limit is per user on macOS, where a busy machine already holds more + // than a unit may start: fork can fail for that before the sandbox is asked. + // `run program` probes the sandbox itself. + assert!(!sandboxed.starts_with("ok"), "{name}: {sandboxed}"); + return; + } + if name == "run program" { + // Unsandboxed, it replaces the process and never answers: no control to compare. + assert_eq!(sandboxed, "PermissionError", "{name}: {sandboxed}"); + return; + } + assert!( + open.starts_with("ok"), + "{name}: the probe must work unsandboxed, said {open}" + ); + assert!( + !sandboxed.starts_with("ok"), + "{name}: the sandbox let it through: {sandboxed}" + ); +} + +fn allowed(probes: &[(String, String, String)], name: &str) { + let (_, sandboxed, _) = probes.iter().find(|(n, ..)| n == name).unwrap(); + assert!( + sandboxed.starts_with("ok"), + "{name}: the sandbox refused it: {sandboxed}" + ); +} + +#[tokio::test] +async fn test_the_student_side_reaches_only_its_own_directory() { + let bench = Bench::new(); + let home = std::env::var("HOME").unwrap(); + let path = |p: &Path| json!(p.to_string_lossy()); + let mut probes = vec![ + ("bundle", function("read", vec![path(&bench.answers)])), + ( + "bundle listing", + function("listing", vec![path(&bench.bundle)]), + ), + ( + "teacher module", + function("read", vec![path(&bench.teacher)]), + ), + ("other student", function("read", vec![path(&bench.other)])), + ( + "submissions", + function("listing", vec![path(&bench.root.join("submissions"))]), + ), + ("home", function("listing", vec![json!(home)])), + ( + "write beside the bundle", + function("write", vec![path(&bench.bundle.join("planted.py"))]), + ), + ("network", function("connect", vec![json!(bench.port)])), + ("spawn", function("spawn", vec![])), + ("signal", function("signal_grader", vec![])), + ("own directory", function("write", vec![json!("mine.txt")])), + ]; + let mut sandboxed_only = vec![("run program", function("run_program", vec![]))]; + if cfg!(target_os = "macos") { + // Seatbelt does not mediate reading another process's arguments and environment; + // refusing native calls is what keeps the grader's out of reach. + probes.push(("native calls", function("native", vec![]))); + } + if cfg!(target_os = "linux") { + probes.push(( + "grader's environment", + function("environ_of_grader", vec![]), + )); + } + let names: Vec<&str> = probes.iter().map(|(n, _)| *n).collect(); + let plan = bench.plan(probes.into_iter().map(|(_, c)| c).collect()); + let (sandboxed, open) = both(&plan).await; + let results = compare(&names, &sandboxed, &open); + for name in names.iter().filter(|n| **n != "own directory") { + denied(&results, name); + } + allowed(&results, "own directory"); + + let (name, probe) = sandboxed_only.remove(0); + let obs = PythonExecutor::new().run(&bench.plan(vec![probe])).await; + denied(&[(name.to_string(), said(&obs, 0), String::new())], name); +} + +#[tokio::test] +async fn test_the_teacher_side_reads_the_bundle_and_the_source_and_nothing_else() { + let bench = Bench::new(); + let home = std::env::var("HOME").unwrap(); + let path = |p: &Path| json!(p.to_string_lossy()); + let probes = vec![ + ("bundle", teacher("read", vec![path(&bench.answers)])), + ("other student", teacher("read", vec![path(&bench.other)])), + ( + "submissions", + teacher("listing", vec![path(&bench.root.join("submissions"))]), + ), + ("home", teacher("listing", vec![json!(home)])), + ( + "write the bundle", + teacher("write", vec![path(&bench.bundle.join("planted.py"))]), + ), + ("network", teacher("connect", vec![json!(bench.port)])), + ("spawn", teacher("spawn", vec![])), + ("signal", teacher("signal_grader", vec![])), + ]; + let exec = teacher("run_program", vec![]); + let names: Vec<&str> = probes.iter().map(|(n, _)| *n).collect(); + let plan = bench.plan(probes.into_iter().map(|(_, c)| c).collect()); + let (sandboxed, open) = both(&plan).await; + let results = compare(&names, &sandboxed, &open); + allowed(&results, "bundle"); + for name in names.iter().filter(|n| **n != "bundle") { + denied(&results, name); + } + let obs = PythonExecutor::new().run(&bench.plan(vec![exec])).await; + denied( + &[("run program".into(), said(&obs, 0), String::new())], + "run program", + ); +} + +#[tokio::test] +async fn test_neither_side_sees_the_graders_environment() { + let bench = Bench::new(); + let plan = bench.plan(vec![function("env", vec![]), teacher("env", vec![])]); + let obs = PythonExecutor::new().run(&plan).await; + for index in 0..2 { + let seen = said(&obs, index); + assert!( + !seen.contains("CANVAS") && !seen.contains("CARGO"), + "step {index} saw {seen}" + ); + } +} From 62f8a2e69839840425605d7dd9365161f3b0bf47 Mon Sep 17 00:00:00 2001 From: Acture Date: Thu, 15 Oct 2026 18:18:30 +0800 Subject: [PATCH 04/20] feat: shared classes, the source parameter, and teacher tools in the teacher's sandbox (OSS-10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `[meta] shared = ["common.py"]` names teacher modules whose classes cross by value. Each is staged beside the student's file, where it imports without an `allowed_imports` entry, and the teacher side imports its own copy from the bundle: an instance is rebuilt from the receiving side's copy of its class, without `__init__`, a student subclass as the shared base, cycles and shared nodes intact. Spec validation keeps them importable Python files of their own, apart from data files, teacher modules and the student's file. - Function checkers may take `source`, the student's source text, reserved beside `stdout`. - Python checker scripts run with a cleared environment and a private working directory, and, sandboxed, in the teacher's sandbox: they read their interpreter and the bundle only. - Lint runs on a read-only copy of each graded file, in the teacher's sandbox (the system's programs, the tool's own installation, the bundle), with a cleared environment and a deadline. Its command is split as a shell would, quotes included, and every graded file of a spec with `[lint]` is linted: the style score is their mean. - `prepare` refuses a path into the bundle handed to student code — a [vars] entry, an argument, or a teacher module's constant — since student code can never open it; `data_files` stages such a file instead. - Sandbox hardening from review: a confined unit keeps a high per-user process cap against a thread bomb; the interpreter's read set is its prefixes and sysconfig's library paths, not every `sys.path` entry; native code is never mapped from a directory a side can write, and hard links are refused; x32 syscall numbers are refused on x86-64. --- src/core/Cargo.toml | 1 + src/core/src/checker/python_checker.rs | 174 +++++++++++----- src/core/src/models/spec.rs | 7 + src/core/src/record.rs | 1 + src/core/src/runner/answers.rs | 6 + src/core/src/runner/executor.rs | 6 + src/core/src/runner/harness/teacher.py | 24 ++- src/core/src/runner/judge.rs | 75 ++++--- src/core/src/runner/linter.rs | 274 ++++++++++++++++++++++--- src/core/src/runner/oracle.rs | 6 + src/core/src/runner/orchestrator.rs | 106 +++++++--- src/core/src/runner/prepare.rs | 108 +++++++++- src/core/src/runner/python.rs | 134 +++++++++--- src/core/src/runner/sandbox.rs | 89 ++++++-- src/core/src/spec_loader.rs | 185 +++++++++++++++-- src/core/tests/harness.rs | 145 +++++++++++++ src/core/tests/integration.rs | 62 +++++- src/core/tests/sandbox.rs | 89 +++++++- 18 files changed, 1276 insertions(+), 216 deletions(-) diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 04d1e40..95b6e16 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -37,6 +37,7 @@ rars = "0.9.4" bzip2-rs = "0.1.2" lzma-rs = "0.3.0" tempfile = "3" +shlex = "2" sha2 = "0.10" [target.'cfg(target_os = "linux")'.dependencies] diff --git a/src/core/src/checker/python_checker.rs b/src/core/src/checker/python_checker.rs index bfbc2e4..b057624 100644 --- a/src/core/src/checker/python_checker.rs +++ b/src/core/src/checker/python_checker.rs @@ -1,5 +1,5 @@ use std::io::{Read, Write}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::time::Duration; @@ -13,10 +13,16 @@ use super::{CheckError, CheckInput, CheckOutput, Checker}; /// /// The script's existence is checked when the spec loads, so a script that then times out, /// crashes or prints no verdict did so on this student's value: the answer is rejected. +/// +/// It runs with a cleared environment and a private directory for its working directory, +/// `HOME` and `TMPDIR`; confined, in the teacher's sandbox, it reads only its interpreter +/// and the places it is given — the test bundle — and writes only that directory. pub struct PythonChecker { pub script_path: PathBuf, pub python_cmd: String, pub timeout_secs: u64, + /// What it may read besides its interpreter, when it runs in the teacher's sandbox. + pub confined: Option>, } impl PythonChecker { @@ -25,6 +31,7 @@ impl PythonChecker { script_path: script_path.into(), python_cmd: "python3".to_string(), timeout_secs: 10, + confined: None, } } @@ -32,6 +39,41 @@ impl PythonChecker { self.python_cmd = cmd.into(); self } + + /// Run in the teacher's sandbox, reading `read` besides the interpreter. + pub fn confined(mut self, read: Vec) -> Self { + self.confined = Some(read); + self + } + + fn command(&self, scratch: &Path) -> Result { + let mut cmd = match &self.confined { + Some(read) => { + let confinement = crate::runner::python::teacher_confinement( + &self.python_cmd, + read.clone(), + vec![scratch.to_path_buf()], + false, + )?; + crate::runner::sandbox::confined(Path::new(&self.python_cmd), &confinement)? + } + None => Command::new(&self.python_cmd), + }; + cmd.env_clear() + .env("PATH", "/usr/bin:/usr/local/bin:/opt/homebrew/bin") + .env("HOME", scratch) + .env("TMPDIR", scratch) + .current_dir(scratch) + // No user site-packages and no bytecode beside the script; the script's own + // directory stays on sys.path, for the helpers beside it. + .args(["-s", "-B"]) + .arg(&self.script_path); + #[cfg(windows)] + if let Some(root) = std::env::var_os("SYSTEMROOT") { + cmd.env("SYSTEMROOT", root); + } + Ok(cmd) + } } impl Checker for PythonChecker { @@ -40,61 +82,34 @@ impl Checker for PythonChecker { let input_json = serde_json::to_string(input) .map_err(|e| CheckError::environment(format!("could not encode checker input: {e}")))?; - let mut child = Command::new(&self.python_cmd) - .arg(&self.script_path) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() + let scratch = tempfile::Builder::new() + .prefix("scriptmark-check-") + .tempdir() + .and_then(|dir| std::fs::canonicalize(dir.path()).map(|path| (dir, path))) .map_err(|e| { - CheckError::environment(format!("could not start checker '{script}': {e}")) + CheckError::environment(format!("could not stage checker '{script}': {e}")) })?; - - // Write and read on their own threads: a checker that ignores its input, or prints - // more than a pipe holds, must not wedge the grader until the timeout. - if let Some(mut stdin) = child.stdin.take() { - std::thread::spawn(move || { - // A checker that exits without reading its input closes the pipe; its output decides. - let _ = stdin.write_all(input_json.as_bytes()); - }); - } - let drain = |pipe: Option>| { - std::thread::spawn(move || { - let mut buf = String::new(); - if let Some(mut pipe) = pipe { - let _ = pipe.read_to_string(&mut buf); - } - buf - }) - }; - let stdout = drain( - child - .stdout - .take() - .map(|p| Box::new(p) as Box), - ); - let stderr = drain( - child - .stderr - .take() - .map(|p| Box::new(p) as Box), - ); - - let status = child - .wait_timeout(Duration::from_secs(self.timeout_secs)) - .map_err(|e| CheckError::environment(format!("lost checker '{script}': {e}")))?; - let Some(status) = status else { - let _ = child.kill(); - let _ = child.wait(); + let cmd = self.command(&scratch.1).map_err(|e| { + CheckError::environment(format!("cannot sandbox checker '{script}': {e}")) + })?; + let ran = run_with_timeout( + cmd, + Some(input_json), + Duration::from_secs(self.timeout_secs), + ) + .map_err(|e| CheckError::environment(format!("could not start checker '{script}': {e}")))?; + let Some(Ran { + status, + stdout, + stderr, + }) = ran + else { return Err(CheckError::student(format!( "checker '{script}' timed out after {}s", self.timeout_secs ))); }; - - let stdout = stdout.join().unwrap_or_default(); if !status.success() && stdout.trim().is_empty() { - let stderr = stderr.join().unwrap_or_default(); return Err(CheckError::student(format!( "checker '{script}' exited with {status}: {}", stderr.trim() @@ -109,6 +124,69 @@ impl Checker for PythonChecker { } } +/// What a finished tool left: how it ended, and what it printed. +pub(crate) struct Ran { + pub status: std::process::ExitStatus, + pub stdout: String, + pub stderr: String, +} + +/// Run `cmd` with `stdin` written to it, killing it at `timeout`: `None` then. Its input is +/// written and its output read on their own threads, so a tool that ignores its input, or +/// prints more than a pipe holds, cannot wedge the grader until the timeout. +pub(crate) fn run_with_timeout( + mut cmd: Command, + stdin: Option, + timeout: Duration, +) -> std::io::Result> { + let mut child = cmd + .stdin(if stdin.is_some() { + Stdio::piped() + } else { + Stdio::null() + }) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + if let (Some(text), Some(mut pipe)) = (stdin, child.stdin.take()) { + std::thread::spawn(move || { + // A tool that exits without reading its input closes the pipe; its output decides. + let _ = pipe.write_all(text.as_bytes()); + }); + } + let drain = |pipe: Option>| { + std::thread::spawn(move || { + let mut buf = String::new(); + if let Some(mut pipe) = pipe { + let _ = pipe.read_to_string(&mut buf); + } + buf + }) + }; + let stdout = drain( + child + .stdout + .take() + .map(|p| Box::new(p) as Box), + ); + let stderr = drain( + child + .stderr + .take() + .map(|p| Box::new(p) as Box), + ); + let Some(status) = child.wait_timeout(timeout)? else { + let _ = child.kill(); + let _ = child.wait(); + return Ok(None); + }; + Ok(Some(Ran { + status, + stdout: stdout.join().unwrap_or_default(), + stderr: stderr.join().unwrap_or_default(), + })) +} + // wait_timeout is not in std — implement using a thread trait WaitTimeout { fn wait_timeout( diff --git a/src/core/src/models/spec.rs b/src/core/src/models/spec.rs index ea8af13..265d9d5 100644 --- a/src/core/src/models/spec.rs +++ b/src/core/src/models/spec.rs @@ -568,6 +568,13 @@ pub struct TestMeta { #[serde(default)] pub data_files: Vec, + /// Teacher modules whose classes cross between the two sides of a unit by value. Each + /// is copied beside the student's file, where student code imports it without an + /// `allowed_imports` entry, and the teacher side imports its own copy from the bundle: + /// an instance one side sends is rebuilt from the other side's copy of its class. + #[serde(default)] + pub shared: Vec, + /// Refused: every case now has its own process. #[serde(default)] pub copy_refs: Option, diff --git a/src/core/src/record.rs b/src/core/src/record.rs index 8ef8b65..010418e 100644 --- a/src/core/src/record.rs +++ b/src/core/src/record.rs @@ -619,6 +619,7 @@ fn sources(spec: &TestSpec) -> Result, String> { .map(|p| spec.dir.join(p)) .collect(); let mut python: BTreeSet = spec.meta.imports.iter().map(PathBuf::from).collect(); + python.extend(spec.meta.shared.iter().map(|p| spec.dir.join(p))); for case in spec .cases .iter() diff --git a/src/core/src/runner/answers.rs b/src/core/src/runner/answers.rs index dfc892a..93e658c 100644 --- a/src/core/src/runner/answers.rs +++ b/src/core/src/runner/answers.rs @@ -70,6 +70,12 @@ impl Contract { pub fn of(spec: &TestSpec, executor: &E, timeout: u64) -> Result { let mut sources: BTreeMap = BTreeMap::new(); let mut paths: BTreeSet = spec.meta.imports.iter().cloned().collect(); + paths.extend( + spec.meta + .shared + .iter() + .map(|p| spec.dir.join(p).to_string_lossy().into_owned()), + ); paths.extend( spec.meta .data_files diff --git a/src/core/src/runner/executor.rs b/src/core/src/runner/executor.rs index 7cb4f03..1e1e2ab 100644 --- a/src/core/src/runner/executor.rs +++ b/src/core/src/runner/executor.rs @@ -56,6 +56,9 @@ pub struct UnitPlan { pub data_files: Vec<(PathBuf, String)>, /// The test bundle's directory: what teacher code may read, besides its own modules. pub bundle: PathBuf, + /// Shared modules, as `(source in the bundle, relative destination)`: staged beside the + /// student's file, and imported by the teacher side from the bundle. + pub shared: Vec<(PathBuf, String)>, pub allowed_imports: Vec, pub load_timeout: u64, pub setup: Vec, @@ -140,6 +143,9 @@ pub struct Export { /// The parameters, when the export is callable and introspectable. #[serde(default)] pub params: Option>, + /// A path into the test bundle: teacher code may open it, student code cannot. + #[serde(default)] + pub bundle_path: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/src/core/src/runner/harness/teacher.py b/src/core/src/runner/harness/teacher.py index e5e8d87..d75586c 100644 --- a/src/core/src/runner/harness/teacher.py +++ b/src/core/src/runner/harness/teacher.py @@ -341,6 +341,22 @@ def load_teacher_modules(): return exports, duplicates +def into_bundle(value): + """Whether `value` is an absolute path into the test bundle, which only teacher code can + open: student code gets data through `data_files`, by a path relative to it.""" + bundle = PAYLOAD.get("bundle") + if not bundle or not isinstance(value, (str, os.PathLike)): + return False + try: + path = os.fspath(value) + if not isinstance(path, str) or not os.path.isabs(path): + return False + path, bundle = os.path.realpath(path), os.path.realpath(bundle) + except (TypeError, ValueError, OSError): + return False + return path == bundle or path.startswith(bundle.rstrip(os.sep) + os.sep) + + def _same(a, b): """One object, or two equal ones: two modules that both define `TOL = 1e-6` agree.""" if a is b: @@ -469,7 +485,7 @@ def run_check(check, value, stdout, timeout, student): continue if name == "stdout": deps[name] = stdout - elif name == "source" and SOURCE is not None and name not in context and name not in held: + elif name == "source": deps[name] = SOURCE elif name in context: deps[name] = context[name] @@ -719,7 +735,11 @@ def main(): finish() if PAYLOAD["mode"] == "inspect": emit({"kind": "inspect", "duplicates": duplicates, "exports": { - name: {"callable": callable(value), "params": params_of(value) if callable(value) else None} + name: { + "callable": callable(value), + "params": params_of(value) if callable(value) else None, + "bundle_path": not callable(value) and into_bundle(value), + } for name, value in exports.items() }}) finish() diff --git a/src/core/src/runner/judge.rs b/src/core/src/runner/judge.rs index e9b24df..ac8bae8 100644 --- a/src/core/src/runner/judge.rs +++ b/src/core/src/runner/judge.rs @@ -18,6 +18,7 @@ use crate::runner::executor::{ CallObservation, CheckObservation, Exit, Outcome, Phase, ProtocolError, StopReason, Stopped, UnitObservation, UnitPlan, }; +use crate::runner::sandbox::Isolation; /// A scored call: the name its result is reported under, and what it expects. #[derive(Debug, Clone, Copy)] @@ -36,12 +37,26 @@ struct Blanket { } /// Judge one unit. `scored[i]` is `plan.steps[i]` — or, in script mode, the script run. +/// How the teacher's Python checker scripts run. +#[derive(Debug, Clone, Copy)] +struct Scripts<'a> { + python: &'a str, + isolation: Isolation, + bundle: &'a std::path::Path, +} + pub fn judge( plan: &UnitPlan, scored: &[Scored], obs: &UnitObservation, python_cmd: &str, + isolation: Isolation, ) -> Vec { + let scripts = Scripts { + python: python_cmd, + isolation, + bundle: &plan.bundle, + }; let target = |index: usize| plan.steps.get(index).map(|c| target_name(&c.target)); if let Some(blanket) = unit_failure(plan, obs) { return scored @@ -94,7 +109,7 @@ pub fn judge( record, check, plan.script.is_some(), - python_cmd, + scripts, )); } None => { @@ -589,7 +604,7 @@ fn judge_call( record: &CallObservation, check: Option<&CheckObservation>, script: bool, - python_cmd: &str, + scripts: Scripts, ) -> CaseResult { let case = s.case; let mut verdict = Verdict { @@ -671,7 +686,7 @@ fn judge_call( case.expected_stdout.clone().map(Value::String), record, check, - python_cmd, + scripts, ) { return failed; } @@ -714,15 +729,9 @@ fn judge_call( } else { (value.clone(), case.expect.clone()) }; - if let Some(failed) = check_value( - &mut verdict, - case, - &value, - expected, - record, - check, - python_cmd, - ) { + if let Some(failed) = + check_value(&mut verdict, case, &value, expected, record, check, scripts) + { return failed; } } @@ -779,7 +788,7 @@ fn check_value( expected: Option, record: &CallObservation, check: Option<&CheckObservation>, - python_cmd: &str, + scripts: Scripts, ) -> Option { let take = |verdict: &mut Verdict| { std::mem::replace( @@ -852,7 +861,17 @@ fn check_value( } Check::Rhai(expr) => Box::new(RhaiChecker::new(expr)), Check::Python(script) => { - Box::new(PythonChecker::new(script).with_python_cmd(python_cmd)) + let checker = PythonChecker::new(script).with_python_cmd(scripts.python); + Box::new(if scripts.isolation.is_sandboxed() { + let beside = std::path::Path::new(script).parent().map(Into::into); + checker.confined( + std::iter::once(scripts.bundle.to_path_buf()) + .chain(beside) + .collect(), + ) + } else { + checker + }) } Check::Function(_) => unreachable!("handled above"), }; @@ -913,6 +932,7 @@ mod tests { vars: Arc::new(BTreeMap::new()), data_files: Vec::new(), bundle: std::path::PathBuf::new(), + shared: Vec::new(), allowed_imports: Vec::new(), load_timeout: 10, setup: Vec::new(), @@ -982,7 +1002,14 @@ mod tests { } fn one(case: &TestCase, plan: &UnitPlan, obs: &UnitObservation) -> CaseResult { - judge(plan, &[Scored { name: "c", case }], obs, "python3").remove(0) + judge( + plan, + &[Scored { name: "c", case }], + obs, + "python3", + Isolation::Sandboxed, + ) + .remove(0) } fn verdict(r: &CaseResult) -> (TestStatus, Option, Option) { @@ -1227,7 +1254,7 @@ mod tests { message: "the student side ended".into(), }); obs.exit = Exit::Signal(libc::SIGXCPU); - let r = judge(&p, &scored, &obs, "python3"); + let r = judge(&p, &scored, &obs, "python3", Isolation::Sandboxed); assert_eq!(r[0].status, TestStatus::Passed); assert_eq!( verdict(&r[1]), @@ -1243,14 +1270,14 @@ mod tests { ); obs.exit = Exit::Signal(libc::SIGSEGV); - let r = judge(&p, &scored, &obs, "python3"); + let r = judge(&p, &scored, &obs, "python3", Isolation::Sandboxed); assert_eq!( verdict(&r[1]), (TestStatus::Error, Some(Fault::Student), Some(Cause::Killed)) ); obs.exit = Exit::Memory; - let r = judge(&p, &scored, &obs, "python3"); + let r = judge(&p, &scored, &obs, "python3", Isolation::Sandboxed); assert_eq!( verdict(&r[1]), (TestStatus::Error, Some(Fault::Student), Some(Cause::Memory)) @@ -1262,7 +1289,7 @@ mod tests { // A step that answered too much, or nonsense, is the student's. obs.stopped.as_mut().unwrap().reason = StopReason::Flooded; - let r = judge(&p, &scored, &obs, "python3"); + let r = judge(&p, &scored, &obs, "python3", Isolation::Sandboxed); assert_eq!( verdict(&r[1]), ( @@ -1285,7 +1312,7 @@ mod tests { message: String::new(), }); late.exit = Exit::Stopped; - let r = judge(&p, &scored, &late, "python3"); + let r = judge(&p, &scored, &late, "python3", Isolation::Sandboxed); assert_eq!( verdict(&r[1]), ( @@ -1309,7 +1336,7 @@ mod tests { let mut lost = finished(vec![record(0, returned(json!(1)))]); lost.done = false; lost.teacher_exit = Exit::Signal(libc::SIGKILL); - let r = judge(&p, &scored, &lost, "python3"); + let r = judge(&p, &scored, &lost, "python3", Isolation::Sandboxed); assert_eq!( verdict(&r[1]), ( @@ -1320,7 +1347,7 @@ mod tests { ); // ...but unsandboxed, student code could have signalled it. lost.sandboxed = false; - let r = judge(&p, &scored, &lost, "python3"); + let r = judge(&p, &scored, &lost, "python3", Isolation::Sandboxed); assert_eq!( verdict(&r[1]), ( @@ -1531,7 +1558,7 @@ mod tests { let mut obs = finished(vec![record(0, returned(json!(1)))]); obs.done = false; broke(&mut obs); - let r = judge(&p, &scored, &obs, "python3"); + let r = judge(&p, &scored, &obs, "python3", Isolation::Sandboxed); assert_eq!(r[0].status, TestStatus::Passed, "an earlier pass stands"); assert_eq!( verdict(&r[1]), @@ -1691,7 +1718,7 @@ mod tests { let mut obs = finished(vec![record(0, returned(json!(1)))]); obs.done = false; obs.protocol_error = Some(ProtocolError::Flooded("a record of 64 MiB".into())); - let r = judge(&p, &scored, &obs, "python3"); + let r = judge(&p, &scored, &obs, "python3", Isolation::Sandboxed); assert_eq!(r[0].status, TestStatus::Passed); assert_eq!( verdict(&r[1]), diff --git a/src/core/src/runner/linter.rs b/src/core/src/runner/linter.rs index d59ba5d..7d86e94 100644 --- a/src/core/src/runner/linter.rs +++ b/src/core/src/runner/linter.rs @@ -1,31 +1,119 @@ -use std::path::Path; -use std::process::Command; +//! Style scores from a teacher's lint tool. +//! +//! The tool runs on a read-only copy of the student's file, in a private directory, with a +//! cleared environment and a deadline. Sandboxed, it runs in the teacher's sandbox: it reads +//! the system's programs, its own installation, the test bundle and that copy, writes only +//! its directory, and reaches no network. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Duration; use crate::models::LintConfig; +use crate::runner::sandbox::{Confinement, Isolation}; + +/// How long a lint tool may run on one file. +const LINT_TIMEOUT: Duration = Duration::from_secs(120); -/// Run a lint command on a student file and return its style score, 0 to 100. +/// Run a lint command on a copy of `file_path` and return its style score, 0 to 100. /// -/// Errs when the tool did not run properly — it could not start, or exited with a code -/// outside `ok_exit_codes`. A tool that failed has found nothing, and reading that as a -/// clean file would hand out full marks for a broken grader. -pub fn run_lint(config: &LintConfig, file_path: &Path) -> Result { - // Split the template before filling in the path, so a path with a space in it stays one - // argument. - let file = file_path.display().to_string(); - let parts: Vec = config - .command - .split_whitespace() +/// Errs when the tool did not run properly — it could not start, ran past its deadline, or +/// exited with a code outside `ok_exit_codes`. A tool that failed has found nothing, and +/// reading that as a clean file would hand out full marks for a broken grader. +pub fn run_lint( + config: &LintConfig, + file_path: &Path, + bundle: &Path, + isolation: Isolation, +) -> Result { + // Split as a shell would, quotes included, before filling in the path: a path with a + // space in it stays one argument. + let parts = shlex::split(&config.command) + .ok_or_else(|| format!("the lint command has an unclosed quote: {}", config.command))?; + let Some((program, _)) = parts.split_first() else { + return Err("the lint command is empty".into()); + }; + let program = locate(program)?; + + let scratch = tempfile::Builder::new() + .prefix("scriptmark-lint-") + .tempdir() + .and_then(|dir| std::fs::canonicalize(dir.path()).map(|path| (dir, path))) + .map_err(|e| format!("could not stage the file to lint: {e}"))?; + let (_keep, base) = scratch; + let work = base.join("work"); + let copy = base.join("submission").join( + file_path + .file_name() + .ok_or_else(|| format!("'{}' names no file", file_path.display()))?, + ); + std::fs::create_dir_all(&work) + .and_then(|_| std::fs::create_dir_all(copy.parent().expect("a parent"))) + .and_then(|_| std::fs::copy(file_path, ©)) + .and_then(|_| { + let mut readonly = std::fs::metadata(©)?.permissions(); + readonly.set_readonly(true); + std::fs::set_permissions(©, readonly) + }) + .map_err(|e| format!("could not stage the file to lint: {e}"))?; + + let file = copy.display().to_string(); + let args: Vec = parts[1..] + .iter() .map(|part| part.replace("{file}", &file)) .collect(); - let command = parts.join(" "); - let Some((program, args)) = parts.split_first() else { - return Err("the lint command is empty".into()); + let command = std::iter::once(program.display().to_string()) + .chain(args.iter().cloned()) + .collect::>() + .join(" "); + + let mut cmd = if isolation.is_sandboxed() { + let confinement = Confinement { + read: crate::runner::sandbox::covering( + crate::runner::sandbox::system_programs() + .into_iter() + .chain(tool_reads(&program)) + .chain([bundle.to_path_buf(), base.join("submission")]) + .filter_map(|p| std::fs::canonicalize(p).ok()) + .collect(), + ), + write: vec![work.clone()], + exec: Vec::new(), + spawn: true, + no_ffi: false, + }; + crate::runner::sandbox::confined(&program, &confinement) + .map_err(|e| format!("cannot sandbox `{command}`: {e}"))? + } else { + Command::new(&program) }; + let mut path = std::env::join_paths( + crate::runner::sandbox::system_programs() + .iter() + .map(|p| p.join("bin")), + ) + .unwrap_or_default(); + if let Some(dir) = program.parent() { + path = std::env::join_paths( + std::iter::once(dir.to_path_buf()).chain(std::env::split_paths(&path)), + ) + .unwrap_or(path); + } + cmd.args(&args) + .env_clear() + .env("PATH", path) + .env("HOME", &work) + .env("TMPDIR", &work) + .current_dir(&work) + .stdin(Stdio::null()); + #[cfg(windows)] + if let Some(root) = std::env::var_os("SYSTEMROOT") { + cmd.env("SYSTEMROOT", root); + } - let out = Command::new(program) - .args(args) - .output() - .map_err(|e| format!("could not run `{program}`: {e}"))?; + let out = crate::checker::python_checker::run_with_timeout(cmd, None, LINT_TIMEOUT) + .map_err(|e| format!("could not run `{command}`: {e}"))? + .ok_or_else(|| format!("`{command}` ran longer than {}s", LINT_TIMEOUT.as_secs()))?; if !out .status .code() @@ -34,18 +122,92 @@ pub fn run_lint(config: &LintConfig, file_path: &Path) -> Result { return Err(format!( "`{command}` exited with {}: {}", out.status, - String::from_utf8_lossy(&out.stderr).trim() + out.stderr.trim() )); } - let stdout = String::from_utf8_lossy(&out.stdout); - let warnings = match serde_json::from_str::>(&stdout) { + let warnings = match serde_json::from_str::>(&out.stdout) { Ok(findings) => findings.len(), - Err(_) => stdout.lines().filter(|l| !l.trim().is_empty()).count(), + Err(_) => out.stdout.lines().filter(|l| !l.trim().is_empty()).count(), }; Ok(style_score(warnings, config.max_warnings)) } +/// The program a command names: a path as given, or the first match on the grader's PATH. +fn locate(program: &str) -> Result { + let given = Path::new(program); + if given.components().count() > 1 { + return std::path::absolute(given).map_err(|e| format!("cannot find `{program}`: {e}")); + } + std::env::var_os("PATH") + .iter() + .flat_map(std::env::split_paths) + .map(|dir| dir.join(program)) + .find(|candidate| candidate.is_file()) + .ok_or_else(|| format!("could not run `{program}`: not found on PATH")) +} + +/// What a tool reads to run: its installation — the directory above its `bin`, or its own — +/// and, for a script, its interpreter's, a Python interpreter's prefixes and packages +/// included. +fn tool_reads(program: &Path) -> Vec { + let Ok(real) = std::fs::canonicalize(program) else { + return Vec::new(); + }; + let mut reads = vec![program.to_path_buf(), installation(&real)]; + let interpreter = std::fs::read(&real).ok().and_then(|bytes| { + let line = bytes + .split(|b| *b == b'\n') + .next()? + .strip_prefix(b"#!")? + .to_vec(); + let line = String::from_utf8(line).ok()?; + let mut words = line.split_whitespace(); + let first = words.next()?; + // `#!/usr/bin/env python3` names its interpreter by the word after `env`. + match Path::new(first).file_name()?.to_str()? { + "env" => locate(words.next()?).ok(), + _ => Some(PathBuf::from(first)), + } + }); + if let Some(interpreter) = interpreter { + let is_python = interpreter + .file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.starts_with("python")); + if is_python + && let Ok(confinement) = crate::runner::python::teacher_confinement( + &interpreter.to_string_lossy(), + Vec::new(), + Vec::new(), + true, + ) { + reads.extend(confinement.read); + } + if let Ok(real) = std::fs::canonicalize(&interpreter) { + reads.push(installation(&real)); + } + reads.push(interpreter); + } + reads +} + +/// The directory a program is installed in: above its `bin` — a virtual environment or a +/// Homebrew keg — or its own. Never a prefix many programs share, such as `/`, +/// `/usr/local` or `/opt/homebrew`: those hold configuration and data in `etc` and `var`. +fn installation(real: &Path) -> PathBuf { + let dir = real.parent().unwrap_or(real); + let above = dir.parent().filter(|above| { + matches!( + dir.file_name().and_then(|n| n.to_str()), + Some("bin" | "sbin") + ) && !["etc", "var"] + .iter() + .any(|shared| above.join(shared).exists()) + }); + above.unwrap_or(dir).to_path_buf() +} + /// `max(0, 1 - warnings / max_warnings) * 100`; with `max_warnings = 0`, any warning is 0. fn style_score(warnings: usize, max_warnings: usize) -> f64 { if max_warnings == 0 { @@ -73,6 +235,27 @@ mod tests { (dir, file) } + /// Unconfined, so these run inside any sandbox the test itself is in; the sandboxed + /// runs are exercised end to end in the integration tests. + fn lint(config: &LintConfig, file: &Path) -> Result { + run_lint(config, file, file.parent().unwrap(), Isolation::Unisolated) + } + + #[test] + fn test_a_tool_is_read_from_its_own_installation_never_a_shared_prefix() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + for made in ["venv/bin", "prefix/bin", "prefix/etc"] { + std::fs::create_dir_all(root.join(made)).unwrap(); + } + assert_eq!(installation(&root.join("venv/bin/ruff")), root.join("venv")); + assert_eq!( + installation(&root.join("prefix/bin/ruff")), + root.join("prefix/bin") + ); + assert_eq!(installation(Path::new("/bin/sh")), PathBuf::from("/bin")); + } + #[test] fn test_style_score() { assert!((style_score(3, 10) - 70.0).abs() < 1e-9); @@ -85,38 +268,61 @@ mod tests { #[test] fn test_run_lint_counts_output_lines() { let (_dir, file) = student_file(); - assert_eq!(run_lint(&config("echo warning1", 10), &file), Ok(90.0)); - assert_eq!(run_lint(&config("true", 0), &file), Ok(100.0)); + assert_eq!(lint(&config("echo warning1", 10), &file), Ok(90.0)); + assert_eq!(lint(&config("true", 0), &file), Ok(100.0)); } #[test] fn test_a_path_with_a_space_stays_one_argument() { let dir = tempfile::tempdir().unwrap(); - let file = dir.path().join("Zhang San_1").join("lab.py"); + let file = dir.path().join("Zhang San_1").join("lab one.py"); std::fs::create_dir_all(file.parent().unwrap()).unwrap(); std::fs::write(&file, "x = 1\n").unwrap(); // `test -f` fails, exit 1 outside [0], unless it gets the path whole. - let lint = LintConfig { + let lint_config = LintConfig { ok_exit_codes: vec![0], ..config("test -f {file}", 10) }; - assert_eq!(run_lint(&lint, &file), Ok(100.0)); + assert_eq!(lint(&lint_config, &file), Ok(100.0)); + } + + #[test] + fn test_quoted_arguments_are_split_as_a_shell_would() { + let (_dir, file) = student_file(); + // Two arguments to `sh -c`, the script and $0: one warning line. + let quoted = config("sh -c 'echo \"one warning\"' sh", 10); + assert_eq!(lint("ed, &file), Ok(90.0)); + assert!( + lint(&config("echo 'unclosed", 10), &file) + .unwrap_err() + .contains("unclosed quote") + ); + } + + #[test] + fn test_the_tool_sees_a_read_only_copy_and_no_environment() { + let (_dir, file) = student_file(); + let probe = config( + "sh -c 'test ! -w \"$1\" && test -z \"$CANVAS_TOKEN\" && echo seen' sh {file}", + 10, + ); + assert_eq!(lint(&probe, &file), Ok(90.0)); } #[test] fn test_a_linter_reporting_findings_with_exit_1_is_scored() { let (_dir, file) = student_file(); - let sh = config("sh -c echo${IFS}w;exit${IFS}1", 10); - assert_eq!(run_lint(&sh, &file), Ok(90.0)); + let sh = config("sh -c 'echo w; exit 1'", 10); + assert_eq!(lint(&sh, &file), Ok(90.0)); } #[test] fn test_a_linter_that_fails_is_an_error_not_100() { let (_dir, file) = student_file(); - let crashed = run_lint(&config("sh -c exit${IFS}2", 10), &file); + let crashed = lint(&config("sh -c 'exit 2'", 10), &file); assert!(crashed.unwrap_err().contains("exited with")); - let missing = run_lint(&config("/nonexistent/ruff {file}", 10), &file); + let missing = lint(&config("/nonexistent/ruff {file}", 10), &file); assert!(missing.unwrap_err().contains("could not run")); - assert!(run_lint(&config(" ", 10), &file).is_err()); + assert!(lint(&config(" ", 10), &file).is_err()); } } diff --git a/src/core/src/runner/oracle.rs b/src/core/src/runner/oracle.rs index befaf3e..2ba2ca5 100644 --- a/src/core/src/runner/oracle.rs +++ b/src/core/src/runner/oracle.rs @@ -37,6 +37,12 @@ pub async fn resolve_oracle( .map(|rel| (spec.dir.join(rel), rel.clone())) .collect(), bundle: spec.dir.clone(), + shared: spec + .meta + .shared + .iter() + .map(|rel| (spec.dir.join(rel), rel.clone())) + .collect(), allowed_imports: spec.meta.allowed_imports.clone(), load_timeout: timeout_secs, setup: Vec::new(), diff --git a/src/core/src/runner/orchestrator.rs b/src/core/src/runner/orchestrator.rs index 49f976f..d626dff 100644 --- a/src/core/src/runner/orchestrator.rs +++ b/src/core/src/runner/orchestrator.rs @@ -37,6 +37,13 @@ impl Default for RunOptions { } } +/// How teacher scripts and tools run outside a unit: checker scripts and lint. +#[derive(Debug, Clone)] +struct Tools { + python: Arc, + isolation: crate::runner::sandbox::Isolation, +} + /// Run every prepared bundle against every student. /// /// Takes `&[StudentSubmission]` rather than the whole `AssignmentInput` so that the roster, @@ -63,7 +70,10 @@ pub async fn run_all( }) .clamp(1, Semaphore::MAX_PERMITS); let semaphore = Arc::new(Semaphore::new(concurrency)); - let python: Arc = options.python.as_str().into(); + let tools = Tools { + python: options.python.as_str().into(), + isolation: options.isolation, + }; let matching = Arc::new(options.matching.clone()); let mut handles = Vec::new(); @@ -81,11 +91,11 @@ pub async fn run_all( .map(|bundle| crate::matching::item_match(&matching, student, &bundle.spec)) .collect(); let matching = matching.clone(); - let (bundles, executor, semaphore, python) = ( + let (bundles, executor, semaphore, tools) = ( bundles.clone(), executor.clone(), semaphore.clone(), - python.clone(), + tools.clone(), ); let handle = tokio::spawn(async move { @@ -99,7 +109,7 @@ pub async fn run_all( bundles, executor, semaphore, - python, + tools, matching, ) .await; @@ -139,7 +149,7 @@ async fn run_student( bundles: Arc<[Bundle]>, executor: Arc, semaphore: Arc, - python: Arc, + tools: Tools, matching: Arc, ) { let mut slots: Vec>>> = @@ -195,11 +205,11 @@ async fn run_student( let functions = matching.functions(&report.student_id, &bundle.spec.meta.name); for u in 0..bundle.units.len() { let functions = functions.clone(); - let (bundles, executor, semaphore, python, path) = ( + let (bundles, executor, semaphore, tools, path) = ( bundles.clone(), executor.clone(), semaphore.clone(), - python.clone(), + tools.clone(), path.clone(), ); let handle = tasks.spawn(async move { @@ -227,7 +237,13 @@ async fn run_student( } } ( - judge(&plan, &unit.scored(), &observation, &python), + judge( + &plan, + &unit.scored(), + &observation, + &tools.python, + tools.isolation, + ), functions, ) }) @@ -289,7 +305,7 @@ async fn run_student( } } } - report.lint = lint(&bundles, &graded_files, &semaphore).await; + report.lint = lint(&bundles, &graded_files, &semaphore, tools.isolation).await; report.test_results = bundles .iter() .zip(slots) @@ -303,30 +319,66 @@ async fn run_student( report.backend_name = Some(executor.language().to_string()); } -/// Lint the file of the first spec that asks for it — that item's file, not whichever the -/// student happened to hand in first. +/// Lint every file a spec asks to — each item's own file, with its own `[lint]` — and score +/// the student by their mean: an item whose file is missing counts 0. A file two items +/// share under the same command is linted once. async fn lint( bundles: &[Bundle], graded_files: &[Option], semaphore: &Semaphore, + isolation: crate::runner::sandbox::Isolation, ) -> Option { - let (b, config) = bundles - .iter() - .enumerate() - .find_map(|(b, bundle)| bundle.spec.lint.clone().map(|c| (b, c)))?; - let Some(file) = graded_files[b].clone() else { + let mut asked: Vec<( + (Option, String), + crate::models::LintConfig, + std::path::PathBuf, + )> = Vec::new(); + for (bundle, file) in bundles.iter().zip(graded_files) { + if let Some(config) = &bundle.spec.lint { + let key = (file.clone(), config.command.clone()); + if !asked.iter().any(|(k, ..)| *k == key) { + asked.push((key, config.clone(), bundle.spec.dir.clone())); + } + } + } + if asked.is_empty() { + return None; + } + if asked.iter().all(|((file, _), ..)| file.is_none()) { return Some(LintOutcome::NoFile); - }; - let _permit = semaphore.acquire().await.expect("semaphore closed"); - let outcome = tokio::task::spawn_blocking(move || { - crate::runner::linter::run_lint(&config, std::path::Path::new(&file)) - }) - .await - .map_err(|e| format!("linting failed: {e}")) - .and_then(|scored| scored); - Some(match outcome { - Ok(score) => LintOutcome::Scored { score }, - Err(message) => LintOutcome::Failed { message }, + } + let mut scores = Vec::new(); + let mut failures = Vec::new(); + for ((file, _), config, bundle) in asked { + let Some(file) = file else { + scores.push(0.0); + continue; + }; + let _permit = semaphore.acquire().await.expect("semaphore closed"); + let outcome = tokio::task::spawn_blocking(move || { + crate::runner::linter::run_lint( + &config, + std::path::Path::new(&file), + &bundle, + isolation, + ) + }) + .await + .map_err(|e| format!("linting failed: {e}")) + .and_then(|scored| scored); + match outcome { + Ok(score) => scores.push(score), + Err(message) => failures.push(message), + } + } + Some(if failures.is_empty() { + LintOutcome::Scored { + score: scores.iter().sum::() / scores.len() as f64, + } + } else { + LintOutcome::Failed { + message: failures.join("; "), + } }) } diff --git a/src/core/src/runner/prepare.rs b/src/core/src/runner/prepare.rs index 416364c..ac39de4 100644 --- a/src/core/src/runner/prepare.rs +++ b/src/core/src/runner/prepare.rs @@ -26,7 +26,7 @@ use crate::runner::frozen::{Generation, replay_entry}; use crate::runner::generation::{DrawSeed, Generated, generate, seed_for}; use crate::runner::judge::Scored; use crate::runner::oracle::{resolve_oracle, validate_references}; -use crate::spec_loader::{MAX_TIMEOUT_SECS, RESERVED, refs, validate, validate_expanded}; +use crate::spec_loader::{MAX_TIMEOUT_SECS, refs, reserved, validate, validate_expanded}; /// A prepared test bundle: one spec, ready to run against any student. #[derive(Debug, Serialize)] @@ -340,6 +340,7 @@ async fn prepare_one( } problems.extend(check_names(&spec, &teacher)); + problems.extend(check_bundle_paths(&spec, &teacher)); problems.extend(dry_run_rhai(&spec)); if !problems.is_empty() { return Err(fail(problems)); @@ -367,10 +368,12 @@ fn check_names(spec: &TestSpec, teacher: &TeacherRuntime) -> Vec { ) }) .collect(); - if teacher.exports.contains_key(RESERVED) { - problems.push(format!( - "a teacher module exports '{RESERVED}', a name checkers reserve for the call's output" - )); + for name in teacher.exports.keys() { + if let Some(meaning) = reserved(name) { + problems.push(format!( + "a teacher module exports '{name}', a name checkers reserve for {meaning}" + )); + } } let exports: BTreeSet<&str> = teacher.exports.keys().map(String::as_str).collect(); let mut top: BTreeSet = BTreeSet::new(); @@ -422,6 +425,93 @@ fn check_names(spec: &TestSpec, teacher: &TeacherRuntime) -> Vec { problems } +/// Student code runs in a sandbox that cannot read the test bundle: a path into it, handed to +/// student code, names a file the student can never open. Such a file belongs in +/// `data_files`, which stages it beside the student's file, under a relative path. +fn check_bundle_paths(spec: &TestSpec, teacher: &TeacherRuntime) -> Vec { + let Ok(bundle) = std::fs::canonicalize(&spec.dir) else { + return Vec::new(); + }; + let fix = "list it in [meta] data_files and pass its path relative to the student's directory"; + let mut problems = Vec::new(); + let mut literals = |at: &str, value: &Value| { + for path in bundle_paths(value, &bundle) { + problems.push(format!( + "{at}: '{path}' is a path into the test bundle, which student code cannot read: {fix}" + )); + } + }; + for (name, value) in &spec.vars { + literals(&format!("[vars] '{name}'"), value); + } + let student_setup = |steps: &[SetupStep]| -> Vec<(String, Vec)> { + steps + .iter() + .filter(|s| s.teacher.is_none()) + .map(|s| (format!("setup '{}'", s.id), s.args.clone())) + .collect() + }; + let mut calls: Vec<(String, Vec)> = student_setup(&spec.setup); + calls.extend( + spec.cases + .iter() + .map(|c| (format!("case '{}'", c.name), c.args.clone())), + ); + for scenario in &spec.scenarios { + calls.extend(student_setup(&scenario.setup)); + calls.extend(scenario.steps.iter().map(|s| { + ( + format!("scenario '{}' step '{}'", scenario.name, s.name), + s.args.clone(), + ) + })); + } + for (at, args) in &calls { + for arg in args { + literals(at, arg); + } + } + for (at, args) in calls { + for name in refs(&args) { + if teacher.exports.get(&name).is_some_and(|e| e.bundle_path) { + problems.push(format!( + "{at}: '${name}' is a path into the test bundle, which student code cannot read: {fix}" + )); + } + } + } + problems +} + +/// `path` with its links resolved as far as it exists: a file the student would write +/// still resolves through its directory. +fn resolved(path: &std::path::Path) -> PathBuf { + if let Ok(real) = std::fs::canonicalize(path) { + return real; + } + match (path.parent(), path.file_name()) { + (Some(parent), Some(name)) => resolved(parent).join(name), + _ => path.to_path_buf(), + } +} + +/// Every string in `value` that is an absolute path into `bundle`. +fn bundle_paths(value: &Value, bundle: &std::path::Path) -> Vec { + match value { + Value::String(text) => { + let path = std::path::Path::new(text); + if path.is_absolute() && resolved(path).starts_with(bundle) { + vec![text.clone()] + } else { + Vec::new() + } + } + Value::Array(items) => items.iter().flat_map(|v| bundle_paths(v, bundle)).collect(), + Value::Object(map) => map.values().flat_map(|v| bundle_paths(v, bundle)).collect(), + _ => Vec::new(), + } +} + fn unknown_refs(args: &[Value], scope: &BTreeSet, at: &str) -> Vec { refs(args) .into_iter() @@ -471,7 +561,7 @@ fn check_call( for param in params.iter().skip(2) { // *args, **kwargs and defaulted parameters are filled only when named. let filled = param.is_variadic() - || param.default || param.name == RESERVED + || param.default || reserved(¶m.name).is_some() || scope.contains(¶m.name); if !filled { problems.push(format!( @@ -556,6 +646,12 @@ fn plan_units(spec: &TestSpec, timeout_secs: u64) -> Vec { .map(|rel| (spec.dir.join(rel), rel.clone())) .collect(), bundle: spec.dir.clone(), + shared: spec + .meta + .shared + .iter() + .map(|rel| (spec.dir.join(rel), rel.clone())) + .collect(), allowed_imports: spec.meta.allowed_imports.clone(), load_timeout: timeout_secs, setup: Vec::new(), diff --git a/src/core/src/runner/python.rs b/src/core/src/runner/python.rs index 296e1fd..a93de29 100644 --- a/src/core/src/runner/python.rs +++ b/src/core/src/runner/python.rs @@ -6,7 +6,7 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, LazyLock, Mutex, OnceLock}; +use std::sync::{Arc, LazyLock, Mutex}; use std::time::Duration; use serde::Serialize; @@ -45,8 +45,6 @@ pub struct PythonExecutor { /// The most memory the student side may hold before it is ended, in MiB. memory_mb: u64, isolation: Isolation, - /// What the interpreter needs to run, found once, the first time a unit is confined. - interpreter: OnceLock>, } impl PythonExecutor { @@ -59,7 +57,6 @@ impl PythonExecutor { python_cmd: resolve_python_path(&python_cmd.into()), memory_mb: crate::runner::sandbox::SandboxConfig::default().mem_mb, isolation: Isolation::default(), - interpreter: OnceLock::new(), } } @@ -83,13 +80,45 @@ impl PythonExecutor { pub fn python_cmd(&self) -> &str { &self.python_cmd } +} - fn interpreter(&self) -> Result<&Interpreter, String> { - self.interpreter - .get_or_init(|| Interpreter::probe(&self.python_cmd)) - .as_ref() - .map_err(Clone::clone) - } +/// What each interpreter needs to run, found once per interpreter, the first time +/// anything it runs is confined. +static INTERPRETERS: LazyLock< + Mutex>>, +> = LazyLock::new(Default::default); + +fn interpreter(python_cmd: &str) -> Result { + INTERPRETERS + .lock() + .expect("interpreter cache poisoned") + .entry(python_cmd.to_string()) + .or_insert_with(|| Interpreter::probe(python_cmd)) + .clone() +} + +/// A confinement for teacher code `python_cmd` runs outside a unit — a Python checker +/// script, or a lint tool written in Python: the interpreter, `read`, and `write`. +pub fn teacher_confinement( + python_cmd: &str, + read: Vec, + write: Vec, + spawn: bool, +) -> Result { + let interpreter = interpreter(python_cmd)?; + Ok(Confinement { + read: covering( + interpreter + .read + .into_iter() + .chain(canonical(read)) + .collect(), + ), + write, + exec: interpreter.exec, + spawn, + no_ffi: !spawn, + }) } /// What one Python interpreter reads and runs: its prefixes, its standard library and @@ -100,9 +129,12 @@ struct Interpreter { exec: Vec, } -/// Asks the interpreter where it lives, unconfined and once. +/// Asks the interpreter where it lives, unconfined and once: its prefixes, and where its +/// standard library and installed packages are — not every `sys.path` entry, which a +/// package's `.pth` file can point anywhere, a home directory included. const PROBE: &str = "import json, os, sys, sysconfig -paths = [sys.prefix, sys.base_prefix, sys.exec_prefix, sys.base_exec_prefix, *sysconfig.get_paths().values(), *sys.path] +places = sysconfig.get_paths() +paths = [sys.prefix, sys.base_prefix, sys.exec_prefix, sys.base_exec_prefix, *(places[k] for k in ('stdlib', 'platstdlib', 'purelib', 'platlib'))] print(json.dumps({ 'executable': os.path.realpath(sys.executable), 'base': os.path.realpath(sys.base_prefix), @@ -321,6 +353,7 @@ fn side(root: &Path, name: &str) -> std::io::Result { fn stage( subject: Option<&Path>, data_files: &[(PathBuf, String)], + shared: &[(PathBuf, String)], payload: impl FnOnce(&Staged) -> serde_json::Value, ) -> std::io::Result { let root = tempfile::Builder::new() @@ -337,6 +370,10 @@ fn stage( copy_into(src, &student.work.join(rel))?; copy_into(src, &teacher.work.join(rel))?; } + // The student side's own copy of each shared module; the teacher side's is the bundle's. + for (src, rel) in shared { + copy_into(src, &student.work.join(rel))?; + } let (copied, source) = match subject { Some(file) => { let name = file @@ -399,8 +436,10 @@ impl PythonExecutor { confinement: Option<&Confinement>, ) -> Result { let mut cmd = match confinement { - Some(c) => crate::runner::sandbox::confined(Path::new(&self.python_cmd), c) - .map_err(|e| format!("cannot sandbox this unit: {e}"))?, + Some(c) => Command::from( + crate::runner::sandbox::confined(Path::new(&self.python_cmd), c) + .map_err(|e| format!("cannot sandbox this unit: {e}"))?, + ), None => Command::new(&self.python_cmd), }; cmd.env_clear() @@ -441,7 +480,7 @@ impl PythonExecutor { if !self.isolation.is_sandboxed() { return Ok(None); } - let interpreter = self.interpreter()?; + let interpreter = interpreter(&self.python_cmd)?; let confine = |side: &Side, extra: Vec| Confinement { read: covering(interpreter.read.iter().cloned().chain(extra).collect()), write: vec![side.work.clone(), side.home.clone(), side.tmp.clone()], @@ -770,13 +809,18 @@ fn resident_bytes(_pid: u32) -> Option { /// The limits for a unit: its CPU limit sits above its deadline, a backstop for one busy /// core rather than a second, shorter timeout. Confined, it cannot start processes, so the -/// per-user process limit — which counts threads on Linux — is left alone. +/// per-user process limit — which counts threads on Linux — only stops a thread bomb, and +/// leaves a busy machine's own threads room. #[cfg(unix)] fn sandbox_for(deadline_secs: u64, confined: bool) -> crate::runner::sandbox::SandboxConfig { let defaults = crate::runner::sandbox::SandboxConfig::default(); crate::runner::sandbox::SandboxConfig { cpu_secs: deadline_secs.saturating_add(1), - nproc: if confined { None } else { defaults.nproc }, + nproc: if confined { + Some(crate::runner::sandbox::CONFINED_NPROC) + } else { + defaults.nproc + }, ..defaults } } @@ -951,12 +995,24 @@ impl Executor for PythonExecutor { .map(|rel| (spec.dir.join(rel), rel.clone())) .collect::>(); let imports = spec.meta.imports.clone(); + let shared: Vec<(PathBuf, String)> = spec + .meta + .shared + .iter() + .map(|rel| (spec.dir.join(rel), rel.clone())) + .collect(); + let teacher_shared: Vec = shared.iter().map(|(src, _)| src.clone()).collect(); + let bundle = spec.dir.clone(); let sandboxed = self.isolation.is_sandboxed(); let staged = tokio::task::spawn_blocking(move || { - stage(None, &data, |staged| { + stage(None, &data, &shared, |staged| { extend( base_payload("inspect", staged, sandboxed), - serde_json::json!({ "imports": imports }), + serde_json::json!({ + "imports": imports, + "shared": teacher_shared, + "bundle": bundle, + }), ) }) }) @@ -1001,6 +1057,7 @@ impl Executor for PythonExecutor { async fn run(&self, plan: &UnitPlan) -> UnitObservation { let data = plan.data_files.clone(); + let shared = plan.shared.clone(); let file = plan.file.clone(); let payload = { let plan = plan.clone(); @@ -1025,6 +1082,8 @@ impl Executor for PythonExecutor { "files": s.files, })), "imports": plan.imports, + "shared": plan.shared.iter().map(|(src, _)| src).collect::>(), + "student_shared": plan.shared.iter().map(|(_, rel)| rel).collect::>(), "vars": &*plan.vars, "allowed_imports": plan.allowed_imports, "load_timeout": plan.load_timeout, @@ -1035,19 +1094,22 @@ impl Executor for PythonExecutor { ) } }; - let staged = match tokio::task::spawn_blocking(move || stage(Some(&file), &data, payload)) - .await - { - Ok(Ok(staged)) => staged, - Ok(Err(e)) => { - return UnitObservation::not_started(Exit::Spawn(format!( - "could not stage the unit: {e}" - ))); - } - Err(e) => { - return UnitObservation::not_started(Exit::Spawn(format!("staging panicked: {e}"))); - } - }; + let staged = + match tokio::task::spawn_blocking(move || stage(Some(&file), &data, &shared, payload)) + .await + { + Ok(Ok(staged)) => staged, + Ok(Err(e)) => { + return UnitObservation::not_started(Exit::Spawn(format!( + "could not stage the unit: {e}" + ))); + } + Err(e) => { + return UnitObservation::not_started(Exit::Spawn(format!( + "staging panicked: {e}" + ))); + } + }; debug_assert!(staged.subject.as_ref().is_some_and(|s| s.is_file())); // A reference implementation is teacher code: it may read the bundle, as teacher @@ -1114,8 +1176,11 @@ mod tests { assert!(sandbox_for(deadline, true).cpu_secs > deadline); } assert_eq!(sandbox_for(u64::MAX, true).cpu_secs, u64::MAX, "saturates"); - assert_eq!(sandbox_for(3, true).nproc, None); - assert!(sandbox_for(3, false).nproc.is_some()); + assert_eq!( + sandbox_for(3, true).nproc, + Some(crate::runner::sandbox::CONFINED_NPROC) + ); + assert_eq!(sandbox_for(3, false).nproc, Some(64)); } /// Each record is bounded by the largest a call can make, and the stream by the number @@ -1141,6 +1206,7 @@ mod tests { vars: Default::default(), data_files: Vec::new(), bundle: PathBuf::new(), + shared: Vec::new(), allowed_imports: Vec::new(), load_timeout: 1, setup: Vec::new(), diff --git a/src/core/src/runner/sandbox.rs b/src/core/src/runner/sandbox.rs index 07d213f..0e38f59 100644 --- a/src/core/src/runner/sandbox.rs +++ b/src/core/src/runner/sandbox.rs @@ -10,7 +10,6 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; -use tokio::process::Command; /// Resource limits for student code execution. /// @@ -28,12 +27,17 @@ pub struct SandboxConfig { /// Max open file descriptors (RLIMIT_NOFILE). Default: 64. pub nofile: u64, /// Max processes for this UID (RLIMIT_NPROC). Default: 64. - /// It counts every process of the user — and on Linux every thread — so on a busy - /// machine it refuses even a thread. A sandboxed unit cannot start processes at all, - /// and leaves it unset (`None`). + /// It counts every process of the user — and on Linux every thread — so a low cap + /// refuses even a thread on a busy machine. A sandboxed unit cannot start processes at + /// all, so its cap only stops a runaway of threads, and sits far higher + /// (`CONFINED_NPROC`); the harness itself starts none. pub nproc: Option, } +/// The per-user process cap of a confined unit: room for a busy desktop's own threads, and +/// a stop to a student's thread bomb. +pub const CONFINED_NPROC: u64 = 4096; + impl Default for SandboxConfig { fn default() -> Self { Self { @@ -52,7 +56,7 @@ impl Default for SandboxConfig { /// in the forked child process before exec, so it only affects the /// student Python process. #[cfg(unix)] -pub fn apply_sandbox(cmd: &mut Command, config: &SandboxConfig) { +pub fn apply_sandbox(cmd: &mut tokio::process::Command, config: &SandboxConfig) { let cpu = config.cpu_secs; #[cfg(not(target_os = "macos"))] let mem = config.mem_mb * 1024 * 1024; @@ -151,10 +155,35 @@ pub fn available() -> Result<(), String> { } /// A command that runs `program` held to `confinement`. Its arguments are the caller's to add. -pub fn confined(program: &Path, confinement: &Confinement) -> Result { +pub fn confined( + program: &Path, + confinement: &Confinement, +) -> Result { platform::command(program, confinement) } +/// Where the operating system keeps the programs a teacher's tool may run, and what they +/// read: never a home directory. +pub fn system_programs() -> Vec { + #[cfg(target_os = "macos")] + let places = [ + "/usr", + "/bin", + "/sbin", + "/System", + "/Library/Frameworks", + "/private/etc", + "/private/var/db/timezone", + ] + .as_slice(); + #[cfg(not(target_os = "macos"))] + let places = ["/usr", "/bin", "/sbin", "/lib", "/lib64", "/etc"].as_slice(); + places + .iter() + .filter_map(|p| std::fs::canonicalize(p).ok()) + .collect() +} + /// The program the kernel loads to start `executable`: its ELF interpreter, which a /// confined process must be allowed to run too. #[cfg(not(target_os = "macos"))] @@ -222,8 +251,8 @@ mod platform { )) } - pub fn command(program: &Path, c: &Confinement) -> Result { - let mut cmd = Command::new(SANDBOX_EXEC); + pub fn command(program: &Path, c: &Confinement) -> Result { + let mut cmd = std::process::Command::new(SANDBOX_EXEC); cmd.arg("-p").arg(profile(c)?).arg(program); Ok(cmd) } @@ -288,6 +317,15 @@ mod platform { r#"(deny file-read* (regex #"/_ctypes[^/]*\.so$") (regex #"/_cffi_backend[^/]*\.so$"))"# .into(), ); + // Nor by another name: no hard link to the library, and no native code loaded + // from anywhere it can write. + rules.push("(deny file-link)".into()); + if !c.write.is_empty() { + rules.push(format!( + "(deny file-map-executable {})", + subpaths(&c.write)? + )); + } } Ok(rules.join("\n")) } @@ -344,12 +382,13 @@ mod platform { abi().map(|_| ()) } - pub fn command(program: &Path, c: &Confinement) -> Result { + pub fn command(program: &Path, c: &Confinement) -> Result { + use std::os::unix::process::CommandExt; let abi = abi()?; let ruleset = landlock_ruleset(abi, c).map_err(|e| format!("Landlock: {e}"))?; // Signals and abstract sockets are Landlock's from ABI 6; before it, seccomp's. let filters = filters(c, abi < ABI::V6)?; - let mut cmd = Command::new(program); + let mut cmd = std::process::Command::new(program); let mut ruleset = Some(ruleset); // SAFETY: between fork and exec this only makes the syscalls that apply a prepared // ruleset and prepared filters; nothing is allocated unless one fails. @@ -553,10 +592,30 @@ mod platform { // `clone3` takes its flags by pointer, where a filter cannot look: refuse it as // unknown, and the C library falls back to `clone`. let clone3 = BTreeMap::from([(libc::SYS_clone3, Vec::new())]); - Ok(vec![ - build(denied, libc::EPERM)?, - build(clone3, libc::ENOSYS)?, - ]) + let mut filters = vec![build(denied, libc::EPERM)?, build(clone3, libc::ENOSYS)?]; + #[cfg(target_arch = "x86_64")] + filters.push(no_x32()); + Ok(filters) + } + + /// The x32 calling convention numbers every syscall again, past `0x40000000`: a filter + /// keyed on the x86-64 numbers would let them through, so they are refused outright. + #[cfg(target_arch = "x86_64")] + fn no_x32() -> BpfProgram { + use seccompiler::sock_filter; + const X32_SYSCALL_BIT: u32 = 0x4000_0000; + const SECCOMP_RET_ALLOW: u32 = 0x7fff_0000; + const SECCOMP_RET_ERRNO: u32 = 0x0005_0000; + const LOAD_NR: u16 = 0x20; // BPF_LD | BPF_W | BPF_ABS, at seccomp_data.nr + const JUMP_GE: u16 = 0x35; // BPF_JMP | BPF_JGE | BPF_K + const RETURN: u16 = 0x06; // BPF_RET | BPF_K + let op = |code, jt, jf, k| sock_filter { code, jt, jf, k }; + vec![ + op(LOAD_NR, 0, 0, 0), + op(JUMP_GE, 0, 1, X32_SYSCALL_BIT), + op(RETURN, 0, 0, SECCOMP_RET_ERRNO | libc::EPERM as u32), + op(RETURN, 0, 0, SECCOMP_RET_ALLOW), + ] } } @@ -568,7 +627,7 @@ mod platform { Err("ScriptMark cannot sandbox student code on this platform yet".into()) } - pub fn command(_program: &Path, _c: &Confinement) -> Result { + pub fn command(_program: &Path, _c: &Confinement) -> Result { Err(available().unwrap_err()) } } diff --git a/src/core/src/spec_loader.rs b/src/core/src/spec_loader.rs index d7d3609..1e61f40 100644 --- a/src/core/src/spec_loader.rs +++ b/src/core/src/spec_loader.rs @@ -115,8 +115,19 @@ fn resolve_paths(spec: &mut TestSpec) { } } -/// The name a function checker's `stdout` parameter always receives: the call's output. -pub const RESERVED: &str = "stdout"; +/// The names a function checker's extra parameters always receive, and what they hold. +pub const RESERVED: [(&str, &str); 2] = [ + ("stdout", "the call's output"), + ("source", "the student's source text"), +]; + +/// What `name` always means to a function checker, if it is reserved. +pub fn reserved(name: &str) -> Option<&'static str> { + RESERVED + .iter() + .find(|(reserved, _)| *reserved == name) + .map(|(_, meaning)| *meaning) +} /// The longest timeout a spec or the command line may ask for. pub const MAX_TIMEOUT_SECS: u64 = 86_400; @@ -217,14 +228,15 @@ impl<'a> Validator<'a> { self.problem("[meta]", format!("data file '{data}' does not exist")); } } - if self.spec.vars.contains_key(RESERVED) { - self.problem( - "[vars]", - format!( - "'{RESERVED}' is reserved: function checkers receive the call's output under it" - ), - ); + for name in self.spec.vars.keys() { + if let Some(meaning) = reserved(name) { + self.problem( + "[vars]", + format!("'{name}' is reserved: function checkers receive {meaning} under it"), + ); + } } + self.shared(); for (name, value) in &self.spec.vars { if contains_null(value) { self.problem( @@ -235,6 +247,70 @@ impl<'a> Validator<'a> { } } + /// Shared modules: Python files in the bundle, each importable under a name of its own + /// on both sides, and neither data nor a teacher module besides. + fn shared(&mut self) { + let meta = &self.spec.meta; + let student = Path::new(&meta.file) + .file_name() + .map(|n| n.to_string_lossy().to_lowercase()); + let mut names = BTreeSet::new(); + for shared in &meta.shared { + let at = format!("shared module '{shared}'"); + if !is_contained(shared) { + self.problem( + "[meta]", + format!("{at} must be a relative path without '..'"), + ); + continue; + } + let path = Path::new(shared); + let module = path.file_stem().map(|s| s.to_string_lossy().into_owned()); + if path.extension().is_none_or(|e| e != "py") + || !module.as_deref().is_some_and(is_identifier) + { + self.problem( + "[meta]", + format!("{at} must be a .py file whose name can be imported"), + ); + } else if !self.spec.dir.join(path).is_file() { + self.problem("[meta]", format!("{at} does not exist")); + } + let name = path.file_name().map(|n| n.to_string_lossy().to_lowercase()); + if name.is_some() && name == student { + self.problem( + "[meta]", + format!("{at} has the student file's name: it would replace the submission"), + ); + } + if let Some(module) = module + && !names.insert(module.clone()) + { + self.problem( + "[meta]", + format!("two shared modules are both imported as '{module}'"), + ); + } + let resolved = self.spec.dir.join(path); + if meta.imports.iter().any(|i| Path::new(i) == resolved) { + self.problem( + "[meta]", + format!("{at} is also a teacher module: list it in shared only"), + ); + } + if meta + .data_files + .iter() + .any(|d| path.starts_with(d) || Path::new(d).starts_with(path)) + { + self.problem( + "[meta]", + format!("{at} is also a data file: list it in shared only"), + ); + } + } + } + fn body(&mut self) { let spec = self.spec; if spec.cases.is_empty() && spec.scenarios.is_empty() { @@ -311,11 +387,8 @@ impl<'a> Validator<'a> { if bound.all.contains(id) || self.spec.vars.contains_key(id) { self.problem(&step_at, format!("id '{id}' is already bound")); } - if id == RESERVED { - self.problem( - &step_at, - format!("id '{RESERVED}' is reserved for the call's output"), - ); + if let Some(meaning) = reserved(id) { + self.problem(&step_at, format!("id '{id}' is reserved for {meaning}")); } if let Some(target) = step.target(self.spec.meta.function.as_deref()) { bound.bind(id, &target); @@ -332,11 +405,8 @@ impl<'a> Validator<'a> { if bound.all.contains(&step.id) || self.spec.vars.contains_key(&step.id) { self.problem(at, format!("id '{}' is already bound", step.id)); } - if step.id == RESERVED { - self.problem( - at, - format!("id '{RESERVED}' is reserved for the call's output"), - ); + if let Some(meaning) = reserved(&step.id) { + self.problem(at, format!("id '{}' is reserved for {meaning}", step.id)); } if step.args.iter().any(contains_null) { self.problem( @@ -1812,6 +1882,83 @@ expect = 150 ); } + #[test] + fn test_shared_modules_are_importable_python_files_of_their_own() { + let dir = tempfile::tempdir().unwrap(); + for file in [ + "common.py", + "other/common.py", + "data/extra.py", + "lab.py", + "notes.txt", + "helpers/t.py", + "my-module.py", + ] { + let path = dir.path().join(file); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, "X = 1\n").unwrap(); + } + let problems = |shared: &str, extra: &str| -> Vec { + let toml = format!( + "[meta]\nname = \"t\"\nfile = \"lab.py\"\nfunction = \"f\"\nlanguage = \"python\"\nshared = [{shared}]\n{extra}\n[[cases]]\nname = \"x\"\nexpect = 1\n" + ); + match load_spec_str(&toml, dir.path()) { + Ok(_) => Vec::new(), + Err(SpecError::Invalid { problems, .. }) => problems, + Err(other) => vec![other.to_string()], + } + }; + assert!(problems("\"common.py\"", "").is_empty()); + for (shared, extra, needle) in [ + ("\"../x.py\"", "", "must be a relative path"), + ( + "\"notes.txt\"", + "", + "must be a .py file whose name can be imported", + ), + ( + "\"my-module.py\"", + "", + "must be a .py file whose name can be imported", + ), + ("\"missing.py\"", "", "does not exist"), + ("\"lab.py\"", "", "the student file's name"), + ( + "\"common.py\", \"other/common.py\"", + "", + "both imported as 'common'", + ), + ( + "\"helpers/t.py\"", + "imports = [\"helpers/t.py\"]", + "also a teacher module", + ), + ( + "\"data/extra.py\"", + "data_files = [\"data\"]", + "also a data file", + ), + ] { + let found = problems(shared, extra); + assert!( + found.iter().any(|p| p.contains(needle)), + "{shared}: expected {needle:?}, got {found:?}" + ); + } + } + + #[test] + fn test_source_is_reserved_like_stdout() { + refused( + "[vars]\nsource = 1\n[[cases]]\nname = \"x\"\nexpect = 1\n", + "'source' is reserved: function checkers receive the student's source text", + ); + refused( + "[[setup]]\nid = \"source\"\nfunction = \"a\"\n[[cases]]\nname = \"x\"\nexpect = 1\n", + "id 'source' is reserved", + ); + } + #[test] fn test_a_script_case_refuses_a_function_checker() { refused( diff --git a/src/core/tests/harness.rs b/src/core/tests/harness.rs index 1fb6b69..e4d5128 100644 --- a/src/core/tests/harness.rs +++ b/src/core/tests/harness.rs @@ -53,6 +53,7 @@ fn unit(file: &Path) -> UnitPlan { vars: Arc::new(BTreeMap::new()), data_files: Vec::new(), bundle: file.parent().unwrap().to_path_buf(), + shared: Vec::new(), allowed_imports: Vec::new(), load_timeout: 5, setup: Vec::new(), @@ -1135,3 +1136,147 @@ async fn test_teacher_values_cross_by_value_and_student_values_are_read_when_che assert_eq!(obs.checks[&0], pass("the teacher still holds ['teacher']")); assert_eq!(obs.checks[&2], pass("balance 30")); } + +const COMMON: &str = "class ListNode:\n def __init__(self, value, next=None):\n self.value = value\n self.next = next\n\n def length(self):\n n, node, seen = 0, self, set()\n while node is not None and id(node) not in seen:\n seen.add(id(node))\n n += 1\n node = node.next\n return n\n"; + +/// A shared class crosses by value both ways, rebuilt from the receiving side's own copy: +/// the teacher's checker gets the teacher's class, cycles and all, and what a student does +/// to their copy stays on their side. +#[tokio::test] +async fn test_shared_classes_cross_by_value_into_each_sides_own_copy() { + let dir = tempfile::tempdir().unwrap(); + let common = write(dir.path(), "common.py", COMMON); + let teacher = write( + dir.path(), + "teacher.py", + "from common import ListNode\n\ndef is_ring(result, expected):\n assert type(result) is ListNode, type(result)\n assert result.next.next is result, 'the cycle survives'\n return result.length() == 2\n\ndef make():\n return ListNode(1, ListNode(2))\n\ndef base(result, expected):\n return type(result) is ListNode and result.length() == 1, f'{type(result)} {result.length()}'\n", + ); + let student = write( + dir.path(), + "lab.py", + "from common import ListNode\n\nclass Mine(ListNode):\n def length(self):\n return -1\n\ndef ring():\n a, b = ListNode(1), ListNode(2)\n a.next, b.next = b, a\n return a\n\ndef sub():\n return Mine(9)\n\ndef walk(head):\n return [isinstance(head, ListNode), head.value, head.next.value, head.length()]\n\ndef tamper():\n ListNode.length = lambda self: 42\n return ListNode(5)\n", + ); + let mut plan = unit(&student); + plan.imports = vec![teacher.to_string_lossy().into_owned()]; + plan.shared = vec![(common, "common.py".into())]; + let checked = |name: &str, check: &str| { + let mut c = function(name, vec![]); + c.check = Some(InProcessCheck { + function: check.into(), + expected: None, + }); + c + }; + let mut made = call( + Target::Teacher { + name: "make".into(), + }, + vec![], + ); + made.id = Some("made".into()); + plan.setup = vec![made]; + plan.steps = vec![ + checked("ring", "is_ring"), + checked("sub", "base"), + function("walk", vec![json!("$made")]), + checked("tamper", "base"), + ]; + let obs = run(&plan).await; + let pass = CheckObservation::Verdict { + pass: true, + message: String::new(), + }; + assert_eq!(obs.checks[&0], pass, "{obs:?}"); + assert!( + matches!( + &obs.checks[&1], + CheckObservation::Verdict { pass: true, .. } + ), + "a student subclass arrives as the shared base: {:?}", + obs.checks[&1] + ); + assert_eq!( + returned(&obs.steps[2].outcome), + &json!([true, 1, 2, 2]), + "the teacher's list arrives as the student's own class" + ); + assert!( + matches!( + &obs.checks[&3], + CheckObservation::Verdict { pass: true, .. } + ), + "the student's patch stays on the student's side: {:?}", + obs.checks[&3] + ); +} + +/// A shared module imports without an `allowed_imports` entry, and a student file that +/// shares its name with a teacher module never stands in for it. +#[tokio::test] +async fn test_a_shared_module_is_allowed_and_namespaces_stay_apart() { + let dir = tempfile::tempdir().unwrap(); + let common = write(dir.path(), "bundle/common.py", COMMON); + write(dir.path(), "bundle/helper.py", "WHO = 'teacher'\n"); + let teacher = write( + dir.path(), + "bundle/teacher.py", + "import helper\n\ndef who(result, expected):\n return helper.WHO == 'teacher', helper.WHO\n", + ); + let student = write( + dir.path(), + "submission/helper.py", + "import common\nWHO = 'student'\n\ndef f():\n return common.ListNode(1).length()\n", + ); + let mut plan = unit(&student); + plan.bundle = dir.path().join("bundle"); + plan.imports = vec![teacher.to_string_lossy().into_owned()]; + plan.shared = vec![(common, "common.py".into())]; + let mut f = function("f", vec![]); + f.check = Some(InProcessCheck { + function: "who".into(), + expected: None, + }); + plan.steps = vec![f]; + let obs = run(&plan).await; + assert_eq!(returned(&obs.steps[0].outcome), &json!(1), "{obs:?}"); + assert_eq!( + obs.checks[&0], + CheckObservation::Verdict { + pass: true, + message: "teacher".into() + } + ); +} + +/// A checker may take the student's source, to judge it with `ast`. +#[tokio::test] +async fn test_a_checker_can_read_the_students_source() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "def f(xs):\n total = 0\n for x in xs:\n total += x\n return total\n", + ); + let teacher = write( + dir.path(), + "t.py", + "import ast\n\ndef no_sum(result, expected, source):\n calls = [n.func.id for n in ast.walk(ast.parse(source)) if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)]\n return 'sum' not in calls and result == expected, f'calls {calls}'\n", + ); + let mut plan = unit(&student); + plan.imports = vec![teacher.to_string_lossy().into_owned()]; + let mut f = function("f", vec![json!([1, 2, 3])]); + f.check = Some(InProcessCheck { + function: "no_sum".into(), + expected: Some(json!(6)), + }); + plan.steps = vec![f]; + let obs = run(&plan).await; + assert_eq!( + obs.checks[&0], + CheckObservation::Verdict { + pass: true, + message: "calls []".into() + }, + "{obs:?}" + ); +} diff --git a/src/core/tests/integration.rs b/src/core/tests/integration.rs index a218c00..d6b8677 100644 --- a/src/core/tests/integration.rs +++ b/src/core/tests/integration.rs @@ -652,8 +652,10 @@ expect = 2 #[tokio::test] async fn test_setup_runs_once_per_unit_and_once_per_scenario() { let bench = Bench::new(); - let unit_log = bench.path().join("unit.log"); - let scenario_log = bench.path().join("scenario.log"); + // Outside the bundle: a path into it is refused before it reaches student code. + let logs = tempfile::tempdir().unwrap(); + let unit_log = logs.path().join("unit.log"); + let scenario_log = logs.path().join("scenario.log"); let spec = bench.spec(&format!( r#" [meta] @@ -1407,3 +1409,59 @@ impl TapImports for TestSpec { self } } + +/// OSS-10: student code cannot read the bundle, so a path into it handed to student code +/// is refused before anyone is graded — whether a [vars] entry, an argument, or a teacher +/// module's constant — and `data_files` is the way to give the student a file. +#[tokio::test] +async fn test_a_path_into_the_bundle_is_never_handed_to_student_code() { + let bench = Bench::new(); + let data = bench.write("data/numbers.txt", "1 2 3\n"); + bench.write( + "helpers/paths.py", + "from pathlib import Path\nDATA = Path(__file__).parent.parent / 'data' / 'numbers.txt'\nNAME = 'numbers.txt'\n", + ); + let spec = |body: &str| { + bench.spec(&format!( + "[meta]\nname = \"t\"\nfile = \"lab.py\"\nfunction = \"f\"\nlanguage = \"python\"\nimports = [\"helpers/paths.py\"]\ndata_files = [\"data\"]\n{body}" + )) + }; + let path = data.to_string_lossy().into_owned(); + // A file the student would write, which does not exist yet, through the same links + // (`/var` is `/private/var` on macOS). + let unborn = bench.path().join("out.log").to_string_lossy().into_owned(); + for (body, wanted) in [ + ( + format!("[vars]\nOUT = {unborn:?}\n[[cases]]\nname = \"x\"\nexpect = 1\n"), + "[vars] 'OUT'", + ), + ( + format!("[vars]\nINPUT = {path:?}\n[[cases]]\nname = \"x\"\nexpect = 1\n"), + "[vars] 'INPUT'", + ), + ( + format!("[[cases]]\nname = \"x\"\nargs = [{path:?}]\nexpect = 1\n"), + "case 'x'", + ), + ( + "[[cases]]\nname = \"x\"\nargs = [\"$DATA\"]\nexpect = 1\n".to_string(), + "case 'x': '$DATA' is a path into the test bundle", + ), + ] { + let refused = refusal(spec(&body)).await; + assert!(refused.contains(wanted), "{refused}"); + assert!(refused.contains("data_files"), "{refused}"); + } + + // Staged by data_files and named relatively, the same file is the student's to read. + let spec = spec( + "[vars]\nINPUT = \"data/numbers.txt\"\n[[cases]]\nname = \"x\"\nargs = [\"$NAME\"]\nexpect = 6\n", + ); + let student = bench.student( + "alice", + "lab.py", + "def f(name):\n with open(INPUT) as fh:\n return sum(int(n) for n in fh.read().split())\n", + ); + let results = grade(vec![spec], &[student]).await; + assert_eq!(results[0].total_passed(), 1, "{:#?}", results[0]); +} diff --git a/src/core/tests/sandbox.rs b/src/core/tests/sandbox.rs index efe5f62..9bbfcdb 100644 --- a/src/core/tests/sandbox.rs +++ b/src/core/tests/sandbox.rs @@ -83,6 +83,14 @@ def run_program(): # In place of this process: no fork, so no process limit stands in for the sandbox. return attempt(lambda: os.execv('/bin/echo', ['/bin/echo', 'escaped'])) +def link_native(): + # The native-call module under a name of the student's own: its directory can be + # listed, though the module itself cannot be read. + sysconfig = importlib.import_module('sysconfig') + where = os.path.join(sysconfig.get_paths()['platstdlib'], 'lib-dynload') + name = next(n for n in os.listdir(where) if n.startswith('_ctypes')) + return attempt(lambda: os.link(os.path.join(where, name), 'mine.so')) + def signal_grader(): return attempt(lambda: os.kill(os.getppid(), 0)) @@ -142,6 +150,7 @@ impl Bench { vars: Arc::new(BTreeMap::new()), data_files: Vec::new(), bundle: self.bundle.clone(), + shared: Vec::new(), allowed_imports: Vec::new(), load_timeout: 5, setup: Vec::new(), @@ -190,8 +199,9 @@ fn denied(probes: &[(String, String, String)], name: &str) { assert!(!sandboxed.starts_with("ok"), "{name}: {sandboxed}"); return; } - if name == "run program" { - // Unsandboxed, it replaces the process and never answers: no control to compare. + if name == "run program" || name == "native by another name" { + // Unsandboxed, the first replaces the process and never answers, and the second + // depends on where the interpreter lives: no control to compare. assert_eq!(sandboxed, "PermissionError", "{name}: {sandboxed}"); return; } @@ -244,6 +254,9 @@ async fn test_the_student_side_reaches_only_its_own_directory() { ("own directory", function("write", vec![json!("mine.txt")])), ]; let mut sandboxed_only = vec![("run program", function("run_program", vec![]))]; + if cfg!(target_os = "macos") { + sandboxed_only.push(("native by another name", function("link_native", vec![]))); + } if cfg!(target_os = "macos") { // Seatbelt does not mediate reading another process's arguments and environment; // refusing native calls is what keeps the grader's out of reach. @@ -264,9 +277,10 @@ async fn test_the_student_side_reaches_only_its_own_directory() { } allowed(&results, "own directory"); - let (name, probe) = sandboxed_only.remove(0); - let obs = PythonExecutor::new().run(&bench.plan(vec![probe])).await; - denied(&[(name.to_string(), said(&obs, 0), String::new())], name); + for (name, probe) in sandboxed_only { + let obs = PythonExecutor::new().run(&bench.plan(vec![probe])).await; + denied(&[(name.to_string(), said(&obs, 0), String::new())], name); + } } #[tokio::test] @@ -319,3 +333,68 @@ async fn test_neither_side_sees_the_graders_environment() { ); } } + +/// A lint tool runs in the teacher's sandbox: it reads the submission's copy and the bundle, +/// and not another student's work. +#[test] +fn test_a_lint_tool_reads_the_bundle_and_the_copy_and_nothing_else() { + use scriptmark_core::models::LintConfig; + use scriptmark_core::runner::linter::run_lint; + let bench = Bench::new(); + let reading = |path: &Path| LintConfig { + // One finding line, if both reads succeed; a failed read exits 1, outside [0]. + command: format!( + "sh -c 'cat \"$1\" > /dev/null && cat \"$2\" > /dev/null && echo finding' sh {{file}} {}", + path.display() + ), + max_warnings: 10, + ok_exit_codes: vec![0], + }; + let lint = + |config: &LintConfig, isolation| run_lint(config, &bench.student, &bench.bundle, isolation); + assert_eq!( + lint(&reading(&bench.answers), Isolation::Sandboxed), + Ok(90.0) + ); + assert_eq!( + lint(&reading(&bench.other), Isolation::Unisolated), + Ok(90.0) + ); + let refused = lint(&reading(&bench.other), Isolation::Sandboxed).unwrap_err(); + assert!(refused.contains("exited with"), "{refused}"); +} + +/// A Python checker script runs in the teacher's sandbox too, with a cleared environment. +#[test] +fn test_a_python_checker_script_reads_the_bundle_and_nothing_else() { + use scriptmark_core::checker::python_checker::PythonChecker; + use scriptmark_core::checker::{CheckInput, Checker}; + let bench = Bench::new(); + let script = write( + &bench.bundle, + "verify.py", + &format!( + "import json, os, sys\njson.load(sys.stdin)\ndef can(path):\n try:\n open(path).read()\n return 'yes'\n except OSError:\n return 'no'\nbundle, other = can({:?}), can({:?})\nprint(json.dumps({{'pass': True, 'message': f'bundle={{bundle}} other={{other}} env={{sorted(os.environ)}}'}}))\n", + bench.answers, bench.other + ), + ); + let input = CheckInput { + result: json!(1), + expected: json!(1), + context: json!({}), + }; + let message = |checker: PythonChecker| checker.check(&input).unwrap().message; + let python = PythonExecutor::new().python_cmd().to_string(); + let open = message(PythonChecker::new(&script).with_python_cmd(&python)); + let confined = message( + PythonChecker::new(&script) + .with_python_cmd(&python) + .confined(vec![bench.bundle.clone()]), + ); + assert!(open.starts_with("bundle=yes other=yes"), "{open}"); + assert!(confined.starts_with("bundle=yes other=no"), "{confined}"); + assert!( + !confined.contains("CARGO"), + "the environment is cleared: {confined}" + ); +} From 1d38206969446aeba960729164841e1212c2da5d Mon Sep 17 00:00:00 2001 From: Acture Date: Thu, 15 Oct 2026 18:24:17 +0800 Subject: [PATCH 05/20] fix(lint): read only a tool's own installation, and stage the file under a safe name (OSS-10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A tool's read grant widened from its `bin` to the directory above whenever that directory held no `etc` or `var`, which `~/bin/tool` turns into the whole home directory. It now widens only for a self-contained installation — a virtual environment or a Homebrew keg — and otherwise grants the program file alone. The read-only copy kept the student's file name, which reaches the tool's arguments and, through a command such as `sh -c '… {file}'`, a shell. It is staged under that name with everything but letters, digits, `.`, `_` and `-` replaced, so a linter judging module names still judges the same one. --- src/core/src/runner/linter.rs | 87 +++++++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 18 deletions(-) diff --git a/src/core/src/runner/linter.rs b/src/core/src/runner/linter.rs index 7d86e94..f31be51 100644 --- a/src/core/src/runner/linter.rs +++ b/src/core/src/runner/linter.rs @@ -42,11 +42,12 @@ pub fn run_lint( .map_err(|e| format!("could not stage the file to lint: {e}"))?; let (_keep, base) = scratch; let work = base.join("work"); - let copy = base.join("submission").join( - file_path - .file_name() - .ok_or_else(|| format!("'{}' names no file", file_path.display()))?, - ); + let name = file_path + .file_name() + .ok_or_else(|| format!("'{}' names no file", file_path.display()))?; + let copy = base + .join("submission") + .join(safe_name(&name.to_string_lossy())); std::fs::create_dir_all(&work) .and_then(|_| std::fs::create_dir_all(copy.parent().expect("a parent"))) .and_then(|_| std::fs::copy(file_path, ©)) @@ -133,6 +134,28 @@ pub fn run_lint( Ok(style_score(warnings, config.max_warnings)) } +/// The student's file name with anything a shell would read as syntax replaced: the name +/// reaches the tool's arguments, and a teacher's command may hand it to `sh -c`. Letters, +/// digits, `.`, `_` and `-` stay, so a linter judging module names judges the same one. +fn safe_name(name: &str) -> String { + let safe: String = name + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-') { + c + } else { + '_' + } + }) + .collect(); + // Never an option, and never `.` or `..`. + if safe.starts_with(['-', '.']) { + format!("_{safe}") + } else { + safe + } +} + /// The program a command names: a path as given, or the first match on the grader's PATH. fn locate(program: &str) -> Result { let given = Path::new(program); @@ -192,20 +215,27 @@ fn tool_reads(program: &Path) -> Vec { reads } -/// The directory a program is installed in: above its `bin` — a virtual environment or a -/// Homebrew keg — or its own. Never a prefix many programs share, such as `/`, -/// `/usr/local` or `/opt/homebrew`: those hold configuration and data in `etc` and `var`. +/// What a program's installation is: the whole of it when it is self-contained — a virtual +/// environment, as `uv tool` and `pipx` make, or a Homebrew keg — and otherwise only the +/// program itself. A directory it merely sits in, such as `~/bin` or `/usr/local/bin`, may +/// hold anything. fn installation(real: &Path) -> PathBuf { let dir = real.parent().unwrap_or(real); let above = dir.parent().filter(|above| { - matches!( + let in_bin = matches!( dir.file_name().and_then(|n| n.to_str()), Some("bin" | "sbin") - ) && !["etc", "var"] - .iter() - .any(|shared| above.join(shared).exists()) + ); + let venv = above.join("pyvenv.cfg").is_file(); + // …/Cellar///bin/ + let keg = above + .parent() + .and_then(Path::parent) + .and_then(Path::file_name) + .is_some_and(|n| n == "Cellar"); + in_bin && (venv || keg) }); - above.unwrap_or(dir).to_path_buf() + above.unwrap_or(real).to_path_buf() } /// `max(0, 1 - warnings / max_warnings) * 100`; with `max_warnings = 0`, any warning is 0. @@ -242,18 +272,39 @@ mod tests { } #[test] - fn test_a_tool_is_read_from_its_own_installation_never_a_shared_prefix() { + fn test_a_tool_is_read_whole_only_when_its_installation_is_its_own() { let dir = tempfile::tempdir().unwrap(); let root = dir.path(); - for made in ["venv/bin", "prefix/bin", "prefix/etc"] { + for made in ["venv/bin", "Cellar/ruff/0.6/bin", "home/bin"] { std::fs::create_dir_all(root.join(made)).unwrap(); } + std::fs::write(root.join("venv/pyvenv.cfg"), "home = /usr/bin\n").unwrap(); assert_eq!(installation(&root.join("venv/bin/ruff")), root.join("venv")); assert_eq!( - installation(&root.join("prefix/bin/ruff")), - root.join("prefix/bin") + installation(&root.join("Cellar/ruff/0.6/bin/ruff")), + root.join("Cellar/ruff/0.6") ); - assert_eq!(installation(Path::new("/bin/sh")), PathBuf::from("/bin")); + // `~/bin` holds whatever its owner put there: only the tool itself. + assert_eq!( + installation(&root.join("home/bin/ruff")), + root.join("home/bin/ruff") + ); + assert_eq!(installation(Path::new("/bin/sh")), PathBuf::from("/bin/sh")); + } + + #[test] + fn test_a_students_file_name_never_reaches_the_tool_as_syntax() { + assert_eq!(safe_name("Lab5_1.py"), "Lab5_1.py"); + assert_eq!(safe_name("x'; rm -rf ~ #.py"), "x___rm_-rf____.py"); + assert_eq!(safe_name("-rf.py"), "_-rf.py"); + assert_eq!(safe_name("..py"), "_..py"); + assert_eq!(safe_name("实验一.py"), "___.py"); + // The tool sees the safe name, whatever the submission was called: one line. + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("a;echo injected;b.py"); + std::fs::write(&file, "x = 1\n").unwrap(); + let probe = config("sh -c 'echo {file}' sh", 10); + assert_eq!(lint(&probe, &file), Ok(90.0)); } #[test] From 2c59d878bf97d89c5b9991b93a9541be13aa9886 Mon Sep 17 00:00:00 2001 From: Acture Date: Thu, 15 Oct 2026 18:29:50 +0800 Subject: [PATCH 06/20] perf(runner): stop watching a unit's memory once its student side has ended (OSS-10) The watch ticks every 50 ms and was among the tasks a unit waited for after both sides ended, so every unit waited for its next tick: about 40 ms a unit when sandboxed. It is aborted as soon as the student side's exit is settled. --- src/core/src/runner/python.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/core/src/runner/python.rs b/src/core/src/runner/python.rs index a93de29..274499a 100644 --- a/src/core/src/runner/python.rs +++ b/src/core/src/runner/python.rs @@ -625,6 +625,7 @@ impl PythonExecutor { pumps.spawn(pump(pipe, teacher_tail.clone())); } let over_memory = Arc::new(AtomicBool::new(false)); + let mut watch = None; if let Some(child) = student.as_mut() { if let Some(pipe) = child.stderr.take() { pumps.spawn(pump(pipe, student_tail.clone())); @@ -633,7 +634,7 @@ impl PythonExecutor { let limit = self.memory_mb.saturating_mul(1 << 20); let over = over_memory.clone(); let group = student_group.0; - pumps.spawn(async move { watch_memory(pid, group, limit, over).await }); + watch = Some(tokio::spawn(watch_memory(pid, group, limit, over))); } } @@ -663,6 +664,10 @@ impl PythonExecutor { } } }; + // Its exit is settled: the watch has nothing left to end, and is not waited for. + if let Some(watch) = watch { + watch.abort(); + } // Whatever is left of either side — a hung student side, or a process either left // behind holding a pipe — goes now. drop(_teacher_group); From 2c480893ee29c878bc887f33b24ba8d17f5386c2 Mon Sep 17 00:00:00 2001 From: Acture Date: Thu, 15 Oct 2026 19:07:02 +0800 Subject: [PATCH 07/20] fix: harden the split and the sandboxes after review (OSS-10) From a fourteen-agent review, each finding checked by an adversarial verifier: - seccomp also refuses changing another process's limits or scheduling, any file's mode, owner or extended attributes, and, below Landlock ABI 3, truncating by path. - A student answer is checked against what its record may hold before it is written: a padded answer stops the student side, on the student's account, instead of breaking the record stream. - Checker scripts and lint tools run in their own process group under CPU and file-size limits, and are ended as a group. - The transfer encoding writes a way back into a tuple or frozenset as a cycle, and bounds integers by the frame alone. - Judging: a student value a teacher setup call could not read is the student's dependency; a stop during setup keeps its cause; a student side that dies while a checker reads its values is judged by how it died. - A relative --python is made absolute; Python older than 3.10 is refused with its version, sandboxed or not; Ctrl-C during prepare stops its units. - Lint: a {bundle} placeholder, a real PATH, PATHEXT and whitespace splitting on Windows, and an injectable deadline. - Records always carry their isolation; one made before this change has none, keeps its digest, and reads as unisolated. An empty `shared` stays out of the spec digest. Teacher exports holding bundle paths in lists or dicts are refused too, and a spec with only shared modules imports them at prepare. --- src/cli/src/main.rs | 34 ++++--- src/cli/tests/record.rs | 14 ++- src/core/src/checker/python_checker.rs | 18 ++++ src/core/src/export.rs | 6 +- src/core/src/models/spec.rs | 2 +- src/core/src/record.rs | 29 ++++-- src/core/src/runner/harness/common.py | 29 ++++-- src/core/src/runner/harness/teacher.py | 43 +++++++-- src/core/src/runner/judge.rs | 123 ++++++++++++++++++++++++- src/core/src/runner/linter.rs | 113 +++++++++++++++++++---- src/core/src/runner/orchestrator.rs | 1 + src/core/src/runner/python.rs | 27 +++++- src/core/src/runner/sandbox.rs | 67 +++++++++++++- src/core/tests/harness.rs | 75 +++++++++++++++ src/core/tests/sandbox.rs | 54 ++++++++++- 15 files changed, 563 insertions(+), 72 deletions(-) diff --git a/src/cli/src/main.rs b/src/cli/src/main.rs index 39cdf65..63bdb17 100644 --- a/src/cli/src/main.rs +++ b/src/cli/src/main.rs @@ -682,10 +682,13 @@ fn shown(path: &Path, view: &View) -> String { Some(n) => format!("{} (revision {n} of {})", path.display(), view.of), None => format!("{} (unscored)", path.display()), }; - if view.isolation.is_sandboxed() { + if view.sandboxed() { title } else { - format!("{title} — UNISOLATED: student code ran without a sandbox") + format!( + "{title} — UNISOLATED: {}", + record::isolation_label(view.isolation) + ) } } @@ -721,9 +724,15 @@ async fn run_bundles( let executor = Arc::new( PythonExecutor::with_python_cmd(&run_options.python).with_isolation(run_options.isolation), ); - let bundles = prepare(specs, &generation, executor.clone(), timeout) - .await - .context("refusing to grade: the test bundle is not ready")?; + // Preparing runs teacher code in units of its own process groups too: Ctrl-C ends them. + let bundles = tokio::select! { + prepared = prepare(specs, &generation, executor.clone(), timeout) => prepared + .context("refusing to grade: the test bundle is not ready")?, + _ = tokio::signal::ctrl_c() => { + scriptmark_core::runner::python::kill_all_units(); + anyhow::bail!("interrupted while preparing: every running unit was stopped") + } + }; println!( "Prepared {} test bundle(s): {} unit(s) per student", bundles.len(), @@ -1462,10 +1471,11 @@ async fn cmd_grades_push(args: GradesPushArgs) -> Result<()> { } } let view = record.view(args.revision)?; - if !view.isolation.is_sandboxed() { + if !view.sandboxed() { eprintln!( - "warning: {} was graded with --no-sandbox: these grades come from runs where student code was not isolated", - args.results.display() + "warning: {} is {}: student code was not isolated when these grades were made", + args.results.display(), + record::isolation_label(view.isolation) ); } let revision = scored(&view, &args.results)?; @@ -1583,6 +1593,7 @@ fn cmd_similarity(args: SimilarityArgs) -> Result<()> { fn cmd_report(args: ReportArgs) -> Result<()> { let (_, view) = load_view(&args.results, args.revision.revision)?; + let (sandboxed, isolation) = (view.sandboxed(), view.isolation); let reports = view.reports; let similarity = if let Some(sim_dir) = &args.similarity_dir { @@ -1609,12 +1620,13 @@ fn cmd_report(args: ReportArgs) -> Result<()> { None }; - let title = if view.isolation.is_sandboxed() { + let title = if sandboxed { args.title.clone() } else { format!( - "{} — UNISOLATED: student code ran without a sandbox", - args.title + "{} — UNISOLATED: {}", + args.title, + record::isolation_label(isolation) ) }; report::generate_html_report(&reports, similarity.as_deref(), &title, &args.output)?; diff --git a/src/cli/tests/record.rs b/src/cli/tests/record.rs index ee9de25..b65bca8 100644 --- a/src/cli/tests/record.rs +++ b/src/cli/tests/record.rs @@ -6,6 +6,7 @@ use std::path::Path; use std::process::{Command, Output}; use scriptmark_core::record::Record; +use scriptmark_core::runner::sandbox::Isolation; const SPEC: &str = r#" [meta] @@ -878,8 +879,8 @@ fn a_canvas_record_rescores_from_its_bundle_and_refuses_another_attempt_or_excus refused(rescore(dir, &[]), "2024002 is excused now"); } -/// OSS-10: a run without the sandbox says so in its record, and only then: a sandboxed -/// record's bytes do not mention it. +/// OSS-10: a record says whether student code ran in its sandbox, and a summary of one +/// that did not says so. #[test] fn a_run_without_the_sandbox_is_recorded_as_unisolated() { let dir = bench(); @@ -902,8 +903,13 @@ fn a_run_without_the_sandbox_is_recorded_as_unisolated() { ); let text = std::fs::read_to_string(dir.path().join("out/result.json")).unwrap(); let record = Record::load(&dir.path().join("out/result.json")).unwrap(); - assert_eq!(record.evidence.bundle.isolation.is_sandboxed(), isolated); - assert_eq!(text.contains("\"isolation\""), !isolated, "{text}"); + let expected = if isolated { + Isolation::Sandboxed + } else { + Isolation::Unisolated + }; + assert_eq!(record.evidence.bundle.isolation, Some(expected)); + assert!(text.contains("\"isolation\""), "{text}"); let summary = scriptmark(dir.path(), &["summarize", "out/result.json"]); assert_eq!( String::from_utf8_lossy(&summary.stdout).contains("UNISOLATED"), diff --git a/src/core/src/checker/python_checker.rs b/src/core/src/checker/python_checker.rs index b057624..d61e28a 100644 --- a/src/core/src/checker/python_checker.rs +++ b/src/core/src/checker/python_checker.rs @@ -139,6 +139,8 @@ pub(crate) fn run_with_timeout( stdin: Option, timeout: Duration, ) -> std::io::Result> { + #[cfg(unix)] + crate::runner::sandbox::limit_tool(&mut cmd, timeout.as_secs()); let mut child = cmd .stdin(if stdin.is_some() { Stdio::piped() @@ -175,11 +177,27 @@ pub(crate) fn run_with_timeout( .take() .map(|p| Box::new(p) as Box), ); + // Its own process group: whatever it started goes with it, so nothing left behind + // holds its pipes open. + let group = child.id(); + let end_group = || { + #[cfg(unix)] + if let Ok(group) = i32::try_from(group) { + // SAFETY: killpg only sends a signal; the group may already be gone. + unsafe { + libc::killpg(group, libc::SIGKILL); + } + } + #[cfg(not(unix))] + let _ = group; + }; let Some(status) = child.wait_timeout(timeout)? else { + end_group(); let _ = child.kill(); let _ = child.wait(); return Ok(None); }; + end_group(); Ok(Some(Ran { status, stdout: stdout.join().unwrap_or_default(), diff --git a/src/core/src/export.rs b/src/core/src/export.rs index f9cef29..04d4d1f 100644 --- a/src/core/src/export.rs +++ b/src/core/src/export.rs @@ -474,11 +474,7 @@ pub fn about(record: &Record, n: u32) -> Result
{ ("run_by".into(), Cell::text(&evidence.scriptmark)), ( "isolation".into(), - Cell::text(if evidence.bundle.isolation.is_sandboxed() { - "sandboxed" - } else { - "unisolated (--no-sandbox)" - }), + Cell::text(crate::record::isolation_label(evidence.bundle.isolation)), ), ("scored_by".into(), Cell::text(&revision.scriptmark)), ( diff --git a/src/core/src/models/spec.rs b/src/core/src/models/spec.rs index 265d9d5..44e6327 100644 --- a/src/core/src/models/spec.rs +++ b/src/core/src/models/spec.rs @@ -572,7 +572,7 @@ pub struct TestMeta { /// is copied beside the student's file, where student code imports it without an /// `allowed_imports` entry, and the teacher side imports its own copy from the bundle: /// an instance one side sends is rebuilt from the other side's copy of its class. - #[serde(default)] + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub shared: Vec, /// Refused: every case now has its own process. diff --git a/src/core/src/record.rs b/src/core/src/record.rs index 010418e..de0a45d 100644 --- a/src/core/src/record.rs +++ b/src/core/src/record.rs @@ -122,14 +122,20 @@ pub struct BundleVersion { /// The frozen inputs and answers the batch used, when it had any. #[serde(default, skip_serializing_if = "Option::is_none")] pub frozen: Option, - /// Whether student and teacher code ran in their sandboxes. Written only when they did - /// not (`--no-sandbox`), so every sandboxed record keeps its bytes and its digest. - #[serde(default, skip_serializing_if = "is_sandboxed")] - pub isolation: Isolation, + /// Whether student and teacher code ran in their sandboxes. Absent from a record made + /// before ScriptMark sandboxed them, whose bytes and digest it keeps: nothing was + /// isolated then. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub isolation: Option, } -fn is_sandboxed(isolation: &Isolation) -> bool { - isolation.is_sandboxed() +/// How a record's evidence was gathered, in words. +pub fn isolation_label(isolation: Option) -> &'static str { + match isolation { + Some(Isolation::Sandboxed) => "sandboxed", + Some(Isolation::Unisolated) => "unisolated (--no-sandbox)", + None => "unisolated (graded before ScriptMark sandboxed student code)", + } } /// One test spec as it ran. @@ -197,7 +203,14 @@ pub struct View { /// The items the revision scored, in declaration order; empty when unscored. pub items: Vec, /// How the evidence was gathered: an unisolated run is shown as one. - pub isolation: Isolation, + pub isolation: Option, +} + +impl View { + /// Student code ran in its sandbox. + pub fn sandboxed(&self) -> bool { + self.isolation == Some(Isolation::Sandboxed) + } } #[derive(Debug, thiserror::Error)] @@ -693,7 +706,7 @@ pub fn bundle_version( digest: digest(&serde_json::to_vec(&(timeout, semantics)).expect("plain JSON")), timeout, python, - isolation, + isolation: Some(isolation), frozen: (!frozen.is_empty()).then(|| FrozenLink { path: frozen_path, digest: digest(frozen.to_json().as_bytes()), diff --git a/src/core/src/runner/harness/common.py b/src/core/src/runner/harness/common.py index 1673911..2fceaf6 100644 --- a/src/core/src/runner/harness/common.py +++ b/src/core/src/runner/harness/common.py @@ -49,8 +49,6 @@ MESSAGE_LIMIT = 64 * 1024 TRANSFER_LIMIT = 32 * 1024 * 1024 DEPTH_LIMIT = 100 -# The longest integer the transfer encoding carries, in hexadecimal digits. -INT_DIGITS = 1 << 20 def configure(limits): @@ -463,13 +461,14 @@ def _properties(value): class _Node: - __slots__ = ("kind", "body", "refs", "number") + __slots__ = ("kind", "body", "refs", "number", "building") def __init__(self, kind): self.kind = kind self.body = None self.refs = 1 self.number = None + self.building = False def encode(value): @@ -565,16 +564,31 @@ def _emit(item, numbers): return {"$": "text", "v": item[1], "type": item[2]} if item.number is not None: return {"$": "ref", "#": item.number} + kind = item.kind + if kind in ("tuple", "frozenset"): + # Made from their items, so they are numbered once those are: a way back into one + # while its items are written cannot be rebuilt, and reads as a cycle, as the + # record encoding says. + if item.building: + return {"$": "text", "v": "", "type": kind} + item.building = True + items = [_emit(v, numbers) for v in item.body] + item.building = False + tagged = {} + if item.refs > 1: + item.number = numbers[0] + numbers[0] += 1 + tagged["#"] = item.number + return {"$": kind, **tagged, "v": items} tagged = {} if item.refs > 1: item.number = numbers[0] numbers[0] += 1 tagged["#"] = item.number - kind = item.kind if kind == "list": items = [_emit(v, numbers) for v in item.body] return items if not tagged else {"$": "list", **tagged, "v": items} - if kind in ("tuple", "set", "frozenset"): + if kind == "set": return {"$": kind, **tagged, "v": [_emit(v, numbers) for v in item.body]} if kind == "dict": pairs = [(_emit(k, numbers), _emit(v, numbers)) for k, v in item.body] @@ -643,8 +657,9 @@ def _tagged(kind, tree, made, resolve, depth): raise Garbled("a reference to a value this side does not hold") return resolve(tree["name"]) if kind == "int": - if not isinstance(body, str) or len(body) > INT_DIGITS: - raise Garbled("an integer too long to carry") + # Its length is the frame's, already bounded; hexadecimal parses in linear time. + if not isinstance(body, str): + raise Garbled("an integer that is not text") return int(body, 16) if kind == "float": return float(body) diff --git a/src/core/src/runner/harness/teacher.py b/src/core/src/runner/harness/teacher.py index d75586c..e0b3988 100644 --- a/src/core/src/runner/harness/teacher.py +++ b/src/core/src/runner/harness/teacher.py @@ -29,6 +29,16 @@ def emit(record): write_all(_RECORDS, frame(record)) +def emit_bounded(record, bound, student): + """Emit a record built from a student's answer, if it fits what a record of its call + may hold: an answer padded past that is the student's, not a record the grader + cannot read.""" + data = frame(record) + if len(data) - HEADER.size > bound: + raise student.garbled(f"an answer that makes a record of {len(data)} bytes, where at most {bound} may") + write_all(_RECORDS, data) + + def finish(student=None): """Stop now: threads teacher code left running must not hold the unit open.""" if student is not None: @@ -341,11 +351,18 @@ def load_teacher_modules(): return exports, duplicates -def into_bundle(value): - """Whether `value` is an absolute path into the test bundle, which only teacher code can - open: student code gets data through `data_files`, by a path relative to it.""" +def into_bundle(value, depth=0): + """Whether `value` is, or holds, an absolute path into the test bundle, which only + teacher code can open: student code gets data through `data_files`, by a path relative + to it.""" bundle = PAYLOAD.get("bundle") - if not bundle or not isinstance(value, (str, os.PathLike)): + if not bundle or depth > DEPTH_LIMIT: + return False + if isinstance(value, (list, tuple, set, frozenset)): + return any(into_bundle(v, depth + 1) for v in value) + if isinstance(value, dict): + return any(into_bundle(v, depth + 1) for v in [*value.keys(), *value.values()]) + if not isinstance(value, (str, os.PathLike)): return False try: path = os.fspath(value) @@ -611,7 +628,11 @@ def student_call(plan, phase, index, student): }) stopped(phase, index, stop) return None - emit({"kind": "call", "phase": phase, "index": index, **record}) + try: + emit_bounded({"kind": "call", "phase": phase, "index": index, **record}, call_bound(len(files)), student) + except Stop as stop: + stopped(phase, index, stop) + return None if returned and plan.get("id"): bind(plan["id"], student=True) if returned and check is not None: @@ -656,7 +677,11 @@ def run_load(student): "stdout": "", "stdout_truncated": False, "elapsed_ms": int((PAYLOAD["load_timeout"] + GRACE) * 1000)}) stopped("load", 0, stop) return False - emit({"kind": "call", "phase": "load", "index": 0, **record}) + try: + emit_bounded({"kind": "call", "phase": "load", "index": 0, **record}, call_bound(value=False), student) + except Stop as stop: + stopped("load", 0, stop) + return False return "returned" in record["outcome"] @@ -681,7 +706,11 @@ def run_script(student): stopped("step", 0, stop) return # A script that does not compile never ran: that is its load failing. - emit({"kind": "call", "phase": "load" if compile_error else "step", "index": 0, **record}) + phase = "load" if compile_error else "step" + try: + emit_bounded({"kind": "call", "phase": phase, "index": 0, **record}, call_bound(len(files), value=False), student) + except Stop as stop: + stopped("step", 0, stop) def start_student(): diff --git a/src/core/src/runner/judge.rs b/src/core/src/runner/judge.rs index ac8bae8..6dd0404 100644 --- a/src/core/src/runner/judge.rs +++ b/src/core/src/runner/judge.rs @@ -82,6 +82,18 @@ pub fn judge( match obs.steps.get(index) { Some(record) => { let check = obs.checks.get(&index); + if let (Some(stop), Some(CheckObservation::Error(e))) = (stop, check) + && stop.index == index + && e.type_name == "StudentEnded" + { + // The student side went while the checker read its values: how it + // went decides, not the checker. + let blanket = + student_stop(obs, stop, "the reading of its values for the checker"); + results.push(blanket_result(s, &blanket, target(index))); + stopped = Some(not_run(&blanket, s.name)); + continue; + } if check.is_none() && call.is_some_and(|c| c.check.is_some()) && matches!(record.outcome, Outcome::Returned { .. }) @@ -310,6 +322,17 @@ fn unit_failure(plan: &UnitPlan, obs: &UnitObservation) -> Option { message.clone(), ); } + // Every name is resolved before the call: a setup call that produced one + // failing stops the loop first, so what is unresolved here is a student's + // value that could not be read — whoever's call needed it. + outcome @ Outcome::Unresolved { .. } => { + return blanket( + TestStatus::Error, + Fault::Student, + Cause::Dependency, + format!("not run: {label} {}", describe(outcome, call.timeout)), + ); + } outcome => { return blanket( TestStatus::Error, @@ -324,7 +347,15 @@ fn unit_failure(plan: &UnitPlan, obs: &UnitObservation) -> Option { return Some(broken); } let killed = match &obs.stopped { - Some(stop) => student_stop(obs, stop, &label), + // How the student side stopped says more than "setup": a timeout stays + // a timeout, a memory kill a memory kill. + Some(stop) => { + let stopped = student_stop(obs, stop, &label); + return Some(Blanket { + message: format!("not run: {}", stopped.message), + ..stopped + }); + } None => unfinished(obs, owner, Cause::Setup, &label), }; if killed.fault != owner { @@ -1362,6 +1393,96 @@ mod tests { ); } + /// A student value that cannot be read for a teacher setup call is the student's; a + /// student side that stops during setup keeps how it stopped; and one that goes while a + /// checker reads its values is judged by how it went, not as a failed checker. + #[test] + fn test_what_the_student_side_does_outside_a_scored_call_is_still_the_students() { + let c = case("expect = 1"); + let mut p = plan(1); + let mut student_setup = call(Target::Function { + name: "make".into(), + }); + student_setup.id = Some("obj".into()); + let mut teacher_setup = call(Target::Teacher { + name: "wrap".into(), + }); + teacher_setup.id = Some("wrapped".into()); + p.setup = vec![student_setup, teacher_setup]; + + let mut unreadable = finished(vec![]); + unreadable.setup = vec![ + record(0, returned(Value::Null)), + record(1, Outcome::Unresolved { name: "obj".into() }), + ]; + assert_eq!( + verdict(&one(&c, &p, &unreadable)), + ( + TestStatus::Error, + Some(Fault::Student), + Some(Cause::Dependency) + ) + ); + + let mut hog = finished(vec![]); + hog.setup = Vec::new(); + hog.stopped = Some(Stopped { + phase: Phase::Setup, + index: 0, + reason: StopReason::Exited, + message: String::new(), + }); + hog.exit = Exit::Memory; + let r = one(&c, &p, &hog); + assert_eq!( + verdict(&r), + (TestStatus::Error, Some(Fault::Student), Some(Cause::Memory)) + ); + assert!(r.failure.unwrap().message.starts_with("not run:")); + + let mut p = plan(2); + p.steps[0].check = Some(InProcessCheck { + function: "chk".into(), + expected: None, + }); + let c = case("expect = 1\ncheck = { function = \"chk\" }"); + let scored = [ + Scored { + name: "a", + case: &c, + }, + Scored { + name: "b", + case: &c, + }, + ]; + let mut gone = finished(vec![record(0, returned(json!(1)))]); + gone.checks.insert( + 0, + CheckObservation::Error(ErrorInfo { + type_name: "StudentEnded".into(), + types: vec!["StudentEnded".into()], + message: "the student side ended".into(), + }), + ); + gone.stopped = Some(Stopped { + phase: Phase::Step, + index: 0, + reason: StopReason::Exited, + message: String::new(), + }); + gone.exit = Exit::Memory; + let r = judge(&p, &scored, &gone, "python3", Isolation::Sandboxed); + assert_eq!( + verdict(&r[0]), + (TestStatus::Error, Some(Fault::Student), Some(Cause::Memory)) + ); + assert_eq!( + verdict(&r[1]), + (TestStatus::Error, Some(Fault::Student), Some(Cause::NotRun)) + ); + } + #[test] fn test_unit_failures_reach_every_case() { let c = case("expect = 1"); diff --git a/src/core/src/runner/linter.rs b/src/core/src/runner/linter.rs index f31be51..8d68933 100644 --- a/src/core/src/runner/linter.rs +++ b/src/core/src/runner/linter.rs @@ -13,7 +13,7 @@ use crate::models::LintConfig; use crate::runner::sandbox::{Confinement, Isolation}; /// How long a lint tool may run on one file. -const LINT_TIMEOUT: Duration = Duration::from_secs(120); +pub const LINT_TIMEOUT: Duration = Duration::from_secs(120); /// Run a lint command on a copy of `file_path` and return its style score, 0 to 100. /// @@ -25,11 +25,11 @@ pub fn run_lint( file_path: &Path, bundle: &Path, isolation: Isolation, + timeout: Duration, ) -> Result { // Split as a shell would, quotes included, before filling in the path: a path with a // space in it stays one argument. - let parts = shlex::split(&config.command) - .ok_or_else(|| format!("the lint command has an unclosed quote: {}", config.command))?; + let parts = split(&config.command)?; let Some((program, _)) = parts.split_first() else { return Err("the lint command is empty".into()); }; @@ -59,9 +59,14 @@ pub fn run_lint( .map_err(|e| format!("could not stage the file to lint: {e}"))?; let file = copy.display().to_string(); + // `{bundle}` names the test bundle, for a configuration file kept in it. + let bundle_dir = bundle.display().to_string(); let args: Vec = parts[1..] .iter() - .map(|part| part.replace("{file}", &file)) + .map(|part| { + part.replace("{file}", &file) + .replace("{bundle}", &bundle_dir) + }) .collect(); let command = std::iter::once(program.display().to_string()) .chain(args.iter().cloned()) @@ -88,18 +93,16 @@ pub fn run_lint( } else { Command::new(&program) }; - let mut path = std::env::join_paths( - crate::runner::sandbox::system_programs() - .iter() - .map(|p| p.join("bin")), + // The tool's own directory, then the system's: what a wrapper script it runs may need. + let path = std::env::join_paths( + program.parent().map(Path::to_path_buf).into_iter().chain( + PROGRAM_DIRS + .iter() + .map(PathBuf::from) + .filter(|d| d.is_dir()), + ), ) .unwrap_or_default(); - if let Some(dir) = program.parent() { - path = std::env::join_paths( - std::iter::once(dir.to_path_buf()).chain(std::env::split_paths(&path)), - ) - .unwrap_or(path); - } cmd.args(&args) .env_clear() .env("PATH", path) @@ -112,9 +115,9 @@ pub fn run_lint( cmd.env("SYSTEMROOT", root); } - let out = crate::checker::python_checker::run_with_timeout(cmd, None, LINT_TIMEOUT) + let out = crate::checker::python_checker::run_with_timeout(cmd, None, timeout) .map_err(|e| format!("could not run `{command}`: {e}"))? - .ok_or_else(|| format!("`{command}` ran longer than {}s", LINT_TIMEOUT.as_secs()))?; + .ok_or_else(|| format!("`{command}` ran longer than {}s", timeout.as_secs()))?; if !out .status .code() @@ -156,16 +159,56 @@ fn safe_name(name: &str) -> String { } } -/// The program a command names: a path as given, or the first match on the grader's PATH. +/// Where a tool finds the system's programs, in the order a shell would look. +#[cfg(unix)] +const PROGRAM_DIRS: &[&str] = &[ + "/usr/local/bin", + "/opt/homebrew/bin", + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", +]; +#[cfg(not(unix))] +const PROGRAM_DIRS: &[&str] = &[]; + +/// A command split as a shell would, quotes included. On Windows, where a path is written +/// with backslashes a POSIX shell would read as escapes, on whitespace. +fn split(command: &str) -> Result, String> { + if cfg!(windows) { + return Ok(command.split_whitespace().map(String::from).collect()); + } + shlex::split(command) + .ok_or_else(|| format!("the lint command has an unclosed quote: {command}")) +} + +/// The program a command names: a path as given, or the first match on the grader's PATH +/// (with Windows' executable extensions there). fn locate(program: &str) -> Result { let given = Path::new(program); if given.components().count() > 1 { return std::path::absolute(given).map_err(|e| format!("cannot find `{program}`: {e}")); } + let extensions: Vec = if cfg!(windows) { + std::iter::once(String::new()) + .chain( + std::env::var("PATHEXT") + .unwrap_or_else(|_| ".EXE;.CMD;.BAT".into()) + .split(';') + .map(String::from), + ) + .collect() + } else { + vec![String::new()] + }; std::env::var_os("PATH") .iter() .flat_map(std::env::split_paths) - .map(|dir| dir.join(program)) + .flat_map(|dir| { + extensions + .iter() + .map(move |ext| dir.join(format!("{program}{ext}"))) + }) .find(|candidate| candidate.is_file()) .ok_or_else(|| format!("could not run `{program}`: not found on PATH")) } @@ -268,7 +311,13 @@ mod tests { /// Unconfined, so these run inside any sandbox the test itself is in; the sandboxed /// runs are exercised end to end in the integration tests. fn lint(config: &LintConfig, file: &Path) -> Result { - run_lint(config, file, file.parent().unwrap(), Isolation::Unisolated) + run_lint( + config, + file, + file.parent().unwrap(), + Isolation::Unisolated, + LINT_TIMEOUT, + ) } #[test] @@ -307,6 +356,32 @@ mod tests { assert_eq!(lint(&probe, &file), Ok(90.0)); } + #[test] + fn test_a_tool_past_its_deadline_has_failed_and_its_children_go_with_it() { + let (_dir, file) = student_file(); + let config = config("sh -c 'sleep 30 & sleep 30'", 10); + let started = std::time::Instant::now(); + let late = run_lint( + &config, + &file, + file.parent().unwrap(), + Isolation::Unisolated, + Duration::from_secs(1), + ); + assert!(late.unwrap_err().contains("ran longer than 1s")); + assert!( + started.elapsed() < Duration::from_secs(10), + "a child holding the pipes was ended with its group" + ); + } + + #[test] + fn test_bundle_names_the_test_bundle() { + let (_dir, file) = student_file(); + let probe = config("sh -c 'test -d \"$1\" && echo found' sh {bundle}", 10); + assert_eq!(lint(&probe, &file), Ok(90.0)); + } + #[test] fn test_style_score() { assert!((style_score(3, 10) - 70.0).abs() < 1e-9); diff --git a/src/core/src/runner/orchestrator.rs b/src/core/src/runner/orchestrator.rs index d626dff..80455e6 100644 --- a/src/core/src/runner/orchestrator.rs +++ b/src/core/src/runner/orchestrator.rs @@ -361,6 +361,7 @@ async fn lint( std::path::Path::new(&file), &bundle, isolation, + crate::runner::linter::LINT_TIMEOUT, ) }) .await diff --git a/src/core/src/runner/python.rs b/src/core/src/runner/python.rs index 274499a..738d9a3 100644 --- a/src/core/src/runner/python.rs +++ b/src/core/src/runner/python.rs @@ -24,9 +24,16 @@ use crate::runner::sandbox::{Confinement, Isolation, covering}; /// Resolve a python command to an absolute path so env_clear() doesn't change /// which interpreter runs. Falls back to the original string if resolution fails. fn resolve_python_path(cmd: &str) -> String { - if cmd.starts_with('/') { + let path = Path::new(cmd); + if path.is_absolute() { return cmd.to_string(); } + // A relative path, `.venv/bin/python`, is the grader's: each side starts elsewhere. + if path.components().count() > 1 { + return std::path::absolute(path) + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_else(|_| cmd.to_string()); + } if let Ok(output) = std::process::Command::new("which").arg(cmd).output() && output.status.success() && let Ok(path) = String::from_utf8(output.stdout) @@ -129,6 +136,9 @@ struct Interpreter { exec: Vec, } +/// The oldest Python the harness runs on. +const MINIMUM_PYTHON: (u32, u32) = (3, 10); + /// Asks the interpreter where it lives, unconfined and once: its prefixes, and where its /// standard library and installed packages are — not every `sys.path` entry, which a /// package's `.pth` file can point anywhere, a home directory included. @@ -136,6 +146,7 @@ const PROBE: &str = "import json, os, sys, sysconfig places = sysconfig.get_paths() paths = [sys.prefix, sys.base_prefix, sys.exec_prefix, sys.base_exec_prefix, *(places[k] for k in ('stdlib', 'platstdlib', 'purelib', 'platlib'))] print(json.dumps({ + 'version': list(sys.version_info[:2]), 'executable': os.path.realpath(sys.executable), 'base': os.path.realpath(sys.base_prefix), 'paths': [os.path.realpath(p) for p in paths if p and os.path.exists(p)], @@ -145,6 +156,7 @@ impl Interpreter { fn probe(python_cmd: &str) -> Result { #[derive(serde::Deserialize)] struct Found { + version: (u32, u32), executable: PathBuf, base: PathBuf, paths: Vec, @@ -160,6 +172,13 @@ impl Interpreter { String::from_utf8_lossy(&output.stderr).trim() ) })?; + // The harness uses `X | None` and `match`-era typing: older interpreters die starting it. + if found.version < MINIMUM_PYTHON { + return Err(format!( + "{python_cmd} is Python {}.{}; ScriptMark needs Python {}.{} or newer", + found.version.0, found.version.1, MINIMUM_PYTHON.0, MINIMUM_PYTHON.1 + )); + } let command = PathBuf::from(python_cmd); let mut read = found.paths; read.extend(system_libraries(&found.base)); @@ -477,10 +496,12 @@ impl PythonExecutor { teacher: Vec, student: Vec, ) -> Result, String> { + // Asked even unsandboxed: an interpreter too old for the harness is refused here, + // with its version, rather than by both sides failing to start. + let interpreter = interpreter(&self.python_cmd)?; if !self.isolation.is_sandboxed() { return Ok(None); } - let interpreter = interpreter(&self.python_cmd)?; let confine = |side: &Side, extra: Vec| Confinement { read: covering(interpreter.read.iter().cloned().chain(extra).collect()), write: vec![side.work.clone(), side.home.clone(), side.tmp.clone()], @@ -990,7 +1011,7 @@ impl Executor for PythonExecutor { } async fn inspect(&self, spec: &TestSpec, timeout_secs: u64) -> Result { - if spec.meta.imports.is_empty() { + if spec.meta.imports.is_empty() && spec.meta.shared.is_empty() { return Ok(TeacherRuntime::default()); } let data = spec diff --git a/src/core/src/runner/sandbox.rs b/src/core/src/runner/sandbox.rs index 0e38f59..95b787b 100644 --- a/src/core/src/runner/sandbox.rs +++ b/src/core/src/runner/sandbox.rs @@ -88,6 +88,26 @@ pub fn apply_sandbox(cmd: &mut tokio::process::Command, config: &SandboxConfig) } } +/// Limits for a teacher's tool — a checker script or a lint tool — run outside a unit: its +/// own process group, so it ends with everything it started, a CPU limit a second past +/// its deadline, and no file larger than `SandboxConfig`'s or any core dump. +#[cfg(unix)] +pub fn limit_tool(cmd: &mut std::process::Command, deadline_secs: u64) { + use std::os::unix::process::CommandExt; + let cpu = deadline_secs.saturating_add(1); + let fsize = SandboxConfig::default().fsize_mb * 1024 * 1024; + cmd.process_group(0); + // SAFETY: setrlimit is async-signal-safe; this runs between fork and exec. + unsafe { + cmd.pre_exec(move || { + let _ = set_rlimits(libc::RLIMIT_CPU, cpu, cpu.saturating_add(1)); + let _ = set_rlimit(libc::RLIMIT_FSIZE, fsize); + let _ = set_rlimit(libc::RLIMIT_CORE, 0); + Ok(()) + }); + } +} + #[cfg(target_os = "macos")] type RlimitResource = libc::c_int; #[cfg(target_os = "linux")] @@ -387,7 +407,7 @@ mod platform { let abi = abi()?; let ruleset = landlock_ruleset(abi, c).map_err(|e| format!("Landlock: {e}"))?; // Signals and abstract sockets are Landlock's from ABI 6; before it, seccomp's. - let filters = filters(c, abi < ABI::V6)?; + let filters = filters(c, abi < ABI::V6, abi < ABI::V3)?; let mut cmd = std::process::Command::new(program); let mut ruleset = Some(ruleset); // SAFETY: between fork and exec this only makes the syscalls that apply a prepared @@ -505,9 +525,12 @@ mod platform { /// The seccomp filters for `c`. Landlock governs paths; these govern what it cannot: /// sockets other than a pair the process makes for itself, connecting or binding - /// anything, sending to an address, starting processes, tracing, and the kernel's other - /// ways out. `signals` adds the kill family, when Landlock cannot scope signals. - fn filters(c: &Confinement, signals: bool) -> Result, String> { + /// anything, sending to an address, starting processes, tracing, changing another + /// process's limits or scheduling, changing any file's mode, owner or extended + /// attributes, and the kernel's other ways out. `signals` adds the kill family, when + /// Landlock cannot scope signals; `truncate` adds truncating by path, before Landlock + /// governs it (ABI 3). + fn filters(c: &Confinement, signals: bool, truncate: bool) -> Result, String> { let mut denied: BTreeMap> = BTreeMap::new(); for call in [ libc::SYS_connect, @@ -566,6 +589,42 @@ mod platform { denied.insert(call, Vec::new()); } } + // Its own limits and priority stay its own (pid 0, who 0); another process's are not. + for (call, arg) in [ + (libc::SYS_prlimit64, 0), + (libc::SYS_setpriority, 1), + (libc::SYS_sched_setaffinity, 0), + (libc::SYS_sched_setscheduler, 0), + (libc::SYS_sched_setparam, 0), + (libc::SYS_sched_setattr, 0), + ] { + denied.insert(call, vec![condition(arg, SeccompCmpOp::Ne, 0)?]); + } + // Landlock has no right for mode, owner or extended attributes: a student could + // otherwise lock another file of the user's away, even one it cannot read. + const SYS_FCHMODAT2: i64 = 452; + for call in [ + libc::SYS_fchmod, + libc::SYS_fchmodat, + SYS_FCHMODAT2, + libc::SYS_fchown, + libc::SYS_fchownat, + libc::SYS_setxattr, + libc::SYS_lsetxattr, + libc::SYS_fsetxattr, + libc::SYS_removexattr, + libc::SYS_lremovexattr, + libc::SYS_fremovexattr, + ] { + denied.insert(call, Vec::new()); + } + #[cfg(target_arch = "x86_64")] + for call in [libc::SYS_chmod, libc::SYS_chown, libc::SYS_lchown] { + denied.insert(call, Vec::new()); + } + if truncate { + denied.insert(libc::SYS_truncate, Vec::new()); + } if signals { for call in [ libc::SYS_kill, diff --git a/src/core/tests/harness.rs b/src/core/tests/harness.rs index e4d5128..39fe44a 100644 --- a/src/core/tests/harness.rs +++ b/src/core/tests/harness.rs @@ -1280,3 +1280,78 @@ async fn test_a_checker_can_read_the_students_source() { "{obs:?}" ); } + +/// Values the transfer encoding once could not rebuild — a cycle back through a tuple, an +/// integer of millions of digits — still reach the checker. +#[tokio::test] +async fn test_a_tuple_cycle_and_a_huge_integer_reach_the_checker() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "def looped():\n inner = []\n outer = (inner, 1)\n inner.append(outer)\n return outer\n\ndef huge():\n return 2 ** 5_000_000\n", + ); + let teacher = write( + dir.path(), + "t.py", + "def looped(result, expected):\n inner, one = result\n return one == 1 and inner == [''], repr(result)[:80]\n\ndef huge(result, expected):\n return result == 2 ** 5_000_000\n", + ); + let mut plan = unit(&student); + plan.imports = vec![teacher.to_string_lossy().into_owned()]; + plan.steps = ["looped", "huge"] + .iter() + .map(|name| { + let mut c = function(name, vec![]); + c.check = Some(InProcessCheck { + function: name.to_string(), + expected: None, + }); + c + }) + .collect(); + let obs = run(&plan).await; + assert!(obs.stopped.is_none(), "{:?}", obs.stopped); + for index in 0..2 { + assert!( + matches!( + &obs.checks[&index], + CheckObservation::Verdict { pass: true, .. } + ), + "{:?}", + obs.checks[&index] + ); + } +} + +/// A forged answer padded past what a record may hold is the student's broken channel, +/// not a record stream the grader cannot read. +#[tokio::test] +async fn test_a_padded_answer_is_the_students_not_the_harnesss() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "import importlib, json, struct\nos = importlib.import_module('os')\n\ndef pad():\n body = json.dumps({'outcome': {'returned': {'value': 1, 'type': 'int'}}, 'stdout': 'x' * (8 * 1024 * 1024), 'stdout_truncated': False, 'elapsed_ms': 0}).encode()\n for fd in range(3, 64):\n try:\n os.write(fd, struct.pack('>I', len(body)) + body)\n except OSError:\n pass\n return 1\n", + ); + let teacher = write( + dir.path(), + "t.py", + "def any(result, expected):\n return True\n", + ); + let mut plan = unit(&student); + plan.imports = vec![teacher.to_string_lossy().into_owned()]; + let mut pad = function("pad", vec![]); + pad.check = Some(InProcessCheck { + function: "any".into(), + expected: None, + }); + plan.steps = vec![pad]; + let obs = run(&plan).await; + assert!(obs.protocol_error.is_none(), "{:?}", obs.protocol_error); + assert!(obs.done, "{obs:?}"); + let stopped = obs.stopped.as_ref().expect("the student side is stopped"); + assert!( + matches!(stopped.reason, StopReason::Garbled | StopReason::Flooded), + "{stopped:?}" + ); +} diff --git a/src/core/tests/sandbox.rs b/src/core/tests/sandbox.rs index 9bbfcdb..c09e974 100644 --- a/src/core/tests/sandbox.rs +++ b/src/core/tests/sandbox.rs @@ -102,6 +102,20 @@ def environ_of_grader(): def env(): return attempt(lambda: sorted(os.environ)) + +def chmod(path): + stat = importlib.import_module('stat') + def change(): + mode = os.stat(path).st_mode + os.chmod(path, stat.S_IMODE(mode)) # the mode it already has: nothing to undo + return attempt(change) + +def unix_socket(path): + return attempt(lambda: socket.socket(socket.AF_UNIX).connect(path)) + +def limits_of_grader(): + resource = importlib.import_module('resource') + return attempt(lambda: resource.prlimit(os.getppid(), resource.RLIMIT_CPU)) "#; struct Bench { @@ -115,6 +129,9 @@ struct Bench { /// A port the grader listens on, which no unit may connect to. port: u16, _listener: std::net::TcpListener, + /// A Unix socket the grader listens on, likewise. + socket: PathBuf, + _unix: std::os::unix::net::UnixListener, } impl Bench { @@ -127,7 +144,19 @@ impl Bench { let other = write(&root, "submissions/bob/lab.py", "SECRET = 'bob'\n"); let student = write(&root, "submissions/alice/lab.py", PROBES); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + // Short: a socket path has a length limit, and a temporary directory can be deep. + let unique = dir + .path() + .file_name() + .unwrap() + .to_string_lossy() + .into_owned(); + let socket = std::env::temp_dir().join(format!("sm{unique}.sock")); + let _ = std::fs::remove_file(&socket); + let unix = std::os::unix::net::UnixListener::bind(&socket).unwrap(); Self { + socket, + _unix: unix, port: listener.local_addr().unwrap().port(), _listener: listener, _dir: dir, @@ -159,6 +188,12 @@ impl Bench { } } +impl Drop for Bench { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.socket); + } +} + async fn both(plan: &UnitPlan) -> (UnitObservation, UnitObservation) { let sandboxed = PythonExecutor::new().run(plan).await; let open = PythonExecutor::new() @@ -252,6 +287,14 @@ async fn test_the_student_side_reaches_only_its_own_directory() { ("spawn", function("spawn", vec![])), ("signal", function("signal_grader", vec![])), ("own directory", function("write", vec![json!("mine.txt")])), + ( + "mode of a bundle file", + function("chmod", vec![path(&bench.answers)]), + ), + ( + "unix socket", + function("unix_socket", vec![path(&bench.socket)]), + ), ]; let mut sandboxed_only = vec![("run program", function("run_program", vec![]))]; if cfg!(target_os = "macos") { @@ -350,8 +393,15 @@ fn test_a_lint_tool_reads_the_bundle_and_the_copy_and_nothing_else() { max_warnings: 10, ok_exit_codes: vec![0], }; - let lint = - |config: &LintConfig, isolation| run_lint(config, &bench.student, &bench.bundle, isolation); + let lint = |config: &LintConfig, isolation| { + run_lint( + config, + &bench.student, + &bench.bundle, + isolation, + scriptmark_core::runner::linter::LINT_TIMEOUT, + ) + }; assert_eq!( lint(&reading(&bench.answers), Isolation::Sandboxed), Ok(90.0) From 1b2b2442657da870b774f5475a57332d41b3202f Mon Sep 17 00:00:00 2001 From: Acture Date: Thu, 15 Oct 2026 19:08:25 +0800 Subject: [PATCH 08/20] fix(sandbox): drop the conversions clippy refuses on Linux (OSS-10) --- src/core/src/runner/sandbox.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/src/runner/sandbox.rs b/src/core/src/runner/sandbox.rs index 95b787b..9b609f6 100644 --- a/src/core/src/runner/sandbox.rs +++ b/src/core/src/runner/sandbox.rs @@ -446,7 +446,7 @@ mod platform { let readable = if c.spawn { read | AccessFs::Execute } else { - read.into() + read }; let mut ruleset = Ruleset::default() .set_compatibility(CompatLevel::HardRequirement) @@ -475,9 +475,9 @@ mod platform { }; let devices = [ ("/dev/null", read | AccessFs::WriteFile), - ("/dev/zero", read.into()), - ("/dev/urandom", read.into()), - ("/dev/random", read.into()), + ("/dev/zero", read), + ("/dev/urandom", read), + ("/dev/random", read), ] .into_iter() .filter_map(|(path, access)| rule(&PathBuf::from(path), access)); From 21f759075868b9c55b36439033150b4aa187025a Mon Sep 17 00:00:00 2001 From: Acture Date: Thu, 15 Oct 2026 19:11:48 +0800 Subject: [PATCH 09/20] fix(sandbox): refuse acting on other processes by kind, and syscalls newer than the filter (OSS-10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `setpriority` with PRIO_USER or PRIO_PGRP and who 0 reaches every process of the user: it is refused unless it names the calling process, as is `ioprio_set`; `migrate_pages` and `move_pages` on another process are refused. A denylist lets through whatever the kernel adds after it was written, so every syscall numbered from `setxattrat` (463) on — and x32's numbering with them — is refused as unknown, which the C library treats as an older kernel. --- src/core/src/runner/sandbox.rs | 53 ++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/src/core/src/runner/sandbox.rs b/src/core/src/runner/sandbox.rs index 9b609f6..84815cf 100644 --- a/src/core/src/runner/sandbox.rs +++ b/src/core/src/runner/sandbox.rs @@ -589,16 +589,27 @@ mod platform { denied.insert(call, Vec::new()); } } - // Its own limits and priority stay its own (pid 0, who 0); another process's are not. - for (call, arg) in [ - (libc::SYS_prlimit64, 0), - (libc::SYS_setpriority, 1), - (libc::SYS_sched_setaffinity, 0), - (libc::SYS_sched_setscheduler, 0), - (libc::SYS_sched_setparam, 0), - (libc::SYS_sched_setattr, 0), + // Its own limits, priority, scheduling and memory stay its own — pid 0, or the + // calling process by kind and id — and another process's are not. Rules for one call + // refuse it when any of them matches. + const PRIO_PROCESS: u64 = 0; + const IOPRIO_WHO_PROCESS: u64 = 1; + for (call, rules) in [ + (libc::SYS_prlimit64, vec![(0, 0)]), + (libc::SYS_setpriority, vec![(0, PRIO_PROCESS), (1, 0)]), + (libc::SYS_ioprio_set, vec![(0, IOPRIO_WHO_PROCESS), (1, 0)]), + (libc::SYS_sched_setaffinity, vec![(0, 0)]), + (libc::SYS_sched_setscheduler, vec![(0, 0)]), + (libc::SYS_sched_setparam, vec![(0, 0)]), + (libc::SYS_sched_setattr, vec![(0, 0)]), + (libc::SYS_migrate_pages, vec![(0, 0)]), + (libc::SYS_move_pages, vec![(0, 0)]), ] { - denied.insert(call, vec![condition(arg, SeccompCmpOp::Ne, 0)?]); + let rules = rules + .into_iter() + .map(|(arg, own)| condition(arg, SeccompCmpOp::Ne, own)) + .collect::, _>>()?; + denied.insert(call, rules); } // Landlock has no right for mode, owner or extended attributes: a student could // otherwise lock another file of the user's away, even one it cannot read. @@ -651,18 +662,22 @@ mod platform { // `clone3` takes its flags by pointer, where a filter cannot look: refuse it as // unknown, and the C library falls back to `clone`. let clone3 = BTreeMap::from([(libc::SYS_clone3, Vec::new())]); - let mut filters = vec![build(denied, libc::EPERM)?, build(clone3, libc::ENOSYS)?]; - #[cfg(target_arch = "x86_64")] - filters.push(no_x32()); + let filters = vec![ + build(denied, libc::EPERM)?, + build(clone3, libc::ENOSYS)?, + unknown_calls(), + ]; Ok(filters) } - /// The x32 calling convention numbers every syscall again, past `0x40000000`: a filter - /// keyed on the x86-64 numbers would let them through, so they are refused outright. - #[cfg(target_arch = "x86_64")] - fn no_x32() -> BpfProgram { + /// A list of what to refuse lets through whatever the kernel adds after it was written + /// — `setxattrat` and `removexattrat` came in 6.13. Every syscall numbered from the + /// first this list does not know is refused as unknown (`ENOSYS`), which the C library + /// and Python treat as an older kernel. They share one numbering on every architecture; + /// x32's numbering on x86-64, which starts at `0x40000000`, is refused with them. + fn unknown_calls() -> BpfProgram { use seccompiler::sock_filter; - const X32_SYSCALL_BIT: u32 = 0x4000_0000; + const FIRST_UNKNOWN: u32 = 463; // setxattrat const SECCOMP_RET_ALLOW: u32 = 0x7fff_0000; const SECCOMP_RET_ERRNO: u32 = 0x0005_0000; const LOAD_NR: u16 = 0x20; // BPF_LD | BPF_W | BPF_ABS, at seccomp_data.nr @@ -671,8 +686,8 @@ mod platform { let op = |code, jt, jf, k| sock_filter { code, jt, jf, k }; vec![ op(LOAD_NR, 0, 0, 0), - op(JUMP_GE, 0, 1, X32_SYSCALL_BIT), - op(RETURN, 0, 0, SECCOMP_RET_ERRNO | libc::EPERM as u32), + op(JUMP_GE, 0, 1, FIRST_UNKNOWN), + op(RETURN, 0, 0, SECCOMP_RET_ERRNO | libc::ENOSYS as u32), op(RETURN, 0, 0, SECCOMP_RET_ALLOW), ] } From f73338d8e6f8c0dd047387fe1c88e583e5094cf3 Mon Sep 17 00:00:00 2001 From: Acture Date: Thu, 15 Oct 2026 19:12:32 +0800 Subject: [PATCH 10/20] docs: describe the sandboxed teacher and student sides (OSS-10) README and CLAUDE.md describe the two processes per unit, the sandboxes and --no-sandbox, Python 3.10 as the minimum, and the files that hold them; the notes gitlink brings the teacher contract and design record. --- CLAUDE.md | 17 ++++++++++------- README.md | 14 ++++++++++---- notes | 2 +- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 98d7f3f..1d7ab41 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,8 +82,9 @@ revisions) → Display/DB/HTML/Canvas. `rescore` adds a revision from saved evid ``` core/src/models/ Data models, TOML spec parsing, grading policies core/src/discovery Student file discovery + archive extraction -core/src/runner/ PythonExecutor (subprocess), orchestrator, sandbox (setrlimit), - parametrize, expander, oracle, linter +core/src/runner/ PythonExecutor (teacher + student processes), harness/*.py, + orchestrator, sandbox (Seatbelt / Landlock + seccomp, setrlimit), + records (framed), parametrize, expander, oracle, linter core/src/checker/ Checker trait + Rhai + Python checkers core/src/record Versioned evidence, score revisions, rescore reuse checks cli/src/db/ SQLite: students, sessions (= revisions), results, similarity @@ -93,11 +94,11 @@ cli/src/tui/ ratatui terminal UI: students/sessions/similarity tabs ## Key Design Decisions -**No pytest.** ScriptMark is its own test engine. Student code runs in subprocess via embedded Python helper script (`python.rs::HELPER_SCRIPT`). The helper: syntax-checks with py_compile, suppresses input()/print() during module load, calls target function, returns JSON on stdout. +**No pytest.** ScriptMark is its own test engine. A unit (a case or a scenario) runs as two Python processes from embedded scripts (`runner/harness/`): the teacher side (`teacher.py`: teacher modules, function checkers, the plan, per-request deadlines, the records) and the student side (`student.py`: the student module only, one call per request, under its timer and import guard). They share `common.py` (framing, the record and transfer value encodings, `call()`). ScriptMark joins the two with a pair of pipes and reads length-prefixed JSON records from the teacher side; values cross by value (shared-class instances rebuilt from the receiver's copy, other objects as snapshots). -**Sandboxed execution.** Student subprocess runs with: (1) `env_clear()` — no access to host env vars, (2) import allowlist — only safe stdlib + teacher-specified `allowed_imports`, (3) `setrlimit` — kernel-enforced CPU/file size/fd/process limits, (4) `tokio::join!` pipe reading + explicit `child.kill()` on timeout. Python interpreter resolved to absolute path at construction time so `env_clear()` doesn't change which interpreter runs. +**Sandboxed execution.** Each side runs confined by default (`runner/sandbox.rs`): macOS Seatbelt via `sandbox-exec` with a generated profile, Linux Landlock + seccomp applied in `pre_exec`. The student side reads its interpreter and its own directory only; the teacher side also reads the bundle and a read-only source copy. Fail closed: no sandbox, no grading, unless `--no-sandbox` (recorded as `isolation: unisolated`). Both also get `env_clear()`, `setrlimit`, a memory watch on the student side, and an absolute interpreter path. The import allowlist is a teaching rule, not a boundary. Sandbox tests need a real sandbox: run `cargo test` outside any other Seatbelt sandbox (Claude Code's included), and on Linux from a filesystem with stable inodes (not a Docker Desktop bind mount). -**TOML test specs** have three sections: `[vars]` (constants injected as Python globals), `[[setup]]` (function calls storing results for `$ref`), `[[cases]]` (scored test cases). Cases support `expect`, `expect_error`, checkers (`check = "sorted"` / `{ rhai = "..." }` / `{ python = "verifiers/check.py" }`), and `[cases.parametrize]` with oracle. `[meta]` supports `allowed_imports` for teacher-controlled package access. +**TOML test specs** have three sections: `[vars]` (constants injected as Python globals), `[[setup]]` (function calls storing results for `$ref`), `[[cases]]` (scored test cases). Cases support `expect`, `expect_error`, checkers (`check = "sorted"` / `{ rhai = "..." }` / `{ python = "verifiers/check.py" }` / `{ function = "..." }`, which may take the reserved `stdout` and `source`), and `[cases.parametrize]` with oracle. `[meta]` supports `allowed_imports` (a teaching rule) and `shared` (modules whose classes cross between the two sides by value). Prepare refuses a path into the bundle handed to student code: `data_files` stages it instead. **Grading** uses declared per-item points and aggregation in `assignment.toml`. The default grade is the raw score scaled to the configured total; curves are opt-in. @@ -109,8 +110,10 @@ Zero and withheld grades stay distinct. See the notes' teacher contract for poli ## Critical Files -- `src/core/src/runner/python.rs` — PythonExecutor and embedded harness. Starts the isolated Python subprocess and checks its output. -- `src/core/src/runner/sandbox.rs` — SandboxConfig + setrlimit application. Platform-conditional: RLIMIT_AS skipped on macOS, RLIMIT_* constants differ between Linux (c_uint) and macOS (c_int). +- `src/core/src/runner/python.rs` — PythonExecutor: stages a unit, starts and joins both sides, reads the records, watches memory, probes the interpreter's read roots. +- `src/core/src/runner/harness/{common,teacher,student}.py` — the two sides of a unit and what they share. +- `src/core/src/runner/sandbox.rs` — `Isolation`, `Confinement`, the Seatbelt profile and Landlock/seccomp rules, plus setrlimit. Platform-conditional: RLIMIT_AS skipped on macOS, RLIMIT_* constants differ between Linux (c_uint) and macOS (c_int). +- `src/core/src/runner/records.rs` — framed record reader with plan-derived bounds. - `src/core/src/runner/orchestrator.rs` — Runs prepared bundles per student, tokio parallel. - `src/core/src/models/spec.rs` — TOML test specification types. - `src/core/src/discovery.rs` — File discovery and archive extraction with size/count limits. diff --git a/README.md b/README.md index 4fd1601..4fe653d 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,16 @@ notes/ Private project documentation submodule ``` Run Cargo commands from the repository root. The CLI depends on `scriptmark-core`; -the core has no CLI, network, database or terminal UI dependencies. Python 3 is still -required to execute student submissions, teacher checkers and reference solutions. +the core has no CLI, network, database or terminal UI dependencies. Python 3.10 or newer +is still required to execute student submissions, teacher checkers and reference +solutions. ScriptMark is distributed as a Rust CLI; it does not expose a Python package/API. +Student code and teacher code run in separate processes, each in the operating system's +sandbox: Seatbelt on macOS, Landlock and seccomp on Linux (5.13 or later, Landlock +enabled). Grading refuses where no sandbox can be applied; `--no-sandbox` grades without +one — on Windows, for now — and the results are marked unisolated. + ## Documentation and checkout Public documentation belongs in `docs/`. Private project documentation lives in @@ -387,13 +393,13 @@ See [the scoring contract](https://github.com/Acture/obsidian-vault/blob/project | `sorted` / `set_eq` / `contains` | Collection checks | | Rhai expression | `check = { rhai = "result != () && result > 0" }` | | Python script | `check = { python = "verifiers/check.py" }` | -| Teacher function | `check = { function = "is_valid" }`, run on the live value | +| Teacher function | `check = { function = "is_valid" }`, run on the returned value | | Output and files | `expected_stdout = "..."`, `expect_files = { "out.txt" = "..." }` | ## Features - **Custom test engine** -- subprocess execution, no pytest dependency -- **Isolated units** -- a process and directory per case, env isolation, import allowlist, setrlimit, timeout with kill; contains accidents, not a security sandbox ([details](https://github.com/Acture/obsidian-vault/blob/project/scriptmark/docs/test-bundles.md#what-grading-does-not-defend-against)) +- **Sandboxed units** -- every case runs as a teacher process and a student process, each in the OS sandbox: the student's reads only its own directory, with no network, no new programs and no signals to others; the teacher's reads the bundle but not other students' work or your environment. Resource limits, a memory watch and per-call deadlines contain accidents; built for introductory courses, not against a student attacking the grader ([details](https://github.com/Acture/obsidian-vault/blob/project/scriptmark/docs/test-bundles.md#what-grading-does-not-defend-against)) - **Parallel** -- tokio orchestrator, grades 80+ students in seconds - **Parametrize + oracle** -- random inputs with teacher reference implementations - **Per-item scoring** -- declared points and aggregation; zero and withheld kept apart in every export diff --git a/notes b/notes index 785624f..4d626ef 160000 --- a/notes +++ b/notes @@ -1 +1 @@ -Subproject commit 785624fda41ce4229b2ad8cedd6c354853af5200 +Subproject commit 4d626efd4257b744c2c39f2db9a2b890bda60b13 From 9165cc9c34bcf8e3656b658d1e6147f487d60032 Mon Sep 17 00:00:00 2001 From: Acture Date: Thu, 15 Oct 2026 20:01:27 +0800 Subject: [PATCH 11/20] feat: declare a unit's memory with memory_mb, 256 MiB by default (OSS-10) A case, a scenario (its steps share one process) or `[meta]` may declare `memory_mb`, 64 to 65536; the narrowest applies, and an unset one stays out of the spec's digest. The student side is held to it by the grader's watch of its resident memory, and the message of a unit ended for it names the limit. The default drops from 512 to 256 MiB: Python with the harness takes about 24 MiB, a list of a million integers about 62. On Linux RLIMIT_AS becomes a backstop at four times the limit and never under 2 GiB, for both sides: address space is not memory, and a library such as OpenBLAS reserves far more than it touches. --- src/core/src/models/spec.rs | 14 +++++++ src/core/src/runner/executor.rs | 6 ++- src/core/src/runner/judge.rs | 13 +++--- src/core/src/runner/oracle.rs | 4 ++ src/core/src/runner/prepare.rs | 10 ++++- src/core/src/runner/python.rs | 71 ++++++++++++++++++++++----------- src/core/src/runner/sandbox.rs | 8 ++-- src/core/src/spec_loader.rs | 57 ++++++++++++++++++++++++++ src/core/tests/harness.rs | 6 ++- src/core/tests/integration.rs | 33 +++++++++++++++ src/core/tests/sandbox.rs | 1 + 11 files changed, 184 insertions(+), 39 deletions(-) diff --git a/src/core/src/models/spec.rs b/src/core/src/models/spec.rs index 44e6327..9258154 100644 --- a/src/core/src/models/spec.rs +++ b/src/core/src/models/spec.rs @@ -495,6 +495,11 @@ pub struct TestCase { #[serde(default)] pub timeout: Option, + /// The most memory the case's student process may hold, in MiB. Not on a scenario + /// step: its steps share their scenario's process. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory_mb: Option, + /// Generate concrete cases from this one. #[serde(default)] pub parametrize: Option, @@ -575,6 +580,11 @@ pub struct TestMeta { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub shared: Vec, + /// The most memory a unit's student process may hold, in MiB, where a case or scenario + /// does not say: `DEFAULT_MEMORY_MB` when neither does. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory_mb: Option, + /// Refused: every case now has its own process. #[serde(default)] pub copy_refs: Option, @@ -649,6 +659,10 @@ pub struct Scenario { #[serde(default)] pub timeout: Option, + /// The most memory the scenario's student process may hold, in MiB. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory_mb: Option, + /// Runs once, at the start of the scenario, after the top-level setup. #[serde(default)] pub setup: Vec, diff --git a/src/core/src/runner/executor.rs b/src/core/src/runner/executor.rs index 1e1e2ab..61001a2 100644 --- a/src/core/src/runner/executor.rs +++ b/src/core/src/runner/executor.rs @@ -59,6 +59,8 @@ pub struct UnitPlan { /// Shared modules, as `(source in the bundle, relative destination)`: staged beside the /// student's file, and imported by the teacher side from the bundle. pub shared: Vec<(PathBuf, String)>, + /// The most memory the student side may hold, in MiB, before it is ended. + pub memory_mb: u64, pub allowed_imports: Vec, pub load_timeout: u64, pub setup: Vec, @@ -270,8 +272,8 @@ pub enum Exit { Signal(i32), /// Killed by the grader at the unit's deadline. Deadline, - /// Killed by the grader for going past the unit's memory limit. - Memory, + /// Killed by the grader for going past the unit's memory limit, in MiB. + Memory(u64), /// Still running when the teacher side finished, and ended by the grader then. Stopped, /// Never started. diff --git a/src/core/src/runner/judge.rs b/src/core/src/runner/judge.rs index 6dd0404..c472ac0 100644 --- a/src/core/src/runner/judge.rs +++ b/src/core/src/runner/judge.rs @@ -384,7 +384,7 @@ fn in_flight(exit: &Exit, fault: Fault, cause: Cause, what: &str) -> Blanket { TestStatus::Error }, fault, - cause: if *exit == Exit::Memory { + cause: if matches!(exit, Exit::Memory(_)) { Cause::Memory } else { cause @@ -465,9 +465,9 @@ fn death(exit: &Exit) -> (TestStatus, String) { TestStatus::Timeout, "the unit's deadline passed during".into(), ), - Exit::Memory => ( + Exit::Memory(limit) => ( TestStatus::Error, - "the memory limit was exceeded during".into(), + format!("the {limit} MiB memory limit (`memory_mb`) was exceeded during"), ), Exit::Stopped => ( TestStatus::Error, @@ -964,6 +964,7 @@ mod tests { data_files: Vec::new(), bundle: std::path::PathBuf::new(), shared: Vec::new(), + memory_mb: 256, allowed_imports: Vec::new(), load_timeout: 10, setup: Vec::new(), @@ -1307,7 +1308,7 @@ mod tests { (TestStatus::Error, Some(Fault::Student), Some(Cause::Killed)) ); - obs.exit = Exit::Memory; + obs.exit = Exit::Memory(256); let r = judge(&p, &scored, &obs, "python3", Isolation::Sandboxed); assert_eq!( verdict(&r[1]), @@ -1432,7 +1433,7 @@ mod tests { reason: StopReason::Exited, message: String::new(), }); - hog.exit = Exit::Memory; + hog.exit = Exit::Memory(256); let r = one(&c, &p, &hog); assert_eq!( verdict(&r), @@ -1471,7 +1472,7 @@ mod tests { reason: StopReason::Exited, message: String::new(), }); - gone.exit = Exit::Memory; + gone.exit = Exit::Memory(256); let r = judge(&p, &scored, &gone, "python3", Isolation::Sandboxed); assert_eq!( verdict(&r[0]), diff --git a/src/core/src/runner/oracle.rs b/src/core/src/runner/oracle.rs index 2ba2ca5..ea52f47 100644 --- a/src/core/src/runner/oracle.rs +++ b/src/core/src/runner/oracle.rs @@ -44,6 +44,10 @@ pub async fn resolve_oracle( .map(|rel| (spec.dir.join(rel), rel.clone())) .collect(), allowed_imports: spec.meta.allowed_imports.clone(), + memory_mb: case + .memory_mb + .or(spec.meta.memory_mb) + .unwrap_or(crate::spec_loader::DEFAULT_MEMORY_MB), load_timeout: timeout_secs, setup: Vec::new(), steps: vec![CallPlan { diff --git a/src/core/src/runner/prepare.rs b/src/core/src/runner/prepare.rs index ac39de4..89f3cf6 100644 --- a/src/core/src/runner/prepare.rs +++ b/src/core/src/runner/prepare.rs @@ -26,7 +26,9 @@ use crate::runner::frozen::{Generation, replay_entry}; use crate::runner::generation::{DrawSeed, Generated, generate, seed_for}; use crate::runner::judge::Scored; use crate::runner::oracle::{resolve_oracle, validate_references}; -use crate::spec_loader::{MAX_TIMEOUT_SECS, refs, reserved, validate, validate_expanded}; +use crate::spec_loader::{ + DEFAULT_MEMORY_MB, MAX_TIMEOUT_SECS, refs, reserved, validate, validate_expanded, +}; /// A prepared test bundle: one spec, ready to run against any student. #[derive(Debug, Serialize)] @@ -653,6 +655,7 @@ fn plan_units(spec: &TestSpec, timeout_secs: u64) -> Vec { .map(|rel| (spec.dir.join(rel), rel.clone())) .collect(), allowed_imports: spec.meta.allowed_imports.clone(), + memory_mb: spec.meta.memory_mb.unwrap_or(DEFAULT_MEMORY_MB), load_timeout: timeout_secs, setup: Vec::new(), steps: Vec::new(), @@ -665,6 +668,10 @@ fn plan_units(spec: &TestSpec, timeout_secs: u64) -> Vec { .collect(); let cases = spec.cases.iter().map(|case| { + let base = UnitPlan { + memory_mb: case.memory_mb.unwrap_or(base.memory_mb), + ..base.clone() + }; let plan = if case.script { UnitPlan { script: Some(ScriptRun { @@ -690,6 +697,7 @@ fn plan_units(spec: &TestSpec, timeout_secs: u64) -> Vec { let timeout = scenario.timeout.unwrap_or(timeout_secs); Unit { plan: UnitPlan { + memory_mb: scenario.memory_mb.unwrap_or(base.memory_mb), setup: top_setup .iter() .cloned() diff --git a/src/core/src/runner/python.rs b/src/core/src/runner/python.rs index 738d9a3..9fb75dc 100644 --- a/src/core/src/runner/python.rs +++ b/src/core/src/runner/python.rs @@ -50,7 +50,6 @@ fn resolve_python_path(cmd: &str) -> String { pub struct PythonExecutor { python_cmd: String, /// The most memory the student side may hold before it is ended, in MiB. - memory_mb: u64, isolation: Isolation, } @@ -62,17 +61,10 @@ impl PythonExecutor { pub fn with_python_cmd(python_cmd: impl Into) -> Self { Self { python_cmd: resolve_python_path(&python_cmd.into()), - memory_mb: crate::runner::sandbox::SandboxConfig::default().mem_mb, isolation: Isolation::default(), } } - /// End the student side once it holds more than `mb` MiB. - pub fn with_memory_limit(mut self, mb: u64) -> Self { - self.memory_mb = mb; - self - } - /// Whether each side runs in its own sandbox. Sandboxed is the default; a unit whose /// sandbox cannot be applied is not graded. pub fn with_isolation(mut self, isolation: Isolation) -> Self { @@ -452,6 +444,7 @@ impl PythonExecutor { script: &str, side: &Side, deadline_secs: u64, + memory_mb: u64, confinement: Option<&Confinement>, ) -> Result { let mut cmd = match confinement { @@ -480,10 +473,13 @@ impl PythonExecutor { { // Its own process group, so anything it spawns dies with it. cmd.process_group(0); - apply_sandbox(&mut cmd, &sandbox_for(deadline_secs, confinement.is_some())); + apply_sandbox( + &mut cmd, + &sandbox_for(deadline_secs, memory_mb, confinement.is_some()), + ); } #[cfg(not(unix))] - let _ = deadline_secs; + let _ = (deadline_secs, memory_mb); Ok(cmd) } @@ -524,11 +520,13 @@ impl PythonExecutor { } /// Run a staged unit — the teacher side, and the student side when `with_student` — - /// and end both at `deadline_secs`. + /// end both at `deadline_secs`, and the student side past `memory_mb` MiB. + #[allow(clippy::too_many_arguments)] async fn run_unit( &self, staged: &Staged, deadline_secs: u64, + memory_mb: u64, bounds: Bounds, with_student: bool, confinements: Option<[Confinement; 2]>, @@ -541,6 +539,7 @@ impl PythonExecutor { &TEACHER_SCRIPT, &staged.teacher, deadline_secs, + memory_mb, teacher_confinement.as_ref(), ) { Ok(cmd) => cmd, @@ -561,6 +560,7 @@ impl PythonExecutor { &STUDENT_SCRIPT, &staged.student, deadline_secs, + memory_mb, student_confinement.as_ref(), ) { Ok(cmd) => cmd, @@ -652,7 +652,7 @@ impl PythonExecutor { pumps.spawn(pump(pipe, student_tail.clone())); } if let Some(pid) = child.id() { - let limit = self.memory_mb.saturating_mul(1 << 20); + let limit = memory_mb.saturating_mul(1 << 20); let over = over_memory.clone(); let group = student_group.0; watch = Some(tokio::spawn(watch_memory(pid, group, limit, over))); @@ -676,10 +676,10 @@ impl PythonExecutor { let ended = tokio::time::timeout(linger, child.wait()).await; let over = || over_memory.load(Ordering::SeqCst); match ended { - Ok(Ok(_)) if over() => Exit::Memory, + Ok(Ok(_)) if over() => Exit::Memory(memory_mb), Ok(Ok(status)) => exit_of(status), Ok(Err(e)) => Exit::Spawn(format!("lost the student side: {e}")), - Err(_) if over() => Exit::Memory, + Err(_) if over() => Exit::Memory(memory_mb), Err(_) if teacher_exit == Exit::Deadline => Exit::Deadline, Err(_) => Exit::Stopped, } @@ -834,14 +834,22 @@ fn resident_bytes(_pid: u32) -> Option { } /// The limits for a unit: its CPU limit sits above its deadline, a backstop for one busy -/// core rather than a second, shorter timeout. Confined, it cannot start processes, so the -/// per-user process limit — which counts threads on Linux — only stops a thread bomb, and -/// leaves a busy machine's own threads room. +/// core rather than a second, shorter timeout. Its address space (Linux) sits far above +/// the memory the grader's watch holds it to: a backstop for an allocation faster than the +/// watch, which a library reserving address space it never uses (OpenBLAS) must not hit. +/// Confined, it cannot start processes, so the per-user process limit — which counts +/// threads on Linux — only stops a thread bomb, and leaves a busy machine's own threads +/// room. #[cfg(unix)] -fn sandbox_for(deadline_secs: u64, confined: bool) -> crate::runner::sandbox::SandboxConfig { +fn sandbox_for( + deadline_secs: u64, + memory_mb: u64, + confined: bool, +) -> crate::runner::sandbox::SandboxConfig { let defaults = crate::runner::sandbox::SandboxConfig::default(); crate::runner::sandbox::SandboxConfig { cpu_secs: deadline_secs.saturating_add(1), + mem_mb: memory_mb.saturating_mul(4).max(defaults.mem_mb), nproc: if confined { Some(crate::runner::sandbox::CONFINED_NPROC) } else { @@ -1052,8 +1060,17 @@ impl Executor for PythonExecutor { Vec::new(), ) { Ok(confinements) => { - self.run_unit(&staged, deadline, INSPECT_BOUNDS, false, confinements) - .await + self.run_unit( + &staged, + deadline, + spec.meta + .memory_mb + .unwrap_or(crate::spec_loader::DEFAULT_MEMORY_MB), + INSPECT_BOUNDS, + false, + confinements, + ) + .await } Err(e) => Finished::spawn_failed(e), }; @@ -1154,6 +1171,7 @@ impl Executor for PythonExecutor { self.run_unit( &staged, plan.deadline_secs(), + plan.memory_mb, record_bounds(plan), true, confinements, @@ -1199,14 +1217,18 @@ mod tests { #[test] fn test_the_cpu_limit_sits_above_any_units_deadline() { for deadline in [3, 45, 4 * 10 + 2, 86_400 * 4] { - assert!(sandbox_for(deadline, true).cpu_secs > deadline); + assert!(sandbox_for(deadline, 256, true).cpu_secs > deadline); } - assert_eq!(sandbox_for(u64::MAX, true).cpu_secs, u64::MAX, "saturates"); assert_eq!( - sandbox_for(3, true).nproc, + sandbox_for(u64::MAX, 256, true).cpu_secs, + u64::MAX, + "saturates" + ); + assert_eq!( + sandbox_for(3, 256, true).nproc, Some(crate::runner::sandbox::CONFINED_NPROC) ); - assert_eq!(sandbox_for(3, false).nproc, Some(64)); + assert_eq!(sandbox_for(3, 256, false).nproc, Some(64)); } /// Each record is bounded by the largest a call can make, and the stream by the number @@ -1233,6 +1255,7 @@ mod tests { data_files: Vec::new(), bundle: PathBuf::new(), shared: Vec::new(), + memory_mb: 256, allowed_imports: Vec::new(), load_timeout: 1, setup: Vec::new(), diff --git a/src/core/src/runner/sandbox.rs b/src/core/src/runner/sandbox.rs index 84815cf..d263a32 100644 --- a/src/core/src/runner/sandbox.rs +++ b/src/core/src/runner/sandbox.rs @@ -19,8 +19,8 @@ use serde::{Deserialize, Serialize}; pub struct SandboxConfig { /// CPU time limit in seconds (RLIMIT_CPU). Default: 30. pub cpu_secs: u64, - /// Memory limit in MB: RLIMIT_AS on Linux, and the grader's watch on the student side - /// everywhere. Default: 512. + /// Address-space limit in MB (RLIMIT_AS, Linux only): a backstop far above the memory a + /// unit is held to by the grader's watch of its resident memory. Default: 2048. pub mem_mb: u64, /// Max file write size in MB (RLIMIT_FSIZE). Default: 10. pub fsize_mb: u64, @@ -42,7 +42,7 @@ impl Default for SandboxConfig { fn default() -> Self { Self { cpu_secs: 30, - mem_mb: 512, + mem_mb: 2048, fsize_mb: 10, nofile: 64, nproc: Some(64), @@ -714,7 +714,7 @@ mod tests { fn test_default_config() { let config = SandboxConfig::default(); assert_eq!(config.cpu_secs, 30); - assert_eq!(config.mem_mb, 512); + assert_eq!(config.mem_mb, 2048); assert_eq!(config.fsize_mb, 10); assert_eq!(config.nofile, 64); assert_eq!(config.nproc, Some(64)); diff --git a/src/core/src/spec_loader.rs b/src/core/src/spec_loader.rs index 1e61f40..42b6831 100644 --- a/src/core/src/spec_loader.rs +++ b/src/core/src/spec_loader.rs @@ -132,6 +132,15 @@ pub fn reserved(name: &str) -> Option<&'static str> { /// The longest timeout a spec or the command line may ask for. pub const MAX_TIMEOUT_SECS: u64 = 86_400; +/// The memory a unit's student process may hold when its spec does not say, in MiB: +/// several times what an introductory exercise needs (Python itself takes about 25 MiB, a +/// list of a million integers about 40 more). +pub const DEFAULT_MEMORY_MB: u64 = 256; + +/// The memory a spec may give a unit, in MiB: from what Python and the harness need to +/// start, to 64 GiB. +pub const MEMORY_MB: std::ops::RangeInclusive = 64..=65_536; + /// What the harness keeps of a call's stdout; an expectation longer than this can never match. pub const STDOUT_LIMIT: usize = 64 * 1024; @@ -237,6 +246,7 @@ impl<'a> Validator<'a> { } } self.shared(); + self.memory(self.spec.meta.memory_mb, "[meta]"); for (name, value) in &self.spec.vars { if contains_null(value) { self.problem( @@ -358,6 +368,19 @@ impl<'a> Validator<'a> { } } + fn memory(&mut self, memory: Option, at: &str) { + if memory.is_some_and(|m| !MEMORY_MB.contains(&m)) { + self.problem( + at, + format!( + "memory_mb must be between {} and {} MiB", + MEMORY_MB.start(), + MEMORY_MB.end() + ), + ); + } + } + fn unique_name(&mut self, names: &mut BTreeSet, name: String) { if !names.insert(name.clone()) { self.problem( @@ -376,6 +399,7 @@ impl<'a> Validator<'a> { self.problem(&at, "has no steps"); } self.timeout(scenario.timeout, &at); + self.memory(scenario.memory_mb, &at); let mut bound = top.clone(); for step in &scenario.setup { self.setup(step, &mut bound, &format!("{at} setup '{}'", step.id)); @@ -475,6 +499,14 @@ impl<'a> Validator<'a> { ); } self.timeout(case.timeout, at); + if is_step && case.memory_mb.is_some() { + self.problem( + at, + "memory_mb belongs to the scenario: its steps share one process", + ); + } else { + self.memory(case.memory_mb, at); + } if case.args.iter().any(contains_null) { self.problem( at, @@ -1947,6 +1979,31 @@ expect = 150 } } + #[test] + fn test_memory_is_declared_per_process_within_bounds() { + refused( + "[[cases]]\nname = \"x\"\nexpect = 1\nmemory_mb = 16\n", + "memory_mb must be between 64 and 65536 MiB", + ); + refused( + "[[scenarios]]\nname = \"s\"\n[[scenarios.steps]]\nname = \"a\"\nexpect = 1\nmemory_mb = 512\n", + "memory_mb belongs to the scenario", + ); + assert!( + problems( + "[[scenarios]]\nname = \"s\"\nmemory_mb = 512\n[[scenarios.steps]]\nname = \"a\"\nexpect = 1\n" + ) + .is_empty() + ); + // Unset, it is not part of what a spec's digest covers: older records still match. + let spec = load_spec_str( + &format!("{META}[[cases]]\nname = \"x\"\nexpect = 1\n"), + Path::new("."), + ) + .unwrap(); + assert!(!serde_json::to_string(&spec).unwrap().contains("memory_mb")); + } + #[test] fn test_source_is_reserved_like_stdout() { refused( diff --git a/src/core/tests/harness.rs b/src/core/tests/harness.rs index 39fe44a..44162c2 100644 --- a/src/core/tests/harness.rs +++ b/src/core/tests/harness.rs @@ -54,6 +54,7 @@ fn unit(file: &Path) -> UnitPlan { data_files: Vec::new(), bundle: file.parent().unwrap().to_path_buf(), shared: Vec::new(), + memory_mb: 256, allowed_imports: Vec::new(), load_timeout: 5, setup: Vec::new(), @@ -1030,8 +1031,9 @@ async fn test_a_student_side_past_its_memory_limit_is_ended() { ); let mut plan = unit(&student); plan.steps = vec![function("hog", vec![]), function("hog", vec![])]; - let obs = PythonExecutor::new().with_memory_limit(96).run(&plan).await; - assert_eq!(obs.exit, Exit::Memory, "{obs:?}"); + plan.memory_mb = 96; + let obs = run(&plan).await; + assert_eq!(obs.exit, Exit::Memory(96), "{obs:?}"); assert!(obs.steps.is_empty()); assert_eq!( obs.stopped.as_ref().map(|s| s.reason), diff --git a/src/core/tests/integration.rs b/src/core/tests/integration.rs index d6b8677..7d6b0a5 100644 --- a/src/core/tests/integration.rs +++ b/src/core/tests/integration.rs @@ -1465,3 +1465,36 @@ async fn test_a_path_into_the_bundle_is_never_handed_to_student_code() { let results = grade(vec![spec], &[student]).await; assert_eq!(results[0].total_passed(), 1, "{:#?}", results[0]); } + +/// A unit's student process is held to `memory_mb`: 256 MiB unless the spec, the case or +/// the scenario says otherwise, and a case that needs more declares it. +#[tokio::test] +async fn test_memory_is_the_default_unless_declared() { + let bench = Bench::new(); + let spec = |meta: &str, case: &str| { + bench.spec(&format!( + "[meta]\nname = \"m\"\nfile = \"lab.py\"\nfunction = \"big\"\nlanguage = \"python\"\n{meta}\n[[cases]]\nname = \"big\"\nexpect = 8000000\n{case}\n" + )) + }; + let student = bench.student( + "alice", + "lab.py", + "def big():\n return len(list(range(8_000_000)))\n", + ); + let tight = grade(vec![spec("", "")], std::slice::from_ref(&student)).await; + let c = &tight[0].test_results[0].cases[0]; + assert_eq!( + (c.status, c.fault, c.cause), + (TestStatus::Error, Some(Fault::Student), Some(Cause::Memory)), + "{c:?}" + ); + assert!( + c.failure.as_ref().unwrap().message.contains("256 MiB"), + "{:?}", + c.failure + ); + for (meta, case) in [("memory_mb = 1024", ""), ("", "memory_mb = 1024")] { + let roomy = grade(vec![spec(meta, case)], std::slice::from_ref(&student)).await; + assert_eq!(roomy[0].total_passed(), 1, "{meta}{case}: {:#?}", roomy[0]); + } +} diff --git a/src/core/tests/sandbox.rs b/src/core/tests/sandbox.rs index c09e974..49cc34d 100644 --- a/src/core/tests/sandbox.rs +++ b/src/core/tests/sandbox.rs @@ -180,6 +180,7 @@ impl Bench { data_files: Vec::new(), bundle: self.bundle.clone(), shared: Vec::new(), + memory_mb: 256, allowed_imports: Vec::new(), load_timeout: 5, setup: Vec::new(), From 7a8a40606b1b376aa43eff971413463640291abf Mon Sep 17 00:00:00 2001 From: Acture Date: Thu, 15 Oct 2026 20:01:58 +0800 Subject: [PATCH 12/20] docs: describe memory_mb (OSS-10) --- CLAUDE.md | 2 +- notes | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1d7ab41..72f78b1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,7 +96,7 @@ cli/src/tui/ ratatui terminal UI: students/sessions/similarity tabs **No pytest.** ScriptMark is its own test engine. A unit (a case or a scenario) runs as two Python processes from embedded scripts (`runner/harness/`): the teacher side (`teacher.py`: teacher modules, function checkers, the plan, per-request deadlines, the records) and the student side (`student.py`: the student module only, one call per request, under its timer and import guard). They share `common.py` (framing, the record and transfer value encodings, `call()`). ScriptMark joins the two with a pair of pipes and reads length-prefixed JSON records from the teacher side; values cross by value (shared-class instances rebuilt from the receiver's copy, other objects as snapshots). -**Sandboxed execution.** Each side runs confined by default (`runner/sandbox.rs`): macOS Seatbelt via `sandbox-exec` with a generated profile, Linux Landlock + seccomp applied in `pre_exec`. The student side reads its interpreter and its own directory only; the teacher side also reads the bundle and a read-only source copy. Fail closed: no sandbox, no grading, unless `--no-sandbox` (recorded as `isolation: unisolated`). Both also get `env_clear()`, `setrlimit`, a memory watch on the student side, and an absolute interpreter path. The import allowlist is a teaching rule, not a boundary. Sandbox tests need a real sandbox: run `cargo test` outside any other Seatbelt sandbox (Claude Code's included), and on Linux from a filesystem with stable inodes (not a Docker Desktop bind mount). +**Sandboxed execution.** Each side runs confined by default (`runner/sandbox.rs`): macOS Seatbelt via `sandbox-exec` with a generated profile, Linux Landlock + seccomp applied in `pre_exec`. The student side reads its interpreter and its own directory only; the teacher side also reads the bundle and a read-only source copy. Fail closed: no sandbox, no grading, unless `--no-sandbox` (recorded as `isolation: unisolated`). Both also get `env_clear()`, `setrlimit` (RLIMIT_AS only as a backstop), and an absolute interpreter path; the student side is watched against `memory_mb` (spec, case or scenario; default 256 MiB). The import allowlist is a teaching rule, not a boundary. Sandbox tests need a real sandbox: run `cargo test` outside any other Seatbelt sandbox (Claude Code's included), and on Linux from a filesystem with stable inodes (not a Docker Desktop bind mount). **TOML test specs** have three sections: `[vars]` (constants injected as Python globals), `[[setup]]` (function calls storing results for `$ref`), `[[cases]]` (scored test cases). Cases support `expect`, `expect_error`, checkers (`check = "sorted"` / `{ rhai = "..." }` / `{ python = "verifiers/check.py" }` / `{ function = "..." }`, which may take the reserved `stdout` and `source`), and `[cases.parametrize]` with oracle. `[meta]` supports `allowed_imports` (a teaching rule) and `shared` (modules whose classes cross between the two sides by value). Prepare refuses a path into the bundle handed to student code: `data_files` stages it instead. diff --git a/notes b/notes index 4d626ef..f2a4f03 160000 --- a/notes +++ b/notes @@ -1 +1 @@ -Subproject commit 4d626efd4257b744c2c39f2db9a2b890bda60b13 +Subproject commit f2a4f03bea079f06ea73973b6b23b1f4de2e2b58 From 45c6f84d050b651bdca14a3f650d7402f7d70d17 Mon Sep 17 00:00:00 2001 From: Acture Date: Thu, 15 Oct 2026 20:37:40 +0800 Subject: [PATCH 13/20] feat: default memory_mb to 128 MiB (OSS-10) The student side takes about 25 MiB idle (Python alone: 15 MiB on macOS, 7 MiB on Linux). The watch also counts the harness's copies of a returned value, a few times its size: at 64 MiB a list of a million integers is ended even unchecked, at 96 MiB when handed to a checker; at 128 MiB only 100,000 dictionaries handed to a checker are. Test plans use the default; the 1.5-million-integer checker test declares 512. The two tests whose student defeats its timer by catching the timer's exception escaped it when the alarm landed in the handler, now and then under load. Their student now loops in C, where the timer never fires. --- CLAUDE.md | 2 +- notes | 2 +- src/core/src/runner/judge.rs | 2 +- src/core/src/runner/python.rs | 2 +- src/core/src/spec_loader.rs | 9 +++++---- src/core/tests/harness.rs | 7 +++++-- src/core/tests/integration.rs | 8 +++++--- src/core/tests/sandbox.rs | 2 +- 8 files changed, 20 insertions(+), 14 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 72f78b1..7b62c0a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,7 +96,7 @@ cli/src/tui/ ratatui terminal UI: students/sessions/similarity tabs **No pytest.** ScriptMark is its own test engine. A unit (a case or a scenario) runs as two Python processes from embedded scripts (`runner/harness/`): the teacher side (`teacher.py`: teacher modules, function checkers, the plan, per-request deadlines, the records) and the student side (`student.py`: the student module only, one call per request, under its timer and import guard). They share `common.py` (framing, the record and transfer value encodings, `call()`). ScriptMark joins the two with a pair of pipes and reads length-prefixed JSON records from the teacher side; values cross by value (shared-class instances rebuilt from the receiver's copy, other objects as snapshots). -**Sandboxed execution.** Each side runs confined by default (`runner/sandbox.rs`): macOS Seatbelt via `sandbox-exec` with a generated profile, Linux Landlock + seccomp applied in `pre_exec`. The student side reads its interpreter and its own directory only; the teacher side also reads the bundle and a read-only source copy. Fail closed: no sandbox, no grading, unless `--no-sandbox` (recorded as `isolation: unisolated`). Both also get `env_clear()`, `setrlimit` (RLIMIT_AS only as a backstop), and an absolute interpreter path; the student side is watched against `memory_mb` (spec, case or scenario; default 256 MiB). The import allowlist is a teaching rule, not a boundary. Sandbox tests need a real sandbox: run `cargo test` outside any other Seatbelt sandbox (Claude Code's included), and on Linux from a filesystem with stable inodes (not a Docker Desktop bind mount). +**Sandboxed execution.** Each side runs confined by default (`runner/sandbox.rs`): macOS Seatbelt via `sandbox-exec` with a generated profile, Linux Landlock + seccomp applied in `pre_exec`. The student side reads its interpreter and its own directory only; the teacher side also reads the bundle and a read-only source copy. Fail closed: no sandbox, no grading, unless `--no-sandbox` (recorded as `isolation: unisolated`). Both also get `env_clear()`, `setrlimit` (RLIMIT_AS only as a backstop), and an absolute interpreter path; the student side is watched against `memory_mb` (spec, case or scenario; default 128 MiB). The import allowlist is a teaching rule, not a boundary. Sandbox tests need a real sandbox: run `cargo test` outside any other Seatbelt sandbox (Claude Code's included), and on Linux from a filesystem with stable inodes (not a Docker Desktop bind mount). **TOML test specs** have three sections: `[vars]` (constants injected as Python globals), `[[setup]]` (function calls storing results for `$ref`), `[[cases]]` (scored test cases). Cases support `expect`, `expect_error`, checkers (`check = "sorted"` / `{ rhai = "..." }` / `{ python = "verifiers/check.py" }` / `{ function = "..." }`, which may take the reserved `stdout` and `source`), and `[cases.parametrize]` with oracle. `[meta]` supports `allowed_imports` (a teaching rule) and `shared` (modules whose classes cross between the two sides by value). Prepare refuses a path into the bundle handed to student code: `data_files` stages it instead. diff --git a/notes b/notes index f2a4f03..8d80e03 160000 --- a/notes +++ b/notes @@ -1 +1 @@ -Subproject commit f2a4f03bea079f06ea73973b6b23b1f4de2e2b58 +Subproject commit 8d80e0350242c93d9b6142eb4654c451bd8ec857 diff --git a/src/core/src/runner/judge.rs b/src/core/src/runner/judge.rs index c472ac0..75e6c65 100644 --- a/src/core/src/runner/judge.rs +++ b/src/core/src/runner/judge.rs @@ -964,7 +964,7 @@ mod tests { data_files: Vec::new(), bundle: std::path::PathBuf::new(), shared: Vec::new(), - memory_mb: 256, + memory_mb: crate::spec_loader::DEFAULT_MEMORY_MB, allowed_imports: Vec::new(), load_timeout: 10, setup: Vec::new(), diff --git a/src/core/src/runner/python.rs b/src/core/src/runner/python.rs index 9fb75dc..5245810 100644 --- a/src/core/src/runner/python.rs +++ b/src/core/src/runner/python.rs @@ -1255,7 +1255,7 @@ mod tests { data_files: Vec::new(), bundle: PathBuf::new(), shared: Vec::new(), - memory_mb: 256, + memory_mb: crate::spec_loader::DEFAULT_MEMORY_MB, allowed_imports: Vec::new(), load_timeout: 1, setup: Vec::new(), diff --git a/src/core/src/spec_loader.rs b/src/core/src/spec_loader.rs index 42b6831..a89d2b2 100644 --- a/src/core/src/spec_loader.rs +++ b/src/core/src/spec_loader.rs @@ -132,10 +132,11 @@ pub fn reserved(name: &str) -> Option<&'static str> { /// The longest timeout a spec or the command line may ask for. pub const MAX_TIMEOUT_SECS: u64 = 86_400; -/// The memory a unit's student process may hold when its spec does not say, in MiB: -/// several times what an introductory exercise needs (Python itself takes about 25 MiB, a -/// list of a million integers about 40 more). -pub const DEFAULT_MEMORY_MB: u64 = 256; +/// The memory a unit's student process may hold when its spec does not say, in MiB: five +/// times what it takes idle (about 25 MiB), and room to return a list of a million +/// integers (about 40 MiB) to a function checker. The limit also counts the harness's +/// copies of a returned value while it encodes it, a few times the value's own size. +pub const DEFAULT_MEMORY_MB: u64 = 128; /// The memory a spec may give a unit, in MiB: from what Python and the harness need to /// start, to 64 GiB. diff --git a/src/core/tests/harness.rs b/src/core/tests/harness.rs index 44162c2..aa43b7e 100644 --- a/src/core/tests/harness.rs +++ b/src/core/tests/harness.rs @@ -54,7 +54,7 @@ fn unit(file: &Path) -> UnitPlan { data_files: Vec::new(), bundle: file.parent().unwrap().to_path_buf(), shared: Vec::new(), - memory_mb: 256, + memory_mb: scriptmark_core::spec_loader::DEFAULT_MEMORY_MB, allowed_imports: Vec::new(), load_timeout: 5, setup: Vec::new(), @@ -146,7 +146,8 @@ async fn test_a_call_that_never_yields_is_given_up_on_and_nothing_after_it_runs( let student = write( dir.path(), "lab.py", - "def forever():\n while True:\n try:\n while True:\n pass\n except BaseException:\n pass\n\ndef ok():\n return 1\n", + // Never returns to the interpreter, where the timer would fire. + "import itertools\n\ndef forever():\n return sum(itertools.count())\n\ndef ok():\n return 1\n", ); let mut plan = unit(&student); plan.load_timeout = 1; @@ -671,6 +672,8 @@ async fn test_serialisation_edges_found_in_rereview() { ); let mut plan = unit(&student); plan.imports = vec![teacher.to_string_lossy().into_owned()]; + // 1.5 million integers, and the harness's copies of them for the checker. + plan.memory_mb = 512; let mut big = function("big_list", vec![]); big.check = Some(InProcessCheck { function: "length".into(), diff --git a/src/core/tests/integration.rs b/src/core/tests/integration.rs index 7d6b0a5..e3a4353 100644 --- a/src/core/tests/integration.rs +++ b/src/core/tests/integration.rs @@ -1377,7 +1377,9 @@ expect = 1 let student = bench.student( "alice", "lab.py", - "def ok():\n return 1\n\ndef forever():\n while True:\n try:\n while True:\n pass\n except BaseException:\n pass\n", + // A loop that never returns to the interpreter, so the timer never fires in it. One + // that catches the timer's exception escapes when the timer lands in its handler. + "import itertools\n\ndef ok():\n return 1\n\ndef forever():\n return sum(itertools.count())\n", ); let results = grade_with(vec![spec], &[student], PythonExecutor::new(), 1).await; let r = &results[0]; @@ -1466,7 +1468,7 @@ async fn test_a_path_into_the_bundle_is_never_handed_to_student_code() { assert_eq!(results[0].total_passed(), 1, "{:#?}", results[0]); } -/// A unit's student process is held to `memory_mb`: 256 MiB unless the spec, the case or +/// A unit's student process is held to `memory_mb`: 128 MiB unless the spec, the case or /// the scenario says otherwise, and a case that needs more declares it. #[tokio::test] async fn test_memory_is_the_default_unless_declared() { @@ -1489,7 +1491,7 @@ async fn test_memory_is_the_default_unless_declared() { "{c:?}" ); assert!( - c.failure.as_ref().unwrap().message.contains("256 MiB"), + c.failure.as_ref().unwrap().message.contains("128 MiB"), "{:?}", c.failure ); diff --git a/src/core/tests/sandbox.rs b/src/core/tests/sandbox.rs index 49cc34d..c0f4141 100644 --- a/src/core/tests/sandbox.rs +++ b/src/core/tests/sandbox.rs @@ -180,7 +180,7 @@ impl Bench { data_files: Vec::new(), bundle: self.bundle.clone(), shared: Vec::new(), - memory_mb: 256, + memory_mb: scriptmark_core::spec_loader::DEFAULT_MEMORY_MB, allowed_imports: Vec::new(), load_timeout: 5, setup: Vec::new(), From cfc23481724a7407d4033d295cec629df18a5507 Mon Sep 17 00:00:00 2001 From: Acture Date: Thu, 15 Oct 2026 21:51:00 +0800 Subject: [PATCH 14/20] feat: count memory_mb beyond what the interpreter takes idle (OSS-10) A teacher sizes memory_mb from the algorithm; what Python and the harness take idle is ScriptMark's to add. It is measured once per interpreter and isolation by the same watch that enforces the limit, on a unit whose student is ScriptMark's own and sleeps while it is sampled: 24 MiB on macOS, 16 MiB on Linux. The student process cannot report it itself: student code could forge whatever it says. A failed measurement is not kept; the unit does not start, and the next measures again. Platforms without a watch add nothing. The default drops to 64 MiB and the floor to 16. Exit::Memory carries both numbers, and the message names them: "the 64 MiB memory limit (`memory_mb`, beyond the interpreter's 24 MiB)". --- CLAUDE.md | 2 +- notes | 2 +- src/core/src/models/spec.rs | 11 +- src/core/src/runner/executor.rs | 11 +- src/core/src/runner/judge.rs | 27 ++++- src/core/src/runner/python.rs | 207 ++++++++++++++++++++++++++++---- src/core/src/spec_loader.rs | 20 +-- src/core/tests/harness.rs | 5 +- src/core/tests/integration.rs | 10 +- 9 files changed, 242 insertions(+), 53 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7b62c0a..88f7e1b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,7 +96,7 @@ cli/src/tui/ ratatui terminal UI: students/sessions/similarity tabs **No pytest.** ScriptMark is its own test engine. A unit (a case or a scenario) runs as two Python processes from embedded scripts (`runner/harness/`): the teacher side (`teacher.py`: teacher modules, function checkers, the plan, per-request deadlines, the records) and the student side (`student.py`: the student module only, one call per request, under its timer and import guard). They share `common.py` (framing, the record and transfer value encodings, `call()`). ScriptMark joins the two with a pair of pipes and reads length-prefixed JSON records from the teacher side; values cross by value (shared-class instances rebuilt from the receiver's copy, other objects as snapshots). -**Sandboxed execution.** Each side runs confined by default (`runner/sandbox.rs`): macOS Seatbelt via `sandbox-exec` with a generated profile, Linux Landlock + seccomp applied in `pre_exec`. The student side reads its interpreter and its own directory only; the teacher side also reads the bundle and a read-only source copy. Fail closed: no sandbox, no grading, unless `--no-sandbox` (recorded as `isolation: unisolated`). Both also get `env_clear()`, `setrlimit` (RLIMIT_AS only as a backstop), and an absolute interpreter path; the student side is watched against `memory_mb` (spec, case or scenario; default 128 MiB). The import allowlist is a teaching rule, not a boundary. Sandbox tests need a real sandbox: run `cargo test` outside any other Seatbelt sandbox (Claude Code's included), and on Linux from a filesystem with stable inodes (not a Docker Desktop bind mount). +**Sandboxed execution.** Each side runs confined by default (`runner/sandbox.rs`): macOS Seatbelt via `sandbox-exec` with a generated profile, Linux Landlock + seccomp applied in `pre_exec`. The student side reads its interpreter and its own directory only; the teacher side also reads the bundle and a read-only source copy. Fail closed: no sandbox, no grading, unless `--no-sandbox` (recorded as `isolation: unisolated`). Both also get `env_clear()`, `setrlimit` (RLIMIT_AS only as a backstop), and an absolute interpreter path; the student side is watched against `memory_mb` beyond what it takes idle (spec, case or scenario; default 64 MiB; the idle amount is measured once per interpreter and isolation). The import allowlist is a teaching rule, not a boundary. Sandbox tests need a real sandbox: run `cargo test` outside any other Seatbelt sandbox (Claude Code's included), and on Linux from a filesystem with stable inodes (not a Docker Desktop bind mount). **TOML test specs** have three sections: `[vars]` (constants injected as Python globals), `[[setup]]` (function calls storing results for `$ref`), `[[cases]]` (scored test cases). Cases support `expect`, `expect_error`, checkers (`check = "sorted"` / `{ rhai = "..." }` / `{ python = "verifiers/check.py" }` / `{ function = "..." }`, which may take the reserved `stdout` and `source`), and `[cases.parametrize]` with oracle. `[meta]` supports `allowed_imports` (a teaching rule) and `shared` (modules whose classes cross between the two sides by value). Prepare refuses a path into the bundle handed to student code: `data_files` stages it instead. diff --git a/notes b/notes index 8d80e03..0051f73 160000 --- a/notes +++ b/notes @@ -1 +1 @@ -Subproject commit 8d80e0350242c93d9b6142eb4654c451bd8ec857 +Subproject commit 0051f73140630b576daeecad11d9899b74a26bc2 diff --git a/src/core/src/models/spec.rs b/src/core/src/models/spec.rs index 9258154..7ba97be 100644 --- a/src/core/src/models/spec.rs +++ b/src/core/src/models/spec.rs @@ -495,8 +495,8 @@ pub struct TestCase { #[serde(default)] pub timeout: Option, - /// The most memory the case's student process may hold, in MiB. Not on a scenario - /// step: its steps share their scenario's process. + /// The most memory the case's student process may hold beyond what it takes idle, in + /// MiB. Not on a scenario step: its steps share their scenario's process. #[serde(default, skip_serializing_if = "Option::is_none")] pub memory_mb: Option, @@ -580,8 +580,8 @@ pub struct TestMeta { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub shared: Vec, - /// The most memory a unit's student process may hold, in MiB, where a case or scenario - /// does not say: `DEFAULT_MEMORY_MB` when neither does. + /// The most memory a unit's student process may hold beyond what it takes idle, in MiB, + /// where a case or scenario does not say: `DEFAULT_MEMORY_MB` when neither does. #[serde(default, skip_serializing_if = "Option::is_none")] pub memory_mb: Option, @@ -659,7 +659,8 @@ pub struct Scenario { #[serde(default)] pub timeout: Option, - /// The most memory the scenario's student process may hold, in MiB. + /// The most memory the scenario's student process may hold beyond what it takes idle, + /// in MiB. #[serde(default, skip_serializing_if = "Option::is_none")] pub memory_mb: Option, diff --git a/src/core/src/runner/executor.rs b/src/core/src/runner/executor.rs index 61001a2..1227c48 100644 --- a/src/core/src/runner/executor.rs +++ b/src/core/src/runner/executor.rs @@ -59,7 +59,8 @@ pub struct UnitPlan { /// Shared modules, as `(source in the bundle, relative destination)`: staged beside the /// student's file, and imported by the teacher side from the bundle. pub shared: Vec<(PathBuf, String)>, - /// The most memory the student side may hold, in MiB, before it is ended. + /// The most memory the student side may hold beyond what it takes idle, in MiB, before + /// it is ended. pub memory_mb: u64, pub allowed_imports: Vec, pub load_timeout: u64, @@ -272,8 +273,12 @@ pub enum Exit { Signal(i32), /// Killed by the grader at the unit's deadline. Deadline, - /// Killed by the grader for going past the unit's memory limit, in MiB. - Memory(u64), + /// Killed by the grader for holding more than the unit's `memory_mb` beyond what the + /// student side takes idle (`interpreter_mb`). + Memory { + limit_mb: u64, + interpreter_mb: u64, + }, /// Still running when the teacher side finished, and ended by the grader then. Stopped, /// Never started. diff --git a/src/core/src/runner/judge.rs b/src/core/src/runner/judge.rs index 75e6c65..bec2ffb 100644 --- a/src/core/src/runner/judge.rs +++ b/src/core/src/runner/judge.rs @@ -384,7 +384,7 @@ fn in_flight(exit: &Exit, fault: Fault, cause: Cause, what: &str) -> Blanket { TestStatus::Error }, fault, - cause: if matches!(exit, Exit::Memory(_)) { + cause: if matches!(exit, Exit::Memory { .. }) { Cause::Memory } else { cause @@ -465,9 +465,15 @@ fn death(exit: &Exit) -> (TestStatus, String) { TestStatus::Timeout, "the unit's deadline passed during".into(), ), - Exit::Memory(limit) => ( + Exit::Memory { + limit_mb, + interpreter_mb, + } => ( TestStatus::Error, - format!("the {limit} MiB memory limit (`memory_mb`) was exceeded during"), + format!( + "the {limit_mb} MiB memory limit (`memory_mb`, beyond the interpreter's \ + {interpreter_mb} MiB) was exceeded during" + ), ), Exit::Stopped => ( TestStatus::Error, @@ -1308,7 +1314,10 @@ mod tests { (TestStatus::Error, Some(Fault::Student), Some(Cause::Killed)) ); - obs.exit = Exit::Memory(256); + obs.exit = Exit::Memory { + limit_mb: 64, + interpreter_mb: 24, + }; let r = judge(&p, &scored, &obs, "python3", Isolation::Sandboxed); assert_eq!( verdict(&r[1]), @@ -1433,7 +1442,10 @@ mod tests { reason: StopReason::Exited, message: String::new(), }); - hog.exit = Exit::Memory(256); + hog.exit = Exit::Memory { + limit_mb: 64, + interpreter_mb: 24, + }; let r = one(&c, &p, &hog); assert_eq!( verdict(&r), @@ -1472,7 +1484,10 @@ mod tests { reason: StopReason::Exited, message: String::new(), }); - gone.exit = Exit::Memory(256); + gone.exit = Exit::Memory { + limit_mb: 64, + interpreter_mb: 24, + }; let r = judge(&p, &scored, &gone, "python3", Isolation::Sandboxed); assert_eq!( verdict(&r[0]), diff --git a/src/core/src/runner/python.rs b/src/core/src/runner/python.rs index 5245810..18670ab 100644 --- a/src/core/src/runner/python.rs +++ b/src/core/src/runner/python.rs @@ -5,7 +5,7 @@ //! teacher side's records; it never relays what passes between them. use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, LazyLock, Mutex}; use std::time::Duration; @@ -423,6 +423,8 @@ struct Finished { exit: Exit, teacher_exit: Exit, stderr: String, + /// The most the student side was seen holding, in bytes. + peak: u64, } impl Finished { @@ -432,10 +434,53 @@ impl Finished { exit: Exit::Spawn(message.clone()), teacher_exit: Exit::Spawn(message), stderr: String::new(), + peak: 0, } } } +/// What the student side may hold: `limit_mb` beyond `interpreter`, the bytes it takes idle. +#[derive(Debug, Clone, Copy)] +struct Memory { + limit_mb: u64, + interpreter: u64, +} + +impl Memory { + fn watched(self) -> u64 { + self.interpreter + .saturating_add(self.limit_mb.saturating_mul(1 << 20)) + } + + fn total_mb(self) -> u64 { + self.watched().div_ceil(1 << 20) + } + + fn exceeded(self) -> Exit { + Exit::Memory { + limit_mb: self.limit_mb, + interpreter_mb: self.interpreter.div_ceil(1 << 20), + } + } +} + +/// An interpreter, and whether it runs sandboxed. +#[cfg(any(target_os = "macos", target_os = "linux"))] +type IdleKey = (String, bool); +#[cfg(any(target_os = "macos", target_os = "linux"))] +type Idle = tokio::sync::OnceCell; + +/// What the student side takes idle — Python, the harness and a loaded module — per +/// interpreter and isolation, measured once by the instrument that enforces the limit. +/// A failed measurement is not kept: the next unit measures again. +#[cfg(any(target_os = "macos", target_os = "linux"))] +static IDLE_MEMORY: LazyLock>>> = + LazyLock::new(Default::default); + +/// The student whose idle memory is measured: it waits while the watch samples it. +#[cfg(any(target_os = "macos", target_os = "linux"))] +const IDLE_SUBJECT: &str = "import time\n\ndef idle():\n time.sleep(0.3)\n"; + impl PythonExecutor { /// A command for one side: confined when sandboxed, with a cleared environment, the /// side's own `HOME` and `TMPDIR`, and the kernel's limits. @@ -447,6 +492,7 @@ impl PythonExecutor { memory_mb: u64, confinement: Option<&Confinement>, ) -> Result { + // `memory_mb`: all the side may hold, interpreter included. let mut cmd = match confinement { Some(c) => Command::from( crate::runner::sandbox::confined(Path::new(&self.python_cmd), c) @@ -520,13 +566,13 @@ impl PythonExecutor { } /// Run a staged unit — the teacher side, and the student side when `with_student` — - /// end both at `deadline_secs`, and the student side past `memory_mb` MiB. + /// end both at `deadline_secs`, and the student side past `memory`. #[allow(clippy::too_many_arguments)] async fn run_unit( &self, staged: &Staged, deadline_secs: u64, - memory_mb: u64, + memory: Memory, bounds: Bounds, with_student: bool, confinements: Option<[Confinement; 2]>, @@ -539,7 +585,7 @@ impl PythonExecutor { &TEACHER_SCRIPT, &staged.teacher, deadline_secs, - memory_mb, + memory.total_mb(), teacher_confinement.as_ref(), ) { Ok(cmd) => cmd, @@ -560,7 +606,7 @@ impl PythonExecutor { &STUDENT_SCRIPT, &staged.student, deadline_secs, - memory_mb, + memory.total_mb(), student_confinement.as_ref(), ) { Ok(cmd) => cmd, @@ -646,16 +692,22 @@ impl PythonExecutor { pumps.spawn(pump(pipe, teacher_tail.clone())); } let over_memory = Arc::new(AtomicBool::new(false)); + let peak = Arc::new(AtomicU64::new(0)); let mut watch = None; if let Some(child) = student.as_mut() { if let Some(pipe) = child.stderr.take() { pumps.spawn(pump(pipe, student_tail.clone())); } if let Some(pid) = child.id() { - let limit = memory_mb.saturating_mul(1 << 20); - let over = over_memory.clone(); + let (over, held) = (over_memory.clone(), peak.clone()); let group = student_group.0; - watch = Some(tokio::spawn(watch_memory(pid, group, limit, over))); + watch = Some(tokio::spawn(watch_memory( + pid, + group, + memory.watched(), + over, + held, + ))); } } @@ -676,10 +728,10 @@ impl PythonExecutor { let ended = tokio::time::timeout(linger, child.wait()).await; let over = || over_memory.load(Ordering::SeqCst); match ended { - Ok(Ok(_)) if over() => Exit::Memory(memory_mb), + Ok(Ok(_)) if over() => memory.exceeded(), Ok(Ok(status)) => exit_of(status), Ok(Err(e)) => Exit::Spawn(format!("lost the student side: {e}")), - Err(_) if over() => Exit::Memory(memory_mb), + Err(_) if over() => memory.exceeded(), Err(_) if teacher_exit == Exit::Deadline => Exit::Deadline, Err(_) => Exit::Stopped, } @@ -731,6 +783,7 @@ impl PythonExecutor { exit, teacher_exit, stderr, + peak: peak.load(Ordering::SeqCst), } } } @@ -787,9 +840,16 @@ async fn pump_records(mut pipe: R, reader: Arc< /// End the student side once it holds more than `limit` bytes: the operating system's /// sandbox cannot bound its memory everywhere, so the grader watches it. -async fn watch_memory(pid: u32, group: Option, limit: u64, over: Arc) { +async fn watch_memory( + pid: u32, + group: Option, + limit: u64, + over: Arc, + peak: Arc, +) { let mut tick = tokio::time::interval(Duration::from_millis(50)); while let Some(held) = resident_bytes(pid) { + peak.fetch_max(held, Ordering::SeqCst); if held > limit { over.store(true, Ordering::SeqCst); if let Some(group) = group { @@ -1063,9 +1123,13 @@ impl Executor for PythonExecutor { self.run_unit( &staged, deadline, - spec.meta - .memory_mb - .unwrap_or(crate::spec_loader::DEFAULT_MEMORY_MB), + Memory { + limit_mb: spec + .meta + .memory_mb + .unwrap_or(crate::spec_loader::DEFAULT_MEMORY_MB), + interpreter: 0, + }, INSPECT_BOUNDS, false, confinements, @@ -1099,6 +1163,104 @@ impl Executor for PythonExecutor { } async fn run(&self, plan: &UnitPlan) -> UnitObservation { + let interpreter = match self.idle_memory().await { + Ok(bytes) => bytes, + Err(e) => { + let mut obs = UnitObservation::not_started(Exit::Spawn(e)); + obs.sandboxed = self.isolation.is_sandboxed(); + return obs; + } + }; + self.observe( + plan, + Memory { + limit_mb: plan.memory_mb, + interpreter, + }, + ) + .await + .0 + } +} + +impl PythonExecutor { + /// What the student side takes idle, in bytes: what `memory_mb` is counted beyond. + #[cfg(any(target_os = "macos", target_os = "linux"))] + async fn idle_memory(&self) -> Result { + let cell = IDLE_MEMORY + .lock() + .expect("idle memory cache poisoned") + .entry((self.python_cmd.clone(), self.isolation.is_sandboxed())) + .or_default() + .clone(); + cell.get_or_try_init(|| self.measure_idle_memory()) + .await + .copied() + } + + /// Nothing to count it by where the student side's memory is not watched. + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + async fn idle_memory(&self) -> Result { + Ok(0) + } + + #[cfg(any(target_os = "macos", target_os = "linux"))] + async fn measure_idle_memory(&self) -> Result { + let dir = tempfile::tempdir().map_err(|e| format!("could not stage the idle unit: {e}"))?; + let file = dir.path().join("idle.py"); + std::fs::write(&file, IDLE_SUBJECT) + .map_err(|e| format!("could not stage the idle unit: {e}"))?; + let plan = UnitPlan { + subject: Subject::Student, + functions: crate::matching::Functions::default(), + file, + script: None, + imports: Vec::new(), + vars: Arc::default(), + data_files: Vec::new(), + bundle: dir.path().to_path_buf(), + shared: Vec::new(), + memory_mb: *crate::spec_loader::MEMORY_MB.end(), + allowed_imports: Vec::new(), + load_timeout: STARTUP_SECS, + setup: Vec::new(), + steps: vec![crate::runner::executor::CallPlan { + target: crate::models::Target::Function { + name: "idle".into(), + }, + args: Vec::new(), + stdin: None, + timeout: STARTUP_SECS, + id: None, + files: Vec::new(), + check: None, + }], + }; + let memory = Memory { + limit_mb: plan.memory_mb, + interpreter: 0, + }; + let (obs, peak) = self.observe(&plan, memory).await; + if let Exit::Spawn(e) = &obs.exit { + return Err(e.clone()); + } + let idled = obs.done + && obs.steps.first().is_some_and(|s| { + matches!(s.outcome, crate::runner::executor::Outcome::Returned { .. }) + }); + if !idled || peak == 0 { + return Err(format!( + "could not measure what Python takes idle ({:?}): {}", + obs.exit, + obs.stderr.trim() + )); + } + Ok(peak) + } + + /// Run `plan` with the student side held to `memory`: what it reported, and the most + /// the student side was seen holding. + async fn observe(&self, plan: &UnitPlan, memory: Memory) -> (UnitObservation, u64) { let data = plan.data_files.clone(); let shared = plan.shared.clone(); let file = plan.file.clone(); @@ -1143,14 +1305,12 @@ impl Executor for PythonExecutor { { Ok(Ok(staged)) => staged, Ok(Err(e)) => { - return UnitObservation::not_started(Exit::Spawn(format!( - "could not stage the unit: {e}" - ))); + let exit = Exit::Spawn(format!("could not stage the unit: {e}")); + return (UnitObservation::not_started(exit), 0); } Err(e) => { - return UnitObservation::not_started(Exit::Spawn(format!( - "staging panicked: {e}" - ))); + let exit = Exit::Spawn(format!("staging panicked: {e}")); + return (UnitObservation::not_started(exit), 0); } }; debug_assert!(staged.subject.as_ref().is_some_and(|s| s.is_file())); @@ -1171,7 +1331,7 @@ impl Executor for PythonExecutor { self.run_unit( &staged, plan.deadline_secs(), - plan.memory_mb, + memory, record_bounds(plan), true, confinements, @@ -1183,7 +1343,7 @@ impl Executor for PythonExecutor { discard(staged).await; let records = finished.records; - UnitObservation { + let obs = UnitObservation { ready: records.ready, load: records.load, setup: records.setup, @@ -1197,7 +1357,8 @@ impl Executor for PythonExecutor { protocol_error: records.protocol_error, stderr: finished.stderr, sandboxed: self.isolation.is_sandboxed(), - } + }; + (obs, finished.peak) } } diff --git a/src/core/src/spec_loader.rs b/src/core/src/spec_loader.rs index a89d2b2..ea6c373 100644 --- a/src/core/src/spec_loader.rs +++ b/src/core/src/spec_loader.rs @@ -132,15 +132,15 @@ pub fn reserved(name: &str) -> Option<&'static str> { /// The longest timeout a spec or the command line may ask for. pub const MAX_TIMEOUT_SECS: u64 = 86_400; -/// The memory a unit's student process may hold when its spec does not say, in MiB: five -/// times what it takes idle (about 25 MiB), and room to return a list of a million -/// integers (about 40 MiB) to a function checker. The limit also counts the harness's -/// copies of a returned value while it encodes it, a few times the value's own size. -pub const DEFAULT_MEMORY_MB: u64 = 128; +/// The memory a unit's student process may hold beyond what it takes idle, when its spec +/// does not say, in MiB: what the student's code holds, and the harness's copies of what +/// it returns while encoding them — a few times the value's own size. What Python and the +/// harness take idle is measured, and added. +pub const DEFAULT_MEMORY_MB: u64 = 64; -/// The memory a spec may give a unit, in MiB: from what Python and the harness need to -/// start, to 64 GiB. -pub const MEMORY_MB: std::ops::RangeInclusive = 64..=65_536; +/// The memory a spec may give a unit beyond what it takes idle, in MiB: from what a small +/// exercise holds, above the noise of loading a module, to 64 GiB. +pub const MEMORY_MB: std::ops::RangeInclusive = 16..=65_536; /// What the harness keeps of a call's stdout; an expectation longer than this can never match. pub const STDOUT_LIMIT: usize = 64 * 1024; @@ -1983,8 +1983,8 @@ expect = 150 #[test] fn test_memory_is_declared_per_process_within_bounds() { refused( - "[[cases]]\nname = \"x\"\nexpect = 1\nmemory_mb = 16\n", - "memory_mb must be between 64 and 65536 MiB", + "[[cases]]\nname = \"x\"\nexpect = 1\nmemory_mb = 8\n", + "memory_mb must be between 16 and 65536 MiB", ); refused( "[[scenarios]]\nname = \"s\"\n[[scenarios.steps]]\nname = \"a\"\nexpect = 1\nmemory_mb = 512\n", diff --git a/src/core/tests/harness.rs b/src/core/tests/harness.rs index aa43b7e..f243a6a 100644 --- a/src/core/tests/harness.rs +++ b/src/core/tests/harness.rs @@ -1036,7 +1036,10 @@ async fn test_a_student_side_past_its_memory_limit_is_ended() { plan.steps = vec![function("hog", vec![]), function("hog", vec![])]; plan.memory_mb = 96; let obs = run(&plan).await; - assert_eq!(obs.exit, Exit::Memory(96), "{obs:?}"); + assert!( + matches!(obs.exit, Exit::Memory { limit_mb: 96, interpreter_mb } if interpreter_mb > 0), + "{obs:?}" + ); assert!(obs.steps.is_empty()); assert_eq!( obs.stopped.as_ref().map(|s| s.reason), diff --git a/src/core/tests/integration.rs b/src/core/tests/integration.rs index e3a4353..23f5659 100644 --- a/src/core/tests/integration.rs +++ b/src/core/tests/integration.rs @@ -1468,8 +1468,8 @@ async fn test_a_path_into_the_bundle_is_never_handed_to_student_code() { assert_eq!(results[0].total_passed(), 1, "{:#?}", results[0]); } -/// A unit's student process is held to `memory_mb`: 128 MiB unless the spec, the case or -/// the scenario says otherwise, and a case that needs more declares it. +/// A unit's student process is held to `memory_mb` beyond what it takes idle: 64 MiB unless +/// the spec, the case or the scenario says otherwise, and a case that needs more declares it. #[tokio::test] async fn test_memory_is_the_default_unless_declared() { let bench = Bench::new(); @@ -1491,7 +1491,11 @@ async fn test_memory_is_the_default_unless_declared() { "{c:?}" ); assert!( - c.failure.as_ref().unwrap().message.contains("128 MiB"), + c.failure + .as_ref() + .unwrap() + .message + .contains("64 MiB memory limit"), "{:?}", c.failure ); From 26c478e5a1bf123db286f5c39ec704d1358015b6 Mon Sep 17 00:00:00 2001 From: Acture Date: Thu, 15 Oct 2026 22:28:48 +0800 Subject: [PATCH 15/20] feat: count allowed imports as what a unit takes idle (OSS-10) A library the teacher allows is environment, not the algorithm's: import pandas alone takes more than the 64 MiB default. The idle unit now imports each module in the spec's allowed_imports, skipping one that does not load, and the measurement is kept per interpreter, isolation and set of allowed imports: 120 MiB on macOS with pandas and matplotlib.pyplot, against 24 with nothing allowed. An entry may be dotted: matplotlib.pyplot allows the matplotlib package, as matplotlib did, and names the module to load (matplotlib alone is 46 MiB, with pyplot 77). Entries must be module names; the idle student takes them as data. Exit::Memory's idle_mb and its message say what the idle share covers. --- CLAUDE.md | 2 +- notes | 2 +- src/core/src/models/spec.rs | 5 +- src/core/src/runner/executor.rs | 4 +- src/core/src/runner/harness/student.py | 2 +- src/core/src/runner/judge.rs | 15 ++-- src/core/src/runner/python.rs | 95 +++++++++++++++++++------- src/core/src/spec_loader.rs | 36 ++++++++++ src/core/tests/harness.rs | 49 ++++++++++++- 9 files changed, 169 insertions(+), 41 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 88f7e1b..af89501 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,7 +96,7 @@ cli/src/tui/ ratatui terminal UI: students/sessions/similarity tabs **No pytest.** ScriptMark is its own test engine. A unit (a case or a scenario) runs as two Python processes from embedded scripts (`runner/harness/`): the teacher side (`teacher.py`: teacher modules, function checkers, the plan, per-request deadlines, the records) and the student side (`student.py`: the student module only, one call per request, under its timer and import guard). They share `common.py` (framing, the record and transfer value encodings, `call()`). ScriptMark joins the two with a pair of pipes and reads length-prefixed JSON records from the teacher side; values cross by value (shared-class instances rebuilt from the receiver's copy, other objects as snapshots). -**Sandboxed execution.** Each side runs confined by default (`runner/sandbox.rs`): macOS Seatbelt via `sandbox-exec` with a generated profile, Linux Landlock + seccomp applied in `pre_exec`. The student side reads its interpreter and its own directory only; the teacher side also reads the bundle and a read-only source copy. Fail closed: no sandbox, no grading, unless `--no-sandbox` (recorded as `isolation: unisolated`). Both also get `env_clear()`, `setrlimit` (RLIMIT_AS only as a backstop), and an absolute interpreter path; the student side is watched against `memory_mb` beyond what it takes idle (spec, case or scenario; default 64 MiB; the idle amount is measured once per interpreter and isolation). The import allowlist is a teaching rule, not a boundary. Sandbox tests need a real sandbox: run `cargo test` outside any other Seatbelt sandbox (Claude Code's included), and on Linux from a filesystem with stable inodes (not a Docker Desktop bind mount). +**Sandboxed execution.** Each side runs confined by default (`runner/sandbox.rs`): macOS Seatbelt via `sandbox-exec` with a generated profile, Linux Landlock + seccomp applied in `pre_exec`. The student side reads its interpreter and its own directory only; the teacher side also reads the bundle and a read-only source copy. Fail closed: no sandbox, no grading, unless `--no-sandbox` (recorded as `isolation: unisolated`). Both also get `env_clear()`, `setrlimit` (RLIMIT_AS only as a backstop), and an absolute interpreter path; the student side is watched against `memory_mb` beyond what it takes idle with the spec's `allowed_imports` loaded (spec, case or scenario; default 64 MiB; the idle amount is measured once per interpreter, isolation and set of allowed imports). The import allowlist is a teaching rule, not a boundary. Sandbox tests need a real sandbox: run `cargo test` outside any other Seatbelt sandbox (Claude Code's included), and on Linux from a filesystem with stable inodes (not a Docker Desktop bind mount). **TOML test specs** have three sections: `[vars]` (constants injected as Python globals), `[[setup]]` (function calls storing results for `$ref`), `[[cases]]` (scored test cases). Cases support `expect`, `expect_error`, checkers (`check = "sorted"` / `{ rhai = "..." }` / `{ python = "verifiers/check.py" }` / `{ function = "..." }`, which may take the reserved `stdout` and `source`), and `[cases.parametrize]` with oracle. `[meta]` supports `allowed_imports` (a teaching rule) and `shared` (modules whose classes cross between the two sides by value). Prepare refuses a path into the bundle handed to student code: `data_files` stages it instead. diff --git a/notes b/notes index 0051f73..3f93e29 160000 --- a/notes +++ b/notes @@ -1 +1 @@ -Subproject commit 0051f73140630b576daeecad11d9899b74a26bc2 +Subproject commit 3f93e2954bd08b93ad66b0b8d0046390a27f6253 diff --git a/src/core/src/models/spec.rs b/src/core/src/models/spec.rs index 7ba97be..cf12801 100644 --- a/src/core/src/models/spec.rs +++ b/src/core/src/models/spec.rs @@ -589,7 +589,10 @@ pub struct TestMeta { #[serde(default)] pub copy_refs: Option, - /// Extra packages allowed in student code (beyond safe stdlib). + /// Extra packages allowed in student code (beyond safe stdlib). An entry allows its + /// top-level package; a dotted one (`matplotlib.pyplot`) also names the module loaded + /// when ScriptMark measures what the student side takes idle, which `memory_mb` is + /// counted beyond. #[serde(default)] pub allowed_imports: Vec, } diff --git a/src/core/src/runner/executor.rs b/src/core/src/runner/executor.rs index 1227c48..86c1ba1 100644 --- a/src/core/src/runner/executor.rs +++ b/src/core/src/runner/executor.rs @@ -274,10 +274,10 @@ pub enum Exit { /// Killed by the grader at the unit's deadline. Deadline, /// Killed by the grader for holding more than the unit's `memory_mb` beyond what the - /// student side takes idle (`interpreter_mb`). + /// student side takes idle with its allowed imports loaded (`idle_mb`). Memory { limit_mb: u64, - interpreter_mb: u64, + idle_mb: u64, }, /// Still running when the teacher side finished, and ended by the grader then. Stopped, diff --git a/src/core/src/runner/harness/student.py b/src/core/src/runner/harness/student.py index 077f908..8f7ff40 100644 --- a/src/core/src/runner/harness/student.py +++ b/src/core/src/runner/harness/student.py @@ -74,7 +74,7 @@ def reply(message): "gzip", "bz2", "lzma", "zipfile", "tarfile", "xml", "html", "email", "logging", "argparse", "getopt", "configparser", "tomllib", "threading", "concurrent", } -) | frozenset(INIT.get("allowed_imports", [])) +) | frozenset(name.split(".")[0] for name in INIT.get("allowed_imports", [])) STUDENT_MARK = "__scriptmark_student__" _original_import = builtins.__import__ diff --git a/src/core/src/runner/judge.rs b/src/core/src/runner/judge.rs index bec2ffb..e6ad31e 100644 --- a/src/core/src/runner/judge.rs +++ b/src/core/src/runner/judge.rs @@ -465,14 +465,11 @@ fn death(exit: &Exit) -> (TestStatus, String) { TestStatus::Timeout, "the unit's deadline passed during".into(), ), - Exit::Memory { - limit_mb, - interpreter_mb, - } => ( + Exit::Memory { limit_mb, idle_mb } => ( TestStatus::Error, format!( - "the {limit_mb} MiB memory limit (`memory_mb`, beyond the interpreter's \ - {interpreter_mb} MiB) was exceeded during" + "the {limit_mb} MiB memory limit (`memory_mb`, on top of the {idle_mb} MiB \ + Python and the allowed imports take) was exceeded during" ), ), Exit::Stopped => ( @@ -1316,7 +1313,7 @@ mod tests { obs.exit = Exit::Memory { limit_mb: 64, - interpreter_mb: 24, + idle_mb: 24, }; let r = judge(&p, &scored, &obs, "python3", Isolation::Sandboxed); assert_eq!( @@ -1444,7 +1441,7 @@ mod tests { }); hog.exit = Exit::Memory { limit_mb: 64, - interpreter_mb: 24, + idle_mb: 24, }; let r = one(&c, &p, &hog); assert_eq!( @@ -1486,7 +1483,7 @@ mod tests { }); gone.exit = Exit::Memory { limit_mb: 64, - interpreter_mb: 24, + idle_mb: 24, }; let r = judge(&p, &scored, &gone, "python3", Isolation::Sandboxed); assert_eq!( diff --git a/src/core/src/runner/python.rs b/src/core/src/runner/python.rs index 18670ab..e0f7f35 100644 --- a/src/core/src/runner/python.rs +++ b/src/core/src/runner/python.rs @@ -439,16 +439,16 @@ impl Finished { } } -/// What the student side may hold: `limit_mb` beyond `interpreter`, the bytes it takes idle. +/// What the student side may hold: `limit_mb` beyond `idle`, the bytes it takes idle. #[derive(Debug, Clone, Copy)] struct Memory { limit_mb: u64, - interpreter: u64, + idle: u64, } impl Memory { fn watched(self) -> u64 { - self.interpreter + self.idle .saturating_add(self.limit_mb.saturating_mul(1 << 20)) } @@ -459,27 +459,36 @@ impl Memory { fn exceeded(self) -> Exit { Exit::Memory { limit_mb: self.limit_mb, - interpreter_mb: self.interpreter.div_ceil(1 << 20), + idle_mb: self.idle.div_ceil(1 << 20), } } } -/// An interpreter, and whether it runs sandboxed. +/// An interpreter, whether it runs sandboxed, and the allowed imports loaded idle. #[cfg(any(target_os = "macos", target_os = "linux"))] -type IdleKey = (String, bool); +type IdleKey = (String, bool, Vec); #[cfg(any(target_os = "macos", target_os = "linux"))] type Idle = tokio::sync::OnceCell; -/// What the student side takes idle — Python, the harness and a loaded module — per -/// interpreter and isolation, measured once by the instrument that enforces the limit. -/// A failed measurement is not kept: the next unit measures again. +/// What the student side takes idle — Python, the harness, the allowed imports and a loaded +/// module — per interpreter, isolation and set of allowed imports, measured once by the +/// instrument that enforces the limit. A failed measurement is not kept: the next unit +/// measures again. #[cfg(any(target_os = "macos", target_os = "linux"))] static IDLE_MEMORY: LazyLock>>> = LazyLock::new(Default::default); -/// The student whose idle memory is measured: it waits while the watch samples it. +/// The student whose idle memory is measured: it imports what the teacher allows, as a +/// student would, and waits while the watch samples it. A library that cannot load is +/// skipped: it costs a student nothing either. #[cfg(any(target_os = "macos", target_os = "linux"))] -const IDLE_SUBJECT: &str = "import time\n\ndef idle():\n time.sleep(0.3)\n"; +fn idle_subject(imports: &[String]) -> String { + let names = serde_json::to_string(imports).expect("module names serialise"); + format!( + "import time\n\nfor _name in {names}:\n try:\n __import__(_name)\n \ + except Exception:\n pass\n\n\ndef idle():\n time.sleep(0.3)\n" + ) +} impl PythonExecutor { /// A command for one side: confined when sandboxed, with a cleared environment, the @@ -1128,7 +1137,7 @@ impl Executor for PythonExecutor { .meta .memory_mb .unwrap_or(crate::spec_loader::DEFAULT_MEMORY_MB), - interpreter: 0, + idle: 0, }, INSPECT_BOUNDS, false, @@ -1163,7 +1172,10 @@ impl Executor for PythonExecutor { } async fn run(&self, plan: &UnitPlan) -> UnitObservation { - let interpreter = match self.idle_memory().await { + let idle = match self + .idle_memory(&plan.allowed_imports, plan.load_timeout) + .await + { Ok(bytes) => bytes, Err(e) => { let mut obs = UnitObservation::not_started(Exit::Spawn(e)); @@ -1175,7 +1187,7 @@ impl Executor for PythonExecutor { plan, Memory { limit_mb: plan.memory_mb, - interpreter, + idle, }, ) .await @@ -1184,31 +1196,53 @@ impl Executor for PythonExecutor { } impl PythonExecutor { - /// What the student side takes idle, in bytes: what `memory_mb` is counted beyond. + /// What the student side takes idle with `allowed_imports` loaded, in bytes: what + /// `memory_mb` is counted beyond. #[cfg(any(target_os = "macos", target_os = "linux"))] - async fn idle_memory(&self) -> Result { + async fn idle_memory( + &self, + allowed_imports: &[String], + load_timeout: u64, + ) -> Result { + let mut imports = allowed_imports.to_vec(); + imports.sort(); + imports.dedup(); + if let Some(name) = imports + .iter() + .find(|n| !crate::spec_loader::is_module_name(n)) + { + return Err(format!("allowed import '{name}' is not a module name")); + } let cell = IDLE_MEMORY .lock() .expect("idle memory cache poisoned") - .entry((self.python_cmd.clone(), self.isolation.is_sandboxed())) + .entry(( + self.python_cmd.clone(), + self.isolation.is_sandboxed(), + imports.clone(), + )) .or_default() .clone(); - cell.get_or_try_init(|| self.measure_idle_memory()) + cell.get_or_try_init(|| self.measure_idle_memory(imports, load_timeout)) .await .copied() } /// Nothing to count it by where the student side's memory is not watched. #[cfg(not(any(target_os = "macos", target_os = "linux")))] - async fn idle_memory(&self) -> Result { + async fn idle_memory(&self, _: &[String], _: u64) -> Result { Ok(0) } #[cfg(any(target_os = "macos", target_os = "linux"))] - async fn measure_idle_memory(&self) -> Result { + async fn measure_idle_memory( + &self, + imports: Vec, + load_timeout: u64, + ) -> Result { let dir = tempfile::tempdir().map_err(|e| format!("could not stage the idle unit: {e}"))?; let file = dir.path().join("idle.py"); - std::fs::write(&file, IDLE_SUBJECT) + std::fs::write(&file, idle_subject(&imports)) .map_err(|e| format!("could not stage the idle unit: {e}"))?; let plan = UnitPlan { subject: Subject::Student, @@ -1221,8 +1255,8 @@ impl PythonExecutor { bundle: dir.path().to_path_buf(), shared: Vec::new(), memory_mb: *crate::spec_loader::MEMORY_MB.end(), - allowed_imports: Vec::new(), - load_timeout: STARTUP_SECS, + allowed_imports: imports.clone(), + load_timeout: load_timeout.max(STARTUP_SECS), setup: Vec::new(), steps: vec![crate::runner::executor::CallPlan { target: crate::models::Target::Function { @@ -1238,7 +1272,7 @@ impl PythonExecutor { }; let memory = Memory { limit_mb: plan.memory_mb, - interpreter: 0, + idle: 0, }; let (obs, peak) = self.observe(&plan, memory).await; if let Exit::Spawn(e) = &obs.exit { @@ -1250,7 +1284,7 @@ impl PythonExecutor { }); if !idled || peak == 0 { return Err(format!( - "could not measure what Python takes idle ({:?}): {}", + "could not measure what Python takes idle with {imports:?} ({:?}): {}", obs.exit, obs.stderr.trim() )); @@ -1373,6 +1407,17 @@ mod tests { use super::*; use crate::runner::executor::CallPlan; + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn test_the_idle_student_imports_each_allowed_module_by_name() { + assert_eq!( + idle_subject(&["matplotlib.pyplot".into(), "numpy".into()]), + "import time\n\nfor _name in [\"matplotlib.pyplot\",\"numpy\"]:\n try:\n \ + __import__(_name)\n except Exception:\n pass\n\n\ndef idle():\n \ + time.sleep(0.3)\n" + ); + } + /// The CPU limit is derived from the unit's own deadline — never the fixed 30 s it used /// to be, which a scenario of several long steps would outrun and be killed by. #[test] diff --git a/src/core/src/spec_loader.rs b/src/core/src/spec_loader.rs index ea6c373..d50ed12 100644 --- a/src/core/src/spec_loader.rs +++ b/src/core/src/spec_loader.rs @@ -247,6 +247,14 @@ impl<'a> Validator<'a> { } } self.shared(); + for name in &self.spec.meta.allowed_imports { + if !is_module_name(name) { + let message = format!( + "allowed import '{name}' must be a module name, such as 'numpy' or 'matplotlib.pyplot'" + ); + self.problem("[meta]", message); + } + } self.memory(self.spec.meta.memory_mb, "[meta]"); for (name, value) in &self.spec.vars { if contains_null(value) { @@ -1073,6 +1081,11 @@ fn is_contained(path: &str) -> bool { } /// A name a Rhai oracle can bind: letters, digits and `_`, not starting with a digit. +/// A module Python can import by this name: `numpy`, `matplotlib.pyplot`. +pub fn is_module_name(name: &str) -> bool { + name.split('.').all(is_identifier) +} + fn is_identifier(name: &str) -> bool { let mut chars = name.chars(); chars @@ -2005,6 +2018,29 @@ expect = 150 assert!(!serde_json::to_string(&spec).unwrap().contains("memory_mb")); } + #[test] + fn test_an_allowed_import_is_a_module_name() { + for bad in [ + "os; import shutil", + "numpy.", + ".numpy", + "matplotlib..pyplot", + "1abc", + "", + ] { + refused( + &format!("allowed_imports = [{bad:?}]\n[[cases]]\nname = \"x\"\nexpect = 1\n"), + "must be a module name", + ); + } + assert!( + problems( + "allowed_imports = [\"numpy\", \"matplotlib.pyplot\", \"_private\"]\n[[cases]]\nname = \"x\"\nexpect = 1\n" + ) + .is_empty() + ); + } + #[test] fn test_source_is_reserved_like_stdout() { refused( diff --git a/src/core/tests/harness.rs b/src/core/tests/harness.rs index f243a6a..53ae056 100644 --- a/src/core/tests/harness.rs +++ b/src/core/tests/harness.rs @@ -1024,6 +1024,53 @@ async fn test_a_student_who_writes_into_the_pipe_only_breaks_their_own_unit() { assert!(obs.done, "{obs:?}"); } +/// What the teacher allows is loaded when ScriptMark measures what the student side takes +/// idle, so a student is not charged for importing it. +#[tokio::test] +async fn test_allowed_imports_count_as_what_a_unit_takes_idle() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "import time\n\ndef hog():\n held = b'x' * (256 * 1024 * 1024)\n time.sleep(5)\n return len(held)\n", + ); + let mut idle = Vec::new(); + for allowed in [vec![], vec!["asyncio".to_string()]] { + let mut plan = unit(&student); + plan.allowed_imports = allowed; + plan.steps = vec![function("hog", vec![])]; + plan.memory_mb = 16; + match run(&plan).await.exit { + Exit::Memory { idle_mb, .. } => idle.push(idle_mb), + other => panic!("{:?}: {other:?}", plan.allowed_imports), + } + } + let (bare, with_asyncio) = (idle[0], idle[1]); + // asyncio, which the harness does not load, takes 7 to 10 MiB to import. + assert!( + with_asyncio >= bare + 4, + "{bare} MiB, {with_asyncio} MiB with asyncio" + ); +} + +/// An entry names a module: a dotted one allows its whole package, and one that is not +/// installed does not keep the unit from running. +#[tokio::test] +async fn test_a_dotted_allowed_import_allows_its_package() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "import wsgiref.util\n\ndef scheme():\n return wsgiref.util.guess_scheme({})\n", + ); + let mut plan = unit(&student); + plan.allowed_imports = vec!["wsgiref.util".into(), "not_installed_anywhere".into()]; + plan.steps = vec![function("scheme", vec![])]; + let obs = run(&plan).await; + assert!(obs.done, "{obs:?}"); + assert_eq!(returned(&obs.steps[0].outcome), &json!("http")); +} + #[tokio::test] async fn test_a_student_side_past_its_memory_limit_is_ended() { let dir = tempfile::tempdir().unwrap(); @@ -1037,7 +1084,7 @@ async fn test_a_student_side_past_its_memory_limit_is_ended() { plan.memory_mb = 96; let obs = run(&plan).await; assert!( - matches!(obs.exit, Exit::Memory { limit_mb: 96, interpreter_mb } if interpreter_mb > 0), + matches!(obs.exit, Exit::Memory { limit_mb: 96, idle_mb } if idle_mb > 0), "{obs:?}" ); assert!(obs.steps.is_empty()); From 8681a251031fb47f749d7915e9ed833ff8806bc1 Mon Sep 17 00:00:00 2001 From: Acture Date: Thu, 15 Oct 2026 23:18:42 +0800 Subject: [PATCH 16/20] feat: no import counts against a student (OSS-10) What a unit takes idle is now measured per submission: harness/idle.py reads the submission's source with ast, never running it, imports every module an import names anywhere in the file (skipping what the guard refuses or what does not load), with the teacher's vars, shared modules and data files, and sleeps 50 ms while the watch samples it every 2 ms. memory_mb counts beyond that, so imports - stdlib or allowed library - are never the student's. Keyed by a digest of the submission and what the teacher gives it. Sampled, not reported: on Linux ru_maxrss carries the grader's own memory from between fork and exec, where the sandbox is applied. 24 students x 3 cases: 2.2-3.0 s, against 1.9-2.0 s measured once per interpreter. --- CLAUDE.md | 4 +- notes | 2 +- src/core/src/models/spec.rs | 6 +- src/core/src/runner/executor.rs | 6 +- src/core/src/runner/harness/idle.py | 35 +++++++ src/core/src/runner/judge.rs | 2 +- src/core/src/runner/python.rs | 142 +++++++++++++--------------- src/core/tests/harness.rs | 30 +++--- 8 files changed, 125 insertions(+), 102 deletions(-) create mode 100644 src/core/src/runner/harness/idle.py diff --git a/CLAUDE.md b/CLAUDE.md index af89501..ff7776f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,7 +96,7 @@ cli/src/tui/ ratatui terminal UI: students/sessions/similarity tabs **No pytest.** ScriptMark is its own test engine. A unit (a case or a scenario) runs as two Python processes from embedded scripts (`runner/harness/`): the teacher side (`teacher.py`: teacher modules, function checkers, the plan, per-request deadlines, the records) and the student side (`student.py`: the student module only, one call per request, under its timer and import guard). They share `common.py` (framing, the record and transfer value encodings, `call()`). ScriptMark joins the two with a pair of pipes and reads length-prefixed JSON records from the teacher side; values cross by value (shared-class instances rebuilt from the receiver's copy, other objects as snapshots). -**Sandboxed execution.** Each side runs confined by default (`runner/sandbox.rs`): macOS Seatbelt via `sandbox-exec` with a generated profile, Linux Landlock + seccomp applied in `pre_exec`. The student side reads its interpreter and its own directory only; the teacher side also reads the bundle and a read-only source copy. Fail closed: no sandbox, no grading, unless `--no-sandbox` (recorded as `isolation: unisolated`). Both also get `env_clear()`, `setrlimit` (RLIMIT_AS only as a backstop), and an absolute interpreter path; the student side is watched against `memory_mb` beyond what it takes idle with the spec's `allowed_imports` loaded (spec, case or scenario; default 64 MiB; the idle amount is measured once per interpreter, isolation and set of allowed imports). The import allowlist is a teaching rule, not a boundary. Sandbox tests need a real sandbox: run `cargo test` outside any other Seatbelt sandbox (Claude Code's included), and on Linux from a filesystem with stable inodes (not a Docker Desktop bind mount). +**Sandboxed execution.** Each side runs confined by default (`runner/sandbox.rs`): macOS Seatbelt via `sandbox-exec` with a generated profile, Linux Landlock + seccomp applied in `pre_exec`. The student side reads its interpreter and its own directory only; the teacher side also reads the bundle and a read-only source copy. Fail closed: no sandbox, no grading, unless `--no-sandbox` (recorded as `isolation: unisolated`). Both also get `env_clear()`, `setrlimit` (RLIMIT_AS only as a backstop), and an absolute interpreter path; the student side is watched against `memory_mb` beyond what it takes idle with every module the submission imports loaded (spec, case or scenario; default 64 MiB; `harness/idle.py` measures it once per submission without running the submission). The import allowlist is a teaching rule, not a boundary. Sandbox tests need a real sandbox: run `cargo test` outside any other Seatbelt sandbox (Claude Code's included), and on Linux from a filesystem with stable inodes (not a Docker Desktop bind mount). **TOML test specs** have three sections: `[vars]` (constants injected as Python globals), `[[setup]]` (function calls storing results for `$ref`), `[[cases]]` (scored test cases). Cases support `expect`, `expect_error`, checkers (`check = "sorted"` / `{ rhai = "..." }` / `{ python = "verifiers/check.py" }` / `{ function = "..." }`, which may take the reserved `stdout` and `source`), and `[cases.parametrize]` with oracle. `[meta]` supports `allowed_imports` (a teaching rule) and `shared` (modules whose classes cross between the two sides by value). Prepare refuses a path into the bundle handed to student code: `data_files` stages it instead. @@ -111,7 +111,7 @@ Zero and withheld grades stay distinct. See the notes' teacher contract for poli ## Critical Files - `src/core/src/runner/python.rs` — PythonExecutor: stages a unit, starts and joins both sides, reads the records, watches memory, probes the interpreter's read roots. -- `src/core/src/runner/harness/{common,teacher,student}.py` — the two sides of a unit and what they share. +- `src/core/src/runner/harness/{common,teacher,student}.py` — the two sides of a unit and what they share; `idle.py` is the student that measures what a submission's imports take. - `src/core/src/runner/sandbox.rs` — `Isolation`, `Confinement`, the Seatbelt profile and Landlock/seccomp rules, plus setrlimit. Platform-conditional: RLIMIT_AS skipped on macOS, RLIMIT_* constants differ between Linux (c_uint) and macOS (c_int). - `src/core/src/runner/records.rs` — framed record reader with plan-derived bounds. - `src/core/src/runner/orchestrator.rs` — Runs prepared bundles per student, tokio parallel. diff --git a/notes b/notes index 3f93e29..ef211b7 160000 --- a/notes +++ b/notes @@ -1 +1 @@ -Subproject commit 3f93e2954bd08b93ad66b0b8d0046390a27f6253 +Subproject commit ef211b7a9720cf24eadec298d50bdb15536a6d0d diff --git a/src/core/src/models/spec.rs b/src/core/src/models/spec.rs index cf12801..f47510f 100644 --- a/src/core/src/models/spec.rs +++ b/src/core/src/models/spec.rs @@ -589,10 +589,8 @@ pub struct TestMeta { #[serde(default)] pub copy_refs: Option, - /// Extra packages allowed in student code (beyond safe stdlib). An entry allows its - /// top-level package; a dotted one (`matplotlib.pyplot`) also names the module loaded - /// when ScriptMark measures what the student side takes idle, which `memory_mb` is - /// counted beyond. + /// Extra packages allowed in student code (beyond safe stdlib). An entry is a module + /// name, and allows its top-level package: `numpy`, `matplotlib.pyplot`. #[serde(default)] pub allowed_imports: Vec, } diff --git a/src/core/src/runner/executor.rs b/src/core/src/runner/executor.rs index 86c1ba1..5ca9c93 100644 --- a/src/core/src/runner/executor.rs +++ b/src/core/src/runner/executor.rs @@ -59,8 +59,8 @@ pub struct UnitPlan { /// Shared modules, as `(source in the bundle, relative destination)`: staged beside the /// student's file, and imported by the teacher side from the bundle. pub shared: Vec<(PathBuf, String)>, - /// The most memory the student side may hold beyond what it takes idle, in MiB, before - /// it is ended. + /// The most memory the student side may hold beyond what it takes idle with the + /// submission's imports loaded, in MiB, before it is ended. pub memory_mb: u64, pub allowed_imports: Vec, pub load_timeout: u64, @@ -274,7 +274,7 @@ pub enum Exit { /// Killed by the grader at the unit's deadline. Deadline, /// Killed by the grader for holding more than the unit's `memory_mb` beyond what the - /// student side takes idle with its allowed imports loaded (`idle_mb`). + /// student side takes idle with the submission's imports loaded (`idle_mb`). Memory { limit_mb: u64, idle_mb: u64, diff --git a/src/core/src/runner/harness/idle.py b/src/core/src/runner/harness/idle.py new file mode 100644 index 0000000..d9af275 --- /dev/null +++ b/src/core/src/runner/harness/idle.py @@ -0,0 +1,35 @@ +# The student ScriptMark measures a unit's idle memory with: it imports what the submission +# imports, read from its source without running any of it, and waits while the grader +# samples it. `memory_mb` counts beyond what this holds, so no import is the student's. +# The grader samples rather than asking: on Linux `ru_maxrss` also counts the grader +# itself, whose memory the process held between fork and exec. +import ast +import time + +SOURCE = ".scriptmark-source" + + +def _imports(path): + try: + with open(path, encoding="utf-8", errors="replace") as fh: + tree = ast.parse(fh.read()) + except Exception: # a submission that does not parse imports nothing here + return [] + found = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + found.extend((alias.name, ()) for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + found.append((node.module, tuple(a.name for a in node.names if a.name != "*"))) + return found + + +for _name, _from in _imports(SOURCE): + try: + __import__(_name, fromlist=_from) + except Exception: # refused, missing or broken: it costs the student nothing either + pass + + +def idle(): + time.sleep(0.05) diff --git a/src/core/src/runner/judge.rs b/src/core/src/runner/judge.rs index e6ad31e..1cfa7bc 100644 --- a/src/core/src/runner/judge.rs +++ b/src/core/src/runner/judge.rs @@ -469,7 +469,7 @@ fn death(exit: &Exit) -> (TestStatus, String) { TestStatus::Error, format!( "the {limit_mb} MiB memory limit (`memory_mb`, on top of the {idle_mb} MiB \ - Python and the allowed imports take) was exceeded during" + Python and the submission's imports take) was exceeded during" ), ), Exit::Stopped => ( diff --git a/src/core/src/runner/python.rs b/src/core/src/runner/python.rs index e0f7f35..8d81e36 100644 --- a/src/core/src/runner/python.rs +++ b/src/core/src/runner/python.rs @@ -439,13 +439,18 @@ impl Finished { } } -/// What the student side may hold: `limit_mb` beyond `idle`, the bytes it takes idle. +/// What the student side may hold: `limit_mb` beyond `idle`, the bytes it takes idle, +/// sampled `every` so often. #[derive(Debug, Clone, Copy)] struct Memory { limit_mb: u64, idle: u64, + every: Duration, } +/// How often a unit's student side is sampled. +const SAMPLE_EVERY: Duration = Duration::from_millis(50); + impl Memory { fn watched(self) -> u64 { self.idle @@ -464,31 +469,20 @@ impl Memory { } } -/// An interpreter, whether it runs sandboxed, and the allowed imports loaded idle. -#[cfg(any(target_os = "macos", target_os = "linux"))] -type IdleKey = (String, bool, Vec); +/// What the student side takes idle, by a digest of everything it is measured from. A +/// failed measurement is not kept: the next unit measures again. #[cfg(any(target_os = "macos", target_os = "linux"))] -type Idle = tokio::sync::OnceCell; +static IDLE_MEMORY: LazyLock< + Mutex>>>, +> = LazyLock::new(Default::default); -/// What the student side takes idle — Python, the harness, the allowed imports and a loaded -/// module — per interpreter, isolation and set of allowed imports, measured once by the -/// instrument that enforces the limit. A failed measurement is not kept: the next unit -/// measures again. +/// The student whose idle memory is measured (`harness/idle.py`). #[cfg(any(target_os = "macos", target_os = "linux"))] -static IDLE_MEMORY: LazyLock>>> = - LazyLock::new(Default::default); +const IDLE: &str = include_str!("harness/idle.py"); -/// The student whose idle memory is measured: it imports what the teacher allows, as a -/// student would, and waits while the watch samples it. A library that cannot load is -/// skipped: it costs a student nothing either. +/// Where the idle unit finds the submission's source, which it reads and never runs. #[cfg(any(target_os = "macos", target_os = "linux"))] -fn idle_subject(imports: &[String]) -> String { - let names = serde_json::to_string(imports).expect("module names serialise"); - format!( - "import time\n\nfor _name in {names}:\n try:\n __import__(_name)\n \ - except Exception:\n pass\n\n\ndef idle():\n time.sleep(0.3)\n" - ) -} +const IDLE_SOURCE: &str = ".scriptmark-source"; impl PythonExecutor { /// A command for one side: confined when sandboxed, with a cleared environment, the @@ -714,6 +708,7 @@ impl PythonExecutor { pid, group, memory.watched(), + memory.every, over, held, ))); @@ -853,10 +848,11 @@ async fn watch_memory( pid: u32, group: Option, limit: u64, + every: Duration, over: Arc, peak: Arc, ) { - let mut tick = tokio::time::interval(Duration::from_millis(50)); + let mut tick = tokio::time::interval(every); while let Some(held) = resident_bytes(pid) { peak.fetch_max(held, Ordering::SeqCst); if held > limit { @@ -1138,6 +1134,7 @@ impl Executor for PythonExecutor { .memory_mb .unwrap_or(crate::spec_loader::DEFAULT_MEMORY_MB), idle: 0, + every: SAMPLE_EVERY, }, INSPECT_BOUNDS, false, @@ -1172,10 +1169,7 @@ impl Executor for PythonExecutor { } async fn run(&self, plan: &UnitPlan) -> UnitObservation { - let idle = match self - .idle_memory(&plan.allowed_imports, plan.load_timeout) - .await - { + let idle = match self.idle_memory(plan).await { Ok(bytes) => bytes, Err(e) => { let mut obs = UnitObservation::not_started(Exit::Spawn(e)); @@ -1188,6 +1182,7 @@ impl Executor for PythonExecutor { Memory { limit_mb: plan.memory_mb, idle, + every: SAMPLE_EVERY, }, ) .await @@ -1196,67 +1191,74 @@ impl Executor for PythonExecutor { } impl PythonExecutor { - /// What the student side takes idle with `allowed_imports` loaded, in bytes: what - /// `memory_mb` is counted beyond. + /// What the student side of `plan` takes idle, in bytes: Python, the harness, the + /// teacher's values and modules, and every module the submission imports — the most a + /// unit whose student is ScriptMark's own was seen holding, sampled every 2 ms. + /// `memory_mb` counts beyond it. Once per submission and what the teacher gives it. #[cfg(any(target_os = "macos", target_os = "linux"))] - async fn idle_memory( - &self, - allowed_imports: &[String], - load_timeout: u64, - ) -> Result { - let mut imports = allowed_imports.to_vec(); - imports.sort(); - imports.dedup(); - if let Some(name) = imports - .iter() - .find(|n| !crate::spec_loader::is_module_name(n)) - { - return Err(format!("allowed import '{name}' is not a module name")); - } + async fn idle_memory(&self, plan: &UnitPlan) -> Result { + let source = tokio::fs::read(&plan.file) + .await + .map_err(|e| format!("could not read {}: {e}", plan.file.display()))?; + let key = self.idle_key(plan, &source)?; let cell = IDLE_MEMORY .lock() .expect("idle memory cache poisoned") - .entry(( - self.python_cmd.clone(), - self.isolation.is_sandboxed(), - imports.clone(), - )) + .entry(key) .or_default() .clone(); - cell.get_or_try_init(|| self.measure_idle_memory(imports, load_timeout)) + cell.get_or_try_init(|| self.measure_idle_memory(plan)) .await .copied() } /// Nothing to count it by where the student side's memory is not watched. #[cfg(not(any(target_os = "macos", target_os = "linux")))] - async fn idle_memory(&self, _: &[String], _: u64) -> Result { + async fn idle_memory(&self, _: &UnitPlan) -> Result { Ok(0) } + /// Everything the idle measurement of `plan` depends on. #[cfg(any(target_os = "macos", target_os = "linux"))] - async fn measure_idle_memory( - &self, - imports: Vec, - load_timeout: u64, - ) -> Result { + fn idle_key(&self, plan: &UnitPlan, source: &[u8]) -> Result { + use sha2::{Digest, Sha256}; + let given = serde_json::to_vec(&serde_json::json!({ + "python": self.python_cmd, + "sandboxed": self.isolation.is_sandboxed(), + "subject": plan.subject, + "allowed_imports": plan.allowed_imports, + "vars": &*plan.vars, + "shared": plan.shared, + "data_files": plan.data_files, + })) + .map_err(|e| format!("could not key the idle unit: {e}"))?; + let mut digest = Sha256::new(); + digest.update((given.len() as u64).to_le_bytes()); + digest.update(&given); + digest.update(source); + Ok(format!("{:x}", digest.finalize())) + } + + #[cfg(any(target_os = "macos", target_os = "linux"))] + async fn measure_idle_memory(&self, given: &UnitPlan) -> Result { let dir = tempfile::tempdir().map_err(|e| format!("could not stage the idle unit: {e}"))?; let file = dir.path().join("idle.py"); - std::fs::write(&file, idle_subject(&imports)) - .map_err(|e| format!("could not stage the idle unit: {e}"))?; + std::fs::write(&file, IDLE).map_err(|e| format!("could not stage the idle unit: {e}"))?; + let mut data_files = given.data_files.clone(); + data_files.push((given.file.clone(), IDLE_SOURCE.into())); let plan = UnitPlan { - subject: Subject::Student, + subject: given.subject, functions: crate::matching::Functions::default(), file, script: None, imports: Vec::new(), - vars: Arc::default(), - data_files: Vec::new(), - bundle: dir.path().to_path_buf(), - shared: Vec::new(), + vars: given.vars.clone(), + data_files, + bundle: given.bundle.clone(), + shared: given.shared.clone(), memory_mb: *crate::spec_loader::MEMORY_MB.end(), - allowed_imports: imports.clone(), - load_timeout: load_timeout.max(STARTUP_SECS), + allowed_imports: given.allowed_imports.clone(), + load_timeout: given.load_timeout.max(STARTUP_SECS), setup: Vec::new(), steps: vec![crate::runner::executor::CallPlan { target: crate::models::Target::Function { @@ -1273,6 +1275,7 @@ impl PythonExecutor { let memory = Memory { limit_mb: plan.memory_mb, idle: 0, + every: Duration::from_millis(2), }; let (obs, peak) = self.observe(&plan, memory).await; if let Exit::Spawn(e) = &obs.exit { @@ -1284,7 +1287,7 @@ impl PythonExecutor { }); if !idled || peak == 0 { return Err(format!( - "could not measure what Python takes idle with {imports:?} ({:?}): {}", + "could not measure what Python takes idle ({:?}): {}", obs.exit, obs.stderr.trim() )); @@ -1407,17 +1410,6 @@ mod tests { use super::*; use crate::runner::executor::CallPlan; - #[cfg(any(target_os = "macos", target_os = "linux"))] - #[test] - fn test_the_idle_student_imports_each_allowed_module_by_name() { - assert_eq!( - idle_subject(&["matplotlib.pyplot".into(), "numpy".into()]), - "import time\n\nfor _name in [\"matplotlib.pyplot\",\"numpy\"]:\n try:\n \ - __import__(_name)\n except Exception:\n pass\n\n\ndef idle():\n \ - time.sleep(0.3)\n" - ); - } - /// The CPU limit is derived from the unit's own deadline — never the fixed 30 s it used /// to be, which a scenario of several long steps would outrun and be killed by. #[test] diff --git a/src/core/tests/harness.rs b/src/core/tests/harness.rs index 53ae056..93ec695 100644 --- a/src/core/tests/harness.rs +++ b/src/core/tests/harness.rs @@ -1024,33 +1024,31 @@ async fn test_a_student_who_writes_into_the_pipe_only_breaks_their_own_unit() { assert!(obs.done, "{obs:?}"); } -/// What the teacher allows is loaded when ScriptMark measures what the student side takes -/// idle, so a student is not charged for importing it. +/// What a submission imports, wherever in its file, is loaded when ScriptMark measures what +/// its student side takes idle: no import is the student's. #[tokio::test] -async fn test_allowed_imports_count_as_what_a_unit_takes_idle() { +async fn test_what_a_submission_imports_counts_as_idle() { let dir = tempfile::tempdir().unwrap(); - let student = write( - dir.path(), - "lab.py", - "import time\n\ndef hog():\n held = b'x' * (256 * 1024 * 1024)\n time.sleep(5)\n return len(held)\n", - ); let mut idle = Vec::new(); - for allowed in [vec![], vec!["asyncio".to_string()]] { + for (name, import) in [("bare.py", ""), ("imports.py", " import asyncio\n")] { + let student = write( + dir.path(), + name, + &format!( + "import time\n\ndef hog():\n{import} held = b'x' * (256 * 1024 * 1024)\n time.sleep(5)\n return len(held)\n" + ), + ); let mut plan = unit(&student); - plan.allowed_imports = allowed; + plan.allowed_imports = vec!["asyncio".into()]; plan.steps = vec![function("hog", vec![])]; plan.memory_mb = 16; match run(&plan).await.exit { Exit::Memory { idle_mb, .. } => idle.push(idle_mb), - other => panic!("{:?}: {other:?}", plan.allowed_imports), + other => panic!("{name}: {other:?}"), } } - let (bare, with_asyncio) = (idle[0], idle[1]); // asyncio, which the harness does not load, takes 7 to 10 MiB to import. - assert!( - with_asyncio >= bare + 4, - "{bare} MiB, {with_asyncio} MiB with asyncio" - ); + assert!(idle[1] >= idle[0] + 4, "{idle:?} MiB"); } /// An entry names a module: a dotted one allows its whole package, and one that is not From 67074a1c2ae62920e872252ecacda8d18cb3857c Mon Sep 17 00:00:00 2001 From: Acture Date: Thu, 15 Oct 2026 23:36:56 +0800 Subject: [PATCH 17/20] fix: read a submission's imports outside the measured process (OSS-10) idle.py parsed the submission with ast in the very process whose peak became the idle share. Parsing takes several hundred times the source - 500 KB of constants, about 290 MiB - so a student could raise their own limit by that much with a padded file. harness/imports.py now reads the imports in a process of its own, confined and not watched, and skips a source over 256 KiB, whose imports then count; the idle unit only receives the module names, as an argument to idle(). Imports are read once per source and the idle unit runs once per set of imports and what the teacher gives, so students importing the same modules share it: 24 students x 3 cases, 2.1-2.8 s against 2.0-2.6 s. --- CLAUDE.md | 4 +- notes | 2 +- src/core/src/runner/harness/idle.py | 42 ++---- src/core/src/runner/harness/imports.py | 24 ++++ src/core/src/runner/python.rs | 173 +++++++++++++++++++------ src/core/tests/harness.rs | 24 ++++ 6 files changed, 197 insertions(+), 72 deletions(-) create mode 100644 src/core/src/runner/harness/imports.py diff --git a/CLAUDE.md b/CLAUDE.md index ff7776f..cbd10c9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,7 +96,7 @@ cli/src/tui/ ratatui terminal UI: students/sessions/similarity tabs **No pytest.** ScriptMark is its own test engine. A unit (a case or a scenario) runs as two Python processes from embedded scripts (`runner/harness/`): the teacher side (`teacher.py`: teacher modules, function checkers, the plan, per-request deadlines, the records) and the student side (`student.py`: the student module only, one call per request, under its timer and import guard). They share `common.py` (framing, the record and transfer value encodings, `call()`). ScriptMark joins the two with a pair of pipes and reads length-prefixed JSON records from the teacher side; values cross by value (shared-class instances rebuilt from the receiver's copy, other objects as snapshots). -**Sandboxed execution.** Each side runs confined by default (`runner/sandbox.rs`): macOS Seatbelt via `sandbox-exec` with a generated profile, Linux Landlock + seccomp applied in `pre_exec`. The student side reads its interpreter and its own directory only; the teacher side also reads the bundle and a read-only source copy. Fail closed: no sandbox, no grading, unless `--no-sandbox` (recorded as `isolation: unisolated`). Both also get `env_clear()`, `setrlimit` (RLIMIT_AS only as a backstop), and an absolute interpreter path; the student side is watched against `memory_mb` beyond what it takes idle with every module the submission imports loaded (spec, case or scenario; default 64 MiB; `harness/idle.py` measures it once per submission without running the submission). The import allowlist is a teaching rule, not a boundary. Sandbox tests need a real sandbox: run `cargo test` outside any other Seatbelt sandbox (Claude Code's included), and on Linux from a filesystem with stable inodes (not a Docker Desktop bind mount). +**Sandboxed execution.** Each side runs confined by default (`runner/sandbox.rs`): macOS Seatbelt via `sandbox-exec` with a generated profile, Linux Landlock + seccomp applied in `pre_exec`. The student side reads its interpreter and its own directory only; the teacher side also reads the bundle and a read-only source copy. Fail closed: no sandbox, no grading, unless `--no-sandbox` (recorded as `isolation: unisolated`). Both also get `env_clear()`, `setrlimit` (RLIMIT_AS only as a backstop), and an absolute interpreter path; the student side is watched against `memory_mb` beyond what it takes idle with every module the submission imports loaded (spec, case or scenario; default 64 MiB; `harness/idle.py` measures it once per set of imports, never running the submission). The import allowlist is a teaching rule, not a boundary. Sandbox tests need a real sandbox: run `cargo test` outside any other Seatbelt sandbox (Claude Code's included), and on Linux from a filesystem with stable inodes (not a Docker Desktop bind mount). **TOML test specs** have three sections: `[vars]` (constants injected as Python globals), `[[setup]]` (function calls storing results for `$ref`), `[[cases]]` (scored test cases). Cases support `expect`, `expect_error`, checkers (`check = "sorted"` / `{ rhai = "..." }` / `{ python = "verifiers/check.py" }` / `{ function = "..." }`, which may take the reserved `stdout` and `source`), and `[cases.parametrize]` with oracle. `[meta]` supports `allowed_imports` (a teaching rule) and `shared` (modules whose classes cross between the two sides by value). Prepare refuses a path into the bundle handed to student code: `data_files` stages it instead. @@ -111,7 +111,7 @@ Zero and withheld grades stay distinct. See the notes' teacher contract for poli ## Critical Files - `src/core/src/runner/python.rs` — PythonExecutor: stages a unit, starts and joins both sides, reads the records, watches memory, probes the interpreter's read roots. -- `src/core/src/runner/harness/{common,teacher,student}.py` — the two sides of a unit and what they share; `idle.py` is the student that measures what a submission's imports take. +- `src/core/src/runner/harness/{common,teacher,student}.py` — the two sides of a unit and what they share; `idle.py` is the student that measures what a submission's imports take, and `imports.py` reads those imports in a process of its own. - `src/core/src/runner/sandbox.rs` — `Isolation`, `Confinement`, the Seatbelt profile and Landlock/seccomp rules, plus setrlimit. Platform-conditional: RLIMIT_AS skipped on macOS, RLIMIT_* constants differ between Linux (c_uint) and macOS (c_int). - `src/core/src/runner/records.rs` — framed record reader with plan-derived bounds. - `src/core/src/runner/orchestrator.rs` — Runs prepared bundles per student, tokio parallel. diff --git a/notes b/notes index ef211b7..0970307 160000 --- a/notes +++ b/notes @@ -1 +1 @@ -Subproject commit ef211b7a9720cf24eadec298d50bdb15536a6d0d +Subproject commit 0970307e9a4751c6d0d6e75e26f3d85da7290548 diff --git a/src/core/src/runner/harness/idle.py b/src/core/src/runner/harness/idle.py index d9af275..44f1acb 100644 --- a/src/core/src/runner/harness/idle.py +++ b/src/core/src/runner/harness/idle.py @@ -1,35 +1,15 @@ -# The student ScriptMark measures a unit's idle memory with: it imports what the submission -# imports, read from its source without running any of it, and waits while the grader -# samples it. `memory_mb` counts beyond what this holds, so no import is the student's. -# The grader samples rather than asking: on Linux `ru_maxrss` also counts the grader -# itself, whose memory the process held between fork and exec. -import ast +# The student ScriptMark measures a unit's idle memory with: it imports the modules the +# submission imports — read from its source beforehand, in another process — and waits +# while the grader samples it. `memory_mb` counts beyond what this holds, so no import is +# the student's. The grader samples rather than asking: on Linux `ru_maxrss` also counts +# the grader itself, whose memory the process held between fork and exec. import time -SOURCE = ".scriptmark-source" - -def _imports(path): - try: - with open(path, encoding="utf-8", errors="replace") as fh: - tree = ast.parse(fh.read()) - except Exception: # a submission that does not parse imports nothing here - return [] - found = [] - for node in ast.walk(tree): - if isinstance(node, ast.Import): - found.extend((alias.name, ()) for alias in node.names) - elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: - found.append((node.module, tuple(a.name for a in node.names if a.name != "*"))) - return found - - -for _name, _from in _imports(SOURCE): - try: - __import__(_name, fromlist=_from) - except Exception: # refused, missing or broken: it costs the student nothing either - pass - - -def idle(): +def idle(imports): + for name, names in imports: + try: + __import__(name, fromlist=names) + except Exception: # refused, missing or broken: it costs the student nothing either + pass time.sleep(0.05) diff --git a/src/core/src/runner/harness/imports.py b/src/core/src/runner/harness/imports.py new file mode 100644 index 0000000..73d2242 --- /dev/null +++ b/src/core/src/runner/harness/imports.py @@ -0,0 +1,24 @@ +# Reads a submission's source on stdin and prints, as JSON, each module an import in it +# names, wherever in the file, with what `from … import` takes from it. Nothing in the +# source runs. It runs in a process of its own, never the one whose memory is measured: +# what parsing takes is the source's to decide. +import ast +import json +import sys + + +def imports(text): + found = [] + for node in ast.walk(ast.parse(text)): + if isinstance(node, ast.Import): + found.extend([alias.name, []] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + found.append([node.module, [a.name for a in node.names if a.name != "*"]]) + return found + + +try: + found = imports(sys.stdin.buffer.read().decode("utf-8", "replace")) +except (SyntaxError, ValueError, RecursionError, MemoryError): # it imports nothing beforehand + found = [] +print(json.dumps(found)) diff --git a/src/core/src/runner/python.rs b/src/core/src/runner/python.rs index 8d81e36..1005c38 100644 --- a/src/core/src/runner/python.rs +++ b/src/core/src/runner/python.rs @@ -469,20 +469,52 @@ impl Memory { } } -/// What the student side takes idle, by a digest of everything it is measured from. A -/// failed measurement is not kept: the next unit measures again. +/// Values worked out once per key, by a digest of everything they depend on. A failure is +/// not kept: the next caller works it out again. #[cfg(any(target_os = "macos", target_os = "linux"))] -static IDLE_MEMORY: LazyLock< - Mutex>>>, -> = LazyLock::new(Default::default); +type Once = LazyLock>>>>; + +/// The modules each submission imports, by its source. +#[cfg(any(target_os = "macos", target_os = "linux"))] +static SOURCE_IMPORTS: Once = LazyLock::new(Default::default); + +/// What the student side takes idle, by what it imports and what the teacher gives it. +#[cfg(any(target_os = "macos", target_os = "linux"))] +static IDLE_MEMORY: Once = LazyLock::new(Default::default); + +/// Modules, each with what `from … import` takes from it. +#[cfg(any(target_os = "macos", target_os = "linux"))] +type Imports = Vec<(String, Vec)>; /// The student whose idle memory is measured (`harness/idle.py`). #[cfg(any(target_os = "macos", target_os = "linux"))] const IDLE: &str = include_str!("harness/idle.py"); -/// Where the idle unit finds the submission's source, which it reads and never runs. +/// Reads a source's imports without running it (`harness/imports.py`). +#[cfg(any(target_os = "macos", target_os = "linux"))] +const IMPORTS: &str = include_str!("harness/imports.py"); + +/// The largest source whose imports are read beforehand: a larger one's count. Parsing +/// takes several hundred times the source (500 KB of constants: about 290 MiB), and the +/// reader is not watched; a submission is a few kilobytes. +#[cfg(any(target_os = "macos", target_os = "linux"))] +const IMPORTS_SOURCE_LIMIT: usize = 256 << 10; + +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn digest(given: &serde_json::Value) -> String { + use sha2::{Digest, Sha256}; + format!("{:x}", Sha256::digest(given.to_string())) +} + #[cfg(any(target_os = "macos", target_os = "linux"))] -const IDLE_SOURCE: &str = ".scriptmark-source"; +fn once(cache: &Once, key: String) -> Arc> { + cache + .lock() + .expect("once cache poisoned") + .entry(key) + .or_default() + .clone() +} impl PythonExecutor { /// A command for one side: confined when sandboxed, with a cleared environment, the @@ -1194,20 +1226,48 @@ impl PythonExecutor { /// What the student side of `plan` takes idle, in bytes: Python, the harness, the /// teacher's values and modules, and every module the submission imports — the most a /// unit whose student is ScriptMark's own was seen holding, sampled every 2 ms. - /// `memory_mb` counts beyond it. Once per submission and what the teacher gives it. + /// `memory_mb` counts beyond it. Once per set of imports and what the teacher gives. #[cfg(any(target_os = "macos", target_os = "linux"))] async fn idle_memory(&self, plan: &UnitPlan) -> Result { let source = tokio::fs::read(&plan.file) .await .map_err(|e| format!("could not read {}: {e}", plan.file.display()))?; - let key = self.idle_key(plan, &source)?; - let cell = IDLE_MEMORY - .lock() - .expect("idle memory cache poisoned") - .entry(key) - .or_default() + let sandboxed = self.isolation.is_sandboxed(); + let read = once( + &SOURCE_IMPORTS, + digest(&serde_json::json!({ + "python": self.python_cmd, + "sandboxed": sandboxed, + "source": String::from_utf8_lossy(&source), + })), + ); + let imports = read + .get_or_try_init(|| async { + let reader = Self { + python_cmd: self.python_cmd.clone(), + isolation: self.isolation, + }; + tokio::task::spawn_blocking(move || reader.source_imports(&source)) + .await + .map_err(|e| format!("reading the submission's imports panicked: {e}"))? + }) + .await? .clone(); - cell.get_or_try_init(|| self.measure_idle_memory(plan)) + let measured = once( + &IDLE_MEMORY, + digest(&serde_json::json!({ + "python": self.python_cmd, + "sandboxed": sandboxed, + "subject": plan.subject, + "allowed_imports": plan.allowed_imports, + "vars": &*plan.vars, + "shared": plan.shared, + "data_files": plan.data_files, + "imports": imports, + })), + ); + measured + .get_or_try_init(|| self.measure_idle_memory(plan, &imports)) .await .copied() } @@ -1218,34 +1278,71 @@ impl PythonExecutor { Ok(0) } - /// Everything the idle measurement of `plan` depends on. + /// The modules `source` imports, wherever in it: read by Python's own parser in a process + /// of its own, confined and not watched — what parsing takes is the source's to decide, + /// and must never be part of what a unit takes idle. Nothing in the source runs. One + /// that does not parse, is past `IMPORTS_SOURCE_LIMIT` or takes too long imports nothing + /// beforehand: its imports count. #[cfg(any(target_os = "macos", target_os = "linux"))] - fn idle_key(&self, plan: &UnitPlan, source: &[u8]) -> Result { - use sha2::{Digest, Sha256}; - let given = serde_json::to_vec(&serde_json::json!({ - "python": self.python_cmd, - "sandboxed": self.isolation.is_sandboxed(), - "subject": plan.subject, - "allowed_imports": plan.allowed_imports, - "vars": &*plan.vars, - "shared": plan.shared, - "data_files": plan.data_files, - })) - .map_err(|e| format!("could not key the idle unit: {e}"))?; - let mut digest = Sha256::new(); - digest.update((given.len() as u64).to_le_bytes()); - digest.update(&given); - digest.update(source); - Ok(format!("{:x}", digest.finalize())) + fn source_imports(&self, source: &[u8]) -> Result { + if source.len() > IMPORTS_SOURCE_LIMIT { + return Ok(Vec::new()); + } + let (_dir, scratch) = tempfile::Builder::new() + .prefix("scriptmark-imports-") + .tempdir() + .and_then(|dir| std::fs::canonicalize(dir.path()).map(|path| (dir, path))) + .map_err(|e| format!("could not stage the import reader: {e}"))?; + let mut cmd = if self.isolation.is_sandboxed() { + let confinement = + teacher_confinement(&self.python_cmd, Vec::new(), vec![scratch.clone()], false)?; + crate::runner::sandbox::confined(Path::new(&self.python_cmd), &confinement)? + } else { + std::process::Command::new(&self.python_cmd) + }; + cmd.env_clear() + .env("HOME", &scratch) + .env("TMPDIR", &scratch) + .current_dir(&scratch) + .args(["-I", "-B", "-c", IMPORTS]); + let ran = crate::checker::python_checker::run_with_timeout( + cmd, + Some(String::from_utf8_lossy(source).into_owned()), + Duration::from_secs(STARTUP_SECS), + ) + .map_err(|e| format!("could not start the import reader: {e}"))?; + let Some(ran) = ran else { + return Ok(Vec::new()); + }; + let found: Vec<(String, Vec)> = + serde_json::from_str(ran.stdout.trim()).map_err(|e| { + format!( + "the import reader printed no imports ({e}, {}): {}", + ran.status, + ran.stderr.trim() + ) + })?; + let mut imports: Imports = found + .into_iter() + .filter(|(name, names)| { + crate::spec_loader::is_module_name(name) + && names.iter().all(|n| crate::spec_loader::is_module_name(n)) + }) + .collect(); + imports.sort(); + imports.dedup(); + Ok(imports) } #[cfg(any(target_os = "macos", target_os = "linux"))] - async fn measure_idle_memory(&self, given: &UnitPlan) -> Result { + async fn measure_idle_memory( + &self, + given: &UnitPlan, + imports: &Imports, + ) -> Result { let dir = tempfile::tempdir().map_err(|e| format!("could not stage the idle unit: {e}"))?; let file = dir.path().join("idle.py"); std::fs::write(&file, IDLE).map_err(|e| format!("could not stage the idle unit: {e}"))?; - let mut data_files = given.data_files.clone(); - data_files.push((given.file.clone(), IDLE_SOURCE.into())); let plan = UnitPlan { subject: given.subject, functions: crate::matching::Functions::default(), @@ -1253,7 +1350,7 @@ impl PythonExecutor { script: None, imports: Vec::new(), vars: given.vars.clone(), - data_files, + data_files: given.data_files.clone(), bundle: given.bundle.clone(), shared: given.shared.clone(), memory_mb: *crate::spec_loader::MEMORY_MB.end(), @@ -1264,7 +1361,7 @@ impl PythonExecutor { target: crate::models::Target::Function { name: "idle".into(), }, - args: Vec::new(), + args: vec![serde_json::json!(imports)], stdin: None, timeout: STARTUP_SECS, id: None, diff --git a/src/core/tests/harness.rs b/src/core/tests/harness.rs index 93ec695..7c09c02 100644 --- a/src/core/tests/harness.rs +++ b/src/core/tests/harness.rs @@ -1051,6 +1051,30 @@ async fn test_what_a_submission_imports_counts_as_idle() { assert!(idle[1] >= idle[0] + 4, "{idle:?} MiB"); } +/// Reading a submission's imports takes memory its source decides — about 140 MiB for the +/// 120,000 constants here — and none of it is part of what its unit takes idle. +#[tokio::test] +async fn test_what_reading_a_submission_takes_is_not_idle() { + let dir = tempfile::tempdir().unwrap(); + let hog = "import time\n\ndef hog():\n held = b'x' * (256 * 1024 * 1024)\n time.sleep(5)\n return len(held)\n"; + let mut idle = Vec::new(); + for (name, extra) in [ + ("plain.py", String::new()), + ("padded.py", format!("PAD = [{}]\n", "1,".repeat(120_000))), + ] { + let student = write(dir.path(), name, &format!("{extra}{hog}")); + let mut plan = unit(&student); + plan.steps = vec![function("hog", vec![])]; + plan.memory_mb = 16; + plan.load_timeout = 30; + match run(&plan).await.exit { + Exit::Memory { idle_mb, .. } => idle.push(idle_mb), + other => panic!("{name}: {other:?}"), + } + } + assert!(idle[1] <= idle[0] + 3, "{idle:?} MiB"); +} + /// An entry names a module: a dotted one allows its whole package, and one that is not /// installed does not keep the unit from running. #[tokio::test] From 003a7b8de53b4106ab901ffdcf84334436aac48a Mon Sep 17 00:00:00 2001 From: Acture Date: Fri, 16 Oct 2026 00:00:41 +0800 Subject: [PATCH 18/20] feat: estimate an undeclared memory limit from space and data size (OSS-10) Where a spec declares no memory_mb, a unit's limit is estimated from the space the teacher allows the algorithm - `space = "O(n)"` and the like, O(n^2) where nothing is declared - over the size of its test data: the elements of arguments and expected values, the characters of strings, stdin and expected output, the bytes of data files, and an integer argument as its value (fib(30), primes(10**6)). 512 bytes per step of the growth, kept between 16 MiB and 1 GiB; a unit whose arguments hold a setup's result cannot be sized before grading and gets 1 GiB. The narrowest declaration of either applies; both on one item, or either on a scenario step, is refused. Reference implementations are held the same way, over their arguments. DEFAULT_MEMORY_MB goes. UnitPlan.memory_basis and Exit::Memory's basis carry what an estimate came from, and the message names it. The example bundles grade identically. --- CLAUDE.md | 2 +- notes | 2 +- src/core/src/models/spec.rs | 18 +- src/core/src/runner/executor.rs | 7 +- src/core/src/runner/judge.rs | 30 +++- src/core/src/runner/memory.rs | 285 ++++++++++++++++++++++++++++++++ src/core/src/runner/mod.rs | 1 + src/core/src/runner/oracle.rs | 15 +- src/core/src/runner/prepare.rs | 65 +++++--- src/core/src/runner/python.rs | 21 ++- src/core/src/spec_loader.rs | 61 +++++-- src/core/tests/harness.rs | 5 +- src/core/tests/integration.rs | 52 +++--- src/core/tests/sandbox.rs | 3 +- 14 files changed, 482 insertions(+), 85 deletions(-) create mode 100644 src/core/src/runner/memory.rs diff --git a/CLAUDE.md b/CLAUDE.md index cbd10c9..cd1b511 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -96,7 +96,7 @@ cli/src/tui/ ratatui terminal UI: students/sessions/similarity tabs **No pytest.** ScriptMark is its own test engine. A unit (a case or a scenario) runs as two Python processes from embedded scripts (`runner/harness/`): the teacher side (`teacher.py`: teacher modules, function checkers, the plan, per-request deadlines, the records) and the student side (`student.py`: the student module only, one call per request, under its timer and import guard). They share `common.py` (framing, the record and transfer value encodings, `call()`). ScriptMark joins the two with a pair of pipes and reads length-prefixed JSON records from the teacher side; values cross by value (shared-class instances rebuilt from the receiver's copy, other objects as snapshots). -**Sandboxed execution.** Each side runs confined by default (`runner/sandbox.rs`): macOS Seatbelt via `sandbox-exec` with a generated profile, Linux Landlock + seccomp applied in `pre_exec`. The student side reads its interpreter and its own directory only; the teacher side also reads the bundle and a read-only source copy. Fail closed: no sandbox, no grading, unless `--no-sandbox` (recorded as `isolation: unisolated`). Both also get `env_clear()`, `setrlimit` (RLIMIT_AS only as a backstop), and an absolute interpreter path; the student side is watched against `memory_mb` beyond what it takes idle with every module the submission imports loaded (spec, case or scenario; default 64 MiB; `harness/idle.py` measures it once per set of imports, never running the submission). The import allowlist is a teaching rule, not a boundary. Sandbox tests need a real sandbox: run `cargo test` outside any other Seatbelt sandbox (Claude Code's included), and on Linux from a filesystem with stable inodes (not a Docker Desktop bind mount). +**Sandboxed execution.** Each side runs confined by default (`runner/sandbox.rs`): macOS Seatbelt via `sandbox-exec` with a generated profile, Linux Landlock + seccomp applied in `pre_exec`. The student side reads its interpreter and its own directory only; the teacher side also reads the bundle and a read-only source copy. Fail closed: no sandbox, no grading, unless `--no-sandbox` (recorded as `isolation: unisolated`). Both also get `env_clear()`, `setrlimit` (RLIMIT_AS only as a backstop), and an absolute interpreter path; the student side is watched against `memory_mb` beyond what it takes idle with every module the submission imports loaded (spec, case or scenario; undeclared, estimated by `runner/memory.rs` from `space` — O(n^2) by default — and the test data's size, 16 MiB–1 GiB; `harness/idle.py` measures the idle share once per set of imports, never running the submission). The import allowlist is a teaching rule, not a boundary. Sandbox tests need a real sandbox: run `cargo test` outside any other Seatbelt sandbox (Claude Code's included), and on Linux from a filesystem with stable inodes (not a Docker Desktop bind mount). **TOML test specs** have three sections: `[vars]` (constants injected as Python globals), `[[setup]]` (function calls storing results for `$ref`), `[[cases]]` (scored test cases). Cases support `expect`, `expect_error`, checkers (`check = "sorted"` / `{ rhai = "..." }` / `{ python = "verifiers/check.py" }` / `{ function = "..." }`, which may take the reserved `stdout` and `source`), and `[cases.parametrize]` with oracle. `[meta]` supports `allowed_imports` (a teaching rule) and `shared` (modules whose classes cross between the two sides by value). Prepare refuses a path into the bundle handed to student code: `data_files` stages it instead. diff --git a/notes b/notes index 0970307..939bca1 160000 --- a/notes +++ b/notes @@ -1 +1 @@ -Subproject commit 0970307e9a4751c6d0d6e75e26f3d85da7290548 +Subproject commit 939bca151c9f12ae54672bf2a67118b445ebff2d diff --git a/src/core/src/models/spec.rs b/src/core/src/models/spec.rs index f47510f..9dea244 100644 --- a/src/core/src/models/spec.rs +++ b/src/core/src/models/spec.rs @@ -500,6 +500,11 @@ pub struct TestCase { #[serde(default, skip_serializing_if = "Option::is_none")] pub memory_mb: Option, + /// The space the algorithm may use, as `O(n)` and the like: the memory is then estimated + /// from it and the size of the test data. Not beside `memory_mb`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub space: Option, + /// Generate concrete cases from this one. #[serde(default)] pub parametrize: Option, @@ -581,10 +586,16 @@ pub struct TestMeta { pub shared: Vec, /// The most memory a unit's student process may hold beyond what it takes idle, in MiB, - /// where a case or scenario does not say: `DEFAULT_MEMORY_MB` when neither does. + /// where a case or scenario does not say. Where nothing does, it is estimated from + /// `space`, or `DEFAULT_SPACE`, and the size of the unit's test data. #[serde(default, skip_serializing_if = "Option::is_none")] pub memory_mb: Option, + /// The space the algorithm may use, as `O(n)` and the like: the memory is then estimated + /// from it and the size of the test data. Not beside `memory_mb`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub space: Option, + /// Refused: every case now has its own process. #[serde(default)] pub copy_refs: Option, @@ -665,6 +676,11 @@ pub struct Scenario { #[serde(default, skip_serializing_if = "Option::is_none")] pub memory_mb: Option, + /// The space the algorithm may use, as `O(n)` and the like: the memory is then estimated + /// from it and the size of the test data. Not beside `memory_mb`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub space: Option, + /// Runs once, at the start of the scenario, after the top-level setup. #[serde(default)] pub setup: Vec, diff --git a/src/core/src/runner/executor.rs b/src/core/src/runner/executor.rs index 5ca9c93..0ae3e2a 100644 --- a/src/core/src/runner/executor.rs +++ b/src/core/src/runner/executor.rs @@ -62,6 +62,9 @@ pub struct UnitPlan { /// The most memory the student side may hold beyond what it takes idle with the /// submission's imports loaded, in MiB, before it is ended. pub memory_mb: u64, + /// How `memory_mb` was estimated, where the spec does not declare it: + /// `O(n^2) over 1000 elements`. + pub memory_basis: Option, pub allowed_imports: Vec, pub load_timeout: u64, pub setup: Vec, @@ -274,10 +277,12 @@ pub enum Exit { /// Killed by the grader at the unit's deadline. Deadline, /// Killed by the grader for holding more than the unit's `memory_mb` beyond what the - /// student side takes idle with the submission's imports loaded (`idle_mb`). + /// student side takes idle with the submission's imports loaded (`idle_mb`); `basis` + /// says how an undeclared limit was estimated. Memory { limit_mb: u64, idle_mb: u64, + basis: Option, }, /// Still running when the teacher side finished, and ended by the grader then. Stopped, diff --git a/src/core/src/runner/judge.rs b/src/core/src/runner/judge.rs index 1cfa7bc..2535ed6 100644 --- a/src/core/src/runner/judge.rs +++ b/src/core/src/runner/judge.rs @@ -465,13 +465,23 @@ fn death(exit: &Exit) -> (TestStatus, String) { TestStatus::Timeout, "the unit's deadline passed during".into(), ), - Exit::Memory { limit_mb, idle_mb } => ( - TestStatus::Error, - format!( - "the {limit_mb} MiB memory limit (`memory_mb`, on top of the {idle_mb} MiB \ - Python and the submission's imports take) was exceeded during" - ), - ), + Exit::Memory { + limit_mb, + idle_mb, + basis, + } => { + let how = match basis { + Some(basis) => format!("estimated from {basis}"), + None => "`memory_mb`".into(), + }; + ( + TestStatus::Error, + format!( + "the {limit_mb} MiB memory limit ({how}, on top of the {idle_mb} MiB Python \ + and the submission's imports take) was exceeded during" + ), + ) + } Exit::Stopped => ( TestStatus::Error, "the process was ended with its unit during".into(), @@ -967,7 +977,8 @@ mod tests { data_files: Vec::new(), bundle: std::path::PathBuf::new(), shared: Vec::new(), - memory_mb: crate::spec_loader::DEFAULT_MEMORY_MB, + memory_mb: 64, + memory_basis: None, allowed_imports: Vec::new(), load_timeout: 10, setup: Vec::new(), @@ -1314,6 +1325,7 @@ mod tests { obs.exit = Exit::Memory { limit_mb: 64, idle_mb: 24, + basis: None, }; let r = judge(&p, &scored, &obs, "python3", Isolation::Sandboxed); assert_eq!( @@ -1442,6 +1454,7 @@ mod tests { hog.exit = Exit::Memory { limit_mb: 64, idle_mb: 24, + basis: None, }; let r = one(&c, &p, &hog); assert_eq!( @@ -1484,6 +1497,7 @@ mod tests { gone.exit = Exit::Memory { limit_mb: 64, idle_mb: 24, + basis: None, }; let r = judge(&p, &scored, &gone, "python3", Isolation::Sandboxed); assert_eq!( diff --git a/src/core/src/runner/memory.rs b/src/core/src/runner/memory.rs new file mode 100644 index 0000000..75f1a26 --- /dev/null +++ b/src/core/src/runner/memory.rs @@ -0,0 +1,285 @@ +//! What a unit's student side may hold beyond what it takes idle, when the spec does not +//! say in MiB: estimated from the space the teacher allows the algorithm (`space`, O(n²) +//! unless declared) and the size of the unit's test data. + +use std::collections::BTreeMap; +use std::fmt; +use std::path::Path; + +use serde_json::Value; + +use crate::models::{TestCase, TestSpec}; +use crate::runner::executor::UnitPlan; + +/// How much memory an algorithm may use, as the size of its data grows. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Space { + Constant, + Logarithmic, + Linear, + Linearithmic, + Quadratic, +} + +/// What a unit may hold where nothing is declared: lenient, as an exercise's data is small. +pub const DEFAULT_SPACE: Space = Space::Quadratic; + +/// The least an estimate gives, in MiB: what a small exercise holds, above the noise of +/// loading a module. +pub const ESTIMATE_FLOOR_MB: u64 = 16; + +/// The most an estimate gives, in MiB, and what a unit whose data cannot be sized before +/// grading gets: past it, a teacher declares `memory_mb`. +pub const ESTIMATE_CAP_MB: u64 = 1024; + +/// Bytes per step of the growth: about 64 bytes a Python value takes, eight times over +/// for the input held, copies of it, the result and ScriptMark's encodings of it. +const BYTES_PER_STEP: f64 = 512.0; + +impl Space { + /// `O(1)`, `O(log n)`, `O(n)`, `O(n log n)` or `O(n^2)` (also `O(n²)`, `O(n**2)`). + pub fn parse(text: &str) -> Option { + let squeezed: String = text + .chars() + .filter(|c| !c.is_whitespace()) + .collect::() + .to_lowercase(); + let inner = squeezed.strip_prefix("o(")?.strip_suffix(')')?; + Some(match inner { + "1" => Space::Constant, + "logn" => Space::Logarithmic, + "n" => Space::Linear, + "nlogn" => Space::Linearithmic, + "n^2" | "n²" | "n**2" | "n*n" => Space::Quadratic, + _ => return None, + }) + } + + fn grow(self, n: f64) -> f64 { + let log = n.max(2.0).log2(); + match self { + Space::Constant => 1.0, + Space::Logarithmic => log, + Space::Linear => n, + Space::Linearithmic => n * log, + Space::Quadratic => n * n, + } + } +} + +impl fmt::Display for Space { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Space::Constant => "O(1)", + Space::Logarithmic => "O(log n)", + Space::Linear => "O(n)", + Space::Linearithmic => "O(n log n)", + Space::Quadratic => "O(n^2)", + }) + } +} + +/// The MiB `space` allows over data of `size` elements, and how it was reached. Data that +/// cannot be sized before grading gets the cap. +pub fn estimate(space: Space, size: Option) -> (u64, String) { + let Some(n) = size else { + return ( + ESTIMATE_CAP_MB, + format!("{space}, the data's size unknown before grading"), + ); + }; + let mb = (BYTES_PER_STEP * space.grow(n as f64) / f64::from(1 << 20)).ceil(); + let mb = mb.clamp(ESTIMATE_FLOOR_MB as f64, ESTIMATE_CAP_MB as f64) as u64; + let elements = if n == 1 { "element" } else { "elements" }; + (mb, format!("{space} over {n} {elements}")) +} + +/// What a unit may hold, in MiB, and how an undeclared limit was estimated: the narrowest +/// declaration first — `memory_mb` as written, or `space` estimated over `size` — and +/// `DEFAULT_SPACE` where none is. +pub fn resolve( + declared: [(Option, Option<&str>); 2], + size: Option, +) -> (u64, Option) { + let space = declared + .into_iter() + .find_map( + |(memory_mb, space)| match (memory_mb, space.and_then(Space::parse)) { + (Some(mb), _) => Some(Err(mb)), + (None, Some(space)) => Some(Ok(space)), + (None, None) => None, + }, + ) + .unwrap_or(Ok(DEFAULT_SPACE)); + match space { + Err(mb) => (mb, None), + Ok(space) => { + let (mb, basis) = estimate(space, size); + (mb, Some(basis)) + } + } +} + +/// The size of a unit's test data, in elements: its calls' arguments and stdin, what its +/// `expected` cases expect, and the bytes of the data files staged for it. None where an +/// argument holds a setup's result, which is not known before grading. +pub fn unit_size<'a>( + spec: &TestSpec, + plan: &UnitPlan, + expected: impl IntoIterator, +) -> Option { + let text = |s: &Option| s.as_ref().map_or(0, |s| s.chars().count() as u64); + let mut total = spec + .meta + .data_files + .iter() + .map(|rel| bytes_under(&spec.dir.join(rel))) + .fold(0u64, u64::saturating_add); + total = total.saturating_add(plan.script.as_ref().map_or(0, |s| text(&s.stdin))); + for call in plan.setup.iter().chain(&plan.steps) { + total = total + .saturating_add(args_size(&call.args, &spec.vars)?) + .saturating_add(text(&call.stdin)); + } + for case in expected { + if let Some(expect) = &case.expect { + total = total.saturating_add(value_size(expect, &spec.vars)?); + } + total = total.saturating_add(text(&case.expected_stdout)); + } + Some(total) +} + +fn bytes_under(path: &Path) -> u64 { + match std::fs::metadata(path) { + Ok(meta) if meta.is_dir() => std::fs::read_dir(path).map_or(0, |entries| { + entries + .filter_map(Result::ok) + .map(|entry| bytes_under(&entry.path())) + .fold(0, u64::saturating_add) + }), + Ok(meta) => meta.len(), + Err(_) => 0, + } +} + +/// The size of a call's arguments, in elements: a container counts its elements, a string +/// its characters, and an integer argument its value — `fib(30)` and `primes(10**6)` are +/// sized by their number. `$name` reads the var `name`; a reference to a setup result +/// cannot be sized before grading: None. +pub fn args_size(args: &[Value], vars: &BTreeMap) -> Option { + args.iter().try_fold(0u64, |total, arg| { + Some(total.saturating_add(size(arg, true, vars)?)) + }) +} + +/// The size of an expected value: as an argument's, but an integer is one element. +pub fn value_size(value: &Value, vars: &BTreeMap) -> Option { + size(value, false, vars) +} + +fn size(value: &Value, top: bool, vars: &BTreeMap) -> Option { + Some(match value { + Value::String(s) if s.starts_with('$') && !s.starts_with("$$") => { + return size(vars.get(&s[1..])?, top, vars); + } + Value::String(s) => s.chars().count() as u64, + Value::Number(number) if top => number + .as_i64() + .map(i64::unsigned_abs) + .or_else(|| number.as_u64()) + .unwrap_or(1), + Value::Array(items) => items.iter().try_fold(0u64, |total, item| { + Some(total.saturating_add(size(item, false, vars)?.max(1))) + })?, + Value::Object(map) => map.values().try_fold(0u64, |total, item| { + Some(total.saturating_add(size(item, false, vars)?.saturating_add(1))) + })?, + _ => 1, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_space_is_written_as_big_o() { + for (text, space) in [ + ("O(1)", Space::Constant), + ("O(log n)", Space::Logarithmic), + ("o(n)", Space::Linear), + ("O(n log n)", Space::Linearithmic), + ("O(n^2)", Space::Quadratic), + ("O(n²)", Space::Quadratic), + ("O( n**2 )", Space::Quadratic), + ] { + assert_eq!(Space::parse(text), Some(space), "{text}"); + } + for text in ["n", "O(n^3)", "O(2^n)", "O()", "linear"] { + assert_eq!(Space::parse(text), None, "{text}"); + } + } + + #[test] + fn test_an_estimate_grows_with_the_data_between_floor_and_cap() { + assert_eq!(estimate(Space::Quadratic, Some(10)).0, ESTIMATE_FLOOR_MB); + assert_eq!( + estimate(Space::Quadratic, Some(1000)), + (489, "O(n^2) over 1000 elements".to_string()) + ); + assert_eq!(estimate(Space::Linear, Some(1_000_000)).0, 489); + assert_eq!(estimate(Space::Quadratic, Some(100_000)).0, ESTIMATE_CAP_MB); + assert_eq!( + estimate(Space::Constant, Some(u64::MAX)).0, + ESTIMATE_FLOOR_MB + ); + assert_eq!(estimate(Space::Linear, None).0, ESTIMATE_CAP_MB); + assert_eq!(estimate(Space::Linear, Some(1)).1, "O(n) over 1 element"); + } + + #[test] + fn test_the_narrowest_declaration_applies() { + let n = Some(1000); + assert_eq!( + resolve([(None, None), (None, None)], n), + (489, Some("O(n^2) over 1000 elements".into())) + ); + assert_eq!( + resolve([(Some(32), None), (None, Some("O(1)"))], n), + (32, None) + ); + assert_eq!( + resolve([(None, Some("O(1)")), (Some(512), None)], n).0, + ESTIMATE_FLOOR_MB + ); + assert_eq!(resolve([(None, None), (Some(512), None)], n), (512, None)); + assert_eq!( + resolve([(None, None), (None, Some("O(n)"))], None).0, + ESTIMATE_CAP_MB + ); + } + + #[test] + fn test_data_is_sized_by_elements_characters_and_integer_arguments() { + let vars = BTreeMap::from([ + ("N".to_string(), json!(30)), + ("XS".to_string(), json!([1, 2])), + ]); + assert_eq!(args_size(&[json!([3, 1, 5, 2])], &vars), Some(4)); + assert_eq!(args_size(&[json!(3), json!(5)], &vars), Some(8)); + assert_eq!(args_size(&[json!(1_000_000)], &vars), Some(1_000_000)); + assert_eq!(args_size(&[json!(-7), json!(2.5)], &vars), Some(8)); + assert_eq!(args_size(&[json!("hello")], &vars), Some(5)); + assert_eq!( + args_size(&[json!({"a": [1, 2], "b": "xy"})], &vars), + Some(6) + ); + assert_eq!(args_size(&[json!("$N"), json!("$XS")], &vars), Some(32)); + assert_eq!(args_size(&[json!("$$literal")], &vars), Some(9)); + assert_eq!(args_size(&[json!("$data")], &vars), None); + assert_eq!(value_size(&json!(8_000_000), &vars), Some(1)); + assert_eq!(value_size(&json!([[1, 2], []]), &vars), Some(3)); + } +} diff --git a/src/core/src/runner/mod.rs b/src/core/src/runner/mod.rs index 098f8f2..422a366 100644 --- a/src/core/src/runner/mod.rs +++ b/src/core/src/runner/mod.rs @@ -5,6 +5,7 @@ pub mod generation; pub mod generator; pub mod judge; pub mod linter; +pub mod memory; pub mod oracle; pub mod orchestrator; pub mod prepare; diff --git a/src/core/src/runner/oracle.rs b/src/core/src/runner/oracle.rs index ea52f47..273123f 100644 --- a/src/core/src/runner/oracle.rs +++ b/src/core/src/runner/oracle.rs @@ -44,10 +44,9 @@ pub async fn resolve_oracle( .map(|rel| (spec.dir.join(rel), rel.clone())) .collect(), allowed_imports: spec.meta.allowed_imports.clone(), - memory_mb: case - .memory_mb - .or(spec.meta.memory_mb) - .unwrap_or(crate::spec_loader::DEFAULT_MEMORY_MB), + // From the case's declarations and arguments, below: its answer is what this finds. + memory_mb: 0, + memory_basis: None, load_timeout: timeout_secs, setup: Vec::new(), steps: vec![CallPlan { @@ -60,6 +59,14 @@ pub async fn resolve_oracle( check: None, }], }; + let mut plan = plan; + (plan.memory_mb, plan.memory_basis) = crate::runner::memory::resolve( + [ + (case.memory_mb, case.space.as_deref()), + (spec.meta.memory_mb, spec.meta.space.as_deref()), + ], + crate::runner::memory::unit_size(spec, &plan, []), + ); let obs = executor.run(&plan).await; if !obs.ready || !obs.done diff --git a/src/core/src/runner/prepare.rs b/src/core/src/runner/prepare.rs index 89f3cf6..1ca0cb2 100644 --- a/src/core/src/runner/prepare.rs +++ b/src/core/src/runner/prepare.rs @@ -25,10 +25,9 @@ use crate::runner::executor::{ use crate::runner::frozen::{Generation, replay_entry}; use crate::runner::generation::{DrawSeed, Generated, generate, seed_for}; use crate::runner::judge::Scored; +use crate::runner::memory; use crate::runner::oracle::{resolve_oracle, validate_references}; -use crate::spec_loader::{ - DEFAULT_MEMORY_MB, MAX_TIMEOUT_SECS, refs, reserved, validate, validate_expanded, -}; +use crate::spec_loader::{MAX_TIMEOUT_SECS, refs, reserved, validate, validate_expanded}; /// A prepared test bundle: one spec, ready to run against any student. #[derive(Debug, Serialize)] @@ -655,7 +654,9 @@ fn plan_units(spec: &TestSpec, timeout_secs: u64) -> Vec { .map(|rel| (spec.dir.join(rel), rel.clone())) .collect(), allowed_imports: spec.meta.allowed_imports.clone(), - memory_mb: spec.meta.memory_mb.unwrap_or(DEFAULT_MEMORY_MB), + // Each unit's, from its own declarations and data: below. + memory_mb: 0, + memory_basis: None, load_timeout: timeout_secs, setup: Vec::new(), steps: Vec::new(), @@ -667,11 +668,15 @@ fn plan_units(spec: &TestSpec, timeout_secs: u64) -> Vec { .map(|s| setup_call(s, timeout_secs)) .collect(); - let cases = spec.cases.iter().map(|case| { - let base = UnitPlan { - memory_mb: case.memory_mb.unwrap_or(base.memory_mb), - ..base.clone() + let meta = (spec.meta.memory_mb, spec.meta.space.as_deref()); + let held = + |mut plan: UnitPlan, narrow: (Option, Option<&str>), scored: &[(String, TestCase)]| { + let size = memory::unit_size(spec, &plan, scored.iter().map(|(_, c)| c)); + (plan.memory_mb, plan.memory_basis) = memory::resolve([narrow, meta], size); + plan }; + + let cases = spec.cases.iter().map(|case| { let plan = if case.script { UnitPlan { script: Some(ScriptRun { @@ -688,33 +693,39 @@ fn plan_units(spec: &TestSpec, timeout_secs: u64) -> Vec { ..base.clone() } }; + let scored = vec![(case.name.clone(), case.clone())]; Unit { - plan, - scored: vec![(case.name.clone(), case.clone())], + plan: held(plan, (case.memory_mb, case.space.as_deref()), &scored), + scored, } }); let scenarios = spec.scenarios.iter().map(|scenario| { let timeout = scenario.timeout.unwrap_or(timeout_secs); - Unit { - plan: UnitPlan { - memory_mb: scenario.memory_mb.unwrap_or(base.memory_mb), - setup: top_setup - .iter() - .cloned() - .chain(scenario.setup.iter().map(|s| setup_call(s, timeout))) - .collect(), - steps: scenario - .steps - .iter() - .map(|s| call_of(s, default_function, timeout)) - .collect(), - ..base.clone() - }, - scored: scenario + let scored: Vec<(String, TestCase)> = scenario + .steps + .iter() + .map(|s| (scenario.step_name(s), s.clone())) + .collect(); + let plan = UnitPlan { + setup: top_setup + .iter() + .cloned() + .chain(scenario.setup.iter().map(|s| setup_call(s, timeout))) + .collect(), + steps: scenario .steps .iter() - .map(|s| (scenario.step_name(s), s.clone())) + .map(|s| call_of(s, default_function, timeout)) .collect(), + ..base.clone() + }; + Unit { + plan: held( + plan, + (scenario.memory_mb, scenario.space.as_deref()), + &scored, + ), + scored, } }); cases.chain(scenarios).collect() diff --git a/src/core/src/runner/python.rs b/src/core/src/runner/python.rs index 1005c38..a716a1b 100644 --- a/src/core/src/runner/python.rs +++ b/src/core/src/runner/python.rs @@ -440,31 +440,33 @@ impl Finished { } /// What the student side may hold: `limit_mb` beyond `idle`, the bytes it takes idle, -/// sampled `every` so often. -#[derive(Debug, Clone, Copy)] +/// sampled `every` so often; `basis` says how an undeclared limit was estimated. +#[derive(Debug, Clone)] struct Memory { limit_mb: u64, idle: u64, every: Duration, + basis: Option, } /// How often a unit's student side is sampled. const SAMPLE_EVERY: Duration = Duration::from_millis(50); impl Memory { - fn watched(self) -> u64 { + fn watched(&self) -> u64 { self.idle .saturating_add(self.limit_mb.saturating_mul(1 << 20)) } - fn total_mb(self) -> u64 { + fn total_mb(&self) -> u64 { self.watched().div_ceil(1 << 20) } - fn exceeded(self) -> Exit { + fn exceeded(&self) -> Exit { Exit::Memory { limit_mb: self.limit_mb, idle_mb: self.idle.div_ceil(1 << 20), + basis: self.basis.clone(), } } } @@ -1164,9 +1166,10 @@ impl Executor for PythonExecutor { limit_mb: spec .meta .memory_mb - .unwrap_or(crate::spec_loader::DEFAULT_MEMORY_MB), + .unwrap_or(crate::runner::memory::ESTIMATE_CAP_MB), idle: 0, every: SAMPLE_EVERY, + basis: None, }, INSPECT_BOUNDS, false, @@ -1215,6 +1218,7 @@ impl Executor for PythonExecutor { limit_mb: plan.memory_mb, idle, every: SAMPLE_EVERY, + basis: plan.memory_basis.clone(), }, ) .await @@ -1354,6 +1358,7 @@ impl PythonExecutor { bundle: given.bundle.clone(), shared: given.shared.clone(), memory_mb: *crate::spec_loader::MEMORY_MB.end(), + memory_basis: None, allowed_imports: given.allowed_imports.clone(), load_timeout: given.load_timeout.max(STARTUP_SECS), setup: Vec::new(), @@ -1373,6 +1378,7 @@ impl PythonExecutor { limit_mb: plan.memory_mb, idle: 0, every: Duration::from_millis(2), + basis: None, }; let (obs, peak) = self.observe(&plan, memory).await; if let Exit::Spawn(e) = &obs.exit { @@ -1550,7 +1556,8 @@ mod tests { data_files: Vec::new(), bundle: PathBuf::new(), shared: Vec::new(), - memory_mb: crate::spec_loader::DEFAULT_MEMORY_MB, + memory_mb: 64, + memory_basis: None, allowed_imports: Vec::new(), load_timeout: 1, setup: Vec::new(), diff --git a/src/core/src/spec_loader.rs b/src/core/src/spec_loader.rs index d50ed12..cc11e5c 100644 --- a/src/core/src/spec_loader.rs +++ b/src/core/src/spec_loader.rs @@ -6,6 +6,7 @@ use serde_json::Value; use crate::checker::rhai_checker; use crate::runner::generation::{MAX_COUNT, MAX_GENERATED, concrete_name, origins}; use crate::runner::generator::Rule; +use crate::runner::memory::Space; use crate::models::{ AssignmentConfig, Check, CourseConfig, Scenario, SetupStep, Target, TestCase, TestSpec, @@ -132,12 +133,6 @@ pub fn reserved(name: &str) -> Option<&'static str> { /// The longest timeout a spec or the command line may ask for. pub const MAX_TIMEOUT_SECS: u64 = 86_400; -/// The memory a unit's student process may hold beyond what it takes idle, when its spec -/// does not say, in MiB: what the student's code holds, and the harness's copies of what -/// it returns while encoding them — a few times the value's own size. What Python and the -/// harness take idle is measured, and added. -pub const DEFAULT_MEMORY_MB: u64 = 64; - /// The memory a spec may give a unit beyond what it takes idle, in MiB: from what a small /// exercise holds, above the noise of loading a module, to 64 GiB. pub const MEMORY_MB: std::ops::RangeInclusive = 16..=65_536; @@ -255,7 +250,11 @@ impl<'a> Validator<'a> { self.problem("[meta]", message); } } - self.memory(self.spec.meta.memory_mb, "[meta]"); + self.memory( + self.spec.meta.memory_mb, + self.spec.meta.space.as_deref(), + "[meta]", + ); for (name, value) in &self.spec.vars { if contains_null(value) { self.problem( @@ -377,7 +376,21 @@ impl<'a> Validator<'a> { } } - fn memory(&mut self, memory: Option, at: &str) { + fn memory(&mut self, memory: Option, space: Option<&str>, at: &str) { + if let Some(space) = space { + if memory.is_some() { + self.problem( + at, + "memory_mb and space both say what it may hold: keep one", + ); + } + if Space::parse(space).is_none() { + self.problem( + at, + format!("space '{space}' must be O(1), O(log n), O(n), O(n log n) or O(n^2)"), + ); + } + } if memory.is_some_and(|m| !MEMORY_MB.contains(&m)) { self.problem( at, @@ -408,7 +421,7 @@ impl<'a> Validator<'a> { self.problem(&at, "has no steps"); } self.timeout(scenario.timeout, &at); - self.memory(scenario.memory_mb, &at); + self.memory(scenario.memory_mb, scenario.space.as_deref(), &at); let mut bound = top.clone(); for step in &scenario.setup { self.setup(step, &mut bound, &format!("{at} setup '{}'", step.id)); @@ -508,13 +521,13 @@ impl<'a> Validator<'a> { ); } self.timeout(case.timeout, at); - if is_step && case.memory_mb.is_some() { + if is_step && (case.memory_mb.is_some() || case.space.is_some()) { self.problem( at, - "memory_mb belongs to the scenario: its steps share one process", + "memory_mb and space belong to the scenario: its steps share one process", ); } else { - self.memory(case.memory_mb, at); + self.memory(case.memory_mb, case.space.as_deref(), at); } if case.args.iter().any(contains_null) { self.problem( @@ -2001,7 +2014,7 @@ expect = 150 ); refused( "[[scenarios]]\nname = \"s\"\n[[scenarios.steps]]\nname = \"a\"\nexpect = 1\nmemory_mb = 512\n", - "memory_mb belongs to the scenario", + "memory_mb and space belong to the scenario", ); assert!( problems( @@ -2018,6 +2031,28 @@ expect = 150 assert!(!serde_json::to_string(&spec).unwrap().contains("memory_mb")); } + #[test] + fn test_space_is_big_o_and_not_beside_memory_mb() { + refused( + "[[cases]]\nname = \"x\"\nexpect = 1\nspace = \"O(n^3)\"\n", + "space 'O(n^3)' must be O(1), O(log n), O(n), O(n log n) or O(n^2)", + ); + refused( + "[[cases]]\nname = \"x\"\nexpect = 1\nspace = \"O(n)\"\nmemory_mb = 64\n", + "memory_mb and space both say what it may hold", + ); + refused( + "[[scenarios]]\nname = \"s\"\n[[scenarios.steps]]\nname = \"a\"\nexpect = 1\nspace = \"O(n)\"\n", + "memory_mb and space belong to the scenario", + ); + assert!( + problems( + "space = \"O(n log n)\"\n[[scenarios]]\nname = \"s\"\nspace = \"O(1)\"\n[[scenarios.steps]]\nname = \"a\"\nexpect = 1\n[[cases]]\nname = \"x\"\nexpect = 1\nmemory_mb = 32\n" + ) + .is_empty() + ); + } + #[test] fn test_an_allowed_import_is_a_module_name() { for bad in [ diff --git a/src/core/tests/harness.rs b/src/core/tests/harness.rs index 7c09c02..674980b 100644 --- a/src/core/tests/harness.rs +++ b/src/core/tests/harness.rs @@ -54,7 +54,8 @@ fn unit(file: &Path) -> UnitPlan { data_files: Vec::new(), bundle: file.parent().unwrap().to_path_buf(), shared: Vec::new(), - memory_mb: scriptmark_core::spec_loader::DEFAULT_MEMORY_MB, + memory_mb: 64, + memory_basis: None, allowed_imports: Vec::new(), load_timeout: 5, setup: Vec::new(), @@ -1106,7 +1107,7 @@ async fn test_a_student_side_past_its_memory_limit_is_ended() { plan.memory_mb = 96; let obs = run(&plan).await; assert!( - matches!(obs.exit, Exit::Memory { limit_mb: 96, idle_mb } if idle_mb > 0), + matches!(obs.exit, Exit::Memory { limit_mb: 96, idle_mb, basis: None } if idle_mb > 0), "{obs:?}" ); assert!(obs.steps.is_empty()); diff --git a/src/core/tests/integration.rs b/src/core/tests/integration.rs index 23f5659..53a704f 100644 --- a/src/core/tests/integration.rs +++ b/src/core/tests/integration.rs @@ -1468,38 +1468,52 @@ async fn test_a_path_into_the_bundle_is_never_handed_to_student_code() { assert_eq!(results[0].total_passed(), 1, "{:#?}", results[0]); } -/// A unit's student process is held to `memory_mb` beyond what it takes idle: 64 MiB unless -/// the spec, the case or the scenario says otherwise, and a case that needs more declares it. +/// A unit's student process is held to `memory_mb` beyond what it takes idle. Undeclared, +/// it is estimated from `space` — O(n^2) unless declared — and the size of the test data, +/// an integer argument counting as its value; the narrowest declaration applies. #[tokio::test] -async fn test_memory_is_the_default_unless_declared() { +async fn test_memory_is_estimated_from_space_and_data_unless_declared() { let bench = Bench::new(); let spec = |meta: &str, case: &str| { bench.spec(&format!( - "[meta]\nname = \"m\"\nfile = \"lab.py\"\nfunction = \"big\"\nlanguage = \"python\"\n{meta}\n[[cases]]\nname = \"big\"\nexpect = 8000000\n{case}\n" + "[meta]\nname = \"m\"\nfile = \"lab.py\"\nfunction = \"big\"\nlanguage = \"python\"\n{meta}\n[[cases]]\nname = \"big\"\nargs = [8000000]\nexpect = 8000000\n{case}\n" )) }; let student = bench.student( "alice", "lab.py", - "def big():\n return len(list(range(8_000_000)))\n", + "def big(n):\n return len(list(range(n)))\n", ); - let tight = grade(vec![spec("", "")], std::slice::from_ref(&student)).await; - let c = &tight[0].test_results[0].cases[0]; - assert_eq!( - (c.status, c.fault, c.cause), - (TestStatus::Error, Some(Fault::Student), Some(Cause::Memory)), - "{c:?}" + let failure = |meta: &str, case: &str| { + let spec = spec(meta, case); + let student = student.clone(); + async move { + let r = grade(vec![spec], &[student]).await; + let c = r[0].test_results[0].cases[0].clone(); + assert_eq!( + (c.status, c.fault, c.cause), + (TestStatus::Error, Some(Fault::Student), Some(Cause::Memory)), + "{c:?}" + ); + c.failure.unwrap().message + } + }; + let estimated = failure("", "space = \"O(1)\"").await; + assert!( + estimated.contains("16 MiB memory limit (estimated from O(1) over 8000001 elements"), + "{estimated}" ); + let declared = failure("memory_mb = 16", "").await; assert!( - c.failure - .as_ref() - .unwrap() - .message - .contains("64 MiB memory limit"), - "{:?}", - c.failure + declared.contains("16 MiB memory limit (`memory_mb`"), + "{declared}" ); - for (meta, case) in [("memory_mb = 1024", ""), ("", "memory_mb = 1024")] { + for (meta, case) in [ + // O(n^2) over eight million elements: the 1 GiB cap. + ("", ""), + ("space = \"O(1)\"", "memory_mb = 1024"), + ("memory_mb = 16", "space = \"O(n)\""), + ] { let roomy = grade(vec![spec(meta, case)], std::slice::from_ref(&student)).await; assert_eq!(roomy[0].total_passed(), 1, "{meta}{case}: {:#?}", roomy[0]); } diff --git a/src/core/tests/sandbox.rs b/src/core/tests/sandbox.rs index c0f4141..8939e2b 100644 --- a/src/core/tests/sandbox.rs +++ b/src/core/tests/sandbox.rs @@ -180,7 +180,8 @@ impl Bench { data_files: Vec::new(), bundle: self.bundle.clone(), shared: Vec::new(), - memory_mb: scriptmark_core::spec_loader::DEFAULT_MEMORY_MB, + memory_mb: 64, + memory_basis: None, allowed_imports: Vec::new(), load_timeout: 5, setup: Vec::new(), From 7549b0a0861f5360c85f246d7b484770c9d83900 Mon Sep 17 00:00:00 2001 From: Acture Date: Fri, 16 Oct 2026 00:13:34 +0800 Subject: [PATCH 19/20] ci: give the macOS job Python 3.12 (OSS-10) --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 553bbde..9db2b94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,10 @@ jobs: - uses: actions/checkout@v7 with: persist-credentials: false + # The runner's own python3 may be older than the 3.10 the harness needs. + - uses: actions/setup-python@v6 + with: + python-version: "3.12" - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - run: cargo test --workspace From 8733b2aa3dfeefc084cf3e5a7c88956ae9a87f06 Mon Sep 17 00:00:00 2001 From: Acture Date: Fri, 16 Oct 2026 00:56:50 +0800 Subject: [PATCH 20/20] fix: close what the review of the memory work found (OSS-10) - harness/imports.py reads a submission's imports from its tokens, keeping only those of import statements: 4 MB of source takes 31 MiB, where ast took about 2 GB. No size cap; a reader that cannot finish finds no imports, and they count, rather than failing the unit. - An import that exits or breaks the idle unit (import unittest.__main__) no longer turns the submission's units into the environment's failure: idle.py skips SystemExit too, and the unit is otherwise measured without its imports. - A var's own `$...` strings are text when sizing data, as the teacher side treats them: a var naming itself no longer recurses without end. - On Windows the teacher side moves its stderr off the record stream, which there is fd 2: a teacher's warning no longer floods the records. - The Landlock and seccomp pre_exec fails with a constant errno instead of allocating an error after fork; spawn errors say when the sandbox could not be applied. - shlex, landlock and seccompiler are workspace dependencies. - The idle test imports a teacher data module holding a fixed 32 MiB, not asyncio; the record-flood test, which macOS CI ended for memory, declares 512 MiB and reports why if steps go missing. --- Cargo.toml | 3 + notes | 2 +- src/core/Cargo.toml | 6 +- src/core/src/checker/python_checker.rs | 7 ++- src/core/src/runner/harness/idle.py | 2 +- src/core/src/runner/harness/imports.py | 75 +++++++++++++++++++----- src/core/src/runner/harness/teacher.py | 7 +++ src/core/src/runner/linter.rs | 7 ++- src/core/src/runner/memory.rs | 15 +++-- src/core/src/runner/python.rs | 80 ++++++++++++++++---------- src/core/src/runner/sandbox.rs | 26 +++++++-- src/core/tests/harness.rs | 36 +++++++++--- 12 files changed, 199 insertions(+), 67 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4efc021..4a457c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ clap = { version = "4", features = ["derive", "env"] } # Scripting rhai = "1" +shlex = "2" # Display comfy-table = "7" @@ -53,6 +54,8 @@ calamine = "0.36" # System libc = "0.2" +landlock = "0.4" +seccompiler = "0.5" # Random rand = "0.9" diff --git a/notes b/notes index 939bca1..a7e30b6 160000 --- a/notes +++ b/notes @@ -1 +1 @@ -Subproject commit 939bca151c9f12ae54672bf2a67118b445ebff2d +Subproject commit a7e30b64dd7b259b1224fc0c4f35414a7524eb14 diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 95b6e16..387f309 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -37,9 +37,9 @@ rars = "0.9.4" bzip2-rs = "0.1.2" lzma-rs = "0.3.0" tempfile = "3" -shlex = "2" +shlex = { workspace = true } sha2 = "0.10" [target.'cfg(target_os = "linux")'.dependencies] -landlock = "0.4" -seccompiler = "0.5" +landlock = { workspace = true } +seccompiler = { workspace = true } diff --git a/src/core/src/checker/python_checker.rs b/src/core/src/checker/python_checker.rs index d61e28a..587fade 100644 --- a/src/core/src/checker/python_checker.rs +++ b/src/core/src/checker/python_checker.rs @@ -97,7 +97,12 @@ impl Checker for PythonChecker { Some(input_json), Duration::from_secs(self.timeout_secs), ) - .map_err(|e| CheckError::environment(format!("could not start checker '{script}': {e}")))?; + .map_err(|e| { + CheckError::environment(format!( + "could not start checker '{script}': {}", + crate::runner::sandbox::spawn_error(&e) + )) + })?; let Some(Ran { status, stdout, diff --git a/src/core/src/runner/harness/idle.py b/src/core/src/runner/harness/idle.py index 44f1acb..82ece93 100644 --- a/src/core/src/runner/harness/idle.py +++ b/src/core/src/runner/harness/idle.py @@ -10,6 +10,6 @@ def idle(imports): for name, names in imports: try: __import__(name, fromlist=names) - except Exception: # refused, missing or broken: it costs the student nothing either + except (Exception, SystemExit): # refused, missing, broken or exiting: skipped pass time.sleep(0.05) diff --git a/src/core/src/runner/harness/imports.py b/src/core/src/runner/harness/imports.py index 73d2242..2633610 100644 --- a/src/core/src/runner/harness/imports.py +++ b/src/core/src/runner/harness/imports.py @@ -1,24 +1,71 @@ # Reads a submission's source on stdin and prints, as JSON, each module an import in it # names, wherever in the file, with what `from … import` takes from it. Nothing in the -# source runs. It runs in a process of its own, never the one whose memory is measured: -# what parsing takes is the source's to decide. -import ast +# source runs. It reads tokens as they come and keeps only those of import statements, so +# what it holds does not grow with the source, whatever the source holds. import json import sys +import tokenize +_STARTS = {tokenize.NEWLINE, tokenize.INDENT, tokenize.DEDENT} +_SKIPPED = {tokenize.NL, tokenize.COMMENT, tokenize.ENCODING} -def imports(text): + +def _module(words): + """`import a.b as c, d` or `from a.b import (x as y, z)`: the module, and what it takes.""" + if words[0] == "import": + modules, name, renamed = [], [], False + for word in words[1:] + [","]: + if word == ",": + if name: + modules.append(["".join(name), []]) + name, renamed = [], False + elif word == "as": + renamed = True + elif not renamed: + name.append(word) + return modules + if "import" not in words or words[1] in (".", "..."): # relative: the submission's own + return [] + at = words.index("import") + names, renamed = [], False + for word in words[at + 1 :]: + if word in ("(", ")", "*"): + continue + if word == ",": + renamed = False + elif word == "as": + renamed = True + elif not renamed: + names.append(word) + return [["".join(words[1:at]), names]] + + +def imports(readline): found = [] - for node in ast.walk(ast.parse(text)): - if isinstance(node, ast.Import): - found.extend([alias.name, []] for alias in node.names) - elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: - found.append([node.module, [a.name for a in node.names if a.name != "*"]]) + start, nesting, statement = True, 0, None + try: + for token in tokenize.tokenize(readline): + kind, text = token.type, token.string + if kind in _SKIPPED: + continue + ends = kind in _STARTS or kind == tokenize.ENDMARKER or (kind == tokenize.OP and text == ";" and not nesting) + if ends: + if statement: + found.extend(_module(statement)) + start, statement = True, None + continue + if kind == tokenize.OP: + nesting += text in "([{" + nesting -= text in ")]}" + if statement is not None: + statement.append(text) + elif start and kind == tokenize.NAME and text in ("import", "from"): + statement = [text] + # A statement after a colon on the same line: `if x: import y`. + start = kind == tokenize.OP and text == ":" and not nesting + except (tokenize.TokenError, SyntaxError, ValueError): # what was read so far stands + pass return found -try: - found = imports(sys.stdin.buffer.read().decode("utf-8", "replace")) -except (SyntaxError, ValueError, RecursionError, MemoryError): # it imports nothing beforehand - found = [] -print(json.dumps(found)) +print(json.dumps(imports(sys.stdin.buffer.readline))) diff --git a/src/core/src/runner/harness/teacher.py b/src/core/src/runner/harness/teacher.py index e0b3988..07d9a97 100644 --- a/src/core/src/runner/harness/teacher.py +++ b/src/core/src/runner/harness/teacher.py @@ -16,6 +16,13 @@ configure(PAYLOAD["limits"]) _RECORDS = PAYLOAD["records"] +if _RECORDS == 2: + # Windows has no third pipe: records take stderr, so the teacher's own stderr — a + # module's warning, a checker's print — goes nowhere rather than into them. + _RECORDS = os.dup(2) + _null = os.open(os.devnull, os.O_WRONLY) + os.dup2(_null, 2) + os.close(_null) # Seconds a request may run past its own timeout before the student side is given up on, # and how long the student side has to start. GRACE = PAYLOAD["grace"] diff --git a/src/core/src/runner/linter.rs b/src/core/src/runner/linter.rs index 8d68933..7e283ee 100644 --- a/src/core/src/runner/linter.rs +++ b/src/core/src/runner/linter.rs @@ -116,7 +116,12 @@ pub fn run_lint( } let out = crate::checker::python_checker::run_with_timeout(cmd, None, timeout) - .map_err(|e| format!("could not run `{command}`: {e}"))? + .map_err(|e| { + format!( + "could not run `{command}`: {}", + crate::runner::sandbox::spawn_error(&e) + ) + })? .ok_or_else(|| format!("`{command}` ran longer than {}s", timeout.as_secs()))?; if !out .status diff --git a/src/core/src/runner/memory.rs b/src/core/src/runner/memory.rs index 75f1a26..d7e2b59 100644 --- a/src/core/src/runner/memory.rs +++ b/src/core/src/runner/memory.rs @@ -169,19 +169,21 @@ fn bytes_under(path: &Path) -> u64 { /// cannot be sized before grading: None. pub fn args_size(args: &[Value], vars: &BTreeMap) -> Option { args.iter().try_fold(0u64, |total, arg| { - Some(total.saturating_add(size(arg, true, vars)?)) + Some(total.saturating_add(size(arg, true, Some(vars))?)) }) } /// The size of an expected value: as an argument's, but an integer is one element. pub fn value_size(value: &Value, vars: &BTreeMap) -> Option { - size(value, false, vars) + size(value, false, Some(vars)) } -fn size(value: &Value, top: bool, vars: &BTreeMap) -> Option { +/// `vars` resolves `$name`; a var's own value, None, holds text where it says `$…`. +fn size(value: &Value, top: bool, vars: Option<&BTreeMap>) -> Option { Some(match value { - Value::String(s) if s.starts_with('$') && !s.starts_with("$$") => { - return size(vars.get(&s[1..])?, top, vars); + // Resolved once, as the teacher side does: a var's own `$…` strings are text. + Value::String(s) if vars.is_some() && s.starts_with('$') && !s.starts_with("$$") => { + return size(vars?.get(&s[1..])?, top, None); } Value::String(s) => s.chars().count() as u64, Value::Number(number) if top => number @@ -279,6 +281,9 @@ mod tests { assert_eq!(args_size(&[json!("$N"), json!("$XS")], &vars), Some(32)); assert_eq!(args_size(&[json!("$$literal")], &vars), Some(9)); assert_eq!(args_size(&[json!("$data")], &vars), None); + // A var that names itself is text, not a loop. + let looped = BTreeMap::from([("A".to_string(), json!(["$A", "$A"]))]); + assert_eq!(args_size(&[json!("$A")], &looped), Some(4)); assert_eq!(value_size(&json!(8_000_000), &vars), Some(1)); assert_eq!(value_size(&json!([[1, 2], []]), &vars), Some(3)); } diff --git a/src/core/src/runner/python.rs b/src/core/src/runner/python.rs index a716a1b..c625317 100644 --- a/src/core/src/runner/python.rs +++ b/src/core/src/runner/python.rs @@ -496,12 +496,6 @@ const IDLE: &str = include_str!("harness/idle.py"); #[cfg(any(target_os = "macos", target_os = "linux"))] const IMPORTS: &str = include_str!("harness/imports.py"); -/// The largest source whose imports are read beforehand: a larger one's count. Parsing -/// takes several hundred times the source (500 KB of constants: about 290 MiB), and the -/// reader is not watched; a submission is a few kilobytes. -#[cfg(any(target_os = "macos", target_os = "linux"))] -const IMPORTS_SOURCE_LIMIT: usize = 256 << 10; - #[cfg(any(target_os = "macos", target_os = "linux"))] fn digest(given: &serde_json::Value) -> String { use sha2::{Digest, Sha256}; @@ -657,8 +651,9 @@ impl PythonExecutor { Ok(child) => student = Some(child), Err(e) => { return Finished::spawn_failed(format!( - "could not start {}: {e}", - self.python_cmd + "could not start {}: {}", + self.python_cmd, + crate::runner::sandbox::spawn_error(&e) )); } } @@ -689,7 +684,11 @@ impl PythonExecutor { let mut teacher = match teacher_cmd.spawn() { Ok(child) => child, Err(e) => { - return Finished::spawn_failed(format!("could not start {}: {e}", self.python_cmd)); + return Finished::spawn_failed(format!( + "could not start {}: {}", + self.python_cmd, + crate::runner::sandbox::spawn_error(&e) + )); } }; // The parent's copies of every pipe end go, so each pipe ends when its side does. @@ -1271,7 +1270,15 @@ impl PythonExecutor { })), ); measured - .get_or_try_init(|| self.measure_idle_memory(plan, &imports)) + .get_or_try_init(|| async { + match self.measure_idle_memory(plan, &imports).await { + // An import that keeps the idle unit from finishing — one that exits, say — + // is the submission's doing: it is then measured without its imports, and + // they count. + Err(_) if !imports.is_empty() => self.measure_idle_memory(plan, &[]).await, + measured => measured, + } + }) .await .copied() } @@ -1282,16 +1289,13 @@ impl PythonExecutor { Ok(0) } - /// The modules `source` imports, wherever in it: read by Python's own parser in a process - /// of its own, confined and not watched — what parsing takes is the source's to decide, - /// and must never be part of what a unit takes idle. Nothing in the source runs. One - /// that does not parse, is past `IMPORTS_SOURCE_LIMIT` or takes too long imports nothing - /// beforehand: its imports count. + /// The modules `source` imports, wherever in it: read from its tokens in a process of its + /// own, confined and not watched, so nothing the source holds is part of what a unit + /// takes idle. Nothing in the source runs. A reader that cannot finish — the source is + /// too strange, or too long to read in time — finds no imports, and they count: only a + /// reader that cannot start is an error. #[cfg(any(target_os = "macos", target_os = "linux"))] fn source_imports(&self, source: &[u8]) -> Result { - if source.len() > IMPORTS_SOURCE_LIMIT { - return Ok(Vec::new()); - } let (_dir, scratch) = tempfile::Builder::new() .prefix("scriptmark-imports-") .tempdir() @@ -1314,18 +1318,17 @@ impl PythonExecutor { Some(String::from_utf8_lossy(source).into_owned()), Duration::from_secs(STARTUP_SECS), ) - .map_err(|e| format!("could not start the import reader: {e}"))?; - let Some(ran) = ran else { + .map_err(|e| { + format!( + "could not start the import reader: {}", + crate::runner::sandbox::spawn_error(&e) + ) + })?; + let Some(found) = ran.and_then(|ran| { + serde_json::from_str::)>>(ran.stdout.trim()).ok() + }) else { return Ok(Vec::new()); }; - let found: Vec<(String, Vec)> = - serde_json::from_str(ran.stdout.trim()).map_err(|e| { - format!( - "the import reader printed no imports ({e}, {}): {}", - ran.status, - ran.stderr.trim() - ) - })?; let mut imports: Imports = found .into_iter() .filter(|(name, names)| { @@ -1342,7 +1345,7 @@ impl PythonExecutor { async fn measure_idle_memory( &self, given: &UnitPlan, - imports: &Imports, + imports: &[(String, Vec)], ) -> Result { let dir = tempfile::tempdir().map_err(|e| format!("could not stage the idle unit: {e}"))?; let file = dir.path().join("idle.py"); @@ -1513,6 +1516,25 @@ mod tests { use super::*; use crate::runner::executor::CallPlan; + /// The reader finds every import by its tokens — wherever it is, nothing it does not + /// spell out — and runs none of the source. + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn test_imports_are_read_from_the_source_without_running_it() { + let source = "import os, sys as system\nfrom x.y import (p as q,\n r)\nfrom . import mine\n\ + if False: import padded\ndef f():\n import inner; s = 'import fake'\n # import no\n\ + open('ran', 'w')\nPAD = [1,\n 2]\nimport last\n"; + let reader = PythonExecutor::new().with_isolation(Isolation::Unisolated); + let found = reader.source_imports(source.as_bytes()).unwrap(); + let names: Vec<&str> = found.iter().map(|(m, _)| m.as_str()).collect(); + assert_eq!(names, ["inner", "last", "os", "padded", "sys", "x.y"]); + assert!(found.contains(&("x.y".into(), vec!["p".into(), "r".into()]))); + let garbled = reader + .source_imports(b"import ok\nx = '''never ends") + .unwrap(); + assert_eq!(garbled, [("ok".to_string(), Vec::::new())]); + } + /// The CPU limit is derived from the unit's own deadline — never the fixed 30 s it used /// to be, which a scenario of several long steps would outrun and be killed by. #[test] diff --git a/src/core/src/runner/sandbox.rs b/src/core/src/runner/sandbox.rs index d263a32..74a1f6c 100644 --- a/src/core/src/runner/sandbox.rs +++ b/src/core/src/runner/sandbox.rs @@ -174,6 +174,20 @@ pub fn available() -> Result<(), String> { platform::available() } +/// The errno a confined command fails to start with when its sandbox could not be applied +/// between fork and exec, where nothing may allocate to say more. +#[cfg(target_os = "linux")] +const SANDBOX_REFUSED: i32 = libc::ENOTSUP; + +/// Why a command could not start, in words: a sandbox that could not be applied says so. +pub fn spawn_error(e: &std::io::Error) -> String { + #[cfg(target_os = "linux")] + if e.raw_os_error() == Some(SANDBOX_REFUSED) { + return format!("its sandbox could not be applied ({e})"); + } + e.to_string() +} + /// A command that runs `program` held to `confinement`. Its arguments are the caller's to add. pub fn confined( program: &Path, @@ -411,17 +425,19 @@ mod platform { let mut cmd = std::process::Command::new(program); let mut ruleset = Some(ruleset); // SAFETY: between fork and exec this only makes the syscalls that apply a prepared - // ruleset and prepared filters; nothing is allocated unless one fails. + // ruleset and prepared filters, and fails with a constant errno: nothing allocates, + // as another thread of the grader may have held the allocator's lock at the fork. unsafe { cmd.pre_exec(move || { + let refused = || std::io::Error::from_raw_os_error(SANDBOX_REFUSED); if let Some(ruleset) = ruleset.take() { - let status = ruleset.restrict_self().map_err(std::io::Error::other)?; - if status.ruleset != RulesetStatus::FullyEnforced { - return Err(std::io::Error::other("Landlock was not fully enforced")); + match ruleset.restrict_self() { + Ok(status) if status.ruleset == RulesetStatus::FullyEnforced => {} + _ => return Err(refused()), } } for filter in &filters { - seccompiler::apply_filter(filter).map_err(std::io::Error::other)?; + seccompiler::apply_filter(filter).map_err(|_| refused())?; } Ok(()) }); diff --git a/src/core/tests/harness.rs b/src/core/tests/harness.rs index 674980b..be1fd59 100644 --- a/src/core/tests/harness.rs +++ b/src/core/tests/harness.rs @@ -866,12 +866,14 @@ async fn test_a_long_scenario_of_large_legal_values_is_not_a_flood() { function: "picky".into(), expected: None, }); - // About 10.75 MiB each on the wire: 75 MiB in all, past a fixed 64 MiB cap. + // About 10.75 MiB each on the wire: 75 MiB in all, past a fixed 64 MiB cap. This is + // about the record channel: the student side's copies of each value get room. plan.steps = vec![step; 7]; + plan.memory_mb = 512; let obs = run(&plan).await; assert!(obs.protocol_error.is_none(), "{:?}", obs.protocol_error); assert!(obs.done, "{obs:?}"); - assert_eq!(obs.steps.len(), 7); + assert_eq!(obs.steps.len(), 7, "{:?} {:?}", obs.exit, obs.stopped); assert_eq!(obs.checks.len(), 7); for step in &obs.steps { // The value itself arrived, at the limit rather than replaced by its size. @@ -1026,12 +1028,14 @@ async fn test_a_student_who_writes_into_the_pipe_only_breaks_their_own_unit() { } /// What a submission imports, wherever in its file, is loaded when ScriptMark measures what -/// its student side takes idle: no import is the student's. +/// its student side takes idle: no import is the student's. The module here, a teacher's +/// data file the spec allows, holds 32 MiB from the moment it is imported. #[tokio::test] async fn test_what_a_submission_imports_counts_as_idle() { let dir = tempfile::tempdir().unwrap(); + let heavy = write(dir.path(), "heavy.py", "BLOB = b'x' * (32 * 1024 * 1024)\n"); let mut idle = Vec::new(); - for (name, import) in [("bare.py", ""), ("imports.py", " import asyncio\n")] { + for (name, import) in [("bare.py", ""), ("imports.py", " import heavy\n")] { let student = write( dir.path(), name, @@ -1040,7 +1044,8 @@ async fn test_what_a_submission_imports_counts_as_idle() { ), ); let mut plan = unit(&student); - plan.allowed_imports = vec!["asyncio".into()]; + plan.data_files = vec![(heavy.clone(), "heavy.py".into())]; + plan.allowed_imports = vec!["heavy".into()]; plan.steps = vec![function("hog", vec![])]; plan.memory_mb = 16; match run(&plan).await.exit { @@ -1048,8 +1053,7 @@ async fn test_what_a_submission_imports_counts_as_idle() { other => panic!("{name}: {other:?}"), } } - // asyncio, which the harness does not load, takes 7 to 10 MiB to import. - assert!(idle[1] >= idle[0] + 4, "{idle:?} MiB"); + assert!(idle[1] >= idle[0] + 30, "{idle:?} MiB"); } /// Reading a submission's imports takes memory its source decides — about 140 MiB for the @@ -1076,6 +1080,24 @@ async fn test_what_reading_a_submission_takes_is_not_idle() { assert!(idle[1] <= idle[0] + 3, "{idle:?} MiB"); } +/// An import that exits when loaded — `unittest.__main__` runs the test runner — cannot +/// stop a submission's idle share from being measured, nor turn its units into the +/// environment's failure: it is measured without its imports. +#[tokio::test] +async fn test_an_import_that_exits_does_not_stop_the_measurement() { + let dir = tempfile::tempdir().unwrap(); + let student = write( + dir.path(), + "lab.py", + "def never():\n import unittest.__main__\n\ndef two():\n return 2\n", + ); + let mut plan = unit(&student); + plan.steps = vec![function("two", vec![])]; + let obs = run(&plan).await; + assert!(obs.done && obs.exit == Exit::Code(0), "{obs:?}"); + assert_eq!(returned(&obs.steps[0].outcome), &json!(2)); +} + /// An entry names a module: a dotted one allows its whole package, and one that is not /// installed does not keep the unit from running. #[tokio::test]