From e4accdf3c800464eac4fa4ee52fa1b63fe82480f Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Mon, 9 Nov 2026 17:56:10 -0500 Subject: [PATCH 1/4] Add data_files setting and tests_dir fixture ConfigObject(data_files=[...]) copies the listed tests-folder files into a new read-only folder that the sandbox starts in, so leading code and student code can read them by relative path, as in a Jupyter workspace. A missing listed file makes the submission ungradable. The tests_dir fixture gives tests the tests folder for reading the originals. run.sh now makes the merged run folder root-owned and not writable by group or others, so student code can't change files that tests read later when the host's file owner has ag's uid. Co-Authored-By: Claude Opus 5.5 --- .github/copilot-instructions.md | 4 ++ docker/run.sh | 9 ++++ quick_start.md | 48 ++++++++++++++++- src/pytest_prairielearn_grader/config.py | 36 +++++++++++++ src/pytest_prairielearn_grader/fixture.py | 53 ++++++++++++++++++- src/pytest_prairielearn_grader/plugin.py | 42 +++++++++++++++ .../scenario_root/test_data_files/__init__.py | 0 .../test_data_files/data/weights.txt | 2 + tests/scenario_root/test_data_files/edges.txt | 3 ++ .../test_data_files/expected_outcome.json | 38 +++++++++++++ .../test_data_files/leading_code.py | 3 ++ .../scenario_root/test_data_files/scenario.py | 35 ++++++++++++ .../test_data_files/student_code.py | 15 ++++++ .../test_data_files_missing/__init__.py | 0 .../expected_outcome.json | 13 +++++ .../test_data_files_missing/scenario.py | 13 +++++ .../test_data_files_missing/student_code.py | 1 + .../test_data_files_notebook/__init__.py | 0 .../test_data_files_notebook/edges.txt | 3 ++ .../expected_outcome.json | 20 +++++++ .../test_data_files_notebook/leading_code.py | 3 ++ .../test_data_files_notebook/scenario.py | 18 +++++++ .../student_code.ipynb | 42 +++++++++++++++ .../test_data_files_workspace/__init__.py | 0 .../test_data_files_workspace/edges.txt | 3 ++ .../expected_outcome.json | 20 +++++++ .../test_data_files_workspace/scenario.py | 16 ++++++ .../student/analysis.py | 5 ++ tests/test_config.py | 46 ++++++++++++++++ 29 files changed, 488 insertions(+), 3 deletions(-) create mode 100644 tests/scenario_root/test_data_files/__init__.py create mode 100644 tests/scenario_root/test_data_files/data/weights.txt create mode 100644 tests/scenario_root/test_data_files/edges.txt create mode 100644 tests/scenario_root/test_data_files/expected_outcome.json create mode 100644 tests/scenario_root/test_data_files/leading_code.py create mode 100644 tests/scenario_root/test_data_files/scenario.py create mode 100644 tests/scenario_root/test_data_files/student_code.py create mode 100644 tests/scenario_root/test_data_files_missing/__init__.py create mode 100644 tests/scenario_root/test_data_files_missing/expected_outcome.json create mode 100644 tests/scenario_root/test_data_files_missing/scenario.py create mode 100644 tests/scenario_root/test_data_files_missing/student_code.py create mode 100644 tests/scenario_root/test_data_files_notebook/__init__.py create mode 100644 tests/scenario_root/test_data_files_notebook/edges.txt create mode 100644 tests/scenario_root/test_data_files_notebook/expected_outcome.json create mode 100644 tests/scenario_root/test_data_files_notebook/leading_code.py create mode 100644 tests/scenario_root/test_data_files_notebook/scenario.py create mode 100644 tests/scenario_root/test_data_files_notebook/student_code.ipynb create mode 100644 tests/scenario_root/test_data_files_workspace/__init__.py create mode 100644 tests/scenario_root/test_data_files_workspace/edges.txt create mode 100644 tests/scenario_root/test_data_files_workspace/expected_outcome.json create mode 100644 tests/scenario_root/test_data_files_workspace/scenario.py create mode 100644 tests/scenario_root/test_data_files_workspace/student/analysis.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index c939487..ab57a93 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -146,6 +146,10 @@ result = sandbox.query_function("student_func", arg1, arg2, timeout=5) these modules, so they must not contain answers. In workspace mode, their names are reserved: a student file or folder with one fails the submission with a message asking the student to rename it. +`ConfigObject(data_files=["pagerank_large.txt"])` starts the sandbox in a new folder holding read-only +copies of the listed tests-folder files, so code in the sandbox can read them by relative path. Tests read +the originals through the `tests_dir` fixture. A missing listed file makes the submission ungradable. + ### 5. Serialization - Uses `dill` for complex Python objects (numpy arrays, pandas DataFrames, matplotlib plots) diff --git a/docker/run.sh b/docker/run.sh index 9d7aa4a..68e17b2 100644 --- a/docker/run.sh +++ b/docker/run.sh @@ -34,6 +34,15 @@ mv "$STUDENT_DIR" "$MERGE_DIR/test_student/student" mv $TEST_DIR/* $MERGE_DIR/test_student/ mv $MERGE_DIR/test_student/test_student.py $MERGE_DIR/test_student.py +# Student code runs as ag, whose uid may match the owner of files mounted from the host. Stop it from +# changing the tests and the submission, since tests read them after student code has run (for +# example, a data file the tests_dir fixture reads to compute expected values). +if ! { chown -R root:root "$MERGE_DIR" && chmod -R go-w "$MERGE_DIR"; }; then + echo '{"succeeded": false, "score": 0.0, "message": "The autograder could not protect its grading files. Please contact course staff and have them check the logs for this submission."}' > "$OUT_DIR/results.json" + echo "ERROR: could not protect $MERGE_DIR" + exit 1 +fi + # PrairieLearn writes data.json to the data directory, and pytest is told to read it from there. # It can hold correct_answers, so it's for the grader only. It must be owned by root with # owner-only modes: student code keeps root's supplementary groups, and its uid may match the owner diff --git a/quick_start.md b/quick_start.md index f90c6f9..3ad43dc 100644 --- a/quick_start.md +++ b/quick_start.md @@ -123,7 +123,7 @@ def test_with_output(sandbox: StudentFixture) -> None: ### Available Fixtures -Three fixtures are provided by the `pytest-prairielearn-grader` package: +These fixtures are provided by the `pytest-prairielearn-grader` package: 1. **`sandbox: StudentFixture`**: Provides sandboxed access to student code. Use this to query variables and call functions from the student's submission. @@ -137,6 +137,10 @@ Three fixtures are provided by the `pytest-prairielearn-grader` package: 4. **`data_json: DataFixture`**: Provides access to parameters from PrairieLearn's `data.json` file (generated via `pl-external-grader-variables` and other elements). +5. **`tests_dir: Path`**: The question's tests folder, which holds `leading_code.py` and any data files. + Use it to read instructor files in a test, for example `np.loadtxt(tests_dir / "pagerank_large.txt")`, + instead of building the path from `__file__`. + For implementation details, see the [fixture source code](https://github.com/prairielearn/grader-python-v2/blob/main/src/pytest_prairielearn_grader/fixture.py). ## Querying Student Code @@ -349,6 +353,7 @@ All parameters are optional with sensible defaults: - **`import_blacklist`** (list[str] | None): Blocked import modules (blacklist mode) - **`builtin_whitelist`** (list[str] | None): Allowed builtin functions - **`helper_modules`** (list[str] | None): Modules from the tests folder that can be imported (see [Helper Modules](#helper-modules)) +- **`data_files`** (list[str] | None): Files from the tests folder that code in the sandbox can read by relative path (see [Data Files](#data-files)) - **`names_for_user`** (list[str] | None): List of variable names to inject into student code - **`student_code_pattern`** (str, default="student_code\*.py"): Glob pattern for finding student files - **`starting_vars`** (dict[str, Any], default={}): Dictionary of variable values to provide for injection @@ -393,6 +398,47 @@ import helper_function as hf > setting also doesn't make the rest of the tests folder private. `importlib` isn't blacklisted by > default, so student code can load files in that folder by path. +#### Data Files + +In a workspace, code such as `np.loadtxt("pagerank_large.txt")` reads a file next to the notebook. To +make the same code work during grading, put the file in the tests folder and name it in `data_files`: + +```python +# tests/test_student.py, with the data in tests/pagerank_large.txt +autograder_config = ConfigObject( + data_files=["pagerank_large.txt"], + student_code_pattern="Markov-Chains-3.ipynb", + notebook_cell_tag="#grade", +) + + +def test_pagerank(sandbox: StudentFixture, tests_dir: Path) -> None: + edges = np.loadtxt(tests_dir / "pagerank_large.txt") # the instructor's original + ... +``` + +```python +# tests/leading_code.py, or the student's code +edges = np.loadtxt("pagerank_large.txt") +``` + +- **Paths:** each entry is relative to the tests folder, such as `"pagerank_large.txt"` or + `"data/points.csv"`, and keeps that relative path in the sandbox. Absolute paths and `..` aren't + accepted. +- **Starting folder:** the sandbox starts in a new folder that holds only copies of the listed files. + Without `data_files`, the starting folder isn't changed. +- **Read-only copies:** the copies and their folder are read-only for student code, and a fresh copy is + made for each sandbox. Tests should read the originals through the `tests_dir` fixture. +- **Reading, not importing:** the folder isn't on the import path, so a listed `.py` file can't be + imported. Use `helper_modules` for that. +- **Configuration errors:** if a listed file is missing, the submission is ungradable. The students + don't lose an attempt, and each test's message says what's wrong. +- **Fixtures:** it works with `sandbox` and `module_sandbox`, for `.py` and `.ipynb` submissions, and + with `workspace_sandbox`. In workspace mode, relative paths resolve against the data files' folder, + not the student's workspace folder. + +> **Warning:** Students can read the listed files, so they must not contain answers. + **How variable injection works**: - Only variables listed in `names_for_user` are injected into the student code namespace diff --git a/src/pytest_prairielearn_grader/config.py b/src/pytest_prairielearn_grader/config.py index 300c562..196eb9a 100644 --- a/src/pytest_prairielearn_grader/config.py +++ b/src/pytest_prairielearn_grader/config.py @@ -7,6 +7,12 @@ from typing import Literal +def _is_plain_relative_path(name: str) -> bool: + """Whether name is a relative file path with no '..', '.', empty or backslash parts.""" + # An empty part also rules out absolute paths, "a//b" and a trailing "/". + return bool(name.strip()) and "\\" not in name and all(part not in ("", ".", "..") for part in name.split("/")) + + @dataclass(frozen=True, slots=True, kw_only=True) class ConfigObject: """ @@ -87,6 +93,24 @@ def test_student_code(sandbox): Example: ["helper_function"] """ + data_files: list[str] | None = None + """Files from the tests folder that setup code, leading code and student code can read by relative path. + + Each entry is a path relative to the folder that holds leading_code.py, such as + "pagerank_large.txt" or "data/points.csv". Absolute paths and ".." aren't accepted. + + The sandbox starts in a fresh folder that holds a copy of each listed file at the same + relative path, so ``np.loadtxt("pagerank_large.txt")`` works as it does in a Jupyter + workspace. Student code can't change the copies, and changes to them couldn't reach the + originals anyway, which tests read through the ``tests_dir`` fixture. Without this setting + the sandbox's starting folder isn't changed. + + A listed file that doesn't exist makes the submission ungradable, with a message for the + instructor. Students can read the listed files, so they must not contain answers. + + Example: ["pagerank_large.txt"] + """ + builtin_whitelist: list[str] | None = None """List of allowed Python builtin functions that student code can use. @@ -264,6 +288,18 @@ def __post_init__(self) -> None: if not isinstance(module_name, str) or not module_name.isidentifier() or keyword.iskeyword(module_name): raise ValueError(f"helper_modules must contain plain module names without dots, paths or '.py', got: {module_name!r}") + # Validate data_files contains relative paths that stay inside the tests folder + if self.data_files is not None: + if not isinstance(self.data_files, list): + raise TypeError(f"data_files must be a list, got {type(self.data_files).__name__}") + if not self.data_files: + raise ValueError("data_files cannot be empty (use None for no data files)") + for file_name in self.data_files: + if not isinstance(file_name, str) or not _is_plain_relative_path(file_name): + raise ValueError(f"data_files must contain relative paths inside the tests folder, without '..', got: {file_name!r}") + if len(set(self.data_files)) != len(self.data_files): + raise ValueError(f"data_files lists the same file more than once: {self.data_files!r}") + # Validate builtin_whitelist contains non-empty strings if self.builtin_whitelist is not None: if not isinstance(self.builtin_whitelist, list): diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index a2c5d7d..d63b8c3 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -1,9 +1,11 @@ import json import logging import os +import shutil import socket import subprocess import sys +import tempfile from importlib.resources import files from pathlib import Path from typing import Any @@ -76,6 +78,33 @@ def _extract_notebook_code(notebook_path: Path, cell_tag: str | None = None) -> return "\n\n".join(code_parts) +def _make_data_files_dir(data_files: dict[str, Path]) -> Path: + """Copy each data file to its relative path in a new folder that only its owner can change. + + The folder is made by the grader process, so in the Docker image it's owned by root and the + sandbox user can read the copies but not change, add or remove anything. + """ + root = Path(tempfile.mkdtemp(prefix="pl-data-files-")) + try: + for relative_path, source in data_files.items(): + destination = root / relative_path + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + destination.chmod(0o444) + for directory in [root, *(path for path in root.rglob("*") if path.is_dir())]: + directory.chmod(0o555) + except BaseException: + _remove_data_files_dir(root) + raise + return root + + +def _remove_data_files_dir(root: Path) -> None: + for directory in [root, *(path for path in root.rglob("*") if path.is_dir())]: + directory.chmod(0o755) + shutil.rmtree(root, ignore_errors=True) + + class StudentFiles(NamedTuple): leading_file: Path trailing_file: Path @@ -139,6 +168,8 @@ class _SandboxBase: builtin_whitelist: list[str] | None names_for_user_list: list[str] | None worker_username: str | None + data_files: dict[str, Path] | None + _data_files_dir: Path | None _accumulated_stdout: list[str] _server_failure: str | None @@ -150,6 +181,7 @@ def __init__( builtin_whitelist: list[str] | None, names_for_user_list: list[str] | None, worker_username: str | None, + data_files: dict[str, Path] | None = None, ) -> None: self.import_whitelist = import_whitelist self.import_blacklist = import_blacklist @@ -157,6 +189,8 @@ def __init__( self.builtin_whitelist = builtin_whitelist self.names_for_user_list = names_for_user_list self.worker_username = worker_username + self.data_files = data_files + self._data_files_dir = None self.process = None self.student_socket = None self._accumulated_stdout = [] @@ -213,11 +247,17 @@ def try_drop_privileges() -> None: if self.worker_username is not None: drop_privileges(self.worker_username) + # With data files, the sandbox starts in a folder holding copies of them, so student code can + # read them by relative path. Without them, the starting folder is left as it was. + if self.data_files is not None: + self._data_files_dir = _make_data_files_dir(self.data_files) + self.process = subprocess.Popen( args=(sys.executable, SCRIPT_PATH), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=self._data_files_dir, preexec_fn=None if sys.platform == "win32" else try_drop_privileges, ) self._assert_process_running() @@ -337,6 +377,9 @@ def _cleanup(self) -> None: self.process.terminate() self.process.wait() self.process = None + if self._data_files_dir is not None: + _remove_data_files_dir(self._data_files_dir) + self._data_files_dir = None class StudentFixture(_SandboxBase): @@ -356,8 +399,11 @@ def __init__( worker_username: str | None, notebook_cell_tag: str | None = None, helper_modules: dict[str, str] | None = None, + data_files: dict[str, Path] | None = None, ) -> None: - super().__init__(import_whitelist, import_blacklist, starting_vars, builtin_whitelist, names_for_user_list, worker_username) + super().__init__( + import_whitelist, import_blacklist, starting_vars, builtin_whitelist, names_for_user_list, worker_username, data_files + ) self.leading_file = file_names.leading_file self.trailing_file = file_names.trailing_file self.student_code_file = file_names.student_code_file @@ -466,8 +512,11 @@ def __init__( worker_username: str | None, exec_entry: str | None = None, helper_modules: dict[str, str] | None = None, + data_files: dict[str, Path] | None = None, ) -> None: - super().__init__(import_whitelist, import_blacklist, starting_vars, builtin_whitelist, names_for_user_list, worker_username) + super().__init__( + import_whitelist, import_blacklist, starting_vars, builtin_whitelist, names_for_user_list, worker_username, data_files + ) self.workspace_dir = workspace_dir self.setup_code_file = setup_code_file self.exec_entry = exec_entry diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index 93e5ce4..fd1fe37 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -145,6 +145,42 @@ def _load_helper_modules( return sources +def _resolve_data_files(request: pytest.FixtureRequest, names: list[str] | None) -> dict[str, Path] | None: + """ + Find each file named by the data_files setting in the tests folder. + + A missing file is the instructor's to fix, so it makes the submission ungradable instead of + costing the student an attempt. + """ + if names is None: + return None + + data_dir = get_datadir(request.module) + paths: dict[str, Path] = {} + for name in names: + path = None if data_dir is None else data_dir / name + if path is None or not path.is_file(): + message = f"Autograder configuration error in data_files: can't use '{name}': {path} doesn't exist" + request.config.result_collector_plugin.collection_errors.append(message) # type: ignore[attr-defined] + raise SandboxStartupFailed(message) + paths[name] = path + return paths + + +@pytest.fixture(scope="module") +def tests_dir(request: pytest.FixtureRequest) -> Path: + """ + The question's tests folder, which holds leading_code.py, helper modules and data files. + + Tests can read instructor files from it, for example to compute expected values from the + same data file that data_files gives the student: ``np.loadtxt(tests_dir / "data.txt")``. + """ + data_dir = get_datadir(request.module) + if data_dir is None: + pytest.fail("tests_dir: the test module has no file, so its tests folder can't be found", pytrace=False) + return data_dir + + def _initialize_sandbox_fixture( request: pytest.FixtureRequest, data_json: dict[str, Any] | None, @@ -174,6 +210,7 @@ def _initialize_sandbox_fixture( config_dict: dict[str, Any] = {} notebook_cell_tag: str | None = None helper_module_names: list[str] | None = None + data_file_names: list[str] | None = None if hasattr(request, "module") and hasattr(request.module, "autograder_config"): config_object = request.module.autograder_config if not isinstance(config_object, ConfigObject): @@ -185,6 +222,7 @@ def _initialize_sandbox_fixture( config_starting_vars = config_object.starting_vars notebook_cell_tag = config_object.notebook_cell_tag helper_module_names = config_object.helper_modules + data_file_names = config_object.data_files else: # Check for module-level timeout variable (works for module-scoped fixtures) # This allows setting: sandbox_timeout = 0.5 at module level @@ -247,6 +285,7 @@ def _initialize_sandbox_fixture( worker_username=request.config.getoption("--worker-username"), notebook_cell_tag=notebook_cell_tag, helper_modules=_load_helper_modules(request, helper_module_names, import_blacklist), + data_files=_resolve_data_files(request, data_file_names), ) return fixture, initialization_timeout @@ -511,6 +550,7 @@ def _initialize_workspace_fixture( config_dict: dict[str, Any] = {} exec_entry: str | None = None helper_module_names: list[str] | None = None + data_file_names: list[str] | None = None if hasattr(request, "module") and hasattr(request.module, "autograder_config"): config_object = request.module.autograder_config if not isinstance(config_object, ConfigObject): @@ -520,6 +560,7 @@ def _initialize_workspace_fixture( config_starting_vars = config_object.starting_vars exec_entry = config_object.workspace_exec_entry helper_module_names = config_object.helper_modules + data_file_names = config_object.data_files else: if hasattr(request, "module") and hasattr(request.module, "sandbox_timeout"): initialization_timeout = request.module.sandbox_timeout @@ -566,6 +607,7 @@ def _initialize_workspace_fixture( worker_username=request.config.getoption("--worker-username"), exec_entry=exec_entry, helper_modules=_load_helper_modules(request, helper_module_names, import_blacklist), + data_files=_resolve_data_files(request, data_file_names), ) return fixture, initialization_timeout diff --git a/tests/scenario_root/test_data_files/__init__.py b/tests/scenario_root/test_data_files/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_data_files/data/weights.txt b/tests/scenario_root/test_data_files/data/weights.txt new file mode 100644 index 0000000..1ddc01a --- /dev/null +++ b/tests/scenario_root/test_data_files/data/weights.txt @@ -0,0 +1,2 @@ +1.5 +2.5 diff --git a/tests/scenario_root/test_data_files/edges.txt b/tests/scenario_root/test_data_files/edges.txt new file mode 100644 index 0000000..a329001 --- /dev/null +++ b/tests/scenario_root/test_data_files/edges.txt @@ -0,0 +1,3 @@ +0 1 +1 2 +2 0 diff --git a/tests/scenario_root/test_data_files/expected_outcome.json b/tests/scenario_root/test_data_files/expected_outcome.json new file mode 100644 index 0000000..c5b4caa --- /dev/null +++ b/tests/scenario_root/test_data_files/expected_outcome.json @@ -0,0 +1,38 @@ +{ + "expected_data_object": { + "score": 1.0, + "tests": [ + { + "max_points": 1, + "message": "", + "name": "relative paths", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_data_files.py::test_relative_paths[student_code]" + }, + { + "max_points": 1, + "message": "", + "name": "copies are read-only", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_data_files.py::test_copies_read_only[student_code]" + }, + { + "max_points": 1, + "message": "", + "name": "module sandbox", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_data_files.py::test_module_sandbox[student_code]" + } + ] + }, + "expected_outcome": { + "passed": 3, + "failed": 0 + } +} diff --git a/tests/scenario_root/test_data_files/leading_code.py b/tests/scenario_root/test_data_files/leading_code.py new file mode 100644 index 0000000..b40ae92 --- /dev/null +++ b/tests/scenario_root/test_data_files/leading_code.py @@ -0,0 +1,3 @@ +import numpy as np + +edges = np.loadtxt("edges.txt") diff --git a/tests/scenario_root/test_data_files/scenario.py b/tests/scenario_root/test_data_files/scenario.py new file mode 100644 index 0000000..0a30a7b --- /dev/null +++ b/tests/scenario_root/test_data_files/scenario.py @@ -0,0 +1,35 @@ +"""Leading code and student code read data files by relative path; tests read them from tests_dir.""" + +# No numpy here: pytester runs scenarios in this process and unloads their modules, and numpy fails to load twice. +import os +from pathlib import Path + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import StudentFixture + +autograder_config = ConfigObject(data_files=["edges.txt", "data/weights.txt"], sandbox_timeout=5) + +# Root can write to read-only files, so the write checks only hold for other users. +RUNNING_AS_ROOT = hasattr(os, "geteuid") and os.geteuid() == 0 + + +@pytest.mark.grading_data(name="relative paths", points=1) +def test_relative_paths(sandbox: StudentFixture, tests_dir: Path) -> None: + assert sandbox.query("edge_total") == sum(float(x) for x in (tests_dir / "edges.txt").read_text().split()) + assert sandbox.query("weights") == [1.5, 2.5] + + +@pytest.mark.grading_data(name="copies are read-only", points=1) +def test_copies_read_only(sandbox: StudentFixture, tests_dir: Path) -> None: + if not RUNNING_AS_ROOT: + assert sandbox.query("overwrite_error") == "PermissionError" + assert sandbox.query("create_error") == "PermissionError" + # The originals are never touched. + assert (tests_dir / "edges.txt").read_text() == "0 1\n1 2\n2 0\n" + + +@pytest.mark.grading_data(name="module sandbox", points=1) +def test_module_sandbox(module_sandbox: StudentFixture) -> None: + assert module_sandbox.query("weights") == [1.5, 2.5] diff --git a/tests/scenario_root/test_data_files/student_code.py b/tests/scenario_root/test_data_files/student_code.py new file mode 100644 index 0000000..64cb5a2 --- /dev/null +++ b/tests/scenario_root/test_data_files/student_code.py @@ -0,0 +1,15 @@ +edge_total = float(edges.sum()) +weights = np.loadtxt("data/weights.txt").tolist() + +# Student code must not be able to change the copies it reads. +try: + np.savetxt("edges.txt", np.zeros(2)) + overwrite_error = None +except OSError as e: + overwrite_error = e.__class__.__name__ + +try: + np.savetxt("new_file.txt", np.zeros(2)) + create_error = None +except OSError as e: + create_error = e.__class__.__name__ diff --git a/tests/scenario_root/test_data_files_missing/__init__.py b/tests/scenario_root/test_data_files_missing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_data_files_missing/expected_outcome.json b/tests/scenario_root/test_data_files_missing/expected_outcome.json new file mode 100644 index 0000000..972a99c --- /dev/null +++ b/tests/scenario_root/test_data_files_missing/expected_outcome.json @@ -0,0 +1,13 @@ +{ + "expected_data_object": { + "gradable": false, + "message": "The autograder encountered an internal error.", + "tests": [ + { + "test_id": "test_data_files_missing.py::test_query[student_code]", + "outcome": "error", + "message": "Autograder configuration error in data_files: can't use 'edges.txt':" + } + ] + } +} diff --git a/tests/scenario_root/test_data_files_missing/scenario.py b/tests/scenario_root/test_data_files_missing/scenario.py new file mode 100644 index 0000000..edd610e --- /dev/null +++ b/tests/scenario_root/test_data_files_missing/scenario.py @@ -0,0 +1,13 @@ +"""A listed data file that doesn't exist makes the submission ungradable, with a message for the instructor.""" + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import StudentFixture + +autograder_config = ConfigObject(data_files=["edges.txt"]) + + +@pytest.mark.grading_data(name="query", points=1) +def test_query(sandbox: StudentFixture) -> None: + assert sandbox.query("value") == 1 diff --git a/tests/scenario_root/test_data_files_missing/student_code.py b/tests/scenario_root/test_data_files_missing/student_code.py new file mode 100644 index 0000000..0a979d4 --- /dev/null +++ b/tests/scenario_root/test_data_files_missing/student_code.py @@ -0,0 +1 @@ +value = 1 diff --git a/tests/scenario_root/test_data_files_notebook/__init__.py b/tests/scenario_root/test_data_files_notebook/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_data_files_notebook/edges.txt b/tests/scenario_root/test_data_files_notebook/edges.txt new file mode 100644 index 0000000..a329001 --- /dev/null +++ b/tests/scenario_root/test_data_files_notebook/edges.txt @@ -0,0 +1,3 @@ +0 1 +1 2 +2 0 diff --git a/tests/scenario_root/test_data_files_notebook/expected_outcome.json b/tests/scenario_root/test_data_files_notebook/expected_outcome.json new file mode 100644 index 0000000..3a92805 --- /dev/null +++ b/tests/scenario_root/test_data_files_notebook/expected_outcome.json @@ -0,0 +1,20 @@ +{ + "expected_data_object": { + "score": 1.0, + "tests": [ + { + "max_points": 1, + "message": "", + "name": "notebook", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_data_files_notebook.py::test_notebook[student_code]" + } + ] + }, + "expected_outcome": { + "passed": 1, + "failed": 0 + } +} diff --git a/tests/scenario_root/test_data_files_notebook/leading_code.py b/tests/scenario_root/test_data_files_notebook/leading_code.py new file mode 100644 index 0000000..b40ae92 --- /dev/null +++ b/tests/scenario_root/test_data_files_notebook/leading_code.py @@ -0,0 +1,3 @@ +import numpy as np + +edges = np.loadtxt("edges.txt") diff --git a/tests/scenario_root/test_data_files_notebook/scenario.py b/tests/scenario_root/test_data_files_notebook/scenario.py new file mode 100644 index 0000000..9e46474 --- /dev/null +++ b/tests/scenario_root/test_data_files_notebook/scenario.py @@ -0,0 +1,18 @@ +"""Leading code and a tagged notebook cell both read a data file by relative path.""" + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import StudentFixture + +autograder_config = ConfigObject( + data_files=["edges.txt"], + student_code_pattern="student_code.ipynb", + notebook_cell_tag="#grade", + sandbox_timeout=5, +) + + +@pytest.mark.grading_data(name="notebook", points=1) +def test_notebook(sandbox: StudentFixture) -> None: + assert sandbox.query("edge_count") == 6 diff --git a/tests/scenario_root/test_data_files_notebook/student_code.ipynb b/tests/scenario_root/test_data_files_notebook/student_code.ipynb new file mode 100644 index 0000000..8a983be --- /dev/null +++ b/tests/scenario_root/test_data_files_notebook/student_code.ipynb @@ -0,0 +1,42 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "cell0", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "edges = np.loadtxt(\"edges.txt\")\n", + "raise ValueError(\"Not tagged, so this must not run\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cell1", + "metadata": {}, + "outputs": [], + "source": [ + "#grade\n", + "own_edges = np.loadtxt(\"edges.txt\")\n", + "edge_count = len(own_edges) + len(edges)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/scenario_root/test_data_files_workspace/__init__.py b/tests/scenario_root/test_data_files_workspace/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_data_files_workspace/edges.txt b/tests/scenario_root/test_data_files_workspace/edges.txt new file mode 100644 index 0000000..a329001 --- /dev/null +++ b/tests/scenario_root/test_data_files_workspace/edges.txt @@ -0,0 +1,3 @@ +0 1 +1 2 +2 0 diff --git a/tests/scenario_root/test_data_files_workspace/expected_outcome.json b/tests/scenario_root/test_data_files_workspace/expected_outcome.json new file mode 100644 index 0000000..3008a1d --- /dev/null +++ b/tests/scenario_root/test_data_files_workspace/expected_outcome.json @@ -0,0 +1,20 @@ +{ + "expected_data_object": { + "score": 1.0, + "tests": [ + { + "max_points": 1, + "message": "", + "name": "relative path", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_data_files_workspace.py::test_relative_path" + } + ] + }, + "expected_outcome": { + "passed": 1, + "failed": 0 + } +} diff --git a/tests/scenario_root/test_data_files_workspace/scenario.py b/tests/scenario_root/test_data_files_workspace/scenario.py new file mode 100644 index 0000000..a51b63d --- /dev/null +++ b/tests/scenario_root/test_data_files_workspace/scenario.py @@ -0,0 +1,16 @@ +"""Student modules in workspace mode read data files by relative path.""" + +from pathlib import Path + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import WorkspaceFixture + +autograder_config = ConfigObject(workspace_mode=True, data_files=["edges.txt"], sandbox_timeout=5) + + +@pytest.mark.grading_data(name="relative path", points=1) +def test_relative_path(workspace_sandbox: WorkspaceFixture, tests_dir: Path) -> None: + expected = sum(float(x) for x in (tests_dir / "edges.txt").read_text().split()) + assert workspace_sandbox.query_function("analysis.edge_total") == expected diff --git a/tests/scenario_root/test_data_files_workspace/student/analysis.py b/tests/scenario_root/test_data_files_workspace/student/analysis.py new file mode 100644 index 0000000..43044dc --- /dev/null +++ b/tests/scenario_root/test_data_files_workspace/student/analysis.py @@ -0,0 +1,5 @@ +import numpy as np + + +def edge_total(): + return float(np.loadtxt("edges.txt").sum()) diff --git a/tests/test_config.py b/tests/test_config.py index c82357e..020e614 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -307,3 +307,49 @@ def test_helper_modules_cannot_be_empty() -> None: def test_helper_modules_in_workspace_mode() -> None: config = ConfigObject(workspace_mode=True, helper_modules=["helper_function"]) assert config.helper_modules == ["helper_function"] + + +# --------------------------------------------------------------------------- +# data_files field +# --------------------------------------------------------------------------- + + +def test_data_files_default_is_none() -> None: + assert ConfigObject().data_files is None + + +def test_data_files_valid() -> None: + names = ["pagerank_large.txt", "data/points.csv", ".hidden"] + assert ConfigObject(data_files=names).data_files == names + + +def test_data_files_in_workspace_mode() -> None: + assert ConfigObject(workspace_mode=True, data_files=["a.txt"]).data_files == ["a.txt"] + + +def test_data_files_not_in_params() -> None: + assert "data_files" not in ConfigObject(data_files=["a.txt"]).to_dict() + + +@pytest.mark.parametrize( + "name", + ["", " ", ".", "..", "../ans.py", "data/../x.txt", "./a.txt", "/grade/tests/a.txt", "a//b", "data/", "a\\b", 5, None], +) +def test_data_files_rejects_invalid_paths(name: object) -> None: + with pytest.raises(ValueError, match="data_files must contain relative paths"): + ConfigObject(data_files=[name]) # type: ignore[list-item] + + +def test_data_files_must_be_a_list() -> None: + with pytest.raises(TypeError, match="data_files must be a list"): + ConfigObject(data_files="a.txt") # type: ignore[arg-type] + + +def test_data_files_cannot_be_empty() -> None: + with pytest.raises(ValueError, match="data_files cannot be empty"): + ConfigObject(data_files=[]) + + +def test_data_files_rejects_duplicates() -> None: + with pytest.raises(ValueError, match="more than once"): + ConfigObject(data_files=["a.txt", "a.txt"]) From 1ec00ba836632930e5b757dd6dd572231a0fa391 Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Tue, 10 Nov 2026 09:31:28 -0500 Subject: [PATCH 2/4] Check data_files paths with pathlib Use PurePosixPath so "/" is the separator on every OS and "." and repeated "/" parts are normalized, and compare normalized paths for duplicates. Keep rejecting backslashes and Windows drives, which would be a separator or an anchor on Windows. Co-Authored-By: Claude Opus 5.5 --- src/pytest_prairielearn_grader/config.py | 17 +++++++++++++---- tests/test_config.py | 12 +++++++++--- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/pytest_prairielearn_grader/config.py b/src/pytest_prairielearn_grader/config.py index 196eb9a..f80804a 100644 --- a/src/pytest_prairielearn_grader/config.py +++ b/src/pytest_prairielearn_grader/config.py @@ -3,14 +3,23 @@ import keyword from dataclasses import dataclass from dataclasses import field +from pathlib import PurePosixPath +from pathlib import PureWindowsPath from typing import Any from typing import Literal def _is_plain_relative_path(name: str) -> bool: - """Whether name is a relative file path with no '..', '.', empty or backslash parts.""" - # An empty part also rules out absolute paths, "a//b" and a trailing "/". - return bool(name.strip()) and "\\" not in name and all(part not in ("", ".", "..") for part in name.split("/")) + """Whether name is a relative path to a file that stays inside the folder it's resolved against. + + Entries always use "/", whatever the OS, so a config means the same thing on every machine. + Backslashes and drives are rejected because on Windows they'd be a separator or an anchor. + """ + if not name.strip() or "\\" in name or PureWindowsPath(name).drive: + return False + path = PurePosixPath(name) + # PurePosixPath drops "." and empty parts, so "." and "" have no parts left. + return not path.is_absolute() and bool(path.parts) and ".." not in path.parts @dataclass(frozen=True, slots=True, kw_only=True) @@ -297,7 +306,7 @@ def __post_init__(self) -> None: for file_name in self.data_files: if not isinstance(file_name, str) or not _is_plain_relative_path(file_name): raise ValueError(f"data_files must contain relative paths inside the tests folder, without '..', got: {file_name!r}") - if len(set(self.data_files)) != len(self.data_files): + if len({PurePosixPath(name) for name in self.data_files}) != len(self.data_files): raise ValueError(f"data_files lists the same file more than once: {self.data_files!r}") # Validate builtin_whitelist contains non-empty strings diff --git a/tests/test_config.py b/tests/test_config.py index 020e614..f772dbd 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -333,7 +333,7 @@ def test_data_files_not_in_params() -> None: @pytest.mark.parametrize( "name", - ["", " ", ".", "..", "../ans.py", "data/../x.txt", "./a.txt", "/grade/tests/a.txt", "a//b", "data/", "a\\b", 5, None], + ["", " ", ".", "./", "..", "../ans.py", "data/../x.txt", "/grade/tests/a.txt", "a\\b", "..\\a.txt", "C:a.txt", "C:/a.txt", 5, None], ) def test_data_files_rejects_invalid_paths(name: object) -> None: with pytest.raises(ValueError, match="data_files must contain relative paths"): @@ -350,6 +350,12 @@ def test_data_files_cannot_be_empty() -> None: ConfigObject(data_files=[]) -def test_data_files_rejects_duplicates() -> None: +def test_data_files_normalizes_paths() -> None: + # "/" is the separator on every OS, and "." and repeated "/" parts are dropped. + assert ConfigObject(data_files=["./a.txt", "data//b.txt"]).data_files == ["./a.txt", "data//b.txt"] + + +@pytest.mark.parametrize("names", [["a.txt", "a.txt"], ["a.txt", "./a.txt"], ["data/b.txt", "data//b.txt"]]) +def test_data_files_rejects_duplicates(names: list[str]) -> None: with pytest.raises(ValueError, match="more than once"): - ConfigObject(data_files=["a.txt", "a.txt"]) + ConfigObject(data_files=names) From 85f0a315f14e88750fcea449e5be67a322edda4c Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Tue, 10 Nov 2026 09:35:02 -0500 Subject: [PATCH 3/4] Name the data files folder's permission modes Build them from stat flags instead of octal literals. Co-Authored-By: Claude Opus 5.5 --- src/pytest_prairielearn_grader/fixture.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/pytest_prairielearn_grader/fixture.py b/src/pytest_prairielearn_grader/fixture.py index d63b8c3..253ebe5 100644 --- a/src/pytest_prairielearn_grader/fixture.py +++ b/src/pytest_prairielearn_grader/fixture.py @@ -3,6 +3,7 @@ import os import shutil import socket +import stat import subprocess import sys import tempfile @@ -35,6 +36,13 @@ SCRIPT_PATH = str(files("pytest_prairielearn_grader").joinpath("_student_code_runner.py")) DEFAULT_TIMEOUT = 1.0 +# Modes for the data files folder (see _make_data_files_dir). Everyone can read the files and list the +# folders, and nobody can change them; only the owner (root, in the Docker image) can make them writable again. +_READ_ONLY_FILE_MODE = stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH # r--r--r-- +_READ_ONLY_DIR_MODE = _READ_ONLY_FILE_MODE | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH # r-xr-xr-x +# Lets the owner delete the folder's contents at cleanup. +_OWNER_WRITABLE_DIR_MODE = _READ_ONLY_DIR_MODE | stat.S_IWUSR # rwxr-xr-x + logger = logging.getLogger(__name__) @@ -90,9 +98,9 @@ def _make_data_files_dir(data_files: dict[str, Path]) -> Path: destination = root / relative_path destination.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(source, destination) - destination.chmod(0o444) + destination.chmod(_READ_ONLY_FILE_MODE) for directory in [root, *(path for path in root.rglob("*") if path.is_dir())]: - directory.chmod(0o555) + directory.chmod(_READ_ONLY_DIR_MODE) except BaseException: _remove_data_files_dir(root) raise @@ -101,7 +109,7 @@ def _make_data_files_dir(data_files: dict[str, Path]) -> Path: def _remove_data_files_dir(root: Path) -> None: for directory in [root, *(path for path in root.rglob("*") if path.is_dir())]: - directory.chmod(0o755) + directory.chmod(_OWNER_WRITABLE_DIR_MODE) shutil.rmtree(root, ignore_errors=True) From 4c294e5d352b602ce560a6793ab6fa79603dbdcb Mon Sep 17 00:00:00 2001 From: Eliot Robson Date: Tue, 10 Nov 2026 09:53:21 -0500 Subject: [PATCH 4/4] Rename get_datadir to get_tests_dir Call the folder named after the test module the tests folder everywhere, to match the tests_dir fixture, and fix get_tests_dir's docstring, which described a 'data' directory. Co-Authored-By: Claude Opus 5.5 --- quick_start.md | 4 +- src/pytest_prairielearn_grader/plugin.py | 79 ++++++++++--------- .../test_docker_layout/scenario.py | 2 +- 3 files changed, 43 insertions(+), 42 deletions(-) diff --git a/quick_start.md b/quick_start.md index 3ad43dc..3996040 100644 --- a/quick_start.md +++ b/quick_start.md @@ -1374,7 +1374,7 @@ def test_result_variable(sandbox: StudentFixture, feedback: FeedbackFixture) -> In production PrairieLearn, the submitted `gradedFiles` end up in `/grade/student/`. The grader searches for matching files there automatically. For local development, place the notebook in the -scenario data directory next to your test file: +tests folder next to your test file (the folder named after it): ``` questions/my_notebook_question/ @@ -1435,7 +1435,7 @@ questions/my_workspace_question/ └── tests/ ├── test_student.py # your test file ├── setup_code.py # (optional) - └── test_student/ # ← mirrors the data directory layout + └── test_student/ # ← mirrors the tests folder layout └── student/ # ← local stand-in for /grade/student ├── calculator.py └── utils/ diff --git a/src/pytest_prairielearn_grader/plugin.py b/src/pytest_prairielearn_grader/plugin.py index fd1fe37..d9f1c4c 100644 --- a/src/pytest_prairielearn_grader/plugin.py +++ b/src/pytest_prairielearn_grader/plugin.py @@ -56,9 +56,13 @@ class TestResult(NamedTuple): stdout: str -def get_datadir(test_module: ModuleType) -> Path | None: +def get_tests_dir(test_module: ModuleType) -> Path | None: """ - Get the data directory for the current test module. + Get the tests folder for a test module: the folder named after the module, next to it. + + For /grade/run/test_student.py that's /grade/run/test_student/, which holds the question's + tests files (leading_code.py, helper modules, data files) and the student/ folder. + The tests_dir fixture exposes this to tests. """ if test_module is None: @@ -75,10 +79,7 @@ def get_datadir(test_module: ModuleType) -> Path | None: # Convert it to a pathlib.Path object for easier manipulation module_path = Path(module_filepath_str) - # Let's assume you have a 'data' directory next to your test file - data_dir = module_path.parent / module_path.stem - - return data_dir + return module_path.parent / module_path.stem def _find_data_json(request: pytest.FixtureRequest) -> Path | None: @@ -87,8 +88,8 @@ def _find_data_json(request: pytest.FixtureRequest) -> Path | None: if data_json_path is not None and Path(data_json_path).is_file(): return Path(data_json_path) - datadir = get_datadir(request.module) - return None if datadir is None else datadir / "data.json" + tests_dir = get_tests_dir(request.module) + return None if tests_dir is None else tests_dir / "data.json" @pytest.fixture(scope="module") @@ -106,11 +107,11 @@ def data_json(request: pytest.FixtureRequest) -> dict[str, Any] | None: except (OSError, ValueError) as e: raise RuntimeError(f"Could not load data.json: {e}") from e - # if datadir is None or not datadir.is_dir(): - # raise ValueError(f"Data directory '{datadir}' not found or is not a directory.") + # if tests_dir is None or not tests_dir.is_dir(): + # raise ValueError(f"Tests folder '{tests_dir}' not found or is not a directory.") # if not data_file.is_file(): - # raise ValueError(f"Data file '{data_file.name}' not found in '{datadir}'.") + # raise ValueError(f"Data file '{data_file.name}' not found in '{tests_dir}'.") def _load_helper_modules( @@ -127,10 +128,10 @@ def _load_helper_modules( if names is None: return None - data_dir = get_datadir(request.module) + tests_dir = get_tests_dir(request.module) sources: dict[str, str] = {} for name in names: - path = None if data_dir is None else data_dir / f"{name}.py" + path = None if tests_dir is None else tests_dir / f"{name}.py" try: if import_blacklist is not None and name in import_blacklist: raise ValueError("it's also in import_blacklist, so it can't be imported") @@ -155,10 +156,10 @@ def _resolve_data_files(request: pytest.FixtureRequest, names: list[str] | None) if names is None: return None - data_dir = get_datadir(request.module) + tests_dir = get_tests_dir(request.module) paths: dict[str, Path] = {} for name in names: - path = None if data_dir is None else data_dir / name + path = None if tests_dir is None else tests_dir / name if path is None or not path.is_file(): message = f"Autograder configuration error in data_files: can't use '{name}': {path} doesn't exist" request.config.result_collector_plugin.collection_errors.append(message) # type: ignore[attr-defined] @@ -175,10 +176,10 @@ def tests_dir(request: pytest.FixtureRequest) -> Path: Tests can read instructor files from it, for example to compute expected values from the same data file that data_files gives the student: ``np.loadtxt(tests_dir / "data.txt")``. """ - data_dir = get_datadir(request.module) - if data_dir is None: + path = get_tests_dir(request.module) + if path is None: pytest.fail("tests_dir: the test module has no file, so its tests folder can't be found", pytrace=False) - return data_dir + return path def _initialize_sandbox_fixture( @@ -453,10 +454,10 @@ def sandbox(request: pytest.FixtureRequest, data_json: dict[str, Any] | None) -> def _find_student_files(module: ModuleType) -> list[StudentFiles]: """ Find all student code files for a module and return StudentFiles objects. - Returns empty list if no data directory or no matching files found. + Returns empty list if no tests folder or no matching files found. """ - data_dir = get_datadir(module) - if data_dir is None or not data_dir.is_dir(): + tests_dir = get_tests_dir(module) + if tests_dir is None or not tests_dir.is_dir(): return [] # Check for ConfigObject first, then module-level pattern, then default @@ -464,18 +465,18 @@ def _find_student_files(module: ModuleType) -> list[StudentFiles]: student_code_pattern = module.autograder_config.student_code_pattern else: student_code_pattern = getattr(module, "student_code_pattern", "student_code*.py") - logger.debug(f"Looking for student code files in {data_dir} with pattern '{student_code_pattern}'") + logger.debug(f"Looking for student code files in {tests_dir} with pattern '{student_code_pattern}'") # Set up file paths - leading_file = data_dir / "leading_code.py" - trailing_file = data_dir / "trailing_code.py" - setup_code_file = data_dir / "setup_code.py" + leading_file = tests_dir / "leading_code.py" + trailing_file = tests_dir / "trailing_code.py" + setup_code_file = tests_dir / "setup_code.py" - student_code_files = list(data_dir.glob(student_code_pattern)) + student_code_files = list(tests_dir.glob(student_code_pattern)) # Fall back to student/ subdirectory — matches the production Docker layout where # run.sh moves /grade/student → test_student/student/ as a whole directory. if not student_code_files: - student_subdir = data_dir / "student" + student_subdir = tests_dir / "student" if student_subdir.is_dir(): student_code_files = list(student_subdir.glob(student_code_pattern)) logger.debug(f"Found {len(student_code_files)} student code file(s): {[f.name for f in student_code_files]}") @@ -490,8 +491,8 @@ def _find_workspace_dir(module: ModuleType) -> Path | None: Resolution order: 1. ``autograder_config.workspace_student_dir`` if set as an absolute path. 2. ``autograder_config.workspace_student_dir`` resolved relative to the - module's data directory (``//``). if set as a relative path. - 3. A sub-directory named ``"student"`` inside the module's data directory + module's tests folder (``//``). if set as a relative path. + 3. A sub-directory named ``"student"`` inside the module's tests folder (mirrors the ``/grade/student`` directory provided by PrairieLearn and the layout created by the pytester-based test runner). 4. A sub-directory named ``"student"`` in the same directory as the test @@ -505,10 +506,10 @@ def _find_workspace_dir(module: ModuleType) -> Path | None: p = Path(config_workspace_dir) if p.is_absolute(): return p if p.is_dir() else None - # Relative path: try resolving from data_dir first, then module parent - data_dir = get_datadir(module) - if data_dir is not None: - resolved = data_dir / p + # Relative path: try resolving from tests_dir first, then module parent + tests_dir = get_tests_dir(module) + if tests_dir is not None: + resolved = tests_dir / p if resolved.is_dir(): return resolved module_file = getattr(module, "__file__", None) @@ -522,9 +523,9 @@ def _find_workspace_dir(module: ModuleType) -> Path | None: # Check the data subdir first (matches the pytester layout used by the automated # scenario runner), then fall back to the module's own parent directory (the # natural layout when running a test file directly from its source location). - data_dir = get_datadir(module) - if data_dir is not None and data_dir.is_dir(): - candidate = data_dir / "student" + tests_dir = get_tests_dir(module) + if tests_dir is not None and tests_dir.is_dir(): + candidate = tests_dir / "student" if candidate.is_dir(): return candidate @@ -592,9 +593,9 @@ def _initialize_workspace_fixture( value = params_dict.get(name, None) starting_vars[name] = value - # Resolve setup_code_file from the test module's data directory - data_dir = get_datadir(request.module) - setup_code_file = (data_dir / "setup_code.py") if data_dir is not None else Path("setup_code.py") + # Resolve setup_code_file from the test module's tests folder + tests_dir = get_tests_dir(request.module) + setup_code_file = (tests_dir / "setup_code.py") if tests_dir is not None else Path("setup_code.py") fixture = WorkspaceFixture( workspace_dir=workspace_dir, diff --git a/tests/scenario_root/test_docker_layout/scenario.py b/tests/scenario_root/test_docker_layout/scenario.py index 22b17df..df5c433 100644 --- a/tests/scenario_root/test_docker_layout/scenario.py +++ b/tests/scenario_root/test_docker_layout/scenario.py @@ -1,7 +1,7 @@ """ Tests that the autograder correctly handles the Docker production layout, where run.sh places the student submission directory inside 'student/' rather than -scattering files at the data directory root. +scattering files at the tests folder root. Resulting directory structure (mirrors what run.sh produces):