diff --git a/src/pytest_prairielearn_grader/_student_code_runner.py b/src/pytest_prairielearn_grader/_student_code_runner.py index 7e5b842..79bd0f1 100644 --- a/src/pytest_prairielearn_grader/_student_code_runner.py +++ b/src/pytest_prairielearn_grader/_student_code_runner.py @@ -209,6 +209,36 @@ def _find_blacklisted(name: str, fromlist: Sequence[str], import_blacklist: list return None +def _block_blacklisted_functions(import_blacklist: list[str] | None) -> None: + """ + Replaces each blacklisted function, such as "numpy.linalg.inv", with one that raises. + + The importer only sees import statements, so on its own it can't stop `np.linalg.inv(A)`. + Modules are shared by everything in the sandbox, so this covers every way student code can + reach the function through its module. Entries whose module isn't installed, or that name a + module or a value that isn't callable, are left to the importer. + """ + for entry in import_blacklist or []: + module_name, _, attribute = entry.rpartition(".") + if not module_name: + continue + try: + module = importlib.import_module(module_name) + except ImportError: + continue + value = getattr(module, attribute, None) + if value is None or isinstance(value, types.ModuleType) or not callable(value): + continue + setattr(module, attribute, _make_blocked_function(entry)) + + +def _make_blocked_function(entry: str) -> Callable[..., Any]: + def blocked(*args: Any, **kwargs: Any) -> Any: + raise RuntimeError(f"'{entry}' is blacklisted and cannot be used.") + + return blocked + + class _HelperModules: """ Loads the modules named by the helper_modules setting from the source sent by the grader. @@ -407,6 +437,7 @@ async def student_code_runner( import_blacklist, helper_modules=None if helper_modules is None else _HelperModules(helper_modules, student_code_vars["__builtins__"]), ) + _block_blacklisted_functions(import_blacklist) try: # First, execute the setup code if provided @@ -537,6 +568,7 @@ async def workspace_runner( workspace_dir, helper_modules=None if helper_modules is None else _HelperModules(helper_modules, student_code_vars["__builtins__"]), ) + _block_blacklisted_functions(import_blacklist) student_code_vars["__builtins__"]["__import__"] = workspace_importer # Patch the real builtins.__import__ so that workspace modules loaded via # importlib.import_module() (which use the real builtins, not student_code_vars) diff --git a/src/pytest_prairielearn_grader/config.py b/src/pytest_prairielearn_grader/config.py index f80804a..a71b910 100644 --- a/src/pytest_prairielearn_grader/config.py +++ b/src/pytest_prairielearn_grader/config.py @@ -75,10 +75,13 @@ def test_student_code(sandbox): Modules in the blacklist are always blocked, even if in the whitelist, and so are their submodules: "os" also blocks `import os.path`. + An entry can also name a function in a module, such as "numpy.linalg.inv". The function then + raises a RuntimeError when it's called, however student code reached it, and importing it by + name is blocked. Setup code and helper modules can't call it either. Can be combined with import_whitelist for fine-grained control. Default blocks dangerous system operations: ["os", "sys", "subprocess", "pathlib", "shutil"]. - Example: ["requests", "socket"] + Example: ["requests", "socket", "numpy.linalg.inv"] """ helper_modules: list[str] | None = None diff --git a/tests/scenario_root/test_import_blacklist_functions/__init__.py b/tests/scenario_root/test_import_blacklist_functions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/scenario_root/test_import_blacklist_functions/expected_outcome.json b/tests/scenario_root/test_import_blacklist_functions/expected_outcome.json new file mode 100644 index 0000000..59e40a1 --- /dev/null +++ b/tests/scenario_root/test_import_blacklist_functions/expected_outcome.json @@ -0,0 +1,43 @@ +{ + "expected_data_object": { + "score": 1.0, + "tests": [ + { + "max_points": 1, + "message": "", + "name": "called", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_import_blacklist_functions.py::test_called[student_code-numpy_attribute]" + }, + { + "max_points": 1, + "message": "", + "name": "called", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_import_blacklist_functions.py::test_called[student_code-module_alias]" + }, + { + "max_points": 1, + "message": "", + "name": "imported", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_import_blacklist_functions.py::test_imported[student_code]" + }, + { + "max_points": 1, + "message": "", + "name": "allowed", + "outcome": "passed", + "points": 1.0, + "points_frac": 1.0, + "test_id": "test_import_blacklist_functions.py::test_allowed[student_code]" + } + ] + } +} diff --git a/tests/scenario_root/test_import_blacklist_functions/scenario.py b/tests/scenario_root/test_import_blacklist_functions/scenario.py new file mode 100644 index 0000000..33a7b74 --- /dev/null +++ b/tests/scenario_root/test_import_blacklist_functions/scenario.py @@ -0,0 +1,28 @@ +"""A blacklisted function is blocked however student code reaches it, and the rest of its module isn't.""" + +import pytest + +from pytest_prairielearn_grader import ConfigObject +from pytest_prairielearn_grader.fixture import StudentFixture + +autograder_config = ConfigObject(import_blacklist=["numpy.linalg.inv"], sandbox_timeout=30) + + +@pytest.mark.grading_data(name="called", points=1) +@pytest.mark.parametrize("function_name", ["numpy_attribute", "module_alias"]) +def test_called(sandbox: StudentFixture, function_name: str) -> None: + response = sandbox.query_function_raw(function_name, query_timeout=30) + assert response["exception_name"] == "RuntimeError" + assert response["exception_message"] == "'numpy.linalg.inv' is blacklisted and cannot be used." + + +@pytest.mark.grading_data(name="imported", points=1) +def test_imported(sandbox: StudentFixture) -> None: + response = sandbox.query_function_raw("from_import", query_timeout=30) + assert response["exception_name"] == "ImportError" + assert "Module 'numpy.linalg.inv' is blacklisted" in str(response["exception_message"]) + + +@pytest.mark.grading_data(name="allowed", points=1) +def test_allowed(sandbox: StudentFixture) -> None: + assert sandbox.query_function("other_function", query_timeout=30) == [0.5] diff --git a/tests/scenario_root/test_import_blacklist_functions/student_code.py b/tests/scenario_root/test_import_blacklist_functions/student_code.py new file mode 100644 index 0000000..5783a48 --- /dev/null +++ b/tests/scenario_root/test_import_blacklist_functions/student_code.py @@ -0,0 +1,20 @@ +import numpy as np +import numpy.linalg as la + + +def numpy_attribute(): + return np.linalg.inv([[2.0]]).tolist() + + +def module_alias(): + return la.inv([[2.0]]).tolist() + + +def from_import(): + from numpy.linalg import inv + + return inv([[2.0]]).tolist() + + +def other_function(): + return la.solve([[2.0]], [1.0]).tolist() diff --git a/tests/test_import_blacklist.py b/tests/test_import_blacklist.py index 2232db7..6632173 100644 --- a/tests/test_import_blacklist.py +++ b/tests/test_import_blacklist.py @@ -1,7 +1,11 @@ """How import_blacklist entries match the module an import would load.""" +import sys +import types + import pytest +from pytest_prairielearn_grader._student_code_runner import _block_blacklisted_functions from pytest_prairielearn_grader._student_code_runner import _find_blacklisted @@ -25,3 +29,17 @@ ) def test_find_blacklisted(name: str, fromlist: tuple[str, ...], blacklist: list[str], expected: str | None) -> None: assert _find_blacklisted(name, fromlist, blacklist) == expected + + +def test_block_blacklisted_functions(monkeypatch: pytest.MonkeyPatch) -> None: + module = types.ModuleType("fake_linalg") + attributes = vars(module) + attributes.update(inv=lambda matrix: matrix, solve=lambda matrix, vector: vector, EPSILON=1e-8) + monkeypatch.setitem(sys.modules, "fake_linalg", module) + + _block_blacklisted_functions(["os", "fake_linalg.inv", "fake_linalg.EPSILON", "missing_module.f"]) + + with pytest.raises(RuntimeError, match="'fake_linalg.inv' is blacklisted and cannot be used."): + attributes["inv"]([[1.0]]) + assert attributes["solve"]([[1.0]], [2.0]) == [2.0] + assert attributes["EPSILON"] == 1e-8