Block blacklisted functions however student code reaches them

src/pytest_prairielearn_grader/_student_code_runner.py

@@ -209,6 +209,36 @@ def _find_blacklisted(name: str, fromlist: Sequence[str], import_blacklist: list
     return None
 
 
+def _block_blacklisted_functions(import_blacklist: list[str] | None) -> None:
+    """
+    Replaces each blacklisted function, such as "numpy.linalg.inv", with one that raises.
+
+    The importer only sees import statements, so on its own it can't stop `np.linalg.inv(A)`.
+    Modules are shared by everything in the sandbox, so this covers every way student code can
+    reach the function through its module. Entries whose module isn't installed, or that name a
+    module or a value that isn't callable, are left to the importer.
+    """
+    for entry in import_blacklist or []:
+        module_name, _, attribute = entry.rpartition(".")
+        if not module_name:
+            continue
+        try:
+            module = importlib.import_module(module_name)
+        except ImportError:
+            continue
+        value = getattr(module, attribute, None)
+        if value is None or isinstance(value, types.ModuleType) or not callable(value):
+            continue
+        setattr(module, attribute, _make_blocked_function(entry))
+
+
+def _make_blocked_function(entry: str) -> Callable[..., Any]:
+    def blocked(*args: Any, **kwargs: Any) -> Any:
+        raise RuntimeError(f"'{entry}' is blacklisted and cannot be used.")
+
+    return blocked
+
+
 class _HelperModules:
     """
     Loads the modules named by the helper_modules setting from the source sent by the grader.
@@ -407,6 +437,7 @@ async def student_code_runner(
         import_blacklist,
         helper_modules=None if helper_modules is None else _HelperModules(helper_modules, student_code_vars["__builtins__"]),
     )
+    _block_blacklisted_functions(import_blacklist)
 
     try:
         # First, execute the setup code if provided
@@ -537,6 +568,7 @@ async def workspace_runner(
         workspace_dir,
         helper_modules=None if helper_modules is None else _HelperModules(helper_modules, student_code_vars["__builtins__"]),
     )
+    _block_blacklisted_functions(import_blacklist)
     student_code_vars["__builtins__"]["__import__"] = workspace_importer
     # Patch the real builtins.__import__ so that workspace modules loaded via
     # importlib.import_module() (which use the real builtins, not student_code_vars)

src/pytest_prairielearn_grader/config.py

@@ -75,10 +75,13 @@ def test_student_code(sandbox):
 
     Modules in the blacklist are always blocked, even if in the whitelist, and so are their
     submodules: "os" also blocks `import os.path`.
+    An entry can also name a function in a module, such as "numpy.linalg.inv". The function then
+    raises a RuntimeError when it's called, however student code reached it, and importing it by
+    name is blocked. Setup code and helper modules can't call it either.
     Can be combined with import_whitelist for fine-grained control.
     Default blocks dangerous system operations: ["os", "sys", "subprocess", "pathlib", "shutil"].
 
-    Example: ["requests", "socket"]
+    Example: ["requests", "socket", "numpy.linalg.inv"]
     """
 
     helper_modules: list[str] | None = None

tests/scenario_root/test_import_blacklist_functions/expected_outcome.json

@@ -0,0 +1,43 @@
+{
+  "expected_data_object": {
+    "score": 1.0,
+    "tests": [
+      {
+        "max_points": 1,
+        "message": "",
+        "name": "called",
+        "outcome": "passed",
+        "points": 1.0,
+        "points_frac": 1.0,
+        "test_id": "test_import_blacklist_functions.py::test_called[student_code-numpy_attribute]"
+      },
+      {
+        "max_points": 1,
+        "message": "",
+        "name": "called",
+        "outcome": "passed",
+        "points": 1.0,
+        "points_frac": 1.0,
+        "test_id": "test_import_blacklist_functions.py::test_called[student_code-module_alias]"
+      },
+      {
+        "max_points": 1,
+        "message": "",
+        "name": "imported",
+        "outcome": "passed",
+        "points": 1.0,
+        "points_frac": 1.0,
+        "test_id": "test_import_blacklist_functions.py::test_imported[student_code]"
+      },
+      {
+        "max_points": 1,
+        "message": "",
+        "name": "allowed",
+        "outcome": "passed",
+        "points": 1.0,
+        "points_frac": 1.0,
+        "test_id": "test_import_blacklist_functions.py::test_allowed[student_code]"
+      }
+    ]
+  }
+}

tests/scenario_root/test_import_blacklist_functions/scenario.py

@@ -0,0 +1,28 @@
+"""A blacklisted function is blocked however student code reaches it, and the rest of its module isn't."""
+
+import pytest
+
+from pytest_prairielearn_grader import ConfigObject
+from pytest_prairielearn_grader.fixture import StudentFixture
+
+autograder_config = ConfigObject(import_blacklist=["numpy.linalg.inv"], sandbox_timeout=30)
+
+
+@pytest.mark.grading_data(name="called", points=1)
+@pytest.mark.parametrize("function_name", ["numpy_attribute", "module_alias"])
+def test_called(sandbox: StudentFixture, function_name: str) -> None:
+    response = sandbox.query_function_raw(function_name, query_timeout=30)
+    assert response["exception_name"] == "RuntimeError"
+    assert response["exception_message"] == "'numpy.linalg.inv' is blacklisted and cannot be used."
+
+
+@pytest.mark.grading_data(name="imported", points=1)
+def test_imported(sandbox: StudentFixture) -> None:
+    response = sandbox.query_function_raw("from_import", query_timeout=30)
+    assert response["exception_name"] == "ImportError"
+    assert "Module 'numpy.linalg.inv' is blacklisted" in str(response["exception_message"])
+
+
+@pytest.mark.grading_data(name="allowed", points=1)
+def test_allowed(sandbox: StudentFixture) -> None:
+    assert sandbox.query_function("other_function", query_timeout=30) == [0.5]

tests/scenario_root/test_import_blacklist_functions/student_code.py

@@ -0,0 +1,20 @@
+import numpy as np
+import numpy.linalg as la
+
+
+def numpy_attribute():
+    return np.linalg.inv([[2.0]]).tolist()
+
+
+def module_alias():
+    return la.inv([[2.0]]).tolist()
+
+
+def from_import():
+    from numpy.linalg import inv
+
+    return inv([[2.0]]).tolist()
+
+
+def other_function():
+    return la.solve([[2.0]], [1.0]).tolist()

tests/test_import_blacklist.py

@@ -1,7 +1,11 @@
 """How import_blacklist entries match the module an import would load."""
 
+import sys
+import types
+
 import pytest
 
+from pytest_prairielearn_grader._student_code_runner import _block_blacklisted_functions
 from pytest_prairielearn_grader._student_code_runner import _find_blacklisted
 
 
@@ -25,3 +29,17 @@
 )
 def test_find_blacklisted(name: str, fromlist: tuple[str, ...], blacklist: list[str], expected: str | None) -> None:
     assert _find_blacklisted(name, fromlist, blacklist) == expected
+
+
+def test_block_blacklisted_functions(monkeypatch: pytest.MonkeyPatch) -> None:
+    module = types.ModuleType("fake_linalg")
+    attributes = vars(module)
+    attributes.update(inv=lambda matrix: matrix, solve=lambda matrix, vector: vector, EPSILON=1e-8)
+    monkeypatch.setitem(sys.modules, "fake_linalg", module)
+
+    _block_blacklisted_functions(["os", "fake_linalg.inv", "fake_linalg.EPSILON", "missing_module.f"])
+
+    with pytest.raises(RuntimeError, match="'fake_linalg.inv' is blacklisted and cannot be used."):
+        attributes["inv"]([[1.0]])
+    assert attributes["solve"]([[1.0]], [2.0]) == [2.0]
+    assert attributes["EPSILON"] == 1e-8