Repository navigation
Expand file tree
/
Copy pathrunner.py
More file actions
105 lines (87 loc) · 3.97 KB
/
Copy pathrunner.py
File metadata and controls
105 lines (87 loc) · 3.97 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
"""Executes a learner's code and test cases in an isolated subprocess.
Usage: python -I runner.py <payload.json>
payload = {"code": str, "tests": [{"call": str, "expected": str}]}
Prints a single line starting with RESULT: followed by JSON.
Timeouts use a worker thread (not signals) so this works on Windows, macOS and Linux.
A stuck thread can't be killed, so the process exits with os._exit() once results are printed.
"""
import ast
import json
import os
import sys
import threading
import traceback
PER_TEST_SECONDS = 3
def run_with_timeout(fn, seconds):
"""Returns ("ok", value) | ("error", exc_or_traceback) | ("timeout", None)."""
box = {}
def target():
try:
box["ok"] = fn()
except BaseException as e:
box["error"] = e
box["tb"] = clean_tb(e)
t = threading.Thread(target=target, daemon=True)
t.start()
t.join(seconds)
if t.is_alive():
return "timeout", None
if "error" in box:
return "error", box
return "ok", box["ok"]
def clean_tb(exc):
"""Traceback limited to the learner's own code (cells / tests), without this runner's internals."""
frames = [f for f in traceback.extract_tb(exc.__traceback__) if f.filename.startswith("<")]
lines = ["Traceback (most recent call last):"] if frames else []
lines += [l.rstrip("\n") for l in traceback.format_list(frames)]
lines += [l.rstrip("\n") for l in traceback.format_exception_only(type(exc), exc)]
return "\n".join(lines)
def evaluate(src, ns):
"""Evaluates a test snippet. A plain expression works; so does a multi-line snippet whose
last line is an expression (earlier lines are executed first, e.g. to capture printed output)."""
tree = ast.parse(src.strip(), mode="exec")
if not tree.body or not isinstance(tree.body[-1], ast.Expr):
raise SyntaxError("a test must end with an expression whose value is compared")
last = ast.Expression(tree.body[-1].value)
tree.body = tree.body[:-1]
exec(compile(tree, "<test>", "exec"), ns)
return eval(compile(last, "<test>", "eval"), ns)
def main():
with open(sys.argv[1], encoding="utf-8") as f:
payload = json.load(f)
ns = {"__name__": "__main__"}
out = {"load_error": None, "cell_errors": [], "results": []}
# Run each notebook cell on its own, in one shared namespace, like Jupyter does. A scratch cell that
# raises (e.g. uses an undefined variable) is reported but must not stop the other cells or the tests.
cells = payload.get("cells") or [payload.get("code", "")]
for i, src in enumerate(cells, 1):
if not src.strip():
continue
status, val = run_with_timeout(lambda: exec(compile(src, f"<cell {i}>", "exec"), ns), PER_TEST_SECONDS * 2)
if status == "timeout":
out["cell_errors"].append({"cell": i, "error": f"Cell {i} did not finish within {PER_TEST_SECONDS * 2}s "
"(infinite loop outside a function?)."})
elif status == "error":
out["cell_errors"].append({"cell": i, "error": val["tb"]})
for t in payload["tests"]:
r = {"call": t["call"], "expected": t["expected"], "passed": False, "actual": None, "error": None}
try:
expected = evaluate(t["expected"], ns)
status, val = run_with_timeout(lambda: evaluate(t["call"], ns), PER_TEST_SECONDS)
if status == "timeout":
r["error"] = f"Timed out after {PER_TEST_SECONDS}s (infinite loop?)"
elif status == "error":
e = val["error"]
r["error"] = f"{type(e).__name__}: {e}"
else:
r["actual"] = repr(val)
r["passed"] = bool(val == expected)
except BaseException as e:
r["error"] = f"{type(e).__name__}: {e}"
out["results"].append(r)
return out
if __name__ == "__main__":
result = main()
print("RESULT:" + json.dumps(result), flush=True)
sys.stdout.flush()
os._exit(0)