diff --git a/app/docs/criteria_graphs/physical_quantity_matches.svg b/app/docs/criteria_graphs/physical_quantity_matches.svg
new file mode 100644
index 0000000..f9c34eb
--- /dev/null
+++ b/app/docs/criteria_graphs/physical_quantity_matches.svg
@@ -0,0 +1,278 @@
+
+
+
+
+
diff --git a/app/docs/criteria_graphs/physical_quantity_matches_mermaid.svg b/app/docs/criteria_graphs/physical_quantity_matches_mermaid.svg
new file mode 100644
index 0000000..70b3ee7
--- /dev/null
+++ b/app/docs/criteria_graphs/physical_quantity_matches_mermaid.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/app/docs/criteria_graphs/physical_quantity_order.svg b/app/docs/criteria_graphs/physical_quantity_order.svg
new file mode 100644
index 0000000..9ed3f2c
--- /dev/null
+++ b/app/docs/criteria_graphs/physical_quantity_order.svg
@@ -0,0 +1,68 @@
+
+
+
+
+
diff --git a/app/docs/criteria_graphs/physical_quantity_order_mermaid.svg b/app/docs/criteria_graphs/physical_quantity_order_mermaid.svg
new file mode 100644
index 0000000..0b37479
--- /dev/null
+++ b/app/docs/criteria_graphs/physical_quantity_order_mermaid.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/app/docs/criteria_graphs/symbolic_equality.svg b/app/docs/criteria_graphs/symbolic_equality.svg
new file mode 100644
index 0000000..013c93a
--- /dev/null
+++ b/app/docs/criteria_graphs/symbolic_equality.svg
@@ -0,0 +1,133 @@
+
+
+
+
+
diff --git a/app/docs/criteria_graphs/symbolic_equality_mermaid.svg b/app/docs/criteria_graphs/symbolic_equality_mermaid.svg
new file mode 100644
index 0000000..3a08303
--- /dev/null
+++ b/app/docs/criteria_graphs/symbolic_equality_mermaid.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/app/docs/criteria_graphs/symbolic_equality_of_equations.svg b/app/docs/criteria_graphs/symbolic_equality_of_equations.svg
new file mode 100644
index 0000000..01f2785
--- /dev/null
+++ b/app/docs/criteria_graphs/symbolic_equality_of_equations.svg
@@ -0,0 +1,123 @@
+
+
+
+
+
diff --git a/app/docs/criteria_graphs/symbolic_equality_of_equations_mermaid.svg b/app/docs/criteria_graphs/symbolic_equality_of_equations_mermaid.svg
new file mode 100644
index 0000000..9b93b95
--- /dev/null
+++ b/app/docs/criteria_graphs/symbolic_equality_of_equations_mermaid.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/app/docs/criteria_graphs/symbolic_equality_of_sets.svg b/app/docs/criteria_graphs/symbolic_equality_of_sets.svg
new file mode 100644
index 0000000..3c224d9
--- /dev/null
+++ b/app/docs/criteria_graphs/symbolic_equality_of_sets.svg
@@ -0,0 +1,99 @@
+
+
+
+
+
diff --git a/app/docs/criteria_graphs/symbolic_equality_of_sets_mermaid.svg b/app/docs/criteria_graphs/symbolic_equality_of_sets_mermaid.svg
new file mode 100644
index 0000000..8c39155
--- /dev/null
+++ b/app/docs/criteria_graphs/symbolic_equality_of_sets_mermaid.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/app/docs/criteria_graphs/symbolic_other.svg b/app/docs/criteria_graphs/symbolic_other.svg
new file mode 100644
index 0000000..c3482ca
--- /dev/null
+++ b/app/docs/criteria_graphs/symbolic_other.svg
@@ -0,0 +1,84 @@
+
+
+
+
+
diff --git a/app/docs/criteria_graphs/symbolic_other_mermaid.svg b/app/docs/criteria_graphs/symbolic_other_mermaid.svg
new file mode 100644
index 0000000..2d1708b
--- /dev/null
+++ b/app/docs/criteria_graphs/symbolic_other_mermaid.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/app/docs/criteria_graphs/symbolic_where.svg b/app/docs/criteria_graphs/symbolic_where.svg
new file mode 100644
index 0000000..c0bf656
--- /dev/null
+++ b/app/docs/criteria_graphs/symbolic_where.svg
@@ -0,0 +1,59 @@
+
+
+
+
+
diff --git a/app/docs/criteria_graphs/symbolic_where_mermaid.svg b/app/docs/criteria_graphs/symbolic_where_mermaid.svg
new file mode 100644
index 0000000..f187891
--- /dev/null
+++ b/app/docs/criteria_graphs/symbolic_where_mermaid.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/app/docs/criteria_graphs/symbolic_written_as.svg b/app/docs/criteria_graphs/symbolic_written_as.svg
new file mode 100644
index 0000000..5ec41f0
--- /dev/null
+++ b/app/docs/criteria_graphs/symbolic_written_as.svg
@@ -0,0 +1,221 @@
+
+
+
+
+
diff --git a/app/docs/criteria_graphs/symbolic_written_as_mermaid.svg b/app/docs/criteria_graphs/symbolic_written_as_mermaid.svg
new file mode 100644
index 0000000..6455ff5
--- /dev/null
+++ b/app/docs/criteria_graphs/symbolic_written_as_mermaid.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/app/docs/dev.md b/app/docs/dev.md
index 2b6c2bc..a5acd55 100644
--- a/app/docs/dev.md
+++ b/app/docs/dev.md
@@ -1,4 +1,3 @@
-=======
# CompareExpressions
This function utilises the [`SymPy`](https://docs.sympy.org/latest/index.html) to provide a maths-aware evaluation of a learner's response.
@@ -45,6 +44,57 @@ The overall flow of the evaluation procedure can be described as follows:
**TODO** Describe what further information is supplied when `include_test_data` is set to true.
+#### Process graphs
+
+These graphs come from running `evaluation_function` on example inputs. `_evaluate` logs a `stage_start` event as it enters each stage, and these are the stages:
+
+- `setup`: copies the tolerance aliases to `rtol` and `atol` and creates the `EvaluationResult`.
+- `determine_context`: `select_context` chooses the context from `params`, and the context's default parameters and criteria are added.
+- `parse_latex`: runs only when `is_latex` is set, and reads the LaTeX response as SymPy syntax.
+- `parse_reserved_expressions`: preprocesses and parses the response and answer. If either can't be parsed, the result is serialised here with its feedback and returned early.
+- `preview`: sets the LaTeX and simplified forms of the response.
+- `parse_criteria`: parses the criteria with the context's criteria parser.
+- `generate_feedback_procedures`: the context builds a criteria graph for each criterion.
+- `generate_feedback`: runs each criteria graph on the response and collects the feedback and tags.
+- `serialise`: serialises the result.
+
+Each example has two graphs:
+- The stages, with the criteria graph the response went through. The path the response took through it is highlighted in orange. Stages that didn't run are greyed out.
+- The `app` and `lambdafeedback` functions called in each stage, up to two calls below `_evaluate`. Repeated calls from the same caller are shown once and marked ×n.
+
+**Symbolic:** response `1 + x`, answer `x + 1`, default criteria.
+
+
+
+Function calls
+
+
+
+
+**Physical quantity:** response `100 cm`, answer `1 m`, `physical_quantity: true`.
+
+
+
+Function calls
+
+
+
+
+**Parse error:** response `x +* 1`, answer `x + 1`. The response can't be parsed, so the evaluation returns after `parse_reserved_expressions`.
+
+
+
+Function calls
+
+
+
+
+To regenerate the images after changing the evaluation, run this from the repository root (it needs graphviz, i.e. `dot`):
+
+```bash
+python -m scripts.generate_process_graphs
+```
+
### Context
The context is a frozen `Context` dataclass, defined in `context/base.py`, with the following fields:
@@ -196,6 +246,76 @@ along the the following base tokens:
- For each criterion; run the corresponding procedure and store the result, the feedback string and the list of tags
- If all criteria are found to be true, then the response is considered correct
+### Criteria graphs
+Each criterion's feedback procedure is a `CriteriaGraph` (from `lambdafeedback.criteria`), built by the graph template the context picks for the criterion. A graph has three kinds of node:
+- **Evaluation nodes** (rounded, blue) run a check on the response and report which of the criterion nodes after them are reached.
+- **Criterion nodes** (rectangles) are the outcomes of an evaluation; each one produces the feedback for that outcome and its label is added to the tags.
+- **Output nodes** (hexagons) end the evaluation, usually `END`.
+
+A dotted edge is a sufficiency: when deciding where to start, an evaluation can be replaced by the evaluations leading to the criteria it points to. The response satisfies a criterion when `_TRUE` is reached.
+
+The graphs below are for one example answer each, since the shape can depend on the answer (e.g. whether it is an expression, an equation or a set). Each image is drawn with graphviz from the graph's nodes; the mermaid version, from `CriteriaGraph.to_mermaid` (the `criteria_graphs_vis` field in the test data), is under each image. Regenerate them with `python -m scripts.generate_criteria_graphs` from the repository root; it needs graphviz, and Node.js for the mermaid images (if puppeteer can't find Chrome, set `PUPPETEER_EXECUTABLE_PATH` to a local Chrome).
+
+#### `symbolic`: `response = answer`, answer `x + 1`
+
+Mermaid
+
+
+
+
+#### `symbolic`: `response = answer`, answer `x = 1`
+
+Mermaid
+
+
+
+
+#### `symbolic`: `response = answer`, answer `{1, 2}`
+
+Mermaid
+
+
+
+
+#### `symbolic`: `response = answer where x = 0`
+
+Mermaid
+
+
+
+
+#### `symbolic`: `response written as answer`
+
+Mermaid
+
+
+
+
+#### `symbolic`: other criteria, e.g. `response contains x`
+Criteria without their own template (e.g. `contains` and order comparisons such as `response > answer`) use the generic graph.
+
+
+Mermaid
+
+
+
+
+#### `physical_quantity`: `response matches answer`, answer `1 m`
+
+Mermaid
+
+
+
+
+#### `physical_quantity`: `response > answer`, answer `1 m`
+`>=`, `<` and `<=` give the same graph with their own operator.
+
+
+Mermaid
+
+
+
+
### Tag conventions
The feedback procedures consists of a series of function calls, the specifics are determined by the particular criteria, that each return a list of strings (called tags). Each tag then indicates what further function calls must be performed to continue the evaluation, as well as what feedback string (if any) should be generated. When there are no remaining function calls the feedback procedure is completed. The tags are formatted according as *criteria*`_`*name of function call outcome*. For tags that are not connected to a specific criteria (e.g. tags that indicate an issue with expression parsing) the criteria name and underscore is omitted.
diff --git a/app/docs/process_graphs/process_physical_quantity.svg b/app/docs/process_graphs/process_physical_quantity.svg
new file mode 100644
index 0000000..8aa84c5
--- /dev/null
+++ b/app/docs/process_graphs/process_physical_quantity.svg
@@ -0,0 +1,421 @@
+
+
+
+
+
diff --git a/app/docs/process_graphs/process_physical_quantity_calls.svg b/app/docs/process_graphs/process_physical_quantity_calls.svg
new file mode 100644
index 0000000..a091b1d
--- /dev/null
+++ b/app/docs/process_graphs/process_physical_quantity_calls.svg
@@ -0,0 +1,671 @@
+
+
+
+
+
diff --git a/app/docs/process_graphs/process_symbolic.svg b/app/docs/process_graphs/process_symbolic.svg
new file mode 100644
index 0000000..36815d4
--- /dev/null
+++ b/app/docs/process_graphs/process_symbolic.svg
@@ -0,0 +1,275 @@
+
+
+
+
+
diff --git a/app/docs/process_graphs/process_symbolic_calls.svg b/app/docs/process_graphs/process_symbolic_calls.svg
new file mode 100644
index 0000000..c4067a2
--- /dev/null
+++ b/app/docs/process_graphs/process_symbolic_calls.svg
@@ -0,0 +1,670 @@
+
+
+
+
+
diff --git a/app/docs/process_graphs/process_symbolic_parse_error.svg b/app/docs/process_graphs/process_symbolic_parse_error.svg
new file mode 100644
index 0000000..9da48c4
--- /dev/null
+++ b/app/docs/process_graphs/process_symbolic_parse_error.svg
@@ -0,0 +1,141 @@
+
+
+
+
+
diff --git a/app/docs/process_graphs/process_symbolic_parse_error_calls.svg b/app/docs/process_graphs/process_symbolic_parse_error_calls.svg
new file mode 100644
index 0000000..d6c9853
--- /dev/null
+++ b/app/docs/process_graphs/process_symbolic_parse_error_calls.svg
@@ -0,0 +1,400 @@
+
+
+
+
+
diff --git a/scripts/generate_criteria_graphs.py b/scripts/generate_criteria_graphs.py
new file mode 100644
index 0000000..0854598
--- /dev/null
+++ b/scripts/generate_criteria_graphs.py
@@ -0,0 +1,171 @@
+"""Generates the criteria graph images in `app/docs/criteria_graphs/` for the developer documentation.
+
+Each graph is built the way `evaluation_function` builds it (without generating any feedback) and drawn twice:
+- `_mermaid.svg`: the mermaid source from `CriteriaGraph.to_mermaid`, rendered with mermaid-cli
+ (`npx @mermaid-js/mermaid-cli`).
+- `.svg`: drawn with graphviz straight from the graph's evaluation, criterion and output nodes.
+
+Run from the repository root with `python -m scripts.generate_criteria_graphs`.
+"""
+from __future__ import annotations
+
+import html
+import subprocess
+import tempfile
+import textwrap
+from collections.abc import Collection
+from pathlib import Path
+from typing import Any
+
+import pydot
+from lambdafeedback.criteria import CriteriaGraph, EvaluationNode
+
+from app.evaluation import (
+ EvaluationResult,
+ _build_evaluation_parameters,
+ _build_parameters,
+ create_criteria_dict,
+ parse_reserved_expressions,
+ select_context,
+)
+
+OUTPUT_DIRECTORY = Path(__file__).resolve().parent.parent / "app" / "docs" / "criteria_graphs"
+
+# (file name, answer, params): one example of each shape of graph the contexts build.
+EXAMPLES: list[tuple[str, str, dict[str, Any]]] = [
+ ("symbolic_equality", "x + 1", {"criteria": "response = answer"}),
+ ("symbolic_equality_of_equations", "x = 1", {"criteria": "response = answer"}),
+ ("symbolic_equality_of_sets", "{1, 2}", {"criteria": "response = answer"}),
+ ("symbolic_where", "x", {"criteria": "response = answer where x = 0"}),
+ ("symbolic_written_as", "x + 1", {"criteria": "response written as answer"}),
+ ("symbolic_other", "x", {"criteria": "response contains x"}),
+ ("physical_quantity_matches", "1 m", {"physical_quantity": True, "criteria": "response matches answer"}),
+ ("physical_quantity_order", "1 m", {"physical_quantity": True, "criteria": "response > answer"}),
+]
+
+# Graphviz style for each kind of node, close to the shapes `CriteriaGraph.to_mermaid` uses.
+NODE_STYLES = {
+ "evaluation": {"shape": "box", "style": "rounded,filled", "fillcolor": "#E5F8FB", "color": "#00B8D4"},
+ "criterion": {"shape": "box", "style": "filled", "fillcolor": "#FFFFFF", "color": "#212121"},
+ "output": {"shape": "hexagon", "style": "filled", "fillcolor": "#C5CAE9", "color": "#2F3C86"},
+}
+
+# Extra style for the nodes and edges on the path a response took (see `add_criteria_graph`).
+REACHED_NODE_STYLE = {"color": "#E65100", "penwidth": "3"}
+REACHED_EDGE_STYLE = {"color": "#E65100", "penwidth": "2.5"}
+
+# Characters per line of a node's details in the graphviz drawings (mermaid wraps them itself).
+DETAILS_WIDTH = 45
+
+
+def build_graph(answer: str, params: dict[str, Any]) -> CriteriaGraph:
+ """The criteria graph `evaluation_function` builds for `answer` and the single criterion in `params`.
+
+ The answer is used as the response too: the graph's shape depends only on the answer, not on the response.
+ """
+ params = dict(params)
+ evaluation_result = EvaluationResult()
+ context = select_context(params)
+ parameters = _build_parameters(context, params, answer, answer)
+ success, reserved_expressions = parse_reserved_expressions(
+ context, parameters["reserved_expressions_strings"], parameters, evaluation_result
+ )
+ if success is False:
+ raise ValueError(f"Cannot parse the answer `{answer}`.")
+ criteria_parser = context.generate_criteria_parser(reserved_expressions)
+ criteria = create_criteria_dict(context, criteria_parser, parameters)
+ evaluation_parameters = _build_evaluation_parameters(parameters, reserved_expressions, criteria, evaluation_result)
+ graphs = context.feedback_procedure_generator(evaluation_parameters)
+ if len(graphs) != 1:
+ raise ValueError(f"Expected one criterion, got {len(graphs)}.")
+ return next(iter(graphs.values()))
+
+
+def _node_label(label: str, details: str) -> str:
+ """A graphviz HTML-like label with the node's label in bold above its details, wrapped to `DETAILS_WIDTH`."""
+ details_lines = " ".join(html.escape(line) for line in textwrap.wrap(details, DETAILS_WIDTH))
+ return f"<{html.escape(label)} {details_lines}>"
+
+
+def reached_nodes(graph: CriteriaGraph, reached_criteria: Collection[str]) -> set[str]:
+ """The labels of the nodes on the path through `graph` that reached `reached_criteria`.
+
+ These are the reached criteria, the evaluations that reported them and the output nodes they lead to.
+ """
+ reached = {label for label in graph.criteria if label in reached_criteria}
+ for (label, node) in graph.evaluations.items():
+ if any(edge.target.label in reached for edge in node.outgoing):
+ reached.add(label)
+ for label in graph.outputs:
+ if any(edge.source.label in reached for edge in graph.outputs[label].incoming):
+ reached.add(label)
+ return reached
+
+
+def add_criteria_graph(
+ container: pydot.Graph, graph: CriteriaGraph, prefix: str = "", reached_criteria: Collection[str] = ()
+) -> dict[str, str]:
+ """Adds the nodes and edges of `graph` to `container` and returns the graphviz identifier of each node label.
+
+ Sufficiencies are dotted edges. The path that reached `reached_criteria` is highlighted, and `prefix` keeps the
+ identifiers apart from other nodes in the same drawing.
+ """
+ groups = [("evaluation", graph.evaluations), ("criterion", graph.criteria), ("output", graph.outputs)]
+ identifiers = {label: f"{prefix}{kind}_{index}" for (kind, nodes) in groups for (index, label) in enumerate(nodes)}
+ reached = reached_nodes(graph, reached_criteria)
+ for (kind, nodes) in groups:
+ for (label, node) in nodes.items():
+ style = dict(NODE_STYLES[kind])
+ if label in reached:
+ style.update(REACHED_NODE_STYLE)
+ container.add_node(pydot.Node(identifiers[label], label=_node_label(label, node.details), **style))
+ for (kind, nodes) in groups:
+ for (label, node) in nodes.items():
+ for edge in node.outgoing:
+ if label in reached and edge.target.label in reached:
+ container.add_edge(pydot.Edge(identifiers[label], identifiers[edge.target.label], **REACHED_EDGE_STYLE))
+ else:
+ container.add_edge(pydot.Edge(identifiers[label], identifiers[edge.target.label]))
+ if isinstance(node, EvaluationNode):
+ for sufficiency in node.sufficiencies:
+ if sufficiency in identifiers:
+ container.add_edge(pydot.Edge(identifiers[label], identifiers[sufficiency], style="dotted"))
+ return identifiers
+
+
+def new_drawing() -> pydot.Dot:
+ """An empty top-to-bottom graphviz drawing with the styles shared by the documentation graphs."""
+ dot = pydot.Dot(graph_type="digraph", rankdir="TB", fontname="Helvetica", bgcolor="white")
+ dot.set_node_defaults(fontname="Helvetica", fontsize="12")
+ return dot
+
+
+def graph_to_dot(graph: CriteriaGraph) -> pydot.Dot:
+ """`graph` drawn with graphviz from its nodes and edges; sufficiencies are dotted edges."""
+ dot = new_drawing()
+ add_criteria_graph(dot, graph)
+ return dot
+
+
+def render_mermaid(mermaid: str, target: Path) -> None:
+ """Renders the mermaid source `mermaid` to the SVG file `target`; the source itself is not kept."""
+ with tempfile.TemporaryDirectory() as directory:
+ source = Path(directory) / "graph.mmd"
+ source.write_text(mermaid + "\n")
+ subprocess.run(
+ ["npx", "--yes", "@mermaid-js/mermaid-cli@11.4.2", "--quiet", "-i", str(source), "-o", str(target)],
+ check=True,
+ )
+
+
+def main() -> None:
+ OUTPUT_DIRECTORY.mkdir(parents=True, exist_ok=True)
+ for (name, answer, params) in EXAMPLES:
+ graph = build_graph(answer, params)
+ graph_to_dot(graph).write_svg(str(OUTPUT_DIRECTORY / f"{name}.svg"))
+ render_mermaid(graph.to_mermaid(), OUTPUT_DIRECTORY / f"{name}_mermaid.svg")
+ print(f"Generated {name}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/generate_process_graphs.py b/scripts/generate_process_graphs.py
new file mode 100644
index 0000000..a611421
--- /dev/null
+++ b/scripts/generate_process_graphs.py
@@ -0,0 +1,258 @@
+"""Generates the process graph images in `app/docs/process_graphs/` for the developer documentation.
+
+Each example is run through `evaluation_function` while recording what happens, and drawn twice with graphviz:
+- `.svg`: the stages of the evaluation (from the `stage_start` events `_evaluate` logs), with the criteria graph
+ the response went through and the path it took highlighted.
+- `_calls.svg`: the same stages, each with the `app` and `lambdafeedback` functions called during it (recorded
+ with `sys.setprofile`), down to `CALL_DEPTH` calls below `_evaluate`.
+
+Run from the repository root with `python -m scripts.generate_process_graphs`.
+"""
+from __future__ import annotations
+
+import html
+import inspect
+import json
+import logging
+import re
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from types import FrameType
+from typing import Any
+
+import pydot
+from lambdafeedback.criteria import CriteriaGraph
+
+from app import evaluation
+from scripts.generate_criteria_graphs import _node_label, add_criteria_graph, new_drawing
+
+OUTPUT_DIRECTORY = Path(__file__).resolve().parent.parent / "app" / "docs" / "process_graphs"
+
+# (file name, response, answer, params)
+EXAMPLES: list[tuple[str, str, str, dict[str, Any]]] = [
+ ("process_symbolic", "1 + x", "x + 1", {}),
+ ("process_physical_quantity", "100 cm", "1 m", {"physical_quantity": True}),
+ ("process_symbolic_parse_error", "x +* 1", "x + 1", {}),
+]
+
+# How many calls below `_evaluate` the call graphs go.
+CALL_DEPTH = 2
+
+# Functions left out of the call graphs; the functions they call are shown under their caller instead.
+HIDDEN_FUNCTIONS = {"log_event", "elapsed_ms", "_evaluate..enter_stage"}
+HIDDEN_FUNCTION_PARTS = ("", "", "", "", "", "__create_fn__")
+
+# The stage `_evaluate` is in before its first `enter_stage` call.
+FIRST_STAGE = "setup"
+
+STAGE_STYLE = {"shape": "box", "style": "rounded,filled", "fillcolor": "#E8EAF6", "color": "#2F3C86"}
+SKIPPED_STAGE_STYLE = {"shape": "box", "style": "rounded,dashed", "color": "#9E9E9E", "fontcolor": "#9E9E9E"}
+RESULT_STYLE = {"shape": "hexagon", "style": "filled", "fillcolor": "#C5CAE9", "color": "#2F3C86"}
+CALL_STYLE = {"shape": "box", "style": "filled", "fillcolor": "#FFFFFF", "color": "#212121"}
+PIPELINE_EDGE_STYLE = {"penwidth": "2", "weight": "10"}
+
+
+def all_stages() -> list[str]:
+ """Every stage `_evaluate` can enter, in order, read from its `enter_stage` calls."""
+ return [FIRST_STAGE] + re.findall(r'enter_stage\("(\w+)"\)', inspect.getsource(evaluation._evaluate))
+
+
+@dataclass
+class Call:
+ """A function in the call graph: every call of it from the same caller in the same stage."""
+
+ identifier: str
+ module: str
+ qualname: str
+ stage: str
+ parent: Call | None
+ count: int = 1
+
+
+@dataclass
+class Recording:
+ """What happened during one `evaluation_function` call."""
+
+ stages: list[str] = field(default_factory=list)
+ early_return: bool = False
+ calls: dict[tuple[Any, ...], Call] = field(default_factory=dict)
+ criteria_graphs: dict[str, CriteriaGraph] = field(default_factory=dict)
+
+
+class Recorder(logging.Handler):
+ """Records the stages from the evaluation's log events, and the function calls with `sys.setprofile`."""
+
+ def __init__(self) -> None:
+ super().__init__()
+ self.recording = Recording()
+ self.stage = FIRST_STAGE
+ # Each frame being followed: (the Call it belongs to, or None for `_evaluate`, its depth below `_evaluate`).
+ self.frames: dict[FrameType, tuple[Call | None, int]] = {}
+
+ def emit(self, record: logging.LogRecord) -> None:
+ event = json.loads(record.getMessage())
+ if event["event"] == "stage_start":
+ self.stage = event["stage"]
+ self.recording.stages.append(self.stage)
+ elif event["event"] == "evaluation_end":
+ self.recording.early_return = event["early_return"]
+
+ def _caller(self, frame: FrameType) -> tuple[Call | None, int] | None:
+ caller = frame.f_back
+ while caller is not None and caller not in self.frames:
+ caller = caller.f_back
+ if caller is None:
+ return None
+ return self.frames[caller]
+
+ def profile(self, frame: FrameType, event: str, arg: Any) -> None:
+ module = frame.f_globals.get("__name__", "")
+ qualname = frame.f_code.co_qualname
+ if event == "return":
+ if qualname == "feedback_procedure_generator" and module.startswith("app.context."):
+ self.recording.criteria_graphs.update(arg)
+ self.frames.pop(frame, None)
+ return
+ if event != "call":
+ return
+ if module == "app.evaluation" and qualname == "_evaluate":
+ self.frames[frame] = (None, 0)
+ return
+ if not module.startswith(("app.", "lambdafeedback.")):
+ return
+ caller = self._caller(frame)
+ if caller is None:
+ return
+ (parent, depth) = caller
+ if qualname in HIDDEN_FUNCTIONS or any(part in qualname for part in HIDDEN_FUNCTION_PARTS):
+ self.frames[frame] = caller
+ return
+ if depth >= CALL_DEPTH:
+ self.frames[frame] = (parent, depth + 1)
+ return
+ key = (self.stage if parent is None else parent.identifier, module, qualname)
+ call = self.recording.calls.get(key)
+ if call is None:
+ call = Call(f"call_{len(self.recording.calls)}", module, qualname, self.stage, parent)
+ self.recording.calls[key] = call
+ else:
+ call.count += 1
+ self.frames[frame] = (call, depth + 1)
+
+
+def record(response: str, answer: str, params: dict[str, Any]) -> tuple[Recording, dict[str, Any]]:
+ """Runs `evaluation_function` on the example and returns what happened, and the result."""
+ recorder = Recorder()
+ logger = logging.getLogger(evaluation.__name__)
+ logger.addHandler(recorder)
+ sys.setprofile(recorder.profile)
+ try:
+ result = evaluation.evaluation_function(response, answer, dict(params), include_test_data=True)
+ finally:
+ sys.setprofile(None)
+ logger.removeHandler(recorder)
+ return (recorder.recording, result)
+
+
+def stage_description(stage: str, recording: Recording, params: dict[str, Any], result: dict[str, Any]) -> str:
+ """What `stage` did for the example, or why it didn't run."""
+ ran = stage in recording.stages or stage == FIRST_STAGE
+ if stage == "parse_latex" and not ran:
+ return "Not run: is_latex is not set."
+ elif not ran and recording.early_return:
+ return "Not run: the response could not be parsed."
+ elif stage == FIRST_STAGE:
+ return "Copies the tolerance aliases to rtol and atol and creates the EvaluationResult."
+ elif stage == "determine_context":
+ context = "physical_quantity" if params.get("physical_quantity", False) is True else "symbolic"
+ return f"Chooses the {context} context and adds its default parameters and criteria."
+ elif stage == "parse_latex":
+ return "Reads the LaTeX response as SymPy syntax."
+ elif stage == "parse_reserved_expressions":
+ if recording.early_return:
+ return "Preprocesses and parses the response and answer. The response could not be parsed, so the " \
+ "result is returned with its feedback."
+ return "Preprocesses and parses the response and answer."
+ elif stage == "preview":
+ return "Sets the LaTeX and simplified forms of the response."
+ elif stage == "parse_criteria":
+ return "Parses the criteria: " + ", ".join(f"`{criterion}`" for criterion in recording.criteria_graphs) + "."
+ elif stage == "generate_feedback_procedures":
+ return "Builds the criteria graph for each criterion."
+ elif stage == "generate_feedback":
+ return "Runs each criteria graph on the response; the path it took is highlighted."
+ elif stage == "serialise":
+ return "Serialises the result."
+ return ""
+
+
+def result_label(result: dict[str, Any]) -> str:
+ tags = ", ".join(result["tags"])
+ return _node_label("Result", f"is_correct: {result['is_correct']}; tags: {tags}")
+
+
+def add_stage_nodes(dot: pydot.Graph, recording: Recording, params: dict[str, Any], result: dict[str, Any]) -> dict[str, str]:
+ """Adds a node for each stage, in order, then the result, and returns each stage's node identifier."""
+ identifiers = {}
+ previous = None
+ for (index, stage) in enumerate(all_stages()):
+ identifier = f"stage_{index}"
+ identifiers[stage] = identifier
+ ran = stage in recording.stages or stage == FIRST_STAGE
+ style = STAGE_STYLE if ran else SKIPPED_STAGE_STYLE
+ label = _node_label(stage, stage_description(stage, recording, params, result))
+ dot.add_node(pydot.Node(identifier, label=label, **style))
+ if previous is not None:
+ dot.add_edge(pydot.Edge(previous, identifier, **PIPELINE_EDGE_STYLE))
+ previous = identifier
+ dot.add_node(pydot.Node("result", label=result_label(result), **RESULT_STYLE))
+ dot.add_edge(pydot.Edge(previous, "result", **PIPELINE_EDGE_STYLE))
+ return identifiers
+
+
+def process_to_dot(recording: Recording, params: dict[str, Any], result: dict[str, Any]) -> pydot.Dot:
+ """The stages of the evaluation, with the criteria graphs it ran and the path the response took through them."""
+ dot = new_drawing()
+ dot.set("compound", "true")
+ stages = add_stage_nodes(dot, recording, params, result)
+ for (index, (criterion, graph)) in enumerate(recording.criteria_graphs.items()):
+ cluster = pydot.Cluster(f"criteria_{index}", label=f"Criteria graph: {html.escape(criterion)}", color="#9E9E9E")
+ identifiers = add_criteria_graph(cluster, graph, prefix=f"criteria_{index}_", reached_criteria=result["tags"])
+ dot.add_subgraph(cluster)
+ for start in graph.starting_evaluations(criterion + "_TRUE"):
+ dot.add_edge(pydot.Edge(stages["generate_feedback"], identifiers[start], style="dashed", lhead=cluster.get_name()))
+ return dot
+
+
+def calls_to_dot(recording: Recording, params: dict[str, Any], result: dict[str, Any]) -> pydot.Dot:
+ """The stages of the evaluation down the left, each with the functions called during it to its right."""
+ dot = new_drawing()
+ dot.set("rankdir", "LR")
+ pipeline = pydot.Subgraph("pipeline", rank="same")
+ dot.add_subgraph(pipeline)
+ stages = add_stage_nodes(pipeline, recording, params, result)
+ clusters = {}
+ for (index, stage) in enumerate(all_stages()):
+ cluster = pydot.Cluster(f"stage_cluster_{index}", label="", color="#9E9E9E")
+ clusters[stage] = cluster
+ dot.add_subgraph(cluster)
+ for call in recording.calls.values():
+ label = call.qualname if call.count == 1 else f"{call.qualname} ×{call.count}"
+ clusters[call.stage].add_node(pydot.Node(call.identifier, label=_node_label(label, call.module), **CALL_STYLE))
+ caller = stages[call.stage] if call.parent is None else call.parent.identifier
+ dot.add_edge(pydot.Edge(caller, call.identifier))
+ return dot
+
+
+def main() -> None:
+ OUTPUT_DIRECTORY.mkdir(parents=True, exist_ok=True)
+ for (name, response, answer, params) in EXAMPLES:
+ (recording, result) = record(response, answer, params)
+ process_to_dot(recording, params, result).write_svg(str(OUTPUT_DIRECTORY / f"{name}.svg"))
+ calls_to_dot(recording, params, result).write_svg(str(OUTPUT_DIRECTORY / f"{name}_calls.svg"))
+ print(f"Generated {name}: stages {', '.join(recording.stages)}; tags {', '.join(result['tags'])}")
+
+
+if __name__ == "__main__":
+ main()