From 7cda49a3f3f0930db94865287ac76510bb0d1c02 Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Tue, 3 Nov 2026 16:00:37 +0000 Subject: [PATCH 1/5] Add an interactive animation of the evaluation calls graph scripts/generate_process_animation.py writes build/process_animation.html, which grows each example's calls graph in call order. A side panel shows the response's forms, each call's arguments and return value, and the criteria graph traversal, with toggles and a call depth selector. The recorder in generate_process_graphs.py now keeps an ordered event log with argument and return values, records to a configurable depth, and catches criteria graph evaluations. The static figures are unchanged. Co-Authored-By: Claude Opus 5.5 --- scripts/generate_process_animation.py | 1133 +++++++++++++++++++++++++ scripts/generate_process_graphs.py | 175 +++- 2 files changed, 1296 insertions(+), 12 deletions(-) create mode 100644 scripts/generate_process_animation.py diff --git a/scripts/generate_process_animation.py b/scripts/generate_process_animation.py new file mode 100644 index 0000000..84821a3 --- /dev/null +++ b/scripts/generate_process_animation.py @@ -0,0 +1,1133 @@ +"""Generates `build/process_animation.html`: an interactive page that animates the process graphs of +`scripts/generate_process_graphs.py`. + +Each example is run through `evaluation_function` while recording every call down to `MAX_DEPTH` calls below +`_evaluate`, with its arguments and return value, the forms the response takes, and the evaluations each criteria +graph runs. The calls graph is laid out with graphviz once for each depth, with and without the criteria graphs, and +the page grows it in the order the functions were called. A side panel shows each step's details, and these can be +switched on and off on the page. + +The page is a build output and is not committed. Run from the repository root with +`python -m scripts.generate_process_animation` (it needs graphviz, i.e. `dot`), then open the page in a browser. +""" +from __future__ import annotations + +import html +import json +from pathlib import Path +from typing import Any + +import pydot + +from scripts.generate_criteria_graphs import add_criteria_graph +from scripts.generate_process_graphs import ( + EXAMPLES, + FIRST_STAGE, + Recording, + all_stages, + calls_to_dot, + record, + stage_description, +) + +OUTPUT_FILE = Path(__file__).resolve().parent.parent / "build" / "process_animation.html" + +# The deepest calls below `_evaluate` recorded, and the deepest the page can show. +MAX_DEPTH = 5 + +# The depth the page starts at: the depth of the static calls figures. +START_DEPTH = 2 + + +def example_title(name: str) -> str: + """The name shown on the page for the example `name`, e.g. "Physical quantity" for `process_physical_quantity`.""" + return name.removeprefix("process_").replace("_", " ").capitalize() + + +def stage_identifier(stage: str) -> str: + """The graphviz identifier `add_stage_nodes` gives the node of `stage`.""" + return f"stage_{all_stages().index(stage)}" + + +def build_steps(recording: Recording) -> tuple[list[dict[str, Any]], dict[str, list[int]], list[dict[str, Any]]]: + """The steps of the animation, the index of the event of each call of each function, and the forms the response + took, from the events in `recording`. + + Each step is a stage being entered, a function called for the first time from its caller (a node of the calls + graph appearing), a criteria graph evaluation, or the result. Later calls of the same function only raise its + count, so they are not steps of their own. + """ + steps: list[dict[str, Any]] = [{"kind": "stage", "stage": FIRST_STAGE, "event": -1, "depth": 0}] + invocations: dict[str, list[int]] = {} + forms: list[dict[str, Any]] = [] + calls = {call.identifier: call for call in recording.calls.values()} + # The step of each call still running, or None for a call that is not the first of its function. + running: list[dict[str, Any] | None] = [] + evaluation_step: dict[str, Any] = {} + for (index, event) in enumerate(recording.events): + kind = event["kind"] + if kind == "stage": + steps.append({"kind": "stage", "stage": event["stage"], "event": index, "depth": 0}) + elif kind == "call": + invocations.setdefault(event["call"], []).append(index) + if len(invocations[event["call"]]) == 1: + step = { + "kind": "call", + "call": event["call"], + "stage": event["stage"], + "depth": calls[event["call"]].depth, + "event": index, + "arguments": event["arguments"], + "returned": "", + "raised": False, + } + steps.append(step) + running.append(step) + else: + running.append(None) + elif kind == "return": + step = running.pop() + if step is not None: + step["returned"] = event["value"] + step["raised"] = event["raised"] + elif kind == "evaluate": + evaluation_step = { + "kind": "evaluate", + "criterion": event["criterion"], + "node": event["node"], + "depth": 0, + "event": index, + "reached": [], + } + steps.append(evaluation_step) + elif kind == "evaluated": + evaluation_step["reached"] = event["reached"] + elif kind == "form": + forms.append({"event": index, "call": event["call"], "value": event["value"], "type": event["type"]}) + else: + raise ValueError(f"Unknown event kind {kind!r}.") + steps.append({"kind": "result", "event": len(recording.events), "depth": 0}) + for (index, step) in enumerate(steps): + step["index"] = index + return (steps, invocations, forms) + + +def calls_layout( + recording: Recording, params: dict[str, Any], result: dict[str, Any], depth: int, with_criteria: bool +) -> tuple[pydot.Dot, dict[str, dict[str, str]]]: + """The calls graph down to `depth`, and with `with_criteria`, each criteria graph beside the `generate_feedback` + stage, with the graphviz identifier of each of its nodes by criterion and node label. + """ + dot = calls_to_dot(recording, params, result, depth) + identifiers: dict[str, dict[str, str]] = {} + if not with_criteria: + return (dot, identifiers) + dot.set("compound", "true") + for (index, (criterion, graph)) in enumerate(recording.criteria_graphs.items()): + cluster = pydot.Cluster(f"criteria_{index}", label=f"Criteria graph: {html.escape(criterion)}", color="#9E9E9E") + identifiers[criterion] = add_criteria_graph(cluster, graph, prefix=f"criteria_{index}_") + dot.add_subgraph(cluster) + for start in graph.starting_evaluations(criterion + "_TRUE"): + dot.add_edge(pydot.Edge( + stage_identifier("generate_feedback"), + identifiers[criterion][start], + style="dashed", + lhead=cluster.get_name(), + )) + return (dot, identifiers) + + +def svg_markup(dot: pydot.Dot) -> str: + """`dot` drawn as an SVG element, without the XML declaration and doctype in front of it.""" + svg = dot.create(format="svg").decode("utf-8") + return svg[svg.index(" dict[str, Any]: + """The graphviz identifier, kind and details of each node of each criteria graph, by criterion and node label.""" + data: dict[str, Any] = {} + for (criterion, graph) in recording.criteria_graphs.items(): + nodes = {} + groups = [("evaluation", graph.evaluations), ("criterion", graph.criteria), ("output", graph.outputs)] + for (kind, group) in groups: + for (label, node) in group.items(): + nodes[label] = {"id": identifiers[criterion][label], "kind": kind, "details": node.details} + data[criterion] = nodes + return data + + +def example_data(name: str, response: str, answer: str, params: dict[str, Any]) -> dict[str, Any]: + """Everything the page needs to animate one example.""" + (recording, result) = record(response, answer, params, depth=MAX_DEPTH, record_values=True) + (steps, invocations, forms) = build_steps(recording) + layouts = {} + criteria_identifiers: dict[str, dict[str, str]] = {} + for depth in range(1, MAX_DEPTH + 1): + for with_criteria in (False, True): + (dot, identifiers) = calls_layout(recording, params, result, depth, with_criteria) + layouts[f"{depth}{'c' if with_criteria else ''}"] = svg_markup(dot) + if with_criteria: + criteria_identifiers = identifiers + calls = { + call.identifier: { + "qualname": call.qualname, + "module": call.module, + "depth": call.depth, + "stage": call.stage, + "parent": None if call.parent is None else call.parent.identifier, + } + for call in recording.calls.values() + } + stages = all_stages() + return { + "name": name, + "title": example_title(name), + "response": response, + "answer": answer, + "params": params, + "stages": [ + { + "name": stage, + "id": stage_identifier(stage), + "description": stage_description(stage, recording, params, result), + } + for stage in stages + ], + "calls": calls, + "steps": steps, + "invocations": invocations, + "forms": forms, + "criteria": criteria_data(recording, criteria_identifiers), + "layouts": layouts, + "result": { + "is_correct": result["is_correct"], + "tags": result["tags"], + "feedback": result["feedback"], + "response_latex": result.get("response_latex", ""), + "response_simplified": result.get("response_simplified", ""), + }, + } + + +def main() -> None: + examples = [] + for (name, response, answer, params) in EXAMPLES: + examples.append(example_data(name, response, answer, params)) + print(f"Recorded {name}: {len(examples[-1]['steps'])} steps") + data = json.dumps({"examples": examples, "maxDepth": MAX_DEPTH, "startDepth": START_DEPTH}) + # Keeps the data from closing the script element it is in. + data = data.replace(" + + + + +Evaluation process animation + + + +
+

Evaluation process

+ +
+
+
+
+
+ + + +
+
+
+ +
+
+
+ + + + + + +
+
+ + + + + +
+
+ + + + +""" + + +if __name__ == "__main__": + main() diff --git a/scripts/generate_process_graphs.py b/scripts/generate_process_graphs.py index a611421..dee477b 100644 --- a/scripts/generate_process_graphs.py +++ b/scripts/generate_process_graphs.py @@ -10,18 +10,20 @@ """ from __future__ import annotations +import dis import html import inspect import json import logging import re import sys -from dataclasses import dataclass, field +from dataclasses import dataclass, field, is_dataclass from pathlib import Path from types import FrameType from typing import Any import pydot +import sympy from lambdafeedback.criteria import CriteriaGraph from app import evaluation @@ -39,6 +41,9 @@ # How many calls below `_evaluate` the call graphs go. CALL_DEPTH = 2 +# The longest an argument or return value recorded for the process animation gets before it is cut short. +VALUE_LENGTH = 200 + # Functions left out of the call graphs; the functions they call are shown under their caller instead. HIDDEN_FUNCTIONS = {"log_event", "elapsed_ms", "_evaluate..enter_stage"} HIDDEN_FUNCTION_PARTS = ("", "", "", "", "", "__create_fn__") @@ -67,37 +72,166 @@ class Call: qualname: str stage: str parent: Call | None + depth: int count: int = 1 @dataclass class Recording: - """What happened during one `evaluation_function` call.""" + """What happened during one `evaluation_function` call. + + `events` is filled only when values are recorded (see `Recorder`): every stage entered, call made, value returned + and criteria graph evaluation run, in order, as JSON-ready dictionaries. + """ stages: list[str] = field(default_factory=list) early_return: bool = False calls: dict[tuple[Any, ...], Call] = field(default_factory=dict) criteria_graphs: dict[str, CriteriaGraph] = field(default_factory=dict) + events: list[dict[str, Any]] = field(default_factory=list) + + +def short_repr(value: Any) -> str: + """`repr(value)` without memory addresses, cut short at `VALUE_LENGTH` characters.""" + try: + text = repr(value) + except (AttributeError, TypeError, ValueError, KeyError): + # Objects whose `__init__` is still running can fail to give a repr. + text = f"<{type(value).__name__}>" + text = re.sub(r" at 0x[0-9a-f]+", "", text) + if len(text) > VALUE_LENGTH: + return text[:VALUE_LENGTH - 1] + "…" + return text + + +def frame_argument_values(frame: FrameType) -> list[tuple[str, Any]]: + """The name and value of each argument of the function running in `frame`.""" + arguments = inspect.getargvalues(frame) + names = list(arguments.args) + if arguments.varargs is not None: + names.append(arguments.varargs) + if arguments.keywords is not None: + names.append(arguments.keywords) + return [(name, arguments.locals.get(name)) for name in names] + + +def is_response_form(value: Any) -> bool: + """Whether `value` is the kind of value the response is turned into: a string, a SymPy expression, an `app` or + `lambdafeedback` dataclass such as a parsed quantity or a token, or a list or tuple of these. + """ + if isinstance(value, (str, sympy.Basic)): + return True + elif isinstance(value, (list, tuple)): + return len(value) > 0 and all(is_response_form(item) for item in value) + elif type(value).__module__.startswith("lambdafeedback."): + return True + return type(value).__module__.startswith("app.") and is_dataclass(value) + + +def returned_normally(frame: FrameType) -> bool: + """Whether `frame` is returning (or yielding) a value, rather than stopping because of an exception.""" + instruction = dis.opname[frame.f_code.co_code[frame.f_lasti]] + return instruction in ("RETURN_VALUE", "RETURN_CONST", "YIELD_VALUE") class Recorder(logging.Handler): - """Records the stages from the evaluation's log events, and the function calls with `sys.setprofile`.""" + """Records the stages from the evaluation's log events, and the function calls with `sys.setprofile`. - def __init__(self) -> None: + Calls are followed down to `depth` calls below `_evaluate`. With `record_values`, these are kept in order in + `Recording.events`: + - the arguments and return value of each call, and whether it raised an exception instead of returning; + - the evaluations each criteria graph runs, and the criteria each one reports; + - each new form of the response (see `is_response_form`): the value returned by a call that was given the response, + or an earlier form of it, as an argument. + """ + + def __init__(self, depth: int = CALL_DEPTH, record_values: bool = False, response: str = "") -> None: super().__init__() self.recording = Recording() self.stage = FIRST_STAGE + self.depth = depth + self.record_values = record_values # Each frame being followed: (the Call it belongs to, or None for `_evaluate`, its depth below `_evaluate`). self.frames: dict[FrameType, tuple[Call | None, int]] = {} + # The frames whose return is recorded: each frame's Call, or the criterion and evaluation node it runs. + self.call_frames: dict[FrameType, Call] = {} + self.evaluation_frames: dict[FrameType, tuple[str, str]] = {} + # The reprs of the forms the response has taken so far, and the frames given one of them as an argument. + self.response_forms = {repr(response)} + self.response_frames: set[FrameType] = set() def emit(self, record: logging.LogRecord) -> None: event = json.loads(record.getMessage()) if event["event"] == "stage_start": self.stage = event["stage"] self.recording.stages.append(self.stage) + if self.record_values: + self.recording.events.append({"kind": "stage", "stage": self.stage}) elif event["event"] == "evaluation_end": self.recording.early_return = event["early_return"] + def _record_evaluation(self, frame: FrameType) -> None: + """Records the start of a criteria graph evaluation, when `frame` is the `evaluate` function of one.""" + runner = frame.f_back + if runner is None or runner.f_code.co_qualname != "CriteriaGraph.generate_feedback": + return + evaluate = runner.f_locals.get("evaluate") + # `evaluate` can be a `functools.partial` of the function that runs + function = getattr(evaluate, "func", evaluate) + if getattr(function, "__code__", None) is not frame.f_code: + return + criterion = runner.f_locals["main_criterion"].removesuffix("_TRUE") + node = runner.f_locals["label"] + self.evaluation_frames[frame] = (criterion, node) + self.recording.events.append({"kind": "evaluate", "stage": self.stage, "criterion": criterion, "node": node}) + + def _record_call(self, frame: FrameType, call: Call) -> None: + self.call_frames[frame] = call + arguments = frame_argument_values(frame) + for (name, value) in arguments: + if name != "self" and is_response_form(value) and repr(value) in self.response_forms: + self.response_frames.add(frame) + break + shown_arguments = [] + for (name, value) in arguments: + if name == "self": + shown_arguments.append((name, f"<{type(value).__name__}>")) + else: + shown_arguments.append((name, short_repr(value))) + self.recording.events.append({ + "kind": "call", "stage": self.stage, "call": call.identifier, "arguments": shown_arguments + }) + + def _record_return(self, frame: FrameType, value: Any) -> None: + call = self.call_frames.pop(frame, None) + given_response = frame in self.response_frames + self.response_frames.discard(frame) + if call is not None: + raised = not returned_normally(frame) + self.recording.events.append({ + "kind": "return", + "stage": self.stage, + "call": call.identifier, + "value": "" if raised else short_repr(value), + "raised": raised, + }) + if given_response and not raised and is_response_form(value) and repr(value) not in self.response_forms: + self.response_forms.add(repr(value)) + self.recording.events.append({ + "kind": "form", + "stage": self.stage, + "call": call.identifier, + "value": short_repr(value), + "type": type(value).__name__, + }) + evaluation = self.evaluation_frames.pop(frame, None) + if evaluation is not None: + (criterion, node) = evaluation + reached = list(value) if isinstance(value, dict) else [] + self.recording.events.append({ + "kind": "evaluated", "stage": self.stage, "criterion": criterion, "node": node, "reached": reached + }) + def _caller(self, frame: FrameType) -> tuple[Call | None, int] | None: caller = frame.f_back while caller is not None and caller not in self.frames: @@ -112,6 +246,8 @@ def profile(self, frame: FrameType, event: str, arg: Any) -> None: if event == "return": if qualname == "feedback_procedure_generator" and module.startswith("app.context."): self.recording.criteria_graphs.update(arg) + if self.record_values: + self._record_return(frame, arg) self.frames.pop(frame, None) return if event != "call": @@ -119,6 +255,8 @@ def profile(self, frame: FrameType, event: str, arg: Any) -> None: if module == "app.evaluation" and qualname == "_evaluate": self.frames[frame] = (None, 0) return + if self.record_values: + self._record_evaluation(frame) if not module.startswith(("app.", "lambdafeedback.")): return caller = self._caller(frame) @@ -128,22 +266,29 @@ def profile(self, frame: FrameType, event: str, arg: Any) -> None: if qualname in HIDDEN_FUNCTIONS or any(part in qualname for part in HIDDEN_FUNCTION_PARTS): self.frames[frame] = caller return - if depth >= CALL_DEPTH: + if depth >= self.depth: self.frames[frame] = (parent, depth + 1) return key = (self.stage if parent is None else parent.identifier, module, qualname) call = self.recording.calls.get(key) if call is None: - call = Call(f"call_{len(self.recording.calls)}", module, qualname, self.stage, parent) + call = Call(f"call_{len(self.recording.calls)}", module, qualname, self.stage, parent, depth + 1) self.recording.calls[key] = call else: call.count += 1 self.frames[frame] = (call, depth + 1) + if self.record_values: + self._record_call(frame, call) + +def record( + response: str, answer: str, params: dict[str, Any], depth: int = CALL_DEPTH, record_values: bool = False +) -> tuple[Recording, dict[str, Any]]: + """Runs `evaluation_function` on the example and returns what happened, and the result. -def record(response: str, answer: str, params: dict[str, Any]) -> tuple[Recording, dict[str, Any]]: - """Runs `evaluation_function` on the example and returns what happened, and the result.""" - recorder = Recorder() + See `Recorder` for `depth` and `record_values`. + """ + recorder = Recorder(depth, record_values, response) logger = logging.getLogger(evaluation.__name__) logger.addHandler(recorder) sys.setprofile(recorder.profile) @@ -225,8 +370,12 @@ def process_to_dot(recording: Recording, params: dict[str, Any], result: dict[st return dot -def calls_to_dot(recording: Recording, params: dict[str, Any], result: dict[str, Any]) -> pydot.Dot: - """The stages of the evaluation down the left, each with the functions called during it to its right.""" +def calls_to_dot( + recording: Recording, params: dict[str, Any], result: dict[str, Any], depth: int = CALL_DEPTH +) -> pydot.Dot: + """The stages of the evaluation down the left, each with the functions called during it to its right, down to + `depth` calls below `_evaluate`. + """ dot = new_drawing() dot.set("rankdir", "LR") pipeline = pydot.Subgraph("pipeline", rank="same") @@ -238,7 +387,9 @@ def calls_to_dot(recording: Recording, params: dict[str, Any], result: dict[str, clusters[stage] = cluster dot.add_subgraph(cluster) for call in recording.calls.values(): - label = call.qualname if call.count == 1 else f"{call.qualname} ×{call.count}" + if call.depth > depth: + continue + label =call.qualname if call.count == 1 else f"{call.qualname} ×{call.count}" clusters[call.stage].add_node(pydot.Node(call.identifier, label=_node_label(label, call.module), **CALL_STYLE)) caller = stages[call.stage] if call.parent is None else call.parent.identifier dot.add_edge(pydot.Edge(caller, call.identifier)) From a49dd58ef52944c562afcbff64d98ce84360931c Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Tue, 3 Nov 2026 16:37:32 +0000 Subject: [PATCH 2/5] Show how each criteria graph is chosen, and make the animation's nodes clickable The recorder now records the criteria create_criteria_dict parses and the template feedback_procedure_generator chooses for each one. With the criteria graph shown, the animation adds a node for each criterion in the parse_criteria stage, an edge labelled with its parse label to the template it chooses, and edges to its criteria graph from the template that builds it and the stage that runs it. A Criteria panel section follows the chain from the criteria string to whether each graph reached its main criterion. Clicking a node pins its details in the panel: a call's invocations with their arguments and return values, a stage's calls, a criteria graph node's details and whether it was reached, or a criterion's chain. Co-Authored-By: Claude Opus 5.5 --- scripts/generate_process_animation.py | 532 ++++++++++++++++++++++++-- scripts/generate_process_graphs.py | 49 ++- 2 files changed, 544 insertions(+), 37 deletions(-) diff --git a/scripts/generate_process_animation.py b/scripts/generate_process_animation.py index 84821a3..6a19b9e 100644 --- a/scripts/generate_process_animation.py +++ b/scripts/generate_process_animation.py @@ -19,7 +19,7 @@ import pydot -from scripts.generate_criteria_graphs import add_criteria_graph +from scripts.generate_criteria_graphs import _node_label, add_criteria_graph from scripts.generate_process_graphs import ( EXAMPLES, FIRST_STAGE, @@ -38,6 +38,12 @@ # The depth the page starts at: the depth of the static calls figures. START_DEPTH = 2 +# How many calls of each function keep their arguments and return value, for the details of a clicked node. +INVOCATION_VALUES = 20 + +CRITERION_STYLE = {"shape": "note", "style": "filled", "fillcolor": "#FFF8E1", "color": "#8D6E63"} +CRITERION_EDGE_STYLE = {"color": "#8D6E63", "fontcolor": "#5D4037", "fontname": "Helvetica", "fontsize": "10"} + def example_title(name: str) -> str: """The name shown on the page for the example `name`, e.g. "Physical quantity" for `process_physical_quantity`.""" @@ -49,28 +55,35 @@ def stage_identifier(stage: str) -> str: return f"stage_{all_stages().index(stage)}" -def build_steps(recording: Recording) -> tuple[list[dict[str, Any]], dict[str, list[int]], list[dict[str, Any]]]: +def build_steps(recording: Recording) -> tuple[list[dict[str, Any]], dict[str, dict[str, list[Any]]], list[dict[str, Any]]]: """The steps of the animation, the index of the event of each call of each function, and the forms the response took, from the events in `recording`. Each step is a stage being entered, a function called for the first time from its caller (a node of the calls - graph appearing), a criteria graph evaluation, or the result. Later calls of the same function only raise its - count, so they are not steps of their own. + graph appearing), the criteria being parsed, a template building a criteria graph, a criteria graph evaluation, or + the result. Later calls of the same function only raise its count, so they are not steps of their own. + + The invocations of each call are the index of the event of each, and the values of the first `INVOCATION_VALUES`. """ steps: list[dict[str, Any]] = [{"kind": "stage", "stage": FIRST_STAGE, "event": -1, "depth": 0}] - invocations: dict[str, list[int]] = {} + invocations: dict[str, dict[str, list[Any]]] = {} forms: list[dict[str, Any]] = [] calls = {call.identifier: call for call in recording.calls.values()} - # The step of each call still running, or None for a call that is not the first of its function. - running: list[dict[str, Any] | None] = [] + # The values of each call still running, and its step, or None for a call that is not the first of its function. + running: list[tuple[dict[str, Any], dict[str, Any] | None]] = [] evaluation_step: dict[str, Any] = {} for (index, event) in enumerate(recording.events): kind = event["kind"] if kind == "stage": steps.append({"kind": "stage", "stage": event["stage"], "event": index, "depth": 0}) elif kind == "call": - invocations.setdefault(event["call"], []).append(index) - if len(invocations[event["call"]]) == 1: + call_invocations = invocations.setdefault(event["call"], {"events": [], "values": []}) + call_invocations["events"].append(index) + values = {"event": index, "arguments": event["arguments"], "returned": "", "raised": False} + # Only the first calls keep their values, as some functions are called over a thousand times + if len(call_invocations["values"]) < INVOCATION_VALUES: + call_invocations["values"].append(values) + if len(call_invocations["events"]) == 1: step = { "kind": "call", "call": event["call"], @@ -82,14 +95,25 @@ def build_steps(recording: Recording) -> tuple[list[dict[str, Any]], dict[str, l "raised": False, } steps.append(step) - running.append(step) + running.append((values, step)) else: - running.append(None) + running.append((values, None)) elif kind == "return": - step = running.pop() + (values, step) = running.pop() + values["returned"] = event["value"] + values["raised"] = event["raised"] if step is not None: step["returned"] = event["value"] step["raised"] = event["raised"] + elif kind == "criteria": + steps.append({ + "kind": "criteria", + "depth": 0, + "event": index, + "criteria": [criterion["identifier"] for criterion in event["criteria"]], + }) + elif kind == "template": + steps.append({"kind": "template", "depth": 0, "event": index, "criterion": event["criterion"]}) elif kind == "evaluate": evaluation_step = { "kind": "evaluate", @@ -112,27 +136,111 @@ def build_steps(recording: Recording) -> tuple[list[dict[str, Any]], dict[str, l return (steps, invocations, forms) +def criteria_chain(recording: Recording, params: dict[str, Any]) -> dict[str, Any]: + """How the example's criteria graphs came about: where the criteria string came from, the string before and after + preprocessing, and for each criterion (by its identifier) its parse, the template that built its graph and the + index of its graph's cluster. + + Empty when the evaluation stopped before parsing the criteria. + """ + criteria_event = next((event for event in recording.events if event["kind"] == "criteria"), None) + if criteria_event is None: + return {} + if "criteria" in params: + source = 'Given in params["criteria"].' + else: + context = "physical_quantity" if params.get("physical_quantity", False) is True else "symbolic" + source = f"Not given in params, so the default criteria of the {context} context." + if params.get("syntactical_comparison", False) is True: + source += ' syntactical_comparison adds "response written as answer".' + indices = {criterion: index for (index, criterion) in enumerate(recording.criteria_graphs)} + criteria = {} + for criterion in criteria_event["criteria"]: + criteria[criterion["identifier"]] = { + "index": indices.get(criterion["identifier"]), + "label": criterion["label"], + "tree": criterion["tree"], + } + for event in recording.events: + if event["kind"] == "template": + criteria[event["criterion"]].update({ + "template": event["template"], + "templateLabel": event["label"], + "call": event["call"], + "evaluations": event["evaluations"], + }) + return { + "source": source, + "original": criteria_event["original"], + "preprocessed": criteria_event["preprocessed"], + "criteria": criteria, + } + + +def visible_call(recording: Recording, identifier: str, depth: int) -> str: + """The identifier of the call `identifier`, or of its nearest caller, that is drawn down to `depth`.""" + calls = {call.identifier: call for call in recording.calls.values()} + call = calls[identifier] + while call.depth > depth and call.parent is not None: + call = call.parent + return call.identifier + + def calls_layout( - recording: Recording, params: dict[str, Any], result: dict[str, Any], depth: int, with_criteria: bool + recording: Recording, + params: dict[str, Any], + result: dict[str, Any], + chain: dict[str, Any], + depth: int, + with_criteria: bool, ) -> tuple[pydot.Dot, dict[str, dict[str, str]]]: - """The calls graph down to `depth`, and with `with_criteria`, each criteria graph beside the `generate_feedback` - stage, with the graphviz identifier of each of its nodes by criterion and node label. + """The calls graph down to `depth`, with the graphviz identifier of each criteria graph node by criterion and node + label. + + With `with_criteria`, each criterion gets a node in the `parse_criteria` stage, with an edge to the template call + its parse label chose, and each criteria graph is drawn with an edge from that template call, which built it, and + from the `generate_feedback` stage, which ran it. """ dot = calls_to_dot(recording, params, result, depth) identifiers: dict[str, dict[str, str]] = {} - if not with_criteria: + # Without criteria, i.e. when the evaluation stopped before parsing them, there is nothing to add + if not with_criteria or not chain: return (dot, identifiers) dot.set("compound", "true") + stages = all_stages() + parse_cluster = dot.get_subgraph(f"cluster_stage_cluster_{stages.index('parse_criteria')}")[0] + parser_call = next(call.identifier for call in recording.calls.values() if call.qualname == "create_criteria_dict") for (index, (criterion, graph)) in enumerate(recording.criteria_graphs.items()): - cluster = pydot.Cluster(f"criteria_{index}", label=f"Criteria graph: {html.escape(criterion)}", color="#9E9E9E") + details = chain["criteria"][criterion] + template_call = visible_call(recording, details["call"], depth) + criterion_node = f"criterion_{index}" + parse_cluster.add_node(pydot.Node( + criterion_node, label=_node_label(criterion, f"parse label: {details['label']}"), **CRITERION_STYLE + )) + dot.add_edge(pydot.Edge(parser_call, criterion_node, **CRITERION_EDGE_STYLE)) + choice = details["label"] if details["templateLabel"] is not None else "default" + dot.add_edge(pydot.Edge(criterion_node, template_call, label=html.escape(choice), **CRITERION_EDGE_STYLE)) + title = f"Criteria graph: {criterion} (built by {details['template']})" + cluster = pydot.Cluster(f"criteria_{index}", label=html.escape(title), color="#9E9E9E") identifiers[criterion] = add_criteria_graph(cluster, graph, prefix=f"criteria_{index}_") dot.add_subgraph(cluster) for start in graph.starting_evaluations(criterion + "_TRUE"): + dot.add_edge(pydot.Edge( + template_call, + identifiers[criterion][start], + label="builds", + style="dashed", + lhead=cluster.get_name(), + **CRITERION_EDGE_STYLE, + )) dot.add_edge(pydot.Edge( stage_identifier("generate_feedback"), identifiers[criterion][start], + label="runs", style="dashed", lhead=cluster.get_name(), + fontname="Helvetica", + fontsize="10", )) return (dot, identifiers) @@ -151,7 +259,9 @@ def criteria_data(recording: Recording, identifiers: dict[str, dict[str, str]]) groups = [("evaluation", graph.evaluations), ("criterion", graph.criteria), ("output", graph.outputs)] for (kind, group) in groups: for (label, node) in group.items(): - nodes[label] = {"id": identifiers[criterion][label], "kind": kind, "details": node.details} + nodes[label] = { + "id": identifiers[criterion][label], "kind": kind, "summary": node.summary, "details": node.details + } data[criterion] = nodes return data @@ -160,11 +270,12 @@ def example_data(name: str, response: str, answer: str, params: dict[str, Any]) """Everything the page needs to animate one example.""" (recording, result) = record(response, answer, params, depth=MAX_DEPTH, record_values=True) (steps, invocations, forms) = build_steps(recording) + chain = criteria_chain(recording, params) layouts = {} criteria_identifiers: dict[str, dict[str, str]] = {} for depth in range(1, MAX_DEPTH + 1): for with_criteria in (False, True): - (dot, identifiers) = calls_layout(recording, params, result, depth, with_criteria) + (dot, identifiers) = calls_layout(recording, params, result, chain, depth, with_criteria) layouts[f"{depth}{'c' if with_criteria else ''}"] = svg_markup(dot) if with_criteria: criteria_identifiers = identifiers @@ -198,6 +309,7 @@ def example_data(name: str, response: str, answer: str, params: dict[str, Any]) "invocations": invocations, "forms": forms, "criteria": criteria_data(recording, criteria_identifiers), + "chain": chain, "layouts": layouts, "result": { "is_correct": result["is_correct"], @@ -434,6 +546,32 @@ def main() -> None: #graph g.is-reached > :is(polygon, path, ellipse):first-of-type { stroke: #e65100; stroke-width: 3px; } #graph g.edge.is-reached > path { stroke: #e65100; stroke-width: 2.5px; } #graph g.edge.is-reached > polygon { stroke: #e65100; fill: #e65100; } +#graph g.node:not(.is-hidden) { cursor: pointer; } +#graph g.node:not(.is-hidden):hover > :is(polygon, path, ellipse):first-of-type { stroke-width: 2.5px; } +#graph g.is-pinned > :is(polygon, path, ellipse):first-of-type { stroke: #7b1fa2; stroke-width: 4px; stroke-dasharray: none; } + +.pinned { border: 1px solid var(--border); border-left: 3px solid #ab47bc; border-radius: 8px; padding: 10px 12px; } +.pinned-head { display: flex; align-items: flex-start; justify-content: space-between; gap: 8px; } +.pinned-head button { min-height: 26px; padding: 0 8px; } +.link { color: var(--accent); background: none; border: none; padding: 0; min-height: 0; text-decoration: underline; cursor: pointer; } +.invocations { list-style: none; margin: 8px 0 0; padding: 0; display: grid; gap: 8px; } +.invocations li { border-top: 1px solid var(--border); padding-top: 6px; } +.chain { list-style: none; margin: 0; padding: 0; display: grid; gap: 10px; } +.chain > li { padding-left: 12px; border-left: 2px solid var(--border); } +.chain > li.is-focus { border-left-color: var(--highlight); } +.chain .step-label { font-size: 12px; color: var(--muted); } +.chain code { background: var(--code-bg); padding: 1px 6px; border-radius: 4px; word-break: break-word; } +.tree { + margin: 4px 0 0; + padding: 6px 8px; + background: var(--code-bg); + border-radius: 4px; + font-size: 12px; + white-space: pre-wrap; + word-break: break-word; +} +.status-good { color: var(--good); font-weight: 600; } +.status-bad { color: var(--bad); font-weight: 600; } @media (max-width: 900px) { main { grid-template-columns: 1fr; grid-template-rows: minmax(320px, 55vh) auto; overflow-y: auto; } @@ -504,6 +642,8 @@ def main() -> None: zoom: 0.9, // The index in the example's steps of the step shown. step: 0, + // The graphviz title of the node whose details are pinned in the panel, or null. + pinned: null, playing: false, timer: null, layout: null, @@ -553,7 +693,7 @@ def main() -> None: } function isShown(step) { - if (step.kind === "evaluate") { + if (step.kind === "evaluate" || step.kind === "criteria" || step.kind === "template") { return state.criteria; } return step.depth <= state.depth; @@ -585,7 +725,7 @@ def main() -> None: function invocationsUpTo(callId, event) { let count = 0; - for (const index of example().invocations[callId] || []) { + for (const index of invocationEvents(callId)) { if (index <= event) { count += 1; } @@ -593,6 +733,20 @@ def main() -> None: return count; } +function invocationEvents(callId) { + const invocations = example().invocations[callId]; + return invocations === undefined ? [] : invocations.events; +} + +// The chain of the criterion `identifier`: its parse, template and graph cluster index. +function chainOf(identifier) { + return example().chain.criteria[identifier]; +} + +function firstStepOf(kind) { + return example().steps.find((step) => step.kind === kind); +} + // Drawing function drawLayout() { @@ -648,22 +802,29 @@ def main() -> None: saveSettings(); } -const STATE_CLASSES = ["is-hidden", "is-waiting", "is-current", "is-stack", "is-visited", "is-reached", "appear", "draw"]; +const STATE_CLASSES = [ + "is-hidden", "is-waiting", "is-current", "is-stack", "is-visited", "is-reached", "is-pinned", "appear", "draw", +]; + +// Whether the node `name` is only drawn once a step reveals it: calls, criteria and criteria graph nodes. +function isRevealedNode(name) { + return name.startsWith("call_") || name.startsWith("criteria_") || name.startsWith("criterion_"); +} function resetGraph() { for (const group of [...nodes.values(), ...edges.values(), ...clusters.values()]) { group.classList.remove(...STATE_CLASSES); } for (const [name, group] of nodes) { - if (name.startsWith("call_") || name.startsWith("criteria_")) { + if (isRevealedNode(name)) { group.classList.add("is-hidden"); } else { group.classList.add("is-waiting"); } } for (const [name, group] of edges) { - const target = name.split("->")[1]; - if (target.startsWith("call_") || target.startsWith("criteria_")) { + const [source, target] = name.split("->"); + if (isRevealedNode(target) || source.startsWith("criterion_")) { group.classList.add("is-hidden"); } } @@ -687,20 +848,66 @@ def main() -> None: } } -function showCriteriaGraphs(isCurrent) { - for (const [name, group] of clusters) { - if (name.startsWith("cluster_criteria_")) { - reveal(group, isCurrent, "appear"); +// The criteria parsed: each criterion's node, and the edge to it from the call that parsed it. +function applyCriteria(step, isCurrent) { + for (const identifier of step.criteria) { + const node = "criterion_" + chainOf(identifier).index; + reveal(nodes.get(node), isCurrent, "appear"); + for (const [name, group] of edges) { + if (name.split("->")[1] === node) { + reveal(group, isCurrent, "draw"); + } + } + if (isCurrent) { + nodes.get(node)?.classList.add("is-current"); } } +} + +// A template building a criteria graph: the edge from the criterion to the template its label chose, and the graph, +// dimmed until `generate_feedback` runs it. +function applyTemplate(step, isCurrent) { + const index = chainOf(step.criterion).index; + const node = "criterion_" + index; + const prefix = "criteria_" + index + "_"; + reveal(clusters.get("cluster_criteria_" + index), isCurrent, "appear"); for (const [name, group] of nodes) { - if (name.startsWith("criteria_")) { + if (name.startsWith(prefix)) { reveal(group, isCurrent, "appear"); + group.classList.add("is-waiting"); } } for (const [name, group] of edges) { - if (name.split("->")[1].startsWith("criteria_")) { + const [source, target] = name.split("->"); + if (source === node) { + reveal(group, isCurrent, "draw"); + } else if (source.startsWith(prefix)) { reveal(group, isCurrent, "appear"); + group.classList.add("is-waiting"); + } else if (target.startsWith(prefix) && source.startsWith("call_")) { + // The template call building the graph + reveal(group, isCurrent, "draw"); + } + } + if (isCurrent) { + nodes.get(node)?.classList.add("is-current"); + } +} + +// `generate_feedback` starting: the criteria graphs are no longer dimmed, and the edges to them from the stage show. +function runCriteriaGraphs(stageId, isCurrent) { + for (const [name, group] of nodes) { + if (name.startsWith("criteria_")) { + group.classList.remove("is-waiting"); + } + } + for (const [name, group] of edges) { + const [source, target] = name.split("->"); + if (target.startsWith("criteria_")) { + group.classList.remove("is-waiting"); + if (source === stageId) { + reveal(group, isCurrent, "draw"); + } } } } @@ -749,8 +956,8 @@ def main() -> None: const stageGroup = nodes.get(stageOf(step.stage).id); reveal(stageGroup, false, "appear"); reveal(stageCluster(stageOf(step.stage).id), isCurrent, "appear"); - if (step.stage === "generate_feedback_procedures") { - showCriteriaGraphs(isCurrent); + if (step.stage === "generate_feedback") { + runCriteriaGraphs(stageOf(step.stage).id, isCurrent); } if (isCurrent && stageGroup !== undefined) { stageGroup.classList.add("is-current"); @@ -763,6 +970,10 @@ def main() -> None: callGroup.classList.add("is-current"); markCallStack(step.call); } + } else if (step.kind === "criteria") { + applyCriteria(step, isCurrent); + } else if (step.kind === "template") { + applyTemplate(step, isCurrent); } else if (step.kind === "evaluate") { applyEvaluation(step, isCurrent); } else if (step.kind === "result") { @@ -856,7 +1067,7 @@ def main() -> None: const call = example().calls[step.call]; const stack = [stepStage(step), ...callStack(step.call).map((id) => example().calls[id].qualname), call.qualname]; const count = invocationsUpTo(step.call, step.event); - const total = (example().invocations[step.call] || []).length; + const total = invocationEvents(step.call).length; let body = `
${escapeHtml(call.qualname)}
${escapeHtml(call.module)} · depth ${call.depth}
${stack.map((name) => `${escapeHtml(name)}`).join("")}
`; @@ -935,8 +1146,229 @@ def main() -> None: `; } +// The template that built the graph of the criterion `identifier`, and why that one. +function templateChoice(details) { + if (details.templateLabel !== null) { + return `Its parse label ${escapeHtml(details.label)} is in graph_templates, which maps it + to ${escapeHtml(details.template)}.`; + } + return `Its parse label ${escapeHtml(details.label)} is not in graph_templates, so the + context's default template ${escapeHtml(details.template)} is used.`; +} + +// Whether the criteria graph of `identifier` reached its main criterion by `step`: "reached", "missed" once +// `generate_feedback` has finished without reaching it, "running", "built" or "" before the graph is built. +function mainCriterionStatus(identifier, step) { + const main = identifier + "_TRUE"; + const evaluations = example().steps.filter( + (candidate) => candidate.kind === "evaluate" && candidate.criterion === identifier && candidate.event <= step.event + ); + if (evaluations.some((evaluation) => evaluation.reached.includes(main))) { + return "reached"; + } + const serialise = example().steps.find((candidate) => candidate.kind === "stage" && candidate.stage === "serialise"); + if (serialise !== undefined && step.event >= serialise.event) { + return "missed"; + } + if (evaluations.length > 0) { + return "running"; + } + const built = example().steps.some( + (candidate) => candidate.kind === "template" && candidate.criterion === identifier && candidate.event <= step.event + ); + return built ? "built" : ""; +} + +function criterionChainItems(identifier, step) { + const details = chainOf(identifier); + const items = [`
  • +
    Parsed by the criteria parser into
    + 
    ${escapeHtml(details.tree)}
    +
    Parse label ${escapeHtml(details.label)}
    • `]; + const template = firstStepOf("template") === undefined ? undefined : example().steps.find( + (candidate) => candidate.kind === "template" && candidate.criterion === identifier + ); + if (template !== undefined && template.event <= step.event) { + items.push(`
  • feedback_procedure_generator chooses its template
    +
    ${templateChoice(details)}
    +
    It builds a criteria graph with ${details.evaluations} evaluation + node${details.evaluations === 1 ? "" : "s"}.
    • `); + } + const status = mainCriterionStatus(identifier, step); + const main = `${escapeHtml(identifier + "_TRUE")}`; + if (status === "reached") { + items.push(`
  • generate_feedback runs the graph
    + ✓ reached ${main}
    • `); + } else if (status === "missed") { + items.push(`
  • generate_feedback runs the graph
    + ✗ did not reach ${main}
    • `); + } else if (status === "running") { + items.push(`
  • generate_feedback runs the graph
    + Running; the main criterion is ${main}.
    • `); + } else if (status === "built") { + items.push(`
  • Waiting for generate_feedback
    + The response is correct for this criterion if the graph reaches ${main}.
    • `); + } + return items; +} + +// How the criteria graphs came about, up to `step`, from the criteria string to whether each reached its main +// criterion; empty before the criteria are parsed. +function criteriaSection(step) { + const chain = example().chain; + const criteriaStep = firstStepOf("criteria"); + if (chain.criteria === undefined || criteriaStep === undefined || step.event < criteriaStep.event) { + return ""; + } + const focus = step.kind === "template" ? step.criterion : null; + const identifiers = Object.keys(chain.criteria); + const items = [`
  • +
    The criteria string
    + ${escapeHtml(chain.original)} +
    ${escapeHtml(chain.source)}
    • `, + `
  • ${chain.preprocessed === chain.original ? "Preprocessing leaves it unchanged; " + : `Preprocessed to ${escapeHtml(chain.preprocessed)}, then `}split at the top-level commas into + ${identifiers.length} criteri${identifiers.length === 1 ? "on" : "a"}, one criteria graph each
    + ${identifiers.map((identifier) => `${escapeHtml(identifier)}`).join(" ")}
    • `]; + for (const identifier of identifiers) { + items.push(`
  • +
    +
      ${criterionChainItems(identifier, step).join("")}
    • `); + } + const statuses = identifiers.map((identifier) => mainCriterionStatus(identifier, step)); + if (statuses.every((status) => status === "reached" || status === "missed")) { + const correct = statuses.every((status) => status === "reached"); + items.push(`
  • is_correct
    + ${correct}: + true only if every criteria graph reached its main criterion.
    • `); + } + return `

    Criteria

      ${items.join("")}
    `; +} + +// Pinned details + +function pinnedCall(callId, step) { + const call = example().calls[callId]; + const caller = callerOf(callId); + const callerName = call.parent === null ? "the " + call.stage + " stage" : example().calls[call.parent].qualname; + const count = invocationsUpTo(callId, step.event); + const values = (example().invocations[callId]?.values || []).filter((invocation) => invocation.event <= step.event); + let body = `
    ${escapeHtml(call.qualname)}
    +
    ${escapeHtml(call.module)} · depth ${call.depth} · stage ${escapeHtml(call.stage)}
    +
    Called by , + ${count} time${count === 1 ? "" : "s"} so far.
    `; + if (values.length > 0) { + const items = values.map((invocation, position) => { + const argumentList = invocation.arguments.length === 0 ? `

    No arguments.

    ` + : `
    ${invocation.arguments.map(([name, value]) => + `
    ${escapeHtml(name)}
    ${escapeHtml(value)}
    `).join("")}
    `; + let returned; + if (invocation.event > step.event) { + returned = ""; + } else if (invocation.raised) { + returned = `
    raised an exception
    `; + } else { + returned = `
    ↩ ${escapeHtml(invocation.returned)}
    `; + } + return `
  • Call ${position + 1}
    ${argumentList}${returned}
    • `; + }); + body += `
      ${items.join("")}
    `; + if (count > values.length) { + body += `

    Only the first ${values.length} calls keep their values.

    `; + } + } + return body; +} + +function pinnedStage(stageId, step) { + const stage = example().stages.find((candidate) => candidate.id === stageId); + const calls = Object.entries(example().calls).filter(([callId, call]) => + call.stage === stage.name && call.parent === null && invocationsUpTo(callId, step.event) > 0); + const list = calls.length === 0 ? `

    No calls made in this stage yet.

    ` + : `
    Calls:
      ${calls.map(([callId, call]) => + `
      • `).join("")}
    `; + return `${escapeHtml(stage.name)} +

    ${escapeHtml(stage.description)}

    ${list}`; +} + +function pinnedCriteriaNode(nodeId, step) { + for (const [identifier, criterionNodes] of Object.entries(example().criteria)) { + for (const [label, node] of Object.entries(criterionNodes)) { + if (node.id !== nodeId) { + continue; + } + const evaluations = example().steps.filter((candidate) => candidate.kind === "evaluate" + && candidate.criterion === identifier && candidate.event <= step.event); + let status = ""; + if (node.kind === "evaluation") { + const visit = evaluations.find((evaluation) => evaluation.node === label); + status = visit === undefined ? `

    Not evaluated by this step.

    ` + : `

    Evaluated; it reported ${visit.reached.map((reached) => + `${escapeHtml(reached)}`).join(", ") || "no criteria"}.

    `; + } else if (node.kind === "criterion") { + const reached = evaluations.some((evaluation) => evaluation.reached.includes(label)); + status = reached ? `

    Reached.

    ` : `

    Not reached by this step.

    `; + } + return `
    Criteria graph of ${escapeHtml(identifier)} · ${node.kind} node
    +
    ${escapeHtml(label)}
    + ${node.summary && node.summary !== label ? `

    ${escapeHtml(node.summary)}

    ` : ""} + ${node.details ? `

    ${escapeHtml(node.details)}

    ` : ""} + ${status}`; + } + } + return `

    No details.

    `; +} + +function pinnedCriterion(nodeId, step) { + const index = Number(nodeId.replace("criterion_", "")); + const identifier = Object.keys(example().chain.criteria).find((key) => chainOf(key).index === index); + return `
    Criterion
    +
    ${escapeHtml(identifier)}
    +
      ${criterionChainItems(identifier, step).join("")}
    `; +} + +function pinnedSection(step) { + const name = state.pinned; + let body; + if (name.startsWith("call_") && example().calls[name] !== undefined) { + body = pinnedCall(name, step); + } else if (name.startsWith("stage_")) { + body = pinnedStage(name, step); + } else if (name.startsWith("criterion_")) { + body = pinnedCriterion(name, step); + } else if (name.startsWith("criteria_")) { + body = pinnedCriteriaNode(name, step); + } else if (name === "result") { + body = resultSection(); + } else { + body = `

    No details.

    `; + } + return `
    +

    Pinned

    +
    + ${body} +
    `; +} + +function markPinned() { + for (const group of nodes.values()) { + group.classList.remove("is-pinned"); + } + if (state.pinned !== null) { + nodes.get(state.pinned)?.classList.add("is-pinned"); + } +} + +function pin(name) { + state.pinned = name; + markPinned(); + renderPanel(example().steps[state.step]); + document.getElementById("panel").scrollTo(0, 0); +} + function renderPanel(step) { - let content = stageSection(step); + let content = state.pinned === null ? "" : pinnedSection(step); + content += stageSection(step); if (step.kind === "call") { content += callSection(step); } else if (step.kind === "evaluate") { @@ -944,6 +1376,9 @@ def main() -> None: } else if (step.kind === "result") { content += resultSection(); } + if (state.criteria) { + content += criteriaSection(step); + } if (state.forms) { content += formsSection(step); } @@ -964,6 +1399,7 @@ def main() -> None: applyStep(shown, shown.index === state.step); } updateCounts(step.event); + markPinned(); renderPanel(step); const shown = shownSteps(); const position = shown.findIndex((candidate) => candidate.index === state.step); @@ -985,6 +1421,11 @@ def main() -> None: function setPlaying(playing) { state.playing = playing; + if (playing && state.pinned !== null) { + state.pinned = null; + markPinned(); + renderPanel(example().steps[state.step]); + } const button = document.getElementById("play"); button.textContent = playing ? "⏸ Pause" : "▶ Play"; button.setAttribute("aria-label", playing ? "Pause" : "Play"); @@ -1009,6 +1450,7 @@ def main() -> None: state.example = index; state.layout = null; state.step = 0; + state.pinned = null; const current = example(); document.getElementById("inputs").innerHTML = ` response ${escapeHtml(current.response)} @@ -1087,9 +1529,29 @@ def main() -> None: }); document.getElementById("scrubber").addEventListener("input", (event) => { setPlaying(false); + state.pinned = null; state.step = shownSteps()[Number(event.target.value)].index; render(); }); + document.getElementById("graph").addEventListener("click", (event) => { + const group = event.target.closest("g.node"); + if (group === null || group.classList.contains("is-hidden")) { + return; + } + pin(group.querySelector(":scope > title").textContent); + }); + document.getElementById("panel").addEventListener("click", (event) => { + if (event.target.closest("#unpin") !== null) { + state.pinned = null; + markPinned(); + renderPanel(example().steps[state.step]); + return; + } + const link = event.target.closest("[data-pin]"); + if (link !== null) { + pin(link.dataset.pin); + } + }); document.getElementById("zoom-in").addEventListener("click", () => { state.zoom = Math.min(4, state.zoom * 1.25); applyZoom(); diff --git a/scripts/generate_process_graphs.py b/scripts/generate_process_graphs.py index dee477b..d791eeb 100644 --- a/scripts/generate_process_graphs.py +++ b/scripts/generate_process_graphs.py @@ -142,7 +142,8 @@ class Recorder(logging.Handler): - the arguments and return value of each call, and whether it raised an exception instead of returning; - the evaluations each criteria graph runs, and the criteria each one reports; - each new form of the response (see `is_response_form`): the value returned by a call that was given the response, - or an earlier form of it, as an argument. + or an earlier form of it, as an argument; + - the criteria `create_criteria_dict` parses, and the template that builds each one's criteria graph. """ def __init__(self, depth: int = CALL_DEPTH, record_values: bool = False, response: str = "") -> None: @@ -202,10 +203,54 @@ def _record_call(self, frame: FrameType, call: Call) -> None: "kind": "call", "stage": self.stage, "call": call.identifier, "arguments": shown_arguments }) + def _record_criteria(self, frame: FrameType, value: Any) -> None: + """Records the criteria when `frame` is `create_criteria_dict` returning them: the criteria string before and + after preprocessing, and each criterion with the label and tree its parse gave. + """ + if frame.f_code.co_qualname != "create_criteria_dict" or not isinstance(value, dict): + return + self.recording.events.append({ + "kind": "criteria", + "stage": self.stage, + "original": frame.f_locals.get("original_criteria_string", ""), + "preprocessed": frame.f_locals.get("criteria_string", ""), + "criteria": [ + {"identifier": identifier, "label": node.label.strip(), "tree": node.tree_string()} + for (identifier, node) in value.items() + ], + }) + + def _record_template(self, frame: FrameType, value: Any, call: Call | None) -> None: + """Records which template built which criteria graph, when `frame` is a template `feedback_procedure_generator` + called to build the graph of one criterion. + """ + generator = frame.f_back + if generator is None or generator.f_code.co_qualname != "feedback_procedure_generator": + return + if not isinstance(value, CriteriaGraph): + return + # The key of `graph_templates` that chose this template, or None when the template is the context's default + label = None + for (key, template) in generator.f_globals.get("graph_templates", {}).items(): + if getattr(template, "__code__", None) is frame.f_code: + label = key.strip() + self.recording.events.append({ + "kind": "template", + "stage": self.stage, + "call": None if call is None else call.identifier, + "criterion": generator.f_locals["label"], + "template": frame.f_code.co_qualname, + "label": label, + "evaluations": len(value.evaluations), + }) + def _record_return(self, frame: FrameType, value: Any) -> None: call = self.call_frames.pop(frame, None) given_response = frame in self.response_frames self.response_frames.discard(frame) + if returned_normally(frame): + self._record_criteria(frame, value) + self._record_template(frame, value, call) if call is not None: raised = not returned_normally(frame) self.recording.events.append({ @@ -389,7 +434,7 @@ def calls_to_dot( for call in recording.calls.values(): if call.depth > depth: continue - label =call.qualname if call.count == 1 else f"{call.qualname} ×{call.count}" + label = call.qualname if call.count == 1 else f"{call.qualname} ×{call.count}" clusters[call.stage].add_node(pydot.Node(call.identifier, label=_node_label(label, call.module), **CALL_STYLE)) caller = stages[call.stage] if call.parent is None else call.parent.identifier dot.add_edge(pydot.Edge(caller, call.identifier)) From e5ce1cd8f2421ea012a3c732c15985a652b0f47d Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Tue, 3 Nov 2026 17:51:04 +0000 Subject: [PATCH 3/5] Simplify the evaluation animation and show more of what each call does Internal dunder calls are hidden by default, values and parse trees are summarised, and the panel's sections can be collapsed. The criteria graph is drawn beside the generate_feedback stage. The page gains a legend, the parameters the context added, where each feedback tag came from, stage timings, the type of a raised exception, and each call's docstring and source location. Three animation-only examples cover an incorrect response, syntactical comparison and the default template. The layouts are gzipped to keep the page small. Co-Authored-By: Claude Opus 5.5 --- scripts/generate_process_animation.py | 467 ++++++++++++++++++++------ scripts/generate_process_graphs.py | 267 ++++++++++++++- 2 files changed, 623 insertions(+), 111 deletions(-) diff --git a/scripts/generate_process_animation.py b/scripts/generate_process_animation.py index 6a19b9e..5197501 100644 --- a/scripts/generate_process_animation.py +++ b/scripts/generate_process_animation.py @@ -1,19 +1,23 @@ """Generates `build/process_animation.html`: an interactive page that animates the process graphs of `scripts/generate_process_graphs.py`. -Each example is run through `evaluation_function` while recording every call down to `MAX_DEPTH` calls below -`_evaluate`, with its arguments and return value, the forms the response takes, and the evaluations each criteria -graph runs. The calls graph is laid out with graphviz once for each depth, with and without the criteria graphs, and -the page grows it in the order the functions were called. A side panel shows each step's details, and these can be -switched on and off on the page. +Each example (those of the static figures, and `EXTRA_EXAMPLES`) is run through `evaluation_function` while recording +every call down to `MAX_DEPTH` calls below `_evaluate`, with its arguments and return value or exception, the forms the +response takes, the parameters, the criteria and the evaluations each criteria graph runs. The calls graph is laid out +with graphviz once for each depth, with and without the criteria graphs and the internal (dunder) calls, and the page +grows it in the order the functions were called. A side panel shows each step's details, and these can be switched on +and off on the page. The page is a build output and is not committed. Run from the repository root with `python -m scripts.generate_process_animation` (it needs graphviz, i.e. `dot`), then open the page in a browser. """ from __future__ import annotations +import base64 +import gzip import html import json +import re from pathlib import Path from typing import Any @@ -27,11 +31,20 @@ all_stages, calls_to_dot, record, + shown_parent, stage_description, ) OUTPUT_FILE = Path(__file__).resolve().parent.parent / "build" / "process_animation.html" +# Examples shown only in the animation, for the parts of the criteria chain the static figures' examples don't reach: +# a response that is not correct, more than one criterion, and a criterion without a template of its own. +EXTRA_EXAMPLES: list[tuple[str, str, str, dict[str, Any]]] = [ + ("process_symbolic_incorrect", "x + 2", "x + 1", {}), + ("process_symbolic_syntactical_comparison", "1 + x", "x + 1", {"syntactical_comparison": True}), + ("process_symbolic_default_template", "2*x + 1", "x + 1", {"criteria": "response contains x"}), +] + # The deepest calls below `_evaluate` recorded, and the deepest the page can show. MAX_DEPTH = 5 @@ -100,11 +113,14 @@ def build_steps(recording: Recording) -> tuple[list[dict[str, Any]], dict[str, d running.append((values, None)) elif kind == "return": (values, step) = running.pop() - values["returned"] = event["value"] - values["raised"] = event["raised"] + outcome = {"returned": event["value"], "raised": event["raised"]} + if "full" in event: + outcome["returnedFull"] = event["full"] + if "exception" in event: + outcome["exception"] = event["exception"] + values.update(outcome) if step is not None: - step["returned"] = event["value"] - step["raised"] = event["raised"] + step.update(outcome) elif kind == "criteria": steps.append({ "kind": "criteria", @@ -128,6 +144,9 @@ def build_steps(recording: Recording) -> tuple[list[dict[str, Any]], dict[str, d evaluation_step["reached"] = event["reached"] elif kind == "form": forms.append({"event": index, "call": event["call"], "value": event["value"], "type": event["type"]}) + elif kind == "parameters" or kind == "feedback": + # Shown in the panel rather than as steps (see `parameters_data` and `feedback_data`) + continue else: raise ValueError(f"Unknown event kind {kind!r}.") steps.append({"kind": "result", "event": len(recording.events), "depth": 0}) @@ -177,11 +196,13 @@ def criteria_chain(recording: Recording, params: dict[str, Any]) -> dict[str, An } -def visible_call(recording: Recording, identifier: str, depth: int) -> str: - """The identifier of the call `identifier`, or of its nearest caller, that is drawn down to `depth`.""" +def visible_call(recording: Recording, identifier: str, depth: int, hide_internals: bool) -> str: + """The identifier of the call `identifier`, or of its nearest caller, that is drawn down to `depth`, with or + without the internal calls. + """ calls = {call.identifier: call for call in recording.calls.values()} call = calls[identifier] - while call.depth > depth and call.parent is not None: + while call.parent is not None and (call.depth > depth or (hide_internals and call.internal)): call = call.parent return call.identifier @@ -193,15 +214,19 @@ def calls_layout( chain: dict[str, Any], depth: int, with_criteria: bool, + hide_internals: bool, ) -> tuple[pydot.Dot, dict[str, dict[str, str]]]: """The calls graph down to `depth`, with the graphviz identifier of each criteria graph node by criterion and node label. With `with_criteria`, each criterion gets a node in the `parse_criteria` stage, with an edge to the template call its parse label chose, and each criteria graph is drawn with an edge from that template call, which built it, and - from the `generate_feedback` stage, which ran it. + from the `generate_feedback` stage, which ran it. Only the edge from the stage places the graph, so it is drawn + next to the stage rather than past the template call. + + With `hide_internals`, the internal calls are left out (see `calls_to_dot`). """ - dot = calls_to_dot(recording, params, result, depth) + dot = calls_to_dot(recording, params, result, depth, hide_internals) identifiers: dict[str, dict[str, str]] = {} # Without criteria, i.e. when the evaluation stopped before parsing them, there is nothing to add if not with_criteria or not chain: @@ -212,7 +237,7 @@ def calls_layout( parser_call = next(call.identifier for call in recording.calls.values() if call.qualname == "create_criteria_dict") for (index, (criterion, graph)) in enumerate(recording.criteria_graphs.items()): details = chain["criteria"][criterion] - template_call = visible_call(recording, details["call"], depth) + template_call = visible_call(recording, details["call"], depth, hide_internals) criterion_node = f"criterion_{index}" parse_cluster.add_node(pydot.Node( criterion_node, label=_node_label(criterion, f"parse label: {details['label']}"), **CRITERION_STYLE @@ -231,6 +256,7 @@ def calls_layout( label="builds", style="dashed", lhead=cluster.get_name(), + constraint="false", **CRITERION_EDGE_STYLE, )) dot.add_edge(pydot.Edge( @@ -246,9 +272,15 @@ def calls_layout( def svg_markup(dot: pydot.Dot) -> str: - """`dot` drawn as an SVG element, without the XML declaration and doctype in front of it.""" + """`dot` drawn as an SVG element, gzipped and base64 encoded (the page unpacks it). + + The SVG is left without the XML declaration and doctype in front of it, or the comments graphviz writes before each + node and edge (their titles hold the same names). The 20 layouts of an example are mostly the same markup, so + gzipping them makes the page about a tenth of the size. + """ svg = dot.create(format="svg").decode("utf-8") - return svg[svg.index("\n?", "", svg[svg.index(" dict[str, Any]: @@ -266,6 +298,48 @@ def criteria_data(recording: Recording, identifiers: dict[str, dict[str, str]]) return data +def parameters_data(recording: Recording) -> dict[str, Any]: + """The parameters `_build_parameters` completed: the index of its event, the ones it was given and the ones it + added. Empty when it wasn't called. + """ + for (index, event) in enumerate(recording.events): + if event["kind"] == "parameters": + return {"event": index, "given": event["given"], "added": event["added"]} + return {} + + +def feedback_data(recording: Recording, identifiers: dict[str, dict[str, str]]) -> dict[str, dict[str, Any]]: + """For each tag a criteria graph exported, the criterion whose graph exported it first, its feedback text and the + graphviz identifier of its criteria node. + """ + feedback: dict[str, dict[str, Any]] = {} + for event in recording.events: + if event["kind"] != "feedback" or event["criterion"] is None: + continue + for (tag, text) in event["feedback"]: + if tag not in feedback: + feedback[tag] = { + "criterion": event["criterion"], + "text": text, + "node": identifiers.get(event["criterion"], {}).get(tag), + } + return feedback + + +def stage_durations(recording: Recording) -> dict[str, float]: + """How long each stage that ran took, in milliseconds: from its start to the next stage's, or to the end of the + evaluation for the last one. + """ + starts = [(FIRST_STAGE, 0.0)] + list(recording.stage_times.items()) + durations = {} + for (position, (stage, start)) in enumerate(starts): + if position + 1 < len(starts): + durations[stage] = round(starts[position + 1][1] - start, 1) + elif recording.duration_ms is not None: + durations[stage] = round(recording.duration_ms - start, 1) + return durations + + def example_data(name: str, response: str, answer: str, params: dict[str, Any]) -> dict[str, Any]: """Everything the page needs to animate one example.""" (recording, result) = record(response, answer, params, depth=MAX_DEPTH, record_values=True) @@ -275,21 +349,30 @@ def example_data(name: str, response: str, answer: str, params: dict[str, Any]) criteria_identifiers: dict[str, dict[str, str]] = {} for depth in range(1, MAX_DEPTH + 1): for with_criteria in (False, True): - (dot, identifiers) = calls_layout(recording, params, result, chain, depth, with_criteria) - layouts[f"{depth}{'c' if with_criteria else ''}"] = svg_markup(dot) - if with_criteria: - criteria_identifiers = identifiers - calls = { - call.identifier: { + for hide_internals in (False, True): + (dot, identifiers) = calls_layout( + recording, params, result, chain, depth, with_criteria, hide_internals + ) + key = f"{depth}{'c' if with_criteria else ''}{'h' if hide_internals else ''}" + layouts[key] = svg_markup(dot) + if with_criteria: + criteria_identifiers = identifiers + calls = {} + for call in recording.calls.values(): + parent = shown_parent(call, hide_internals=True) + calls[call.identifier] = { "qualname": call.qualname, "module": call.module, "depth": call.depth, "stage": call.stage, "parent": None if call.parent is None else call.parent.identifier, + # The caller drawn when the internal calls are hidden + "shownParent": None if parent is None else parent.identifier, + "internal": call.internal, + "doc": call.doc, + "source": call.source, } - for call in recording.calls.values() - } - stages = all_stages() + durations = stage_durations(recording) return { "name": name, "title": example_title(name), @@ -301,15 +384,19 @@ def example_data(name: str, response: str, answer: str, params: dict[str, Any]) "name": stage, "id": stage_identifier(stage), "description": stage_description(stage, recording, params, result), + "duration": durations.get(stage), } - for stage in stages + for stage in all_stages() ], + "duration": recording.duration_ms, "calls": calls, "steps": steps, "invocations": invocations, "forms": forms, "criteria": criteria_data(recording, criteria_identifiers), "chain": chain, + "parameters": parameters_data(recording), + "feedback": feedback_data(recording, criteria_identifiers), "layouts": layouts, "result": { "is_correct": result["is_correct"], @@ -323,7 +410,7 @@ def example_data(name: str, response: str, answer: str, params: dict[str, Any]) def main() -> None: examples = [] - for (name, response, answer, params) in EXAMPLES: + for (name, response, answer, params) in EXAMPLES + EXTRA_EXAMPLES: examples.append(example_data(name, response, answer, params)) print(f"Recorded {name}: {len(examples[-1]['steps'])} steps") data = json.dumps({"examples": examples, "maxDepth": MAX_DEPTH, "startDepth": START_DEPTH}) @@ -439,14 +526,7 @@ def main() -> None: filter: var(--graph-filter); } #graph svg { display: block; height: auto; } -.zoom { - position: absolute; - right: 12px; - top: 12px; - display: flex; - gap: 4px; - z-index: 1; -} +.zoom { display: flex; gap: 4px; } .zoom button { min-width: 32px; padding: 4px 8px; box-shadow: 0 1px 3px rgba(0, 0, 0, 0.15); } aside { min-height: 0; @@ -572,6 +652,47 @@ def main() -> None: } .status-good { color: var(--good); font-weight: 600; } .status-bad { color: var(--bad); font-weight: 600; } +.doc { margin: 6px 0 0; } +.source { font-size: 12px; color: var(--muted); } +details.section { margin-bottom: 18px; } +details.section > summary { cursor: pointer; list-style: none; display: flex; align-items: center; gap: 6px; margin-bottom: 6px; } +details.section > summary::-webkit-details-marker { display: none; } +details.section > summary::before { content: "▸"; color: var(--muted); font-size: 11px; transition: transform 0.15s; } +details.section[open] > summary::before { transform: rotate(90deg); } +details.section > summary h2 { margin: 0; } +details.more > summary { cursor: pointer; color: var(--accent); font-family: system-ui, sans-serif; font-size: 12px; } +details.more pre { margin: 4px 0 0; white-space: pre-wrap; word-break: break-all; } +.group-label { font-size: 12px; color: var(--muted); margin-top: 8px; } +.feedback-list { list-style: none; margin: 6px 0 0; padding: 0; display: grid; gap: 8px; } +.feedback-list li { padding-left: 10px; border-left: 2px solid var(--border); } +.feedback-text { margin-top: 2px; } + +.graph-tools { + position: absolute; + right: 12px; + top: 12px; + display: flex; + flex-direction: column; + align-items: flex-end; + gap: 6px; + z-index: 1; + max-height: calc(100% - 24px); +} +.legend { + background: var(--surface); + border: 1px solid var(--border); + border-radius: 6px; + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.15); + font-size: 12px; + overflow-y: auto; + min-height: 0; +} +.legend > summary { cursor: pointer; padding: 5px 10px; font-weight: 600; } +.legend-body { padding: 4px 10px 8px; display: grid; gap: 2px; } +.legend-body .group-label { margin-top: 4px; font-weight: 600; } +.legend-row { display: flex; align-items: center; gap: 8px; } +/* The swatches are drawn in the graph's colours, and recoloured with it in dark mode */ +.legend-row svg { flex: none; background: white; filter: var(--graph-filter); border-radius: 2px; } @media (max-width: 900px) { main { grid-template-columns: 1fr; grid-template-rows: minmax(320px, 55vh) auto; overflow-y: auto; } @@ -591,10 +712,37 @@ def main() -> None:
    -
    - - - +
    +
    + + + +
    +
    + Key +
    +
    Nodes
    +
    stage of _evaluate
    +
    function call (×N: calls from that caller)
    +
    criterion parsed from the criteria
    +
    criteria graph evaluation
    +
    criteria graph criterion (a tag)
    +
    output, or the result
    +
    States
    +
    the current step
    +
    its callers (the call stack)
    +
    evaluation run
    +
    criterion reached, and its path
    +
    pinned in the panel
    +
    not reached yet
    +
    Edges
    +
    calls
    +
    criterion chooses its template
    +
    template builds the graph
    +
    generate_feedback runs the graph
    +
    sufficiency in a criteria graph
    +
    +
    @@ -623,6 +771,8 @@ def main() -> None: +
    @@ -638,8 +788,12 @@ def main() -> None: forms: true, values: true, criteria: true, + // Whether the internal (dunder) calls are left out of the graph. + internals: true, speed: 1, zoom: 0.9, + // Whether each collapsible panel section is open, by its key, when it differs from `SECTION_DEFAULTS`. + sections: {}, // The index in the example's steps of the step shown. step: 0, // The graphviz title of the node whose details are pinned in the panel, or null. @@ -654,10 +808,25 @@ def main() -> None: let edges = new Map(); let clusters = new Map(); +// The SVG markup of each layout of the example `layoutsExample`, by its key, unpacked by `unpackLayouts`. +let layouts = new Map(); +let layoutsExample = -1; + +// The layouts of the example `index`, which are gzipped and base64 encoded in the data. +async function unpackLayouts(index) { + const unpacked = new Map(); + for (const [key, packed] of Object.entries(DATA.examples[index].layouts)) { + const bytes = Uint8Array.from(atob(packed), (character) => character.charCodeAt(0)); + const stream = new Blob([bytes]).stream().pipeThrough(new DecompressionStream("gzip")); + unpacked.set(key, await new Response(stream).text()); + } + return unpacked; +} + function loadSettings() { try { const saved = JSON.parse(localStorage.getItem(STORAGE_KEY) || "{}"); - for (const key of ["depth", "forms", "values", "criteria", "speed", "zoom"]) { + for (const key of ["depth", "forms", "values", "criteria", "internals", "speed", "zoom", "sections"]) { if (key in saved) { state[key] = saved[key]; } @@ -669,8 +838,10 @@ def main() -> None: function saveSettings() { try { - const { depth, forms, values, criteria, speed, zoom } = state; - localStorage.setItem(STORAGE_KEY, JSON.stringify({ depth, forms, values, criteria, speed, zoom })); + const { depth, forms, values, criteria, internals, speed, zoom, sections } = state; + localStorage.setItem( + STORAGE_KEY, JSON.stringify({ depth, forms, values, criteria, internals, speed, zoom, sections }) + ); } catch (error) { // The settings are only a convenience, so they are not saved when storage is unavailable. } @@ -695,10 +866,19 @@ def main() -> None: function isShown(step) { if (step.kind === "evaluate" || step.kind === "criteria" || step.kind === "template") { return state.criteria; + } else if (step.kind === "call" && state.internals && example().calls[step.call].internal) { + return false; } return step.depth <= state.depth; } +// The caller drawn for the call `callId`: its parent, or its nearest caller that is not internal when the internal +// calls are hidden; null for a call made by `_evaluate`. +function parentOf(callId) { + const call = example().calls[callId]; + return state.internals ? call.shownParent : call.parent; +} + function shownSteps() { return example().steps.filter(isShown); } @@ -716,11 +896,11 @@ def main() -> None: } function callerOf(callId) { - const call = example().calls[callId]; - if (call.parent === null) { - return stageOf(call.stage).id; + const parent = parentOf(callId); + if (parent === null) { + return stageOf(example().calls[callId].stage).id; } - return call.parent; + return parent; } function invocationsUpTo(callId, event) { @@ -750,14 +930,14 @@ def main() -> None: // Drawing function drawLayout() { - const key = String(state.depth) + (state.criteria ? "c" : ""); + const key = String(state.depth) + (state.criteria ? "c" : "") + (state.internals ? "h" : ""); const layoutId = state.example + ":" + key; if (state.layout === layoutId) { return; } state.layout = layoutId; const graph = document.getElementById("graph"); - graph.innerHTML = example().layouts[key]; + graph.innerHTML = layouts.get(key); nodes = new Map(); edges = new Map(); clusters = new Map(); @@ -991,10 +1171,10 @@ def main() -> None: function callStack(callId) { const stack = []; - let current = example().calls[callId]; - while (current.parent !== null) { - stack.unshift(current.parent); - current = example().calls[current.parent]; + let current = parentOf(callId); + while (current !== null) { + stack.unshift(current); + current = parentOf(current); } return stack; } @@ -1054,42 +1234,90 @@ def main() -> None: return steps[0].stage; } +// Whether each collapsible section is open until it is opened or closed on the page. +const SECTION_DEFAULTS = { stage: true, parameters: false, criteria: true, forms: true }; + +function sectionOpen(key) { + return key in state.sections ? state.sections[key] : SECTION_DEFAULTS[key]; +} + +// A panel section that can be collapsed, and stays open or closed from step to step. +function collapsible(key, title, body) { + return `
    +

    ${escapeHtml(title)}

    ${body}
    `; +} + +// A recorded value; with `withFull`, a summarised value has its full value behind "show more". +function valueHtml(value, full, withFull) { + if (!withFull || full === undefined) { + return escapeHtml(value); + } + return `${escapeHtml(value)}
    show more
    ${escapeHtml(full)}
    `; +} + +// `[name, value, full?]` pairs as a list of names and values. +function namedValues(pairs, withFull) { + return `
    ${pairs.map(([name, value, full]) => + `
    ${escapeHtml(name)}
    ${valueHtml(value, full, withFull)}
    `).join("")}
    `; +} + +function argumentsHtml(argumentList, withFull) { + return argumentList.length === 0 ? `

    No arguments.

    ` : namedValues(argumentList, withFull); +} + +// What a call returned, or the exception it raised. +function outcomeHtml(values, withFull) { + if (values.raised) { + const exception = values.exception === undefined ? "an exception" : values.exception; + return `
    raised ${escapeHtml(exception)}
    `; + } + return `
    ↩ ${valueHtml(values.returned, values.returnedFull, withFull)}
    `; +} + +function durationText(stage) { + return stage.duration === null || stage.duration === undefined ? "" : `took ${stage.duration} ms`; +} + +const TIMING_NOTE = "Times are measured while every call is recorded, so they are far longer than a normal evaluation's."; + function stageSection(step) { const stage = stageOf(stepStage(step)); - return `
    -

    Stage

    - ${escapeHtml(stage.name)} -

    ${escapeHtml(stage.description)}

    -
    `; + const duration = durationText(stage); + return collapsible("stage", "Stage", `${escapeHtml(stage.name)} + ${duration === "" ? "" : `${duration}`} +

    ${escapeHtml(stage.description)}

    `); } function callSection(step) { const call = example().calls[step.call]; const stack = [stepStage(step), ...callStack(step.call).map((id) => example().calls[id].qualname), call.qualname]; - const count = invocationsUpTo(step.call, step.event); const total = invocationEvents(step.call).length; let body = `
    ${escapeHtml(call.qualname)}
    ${escapeHtml(call.module)} · depth ${call.depth}
    -
    ${stack.map((name) => `${escapeHtml(name)}`).join("")}
    `; +
    ${stack.map((name) => `${escapeHtml(name)}`).join("")}
    + ${call.doc === "" ? "" : `

    ${escapeHtml(call.doc)}

    `}`; if (total > 1) { body += `

    First of ${total} calls from this caller; the rest raise the count on its node.

    `; } if (state.values) { - if (step.arguments.length > 0) { - body += `
    ${step.arguments.map(([name, value]) => - `
    ${escapeHtml(name)}
    ${escapeHtml(value)}
    `).join("")}
    `; - } else { - body += `

    No arguments.

    `; - } - if (step.raised) { - body += `
    raised an exception
    `; - } else { - body += `
    ↩ ${escapeHtml(step.returned)}
    `; - } + body += argumentsHtml(step.arguments, false) + outcomeHtml(step, false); } return `

    Call

    ${body}
    `; } +// The parameters `_build_parameters` completed, once it has returned. +function parametersSection(step) { + const parameters = example().parameters; + if (parameters.event === undefined || step.event < parameters.event) { + return ""; + } + const group = (pairs) => pairs.length === 0 ? `

    None.

    ` : namedValues(pairs, true); + return collapsible("parameters", "Parameters", ` +
    Given in params
    ${group(parameters.given)} +
    Added by _build_parameters: the context's defaults and the parsing set-up
    + ${group(parameters.added)}`); +} + function evaluationSection(step) { const criterionNodes = example().criteria[step.criterion]; const evaluation = criterionNodes[step.node]; @@ -1123,7 +1351,23 @@ def main() -> None: ${isNew ? 'new' : ""} ${escapeHtml(form.value)}`); }); - return `

    Response forms

      ${items.join("")}
    `; + return collapsible("forms", "Response forms", `
      ${items.join("")}
    `); +} + +// A tag of the result: the criteria graph criterion it came from (a link that pins it) and its feedback text. +function tagItem(tag) { + const source = example().feedback[tag]; + if (source === undefined) { + return `
  • ${escapeHtml(tag)} +
    Added by the evaluation itself, not by a criteria graph.
    • `; + } + const name = source.node === null ? `${escapeHtml(tag)}` + : ``; + const text = source.text === "" ? `No feedback text.` + : `“${escapeHtml(source.text)}”`; + return `
  • ${name} +
    Reached in the criteria graph of ${escapeHtml(source.criterion)}
    +
    • `; } function resultSection() { @@ -1134,14 +1378,17 @@ def main() -> None: preview = `
    Preview LaTeX: ${escapeHtml(result.response_latex)}
    Simplified: ${escapeHtml(result.response_simplified)}
    `; } + const duration = example().duration === null ? "" + : `
    The evaluation took ${example().duration} ms while recording.
    `; return `

    Result

    is_correct: ${result.is_correct}
    -

    ${escapeHtml(result.feedback)}

    -
    Tags
    -
      ${result.tags.map((tag) => `
    • ${escapeHtml(tag)}
      • `).join("")}
    + ${result.feedback === "" ? `

    No feedback text.

    ` : `

    ${escapeHtml(result.feedback)}

    `} +
    Tags, and where each came from
    + ${preview} + ${duration}
    `; } @@ -1242,7 +1489,7 @@ def main() -> None: ${correct}: true only if every criteria graph reached its main criterion.`); } - return `

    Criteria

      ${items.join("")}
    `; + return collapsible("criteria", "Criteria", `
      ${items.join("")}
    `); } // Pinned details @@ -1250,28 +1497,19 @@ def main() -> None: function pinnedCall(callId, step) { const call = example().calls[callId]; const caller = callerOf(callId); - const callerName = call.parent === null ? "the " + call.stage + " stage" : example().calls[call.parent].qualname; + const parent = parentOf(callId); + const callerName = parent === null ? "the " + call.stage + " stage" : example().calls[parent].qualname; const count = invocationsUpTo(callId, step.event); const values = (example().invocations[callId]?.values || []).filter((invocation) => invocation.event <= step.event); let body = `
    ${escapeHtml(call.qualname)}
    ${escapeHtml(call.module)} · depth ${call.depth} · stage ${escapeHtml(call.stage)}
    +
    ${escapeHtml(call.source)}
    + ${call.doc === "" ? `

    No docstring.

    ` : `

    ${escapeHtml(call.doc)}

    `}
    Called by , ${count} time${count === 1 ? "" : "s"} so far.
    `; if (values.length > 0) { - const items = values.map((invocation, position) => { - const argumentList = invocation.arguments.length === 0 ? `

    No arguments.

    ` - : `
    ${invocation.arguments.map(([name, value]) => - `
    ${escapeHtml(name)}
    ${escapeHtml(value)}
    `).join("")}
    `; - let returned; - if (invocation.event > step.event) { - returned = ""; - } else if (invocation.raised) { - returned = `
    raised an exception
    `; - } else { - returned = `
    ↩ ${escapeHtml(invocation.returned)}
    `; - } - return `
  • Call ${position + 1}
    ${argumentList}${returned}
    • `; - }); + const items = values.map((invocation, position) => + `
  • Call ${position + 1}
    ${argumentsHtml(invocation.arguments, true)}${outcomeHtml(invocation, true)}
    • `); body += `
      ${items.join("")}
    `; if (count > values.length) { body += `

    Only the first ${values.length} calls keep their values.

    `; @@ -1282,13 +1520,16 @@ def main() -> None: function pinnedStage(stageId, step) { const stage = example().stages.find((candidate) => candidate.id === stageId); - const calls = Object.entries(example().calls).filter(([callId, call]) => - call.stage === stage.name && call.parent === null && invocationsUpTo(callId, step.event) > 0); + const calls = Object.entries(example().calls).filter(([callId, call]) => call.stage === stage.name + && parentOf(callId) === null && !(state.internals && call.internal) && invocationsUpTo(callId, step.event) > 0); const list = calls.length === 0 ? `

    No calls made in this stage yet.

    ` : `
    Calls:
      ${calls.map(([callId, call]) => `
      • `).join("")}
    `; + const duration = durationText(stage); return `${escapeHtml(stage.name)} -

    ${escapeHtml(stage.description)}

    ${list}`; + ${duration === "" ? "" : `${duration}`} +

    ${escapeHtml(stage.description)}

    ${list} + ${duration === "" ? "" : `

    ${escapeHtml(TIMING_NOTE)}

    `}`; } function pinnedCriteriaNode(nodeId, step) { @@ -1376,6 +1617,7 @@ def main() -> None: } else if (step.kind === "result") { content += resultSection(); } + content += parametersSection(step); if (state.criteria) { content += criteriaSection(step); } @@ -1388,6 +1630,10 @@ def main() -> None: // Rendering and controls function render() { + // Until the example's layouts are unpacked; `selectExample` renders once they are + if (layoutsExample !== state.example) { + return; + } drawLayout(); const steps = example().steps; const step = steps[state.step]; @@ -1446,7 +1692,7 @@ def main() -> None: }, 1100 / state.speed); } -function selectExample(index) { +async function selectExample(index) { state.example = index; state.layout = null; state.step = 0; @@ -1456,6 +1702,13 @@ def main() -> None: response ${escapeHtml(current.response)} answer ${escapeHtml(current.answer)} params ${escapeHtml(JSON.stringify(current.params))}`; + const unpacked = await unpackLayouts(index); + // Another example can be chosen while these layouts are unpacked + if (state.example !== index) { + return; + } + layouts = unpacked; + layoutsExample = index; render(); document.getElementById("graph").scrollTo(0, 0); } @@ -1475,6 +1728,13 @@ def main() -> None: document.getElementById("show-forms").checked = state.forms; document.getElementById("show-values").checked = state.values; document.getElementById("show-criteria").checked = state.criteria; + document.getElementById("hide-internals").checked = state.internals; + const legend = document.getElementById("legend"); + legend.open = state.sections.legend === true; + legend.addEventListener("toggle", () => { + state.sections.legend = legend.open; + saveSettings(); + }); exampleSelect.addEventListener("change", () => { setPlaying(false); @@ -1506,6 +1766,15 @@ def main() -> None: saveSettings(); render(); }); + document.getElementById("hide-internals").addEventListener("change", (event) => { + state.internals = event.target.checked; + state.step = nearestShownStep(state.step); + if (state.pinned !== null && state.internals && example().calls[state.pinned]?.internal) { + state.pinned = null; + } + saveSettings(); + render(); + }); document.getElementById("restart").addEventListener("click", () => { setPlaying(false); state.step = 0; @@ -1552,6 +1821,14 @@ def main() -> None: pin(link.dataset.pin); } }); + // `toggle` does not bubble, so it is caught on its way down to the section + document.getElementById("panel").addEventListener("toggle", (event) => { + const key = event.target.dataset?.section; + if (key !== undefined && sectionOpen(key) !== event.target.open) { + state.sections[key] = event.target.open; + saveSettings(); + } + }, true); document.getElementById("zoom-in").addEventListener("click", () => { state.zoom = Math.min(4, state.zoom * 1.25); applyZoom(); diff --git a/scripts/generate_process_graphs.py b/scripts/generate_process_graphs.py index d791eeb..265513f 100644 --- a/scripts/generate_process_graphs.py +++ b/scripts/generate_process_graphs.py @@ -27,6 +27,8 @@ from lambdafeedback.criteria import CriteriaGraph from app import evaluation +from app.context.base import Context +from app.result import EvaluationResult from scripts.generate_criteria_graphs import _node_label, add_criteria_graph, new_drawing OUTPUT_DIRECTORY = Path(__file__).resolve().parent.parent / "app" / "docs" / "process_graphs" @@ -41,8 +43,15 @@ # How many calls below `_evaluate` the call graphs go. CALL_DEPTH = 2 -# The longest an argument or return value recorded for the process animation gets before it is cut short. +# The longest an argument or return value recorded for the process animation gets before it is cut short, and the +# longest the full value of a summarised dictionary gets. VALUE_LENGTH = 200 +FULL_VALUE_LENGTH = 1000 + +# The longest an exception's type and message recorded for the process animation get before they are cut short. +EXCEPTION_LENGTH = 160 + +REPOSITORY = Path(__file__).resolve().parent.parent # Functions left out of the call graphs; the functions they call are shown under their caller instead. HIDDEN_FUNCTIONS = {"log_event", "elapsed_ms", "_evaluate..enter_stage"} @@ -74,6 +83,16 @@ class Call: parent: Call | None depth: int count: int = 1 + # The first paragraph of the function's docstring, and where it is defined (`path:line`); only set when values + # are recorded. + doc: str = "" + source: str = "" + + @property + def internal(self) -> bool: + """Whether the function is a dunder method, such as `__init__` or `__getstate__`.""" + name = self.qualname.rsplit(".", 1)[-1] + return name.startswith("__") and name.endswith("__") @dataclass @@ -82,6 +101,9 @@ class Recording: `events` is filled only when values are recorded (see `Recorder`): every stage entered, call made, value returned and criteria graph evaluation run, in order, as JSON-ready dictionaries. + + `stage_times` is the time from the start of the evaluation to the start of each stage, and `duration_ms` the time + the whole evaluation took, in milliseconds, both from the evaluation's log events. """ stages: list[str] = field(default_factory=list) @@ -89,21 +111,146 @@ class Recording: calls: dict[tuple[Any, ...], Call] = field(default_factory=dict) criteria_graphs: dict[str, CriteriaGraph] = field(default_factory=dict) events: list[dict[str, Any]] = field(default_factory=list) + stage_times: dict[str, float] = field(default_factory=dict) + duration_ms: float | None = None -def short_repr(value: Any) -> str: - """`repr(value)` without memory addresses, cut short at `VALUE_LENGTH` characters.""" +def _cut(text: str, length: int) -> str: + if len(text) > length: + return text[:length - 1] + "…" + return text + + +def _plain_repr(value: Any) -> str: + """`repr(value)` without memory addresses.""" try: text = repr(value) except (AttributeError, TypeError, ValueError, KeyError): # Objects whose `__init__` is still running can fail to give a repr. text = f"<{type(value).__name__}>" - text = re.sub(r" at 0x[0-9a-f]+", "", text) - if len(text) > VALUE_LENGTH: - return text[:VALUE_LENGTH - 1] + "…" + return re.sub(r" at 0x[0-9a-f]+", "", text) + + +def short_repr(value: Any) -> str: + """A short description of `value` for the process animation: `repr(value)` without memory addresses, cut short at + `VALUE_LENGTH` characters. + + The values whose repr says little are summarised instead: a context by its name, an `EvaluationResult` by its + correctness and tags, and a dictionary too long to show by its keys. + """ + if isinstance(value, Context): + name = value.feedback_procedure_generator.__module__.removeprefix("app.context.").split(".")[0] + return f"{name} context" + elif isinstance(value, EvaluationResult): + try: + return f"EvaluationResult(is_correct={value.is_correct}, tags={value.tags})" + except AttributeError: + # The result's `__init__` is still running + return "EvaluationResult()" + text = _plain_repr(value) + if isinstance(value, dict) and len(text) > VALUE_LENGTH: + keys = ", ".join(str(key) for key in value) + return _cut(f"{{… {len(value)} keys: {keys} …}}", VALUE_LENGTH) + return _cut(text, VALUE_LENGTH) + + +def full_repr(value: Any) -> str | None: + """The repr of `value`, cut short at `FULL_VALUE_LENGTH` characters, when `short_repr` summarises it as a dictionary + of keys; None otherwise. + """ + if not isinstance(value, dict): + return None + text = _plain_repr(value) + if len(text) <= VALUE_LENGTH: + return None + return _cut(text, FULL_VALUE_LENGTH) + + +def shown_value(name: str, value: Any) -> list[str]: + """The name of an argument and its `short_repr`, followed by its `full_repr` when there is one.""" + full = full_repr(value) + if full is None: + return [name, short_repr(value)] + return [name, short_repr(value), full] + + +def exception_text(exception: BaseException) -> str: + """The type and message of `exception`, cut short at `EXCEPTION_LENGTH` characters.""" + try: + message = str(exception) + except (AttributeError, TypeError, ValueError): + message = "" + name = type(exception).__name__ + return _cut(f"{name}: {message}" if message else name, EXCEPTION_LENGTH) + + +def parse_tree_text(node: Any) -> str: + """The criteria parse tree under `node` drawn with `├─` and `└─` connectors, one node per line as `label: content`, + followed by the node's tags when it has any. + """ + lines = [_parse_node_text(node)] + _add_parse_children(node, "", lines) + return "\n".join(lines) + + +def _parse_node_text(node: Any) -> str: + label = node.label.strip() + content = str(node.content).strip() + text = label if content in ("", label) else f"{label}: {content}" + if node.tags: + text += f" tags: {', '.join(sorted(str(tag) for tag in node.tags))}" return text +def _add_parse_children(node: Any, indent: str, lines: list[str]) -> None: + for (index, child) in enumerate(node.children): + last = index == len(node.children) - 1 + lines.append(indent + ("└─ " if last else "├─ ") + _parse_node_text(child)) + _add_parse_children(child, indent + (" " if last else "│ "), lines) + + +def source_location(frame: FrameType) -> str: + """Where the function running in `frame` is defined, as `path:line`, with the path relative to the repository or, + for an installed package, to `site-packages`. + """ + path = Path(frame.f_code.co_filename) + if path.is_relative_to(REPOSITORY) and "site-packages" not in path.parts: + shown = str(path.relative_to(REPOSITORY)) + elif "site-packages" in path.parts: + shown = "/".join(path.parts[path.parts.index("site-packages") + 1:]) + else: + shown = path.name + return f"{shown}:{frame.f_code.co_firstlineno}" + + +def first_doc_paragraph(frame: FrameType) -> str: + """The first paragraph of the docstring of the function running in `frame`, on one line, found from its module by + its qualified name; empty when it has none of its own, or is defined inside another function. + """ + qualname = frame.f_code.co_qualname + if "" in qualname: + return "" + function: Any = None + for (index, part) in enumerate(qualname.split(".")): + if index == 0: + function = frame.f_globals.get(part) + else: + function = inspect.getattr_static(function, part, None) + if function is None: + return "" + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + elif isinstance(function, property): + function = function.fget + # `__doc__` rather than `inspect.getdoc`, which falls back to the docstring of the method a class inherits from, + # e.g. `object.__init__`'s "Initialize self." + doc = getattr(function, "__doc__", None) + if not isinstance(doc, str): + return "" + paragraph = inspect.cleandoc(doc).split("\n\n")[0] + return _cut(" ".join(paragraph.split()), VALUE_LENGTH) + + def frame_argument_values(frame: FrameType) -> list[tuple[str, Any]]: """The name and value of each argument of the function running in `frame`.""" arguments = inspect.getargvalues(frame) @@ -139,11 +286,14 @@ class Recorder(logging.Handler): Calls are followed down to `depth` calls below `_evaluate`. With `record_values`, these are kept in order in `Recording.events`: - - the arguments and return value of each call, and whether it raised an exception instead of returning; + - the arguments and return value of each call, or the exception it raised instead of returning (see `unwind`); - the evaluations each criteria graph runs, and the criteria each one reports; - each new form of the response (see `is_response_form`): the value returned by a call that was given the response, or an earlier form of it, as an argument; - - the criteria `create_criteria_dict` parses, and the template that builds each one's criteria graph. + - the criteria `create_criteria_dict` parses, and the template that builds each one's criteria graph; + - the parameters `_build_parameters` completes, and the feedback each criteria graph exports for its reached + criteria. + The docstring and source location of each function called are kept on its `Call` too. """ def __init__(self, depth: int = CALL_DEPTH, record_values: bool = False, response: str = "") -> None: @@ -160,16 +310,31 @@ def __init__(self, depth: int = CALL_DEPTH, record_values: bool = False, respons # The reprs of the forms the response has taken so far, and the frames given one of them as an argument. self.response_forms = {repr(response)} self.response_frames: set[FrameType] = set() + # The code of the last call recorded as raising an exception, and its return event, until `unwind` fills in + # the exception. + self.raised: tuple[Any, dict[str, Any]] | None = None def emit(self, record: logging.LogRecord) -> None: event = json.loads(record.getMessage()) if event["event"] == "stage_start": self.stage = event["stage"] self.recording.stages.append(self.stage) + self.recording.stage_times[self.stage] = event["elapsed_ms"] if self.record_values: self.recording.events.append({"kind": "stage", "stage": self.stage}) elif event["event"] == "evaluation_end": self.recording.early_return = event["early_return"] + self.recording.duration_ms = event["duration_ms"] + + def unwind(self, code: Any, offset: int, exception: BaseException) -> None: + """The `sys.monitoring` `PY_UNWIND` callback: a frame of `code` is stopping because of `exception`. + + It runs just after the profile function's `return` event for the same frame, which cannot see the exception, + so the exception is added to the return event recorded there. + """ + if self.raised is not None and self.raised[0] is code: + self.raised[1]["exception"] = exception_text(exception) + self.raised = None def _record_evaluation(self, frame: FrameType) -> None: """Records the start of a criteria graph evaluation, when `frame` is the `evaluate` function of one.""" @@ -196,9 +361,9 @@ def _record_call(self, frame: FrameType, call: Call) -> None: shown_arguments = [] for (name, value) in arguments: if name == "self": - shown_arguments.append((name, f"<{type(value).__name__}>")) + shown_arguments.append([name, f"<{type(value).__name__}>"]) else: - shown_arguments.append((name, short_repr(value))) + shown_arguments.append(shown_value(name, value)) self.recording.events.append({ "kind": "call", "stage": self.stage, "call": call.identifier, "arguments": shown_arguments }) @@ -215,11 +380,43 @@ def _record_criteria(self, frame: FrameType, value: Any) -> None: "original": frame.f_locals.get("original_criteria_string", ""), "preprocessed": frame.f_locals.get("criteria_string", ""), "criteria": [ - {"identifier": identifier, "label": node.label.strip(), "tree": node.tree_string()} + {"identifier": identifier, "label": node.label.strip(), "tree": parse_tree_text(node)} for (identifier, node) in value.items() ], }) + def _record_parameters(self, frame: FrameType, value: Any) -> None: + """Records the parameters when `frame` is `_build_parameters` returning them: the parameters it was given, and + the ones it added. + """ + if frame.f_code.co_qualname != "_build_parameters" or not isinstance(value, dict): + return + given = frame.f_locals.get("params", {}) + self.recording.events.append({ + "kind": "parameters", + "stage": self.stage, + "given": [shown_value(str(key), item) for (key, item) in value.items() if key in given], + "added": [shown_value(str(key), item) for (key, item) in value.items() if key not in given], + }) + + def _record_feedback(self, frame: FrameType, value: Any) -> None: + """Records the feedback text of each reached criterion, when `frame` is `CriteriaGraph.resolve_feedback` + returning it for `CriteriaGraph.export_feedback`. + """ + exporter = frame.f_back + if frame.f_code.co_qualname != "CriteriaGraph.resolve_feedback" or exporter is None: + return + if exporter.f_code.co_qualname != "CriteriaGraph.export_feedback": + return + graph = frame.f_locals.get("self") + criterion = next((key for (key, item) in self.recording.criteria_graphs.items() if item is graph), None) + self.recording.events.append({ + "kind": "feedback", + "stage": self.stage, + "criterion": criterion, + "feedback": [[str(tag), str(text)] for (tag, text) in value], + }) + def _record_template(self, frame: FrameType, value: Any, call: Call | None) -> None: """Records which template built which criteria graph, when `frame` is a template `feedback_procedure_generator` called to build the graph of one criterion. @@ -251,15 +448,23 @@ def _record_return(self, frame: FrameType, value: Any) -> None: if returned_normally(frame): self._record_criteria(frame, value) self._record_template(frame, value, call) + self._record_parameters(frame, value) + self._record_feedback(frame, value) if call is not None: raised = not returned_normally(frame) - self.recording.events.append({ + event = { "kind": "return", "stage": self.stage, "call": call.identifier, "value": "" if raised else short_repr(value), "raised": raised, - }) + } + full = None if raised else full_repr(value) + if full is not None: + event["full"] = full + self.recording.events.append(event) + if raised: + self.raised = (frame.f_code, event) if given_response and not raised and is_response_form(value) and repr(value) not in self.response_forms: self.response_forms.add(repr(value)) self.recording.events.append({ @@ -318,6 +523,9 @@ def profile(self, frame: FrameType, event: str, arg: Any) -> None: call = self.recording.calls.get(key) if call is None: call = Call(f"call_{len(self.recording.calls)}", module, qualname, self.stage, parent, depth + 1) + if self.record_values: + call.doc = first_doc_paragraph(frame) + call.source = source_location(frame) self.recording.calls[key] = call else: call.count += 1 @@ -336,12 +544,22 @@ def record( recorder = Recorder(depth, record_values, response) logger = logging.getLogger(evaluation.__name__) logger.addHandler(recorder) + monitoring = sys.monitoring + unwind = monitoring.events.PY_UNWIND + if record_values: + monitoring.use_tool_id(monitoring.PROFILER_ID, "generate_process_graphs") + monitoring.register_callback(monitoring.PROFILER_ID, unwind, recorder.unwind) + monitoring.set_events(monitoring.PROFILER_ID, unwind) sys.setprofile(recorder.profile) try: result = evaluation.evaluation_function(response, answer, dict(params), include_test_data=True) finally: sys.setprofile(None) logger.removeHandler(recorder) + if record_values: + monitoring.set_events(monitoring.PROFILER_ID, monitoring.events.NO_EVENTS) + monitoring.register_callback(monitoring.PROFILER_ID, unwind, None) + monitoring.free_tool_id(monitoring.PROFILER_ID) return (recorder.recording, result) @@ -415,11 +633,27 @@ def process_to_dot(recording: Recording, params: dict[str, Any], result: dict[st return dot +def shown_parent(call: Call, hide_internals: bool) -> Call | None: + """The caller drawn for `call`: its parent, or with `hide_internals` its nearest caller that is not internal (see + `Call.internal`); None for a call made by `_evaluate`. + """ + parent = call.parent + while hide_internals and parent is not None and parent.internal: + parent = parent.parent + return parent + + def calls_to_dot( - recording: Recording, params: dict[str, Any], result: dict[str, Any], depth: int = CALL_DEPTH + recording: Recording, + params: dict[str, Any], + result: dict[str, Any], + depth: int = CALL_DEPTH, + hide_internals: bool = False, ) -> pydot.Dot: """The stages of the evaluation down the left, each with the functions called during it to its right, down to `depth` calls below `_evaluate`. + + With `hide_internals`, dunder methods are left out, and the calls they made are drawn under their caller. """ dot = new_drawing() dot.set("rankdir", "LR") @@ -432,11 +666,12 @@ def calls_to_dot( clusters[stage] = cluster dot.add_subgraph(cluster) for call in recording.calls.values(): - if call.depth > depth: + if call.depth > depth or (hide_internals and call.internal): continue label = call.qualname if call.count == 1 else f"{call.qualname} ×{call.count}" clusters[call.stage].add_node(pydot.Node(call.identifier, label=_node_label(label, call.module), **CALL_STYLE)) - caller = stages[call.stage] if call.parent is None else call.parent.identifier + parent = shown_parent(call, hide_internals) + caller = stages[call.stage] if parent is None else parent.identifier dot.add_edge(pydot.Edge(caller, call.identifier)) return dot From e2af2b71ab1a6c2c635aa813003821183b77f30a Mon Sep 17 00:00:00 2001 From: Marcus Messer Date: Tue, 3 Nov 2026 19:47:12 +0000 Subject: [PATCH 4/5] Move to a node's step when it is clicked, instead of pinning its details Clicking a node, or a tag or criterion link in the panel, makes that node the current node. A criteria graph criterion or output moves to the evaluation that reached it. The call section takes over the source location, the full values and the later calls from the pinned view. Co-Authored-By: Claude Opus 5.5 --- scripts/generate_process_animation.py | 207 +++++++++----------------- 1 file changed, 74 insertions(+), 133 deletions(-) diff --git a/scripts/generate_process_animation.py b/scripts/generate_process_animation.py index 5197501..016bbb7 100644 --- a/scripts/generate_process_animation.py +++ b/scripts/generate_process_animation.py @@ -51,7 +51,7 @@ # The depth the page starts at: the depth of the static calls figures. START_DEPTH = 2 -# How many calls of each function keep their arguments and return value, for the details of a clicked node. +# How many calls of each function keep their arguments and return value, for the list of calls in the panel. INVOCATION_VALUES = 20 CRITERION_STYLE = {"shape": "note", "style": "filled", "fillcolor": "#FFF8E1", "color": "#8D6E63"} @@ -628,11 +628,7 @@ def main() -> None: #graph g.edge.is-reached > polygon { stroke: #e65100; fill: #e65100; } #graph g.node:not(.is-hidden) { cursor: pointer; } #graph g.node:not(.is-hidden):hover > :is(polygon, path, ellipse):first-of-type { stroke-width: 2.5px; } -#graph g.is-pinned > :is(polygon, path, ellipse):first-of-type { stroke: #7b1fa2; stroke-width: 4px; stroke-dasharray: none; } -.pinned { border: 1px solid var(--border); border-left: 3px solid #ab47bc; border-radius: 8px; padding: 10px 12px; } -.pinned-head { display: flex; align-items: flex-start; justify-content: space-between; gap: 8px; } -.pinned-head button { min-height: 26px; padding: 0 8px; } .link { color: var(--accent); background: none; border: none; padding: 0; min-height: 0; text-decoration: underline; cursor: pointer; } .invocations { list-style: none; margin: 8px 0 0; padding: 0; display: grid; gap: 8px; } .invocations li { border-top: 1px solid var(--border); padding-top: 6px; } @@ -733,7 +729,6 @@ def main() -> None:
    its callers (the call stack)
    evaluation run
    criterion reached, and its path
    -
    pinned in the panel
    not reached yet
    Edges
    calls
    @@ -796,8 +791,6 @@ def main() -> None: sections: {}, // The index in the example's steps of the step shown. step: 0, - // The graphviz title of the node whose details are pinned in the panel, or null. - pinned: null, playing: false, timer: null, layout: null, @@ -983,7 +976,7 @@ def main() -> None: } const STATE_CLASSES = [ - "is-hidden", "is-waiting", "is-current", "is-stack", "is-visited", "is-reached", "is-pinned", "appear", "draw", + "is-hidden", "is-waiting", "is-current", "is-stack", "is-visited", "is-reached", "appear", "draw", ]; // Whether the node `name` is only drawn once a step reveals it: calls, criteria and criteria graph nodes. @@ -1294,17 +1287,34 @@ def main() -> None: const total = invocationEvents(step.call).length; let body = `
    ${escapeHtml(call.qualname)}
    ${escapeHtml(call.module)} · depth ${call.depth}
    +
    ${escapeHtml(call.source)}
    ${stack.map((name) => `${escapeHtml(name)}`).join("")}
    ${call.doc === "" ? "" : `

    ${escapeHtml(call.doc)}

    `}`; + if (state.values) { + body += argumentsHtml(step.arguments, true) + outcomeHtml(step, true); + } if (total > 1) { body += `

    First of ${total} calls from this caller; the rest raise the count on its node.

    `; - } - if (state.values) { - body += argumentsHtml(step.arguments, false) + outcomeHtml(step, false); + if (state.values) { + body += laterCalls(step.call, total); + } } return `

    Call

    ${body}
    `; } +// The calls after the first from the same caller, with the values of those that kept them, behind "show". +function laterCalls(callId, total) { + const values = example().invocations[callId].values.slice(1); + if (values.length === 0) { + return ""; + } + const items = values.map((invocation, position) => `
  • Call ${position + 2}
    + ${argumentsHtml(invocation.arguments, true)}${outcomeHtml(invocation, true)}
    • `); + const kept = values.length + 1 < total ? `

    Only the first ${values.length + 1} calls keep their values.

    ` : ""; + return `
    show the other calls +
      ${items.join("")}
    ${kept}
    `; +} + // The parameters `_build_parameters` completed, once it has returned. function parametersSection(step) { const parameters = example().parameters; @@ -1354,7 +1364,8 @@ def main() -> None: return collapsible("forms", "Response forms", `
      ${items.join("")}
    `); } -// A tag of the result: the criteria graph criterion it came from (a link that pins it) and its feedback text. +// A tag of the result: the criteria graph criterion it came from (a link to the evaluation that reached it) and its +// feedback text. function tagItem(tag) { const source = example().feedback[tag]; if (source === undefined) { @@ -1362,7 +1373,7 @@ def main() -> None:
    Added by the evaluation itself, not by a criteria graph.
    `; } const name = source.node === null ? `${escapeHtml(tag)}` - : ``; + : ``; const text = source.text === "" ? `No feedback text.` : `“${escapeHtml(source.text)}”`; return `
  • ${name} @@ -1479,7 +1490,7 @@ def main() -> None: ${identifiers.map((identifier) => `${escapeHtml(identifier)}`).join(" ")}
    • `]; for (const identifier of identifiers) { items.push(`
  • -
    +
      ${criterionChainItems(identifier, step).join("")}
    • `); } const statuses = identifiers.map((identifier) => mainCriterionStatus(identifier, step)); @@ -1492,124 +1503,71 @@ def main() -> None: return collapsible("criteria", "Criteria", `
      ${items.join("")}
    `); } -// Pinned details +// Clicking a node -function pinnedCall(callId, step) { - const call = example().calls[callId]; - const caller = callerOf(callId); - const parent = parentOf(callId); - const callerName = parent === null ? "the " + call.stage + " stage" : example().calls[parent].qualname; - const count = invocationsUpTo(callId, step.event); - const values = (example().invocations[callId]?.values || []).filter((invocation) => invocation.event <= step.event); - let body = `
    ${escapeHtml(call.qualname)}
    -
    ${escapeHtml(call.module)} · depth ${call.depth} · stage ${escapeHtml(call.stage)}
    -
    ${escapeHtml(call.source)}
    - ${call.doc === "" ? `

    No docstring.

    ` : `

    ${escapeHtml(call.doc)}

    `} -
    Called by , - ${count} time${count === 1 ? "" : "s"} so far.
    `; - if (values.length > 0) { - const items = values.map((invocation, position) => - `
  • Call ${position + 1}
    ${argumentsHtml(invocation.arguments, true)}${outcomeHtml(invocation, true)}
    • `); - body += `
      ${items.join("")}
    `; - if (count > values.length) { - body += `

    Only the first ${values.length} calls keep their values.

    `; - } - } - return body; -} - -function pinnedStage(stageId, step) { - const stage = example().stages.find((candidate) => candidate.id === stageId); - const calls = Object.entries(example().calls).filter(([callId, call]) => call.stage === stage.name - && parentOf(callId) === null && !(state.internals && call.internal) && invocationsUpTo(callId, step.event) > 0); - const list = calls.length === 0 ? `

    No calls made in this stage yet.

    ` - : `
    Calls:
      ${calls.map(([callId, call]) => - `
      • `).join("")}
    `; - const duration = durationText(stage); - return `${escapeHtml(stage.name)} - ${duration === "" ? "" : `${duration}`} -

    ${escapeHtml(stage.description)}

    ${list} - ${duration === "" ? "" : `

    ${escapeHtml(TIMING_NOTE)}

    `}`; -} - -function pinnedCriteriaNode(nodeId, step) { +// The criteria graph `nodes` entry with the graphviz title `name`, with its criterion and label, or null. +function criteriaGraphNode(name) { for (const [identifier, criterionNodes] of Object.entries(example().criteria)) { for (const [label, node] of Object.entries(criterionNodes)) { - if (node.id !== nodeId) { - continue; + if (node.id === name) { + return { identifier: identifier, label: label, node: node }; } - const evaluations = example().steps.filter((candidate) => candidate.kind === "evaluate" - && candidate.criterion === identifier && candidate.event <= step.event); - let status = ""; - if (node.kind === "evaluation") { - const visit = evaluations.find((evaluation) => evaluation.node === label); - status = visit === undefined ? `

    Not evaluated by this step.

    ` - : `

    Evaluated; it reported ${visit.reached.map((reached) => - `${escapeHtml(reached)}`).join(", ") || "no criteria"}.

    `; - } else if (node.kind === "criterion") { - const reached = evaluations.some((evaluation) => evaluation.reached.includes(label)); - status = reached ? `

    Reached.

    ` : `

    Not reached by this step.

    `; - } - return `
    Criteria graph of ${escapeHtml(identifier)} · ${node.kind} node
    -
    ${escapeHtml(label)}
    - ${node.summary && node.summary !== label ? `

    ${escapeHtml(node.summary)}

    ` : ""} - ${node.details ? `

    ${escapeHtml(node.details)}

    ` : ""} - ${status}`; } } - return `

    No details.

    `; + return null; } -function pinnedCriterion(nodeId, step) { - const index = Number(nodeId.replace("criterion_", "")); - const identifier = Object.keys(example().chain.criteria).find((key) => chainOf(key).index === index); - return `
    Criterion
    -
    ${escapeHtml(identifier)}
    -
      ${criterionChainItems(identifier, step).join("")}
    `; +// The shown steps of the evaluations of the criteria graph of `identifier` that reached one of `labels`. +function stepsReaching(identifier, labels) { + return shownSteps().filter((step) => step.kind === "evaluate" && step.criterion === identifier + && step.reached.some((label) => labels.includes(label))); } -function pinnedSection(step) { - const name = state.pinned; - let body; - if (name.startsWith("call_") && example().calls[name] !== undefined) { - body = pinnedCall(name, step); +// The shown steps that make the node `name` the current node. A criteria graph criterion is never current itself, so +// its steps are the evaluations that reached it, and an output's are those that reached a criterion leading to it. +function stepsOfNode(name) { + const steps = shownSteps(); + if (name === "result") { + return steps.filter((step) => step.kind === "result"); } else if (name.startsWith("stage_")) { - body = pinnedStage(name, step); + return steps.filter((step) => step.kind === "stage" && stageOf(step.stage).id === name); + } else if (name.startsWith("call_")) { + return steps.filter((step) => step.kind === "call" && step.call === name); } else if (name.startsWith("criterion_")) { - body = pinnedCriterion(name, step); - } else if (name.startsWith("criteria_")) { - body = pinnedCriteriaNode(name, step); - } else if (name === "result") { - body = resultSection(); - } else { - body = `

    No details.

    `; - } - return `
    -

    Pinned

    -
    - ${body} -
    `; -} - -function markPinned() { - for (const group of nodes.values()) { - group.classList.remove("is-pinned"); + return steps.filter((step) => step.kind === "criteria" + && step.criteria.some((identifier) => "criterion_" + chainOf(identifier).index === name)); } - if (state.pinned !== null) { - nodes.get(state.pinned)?.classList.add("is-pinned"); + const found = criteriaGraphNode(name); + if (found === null) { + return []; } + if (found.node.kind === "evaluation") { + return steps.filter((step) => step.kind === "evaluate" && step.criterion === found.identifier + && step.node === found.label); + } else if (found.node.kind === "criterion") { + return stepsReaching(found.identifier, [found.label]); + } + const sources = [...edges.keys()].filter((edge) => edge.split("->")[1] === name) + .map((edge) => criteriaGraphNode(edge.split("->")[0])).filter((source) => source !== null) + .map((source) => source.label); + return stepsReaching(found.identifier, sources); } -function pin(name) { - state.pinned = name; - markPinned(); - renderPanel(example().steps[state.step]); +// Moves to the step that makes the node `name` current: its latest such step up to the current one, or else its first. +function goToNode(name) { + const steps = stepsOfNode(name); + if (steps.length === 0) { + return; + } + const earlier = steps.filter((step) => step.index <= state.step); + setPlaying(false); + state.step = (earlier.length > 0 ? earlier[earlier.length - 1] : steps[0]).index; + render(); document.getElementById("panel").scrollTo(0, 0); } function renderPanel(step) { - let content = state.pinned === null ? "" : pinnedSection(step); - content += stageSection(step); + let content = stageSection(step); if (step.kind === "call") { content += callSection(step); } else if (step.kind === "evaluate") { @@ -1645,7 +1603,6 @@ def main() -> None: applyStep(shown, shown.index === state.step); } updateCounts(step.event); - markPinned(); renderPanel(step); const shown = shownSteps(); const position = shown.findIndex((candidate) => candidate.index === state.step); @@ -1667,11 +1624,6 @@ def main() -> None: function setPlaying(playing) { state.playing = playing; - if (playing && state.pinned !== null) { - state.pinned = null; - markPinned(); - renderPanel(example().steps[state.step]); - } const button = document.getElementById("play"); button.textContent = playing ? "⏸ Pause" : "▶ Play"; button.setAttribute("aria-label", playing ? "Pause" : "Play"); @@ -1696,7 +1648,6 @@ def main() -> None: state.example = index; state.layout = null; state.step = 0; - state.pinned = null; const current = example(); document.getElementById("inputs").innerHTML = ` response ${escapeHtml(current.response)} @@ -1769,9 +1720,6 @@ def main() -> None: document.getElementById("hide-internals").addEventListener("change", (event) => { state.internals = event.target.checked; state.step = nearestShownStep(state.step); - if (state.pinned !== null && state.internals && example().calls[state.pinned]?.internal) { - state.pinned = null; - } saveSettings(); render(); }); @@ -1798,7 +1746,6 @@ def main() -> None: }); document.getElementById("scrubber").addEventListener("input", (event) => { setPlaying(false); - state.pinned = null; state.step = shownSteps()[Number(event.target.value)].index; render(); }); @@ -1807,18 +1754,12 @@ def main() -> None: if (group === null || group.classList.contains("is-hidden")) { return; } - pin(group.querySelector(":scope > title").textContent); + goToNode(group.querySelector(":scope > title").textContent); }); document.getElementById("panel").addEventListener("click", (event) => { - if (event.target.closest("#unpin") !== null) { - state.pinned = null; - markPinned(); - renderPanel(example().steps[state.step]); - return; - } - const link = event.target.closest("[data-pin]"); + const link = event.target.closest("[data-node]"); if (link !== null) { - pin(link.dataset.pin); + goToNode(link.dataset.node); } }); // `toggle` does not bubble, so it is caught on its way down to the section From d2e743036bee47418b72f0a96410601de75ffb1a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:00:47 +0000 Subject: [PATCH 5/5] Regenerate the documentation graphs --- .../physical_quantity_matches.svg | 234 ++++---- .../physical_quantity_matches_mermaid.svg | 2 +- .../physical_quantity_order.svg | 54 +- .../physical_quantity_order_mermaid.svg | 2 +- .../criteria_graphs/symbolic_equality.svg | 106 ++-- .../symbolic_equality_mermaid.svg | 2 +- .../symbolic_equality_of_equations.svg | 92 ++-- ...symbolic_equality_of_equations_mermaid.svg | 2 +- .../symbolic_equality_of_sets.svg | 76 +-- .../symbolic_equality_of_sets_mermaid.svg | 2 +- app/docs/criteria_graphs/symbolic_other.svg | 62 +-- .../symbolic_other_mermaid.svg | 2 +- app/docs/criteria_graphs/symbolic_where.svg | 44 +- .../symbolic_where_mermaid.svg | 2 +- .../criteria_graphs/symbolic_written_as.svg | 168 +++--- .../symbolic_written_as_mermaid.svg | 2 +- .../process_physical_quantity.svg | 354 ++++++------ .../process_physical_quantity_calls.svg | 516 +++++++++--------- app/docs/process_graphs/process_symbolic.svg | 222 ++++---- .../process_graphs/process_symbolic_calls.svg | 514 ++++++++--------- .../process_symbolic_parse_error.svg | 112 ++-- .../process_symbolic_parse_error_calls.svg | 308 +++++------ 22 files changed, 1439 insertions(+), 1439 deletions(-) diff --git a/app/docs/criteria_graphs/physical_quantity_matches.svg b/app/docs/criteria_graphs/physical_quantity_matches.svg index f9c34eb..1cd8bd2 100644 --- a/app/docs/criteria_graphs/physical_quantity_matches.svg +++ b/app/docs/criteria_graphs/physical_quantity_matches.svg @@ -1,278 +1,278 @@ - - + G - + evaluation_0 - -response matches answer -Converts QUANTITY: response tags: {} and -QUANTITY: answer tags: {} match to a common -set of base units and compares their values. + +response matches answer +Converts QUANTITY: response tags: {} and +QUANTITY: answer tags: {} match to a common +set of base units and compares their values. criterion_0 - -response matches answer_TRUE -The quantities QUANTITY: response tags: {} -and QUANTITY: answer tags: {} match. + +response matches answer_TRUE +The quantities QUANTITY: response tags: {} +and QUANTITY: answer tags: {} match. evaluation_0->criterion_0 - - + + criterion_1 - -response matches answer_FALSE -The quantities QUANTITY: response tags: {} -and QUANTITY: answer tags: {} does not match. + +response matches answer_FALSE +The quantities QUANTITY: response tags: {} +and QUANTITY: answer tags: {} does not match. evaluation_0->criterion_1 - - + + criterion_2 - -response matches answer_MISSING_VALUE -The response is missing a value. + +response matches answer_MISSING_VALUE +The response is missing a value. evaluation_0->criterion_2 - - + + criterion_3 - -response matches answer_UNEXPECTED_VALUE -The response is expected only have unit(s), -no value. + +response matches answer_UNEXPECTED_VALUE +The response is expected only have unit(s), +no value. evaluation_0->criterion_3 - - + + criterion_4 - -response matches answer_MISSING_UNIT -The response is missing unit(s). + +response matches answer_MISSING_UNIT +The response is missing unit(s). evaluation_0->criterion_4 - - + + criterion_5 - -response matches answer_UNEXPECTED_UNIT -The response is expected to be a value -without unit(s). + +response matches answer_UNEXPECTED_UNIT +The response is expected to be a value +without unit(s). evaluation_0->criterion_5 - - + + evaluation_1 - -response matches answer_DIMENSION_MATCH -Do the dimensions of QUANTITY: response tags: -{} and QUANTITY: answer tags: {} match? + +response matches answer_DIMENSION_MATCH +Do the dimensions of QUANTITY: response tags: +{} and QUANTITY: answer tags: {} match? criterion_6 - -response matches answer_DIMENSION_MATCH_TRUE -The quantities QUANTITY: response tags: {} -and QUANTITY: answer tags: {} have the same -dimensions. + +response matches answer_DIMENSION_MATCH_TRUE +The quantities QUANTITY: response tags: {} +and QUANTITY: answer tags: {} have the same +dimensions. evaluation_1->criterion_6 - - + + criterion_7 - -response matches answer_DIMENSION_MATCH_FALSE -The quantities QUANTITY: response tags: {} -and QUANTITY: answer tags: {} have different -dimensions. + +response matches answer_DIMENSION_MATCH_FALSE +The quantities QUANTITY: response tags: {} +and QUANTITY: answer tags: {} have different +dimensions. evaluation_1->criterion_7 - - + + evaluation_2 - -response matches answer_UNIT_COMPARISON -Compares how similar the units of QUANTITY: -response tags: {} and QUANTITY: answer tags: -{} are. + +response matches answer_UNIT_COMPARISON +Compares how similar the units of QUANTITY: +response tags: {} and QUANTITY: answer tags: +{} are. criterion_8 - -response matches answer_UNIT_COMPARISON_IDENTICAL -The units of quantities QUANTITY: response -tags: {} and QUANTITY: answer tags: {} are -identical. + +response matches answer_UNIT_COMPARISON_IDENTICAL +The units of quantities QUANTITY: response +tags: {} and QUANTITY: answer tags: {} are +identical. evaluation_2->criterion_8 - - + + criterion_9 - -response matches answer_UNIT_COMPARISON_SIMILAR -The units of quantities QUANTITY: response -tags: {} and QUANTITY: answer tags: {} are -similar. + +response matches answer_UNIT_COMPARISON_SIMILAR +The units of quantities QUANTITY: response +tags: {} and QUANTITY: answer tags: {} are +similar. evaluation_2->criterion_9 - - + + criterion_10 - -response matches answer_UNIT_COMPARISON_PREFIX_IS_LARGE -The units of QUANTITY: response tags: {} are -at least 1000 times greater than the units of -QUANTITY: answer tags: {}. + +response matches answer_UNIT_COMPARISON_PREFIX_IS_LARGE +The units of QUANTITY: response tags: {} are +at least 1000 times greater than the units of +QUANTITY: answer tags: {}. evaluation_2->criterion_10 - - + + criterion_11 - -response matches answer_UNIT_COMPARISON_PREFIX_IS_SMALL -The units of QUANTITY: response tags: {} are -at least 1000 times smaller than the units of -QUANTITY: answer tags: {}. + +response matches answer_UNIT_COMPARISON_PREFIX_IS_SMALL +The units of QUANTITY: response tags: {} are +at least 1000 times smaller than the units of +QUANTITY: answer tags: {}. evaluation_2->criterion_11 - - + + criterion_0->evaluation_2 - - + + criterion_1->evaluation_1 - - + + output_0 - -END -Evaluation completed. + +END +Evaluation completed. criterion_2->output_0 - - + + criterion_3->output_0 - - + + criterion_4->output_0 - - + + criterion_5->output_0 - - + + criterion_6->output_0 - - + + criterion_7->output_0 - - + + criterion_10->output_0 - - + + criterion_11->output_0 - - + + diff --git a/app/docs/criteria_graphs/physical_quantity_matches_mermaid.svg b/app/docs/criteria_graphs/physical_quantity_matches_mermaid.svg index 70b3ee7..a62915f 100644 --- a/app/docs/criteria_graphs/physical_quantity_matches_mermaid.svg +++ b/app/docs/criteria_graphs/physical_quantity_matches_mermaid.svg @@ -1 +1 @@ -

    response matches answer
    ---
    Converts QUANTITY: response tags: {} and QUANTITY: answer tags: {} match to a common set of base units and compares their values.

    response matches answer_DIMENSION_MATCH
    ---
    Do the dimensions of QUANTITY: response tags: {} and QUANTITY: answer tags: {} match?

    response matches answer_UNIT_COMPARISON
    ---
    Compares how similar the units of QUANTITY: response tags: {} and QUANTITY: answer tags: {} are.

    response matches answer_TRUE
    ---
    The quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} match.

    response matches answer_FALSE
    ---
    The quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} does not match.

    response matches answer_MISSING_VALUE
    ---
    The response is missing a value.

    response matches answer_UNEXPECTED_VALUE
    ---
    The response is expected only have unit(s), no value.

    response matches answer_MISSING_UNIT
    ---
    The response is missing unit(s).

    response matches answer_UNEXPECTED_UNIT
    ---
    The response is expected to be a value without unit(s).

    response matches answer_DIMENSION_MATCH_TRUE
    ---
    The quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} have the same dimensions.

    response matches answer_DIMENSION_MATCH_FALSE
    ---
    The quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} have different dimensions.

    response matches answer_UNIT_COMPARISON_IDENTICAL
    ---
    The units of quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} are identical.

    response matches answer_UNIT_COMPARISON_SIMILAR
    ---
    The units of quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} are similar.

    response matches answer_UNIT_COMPARISON_PREFIX_IS_LARGE
    ---
    The units of QUANTITY: response tags: {} are at least 1000 times greater than the units of QUANTITY: answer tags: {}.

    response matches answer_UNIT_COMPARISON_PREFIX_IS_SMALL
    ---
    The units of QUANTITY: response tags: {} are at least 1000 times smaller than the units of QUANTITY: answer tags: {}.

    END
    ---
    Evaluation completed.

     \ No newline at end of file +

    response matches answer
    ---
    Converts QUANTITY: response tags: {} and QUANTITY: answer tags: {} match to a common set of base units and compares their values.

    response matches answer_DIMENSION_MATCH
    ---
    Do the dimensions of QUANTITY: response tags: {} and QUANTITY: answer tags: {} match?

    response matches answer_UNIT_COMPARISON
    ---
    Compares how similar the units of QUANTITY: response tags: {} and QUANTITY: answer tags: {} are.

    response matches answer_TRUE
    ---
    The quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} match.

    response matches answer_FALSE
    ---
    The quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} does not match.

    response matches answer_MISSING_VALUE
    ---
    The response is missing a value.

    response matches answer_UNEXPECTED_VALUE
    ---
    The response is expected only have unit(s), no value.

    response matches answer_MISSING_UNIT
    ---
    The response is missing unit(s).

    response matches answer_UNEXPECTED_UNIT
    ---
    The response is expected to be a value without unit(s).

    response matches answer_DIMENSION_MATCH_TRUE
    ---
    The quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} have the same dimensions.

    response matches answer_DIMENSION_MATCH_FALSE
    ---
    The quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} have different dimensions.

    response matches answer_UNIT_COMPARISON_IDENTICAL
    ---
    The units of quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} are identical.

    response matches answer_UNIT_COMPARISON_SIMILAR
    ---
    The units of quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} are similar.

    response matches answer_UNIT_COMPARISON_PREFIX_IS_LARGE
    ---
    The units of QUANTITY: response tags: {} are at least 1000 times greater than the units of QUANTITY: answer tags: {}.

    response matches answer_UNIT_COMPARISON_PREFIX_IS_SMALL
    ---
    The units of QUANTITY: response tags: {} are at least 1000 times smaller than the units of QUANTITY: answer tags: {}.

    END
    ---
    Evaluation completed.

     \ No newline at end of file diff --git a/app/docs/criteria_graphs/physical_quantity_order.svg b/app/docs/criteria_graphs/physical_quantity_order.svg index 9ed3f2c..04488c6 100644 --- a/app/docs/criteria_graphs/physical_quantity_order.svg +++ b/app/docs/criteria_graphs/physical_quantity_order.svg @@ -1,68 +1,68 @@ - - + G - + evaluation_0 - -response > answer -checks if QUANTITY: response tags: {} > -QUANTITY: answer tags: {}. + +response > answer +checks if QUANTITY: response tags: {} > +QUANTITY: answer tags: {}. criterion_0 - -response > answer_TRUE -QUANTITY: response tags: {} > QUANTITY: -answer tags: {} is true. + +response > answer_TRUE +QUANTITY: response tags: {} > QUANTITY: +answer tags: {} is true. evaluation_0->criterion_0 - - + + criterion_1 - -response > answer_FALSE -QUANTITY: response tags: {} > QUANTITY: -answer tags: {} is false. + +response > answer_FALSE +QUANTITY: response tags: {} > QUANTITY: +answer tags: {} is false. evaluation_0->criterion_1 - - + + output_0 - -END -Evaluation completed. + +END +Evaluation completed. criterion_0->output_0 - - + + criterion_1->output_0 - - + + diff --git a/app/docs/criteria_graphs/physical_quantity_order_mermaid.svg b/app/docs/criteria_graphs/physical_quantity_order_mermaid.svg index 0b37479..4c1585a 100644 --- a/app/docs/criteria_graphs/physical_quantity_order_mermaid.svg +++ b/app/docs/criteria_graphs/physical_quantity_order_mermaid.svg @@ -1 +1 @@ -

    response > answer
    ---
    checks if QUANTITY: response tags: {} > QUANTITY: answer tags: {}.

    response > answer_TRUE
    ---
    QUANTITY: response tags: {} > QUANTITY: answer tags: {} is true.

    response > answer_FALSE
    ---
    QUANTITY: response tags: {} > QUANTITY: answer tags: {} is false.

    END
    ---
    Evaluation completed.

     \ No newline at end of file +

    response > answer
    ---
    checks if QUANTITY: response tags: {} > QUANTITY: answer tags: {}.

    response > answer_TRUE
    ---
    QUANTITY: response tags: {} > QUANTITY: answer tags: {} is true.

    response > answer_FALSE
    ---
    QUANTITY: response tags: {} > QUANTITY: answer tags: {} is false.

    END
    ---
    Evaluation completed.

     \ No newline at end of file diff --git a/app/docs/criteria_graphs/symbolic_equality.svg b/app/docs/criteria_graphs/symbolic_equality.svg index 013c93a..c9b0a51 100644 --- a/app/docs/criteria_graphs/symbolic_equality.svg +++ b/app/docs/criteria_graphs/symbolic_equality.svg @@ -1,133 +1,133 @@ - - - + + G - + evaluation_0 - -response = answer -Checks if response=answer. + +response = answer +Checks if response=answer. criterion_0 - -response = answer_TRUE -response is equal to answer. + +response = answer_TRUE +response is equal to answer. evaluation_0->criterion_0 - - + + criterion_3 - -response = answer_FALSE -response is not equal toanswer. + +response = answer_FALSE +response is not equal toanswer. evaluation_0->criterion_3 - - + + criterion_4 - -response = answer_UNKNOWN -Cannot detrmine if response is equal to -answer. + +response = answer_UNKNOWN +Cannot detrmine if response is equal to +answer. evaluation_0->criterion_4 - - + + evaluation_1 - -response = answer_SAME_SYMBOLS -response has the same (free) symbols as -answer. + +response = answer_SAME_SYMBOLS +response has the same (free) symbols as +answer. criterion_1 - -response = answer_SAME_SYMBOLS_TRUE -response has the same (free) symbols as -answer. + +response = answer_SAME_SYMBOLS_TRUE +response has the same (free) symbols as +answer. evaluation_1->criterion_1 - - + + criterion_2 - -response = answer_SAME_SYMBOLS_FALSE -response does note have the same (free) -symbols as answer. + +response = answer_SAME_SYMBOLS_FALSE +response does note have the same (free) +symbols as answer. evaluation_1->criterion_2 - - + + criterion_0->evaluation_1 - - + + output_0 - -END -Evaluation completed. + +END +Evaluation completed. criterion_1->output_0 - - + + criterion_2->output_0 - - + + criterion_3->output_0 - - + + criterion_4->output_0 - - + + diff --git a/app/docs/criteria_graphs/symbolic_equality_mermaid.svg b/app/docs/criteria_graphs/symbolic_equality_mermaid.svg index 3a08303..0111c37 100644 --- a/app/docs/criteria_graphs/symbolic_equality_mermaid.svg +++ b/app/docs/criteria_graphs/symbolic_equality_mermaid.svg @@ -1 +1 @@ -

    response = answer
    ---
    Checks if response=answer.

    response = answer_SAME_SYMBOLS
    ---
    response has the same (free) symbols as answer.

    response = answer_TRUE
    ---
    response is equal to answer.

    response = answer_SAME_SYMBOLS_TRUE
    ---
    response has the same (free) symbols as answer.

    response = answer_SAME_SYMBOLS_FALSE
    ---
    response does note have the same (free) symbols as answer.

    response = answer_FALSE
    ---
    response is not equal toanswer.

    response = answer_UNKNOWN
    ---
    Cannot detrmine if response is equal to answer.

    END
    ---
    Evaluation completed.

     \ No newline at end of file +

    response = answer
    ---
    Checks if response=answer.

    response = answer_SAME_SYMBOLS
    ---
    response has the same (free) symbols as answer.

    response = answer_TRUE
    ---
    response is equal to answer.

    response = answer_SAME_SYMBOLS_TRUE
    ---
    response has the same (free) symbols as answer.

    response = answer_SAME_SYMBOLS_FALSE
    ---
    response does note have the same (free) symbols as answer.

    response = answer_FALSE
    ---
    response is not equal toanswer.

    response = answer_UNKNOWN
    ---
    Cannot detrmine if response is equal to answer.

    END
    ---
    Evaluation completed.

     \ No newline at end of file diff --git a/app/docs/criteria_graphs/symbolic_equality_of_equations.svg b/app/docs/criteria_graphs/symbolic_equality_of_equations.svg index 01f2785..46453a0 100644 --- a/app/docs/criteria_graphs/symbolic_equality_of_equations.svg +++ b/app/docs/criteria_graphs/symbolic_equality_of_equations.svg @@ -1,123 +1,123 @@ - - + G - + evaluation_0 - -response = answer -Checks if response is equivalent to answer. + +response = answer +Checks if response is equivalent to answer. criterion_0 - -response = answer_UNKNOWN -Cannot determine if response is equivalent to -answer. + +response = answer_UNKNOWN +Cannot determine if response is equivalent to +answer. evaluation_0->criterion_0 - - + + criterion_1 - -response = answer_TRUE -response is equivalent to answer. + +response = answer_TRUE +response is equivalent to answer. evaluation_0->criterion_1 - - + + criterion_2 - -response = answer_FALSE -response is not equivalent to answer. + +response = answer_FALSE +response is not equivalent to answer. evaluation_0->criterion_2 - - + + criterion_3 - -response = answer_EXPRESSION_NOT_EQUALITY -response is an expression, not an equality. + +response = answer_EXPRESSION_NOT_EQUALITY +response is an expression, not an equality. evaluation_0->criterion_3 - - + + criterion_4 - -response = answer_EQUALITY_NOT_EXPRESSION -response is an equality, not an expression. + +response = answer_EQUALITY_NOT_EXPRESSION +response is an equality, not an expression. evaluation_0->criterion_4 - - + + output_0 - -END -Evaluation completed. + +END +Evaluation completed. criterion_0->output_0 - - + + criterion_1->output_0 - - + + criterion_2->output_0 - - + + criterion_3->output_0 - - + + criterion_4->output_0 - - + + diff --git a/app/docs/criteria_graphs/symbolic_equality_of_equations_mermaid.svg b/app/docs/criteria_graphs/symbolic_equality_of_equations_mermaid.svg index 9b93b95..771d392 100644 --- a/app/docs/criteria_graphs/symbolic_equality_of_equations_mermaid.svg +++ b/app/docs/criteria_graphs/symbolic_equality_of_equations_mermaid.svg @@ -1 +1 @@ -

    response = answer
    ---
    Checks if response is equivalent to answer.

    response = answer_UNKNOWN
    ---
    Cannot determine if response is equivalent to answer.

    response = answer_TRUE
    ---
    response is equivalent to answer.

    response = answer_FALSE
    ---
    response is not equivalent to answer.

    response = answer_EXPRESSION_NOT_EQUALITY
    ---
    response is an expression, not an equality.

    response = answer_EQUALITY_NOT_EXPRESSION
    ---
    response is an equality, not an expression.

    END
    ---
    Evaluation completed.

     \ No newline at end of file +

    response = answer
    ---
    Checks if response is equivalent to answer.

    response = answer_UNKNOWN
    ---
    Cannot determine if response is equivalent to answer.

    response = answer_TRUE
    ---
    response is equivalent to answer.

    response = answer_FALSE
    ---
    response is not equivalent to answer.

    response = answer_EXPRESSION_NOT_EQUALITY
    ---
    response is an expression, not an equality.

    response = answer_EQUALITY_NOT_EXPRESSION
    ---
    response is an equality, not an expression.

    END
    ---
    Evaluation completed.

     \ No newline at end of file diff --git a/app/docs/criteria_graphs/symbolic_equality_of_sets.svg b/app/docs/criteria_graphs/symbolic_equality_of_sets.svg index 3c224d9..c61d209 100644 --- a/app/docs/criteria_graphs/symbolic_equality_of_sets.svg +++ b/app/docs/criteria_graphs/symbolic_equality_of_sets.svg @@ -1,99 +1,99 @@ - - + G - + evaluation_0 - -response = answer -Checks if response=answer. + +response = answer +Checks if response=answer. criterion_0 - -response = answer_TRUE -response is equal to answer. + +response = answer_TRUE +response is equal to answer. evaluation_0->criterion_0 - - + + criterion_1 - -response = answer_MULTIPLE_ANSWER_FAIL_ALL -At least one answer or response was -incorrect. + +response = answer_MULTIPLE_ANSWER_FAIL_ALL +At least one answer or response was +incorrect. evaluation_0->criterion_1 - - + + criterion_2 - -response = answer_MULTIPLE_ANSWER_FAIL_RESPONSE -At least one response was incorrect. + +response = answer_MULTIPLE_ANSWER_FAIL_RESPONSE +At least one response was incorrect. evaluation_0->criterion_2 - - + + criterion_3 - -response = answer_MULTIPLE_ANSWER_FAIL_ANSWER -At least one answer is missing in the -response. + +response = answer_MULTIPLE_ANSWER_FAIL_ANSWER +At least one answer is missing in the +response. evaluation_0->criterion_3 - - + + output_0 - -END -Evaluation completed. + +END +Evaluation completed. criterion_1->output_0 - - + + criterion_2->output_0 - - + + criterion_3->output_0 - - + + diff --git a/app/docs/criteria_graphs/symbolic_equality_of_sets_mermaid.svg b/app/docs/criteria_graphs/symbolic_equality_of_sets_mermaid.svg index 8c39155..fb6fa7f 100644 --- a/app/docs/criteria_graphs/symbolic_equality_of_sets_mermaid.svg +++ b/app/docs/criteria_graphs/symbolic_equality_of_sets_mermaid.svg @@ -1 +1 @@ -

    response = answer
    ---
    Checks if response=answer.

    response = answer_TRUE
    ---
    response is equal to answer.

    response = answer_MULTIPLE_ANSWER_FAIL_ALL
    ---
    At least one answer or response was incorrect.

    response = answer_MULTIPLE_ANSWER_FAIL_RESPONSE
    ---
    At least one response was incorrect.

    response = answer_MULTIPLE_ANSWER_FAIL_ANSWER
    ---
    At least one answer is missing in the response.

    END
    ---
    Evaluation completed.

     \ No newline at end of file +

    response = answer
    ---
    Checks if response=answer.

    response = answer_TRUE
    ---
    response is equal to answer.

    response = answer_MULTIPLE_ANSWER_FAIL_ALL
    ---
    At least one answer or response was incorrect.

    response = answer_MULTIPLE_ANSWER_FAIL_RESPONSE
    ---
    At least one response was incorrect.

    response = answer_MULTIPLE_ANSWER_FAIL_ANSWER
    ---
    At least one answer is missing in the response.

    END
    ---
    Evaluation completed.

     \ No newline at end of file diff --git a/app/docs/criteria_graphs/symbolic_other.svg b/app/docs/criteria_graphs/symbolic_other.svg index c3482ca..d253e72 100644 --- a/app/docs/criteria_graphs/symbolic_other.svg +++ b/app/docs/criteria_graphs/symbolic_other.svg @@ -1,84 +1,84 @@ - - + G - + evaluation_0 - -response contains x -Checks if response contains x is true. + +response contains x +Checks if response contains x is true. criterion_0 - -response contains x_TRUE -response contains x is true. + +response contains x_TRUE +response contains x is true. evaluation_0->criterion_0 - - + + criterion_1 - -response contains x_FALSE -response contains x is false. + +response contains x_FALSE +response contains x is false. evaluation_0->criterion_1 - - + + criterion_2 - -response contains x_UNKNOWN -response contains x is false. + +response contains x_UNKNOWN +response contains x is false. evaluation_0->criterion_2 - - + + output_0 - -END -Evaluation completed. + +END +Evaluation completed. criterion_0->output_0 - - + + criterion_1->output_0 - - + + criterion_2->output_0 - - + + diff --git a/app/docs/criteria_graphs/symbolic_other_mermaid.svg b/app/docs/criteria_graphs/symbolic_other_mermaid.svg index 2d1708b..6d54d29 100644 --- a/app/docs/criteria_graphs/symbolic_other_mermaid.svg +++ b/app/docs/criteria_graphs/symbolic_other_mermaid.svg @@ -1 +1 @@ -

    response contains x
    ---
    Checks if response contains x is true.

    response contains x_TRUE
    ---
    response contains x is true.

    response contains x_FALSE
    ---
    response contains x is false.

    response contains x_UNKNOWN
    ---
    response contains x is false.

    END
    ---
    Evaluation completed.

     \ No newline at end of file +

    response contains x
    ---
    Checks if response contains x is true.

    response contains x_TRUE
    ---
    response contains x is true.

    response contains x_FALSE
    ---
    response contains x is false.

    response contains x_UNKNOWN
    ---
    response contains x is false.

    END
    ---
    Evaluation completed.

     \ No newline at end of file diff --git a/app/docs/criteria_graphs/symbolic_where.svg b/app/docs/criteria_graphs/symbolic_where.svg index c0bf656..89bd77a 100644 --- a/app/docs/criteria_graphs/symbolic_where.svg +++ b/app/docs/criteria_graphs/symbolic_where.svg @@ -1,59 +1,59 @@ - - + G - + evaluation_0 - -response = answer where x = 0 -Checks if response = answer where x = 0. + +response = answer where x = 0 +Checks if response = answer where x = 0. criterion_0 - -response = answer where x = 0_TRUE -response = answer where x = 0is true. + +response = answer where x = 0_TRUE +response = answer where x = 0is true. evaluation_0->criterion_0 - - + + criterion_1 - -response = answer where x = 0_FALSE -response = answer is not true when x = 0. + +response = answer where x = 0_FALSE +response = answer is not true when x = 0. evaluation_0->criterion_1 - - + + output_0 - -END -Evaluation completed. + +END +Evaluation completed. criterion_0->output_0 - - + + diff --git a/app/docs/criteria_graphs/symbolic_where_mermaid.svg b/app/docs/criteria_graphs/symbolic_where_mermaid.svg index f187891..bedc275 100644 --- a/app/docs/criteria_graphs/symbolic_where_mermaid.svg +++ b/app/docs/criteria_graphs/symbolic_where_mermaid.svg @@ -1 +1 @@ -

    response = answer where x = 0
    ---
    Checks if response = answer where x = 0.

    response = answer where x = 0_TRUE
    ---
    response = answer where x = 0is true.

    response = answer where x = 0_FALSE
    ---
    response = answer is not true when x = 0.

    END
    ---
    Evaluation completed.

     \ No newline at end of file +

    response = answer where x = 0
    ---
    Checks if response = answer where x = 0.

    response = answer where x = 0_TRUE
    ---
    response = answer where x = 0is true.

    response = answer where x = 0_FALSE
    ---
    response = answer is not true when x = 0.

    END
    ---
    Evaluation completed.

     \ No newline at end of file diff --git a/app/docs/criteria_graphs/symbolic_written_as.svg b/app/docs/criteria_graphs/symbolic_written_as.svg index 5ec41f0..dca47d6 100644 --- a/app/docs/criteria_graphs/symbolic_written_as.svg +++ b/app/docs/criteria_graphs/symbolic_written_as.svg @@ -1,221 +1,221 @@ - - + G - + evaluation_0 - -answer_WRITTEN_AS_KNOWN_FORM -Checks if answer is written in a known form. + +answer_WRITTEN_AS_KNOWN_FORM +Checks if answer is written in a known form. criterion_0 - -answer_WRITTEN_AS_UNKNOWN -answer is not written in a known form. + +answer_WRITTEN_AS_UNKNOWN +answer is not written in a known form. evaluation_0->criterion_0 - - + + criterion_3 - -answer_WRITTEN_AS_NUMBER -answer in written in simplified number form. + +answer_WRITTEN_AS_NUMBER +answer in written in simplified number form. evaluation_0->criterion_3 - - + + criterion_4 - -answer_WRITTEN_AS_CARTESIAN -answer in written in cartesian form. + +answer_WRITTEN_AS_CARTESIAN +answer in written in cartesian form. evaluation_0->criterion_4 - - + + criterion_5 - -answer_WRITTEN_AS_EXPONENTIAL -answer in written in exponential form. + +answer_WRITTEN_AS_EXPONENTIAL +answer in written in exponential form. evaluation_0->criterion_5 - - + + evaluation_1 - -response written as answer -Checks if response is written in the same -form as answer. + +response written as answer +Checks if response is written in the same +form as answer. evaluation_1->criterion_0 - - + + criterion_1 - -response written as answer_TRUE -answer written in the same form as answer. + +response written as answer_TRUE +answer written in the same form as answer. evaluation_1->criterion_1 - - + + criterion_2 - -response written as answer_FALSE -answer written in the same form as answer. + +response written as answer_FALSE +answer written in the same form as answer. evaluation_1->criterion_2 - - + + evaluation_2 - -response_COMPARE_FORM_TO_NUMBER -Checks if response is written in simplified -number form. + +response_COMPARE_FORM_TO_NUMBER +Checks if response is written in simplified +number form. evaluation_2->criterion_1 - - + + evaluation_2->criterion_2 - - + + evaluation_2->criterion_3 - - + + evaluation_3 - -response_COMPARE_FORM_TO_CARTESIAN -Checks if response is written in cartesian -form. + +response_COMPARE_FORM_TO_CARTESIAN +Checks if response is written in cartesian +form. evaluation_3->criterion_1 - - + + evaluation_3->criterion_2 - - + + evaluation_3->criterion_4 - - + + evaluation_4 - -response_COMPARE_FORM_TO_EXPONENTIAL -Checks if response is written in exponential -form. + +response_COMPARE_FORM_TO_EXPONENTIAL +Checks if response is written in exponential +form. evaluation_4->criterion_1 - - + + evaluation_4->criterion_2 - - + + evaluation_4->criterion_5 - - + + criterion_0->evaluation_1 - - + + criterion_3->evaluation_2 - - + + criterion_4->evaluation_3 - - + + criterion_5->evaluation_4 - - + + output_0 - -END -Evaluation completed. + +END +Evaluation completed. diff --git a/app/docs/criteria_graphs/symbolic_written_as_mermaid.svg b/app/docs/criteria_graphs/symbolic_written_as_mermaid.svg index 6455ff5..325ad03 100644 --- a/app/docs/criteria_graphs/symbolic_written_as_mermaid.svg +++ b/app/docs/criteria_graphs/symbolic_written_as_mermaid.svg @@ -1 +1 @@ -

    answer_WRITTEN_AS_KNOWN_FORM
    ---
    Checks if answer is written in a known form.

    response written as answer
    ---
    Checks if response is written in the same form as answer.

    response_COMPARE_FORM_TO_NUMBER
    ---
    Checks if response is written in simplified number form.

    response_COMPARE_FORM_TO_CARTESIAN
    ---
    Checks if response is written in cartesian form.

    response_COMPARE_FORM_TO_EXPONENTIAL
    ---
    Checks if response is written in exponential form.

    answer_WRITTEN_AS_UNKNOWN
    ---
    answer is not written in a known form.

    response written as answer_TRUE
    ---
    answer written in the same form as answer.

    response written as answer_FALSE
    ---
    answer written in the same form as answer.

    answer_WRITTEN_AS_NUMBER
    ---
    answer in written in simplified number form.

    answer_WRITTEN_AS_CARTESIAN
    ---
    answer in written in cartesian form.

    answer_WRITTEN_AS_EXPONENTIAL
    ---
    answer in written in exponential form.

    END
    ---
    Evaluation completed.

     \ No newline at end of file +

    answer_WRITTEN_AS_KNOWN_FORM
    ---
    Checks if answer is written in a known form.

    response written as answer
    ---
    Checks if response is written in the same form as answer.

    response_COMPARE_FORM_TO_NUMBER
    ---
    Checks if response is written in simplified number form.

    response_COMPARE_FORM_TO_CARTESIAN
    ---
    Checks if response is written in cartesian form.

    response_COMPARE_FORM_TO_EXPONENTIAL
    ---
    Checks if response is written in exponential form.

    answer_WRITTEN_AS_UNKNOWN
    ---
    answer is not written in a known form.

    response written as answer_TRUE
    ---
    answer written in the same form as answer.

    response written as answer_FALSE
    ---
    answer written in the same form as answer.

    answer_WRITTEN_AS_NUMBER
    ---
    answer in written in simplified number form.

    answer_WRITTEN_AS_CARTESIAN
    ---
    answer in written in cartesian form.

    answer_WRITTEN_AS_EXPONENTIAL
    ---
    answer in written in exponential form.

    END
    ---
    Evaluation completed.

     \ No newline at end of file diff --git a/app/docs/process_graphs/process_physical_quantity.svg b/app/docs/process_graphs/process_physical_quantity.svg index 8aa84c5..9342799 100644 --- a/app/docs/process_graphs/process_physical_quantity.svg +++ b/app/docs/process_graphs/process_physical_quantity.svg @@ -1,421 +1,421 @@ - - + G - + cluster_criteria_0 - -Criteria graph: response matches answer + +Criteria graph: response matches answer stage_0 - -setup -Copies the tolerance aliases to rtol and atol -and creates the EvaluationResult. + +setup +Copies the tolerance aliases to rtol and atol +and creates the EvaluationResult. stage_1 - -determine_context -Chooses the physical_quantity context and -adds its default parameters and criteria. + +determine_context +Chooses the physical_quantity context and +adds its default parameters and criteria. stage_0->stage_1 - - + + stage_2 - -parse_latex -Not run: is_latex is not set. + +parse_latex +Not run: is_latex is not set. stage_1->stage_2 - - + + stage_3 - -parse_reserved_expressions -Preprocesses and parses the response and -answer. + +parse_reserved_expressions +Preprocesses and parses the response and +answer. stage_2->stage_3 - - + + stage_4 - -preview -Sets the LaTeX and simplified forms of the -response. + +preview +Sets the LaTeX and simplified forms of the +response. stage_3->stage_4 - - + + stage_5 - -parse_criteria -Parses the criteria: `response matches -answer`. + +parse_criteria +Parses the criteria: `response matches +answer`. stage_4->stage_5 - - + + stage_6 - -generate_feedback_procedures -Builds the criteria graph for each criterion. + +generate_feedback_procedures +Builds the criteria graph for each criterion. stage_5->stage_6 - - + + stage_7 - -generate_feedback -Runs each criteria graph on the response; the -path it took is highlighted. + +generate_feedback +Runs each criteria graph on the response; the +path it took is highlighted. stage_6->stage_7 - - + + stage_8 - -serialise -Serialises the result. + +serialise +Serialises the result. stage_7->stage_8 - - + + criteria_0_evaluation_0 - -response matches answer -Converts QUANTITY: response tags: {} and -QUANTITY: answer tags: {} match to a common -set of base units and compares their values. + +response matches answer +Converts QUANTITY: response tags: {} and +QUANTITY: answer tags: {} match to a common +set of base units and compares their values. stage_7->criteria_0_evaluation_0 - - + + result - -Result -is_correct: True; tags: response matches -answer_TRUE, response matches -answer_UNIT_COMPARISON_SIMILAR + +Result +is_correct: True; tags: response matches +answer_TRUE, response matches +answer_UNIT_COMPARISON_SIMILAR stage_8->result - - + + criteria_0_criterion_0 - -response matches answer_TRUE -The quantities QUANTITY: response tags: {} -and QUANTITY: answer tags: {} match. + +response matches answer_TRUE +The quantities QUANTITY: response tags: {} +and QUANTITY: answer tags: {} match. criteria_0_evaluation_0->criteria_0_criterion_0 - - + + criteria_0_criterion_1 - -response matches answer_FALSE -The quantities QUANTITY: response tags: {} -and QUANTITY: answer tags: {} does not match. + +response matches answer_FALSE +The quantities QUANTITY: response tags: {} +and QUANTITY: answer tags: {} does not match. criteria_0_evaluation_0->criteria_0_criterion_1 - - + + criteria_0_criterion_2 - -response matches answer_MISSING_VALUE -The response is missing a value. + +response matches answer_MISSING_VALUE +The response is missing a value. criteria_0_evaluation_0->criteria_0_criterion_2 - - + + criteria_0_criterion_3 - -response matches answer_UNEXPECTED_VALUE -The response is expected only have unit(s), -no value. + +response matches answer_UNEXPECTED_VALUE +The response is expected only have unit(s), +no value. criteria_0_evaluation_0->criteria_0_criterion_3 - - + + criteria_0_criterion_4 - -response matches answer_MISSING_UNIT -The response is missing unit(s). + +response matches answer_MISSING_UNIT +The response is missing unit(s). criteria_0_evaluation_0->criteria_0_criterion_4 - - + + criteria_0_criterion_5 - -response matches answer_UNEXPECTED_UNIT -The response is expected to be a value -without unit(s). + +response matches answer_UNEXPECTED_UNIT +The response is expected to be a value +without unit(s). criteria_0_evaluation_0->criteria_0_criterion_5 - - + + criteria_0_evaluation_1 - -response matches answer_DIMENSION_MATCH -Do the dimensions of QUANTITY: response tags: -{} and QUANTITY: answer tags: {} match? + +response matches answer_DIMENSION_MATCH +Do the dimensions of QUANTITY: response tags: +{} and QUANTITY: answer tags: {} match? criteria_0_criterion_6 - -response matches answer_DIMENSION_MATCH_TRUE -The quantities QUANTITY: response tags: {} -and QUANTITY: answer tags: {} have the same -dimensions. + +response matches answer_DIMENSION_MATCH_TRUE +The quantities QUANTITY: response tags: {} +and QUANTITY: answer tags: {} have the same +dimensions. criteria_0_evaluation_1->criteria_0_criterion_6 - - + + criteria_0_criterion_7 - -response matches answer_DIMENSION_MATCH_FALSE -The quantities QUANTITY: response tags: {} -and QUANTITY: answer tags: {} have different -dimensions. + +response matches answer_DIMENSION_MATCH_FALSE +The quantities QUANTITY: response tags: {} +and QUANTITY: answer tags: {} have different +dimensions. criteria_0_evaluation_1->criteria_0_criterion_7 - - + + criteria_0_evaluation_2 - -response matches answer_UNIT_COMPARISON -Compares how similar the units of QUANTITY: -response tags: {} and QUANTITY: answer tags: -{} are. + +response matches answer_UNIT_COMPARISON +Compares how similar the units of QUANTITY: +response tags: {} and QUANTITY: answer tags: +{} are. criteria_0_criterion_8 - -response matches answer_UNIT_COMPARISON_IDENTICAL -The units of quantities QUANTITY: response -tags: {} and QUANTITY: answer tags: {} are -identical. + +response matches answer_UNIT_COMPARISON_IDENTICAL +The units of quantities QUANTITY: response +tags: {} and QUANTITY: answer tags: {} are +identical. criteria_0_evaluation_2->criteria_0_criterion_8 - - + + criteria_0_criterion_9 - -response matches answer_UNIT_COMPARISON_SIMILAR -The units of quantities QUANTITY: response -tags: {} and QUANTITY: answer tags: {} are -similar. + +response matches answer_UNIT_COMPARISON_SIMILAR +The units of quantities QUANTITY: response +tags: {} and QUANTITY: answer tags: {} are +similar. criteria_0_evaluation_2->criteria_0_criterion_9 - - + + criteria_0_criterion_10 - -response matches answer_UNIT_COMPARISON_PREFIX_IS_LARGE -The units of QUANTITY: response tags: {} are -at least 1000 times greater than the units of -QUANTITY: answer tags: {}. + +response matches answer_UNIT_COMPARISON_PREFIX_IS_LARGE +The units of QUANTITY: response tags: {} are +at least 1000 times greater than the units of +QUANTITY: answer tags: {}. criteria_0_evaluation_2->criteria_0_criterion_10 - - + + criteria_0_criterion_11 - -response matches answer_UNIT_COMPARISON_PREFIX_IS_SMALL -The units of QUANTITY: response tags: {} are -at least 1000 times smaller than the units of -QUANTITY: answer tags: {}. + +response matches answer_UNIT_COMPARISON_PREFIX_IS_SMALL +The units of QUANTITY: response tags: {} are +at least 1000 times smaller than the units of +QUANTITY: answer tags: {}. criteria_0_evaluation_2->criteria_0_criterion_11 - - + + criteria_0_criterion_0->criteria_0_evaluation_2 - - + + criteria_0_criterion_1->criteria_0_evaluation_1 - - + + criteria_0_output_0 - -END -Evaluation completed. + +END +Evaluation completed. criteria_0_criterion_2->criteria_0_output_0 - - + + criteria_0_criterion_3->criteria_0_output_0 - - + + criteria_0_criterion_4->criteria_0_output_0 - - + + criteria_0_criterion_5->criteria_0_output_0 - - + + criteria_0_criterion_6->criteria_0_output_0 - - + + criteria_0_criterion_7->criteria_0_output_0 - - + + criteria_0_criterion_10->criteria_0_output_0 - - + + criteria_0_criterion_11->criteria_0_output_0 - - + + diff --git a/app/docs/process_graphs/process_physical_quantity_calls.svg b/app/docs/process_graphs/process_physical_quantity_calls.svg index a091b1d..99a79ef 100644 --- a/app/docs/process_graphs/process_physical_quantity_calls.svg +++ b/app/docs/process_graphs/process_physical_quantity_calls.svg @@ -1,671 +1,671 @@ - - + G - + cluster_stage_cluster_0 - + cluster_stage_cluster_1 - + cluster_stage_cluster_3 - + cluster_stage_cluster_4 - + cluster_stage_cluster_5 - + cluster_stage_cluster_6 - + cluster_stage_cluster_7 - + cluster_stage_cluster_8 - + stage_0 - -setup -Copies the tolerance aliases to rtol and atol -and creates the EvaluationResult. + +setup +Copies the tolerance aliases to rtol and atol +and creates the EvaluationResult. stage_1 - -determine_context -Chooses the physical_quantity context and -adds its default parameters and criteria. + +determine_context +Chooses the physical_quantity context and +adds its default parameters and criteria. stage_0->stage_1 - - + + call_0 - -_apply_tolerance_aliases -app.evaluation + +_apply_tolerance_aliases +app.evaluation stage_0->call_0 - - + + call_1 - -EvaluationResult.__init__ -app.result + +EvaluationResult.__init__ +app.result stage_0->call_1 - - + + stage_2 - -parse_latex -Not run: is_latex is not set. + +parse_latex +Not run: is_latex is not set. stage_1->stage_2 - - + + call_2 - -select_context -app.evaluation + +select_context +app.evaluation stage_1->call_2 - - + + call_3 - -_build_parameters -app.evaluation + +_build_parameters +app.evaluation stage_1->call_3 - - + + call_6 - -_add_notation_warnings -app.evaluation + +_add_notation_warnings +app.evaluation stage_1->call_6 - - + + stage_3 - -parse_reserved_expressions -Preprocesses and parses the response and -answer. + +parse_reserved_expressions +Preprocesses and parses the response and +answer. stage_2->stage_3 - - + + stage_4 - -preview -Sets the LaTeX and simplified forms of the -response. + +preview +Sets the LaTeX and simplified forms of the +response. stage_3->stage_4 - - + + call_8 - -parse_reserved_expressions -app.evaluation + +parse_reserved_expressions +app.evaluation stage_3->call_8 - - + + stage_5 - -parse_criteria -Parses the criteria: `response matches -answer`. + +parse_criteria +Parses the criteria: `response matches +answer`. stage_4->stage_5 - - + + call_14 - -_set_preview_fields -app.evaluation + +_set_preview_fields +app.evaluation stage_4->call_14 - - + + stage_6 - -generate_feedback_procedures -Builds the criteria graph for each criterion. + +generate_feedback_procedures +Builds the criteria graph for each criterion. stage_5->stage_6 - - + + call_18 - -build_quantity_criteria_parser -lambdafeedback.criteria.quantity_grammar + +build_quantity_criteria_parser +lambdafeedback.criteria.quantity_grammar stage_5->call_18 - - + + call_20 - -create_criteria_dict -app.evaluation + +create_criteria_dict +app.evaluation stage_5->call_20 - - + + call_27 - -_build_evaluation_parameters -app.evaluation + +_build_evaluation_parameters +app.evaluation stage_5->call_27 - - + + stage_7 - -generate_feedback -Runs each criteria graph on the response; the -path it took is highlighted. + +generate_feedback +Runs each criteria graph on the response; the +path it took is highlighted. stage_6->stage_7 - - + + call_28 - -feedback_procedure_generator -app.context.physical_quantity + +feedback_procedure_generator +app.context.physical_quantity stage_6->call_28 - - + + call_30 - -_disable_evaluation_nodes -app.evaluation + +_disable_evaluation_nodes +app.evaluation stage_6->call_30 - - + + stage_8 - -serialise -Serialises the result. + +serialise +Serialises the result. stage_7->stage_8 - - + + call_31 - -generate_feedback -app.evaluation + +generate_feedback +app.evaluation stage_7->call_31 - - + + result - -Result -is_correct: True; tags: response matches -answer_TRUE, response matches -answer_UNIT_COMPARISON_SIMILAR + +Result +is_correct: True; tags: response matches +answer_TRUE, response matches +answer_UNIT_COMPARISON_SIMILAR stage_8->result - - + + call_34 - -EvaluationResult.serialise -app.result + +EvaluationResult.serialise +app.result stage_8->call_34 - - + + call_4 - -_reserved_keywords -app.evaluation + +_reserved_keywords +app.evaluation call_3->call_4 - - + + call_5 - -parsing_parameters_generator -app.context.physical_quantity + +parsing_parameters_generator +app.context.physical_quantity call_3->call_5 - - + + call_7 - -notation_warnings -lambdafeedback.expression_parsing.notation + +notation_warnings +lambdafeedback.expression_parsing.notation call_6->call_7 - - + + call_9 - -ExpressionParams.__getstate__ ×2 -lambdafeedback.expression_parsing.params + +ExpressionParams.__getstate__ ×2 +lambdafeedback.expression_parsing.params call_8->call_9 - - + + call_10 - -ExpressionParams.__setstate__ ×2 -lambdafeedback.expression_parsing.params + +ExpressionParams.__setstate__ ×2 +lambdafeedback.expression_parsing.params call_8->call_10 - - + + call_11 - -expression_preprocess ×2 -app.context.physical_quantity + +expression_preprocess ×2 +app.context.physical_quantity call_8->call_11 - - + + call_12 - -parse_quantity ×2 -app.context.physical_quantity.parsing + +parse_quantity ×2 +app.context.physical_quantity.parsing call_8->call_12 - - + + call_13 - -is_inequality_or_boolean -app.context.symbolic.comparison + +is_inequality_or_boolean +app.context.symbolic.comparison call_8->call_13 - - + + call_15 - -ExpressionParams.__getstate__ ×2 -lambdafeedback.expression_parsing.params + +ExpressionParams.__getstate__ ×2 +lambdafeedback.expression_parsing.params call_14->call_15 - - + + call_16 - -ExpressionParams.__setstate__ ×2 -lambdafeedback.expression_parsing.params + +ExpressionParams.__setstate__ ×2 +lambdafeedback.expression_parsing.params call_14->call_16 - - + + call_17 - -preview_function -app.preview.physical_quantity + +preview_function +app.preview.physical_quantity call_14->call_17 - - + + call_19 - -SLRParser.__init__ -lambdafeedback.slr_parsing.parser + +SLRParser.__init__ +lambdafeedback.slr_parsing.parser call_18->call_19 - - + + call_21 - -get_criteria_string -app.evaluation + +get_criteria_string +app.evaluation call_20->call_21 - - + + call_22 - -expression_preprocess -app.context.physical_quantity + +expression_preprocess +app.context.physical_quantity call_20->call_22 - - + + call_23 - -split_criteria -lambdafeedback.criteria.split + +split_criteria +lambdafeedback.criteria.split call_20->call_23 - - + + call_24 - -SLRParser.scan -lambdafeedback.slr_parsing.parser + +SLRParser.scan +lambdafeedback.slr_parsing.parser call_20->call_24 - - + + call_25 - -SLRParser.parse -lambdafeedback.slr_parsing.parser + +SLRParser.parse +lambdafeedback.slr_parsing.parser call_20->call_25 - - + + call_26 - -ExprNode.content_string ×2 -lambdafeedback.slr_parsing.tokens + +ExprNode.content_string ×2 +lambdafeedback.slr_parsing.tokens call_20->call_26 - - + + call_29 - -criterion_match_node -app.context.physical_quantity.graphs.match + +criterion_match_node +app.context.physical_quantity.graphs.match call_28->call_29 - - + + call_32 - -CriteriaGraph.generate_feedback -lambdafeedback.criteria.graph + +CriteriaGraph.generate_feedback +lambdafeedback.criteria.graph call_31->call_32 - - + + call_33 - -CriteriaGraph.export_feedback -lambdafeedback.criteria.graph + +CriteriaGraph.export_feedback +lambdafeedback.criteria.graph call_31->call_33 - - + + call_35 - -EvaluationResult._serialise_feedback -app.result + +EvaluationResult._serialise_feedback +app.result call_34->call_35 - - + + call_36 - -EvaluationResult.tags -app.result + +EvaluationResult.tags +app.result call_34->call_36 - - + + call_37 - -CriteriaGraph.test_data -lambdafeedback.criteria.graph + +CriteriaGraph.test_data +lambdafeedback.criteria.graph call_34->call_37 - - + + diff --git a/app/docs/process_graphs/process_symbolic.svg b/app/docs/process_graphs/process_symbolic.svg index 36815d4..0c55463 100644 --- a/app/docs/process_graphs/process_symbolic.svg +++ b/app/docs/process_graphs/process_symbolic.svg @@ -1,275 +1,275 @@ - - + G - + cluster_criteria_0 - -Criteria graph: response = answer + +Criteria graph: response = answer stage_0 - -setup -Copies the tolerance aliases to rtol and atol -and creates the EvaluationResult. + +setup +Copies the tolerance aliases to rtol and atol +and creates the EvaluationResult. stage_1 - -determine_context -Chooses the symbolic context and adds its -default parameters and criteria. + +determine_context +Chooses the symbolic context and adds its +default parameters and criteria. stage_0->stage_1 - - + + stage_2 - -parse_latex -Not run: is_latex is not set. + +parse_latex +Not run: is_latex is not set. stage_1->stage_2 - - + + stage_3 - -parse_reserved_expressions -Preprocesses and parses the response and -answer. + +parse_reserved_expressions +Preprocesses and parses the response and +answer. stage_2->stage_3 - - + + stage_4 - -preview -Sets the LaTeX and simplified forms of the -response. + +preview +Sets the LaTeX and simplified forms of the +response. stage_3->stage_4 - - + + stage_5 - -parse_criteria -Parses the criteria: `response = answer`. + +parse_criteria +Parses the criteria: `response = answer`. stage_4->stage_5 - - + + stage_6 - -generate_feedback_procedures -Builds the criteria graph for each criterion. + +generate_feedback_procedures +Builds the criteria graph for each criterion. stage_5->stage_6 - - + + stage_7 - -generate_feedback -Runs each criteria graph on the response; the -path it took is highlighted. + +generate_feedback +Runs each criteria graph on the response; the +path it took is highlighted. stage_6->stage_7 - - + + stage_8 - -serialise -Serialises the result. + +serialise +Serialises the result. stage_7->stage_8 - - + + criteria_0_evaluation_0 - -response = answer -Checks if response=answer. + +response = answer +Checks if response=answer. stage_7->criteria_0_evaluation_0 - - + + result - -Result -is_correct: True; tags: response = -answer_TRUE, response = -answer_SAME_SYMBOLS_TRUE + +Result +is_correct: True; tags: response = +answer_TRUE, response = +answer_SAME_SYMBOLS_TRUE stage_8->result - - + + criteria_0_criterion_0 - -response = answer_TRUE -response is equal to answer. + +response = answer_TRUE +response is equal to answer. criteria_0_evaluation_0->criteria_0_criterion_0 - - + + criteria_0_criterion_3 - -response = answer_FALSE -response is not equal toanswer. + +response = answer_FALSE +response is not equal toanswer. criteria_0_evaluation_0->criteria_0_criterion_3 - - + + criteria_0_criterion_4 - -response = answer_UNKNOWN -Cannot detrmine if response is equal to -answer. + +response = answer_UNKNOWN +Cannot detrmine if response is equal to +answer. criteria_0_evaluation_0->criteria_0_criterion_4 - - + + criteria_0_evaluation_1 - -response = answer_SAME_SYMBOLS -response has the same (free) symbols as -answer. + +response = answer_SAME_SYMBOLS +response has the same (free) symbols as +answer. criteria_0_criterion_1 - -response = answer_SAME_SYMBOLS_TRUE -response has the same (free) symbols as -answer. + +response = answer_SAME_SYMBOLS_TRUE +response has the same (free) symbols as +answer. criteria_0_evaluation_1->criteria_0_criterion_1 - - + + criteria_0_criterion_2 - -response = answer_SAME_SYMBOLS_FALSE -response does note have the same (free) -symbols as answer. + +response = answer_SAME_SYMBOLS_FALSE +response does note have the same (free) +symbols as answer. criteria_0_evaluation_1->criteria_0_criterion_2 - - + + criteria_0_criterion_0->criteria_0_evaluation_1 - - + + criteria_0_output_0 - -END -Evaluation completed. + +END +Evaluation completed. criteria_0_criterion_1->criteria_0_output_0 - - + + criteria_0_criterion_2->criteria_0_output_0 - - + + criteria_0_criterion_3->criteria_0_output_0 - - + + criteria_0_criterion_4->criteria_0_output_0 - - + + diff --git a/app/docs/process_graphs/process_symbolic_calls.svg b/app/docs/process_graphs/process_symbolic_calls.svg index c4067a2..648c8ce 100644 --- a/app/docs/process_graphs/process_symbolic_calls.svg +++ b/app/docs/process_graphs/process_symbolic_calls.svg @@ -1,670 +1,670 @@ - - + G - + cluster_stage_cluster_0 - + cluster_stage_cluster_1 - + cluster_stage_cluster_3 - + cluster_stage_cluster_4 - + cluster_stage_cluster_5 - + cluster_stage_cluster_6 - + cluster_stage_cluster_7 - + cluster_stage_cluster_8 - + stage_0 - -setup -Copies the tolerance aliases to rtol and atol -and creates the EvaluationResult. + +setup +Copies the tolerance aliases to rtol and atol +and creates the EvaluationResult. stage_1 - -determine_context -Chooses the symbolic context and adds its -default parameters and criteria. + +determine_context +Chooses the symbolic context and adds its +default parameters and criteria. stage_0->stage_1 - - + + call_0 - -_apply_tolerance_aliases -app.evaluation + +_apply_tolerance_aliases +app.evaluation stage_0->call_0 - - + + call_1 - -EvaluationResult.__init__ -app.result + +EvaluationResult.__init__ +app.result stage_0->call_1 - - + + stage_2 - -parse_latex -Not run: is_latex is not set. + +parse_latex +Not run: is_latex is not set. stage_1->stage_2 - - + + call_2 - -select_context -app.evaluation + +select_context +app.evaluation stage_1->call_2 - - + + call_3 - -_build_parameters -app.evaluation + +_build_parameters +app.evaluation stage_1->call_3 - - + + call_6 - -_add_notation_warnings -app.evaluation + +_add_notation_warnings +app.evaluation stage_1->call_6 - - + + stage_3 - -parse_reserved_expressions -Preprocesses and parses the response and -answer. + +parse_reserved_expressions +Preprocesses and parses the response and +answer. stage_2->stage_3 - - + + stage_4 - -preview -Sets the LaTeX and simplified forms of the -response. + +preview +Sets the LaTeX and simplified forms of the +response. stage_3->stage_4 - - + + call_8 - -parse_reserved_expressions -app.evaluation + +parse_reserved_expressions +app.evaluation stage_3->call_8 - - + + stage_5 - -parse_criteria -Parses the criteria: `response = answer`. + +parse_criteria +Parses the criteria: `response = answer`. stage_4->stage_5 - - + + call_14 - -_set_preview_fields -app.evaluation + +_set_preview_fields +app.evaluation stage_4->call_14 - - + + stage_6 - -generate_feedback_procedures -Builds the criteria graph for each criterion. + +generate_feedback_procedures +Builds the criteria graph for each criterion. stage_5->stage_6 - - + + call_18 - -build_criteria_parser -lambdafeedback.criteria.grammar + +build_criteria_parser +lambdafeedback.criteria.grammar stage_5->call_18 - - + + call_20 - -create_criteria_dict -app.evaluation + +create_criteria_dict +app.evaluation stage_5->call_20 - - + + call_27 - -_build_evaluation_parameters -app.evaluation + +_build_evaluation_parameters +app.evaluation stage_5->call_27 - - + + stage_7 - -generate_feedback -Runs each criteria graph on the response; the -path it took is highlighted. + +generate_feedback +Runs each criteria graph on the response; the +path it took is highlighted. stage_6->stage_7 - - + + call_28 - -feedback_procedure_generator -app.context.symbolic + +feedback_procedure_generator +app.context.symbolic stage_6->call_28 - - + + call_30 - -_disable_evaluation_nodes -app.evaluation + +_disable_evaluation_nodes +app.evaluation stage_6->call_30 - - + + stage_8 - -serialise -Serialises the result. + +serialise +Serialises the result. stage_7->stage_8 - - + + call_31 - -generate_feedback -app.evaluation + +generate_feedback +app.evaluation stage_7->call_31 - - + + result - -Result -is_correct: True; tags: response = -answer_TRUE, response = -answer_SAME_SYMBOLS_TRUE + +Result +is_correct: True; tags: response = +answer_TRUE, response = +answer_SAME_SYMBOLS_TRUE stage_8->result - - + + call_34 - -EvaluationResult.serialise -app.result + +EvaluationResult.serialise +app.result stage_8->call_34 - - + + call_4 - -_reserved_keywords -app.evaluation + +_reserved_keywords +app.evaluation call_3->call_4 - - + + call_5 - -parsing_parameters_generator -app.context.symbolic + +parsing_parameters_generator +app.context.symbolic call_3->call_5 - - + + call_7 - -notation_warnings -lambdafeedback.expression_parsing.notation + +notation_warnings +lambdafeedback.expression_parsing.notation call_6->call_7 - - + + call_9 - -ExpressionParams.__getstate__ -lambdafeedback.expression_parsing.params + +ExpressionParams.__getstate__ +lambdafeedback.expression_parsing.params call_8->call_9 - - + + call_10 - -ExpressionParams.__setstate__ -lambdafeedback.expression_parsing.params + +ExpressionParams.__setstate__ +lambdafeedback.expression_parsing.params call_8->call_10 - - + + call_11 - -expression_preprocess ×2 -app.context.symbolic + +expression_preprocess ×2 +app.context.symbolic call_8->call_11 - - + + call_12 - -expression_parse ×2 -app.context.symbolic + +expression_parse ×2 +app.context.symbolic call_8->call_12 - - + + call_13 - -is_inequality_or_boolean -app.context.symbolic.comparison + +is_inequality_or_boolean +app.context.symbolic.comparison call_8->call_13 - - + + call_15 - -ExpressionParams.__getstate__ -lambdafeedback.expression_parsing.params + +ExpressionParams.__getstate__ +lambdafeedback.expression_parsing.params call_14->call_15 - - + + call_16 - -ExpressionParams.__setstate__ -lambdafeedback.expression_parsing.params + +ExpressionParams.__setstate__ +lambdafeedback.expression_parsing.params call_14->call_16 - - + + call_17 - -preview_function -app.preview.symbolic + +preview_function +app.preview.symbolic call_14->call_17 - - + + call_19 - -SLRParser.__init__ -lambdafeedback.slr_parsing.parser + +SLRParser.__init__ +lambdafeedback.slr_parsing.parser call_18->call_19 - - + + call_21 - -get_criteria_string -app.evaluation + +get_criteria_string +app.evaluation call_20->call_21 - - + + call_22 - -expression_preprocess -app.context.symbolic + +expression_preprocess +app.context.symbolic call_20->call_22 - - + + call_23 - -split_criteria -lambdafeedback.criteria.split + +split_criteria +lambdafeedback.criteria.split call_20->call_23 - - + + call_24 - -SLRParser.scan -lambdafeedback.slr_parsing.parser + +SLRParser.scan +lambdafeedback.slr_parsing.parser call_20->call_24 - - + + call_25 - -SLRParser.parse -lambdafeedback.slr_parsing.parser + +SLRParser.parse +lambdafeedback.slr_parsing.parser call_20->call_25 - - + + call_26 - -ExprNode.content_string ×2 -lambdafeedback.slr_parsing.tokens + +ExprNode.content_string ×2 +lambdafeedback.slr_parsing.tokens call_20->call_26 - - + + call_29 - -criterion_equality_node -app.context.symbolic.graphs.equivalence + +criterion_equality_node +app.context.symbolic.graphs.equivalence call_28->call_29 - - + + call_32 - -CriteriaGraph.generate_feedback -lambdafeedback.criteria.graph + +CriteriaGraph.generate_feedback +lambdafeedback.criteria.graph call_31->call_32 - - + + call_33 - -CriteriaGraph.export_feedback -lambdafeedback.criteria.graph + +CriteriaGraph.export_feedback +lambdafeedback.criteria.graph call_31->call_33 - - + + call_35 - -EvaluationResult._serialise_feedback -app.result + +EvaluationResult._serialise_feedback +app.result call_34->call_35 - - + + call_36 - -EvaluationResult.tags -app.result + +EvaluationResult.tags +app.result call_34->call_36 - - + + call_37 - -CriteriaGraph.test_data -lambdafeedback.criteria.graph + +CriteriaGraph.test_data +lambdafeedback.criteria.graph call_34->call_37 - - + + diff --git a/app/docs/process_graphs/process_symbolic_parse_error.svg b/app/docs/process_graphs/process_symbolic_parse_error.svg index 9da48c4..02c0659 100644 --- a/app/docs/process_graphs/process_symbolic_parse_error.svg +++ b/app/docs/process_graphs/process_symbolic_parse_error.svg @@ -1,141 +1,141 @@ - - + G - + stage_0 - -setup -Copies the tolerance aliases to rtol and atol -and creates the EvaluationResult. + +setup +Copies the tolerance aliases to rtol and atol +and creates the EvaluationResult. stage_1 - -determine_context -Chooses the symbolic context and adds its -default parameters and criteria. + +determine_context +Chooses the symbolic context and adds its +default parameters and criteria. stage_0->stage_1 - - + + stage_2 - -parse_latex -Not run: is_latex is not set. + +parse_latex +Not run: is_latex is not set. stage_1->stage_2 - - + + stage_3 - -parse_reserved_expressions -Preprocesses and parses the response and -answer. The response could not be parsed, so -the result is returned with its feedback. + +parse_reserved_expressions +Preprocesses and parses the response and +answer. The response could not be parsed, so +the result is returned with its feedback. stage_2->stage_3 - - + + stage_4 - -preview -Not run: the response could not be parsed. + +preview +Not run: the response could not be parsed. stage_3->stage_4 - - + + stage_5 - -parse_criteria -Not run: the response could not be parsed. + +parse_criteria +Not run: the response could not be parsed. stage_4->stage_5 - - + + stage_6 - -generate_feedback_procedures -Not run: the response could not be parsed. + +generate_feedback_procedures +Not run: the response could not be parsed. stage_5->stage_6 - - + + stage_7 - -generate_feedback -Not run: the response could not be parsed. + +generate_feedback +Not run: the response could not be parsed. stage_6->stage_7 - - + + stage_8 - -serialise -Not run: the response could not be parsed. + +serialise +Not run: the response could not be parsed. stage_7->stage_8 - - + + result - -Result -is_correct: False; tags: PARSE_ERROR_response + +Result +is_correct: False; tags: PARSE_ERROR_response stage_8->result - - + + diff --git a/app/docs/process_graphs/process_symbolic_parse_error_calls.svg b/app/docs/process_graphs/process_symbolic_parse_error_calls.svg index d6c9853..5c70152 100644 --- a/app/docs/process_graphs/process_symbolic_parse_error_calls.svg +++ b/app/docs/process_graphs/process_symbolic_parse_error_calls.svg @@ -1,400 +1,400 @@ - - + G - + cluster_stage_cluster_0 - + cluster_stage_cluster_1 - + cluster_stage_cluster_3 - + stage_0 - -setup -Copies the tolerance aliases to rtol and atol -and creates the EvaluationResult. + +setup +Copies the tolerance aliases to rtol and atol +and creates the EvaluationResult. stage_1 - -determine_context -Chooses the symbolic context and adds its -default parameters and criteria. + +determine_context +Chooses the symbolic context and adds its +default parameters and criteria. stage_0->stage_1 - - + + call_0 - -_apply_tolerance_aliases -app.evaluation + +_apply_tolerance_aliases +app.evaluation stage_0->call_0 - - + + call_1 - -EvaluationResult.__init__ -app.result + +EvaluationResult.__init__ +app.result stage_0->call_1 - - + + stage_2 - -parse_latex -Not run: is_latex is not set. + +parse_latex +Not run: is_latex is not set. stage_1->stage_2 - - + + call_2 - -select_context -app.evaluation + +select_context +app.evaluation stage_1->call_2 - - + + call_3 - -_build_parameters -app.evaluation + +_build_parameters +app.evaluation stage_1->call_3 - - + + call_6 - -_add_notation_warnings -app.evaluation + +_add_notation_warnings +app.evaluation stage_1->call_6 - - + + stage_3 - -parse_reserved_expressions -Preprocesses and parses the response and -answer. The response could not be parsed, so -the result is returned with its feedback. + +parse_reserved_expressions +Preprocesses and parses the response and +answer. The response could not be parsed, so +the result is returned with its feedback. stage_2->stage_3 - - + + stage_4 - -preview -Not run: the response could not be parsed. + +preview +Not run: the response could not be parsed. stage_3->stage_4 - - + + call_8 - -parse_reserved_expressions -app.evaluation + +parse_reserved_expressions +app.evaluation stage_3->call_8 - - + + call_15 - -EvaluationResult.serialise -app.result + +EvaluationResult.serialise +app.result stage_3->call_15 - - + + stage_5 - -parse_criteria -Not run: the response could not be parsed. + +parse_criteria +Not run: the response could not be parsed. stage_4->stage_5 - - + + stage_6 - -generate_feedback_procedures -Not run: the response could not be parsed. + +generate_feedback_procedures +Not run: the response could not be parsed. stage_5->stage_6 - - + + stage_7 - -generate_feedback -Not run: the response could not be parsed. + +generate_feedback +Not run: the response could not be parsed. stage_6->stage_7 - - + + stage_8 - -serialise -Not run: the response could not be parsed. + +serialise +Not run: the response could not be parsed. stage_7->stage_8 - - + + result - -Result -is_correct: False; tags: PARSE_ERROR_response + +Result +is_correct: False; tags: PARSE_ERROR_response stage_8->result - - + + call_4 - -_reserved_keywords -app.evaluation + +_reserved_keywords +app.evaluation call_3->call_4 - - + + call_5 - -parsing_parameters_generator -app.context.symbolic + +parsing_parameters_generator +app.context.symbolic call_3->call_5 - - + + call_7 - -notation_warnings -lambdafeedback.expression_parsing.notation + +notation_warnings +lambdafeedback.expression_parsing.notation call_6->call_7 - - + + call_9 - -ExpressionParams.__getstate__ -lambdafeedback.expression_parsing.params + +ExpressionParams.__getstate__ +lambdafeedback.expression_parsing.params call_8->call_9 - - + + call_10 - -ExpressionParams.__setstate__ -lambdafeedback.expression_parsing.params + +ExpressionParams.__setstate__ +lambdafeedback.expression_parsing.params call_8->call_10 - - + + call_11 - -expression_preprocess ×2 -app.context.symbolic + +expression_preprocess ×2 +app.context.symbolic call_8->call_11 - - + + call_12 - -expression_parse ×2 -app.context.symbolic + +expression_parse ×2 +app.context.symbolic call_8->call_12 - - + + call_13 - -_report -app.evaluation + +_report +app.evaluation call_8->call_13 - - + + call_14 - -is_inequality_or_boolean -app.context.symbolic.comparison + +is_inequality_or_boolean +app.context.symbolic.comparison call_8->call_14 - - + + call_16 - -EvaluationResult._serialise_feedback -app.result + +EvaluationResult._serialise_feedback +app.result call_15->call_16 - - + + call_17 - -EvaluationResult.tags -app.result + +EvaluationResult.tags +app.result call_15->call_17 - - + + call_18 - -CriteriaGraph.test_data -lambdafeedback.criteria.graph + +CriteriaGraph.test_data +lambdafeedback.criteria.graph call_15->call_18 - - + +