diff --git a/app/docs/criteria_graphs/physical_quantity_matches_mermaid.svg b/app/docs/criteria_graphs/physical_quantity_matches_mermaid.svg index 8aa51bf..98d6775 100644 --- a/app/docs/criteria_graphs/physical_quantity_matches_mermaid.svg +++ b/app/docs/criteria_graphs/physical_quantity_matches_mermaid.svg @@ -1 +1 @@ -

response matches answer
---
Converts QUANTITY: response tags: {} and QUANTITY: answer tags: {} match to a common set of base units and compares their values.

response matches answer_DIMENSION_MATCH
---
Do the dimensions of QUANTITY: response tags: {} and QUANTITY: answer tags: {} match?

response matches answer_UNIT_COMPARISON
---
Compares how similar the units of QUANTITY: response tags: {} and QUANTITY: answer tags: {} are.

response matches answer_TRUE
---
The quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} match.

response matches answer_FALSE
---
The quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} does not match.

response matches answer_MISSING_VALUE
---
The response is missing a value.

response matches answer_UNEXPECTED_VALUE
---
The response is expected only have unit(s), no value.

response matches answer_MISSING_UNIT
---
The response is missing unit(s).

response matches answer_UNEXPECTED_UNIT
---
The response is expected to be a value without unit(s).

response matches answer_DIMENSION_MATCH_TRUE
---
The quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} have the same dimensions.

response matches answer_DIMENSION_MATCH_FALSE
---
The quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} have different dimensions.

response matches answer_UNIT_COMPARISON_IDENTICAL
---
The units of quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} are identical.

response matches answer_UNIT_COMPARISON_SIMILAR
---
The units of quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} are similar.

response matches answer_UNIT_COMPARISON_PREFIX_IS_LARGE
---
The units of QUANTITY: response tags: {} are at least 1000 times greater than the units of QUANTITY: answer tags: {}.

response matches answer_UNIT_COMPARISON_PREFIX_IS_SMALL
---
The units of QUANTITY: response tags: {} are at least 1000 times smaller than the units of QUANTITY: answer tags: {}.

END
---
Evaluation completed.

 \ No newline at end of file +

response matches answer
---
Converts QUANTITY: response tags: {} and QUANTITY: answer tags: {} match to a common set of base units and compares their values.

response matches answer_DIMENSION_MATCH
---
Do the dimensions of QUANTITY: response tags: {} and QUANTITY: answer tags: {} match?

response matches answer_UNIT_COMPARISON
---
Compares how similar the units of QUANTITY: response tags: {} and QUANTITY: answer tags: {} are.

response matches answer_TRUE
---
The quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} match.

response matches answer_FALSE
---
The quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} does not match.

response matches answer_MISSING_VALUE
---
The response is missing a value.

response matches answer_UNEXPECTED_VALUE
---
The response is expected only have unit(s), no value.

response matches answer_MISSING_UNIT
---
The response is missing unit(s).

response matches answer_UNEXPECTED_UNIT
---
The response is expected to be a value without unit(s).

response matches answer_DIMENSION_MATCH_TRUE
---
The quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} have the same dimensions.

response matches answer_DIMENSION_MATCH_FALSE
---
The quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} have different dimensions.

response matches answer_UNIT_COMPARISON_IDENTICAL
---
The units of quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} are identical.

response matches answer_UNIT_COMPARISON_SIMILAR
---
The units of quantities QUANTITY: response tags: {} and QUANTITY: answer tags: {} are similar.

response matches answer_UNIT_COMPARISON_PREFIX_IS_LARGE
---
The units of QUANTITY: response tags: {} are at least 1000 times greater than the units of QUANTITY: answer tags: {}.

response matches answer_UNIT_COMPARISON_PREFIX_IS_SMALL
---
The units of QUANTITY: response tags: {} are at least 1000 times smaller than the units of QUANTITY: answer tags: {}.

END
---
Evaluation completed.

 diff --git a/app/docs/criteria_graphs/physical_quantity_order_mermaid.svg b/app/docs/criteria_graphs/physical_quantity_order_mermaid.svg index 467951f..5a60f36 100644 --- a/app/docs/criteria_graphs/physical_quantity_order_mermaid.svg +++ b/app/docs/criteria_graphs/physical_quantity_order_mermaid.svg @@ -1 +1 @@ -

response > answer
---
checks if QUANTITY: response tags: {} > QUANTITY: answer tags: {}.

response > answer_TRUE
---
QUANTITY: response tags: {} > QUANTITY: answer tags: {} is true.

response > answer_FALSE
---
QUANTITY: response tags: {} > QUANTITY: answer tags: {} is false.

END
---
Evaluation completed.

 \ No newline at end of file +

response > answer
---
checks if QUANTITY: response tags: {} > QUANTITY: answer tags: {}.

response > answer_TRUE
---
QUANTITY: response tags: {} > QUANTITY: answer tags: {} is true.

response > answer_FALSE
---
QUANTITY: response tags: {} > QUANTITY: answer tags: {} is false.

END
---
Evaluation completed.

 diff --git a/app/docs/criteria_graphs/symbolic_equality_mermaid.svg b/app/docs/criteria_graphs/symbolic_equality_mermaid.svg index e6bc50a..3abefe8 100644 --- a/app/docs/criteria_graphs/symbolic_equality_mermaid.svg +++ b/app/docs/criteria_graphs/symbolic_equality_mermaid.svg @@ -1 +1 @@ -

response = answer
---
Checks if response=answer.

response = answer_SAME_SYMBOLS
---
response has the same (free) symbols as answer.

response = answer_TRUE
---
response is equal to answer.

response = answer_SAME_SYMBOLS_TRUE
---
response has the same (free) symbols as answer.

response = answer_SAME_SYMBOLS_FALSE
---
response does note have the same (free) symbols as answer.

response = answer_FALSE
---
response is not equal toanswer.

response = answer_UNKNOWN
---
Cannot detrmine if response is equal to answer.

END
---
Evaluation completed.

 \ No newline at end of file +

response = answer
---
Checks if response=answer.

response = answer_SAME_SYMBOLS
---
response has the same (free) symbols as answer.

response = answer_TRUE
---
response is equal to answer.

response = answer_SAME_SYMBOLS_TRUE
---
response has the same (free) symbols as answer.

response = answer_SAME_SYMBOLS_FALSE
---
response does note have the same (free) symbols as answer.

response = answer_FALSE
---
response is not equal toanswer.

response = answer_UNKNOWN
---
Cannot detrmine if response is equal to answer.

END
---
Evaluation completed.

 diff --git a/app/docs/criteria_graphs/symbolic_equality_of_equations_mermaid.svg b/app/docs/criteria_graphs/symbolic_equality_of_equations_mermaid.svg index e9c8b5f..0b9acaa 100644 --- a/app/docs/criteria_graphs/symbolic_equality_of_equations_mermaid.svg +++ b/app/docs/criteria_graphs/symbolic_equality_of_equations_mermaid.svg @@ -1 +1 @@ -

response = answer
---
Checks if response is equivalent to answer.

response = answer_UNKNOWN
---
Cannot determine if response is equivalent to answer.

response = answer_TRUE
---
response is equivalent to answer.

response = answer_FALSE
---
response is not equivalent to answer.

response = answer_EXPRESSION_NOT_EQUALITY
---
response is an expression, not an equality.

response = answer_EQUALITY_NOT_EXPRESSION
---
response is an equality, not an expression.

END
---
Evaluation completed.

 \ No newline at end of file +

response = answer
---
Checks if response is equivalent to answer.

response = answer_UNKNOWN
---
Cannot determine if response is equivalent to answer.

response = answer_TRUE
---
response is equivalent to answer.

response = answer_FALSE
---
response is not equivalent to answer.

response = answer_EXPRESSION_NOT_EQUALITY
---
response is an expression, not an equality.

response = answer_EQUALITY_NOT_EXPRESSION
---
response is an equality, not an expression.

END
---
Evaluation completed.

 diff --git a/app/docs/criteria_graphs/symbolic_equality_of_sets_mermaid.svg b/app/docs/criteria_graphs/symbolic_equality_of_sets_mermaid.svg index 545f872..a341538 100644 --- a/app/docs/criteria_graphs/symbolic_equality_of_sets_mermaid.svg +++ b/app/docs/criteria_graphs/symbolic_equality_of_sets_mermaid.svg @@ -1 +1 @@ -

response = answer
---
Checks if response=answer.

response = answer_TRUE
---
response is equal to answer.

response = answer_MULTIPLE_ANSWER_FAIL_ALL
---
At least one answer or response was incorrect.

response = answer_MULTIPLE_ANSWER_FAIL_RESPONSE
---
At least one response was incorrect.

response = answer_MULTIPLE_ANSWER_FAIL_ANSWER
---
At least one answer is missing in the response.

END
---
Evaluation completed.

 \ No newline at end of file +

response = answer
---
Checks if response=answer.

response = answer_TRUE
---
response is equal to answer.

response = answer_MULTIPLE_ANSWER_FAIL_ALL
---
At least one answer or response was incorrect.

response = answer_MULTIPLE_ANSWER_FAIL_RESPONSE
---
At least one response was incorrect.

response = answer_MULTIPLE_ANSWER_FAIL_ANSWER
---
At least one answer is missing in the response.

END
---
Evaluation completed.

 diff --git a/app/docs/criteria_graphs/symbolic_other_mermaid.svg b/app/docs/criteria_graphs/symbolic_other_mermaid.svg index 7743796..da291d2 100644 --- a/app/docs/criteria_graphs/symbolic_other_mermaid.svg +++ b/app/docs/criteria_graphs/symbolic_other_mermaid.svg @@ -1 +1 @@ -

response contains x
---
Checks if response contains x is true.

response contains x_TRUE
---
response contains x is true.

response contains x_FALSE
---
response contains x is false.

response contains x_UNKNOWN
---
response contains x is false.

END
---
Evaluation completed.

 \ No newline at end of file +

response contains x
---
Checks if response contains x is true.

response contains x_TRUE
---
response contains x is true.

response contains x_FALSE
---
response contains x is false.

response contains x_UNKNOWN
---
response contains x is false.

END
---
Evaluation completed.

 diff --git a/app/docs/criteria_graphs/symbolic_where_mermaid.svg b/app/docs/criteria_graphs/symbolic_where_mermaid.svg index 32715f3..8b6d8ee 100644 --- a/app/docs/criteria_graphs/symbolic_where_mermaid.svg +++ b/app/docs/criteria_graphs/symbolic_where_mermaid.svg @@ -1 +1 @@ -

response = answer where x = 0
---
Checks if response = answer where x = 0.

response = answer where x = 0_TRUE
---
response = answer where x = 0is true.

response = answer where x = 0_FALSE
---
response = answer is not true when x = 0.

END
---
Evaluation completed.

 \ No newline at end of file +

response = answer where x = 0
---
Checks if response = answer where x = 0.

response = answer where x = 0_TRUE
---
response = answer where x = 0is true.

response = answer where x = 0_FALSE
---
response = answer is not true when x = 0.

END
---
Evaluation completed.

 diff --git a/app/docs/criteria_graphs/symbolic_written_as_mermaid.svg b/app/docs/criteria_graphs/symbolic_written_as_mermaid.svg index f8e06d3..f869019 100644 --- a/app/docs/criteria_graphs/symbolic_written_as_mermaid.svg +++ b/app/docs/criteria_graphs/symbolic_written_as_mermaid.svg @@ -1 +1 @@ -

answer_WRITTEN_AS_KNOWN_FORM
---
Checks if answer is written in a known form.

response written as answer
---
Checks if response is written in the same form as answer.

response_COMPARE_FORM_TO_NUMBER
---
Checks if response is written in simplified number form.

response_COMPARE_FORM_TO_CARTESIAN
---
Checks if response is written in cartesian form.

response_COMPARE_FORM_TO_EXPONENTIAL
---
Checks if response is written in exponential form.

answer_WRITTEN_AS_UNKNOWN
---
answer is not written in a known form.

response written as answer_TRUE
---
answer written in the same form as answer.

response written as answer_FALSE
---
answer written in the same form as answer.

answer_WRITTEN_AS_NUMBER
---
answer in written in simplified number form.

answer_WRITTEN_AS_CARTESIAN
---
answer in written in cartesian form.

answer_WRITTEN_AS_EXPONENTIAL
---
answer in written in exponential form.

END
---
Evaluation completed.

 \ No newline at end of file +

answer_WRITTEN_AS_KNOWN_FORM
---
Checks if answer is written in a known form.

response written as answer
---
Checks if response is written in the same form as answer.

response_COMPARE_FORM_TO_NUMBER
---
Checks if response is written in simplified number form.

response_COMPARE_FORM_TO_CARTESIAN
---
Checks if response is written in cartesian form.

response_COMPARE_FORM_TO_EXPONENTIAL
---
Checks if response is written in exponential form.

answer_WRITTEN_AS_UNKNOWN
---
answer is not written in a known form.

response written as answer_TRUE
---
answer written in the same form as answer.

response written as answer_FALSE
---
answer written in the same form as answer.

answer_WRITTEN_AS_NUMBER
---
answer in written in simplified number form.

answer_WRITTEN_AS_CARTESIAN
---
answer in written in cartesian form.

answer_WRITTEN_AS_EXPONENTIAL
---
answer in written in exponential form.

END
---
Evaluation completed.

 diff --git a/scripts/generate_process_animation.py b/scripts/generate_process_animation.py new file mode 100644 index 0000000..016bbb7 --- /dev/null +++ b/scripts/generate_process_animation.py @@ -0,0 +1,1813 @@ +"""Generates `build/process_animation.html`: an interactive page that animates the process graphs of +`scripts/generate_process_graphs.py`. + +Each example (those of the static figures, and `EXTRA_EXAMPLES`) is run through `evaluation_function` while recording +every call down to `MAX_DEPTH` calls below `_evaluate`, with its arguments and return value or exception, the forms the +response takes, the parameters, the criteria and the evaluations each criteria graph runs. The calls graph is laid out +with graphviz once for each depth, with and without the criteria graphs and the internal (dunder) calls, and the page +grows it in the order the functions were called. A side panel shows each step's details, and these can be switched on +and off on the page. + +The page is a build output and is not committed. Run from the repository root with +`python -m scripts.generate_process_animation` (it needs graphviz, i.e. `dot`), then open the page in a browser. +""" +from __future__ import annotations + +import base64 +import gzip +import html +import json +import re +from pathlib import Path +from typing import Any + +import pydot + +from scripts.generate_criteria_graphs import _node_label, add_criteria_graph +from scripts.generate_process_graphs import ( + EXAMPLES, + FIRST_STAGE, + Recording, + all_stages, + calls_to_dot, + record, + shown_parent, + stage_description, +) + +OUTPUT_FILE = Path(__file__).resolve().parent.parent / "build" / "process_animation.html" + +# Examples shown only in the animation, for the parts of the criteria chain the static figures' examples don't reach: +# a response that is not correct, more than one criterion, and a criterion without a template of its own. +EXTRA_EXAMPLES: list[tuple[str, str, str, dict[str, Any]]] = [ + ("process_symbolic_incorrect", "x + 2", "x + 1", {}), + ("process_symbolic_syntactical_comparison", "1 + x", "x + 1", {"syntactical_comparison": True}), + ("process_symbolic_default_template", "2*x + 1", "x + 1", {"criteria": "response contains x"}), +] + +# The deepest calls below `_evaluate` recorded, and the deepest the page can show. +MAX_DEPTH = 5 + +# The depth the page starts at: the depth of the static calls figures. +START_DEPTH = 2 + +# How many calls of each function keep their arguments and return value, for the list of calls in the panel. +INVOCATION_VALUES = 20 + +CRITERION_STYLE = {"shape": "note", "style": "filled", "fillcolor": "#FFF8E1", "color": "#8D6E63"} +CRITERION_EDGE_STYLE = {"color": "#8D6E63", "fontcolor": "#5D4037", "fontname": "Helvetica", "fontsize": "10"} + + +def example_title(name: str) -> str: + """The name shown on the page for the example `name`, e.g. "Physical quantity" for `process_physical_quantity`.""" + return name.removeprefix("process_").replace("_", " ").capitalize() + + +def stage_identifier(stage: str) -> str: + """The graphviz identifier `add_stage_nodes` gives the node of `stage`.""" + return f"stage_{all_stages().index(stage)}" + + +def build_steps(recording: Recording) -> tuple[list[dict[str, Any]], dict[str, dict[str, list[Any]]], list[dict[str, Any]]]: + """The steps of the animation, the index of the event of each call of each function, and the forms the response + took, from the events in `recording`. + + Each step is a stage being entered, a function called for the first time from its caller (a node of the calls + graph appearing), the criteria being parsed, a template building a criteria graph, a criteria graph evaluation, or + the result. Later calls of the same function only raise its count, so they are not steps of their own. + + The invocations of each call are the index of the event of each, and the values of the first `INVOCATION_VALUES`. + """ + steps: list[dict[str, Any]] = [{"kind": "stage", "stage": FIRST_STAGE, "event": -1, "depth": 0}] + invocations: dict[str, dict[str, list[Any]]] = {} + forms: list[dict[str, Any]] = [] + calls = {call.identifier: call for call in recording.calls.values()} + # The values of each call still running, and its step, or None for a call that is not the first of its function. + running: list[tuple[dict[str, Any], dict[str, Any] | None]] = [] + evaluation_step: dict[str, Any] = {} + for (index, event) in enumerate(recording.events): + kind = event["kind"] + if kind == "stage": + steps.append({"kind": "stage", "stage": event["stage"], "event": index, "depth": 0}) + elif kind == "call": + call_invocations = invocations.setdefault(event["call"], {"events": [], "values": []}) + call_invocations["events"].append(index) + values = {"event": index, "arguments": event["arguments"], "returned": "", "raised": False} + # Only the first calls keep their values, as some functions are called over a thousand times + if len(call_invocations["values"]) < INVOCATION_VALUES: + call_invocations["values"].append(values) + if len(call_invocations["events"]) == 1: + step = { + "kind": "call", + "call": event["call"], + "stage": event["stage"], + "depth": calls[event["call"]].depth, + "event": index, + "arguments": event["arguments"], + "returned": "", + "raised": False, + } + steps.append(step) + running.append((values, step)) + else: + running.append((values, None)) + elif kind == "return": + (values, step) = running.pop() + outcome = {"returned": event["value"], "raised": event["raised"]} + if "full" in event: + outcome["returnedFull"] = event["full"] + if "exception" in event: + outcome["exception"] = event["exception"] + values.update(outcome) + if step is not None: + step.update(outcome) + elif kind == "criteria": + steps.append({ + "kind": "criteria", + "depth": 0, + "event": index, + "criteria": [criterion["identifier"] for criterion in event["criteria"]], + }) + elif kind == "template": + steps.append({"kind": "template", "depth": 0, "event": index, "criterion": event["criterion"]}) + elif kind == "evaluate": + evaluation_step = { + "kind": "evaluate", + "criterion": event["criterion"], + "node": event["node"], + "depth": 0, + "event": index, + "reached": [], + } + steps.append(evaluation_step) + elif kind == "evaluated": + evaluation_step["reached"] = event["reached"] + elif kind == "form": + forms.append({"event": index, "call": event["call"], "value": event["value"], "type": event["type"]}) + elif kind == "parameters" or kind == "feedback": + # Shown in the panel rather than as steps (see `parameters_data` and `feedback_data`) + continue + else: + raise ValueError(f"Unknown event kind {kind!r}.") + steps.append({"kind": "result", "event": len(recording.events), "depth": 0}) + for (index, step) in enumerate(steps): + step["index"] = index + return (steps, invocations, forms) + + +def criteria_chain(recording: Recording, params: dict[str, Any]) -> dict[str, Any]: + """How the example's criteria graphs came about: where the criteria string came from, the string before and after + preprocessing, and for each criterion (by its identifier) its parse, the template that built its graph and the + index of its graph's cluster. + + Empty when the evaluation stopped before parsing the criteria. + """ + criteria_event = next((event for event in recording.events if event["kind"] == "criteria"), None) + if criteria_event is None: + return {} + if "criteria" in params: + source = 'Given in params["criteria"].' + else: + context = "physical_quantity" if params.get("physical_quantity", False) is True else "symbolic" + source = f"Not given in params, so the default criteria of the {context} context." + if params.get("syntactical_comparison", False) is True: + source += ' syntactical_comparison adds "response written as answer".' + indices = {criterion: index for (index, criterion) in enumerate(recording.criteria_graphs)} + criteria = {} + for criterion in criteria_event["criteria"]: + criteria[criterion["identifier"]] = { + "index": indices.get(criterion["identifier"]), + "label": criterion["label"], + "tree": criterion["tree"], + } + for event in recording.events: + if event["kind"] == "template": + criteria[event["criterion"]].update({ + "template": event["template"], + "templateLabel": event["label"], + "call": event["call"], + "evaluations": event["evaluations"], + }) + return { + "source": source, + "original": criteria_event["original"], + "preprocessed": criteria_event["preprocessed"], + "criteria": criteria, + } + + +def visible_call(recording: Recording, identifier: str, depth: int, hide_internals: bool) -> str: + """The identifier of the call `identifier`, or of its nearest caller, that is drawn down to `depth`, with or + without the internal calls. + """ + calls = {call.identifier: call for call in recording.calls.values()} + call = calls[identifier] + while call.parent is not None and (call.depth > depth or (hide_internals and call.internal)): + call = call.parent + return call.identifier + + +def calls_layout( + recording: Recording, + params: dict[str, Any], + result: dict[str, Any], + chain: dict[str, Any], + depth: int, + with_criteria: bool, + hide_internals: bool, +) -> tuple[pydot.Dot, dict[str, dict[str, str]]]: + """The calls graph down to `depth`, with the graphviz identifier of each criteria graph node by criterion and node + label. + + With `with_criteria`, each criterion gets a node in the `parse_criteria` stage, with an edge to the template call + its parse label chose, and each criteria graph is drawn with an edge from that template call, which built it, and + from the `generate_feedback` stage, which ran it. Only the edge from the stage places the graph, so it is drawn + next to the stage rather than past the template call. + + With `hide_internals`, the internal calls are left out (see `calls_to_dot`). + """ + dot = calls_to_dot(recording, params, result, depth, hide_internals) + identifiers: dict[str, dict[str, str]] = {} + # Without criteria, i.e. when the evaluation stopped before parsing them, there is nothing to add + if not with_criteria or not chain: + return (dot, identifiers) + dot.set("compound", "true") + stages = all_stages() + parse_cluster = dot.get_subgraph(f"cluster_stage_cluster_{stages.index('parse_criteria')}")[0] + parser_call = next(call.identifier for call in recording.calls.values() if call.qualname == "create_criteria_dict") + for (index, (criterion, graph)) in enumerate(recording.criteria_graphs.items()): + details = chain["criteria"][criterion] + template_call = visible_call(recording, details["call"], depth, hide_internals) + criterion_node = f"criterion_{index}" + parse_cluster.add_node(pydot.Node( + criterion_node, label=_node_label(criterion, f"parse label: {details['label']}"), **CRITERION_STYLE + )) + dot.add_edge(pydot.Edge(parser_call, criterion_node, **CRITERION_EDGE_STYLE)) + choice = details["label"] if details["templateLabel"] is not None else "default" + dot.add_edge(pydot.Edge(criterion_node, template_call, label=html.escape(choice), **CRITERION_EDGE_STYLE)) + title = f"Criteria graph: {criterion} (built by {details['template']})" + cluster = pydot.Cluster(f"criteria_{index}", label=html.escape(title), color="#9E9E9E") + identifiers[criterion] = add_criteria_graph(cluster, graph, prefix=f"criteria_{index}_") + dot.add_subgraph(cluster) + for start in graph.starting_evaluations(criterion + "_TRUE"): + dot.add_edge(pydot.Edge( + template_call, + identifiers[criterion][start], + label="builds", + style="dashed", + lhead=cluster.get_name(), + constraint="false", + **CRITERION_EDGE_STYLE, + )) + dot.add_edge(pydot.Edge( + stage_identifier("generate_feedback"), + identifiers[criterion][start], + label="runs", + style="dashed", + lhead=cluster.get_name(), + fontname="Helvetica", + fontsize="10", + )) + return (dot, identifiers) + + +def svg_markup(dot: pydot.Dot) -> str: + """`dot` drawn as an SVG element, gzipped and base64 encoded (the page unpacks it). + + The SVG is left without the XML declaration and doctype in front of it, or the comments graphviz writes before each + node and edge (their titles hold the same names). The 20 layouts of an example are mostly the same markup, so + gzipping them makes the page about a tenth of the size. + """ + svg = dot.create(format="svg").decode("utf-8") + svg = re.sub(r"\n?", "", svg[svg.index(" dict[str, Any]: + """The graphviz identifier, kind and details of each node of each criteria graph, by criterion and node label.""" + data: dict[str, Any] = {} + for (criterion, graph) in recording.criteria_graphs.items(): + nodes = {} + groups = [("evaluation", graph.evaluations), ("criterion", graph.criteria), ("output", graph.outputs)] + for (kind, group) in groups: + for (label, node) in group.items(): + nodes[label] = { + "id": identifiers[criterion][label], "kind": kind, "summary": node.summary, "details": node.details + } + data[criterion] = nodes + return data + + +def parameters_data(recording: Recording) -> dict[str, Any]: + """The parameters `_build_parameters` completed: the index of its event, the ones it was given and the ones it + added. Empty when it wasn't called. + """ + for (index, event) in enumerate(recording.events): + if event["kind"] == "parameters": + return {"event": index, "given": event["given"], "added": event["added"]} + return {} + + +def feedback_data(recording: Recording, identifiers: dict[str, dict[str, str]]) -> dict[str, dict[str, Any]]: + """For each tag a criteria graph exported, the criterion whose graph exported it first, its feedback text and the + graphviz identifier of its criteria node. + """ + feedback: dict[str, dict[str, Any]] = {} + for event in recording.events: + if event["kind"] != "feedback" or event["criterion"] is None: + continue + for (tag, text) in event["feedback"]: + if tag not in feedback: + feedback[tag] = { + "criterion": event["criterion"], + "text": text, + "node": identifiers.get(event["criterion"], {}).get(tag), + } + return feedback + + +def stage_durations(recording: Recording) -> dict[str, float]: + """How long each stage that ran took, in milliseconds: from its start to the next stage's, or to the end of the + evaluation for the last one. + """ + starts = [(FIRST_STAGE, 0.0)] + list(recording.stage_times.items()) + durations = {} + for (position, (stage, start)) in enumerate(starts): + if position + 1 < len(starts): + durations[stage] = round(starts[position + 1][1] - start, 1) + elif recording.duration_ms is not None: + durations[stage] = round(recording.duration_ms - start, 1) + return durations + + +def example_data(name: str, response: str, answer: str, params: dict[str, Any]) -> dict[str, Any]: + """Everything the page needs to animate one example.""" + (recording, result) = record(response, answer, params, depth=MAX_DEPTH, record_values=True) + (steps, invocations, forms) = build_steps(recording) + chain = criteria_chain(recording, params) + layouts = {} + criteria_identifiers: dict[str, dict[str, str]] = {} + for depth in range(1, MAX_DEPTH + 1): + for with_criteria in (False, True): + for hide_internals in (False, True): + (dot, identifiers) = calls_layout( + recording, params, result, chain, depth, with_criteria, hide_internals + ) + key = f"{depth}{'c' if with_criteria else ''}{'h' if hide_internals else ''}" + layouts[key] = svg_markup(dot) + if with_criteria: + criteria_identifiers = identifiers + calls = {} + for call in recording.calls.values(): + parent = shown_parent(call, hide_internals=True) + calls[call.identifier] = { + "qualname": call.qualname, + "module": call.module, + "depth": call.depth, + "stage": call.stage, + "parent": None if call.parent is None else call.parent.identifier, + # The caller drawn when the internal calls are hidden + "shownParent": None if parent is None else parent.identifier, + "internal": call.internal, + "doc": call.doc, + "source": call.source, + } + durations = stage_durations(recording) + return { + "name": name, + "title": example_title(name), + "response": response, + "answer": answer, + "params": params, + "stages": [ + { + "name": stage, + "id": stage_identifier(stage), + "description": stage_description(stage, recording, params, result), + "duration": durations.get(stage), + } + for stage in all_stages() + ], + "duration": recording.duration_ms, + "calls": calls, + "steps": steps, + "invocations": invocations, + "forms": forms, + "criteria": criteria_data(recording, criteria_identifiers), + "chain": chain, + "parameters": parameters_data(recording), + "feedback": feedback_data(recording, criteria_identifiers), + "layouts": layouts, + "result": { + "is_correct": result["is_correct"], + "tags": result["tags"], + "feedback": result["feedback"], + "response_latex": result.get("response_latex", ""), + "response_simplified": result.get("response_simplified", ""), + }, + } + + +def main() -> None: + examples = [] + for (name, response, answer, params) in EXAMPLES + EXTRA_EXAMPLES: + examples.append(example_data(name, response, answer, params)) + print(f"Recorded {name}: {len(examples[-1]['steps'])} steps") + data = json.dumps({"examples": examples, "maxDepth": MAX_DEPTH, "startDepth": START_DEPTH}) + # Keeps the data from closing the script element it is in. + data = data.replace(" + + + + +Evaluation process animation + + + +
+

Evaluation process

+ +
+
+
+
+
+
+ + + +
+
+ Key +
+
Nodes
+
stage of _evaluate
+
function call (×N: calls from that caller)
+
criterion parsed from the criteria
+
criteria graph evaluation
+
criteria graph criterion (a tag)
+
output, or the result
+
States
+
the current step
+
its callers (the call stack)
+
evaluation run
+
criterion reached, and its path
+
not reached yet
+
Edges
+
calls
+
criterion chooses its template
+
template builds the graph
+
generate_feedback runs the graph
+
sufficiency in a criteria graph
+
+
+
+
+
+ +
+ + + + + +""" + + +if __name__ == "__main__": + main() diff --git a/scripts/generate_process_graphs.py b/scripts/generate_process_graphs.py index a611421..265513f 100644 --- a/scripts/generate_process_graphs.py +++ b/scripts/generate_process_graphs.py @@ -10,21 +10,25 @@ """ from __future__ import annotations +import dis import html import inspect import json import logging import re import sys -from dataclasses import dataclass, field +from dataclasses import dataclass, field, is_dataclass from pathlib import Path from types import FrameType from typing import Any import pydot +import sympy from lambdafeedback.criteria import CriteriaGraph from app import evaluation +from app.context.base import Context +from app.result import EvaluationResult from scripts.generate_criteria_graphs import _node_label, add_criteria_graph, new_drawing OUTPUT_DIRECTORY = Path(__file__).resolve().parent.parent / "app" / "docs" / "process_graphs" @@ -39,6 +43,16 @@ # How many calls below `_evaluate` the call graphs go. CALL_DEPTH = 2 +# The longest an argument or return value recorded for the process animation gets before it is cut short, and the +# longest the full value of a summarised dictionary gets. +VALUE_LENGTH = 200 +FULL_VALUE_LENGTH = 1000 + +# The longest an exception's type and message recorded for the process animation get before they are cut short. +EXCEPTION_LENGTH = 160 + +REPOSITORY = Path(__file__).resolve().parent.parent + # Functions left out of the call graphs; the functions they call are shown under their caller instead. HIDDEN_FUNCTIONS = {"log_event", "elapsed_ms", "_evaluate..enter_stage"} HIDDEN_FUNCTION_PARTS = ("", "", "", "", "", "__create_fn__") @@ -67,36 +81,406 @@ class Call: qualname: str stage: str parent: Call | None + depth: int count: int = 1 + # The first paragraph of the function's docstring, and where it is defined (`path:line`); only set when values + # are recorded. + doc: str = "" + source: str = "" + + @property + def internal(self) -> bool: + """Whether the function is a dunder method, such as `__init__` or `__getstate__`.""" + name = self.qualname.rsplit(".", 1)[-1] + return name.startswith("__") and name.endswith("__") @dataclass class Recording: - """What happened during one `evaluation_function` call.""" + """What happened during one `evaluation_function` call. + + `events` is filled only when values are recorded (see `Recorder`): every stage entered, call made, value returned + and criteria graph evaluation run, in order, as JSON-ready dictionaries. + + `stage_times` is the time from the start of the evaluation to the start of each stage, and `duration_ms` the time + the whole evaluation took, in milliseconds, both from the evaluation's log events. + """ stages: list[str] = field(default_factory=list) early_return: bool = False calls: dict[tuple[Any, ...], Call] = field(default_factory=dict) criteria_graphs: dict[str, CriteriaGraph] = field(default_factory=dict) + events: list[dict[str, Any]] = field(default_factory=list) + stage_times: dict[str, float] = field(default_factory=dict) + duration_ms: float | None = None -class Recorder(logging.Handler): - """Records the stages from the evaluation's log events, and the function calls with `sys.setprofile`.""" +def _cut(text: str, length: int) -> str: + if len(text) > length: + return text[:length - 1] + "…" + return text + + +def _plain_repr(value: Any) -> str: + """`repr(value)` without memory addresses.""" + try: + text = repr(value) + except (AttributeError, TypeError, ValueError, KeyError): + # Objects whose `__init__` is still running can fail to give a repr. + text = f"<{type(value).__name__}>" + return re.sub(r" at 0x[0-9a-f]+", "", text) + + +def short_repr(value: Any) -> str: + """A short description of `value` for the process animation: `repr(value)` without memory addresses, cut short at + `VALUE_LENGTH` characters. + + The values whose repr says little are summarised instead: a context by its name, an `EvaluationResult` by its + correctness and tags, and a dictionary too long to show by its keys. + """ + if isinstance(value, Context): + name = value.feedback_procedure_generator.__module__.removeprefix("app.context.").split(".")[0] + return f"{name} context" + elif isinstance(value, EvaluationResult): + try: + return f"EvaluationResult(is_correct={value.is_correct}, tags={value.tags})" + except AttributeError: + # The result's `__init__` is still running + return "EvaluationResult()" + text = _plain_repr(value) + if isinstance(value, dict) and len(text) > VALUE_LENGTH: + keys = ", ".join(str(key) for key in value) + return _cut(f"{{… {len(value)} keys: {keys} …}}", VALUE_LENGTH) + return _cut(text, VALUE_LENGTH) + + +def full_repr(value: Any) -> str | None: + """The repr of `value`, cut short at `FULL_VALUE_LENGTH` characters, when `short_repr` summarises it as a dictionary + of keys; None otherwise. + """ + if not isinstance(value, dict): + return None + text = _plain_repr(value) + if len(text) <= VALUE_LENGTH: + return None + return _cut(text, FULL_VALUE_LENGTH) + + +def shown_value(name: str, value: Any) -> list[str]: + """The name of an argument and its `short_repr`, followed by its `full_repr` when there is one.""" + full = full_repr(value) + if full is None: + return [name, short_repr(value)] + return [name, short_repr(value), full] + + +def exception_text(exception: BaseException) -> str: + """The type and message of `exception`, cut short at `EXCEPTION_LENGTH` characters.""" + try: + message = str(exception) + except (AttributeError, TypeError, ValueError): + message = "" + name = type(exception).__name__ + return _cut(f"{name}: {message}" if message else name, EXCEPTION_LENGTH) + + +def parse_tree_text(node: Any) -> str: + """The criteria parse tree under `node` drawn with `├─` and `└─` connectors, one node per line as `label: content`, + followed by the node's tags when it has any. + """ + lines = [_parse_node_text(node)] + _add_parse_children(node, "", lines) + return "\n".join(lines) + + +def _parse_node_text(node: Any) -> str: + label = node.label.strip() + content = str(node.content).strip() + text = label if content in ("", label) else f"{label}: {content}" + if node.tags: + text += f" tags: {', '.join(sorted(str(tag) for tag in node.tags))}" + return text + + +def _add_parse_children(node: Any, indent: str, lines: list[str]) -> None: + for (index, child) in enumerate(node.children): + last = index == len(node.children) - 1 + lines.append(indent + ("└─ " if last else "├─ ") + _parse_node_text(child)) + _add_parse_children(child, indent + (" " if last else "│ "), lines) + + +def source_location(frame: FrameType) -> str: + """Where the function running in `frame` is defined, as `path:line`, with the path relative to the repository or, + for an installed package, to `site-packages`. + """ + path = Path(frame.f_code.co_filename) + if path.is_relative_to(REPOSITORY) and "site-packages" not in path.parts: + shown = str(path.relative_to(REPOSITORY)) + elif "site-packages" in path.parts: + shown = "/".join(path.parts[path.parts.index("site-packages") + 1:]) + else: + shown = path.name + return f"{shown}:{frame.f_code.co_firstlineno}" + + +def first_doc_paragraph(frame: FrameType) -> str: + """The first paragraph of the docstring of the function running in `frame`, on one line, found from its module by + its qualified name; empty when it has none of its own, or is defined inside another function. + """ + qualname = frame.f_code.co_qualname + if "" in qualname: + return "" + function: Any = None + for (index, part) in enumerate(qualname.split(".")): + if index == 0: + function = frame.f_globals.get(part) + else: + function = inspect.getattr_static(function, part, None) + if function is None: + return "" + if isinstance(function, (staticmethod, classmethod)): + function = function.__func__ + elif isinstance(function, property): + function = function.fget + # `__doc__` rather than `inspect.getdoc`, which falls back to the docstring of the method a class inherits from, + # e.g. `object.__init__`'s "Initialize self." + doc = getattr(function, "__doc__", None) + if not isinstance(doc, str): + return "" + paragraph = inspect.cleandoc(doc).split("\n\n")[0] + return _cut(" ".join(paragraph.split()), VALUE_LENGTH) + + +def frame_argument_values(frame: FrameType) -> list[tuple[str, Any]]: + """The name and value of each argument of the function running in `frame`.""" + arguments = inspect.getargvalues(frame) + names = list(arguments.args) + if arguments.varargs is not None: + names.append(arguments.varargs) + if arguments.keywords is not None: + names.append(arguments.keywords) + return [(name, arguments.locals.get(name)) for name in names] + + +def is_response_form(value: Any) -> bool: + """Whether `value` is the kind of value the response is turned into: a string, a SymPy expression, an `app` or + `lambdafeedback` dataclass such as a parsed quantity or a token, or a list or tuple of these. + """ + if isinstance(value, (str, sympy.Basic)): + return True + elif isinstance(value, (list, tuple)): + return len(value) > 0 and all(is_response_form(item) for item in value) + elif type(value).__module__.startswith("lambdafeedback."): + return True + return type(value).__module__.startswith("app.") and is_dataclass(value) + + +def returned_normally(frame: FrameType) -> bool: + """Whether `frame` is returning (or yielding) a value, rather than stopping because of an exception.""" + instruction = dis.opname[frame.f_code.co_code[frame.f_lasti]] + return instruction in ("RETURN_VALUE", "RETURN_CONST", "YIELD_VALUE") - def __init__(self) -> None: + +class Recorder(logging.Handler): + """Records the stages from the evaluation's log events, and the function calls with `sys.setprofile`. + + Calls are followed down to `depth` calls below `_evaluate`. With `record_values`, these are kept in order in + `Recording.events`: + - the arguments and return value of each call, or the exception it raised instead of returning (see `unwind`); + - the evaluations each criteria graph runs, and the criteria each one reports; + - each new form of the response (see `is_response_form`): the value returned by a call that was given the response, + or an earlier form of it, as an argument; + - the criteria `create_criteria_dict` parses, and the template that builds each one's criteria graph; + - the parameters `_build_parameters` completes, and the feedback each criteria graph exports for its reached + criteria. + The docstring and source location of each function called are kept on its `Call` too. + """ + + def __init__(self, depth: int = CALL_DEPTH, record_values: bool = False, response: str = "") -> None: super().__init__() self.recording = Recording() self.stage = FIRST_STAGE + self.depth = depth + self.record_values = record_values # Each frame being followed: (the Call it belongs to, or None for `_evaluate`, its depth below `_evaluate`). self.frames: dict[FrameType, tuple[Call | None, int]] = {} + # The frames whose return is recorded: each frame's Call, or the criterion and evaluation node it runs. + self.call_frames: dict[FrameType, Call] = {} + self.evaluation_frames: dict[FrameType, tuple[str, str]] = {} + # The reprs of the forms the response has taken so far, and the frames given one of them as an argument. + self.response_forms = {repr(response)} + self.response_frames: set[FrameType] = set() + # The code of the last call recorded as raising an exception, and its return event, until `unwind` fills in + # the exception. + self.raised: tuple[Any, dict[str, Any]] | None = None def emit(self, record: logging.LogRecord) -> None: event = json.loads(record.getMessage()) if event["event"] == "stage_start": self.stage = event["stage"] self.recording.stages.append(self.stage) + self.recording.stage_times[self.stage] = event["elapsed_ms"] + if self.record_values: + self.recording.events.append({"kind": "stage", "stage": self.stage}) elif event["event"] == "evaluation_end": self.recording.early_return = event["early_return"] + self.recording.duration_ms = event["duration_ms"] + + def unwind(self, code: Any, offset: int, exception: BaseException) -> None: + """The `sys.monitoring` `PY_UNWIND` callback: a frame of `code` is stopping because of `exception`. + + It runs just after the profile function's `return` event for the same frame, which cannot see the exception, + so the exception is added to the return event recorded there. + """ + if self.raised is not None and self.raised[0] is code: + self.raised[1]["exception"] = exception_text(exception) + self.raised = None + + def _record_evaluation(self, frame: FrameType) -> None: + """Records the start of a criteria graph evaluation, when `frame` is the `evaluate` function of one.""" + runner = frame.f_back + if runner is None or runner.f_code.co_qualname != "CriteriaGraph.generate_feedback": + return + evaluate = runner.f_locals.get("evaluate") + # `evaluate` can be a `functools.partial` of the function that runs + function = getattr(evaluate, "func", evaluate) + if getattr(function, "__code__", None) is not frame.f_code: + return + criterion = runner.f_locals["main_criterion"].removesuffix("_TRUE") + node = runner.f_locals["label"] + self.evaluation_frames[frame] = (criterion, node) + self.recording.events.append({"kind": "evaluate", "stage": self.stage, "criterion": criterion, "node": node}) + + def _record_call(self, frame: FrameType, call: Call) -> None: + self.call_frames[frame] = call + arguments = frame_argument_values(frame) + for (name, value) in arguments: + if name != "self" and is_response_form(value) and repr(value) in self.response_forms: + self.response_frames.add(frame) + break + shown_arguments = [] + for (name, value) in arguments: + if name == "self": + shown_arguments.append([name, f"<{type(value).__name__}>"]) + else: + shown_arguments.append(shown_value(name, value)) + self.recording.events.append({ + "kind": "call", "stage": self.stage, "call": call.identifier, "arguments": shown_arguments + }) + + def _record_criteria(self, frame: FrameType, value: Any) -> None: + """Records the criteria when `frame` is `create_criteria_dict` returning them: the criteria string before and + after preprocessing, and each criterion with the label and tree its parse gave. + """ + if frame.f_code.co_qualname != "create_criteria_dict" or not isinstance(value, dict): + return + self.recording.events.append({ + "kind": "criteria", + "stage": self.stage, + "original": frame.f_locals.get("original_criteria_string", ""), + "preprocessed": frame.f_locals.get("criteria_string", ""), + "criteria": [ + {"identifier": identifier, "label": node.label.strip(), "tree": parse_tree_text(node)} + for (identifier, node) in value.items() + ], + }) + + def _record_parameters(self, frame: FrameType, value: Any) -> None: + """Records the parameters when `frame` is `_build_parameters` returning them: the parameters it was given, and + the ones it added. + """ + if frame.f_code.co_qualname != "_build_parameters" or not isinstance(value, dict): + return + given = frame.f_locals.get("params", {}) + self.recording.events.append({ + "kind": "parameters", + "stage": self.stage, + "given": [shown_value(str(key), item) for (key, item) in value.items() if key in given], + "added": [shown_value(str(key), item) for (key, item) in value.items() if key not in given], + }) + + def _record_feedback(self, frame: FrameType, value: Any) -> None: + """Records the feedback text of each reached criterion, when `frame` is `CriteriaGraph.resolve_feedback` + returning it for `CriteriaGraph.export_feedback`. + """ + exporter = frame.f_back + if frame.f_code.co_qualname != "CriteriaGraph.resolve_feedback" or exporter is None: + return + if exporter.f_code.co_qualname != "CriteriaGraph.export_feedback": + return + graph = frame.f_locals.get("self") + criterion = next((key for (key, item) in self.recording.criteria_graphs.items() if item is graph), None) + self.recording.events.append({ + "kind": "feedback", + "stage": self.stage, + "criterion": criterion, + "feedback": [[str(tag), str(text)] for (tag, text) in value], + }) + + def _record_template(self, frame: FrameType, value: Any, call: Call | None) -> None: + """Records which template built which criteria graph, when `frame` is a template `feedback_procedure_generator` + called to build the graph of one criterion. + """ + generator = frame.f_back + if generator is None or generator.f_code.co_qualname != "feedback_procedure_generator": + return + if not isinstance(value, CriteriaGraph): + return + # The key of `graph_templates` that chose this template, or None when the template is the context's default + label = None + for (key, template) in generator.f_globals.get("graph_templates", {}).items(): + if getattr(template, "__code__", None) is frame.f_code: + label = key.strip() + self.recording.events.append({ + "kind": "template", + "stage": self.stage, + "call": None if call is None else call.identifier, + "criterion": generator.f_locals["label"], + "template": frame.f_code.co_qualname, + "label": label, + "evaluations": len(value.evaluations), + }) + + def _record_return(self, frame: FrameType, value: Any) -> None: + call = self.call_frames.pop(frame, None) + given_response = frame in self.response_frames + self.response_frames.discard(frame) + if returned_normally(frame): + self._record_criteria(frame, value) + self._record_template(frame, value, call) + self._record_parameters(frame, value) + self._record_feedback(frame, value) + if call is not None: + raised = not returned_normally(frame) + event = { + "kind": "return", + "stage": self.stage, + "call": call.identifier, + "value": "" if raised else short_repr(value), + "raised": raised, + } + full = None if raised else full_repr(value) + if full is not None: + event["full"] = full + self.recording.events.append(event) + if raised: + self.raised = (frame.f_code, event) + if given_response and not raised and is_response_form(value) and repr(value) not in self.response_forms: + self.response_forms.add(repr(value)) + self.recording.events.append({ + "kind": "form", + "stage": self.stage, + "call": call.identifier, + "value": short_repr(value), + "type": type(value).__name__, + }) + evaluation = self.evaluation_frames.pop(frame, None) + if evaluation is not None: + (criterion, node) = evaluation + reached = list(value) if isinstance(value, dict) else [] + self.recording.events.append({ + "kind": "evaluated", "stage": self.stage, "criterion": criterion, "node": node, "reached": reached + }) def _caller(self, frame: FrameType) -> tuple[Call | None, int] | None: caller = frame.f_back @@ -112,6 +496,8 @@ def profile(self, frame: FrameType, event: str, arg: Any) -> None: if event == "return": if qualname == "feedback_procedure_generator" and module.startswith("app.context."): self.recording.criteria_graphs.update(arg) + if self.record_values: + self._record_return(frame, arg) self.frames.pop(frame, None) return if event != "call": @@ -119,6 +505,8 @@ def profile(self, frame: FrameType, event: str, arg: Any) -> None: if module == "app.evaluation" and qualname == "_evaluate": self.frames[frame] = (None, 0) return + if self.record_values: + self._record_evaluation(frame) if not module.startswith(("app.", "lambdafeedback.")): return caller = self._caller(frame) @@ -128,30 +516,50 @@ def profile(self, frame: FrameType, event: str, arg: Any) -> None: if qualname in HIDDEN_FUNCTIONS or any(part in qualname for part in HIDDEN_FUNCTION_PARTS): self.frames[frame] = caller return - if depth >= CALL_DEPTH: + if depth >= self.depth: self.frames[frame] = (parent, depth + 1) return key = (self.stage if parent is None else parent.identifier, module, qualname) call = self.recording.calls.get(key) if call is None: - call = Call(f"call_{len(self.recording.calls)}", module, qualname, self.stage, parent) + call = Call(f"call_{len(self.recording.calls)}", module, qualname, self.stage, parent, depth + 1) + if self.record_values: + call.doc = first_doc_paragraph(frame) + call.source = source_location(frame) self.recording.calls[key] = call else: call.count += 1 self.frames[frame] = (call, depth + 1) + if self.record_values: + self._record_call(frame, call) + +def record( + response: str, answer: str, params: dict[str, Any], depth: int = CALL_DEPTH, record_values: bool = False +) -> tuple[Recording, dict[str, Any]]: + """Runs `evaluation_function` on the example and returns what happened, and the result. -def record(response: str, answer: str, params: dict[str, Any]) -> tuple[Recording, dict[str, Any]]: - """Runs `evaluation_function` on the example and returns what happened, and the result.""" - recorder = Recorder() + See `Recorder` for `depth` and `record_values`. + """ + recorder = Recorder(depth, record_values, response) logger = logging.getLogger(evaluation.__name__) logger.addHandler(recorder) + monitoring = sys.monitoring + unwind = monitoring.events.PY_UNWIND + if record_values: + monitoring.use_tool_id(monitoring.PROFILER_ID, "generate_process_graphs") + monitoring.register_callback(monitoring.PROFILER_ID, unwind, recorder.unwind) + monitoring.set_events(monitoring.PROFILER_ID, unwind) sys.setprofile(recorder.profile) try: result = evaluation.evaluation_function(response, answer, dict(params), include_test_data=True) finally: sys.setprofile(None) logger.removeHandler(recorder) + if record_values: + monitoring.set_events(monitoring.PROFILER_ID, monitoring.events.NO_EVENTS) + monitoring.register_callback(monitoring.PROFILER_ID, unwind, None) + monitoring.free_tool_id(monitoring.PROFILER_ID) return (recorder.recording, result) @@ -225,8 +633,28 @@ def process_to_dot(recording: Recording, params: dict[str, Any], result: dict[st return dot -def calls_to_dot(recording: Recording, params: dict[str, Any], result: dict[str, Any]) -> pydot.Dot: - """The stages of the evaluation down the left, each with the functions called during it to its right.""" +def shown_parent(call: Call, hide_internals: bool) -> Call | None: + """The caller drawn for `call`: its parent, or with `hide_internals` its nearest caller that is not internal (see + `Call.internal`); None for a call made by `_evaluate`. + """ + parent = call.parent + while hide_internals and parent is not None and parent.internal: + parent = parent.parent + return parent + + +def calls_to_dot( + recording: Recording, + params: dict[str, Any], + result: dict[str, Any], + depth: int = CALL_DEPTH, + hide_internals: bool = False, +) -> pydot.Dot: + """The stages of the evaluation down the left, each with the functions called during it to its right, down to + `depth` calls below `_evaluate`. + + With `hide_internals`, dunder methods are left out, and the calls they made are drawn under their caller. + """ dot = new_drawing() dot.set("rankdir", "LR") pipeline = pydot.Subgraph("pipeline", rank="same") @@ -238,9 +666,12 @@ def calls_to_dot(recording: Recording, params: dict[str, Any], result: dict[str, clusters[stage] = cluster dot.add_subgraph(cluster) for call in recording.calls.values(): + if call.depth > depth or (hide_internals and call.internal): + continue label = call.qualname if call.count == 1 else f"{call.qualname} ×{call.count}" clusters[call.stage].add_node(pydot.Node(call.identifier, label=_node_label(label, call.module), **CALL_STYLE)) - caller = stages[call.stage] if call.parent is None else call.parent.identifier + parent = shown_parent(call, hide_internals) + caller = stages[call.stage] if parent is None else parent.identifier dot.add_edge(pydot.Edge(caller, call.identifier)) return dot