From 3fd788781df6db5f300feefd52adc506c5e49d05 Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Fri, 30 Oct 2026 09:59:22 +0000 Subject: [PATCH 1/4] Accept any test payload, test the answer, and add Greek aliases - Test.payload takes any type, as NUMBER tests in real exports store numbers. - Each response area's answer is written as a passing test, first, unless the answer is empty or a passing test already has it. - InputSymbol.greek builds the aliases of a Greek symbol from its unicode forms, with and without braces around a subscript. Co-Authored-By: Claude Opus 5.5 --- in2lambda/api/response_area.py | 41 +++++++++++++++++++++++++- in2lambda/json_convert/json_convert.py | 13 ++++++-- tests/test_response_area.py | 38 ++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 tests/test_response_area.py diff --git a/in2lambda/api/response_area.py b/in2lambda/api/response_area.py index 430dcac..117e530 100644 --- a/in2lambda/api/response_area.py +++ b/in2lambda/api/response_area.py @@ -1,6 +1,7 @@ """An area for student responses that will receive automated feedback.""" import uuid +from collections.abc import Sequence from dataclasses import dataclass, field from enum import Enum from typing import Any, ClassVar, Optional, Union @@ -13,6 +14,24 @@ def _new_id() -> str: return str(uuid.uuid4()) +_GREEK = { + "alpha": ["α", "𝛼"], + "beta": ["β", "𝛽"], + "gamma": ["γ", "𝛾"], + "delta": ["δ", "𝛿"], + "epsilon": ["ε", "ϵ", "𝜀"], + "theta": ["θ", "𝜃"], + "lambda": ["λ", "𝜆"], + "mu": ["μ", "𝜇"], + "nu": ["ν", "𝜈"], + "rho": ["ρ", "𝜌", "ϱ"], + "sigma": ["σ", "𝜎", "ς"], + "tau": ["τ", "𝜏"], + "phi": ["φ", "ϕ", "𝜙"], + "omega": ["ω", "𝜔"], +} + + @dataclass class InputSymbol: """Defines symbols and how to enter them into a response area. @@ -29,6 +48,26 @@ class InputSymbol: aliases: list[str] = field(default_factory=list) is_visible: bool = True + @classmethod + def greek(cls, symbol: str, code: str, extra: Sequence[str] = ()) -> "InputSymbol": + r"""A Greek symbol whose aliases are its unicode forms. + + ``code`` is a Greek letter name, optionally followed by ``_`` and a subscript. + + Examples: + >>> InputSymbol.greek("$\\rho$", "rho").aliases + ['ρ', '𝜌', 'ϱ'] + """ + name, _, sub = code.partition("_") + if name not in _GREEK: + raise ValueError(f"{name!r} is not a Greek letter name") + aliases = [] + # Never "sigmaf": the subscript could be read as a symbol of its own. + for base in [name, *_GREEK[name]]: + aliases += [f"{base}_{sub}", f"{base}_{{{sub}}}"] if sub else [base] + aliases += extra + return cls(symbol, code, [a for a in dict.fromkeys(aliases) if a != code]) + @dataclass class Test: @@ -37,7 +76,7 @@ class Test: # Its name would otherwise make pytest try to collect it wherever it is imported. __test__ = False - payload: str + payload: Any is_correct: bool id: str = field(default_factory=_new_id) diff --git a/in2lambda/json_convert/json_convert.py b/in2lambda/json_convert/json_convert.py index aa9548f..d581584 100644 --- a/in2lambda/json_convert/json_convert.py +++ b/in2lambda/json_convert/json_convert.py @@ -8,7 +8,7 @@ from typing import Any from in2lambda.api.part import Branch, Part -from in2lambda.api.response_area import Case, ResponseArea +from in2lambda.api.response_area import Case, ResponseArea, Test from in2lambda.api.set import Set @@ -43,6 +43,15 @@ def _case_to_json(case: Case) -> dict[str, Any]: return output +def _tests(area: ResponseArea) -> list[Test]: + """The area's tests, led by its answer passing unless a test already says so.""" + if area.answer in (None, "") or any( + t.payload == area.answer and t.is_correct for t in area.tests + ): + return area.tests + return [Test(area.answer, True), *area.tests] + + def _response_area_to_json(area: ResponseArea, order: int) -> dict[str, Any]: output = { "orderNumber": order, @@ -76,7 +85,7 @@ def _response_area_to_json(area: ResponseArea, order: int) -> dict[str, Any]: "payload": test.payload, "expectedResponse": {"isCorrect": test.is_correct}, } - for test in area.tests + for test in _tests(area) ], "cases": [_case_to_json(case) for case in area.cases], "response": { diff --git a/tests/test_response_area.py b/tests/test_response_area.py new file mode 100644 index 0000000..b086c3c --- /dev/null +++ b/tests/test_response_area.py @@ -0,0 +1,38 @@ +"""Response areas as in2lambda writes them.""" + +import pytest + +from in2lambda.api.response_area import InputSymbol, Number, NumericUnits, Test +from in2lambda.json_convert.json_convert import _response_area_to_json + + +def _tests(area) -> list: + return [ + (t["payload"], t["expectedResponse"]["isCorrect"]) + for t in _response_area_to_json(area, 0)["tests"] + ] + + +def test_a_test_payload_keeps_its_python_type() -> None: + assert _tests(Number(tests=[Test(2.5, True)])) == [(2.5, True)] + + +def test_the_answer_is_written_as_a_passing_test_once() -> None: + assert _tests(NumericUnits(answer="4 m/s")) == [("4 m/s", True)] + assert _tests(NumericUnits(answer="4 m/s", tests=[Test("4 m/s", True)])) == [ + ("4 m/s", True) + ] + + +def test_greek_aliases_cover_subscripts_and_unicode_forms() -> None: + assert InputSymbol.greek("$\\sigma_f$", "sigma_f").aliases == [ + "sigma_{f}", + "σ_f", + "σ_{f}", + "𝜎_f", + "𝜎_{f}", + "ς_f", + "ς_{f}", + ] + with pytest.raises(ValueError): + InputSymbol.greek("$x$", "x") From 9e665c2262993409e2b870feacc9f8c1b40ae6df Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Fri, 30 Oct 2026 10:04:01 +0000 Subject: [PATCH 2/4] Fail an empty answer's test, and refuse an unmarkable multiple choice - An empty-list answer is no answer, so it is written as a failing test. - A set with a multiple choice that marks no option correct raises ValueError before export, ahead of the output directory being cleared, naming the question and part. Co-Authored-By: Claude Opus 5.5 --- in2lambda/json_convert/json_convert.py | 29 ++++++++++++++++++++------ tests/test_response_area.py | 27 +++++++++++++++++++++++- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/in2lambda/json_convert/json_convert.py b/in2lambda/json_convert/json_convert.py index d581584..afc2e1c 100644 --- a/in2lambda/json_convert/json_convert.py +++ b/in2lambda/json_convert/json_convert.py @@ -8,7 +8,7 @@ from typing import Any from in2lambda.api.part import Branch, Part -from in2lambda.api.response_area import Case, ResponseArea, Test +from in2lambda.api.response_area import Case, MultipleChoice, ResponseArea, Test from in2lambda.api.set import Set @@ -44,12 +44,16 @@ def _case_to_json(case: Case) -> dict[str, Any]: def _tests(area: ResponseArea) -> list[Test]: - """The area's tests, led by its answer passing unless a test already says so.""" - if area.answer in (None, "") or any( - t.payload == area.answer and t.is_correct for t in area.tests - ): + """The area's tests, led by its answer passing unless a test already says so. + + An empty list is no answer, so it leads as a failing test instead. + """ + if area.answer in (None, ""): return area.tests - return [Test(area.answer, True), *area.tests] + passes = area.answer != [] + if any(t.payload == area.answer and t.is_correct == passes for t in area.tests): + return area.tests + return [Test(area.answer, passes), *area.tests] def _response_area_to_json(area: ResponseArea, order: int) -> dict[str, Any]: @@ -132,6 +136,18 @@ def _part_to_json(part: Part, order: int) -> dict[str, Any]: return output +def _check(set_questions: Set) -> None: + """Raises ValueError for a set Lambda Feedback could not mark.""" + for i, question in enumerate(set_questions.questions): + for j, part in enumerate(question.parts): + for area in part.response_areas: + if isinstance(area, MultipleChoice) and True not in area.answer: + raise ValueError( + f"Question {i + 1}, part {chr(ord('a') + j)}: " + "a multiple choice answer marks no option correct" + ) + + def converter(SetQuestions: Set, output_dir: str) -> None: """Turns a set of question objects into Lambda Feedback JSON. @@ -231,6 +247,7 @@ def main(set_questions: Set, output_dir: str) -> None: set_questions: A Set object containing questions. output_dir: Where to output the final Lambda Feedback JSON/ZIP files. """ + _check(set_questions) # check if directory exists in file if os.path.isdir(output_dir): try: diff --git a/tests/test_response_area.py b/tests/test_response_area.py index b086c3c..1c230e4 100644 --- a/tests/test_response_area.py +++ b/tests/test_response_area.py @@ -2,7 +2,16 @@ import pytest -from in2lambda.api.response_area import InputSymbol, Number, NumericUnits, Test +from in2lambda.api.part import Part +from in2lambda.api.question import Question +from in2lambda.api.response_area import ( + InputSymbol, + MultipleChoice, + Number, + NumericUnits, + Test, +) +from in2lambda.api.set import Set from in2lambda.json_convert.json_convert import _response_area_to_json @@ -24,6 +33,10 @@ def test_the_answer_is_written_as_a_passing_test_once() -> None: ] +def test_an_empty_list_answer_is_written_as_a_failing_test() -> None: + assert _tests(MultipleChoice(options=["A", "B"])) == [([], False)] + + def test_greek_aliases_cover_subscripts_and_unicode_forms() -> None: assert InputSymbol.greek("$\\sigma_f$", "sigma_f").aliases == [ "sigma_{f}", @@ -36,3 +49,15 @@ def test_greek_aliases_cover_subscripts_and_unicode_forms() -> None: ] with pytest.raises(ValueError): InputSymbol.greek("$x$", "x") + + +def test_a_set_with_no_correct_multiple_choice_option_is_not_written( + tmp_path, +) -> None: + area = MultipleChoice(options=["A", "B"], answer=[False, False]) + s = Set(questions=[Question(parts=[Part(response_areas=[area])])]) + earlier = tmp_path / "earlier.json" + earlier.write_text("{}") + with pytest.raises(ValueError, match="Question 1, part a"): + s.to_json(str(tmp_path)) + assert earlier.exists() From eed843ed94ac9d6181e8363ba21967262ead87a9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 8 Nov 2026 05:17:22 +0000 Subject: [PATCH 3/4] Move each response type's checks onto its class The converter called a MultipleChoice-specific check and so had to import that type. Each ResponseArea now declares payload_types, the types a test payload may take, and a _validate method that subclasses extend; the converter calls _validate on every area without naming any type. Number refuses bool payloads, and MultipleChoice refuses an answer or test whose length differs from its options. An empty answer now adds no test. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JnCY3kcqP1vQ7UixMtuRep --- in2lambda/api/response_area.py | 46 +++++++++++++++++++++++++- in2lambda/json_convert/json_convert.py | 23 ++++++------- tests/test_response_area.py | 40 ++++++++++++++++++++-- 3 files changed, 93 insertions(+), 16 deletions(-) diff --git a/in2lambda/api/response_area.py b/in2lambda/api/response_area.py index 117e530..297b1fa 100644 --- a/in2lambda/api/response_area.py +++ b/in2lambda/api/response_area.py @@ -71,7 +71,13 @@ def greek(cls, symbol: str, code: str, extra: Sequence[str] = ()) -> "InputSymbo @dataclass class Test: - """Teacher author's assertions of how responses should be evaluated.""" + """Teacher author's assertions of how responses should be evaluated. + + ``payload`` is a sample student response, so it takes the type of the area's + answer: a number for ``Number``, a list of booleans for ``MultipleChoice``, a + string for ``NumericUnits``. Each area checks its tests' payloads when the set + is written. + """ # Its name would otherwise make pytest try to collect it wherever it is imported. __test__ = False @@ -124,6 +130,8 @@ class ResponseArea: """ response_type: ClassVar[str] + payload_types: ClassVar[tuple[type, ...]] = (object,) + """The types a test payload, a sample response, may take.""" answer: Any = None evaluation_function: str = UNSET grade_params: Optional[dict[str, Any]] = None @@ -159,9 +167,20 @@ def _grade_params(self) -> Optional[dict[str, Any]]: """The parameters sent to the evaluation function.""" return self.grade_params + def _validate(self) -> None: + """Raises ValueError if Lambda Feedback could not mark the area. + + Subclasses extend this with the rules of their response type. + """ + for test in self.tests: + if not isinstance(test.payload, self.payload_types): + names = " or ".join(t.__name__ for t in self.payload_types) + raise ValueError(f"test payload {test.payload!r} is not {names}") + @dataclass(kw_only=True) class _Maths(ResponseArea): + payload_types: ClassVar[tuple[type, ...]] = (str,) answer: str = "" allow_handwrite: bool = UNSET allow_photo: bool = UNSET @@ -194,14 +213,23 @@ class Number(ResponseArea): """A number.""" response_type: ClassVar[str] = "NUMBER" + payload_types: ClassVar[tuple[type, ...]] = (int, float) answer: Optional[Union[int, float]] = None + def _validate(self) -> None: + super()._validate() + # bool is a subclass of int, but True is not a number response. + for test in self.tests: + if isinstance(test.payload, bool): + raise ValueError(f"test payload {test.payload!r} is not int or float") + @dataclass(kw_only=True) class NumericUnits(ResponseArea): """A number with units, e.g. ``30 N``.""" response_type: ClassVar[str] = "NUMERIC_UNITS" + payload_types: ClassVar[tuple[type, ...]] = (str,) answer: str = "" def _grade_params(self) -> Optional[dict[str, Any]]: @@ -214,6 +242,7 @@ class Text(ResponseArea): """A short piece of text.""" response_type: ClassVar[str] = "TEXT" + payload_types: ClassVar[tuple[type, ...]] = (str,) answer: str = "" @@ -222,6 +251,7 @@ class Boolean(ResponseArea): """True or false.""" response_type: ClassVar[str] = "BOOLEAN" + payload_types: ClassVar[tuple[type, ...]] = (bool,) answer: bool = False @@ -235,6 +265,8 @@ class MultipleChoice(ResponseArea): single: bool = UNSET randomise: bool = UNSET + payload_types: ClassVar[tuple[type, ...]] = (list,) + def _config(self) -> Optional[dict[str, Any]]: return { "options": self.options, @@ -242,12 +274,22 @@ def _config(self) -> Optional[dict[str, Any]]: "single": self.single, } + def _validate(self) -> None: + super()._validate() + if True not in self.answer: + raise ValueError("a multiple choice answer marks no option correct") + n = len(self.options) + for marks in [self.answer, *(t.payload for t in self.tests)]: + if len(marks) != n or not all(isinstance(m, bool) for m in marks): + raise ValueError(f"{marks!r} is not {n} booleans, one per option") + @dataclass(kw_only=True) class Table(ResponseArea): """A grid of cells, one row per row name and one column per column name.""" response_type: ClassVar[str] = "TABLE" + payload_types: ClassVar[tuple[type, ...]] = (list,) answer: list[list[str]] = field(default_factory=list) row_names: list[str] = field(default_factory=list) col_names: list[str] = field(default_factory=list) @@ -266,6 +308,7 @@ class Matrix(ResponseArea): """A table, with parentheses displayed around it, with the shape of ``answer``.""" response_type: ClassVar[str] = "MATRIX" + payload_types: ClassVar[tuple[type, ...]] = (list,) answer: list[list[str]] = field(default_factory=list) def _config(self) -> Optional[dict[str, Any]]: @@ -280,6 +323,7 @@ class Code(ResponseArea): """A monaco code editor.""" response_type: ClassVar[str] = "CODE" + payload_types: ClassVar[tuple[type, ...]] = (str,) answer: str = "" language: str = UNSET diff --git a/in2lambda/json_convert/json_convert.py b/in2lambda/json_convert/json_convert.py index afc2e1c..62f3ae9 100644 --- a/in2lambda/json_convert/json_convert.py +++ b/in2lambda/json_convert/json_convert.py @@ -8,7 +8,7 @@ from typing import Any from in2lambda.api.part import Branch, Part -from in2lambda.api.response_area import Case, MultipleChoice, ResponseArea, Test +from in2lambda.api.response_area import Case, ResponseArea, Test from in2lambda.api.set import Set @@ -44,16 +44,12 @@ def _case_to_json(case: Case) -> dict[str, Any]: def _tests(area: ResponseArea) -> list[Test]: - """The area's tests, led by its answer passing unless a test already says so. - - An empty list is no answer, so it leads as a failing test instead. - """ - if area.answer in (None, ""): + """The area's tests, led by its answer passing unless a test already says so.""" + if area.answer is None or area.answer == "" or area.answer == []: return area.tests - passes = area.answer != [] - if any(t.payload == area.answer and t.is_correct == passes for t in area.tests): + if any(t.payload == area.answer and t.is_correct for t in area.tests): return area.tests - return [Test(area.answer, passes), *area.tests] + return [Test(area.answer, True), *area.tests] def _response_area_to_json(area: ResponseArea, order: int) -> dict[str, Any]: @@ -141,11 +137,12 @@ def _check(set_questions: Set) -> None: for i, question in enumerate(set_questions.questions): for j, part in enumerate(question.parts): for area in part.response_areas: - if isinstance(area, MultipleChoice) and True not in area.answer: + try: + area._validate() + except ValueError as error: raise ValueError( - f"Question {i + 1}, part {chr(ord('a') + j)}: " - "a multiple choice answer marks no option correct" - ) + f"Question {i + 1}, part {chr(ord('a') + j)}: {error}" + ) from None def converter(SetQuestions: Set, output_dir: str) -> None: diff --git a/tests/test_response_area.py b/tests/test_response_area.py index 1c230e4..ad2f775 100644 --- a/tests/test_response_area.py +++ b/tests/test_response_area.py @@ -33,8 +33,44 @@ def test_the_answer_is_written_as_a_passing_test_once() -> None: ] -def test_an_empty_list_answer_is_written_as_a_failing_test() -> None: - assert _tests(MultipleChoice(options=["A", "B"])) == [([], False)] +def test_an_empty_answer_adds_no_test() -> None: + assert _tests(MultipleChoice(options=["A", "B"])) == [] + + +def _write(tmp_path, area) -> None: + Set(questions=[Question(parts=[Part(response_areas=[area])])]).to_json( + str(tmp_path) + ) + + +@pytest.mark.parametrize( + "area", + [ + Number(answer=2.5, tests=[Test("2.5", True)]), + Number(answer=1, tests=[Test(True, True)]), + NumericUnits(answer="4 m/s", tests=[Test(4, True)]), + MultipleChoice( + options=["A", "B"], answer=[True, False], tests=[Test([True], True)] + ), + MultipleChoice(options=["A", "B"], answer=[True]), + MultipleChoice( + options=["A", "B"], answer=[True, False], tests=[Test("A", True)] + ), + ], +) +def test_a_payload_of_the_wrong_type_or_shape_is_refused(tmp_path, area) -> None: + with pytest.raises(ValueError, match="Question 1, part a"): + _write(tmp_path, area) + + +def test_payloads_of_the_answer_type_are_written(tmp_path) -> None: + _write(tmp_path, Number(answer=2.5, tests=[Test(3, False)])) + _write( + tmp_path, + MultipleChoice( + options=["A", "B"], answer=[True, False], tests=[Test([False, True], False)] + ), + ) def test_greek_aliases_cover_subscripts_and_unicode_forms() -> None: From f94af4592dfcd2deef6dc4d657923c032ccc7dd6 Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Fri, 30 Oct 2026 10:22:04 +0000 Subject: [PATCH 4/4] Write a Boolean answer as 1 or 0, compared as a string Lambda Feedback's own teacher test of a Boolean area sends isExactEqual answer 1, submission 1 and gradeParams {type: "str"}. in2lambda wrote the answer as true and no gradeParams, so a correct response of 1 never matched. The Python API keeps bool; the JSON now holds 1 or 0, for the answer and test payloads, with type "str" unless the author sets it. Co-Authored-By: Claude Opus 5.5 --- in2lambda/api/response_area.py | 12 ++++++++++++ in2lambda/json_convert/json_convert.py | 4 ++-- tests/test_boolean.py | 10 ++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 tests/test_boolean.py diff --git a/in2lambda/api/response_area.py b/in2lambda/api/response_area.py index 297b1fa..bcefa69 100644 --- a/in2lambda/api/response_area.py +++ b/in2lambda/api/response_area.py @@ -177,6 +177,10 @@ def _validate(self) -> None: names = " or ".join(t.__name__ for t in self.payload_types) raise ValueError(f"test payload {test.payload!r} is not {names}") + def _serial(self, value: Any) -> Any: + """An answer or test payload as Lambda Feedback stores it.""" + return value + @dataclass(kw_only=True) class _Maths(ResponseArea): @@ -254,6 +258,14 @@ class Boolean(ResponseArea): payload_types: ClassVar[tuple[type, ...]] = (bool,) answer: bool = False + # Lambda Feedback stores the answer, and sends the response, as 1 or 0, and + # isExactEqual compares them as strings. + def _serial(self, value: Any) -> Any: + return int(value) if isinstance(value, bool) else value + + def _grade_params(self) -> Optional[dict[str, Any]]: + return {"type": "str", **(self.grade_params or {})} + @dataclass(kw_only=True) class MultipleChoice(ResponseArea): diff --git a/in2lambda/json_convert/json_convert.py b/in2lambda/json_convert/json_convert.py index 62f3ae9..ef45319 100644 --- a/in2lambda/json_convert/json_convert.py +++ b/in2lambda/json_convert/json_convert.py @@ -82,7 +82,7 @@ def _response_area_to_json(area: ResponseArea, order: int) -> dict[str, Any]: "tests": [ { "id": test.id, - "payload": test.payload, + "payload": area._serial(test.payload), "expectedResponse": {"isCorrect": test.is_correct}, } for test in _tests(area) @@ -91,7 +91,7 @@ def _response_area_to_json(area: ResponseArea, order: int) -> dict[str, Any]: "response": { "responseInput": { "responseType": area.response_type, - "answer": area.answer, + "answer": area._serial(area.answer), "config": area._config(), } }, diff --git a/tests/test_boolean.py b/tests/test_boolean.py new file mode 100644 index 0000000..51a8560 --- /dev/null +++ b/tests/test_boolean.py @@ -0,0 +1,10 @@ +"""A Boolean area is written as Lambda Feedback's own teacher test sends it.""" + +from in2lambda.api.response_area import Boolean +from in2lambda.json_convert.json_convert import _response_area_to_json + + +def test_a_boolean_answer_is_written_as_a_number_compared_as_a_string() -> None: + area = _response_area_to_json(Boolean(answer=True), 0) + assert area["response"]["responseInput"]["answer"] == 1 + assert area["gradeParams"] == {"type": "str"}