From 13dc5037fc11ff716ee2a9c86f588ed090628814 Mon Sep 17 00:00:00 2001 From: "jiaxin.loh" Date: Mon, 9 Nov 2026 12:06:43 +0800 Subject: [PATCH 1/3] feat: add structured JSON verification output Expose grading results for desktop integrations without parsing terminal text while preserving existing verification behavior. --- README.md | 22 ++ app/cli.py | 22 +- app/commands/verify.py | 33 ++- app/utils/click.py | 48 ++++- docs/verify-json-design.md | 100 +++++++++ tests/test_verify_json.py | 418 +++++++++++++++++++++++++++++++++++++ 6 files changed, 639 insertions(+), 4 deletions(-) create mode 100644 docs/verify-json-design.md create mode 100644 tests/test_verify_json.py diff --git a/README.md b/README.md index 2296161..9bf9574 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,28 @@ Refer to the [installation guide](https://git-mastery.org/companion-app/index.ht To develop the app locally, refer to the [development setup guide](https://git-mastery.org/developers/docs/getting-started/setup/) for detailed instructions on how to set up your development environment and contribute to the project. +## JSON verification output + +Run `gitmastery verify --json` inside an exercise to get its grading result on +stdout: + +```json +{"exercise_name": "under-control", "status": "Completed", "comments": []} +``` + +`status` is `Completed`, `Incomplete`, or `Error`. `comments` contains all grader +feedback in order, or an empty array when there is none. Diagnostic messages go +to stderr, so clients can parse the complete stdout as JSON. + +Verification still saves progress and performs any configured remote submission. +Exit `0` means the command finished normally; read `status` to determine whether +the answer is correct. A failure before grading leaves stdout empty and exits +nonzero. A submission failure after grading retains the JSON result and exits +nonzero. Ordinary `gitmastery verify` keeps its existing text output. + +See the [technical design](docs/verify-json-design.md) for the contract and desktop +integration plan. + ## Publishing Tag the PR with the `bump:{major | minor | patch}` label and the CI will automatically perform the publish. diff --git a/app/cli.py b/app/cli.py index 4b10816..753d55f 100644 --- a/app/cli.py +++ b/app/cli.py @@ -8,7 +8,7 @@ from app.commands import check, download, progress, setup, verify from app.commands.repl import repl from app.commands.version import version -from app.utils.click import ClickColor, CliContextKey, warn +from app.utils.click import ClickColor, CliContextKey, json_output_scope, warn from app.utils.version import Version, fetch_latest_release_version from app.version import __version__ @@ -19,6 +19,26 @@ def invoke(self, ctx: click.Context) -> None: logger.info("Running command %s with arguments %s", ctx.command_path, sys.argv) return super().invoke(ctx) + def resolve_command( + self, ctx: click.Context, args: list[str] + ) -> tuple[str | None, click.Command | None, list[str]]: + command_name, command, command_args = super().resolve_command(ctx, args) + if command is verify: + # Resolve aliases and parse options before the group callback prints + # version warnings. Resilient parsing leaves help/errors to Click. + with command.make_context( + command_name or "verify", + list(command_args), + parent=ctx, + resilient_parsing=True, + ) as command_ctx: + requesting_help = any( + arg in command_ctx.help_option_names for arg in command_args + ) + if command_ctx.params.get("json_output") and not requesting_help: + ctx.with_resource(json_output_scope()) + return command_name, command, command_args + CONTEXT_SETTINGS = {"max_content_width": 120} diff --git a/app/commands/verify.py b/app/commands/verify.py index c18c4e7..f33ebec 100644 --- a/app/commands/verify.py +++ b/app/commands/verify.py @@ -22,10 +22,12 @@ from app.utils.click import ( ClickColor, error, + get_json_output_stream, info, must_get_exercise_root_config, must_get_gitmastery_root_config, warn, + with_json_output, ) from app.utils.git import add_all, commit, push from app.utils.github_cli import get_prs, get_username, pull_request @@ -69,6 +71,22 @@ def _print_output(output: GitAutograderOutput) -> None: ) +def _print_json_output(output: GitAutograderOutput, exercise_name: str) -> None: + statuses = { + GitAutograderStatus.SUCCESSFUL: "Completed", + GitAutograderStatus.UNSUCCESSFUL: "Incomplete", + GitAutograderStatus.ERROR: "Error", + } + payload = { + "exercise_name": exercise_name, + "status": statuses[output.status], + "comments": output.comments or [], + } + stdout = get_json_output_stream() + click.echo(json.dumps(payload), file=stdout, color=False) + stdout.flush() + + def _submit_progress(output: GitAutograderOutput) -> None: username = get_username() @@ -183,9 +201,17 @@ def _execute_verify( @click.command() +@click.option( + "--json", + "json_output", + is_flag=True, + default=False, + help="Output the grading result as JSON; send diagnostics to stderr.", +) +@with_json_output @in_exercise_root() @in_gitmastery_root() -def verify() -> None: +def verify(json_output: bool = False) -> None: """ Verifies the state of the exercise attempt. """ @@ -204,5 +230,8 @@ def verify() -> None: output = _execute_verify( exercise_path, exercise_name, formatted_exercise_name, started_at ) - _print_output(output) + if json_output: + _print_json_output(output, exercise_name) + else: + _print_output(output) _submit_progress(output) diff --git a/app/utils/click.py b/app/utils/click.py index 88a4c51..d13f90c 100644 --- a/app/utils/click.py +++ b/app/utils/click.py @@ -1,7 +1,11 @@ import logging import sys +from collections.abc import Callable, Iterator +from contextlib import contextmanager, redirect_stdout +from contextvars import ContextVar from enum import StrEnum -from typing import Any, NoReturn, Optional +from functools import wraps +from typing import Any, NoReturn, Optional, ParamSpec, TextIO, TypeVar import click @@ -10,6 +14,48 @@ logger = logging.getLogger(__name__) +_JSON_STDOUT: ContextVar[Optional[TextIO]] = ContextVar("json_stdout", default=None) +P = ParamSpec("P") +T = TypeVar("T") + + +@contextmanager +def json_output_scope(enabled: bool = True) -> Iterator[TextIO]: + """Reserve stdout for JSON and route Python diagnostics to stderr. + + :param enabled: Whether to enable JSON output routing. + :yield: The original stdout stream for the JSON result. + """ + saved_stdout = _JSON_STDOUT.get() + if not enabled or saved_stdout is not None: + yield saved_stdout if saved_stdout is not None else sys.stdout + return + + saved_stdout = sys.stdout + token = _JSON_STDOUT.set(saved_stdout) + try: + with redirect_stdout(sys.stderr): + yield saved_stdout + finally: + _JSON_STDOUT.reset(token) + + +def with_json_output(func: Callable[P, T]) -> Callable[P, T]: + """Route diagnostics before command guards when ``json_output`` is enabled.""" + + @wraps(func) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> T: + with json_output_scope(bool(kwargs.get("json_output", False))): + return func(*args, **kwargs) + + return wrapper + + +def get_json_output_stream() -> TextIO: + """Return the stdout stream reserved for the JSON result.""" + saved_stdout = _JSON_STDOUT.get() + return saved_stdout if saved_stdout is not None else sys.stdout + class CliContextKey(StrEnum): GITMASTERY_ROOT_CONFIG = "GITMASTERY_ROOT_CONFIG" diff --git a/docs/verify-json-design.md b/docs/verify-json-design.md new file mode 100644 index 0000000..4b19f32 --- /dev/null +++ b/docs/verify-json-design.md @@ -0,0 +1,100 @@ +# Design: JSON output for `gitmastery verify` + +The CLI option is implemented. Desktop adoption described below remains a separate integration step. + +## 1. Goals + +Adding `gitmastery verify --json` is feasible and reasonable. The CLI already receives a structured `GitAutograderOutput`, and [`verify.py`](../app/commands/verify.py) renders its status and comments separately from grading. No grader or dependency change is needed. + +- Give desktop clients a stable subprocess interface for the current attempt's grading result. +- Preserve every feedback comment without depending on terminal formatting or stream chunk boundaries. +- Keep ordinary `gitmastery verify` output, grading, progress persistence, remote submission, and exit codes backward compatible. + +The desktop currently spawns `["verify"]` in [`runVerify`](../../git-mastery-desktop/src/electron/ipc/gitmastery.ts). `_checkCorrectSolution` and `_checkIncorrectSolution` search for `INFO Status:` strings. `_getComments` assumes the header and first comment arrive in the same chunk and extracts only that comment. Node stream chunks do not correspond to output lines, so even unchanged CLI wording can break this parser. + +This is a structured output option for an existing command. It does not introduce a service, change grading rules, or make verification free of side effects. + +## 2. JSON structure + +When a grading result exists, stdout contains exactly one JSON object followed by a newline: + +```json +{ + "exercise_name": "under-control", + "status": "Completed", + "comments": [] +} +``` + +This illustrates the contract for a completed attempt with no feedback. + +| Field | Type | Meaning | +| --- | --- | --- | +| `exercise_name` | string | Exercise identifier from the resolved exercise configuration; lets a client match the result to its request. | +| `status` | string enum | `Completed`, `Incomplete`, or `Error`. | +| `comments` | string[] | All grader comments in their original order; normalize absent comments to `[]`. | + +Use the existing mapping: `SUCCESSFUL` → `Completed`, `UNSUCCESSFUL` → `Incomplete`, `ERROR` → `Error`. These become fixed contract values, independent of future display wording. Preserve comment content, including newlines and Unicode, through JSON escaping. Do not insert the display placeholder `"No comments"`. + +These three fields cover the desktop's grading needs. Timestamps, numeric scores, redundant `correct`/`incorrect` booleans, and a schema version are unnecessary for this first contract. Consumers should ignore additional fields; changing existing field types or status values would be a breaking change. + +**Streams and exit codes:** + +- JSON mode sends diagnostic text to stderr, including version warnings, repository fetching, grader prints, verbose subprocess output, and progress-saving messages. Stdout has no prefixes, ANSI escapes, or progress text. +- Preserve current exit behavior: a returned grading result can have any of the three statuses and still exit `0` when the command finishes normally. Exercise correctness comes from `status`, not the process exit code. +- Emit the result at the same point as today's `_print_output`, before `_submit_progress`. If progress submission subsequently fails, retain the grading JSON and the existing nonzero exit behavior. The result describes grading, not successful persistence. +- Failures before a grading result exists, such as missing exercise configuration or failure entering `ExercisesRepo`, produce empty stdout, diagnostics on stderr, and a nonzero exit. Do not fabricate a grading result. Usage errors and `--help` retain Click's normal behavior. + +## 3. Implementation detail + +### CLI implementation + +| File | Change | +| --- | --- | +| [`app/commands/verify.py`](../app/commands/verify.py) | Add `@click.option("--json", "json_output", is_flag=True, default=False, ...)` and a typed `json_output: bool` callback parameter. Branch only the result renderer: serialize an explicit three-field dictionary or call the existing `_print_output`. Keep `_execute_verify` and `_submit_progress` behavior and ordering. Install the output scope outside the exercise/root guard decorators so their diagnostics are covered. | +| [`app/utils/click.py`](../app/utils/click.py) | Add a reusable, scoped JSON output context: retain the original stdout stream for the JSON writer and redirect ordinary Python stdout to stderr using `contextlib.redirect_stdout`. Nested use should reuse the saved stream, and `finally` must restore it. This covers existing `info`/`warn`/`error`, `click.echo`, and `print` calls without changing their default behavior. | +| [`app/cli.py`](../app/cli.py) | Extend `LoggingGroup.resolve_command` to identify JSON verification before the group callback runs and enter the same output scope around invocation. Resolve the command, including `v`, and inspect its options through Click rather than searching `sys.argv` substrings. This is necessary because `cli()` prints version warnings before `verify()` runs. Help and usage handling remain normal. | +| [`tests/test_verify_json.py`](../tests/test_verify_json.py) | Add deterministic contract tests using `CliRunner`, temporary configurations, and mocked grading/version/GitHub operations, plus a binary smoke test using the existing runner and a local verifier. Cover legacy text output and unchanged progress records in isolated tests. Run this contract suite separately from the existing happy-path E2E workflow. | +| [`README.md`](../README.md) | Document `verify --json`, the three fields, separate streams, exit semantics, and continued progress submission; link this design. | + +The output context also wraps direct invocation of the verify command, which the REPL uses without `LoggingGroup`. The standalone subprocess contract does not include REPL banners or prompts. Avoid retaining JSON mode across subsequent commands. + +Use the standard `json` module and explicitly write and flush the payload to the saved stdout stream. Its default Unicode escaping produces ASCII JSON that round-trips on Windows, Linux, and macOS without relying on the terminal encoding. Do not serialize the entire dependency dataclass, which would expose unnecessary fields and datetime serialization details. + +[`app/utils/command.py`](../app/utils/command.py) already captures subprocess streams; its verbose Python prints are covered by the scope. Any verifier subprocess that inherits stdout directly must explicitly send diagnostic output to stderr: `redirect_stdout` redirects Python writes, not OS file descriptor 1. + +The existing guard decorators, progress-file format, Git/GitHub helpers, packaging, and CI configuration need no edits. + +### Desktop adoption in the sibling repository + +| File | Proposed change | +| --- | --- | +| [`src/electron/ipc/gitmastery.ts`](../../git-mastery-desktop/src/electron/ipc/gitmastery.ts) | Request `["verify", "--json"]` when supported. Collect raw stdout without `[[terminal-line]]`, decode the complete buffer and parse once on process `close`. Keep stderr streaming as diagnostics; a stderr chunk alone must not determine failure. Render the parsed status and every comment into terminal feedback. | +| `src/electron/utils/cli/verification.ts` (new) | Isolate JSON validation and capability detection. Require the three fields, a recognized status, string comments, and the expected exercise identifier. Adapt to existing IPC data: `correct = status === "Completed"`, `incorrect = status !== "Completed"`, and `comments = comments.join("\n")`. | +| `tests/electron/verification.test.ts` (new) | Test validation, legacy capability selection, and mapping to existing desktop feedback. | +| [`tests/e2e/cli.mjs`](../../git-mastery-desktop/tests/e2e/cli.mjs), [`tests/e2e/happy-paths.e2e.ts`](../../git-mastery-desktop/tests/e2e/happy-paths.e2e.ts) | Update fixture commands and invocation expectations for capability probing and JSON verification; exercise split output chunks and preserved completion behavior. | +| [`docs/architecture/cli-output-in-terminal.md`](../../git-mastery-desktop/docs/architecture/cli-output-in-terminal.md), [`docs/development/happy-path-e2e-tests.md`](../../git-mastery-desktop/docs/development/happy-path-e2e-tests.md) | Replace descriptions of grading text parsing with the structured contract and updated fixture behavior. | + +Keep the existing process-success gate and renderer IPC shape. A nonzero exit remains a task failure, even when grading JSON is available; retain that JSON for feedback. Malformed or mismatched JSON on exit `0` is also a task failure and must not patch exercise progress. The existing [`ActivityProvider`](../../git-mastery-desktop/src/ui/providers/ActivityProvider.tsx) and [`types.d.ts`](../../git-mastery-desktop/types.d.ts) can continue consuming the adapted fields without changes. + +Release the CLI option first. Because the desktop resolves a separately installed binary, probe `verify --help` for the advertised `--json` option and cache support until the executable changes. Use the existing text path only for CLIs that lack the option. A failed probe should report a CLI problem. Never retry grading after malformed JSON or a submission failure: verification saves progress and could otherwise run twice. + +## 4. Test cases + +| Case | Expected result | +| --- | --- | +| Ordinary `verify`, including each grading status | Existing text, comment formatting, exit behavior, and progress submission remain unchanged. | +| JSON for each grading status | The entire stdout parses as one object with the specified mapping; normal completion exits `0`. | +| No comments; multiple comments; quotes, newlines, Unicode | `[]` for absent feedback; all other strings round-trip in order without display bullets or placeholders. | +| Version warning, repository message, grader `print`, migration warning, `--verbose` | Diagnostics appear on stderr; stdout remains valid JSON. Test the full group invocation, not just the verify callback. | +| Missing exercise/root; repository initialization failure | Empty stdout, stderr diagnostic, nonzero exit, and no fabricated grade. | +| Wrong-answer, invalid-state, unexpected exception inside grading | Existing `_execute_verify` conversion yields `Incomplete` or `Error` with comments in JSON. | +| Progress save/sync succeeds, warns, or fails after grading | Submission is attempted once with the same output; JSON remains intact. Warnings preserve existing exit behavior; fatal failures remain nonzero. | +| Alias `v --json`, help, invalid arguments, sequential invocations | Alias uses the contract; Click help/errors remain normal; later text commands retain their original stdout behavior. | +| Desktop receives JSON in arbitrary chunks; stderr warning; malformed/unknown/mismatched result | Parse only after complete stdout; valid results map correctly. Invalid results never mark completion or trigger another verify attempt. | +| Desktop uses older CLI; capability probe fails; executable changes | Older CLI uses the legacy path; probe failure is reported; changed executable is probed again. | +| Cross-platform binary smoke test | Separate stdout/stderr, JSON parsing, and unchanged progress persistence pass on the existing Windows, Linux, and macOS matrix. Desktop retains its existing Windows cwd gate. | + +Use isolated progress directories for new cases: the current `verified_exercise_dir` fixture is session-scoped, and its legacy test expects exactly one progress entry. Mock remote operations in contract tests so they do not submit real progress. + +Local validation passed: 25 CLI tests, including the packaged executable smoke test, plus Ruff and mypy checks for changed Python files. Run this contract suite on Windows and Linux as well; the existing CI matrix covers happy-path E2E tests. Desktop cases apply when that integration is implemented. diff --git a/tests/test_verify_json.py b/tests/test_verify_json.py new file mode 100644 index 0000000..0750b12 --- /dev/null +++ b/tests/test_verify_json.py @@ -0,0 +1,418 @@ +import importlib +import json +import sys +from datetime import datetime, timezone +from pathlib import Path +from subprocess import CompletedProcess +from unittest.mock import Mock + +import pytest +from click.testing import CliRunner +from git import Actor, Repo +from git_autograder import ( + GitAutograderInvalidStateException, + GitAutograderStatus, + GitAutograderWrongAnswerException, +) +from git_autograder.output import GitAutograderOutput + +from app.commands.verify import _execute_verify, verify +from app.utils.click import error, info +from app.utils.gitmastery import EXERCISE_UTILS_FILES + +from e2e.constants import EXERCISE_NAME +from e2e.runner import BinaryRunner + +cli_module = importlib.import_module("app.cli") +verify_module = importlib.import_module("app.commands.verify") + + +@pytest.fixture +def runner() -> BinaryRunner: + return BinaryRunner.from_env() + + +@pytest.fixture +def exercise_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + metadata = tmp_path / ".gitmastery" + (metadata / "progress").mkdir(parents=True) + (metadata / "config.json").write_text("{}", encoding="utf-8") + exercise = tmp_path / EXERCISE_NAME + exercise.mkdir() + (exercise / ".gitmastery-exercise.json").write_text( + json.dumps( + { + "exercise_name": EXERCISE_NAME, + "tags": [], + "requires_git": False, + "requires_github": False, + "base_files": {}, + "downloaded_at": None, + "exercise_repo": { + "repo_type": "ignore", + "repo_name": "unused", + "repo_title": None, + "create_fork": False, + "init": False, + }, + } + ), + encoding="utf-8", + ) + monkeypatch.chdir(exercise) + return exercise + + +@pytest.fixture +def graded_output() -> GitAutograderOutput: + now = datetime.now(timezone.utc) + return GitAutograderOutput( + status=GitAutograderStatus.SUCCESSFUL, + exercise_name=EXERCISE_NAME, + started_at=now, + completed_at=now, + comments=["First comment", "Second comment"], + ) + + +@pytest.fixture +def cli_runner( + exercise_dir: Path, + graded_output: GitAutograderOutput, + monkeypatch: pytest.MonkeyPatch, +) -> CliRunner: + monkeypatch.setattr(cli_module.cli, "commands", {}) + monkeypatch.setattr(cli_module.cli, "_aliases", {}) + monkeypatch.setattr(cli_module.cli, "_commands", {}) + cli_module.cli.add_command(verify, aliases=["v"]) + monkeypatch.setattr( + cli_module, + "fetch_latest_release_version", + lambda: (None, "Version check unavailable"), + ) + monkeypatch.setattr(verify_module, "get_username", lambda: "test-user") + + def grade( + exercise_path: Path, + exercise_name: str, + formatted_exercise_name: str, + started_at: datetime, + ) -> GitAutograderOutput: + info("Fetching exercise information") + print("Grader diagnostics") + return graded_output + + monkeypatch.setattr(verify_module, "_execute_verify", grade) + return CliRunner() + + +@pytest.mark.parametrize( + ("status", "text"), + [ + (GitAutograderStatus.SUCCESSFUL, "Completed"), + (GitAutograderStatus.UNSUCCESSFUL, "Incomplete"), + (GitAutograderStatus.ERROR, "Error"), + ], +) +@pytest.mark.parametrize("json_output", [False, True]) +def test_verify_status_and_progress( + cli_runner: CliRunner, + graded_output: GitAutograderOutput, + exercise_dir: Path, + status: GitAutograderStatus, + text: str, + json_output: bool, +) -> None: + graded_output.status = status + result = cli_runner.invoke( + cli_module.cli, ["verify", "--json"] if json_output else ["verify"] + ) + assert result.exit_code == 0, result.output + if json_output: + assert json.loads(result.stdout) == { + "exercise_name": EXERCISE_NAME, + "status": text, + "comments": graded_output.comments, + } + assert result.stdout.count("\n") == 1 + assert "Version check unavailable" in result.stderr + assert "Grader diagnostics" in result.stderr + assert "Saving progress of attempt" in result.stderr + else: + assert f" INFO Status: {text}" in result.stdout + assert ( + " INFO Comments:\n\t- First comment\n\t- Second comment\n" in result.stdout + ) + assert "Version check unavailable" in result.stdout + assert "Grader diagnostics" in result.stdout + assert result.stderr == "" + + entries = json.loads( + (exercise_dir.parent / ".gitmastery" / "progress" / "progress.json").read_text() + ) + assert graded_output.started_at is not None + assert graded_output.completed_at is not None + assert entries == [ + { + "exercise_name": EXERCISE_NAME, + "status": text, + "comments": graded_output.comments, + "started_at": graded_output.started_at.timestamp(), + "completed_at": graded_output.completed_at.timestamp(), + } + ] + + +@pytest.mark.parametrize( + "comments", [None, [], ['Quotes: "hello"', "Two\nlines", "こんにちは 😀"]] +) +def test_json_feedback_round_trips( + cli_runner: CliRunner, + graded_output: GitAutograderOutput, + comments: list[str] | None, +) -> None: + graded_output.comments = comments + graded_output.exercise_name = None + result = cli_runner.invoke(cli_module.cli, ["verify", "--json"], color=True) + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["exercise_name"] == EXERCISE_NAME + assert payload["comments"] == (comments or []) + assert result.stdout.isascii() + assert "\x1b" not in result.stdout + + +def test_verbose_and_alias( + cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + verify_module, + "get_username", + importlib.import_module("app.utils.github_cli").get_username, + ) + monkeypatch.setattr( + "subprocess.run", + lambda *args, **kwargs: CompletedProcess(args[0], 0, "test-user\n", ""), + ) + result = cli_runner.invoke(cli_module.cli, ["--verbose", "v", "--json"]) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout)["status"] == "Completed" + assert "\ttest-user" in result.stderr + + +@pytest.mark.parametrize("missing", ["exercise", "root"]) +def test_missing_configuration( + cli_runner: CliRunner, exercise_dir: Path, missing: str +) -> None: + config = ( + exercise_dir / ".gitmastery-exercise.json" + if missing == "exercise" + else exercise_dir.parent / ".gitmastery" / "config.json" + ) + config.unlink() + result = cli_runner.invoke(cli_module.cli, ["verify", "--json"]) + assert result.exit_code == 1 + assert result.stdout == "" + assert " ERROR " in result.stderr + + +def test_repository_failure_before_grading( + cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch +) -> None: + repo = Mock() + repo.__enter__ = Mock(side_effect=RuntimeError("Repository unavailable")) + repo.__exit__ = Mock(return_value=False) + monkeypatch.setattr(verify_module, "ExercisesRepo", lambda: repo) + monkeypatch.setattr(verify_module, "_execute_verify", _execute_verify) + result = cli_runner.invoke(cli_module.cli, ["verify", "--json"]) + assert result.exit_code != 0 + assert result.stdout == "" + assert isinstance(result.exception, RuntimeError) + + +@pytest.mark.parametrize( + ("exception", "status", "comments"), + [ + ( + GitAutograderWrongAnswerException(["Wrong answer", "Try again"]), + "Incomplete", + ["Wrong answer", "Try again"], + ), + ( + GitAutograderInvalidStateException("Invalid state"), + "Error", + ["Invalid state"], + ), + ( + RuntimeError("Unexpected grading failure"), + "Error", + ["Unexpected grading failure"], + ), + ], +) +def test_grading_exceptions( + cli_runner: CliRunner, + monkeypatch: pytest.MonkeyPatch, + exception: Exception, + status: str, + comments: list[str], +) -> None: + repo = Mock() + repo.__enter__ = Mock(return_value=repo) + repo.__exit__ = Mock(return_value=False) + monkeypatch.setattr(verify_module, "ExercisesRepo", lambda: repo) + monkeypatch.setattr(verify_module, "_execute_verify", _execute_verify) + monkeypatch.setattr( + verify_module.Namespace, "load_file_as_namespace", Mock(side_effect=exception) + ) + result = cli_runner.invoke(cli_module.cli, ["verify", "--json"]) + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert payload["status"] == status + assert payload["comments"] == comments + + +def test_json_survives_submission_failure( + cli_runner: CliRunner, + graded_output: GitAutograderOutput, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fail_submission(output: GitAutograderOutput) -> None: + assert output is graded_output + error("Submission failed") + + submit = Mock(side_effect=fail_submission) + monkeypatch.setattr(verify_module, "_submit_progress", submit) + result = cli_runner.invoke(cli_module.cli, ["verify", "--json"]) + assert result.exit_code == 1 + assert json.loads(result.stdout)["status"] == "Completed" + assert "Submission failed" in result.stderr + submit.assert_called_once_with(graded_output) + + +def test_remote_progress_is_submitted_once( + cli_runner: CliRunner, exercise_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (exercise_dir.parent / ".gitmastery" / "config.json").write_text( + '{"progress_remote": true}', encoding="utf-8" + ) + operations = {} + for name in ("add_all", "commit", "push", "pull_request"): + operations[name] = Mock() + monkeypatch.setattr(verify_module, name, operations[name]) + monkeypatch.setattr(verify_module, "get_prs", Mock(return_value=[])) + result = cli_runner.invoke(cli_module.cli, ["verify", "--json"]) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout)["status"] == "Completed" + assert "No pull request created" in result.stderr + for operation in operations.values(): + operation.assert_called_once() + + +def test_migration_warning(cli_runner: CliRunner, exercise_dir: Path) -> None: + metadata = exercise_dir.parent / ".gitmastery" + (metadata / "config.json").rename(exercise_dir.parent / ".gitmastery.json") + (metadata / "progress").rename(exercise_dir.parent / "progress") + result = cli_runner.invoke(cli_module.cli, ["verify", "--json"]) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout)["status"] == "Completed" + assert "Migrated your Git-Mastery metadata" in result.stderr + + +@pytest.mark.parametrize("args", [["--json", "--help"], ["--help", "--json"]]) +def test_help_remains_on_stdout(cli_runner: CliRunner, args: list[str]) -> None: + result = cli_runner.invoke(cli_module.cli, ["verify", *args]) + assert result.exit_code == 0 + assert "--json" in result.stdout + assert "Usage:" in result.stdout + assert result.stderr == "" + + +def test_usage_error_does_not_grade( + cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch +) -> None: + grade = Mock() + monkeypatch.setattr(verify_module, "_execute_verify", grade) + result = cli_runner.invoke(cli_module.cli, ["verify", "--json", "--unknown"]) + assert result.exit_code == 2 + assert "No such option" in result.stderr + grade.assert_not_called() + + +def test_direct_invocation_and_output_restoration( + cli_runner: CliRunner, exercise_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + stdout = sys.stdout + result = cli_runner.invoke(verify, ["--json"], obj={}) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout)["status"] == "Completed" + assert "Grader diagnostics" in result.stderr + assert sys.stdout is stdout + + monkeypatch.chdir(exercise_dir) + result = cli_runner.invoke(cli_module.cli, ["verify"]) + assert result.exit_code == 0, result.output + assert " INFO Status: Completed" in result.stdout + assert result.stderr == "" + + +def test_output_restoration_after_failure( + cli_runner: CliRunner, exercise_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (exercise_dir / ".gitmastery-exercise.json").unlink() + result = cli_runner.invoke(cli_module.cli, ["verify", "--json"]) + assert result.exit_code == 1 + result = cli_runner.invoke(cli_module.cli, ["verify"]) + assert result.exit_code == 1 + assert " ERROR " in result.stdout + assert result.stderr == "" + + +def test_verify_json_binary( + runner: BinaryRunner, exercise_dir: Path, tmp_path: Path +) -> None: + """Exercise real grading and persistence without a remote exercise source.""" + source = tmp_path / "source" + source.mkdir() + verifier = source / "under_control" + verifier.mkdir() + (verifier / "verify.py").write_text( + "from git_autograder import GitAutograderStatus\n" + "def verify(exercise):\n" + " print('Binary grader diagnostics')\n" + " return exercise.to_output(['First comment', 'Second comment'], " + "GitAutograderStatus.SUCCESSFUL)\n", + encoding="utf-8", + ) + utilities = source / "exercise_utils" + utilities.mkdir() + for name in EXERCISE_UTILS_FILES: + (utilities / f"{name}.py").write_text("", encoding="utf-8") + repo = Repo.init(source) + repo.index.add(["under_control/verify.py", "exercise_utils"]) + actor = Actor("Test verifier", "test@example.com") + repo.index.commit("Test verifier", author=actor, committer=actor) + (tmp_path / ".gitmastery" / "config.json").write_text( + json.dumps({"exercises_source": {"type": "local", "repo_path": str(source)}}), + encoding="utf-8", + ) + gh_config = tmp_path / "gh-config" + gh_config.mkdir() + result = runner.run( + ["verify", "--json"], + cwd=exercise_dir, + env={"GH_TOKEN": "", "GITHUB_TOKEN": "", "GH_CONFIG_DIR": str(gh_config)}, + ) + result.assert_success() + assert json.loads(result.stdout) == { + "exercise_name": EXERCISE_NAME, + "status": "Completed", + "comments": ["First comment", "Second comment"], + } + assert "Using local exercises source" in result.stderr + assert "Binary grader diagnostics" in result.stderr + progress = json.loads( + (tmp_path / ".gitmastery" / "progress" / "progress.json").read_text() + ) + assert len(progress) == 1 + assert progress[0]["status"] == "Completed" From 8a6ce03e6df653ceb8ce22b369cc2811c8b45d1c Mon Sep 17 00:00:00 2001 From: "jiaxin.loh" Date: Mon, 9 Nov 2026 12:13:46 +0800 Subject: [PATCH 2/3] test: focus JSON verification coverage on the new contract Remove redundant checks of unchanged workflows and exercise JSON persistence through the existing E2E fixtures. --- docs/verify-json-design.md | 20 +- tests/e2e/commands/test_verify.py | 33 ++- tests/test_verify_json.py | 388 ++++++------------------------ 3 files changed, 121 insertions(+), 320 deletions(-) diff --git a/docs/verify-json-design.md b/docs/verify-json-design.md index 4b19f32..cf43eba 100644 --- a/docs/verify-json-design.md +++ b/docs/verify-json-design.md @@ -54,7 +54,8 @@ These three fields cover the desktop's grading needs. Timestamps, numeric scores | [`app/commands/verify.py`](../app/commands/verify.py) | Add `@click.option("--json", "json_output", is_flag=True, default=False, ...)` and a typed `json_output: bool` callback parameter. Branch only the result renderer: serialize an explicit three-field dictionary or call the existing `_print_output`. Keep `_execute_verify` and `_submit_progress` behavior and ordering. Install the output scope outside the exercise/root guard decorators so their diagnostics are covered. | | [`app/utils/click.py`](../app/utils/click.py) | Add a reusable, scoped JSON output context: retain the original stdout stream for the JSON writer and redirect ordinary Python stdout to stderr using `contextlib.redirect_stdout`. Nested use should reuse the saved stream, and `finally` must restore it. This covers existing `info`/`warn`/`error`, `click.echo`, and `print` calls without changing their default behavior. | | [`app/cli.py`](../app/cli.py) | Extend `LoggingGroup.resolve_command` to identify JSON verification before the group callback runs and enter the same output scope around invocation. Resolve the command, including `v`, and inspect its options through Click rather than searching `sys.argv` substrings. This is necessary because `cli()` prints version warnings before `verify()` runs. Help and usage handling remain normal. | -| [`tests/test_verify_json.py`](../tests/test_verify_json.py) | Add deterministic contract tests using `CliRunner`, temporary configurations, and mocked grading/version/GitHub operations, plus a binary smoke test using the existing runner and a local verifier. Cover legacy text output and unchanged progress records in isolated tests. Run this contract suite separately from the existing happy-path E2E workflow. | +| [`tests/test_verify_json.py`](../tests/test_verify_json.py) | Cover the JSON contract and output routing with eight focused cases. Stub external grading and username lookup; exercise the real command, guards, rendering, and local progress saving. Run this contract suite separately from the existing E2E workflow. | +| [`tests/e2e/commands/test_verify.py`](../tests/e2e/commands/test_verify.py) | Add one happy-path JSON verification test using the existing `BinaryRunner` and downloaded-exercise fixture. Copy the exercise into an isolated progress root so the extra attempt does not affect shared progress assertions. This test runs in the existing CI matrix. | | [`README.md`](../README.md) | Document `verify --json`, the three fields, separate streams, exit semantics, and continued progress submission; link this design. | The output context also wraps direct invocation of the verify command, which the REPL uses without `LoggingGroup`. The standalone subprocess contract does not include REPL banners or prompts. Avoid retaining JSON mode across subsequent commands. @@ -83,18 +84,17 @@ Release the CLI option first. Because the desktop resolves a separately installe | Case | Expected result | | --- | --- | -| Ordinary `verify`, including each grading status | Existing text, comment formatting, exit behavior, and progress submission remain unchanged. | -| JSON for each grading status | The entire stdout parses as one object with the specified mapping; normal completion exits `0`. | +| Ordinary `verify` after JSON verification | Status and comments remain on stdout; output routing is restored between commands. | +| JSON for each grading status, including alias and verbose invocations | The entire stdout parses as one object with the specified mapping; normal completion exits `0`. | | No comments; multiple comments; quotes, newlines, Unicode | `[]` for absent feedback; all other strings round-trip in order without display bullets or placeholders. | -| Version warning, repository message, grader `print`, migration warning, `--verbose` | Diagnostics appear on stderr; stdout remains valid JSON. Test the full group invocation, not just the verify callback. | -| Missing exercise/root; repository initialization failure | Empty stdout, stderr diagnostic, nonzero exit, and no fabricated grade. | -| Wrong-answer, invalid-state, unexpected exception inside grading | Existing `_execute_verify` conversion yields `Incomplete` or `Error` with comments in JSON. | -| Progress save/sync succeeds, warns, or fails after grading | Submission is attempted once with the same output; JSON remains intact. Warnings preserve existing exit behavior; fatal failures remain nonzero. | -| Alias `v --json`, help, invalid arguments, sequential invocations | Alias uses the contract; Click help/errors remain normal; later text commands retain their original stdout behavior. | +| Version warning, grader `print`, progress-saving messages | Diagnostics appear on stderr; stdout remains valid JSON through the full group invocation. | +| Missing exercise configuration | Empty stdout, stderr diagnostic, nonzero exit, and no fabricated grade. | +| Progress saving fails after grading | JSON remains intact and the command exits nonzero. | +| JSON help and direct command invocation | Help remains on stdout; direct invocation used by the REPL produces the same result as group invocation. | | Desktop receives JSON in arbitrary chunks; stderr warning; malformed/unknown/mismatched result | Parse only after complete stdout; valid results map correctly. Invalid results never mark completion or trigger another verify attempt. | | Desktop uses older CLI; capability probe fails; executable changes | Older CLI uses the legacy path; probe failure is reported; changed executable is probed again. | -| Cross-platform binary smoke test | Separate stdout/stderr, JSON parsing, and unchanged progress persistence pass on the existing Windows, Linux, and macOS matrix. Desktop retains its existing Windows cwd gate. | +| Binary JSON verification with an isolated progress root | Parsed JSON matches the saved grading record without modifying the shared fixture's attempts. Runs on the existing Windows, Linux, and macOS matrix. Desktop retains its existing Windows cwd gate. | Use isolated progress directories for new cases: the current `verified_exercise_dir` fixture is session-scoped, and its legacy test expects exactly one progress entry. Mock remote operations in contract tests so they do not submit real progress. -Local validation passed: 25 CLI tests, including the packaged executable smoke test, plus Ruff and mypy checks for changed Python files. Run this contract suite on Windows and Linux as well; the existing CI matrix covers happy-path E2E tests. Desktop cases apply when that integration is implemented. +The contract suite intentionally avoids retesting unchanged grader exception conversion, metadata migration, and remote PR creation. Those branches are not changed by this flag. The E2E suite contains only the additional happy-path JSON check; error handling stays in the focused contract suite. diff --git a/tests/e2e/commands/test_verify.py b/tests/e2e/commands/test_verify.py index 2a35a6e..eb9333f 100644 --- a/tests/e2e/commands/test_verify.py +++ b/tests/e2e/commands/test_verify.py @@ -1,12 +1,16 @@ import json +import shutil from pathlib import Path from ..constants import EXERCISE_NAME +from ..runner import BinaryRunner def test_verify_exercise(verified_exercise_dir: Path) -> None: """verify writes a progress entry with the expected fields.""" - progress_json = verified_exercise_dir.parent / ".gitmastery" / "progress" / "progress.json" + progress_json = ( + verified_exercise_dir.parent / ".gitmastery" / "progress" / "progress.json" + ) entries = json.loads(progress_json.read_text()) assert len(entries) == 1 entry = entries[0] @@ -14,3 +18,30 @@ def test_verify_exercise(verified_exercise_dir: Path) -> None: assert "started_at" in entry assert "completed_at" in entry assert "status" in entry + + +def test_verify_json( + runner: BinaryRunner, downloaded_exercise_dir: Path, tmp_path: Path +) -> None: + """JSON verification returns the grade while still saving progress.""" + exercise_dir = tmp_path / EXERCISE_NAME + shutil.copytree(downloaded_exercise_dir, exercise_dir) + metadata = tmp_path / ".gitmastery" + (metadata / "progress").mkdir(parents=True) + config = json.loads( + (downloaded_exercise_dir.parent / ".gitmastery" / "config.json").read_text() + ) + config["progress_remote"] = False + (metadata / "config.json").write_text(json.dumps(config), encoding="utf-8") + + result = runner.run(["verify", "--json"], cwd=exercise_dir) + result.assert_success() + grade = json.loads(result.stdout) + entries = json.loads((metadata / "progress" / "progress.json").read_text()) + assert len(entries) == 1 + assert grade == { + "exercise_name": EXERCISE_NAME, + "status": entries[0]["status"], + "comments": entries[0]["comments"] or [], + } + assert grade["status"] in ("Completed", "Incomplete", "Error") diff --git a/tests/test_verify_json.py b/tests/test_verify_json.py index 0750b12..8dc5a1c 100644 --- a/tests/test_verify_json.py +++ b/tests/test_verify_json.py @@ -1,35 +1,20 @@ import importlib import json -import sys -from datetime import datetime, timezone +from datetime import datetime from pathlib import Path -from subprocess import CompletedProcess -from unittest.mock import Mock +import click import pytest from click.testing import CliRunner -from git import Actor, Repo -from git_autograder import ( - GitAutograderInvalidStateException, - GitAutograderStatus, - GitAutograderWrongAnswerException, -) +from git_autograder import GitAutograderStatus from git_autograder.output import GitAutograderOutput -from app.commands.verify import _execute_verify, verify -from app.utils.click import error, info -from app.utils.gitmastery import EXERCISE_UTILS_FILES - -from e2e.constants import EXERCISE_NAME -from e2e.runner import BinaryRunner +from app.cli import LoggingGroup, cli +from app.commands.verify import verify +from app.utils.click import error -cli_module = importlib.import_module("app.cli") verify_module = importlib.import_module("app.commands.verify") - - -@pytest.fixture -def runner() -> BinaryRunner: - return BinaryRunner.from_env() +EXERCISE_NAME = "example" @pytest.fixture @@ -47,7 +32,6 @@ def exercise_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: "requires_git": False, "requires_github": False, "base_files": {}, - "downloaded_at": None, "exercise_repo": { "repo_type": "ignore", "repo_name": "unused", @@ -65,29 +49,25 @@ def exercise_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: @pytest.fixture def graded_output() -> GitAutograderOutput: - now = datetime.now(timezone.utc) return GitAutograderOutput( status=GitAutograderStatus.SUCCESSFUL, exercise_name=EXERCISE_NAME, - started_at=now, - completed_at=now, - comments=["First comment", "Second comment"], + started_at=None, + completed_at=None, + comments=['First "comment"\nwith a newline', "こんにちは 😀"], ) @pytest.fixture -def cli_runner( +def application( exercise_dir: Path, graded_output: GitAutograderOutput, monkeypatch: pytest.MonkeyPatch, -) -> CliRunner: - monkeypatch.setattr(cli_module.cli, "commands", {}) - monkeypatch.setattr(cli_module.cli, "_aliases", {}) - monkeypatch.setattr(cli_module.cli, "_commands", {}) - cli_module.cli.add_command(verify, aliases=["v"]) +) -> click.Group: + group = LoggingGroup("gitmastery", callback=cli.callback, params=cli.params) + group.add_command(verify, aliases=["v"]) monkeypatch.setattr( - cli_module, - "fetch_latest_release_version", + "app.cli.fetch_latest_release_version", lambda: (None, "Version check unavailable"), ) monkeypatch.setattr(verify_module, "get_username", lambda: "test-user") @@ -98,321 +78,111 @@ def grade( formatted_exercise_name: str, started_at: datetime, ) -> GitAutograderOutput: - info("Fetching exercise information") print("Grader diagnostics") return graded_output monkeypatch.setattr(verify_module, "_execute_verify", grade) - return CliRunner() + return group @pytest.mark.parametrize( - ("status", "text"), + ("status", "text", "args"), [ - (GitAutograderStatus.SUCCESSFUL, "Completed"), - (GitAutograderStatus.UNSUCCESSFUL, "Incomplete"), - (GitAutograderStatus.ERROR, "Error"), + (GitAutograderStatus.SUCCESSFUL, "Completed", ["verify", "--json"]), + (GitAutograderStatus.UNSUCCESSFUL, "Incomplete", ["v", "--json"]), + (GitAutograderStatus.ERROR, "Error", ["--verbose", "verify", "--json"]), ], ) -@pytest.mark.parametrize("json_output", [False, True]) -def test_verify_status_and_progress( - cli_runner: CliRunner, +def test_json_grading_result( + application: click.Group, graded_output: GitAutograderOutput, - exercise_dir: Path, status: GitAutograderStatus, text: str, - json_output: bool, + args: list[str], ) -> None: + """Return every comment and a stable status without diagnostic text in JSON.""" graded_output.status = status - result = cli_runner.invoke( - cli_module.cli, ["verify", "--json"] if json_output else ["verify"] - ) + result = CliRunner().invoke(application, args, color=True) assert result.exit_code == 0, result.output - if json_output: - assert json.loads(result.stdout) == { - "exercise_name": EXERCISE_NAME, - "status": text, - "comments": graded_output.comments, - } - assert result.stdout.count("\n") == 1 - assert "Version check unavailable" in result.stderr - assert "Grader diagnostics" in result.stderr - assert "Saving progress of attempt" in result.stderr - else: - assert f" INFO Status: {text}" in result.stdout - assert ( - " INFO Comments:\n\t- First comment\n\t- Second comment\n" in result.stdout - ) - assert "Version check unavailable" in result.stdout - assert "Grader diagnostics" in result.stdout - assert result.stderr == "" - - entries = json.loads( - (exercise_dir.parent / ".gitmastery" / "progress" / "progress.json").read_text() - ) - assert graded_output.started_at is not None - assert graded_output.completed_at is not None - assert entries == [ - { - "exercise_name": EXERCISE_NAME, - "status": text, - "comments": graded_output.comments, - "started_at": graded_output.started_at.timestamp(), - "completed_at": graded_output.completed_at.timestamp(), - } - ] + assert json.loads(result.stdout) == { + "exercise_name": EXERCISE_NAME, + "status": text, + "comments": graded_output.comments, + } + assert "Version check unavailable" in result.stderr + assert "Grader diagnostics" in result.stderr + assert "Saving progress of attempt" in result.stderr -@pytest.mark.parametrize( - "comments", [None, [], ['Quotes: "hello"', "Two\nlines", "こんにちは 😀"]] -) -def test_json_feedback_round_trips( - cli_runner: CliRunner, - graded_output: GitAutograderOutput, - comments: list[str] | None, +def test_json_without_comments( + application: click.Group, graded_output: GitAutograderOutput ) -> None: - graded_output.comments = comments - graded_output.exercise_name = None - result = cli_runner.invoke(cli_module.cli, ["verify", "--json"], color=True) + """Absent feedback is an empty list, without the text renderer's placeholder.""" + graded_output.comments = None + result = CliRunner().invoke(application, ["verify", "--json"]) assert result.exit_code == 0, result.output - payload = json.loads(result.stdout) - assert payload["exercise_name"] == EXERCISE_NAME - assert payload["comments"] == (comments or []) - assert result.stdout.isascii() - assert "\x1b" not in result.stdout + assert json.loads(result.stdout)["comments"] == [] -def test_verbose_and_alias( - cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch +def test_json_before_grading_failure( + application: click.Group, exercise_dir: Path ) -> None: - monkeypatch.setattr( - verify_module, - "get_username", - importlib.import_module("app.utils.github_cli").get_username, - ) - monkeypatch.setattr( - "subprocess.run", - lambda *args, **kwargs: CompletedProcess(args[0], 0, "test-user\n", ""), - ) - result = cli_runner.invoke(cli_module.cli, ["--verbose", "v", "--json"]) - assert result.exit_code == 0, result.output - assert json.loads(result.stdout)["status"] == "Completed" - assert "\ttest-user" in result.stderr - - -@pytest.mark.parametrize("missing", ["exercise", "root"]) -def test_missing_configuration( - cli_runner: CliRunner, exercise_dir: Path, missing: str -) -> None: - config = ( - exercise_dir / ".gitmastery-exercise.json" - if missing == "exercise" - else exercise_dir.parent / ".gitmastery" / "config.json" - ) - config.unlink() - result = cli_runner.invoke(cli_module.cli, ["verify", "--json"]) + """A guard failure must leave stdout empty rather than emit a grading result.""" + (exercise_dir / ".gitmastery-exercise.json").unlink() + result = CliRunner().invoke(application, ["verify", "--json"]) assert result.exit_code == 1 assert result.stdout == "" - assert " ERROR " in result.stderr + assert "not inside a Git-Mastery exercise folder" in result.stderr -def test_repository_failure_before_grading( - cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch -) -> None: - repo = Mock() - repo.__enter__ = Mock(side_effect=RuntimeError("Repository unavailable")) - repo.__exit__ = Mock(return_value=False) - monkeypatch.setattr(verify_module, "ExercisesRepo", lambda: repo) - monkeypatch.setattr(verify_module, "_execute_verify", _execute_verify) - result = cli_runner.invoke(cli_module.cli, ["verify", "--json"]) - assert result.exit_code != 0 - assert result.stdout == "" - assert isinstance(result.exception, RuntimeError) - - -@pytest.mark.parametrize( - ("exception", "status", "comments"), - [ - ( - GitAutograderWrongAnswerException(["Wrong answer", "Try again"]), - "Incomplete", - ["Wrong answer", "Try again"], - ), - ( - GitAutograderInvalidStateException("Invalid state"), - "Error", - ["Invalid state"], - ), - ( - RuntimeError("Unexpected grading failure"), - "Error", - ["Unexpected grading failure"], - ), - ], -) -def test_grading_exceptions( - cli_runner: CliRunner, - monkeypatch: pytest.MonkeyPatch, - exception: Exception, - status: str, - comments: list[str], +def test_json_after_submission_failure( + application: click.Group, monkeypatch: pytest.MonkeyPatch ) -> None: - repo = Mock() - repo.__enter__ = Mock(return_value=repo) - repo.__exit__ = Mock(return_value=False) - monkeypatch.setattr(verify_module, "ExercisesRepo", lambda: repo) - monkeypatch.setattr(verify_module, "_execute_verify", _execute_verify) - monkeypatch.setattr( - verify_module.Namespace, "load_file_as_namespace", Mock(side_effect=exception) - ) - result = cli_runner.invoke(cli_module.cli, ["verify", "--json"]) - assert result.exit_code == 0, result.output - payload = json.loads(result.stdout) - assert payload["status"] == status - assert payload["comments"] == comments - + """Retain the grading result when saving progress fails afterward.""" -def test_json_survives_submission_failure( - cli_runner: CliRunner, - graded_output: GitAutograderOutput, - monkeypatch: pytest.MonkeyPatch, -) -> None: def fail_submission(output: GitAutograderOutput) -> None: - assert output is graded_output - error("Submission failed") + error("Saving progress failed") - submit = Mock(side_effect=fail_submission) - monkeypatch.setattr(verify_module, "_submit_progress", submit) - result = cli_runner.invoke(cli_module.cli, ["verify", "--json"]) + monkeypatch.setattr(verify_module, "_submit_progress", fail_submission) + result = CliRunner().invoke(application, ["verify", "--json"]) assert result.exit_code == 1 assert json.loads(result.stdout)["status"] == "Completed" - assert "Submission failed" in result.stderr - submit.assert_called_once_with(graded_output) - - -def test_remote_progress_is_submitted_once( - cli_runner: CliRunner, exercise_dir: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - (exercise_dir.parent / ".gitmastery" / "config.json").write_text( - '{"progress_remote": true}', encoding="utf-8" - ) - operations = {} - for name in ("add_all", "commit", "push", "pull_request"): - operations[name] = Mock() - monkeypatch.setattr(verify_module, name, operations[name]) - monkeypatch.setattr(verify_module, "get_prs", Mock(return_value=[])) - result = cli_runner.invoke(cli_module.cli, ["verify", "--json"]) - assert result.exit_code == 0, result.output - assert json.loads(result.stdout)["status"] == "Completed" - assert "No pull request created" in result.stderr - for operation in operations.values(): - operation.assert_called_once() - - -def test_migration_warning(cli_runner: CliRunner, exercise_dir: Path) -> None: - metadata = exercise_dir.parent / ".gitmastery" - (metadata / "config.json").rename(exercise_dir.parent / ".gitmastery.json") - (metadata / "progress").rename(exercise_dir.parent / "progress") - result = cli_runner.invoke(cli_module.cli, ["verify", "--json"]) - assert result.exit_code == 0, result.output - assert json.loads(result.stdout)["status"] == "Completed" - assert "Migrated your Git-Mastery metadata" in result.stderr + assert "Saving progress failed" in result.stderr -@pytest.mark.parametrize("args", [["--json", "--help"], ["--help", "--json"]]) -def test_help_remains_on_stdout(cli_runner: CliRunner, args: list[str]) -> None: - result = cli_runner.invoke(cli_module.cli, ["verify", *args]) +def test_json_help(application: click.Group) -> None: + """The early JSON-mode check must leave Click help on stdout.""" + result = CliRunner().invoke(application, ["verify", "--json", "--help"]) assert result.exit_code == 0 - assert "--json" in result.stdout assert "Usage:" in result.stdout + assert "--json" in result.stdout assert result.stderr == "" -def test_usage_error_does_not_grade( - cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch -) -> None: - grade = Mock() - monkeypatch.setattr(verify_module, "_execute_verify", grade) - result = cli_runner.invoke(cli_module.cli, ["verify", "--json", "--unknown"]) - assert result.exit_code == 2 - assert "No such option" in result.stderr - grade.assert_not_called() - - -def test_direct_invocation_and_output_restoration( - cli_runner: CliRunner, exercise_dir: Path, monkeypatch: pytest.MonkeyPatch +def test_direct_json_and_subsequent_invocations( + application: click.Group, + exercise_dir: Path, + graded_output: GitAutograderOutput, + monkeypatch: pytest.MonkeyPatch, ) -> None: - stdout = sys.stdout - result = cli_runner.invoke(verify, ["--json"], obj={}) - assert result.exit_code == 0, result.output - assert json.loads(result.stdout)["status"] == "Completed" - assert "Grader diagnostics" in result.stderr - assert sys.stdout is stdout + """Support REPL-style direct invocation and restore routing for later commands.""" + runner = CliRunner() + direct = runner.invoke(verify, ["--json"], obj={}) + assert direct.exit_code == 0, direct.output + assert json.loads(direct.stdout)["status"] == "Completed" + assert "Grader diagnostics" in direct.stderr monkeypatch.chdir(exercise_dir) - result = cli_runner.invoke(cli_module.cli, ["verify"]) - assert result.exit_code == 0, result.output - assert " INFO Status: Completed" in result.stdout - assert result.stderr == "" - - -def test_output_restoration_after_failure( - cli_runner: CliRunner, exercise_dir: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - (exercise_dir / ".gitmastery-exercise.json").unlink() - result = cli_runner.invoke(cli_module.cli, ["verify", "--json"]) - assert result.exit_code == 1 - result = cli_runner.invoke(cli_module.cli, ["verify"]) - assert result.exit_code == 1 - assert " ERROR " in result.stdout - assert result.stderr == "" - + grouped = runner.invoke(application, ["verify", "--json"]) + assert grouped.exit_code == 0, grouped.output + assert json.loads(grouped.stdout) == json.loads(direct.stdout) -def test_verify_json_binary( - runner: BinaryRunner, exercise_dir: Path, tmp_path: Path -) -> None: - """Exercise real grading and persistence without a remote exercise source.""" - source = tmp_path / "source" - source.mkdir() - verifier = source / "under_control" - verifier.mkdir() - (verifier / "verify.py").write_text( - "from git_autograder import GitAutograderStatus\n" - "def verify(exercise):\n" - " print('Binary grader diagnostics')\n" - " return exercise.to_output(['First comment', 'Second comment'], " - "GitAutograderStatus.SUCCESSFUL)\n", - encoding="utf-8", - ) - utilities = source / "exercise_utils" - utilities.mkdir() - for name in EXERCISE_UTILS_FILES: - (utilities / f"{name}.py").write_text("", encoding="utf-8") - repo = Repo.init(source) - repo.index.add(["under_control/verify.py", "exercise_utils"]) - actor = Actor("Test verifier", "test@example.com") - repo.index.commit("Test verifier", author=actor, committer=actor) - (tmp_path / ".gitmastery" / "config.json").write_text( - json.dumps({"exercises_source": {"type": "local", "repo_path": str(source)}}), - encoding="utf-8", - ) - gh_config = tmp_path / "gh-config" - gh_config.mkdir() - result = runner.run( - ["verify", "--json"], - cwd=exercise_dir, - env={"GH_TOKEN": "", "GITHUB_TOKEN": "", "GH_CONFIG_DIR": str(gh_config)}, - ) - result.assert_success() - assert json.loads(result.stdout) == { - "exercise_name": EXERCISE_NAME, - "status": "Completed", - "comments": ["First comment", "Second comment"], - } - assert "Using local exercises source" in result.stderr - assert "Binary grader diagnostics" in result.stderr - progress = json.loads( - (tmp_path / ".gitmastery" / "progress" / "progress.json").read_text() - ) - assert len(progress) == 1 - assert progress[0]["status"] == "Completed" + monkeypatch.chdir(exercise_dir) + plain = runner.invoke(application, ["verify"]) + assert plain.exit_code == 0, plain.output + assert " INFO Status: Completed" in plain.stdout + assert " INFO Comments:" in plain.stdout + for comment in graded_output.comments or []: + assert comment in plain.stdout + assert "Version check unavailable" in plain.stdout + assert plain.stderr == "" From d7bf091bba3f9c873a7b43f10214a8588126e0de Mon Sep 17 00:00:00 2001 From: "jiaxin.loh" Date: Tue, 10 Nov 2026 10:01:34 +0800 Subject: [PATCH 3/3] test: keep JSON verification coverage minimal Remove the design document and extra tests to match the repository's small test suites. --- README.md | 3 - docs/verify-json-design.md | 100 ------------------------------ tests/e2e/commands/test_verify.py | 29 --------- tests/test_verify_json.py | 83 +------------------------ 4 files changed, 3 insertions(+), 212 deletions(-) delete mode 100644 docs/verify-json-design.md diff --git a/README.md b/README.md index 9bf9574..da1906e 100644 --- a/README.md +++ b/README.md @@ -41,9 +41,6 @@ the answer is correct. A failure before grading leaves stdout empty and exits nonzero. A submission failure after grading retains the JSON result and exits nonzero. Ordinary `gitmastery verify` keeps its existing text output. -See the [technical design](docs/verify-json-design.md) for the contract and desktop -integration plan. - ## Publishing Tag the PR with the `bump:{major | minor | patch}` label and the CI will automatically perform the publish. diff --git a/docs/verify-json-design.md b/docs/verify-json-design.md deleted file mode 100644 index cf43eba..0000000 --- a/docs/verify-json-design.md +++ /dev/null @@ -1,100 +0,0 @@ -# Design: JSON output for `gitmastery verify` - -The CLI option is implemented. Desktop adoption described below remains a separate integration step. - -## 1. Goals - -Adding `gitmastery verify --json` is feasible and reasonable. The CLI already receives a structured `GitAutograderOutput`, and [`verify.py`](../app/commands/verify.py) renders its status and comments separately from grading. No grader or dependency change is needed. - -- Give desktop clients a stable subprocess interface for the current attempt's grading result. -- Preserve every feedback comment without depending on terminal formatting or stream chunk boundaries. -- Keep ordinary `gitmastery verify` output, grading, progress persistence, remote submission, and exit codes backward compatible. - -The desktop currently spawns `["verify"]` in [`runVerify`](../../git-mastery-desktop/src/electron/ipc/gitmastery.ts). `_checkCorrectSolution` and `_checkIncorrectSolution` search for `INFO Status:` strings. `_getComments` assumes the header and first comment arrive in the same chunk and extracts only that comment. Node stream chunks do not correspond to output lines, so even unchanged CLI wording can break this parser. - -This is a structured output option for an existing command. It does not introduce a service, change grading rules, or make verification free of side effects. - -## 2. JSON structure - -When a grading result exists, stdout contains exactly one JSON object followed by a newline: - -```json -{ - "exercise_name": "under-control", - "status": "Completed", - "comments": [] -} -``` - -This illustrates the contract for a completed attempt with no feedback. - -| Field | Type | Meaning | -| --- | --- | --- | -| `exercise_name` | string | Exercise identifier from the resolved exercise configuration; lets a client match the result to its request. | -| `status` | string enum | `Completed`, `Incomplete`, or `Error`. | -| `comments` | string[] | All grader comments in their original order; normalize absent comments to `[]`. | - -Use the existing mapping: `SUCCESSFUL` → `Completed`, `UNSUCCESSFUL` → `Incomplete`, `ERROR` → `Error`. These become fixed contract values, independent of future display wording. Preserve comment content, including newlines and Unicode, through JSON escaping. Do not insert the display placeholder `"No comments"`. - -These three fields cover the desktop's grading needs. Timestamps, numeric scores, redundant `correct`/`incorrect` booleans, and a schema version are unnecessary for this first contract. Consumers should ignore additional fields; changing existing field types or status values would be a breaking change. - -**Streams and exit codes:** - -- JSON mode sends diagnostic text to stderr, including version warnings, repository fetching, grader prints, verbose subprocess output, and progress-saving messages. Stdout has no prefixes, ANSI escapes, or progress text. -- Preserve current exit behavior: a returned grading result can have any of the three statuses and still exit `0` when the command finishes normally. Exercise correctness comes from `status`, not the process exit code. -- Emit the result at the same point as today's `_print_output`, before `_submit_progress`. If progress submission subsequently fails, retain the grading JSON and the existing nonzero exit behavior. The result describes grading, not successful persistence. -- Failures before a grading result exists, such as missing exercise configuration or failure entering `ExercisesRepo`, produce empty stdout, diagnostics on stderr, and a nonzero exit. Do not fabricate a grading result. Usage errors and `--help` retain Click's normal behavior. - -## 3. Implementation detail - -### CLI implementation - -| File | Change | -| --- | --- | -| [`app/commands/verify.py`](../app/commands/verify.py) | Add `@click.option("--json", "json_output", is_flag=True, default=False, ...)` and a typed `json_output: bool` callback parameter. Branch only the result renderer: serialize an explicit three-field dictionary or call the existing `_print_output`. Keep `_execute_verify` and `_submit_progress` behavior and ordering. Install the output scope outside the exercise/root guard decorators so their diagnostics are covered. | -| [`app/utils/click.py`](../app/utils/click.py) | Add a reusable, scoped JSON output context: retain the original stdout stream for the JSON writer and redirect ordinary Python stdout to stderr using `contextlib.redirect_stdout`. Nested use should reuse the saved stream, and `finally` must restore it. This covers existing `info`/`warn`/`error`, `click.echo`, and `print` calls without changing their default behavior. | -| [`app/cli.py`](../app/cli.py) | Extend `LoggingGroup.resolve_command` to identify JSON verification before the group callback runs and enter the same output scope around invocation. Resolve the command, including `v`, and inspect its options through Click rather than searching `sys.argv` substrings. This is necessary because `cli()` prints version warnings before `verify()` runs. Help and usage handling remain normal. | -| [`tests/test_verify_json.py`](../tests/test_verify_json.py) | Cover the JSON contract and output routing with eight focused cases. Stub external grading and username lookup; exercise the real command, guards, rendering, and local progress saving. Run this contract suite separately from the existing E2E workflow. | -| [`tests/e2e/commands/test_verify.py`](../tests/e2e/commands/test_verify.py) | Add one happy-path JSON verification test using the existing `BinaryRunner` and downloaded-exercise fixture. Copy the exercise into an isolated progress root so the extra attempt does not affect shared progress assertions. This test runs in the existing CI matrix. | -| [`README.md`](../README.md) | Document `verify --json`, the three fields, separate streams, exit semantics, and continued progress submission; link this design. | - -The output context also wraps direct invocation of the verify command, which the REPL uses without `LoggingGroup`. The standalone subprocess contract does not include REPL banners or prompts. Avoid retaining JSON mode across subsequent commands. - -Use the standard `json` module and explicitly write and flush the payload to the saved stdout stream. Its default Unicode escaping produces ASCII JSON that round-trips on Windows, Linux, and macOS without relying on the terminal encoding. Do not serialize the entire dependency dataclass, which would expose unnecessary fields and datetime serialization details. - -[`app/utils/command.py`](../app/utils/command.py) already captures subprocess streams; its verbose Python prints are covered by the scope. Any verifier subprocess that inherits stdout directly must explicitly send diagnostic output to stderr: `redirect_stdout` redirects Python writes, not OS file descriptor 1. - -The existing guard decorators, progress-file format, Git/GitHub helpers, packaging, and CI configuration need no edits. - -### Desktop adoption in the sibling repository - -| File | Proposed change | -| --- | --- | -| [`src/electron/ipc/gitmastery.ts`](../../git-mastery-desktop/src/electron/ipc/gitmastery.ts) | Request `["verify", "--json"]` when supported. Collect raw stdout without `[[terminal-line]]`, decode the complete buffer and parse once on process `close`. Keep stderr streaming as diagnostics; a stderr chunk alone must not determine failure. Render the parsed status and every comment into terminal feedback. | -| `src/electron/utils/cli/verification.ts` (new) | Isolate JSON validation and capability detection. Require the three fields, a recognized status, string comments, and the expected exercise identifier. Adapt to existing IPC data: `correct = status === "Completed"`, `incorrect = status !== "Completed"`, and `comments = comments.join("\n")`. | -| `tests/electron/verification.test.ts` (new) | Test validation, legacy capability selection, and mapping to existing desktop feedback. | -| [`tests/e2e/cli.mjs`](../../git-mastery-desktop/tests/e2e/cli.mjs), [`tests/e2e/happy-paths.e2e.ts`](../../git-mastery-desktop/tests/e2e/happy-paths.e2e.ts) | Update fixture commands and invocation expectations for capability probing and JSON verification; exercise split output chunks and preserved completion behavior. | -| [`docs/architecture/cli-output-in-terminal.md`](../../git-mastery-desktop/docs/architecture/cli-output-in-terminal.md), [`docs/development/happy-path-e2e-tests.md`](../../git-mastery-desktop/docs/development/happy-path-e2e-tests.md) | Replace descriptions of grading text parsing with the structured contract and updated fixture behavior. | - -Keep the existing process-success gate and renderer IPC shape. A nonzero exit remains a task failure, even when grading JSON is available; retain that JSON for feedback. Malformed or mismatched JSON on exit `0` is also a task failure and must not patch exercise progress. The existing [`ActivityProvider`](../../git-mastery-desktop/src/ui/providers/ActivityProvider.tsx) and [`types.d.ts`](../../git-mastery-desktop/types.d.ts) can continue consuming the adapted fields without changes. - -Release the CLI option first. Because the desktop resolves a separately installed binary, probe `verify --help` for the advertised `--json` option and cache support until the executable changes. Use the existing text path only for CLIs that lack the option. A failed probe should report a CLI problem. Never retry grading after malformed JSON or a submission failure: verification saves progress and could otherwise run twice. - -## 4. Test cases - -| Case | Expected result | -| --- | --- | -| Ordinary `verify` after JSON verification | Status and comments remain on stdout; output routing is restored between commands. | -| JSON for each grading status, including alias and verbose invocations | The entire stdout parses as one object with the specified mapping; normal completion exits `0`. | -| No comments; multiple comments; quotes, newlines, Unicode | `[]` for absent feedback; all other strings round-trip in order without display bullets or placeholders. | -| Version warning, grader `print`, progress-saving messages | Diagnostics appear on stderr; stdout remains valid JSON through the full group invocation. | -| Missing exercise configuration | Empty stdout, stderr diagnostic, nonzero exit, and no fabricated grade. | -| Progress saving fails after grading | JSON remains intact and the command exits nonzero. | -| JSON help and direct command invocation | Help remains on stdout; direct invocation used by the REPL produces the same result as group invocation. | -| Desktop receives JSON in arbitrary chunks; stderr warning; malformed/unknown/mismatched result | Parse only after complete stdout; valid results map correctly. Invalid results never mark completion or trigger another verify attempt. | -| Desktop uses older CLI; capability probe fails; executable changes | Older CLI uses the legacy path; probe failure is reported; changed executable is probed again. | -| Binary JSON verification with an isolated progress root | Parsed JSON matches the saved grading record without modifying the shared fixture's attempts. Runs on the existing Windows, Linux, and macOS matrix. Desktop retains its existing Windows cwd gate. | - -Use isolated progress directories for new cases: the current `verified_exercise_dir` fixture is session-scoped, and its legacy test expects exactly one progress entry. Mock remote operations in contract tests so they do not submit real progress. - -The contract suite intentionally avoids retesting unchanged grader exception conversion, metadata migration, and remote PR creation. Those branches are not changed by this flag. The E2E suite contains only the additional happy-path JSON check; error handling stays in the focused contract suite. diff --git a/tests/e2e/commands/test_verify.py b/tests/e2e/commands/test_verify.py index eb9333f..3ff91b8 100644 --- a/tests/e2e/commands/test_verify.py +++ b/tests/e2e/commands/test_verify.py @@ -1,9 +1,7 @@ import json -import shutil from pathlib import Path from ..constants import EXERCISE_NAME -from ..runner import BinaryRunner def test_verify_exercise(verified_exercise_dir: Path) -> None: @@ -18,30 +16,3 @@ def test_verify_exercise(verified_exercise_dir: Path) -> None: assert "started_at" in entry assert "completed_at" in entry assert "status" in entry - - -def test_verify_json( - runner: BinaryRunner, downloaded_exercise_dir: Path, tmp_path: Path -) -> None: - """JSON verification returns the grade while still saving progress.""" - exercise_dir = tmp_path / EXERCISE_NAME - shutil.copytree(downloaded_exercise_dir, exercise_dir) - metadata = tmp_path / ".gitmastery" - (metadata / "progress").mkdir(parents=True) - config = json.loads( - (downloaded_exercise_dir.parent / ".gitmastery" / "config.json").read_text() - ) - config["progress_remote"] = False - (metadata / "config.json").write_text(json.dumps(config), encoding="utf-8") - - result = runner.run(["verify", "--json"], cwd=exercise_dir) - result.assert_success() - grade = json.loads(result.stdout) - entries = json.loads((metadata / "progress" / "progress.json").read_text()) - assert len(entries) == 1 - assert grade == { - "exercise_name": EXERCISE_NAME, - "status": entries[0]["status"], - "comments": entries[0]["comments"] or [], - } - assert grade["status"] in ("Completed", "Incomplete", "Error") diff --git a/tests/test_verify_json.py b/tests/test_verify_json.py index 8dc5a1c..44a9909 100644 --- a/tests/test_verify_json.py +++ b/tests/test_verify_json.py @@ -11,7 +11,6 @@ from app.cli import LoggingGroup, cli from app.commands.verify import verify -from app.utils.click import error verify_module = importlib.import_module("app.commands.verify") EXERCISE_NAME = "example" @@ -85,28 +84,16 @@ def grade( return group -@pytest.mark.parametrize( - ("status", "text", "args"), - [ - (GitAutograderStatus.SUCCESSFUL, "Completed", ["verify", "--json"]), - (GitAutograderStatus.UNSUCCESSFUL, "Incomplete", ["v", "--json"]), - (GitAutograderStatus.ERROR, "Error", ["--verbose", "verify", "--json"]), - ], -) def test_json_grading_result( application: click.Group, graded_output: GitAutograderOutput, - status: GitAutograderStatus, - text: str, - args: list[str], ) -> None: - """Return every comment and a stable status without diagnostic text in JSON.""" - graded_output.status = status - result = CliRunner().invoke(application, args, color=True) + """JSON contains the grade and all comments, with diagnostics on stderr.""" + result = CliRunner().invoke(application, ["verify", "--json"], color=True) assert result.exit_code == 0, result.output assert json.loads(result.stdout) == { "exercise_name": EXERCISE_NAME, - "status": text, + "status": "Completed", "comments": graded_output.comments, } assert "Version check unavailable" in result.stderr @@ -122,67 +109,3 @@ def test_json_without_comments( result = CliRunner().invoke(application, ["verify", "--json"]) assert result.exit_code == 0, result.output assert json.loads(result.stdout)["comments"] == [] - - -def test_json_before_grading_failure( - application: click.Group, exercise_dir: Path -) -> None: - """A guard failure must leave stdout empty rather than emit a grading result.""" - (exercise_dir / ".gitmastery-exercise.json").unlink() - result = CliRunner().invoke(application, ["verify", "--json"]) - assert result.exit_code == 1 - assert result.stdout == "" - assert "not inside a Git-Mastery exercise folder" in result.stderr - - -def test_json_after_submission_failure( - application: click.Group, monkeypatch: pytest.MonkeyPatch -) -> None: - """Retain the grading result when saving progress fails afterward.""" - - def fail_submission(output: GitAutograderOutput) -> None: - error("Saving progress failed") - - monkeypatch.setattr(verify_module, "_submit_progress", fail_submission) - result = CliRunner().invoke(application, ["verify", "--json"]) - assert result.exit_code == 1 - assert json.loads(result.stdout)["status"] == "Completed" - assert "Saving progress failed" in result.stderr - - -def test_json_help(application: click.Group) -> None: - """The early JSON-mode check must leave Click help on stdout.""" - result = CliRunner().invoke(application, ["verify", "--json", "--help"]) - assert result.exit_code == 0 - assert "Usage:" in result.stdout - assert "--json" in result.stdout - assert result.stderr == "" - - -def test_direct_json_and_subsequent_invocations( - application: click.Group, - exercise_dir: Path, - graded_output: GitAutograderOutput, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Support REPL-style direct invocation and restore routing for later commands.""" - runner = CliRunner() - direct = runner.invoke(verify, ["--json"], obj={}) - assert direct.exit_code == 0, direct.output - assert json.loads(direct.stdout)["status"] == "Completed" - assert "Grader diagnostics" in direct.stderr - - monkeypatch.chdir(exercise_dir) - grouped = runner.invoke(application, ["verify", "--json"]) - assert grouped.exit_code == 0, grouped.output - assert json.loads(grouped.stdout) == json.loads(direct.stdout) - - monkeypatch.chdir(exercise_dir) - plain = runner.invoke(application, ["verify"]) - assert plain.exit_code == 0, plain.output - assert " INFO Status: Completed" in plain.stdout - assert " INFO Comments:" in plain.stdout - for comment in graded_output.comments or []: - assert comment in plain.stdout - assert "Version check unavailable" in plain.stdout - assert plain.stderr == ""