diff --git a/README.md b/README.md index 2296161..da1906e 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,25 @@ Refer to the [installation guide](https://git-mastery.org/companion-app/index.ht To develop the app locally, refer to the [development setup guide](https://git-mastery.org/developers/docs/getting-started/setup/) for detailed instructions on how to set up your development environment and contribute to the project. +## JSON verification output + +Run `gitmastery verify --json` inside an exercise to get its grading result on +stdout: + +```json +{"exercise_name": "under-control", "status": "Completed", "comments": []} +``` + +`status` is `Completed`, `Incomplete`, or `Error`. `comments` contains all grader +feedback in order, or an empty array when there is none. Diagnostic messages go +to stderr, so clients can parse the complete stdout as JSON. + +Verification still saves progress and performs any configured remote submission. +Exit `0` means the command finished normally; read `status` to determine whether +the answer is correct. A failure before grading leaves stdout empty and exits +nonzero. A submission failure after grading retains the JSON result and exits +nonzero. Ordinary `gitmastery verify` keeps its existing text output. + ## Publishing Tag the PR with the `bump:{major | minor | patch}` label and the CI will automatically perform the publish. diff --git a/app/cli.py b/app/cli.py index 4b10816..753d55f 100644 --- a/app/cli.py +++ b/app/cli.py @@ -8,7 +8,7 @@ from app.commands import check, download, progress, setup, verify from app.commands.repl import repl from app.commands.version import version -from app.utils.click import ClickColor, CliContextKey, warn +from app.utils.click import ClickColor, CliContextKey, json_output_scope, warn from app.utils.version import Version, fetch_latest_release_version from app.version import __version__ @@ -19,6 +19,26 @@ def invoke(self, ctx: click.Context) -> None: logger.info("Running command %s with arguments %s", ctx.command_path, sys.argv) return super().invoke(ctx) + def resolve_command( + self, ctx: click.Context, args: list[str] + ) -> tuple[str | None, click.Command | None, list[str]]: + command_name, command, command_args = super().resolve_command(ctx, args) + if command is verify: + # Resolve aliases and parse options before the group callback prints + # version warnings. Resilient parsing leaves help/errors to Click. + with command.make_context( + command_name or "verify", + list(command_args), + parent=ctx, + resilient_parsing=True, + ) as command_ctx: + requesting_help = any( + arg in command_ctx.help_option_names for arg in command_args + ) + if command_ctx.params.get("json_output") and not requesting_help: + ctx.with_resource(json_output_scope()) + return command_name, command, command_args + CONTEXT_SETTINGS = {"max_content_width": 120} diff --git a/app/commands/verify.py b/app/commands/verify.py index c18c4e7..f33ebec 100644 --- a/app/commands/verify.py +++ b/app/commands/verify.py @@ -22,10 +22,12 @@ from app.utils.click import ( ClickColor, error, + get_json_output_stream, info, must_get_exercise_root_config, must_get_gitmastery_root_config, warn, + with_json_output, ) from app.utils.git import add_all, commit, push from app.utils.github_cli import get_prs, get_username, pull_request @@ -69,6 +71,22 @@ def _print_output(output: GitAutograderOutput) -> None: ) +def _print_json_output(output: GitAutograderOutput, exercise_name: str) -> None: + statuses = { + GitAutograderStatus.SUCCESSFUL: "Completed", + GitAutograderStatus.UNSUCCESSFUL: "Incomplete", + GitAutograderStatus.ERROR: "Error", + } + payload = { + "exercise_name": exercise_name, + "status": statuses[output.status], + "comments": output.comments or [], + } + stdout = get_json_output_stream() + click.echo(json.dumps(payload), file=stdout, color=False) + stdout.flush() + + def _submit_progress(output: GitAutograderOutput) -> None: username = get_username() @@ -183,9 +201,17 @@ def _execute_verify( @click.command() +@click.option( + "--json", + "json_output", + is_flag=True, + default=False, + help="Output the grading result as JSON; send diagnostics to stderr.", +) +@with_json_output @in_exercise_root() @in_gitmastery_root() -def verify() -> None: +def verify(json_output: bool = False) -> None: """ Verifies the state of the exercise attempt. """ @@ -204,5 +230,8 @@ def verify() -> None: output = _execute_verify( exercise_path, exercise_name, formatted_exercise_name, started_at ) - _print_output(output) + if json_output: + _print_json_output(output, exercise_name) + else: + _print_output(output) _submit_progress(output) diff --git a/app/utils/click.py b/app/utils/click.py index 88a4c51..d13f90c 100644 --- a/app/utils/click.py +++ b/app/utils/click.py @@ -1,7 +1,11 @@ import logging import sys +from collections.abc import Callable, Iterator +from contextlib import contextmanager, redirect_stdout +from contextvars import ContextVar from enum import StrEnum -from typing import Any, NoReturn, Optional +from functools import wraps +from typing import Any, NoReturn, Optional, ParamSpec, TextIO, TypeVar import click @@ -10,6 +14,48 @@ logger = logging.getLogger(__name__) +_JSON_STDOUT: ContextVar[Optional[TextIO]] = ContextVar("json_stdout", default=None) +P = ParamSpec("P") +T = TypeVar("T") + + +@contextmanager +def json_output_scope(enabled: bool = True) -> Iterator[TextIO]: + """Reserve stdout for JSON and route Python diagnostics to stderr. + + :param enabled: Whether to enable JSON output routing. + :yield: The original stdout stream for the JSON result. + """ + saved_stdout = _JSON_STDOUT.get() + if not enabled or saved_stdout is not None: + yield saved_stdout if saved_stdout is not None else sys.stdout + return + + saved_stdout = sys.stdout + token = _JSON_STDOUT.set(saved_stdout) + try: + with redirect_stdout(sys.stderr): + yield saved_stdout + finally: + _JSON_STDOUT.reset(token) + + +def with_json_output(func: Callable[P, T]) -> Callable[P, T]: + """Route diagnostics before command guards when ``json_output`` is enabled.""" + + @wraps(func) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> T: + with json_output_scope(bool(kwargs.get("json_output", False))): + return func(*args, **kwargs) + + return wrapper + + +def get_json_output_stream() -> TextIO: + """Return the stdout stream reserved for the JSON result.""" + saved_stdout = _JSON_STDOUT.get() + return saved_stdout if saved_stdout is not None else sys.stdout + class CliContextKey(StrEnum): GITMASTERY_ROOT_CONFIG = "GITMASTERY_ROOT_CONFIG" diff --git a/tests/e2e/commands/test_verify.py b/tests/e2e/commands/test_verify.py index 2a35a6e..3ff91b8 100644 --- a/tests/e2e/commands/test_verify.py +++ b/tests/e2e/commands/test_verify.py @@ -6,7 +6,9 @@ def test_verify_exercise(verified_exercise_dir: Path) -> None: """verify writes a progress entry with the expected fields.""" - progress_json = verified_exercise_dir.parent / ".gitmastery" / "progress" / "progress.json" + progress_json = ( + verified_exercise_dir.parent / ".gitmastery" / "progress" / "progress.json" + ) entries = json.loads(progress_json.read_text()) assert len(entries) == 1 entry = entries[0] diff --git a/tests/test_verify_json.py b/tests/test_verify_json.py new file mode 100644 index 0000000..44a9909 --- /dev/null +++ b/tests/test_verify_json.py @@ -0,0 +1,111 @@ +import importlib +import json +from datetime import datetime +from pathlib import Path + +import click +import pytest +from click.testing import CliRunner +from git_autograder import GitAutograderStatus +from git_autograder.output import GitAutograderOutput + +from app.cli import LoggingGroup, cli +from app.commands.verify import verify + +verify_module = importlib.import_module("app.commands.verify") +EXERCISE_NAME = "example" + + +@pytest.fixture +def exercise_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + metadata = tmp_path / ".gitmastery" + (metadata / "progress").mkdir(parents=True) + (metadata / "config.json").write_text("{}", encoding="utf-8") + exercise = tmp_path / EXERCISE_NAME + exercise.mkdir() + (exercise / ".gitmastery-exercise.json").write_text( + json.dumps( + { + "exercise_name": EXERCISE_NAME, + "tags": [], + "requires_git": False, + "requires_github": False, + "base_files": {}, + "exercise_repo": { + "repo_type": "ignore", + "repo_name": "unused", + "repo_title": None, + "create_fork": False, + "init": False, + }, + } + ), + encoding="utf-8", + ) + monkeypatch.chdir(exercise) + return exercise + + +@pytest.fixture +def graded_output() -> GitAutograderOutput: + return GitAutograderOutput( + status=GitAutograderStatus.SUCCESSFUL, + exercise_name=EXERCISE_NAME, + started_at=None, + completed_at=None, + comments=['First "comment"\nwith a newline', "こんにちは 😀"], + ) + + +@pytest.fixture +def application( + exercise_dir: Path, + graded_output: GitAutograderOutput, + monkeypatch: pytest.MonkeyPatch, +) -> click.Group: + group = LoggingGroup("gitmastery", callback=cli.callback, params=cli.params) + group.add_command(verify, aliases=["v"]) + monkeypatch.setattr( + "app.cli.fetch_latest_release_version", + lambda: (None, "Version check unavailable"), + ) + monkeypatch.setattr(verify_module, "get_username", lambda: "test-user") + + def grade( + exercise_path: Path, + exercise_name: str, + formatted_exercise_name: str, + started_at: datetime, + ) -> GitAutograderOutput: + print("Grader diagnostics") + return graded_output + + monkeypatch.setattr(verify_module, "_execute_verify", grade) + return group + + +def test_json_grading_result( + application: click.Group, + graded_output: GitAutograderOutput, +) -> None: + """JSON contains the grade and all comments, with diagnostics on stderr.""" + result = CliRunner().invoke(application, ["verify", "--json"], color=True) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout) == { + "exercise_name": EXERCISE_NAME, + "status": "Completed", + "comments": graded_output.comments, + } + assert "Version check unavailable" in result.stderr + assert "Grader diagnostics" in result.stderr + assert "Saving progress of attempt" in result.stderr + + +def test_json_without_comments( + application: click.Group, graded_output: GitAutograderOutput +) -> None: + """Absent feedback is an empty list, without the text renderer's placeholder.""" + graded_output.comments = None + result = CliRunner().invoke(application, ["verify", "--json"]) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout)["comments"] == []