Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,25 @@ Refer to the [installation guide](https://git-mastery.org/companion-app/index.ht

To develop the app locally, refer to the [development setup guide](https://git-mastery.org/developers/docs/getting-started/setup/) for detailed instructions on how to set up your development environment and contribute to the project.

## JSON verification output

Run `gitmastery verify --json` inside an exercise to get its grading result on
stdout:

```json
{"exercise_name": "under-control", "status": "Completed", "comments": []}
```

`status` is `Completed`, `Incomplete`, or `Error`. `comments` contains all grader
feedback in order, or an empty array when there is none. Diagnostic messages go
to stderr, so clients can parse the complete stdout as JSON.

Verification still saves progress and performs any configured remote submission.
Exit `0` means the command finished normally; read `status` to determine whether
the answer is correct. A failure before grading leaves stdout empty and exits
nonzero. A submission failure after grading retains the JSON result and exits
nonzero. Ordinary `gitmastery verify` keeps its existing text output.

## Publishing

Tag the PR with the `bump:{major | minor | patch}` label and the CI will automatically perform the publish.
Expand Down
22 changes: 21 additions & 1 deletion app/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from app.commands import check, download, progress, setup, verify
from app.commands.repl import repl
from app.commands.version import version
from app.utils.click import ClickColor, CliContextKey, warn
from app.utils.click import ClickColor, CliContextKey, json_output_scope, warn
from app.utils.version import Version, fetch_latest_release_version
from app.version import __version__

Expand All @@ -19,6 +19,26 @@ def invoke(self, ctx: click.Context) -> None:
logger.info("Running command %s with arguments %s", ctx.command_path, sys.argv)
return super().invoke(ctx)

def resolve_command(
self, ctx: click.Context, args: list[str]
) -> tuple[str | None, click.Command | None, list[str]]:
command_name, command, command_args = super().resolve_command(ctx, args)
if command is verify:
# Resolve aliases and parse options before the group callback prints
# version warnings. Resilient parsing leaves help/errors to Click.
with command.make_context(
command_name or "verify",
list(command_args),
parent=ctx,
resilient_parsing=True,
) as command_ctx:
requesting_help = any(
arg in command_ctx.help_option_names for arg in command_args
)
if command_ctx.params.get("json_output") and not requesting_help:
ctx.with_resource(json_output_scope())
return command_name, command, command_args


CONTEXT_SETTINGS = {"max_content_width": 120}

Expand Down
33 changes: 31 additions & 2 deletions app/commands/verify.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,12 @@
from app.utils.click import (
ClickColor,
error,
get_json_output_stream,
info,
must_get_exercise_root_config,
must_get_gitmastery_root_config,
warn,
with_json_output,
)
from app.utils.git import add_all, commit, push
from app.utils.github_cli import get_prs, get_username, pull_request
Expand Down Expand Up @@ -69,6 +71,22 @@ def _print_output(output: GitAutograderOutput) -> None:
)


def _print_json_output(output: GitAutograderOutput, exercise_name: str) -> None:
statuses = {
GitAutograderStatus.SUCCESSFUL: "Completed",
GitAutograderStatus.UNSUCCESSFUL: "Incomplete",
GitAutograderStatus.ERROR: "Error",
}
payload = {
"exercise_name": exercise_name,
"status": statuses[output.status],
"comments": output.comments or [],
}
stdout = get_json_output_stream()
click.echo(json.dumps(payload), file=stdout, color=False)
stdout.flush()


def _submit_progress(output: GitAutograderOutput) -> None:
username = get_username()

Expand Down Expand Up @@ -183,9 +201,17 @@ def _execute_verify(


@click.command()
@click.option(
"--json",
"json_output",
is_flag=True,
default=False,
help="Output the grading result as JSON; send diagnostics to stderr.",
)
@with_json_output
@in_exercise_root()
@in_gitmastery_root()
def verify() -> None:
def verify(json_output: bool = False) -> None:
"""
Verifies the state of the exercise attempt.
"""
Expand All @@ -204,5 +230,8 @@ def verify() -> None:
output = _execute_verify(
exercise_path, exercise_name, formatted_exercise_name, started_at
)
_print_output(output)
if json_output:
_print_json_output(output, exercise_name)
else:
_print_output(output)
_submit_progress(output)
48 changes: 47 additions & 1 deletion app/utils/click.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
import logging
import sys
from collections.abc import Callable, Iterator
from contextlib import contextmanager, redirect_stdout
from contextvars import ContextVar
from enum import StrEnum
from typing import Any, NoReturn, Optional
from functools import wraps
from typing import Any, NoReturn, Optional, ParamSpec, TextIO, TypeVar

import click

Expand All @@ -10,6 +14,48 @@

logger = logging.getLogger(__name__)

_JSON_STDOUT: ContextVar[Optional[TextIO]] = ContextVar("json_stdout", default=None)
P = ParamSpec("P")
T = TypeVar("T")


@contextmanager
def json_output_scope(enabled: bool = True) -> Iterator[TextIO]:
"""Reserve stdout for JSON and route Python diagnostics to stderr.

:param enabled: Whether to enable JSON output routing.
:yield: The original stdout stream for the JSON result.
"""
saved_stdout = _JSON_STDOUT.get()
if not enabled or saved_stdout is not None:
yield saved_stdout if saved_stdout is not None else sys.stdout
return

saved_stdout = sys.stdout
token = _JSON_STDOUT.set(saved_stdout)
try:
with redirect_stdout(sys.stderr):
yield saved_stdout
finally:
_JSON_STDOUT.reset(token)


def with_json_output(func: Callable[P, T]) -> Callable[P, T]:
"""Route diagnostics before command guards when ``json_output`` is enabled."""

@wraps(func)
def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
with json_output_scope(bool(kwargs.get("json_output", False))):
return func(*args, **kwargs)

return wrapper


def get_json_output_stream() -> TextIO:
"""Return the stdout stream reserved for the JSON result."""
saved_stdout = _JSON_STDOUT.get()
return saved_stdout if saved_stdout is not None else sys.stdout


class CliContextKey(StrEnum):
GITMASTERY_ROOT_CONFIG = "GITMASTERY_ROOT_CONFIG"
Expand Down
4 changes: 3 additions & 1 deletion tests/e2e/commands/test_verify.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@

def test_verify_exercise(verified_exercise_dir: Path) -> None:
"""verify writes a progress entry with the expected fields."""
progress_json = verified_exercise_dir.parent / ".gitmastery" / "progress" / "progress.json"
progress_json = (
verified_exercise_dir.parent / ".gitmastery" / "progress" / "progress.json"
)
entries = json.loads(progress_json.read_text())
assert len(entries) == 1
entry = entries[0]
Expand Down
111 changes: 111 additions & 0 deletions tests/test_verify_json.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
import importlib
import json
from datetime import datetime
from pathlib import Path

import click
import pytest
from click.testing import CliRunner
from git_autograder import GitAutograderStatus
from git_autograder.output import GitAutograderOutput

from app.cli import LoggingGroup, cli
from app.commands.verify import verify

verify_module = importlib.import_module("app.commands.verify")
EXERCISE_NAME = "example"


@pytest.fixture
def exercise_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
metadata = tmp_path / ".gitmastery"
(metadata / "progress").mkdir(parents=True)
(metadata / "config.json").write_text("{}", encoding="utf-8")
exercise = tmp_path / EXERCISE_NAME
exercise.mkdir()
(exercise / ".gitmastery-exercise.json").write_text(
json.dumps(
{
"exercise_name": EXERCISE_NAME,
"tags": [],
"requires_git": False,
"requires_github": False,
"base_files": {},
"exercise_repo": {
"repo_type": "ignore",
"repo_name": "unused",
"repo_title": None,
"create_fork": False,
"init": False,
},
}
),
encoding="utf-8",
)
monkeypatch.chdir(exercise)
return exercise


@pytest.fixture
def graded_output() -> GitAutograderOutput:
return GitAutograderOutput(
status=GitAutograderStatus.SUCCESSFUL,
exercise_name=EXERCISE_NAME,
started_at=None,
completed_at=None,
comments=['First "comment"\nwith a newline', "こんにちは 😀"],
)


@pytest.fixture
def application(
exercise_dir: Path,
graded_output: GitAutograderOutput,
monkeypatch: pytest.MonkeyPatch,
) -> click.Group:
group = LoggingGroup("gitmastery", callback=cli.callback, params=cli.params)
group.add_command(verify, aliases=["v"])
monkeypatch.setattr(
"app.cli.fetch_latest_release_version",
lambda: (None, "Version check unavailable"),
)
monkeypatch.setattr(verify_module, "get_username", lambda: "test-user")

def grade(
exercise_path: Path,
exercise_name: str,
formatted_exercise_name: str,
started_at: datetime,
) -> GitAutograderOutput:
print("Grader diagnostics")
return graded_output

monkeypatch.setattr(verify_module, "_execute_verify", grade)
return group


def test_json_grading_result(
application: click.Group,
graded_output: GitAutograderOutput,
) -> None:
"""JSON contains the grade and all comments, with diagnostics on stderr."""
result = CliRunner().invoke(application, ["verify", "--json"], color=True)
assert result.exit_code == 0, result.output
assert json.loads(result.stdout) == {
"exercise_name": EXERCISE_NAME,
"status": "Completed",
"comments": graded_output.comments,
}
assert "Version check unavailable" in result.stderr
assert "Grader diagnostics" in result.stderr
assert "Saving progress of attempt" in result.stderr


def test_json_without_comments(
application: click.Group, graded_output: GitAutograderOutput
) -> None:
"""Absent feedback is an empty list, without the text renderer's placeholder."""
graded_output.comments = None
result = CliRunner().invoke(application, ["verify", "--json"])
assert result.exit_code == 0, result.output
assert json.loads(result.stdout)["comments"] == []
Loading