diff --git a/src/late/mcp/middleware/tool_telemetry.py b/src/late/mcp/middleware/tool_telemetry.py index 30ce14e6..cac79929 100644 --- a/src/late/mcp/middleware/tool_telemetry.py +++ b/src/late/mcp/middleware/tool_telemetry.py @@ -9,16 +9,19 @@ from typing import Any import sentry_sdk +from fastmcp.exceptions import ValidationError as InvalidArgumentsError from fastmcp.server.auth import AccessToken from fastmcp.server.dependencies import get_access_token from fastmcp.server.middleware import CallNext, Middleware, MiddlewareContext from mcp.types import CallToolRequestParams, TextContent +from pydantic import ValidationError as PydanticValidationError from late.client.exceptions import LateAPIError, LateRateLimitError from late.mcp.auth import token_cache_key from late.mcp.response_format import format_stats from late.mcp.telemetry import ( TOOL_TRANSACTION_OP, + argument_errors_without_input, is_expected_failure, tool_call_logger, ) @@ -90,8 +93,25 @@ def _result_chars(result: Any) -> int: return sum(len(block.text) for block in content if isinstance(block, TextContent)) +def _invalid_arguments_fields(exc: InvalidArgumentsError) -> dict[str, Any]: + # Named apart from ValidationError, which an SDK model rejecting an API + # response also raises: this one never reached the API. + fields: dict[str, Any] = {"error_type": "InvalidArguments"} + if isinstance(exc.__cause__, PydanticValidationError): + errors = exc.__cause__.errors( + include_url=False, include_context=False, include_input=False + ) + fields["invalid_arguments"] = argument_errors_without_input(errors) + return fields + + def _error_fields(exc: Exception) -> dict[str, Any]: + if isinstance(exc, InvalidArgumentsError): + return _invalid_arguments_fields(exc) cause = exc.__cause__ if isinstance(exc.__cause__, Exception) else exc + # FastMCP 4 tags a pydantic error raised by a tool body as _ToolBodyError. + if isinstance(cause.__cause__, PydanticValidationError): + cause = cause.__cause__ fields: dict[str, Any] = {"error_type": type(cause).__name__} if isinstance(cause, LateAPIError): fields["status_code"] = cause.status_code @@ -180,6 +200,11 @@ async def _record( raise except Exception as exc: fields = {"outcome": "error", **_error_fields(exc)} + transaction.set_tag("mcp.error_type", fields["error_type"]) + if "invalid_arguments" in fields: + transaction.set_tag( + "mcp.invalid_arguments", fields["invalid_arguments"] + ) if not is_expected_failure(exc): sentry_sdk.capture_exception(exc) raise diff --git a/src/late/mcp/telemetry.py b/src/late/mcp/telemetry.py index dd6f79b8..80934370 100644 --- a/src/late/mcp/telemetry.py +++ b/src/late/mcp/telemetry.py @@ -70,7 +70,7 @@ def filter(self, record: logging.LogRecord) -> bool: return status_code >= 500 -def _argument_errors_without_input(detail: Any) -> str: +def argument_errors_without_input(detail: Any) -> str: # FastMCP 4 logs its own input-free summary: {"error_count", "error_types"}. if isinstance(detail, dict) and isinstance(detail.get("error_types"), list): return ", ".join(str(error_type) for error_type in detail["error_types"]) @@ -87,7 +87,7 @@ def filter(self, record: logging.LogRecord) -> bool: message = str(record.msg) if message.startswith(_INVALID_ARGUMENTS_LOG_PREFIX): tool, detail = record.args - record.args = (tool, _argument_errors_without_input(detail)) + record.args = (tool, argument_errors_without_input(detail)) return True if not message.startswith(_TOOL_ERROR_LOG_PREFIX): return True @@ -98,7 +98,7 @@ def filter(self, record: logging.LogRecord) -> bool: # An SDK model rejecting an API response: its traceback would log # the response values, so keep only where and why it failed. errors = rejection.errors(include_url=False, include_context=False, include_input=False) - record.msg, record.args = "%s: %s", (message, _argument_errors_without_input(errors)) + record.msg, record.args = "%s: %s", (message, argument_errors_without_input(errors)) record.exc_info = None return True return isinstance(exc, Exception) and not is_expected_failure(exc) diff --git a/tests/test_mcp_telemetry.py b/tests/test_mcp_telemetry.py index d0f1863a..5736609c 100644 --- a/tests/test_mcp_telemetry.py +++ b/tests/test_mcp_telemetry.py @@ -5,7 +5,7 @@ import logging from datetime import datetime from types import SimpleNamespace -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal import httpx import pytest @@ -125,6 +125,66 @@ async def test_every_tool_call_logs_its_outcome_and_latency( assert fields.items() <= line.items() +async def test_rejected_arguments_log_which_fields_failed_without_values( + caplog, +) -> None: + caplog.set_level(logging.INFO, logger="late.mcp.tools") + server = _server() + + @server.tool + def filtered(status: Literal["active", "paused"], limit: int = 10) -> str: + return f"{status} {limit}" + + async with Client(server) as client: + await client.call_tool( + "filtered", {"status": "not_deleted", "limit": "many"}, raise_on_error=False + ) + + [line] = _logged(caplog) + assert line["outcome"] == "error" + assert line["error_type"] == "InvalidArguments" + assert line["invalid_arguments"] == "status: literal_error, limit: int_parsing" + assert "status_code" not in line + assert "not_deleted" not in json.dumps(line) + + +async def test_an_sdk_model_rejecting_a_response_keeps_its_validation_error_type( + caplog, +) -> None: + caplog.set_level(logging.INFO, logger="late.mcp.tools") + server = _server() + + @server.tool + def rejects_response() -> str: + AccountsListResponse.model_validate({"accounts": "nope"}) + return "unreachable" + + async with Client(server) as client: + await client.call_tool("rejects_response", {}, raise_on_error=False) + + [line] = _logged(caplog) + assert line["error_type"] == "ValidationError" + assert "invalid_arguments" not in line + + +async def test_rejected_arguments_tag_the_sentry_transaction(sentry_events) -> None: + server = _server() + + @server.tool + def takes_a_number(value: int) -> int: + return value + + async with Client(server) as client: + await client.call_tool( + "takes_a_number", {"value": "nope"}, raise_on_error=False + ) + + [transaction] = sentry_events.payloads("transaction") + assert transaction["tags"]["mcp.error_type"] == "InvalidArguments" + assert transaction["tags"]["mcp.invalid_arguments"] == "value: int_parsing" + assert sentry_events.payloads("event") == [] + + def test_sentry_stays_off_without_a_dsn(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("SENTRY_DSN", raising=False) assert init_sentry() is False