Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions src/late/mcp/middleware/tool_telemetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,19 @@
from typing import Any

import sentry_sdk
from fastmcp.exceptions import ValidationError as InvalidArgumentsError
from fastmcp.server.auth import AccessToken
from fastmcp.server.dependencies import get_access_token
from fastmcp.server.middleware import CallNext, Middleware, MiddlewareContext
from mcp.types import CallToolRequestParams, TextContent
from pydantic import ValidationError as PydanticValidationError

from late.client.exceptions import LateAPIError, LateRateLimitError
from late.mcp.auth import token_cache_key
from late.mcp.response_format import format_stats
from late.mcp.telemetry import (
TOOL_TRANSACTION_OP,
argument_errors_without_input,
is_expected_failure,
tool_call_logger,
)
Expand Down Expand Up @@ -90,8 +93,25 @@ def _result_chars(result: Any) -> int:
return sum(len(block.text) for block in content if isinstance(block, TextContent))


def _invalid_arguments_fields(exc: InvalidArgumentsError) -> dict[str, Any]:
# Named apart from ValidationError, which an SDK model rejecting an API
# response also raises: this one never reached the API.
fields: dict[str, Any] = {"error_type": "InvalidArguments"}
if isinstance(exc.__cause__, PydanticValidationError):
errors = exc.__cause__.errors(
include_url=False, include_context=False, include_input=False
)
fields["invalid_arguments"] = argument_errors_without_input(errors)
return fields


def _error_fields(exc: Exception) -> dict[str, Any]:
if isinstance(exc, InvalidArgumentsError):
return _invalid_arguments_fields(exc)
cause = exc.__cause__ if isinstance(exc.__cause__, Exception) else exc
# FastMCP 4 tags a pydantic error raised by a tool body as _ToolBodyError.
if isinstance(cause.__cause__, PydanticValidationError):
cause = cause.__cause__
fields: dict[str, Any] = {"error_type": type(cause).__name__}
if isinstance(cause, LateAPIError):
fields["status_code"] = cause.status_code
Expand Down Expand Up @@ -180,6 +200,11 @@ async def _record(
raise
except Exception as exc:
fields = {"outcome": "error", **_error_fields(exc)}
transaction.set_tag("mcp.error_type", fields["error_type"])
if "invalid_arguments" in fields:
transaction.set_tag(
"mcp.invalid_arguments", fields["invalid_arguments"]
)
if not is_expected_failure(exc):
sentry_sdk.capture_exception(exc)
raise
Expand Down
6 changes: 3 additions & 3 deletions src/late/mcp/telemetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ def filter(self, record: logging.LogRecord) -> bool:
return status_code >= 500


def _argument_errors_without_input(detail: Any) -> str:
def argument_errors_without_input(detail: Any) -> str:
# FastMCP 4 logs its own input-free summary: {"error_count", "error_types"}.
if isinstance(detail, dict) and isinstance(detail.get("error_types"), list):
return ", ".join(str(error_type) for error_type in detail["error_types"])
Expand All @@ -87,7 +87,7 @@ def filter(self, record: logging.LogRecord) -> bool:
message = str(record.msg)
if message.startswith(_INVALID_ARGUMENTS_LOG_PREFIX):
tool, detail = record.args
record.args = (tool, _argument_errors_without_input(detail))
record.args = (tool, argument_errors_without_input(detail))
return True
if not message.startswith(_TOOL_ERROR_LOG_PREFIX):
return True
Expand All @@ -98,7 +98,7 @@ def filter(self, record: logging.LogRecord) -> bool:
# An SDK model rejecting an API response: its traceback would log
# the response values, so keep only where and why it failed.
errors = rejection.errors(include_url=False, include_context=False, include_input=False)
record.msg, record.args = "%s: %s", (message, _argument_errors_without_input(errors))
record.msg, record.args = "%s: %s", (message, argument_errors_without_input(errors))
record.exc_info = None
return True
return isinstance(exc, Exception) and not is_expected_failure(exc)
Expand Down
62 changes: 61 additions & 1 deletion tests/test_mcp_telemetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import logging
from datetime import datetime
from types import SimpleNamespace
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Literal

import httpx
import pytest
Expand Down Expand Up @@ -125,6 +125,66 @@ async def test_every_tool_call_logs_its_outcome_and_latency(
assert fields.items() <= line.items()


async def test_rejected_arguments_log_which_fields_failed_without_values(
caplog,
) -> None:
caplog.set_level(logging.INFO, logger="late.mcp.tools")
server = _server()

@server.tool
def filtered(status: Literal["active", "paused"], limit: int = 10) -> str:
return f"{status} {limit}"

async with Client(server) as client:
await client.call_tool(
"filtered", {"status": "not_deleted", "limit": "many"}, raise_on_error=False
)

[line] = _logged(caplog)
assert line["outcome"] == "error"
assert line["error_type"] == "InvalidArguments"
assert line["invalid_arguments"] == "status: literal_error, limit: int_parsing"
assert "status_code" not in line
assert "not_deleted" not in json.dumps(line)


async def test_an_sdk_model_rejecting_a_response_keeps_its_validation_error_type(
caplog,
) -> None:
caplog.set_level(logging.INFO, logger="late.mcp.tools")
server = _server()

@server.tool
def rejects_response() -> str:
AccountsListResponse.model_validate({"accounts": "nope"})
return "unreachable"

async with Client(server) as client:
await client.call_tool("rejects_response", {}, raise_on_error=False)

[line] = _logged(caplog)
assert line["error_type"] == "ValidationError"
assert "invalid_arguments" not in line


async def test_rejected_arguments_tag_the_sentry_transaction(sentry_events) -> None:
server = _server()

@server.tool
def takes_a_number(value: int) -> int:
return value

async with Client(server) as client:
await client.call_tool(
"takes_a_number", {"value": "nope"}, raise_on_error=False
)

[transaction] = sentry_events.payloads("transaction")
assert transaction["tags"]["mcp.error_type"] == "InvalidArguments"
assert transaction["tags"]["mcp.invalid_arguments"] == "value: int_parsing"
assert sentry_events.payloads("event") == []


def test_sentry_stays_off_without_a_dsn(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv("SENTRY_DSN", raising=False)
assert init_sentry() is False
Expand Down
Loading