diff --git a/api/app/settings/common.py b/api/app/settings/common.py index abbd9a1cb700..d36b8f64d5b3 100644 --- a/api/app/settings/common.py +++ b/api/app/settings/common.py @@ -414,7 +414,7 @@ "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", "simple_history.middleware.HistoryRequestMiddleware", - "telemetry.middleware.MCPUsageLoggerMiddleware", # Must come last! + "telemetry.middleware.ClientUsageLoggerMiddleware", # Must come last! ] ADD_NEVER_CACHE_HEADERS = env.bool("ADD_NEVER_CACHE_HEADERS", True) diff --git a/api/telemetry/middleware.py b/api/telemetry/middleware.py index 7754a686f523..d2279e01a0d5 100644 --- a/api/telemetry/middleware.py +++ b/api/telemetry/middleware.py @@ -1,4 +1,6 @@ +import re from collections.abc import Callable +from typing import Literal import structlog from django.http.request import HttpRequest @@ -7,9 +9,14 @@ from telemetry.spans import get_span_attribute +# e.g. `flagsmith-cli/2.0.0 (darwin/arm64)` +CLI_USER_AGENT_PATTERN = re.compile( + r"^flagsmith-cli/(?P[^\s/]+) \((?P[^\s/()]+)/(?P[^\s/()]+)\)" +) -class MCPUsageLoggerMiddleware: - """Emit telemetry events for MCP usage""" + +class ClientUsageLoggerMiddleware: + """Emit telemetry events for usage of Flagsmith clients, i.e. MCP and CLI""" def __init__( self, @@ -20,26 +27,54 @@ def __init__( def __call__(self, request: HttpRequest) -> HttpResponse: response = self.get_response(request) - if baggage.get_baggage("flagsmith.client.name") != "flagsmith-mcp": - return response + if baggage.get_baggage("flagsmith.client.name") == "flagsmith-mcp": + self._log_mcp_usage(request, response) + elif cli := CLI_USER_AGENT_PATTERN.match(request.headers.get("User-Agent", "")): + self._log_cli_usage(request, response, cli) + + return response + def _log_mcp_usage(self, request: HttpRequest, response: HttpResponse) -> None: if not request.user or not request.user.is_authenticated: - return response + return + # NOTE: The following W3C Baggage items are added by downstream processor + # - gen_ai.tool.name + # - flagsmith.mcp.client.name + # - flagsmith.mcp.client.version logger = structlog.get_logger("mcp") - event = { - # NOTE: The following W3C Baggage items are added by downstream processor - # - gen_ai.tool.name - # - flagsmith.mcp.client.name - # - flagsmith.mcp.client.version - "status": "error" if response.status_code >= 400 else "success", - } + logger = logger.bind(status=self._get_status(response)) if (org_id := self._get_organisation_id(request)) is not None: - logger.info("tool.called", organisation__id=org_id, **event) + logger.info("tool.called", organisation__id=org_id) else: - logger.warning("tool.called", organisation__id=None, **event) + logger.warning("tool.called", organisation__id=None) - return response + def _log_cli_usage( + self, + request: HttpRequest, + response: HttpResponse, + cli: re.Match[str], + ) -> None: + if not request.user or not request.user.is_authenticated: + return + + logger = structlog.get_logger("cli") + logger = logger.bind( + cli__version=cli["version"], + cli__os=cli["os"], + cli__arch=cli["arch"], + status=self._get_status(response), + ) + if not getattr(request.user, "is_master_api_key_user", False): + logger = logger.bind(amplitude__user_id=str(request.user.uuid)) + + if (org_id := self._get_organisation_id(request)) is not None: + logger.info("request.made", organisation__id=org_id) + else: + logger.warning("request.made", organisation__id=None) + + def _get_status(self, response: HttpResponse) -> Literal["success", "error"]: + return "error" if response.status_code >= 400 else "success" def _get_organisation_id(self, request: HttpRequest) -> int | None: """Obtain the organisation ID from the request context.""" @@ -51,6 +86,7 @@ def _get_organisation_id(self, request: HttpRequest) -> int | None: assert request.user.is_authenticated # NOTE: protected upstream try: # Most of the time, the user belongs to one organisation + # NOTE: Master API keys belong to exactly one organisation return request.user.organisations.get().id except ( Organisation.DoesNotExist, diff --git a/api/tests/unit/telemetry/test_unit_telemetry_middleware.py b/api/tests/unit/telemetry/test_unit_telemetry_middleware.py index 2ce11324edc0..9e6584299705 100644 --- a/api/tests/unit/telemetry/test_unit_telemetry_middleware.py +++ b/api/tests/unit/telemetry/test_unit_telemetry_middleware.py @@ -1,14 +1,19 @@ import pytest +from django.http import HttpResponse +from django.test import RequestFactory from pytest_structlog import StructuredLogCapture from rest_framework.test import APIClient from environments.models import Environment from organisations.models import Organisation, OrganisationRole +from telemetry.middleware import ClientUsageLoggerMiddleware from users.models import FFAdminUser +CLI_USER_AGENT = "flagsmith-cli/2.0.0 (darwin/arm64)" + @pytest.mark.usefixtures("organisation") -def test_mcp_usage_logger_middleware__no_mcp_baggage__logs_nothing( +def test_client_usage_logger_middleware__no_mcp_baggage__logs_nothing( staff_client: APIClient, log: StructuredLogCapture, ) -> None: @@ -21,7 +26,7 @@ def test_mcp_usage_logger_middleware__no_mcp_baggage__logs_nothing( @pytest.mark.usefixtures("mcp_baggage", "recording_span", "organisation") -def test_mcp_usage_logger_middleware__organisation_id_span_attribute__logs_span_organisation_id( +def test_client_usage_logger_middleware__mcp_organisation_id_span_attribute__logs_span_organisation_id( staff_client: APIClient, staff_user: FFAdminUser, log: StructuredLogCapture, @@ -48,7 +53,7 @@ def test_mcp_usage_logger_middleware__organisation_id_span_attribute__logs_span_ @pytest.mark.usefixtures("mcp_baggage") -def test_mcp_usage_logger_middleware__user_with_single_organisation__logs_user_organisation_id( +def test_client_usage_logger_middleware__mcp_user_with_single_organisation__logs_user_organisation_id( staff_client: APIClient, organisation: Organisation, log: StructuredLogCapture, @@ -69,7 +74,7 @@ def test_mcp_usage_logger_middleware__user_with_single_organisation__logs_user_o @pytest.mark.usefixtures("mcp_baggage") -def test_mcp_usage_logger_middleware__user_without_organisations__logs_warning( +def test_client_usage_logger_middleware__mcp_user_without_organisations__logs_warning( staff_client: APIClient, staff_user: FFAdminUser, log: StructuredLogCapture, @@ -93,7 +98,7 @@ def test_mcp_usage_logger_middleware__user_without_organisations__logs_warning( @pytest.mark.usefixtures("mcp_baggage", "organisation") -def test_mcp_usage_logger_middleware__user_with_multiple_organisations__logs_warning( +def test_client_usage_logger_middleware__mcp_user_with_multiple_organisations__logs_warning( staff_client: APIClient, staff_user: FFAdminUser, log: StructuredLogCapture, @@ -118,7 +123,7 @@ def test_mcp_usage_logger_middleware__user_with_multiple_organisations__logs_war @pytest.mark.usefixtures("mcp_baggage") -def test_mcp_usage_logger_middleware__unauthenticated_request__logs_nothing( +def test_client_usage_logger_middleware__mcp_unauthenticated_request__logs_nothing( api_client: APIClient, log: StructuredLogCapture, ) -> None: @@ -131,7 +136,7 @@ def test_mcp_usage_logger_middleware__unauthenticated_request__logs_nothing( @pytest.mark.usefixtures("mcp_baggage") -def test_mcp_usage_logger_middleware__error_response__logs_error_status( +def test_client_usage_logger_middleware__mcp_error_response__logs_error_status( staff_client: APIClient, organisation: Organisation, log: StructuredLogCapture, @@ -152,7 +157,7 @@ def test_mcp_usage_logger_middleware__error_response__logs_error_status( @pytest.mark.usefixtures("mcp_baggage") -def test_mcp_usage_logger_middleware__sdk_request__logs_nothing( +def test_client_usage_logger_middleware__mcp_sdk_request__logs_nothing( api_client: APIClient, environment: Environment, log: StructuredLogCapture, @@ -166,3 +171,307 @@ def test_mcp_usage_logger_middleware__sdk_request__logs_nothing( # Then assert response.status_code == 200 assert log.events == [] + + +@pytest.mark.usefixtures("organisation") +def test_client_usage_logger_middleware__no_cli_user_agent__logs_nothing( + staff_client: APIClient, + log: StructuredLogCapture, +) -> None: + # Given / When + response = staff_client.get("/api/v1/projects/") + + # Then + assert response.status_code == 200 + assert log.events == [] + + +@pytest.mark.parametrize( + "user_agent", + [ + "python-requests/2.32.3", + "flagsmith-mcp/1.0.0 (linux/amd64)", + "flagsmith-python-sdk/5.0.0", + "flagsmith-cli", + "flagsmith-cli/2.0.0", + "flagsmith-cli/2.0.0 (darwin)", + "flagsmith-cli/ (darwin/arm64)", + "not-flagsmith-cli/2.0.0 (darwin/arm64)", + "Mozilla/5.0 flagsmith-cli/2.0.0 (darwin/arm64)", + ], +) +@pytest.mark.usefixtures("organisation") +def test_client_usage_logger_middleware__user_agent_not_matching_cli_pattern__logs_nothing( + staff_client: APIClient, + log: StructuredLogCapture, + user_agent: str, +) -> None: + # Given / When + response = staff_client.get("/api/v1/projects/", HTTP_USER_AGENT=user_agent) + + # Then + assert response.status_code == 200 + assert log.events == [] + + +@pytest.mark.usefixtures("recording_span", "organisation") +def test_client_usage_logger_middleware__cli_organisation_id_span_attribute__logs_span_organisation_id( + staff_client: APIClient, + staff_user: FFAdminUser, + log: StructuredLogCapture, +) -> None: + # Given + other_organisation = Organisation.objects.create(name="Other Org") + staff_user.add_organisation(other_organisation, role=OrganisationRole.ADMIN) + + # When + response = staff_client.get( + f"/api/v1/organisations/{other_organisation.pk}/invites/", + HTTP_USER_AGENT=CLI_USER_AGENT, + ) + + # Then + assert response.status_code == 200 + assert log.events == [ + { + "level": "info", + "event": "request.made", + "cli__version": "2.0.0", + "cli__os": "darwin", + "cli__arch": "arm64", + "organisation__id": other_organisation.pk, + "status": "success", + "amplitude__user_id": str(staff_user.uuid), + } + ] + + +def test_client_usage_logger_middleware__cli_user_with_single_organisation__logs_user_organisation_id( + staff_client: APIClient, + staff_user: FFAdminUser, + organisation: Organisation, + log: StructuredLogCapture, +) -> None: + # Given / When + response = staff_client.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) + + # Then + assert response.status_code == 200 + assert log.events == [ + { + "level": "info", + "event": "request.made", + "cli__version": "2.0.0", + "cli__os": "darwin", + "cli__arch": "arm64", + "organisation__id": organisation.pk, + "status": "success", + "amplitude__user_id": str(staff_user.uuid), + } + ] + + +def test_client_usage_logger_middleware__cli_user_without_organisations__logs_warning( + staff_client: APIClient, + staff_user: FFAdminUser, + log: StructuredLogCapture, +) -> None: + # Given + assert not staff_user.organisations.exists() + + # When + response = staff_client.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) + + # Then + assert response.status_code == 200 + assert log.events == [ + { + "level": "warning", + "event": "request.made", + "cli__version": "2.0.0", + "cli__os": "darwin", + "cli__arch": "arm64", + "organisation__id": None, + "status": "success", + "amplitude__user_id": str(staff_user.uuid), + } + ] + + +@pytest.mark.usefixtures("organisation") +def test_client_usage_logger_middleware__cli_user_with_multiple_organisations__logs_warning( + staff_client: APIClient, + staff_user: FFAdminUser, + log: StructuredLogCapture, +) -> None: + # Given + other_organisation = Organisation.objects.create(name="Other Org") + staff_user.add_organisation(other_organisation, role=OrganisationRole.USER) + + # When + response = staff_client.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) + + # Then + assert response.status_code == 200 + assert log.events == [ + { + "level": "warning", + "event": "request.made", + "cli__version": "2.0.0", + "cli__os": "darwin", + "cli__arch": "arm64", + "organisation__id": None, + "status": "success", + "amplitude__user_id": str(staff_user.uuid), + } + ] + + +def test_client_usage_logger_middleware__cli_master_api_key__logs_key_organisation_id( + admin_master_api_key_client: APIClient, + organisation: Organisation, + log: StructuredLogCapture, +) -> None: + # Given / When + response = admin_master_api_key_client.get( + "/api/v1/projects/", + HTTP_USER_AGENT=CLI_USER_AGENT, + ) + + # Then + assert response.status_code == 200 + assert log.events == [ + { + "level": "info", + "event": "request.made", + "cli__version": "2.0.0", + "cli__os": "darwin", + "cli__arch": "arm64", + "organisation__id": organisation.pk, + "status": "success", + } + ] + + +def test_client_usage_logger_middleware__cli_unauthenticated_request__logs_nothing( + api_client: APIClient, + log: StructuredLogCapture, +) -> None: + # Given / When + response = api_client.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) + + # Then + assert response.status_code == 401 + assert log.events == [] + + +def test_client_usage_logger_middleware__cli_error_response__logs_error_status( + staff_client: APIClient, + staff_user: FFAdminUser, + organisation: Organisation, + log: StructuredLogCapture, +) -> None: + # Given / When + response = staff_client.get( + f"/api/v1/organisations/{organisation.pk}/invites/", + HTTP_USER_AGENT=CLI_USER_AGENT, + ) + + # Then + assert response.status_code == 403 + assert log.events == [ + { + "level": "info", + "event": "request.made", + "cli__version": "2.0.0", + "cli__os": "darwin", + "cli__arch": "arm64", + "organisation__id": organisation.pk, + "status": "error", + "amplitude__user_id": str(staff_user.uuid), + } + ] + + +def test_client_usage_logger_middleware__cli_server_error_response__logs_error_status( + rf: RequestFactory, + staff_user: FFAdminUser, + organisation: Organisation, + log: StructuredLogCapture, +) -> None: + # Given + request = rf.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) + request.user = staff_user + middleware = ClientUsageLoggerMiddleware(lambda request: HttpResponse(status=500)) + + # When + response = middleware(request) + + # Then + assert response.status_code == 500 + assert log.events == [ + { + "level": "info", + "event": "request.made", + "cli__version": "2.0.0", + "cli__os": "darwin", + "cli__arch": "arm64", + "organisation__id": organisation.pk, + "status": "error", + "amplitude__user_id": str(staff_user.uuid), + } + ] + + +def test_client_usage_logger_middleware__cli_sdk_request__logs_nothing( + api_client: APIClient, + environment: Environment, + log: StructuredLogCapture, +) -> None: + # Given / When + response = api_client.get( + "/api/v1/flags/", + HTTP_X_ENVIRONMENT_KEY=environment.api_key, + HTTP_USER_AGENT=CLI_USER_AGENT, + ) + + # Then + assert response.status_code == 200 + assert log.events == [] + + +@pytest.mark.parametrize( + "user_agent, expected_version, expected_os, expected_arch", + [ + ("flagsmith-cli/2.0.0-beta.3 (linux/amd64)", "2.0.0-beta.3", "linux", "amd64"), + ("flagsmith-cli/dev (windows/386)", "dev", "windows", "386"), + ("flagsmith-cli/99.0.0 (freebsd/riscv64)", "99.0.0", "freebsd", "riscv64"), + ], +) +def test_client_usage_logger_middleware__cli_user_agent__passes_version_through_unchanged( + staff_client: APIClient, + staff_user: FFAdminUser, + organisation: Organisation, + log: StructuredLogCapture, + user_agent: str, + expected_version: str, + expected_os: str, + expected_arch: str, +) -> None: + # Given / When + response = staff_client.get("/api/v1/projects/", HTTP_USER_AGENT=user_agent) + + # Then + assert response.status_code == 200 + assert log.events == [ + { + "level": "info", + "event": "request.made", + "cli__version": expected_version, + "cli__os": expected_os, + "cli__arch": expected_arch, + "organisation__id": organisation.pk, + "status": "success", + "amplitude__user_id": str(staff_user.uuid), + } + ] diff --git a/docs/docs/deployment-self-hosting/observability/_events-catalogue.md b/docs/docs/deployment-self-hosting/observability/_events-catalogue.md index 3a2cd944daea..66ec5db22d43 100644 --- a/docs/docs/deployment-self-hosting/observability/_events-catalogue.md +++ b/docs/docs/deployment-self-hosting/observability/_events-catalogue.md @@ -148,6 +148,20 @@ Attributes: - `seats.previous` - `subscription.id` +### `cli.request.made` + +Logged at `info` from: + - `api/telemetry/middleware.py:72` + - `api/telemetry/middleware.py:74` + +Attributes: + - `amplitude.user_id` + - `cli.arch` + - `cli.os` + - `cli.version` + - `organisation.id` + - `status` + ### `code_references.cleanup_issues.created` Logged at `info` from: @@ -728,11 +742,12 @@ Attributes: ### `mcp.tool.called` Logged at `info` from: - - `api/telemetry/middleware.py:38` - - `api/telemetry/middleware.py:40` + - `api/telemetry/middleware.py:48` + - `api/telemetry/middleware.py:50` Attributes: - `organisation.id` + - `status` ### `oauth2_metadata.registration.rejected`