From 86d52ccdd9f8043d5737b3788988e1b4a7b3c75a Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Wed, 18 Nov 2026 16:17:40 +0100 Subject: [PATCH 1/3] test(Telemetry): Specify CLI usage events middleware Cover the cli.request.made event emitted for authenticated Flagsmith CLI requests: the user-agent pattern, status, organisation resolution for users and master API keys, and that the key paths read no database. --- .../test_unit_telemetry_middleware.py | 357 ++++++++++++++++++ 1 file changed, 357 insertions(+) diff --git a/api/tests/unit/telemetry/test_unit_telemetry_middleware.py b/api/tests/unit/telemetry/test_unit_telemetry_middleware.py index 2ce11324edc0..ff94f827423f 100644 --- a/api/tests/unit/telemetry/test_unit_telemetry_middleware.py +++ b/api/tests/unit/telemetry/test_unit_telemetry_middleware.py @@ -1,11 +1,20 @@ import pytest +from django.http import HttpResponse +from django.test import RequestFactory +from opentelemetry.trace import Span +from pytest_django import DjangoAssertNumQueries from pytest_structlog import StructuredLogCapture from rest_framework.test import APIClient +from api_keys.models import MasterAPIKey +from api_keys.user import APIKeyUser from environments.models import Environment from organisations.models import Organisation, OrganisationRole +from telemetry.middleware import CLIUsageLoggerMiddleware from users.models import FFAdminUser +CLI_USER_AGENT = "flagsmith-cli/2.0.0 (darwin/arm64)" + @pytest.mark.usefixtures("organisation") def test_mcp_usage_logger_middleware__no_mcp_baggage__logs_nothing( @@ -166,3 +175,351 @@ def test_mcp_usage_logger_middleware__sdk_request__logs_nothing( # Then assert response.status_code == 200 assert log.events == [] + + +@pytest.mark.usefixtures("organisation") +def test_cli_usage_logger_middleware__non_cli_request__logs_nothing( + staff_client: APIClient, + log: StructuredLogCapture, +) -> None: + # Given / When + response = staff_client.get("/api/v1/projects/") + + # Then + assert response.status_code == 200 + assert log.events == [] + + +@pytest.mark.parametrize( + "user_agent", + [ + "python-requests/2.32.3", + "flagsmith-mcp/1.0.0 (linux/amd64)", + "flagsmith-python-sdk/5.0.0", + "flagsmith-cli", + "flagsmith-cli/2.0.0", + "flagsmith-cli/2.0.0 (darwin)", + "flagsmith-cli/ (darwin/arm64)", + "not-flagsmith-cli/2.0.0 (darwin/arm64)", + "Mozilla/5.0 flagsmith-cli/2.0.0 (darwin/arm64)", + ], +) +@pytest.mark.usefixtures("organisation") +def test_cli_usage_logger_middleware__user_agent_not_matching_cli_pattern__logs_nothing( + staff_client: APIClient, + log: StructuredLogCapture, + user_agent: str, +) -> None: + # Given / When + response = staff_client.get("/api/v1/projects/", HTTP_USER_AGENT=user_agent) + + # Then + assert response.status_code == 200 + assert log.events == [] + + +def test_cli_usage_logger_middleware__user_with_single_organisation__logs_event( + staff_client: APIClient, + staff_user: FFAdminUser, + organisation: Organisation, + log: StructuredLogCapture, +) -> None: + # Given / When + response = staff_client.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) + + # Then + assert response.status_code == 200 + assert log.events == [ + { + "level": "info", + "event": "cli.request.made", + "cli__version": "2.0.0", + "cli__os": "darwin", + "cli__arch": "arm64", + "organisation__id": organisation.pk, + "status": "success", + "amplitude__user_id": str(staff_user.uuid), + } + ] + + +@pytest.mark.usefixtures("recording_span", "organisation") +def test_cli_usage_logger_middleware__organisation_id_span_attribute__logs_span_organisation_id( + staff_client: APIClient, + staff_user: FFAdminUser, + log: StructuredLogCapture, +) -> None: + # Given + other_organisation = Organisation.objects.create(name="Other Org") + staff_user.add_organisation(other_organisation, role=OrganisationRole.ADMIN) + + # When + response = staff_client.get( + f"/api/v1/organisations/{other_organisation.pk}/invites/", + HTTP_USER_AGENT=CLI_USER_AGENT, + ) + + # Then + assert response.status_code == 200 + assert log.events == [ + { + "level": "info", + "event": "cli.request.made", + "cli__version": "2.0.0", + "cli__os": "darwin", + "cli__arch": "arm64", + "organisation__id": other_organisation.pk, + "status": "success", + "amplitude__user_id": str(staff_user.uuid), + } + ] + + +@pytest.mark.usefixtures("organisation") +def test_cli_usage_logger_middleware__user_with_multiple_organisations__logs_event_without_organisation_id( + staff_client: APIClient, + staff_user: FFAdminUser, + log: StructuredLogCapture, +) -> None: + # Given + other_organisation = Organisation.objects.create(name="Other Org") + staff_user.add_organisation(other_organisation, role=OrganisationRole.USER) + + # When + response = staff_client.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) + + # Then + assert response.status_code == 200 + assert log.events == [ + { + "level": "info", + "event": "cli.request.made", + "cli__version": "2.0.0", + "cli__os": "darwin", + "cli__arch": "arm64", + "status": "success", + "amplitude__user_id": str(staff_user.uuid), + } + ] + + +def test_cli_usage_logger_middleware__user_without_organisations__logs_event_without_organisation_id( + staff_client: APIClient, + staff_user: FFAdminUser, + log: StructuredLogCapture, +) -> None: + # Given + assert not staff_user.organisations.exists() + + # When + response = staff_client.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) + + # Then + assert response.status_code == 200 + assert log.events == [ + { + "level": "info", + "event": "cli.request.made", + "cli__version": "2.0.0", + "cli__os": "darwin", + "cli__arch": "arm64", + "status": "success", + "amplitude__user_id": str(staff_user.uuid), + } + ] + + +def test_cli_usage_logger_middleware__master_api_key__logs_key_organisation_id_without_user_id( + admin_master_api_key_client: APIClient, + organisation: Organisation, + log: StructuredLogCapture, +) -> None: + # Given / When + response = admin_master_api_key_client.get( + "/api/v1/projects/", + HTTP_USER_AGENT=CLI_USER_AGENT, + ) + + # Then + assert response.status_code == 200 + assert log.events == [ + { + "level": "info", + "event": "cli.request.made", + "cli__version": "2.0.0", + "cli__os": "darwin", + "cli__arch": "arm64", + "organisation__id": organisation.pk, + "status": "success", + } + ] + + +def test_cli_usage_logger_middleware__unauthenticated_request__logs_nothing( + api_client: APIClient, + log: StructuredLogCapture, +) -> None: + # Given / When + response = api_client.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) + + # Then + assert response.status_code == 401 + assert log.events == [] + + +def test_cli_usage_logger_middleware__error_response__logs_error_status( + staff_client: APIClient, + staff_user: FFAdminUser, + organisation: Organisation, + log: StructuredLogCapture, +) -> None: + # Given / When + response = staff_client.get( + f"/api/v1/organisations/{organisation.pk}/invites/", + HTTP_USER_AGENT=CLI_USER_AGENT, + ) + + # Then + assert response.status_code == 403 + assert log.events == [ + { + "level": "info", + "event": "cli.request.made", + "cli__version": "2.0.0", + "cli__os": "darwin", + "cli__arch": "arm64", + "organisation__id": organisation.pk, + "status": "error", + "amplitude__user_id": str(staff_user.uuid), + } + ] + + +def test_cli_usage_logger_middleware__server_error_response__logs_error_status( + rf: RequestFactory, + staff_user: FFAdminUser, + organisation: Organisation, + log: StructuredLogCapture, +) -> None: + # Given + request = rf.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) + request.user = staff_user + middleware = CLIUsageLoggerMiddleware(lambda request: HttpResponse(status=500)) + + # When + response = middleware(request) + + # Then + assert response.status_code == 500 + assert log.events == [ + { + "level": "info", + "event": "cli.request.made", + "cli__version": "2.0.0", + "cli__os": "darwin", + "cli__arch": "arm64", + "organisation__id": organisation.pk, + "status": "error", + "amplitude__user_id": str(staff_user.uuid), + } + ] + + +@pytest.mark.parametrize( + "user_agent, expected_version, expected_os, expected_arch", + [ + ("flagsmith-cli/2.0.0-beta.3 (linux/amd64)", "2.0.0-beta.3", "linux", "amd64"), + ("flagsmith-cli/dev (windows/386)", "dev", "windows", "386"), + ("flagsmith-cli/99.0.0 (freebsd/riscv64)", "99.0.0", "freebsd", "riscv64"), + ], +) +def test_cli_usage_logger_middleware__cli_user_agent__passes_version_through_unchanged( + staff_client: APIClient, + staff_user: FFAdminUser, + organisation: Organisation, + log: StructuredLogCapture, + user_agent: str, + expected_version: str, + expected_os: str, + expected_arch: str, +) -> None: + # Given / When + response = staff_client.get("/api/v1/projects/", HTTP_USER_AGENT=user_agent) + + # Then + assert response.status_code == 200 + assert log.events == [ + { + "level": "info", + "event": "cli.request.made", + "cli__version": expected_version, + "cli__os": expected_os, + "cli__arch": expected_arch, + "organisation__id": organisation.pk, + "status": "success", + "amplitude__user_id": str(staff_user.uuid), + } + ] + + +def test_cli_usage_logger_middleware__organisation_id_span_attribute__does_not_query_database( + rf: RequestFactory, + staff_user: FFAdminUser, + organisation: Organisation, + recording_span: Span, + log: StructuredLogCapture, + django_assert_num_queries: DjangoAssertNumQueries, +) -> None: + # Given + recording_span.set_attribute("organisation.id", organisation.pk) + request = rf.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) + request.user = staff_user + middleware = CLIUsageLoggerMiddleware(lambda request: HttpResponse(status=200)) + + # When + with django_assert_num_queries(0): + middleware(request) + + # Then + assert log.events == [ + { + "level": "info", + "event": "cli.request.made", + "cli__version": "2.0.0", + "cli__os": "darwin", + "cli__arch": "arm64", + "organisation__id": organisation.pk, + "status": "success", + "amplitude__user_id": str(staff_user.uuid), + } + ] + + +def test_cli_usage_logger_middleware__master_api_key__does_not_query_database( + rf: RequestFactory, + admin_master_api_key_object: MasterAPIKey, + organisation: Organisation, + log: StructuredLogCapture, + django_assert_num_queries: DjangoAssertNumQueries, +) -> None: + # Given + request = rf.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) + request.user = APIKeyUser(admin_master_api_key_object) # type: ignore[assignment] + middleware = CLIUsageLoggerMiddleware(lambda request: HttpResponse(status=200)) + + # When + with django_assert_num_queries(0): + middleware(request) + + # Then + assert log.events == [ + { + "level": "info", + "event": "cli.request.made", + "cli__version": "2.0.0", + "cli__os": "darwin", + "cli__arch": "arm64", + "organisation__id": organisation.pk, + "status": "success", + } + ] From e0b7ccbc0ffbc10907fab3c47bdc452d60e61a08 Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Wed, 18 Nov 2026 18:32:56 +0100 Subject: [PATCH 2/3] test(Telemetry): Mirror the MCP middleware suite for CLI usage events Log unresolved organisations as None at WARNING, as the MCP middleware does, cover SDK requests carrying a CLI user-agent, and drop the query-count tests now that organisation resolution follows MCP. --- .../test_unit_telemetry_middleware.py | 140 ++++++------------ 1 file changed, 46 insertions(+), 94 deletions(-) diff --git a/api/tests/unit/telemetry/test_unit_telemetry_middleware.py b/api/tests/unit/telemetry/test_unit_telemetry_middleware.py index ff94f827423f..4313780678f0 100644 --- a/api/tests/unit/telemetry/test_unit_telemetry_middleware.py +++ b/api/tests/unit/telemetry/test_unit_telemetry_middleware.py @@ -1,13 +1,9 @@ import pytest from django.http import HttpResponse from django.test import RequestFactory -from opentelemetry.trace import Span -from pytest_django import DjangoAssertNumQueries from pytest_structlog import StructuredLogCapture from rest_framework.test import APIClient -from api_keys.models import MasterAPIKey -from api_keys.user import APIKeyUser from environments.models import Environment from organisations.models import Organisation, OrganisationRole from telemetry.middleware import CLIUsageLoggerMiddleware @@ -178,7 +174,7 @@ def test_mcp_usage_logger_middleware__sdk_request__logs_nothing( @pytest.mark.usefixtures("organisation") -def test_cli_usage_logger_middleware__non_cli_request__logs_nothing( +def test_cli_usage_logger_middleware__no_cli_user_agent__logs_nothing( staff_client: APIClient, log: StructuredLogCapture, ) -> None: @@ -218,14 +214,21 @@ def test_cli_usage_logger_middleware__user_agent_not_matching_cli_pattern__logs_ assert log.events == [] -def test_cli_usage_logger_middleware__user_with_single_organisation__logs_event( +@pytest.mark.usefixtures("recording_span", "organisation") +def test_cli_usage_logger_middleware__organisation_id_span_attribute__logs_span_organisation_id( staff_client: APIClient, staff_user: FFAdminUser, - organisation: Organisation, log: StructuredLogCapture, ) -> None: - # Given / When - response = staff_client.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) + # Given + other_organisation = Organisation.objects.create(name="Other Org") + staff_user.add_organisation(other_organisation, role=OrganisationRole.ADMIN) + + # When + response = staff_client.get( + f"/api/v1/organisations/{other_organisation.pk}/invites/", + HTTP_USER_AGENT=CLI_USER_AGENT, + ) # Then assert response.status_code == 200 @@ -236,28 +239,21 @@ def test_cli_usage_logger_middleware__user_with_single_organisation__logs_event( "cli__version": "2.0.0", "cli__os": "darwin", "cli__arch": "arm64", - "organisation__id": organisation.pk, + "organisation__id": other_organisation.pk, "status": "success", "amplitude__user_id": str(staff_user.uuid), } ] -@pytest.mark.usefixtures("recording_span", "organisation") -def test_cli_usage_logger_middleware__organisation_id_span_attribute__logs_span_organisation_id( +def test_cli_usage_logger_middleware__user_with_single_organisation__logs_user_organisation_id( staff_client: APIClient, staff_user: FFAdminUser, + organisation: Organisation, log: StructuredLogCapture, ) -> None: - # Given - other_organisation = Organisation.objects.create(name="Other Org") - staff_user.add_organisation(other_organisation, role=OrganisationRole.ADMIN) - - # When - response = staff_client.get( - f"/api/v1/organisations/{other_organisation.pk}/invites/", - HTTP_USER_AGENT=CLI_USER_AGENT, - ) + # Given / When + response = staff_client.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) # Then assert response.status_code == 200 @@ -268,22 +264,20 @@ def test_cli_usage_logger_middleware__organisation_id_span_attribute__logs_span_ "cli__version": "2.0.0", "cli__os": "darwin", "cli__arch": "arm64", - "organisation__id": other_organisation.pk, + "organisation__id": organisation.pk, "status": "success", "amplitude__user_id": str(staff_user.uuid), } ] -@pytest.mark.usefixtures("organisation") -def test_cli_usage_logger_middleware__user_with_multiple_organisations__logs_event_without_organisation_id( +def test_cli_usage_logger_middleware__user_without_organisations__logs_warning( staff_client: APIClient, staff_user: FFAdminUser, log: StructuredLogCapture, ) -> None: # Given - other_organisation = Organisation.objects.create(name="Other Org") - staff_user.add_organisation(other_organisation, role=OrganisationRole.USER) + assert not staff_user.organisations.exists() # When response = staff_client.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) @@ -292,24 +286,27 @@ def test_cli_usage_logger_middleware__user_with_multiple_organisations__logs_eve assert response.status_code == 200 assert log.events == [ { - "level": "info", + "level": "warning", "event": "cli.request.made", "cli__version": "2.0.0", "cli__os": "darwin", "cli__arch": "arm64", + "organisation__id": None, "status": "success", "amplitude__user_id": str(staff_user.uuid), } ] -def test_cli_usage_logger_middleware__user_without_organisations__logs_event_without_organisation_id( +@pytest.mark.usefixtures("organisation") +def test_cli_usage_logger_middleware__user_with_multiple_organisations__logs_warning( staff_client: APIClient, staff_user: FFAdminUser, log: StructuredLogCapture, ) -> None: # Given - assert not staff_user.organisations.exists() + other_organisation = Organisation.objects.create(name="Other Org") + staff_user.add_organisation(other_organisation, role=OrganisationRole.USER) # When response = staff_client.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) @@ -318,18 +315,19 @@ def test_cli_usage_logger_middleware__user_without_organisations__logs_event_wit assert response.status_code == 200 assert log.events == [ { - "level": "info", + "level": "warning", "event": "cli.request.made", "cli__version": "2.0.0", "cli__os": "darwin", "cli__arch": "arm64", + "organisation__id": None, "status": "success", "amplitude__user_id": str(staff_user.uuid), } ] -def test_cli_usage_logger_middleware__master_api_key__logs_key_organisation_id_without_user_id( +def test_cli_usage_logger_middleware__master_api_key__logs_key_organisation_id( admin_master_api_key_client: APIClient, organisation: Organisation, log: StructuredLogCapture, @@ -425,6 +423,23 @@ def test_cli_usage_logger_middleware__server_error_response__logs_error_status( ] +def test_cli_usage_logger_middleware__sdk_request__logs_nothing( + api_client: APIClient, + environment: Environment, + log: StructuredLogCapture, +) -> None: + # Given / When + response = api_client.get( + "/api/v1/flags/", + HTTP_X_ENVIRONMENT_KEY=environment.api_key, + HTTP_USER_AGENT=CLI_USER_AGENT, + ) + + # Then + assert response.status_code == 200 + assert log.events == [] + + @pytest.mark.parametrize( "user_agent, expected_version, expected_os, expected_arch", [ @@ -460,66 +475,3 @@ def test_cli_usage_logger_middleware__cli_user_agent__passes_version_through_unc "amplitude__user_id": str(staff_user.uuid), } ] - - -def test_cli_usage_logger_middleware__organisation_id_span_attribute__does_not_query_database( - rf: RequestFactory, - staff_user: FFAdminUser, - organisation: Organisation, - recording_span: Span, - log: StructuredLogCapture, - django_assert_num_queries: DjangoAssertNumQueries, -) -> None: - # Given - recording_span.set_attribute("organisation.id", organisation.pk) - request = rf.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) - request.user = staff_user - middleware = CLIUsageLoggerMiddleware(lambda request: HttpResponse(status=200)) - - # When - with django_assert_num_queries(0): - middleware(request) - - # Then - assert log.events == [ - { - "level": "info", - "event": "cli.request.made", - "cli__version": "2.0.0", - "cli__os": "darwin", - "cli__arch": "arm64", - "organisation__id": organisation.pk, - "status": "success", - "amplitude__user_id": str(staff_user.uuid), - } - ] - - -def test_cli_usage_logger_middleware__master_api_key__does_not_query_database( - rf: RequestFactory, - admin_master_api_key_object: MasterAPIKey, - organisation: Organisation, - log: StructuredLogCapture, - django_assert_num_queries: DjangoAssertNumQueries, -) -> None: - # Given - request = rf.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) - request.user = APIKeyUser(admin_master_api_key_object) # type: ignore[assignment] - middleware = CLIUsageLoggerMiddleware(lambda request: HttpResponse(status=200)) - - # When - with django_assert_num_queries(0): - middleware(request) - - # Then - assert log.events == [ - { - "level": "info", - "event": "cli.request.made", - "cli__version": "2.0.0", - "cli__os": "darwin", - "cli__arch": "arm64", - "organisation__id": organisation.pk, - "status": "success", - } - ] From e61a8922368d5cef726f595efa0d73684295060d Mon Sep 17 00:00:00 2001 From: Kim Gustyr Date: Wed, 18 Nov 2026 18:51:42 +0100 Subject: [PATCH 3/3] feat(Telemetry): Emit CLI usage events from an API middleware Rename MCPUsageLoggerMiddleware to ClientUsageLoggerMiddleware and have it also emit cli.request.made for authenticated requests whose user-agent is the Flagsmith CLI's. Both events share authentication, status and organisation resolution, so they keep a single, last slot in MIDDLEWARE. Emit both events from literal log calls so the events catalogue picks them up, which also documents the status attribute of mcp.tool.called. --- api/app/settings/common.py | 2 +- api/telemetry/middleware.py | 66 ++++++++++++++----- .../test_unit_telemetry_middleware.py | 60 ++++++++--------- .../observability/_events-catalogue.md | 19 +++++- 4 files changed, 99 insertions(+), 48 deletions(-) diff --git a/api/app/settings/common.py b/api/app/settings/common.py index abbd9a1cb700..d36b8f64d5b3 100644 --- a/api/app/settings/common.py +++ b/api/app/settings/common.py @@ -414,7 +414,7 @@ "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", "simple_history.middleware.HistoryRequestMiddleware", - "telemetry.middleware.MCPUsageLoggerMiddleware", # Must come last! + "telemetry.middleware.ClientUsageLoggerMiddleware", # Must come last! ] ADD_NEVER_CACHE_HEADERS = env.bool("ADD_NEVER_CACHE_HEADERS", True) diff --git a/api/telemetry/middleware.py b/api/telemetry/middleware.py index 7754a686f523..d2279e01a0d5 100644 --- a/api/telemetry/middleware.py +++ b/api/telemetry/middleware.py @@ -1,4 +1,6 @@ +import re from collections.abc import Callable +from typing import Literal import structlog from django.http.request import HttpRequest @@ -7,9 +9,14 @@ from telemetry.spans import get_span_attribute +# e.g. `flagsmith-cli/2.0.0 (darwin/arm64)` +CLI_USER_AGENT_PATTERN = re.compile( + r"^flagsmith-cli/(?P[^\s/]+) \((?P[^\s/()]+)/(?P[^\s/()]+)\)" +) -class MCPUsageLoggerMiddleware: - """Emit telemetry events for MCP usage""" + +class ClientUsageLoggerMiddleware: + """Emit telemetry events for usage of Flagsmith clients, i.e. MCP and CLI""" def __init__( self, @@ -20,26 +27,54 @@ def __init__( def __call__(self, request: HttpRequest) -> HttpResponse: response = self.get_response(request) - if baggage.get_baggage("flagsmith.client.name") != "flagsmith-mcp": - return response + if baggage.get_baggage("flagsmith.client.name") == "flagsmith-mcp": + self._log_mcp_usage(request, response) + elif cli := CLI_USER_AGENT_PATTERN.match(request.headers.get("User-Agent", "")): + self._log_cli_usage(request, response, cli) + + return response + def _log_mcp_usage(self, request: HttpRequest, response: HttpResponse) -> None: if not request.user or not request.user.is_authenticated: - return response + return + # NOTE: The following W3C Baggage items are added by downstream processor + # - gen_ai.tool.name + # - flagsmith.mcp.client.name + # - flagsmith.mcp.client.version logger = structlog.get_logger("mcp") - event = { - # NOTE: The following W3C Baggage items are added by downstream processor - # - gen_ai.tool.name - # - flagsmith.mcp.client.name - # - flagsmith.mcp.client.version - "status": "error" if response.status_code >= 400 else "success", - } + logger = logger.bind(status=self._get_status(response)) if (org_id := self._get_organisation_id(request)) is not None: - logger.info("tool.called", organisation__id=org_id, **event) + logger.info("tool.called", organisation__id=org_id) else: - logger.warning("tool.called", organisation__id=None, **event) + logger.warning("tool.called", organisation__id=None) - return response + def _log_cli_usage( + self, + request: HttpRequest, + response: HttpResponse, + cli: re.Match[str], + ) -> None: + if not request.user or not request.user.is_authenticated: + return + + logger = structlog.get_logger("cli") + logger = logger.bind( + cli__version=cli["version"], + cli__os=cli["os"], + cli__arch=cli["arch"], + status=self._get_status(response), + ) + if not getattr(request.user, "is_master_api_key_user", False): + logger = logger.bind(amplitude__user_id=str(request.user.uuid)) + + if (org_id := self._get_organisation_id(request)) is not None: + logger.info("request.made", organisation__id=org_id) + else: + logger.warning("request.made", organisation__id=None) + + def _get_status(self, response: HttpResponse) -> Literal["success", "error"]: + return "error" if response.status_code >= 400 else "success" def _get_organisation_id(self, request: HttpRequest) -> int | None: """Obtain the organisation ID from the request context.""" @@ -51,6 +86,7 @@ def _get_organisation_id(self, request: HttpRequest) -> int | None: assert request.user.is_authenticated # NOTE: protected upstream try: # Most of the time, the user belongs to one organisation + # NOTE: Master API keys belong to exactly one organisation return request.user.organisations.get().id except ( Organisation.DoesNotExist, diff --git a/api/tests/unit/telemetry/test_unit_telemetry_middleware.py b/api/tests/unit/telemetry/test_unit_telemetry_middleware.py index 4313780678f0..9e6584299705 100644 --- a/api/tests/unit/telemetry/test_unit_telemetry_middleware.py +++ b/api/tests/unit/telemetry/test_unit_telemetry_middleware.py @@ -6,14 +6,14 @@ from environments.models import Environment from organisations.models import Organisation, OrganisationRole -from telemetry.middleware import CLIUsageLoggerMiddleware +from telemetry.middleware import ClientUsageLoggerMiddleware from users.models import FFAdminUser CLI_USER_AGENT = "flagsmith-cli/2.0.0 (darwin/arm64)" @pytest.mark.usefixtures("organisation") -def test_mcp_usage_logger_middleware__no_mcp_baggage__logs_nothing( +def test_client_usage_logger_middleware__no_mcp_baggage__logs_nothing( staff_client: APIClient, log: StructuredLogCapture, ) -> None: @@ -26,7 +26,7 @@ def test_mcp_usage_logger_middleware__no_mcp_baggage__logs_nothing( @pytest.mark.usefixtures("mcp_baggage", "recording_span", "organisation") -def test_mcp_usage_logger_middleware__organisation_id_span_attribute__logs_span_organisation_id( +def test_client_usage_logger_middleware__mcp_organisation_id_span_attribute__logs_span_organisation_id( staff_client: APIClient, staff_user: FFAdminUser, log: StructuredLogCapture, @@ -53,7 +53,7 @@ def test_mcp_usage_logger_middleware__organisation_id_span_attribute__logs_span_ @pytest.mark.usefixtures("mcp_baggage") -def test_mcp_usage_logger_middleware__user_with_single_organisation__logs_user_organisation_id( +def test_client_usage_logger_middleware__mcp_user_with_single_organisation__logs_user_organisation_id( staff_client: APIClient, organisation: Organisation, log: StructuredLogCapture, @@ -74,7 +74,7 @@ def test_mcp_usage_logger_middleware__user_with_single_organisation__logs_user_o @pytest.mark.usefixtures("mcp_baggage") -def test_mcp_usage_logger_middleware__user_without_organisations__logs_warning( +def test_client_usage_logger_middleware__mcp_user_without_organisations__logs_warning( staff_client: APIClient, staff_user: FFAdminUser, log: StructuredLogCapture, @@ -98,7 +98,7 @@ def test_mcp_usage_logger_middleware__user_without_organisations__logs_warning( @pytest.mark.usefixtures("mcp_baggage", "organisation") -def test_mcp_usage_logger_middleware__user_with_multiple_organisations__logs_warning( +def test_client_usage_logger_middleware__mcp_user_with_multiple_organisations__logs_warning( staff_client: APIClient, staff_user: FFAdminUser, log: StructuredLogCapture, @@ -123,7 +123,7 @@ def test_mcp_usage_logger_middleware__user_with_multiple_organisations__logs_war @pytest.mark.usefixtures("mcp_baggage") -def test_mcp_usage_logger_middleware__unauthenticated_request__logs_nothing( +def test_client_usage_logger_middleware__mcp_unauthenticated_request__logs_nothing( api_client: APIClient, log: StructuredLogCapture, ) -> None: @@ -136,7 +136,7 @@ def test_mcp_usage_logger_middleware__unauthenticated_request__logs_nothing( @pytest.mark.usefixtures("mcp_baggage") -def test_mcp_usage_logger_middleware__error_response__logs_error_status( +def test_client_usage_logger_middleware__mcp_error_response__logs_error_status( staff_client: APIClient, organisation: Organisation, log: StructuredLogCapture, @@ -157,7 +157,7 @@ def test_mcp_usage_logger_middleware__error_response__logs_error_status( @pytest.mark.usefixtures("mcp_baggage") -def test_mcp_usage_logger_middleware__sdk_request__logs_nothing( +def test_client_usage_logger_middleware__mcp_sdk_request__logs_nothing( api_client: APIClient, environment: Environment, log: StructuredLogCapture, @@ -174,7 +174,7 @@ def test_mcp_usage_logger_middleware__sdk_request__logs_nothing( @pytest.mark.usefixtures("organisation") -def test_cli_usage_logger_middleware__no_cli_user_agent__logs_nothing( +def test_client_usage_logger_middleware__no_cli_user_agent__logs_nothing( staff_client: APIClient, log: StructuredLogCapture, ) -> None: @@ -201,7 +201,7 @@ def test_cli_usage_logger_middleware__no_cli_user_agent__logs_nothing( ], ) @pytest.mark.usefixtures("organisation") -def test_cli_usage_logger_middleware__user_agent_not_matching_cli_pattern__logs_nothing( +def test_client_usage_logger_middleware__user_agent_not_matching_cli_pattern__logs_nothing( staff_client: APIClient, log: StructuredLogCapture, user_agent: str, @@ -215,7 +215,7 @@ def test_cli_usage_logger_middleware__user_agent_not_matching_cli_pattern__logs_ @pytest.mark.usefixtures("recording_span", "organisation") -def test_cli_usage_logger_middleware__organisation_id_span_attribute__logs_span_organisation_id( +def test_client_usage_logger_middleware__cli_organisation_id_span_attribute__logs_span_organisation_id( staff_client: APIClient, staff_user: FFAdminUser, log: StructuredLogCapture, @@ -235,7 +235,7 @@ def test_cli_usage_logger_middleware__organisation_id_span_attribute__logs_span_ assert log.events == [ { "level": "info", - "event": "cli.request.made", + "event": "request.made", "cli__version": "2.0.0", "cli__os": "darwin", "cli__arch": "arm64", @@ -246,7 +246,7 @@ def test_cli_usage_logger_middleware__organisation_id_span_attribute__logs_span_ ] -def test_cli_usage_logger_middleware__user_with_single_organisation__logs_user_organisation_id( +def test_client_usage_logger_middleware__cli_user_with_single_organisation__logs_user_organisation_id( staff_client: APIClient, staff_user: FFAdminUser, organisation: Organisation, @@ -260,7 +260,7 @@ def test_cli_usage_logger_middleware__user_with_single_organisation__logs_user_o assert log.events == [ { "level": "info", - "event": "cli.request.made", + "event": "request.made", "cli__version": "2.0.0", "cli__os": "darwin", "cli__arch": "arm64", @@ -271,7 +271,7 @@ def test_cli_usage_logger_middleware__user_with_single_organisation__logs_user_o ] -def test_cli_usage_logger_middleware__user_without_organisations__logs_warning( +def test_client_usage_logger_middleware__cli_user_without_organisations__logs_warning( staff_client: APIClient, staff_user: FFAdminUser, log: StructuredLogCapture, @@ -287,7 +287,7 @@ def test_cli_usage_logger_middleware__user_without_organisations__logs_warning( assert log.events == [ { "level": "warning", - "event": "cli.request.made", + "event": "request.made", "cli__version": "2.0.0", "cli__os": "darwin", "cli__arch": "arm64", @@ -299,7 +299,7 @@ def test_cli_usage_logger_middleware__user_without_organisations__logs_warning( @pytest.mark.usefixtures("organisation") -def test_cli_usage_logger_middleware__user_with_multiple_organisations__logs_warning( +def test_client_usage_logger_middleware__cli_user_with_multiple_organisations__logs_warning( staff_client: APIClient, staff_user: FFAdminUser, log: StructuredLogCapture, @@ -316,7 +316,7 @@ def test_cli_usage_logger_middleware__user_with_multiple_organisations__logs_war assert log.events == [ { "level": "warning", - "event": "cli.request.made", + "event": "request.made", "cli__version": "2.0.0", "cli__os": "darwin", "cli__arch": "arm64", @@ -327,7 +327,7 @@ def test_cli_usage_logger_middleware__user_with_multiple_organisations__logs_war ] -def test_cli_usage_logger_middleware__master_api_key__logs_key_organisation_id( +def test_client_usage_logger_middleware__cli_master_api_key__logs_key_organisation_id( admin_master_api_key_client: APIClient, organisation: Organisation, log: StructuredLogCapture, @@ -343,7 +343,7 @@ def test_cli_usage_logger_middleware__master_api_key__logs_key_organisation_id( assert log.events == [ { "level": "info", - "event": "cli.request.made", + "event": "request.made", "cli__version": "2.0.0", "cli__os": "darwin", "cli__arch": "arm64", @@ -353,7 +353,7 @@ def test_cli_usage_logger_middleware__master_api_key__logs_key_organisation_id( ] -def test_cli_usage_logger_middleware__unauthenticated_request__logs_nothing( +def test_client_usage_logger_middleware__cli_unauthenticated_request__logs_nothing( api_client: APIClient, log: StructuredLogCapture, ) -> None: @@ -365,7 +365,7 @@ def test_cli_usage_logger_middleware__unauthenticated_request__logs_nothing( assert log.events == [] -def test_cli_usage_logger_middleware__error_response__logs_error_status( +def test_client_usage_logger_middleware__cli_error_response__logs_error_status( staff_client: APIClient, staff_user: FFAdminUser, organisation: Organisation, @@ -382,7 +382,7 @@ def test_cli_usage_logger_middleware__error_response__logs_error_status( assert log.events == [ { "level": "info", - "event": "cli.request.made", + "event": "request.made", "cli__version": "2.0.0", "cli__os": "darwin", "cli__arch": "arm64", @@ -393,7 +393,7 @@ def test_cli_usage_logger_middleware__error_response__logs_error_status( ] -def test_cli_usage_logger_middleware__server_error_response__logs_error_status( +def test_client_usage_logger_middleware__cli_server_error_response__logs_error_status( rf: RequestFactory, staff_user: FFAdminUser, organisation: Organisation, @@ -402,7 +402,7 @@ def test_cli_usage_logger_middleware__server_error_response__logs_error_status( # Given request = rf.get("/api/v1/projects/", HTTP_USER_AGENT=CLI_USER_AGENT) request.user = staff_user - middleware = CLIUsageLoggerMiddleware(lambda request: HttpResponse(status=500)) + middleware = ClientUsageLoggerMiddleware(lambda request: HttpResponse(status=500)) # When response = middleware(request) @@ -412,7 +412,7 @@ def test_cli_usage_logger_middleware__server_error_response__logs_error_status( assert log.events == [ { "level": "info", - "event": "cli.request.made", + "event": "request.made", "cli__version": "2.0.0", "cli__os": "darwin", "cli__arch": "arm64", @@ -423,7 +423,7 @@ def test_cli_usage_logger_middleware__server_error_response__logs_error_status( ] -def test_cli_usage_logger_middleware__sdk_request__logs_nothing( +def test_client_usage_logger_middleware__cli_sdk_request__logs_nothing( api_client: APIClient, environment: Environment, log: StructuredLogCapture, @@ -448,7 +448,7 @@ def test_cli_usage_logger_middleware__sdk_request__logs_nothing( ("flagsmith-cli/99.0.0 (freebsd/riscv64)", "99.0.0", "freebsd", "riscv64"), ], ) -def test_cli_usage_logger_middleware__cli_user_agent__passes_version_through_unchanged( +def test_client_usage_logger_middleware__cli_user_agent__passes_version_through_unchanged( staff_client: APIClient, staff_user: FFAdminUser, organisation: Organisation, @@ -466,7 +466,7 @@ def test_cli_usage_logger_middleware__cli_user_agent__passes_version_through_unc assert log.events == [ { "level": "info", - "event": "cli.request.made", + "event": "request.made", "cli__version": expected_version, "cli__os": expected_os, "cli__arch": expected_arch, diff --git a/docs/docs/deployment-self-hosting/observability/_events-catalogue.md b/docs/docs/deployment-self-hosting/observability/_events-catalogue.md index 3a2cd944daea..66ec5db22d43 100644 --- a/docs/docs/deployment-self-hosting/observability/_events-catalogue.md +++ b/docs/docs/deployment-self-hosting/observability/_events-catalogue.md @@ -148,6 +148,20 @@ Attributes: - `seats.previous` - `subscription.id` +### `cli.request.made` + +Logged at `info` from: + - `api/telemetry/middleware.py:72` + - `api/telemetry/middleware.py:74` + +Attributes: + - `amplitude.user_id` + - `cli.arch` + - `cli.os` + - `cli.version` + - `organisation.id` + - `status` + ### `code_references.cleanup_issues.created` Logged at `info` from: @@ -728,11 +742,12 @@ Attributes: ### `mcp.tool.called` Logged at `info` from: - - `api/telemetry/middleware.py:38` - - `api/telemetry/middleware.py:40` + - `api/telemetry/middleware.py:48` + - `api/telemetry/middleware.py:50` Attributes: - `organisation.id` + - `status` ### `oauth2_metadata.registration.rejected`