Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions api/organisations/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@
FREE_PLAN_ID,
MAX_API_CALLS_IN_FREE_PLAN,
)
from organisations.usage_reporting.services import push_usage_snapshots
from users.models import FFAdminUser

from .constants import (
Expand Down Expand Up @@ -97,11 +96,6 @@ def update_organisation_subscription_information_cache_recurring() -> None:
update_organisation_subscription_information_cache()


@register_recurring_task(run_every=timedelta(hours=1))
def push_usage_to_control_plane() -> None:
push_usage_snapshots()


@register_task_handler()
def update_organisation_subscription_information_api_usage_cache() -> None:
subscription_info_cache.update_caches(SubscriptionCacheEntity.API_USAGE)
Expand Down
81 changes: 0 additions & 81 deletions api/organisations/usage_reporting/mappers.py

This file was deleted.

75 changes: 0 additions & 75 deletions api/organisations/usage_reporting/services.py

This file was deleted.

45 changes: 45 additions & 0 deletions api/telemetry/control_plane/client.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import base64
import dataclasses
import json

import requests
import structlog
from django.core.serializers.json import DjangoJSONEncoder

from telemetry.dataclasses import UsageSnapshot

logger = structlog.get_logger(__name__)

USAGE_ENDPOINT_PATH = "/v1/public/usage"
REQUEST_TIMEOUT_SECONDS = 30


def post_usage_snapshot(
*,
base_url: str,
snapshot: UsageSnapshot,
signature_b64: str,
) -> None:
url = f"{base_url.rstrip('/')}{USAGE_ENDPOINT_PATH}"
headers = {
"Authorization": (f"Bearer {_map_signature_to_bearer_token(signature_b64)}"),
Comment thread
germangarces marked this conversation as resolved.
Resolved
"Content-Type": "application/json",
}
body = json.dumps(dataclasses.asdict(snapshot), cls=DjangoJSONEncoder)

response = requests.post(
url,
data=body,
headers=headers,
timeout=REQUEST_TIMEOUT_SECONDS,
)

if response.ok:
logger.info("snapshot.pushed", status_code=response.status_code)
else:
logger.warning("snapshot.push_failed", status_code=response.status_code)


def _map_signature_to_bearer_token(signature_b64: str) -> str:
raw_signature = base64.b64decode(signature_b64)
return base64.urlsafe_b64encode(raw_signature).decode("ascii").rstrip("=")
106 changes: 106 additions & 0 deletions api/telemetry/services.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
from datetime import timedelta

import structlog
from common.core.utils import get_version
from django.conf import settings
from django.utils import timezone

from app_analytics.analytics_db_service import get_usage_data_for_window
from app_analytics.dataclasses import UsageData
from organisations.models import Organisation
from telemetry.control_plane.client import post_usage_snapshot
from telemetry.dataclasses import ApiCallBreakdown, ProjectUsage, UsageSnapshot

logger = structlog.get_logger(__name__)

MAX_PROJECT_USAGE_ROWS = 5_000


def push_usage_snapshots() -> None:
if not (settings.ENABLE_TELEMETRY):
logger.debug("run.skipped", reason="telemetry_disabled")
return
if not (base_url := settings.CONTROL_PLANE_URL):
logger.debug("run.skipped", reason="control_plane_url_unset")
return
if not (organisations := _get_licensed_organisations()):
logger.debug("run.skipped", reason="no_licensed_organisations")
return

for organisation in organisations:
with structlog.contextvars.bound_contextvars(organisation__id=organisation.id):
try:
post_usage_snapshot(
base_url=base_url,
snapshot=_get_usage_snapshot(organisation),
signature_b64=organisation.licence.signature,
)
except Exception:
logger.exception("snapshot.errored")


def _get_licensed_organisations() -> list[Organisation]:
if settings.LICENSING_INSTALLED: # pragma: no cover
Comment thread
germangarces marked this conversation as resolved.
Resolved
return list(
Organisation.objects.filter(licence__isnull=False).select_related("licence")
)
return []


def _sum_api_calls(usage_data: list[UsageData]) -> int:
return sum(
data.flags + data.identities + data.traits + data.environment_document
for data in usage_data
)


def _map_to_top_project_usage(
usage_data_by_project_id: dict[int, list[UsageData]],
) -> list[ProjectUsage]:
rows = [
ProjectUsage(
project_id=project_id,
api_call_count=_sum_api_calls(usage_data),
)
for project_id, usage_data in usage_data_by_project_id.items()
]
# Highest usage first
rows.sort(key=lambda row: row.api_call_count, reverse=True)
return rows[:MAX_PROJECT_USAGE_ROWS]


def _map_to_api_call_breakdown(
usage_data: list[UsageData],
) -> ApiCallBreakdown:
return ApiCallBreakdown(
flags=sum(data.flags for data in usage_data),
identities=sum(data.identities for data in usage_data),
traits=sum(data.traits for data in usage_data),
environment_documents=sum(data.environment_document for data in usage_data),
)


def _get_usage_snapshot(organisation: Organisation) -> UsageSnapshot:
hour_end = timezone.now().replace(minute=0, second=0, microsecond=0)
hour_start = hour_end - timedelta(hours=24)
seat_count = organisation.num_seats
instance_version = get_version()
usage_data = get_usage_data_for_window(organisation, hour_start, hour_end)
usage_data_by_project_id = {
project.id: get_usage_data_for_window(
organisation, hour_start, hour_end, project_id=project.id
)
for project in organisation.projects.all()
}

# No data access past this point!

return UsageSnapshot(
timestamp=hour_start,
seat_count=seat_count,
api_call_total=_sum_api_calls(usage_data),
api_call_breakdown=_map_to_api_call_breakdown(usage_data),
project_count=len(usage_data_by_project_id),
instance_version=instance_version,
project_usage=_map_to_top_project_usage(usage_data_by_project_id),
)
10 changes: 10 additions & 0 deletions api/telemetry/tasks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
from datetime import timedelta

from task_processor.decorators import register_recurring_task

from telemetry.services import push_usage_snapshots


@register_recurring_task(run_every=timedelta(hours=24))
Comment thread
germangarces marked this conversation as resolved.
Resolved
def push_usage() -> None:
push_usage_snapshots()
Empty file.
Loading
Loading