Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions api/organisations/migrations/0062_add_usage_reporting_state.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Generated by Django 5.2.18 on 2026-11-12 11:08

import django.db.models.deletion
from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
("organisations", "0061_add_user_organisation_is_active"),
]

operations = [
migrations.CreateModel(
name="UsageReportingState",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("reported_until", models.DateTimeField()),
(
"organisation",
models.OneToOneField(
on_delete=django.db.models.deletion.CASCADE,
related_name="usage_reporting_state",
to="organisations.organisation",
),
),
],
),
]
11 changes: 11 additions & 0 deletions api/organisations/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -698,3 +698,14 @@ class tracks the actual rounded count of API calls that are

created_at = models.DateTimeField(null=True, auto_now_add=True)
updated_at = models.DateTimeField(null=True, auto_now=True)


class UsageReportingState(models.Model):
"""
End of the last window pushed successfully
"""

organisation = models.OneToOneField(
Organisation, on_delete=models.CASCADE, related_name="usage_reporting_state"
)
reported_until = models.DateTimeField()
2 changes: 1 addition & 1 deletion api/organisations/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ def update_organisation_subscription_information_cache_recurring() -> None:
update_organisation_subscription_information_cache()


@register_recurring_task(run_every=timedelta(hours=1))
@register_recurring_task(run_every=timedelta(hours=24))
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
def push_usage_to_control_plane() -> None:
push_usage_snapshots()

Expand Down
15 changes: 7 additions & 8 deletions api/organisations/usage_reporting/mappers.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import base64
from datetime import timedelta
from datetime import datetime

from common.core.utils import get_version
from django.utils import timezone

from app_analytics.analytics_db_service import get_usage_data_for_window
from app_analytics.dataclasses import UsageData
Expand Down Expand Up @@ -55,23 +54,23 @@ def map_project_usage_data_to_project_usage(
return rows[:MAX_PROJECT_USAGE_ROWS]


def map_organisation_to_usage_snapshot(organisation: Organisation) -> UsageSnapshot:
hour_end = timezone.now().replace(minute=0, second=0, microsecond=0)
hour_start = hour_end - timedelta(hours=1)
def map_organisation_to_usage_snapshot(
organisation: Organisation, window_start: datetime, window_end: datetime
) -> UsageSnapshot:
seat_count = organisation.num_seats
instance_version = get_version()
usage_data = get_usage_data_for_window(organisation, hour_start, hour_end)
usage_data = get_usage_data_for_window(organisation, window_start, window_end)
usage_data_by_project_id = {
project.id: get_usage_data_for_window(
organisation, hour_start, hour_end, project_id=project.id
organisation, window_start, window_end, project_id=project.id
)
for project in organisation.projects.all()
}

# No data access past this point!

return UsageSnapshot(
timestamp=hour_start,
timestamp=window_start,
seat_count=seat_count,
api_call_total=map_usage_data_to_total_api_calls(usage_data),
api_call_breakdown=map_usage_data_to_api_call_breakdown(usage_data),
Expand Down
65 changes: 51 additions & 14 deletions api/organisations/usage_reporting/services.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
import dataclasses
import json
from datetime import datetime, timedelta

import requests
import structlog
from django.conf import settings
from django.core.serializers.json import DjangoJSONEncoder
from django.utils import timezone
from requests.adapters import HTTPAdapter
from urllib3 import Retry

from organisations.models import Organisation
from organisations.models import Organisation, UsageReportingState
from organisations.usage_reporting.dataclasses import UsageSnapshot
from organisations.usage_reporting.mappers import (
map_organisation_to_usage_snapshot,
Expand All @@ -17,16 +21,35 @@

USAGE_ENDPOINT_PATH = "/v1/public/usage"
REQUEST_TIMEOUT_SECONDS = 30
RETRY_POLICY = Retry(
total=3,
backoff_factor=1,
status_forcelist=(429, 500, 502, 503, 504),
allowed_methods={"POST"},
raise_on_status=False,
)


def get_licensed_organisations() -> list[Organisation]:
if not settings.LICENSING_INSTALLED:
return []
return list(
Organisation.objects.filter(licence__isnull=False).select_related("licence")
Organisation.objects.filter(licence__isnull=False).select_related(
"licence", "usage_reporting_state"
)
)


def get_window_start(organisation: Organisation, window_end: datetime) -> datetime:
try:
return organisation.usage_reporting_state.reported_until
except UsageReportingState.DoesNotExist:
return UsageReportingState.objects.create(
organisation=organisation,
reported_until=window_end - timedelta(hours=24),
).reported_until


def push_snapshot(
*,
base_url: str,
Expand All @@ -42,17 +65,14 @@ def push_snapshot(
}
body = json.dumps(dataclasses.asdict(snapshot), cls=DjangoJSONEncoder)

response = requests.post(
url,
data=body,
headers=headers,
timeout=REQUEST_TIMEOUT_SECONDS,
)
with requests.Session() as session:
session.mount("https://", HTTPAdapter(max_retries=RETRY_POLICY))
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
response = session.post(
url=url, data=body, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS
)

if response.ok:
response.raise_for_status()
logger.info("snapshot.pushed", status_code=response.status_code)
else:
logger.warning("snapshot.push_failed", status_code=response.status_code)


def push_usage_snapshots() -> None:
Expand All @@ -63,13 +83,30 @@ def push_usage_snapshots() -> None:
logger.debug("run.skipped", reason="no_licensed_organisations")
return

window_end = timezone.now().replace(minute=0, second=0, microsecond=0)

for organisation in organisations:
with structlog.contextvars.bound_contextvars(organisation__id=organisation.id):
try:
window_start = get_window_start(organisation, window_end)
Comment thread
germangarces marked this conversation as resolved.
Resolved
Comment thread
germangarces marked this conversation as resolved.
Resolved
if window_start >= window_end:
# The control plane drops snapshots with a repeated timestamp,
# so an empty push would make the next window collide and be lost.
logger.debug("snapshot.skipped", reason="empty_window")
continue

push_snapshot(
base_url=base_url,
snapshot=map_organisation_to_usage_snapshot(organisation),
snapshot=map_organisation_to_usage_snapshot(
organisation=organisation,
window_start=window_start,
window_end=window_end,
),
signature_b64=organisation.licence.signature,
)
except Exception:
logger.exception("snapshot.errored")

UsageReportingState.objects.filter(
organisation=organisation, reported_until__lt=window_end
).update(reported_until=window_end)
except Exception as e:
logger.exception("snapshot.errored", reason=e)
28 changes: 19 additions & 9 deletions api/tests/unit/organisations/usage_reporting/test_mappers.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
from datetime import datetime, timezone

import pytest
from pytest_mock import MockerFixture

from app_analytics.dataclasses import UsageData
Expand All @@ -16,16 +15,18 @@
)
from projects.models import Project

WINDOW_START = datetime(2026, 6, 17, 9, 0, 0, tzinfo=timezone.utc)
WINDOW_END = datetime(2026, 6, 18, 9, 0, 0, tzinfo=timezone.utc)


@pytest.mark.freeze_time("2026-06-18T09:30:00+00:00")
def test_map_organisation_to_usage_snapshot__with_usage__returns_populated_snapshot(
organisation: Organisation,
project: Project,
mocker: MockerFixture,
) -> None:
# Given
mocker.patch.object(mappers, "get_version", return_value="2.142.3")
mocker.patch.object(
mocked_get_usage_data = mocker.patch.object(
mappers,
"get_usage_data_for_window",
return_value=[
Expand All @@ -40,11 +41,13 @@ def test_map_organisation_to_usage_snapshot__with_usage__returns_populated_snaps
)

# When
snapshot = map_organisation_to_usage_snapshot(organisation)
snapshot = map_organisation_to_usage_snapshot(
organisation, window_start=WINDOW_START, window_end=WINDOW_END
)

# Then
assert snapshot == UsageSnapshot(
timestamp=datetime(2026, 6, 18, 8, 0, 0, tzinfo=timezone.utc),
timestamp=WINDOW_START,
seat_count=organisation.num_seats,
api_call_total=10,
api_call_breakdown=ApiCallBreakdown(
Expand All @@ -54,9 +57,12 @@ def test_map_organisation_to_usage_snapshot__with_usage__returns_populated_snaps
instance_version="2.142.3",
project_usage=[ProjectUsage(project_id=project.id, api_call_count=10)],
)
mocked_get_usage_data.assert_any_call(organisation, WINDOW_START, WINDOW_END)
mocked_get_usage_data.assert_any_call(
organisation, WINDOW_START, WINDOW_END, project_id=project.id
)


@pytest.mark.freeze_time("2026-06-18T09:30:00+00:00")
def test_map_organisation_to_usage_snapshot__no_analytics__returns_zeroed_snapshot(
organisation: Organisation,
project: Project,
Expand All @@ -67,11 +73,13 @@ def test_map_organisation_to_usage_snapshot__no_analytics__returns_zeroed_snapsh
mocker.patch.object(mappers, "get_usage_data_for_window", return_value=[])

# When
snapshot = map_organisation_to_usage_snapshot(organisation)
snapshot = map_organisation_to_usage_snapshot(
organisation, window_start=WINDOW_START, window_end=WINDOW_END
)

# Then
assert snapshot == UsageSnapshot(
timestamp=datetime(2026, 6, 18, 8, 0, 0, tzinfo=timezone.utc),
timestamp=WINDOW_START,
seat_count=organisation.num_seats,
api_call_total=0,
api_call_breakdown=ApiCallBreakdown(
Expand Down Expand Up @@ -105,7 +113,9 @@ def test_map_organisation_to_usage_snapshot__excess_projects__keeps_highest_usag
mocker.patch.object(mappers, "MAX_PROJECT_USAGE_ROWS", 1)

# When
snapshot = map_organisation_to_usage_snapshot(organisation)
snapshot = map_organisation_to_usage_snapshot(
organisation, window_start=WINDOW_START, window_end=WINDOW_END
)

# Then - only the highest-usage project survives the cap
assert snapshot.project_usage == [
Expand Down
Loading
Loading