From ffd2b0b2254ad12cc9422bb3dfd5d3d88dcfa0e3 Mon Sep 17 00:00:00 2001 From: TejasLamba2006 Date: Mon, 9 Nov 2026 22:10:46 +0530 Subject: [PATCH 1/6] fix(api): block multivariate option changes during live experiments Multivariate options were only restricted in the frontend. The API accepted create, update and delete on the MV options endpoint, and changed nested multivariate_options on feature update, even when the feature was in a running experiment. Reject these writes with a 400 while any environment has a running experiment for the feature. On feature update, only reject when the submitted options differ from the stored ones, so other feature fields can still be saved. Closes #8766 --- api/experimentation/feature_state_metadata.py | 9 + api/features/constants.py | 3 + api/features/multivariate/views.py | 19 +++ api/features/serializers.py | 31 ++++ .../test_unit_multivariate_live_experiment.py | 158 ++++++++++++++++++ 5 files changed, 220 insertions(+) create mode 100644 api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py diff --git a/api/experimentation/feature_state_metadata.py b/api/experimentation/feature_state_metadata.py index b0af6cddfcbd..1459903017fd 100644 --- a/api/experimentation/feature_state_metadata.py +++ b/api/experimentation/feature_state_metadata.py @@ -9,6 +9,15 @@ from features.models import FeatureState +def has_live_experiment(feature_id: int) -> bool: + # Multivariate options belong to the feature, not an environment, so an + # experiment running in any environment locks them. + return Experiment.objects.filter( + feature_id=feature_id, + status=ExperimentStatus.RUNNING, + ).exists() + + def get_feature_state_metadata_builder( environment: "Environment", ) -> Callable[["FeatureState"], dict[str, Any] | None]: diff --git a/api/features/constants.py b/api/features/constants.py index ad2008ce00fa..37626469dfec 100644 --- a/api/features/constants.py +++ b/api/features/constants.py @@ -10,6 +10,9 @@ # Multivariate variant reported when an identity falls through to the control value CONTROL_VARIANT_KEY = "control" RESERVED_VARIANT_KEY_MESSAGE = f'"{CONTROL_VARIANT_KEY}" is a reserved variant key.' +LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE = ( + "Multivariate options cannot be changed while the feature is in a live experiment." +) # Tag filtering strategy UNION = "UNION" diff --git a/api/features/multivariate/views.py b/api/features/multivariate/views.py index aecb0dee4a6d..ffb3a9c5201a 100644 --- a/api/features/multivariate/views.py +++ b/api/features/multivariate/views.py @@ -6,9 +6,12 @@ from drf_spectacular.utils import extend_schema from rest_framework import viewsets from rest_framework.decorators import api_view +from rest_framework.exceptions import ValidationError from rest_framework.generics import get_object_or_404 from rest_framework.response import Response +from experimentation.feature_state_metadata import has_live_experiment +from features.constants import LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE from features.models import Feature from projects.permissions import NestedProjectPermissions @@ -72,6 +75,22 @@ def create(self, request, *args, **kwargs): # type: ignore[no-untyped-def] get_object_or_404(Feature.objects.filter(project__id=project_pk), pk=feature_pk) return super().create(request, *args, **kwargs) + def perform_create(self, serializer): # type: ignore[no-untyped-def] + self._check_no_live_experiment(serializer.validated_data["feature"]) + super().perform_create(serializer) + + def perform_update(self, serializer): # type: ignore[no-untyped-def] + self._check_no_live_experiment(serializer.instance.feature) + super().perform_update(serializer) + + def perform_destroy(self, instance): # type: ignore[no-untyped-def] + self._check_no_live_experiment(instance.feature) + super().perform_destroy(instance) + + def _check_no_live_experiment(self, feature: Feature) -> None: + if has_live_experiment(feature.id): + raise ValidationError(LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE) + def get_queryset(self): # type: ignore[no-untyped-def] if getattr(self, "swagger_fake_view", False): return MultivariateFeatureOption.objects.none() diff --git a/api/features/serializers.py b/api/features/serializers.py index 19e067b9bbba..40fd5ecf1cf8 100644 --- a/api/features/serializers.py +++ b/api/features/serializers.py @@ -31,7 +31,9 @@ from evaluation.types import EvaluatedFeatureState from experimentation.feature_state_metadata import ( get_feature_state_metadata_builder, + has_live_experiment, ) +from features.constants import LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE from integrations.vcs.services import dispatch_vcs_on_feature_state_changed from metadata.serializers import MetadataSerializer, MetadataSerializerMixin from projects.code_references.serializers import ( @@ -324,8 +326,37 @@ def validate_multivariate_options(self, multivariate_options): # type: ignore[n ) if total_percentage_allocation > 100: raise serializers.ValidationError("Invalid percentage allocation") + if ( + self.instance + and self._changes_multivariate_options(multivariate_options) + and has_live_experiment(self.instance.id) + ): + raise serializers.ValidationError(LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE) return multivariate_options + def _changes_multivariate_options(self, multivariate_options) -> bool: # type: ignore[no-untyped-def] + # The frontend resubmits the whole feature, so only a real change to + # the options counts. Compare on content, not ids. + fields = ( + "key", + "type", + "integer_value", + "string_value", + "boolean_value", + "default_percentage_allocation", + ) + current = NestedMultivariateFeatureOptionSerializer( + self.instance.multivariate_options.all(), many=True + ).data + + def signature(options) -> list[tuple[Any, ...]]: # type: ignore[no-untyped-def] + return sorted( + (tuple(option.get(field) for field in fields) for option in options), + key=repr, + ) + + return signature(current) != signature(multivariate_options or []) + def validate_owners(self, owners: list[FFAdminUser]) -> list[FFAdminUser]: project: Project = self.context["project"] invalid_users = [ diff --git a/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py b/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py new file mode 100644 index 000000000000..dd8430839d16 --- /dev/null +++ b/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py @@ -0,0 +1,158 @@ +import pytest +from rest_framework import status +from rest_framework.test import APIClient + +from experimentation.models import Experiment, ExperimentStatus +from features.constants import LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE +from features.models import Feature +from features.multivariate.models import MultivariateFeatureOption +from projects.models import Project + + +@pytest.fixture() +def live_experiment(environment, multivariate_feature): # type: ignore[no-untyped-def] + return Experiment.objects.create( + environment=environment, + feature=multivariate_feature, + name="live experiment", + hypothesis="new option wins", + status=ExperimentStatus.RUNNING, + ) + + +def _options_url(project: Project, feature: Feature) -> str: + return f"/api/v1/projects/{project.id}/features/{feature.id}/mv-options/" + + +def test_create_mv_option__live_experiment__returns_400( # type: ignore[no-untyped-def] + admin_client: APIClient, project, multivariate_feature, live_experiment +): + # When + response = admin_client.post( + _options_url(project, multivariate_feature), + data={ + "feature": multivariate_feature.id, + "type": "unicode", + "string_value": "new option", + "default_percentage_allocation": 0, + "key": "new", + }, + format="json", + ) + + # Then + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE in str(response.json()) + assert multivariate_feature.multivariate_options.count() == 3 + + +def test_update_mv_option__live_experiment__returns_400( # type: ignore[no-untyped-def] + admin_client: APIClient, project, multivariate_feature, live_experiment +): + # Given + option = multivariate_feature.multivariate_options.first() + + # When + response = admin_client.put( + f"{_options_url(project, multivariate_feature)}{option.id}/", + data={ + "feature": multivariate_feature.id, + "type": "unicode", + "string_value": "edited", + "default_percentage_allocation": option.default_percentage_allocation, + "key": option.key, + }, + format="json", + ) + + # Then + assert response.status_code == status.HTTP_400_BAD_REQUEST + option.refresh_from_db() + assert option.string_value != "edited" + + +def test_delete_mv_option__live_experiment__returns_400( # type: ignore[no-untyped-def] + admin_client: APIClient, project, multivariate_feature, live_experiment +): + # Given + option = multivariate_feature.multivariate_options.first() + + # When + response = admin_client.delete( + f"{_options_url(project, multivariate_feature)}{option.id}/" + ) + + # Then + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert MultivariateFeatureOption.objects.filter(id=option.id).exists() + + +def test_create_mv_option__no_live_experiment__not_blocked_by_experiment_check( # type: ignore[no-untyped-def] + admin_client: APIClient, project, multivariate_feature +): + # When + response = admin_client.post( + _options_url(project, multivariate_feature), + data={ + "feature": multivariate_feature.id, + "type": "unicode", + "string_value": "new option", + "default_percentage_allocation": 0, + "key": "new", + }, + format="json", + ) + + # Then + assert response.status_code == status.HTTP_201_CREATED + + +def test_update_feature__live_experiment__changed_mv_options_returns_400( # type: ignore[no-untyped-def] + admin_client: APIClient, project, multivariate_feature, live_experiment +): + # Given + url = f"/api/v1/projects/{project.id}/features/{multivariate_feature.id}/" + options = [ + { + "type": "unicode", + "string_value": "changed", + "default_percentage_allocation": 100, + }, + ] + + # When + response = admin_client.patch( + url, data={"multivariate_options": options}, format="json" + ) + + # Then + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE in str(response.json()) + + +def test_update_feature__live_experiment__unchanged_mv_options_allowed( # type: ignore[no-untyped-def] + admin_client: APIClient, project, multivariate_feature, live_experiment +): + # Given + url = f"/api/v1/projects/{project.id}/features/{multivariate_feature.id}/" + unchanged = [ + { + "type": option.type, + "string_value": option.string_value, + "default_percentage_allocation": option.default_percentage_allocation, + } + for option in multivariate_feature.multivariate_options.all() + ] + + # When + response = admin_client.patch( + url, + data={ + "description": "only metadata changed", + "multivariate_options": unchanged, + }, + format="json", + ) + + # Then + assert response.status_code == status.HTTP_200_OK From 5b4a677359af2cbd8248f9f60465509bb92357e1 Mon Sep 17 00:00:00 2001 From: TejasLamba2006 Date: Mon, 9 Nov 2026 23:07:25 +0530 Subject: [PATCH 2/6] fix(api): guard multivariate options during experiments --- api/experimentation/feature_state_metadata.py | 10 ++- api/features/import_export/services.py | 14 ++++ api/features/multivariate/views.py | 10 ++- api/features/serializers.py | 34 ++++++-- .../test_unit_features_import_export_tasks.py | 53 ++++++++++++ .../test_unit_multivariate_live_experiment.py | 83 ++++++++++++++++++- 6 files changed, 190 insertions(+), 14 deletions(-) diff --git a/api/experimentation/feature_state_metadata.py b/api/experimentation/feature_state_metadata.py index 1459903017fd..2799a386dd84 100644 --- a/api/experimentation/feature_state_metadata.py +++ b/api/experimentation/feature_state_metadata.py @@ -12,10 +12,12 @@ def has_live_experiment(feature_id: int) -> bool: # Multivariate options belong to the feature, not an environment, so an # experiment running in any environment locks them. - return Experiment.objects.filter( - feature_id=feature_id, - status=ExperimentStatus.RUNNING, - ).exists() + return bool( + Experiment.objects.filter( + feature_id=feature_id, + status=ExperimentStatus.RUNNING, + ).exists() + ) def get_feature_state_metadata_builder( diff --git a/api/features/import_export/services.py b/api/features/import_export/services.py index 4e31b6f35ae6..e6ec645ef5db 100644 --- a/api/features/import_export/services.py +++ b/api/features/import_export/services.py @@ -1,6 +1,9 @@ +from django.core.exceptions import ValidationError from django.db.models import Q from environments.models import Environment +from experimentation.feature_state_metadata import has_live_experiment +from features.constants import LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE from features.import_export.types import FeatureExportData from features.models import Feature, FeatureSegment, FeatureState from features.multivariate.models import MultivariateFeatureOption @@ -15,6 +18,17 @@ def overwrite_feature_for_environment( Apply a destructive feature import to a single environment without affecting other environments' feature states or the Feature definition. """ + imported_option_values = { + (mv_data["type"], mv_data["value"]) for mv_data in feature_data["multivariate"] + } + existing_option_values = { + (option.type, option.value) + for option in existing_feature.multivariate_options.all() + } + creates_option = not imported_option_values.issubset(existing_option_values) + if creates_option and has_live_experiment(existing_feature.id): + raise ValidationError(LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE) + FeatureSegment.objects.filter( feature=existing_feature, environment=environment ).delete() diff --git a/api/features/multivariate/views.py b/api/features/multivariate/views.py index ffb3a9c5201a..4182fb6f0310 100644 --- a/api/features/multivariate/views.py +++ b/api/features/multivariate/views.py @@ -76,11 +76,19 @@ def create(self, request, *args, **kwargs): # type: ignore[no-untyped-def] return super().create(request, *args, **kwargs) def perform_create(self, serializer): # type: ignore[no-untyped-def] - self._check_no_live_experiment(serializer.validated_data["feature"]) + url_feature = get_object_or_404( + Feature.objects.filter(project__id=self.kwargs["project_pk"]), + pk=self.kwargs["feature_pk"], + ) + if serializer.validated_data["feature"] != url_feature: + raise ValidationError({"feature": "Feature must match the URL."}) + self._check_no_live_experiment(url_feature) super().perform_create(serializer) def perform_update(self, serializer): # type: ignore[no-untyped-def] self._check_no_live_experiment(serializer.instance.feature) + if "feature" in serializer.validated_data: + self._check_no_live_experiment(serializer.validated_data["feature"]) super().perform_update(serializer) def perform_destroy(self, instance): # type: ignore[no-untyped-def] diff --git a/api/features/serializers.py b/api/features/serializers.py index 40fd5ecf1cf8..65f949e5cffe 100644 --- a/api/features/serializers.py +++ b/api/features/serializers.py @@ -1,6 +1,6 @@ from datetime import datetime from functools import cached_property -from typing import Any, Callable +from typing import Any, Callable, cast from uuid import UUID import django.core.exceptions @@ -309,6 +309,17 @@ def validate_project_features_limit(self, project: Project) -> None: } ) + def update(self, instance: Feature, validated_data: dict[str, Any]) -> Feature: + if ( + "multivariate_options" in validated_data + and has_live_experiment(instance.id) + and not self._changes_multivariate_options( + validated_data["multivariate_options"], instance + ) + ): + validated_data.pop("multivariate_options") + return super().update(instance, validated_data) # type: ignore[no-any-return] + def validate_multivariate_options(self, multivariate_options): # type: ignore[no-untyped-def] if multivariate_options: user = self.context["request"].user @@ -326,19 +337,26 @@ def validate_multivariate_options(self, multivariate_options): # type: ignore[n ) if total_percentage_allocation > 100: raise serializers.ValidationError("Invalid percentage allocation") + instance = cast(Feature | None, self.instance) if ( - self.instance - and self._changes_multivariate_options(multivariate_options) - and has_live_experiment(self.instance.id) + instance + and self._changes_multivariate_options(multivariate_options, instance) + and has_live_experiment(instance.id) ): raise serializers.ValidationError(LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE) return multivariate_options - def _changes_multivariate_options(self, multivariate_options) -> bool: # type: ignore[no-untyped-def] + def _changes_multivariate_options( + self, + multivariate_options: list[dict[str, Any]] | None, + instance: Feature | None = None, + ) -> bool: + instance = instance or cast(Feature | None, self.instance) + if instance is None: + return False # The frontend resubmits the whole feature, so only a real change to - # the options counts. Compare on content, not ids. + # writable option fields counts. Compare on content, not ids. fields = ( - "key", "type", "integer_value", "string_value", @@ -346,7 +364,7 @@ def _changes_multivariate_options(self, multivariate_options) -> bool: # type: "default_percentage_allocation", ) current = NestedMultivariateFeatureOptionSerializer( - self.instance.multivariate_options.all(), many=True + instance.multivariate_options.all(), many=True ).data def signature(options) -> list[tuple[Any, ...]]: # type: ignore[no-untyped-def] diff --git a/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py b/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py index 702f9a20b841..7a229e66db4d 100644 --- a/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py +++ b/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py @@ -2,6 +2,7 @@ from datetime import timedelta import pytest +from django.core.exceptions import ValidationError from django.db.models import Q from django.utils import timezone from freezegun.api import FrozenDateTimeFactory @@ -9,6 +10,7 @@ from environments.identities.models import Identity from environments.models import Environment +from experimentation.models import Experiment, ExperimentStatus from features.feature_types import MULTIVARIATE, STANDARD from features.import_export.constants import ( FAILED, @@ -22,6 +24,7 @@ FeatureImport, FlagsmithOnFlagsmithFeatureExport, ) +from features.import_export.services import overwrite_feature_for_environment from features.import_export.tasks import ( _create_flagsmith_on_flagsmith_feature_export, clear_stale_feature_imports_and_exports, @@ -29,6 +32,7 @@ import_features_for_environment, retire_stalled_feature_imports_and_exports, ) +from features.import_export.types import FeatureExportData from features.models import Feature, FeatureSegment, FeatureState from features.multivariate.models import MultivariateFeatureOption from features.value_types import STRING @@ -39,6 +43,55 @@ from segments.models import Segment +def test_overwrite_feature_for_environment__live_experiment_requires_new_option__raises_validation_error( + db: None, + environment: Environment, + feature: Feature, +) -> None: + # Given + MultivariateFeatureOption.objects.create( + feature=feature, + type=STRING, + string_value="existing", + default_percentage_allocation=100, + ) + Experiment.objects.create( + environment=environment, + feature=feature, + name="live experiment", + hypothesis="new option wins", + status=ExperimentStatus.RUNNING, + ) + feature_state = feature.feature_states.get(environment=environment) + imported_feature: FeatureExportData = { + "name": feature.name, + "default_enabled": feature.default_enabled, + "is_server_key_only": feature.is_server_key_only, + "initial_value": feature.initial_value, + "value": feature.initial_value, + "type": STRING, + "enabled": feature_state.enabled, + "multivariate": [ + { + "percentage_allocation": 100, + "default_percentage_allocation": 100, + "value": "new", + "type": STRING, + } + ], + } + + # When / Then + with pytest.raises(ValidationError): + overwrite_feature_for_environment(imported_feature, feature, environment) + + assert feature.multivariate_options.count() == 1 + assert ( + feature.feature_states.get(environment=environment).feature_state_value.value + == feature.initial_value + ) + + def test_clear_stale_feature_imports_and_exports__stale_records__deletes_old_keeps_new( # type: ignore[no-untyped-def] db: None, environment: Environment, freezer: FrozenDateTimeFactory ): diff --git a/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py b/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py index dd8430839d16..fd5d3ba7eb36 100644 --- a/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py +++ b/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py @@ -27,6 +27,7 @@ def _options_url(project: Project, feature: Feature) -> str: def test_create_mv_option__live_experiment__returns_400( # type: ignore[no-untyped-def] admin_client: APIClient, project, multivariate_feature, live_experiment ): + # Given # When response = admin_client.post( _options_url(project, multivariate_feature), @@ -71,6 +72,73 @@ def test_update_mv_option__live_experiment__returns_400( # type: ignore[no-unty assert option.string_value != "edited" +def test_update_mv_option__moving_to_live_experiment__returns_400( # type: ignore[no-untyped-def] + admin_client: APIClient, project, environment, multivariate_feature +): + # Given + other_feature = Feature.objects.create(name="other feature", project=project) + option = MultivariateFeatureOption.objects.create( + feature=other_feature, + type="unicode", + string_value="source option", + default_percentage_allocation=0, + ) + for existing_option in multivariate_feature.multivariate_options.all(): + existing_option.default_percentage_allocation = 0 + existing_option.save() + + Experiment.objects.create( + environment=environment, + feature=multivariate_feature, + name="live experiment", + hypothesis="new option wins", + status=ExperimentStatus.RUNNING, + ) + + # When + response = admin_client.put( + f"{_options_url(project, other_feature)}{option.id}/", + data={ + "feature": multivariate_feature.id, + "type": option.type, + "string_value": option.string_value, + "default_percentage_allocation": option.default_percentage_allocation, + }, + format="json", + ) + + # Then + assert response.status_code == status.HTTP_400_BAD_REQUEST + option.refresh_from_db() + assert option.feature == other_feature + assert multivariate_feature.multivariate_options.count() == 3 + + +def test_create_mv_option__body_feature_mismatches_url__returns_400( # type: ignore[no-untyped-def] + admin_client: APIClient, project, multivariate_feature +): + # Given + other_feature = Feature.objects.create(name="other feature", project=project) + + # When + response = admin_client.post( + _options_url(project, multivariate_feature), + data={ + "feature": other_feature.id, + "type": "unicode", + "string_value": "new option", + "default_percentage_allocation": 0, + "key": "new", + }, + format="json", + ) + + # Then + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "Feature must match the URL." in str(response.json()) + assert other_feature.multivariate_options.count() == 0 + + def test_delete_mv_option__live_experiment__returns_400( # type: ignore[no-untyped-def] admin_client: APIClient, project, multivariate_feature, live_experiment ): @@ -90,6 +158,7 @@ def test_delete_mv_option__live_experiment__returns_400( # type: ignore[no-unty def test_create_mv_option__no_live_experiment__not_blocked_by_experiment_check( # type: ignore[no-untyped-def] admin_client: APIClient, project, multivariate_feature ): + # Given # When response = admin_client.post( _options_url(project, multivariate_feature), @@ -130,11 +199,17 @@ def test_update_feature__live_experiment__changed_mv_options_returns_400( # typ assert LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE in str(response.json()) -def test_update_feature__live_experiment__unchanged_mv_options_allowed( # type: ignore[no-untyped-def] +def test_update_feature__live_experiment__unchanged_keyed_options_preserved( # type: ignore[no-untyped-def] admin_client: APIClient, project, multivariate_feature, live_experiment ): # Given + keyed_option = multivariate_feature.multivariate_options.first() + keyed_option.key = "stable-option" + keyed_option.save() url = f"/api/v1/projects/{project.id}/features/{multivariate_feature.id}/" + stored_option_ids = list( + multivariate_feature.multivariate_options.values_list("id", flat=True) + ) unchanged = [ { "type": option.type, @@ -156,3 +231,9 @@ def test_update_feature__live_experiment__unchanged_mv_options_allowed( # type: # Then assert response.status_code == status.HTTP_200_OK + keyed_option.refresh_from_db() + assert keyed_option.key == "stable-option" + assert ( + list(multivariate_feature.multivariate_options.values_list("id", flat=True)) + == stored_option_ids + ) From d564c8d053d14408261767f8c48432435b3adc51 Mon Sep 17 00:00:00 2001 From: TejasLamba2006 Date: Mon, 9 Nov 2026 23:44:32 +0530 Subject: [PATCH 3/6] fix: serialize experiment option changes and imports --- api/experimentation/services.py | 5 +- api/features/import_export/services.py | 11 +--- api/features/import_export/tasks.py | 48 +++++++------- api/features/multivariate/views.py | 37 ++++++++--- api/features/serializers.py | 25 +++++--- .../test_unit_features_import_export_tasks.py | 63 ++++++++++++++++++- .../test_unit_multivariate_live_experiment.py | 8 +-- 7 files changed, 138 insertions(+), 59 deletions(-) diff --git a/api/experimentation/services.py b/api/experimentation/services.py index 9434e0885f93..7f5b8800a06f 100644 --- a/api/experimentation/services.py +++ b/api/experimentation/services.py @@ -79,7 +79,7 @@ from features.feature_states.models import API_VALUE_TYPES from features.future.services import delete_segment_override, update_flag from features.future.types import SegmentOverrideRequest -from features.models import FeatureState +from features.models import Feature, FeatureState from features.value_types import BOOLEAN, STRING from features.versioning.versioning_service import get_environment_flags_list from integrations.flagsmith.client import get_openfeature_client @@ -617,6 +617,9 @@ def transition_experiment_status( experiment.status = target_status if target_status == ExperimentStatus.RUNNING and not experiment.started_at: + Feature.objects.select_for_update().filter( + pk=experiment.feature_id + ).exists() experiment.started_at = timezone.now() _remove_superseded_experiment_rollouts( experiment, AuthorData.from_user(user) diff --git a/api/features/import_export/services.py b/api/features/import_export/services.py index e6ec645ef5db..044ae1b3313a 100644 --- a/api/features/import_export/services.py +++ b/api/features/import_export/services.py @@ -18,17 +18,8 @@ def overwrite_feature_for_environment( Apply a destructive feature import to a single environment without affecting other environments' feature states or the Feature definition. """ - imported_option_values = { - (mv_data["type"], mv_data["value"]) for mv_data in feature_data["multivariate"] - } - existing_option_values = { - (option.type, option.value) - for option in existing_feature.multivariate_options.all() - } - creates_option = not imported_option_values.issubset(existing_option_values) - if creates_option and has_live_experiment(existing_feature.id): + if has_live_experiment(existing_feature.id): raise ValidationError(LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE) - FeatureSegment.objects.filter( feature=existing_feature, environment=environment ).delete() diff --git a/api/features/import_export/tasks.py b/api/features/import_export/tasks.py index cdfe7cfa6fef..bc848d9c9c21 100644 --- a/api/features/import_export/tasks.py +++ b/api/features/import_export/tasks.py @@ -3,6 +3,7 @@ from typing import Optional from django.conf import settings +from django.db import transaction from django.db.models import Q from django.utils import timezone from task_processor.decorators import ( @@ -143,27 +144,32 @@ def import_features_for_environment(feature_import_id: int) -> None: def _import_features_for_environment(feature_import: FeatureImport) -> None: - environment = feature_import.environment - input_data: list[FeatureExportData] = json.loads(feature_import.data) - project = environment.project - - for feature_data in input_data: - existing_feature = Feature.objects.filter( - name=feature_data["name"], - project=project, - ).first() - - if existing_feature and feature_import.strategy == SKIP: - continue - - if existing_feature is None: - existing_feature = map_feature_export_data_to_feature(feature_data, project) - existing_feature.save() - - overwrite_feature_for_environment(feature_data, existing_feature, environment) - - feature_import.status = SUCCESS - feature_import.save() + with transaction.atomic(): + environment = feature_import.environment + input_data: list[FeatureExportData] = json.loads(feature_import.data) + project = environment.project + + for feature_data in input_data: + existing_feature = Feature.objects.filter( + name=feature_data["name"], + project=project, + ).first() + + if existing_feature and feature_import.strategy == SKIP: + continue + + if existing_feature is None: + existing_feature = map_feature_export_data_to_feature( + feature_data, project + ) + existing_feature.save() + + overwrite_feature_for_environment( + feature_data, existing_feature, environment + ) + + feature_import.status = SUCCESS + feature_import.save() # Should only run on official flagsmith instance. diff --git a/api/features/multivariate/views.py b/api/features/multivariate/views.py index 4182fb6f0310..0f3f1fa14363 100644 --- a/api/features/multivariate/views.py +++ b/api/features/multivariate/views.py @@ -2,6 +2,7 @@ CREATE_FEATURE, VIEW_PROJECT, ) +from django.db import transaction from django.utils.decorators import method_decorator from drf_spectacular.utils import extend_schema from rest_framework import viewsets @@ -72,7 +73,13 @@ def get_permissions(self): # type: ignore[no-untyped-def] def create(self, request, *args, **kwargs): # type: ignore[no-untyped-def] feature_pk = self.kwargs.get("feature_pk") project_pk = self.kwargs.get("project_pk") - get_object_or_404(Feature.objects.filter(project__id=project_pk), pk=feature_pk) + url_feature = get_object_or_404( + Feature.objects.filter(project__id=project_pk), pk=feature_pk + ) + if "feature" in request.data and str(request.data.get("feature")) != str( + url_feature.id + ): + raise ValidationError({"feature": "Feature must match the URL."}) return super().create(request, *args, **kwargs) def perform_create(self, serializer): # type: ignore[no-untyped-def] @@ -82,18 +89,32 @@ def perform_create(self, serializer): # type: ignore[no-untyped-def] ) if serializer.validated_data["feature"] != url_feature: raise ValidationError({"feature": "Feature must match the URL."}) - self._check_no_live_experiment(url_feature) - super().perform_create(serializer) + with transaction.atomic(): + Feature.objects.select_for_update().filter(pk=url_feature.pk).exists() + self._check_no_live_experiment(url_feature) + super().perform_create(serializer) def perform_update(self, serializer): # type: ignore[no-untyped-def] - self._check_no_live_experiment(serializer.instance.feature) + features = [serializer.instance.feature] if "feature" in serializer.validated_data: - self._check_no_live_experiment(serializer.validated_data["feature"]) - super().perform_update(serializer) + features.append(serializer.validated_data["feature"]) + feature_ids = sorted({feature.id for feature in features}) + with transaction.atomic(): + list( + Feature.objects.select_for_update() + .filter(pk__in=feature_ids) + .order_by("pk") + .values_list("pk", flat=True) + ) + for feature in features: + self._check_no_live_experiment(feature) + super().perform_update(serializer) def perform_destroy(self, instance): # type: ignore[no-untyped-def] - self._check_no_live_experiment(instance.feature) - super().perform_destroy(instance) + with transaction.atomic(): + Feature.objects.select_for_update().filter(pk=instance.feature_id).exists() + self._check_no_live_experiment(instance.feature) + super().perform_destroy(instance) def _check_no_live_experiment(self, feature: Feature) -> None: if has_live_experiment(feature.id): diff --git a/api/features/serializers.py b/api/features/serializers.py index 65f949e5cffe..c20a81119490 100644 --- a/api/features/serializers.py +++ b/api/features/serializers.py @@ -12,7 +12,7 @@ FeatureStateValueSerializer, ) from common.projects.permissions import VIEW_PROJECT -from django.db import models +from django.db import models, transaction from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema_field from drf_writable_nested import ( # type: ignore[attr-defined] @@ -310,15 +310,20 @@ def validate_project_features_limit(self, project: Project) -> None: ) def update(self, instance: Feature, validated_data: dict[str, Any]) -> Feature: - if ( - "multivariate_options" in validated_data - and has_live_experiment(instance.id) - and not self._changes_multivariate_options( - validated_data["multivariate_options"], instance - ) - ): - validated_data.pop("multivariate_options") - return super().update(instance, validated_data) # type: ignore[no-any-return] + with transaction.atomic(): + Feature.objects.select_for_update().filter(pk=instance.pk).exists() + if ( + has_live_experiment(instance.id) + and "multivariate_options" in validated_data + ): + if self._changes_multivariate_options( + validated_data["multivariate_options"], instance + ): + raise serializers.ValidationError( + {"multivariate_options": LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE} + ) + validated_data.pop("multivariate_options") + return super().update(instance, validated_data) # type: ignore[no-any-return] def validate_multivariate_options(self, multivariate_options): # type: ignore[no-untyped-def] if multivariate_options: diff --git a/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py b/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py index 7a229e66db4d..1e3b576dd9e4 100644 --- a/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py +++ b/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py @@ -43,10 +43,11 @@ from segments.models import Segment -def test_overwrite_feature_for_environment__live_experiment_requires_new_option__raises_validation_error( +def test_overwrite_feature_for_environment__live_experiment__raises_validation_error( db: None, environment: Environment, feature: Feature, + feature_segment: FeatureSegment, ) -> None: # Given MultivariateFeatureOption.objects.create( @@ -75,7 +76,7 @@ def test_overwrite_feature_for_environment__live_experiment_requires_new_option_ { "percentage_allocation": 100, "default_percentage_allocation": 100, - "value": "new", + "value": "existing", "type": STRING, } ], @@ -85,6 +86,7 @@ def test_overwrite_feature_for_environment__live_experiment_requires_new_option_ with pytest.raises(ValidationError): overwrite_feature_for_environment(imported_feature, feature, environment) + assert FeatureSegment.objects.filter(pk=feature_segment.pk).exists() assert feature.multivariate_options.count() == 1 assert ( feature.feature_states.get(environment=environment).feature_state_value.value @@ -92,6 +94,63 @@ def test_overwrite_feature_for_environment__live_experiment_requires_new_option_ ) +def test_import_features__live_experiment_on_later_feature__rolls_back_batch( + db: None, + environment: Environment, + project: Project, + feature: Feature, + feature_segment: FeatureSegment, +) -> None: + # Given + Experiment.objects.create( + environment=environment, + feature=feature, + name="live experiment", + hypothesis="new option wins", + status=ExperimentStatus.RUNNING, + ) + feature_import = FeatureImport.objects.create( + environment=environment, + strategy=OVERWRITE_DESTRUCTIVE, + data=json.dumps( + [ + { + "name": "created before failure", + "default_enabled": False, + "is_server_key_only": False, + "initial_value": "initial", + "value": "imported", + "type": STRING, + "enabled": True, + "multivariate": [], + }, + { + "name": feature.name, + "default_enabled": feature.default_enabled, + "is_server_key_only": feature.is_server_key_only, + "initial_value": feature.initial_value, + "value": feature.initial_value, + "type": STRING, + "enabled": False, + "multivariate": [], + }, + ] + ), + ) + + # When / Then + with pytest.raises(ValidationError): + import_features_for_environment(feature_import.id) + + feature_import.refresh_from_db() + assert feature_import.status == FAILED + assert not Feature.objects.filter( + project=project, name="created before failure" + ).exists() + assert FeatureSegment.objects.filter(pk=feature_segment.pk).exists() + assert feature.feature_states.get(environment=environment).enabled is False + + def test_clear_stale_feature_imports_and_exports__stale_records__deletes_old_keeps_new( # type: ignore[no-untyped-def] db: None, environment: Environment, freezer: FrozenDateTimeFactory ): diff --git a/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py b/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py index fd5d3ba7eb36..1b942a9a052d 100644 --- a/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py +++ b/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py @@ -123,13 +123,7 @@ def test_create_mv_option__body_feature_mismatches_url__returns_400( # type: ig # When response = admin_client.post( _options_url(project, multivariate_feature), - data={ - "feature": other_feature.id, - "type": "unicode", - "string_value": "new option", - "default_percentage_allocation": 0, - "key": "new", - }, + data={"feature": other_feature.id}, format="json", ) From a91b3907949ea4f288643264717e181a5f92efe6 Mon Sep 17 00:00:00 2001 From: TejasLamba2006 Date: Thu, 12 Nov 2026 12:04:53 +0530 Subject: [PATCH 4/6] fix: narrow the experiment lock on variations to real changes - mv-options PUT only checks for an experiment when the variation itself changes (type, value, key or feature). default_percentage_allocation is ignored, so the dashboard can still save other environments. - Paused experiments lock the variations as well as running ones. - Imports keep their original per-feature behaviour. A first pass rejects the import before anything is written, and only when it would add a variation or overwrite the environment running the experiment. - One error shape, {"detail": ...}, naming the experiment environment. - Drop the duplicate check in validate_multivariate_options. - Mention the lock in the experiment docs. --- api/experimentation/feature_state_metadata.py | 30 +- api/features/constants.py | 3 +- api/features/import_export/services.py | 5 - api/features/import_export/tasks.py | 81 +++-- api/features/multivariate/views.py | 42 ++- api/features/serializers.py | 27 +- .../test_unit_features_import_export_tasks.py | 232 +++++++++---- .../test_unit_multivariate_live_experiment.py | 306 +++++++++++++++--- .../observability/_events-catalogue.md | 12 +- .../experimentation/create-an-experiment.md | 4 +- 10 files changed, 554 insertions(+), 188 deletions(-) diff --git a/api/experimentation/feature_state_metadata.py b/api/experimentation/feature_state_metadata.py index 2799a386dd84..4375402479b7 100644 --- a/api/experimentation/feature_state_metadata.py +++ b/api/experimentation/feature_state_metadata.py @@ -1,25 +1,45 @@ from typing import TYPE_CHECKING, Any, Callable from flagsmith_schemas.api import FeatureStateMetadata +from rest_framework.exceptions import ValidationError from experimentation.models import Experiment, ExperimentStatus +from features.constants import LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE if TYPE_CHECKING: # pragma: no cover from environments.models import Environment from features.models import FeatureState -def has_live_experiment(feature_id: int) -> bool: +def get_live_experiment(feature_id: int) -> Experiment | None: # Multivariate options belong to the feature, not an environment, so an - # experiment running in any environment locks them. - return bool( + # experiment in any environment locks them. A paused experiment counts: + # changing the variants before it resumes would skew its results. + return ( # type: ignore[no-any-return] Experiment.objects.filter( feature_id=feature_id, - status=ExperimentStatus.RUNNING, - ).exists() + status__in=(ExperimentStatus.RUNNING, ExperimentStatus.PAUSED), + ) + .select_related("environment") + .first() ) +def live_experiment_error(experiment: Experiment) -> ValidationError: + return ValidationError( + { + "detail": LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE.format( + environment=experiment.environment.name + ) + } + ) + + +def check_no_live_experiment(feature_id: int) -> None: + if experiment := get_live_experiment(feature_id): + raise live_experiment_error(experiment) + + def get_feature_state_metadata_builder( environment: "Environment", ) -> Callable[["FeatureState"], dict[str, Any] | None]: diff --git a/api/features/constants.py b/api/features/constants.py index 37626469dfec..3cf4a30318fa 100644 --- a/api/features/constants.py +++ b/api/features/constants.py @@ -11,7 +11,8 @@ CONTROL_VARIANT_KEY = "control" RESERVED_VARIANT_KEY_MESSAGE = f'"{CONTROL_VARIANT_KEY}" is a reserved variant key.' LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE = ( - "Multivariate options cannot be changed while the feature is in a live experiment." + 'Variations cannot be changed while an experiment is running or paused in the "{environment}" ' + "environment. Complete the experiment to change them." ) # Tag filtering strategy diff --git a/api/features/import_export/services.py b/api/features/import_export/services.py index 044ae1b3313a..4e31b6f35ae6 100644 --- a/api/features/import_export/services.py +++ b/api/features/import_export/services.py @@ -1,9 +1,6 @@ -from django.core.exceptions import ValidationError from django.db.models import Q from environments.models import Environment -from experimentation.feature_state_metadata import has_live_experiment -from features.constants import LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE from features.import_export.types import FeatureExportData from features.models import Feature, FeatureSegment, FeatureState from features.multivariate.models import MultivariateFeatureOption @@ -18,8 +15,6 @@ def overwrite_feature_for_environment( Apply a destructive feature import to a single environment without affecting other environments' feature states or the Feature definition. """ - if has_live_experiment(existing_feature.id): - raise ValidationError(LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE) FeatureSegment.objects.filter( feature=existing_feature, environment=environment ).delete() diff --git a/api/features/import_export/tasks.py b/api/features/import_export/tasks.py index bc848d9c9c21..c02cf671ce1d 100644 --- a/api/features/import_export/tasks.py +++ b/api/features/import_export/tasks.py @@ -3,7 +3,6 @@ from typing import Optional from django.conf import settings -from django.db import transaction from django.db.models import Q from django.utils import timezone from task_processor.decorators import ( @@ -11,6 +10,11 @@ register_task_handler, ) +from environments.models import Environment +from experimentation.feature_state_metadata import ( + get_live_experiment, + live_experiment_error, +) from features.import_export.constants import ( FAILED, PROCESSING, @@ -144,32 +148,55 @@ def import_features_for_environment(feature_import_id: int) -> None: def _import_features_for_environment(feature_import: FeatureImport) -> None: - with transaction.atomic(): - environment = feature_import.environment - input_data: list[FeatureExportData] = json.loads(feature_import.data) - project = environment.project - - for feature_data in input_data: - existing_feature = Feature.objects.filter( - name=feature_data["name"], - project=project, - ).first() - - if existing_feature and feature_import.strategy == SKIP: - continue - - if existing_feature is None: - existing_feature = map_feature_export_data_to_feature( - feature_data, project - ) - existing_feature.save() - - overwrite_feature_for_environment( - feature_data, existing_feature, environment - ) - - feature_import.status = SUCCESS - feature_import.save() + environment = feature_import.environment + input_data: list[FeatureExportData] = json.loads(feature_import.data) + project = environment.project + + if feature_import.strategy != SKIP: + _check_import_against_live_experiments(input_data, environment) + + for feature_data in input_data: + existing_feature = Feature.objects.filter( + name=feature_data["name"], + project=project, + ).first() + + if existing_feature and feature_import.strategy == SKIP: + continue + + if existing_feature is None: + existing_feature = map_feature_export_data_to_feature(feature_data, project) + existing_feature.save() + + overwrite_feature_for_environment(feature_data, existing_feature, environment) + + feature_import.status = SUCCESS + feature_import.save() + + +def _check_import_against_live_experiments( + input_data: list[FeatureExportData], environment: Environment +) -> None: + """ + Reject the whole import, before anything is written, if it would add a + variation to a feature in an experiment, or overwrite the environment the + experiment runs in (which deletes its rollout segment override). + """ + for feature_data in input_data: + feature = Feature.objects.filter( + name=feature_data["name"], project=environment.project + ).first() + if feature is None or not (experiment := get_live_experiment(feature.id)): + continue + existing_variations = { + (option.type, option.value) for option in feature.multivariate_options.all() + } + adds_variation = any( + (mv_data["type"], mv_data["value"]) not in existing_variations + for mv_data in feature_data["multivariate"] + ) + if adds_variation or experiment.environment_id == environment.id: + raise live_experiment_error(experiment) # Should only run on official flagsmith instance. diff --git a/api/features/multivariate/views.py b/api/features/multivariate/views.py index 0f3f1fa14363..5f78e022faf8 100644 --- a/api/features/multivariate/views.py +++ b/api/features/multivariate/views.py @@ -11,14 +11,24 @@ from rest_framework.generics import get_object_or_404 from rest_framework.response import Response -from experimentation.feature_state_metadata import has_live_experiment -from features.constants import LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE +from experimentation.feature_state_metadata import check_no_live_experiment from features.models import Feature from projects.permissions import NestedProjectPermissions from .models import MultivariateFeatureOption from .serializers import MultivariateFeatureOptionSerializer +# What makes a variation. `default_percentage_allocation` is left out: the +# dashboard always sends it as 0, and it only seeds new environments. +VARIATION_FIELDS = ( + "type", + "string_value", + "integer_value", + "boolean_value", + "key", + "feature", +) + @method_decorator( name="list", @@ -91,35 +101,39 @@ def perform_create(self, serializer): # type: ignore[no-untyped-def] raise ValidationError({"feature": "Feature must match the URL."}) with transaction.atomic(): Feature.objects.select_for_update().filter(pk=url_feature.pk).exists() - self._check_no_live_experiment(url_feature) + check_no_live_experiment(url_feature.id) super().perform_create(serializer) def perform_update(self, serializer): # type: ignore[no-untyped-def] - features = [serializer.instance.feature] - if "feature" in serializer.validated_data: - features.append(serializer.validated_data["feature"]) - feature_ids = sorted({feature.id for feature in features}) + instance = serializer.instance + data = serializer.validated_data + # The dashboard PUTs every option on each save, changed or not, so only + # a change to the variation itself is checked against experiments. + if not any( + field in data and data[field] != getattr(instance, field) + for field in VARIATION_FIELDS + ): + super().perform_update(serializer) + return + + features = {instance.feature, data.get("feature", instance.feature)} with transaction.atomic(): list( Feature.objects.select_for_update() - .filter(pk__in=feature_ids) + .filter(pk__in=[feature.pk for feature in features]) .order_by("pk") .values_list("pk", flat=True) ) for feature in features: - self._check_no_live_experiment(feature) + check_no_live_experiment(feature.id) super().perform_update(serializer) def perform_destroy(self, instance): # type: ignore[no-untyped-def] with transaction.atomic(): Feature.objects.select_for_update().filter(pk=instance.feature_id).exists() - self._check_no_live_experiment(instance.feature) + check_no_live_experiment(instance.feature_id) super().perform_destroy(instance) - def _check_no_live_experiment(self, feature: Feature) -> None: - if has_live_experiment(feature.id): - raise ValidationError(LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE) - def get_queryset(self): # type: ignore[no-untyped-def] if getattr(self, "swagger_fake_view", False): return MultivariateFeatureOption.objects.none() diff --git a/api/features/serializers.py b/api/features/serializers.py index c20a81119490..d02167f53c3c 100644 --- a/api/features/serializers.py +++ b/api/features/serializers.py @@ -1,6 +1,6 @@ from datetime import datetime from functools import cached_property -from typing import Any, Callable, cast +from typing import Any, Callable from uuid import UUID import django.core.exceptions @@ -31,9 +31,9 @@ from evaluation.types import EvaluatedFeatureState from experimentation.feature_state_metadata import ( get_feature_state_metadata_builder, - has_live_experiment, + get_live_experiment, + live_experiment_error, ) -from features.constants import LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE from integrations.vcs.services import dispatch_vcs_on_feature_state_changed from metadata.serializers import MetadataSerializer, MetadataSerializerMixin from projects.code_references.serializers import ( @@ -312,16 +312,13 @@ def validate_project_features_limit(self, project: Project) -> None: def update(self, instance: Feature, validated_data: dict[str, Any]) -> Feature: with transaction.atomic(): Feature.objects.select_for_update().filter(pk=instance.pk).exists() - if ( - has_live_experiment(instance.id) - and "multivariate_options" in validated_data + if "multivariate_options" in validated_data and ( + experiment := get_live_experiment(instance.id) ): if self._changes_multivariate_options( validated_data["multivariate_options"], instance ): - raise serializers.ValidationError( - {"multivariate_options": LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE} - ) + raise live_experiment_error(experiment) validated_data.pop("multivariate_options") return super().update(instance, validated_data) # type: ignore[no-any-return] @@ -342,23 +339,13 @@ def validate_multivariate_options(self, multivariate_options): # type: ignore[n ) if total_percentage_allocation > 100: raise serializers.ValidationError("Invalid percentage allocation") - instance = cast(Feature | None, self.instance) - if ( - instance - and self._changes_multivariate_options(multivariate_options, instance) - and has_live_experiment(instance.id) - ): - raise serializers.ValidationError(LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE) return multivariate_options def _changes_multivariate_options( self, multivariate_options: list[dict[str, Any]] | None, - instance: Feature | None = None, + instance: Feature, ) -> bool: - instance = instance or cast(Feature | None, self.instance) - if instance is None: - return False # The frontend resubmits the whole feature, so only a real change to # writable option fields counts. Compare on content, not ids. fields = ( diff --git a/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py b/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py index 1e3b576dd9e4..64f57b86cb5f 100644 --- a/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py +++ b/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py @@ -2,15 +2,16 @@ from datetime import timedelta import pytest -from django.core.exceptions import ValidationError from django.db.models import Q from django.utils import timezone from freezegun.api import FrozenDateTimeFactory from pytest_django.fixtures import SettingsWrapper +from rest_framework.exceptions import ValidationError from environments.identities.models import Identity from environments.models import Environment from experimentation.models import Experiment, ExperimentStatus +from features.constants import LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE from features.feature_types import MULTIVARIATE, STANDARD from features.import_export.constants import ( FAILED, @@ -24,7 +25,6 @@ FeatureImport, FlagsmithOnFlagsmithFeatureExport, ) -from features.import_export.services import overwrite_feature_for_environment from features.import_export.tasks import ( _create_flagsmith_on_flagsmith_feature_export, clear_stale_feature_imports_and_exports, @@ -43,18 +43,103 @@ from segments.models import Segment -def test_overwrite_feature_for_environment__live_experiment__raises_validation_error( +def _feature_import( + environment: Environment, features: list[FeatureExportData] +) -> FeatureImport: + return FeatureImport.objects.create( + environment=environment, + strategy=OVERWRITE_DESTRUCTIVE, + data=json.dumps(features), + ) + + +def _export_data( + feature: Feature, multivariate: list[dict[str, object]] +) -> FeatureExportData: + return { + "name": feature.name, + "default_enabled": feature.default_enabled, + "is_server_key_only": feature.is_server_key_only, + "initial_value": feature.initial_value, + "value": "imported", + "type": STRING, + "enabled": False, + "multivariate": multivariate, # type: ignore[typeddict-item] + } + + +NEW_FEATURE_DATA: FeatureExportData = { + "name": "imported before the rejected feature", + "default_enabled": False, + "is_server_key_only": False, + "initial_value": "initial", + "value": "imported", + "type": STRING, + "enabled": True, + "multivariate": [], +} + + +@pytest.mark.parametrize( + "experiment_status", [ExperimentStatus.RUNNING, ExperimentStatus.PAUSED] +) +def test_import_features__experiment_in_target_environment__changes_nothing( db: None, environment: Environment, + project: Project, feature: Feature, feature_segment: FeatureSegment, + experiment_status: str, +) -> None: + # Given + feature_state = feature.feature_states.get(environment=environment) + feature_state.enabled = True + feature_state.save() + Experiment.objects.create( + environment=environment, + feature=feature, + name="live experiment", + hypothesis="new option wins", + status=experiment_status, + ) + feature_import = _feature_import( + environment, [NEW_FEATURE_DATA, _export_data(feature, [])] + ) + + # When + with pytest.raises(ValidationError) as exc_info: + import_features_for_environment(feature_import.id) + + # Then + assert exc_info.value.detail == { + "detail": LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE.format( + environment=environment.name + ) + } + feature_import.refresh_from_db() + assert feature_import.status == FAILED + assert not Feature.objects.filter( + project=project, name=NEW_FEATURE_DATA["name"] + ).exists() + assert FeatureSegment.objects.filter(pk=feature_segment.pk).exists() + feature_state.refresh_from_db() + assert feature_state.enabled is True + assert feature_state.get_feature_state_value() == feature.initial_value + + +def test_import_features__adds_variation_to_feature_in_experiment__changes_nothing( + db: None, + environment: Environment, + environment_two: Environment, + project: Project, + feature: Feature, ) -> None: # Given MultivariateFeatureOption.objects.create( feature=feature, type=STRING, string_value="existing", - default_percentage_allocation=100, + default_percentage_allocation=0, ) Experiment.objects.create( environment=environment, @@ -63,45 +148,52 @@ def test_overwrite_feature_for_environment__live_experiment__raises_validation_e hypothesis="new option wins", status=ExperimentStatus.RUNNING, ) - feature_state = feature.feature_states.get(environment=environment) - imported_feature: FeatureExportData = { - "name": feature.name, - "default_enabled": feature.default_enabled, - "is_server_key_only": feature.is_server_key_only, - "initial_value": feature.initial_value, - "value": feature.initial_value, - "type": STRING, - "enabled": feature_state.enabled, - "multivariate": [ - { - "percentage_allocation": 100, - "default_percentage_allocation": 100, - "value": "existing", - "type": STRING, - } + feature_import = _feature_import( + environment_two, + [ + NEW_FEATURE_DATA, + _export_data( + feature, + [ + { + "percentage_allocation": 50, + "default_percentage_allocation": 0, + "value": "new variation", + "type": STRING, + } + ], + ), ], - } + ) - # When / Then + # When with pytest.raises(ValidationError): - overwrite_feature_for_environment(imported_feature, feature, environment) + import_features_for_environment(feature_import.id) - assert FeatureSegment.objects.filter(pk=feature_segment.pk).exists() - assert feature.multivariate_options.count() == 1 - assert ( - feature.feature_states.get(environment=environment).feature_state_value.value - == feature.initial_value - ) + # Then + feature_import.refresh_from_db() + assert feature_import.status == FAILED + assert not Feature.objects.filter( + project=project, name=NEW_FEATURE_DATA["name"] + ).exists() + assert list( + feature.multivariate_options.values_list("string_value", flat=True) + ) == ["existing"] -def test_import_features__live_experiment_on_later_feature__rolls_back_batch( +def test_import_features__existing_variations_into_other_environment__succeeds( db: None, environment: Environment, - project: Project, + environment_two: Environment, feature: Feature, - feature_segment: FeatureSegment, ) -> None: # Given + option = MultivariateFeatureOption.objects.create( + feature=feature, + type=STRING, + string_value="existing", + default_percentage_allocation=0, + ) Experiment.objects.create( environment=environment, feature=feature, @@ -109,46 +201,46 @@ def test_import_features__live_experiment_on_later_feature__rolls_back_batch( hypothesis="new option wins", status=ExperimentStatus.RUNNING, ) - feature_import = FeatureImport.objects.create( - environment=environment, - strategy=OVERWRITE_DESTRUCTIVE, - data=json.dumps( - [ - { - "name": "created before failure", - "default_enabled": False, - "is_server_key_only": False, - "initial_value": "initial", - "value": "imported", - "type": STRING, - "enabled": True, - "multivariate": [], - }, - { - "name": feature.name, - "default_enabled": feature.default_enabled, - "is_server_key_only": feature.is_server_key_only, - "initial_value": feature.initial_value, - "value": feature.initial_value, - "type": STRING, - "enabled": False, - "multivariate": [], - }, - ] - ), - ) - - # When / Then - with pytest.raises(ValidationError): - import_features_for_environment(feature_import.id) + feature_import = _feature_import( + environment_two, + [ + _export_data( + feature, + [ + { + "percentage_allocation": 40, + "default_percentage_allocation": 0, + "value": "existing", + "type": STRING, + } + ], + ) + ], + ) + + # When + import_features_for_environment(feature_import.id) + # Then feature_import.refresh_from_db() - assert feature_import.status == FAILED - assert not Feature.objects.filter( - project=project, name="created before failure" - ).exists() - assert FeatureSegment.objects.filter(pk=feature_segment.pk).exists() - assert feature.feature_states.get(environment=environment).enabled is False + assert feature_import.status == SUCCESS + assert list(feature.multivariate_options.all()) == [option] + feature_state_two = feature.feature_states.get( + environment=environment_two, identity=None, feature_segment=None + ) + assert feature_state_two.get_feature_state_value() == "imported" + assert ( + feature_state_two.multivariate_feature_state_values.get( + multivariate_feature_option=option + ).percentage_allocation + == 40 + ) + assert ( + feature.feature_states.get( + environment=environment, identity=None, feature_segment=None + ).get_feature_state_value() + == feature.initial_value + ) def test_clear_stale_feature_imports_and_exports__stale_records__deletes_old_keeps_new( # type: ignore[no-untyped-def] diff --git a/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py b/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py index 1b942a9a052d..f9279b0c6b98 100644 --- a/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py +++ b/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py @@ -1,7 +1,11 @@ +from typing import Any + import pytest +from django.urls import reverse from rest_framework import status from rest_framework.test import APIClient +from environments.models import Environment from experimentation.models import Experiment, ExperimentStatus from features.constants import LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE from features.models import Feature @@ -9,14 +13,18 @@ from projects.models import Project -@pytest.fixture() -def live_experiment(environment, multivariate_feature): # type: ignore[no-untyped-def] - return Experiment.objects.create( +@pytest.fixture(params=[ExperimentStatus.RUNNING, ExperimentStatus.PAUSED]) +def live_experiment( + request: pytest.FixtureRequest, + environment: Environment, + multivariate_feature: Feature, +) -> Experiment: + return Experiment.objects.create( # type: ignore[no-any-return] environment=environment, feature=multivariate_feature, name="live experiment", hypothesis="new option wins", - status=ExperimentStatus.RUNNING, + status=request.param, ) @@ -24,9 +32,34 @@ def _options_url(project: Project, feature: Feature) -> str: return f"/api/v1/projects/{project.id}/features/{feature.id}/mv-options/" -def test_create_mv_option__live_experiment__returns_400( # type: ignore[no-untyped-def] - admin_client: APIClient, project, multivariate_feature, live_experiment -): +def _expected_error(experiment: Experiment) -> dict[str, str]: + return { + "detail": LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE.format( + environment=experiment.environment.name + ) + } + + +def _dashboard_option_payload(option: MultivariateFeatureOption) -> dict[str, Any]: + # What the dashboard PUTs for every existing option on each save. + return { + "id": option.id, + "type": option.type, + "string_value": option.string_value, + "integer_value": option.integer_value, + "boolean_value": option.boolean_value, + "key": option.key, + "default_percentage_allocation": 0, + "feature": option.feature_id, + } + + +def test_create_mv_option__live_experiment__returns_400( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, + live_experiment: Experiment, +) -> None: # Given # When response = admin_client.post( @@ -43,38 +76,216 @@ def test_create_mv_option__live_experiment__returns_400( # type: ignore[no-unty # Then assert response.status_code == status.HTTP_400_BAD_REQUEST - assert LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE in str(response.json()) + assert response.json() == _expected_error(live_experiment) assert multivariate_feature.multivariate_options.count() == 3 -def test_update_mv_option__live_experiment__returns_400( # type: ignore[no-untyped-def] - admin_client: APIClient, project, multivariate_feature, live_experiment -): +def test_update_mv_option__running_experiment_unchanged_payload__returns_200( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, + live_experiment: Experiment, +) -> None: # Given option = multivariate_feature.multivariate_options.first() + assert option is not None + option.key = "variation-a" + option.save() # When response = admin_client.put( f"{_options_url(project, multivariate_feature)}{option.id}/", - data={ - "feature": multivariate_feature.id, - "type": "unicode", - "string_value": "edited", - "default_percentage_allocation": option.default_percentage_allocation, - "key": option.key, - }, + data=_dashboard_option_payload(option), + format="json", + ) + + # Then + assert response.status_code == status.HTTP_200_OK + option.refresh_from_db() + assert option.string_value == "multivariate option for 30% of users." + assert option.key == "variation-a" + + +def test_update_mv_option__running_experiment_default_allocation_only__returns_200( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, + live_experiment: Experiment, +) -> None: + # Given + option = multivariate_feature.multivariate_options.first() + assert option is not None + option.key = "variation-a" + option.save() + payload = {**_dashboard_option_payload(option), "default_percentage_allocation": 10} + + # When + response = admin_client.put( + f"{_options_url(project, multivariate_feature)}{option.id}/", + data=payload, + format="json", + ) + + # Then + assert response.status_code == status.HTTP_200_OK + option.refresh_from_db() + assert option.default_percentage_allocation == 10 + assert option.string_value == "multivariate option for 30% of users." + assert option.key == "variation-a" + + +@pytest.mark.parametrize( + "field, new_value", + [("string_value", "edited"), ("key", "edited-key")], +) +def test_update_mv_option__running_experiment_changed_field__returns_400( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, + live_experiment: Experiment, + field: str, + new_value: str, +) -> None: + # Given + option = multivariate_feature.multivariate_options.first() + assert option is not None + option.key = "variation-a" + option.save() + payload = {**_dashboard_option_payload(option), field: new_value} + + # When + response = admin_client.put( + f"{_options_url(project, multivariate_feature)}{option.id}/", + data=payload, format="json", ) # Then assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json() == _expected_error(live_experiment) + option.refresh_from_db() + assert option.string_value == "multivariate option for 30% of users." + assert option.key == "variation-a" + + +@pytest.mark.parametrize( + "experiment_status", [ExperimentStatus.CREATED, ExperimentStatus.COMPLETED] +) +def test_update_mv_option__experiment_not_running__returns_200( + admin_client: APIClient, + project: Project, + environment: Environment, + multivariate_feature: Feature, + experiment_status: str, +) -> None: + # Given + Experiment.objects.create( + environment=environment, + feature=multivariate_feature, + name="inactive experiment", + hypothesis="new option wins", + status=experiment_status, + ) + option = multivariate_feature.multivariate_options.first() + assert option is not None + payload = {**_dashboard_option_payload(option), "string_value": "edited"} + + # When + response = admin_client.put( + f"{_options_url(project, multivariate_feature)}{option.id}/", + data=payload, + format="json", + ) + + # Then + assert response.status_code == status.HTTP_200_OK option.refresh_from_db() - assert option.string_value != "edited" + assert option.string_value == "edited" + + +def test_save_feature_value__experiment_running_in_other_environment__returns_200( + admin_client: APIClient, + project: Project, + environment: Environment, + environment_two: Environment, + multivariate_feature: Feature, + live_experiment: Experiment, +) -> None: + # Given + options = list(multivariate_feature.multivariate_options.order_by("id")) + feature_state_a = environment.feature_states.get( + feature=multivariate_feature, identity=None, feature_segment=None + ) + feature_state_b = environment_two.feature_states.get( + feature=multivariate_feature, identity=None, feature_segment=None + ) + allocations_a = { + mv.multivariate_feature_option_id: mv.percentage_allocation + for mv in feature_state_a.multivariate_feature_state_values.all() + } + new_allocations_b = dict(zip((option.id for option in options), (10, 20, 30))) + # When + option_responses = [ + admin_client.put( + f"{_options_url(project, multivariate_feature)}{option.id}/", + data=_dashboard_option_payload(option), + format="json", + ) + for option in options + ] + feature_state_response = admin_client.put( + reverse( + "api-v1:environments:environment-featurestates-detail", + args=[environment_two.api_key, feature_state_b.id], + ), + data={ + "id": feature_state_b.id, + "feature_state_value": "new control", + "enabled": True, + "feature": multivariate_feature.id, + "environment": environment_two.id, + "identity": None, + "feature_segment": None, + "multivariate_feature_state_values": [ + { + "id": mv.id, + "multivariate_feature_option": mv.multivariate_feature_option_id, + "percentage_allocation": new_allocations_b[ + mv.multivariate_feature_option_id + ], + } + for mv in feature_state_b.multivariate_feature_state_values.all() + ], + }, + format="json", + ) -def test_update_mv_option__moving_to_live_experiment__returns_400( # type: ignore[no-untyped-def] - admin_client: APIClient, project, environment, multivariate_feature -): + # Then + assert [r.status_code for r in option_responses] == [status.HTTP_200_OK] * 3 + assert feature_state_response.status_code == status.HTTP_200_OK + + feature_state_b.refresh_from_db() + assert feature_state_b.get_feature_state_value() == "new control" + assert { + mv.multivariate_feature_option_id: mv.percentage_allocation + for mv in feature_state_b.multivariate_feature_state_values.all() + } == new_allocations_b + + feature_state_a.refresh_from_db() + assert feature_state_a.get_feature_state_value() == "control" + assert { + mv.multivariate_feature_option_id: mv.percentage_allocation + for mv in feature_state_a.multivariate_feature_state_values.all() + } == allocations_a + + +def test_update_mv_option__moving_to_live_experiment__returns_400( + admin_client: APIClient, + project: Project, + environment: Environment, + multivariate_feature: Feature, +) -> None: # Given other_feature = Feature.objects.create(name="other feature", project=project) option = MultivariateFeatureOption.objects.create( @@ -87,7 +298,7 @@ def test_update_mv_option__moving_to_live_experiment__returns_400( # type: igno existing_option.default_percentage_allocation = 0 existing_option.save() - Experiment.objects.create( + experiment = Experiment.objects.create( environment=environment, feature=multivariate_feature, name="live experiment", @@ -109,14 +320,17 @@ def test_update_mv_option__moving_to_live_experiment__returns_400( # type: igno # Then assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json() == _expected_error(experiment) option.refresh_from_db() assert option.feature == other_feature assert multivariate_feature.multivariate_options.count() == 3 -def test_create_mv_option__body_feature_mismatches_url__returns_400( # type: ignore[no-untyped-def] - admin_client: APIClient, project, multivariate_feature -): +def test_create_mv_option__body_feature_mismatches_url__returns_400( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, +) -> None: # Given other_feature = Feature.objects.create(name="other feature", project=project) @@ -133,11 +347,15 @@ def test_create_mv_option__body_feature_mismatches_url__returns_400( # type: ig assert other_feature.multivariate_options.count() == 0 -def test_delete_mv_option__live_experiment__returns_400( # type: ignore[no-untyped-def] - admin_client: APIClient, project, multivariate_feature, live_experiment -): +def test_delete_mv_option__live_experiment__returns_400( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, + live_experiment: Experiment, +) -> None: # Given option = multivariate_feature.multivariate_options.first() + assert option is not None # When response = admin_client.delete( @@ -146,12 +364,15 @@ def test_delete_mv_option__live_experiment__returns_400( # type: ignore[no-unty # Then assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json() == _expected_error(live_experiment) assert MultivariateFeatureOption.objects.filter(id=option.id).exists() -def test_create_mv_option__no_live_experiment__not_blocked_by_experiment_check( # type: ignore[no-untyped-def] - admin_client: APIClient, project, multivariate_feature -): +def test_create_mv_option__no_live_experiment__returns_201( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, +) -> None: # Given # When response = admin_client.post( @@ -170,9 +391,12 @@ def test_create_mv_option__no_live_experiment__not_blocked_by_experiment_check( assert response.status_code == status.HTTP_201_CREATED -def test_update_feature__live_experiment__changed_mv_options_returns_400( # type: ignore[no-untyped-def] - admin_client: APIClient, project, multivariate_feature, live_experiment -): +def test_update_feature__live_experiment__changed_mv_options_returns_400( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, + live_experiment: Experiment, +) -> None: # Given url = f"/api/v1/projects/{project.id}/features/{multivariate_feature.id}/" options = [ @@ -190,14 +414,18 @@ def test_update_feature__live_experiment__changed_mv_options_returns_400( # typ # Then assert response.status_code == status.HTTP_400_BAD_REQUEST - assert LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE in str(response.json()) + assert response.json() == _expected_error(live_experiment) -def test_update_feature__live_experiment__unchanged_keyed_options_preserved( # type: ignore[no-untyped-def] - admin_client: APIClient, project, multivariate_feature, live_experiment -): +def test_update_feature__live_experiment__unchanged_keyed_options_preserved( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, + live_experiment: Experiment, +) -> None: # Given keyed_option = multivariate_feature.multivariate_options.first() + assert keyed_option is not None keyed_option.key = "stable-option" keyed_option.save() url = f"/api/v1/projects/{project.id}/features/{multivariate_feature.id}/" diff --git a/docs/docs/deployment-self-hosting/observability/_events-catalogue.md b/docs/docs/deployment-self-hosting/observability/_events-catalogue.md index e40b24429464..06badcad69c0 100644 --- a/docs/docs/deployment-self-hosting/observability/_events-catalogue.md +++ b/docs/docs/deployment-self-hosting/observability/_events-catalogue.md @@ -240,7 +240,7 @@ Attributes: ### `experimentation.rollout.applied` Logged at `info` from: - - `api/experimentation/services.py:1081` + - `api/experimentation/services.py:1084` Attributes: - `audience.match` @@ -256,7 +256,7 @@ Attributes: ### `experimentation.rollout.superseded` Logged at `info` from: - - `api/experimentation/services.py:672` + - `api/experimentation/services.py:675` Attributes: - `environment.id` @@ -933,7 +933,7 @@ Attributes: ### `warehouse.connection.connected` Logged at `info` from: - - `api/experimentation/services.py:1257` + - `api/experimentation/services.py:1260` Attributes: - `environment.id` @@ -964,7 +964,7 @@ Attributes: ### `warehouse.connection.test_event_sent` Logged at `info` from: - - `api/experimentation/services.py:1200` + - `api/experimentation/services.py:1203` Attributes: - `environment.id` @@ -973,7 +973,7 @@ Attributes: ### `warehouse.connection.verification_failed` Logged at `warning` from: - - `api/experimentation/services.py:1232` + - `api/experimentation/services.py:1235` Attributes: - `environment.id` @@ -983,7 +983,7 @@ Attributes: ### `warehouse.connection.verification_succeeded` Logged at `info` from: - - `api/experimentation/services.py:1242` + - `api/experimentation/services.py:1245` Attributes: - `environment.id` diff --git a/docs/docs/experimentation/create-an-experiment.md b/docs/docs/experimentation/create-an-experiment.md index 82fccfe996e0..838f6594036b 100644 --- a/docs/docs/experimentation/create-an-experiment.md +++ b/docs/docs/experimentation/create-an-experiment.md @@ -18,7 +18,9 @@ and measures the impact on a metric. - A [connected warehouse](/experimentation/connect-a-warehouse). - A [**multivariate flag**](/managing-flags/core-management) with a variation for each treatment you want to test. The - flag's current value acts as the **control**. A flag can only be in one active experiment at a time. + flag's current value acts as the **control**. A flag can only be in one active experiment at a time. While the + experiment is running or paused, the flag's variations cannot be added, removed or edited; complete the experiment + first if you need to change them. - A [metric](/experimentation/create-metrics). You can also create one from inside the wizard. On the **Experiments** page, click **Create Experiment**. The wizard has four steps. From 8682e20c5fa51c026318bf086d70466d3bd485eb Mon Sep 17 00:00:00 2001 From: TejasLamba2006 Date: Sat, 14 Nov 2026 12:49:16 +0530 Subject: [PATCH 5/6] fix: skip features under live or paused experiments during destructive import Skip individual features in _import_features_for_environment instead of failing the whole import when a feature has a running or paused experiment in the target environment or when the import would add a variation while an experiment is running or paused in any environment. --- api/experimentation/feature_state_metadata.py | 17 +++-- api/features/import_export/tasks.py | 56 +++++++------- .../test_unit_features_import_export_tasks.py | 75 ++++++++++++++----- 3 files changed, 91 insertions(+), 57 deletions(-) diff --git a/api/experimentation/feature_state_metadata.py b/api/experimentation/feature_state_metadata.py index 4375402479b7..f93c799e1fab 100644 --- a/api/experimentation/feature_state_metadata.py +++ b/api/experimentation/feature_state_metadata.py @@ -11,18 +11,19 @@ from features.models import FeatureState -def get_live_experiment(feature_id: int) -> Experiment | None: +def get_live_experiment( + feature_id: int, environment_id: int | None = None +) -> Experiment | None: # Multivariate options belong to the feature, not an environment, so an # experiment in any environment locks them. A paused experiment counts: # changing the variants before it resumes would skew its results. - return ( # type: ignore[no-any-return] - Experiment.objects.filter( - feature_id=feature_id, - status__in=(ExperimentStatus.RUNNING, ExperimentStatus.PAUSED), - ) - .select_related("environment") - .first() + experiments = Experiment.objects.filter( + feature_id=feature_id, + status__in=(ExperimentStatus.RUNNING, ExperimentStatus.PAUSED), ) + if environment_id is not None: + experiments = experiments.filter(environment_id=environment_id) + return experiments.select_related("environment").first() # type: ignore[no-any-return] def live_experiment_error(experiment: Experiment) -> ValidationError: diff --git a/api/features/import_export/tasks.py b/api/features/import_export/tasks.py index c02cf671ce1d..cae92602be4d 100644 --- a/api/features/import_export/tasks.py +++ b/api/features/import_export/tasks.py @@ -11,10 +11,7 @@ ) from environments.models import Environment -from experimentation.feature_state_metadata import ( - get_live_experiment, - live_experiment_error, -) +from experimentation.feature_state_metadata import get_live_experiment from features.import_export.constants import ( FAILED, PROCESSING, @@ -152,16 +149,18 @@ def _import_features_for_environment(feature_import: FeatureImport) -> None: input_data: list[FeatureExportData] = json.loads(feature_import.data) project = environment.project - if feature_import.strategy != SKIP: - _check_import_against_live_experiments(input_data, environment) - for feature_data in input_data: existing_feature = Feature.objects.filter( name=feature_data["name"], project=project, ).first() - if existing_feature and feature_import.strategy == SKIP: + if existing_feature and ( + feature_import.strategy == SKIP + or _should_skip_for_live_experiment( + feature_data, existing_feature, environment + ) + ): continue if existing_feature is None: @@ -174,29 +173,24 @@ def _import_features_for_environment(feature_import: FeatureImport) -> None: feature_import.save() -def _check_import_against_live_experiments( - input_data: list[FeatureExportData], environment: Environment -) -> None: - """ - Reject the whole import, before anything is written, if it would add a - variation to a feature in an experiment, or overwrite the environment the - experiment runs in (which deletes its rollout segment override). - """ - for feature_data in input_data: - feature = Feature.objects.filter( - name=feature_data["name"], project=environment.project - ).first() - if feature is None or not (experiment := get_live_experiment(feature.id)): - continue - existing_variations = { - (option.type, option.value) for option in feature.multivariate_options.all() - } - adds_variation = any( - (mv_data["type"], mv_data["value"]) not in existing_variations - for mv_data in feature_data["multivariate"] - ) - if adds_variation or experiment.environment_id == environment.id: - raise live_experiment_error(experiment) +def _should_skip_for_live_experiment( + feature_data: FeatureExportData, + existing_feature: Feature, + environment: Environment, +) -> bool: + if get_live_experiment(existing_feature.id, environment.id): + return True + if not feature_data["multivariate"]: + return False + existing_variations = { + (option.type, option.value) + for option in existing_feature.multivariate_options.all() + } + adds_variation = any( + (mv_data["type"], mv_data["value"]) not in existing_variations + for mv_data in feature_data["multivariate"] + ) + return bool(adds_variation and get_live_experiment(existing_feature.id)) # Should only run on official flagsmith instance. diff --git a/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py b/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py index 64f57b86cb5f..403b93ea04b6 100644 --- a/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py +++ b/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py @@ -6,12 +6,10 @@ from django.utils import timezone from freezegun.api import FrozenDateTimeFactory from pytest_django.fixtures import SettingsWrapper -from rest_framework.exceptions import ValidationError from environments.identities.models import Identity from environments.models import Environment from experimentation.models import Experiment, ExperimentStatus -from features.constants import LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE from features.feature_types import MULTIVARIATE, STANDARD from features.import_export.constants import ( FAILED, @@ -69,7 +67,7 @@ def _export_data( NEW_FEATURE_DATA: FeatureExportData = { - "name": "imported before the rejected feature", + "name": "imported alongside skipped feature", "default_enabled": False, "is_server_key_only": False, "initial_value": "initial", @@ -83,7 +81,7 @@ def _export_data( @pytest.mark.parametrize( "experiment_status", [ExperimentStatus.RUNNING, ExperimentStatus.PAUSED] ) -def test_import_features__experiment_in_target_environment__changes_nothing( +def test_import_features__experiment_in_target_environment__skips_feature_and_imports_rest( db: None, environment: Environment, project: Project, @@ -107,18 +105,12 @@ def test_import_features__experiment_in_target_environment__changes_nothing( ) # When - with pytest.raises(ValidationError) as exc_info: - import_features_for_environment(feature_import.id) + import_features_for_environment(feature_import.id) # Then - assert exc_info.value.detail == { - "detail": LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE.format( - environment=environment.name - ) - } feature_import.refresh_from_db() - assert feature_import.status == FAILED - assert not Feature.objects.filter( + assert feature_import.status == SUCCESS + assert Feature.objects.filter( project=project, name=NEW_FEATURE_DATA["name"] ).exists() assert FeatureSegment.objects.filter(pk=feature_segment.pk).exists() @@ -127,7 +119,49 @@ def test_import_features__experiment_in_target_environment__changes_nothing( assert feature_state.get_feature_state_value() == feature.initial_value -def test_import_features__adds_variation_to_feature_in_experiment__changes_nothing( +def test_import_features__experiments_in_multiple_environments__skips_target_environment( + db: None, + environment: Environment, + environment_two: Environment, + project: Project, + feature: Feature, + segment: Segment, +) -> None: + # Given + segment_two = FeatureSegment.objects.create( + feature=feature, segment=segment, environment=environment_two + ) + for env in (environment, environment_two): + Experiment.objects.create( + environment=env, + feature=feature, + name=f"live experiment {env.id}", + hypothesis="new option wins", + status=ExperimentStatus.RUNNING, + ) + feature_import = _feature_import( + environment_two, [NEW_FEATURE_DATA, _export_data(feature, [])] + ) + + # When + import_features_for_environment(feature_import.id) + + # Then + feature_import.refresh_from_db() + assert feature_import.status == SUCCESS + assert Feature.objects.filter( + project=project, name=NEW_FEATURE_DATA["name"] + ).exists() + assert FeatureSegment.objects.filter(pk=segment_two.pk).exists() + assert ( + feature.feature_states.get( + environment=environment_two, identity=None, feature_segment=None + ).get_feature_state_value() + == feature.initial_value + ) + + +def test_import_features__adds_variation_to_feature_in_experiment__skips_feature_and_imports_rest( db: None, environment: Environment, environment_two: Environment, @@ -167,18 +201,23 @@ def test_import_features__adds_variation_to_feature_in_experiment__changes_nothi ) # When - with pytest.raises(ValidationError): - import_features_for_environment(feature_import.id) + import_features_for_environment(feature_import.id) # Then feature_import.refresh_from_db() - assert feature_import.status == FAILED - assert not Feature.objects.filter( + assert feature_import.status == SUCCESS + assert Feature.objects.filter( project=project, name=NEW_FEATURE_DATA["name"] ).exists() assert list( feature.multivariate_options.values_list("string_value", flat=True) ) == ["existing"] + assert ( + feature.feature_states.get( + environment=environment_two, identity=None, feature_segment=None + ).get_feature_state_value() + == feature.initial_value + ) def test_import_features__existing_variations_into_other_environment__succeeds( From 7aa28e96bdf881f8e7f574e802e0e1b32bef8087 Mon Sep 17 00:00:00 2001 From: TejasLamba2006 Date: Tue, 17 Nov 2026 22:25:46 +0530 Subject: [PATCH 6/6] fix(features): ignore default_percentage_allocation in feature MV option diff --- api/features/serializers.py | 4 ++-- .../multivariate/test_unit_multivariate_live_experiment.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/features/serializers.py b/api/features/serializers.py index d02167f53c3c..b2d935b17ccf 100644 --- a/api/features/serializers.py +++ b/api/features/serializers.py @@ -347,13 +347,13 @@ def _changes_multivariate_options( instance: Feature, ) -> bool: # The frontend resubmits the whole feature, so only a real change to - # writable option fields counts. Compare on content, not ids. + # variation value/type counts. `default_percentage_allocation` only + # seeds new options or environments and does not affect live variants. fields = ( "type", "integer_value", "string_value", "boolean_value", - "default_percentage_allocation", ) current = NestedMultivariateFeatureOptionSerializer( instance.multivariate_options.all(), many=True diff --git a/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py b/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py index f9279b0c6b98..a2e1ed2f6b8d 100644 --- a/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py +++ b/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py @@ -436,7 +436,7 @@ def test_update_feature__live_experiment__unchanged_keyed_options_preserved( { "type": option.type, "string_value": option.string_value, - "default_percentage_allocation": option.default_percentage_allocation, + "default_percentage_allocation": 0, } for option in multivariate_feature.multivariate_options.all() ]