diff --git a/api/experimentation/feature_state_metadata.py b/api/experimentation/feature_state_metadata.py index b0af6cddfcbd..f93c799e1fab 100644 --- a/api/experimentation/feature_state_metadata.py +++ b/api/experimentation/feature_state_metadata.py @@ -1,14 +1,46 @@ from typing import TYPE_CHECKING, Any, Callable from flagsmith_schemas.api import FeatureStateMetadata +from rest_framework.exceptions import ValidationError from experimentation.models import Experiment, ExperimentStatus +from features.constants import LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE if TYPE_CHECKING: # pragma: no cover from environments.models import Environment from features.models import FeatureState +def get_live_experiment( + feature_id: int, environment_id: int | None = None +) -> Experiment | None: + # Multivariate options belong to the feature, not an environment, so an + # experiment in any environment locks them. A paused experiment counts: + # changing the variants before it resumes would skew its results. + experiments = Experiment.objects.filter( + feature_id=feature_id, + status__in=(ExperimentStatus.RUNNING, ExperimentStatus.PAUSED), + ) + if environment_id is not None: + experiments = experiments.filter(environment_id=environment_id) + return experiments.select_related("environment").first() # type: ignore[no-any-return] + + +def live_experiment_error(experiment: Experiment) -> ValidationError: + return ValidationError( + { + "detail": LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE.format( + environment=experiment.environment.name + ) + } + ) + + +def check_no_live_experiment(feature_id: int) -> None: + if experiment := get_live_experiment(feature_id): + raise live_experiment_error(experiment) + + def get_feature_state_metadata_builder( environment: "Environment", ) -> Callable[["FeatureState"], dict[str, Any] | None]: diff --git a/api/experimentation/services.py b/api/experimentation/services.py index 9434e0885f93..7f5b8800a06f 100644 --- a/api/experimentation/services.py +++ b/api/experimentation/services.py @@ -79,7 +79,7 @@ from features.feature_states.models import API_VALUE_TYPES from features.future.services import delete_segment_override, update_flag from features.future.types import SegmentOverrideRequest -from features.models import FeatureState +from features.models import Feature, FeatureState from features.value_types import BOOLEAN, STRING from features.versioning.versioning_service import get_environment_flags_list from integrations.flagsmith.client import get_openfeature_client @@ -617,6 +617,9 @@ def transition_experiment_status( experiment.status = target_status if target_status == ExperimentStatus.RUNNING and not experiment.started_at: + Feature.objects.select_for_update().filter( + pk=experiment.feature_id + ).exists() experiment.started_at = timezone.now() _remove_superseded_experiment_rollouts( experiment, AuthorData.from_user(user) diff --git a/api/features/constants.py b/api/features/constants.py index ad2008ce00fa..3cf4a30318fa 100644 --- a/api/features/constants.py +++ b/api/features/constants.py @@ -10,6 +10,10 @@ # Multivariate variant reported when an identity falls through to the control value CONTROL_VARIANT_KEY = "control" RESERVED_VARIANT_KEY_MESSAGE = f'"{CONTROL_VARIANT_KEY}" is a reserved variant key.' +LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE = ( + 'Variations cannot be changed while an experiment is running or paused in the "{environment}" ' + "environment. Complete the experiment to change them." +) # Tag filtering strategy UNION = "UNION" diff --git a/api/features/import_export/tasks.py b/api/features/import_export/tasks.py index cdfe7cfa6fef..cae92602be4d 100644 --- a/api/features/import_export/tasks.py +++ b/api/features/import_export/tasks.py @@ -10,6 +10,8 @@ register_task_handler, ) +from environments.models import Environment +from experimentation.feature_state_metadata import get_live_experiment from features.import_export.constants import ( FAILED, PROCESSING, @@ -153,7 +155,12 @@ def _import_features_for_environment(feature_import: FeatureImport) -> None: project=project, ).first() - if existing_feature and feature_import.strategy == SKIP: + if existing_feature and ( + feature_import.strategy == SKIP + or _should_skip_for_live_experiment( + feature_data, existing_feature, environment + ) + ): continue if existing_feature is None: @@ -166,6 +173,26 @@ def _import_features_for_environment(feature_import: FeatureImport) -> None: feature_import.save() +def _should_skip_for_live_experiment( + feature_data: FeatureExportData, + existing_feature: Feature, + environment: Environment, +) -> bool: + if get_live_experiment(existing_feature.id, environment.id): + return True + if not feature_data["multivariate"]: + return False + existing_variations = { + (option.type, option.value) + for option in existing_feature.multivariate_options.all() + } + adds_variation = any( + (mv_data["type"], mv_data["value"]) not in existing_variations + for mv_data in feature_data["multivariate"] + ) + return bool(adds_variation and get_live_experiment(existing_feature.id)) + + # Should only run on official flagsmith instance. if ( settings.FLAGSMITH_ON_FLAGSMITH_FEATURE_EXPORT_ENVIRONMENT_ID diff --git a/api/features/multivariate/views.py b/api/features/multivariate/views.py index aecb0dee4a6d..5f78e022faf8 100644 --- a/api/features/multivariate/views.py +++ b/api/features/multivariate/views.py @@ -2,19 +2,33 @@ CREATE_FEATURE, VIEW_PROJECT, ) +from django.db import transaction from django.utils.decorators import method_decorator from drf_spectacular.utils import extend_schema from rest_framework import viewsets from rest_framework.decorators import api_view +from rest_framework.exceptions import ValidationError from rest_framework.generics import get_object_or_404 from rest_framework.response import Response +from experimentation.feature_state_metadata import check_no_live_experiment from features.models import Feature from projects.permissions import NestedProjectPermissions from .models import MultivariateFeatureOption from .serializers import MultivariateFeatureOptionSerializer +# What makes a variation. `default_percentage_allocation` is left out: the +# dashboard always sends it as 0, and it only seeds new environments. +VARIATION_FIELDS = ( + "type", + "string_value", + "integer_value", + "boolean_value", + "key", + "feature", +) + @method_decorator( name="list", @@ -69,9 +83,57 @@ def get_permissions(self): # type: ignore[no-untyped-def] def create(self, request, *args, **kwargs): # type: ignore[no-untyped-def] feature_pk = self.kwargs.get("feature_pk") project_pk = self.kwargs.get("project_pk") - get_object_or_404(Feature.objects.filter(project__id=project_pk), pk=feature_pk) + url_feature = get_object_or_404( + Feature.objects.filter(project__id=project_pk), pk=feature_pk + ) + if "feature" in request.data and str(request.data.get("feature")) != str( + url_feature.id + ): + raise ValidationError({"feature": "Feature must match the URL."}) return super().create(request, *args, **kwargs) + def perform_create(self, serializer): # type: ignore[no-untyped-def] + url_feature = get_object_or_404( + Feature.objects.filter(project__id=self.kwargs["project_pk"]), + pk=self.kwargs["feature_pk"], + ) + if serializer.validated_data["feature"] != url_feature: + raise ValidationError({"feature": "Feature must match the URL."}) + with transaction.atomic(): + Feature.objects.select_for_update().filter(pk=url_feature.pk).exists() + check_no_live_experiment(url_feature.id) + super().perform_create(serializer) + + def perform_update(self, serializer): # type: ignore[no-untyped-def] + instance = serializer.instance + data = serializer.validated_data + # The dashboard PUTs every option on each save, changed or not, so only + # a change to the variation itself is checked against experiments. + if not any( + field in data and data[field] != getattr(instance, field) + for field in VARIATION_FIELDS + ): + super().perform_update(serializer) + return + + features = {instance.feature, data.get("feature", instance.feature)} + with transaction.atomic(): + list( + Feature.objects.select_for_update() + .filter(pk__in=[feature.pk for feature in features]) + .order_by("pk") + .values_list("pk", flat=True) + ) + for feature in features: + check_no_live_experiment(feature.id) + super().perform_update(serializer) + + def perform_destroy(self, instance): # type: ignore[no-untyped-def] + with transaction.atomic(): + Feature.objects.select_for_update().filter(pk=instance.feature_id).exists() + check_no_live_experiment(instance.feature_id) + super().perform_destroy(instance) + def get_queryset(self): # type: ignore[no-untyped-def] if getattr(self, "swagger_fake_view", False): return MultivariateFeatureOption.objects.none() diff --git a/api/features/serializers.py b/api/features/serializers.py index 19e067b9bbba..b2d935b17ccf 100644 --- a/api/features/serializers.py +++ b/api/features/serializers.py @@ -12,7 +12,7 @@ FeatureStateValueSerializer, ) from common.projects.permissions import VIEW_PROJECT -from django.db import models +from django.db import models, transaction from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema_field from drf_writable_nested import ( # type: ignore[attr-defined] @@ -31,6 +31,8 @@ from evaluation.types import EvaluatedFeatureState from experimentation.feature_state_metadata import ( get_feature_state_metadata_builder, + get_live_experiment, + live_experiment_error, ) from integrations.vcs.services import dispatch_vcs_on_feature_state_changed from metadata.serializers import MetadataSerializer, MetadataSerializerMixin @@ -307,6 +309,19 @@ def validate_project_features_limit(self, project: Project) -> None: } ) + def update(self, instance: Feature, validated_data: dict[str, Any]) -> Feature: + with transaction.atomic(): + Feature.objects.select_for_update().filter(pk=instance.pk).exists() + if "multivariate_options" in validated_data and ( + experiment := get_live_experiment(instance.id) + ): + if self._changes_multivariate_options( + validated_data["multivariate_options"], instance + ): + raise live_experiment_error(experiment) + validated_data.pop("multivariate_options") + return super().update(instance, validated_data) # type: ignore[no-any-return] + def validate_multivariate_options(self, multivariate_options): # type: ignore[no-untyped-def] if multivariate_options: user = self.context["request"].user @@ -326,6 +341,32 @@ def validate_multivariate_options(self, multivariate_options): # type: ignore[n raise serializers.ValidationError("Invalid percentage allocation") return multivariate_options + def _changes_multivariate_options( + self, + multivariate_options: list[dict[str, Any]] | None, + instance: Feature, + ) -> bool: + # The frontend resubmits the whole feature, so only a real change to + # variation value/type counts. `default_percentage_allocation` only + # seeds new options or environments and does not affect live variants. + fields = ( + "type", + "integer_value", + "string_value", + "boolean_value", + ) + current = NestedMultivariateFeatureOptionSerializer( + instance.multivariate_options.all(), many=True + ).data + + def signature(options) -> list[tuple[Any, ...]]: # type: ignore[no-untyped-def] + return sorted( + (tuple(option.get(field) for field in fields) for option in options), + key=repr, + ) + + return signature(current) != signature(multivariate_options or []) + def validate_owners(self, owners: list[FFAdminUser]) -> list[FFAdminUser]: project: Project = self.context["project"] invalid_users = [ diff --git a/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py b/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py index 702f9a20b841..403b93ea04b6 100644 --- a/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py +++ b/api/tests/unit/features/import_export/test_unit_features_import_export_tasks.py @@ -9,6 +9,7 @@ from environments.identities.models import Identity from environments.models import Environment +from experimentation.models import Experiment, ExperimentStatus from features.feature_types import MULTIVARIATE, STANDARD from features.import_export.constants import ( FAILED, @@ -29,6 +30,7 @@ import_features_for_environment, retire_stalled_feature_imports_and_exports, ) +from features.import_export.types import FeatureExportData from features.models import Feature, FeatureSegment, FeatureState from features.multivariate.models import MultivariateFeatureOption from features.value_types import STRING @@ -39,6 +41,247 @@ from segments.models import Segment +def _feature_import( + environment: Environment, features: list[FeatureExportData] +) -> FeatureImport: + return FeatureImport.objects.create( + environment=environment, + strategy=OVERWRITE_DESTRUCTIVE, + data=json.dumps(features), + ) + + +def _export_data( + feature: Feature, multivariate: list[dict[str, object]] +) -> FeatureExportData: + return { + "name": feature.name, + "default_enabled": feature.default_enabled, + "is_server_key_only": feature.is_server_key_only, + "initial_value": feature.initial_value, + "value": "imported", + "type": STRING, + "enabled": False, + "multivariate": multivariate, # type: ignore[typeddict-item] + } + + +NEW_FEATURE_DATA: FeatureExportData = { + "name": "imported alongside skipped feature", + "default_enabled": False, + "is_server_key_only": False, + "initial_value": "initial", + "value": "imported", + "type": STRING, + "enabled": True, + "multivariate": [], +} + + +@pytest.mark.parametrize( + "experiment_status", [ExperimentStatus.RUNNING, ExperimentStatus.PAUSED] +) +def test_import_features__experiment_in_target_environment__skips_feature_and_imports_rest( + db: None, + environment: Environment, + project: Project, + feature: Feature, + feature_segment: FeatureSegment, + experiment_status: str, +) -> None: + # Given + feature_state = feature.feature_states.get(environment=environment) + feature_state.enabled = True + feature_state.save() + Experiment.objects.create( + environment=environment, + feature=feature, + name="live experiment", + hypothesis="new option wins", + status=experiment_status, + ) + feature_import = _feature_import( + environment, [NEW_FEATURE_DATA, _export_data(feature, [])] + ) + + # When + import_features_for_environment(feature_import.id) + + # Then + feature_import.refresh_from_db() + assert feature_import.status == SUCCESS + assert Feature.objects.filter( + project=project, name=NEW_FEATURE_DATA["name"] + ).exists() + assert FeatureSegment.objects.filter(pk=feature_segment.pk).exists() + feature_state.refresh_from_db() + assert feature_state.enabled is True + assert feature_state.get_feature_state_value() == feature.initial_value + + +def test_import_features__experiments_in_multiple_environments__skips_target_environment( + db: None, + environment: Environment, + environment_two: Environment, + project: Project, + feature: Feature, + segment: Segment, +) -> None: + # Given + segment_two = FeatureSegment.objects.create( + feature=feature, segment=segment, environment=environment_two + ) + for env in (environment, environment_two): + Experiment.objects.create( + environment=env, + feature=feature, + name=f"live experiment {env.id}", + hypothesis="new option wins", + status=ExperimentStatus.RUNNING, + ) + feature_import = _feature_import( + environment_two, [NEW_FEATURE_DATA, _export_data(feature, [])] + ) + + # When + import_features_for_environment(feature_import.id) + + # Then + feature_import.refresh_from_db() + assert feature_import.status == SUCCESS + assert Feature.objects.filter( + project=project, name=NEW_FEATURE_DATA["name"] + ).exists() + assert FeatureSegment.objects.filter(pk=segment_two.pk).exists() + assert ( + feature.feature_states.get( + environment=environment_two, identity=None, feature_segment=None + ).get_feature_state_value() + == feature.initial_value + ) + + +def test_import_features__adds_variation_to_feature_in_experiment__skips_feature_and_imports_rest( + db: None, + environment: Environment, + environment_two: Environment, + project: Project, + feature: Feature, +) -> None: + # Given + MultivariateFeatureOption.objects.create( + feature=feature, + type=STRING, + string_value="existing", + default_percentage_allocation=0, + ) + Experiment.objects.create( + environment=environment, + feature=feature, + name="live experiment", + hypothesis="new option wins", + status=ExperimentStatus.RUNNING, + ) + feature_import = _feature_import( + environment_two, + [ + NEW_FEATURE_DATA, + _export_data( + feature, + [ + { + "percentage_allocation": 50, + "default_percentage_allocation": 0, + "value": "new variation", + "type": STRING, + } + ], + ), + ], + ) + + # When + import_features_for_environment(feature_import.id) + + # Then + feature_import.refresh_from_db() + assert feature_import.status == SUCCESS + assert Feature.objects.filter( + project=project, name=NEW_FEATURE_DATA["name"] + ).exists() + assert list( + feature.multivariate_options.values_list("string_value", flat=True) + ) == ["existing"] + assert ( + feature.feature_states.get( + environment=environment_two, identity=None, feature_segment=None + ).get_feature_state_value() + == feature.initial_value + ) + + +def test_import_features__existing_variations_into_other_environment__succeeds( + db: None, + environment: Environment, + environment_two: Environment, + feature: Feature, +) -> None: + # Given + option = MultivariateFeatureOption.objects.create( + feature=feature, + type=STRING, + string_value="existing", + default_percentage_allocation=0, + ) + Experiment.objects.create( + environment=environment, + feature=feature, + name="live experiment", + hypothesis="new option wins", + status=ExperimentStatus.RUNNING, + ) + feature_import = _feature_import( + environment_two, + [ + _export_data( + feature, + [ + { + "percentage_allocation": 40, + "default_percentage_allocation": 0, + "value": "existing", + "type": STRING, + } + ], + ) + ], + ) + + # When + import_features_for_environment(feature_import.id) + + # Then + feature_import.refresh_from_db() + assert feature_import.status == SUCCESS + assert list(feature.multivariate_options.all()) == [option] + feature_state_two = feature.feature_states.get( + environment=environment_two, identity=None, feature_segment=None + ) + assert feature_state_two.get_feature_state_value() == "imported" + assert ( + feature_state_two.multivariate_feature_state_values.get( + multivariate_feature_option=option + ).percentage_allocation + == 40 + ) + assert ( + feature.feature_states.get( + environment=environment, identity=None, feature_segment=None + ).get_feature_state_value() + == feature.initial_value + ) + + def test_clear_stale_feature_imports_and_exports__stale_records__deletes_old_keeps_new( # type: ignore[no-untyped-def] db: None, environment: Environment, freezer: FrozenDateTimeFactory ): diff --git a/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py b/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py new file mode 100644 index 000000000000..a2e1ed2f6b8d --- /dev/null +++ b/api/tests/unit/features/multivariate/test_unit_multivariate_live_experiment.py @@ -0,0 +1,461 @@ +from typing import Any + +import pytest +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APIClient + +from environments.models import Environment +from experimentation.models import Experiment, ExperimentStatus +from features.constants import LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE +from features.models import Feature +from features.multivariate.models import MultivariateFeatureOption +from projects.models import Project + + +@pytest.fixture(params=[ExperimentStatus.RUNNING, ExperimentStatus.PAUSED]) +def live_experiment( + request: pytest.FixtureRequest, + environment: Environment, + multivariate_feature: Feature, +) -> Experiment: + return Experiment.objects.create( # type: ignore[no-any-return] + environment=environment, + feature=multivariate_feature, + name="live experiment", + hypothesis="new option wins", + status=request.param, + ) + + +def _options_url(project: Project, feature: Feature) -> str: + return f"/api/v1/projects/{project.id}/features/{feature.id}/mv-options/" + + +def _expected_error(experiment: Experiment) -> dict[str, str]: + return { + "detail": LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE.format( + environment=experiment.environment.name + ) + } + + +def _dashboard_option_payload(option: MultivariateFeatureOption) -> dict[str, Any]: + # What the dashboard PUTs for every existing option on each save. + return { + "id": option.id, + "type": option.type, + "string_value": option.string_value, + "integer_value": option.integer_value, + "boolean_value": option.boolean_value, + "key": option.key, + "default_percentage_allocation": 0, + "feature": option.feature_id, + } + + +def test_create_mv_option__live_experiment__returns_400( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, + live_experiment: Experiment, +) -> None: + # Given + # When + response = admin_client.post( + _options_url(project, multivariate_feature), + data={ + "feature": multivariate_feature.id, + "type": "unicode", + "string_value": "new option", + "default_percentage_allocation": 0, + "key": "new", + }, + format="json", + ) + + # Then + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json() == _expected_error(live_experiment) + assert multivariate_feature.multivariate_options.count() == 3 + + +def test_update_mv_option__running_experiment_unchanged_payload__returns_200( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, + live_experiment: Experiment, +) -> None: + # Given + option = multivariate_feature.multivariate_options.first() + assert option is not None + option.key = "variation-a" + option.save() + + # When + response = admin_client.put( + f"{_options_url(project, multivariate_feature)}{option.id}/", + data=_dashboard_option_payload(option), + format="json", + ) + + # Then + assert response.status_code == status.HTTP_200_OK + option.refresh_from_db() + assert option.string_value == "multivariate option for 30% of users." + assert option.key == "variation-a" + + +def test_update_mv_option__running_experiment_default_allocation_only__returns_200( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, + live_experiment: Experiment, +) -> None: + # Given + option = multivariate_feature.multivariate_options.first() + assert option is not None + option.key = "variation-a" + option.save() + payload = {**_dashboard_option_payload(option), "default_percentage_allocation": 10} + + # When + response = admin_client.put( + f"{_options_url(project, multivariate_feature)}{option.id}/", + data=payload, + format="json", + ) + + # Then + assert response.status_code == status.HTTP_200_OK + option.refresh_from_db() + assert option.default_percentage_allocation == 10 + assert option.string_value == "multivariate option for 30% of users." + assert option.key == "variation-a" + + +@pytest.mark.parametrize( + "field, new_value", + [("string_value", "edited"), ("key", "edited-key")], +) +def test_update_mv_option__running_experiment_changed_field__returns_400( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, + live_experiment: Experiment, + field: str, + new_value: str, +) -> None: + # Given + option = multivariate_feature.multivariate_options.first() + assert option is not None + option.key = "variation-a" + option.save() + payload = {**_dashboard_option_payload(option), field: new_value} + + # When + response = admin_client.put( + f"{_options_url(project, multivariate_feature)}{option.id}/", + data=payload, + format="json", + ) + + # Then + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json() == _expected_error(live_experiment) + option.refresh_from_db() + assert option.string_value == "multivariate option for 30% of users." + assert option.key == "variation-a" + + +@pytest.mark.parametrize( + "experiment_status", [ExperimentStatus.CREATED, ExperimentStatus.COMPLETED] +) +def test_update_mv_option__experiment_not_running__returns_200( + admin_client: APIClient, + project: Project, + environment: Environment, + multivariate_feature: Feature, + experiment_status: str, +) -> None: + # Given + Experiment.objects.create( + environment=environment, + feature=multivariate_feature, + name="inactive experiment", + hypothesis="new option wins", + status=experiment_status, + ) + option = multivariate_feature.multivariate_options.first() + assert option is not None + payload = {**_dashboard_option_payload(option), "string_value": "edited"} + + # When + response = admin_client.put( + f"{_options_url(project, multivariate_feature)}{option.id}/", + data=payload, + format="json", + ) + + # Then + assert response.status_code == status.HTTP_200_OK + option.refresh_from_db() + assert option.string_value == "edited" + + +def test_save_feature_value__experiment_running_in_other_environment__returns_200( + admin_client: APIClient, + project: Project, + environment: Environment, + environment_two: Environment, + multivariate_feature: Feature, + live_experiment: Experiment, +) -> None: + # Given + options = list(multivariate_feature.multivariate_options.order_by("id")) + feature_state_a = environment.feature_states.get( + feature=multivariate_feature, identity=None, feature_segment=None + ) + feature_state_b = environment_two.feature_states.get( + feature=multivariate_feature, identity=None, feature_segment=None + ) + allocations_a = { + mv.multivariate_feature_option_id: mv.percentage_allocation + for mv in feature_state_a.multivariate_feature_state_values.all() + } + new_allocations_b = dict(zip((option.id for option in options), (10, 20, 30))) + + # When + option_responses = [ + admin_client.put( + f"{_options_url(project, multivariate_feature)}{option.id}/", + data=_dashboard_option_payload(option), + format="json", + ) + for option in options + ] + feature_state_response = admin_client.put( + reverse( + "api-v1:environments:environment-featurestates-detail", + args=[environment_two.api_key, feature_state_b.id], + ), + data={ + "id": feature_state_b.id, + "feature_state_value": "new control", + "enabled": True, + "feature": multivariate_feature.id, + "environment": environment_two.id, + "identity": None, + "feature_segment": None, + "multivariate_feature_state_values": [ + { + "id": mv.id, + "multivariate_feature_option": mv.multivariate_feature_option_id, + "percentage_allocation": new_allocations_b[ + mv.multivariate_feature_option_id + ], + } + for mv in feature_state_b.multivariate_feature_state_values.all() + ], + }, + format="json", + ) + + # Then + assert [r.status_code for r in option_responses] == [status.HTTP_200_OK] * 3 + assert feature_state_response.status_code == status.HTTP_200_OK + + feature_state_b.refresh_from_db() + assert feature_state_b.get_feature_state_value() == "new control" + assert { + mv.multivariate_feature_option_id: mv.percentage_allocation + for mv in feature_state_b.multivariate_feature_state_values.all() + } == new_allocations_b + + feature_state_a.refresh_from_db() + assert feature_state_a.get_feature_state_value() == "control" + assert { + mv.multivariate_feature_option_id: mv.percentage_allocation + for mv in feature_state_a.multivariate_feature_state_values.all() + } == allocations_a + + +def test_update_mv_option__moving_to_live_experiment__returns_400( + admin_client: APIClient, + project: Project, + environment: Environment, + multivariate_feature: Feature, +) -> None: + # Given + other_feature = Feature.objects.create(name="other feature", project=project) + option = MultivariateFeatureOption.objects.create( + feature=other_feature, + type="unicode", + string_value="source option", + default_percentage_allocation=0, + ) + for existing_option in multivariate_feature.multivariate_options.all(): + existing_option.default_percentage_allocation = 0 + existing_option.save() + + experiment = Experiment.objects.create( + environment=environment, + feature=multivariate_feature, + name="live experiment", + hypothesis="new option wins", + status=ExperimentStatus.RUNNING, + ) + + # When + response = admin_client.put( + f"{_options_url(project, other_feature)}{option.id}/", + data={ + "feature": multivariate_feature.id, + "type": option.type, + "string_value": option.string_value, + "default_percentage_allocation": option.default_percentage_allocation, + }, + format="json", + ) + + # Then + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json() == _expected_error(experiment) + option.refresh_from_db() + assert option.feature == other_feature + assert multivariate_feature.multivariate_options.count() == 3 + + +def test_create_mv_option__body_feature_mismatches_url__returns_400( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, +) -> None: + # Given + other_feature = Feature.objects.create(name="other feature", project=project) + + # When + response = admin_client.post( + _options_url(project, multivariate_feature), + data={"feature": other_feature.id}, + format="json", + ) + + # Then + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert "Feature must match the URL." in str(response.json()) + assert other_feature.multivariate_options.count() == 0 + + +def test_delete_mv_option__live_experiment__returns_400( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, + live_experiment: Experiment, +) -> None: + # Given + option = multivariate_feature.multivariate_options.first() + assert option is not None + + # When + response = admin_client.delete( + f"{_options_url(project, multivariate_feature)}{option.id}/" + ) + + # Then + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json() == _expected_error(live_experiment) + assert MultivariateFeatureOption.objects.filter(id=option.id).exists() + + +def test_create_mv_option__no_live_experiment__returns_201( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, +) -> None: + # Given + # When + response = admin_client.post( + _options_url(project, multivariate_feature), + data={ + "feature": multivariate_feature.id, + "type": "unicode", + "string_value": "new option", + "default_percentage_allocation": 0, + "key": "new", + }, + format="json", + ) + + # Then + assert response.status_code == status.HTTP_201_CREATED + + +def test_update_feature__live_experiment__changed_mv_options_returns_400( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, + live_experiment: Experiment, +) -> None: + # Given + url = f"/api/v1/projects/{project.id}/features/{multivariate_feature.id}/" + options = [ + { + "type": "unicode", + "string_value": "changed", + "default_percentage_allocation": 100, + }, + ] + + # When + response = admin_client.patch( + url, data={"multivariate_options": options}, format="json" + ) + + # Then + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.json() == _expected_error(live_experiment) + + +def test_update_feature__live_experiment__unchanged_keyed_options_preserved( + admin_client: APIClient, + project: Project, + multivariate_feature: Feature, + live_experiment: Experiment, +) -> None: + # Given + keyed_option = multivariate_feature.multivariate_options.first() + assert keyed_option is not None + keyed_option.key = "stable-option" + keyed_option.save() + url = f"/api/v1/projects/{project.id}/features/{multivariate_feature.id}/" + stored_option_ids = list( + multivariate_feature.multivariate_options.values_list("id", flat=True) + ) + unchanged = [ + { + "type": option.type, + "string_value": option.string_value, + "default_percentage_allocation": 0, + } + for option in multivariate_feature.multivariate_options.all() + ] + + # When + response = admin_client.patch( + url, + data={ + "description": "only metadata changed", + "multivariate_options": unchanged, + }, + format="json", + ) + + # Then + assert response.status_code == status.HTTP_200_OK + keyed_option.refresh_from_db() + assert keyed_option.key == "stable-option" + assert ( + list(multivariate_feature.multivariate_options.values_list("id", flat=True)) + == stored_option_ids + ) diff --git a/docs/docs/deployment-self-hosting/observability/_events-catalogue.md b/docs/docs/deployment-self-hosting/observability/_events-catalogue.md index e40b24429464..06badcad69c0 100644 --- a/docs/docs/deployment-self-hosting/observability/_events-catalogue.md +++ b/docs/docs/deployment-self-hosting/observability/_events-catalogue.md @@ -240,7 +240,7 @@ Attributes: ### `experimentation.rollout.applied` Logged at `info` from: - - `api/experimentation/services.py:1081` + - `api/experimentation/services.py:1084` Attributes: - `audience.match` @@ -256,7 +256,7 @@ Attributes: ### `experimentation.rollout.superseded` Logged at `info` from: - - `api/experimentation/services.py:672` + - `api/experimentation/services.py:675` Attributes: - `environment.id` @@ -933,7 +933,7 @@ Attributes: ### `warehouse.connection.connected` Logged at `info` from: - - `api/experimentation/services.py:1257` + - `api/experimentation/services.py:1260` Attributes: - `environment.id` @@ -964,7 +964,7 @@ Attributes: ### `warehouse.connection.test_event_sent` Logged at `info` from: - - `api/experimentation/services.py:1200` + - `api/experimentation/services.py:1203` Attributes: - `environment.id` @@ -973,7 +973,7 @@ Attributes: ### `warehouse.connection.verification_failed` Logged at `warning` from: - - `api/experimentation/services.py:1232` + - `api/experimentation/services.py:1235` Attributes: - `environment.id` @@ -983,7 +983,7 @@ Attributes: ### `warehouse.connection.verification_succeeded` Logged at `info` from: - - `api/experimentation/services.py:1242` + - `api/experimentation/services.py:1245` Attributes: - `environment.id` diff --git a/docs/docs/experimentation/create-an-experiment.md b/docs/docs/experimentation/create-an-experiment.md index 82fccfe996e0..838f6594036b 100644 --- a/docs/docs/experimentation/create-an-experiment.md +++ b/docs/docs/experimentation/create-an-experiment.md @@ -18,7 +18,9 @@ and measures the impact on a metric. - A [connected warehouse](/experimentation/connect-a-warehouse). - A [**multivariate flag**](/managing-flags/core-management) with a variation for each treatment you want to test. The - flag's current value acts as the **control**. A flag can only be in one active experiment at a time. + flag's current value acts as the **control**. A flag can only be in one active experiment at a time. While the + experiment is running or paused, the flag's variations cannot be added, removed or edited; complete the experiment + first if you need to change them. - A [metric](/experimentation/create-metrics). You can also create one from inside the wizard. On the **Experiments** page, click **Create Experiment**. The wizard has four steps.