Repository navigation
fix(api): block multivariate option changes during live experiments #8917
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
ffd2b0b
5b4a677
d564c8d
a91b390
8682e20
7aa28e9
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,6 +10,10 @@ | |
| # Multivariate variant reported when an identity falls through to the control value | ||
| CONTROL_VARIANT_KEY = "control" | ||
| RESERVED_VARIANT_KEY_MESSAGE = f'"{CONTROL_VARIANT_KEY}" is a reserved variant key.' | ||
| LIVE_EXPERIMENT_MV_OPTIONS_MESSAGE = ( | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. DRF will return this error in three different shapes: a bare list from the view, and both a list and a string under Could we use one shape, |
||
| 'Variations cannot be changed while an experiment is running or paused in the "{environment}" ' | ||
| "environment. Complete the experiment to change them." | ||
| ) | ||
|
|
||
| # Tag filtering strategy | ||
| UNION = "UNION" | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,6 +10,8 @@ | |
| register_task_handler, | ||
| ) | ||
|
|
||
| from environments.models import Environment | ||
| from experimentation.feature_state_metadata import get_live_experiment | ||
| from features.import_export.constants import ( | ||
| FAILED, | ||
| PROCESSING, | ||
|
|
@@ -153,7 +155,12 @@ def _import_features_for_environment(feature_import: FeatureImport) -> None: | |
| project=project, | ||
| ).first() | ||
|
|
||
| if existing_feature and feature_import.strategy == SKIP: | ||
| if existing_feature and ( | ||
| feature_import.strategy == SKIP | ||
| or _should_skip_for_live_experiment( | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P1: This per-row check can skip a conflicting feature after earlier rows have already been overwritten, then mark the import (Based on your team's feedback about preflighting imports with experiment conflicts.) Prompt for AI agents |
||
| feature_data, existing_feature, environment | ||
| ) | ||
| ): | ||
| continue | ||
|
|
||
| if existing_feature is None: | ||
|
|
@@ -166,6 +173,26 @@ def _import_features_for_environment(feature_import: FeatureImport) -> None: | |
| feature_import.save() | ||
|
|
||
|
|
||
| def _should_skip_for_live_experiment( | ||
| feature_data: FeatureExportData, | ||
| existing_feature: Feature, | ||
| environment: Environment, | ||
| ) -> bool: | ||
| if get_live_experiment(existing_feature.id, environment.id): | ||
| return True | ||
| if not feature_data["multivariate"]: | ||
| return False | ||
| existing_variations = { | ||
| (option.type, option.value) | ||
| for option in existing_feature.multivariate_options.all() | ||
| } | ||
| adds_variation = any( | ||
| (mv_data["type"], mv_data["value"]) not in existing_variations | ||
| for mv_data in feature_data["multivariate"] | ||
| ) | ||
| return bool(adds_variation and get_live_experiment(existing_feature.id)) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P1: The import checks for a live experiment without locking the feature, then can create a variation after a concurrent start commits. Hold the feature-row lock across the experiment check and the import write so the start transition and import serialize. Prompt for AI agents |
||
|
|
||
|
|
||
| # Should only run on official flagsmith instance. | ||
| if ( | ||
| settings.FLAGSMITH_ON_FLAGSMITH_FEATURE_EXPORT_ENVIRONMENT_ID | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -2,19 +2,33 @@ | |||||||||||||
| CREATE_FEATURE, | ||||||||||||||
| VIEW_PROJECT, | ||||||||||||||
| ) | ||||||||||||||
| from django.db import transaction | ||||||||||||||
| from django.utils.decorators import method_decorator | ||||||||||||||
| from drf_spectacular.utils import extend_schema | ||||||||||||||
| from rest_framework import viewsets | ||||||||||||||
| from rest_framework.decorators import api_view | ||||||||||||||
| from rest_framework.exceptions import ValidationError | ||||||||||||||
| from rest_framework.generics import get_object_or_404 | ||||||||||||||
| from rest_framework.response import Response | ||||||||||||||
|
|
||||||||||||||
| from experimentation.feature_state_metadata import check_no_live_experiment | ||||||||||||||
| from features.models import Feature | ||||||||||||||
| from projects.permissions import NestedProjectPermissions | ||||||||||||||
|
|
||||||||||||||
| from .models import MultivariateFeatureOption | ||||||||||||||
| from .serializers import MultivariateFeatureOptionSerializer | ||||||||||||||
|
|
||||||||||||||
| # What makes a variation. `default_percentage_allocation` is left out: the | ||||||||||||||
| # dashboard always sends it as 0, and it only seeds new environments. | ||||||||||||||
| VARIATION_FIELDS = ( | ||||||||||||||
| "type", | ||||||||||||||
| "string_value", | ||||||||||||||
| "integer_value", | ||||||||||||||
| "boolean_value", | ||||||||||||||
| "key", | ||||||||||||||
| "feature", | ||||||||||||||
| ) | ||||||||||||||
|
|
||||||||||||||
|
|
||||||||||||||
| @method_decorator( | ||||||||||||||
| name="list", | ||||||||||||||
|
|
@@ -69,9 +83,57 @@ def get_permissions(self): # type: ignore[no-untyped-def] | |||||||||||||
| def create(self, request, *args, **kwargs): # type: ignore[no-untyped-def] | ||||||||||||||
| feature_pk = self.kwargs.get("feature_pk") | ||||||||||||||
| project_pk = self.kwargs.get("project_pk") | ||||||||||||||
| get_object_or_404(Feature.objects.filter(project__id=project_pk), pk=feature_pk) | ||||||||||||||
| url_feature = get_object_or_404( | ||||||||||||||
| Feature.objects.filter(project__id=project_pk), pk=feature_pk | ||||||||||||||
| ) | ||||||||||||||
| if "feature" in request.data and str(request.data.get("feature")) != str( | ||||||||||||||
| url_feature.id | ||||||||||||||
| ): | ||||||||||||||
| raise ValidationError({"feature": "Feature must match the URL."}) | ||||||||||||||
| return super().create(request, *args, **kwargs) | ||||||||||||||
|
|
||||||||||||||
| def perform_create(self, serializer): # type: ignore[no-untyped-def] | ||||||||||||||
| url_feature = get_object_or_404( | ||||||||||||||
| Feature.objects.filter(project__id=self.kwargs["project_pk"]), | ||||||||||||||
| pk=self.kwargs["feature_pk"], | ||||||||||||||
| ) | ||||||||||||||
| if serializer.validated_data["feature"] != url_feature: | ||||||||||||||
| raise ValidationError({"feature": "Feature must match the URL."}) | ||||||||||||||
| with transaction.atomic(): | ||||||||||||||
| Feature.objects.select_for_update().filter(pk=url_feature.pk).exists() | ||||||||||||||
| check_no_live_experiment(url_feature.id) | ||||||||||||||
| super().perform_create(serializer) | ||||||||||||||
|
|
||||||||||||||
| def perform_update(self, serializer): # type: ignore[no-untyped-def] | ||||||||||||||
| instance = serializer.instance | ||||||||||||||
| data = serializer.validated_data | ||||||||||||||
| # The dashboard PUTs every option on each save, changed or not, so only | ||||||||||||||
| # a change to the variation itself is checked against experiments. | ||||||||||||||
| if not any( | ||||||||||||||
| field in data and data[field] != getattr(instance, field) | ||||||||||||||
| for field in VARIATION_FIELDS | ||||||||||||||
| ): | ||||||||||||||
| super().perform_update(serializer) | ||||||||||||||
|
Comment on lines
+112
to
+116
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Lock and reload the option before taking the unchanged-update path. Suppose an unchanged PUT loads value
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Wrong qualification of major, kind of rare scenario There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @Zaimwa9 I agree. The race requires a narrow overlap, so Minor is a better qualification than Major. The race could still overwrite a variation after an experiment starts. I’ll leave the decision on whether to address it to you. I’m not marking the finding as addressed. You are interacting with an AI system. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P1: This fast path saves without locking or reloading the option, so its “unchanged” comparison can become stale. A concurrent variant edit followed by experiment start lets this PUT overwrite the variant during the live run; recheck against locked current state before bypassing the experiment guard. (Based on your team's feedback about locking experiment variants.) Prompt for AI agents |
||||||||||||||
| return | ||||||||||||||
|
|
||||||||||||||
| features = {instance.feature, data.get("feature", instance.feature)} | ||||||||||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Authorise the replacement feature before moving an option. If a requester can edit an option in project A, the requester can supply a feature ID from project B without permission to edit project B. The new branch checks that target for experiments, but neither it nor the source-object permission check authorises the target. The update can attach the option to project B. Require permission on the target project, or reject cross-project moves. The serializer’s relation uses the related model’s default manager. (raw.githubusercontent.com)
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Out of scope, it's already existing and should be addressed as a follow up There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @Zaimwa9 Understood. We can address the authorisation concern in a follow-up. Would you like me to open a GitHub issue for it? I’m not marking the finding as addressed. You are interacting with an AI system. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P1: This feature set can use the option’s stale source association: after a concurrent reassignment from A to B, the lock set can omit B while the save mutates the option now attached to B. (Based on your team's feedback about locking experiment variants.) Prompt for AI agents |
||||||||||||||
| with transaction.atomic(): | ||||||||||||||
| list( | ||||||||||||||
| Feature.objects.select_for_update() | ||||||||||||||
| .filter(pk__in=[feature.pk for feature in features]) | ||||||||||||||
| .order_by("pk") | ||||||||||||||
| .values_list("pk", flat=True) | ||||||||||||||
| ) | ||||||||||||||
| for feature in features: | ||||||||||||||
| check_no_live_experiment(feature.id) | ||||||||||||||
| super().perform_update(serializer) | ||||||||||||||
|
Comment on lines
+107
to
+129
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for the work on this, the direction is right: variants are shared across the project, so adding, deleting or changing a variant's value or key should be blocked while an experiment is running. There is one case we need to keep working, though. Users must still be able to change the flag's value and variant percentages in the other environments, where no experiment is running.
The cause is that the dashboard sends a Could you make
Create and delete stay blocked as they are. Here are some test ideas that could help implementing the behavior imo.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There is another related change to do here. We need to lock it on a We will ask the users to complete experiments if they need to change the variants so in the code that means:
|
||||||||||||||
|
|
||||||||||||||
| def perform_destroy(self, instance): # type: ignore[no-untyped-def] | ||||||||||||||
| with transaction.atomic(): | ||||||||||||||
| Feature.objects.select_for_update().filter(pk=instance.feature_id).exists() | ||||||||||||||
| check_no_live_experiment(instance.feature_id) | ||||||||||||||
| super().perform_destroy(instance) | ||||||||||||||
|
|
||||||||||||||
| def get_queryset(self): # type: ignore[no-untyped-def] | ||||||||||||||
| if getattr(self, "swagger_fake_view", False): | ||||||||||||||
| return MultivariateFeatureOption.objects.none() | ||||||||||||||
|
|
||||||||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,7 +12,7 @@ | |
| FeatureStateValueSerializer, | ||
| ) | ||
| from common.projects.permissions import VIEW_PROJECT | ||
| from django.db import models | ||
| from django.db import models, transaction | ||
| from drf_spectacular.types import OpenApiTypes | ||
| from drf_spectacular.utils import extend_schema_field | ||
| from drf_writable_nested import ( # type: ignore[attr-defined] | ||
|
|
@@ -31,6 +31,8 @@ | |
| from evaluation.types import EvaluatedFeatureState | ||
| from experimentation.feature_state_metadata import ( | ||
| get_feature_state_metadata_builder, | ||
| get_live_experiment, | ||
| live_experiment_error, | ||
| ) | ||
| from integrations.vcs.services import dispatch_vcs_on_feature_state_changed | ||
| from metadata.serializers import MetadataSerializer, MetadataSerializerMixin | ||
|
|
@@ -307,6 +309,19 @@ def validate_project_features_limit(self, project: Project) -> None: | |
| } | ||
| ) | ||
|
|
||
| def update(self, instance: Feature, validated_data: dict[str, Any]) -> Feature: | ||
| with transaction.atomic(): | ||
| Feature.objects.select_for_update().filter(pk=instance.pk).exists() | ||
| if "multivariate_options" in validated_data and ( | ||
| experiment := get_live_experiment(instance.id) | ||
| ): | ||
| if self._changes_multivariate_options( | ||
| validated_data["multivariate_options"], instance | ||
| ): | ||
| raise live_experiment_error(experiment) | ||
| validated_data.pop("multivariate_options") | ||
| return super().update(instance, validated_data) # type: ignore[no-any-return] | ||
|
|
||
| def validate_multivariate_options(self, multivariate_options): # type: ignore[no-untyped-def] | ||
| if multivariate_options: | ||
| user = self.context["request"].user | ||
|
|
@@ -326,6 +341,32 @@ def validate_multivariate_options(self, multivariate_options): # type: ignore[n | |
| raise serializers.ValidationError("Invalid percentage allocation") | ||
| return multivariate_options | ||
|
|
||
| def _changes_multivariate_options( | ||
| self, | ||
| multivariate_options: list[dict[str, Any]] | None, | ||
| instance: Feature, | ||
| ) -> bool: | ||
| # The frontend resubmits the whole feature, so only a real change to | ||
| # variation value/type counts. `default_percentage_allocation` only | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: A feature PATCH that changes only Prompt for AI agents |
||
| # seeds new options or environments and does not affect live variants. | ||
| fields = ( | ||
| "type", | ||
| "integer_value", | ||
| "string_value", | ||
| "boolean_value", | ||
| ) | ||
| current = NestedMultivariateFeatureOptionSerializer( | ||
| instance.multivariate_options.all(), many=True | ||
| ).data | ||
|
|
||
| def signature(options) -> list[tuple[Any, ...]]: # type: ignore[no-untyped-def] | ||
| return sorted( | ||
| (tuple(option.get(field) for field in fields) for option in options), | ||
| key=repr, | ||
| ) | ||
|
|
||
| return signature(current) != signature(multivariate_options or []) | ||
|
|
||
| def validate_owners(self, owners: list[FFAdminUser]) -> list[FFAdminUser]: | ||
| project: Project = self.context["project"] | ||
| invalid_users = [ | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Script executed:
Repository: Flagsmith/flagsmith
Length of output: 3106
Length of output: 3972
Script executed:
Repository: Flagsmith/flagsmith
Length of output: 33851
Script executed:
Repository: Flagsmith/flagsmith
Length of output: 4823
Script executed:
Repository: Flagsmith/flagsmith
Length of output: 24983
Script executed:
Repository: Flagsmith/flagsmith
Length of output: 13459
Script executed:
Repository: Flagsmith/flagsmith
Length of output: 10425
Web query:
PostgreSQL foreign key update unchanged key skip referential integrity trigger RI_FKey_fk_upd_check_required select for key shareResult:
Script executed:
Repository: Flagsmith/flagsmith
Length of output: 2589
Script executed:
Repository: Flagsmith/flagsmith
Length of output: 3544
Lock the feature when resuming an experiment.
If an option update passes its check while an experiment is paused, a concurrent resume can commit before the update. The option change can then commit while the experiment is running. Acquire the feature lock for every transition to
RUNNING.