From fb67a5caa5d874a3b2499b2dc4016264ebb683a2 Mon Sep 17 00:00:00 2001 From: Gagan Trivedi Date: Wed, 4 Nov 2026 13:50:14 +0530 Subject: [PATCH 1/7] feat: record exposures under local evaluation Carry each feature state's experiment metadata from the environment document into the engine context, and set it on flags built from evaluation results, so get_experiment_flag records exposures under local evaluation as it does under remote evaluation. Environment flags carry no experiment, matching the API, which only sends experiment metadata for identity flags. --- flagsmith/api/types.py | 11 ++ flagsmith/local_evaluation.py | 8 +- flagsmith/mappers.py | 2 + flagsmith/models.py | 1 + flagsmith/types.py | 4 + tests/test_flagsmith.py | 194 ++++++++++++++++++++++++++++++++++ tests/test_mappers.py | 113 +++++++++++++++++++- tests/test_models.py | 53 ++++++++++ 8 files changed, 382 insertions(+), 4 deletions(-) diff --git a/flagsmith/api/types.py b/flagsmith/api/types.py index effe979..8bc81e4 100644 --- a/flagsmith/api/types.py +++ b/flagsmith/api/types.py @@ -49,6 +49,16 @@ class MultivariateFeatureStateValueModel(typing.TypedDict): percentage_allocation: float +class ExperimentModel(typing.TypedDict): + id: int + name: str + in_experiment: bool + + +class FeatureStateMetadataModel(typing.TypedDict): + experiment: NotRequired[ExperimentModel] + + class FeatureStateModel(typing.TypedDict): enabled: bool feature_segment: NotRequired[typing.Optional[FeatureSegmentModel]] @@ -56,6 +66,7 @@ class FeatureStateModel(typing.TypedDict): feature: FeatureModel featurestate_uuid: str multivariate_feature_state_values: list[MultivariateFeatureStateValueModel] + metadata: NotRequired[typing.Optional[FeatureStateMetadataModel]] class IdentityModel(typing.TypedDict): diff --git a/flagsmith/local_evaluation.py b/flagsmith/local_evaluation.py index 074bde4..57c65f2 100644 --- a/flagsmith/local_evaluation.py +++ b/flagsmith/local_evaluation.py @@ -1,5 +1,6 @@ from __future__ import annotations +import dataclasses import typing from collections import ChainMap from functools import cached_property @@ -77,7 +78,8 @@ def environment_flags(self) -> typing.Dict[str, Flag]: """Environment flags, which depend on the environment document only. Evaluated without an identity, so segments reading traits or identity - context are left out. + context are left out, and flags carry no experiment: only an identity + is enrolled in one, as with remote evaluation. """ environment_context = self.context.copy() if segments := environment_context.get("segments"): @@ -90,7 +92,9 @@ def environment_flags(self) -> typing.Dict[str, Flag]: context=environment_context, ) return { - feature_name: Flag.from_evaluation_result(flag_result) + feature_name: dataclasses.replace( + Flag.from_evaluation_result(flag_result), experiment=None + ) for feature_name, flag_result in evaluation_result["flags"].items() } diff --git a/flagsmith/mappers.py b/flagsmith/mappers.py index 7e22a9b..e8233b0 100644 --- a/flagsmith/mappers.py +++ b/flagsmith/mappers.py @@ -199,6 +199,8 @@ def _map_environment_document_feature_states_to_feature_contexts( ) -> typing.Iterable[FeatureContext[FeatureMetadata]]: for feature_state in feature_states: metadata: FeatureMetadata = {"id": feature_state["feature"]["id"]} + if experiment := (feature_state.get("metadata") or {}).get("experiment"): + metadata["experiment"] = experiment feature_context = FeatureContext[FeatureMetadata]( key=str( feature_state.get("django_id") or feature_state["featurestate_uuid"] diff --git a/flagsmith/models.py b/flagsmith/models.py index 929c908..b2019ac 100644 --- a/flagsmith/models.py +++ b/flagsmith/models.py @@ -130,6 +130,7 @@ def from_evaluation_result( feature_id=metadata["id"], variant=flag_result.get("variant"), reason=flag_result["reason"], + experiment=ExperimentMetadata.from_api_metadata(metadata), ) raise ValueError( "FlagResult metadata is missing. Cannot create Flag instance. " diff --git a/flagsmith/types.py b/flagsmith/types.py index 8a50741..80529f3 100644 --- a/flagsmith/types.py +++ b/flagsmith/types.py @@ -6,6 +6,8 @@ from flag_engine.result.types import EvaluationResult, FlagResult from typing_extensions import NotRequired, TypeAlias +from flagsmith.api.types import ExperimentModel + _JsonScalarType: TypeAlias = typing.Union[ int, str, @@ -47,6 +49,8 @@ class SegmentMetadata(typing.TypedDict): class FeatureMetadata(typing.TypedDict): id: int """The ID of the feature used in Flagsmith API.""" + experiment: NotRequired[ExperimentModel] + """The running experiment the feature state belongs to, if any.""" SDKEvaluationContext = EvaluationContext[SegmentMetadata, FeatureMetadata] diff --git a/tests/test_flagsmith.py b/tests/test_flagsmith.py index 320a997..fc453ef 100644 --- a/tests/test_flagsmith.py +++ b/tests/test_flagsmith.py @@ -1471,3 +1471,197 @@ def test_get_experiment_flag__disabled_feature__skips_exposure( "Not sending $flag_exposure for feature checkout_v2: feature is disabled." in caplog.messages ) + + +def _environment_with_running_experiment() -> EnvironmentModel: + """`banner` runs an experiment whose rollout segment is identities on the + premium plan, serving them the `treatment` variant. `user_overridden` has + an identity override for it, and `plain` runs no experiment.""" + experiment = {"id": 42, "name": "Banner experiment", "in_experiment": False} + + def feature_state( + feature_name: str, + value: str, + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None, + **extra: typing.Any, + ) -> typing.Dict[str, typing.Any]: + return { + "enabled": True, + "feature": { + "id": 1 if feature_name == "banner" else 2, + "name": feature_name, + }, + "feature_state_value": value, + "featurestate_uuid": f"00000000-0000-0000-0000-00000000000{len(value)}", + "multivariate_feature_state_values": [], + **({"metadata": metadata} if metadata is not None else {}), + **extra, + } + + environment = { + "api_key": "test-key", + "name": "Test", + "project": { + "segments": [ + { + "id": 7, + "name": "Banner experiment rollout", + "rules": [ + { + "type": "ALL", + "rules": [], + "conditions": [ + { + "operator": "EQUAL", + "property_": "plan", + "value": "premium", + } + ], + } + ], + "feature_states": [ + feature_state( + "banner", + "control", + {"experiment": {**experiment, "in_experiment": True}}, + feature_segment={"priority": 0}, + multivariate_feature_state_values=[ + { + "id": 10, + "mv_fs_value_uuid": "00000000-0000-0000-0000-000000000010", + "percentage_allocation": 100, + "multivariate_feature_option": { + "value": "hello treatment", + "key": "treatment", + }, + } + ], + ) + ], + } + ] + }, + "identity_overrides": [ + { + "identifier": "user_overridden", + "identity_features": [feature_state("banner", "hello override")], + } + ], + "feature_states": [ + feature_state("banner", "control", {"experiment": experiment}), + feature_state("plain", "hello"), + ], + } + return typing.cast(EnvironmentModel, environment) + + +@pytest.fixture() +def local_experiment_flagsmith(api_key: str) -> Flagsmith: + flagsmith = Flagsmith( + environment_key=api_key, + enable_events=True, + event_processor_config=EventProcessorConfig(events_api_url="http://test/"), + ) + flagsmith._local_evaluation = LocalEvaluation.from_environment_document( + _environment_with_running_experiment() + ) + flagsmith.enable_local_evaluation = True + return flagsmith + + +def test_get_experiment_flag__local_evaluation__enrolled__tracks_exposure( + mocker: MockerFixture, local_experiment_flagsmith: Flagsmith +) -> None: + # Given + assert local_experiment_flagsmith._event_processor + mock_track = mocker.patch.object( + local_experiment_flagsmith._event_processor, "track_exposure_event" + ) + + # When + flag = local_experiment_flagsmith.get_experiment_flag( + feature_name="banner", identifier="user_123", traits={"plan": "premium"} + ) + + # Then + assert isinstance(flag, Flag) + assert flag.value == "hello treatment" + assert flag.variant == "treatment" + assert flag.experiment == ExperimentMetadata( + id=42, name="Banner experiment", in_experiment=True + ) + mock_track.assert_called_once_with( + feature_name="banner", + identifier="user_123", + value="treatment", + traits={"plan": "premium"}, + metadata={"experiment_id": 42}, + ) + + +@pytest.mark.parametrize( + "feature_name, identifier, traits, expected_value, expected_experiment", + [ + pytest.param( + "banner", + "user_123", + {"plan": "free"}, + "control", + ExperimentMetadata(id=42, name="Banner experiment", in_experiment=False), + id="not-enrolled", + ), + pytest.param( + "banner", + "user_overridden", + {"plan": "premium"}, + "hello override", + None, + id="identity-override", + ), + pytest.param( + "plain", + "user_123", + {"plan": "premium"}, + "hello", + None, + id="no-experiment", + ), + ], +) +def test_get_experiment_flag__local_evaluation__not_enrolled__skips_exposure( + mocker: MockerFixture, + local_experiment_flagsmith: Flagsmith, + feature_name: str, + identifier: str, + traits: typing.Dict[str, typing.Any], + expected_value: str, + expected_experiment: typing.Optional[ExperimentMetadata], +) -> None: + # Given + assert local_experiment_flagsmith._event_processor + mock_track = mocker.patch.object( + local_experiment_flagsmith._event_processor, "track_exposure_event" + ) + + # When + flag = local_experiment_flagsmith.get_experiment_flag( + feature_name=feature_name, identifier=identifier, traits=traits + ) + + # Then + assert isinstance(flag, Flag) + assert flag.value == expected_value + assert flag.experiment == expected_experiment + mock_track.assert_not_called() + + +def test_get_environment_flags__local_evaluation__running_experiment__no_experiment( + local_experiment_flagsmith: Flagsmith, +) -> None: + # When + flag = local_experiment_flagsmith.get_environment_flags().get_flag("banner") + + # Then - as with remote evaluation, only an identity is enrolled in one + assert isinstance(flag, Flag) + assert flag.value == "control" + assert flag.experiment is None diff --git a/tests/test_mappers.py b/tests/test_mappers.py index fd73dca..6d8b2f6 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -1,5 +1,15 @@ -from flagsmith.api.types import EnvironmentModel -from flagsmith.mappers import map_environment_document_to_context +import typing + +from flagsmith.api.types import ( + EnvironmentModel, + ExperimentModel, + FeatureStateMetadataModel, + FeatureStateModel, +) +from flagsmith.mappers import ( + map_environment_document_to_context, + map_environment_document_to_identity_overrides_segments, +) def _environment_with_keyed_variant() -> EnvironmentModel: @@ -55,3 +65,102 @@ def test_map_environment_document_to_context__null_variant_key__drops_key() -> N # Then - the null key is dropped, treated as no key variants = context["features"]["mv_feature"]["variants"] assert "key" not in variants[0] + + +EXPERIMENT: ExperimentModel = { + "id": 42, + "name": "New checkout CTA", + "in_experiment": False, +} + + +def _feature_state( + feature_name: str, + metadata: typing.Optional[FeatureStateMetadataModel] = None, +) -> FeatureStateModel: + feature_state: FeatureStateModel = { + "enabled": True, + "feature": {"id": 1, "name": feature_name}, + "feature_state_value": "control", + "featurestate_uuid": "00000000-0000-0000-0000-000000000001", + "multivariate_feature_state_values": [], + } + if metadata is not None: + feature_state["metadata"] = metadata + return feature_state + + +def _environment_with_experiment() -> EnvironmentModel: + return { + "api_key": "test-key", + "name": "Test", + "project": { + "segments": [ + { + "id": 7, + "name": "rollout", + "rules": [], + "feature_states": [ + _feature_state( + "banner", + {"experiment": {**EXPERIMENT, "in_experiment": True}}, + ) + ], + } + ] + }, + "identity_overrides": [ + { + "identifier": "user_123", + "identity_features": [_feature_state("banner")], + } + ], + "feature_states": [ + _feature_state("banner", {"experiment": EXPERIMENT}), + _feature_state("plain", {}), + ], + } + + +def test_map_environment_document_to_context__experiment__carried_in_metadata() -> None: + # Given + environment = _environment_with_experiment() + + # When + context = map_environment_document_to_context(environment) + + # Then + assert context["features"]["banner"]["metadata"] == { + "id": 1, + "experiment": EXPERIMENT, + } + assert context["segments"]["7"]["overrides"][0]["metadata"] == { + "id": 1, + "experiment": {**EXPERIMENT, "in_experiment": True}, + } + + +def test_map_environment_document_to_context__no_experiment__metadata_has_id_only() -> ( + None +): + # Given + environment = _environment_with_experiment() + + # When + context = map_environment_document_to_context(environment) + + # Then + assert context["features"]["plain"]["metadata"] == {"id": 1} + + +def test_map_environment_document_to_identity_overrides_segments__no_experiment() -> ( + None +): + # Given + environment = _environment_with_experiment() + + # When + segments = map_environment_document_to_identity_overrides_segments(environment) + + # Then + assert segments["user_123"]["overrides"][0]["metadata"] == {"id": 1} diff --git a/tests/test_models.py b/tests/test_models.py index c6dbdbd..9946301 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -41,6 +41,59 @@ def test_flag_from_evaluation_result() -> None: assert flag.reason == "DEFAULT" +@pytest.mark.parametrize( + "in_experiment", + [pytest.param(True, id="enrolled"), pytest.param(False, id="not-enrolled")], +) +def test_flag_from_evaluation_result__experiment_metadata__sets_experiment( + in_experiment: bool, +) -> None: + # Given + flag_result: SDKFlagResult = { + "enabled": True, + "name": "test_feature", + "reason": "DEFAULT", + "value": "test-value", + "variant": None, + "metadata": { + "id": 123, + "experiment": { + "id": 42, + "name": "New checkout CTA", + "in_experiment": in_experiment, + }, + }, + } + + # When + flag = Flag.from_evaluation_result(flag_result) + + # Then + assert flag.experiment == ExperimentMetadata( + id=42, name="New checkout CTA", in_experiment=in_experiment + ) + + +def test_flag_from_evaluation_result__no_experiment_metadata__experiment_is_none() -> ( + None +): + # Given + flag_result: SDKFlagResult = { + "enabled": True, + "name": "test_feature", + "reason": "DEFAULT", + "value": "test-value", + "variant": None, + "metadata": {"id": 123}, + } + + # When + flag = Flag.from_evaluation_result(flag_result) + + # Then + assert flag.experiment is None + + def test_flag_from_evaluation_result__no_variant__is_none() -> None: # Given flag_result: SDKFlagResult = { From 97aefbd8fc4ad412830922fe201d1ddbd46d10fb Mon Sep 17 00:00:00 2001 From: Gagan Trivedi Date: Wed, 4 Nov 2026 14:05:20 +0530 Subject: [PATCH 2/7] test: share the experiment environment fixture via conftest --- tests/conftest.py | 118 ++++++++++++++++++++++++++++++++++++++++ tests/test_flagsmith.py | 96 -------------------------------- tests/test_mappers.py | 106 +++++++----------------------------- 3 files changed, 138 insertions(+), 182 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b59a44c..fdca45a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,6 +17,7 @@ EventProcessorConfig, ) from flagsmith.api.types import EnvironmentModel +from flagsmith.local_evaluation import LocalEvaluation from flagsmith.mappers import map_environment_document_to_context from flagsmith.types import SDKEvaluationContext @@ -307,3 +308,120 @@ def flag_dependency_context() -> SDKEvaluationContext: }, }, } + + +@pytest.fixture +def experiment_environment() -> EnvironmentModel: + """An environment document with a running experiment. + + `banner` runs experiment 42, whose rollout segment is identities on the + premium plan, serving them the `treatment` variant. `user_overridden` has + an identity override for `banner`, and `plain` runs no experiment. + """ + return { + "api_key": "test-key", + "name": "Test", + "project": { + "segments": [ + { + "id": 7, + "name": "Banner experiment rollout", + "rules": [ + { + "type": "ALL", + "rules": [], + "conditions": [ + { + "operator": "EQUAL", + "property_": "plan", + "value": "premium", + } + ], + } + ], + "feature_states": [ + { + "enabled": True, + "feature": {"id": 1, "name": "banner"}, + "feature_state_value": "control", + "featurestate_uuid": "00000000-0000-0000-0000-000000000003", + "feature_segment": {"priority": 0}, + "multivariate_feature_state_values": [ + { + "id": 10, + "mv_fs_value_uuid": "00000000-0000-0000-0000-000000000010", + "percentage_allocation": 100, + "multivariate_feature_option": { + "value": "hello treatment", + "key": "treatment", + }, + } + ], + "metadata": { + "experiment": { + "id": 42, + "name": "Banner experiment", + "in_experiment": True, + } + }, + } + ], + } + ] + }, + "identity_overrides": [ + { + "identifier": "user_overridden", + "identity_features": [ + { + "enabled": True, + "feature": {"id": 1, "name": "banner"}, + "feature_state_value": "hello override", + "featurestate_uuid": "00000000-0000-0000-0000-000000000004", + "multivariate_feature_state_values": [], + } + ], + } + ], + "feature_states": [ + { + "enabled": True, + "feature": {"id": 1, "name": "banner"}, + "feature_state_value": "control", + "featurestate_uuid": "00000000-0000-0000-0000-000000000001", + "multivariate_feature_state_values": [], + "metadata": { + "experiment": { + "id": 42, + "name": "Banner experiment", + "in_experiment": False, + } + }, + }, + { + "enabled": True, + "feature": {"id": 2, "name": "plain"}, + "feature_state_value": "hello", + "featurestate_uuid": "00000000-0000-0000-0000-000000000002", + "multivariate_feature_state_values": [], + }, + ], + } + + +@pytest.fixture +def local_experiment_flagsmith( + api_key: str, + event_processor_config: EventProcessorConfig, + experiment_environment: EnvironmentModel, +) -> Flagsmith: + flagsmith = Flagsmith( + environment_key=api_key, + enable_events=True, + event_processor_config=event_processor_config, + ) + flagsmith._local_evaluation = LocalEvaluation.from_environment_document( + experiment_environment + ) + flagsmith.enable_local_evaluation = True + return flagsmith diff --git a/tests/test_flagsmith.py b/tests/test_flagsmith.py index fc453ef..4ec1245 100644 --- a/tests/test_flagsmith.py +++ b/tests/test_flagsmith.py @@ -1473,102 +1473,6 @@ def test_get_experiment_flag__disabled_feature__skips_exposure( ) -def _environment_with_running_experiment() -> EnvironmentModel: - """`banner` runs an experiment whose rollout segment is identities on the - premium plan, serving them the `treatment` variant. `user_overridden` has - an identity override for it, and `plain` runs no experiment.""" - experiment = {"id": 42, "name": "Banner experiment", "in_experiment": False} - - def feature_state( - feature_name: str, - value: str, - metadata: typing.Optional[typing.Dict[str, typing.Any]] = None, - **extra: typing.Any, - ) -> typing.Dict[str, typing.Any]: - return { - "enabled": True, - "feature": { - "id": 1 if feature_name == "banner" else 2, - "name": feature_name, - }, - "feature_state_value": value, - "featurestate_uuid": f"00000000-0000-0000-0000-00000000000{len(value)}", - "multivariate_feature_state_values": [], - **({"metadata": metadata} if metadata is not None else {}), - **extra, - } - - environment = { - "api_key": "test-key", - "name": "Test", - "project": { - "segments": [ - { - "id": 7, - "name": "Banner experiment rollout", - "rules": [ - { - "type": "ALL", - "rules": [], - "conditions": [ - { - "operator": "EQUAL", - "property_": "plan", - "value": "premium", - } - ], - } - ], - "feature_states": [ - feature_state( - "banner", - "control", - {"experiment": {**experiment, "in_experiment": True}}, - feature_segment={"priority": 0}, - multivariate_feature_state_values=[ - { - "id": 10, - "mv_fs_value_uuid": "00000000-0000-0000-0000-000000000010", - "percentage_allocation": 100, - "multivariate_feature_option": { - "value": "hello treatment", - "key": "treatment", - }, - } - ], - ) - ], - } - ] - }, - "identity_overrides": [ - { - "identifier": "user_overridden", - "identity_features": [feature_state("banner", "hello override")], - } - ], - "feature_states": [ - feature_state("banner", "control", {"experiment": experiment}), - feature_state("plain", "hello"), - ], - } - return typing.cast(EnvironmentModel, environment) - - -@pytest.fixture() -def local_experiment_flagsmith(api_key: str) -> Flagsmith: - flagsmith = Flagsmith( - environment_key=api_key, - enable_events=True, - event_processor_config=EventProcessorConfig(events_api_url="http://test/"), - ) - flagsmith._local_evaluation = LocalEvaluation.from_environment_document( - _environment_with_running_experiment() - ) - flagsmith.enable_local_evaluation = True - return flagsmith - - def test_get_experiment_flag__local_evaluation__enrolled__tracks_exposure( mocker: MockerFixture, local_experiment_flagsmith: Flagsmith ) -> None: diff --git a/tests/test_mappers.py b/tests/test_mappers.py index 6d8b2f6..f991a4b 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -1,11 +1,4 @@ -import typing - -from flagsmith.api.types import ( - EnvironmentModel, - ExperimentModel, - FeatureStateMetadataModel, - FeatureStateModel, -) +from flagsmith.api.types import EnvironmentModel from flagsmith.mappers import ( map_environment_document_to_context, map_environment_document_to_identity_overrides_segments, @@ -67,100 +60,41 @@ def test_map_environment_document_to_context__null_variant_key__drops_key() -> N assert "key" not in variants[0] -EXPERIMENT: ExperimentModel = { - "id": 42, - "name": "New checkout CTA", - "in_experiment": False, -} - - -def _feature_state( - feature_name: str, - metadata: typing.Optional[FeatureStateMetadataModel] = None, -) -> FeatureStateModel: - feature_state: FeatureStateModel = { - "enabled": True, - "feature": {"id": 1, "name": feature_name}, - "feature_state_value": "control", - "featurestate_uuid": "00000000-0000-0000-0000-000000000001", - "multivariate_feature_state_values": [], - } - if metadata is not None: - feature_state["metadata"] = metadata - return feature_state - - -def _environment_with_experiment() -> EnvironmentModel: - return { - "api_key": "test-key", - "name": "Test", - "project": { - "segments": [ - { - "id": 7, - "name": "rollout", - "rules": [], - "feature_states": [ - _feature_state( - "banner", - {"experiment": {**EXPERIMENT, "in_experiment": True}}, - ) - ], - } - ] - }, - "identity_overrides": [ - { - "identifier": "user_123", - "identity_features": [_feature_state("banner")], - } - ], - "feature_states": [ - _feature_state("banner", {"experiment": EXPERIMENT}), - _feature_state("plain", {}), - ], - } - - -def test_map_environment_document_to_context__experiment__carried_in_metadata() -> None: - # Given - environment = _environment_with_experiment() - +def test_map_environment_document_to_context__experiment__carried_in_metadata( + experiment_environment: EnvironmentModel, +) -> None: # When - context = map_environment_document_to_context(environment) + context = map_environment_document_to_context(experiment_environment) # Then + experiment = {"id": 42, "name": "Banner experiment"} assert context["features"]["banner"]["metadata"] == { "id": 1, - "experiment": EXPERIMENT, + "experiment": {**experiment, "in_experiment": False}, } assert context["segments"]["7"]["overrides"][0]["metadata"] == { "id": 1, - "experiment": {**EXPERIMENT, "in_experiment": True}, + "experiment": {**experiment, "in_experiment": True}, } -def test_map_environment_document_to_context__no_experiment__metadata_has_id_only() -> ( - None -): - # Given - environment = _environment_with_experiment() - +def test_map_environment_document_to_context__no_experiment__metadata_has_id_only( + experiment_environment: EnvironmentModel, +) -> None: # When - context = map_environment_document_to_context(environment) + context = map_environment_document_to_context(experiment_environment) # Then - assert context["features"]["plain"]["metadata"] == {"id": 1} - + assert context["features"]["plain"]["metadata"] == {"id": 2} -def test_map_environment_document_to_identity_overrides_segments__no_experiment() -> ( - None -): - # Given - environment = _environment_with_experiment() +def test_map_environment_document_to_identity_overrides_segments__no_experiment( + experiment_environment: EnvironmentModel, +) -> None: # When - segments = map_environment_document_to_identity_overrides_segments(environment) + segments = map_environment_document_to_identity_overrides_segments( + experiment_environment + ) # Then - assert segments["user_123"]["overrides"][0]["metadata"] == {"id": 1} + assert segments["user_overridden"]["overrides"][0]["metadata"] == {"id": 1} From adfa3ffe68d9082258fcf245cf604678a0892915 Mon Sep 17 00:00:00 2001 From: Gagan Trivedi Date: Wed, 4 Nov 2026 14:15:30 +0530 Subject: [PATCH 3/7] test: rename the experiment fixture's feature to checkout --- tests/conftest.py | 16 ++++++++-------- tests/test_flagsmith.py | 14 +++++++------- tests/test_mappers.py | 4 ++-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index fdca45a..ac22f65 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -314,9 +314,9 @@ def flag_dependency_context() -> SDKEvaluationContext: def experiment_environment() -> EnvironmentModel: """An environment document with a running experiment. - `banner` runs experiment 42, whose rollout segment is identities on the + `checkout` runs experiment 42, whose rollout segment is identities on the premium plan, serving them the `treatment` variant. `user_overridden` has - an identity override for `banner`, and `plain` runs no experiment. + an identity override for `checkout`, and `plain` runs no experiment. """ return { "api_key": "test-key", @@ -325,7 +325,7 @@ def experiment_environment() -> EnvironmentModel: "segments": [ { "id": 7, - "name": "Banner experiment rollout", + "name": "Checkout experiment rollout", "rules": [ { "type": "ALL", @@ -342,7 +342,7 @@ def experiment_environment() -> EnvironmentModel: "feature_states": [ { "enabled": True, - "feature": {"id": 1, "name": "banner"}, + "feature": {"id": 1, "name": "checkout"}, "feature_state_value": "control", "featurestate_uuid": "00000000-0000-0000-0000-000000000003", "feature_segment": {"priority": 0}, @@ -360,7 +360,7 @@ def experiment_environment() -> EnvironmentModel: "metadata": { "experiment": { "id": 42, - "name": "Banner experiment", + "name": "Checkout experiment", "in_experiment": True, } }, @@ -375,7 +375,7 @@ def experiment_environment() -> EnvironmentModel: "identity_features": [ { "enabled": True, - "feature": {"id": 1, "name": "banner"}, + "feature": {"id": 1, "name": "checkout"}, "feature_state_value": "hello override", "featurestate_uuid": "00000000-0000-0000-0000-000000000004", "multivariate_feature_state_values": [], @@ -386,14 +386,14 @@ def experiment_environment() -> EnvironmentModel: "feature_states": [ { "enabled": True, - "feature": {"id": 1, "name": "banner"}, + "feature": {"id": 1, "name": "checkout"}, "feature_state_value": "control", "featurestate_uuid": "00000000-0000-0000-0000-000000000001", "multivariate_feature_state_values": [], "metadata": { "experiment": { "id": 42, - "name": "Banner experiment", + "name": "Checkout experiment", "in_experiment": False, } }, diff --git a/tests/test_flagsmith.py b/tests/test_flagsmith.py index 4ec1245..7998c23 100644 --- a/tests/test_flagsmith.py +++ b/tests/test_flagsmith.py @@ -1484,7 +1484,7 @@ def test_get_experiment_flag__local_evaluation__enrolled__tracks_exposure( # When flag = local_experiment_flagsmith.get_experiment_flag( - feature_name="banner", identifier="user_123", traits={"plan": "premium"} + feature_name="checkout", identifier="user_123", traits={"plan": "premium"} ) # Then @@ -1492,10 +1492,10 @@ def test_get_experiment_flag__local_evaluation__enrolled__tracks_exposure( assert flag.value == "hello treatment" assert flag.variant == "treatment" assert flag.experiment == ExperimentMetadata( - id=42, name="Banner experiment", in_experiment=True + id=42, name="Checkout experiment", in_experiment=True ) mock_track.assert_called_once_with( - feature_name="banner", + feature_name="checkout", identifier="user_123", value="treatment", traits={"plan": "premium"}, @@ -1507,15 +1507,15 @@ def test_get_experiment_flag__local_evaluation__enrolled__tracks_exposure( "feature_name, identifier, traits, expected_value, expected_experiment", [ pytest.param( - "banner", + "checkout", "user_123", {"plan": "free"}, "control", - ExperimentMetadata(id=42, name="Banner experiment", in_experiment=False), + ExperimentMetadata(id=42, name="Checkout experiment", in_experiment=False), id="not-enrolled", ), pytest.param( - "banner", + "checkout", "user_overridden", {"plan": "premium"}, "hello override", @@ -1563,7 +1563,7 @@ def test_get_environment_flags__local_evaluation__running_experiment__no_experim local_experiment_flagsmith: Flagsmith, ) -> None: # When - flag = local_experiment_flagsmith.get_environment_flags().get_flag("banner") + flag = local_experiment_flagsmith.get_environment_flags().get_flag("checkout") # Then - as with remote evaluation, only an identity is enrolled in one assert isinstance(flag, Flag) diff --git a/tests/test_mappers.py b/tests/test_mappers.py index f991a4b..99ad67f 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -67,8 +67,8 @@ def test_map_environment_document_to_context__experiment__carried_in_metadata( context = map_environment_document_to_context(experiment_environment) # Then - experiment = {"id": 42, "name": "Banner experiment"} - assert context["features"]["banner"]["metadata"] == { + experiment = {"id": 42, "name": "Checkout experiment"} + assert context["features"]["checkout"]["metadata"] == { "id": 1, "experiment": {**experiment, "in_experiment": False}, } From 0b39bc83d3348d5608c485ca22cbe47382e1b8a1 Mon Sep 17 00:00:00 2001 From: Gagan Trivedi Date: Wed, 4 Nov 2026 14:18:28 +0530 Subject: [PATCH 4/7] test: reword the experiment fixture docstring --- tests/conftest.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ac22f65..0663be9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -314,9 +314,10 @@ def flag_dependency_context() -> SDKEvaluationContext: def experiment_environment() -> EnvironmentModel: """An environment document with a running experiment. - `checkout` runs experiment 42, whose rollout segment is identities on the - premium plan, serving them the `treatment` variant. `user_overridden` has - an identity override for `checkout`, and `plain` runs no experiment. + `checkout` has a running experiment (id 42). Its rollout segment is + identities on the premium plan, who are served the `treatment` variant. + `user_overridden` has an identity override for `checkout`, and `plain` has + no experiment. """ return { "api_key": "test-key", From 92b502ebe140e22cfe487eeff0ce34a13b4802c5 Mon Sep 17 00:00:00 2001 From: Gagan Trivedi Date: Wed, 4 Nov 2026 14:44:00 +0530 Subject: [PATCH 5/7] deps: bump flagsmith-flag-engine to 11.1.2 --- poetry.lock | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/poetry.lock b/poetry.lock index 48d1483..67f42df 100644 --- a/poetry.lock +++ b/poetry.lock @@ -328,14 +328,14 @@ files = [ [[package]] name = "flagsmith-flag-engine" -version = "11.1.1" +version = "11.1.2" description = "Flag engine for the Flagsmith API." optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "flagsmith_flag_engine-11.1.1-py3-none-any.whl", hash = "sha256:17767e9e16859841772a895bc9c660d593499da40d181b95e838bd61441dee92"}, - {file = "flagsmith_flag_engine-11.1.1.tar.gz", hash = "sha256:effad6d0f3d7aa797272333b133cf1cf8c0f524cf248e2f82e748bee2fd97d34"}, + {file = "flagsmith_flag_engine-11.1.2-py3-none-any.whl", hash = "sha256:65c7abb707f7d4fade284ba8723a7f365ee2ca870381eb79e1e16d9820c5f3a8"}, + {file = "flagsmith_flag_engine-11.1.2.tar.gz", hash = "sha256:7130fdf8f3e5ce8facbe25043ad58ecdf11c7bb4c59e5db624cf84620ec8aeac"}, ] [package.dependencies] @@ -1337,4 +1337,4 @@ typing-extensions = {version = ">=4.13.2", markers = "python_version < \"3.11\"" [metadata] lock-version = "2.1" python-versions = ">=3.10,<4" -content-hash = "7a1ee8aaa27e20a95880eae1b2322cdfd3ec4a9ed72da08bd4e83461975afbfb" +content-hash = "f7b2ef9622b5ec4326b76aab60c3f4f1c323eb381f1295976f9053ea894421be" diff --git a/pyproject.toml b/pyproject.toml index 24d3783..ea889ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ documentation = "https://docs.flagsmith.com" packages = [{ include = "flagsmith" }] [tool.poetry.dependencies] -flagsmith-flag-engine = "^11.1.1" +flagsmith-flag-engine = "^11.1.2" iso8601 = { version = "^2.1.0", python = "<3.11" } python = ">=3.10,<4" requests = "^2.32.3" From 5ebb5d48a18a378bbb784411b2820f00e493cd3b Mon Sep 17 00:00:00 2001 From: Gagan Trivedi Date: Wed, 4 Nov 2026 14:47:24 +0530 Subject: [PATCH 6/7] test: bump engine-test-data to v3.13.0 --- tests/engine_tests/engine-test-data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/engine_tests/engine-test-data b/tests/engine_tests/engine-test-data index cb06740..45e01f6 160000 --- a/tests/engine_tests/engine-test-data +++ b/tests/engine_tests/engine-test-data @@ -1 +1 @@ -Subproject commit cb067401093766e55c3354c1de7c73d9331334df +Subproject commit 45e01f65b115a676190c16b117e1f866bf45d5df From a6e721bf4895ccb5f48224c06aa9fe2411ab66cc Mon Sep 17 00:00:00 2001 From: Gagan Trivedi Date: Wed, 4 Nov 2026 14:48:19 +0530 Subject: [PATCH 7/7] test: check variant and experiment against engine test data --- tests/test_engine_test_data.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/test_engine_test_data.py b/tests/test_engine_test_data.py index 48c6042..fca9e4d 100644 --- a/tests/test_engine_test_data.py +++ b/tests/test_engine_test_data.py @@ -6,6 +6,7 @@ import pytest from flagsmith.models import ( + ExperimentMetadata, Flag, Flags, build_segment_overrides_index, @@ -65,12 +66,24 @@ def test_engine_test_data__lazy_flags__match_expected_result( # Then assert all(isinstance(flag, Flag) for flag in actual_flags.values()) actual = { - feature_name: (flag.enabled, flag.value, flag.reason) + feature_name: ( + flag.enabled, + flag.value, + flag.reason, + flag.variant, + flag.experiment, + ) for feature_name, flag in actual_flags.items() if isinstance(flag, Flag) } assert actual == { - feature_name: (expected["enabled"], expected["value"], expected["reason"]) + feature_name: ( + expected["enabled"], + expected["value"], + expected["reason"], + expected["variant"], + ExperimentMetadata.from_api_metadata(expected.get("metadata")), + ) for feature_name, expected in test_case["result"]["flags"].items() }