From 406ed3a71bceb8485d0e20aecd439416a4739198 Mon Sep 17 00:00:00 2001 From: Gagan Trivedi Date: Wed, 4 Nov 2026 15:19:43 +0530 Subject: [PATCH 1/2] feat: Spec getExperimentFlag under local evaluation Add a client step creating a client with both local evaluation and events, which the runner asks for when a scenario needs both; until now it dropped local evaluation. Let a running experiment enroll a given percentage of identities, so that a scenario can run one enrolling none. --- README.md | 3 ++ features/local__get_experiment_flag.feature | 49 +++++++++++++++++++++ runner/sdk_spec/flagsmith_admin.py | 16 ++++--- runner/tests/conftest.py | 30 ++++++++++--- 4 files changed, 88 insertions(+), 10 deletions(-) create mode 100644 features/local__get_experiment_flag.feature diff --git a/README.md b/README.md index f511df1..5d727e3 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,9 @@ Each release also pins the Flagsmith version in `compose.yml` that it was verifi - `Create the SDK client with local evaluation` receives the same two as `Create the SDK client`, and returns a client that evaluates flags locally, with its environment already loaded. Only `@local-evaluation` scenarios use it, so an SDK without local evaluation can leave it out. + - `Create the SDK client with local evaluation and events enabled` receives the same three as + `Create the SDK client with events enabled`, and returns a client that does both. Only + scenarios tagged both `@local-evaluation` and `@events` use it. - A `When` step receives the client, then the step's placeholders in order, then its data table. A ` | value` table arrives as a map, e.g. `{"plan": "premium", "age": 30}`. - Values are JSON values. Pass flag and trait values through with their types intact. diff --git a/features/local__get_experiment_flag.feature b/features/local__get_experiment_flag.feature new file mode 100644 index 0000000..bf11f20 --- /dev/null +++ b/features/local__get_experiment_flag.feature @@ -0,0 +1,49 @@ +@local-evaluation @events +Feature: Get an experiment flag with local evaluation + A server-side SDK resolves a feature for an identity against the environment document, and + records the exposure as it would with remote evaluation. + + Background: + Given a Flagsmith environment + And the SDK evaluates flags locally + And the SDK sends events to Flagsmith + + Scenario: An identity in a running experiment is exposed to the variant it was served + Given the environment has the following features: + | feature | enabled | value | + | checkout | true | "control" | + And a running experiment on feature "checkout" serving variant "treatment" with value "new checkout" + When the application gets the experiment flag "checkout" for identity "user_123" with traits: + | trait | value | + | plan | "premium" | + Then the application receives the following flags: + | feature | enabled | value | + | checkout | true | "new checkout" | + And Flagsmith has received the following events: + | event | feature | identifier | value | traits | + | $flag_exposure | checkout | user_123 | "treatment" | {"plan": "premium"} | + + Scenario: An identity a running experiment does not enroll is not exposed + Given the environment has the following features: + | feature | enabled | value | + | checkout | true | "control" | + And a running experiment on feature "checkout" serving variant "treatment" with value "new checkout" to 0% of identities + When the application gets the experiment flag "checkout" for identity "user_123" with traits: + | trait | value | + | plan | "premium" | + Then the application receives the following flags: + | feature | enabled | value | + | checkout | true | "control" | + And Flagsmith has received no events + + Scenario: An identity outside a running experiment is not exposed + Given the environment has the following features: + | feature | enabled | value | + | checkout | true | "control" | + When the application gets the experiment flag "checkout" for identity "user_123" with traits: + | trait | value | + | plan | "premium" | + Then the application receives the following flags: + | feature | enabled | value | + | checkout | true | "control" | + And Flagsmith has received no events diff --git a/runner/sdk_spec/flagsmith_admin.py b/runner/sdk_spec/flagsmith_admin.py index c5fc855..1b146ed 100644 --- a/runner/sdk_spec/flagsmith_admin.py +++ b/runner/sdk_spec/flagsmith_admin.py @@ -103,12 +103,18 @@ def create_feature( environment.features[name] = Feature(id=feature["id"], value=value) def start_experiment( - self, environment: Environment, feature: str, variant: str, value: Any + self, + environment: Environment, + feature: str, + variant: str, + value: Any, + rollout_percentage: int = 100, ) -> None: - """Runs an experiment on the feature, serving `variant` to every identity. + """Runs an experiment on the feature, serving `variant` to the identities it enrolls. - A single variant at 100% keeps the scenario deterministic: every identity is - enrolled, and every one of them is served that variant. + It enrolls `rollout_percentage` of identities. A rollout of 100% or 0%, with a single + variant at 100%, keeps the scenario deterministic: every identity is enrolled and served + that variant, or none is. """ control = environment.features[feature] option = self._request( @@ -131,7 +137,7 @@ def start_experiment( "hypothesis": f"Serving {variant} changes what identities do", "experiment_rollout": { "enabled": True, - "rollout_percentage": 100, + "rollout_percentage": rollout_percentage, # What an enrolled identity is served when it is allocated no variant. "feature_state_value": _rollout_value(control.value), "multivariate_feature_state_values": [ diff --git a/runner/tests/conftest.py b/runner/tests/conftest.py index ed3c9c0..5571003 100644 --- a/runner/tests/conftest.py +++ b/runner/tests/conftest.py @@ -162,6 +162,10 @@ def require_environment(state: ScenarioState) -> Environment: CREATE_CLIENT_WITH_EVENTS = "Create the SDK client with events enabled" # A client evaluating flags against the environment document, for server-side SDKs only. CREATE_CLIENT_WITH_LOCAL_EVALUATION = "Create the SDK client with local evaluation" +# Both of the above, for scenarios about the events an SDK sends under local evaluation. +CREATE_CLIENT_WITH_LOCAL_EVALUATION_AND_EVENTS = ( + "Create the SDK client with local evaluation and events enabled" +) SDK_STEPS: list[str] = [] PLACEHOLDER = re.compile(r'("?)\{(\w+)\}\1') @@ -174,7 +178,11 @@ def sdk_client(state: ScenarioState, sdk: SDK) -> str: args = [environment.server_side_key, f"{FLAGSMITH_SDK_URL.rstrip('/')}/api/v1/"] if state.events: args.append(f"{EVENTS_SDK_URL.rstrip('/')}/") - step = CREATE_CLIENT_WITH_EVENTS + step = ( + CREATE_CLIENT_WITH_LOCAL_EVALUATION_AND_EVENTS + if state.local_evaluation + else CREATE_CLIENT_WITH_EVENTS + ) elif state.local_evaluation: step = CREATE_CLIENT_WITH_LOCAL_EVALUATION else: @@ -249,6 +257,7 @@ def sdk_sends_events(state: ScenarioState, flagsmith: FlagsmithAdmin) -> None: @given("the SDK evaluates flags locally") def sdk_evaluates_flags_locally(state: ScenarioState) -> None: """Asks for a client with local evaluation, which the runner creates for the first SDK step.""" + assert state.client is None, "The SDK client has already been created without local evaluation" state.local_evaluation = True @@ -267,15 +276,26 @@ def environment_features( @given( - parsers.parse( - 'a running experiment on feature "{feature}" serving variant "{variant}" with value {value}' + parsers.re( + r'a running experiment on feature "(?P[^"]+)" serving variant "(?P[^"]+)"' + r" with value (?P.+?)(?: to (?P\d+)% of identities)?" ) ) def running_experiment( - state: ScenarioState, flagsmith: FlagsmithAdmin, feature: str, variant: str, value: str + state: ScenarioState, + flagsmith: FlagsmithAdmin, + feature: str, + variant: str, + value: str, + rollout: str | None, ) -> None: + """Runs an experiment on the feature, enrolling every identity unless a rollout is given.""" flagsmith.start_experiment( - require_environment(state), feature, variant, parse_value(value) + require_environment(state), + feature, + variant, + parse_value(value), + 100 if rollout is None else int(rollout), ) From cb589d7d655493e3d577c4236c3736f18220fee9 Mon Sep 17 00:00:00 2001 From: Gagan Trivedi Date: Tue, 10 Nov 2026 09:23:22 +0530 Subject: [PATCH 2/2] feat: Spec the control exposure and experiment id under local evaluation An experiment now states both its rollout and the share of enrolled identities served the variant, so a scenario can enrol an identity into control. Event expectations can match the scenario's experiment id with "". --- features/get_experiment_flag.feature | 2 +- features/local__get_experiment_flag.feature | 23 +++++++++--- runner/sdk_spec/flagsmith_admin.py | 21 ++++++----- runner/tests/conftest.py | 39 +++++++++++++++------ 4 files changed, 61 insertions(+), 24 deletions(-) diff --git a/features/get_experiment_flag.feature b/features/get_experiment_flag.feature index fe3bb7d..79325fe 100644 --- a/features/get_experiment_flag.feature +++ b/features/get_experiment_flag.feature @@ -26,7 +26,7 @@ Feature: Get an experiment flag Given the environment has the following features: | feature | enabled | value | | banner | true | "control" | - And a running experiment on feature "banner" serving variant "treatment" with value "hello treatment" + And a running experiment on feature "banner" enrolling 100% of identities, serving variant "treatment" with value "hello treatment" to 100% of them When the application gets the experiment flag "banner" for identity "user_123" with traits: | trait | value | | plan | "premium" | diff --git a/features/local__get_experiment_flag.feature b/features/local__get_experiment_flag.feature index bf11f20..b341ca8 100644 --- a/features/local__get_experiment_flag.feature +++ b/features/local__get_experiment_flag.feature @@ -12,7 +12,7 @@ Feature: Get an experiment flag with local evaluation Given the environment has the following features: | feature | enabled | value | | checkout | true | "control" | - And a running experiment on feature "checkout" serving variant "treatment" with value "new checkout" + And a running experiment on feature "checkout" enrolling 100% of identities, serving variant "treatment" with value "new checkout" to 100% of them When the application gets the experiment flag "checkout" for identity "user_123" with traits: | trait | value | | plan | "premium" | @@ -20,14 +20,29 @@ Feature: Get an experiment flag with local evaluation | feature | enabled | value | | checkout | true | "new checkout" | And Flagsmith has received the following events: - | event | feature | identifier | value | traits | - | $flag_exposure | checkout | user_123 | "treatment" | {"plan": "premium"} | + | event | feature | identifier | value | traits | metadata | + | $flag_exposure | checkout | user_123 | "treatment" | {"plan": "premium"} | {"experiment_id": ""} | + + Scenario: An identity in a running experiment served no variant is exposed to control + Given the environment has the following features: + | feature | enabled | value | + | checkout | true | "control" | + And a running experiment on feature "checkout" enrolling 100% of identities, serving variant "treatment" with value "new checkout" to 0% of them + When the application gets the experiment flag "checkout" for identity "user_123" with traits: + | trait | value | + | plan | "premium" | + Then the application receives the following flags: + | feature | enabled | value | + | checkout | true | "control" | + And Flagsmith has received the following events: + | event | feature | identifier | value | traits | metadata | + | $flag_exposure | checkout | user_123 | "control" | {"plan": "premium"} | {"experiment_id": ""} | Scenario: An identity a running experiment does not enroll is not exposed Given the environment has the following features: | feature | enabled | value | | checkout | true | "control" | - And a running experiment on feature "checkout" serving variant "treatment" with value "new checkout" to 0% of identities + And a running experiment on feature "checkout" enrolling 0% of identities, serving variant "treatment" with value "new checkout" to 100% of them When the application gets the experiment flag "checkout" for identity "user_123" with traits: | trait | value | | plan | "premium" | diff --git a/runner/sdk_spec/flagsmith_admin.py b/runner/sdk_spec/flagsmith_admin.py index 1b146ed..b00e493 100644 --- a/runner/sdk_spec/flagsmith_admin.py +++ b/runner/sdk_spec/flagsmith_admin.py @@ -108,13 +108,14 @@ def start_experiment( feature: str, variant: str, value: Any, - rollout_percentage: int = 100, - ) -> None: - """Runs an experiment on the feature, serving `variant` to the identities it enrolls. - - It enrolls `rollout_percentage` of identities. A rollout of 100% or 0%, with a single - variant at 100%, keeps the scenario deterministic: every identity is enrolled and served - that variant, or none is. + rollout_percentage: int, + variant_percentage: int, + ) -> int: + """Runs an experiment on the feature, and returns its id. + + It enrolls `rollout_percentage` of identities, and serves `variant` to + `variant_percentage` of those; the rest are served the feature's value as control. + Percentages of 100% or 0% keep the scenario deterministic. """ control = environment.features[feature] option = self._request( @@ -141,7 +142,10 @@ def start_experiment( # What an enrolled identity is served when it is allocated no variant. "feature_state_value": _rollout_value(control.value), "multivariate_feature_state_values": [ - {"multivariate_feature_option": option["id"], "percentage_allocation": 100} + { + "multivariate_feature_option": option["id"], + "percentage_allocation": variant_percentage, + } ], # No segments: the experiment targets every identity in the environment. "audience": {"match": "any", "segment_ids": []}, @@ -151,6 +155,7 @@ def start_experiment( self._request( "POST", f"/environments/{environment.api_key}/experiments/{experiment['id']}/start/" ) + return experiment["id"] def create_trait_segment( self, environment: Environment, name: str, trait: str, operator: str, value: Any diff --git a/runner/tests/conftest.py b/runner/tests/conftest.py index 5571003..dbc8022 100644 --- a/runner/tests/conftest.py +++ b/runner/tests/conftest.py @@ -68,6 +68,7 @@ class ScenarioState: client: str | None = None events: bool = False local_evaluation: bool = False + experiment_id: int | None = None result: Any = None @@ -277,9 +278,12 @@ def environment_features( @given( parsers.re( - r'a running experiment on feature "(?P[^"]+)" serving variant "(?P[^"]+)"' - r" with value (?P.+?)(?: to (?P\d+)% of identities)?" - ) + r'a running experiment on feature "(?P[^"]+)"' + r" enrolling (?P\d+)% of identities," + r' serving variant "(?P[^"]+)" with value (?P.+)' + r" to (?P\d+)% of them" + ), + converters={"rollout": int, "allocation": int}, ) def running_experiment( state: ScenarioState, @@ -287,15 +291,12 @@ def running_experiment( feature: str, variant: str, value: str, - rollout: str | None, + rollout: int, + allocation: int, ) -> None: - """Runs an experiment on the feature, enrolling every identity unless a rollout is given.""" - flagsmith.start_experiment( - require_environment(state), - feature, - variant, - parse_value(value), - 100 if rollout is None else int(rollout), + """Runs an experiment on the feature. Identities not served the variant are its control.""" + state.experiment_id = flagsmith.start_experiment( + require_environment(state), feature, variant, parse_value(value), rollout, allocation ) @@ -367,6 +368,19 @@ def persisted_traits( EVENT_VALUES = ("value", "traits", "metadata") +# A metadata value standing for the id of the scenario's experiment, which Flagsmith assigns. +EXPERIMENT_ID = "" + + +def with_experiment_id(metadata: Any, state: ScenarioState) -> Any: + if not isinstance(metadata, dict): + return metadata + return { + key: state.experiment_id if value == EXPERIMENT_ID else value + for key, value in metadata.items() + } + + def event_as_expected(event: dict[str, Any], expectation: dict[str, Any]) -> dict[str, Any]: """The part of a delivered event its expectation names, in the expectation's terms.""" actual = {column: event.get(EVENT_FIELDS.get(column, column)) for column in expectation} @@ -386,6 +400,9 @@ def received_events( {key: parse_value(value) if key in EVENT_VALUES else value for key, value in row.items()} for row in table_rows(datatable) ] + for expectation in expected: + if "metadata" in expectation: + expectation["metadata"] = with_experiment_id(expectation["metadata"], state) delivered = warehouse.wait_for_events(require_environment(state).api_key, len(expected)) assert len(delivered) == len(expected), f"Flagsmith received {delivered}" for event, expectation in zip(delivered, expected):