diff --git a/README.md b/README.md index f511df1..5d727e3 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,9 @@ Each release also pins the Flagsmith version in `compose.yml` that it was verifi - `Create the SDK client with local evaluation` receives the same two as `Create the SDK client`, and returns a client that evaluates flags locally, with its environment already loaded. Only `@local-evaluation` scenarios use it, so an SDK without local evaluation can leave it out. + - `Create the SDK client with local evaluation and events enabled` receives the same three as + `Create the SDK client with events enabled`, and returns a client that does both. Only + scenarios tagged both `@local-evaluation` and `@events` use it. - A `When` step receives the client, then the step's placeholders in order, then its data table. A ` | value` table arrives as a map, e.g. `{"plan": "premium", "age": 30}`. - Values are JSON values. Pass flag and trait values through with their types intact. diff --git a/features/get_experiment_flag.feature b/features/get_experiment_flag.feature index fe3bb7d..79325fe 100644 --- a/features/get_experiment_flag.feature +++ b/features/get_experiment_flag.feature @@ -26,7 +26,7 @@ Feature: Get an experiment flag Given the environment has the following features: | feature | enabled | value | | banner | true | "control" | - And a running experiment on feature "banner" serving variant "treatment" with value "hello treatment" + And a running experiment on feature "banner" enrolling 100% of identities, serving variant "treatment" with value "hello treatment" to 100% of them When the application gets the experiment flag "banner" for identity "user_123" with traits: | trait | value | | plan | "premium" | diff --git a/features/local__get_experiment_flag.feature b/features/local__get_experiment_flag.feature new file mode 100644 index 0000000..b341ca8 --- /dev/null +++ b/features/local__get_experiment_flag.feature @@ -0,0 +1,64 @@ +@local-evaluation @events +Feature: Get an experiment flag with local evaluation + A server-side SDK resolves a feature for an identity against the environment document, and + records the exposure as it would with remote evaluation. + + Background: + Given a Flagsmith environment + And the SDK evaluates flags locally + And the SDK sends events to Flagsmith + + Scenario: An identity in a running experiment is exposed to the variant it was served + Given the environment has the following features: + | feature | enabled | value | + | checkout | true | "control" | + And a running experiment on feature "checkout" enrolling 100% of identities, serving variant "treatment" with value "new checkout" to 100% of them + When the application gets the experiment flag "checkout" for identity "user_123" with traits: + | trait | value | + | plan | "premium" | + Then the application receives the following flags: + | feature | enabled | value | + | checkout | true | "new checkout" | + And Flagsmith has received the following events: + | event | feature | identifier | value | traits | metadata | + | $flag_exposure | checkout | user_123 | "treatment" | {"plan": "premium"} | {"experiment_id": ""} | + + Scenario: An identity in a running experiment served no variant is exposed to control + Given the environment has the following features: + | feature | enabled | value | + | checkout | true | "control" | + And a running experiment on feature "checkout" enrolling 100% of identities, serving variant "treatment" with value "new checkout" to 0% of them + When the application gets the experiment flag "checkout" for identity "user_123" with traits: + | trait | value | + | plan | "premium" | + Then the application receives the following flags: + | feature | enabled | value | + | checkout | true | "control" | + And Flagsmith has received the following events: + | event | feature | identifier | value | traits | metadata | + | $flag_exposure | checkout | user_123 | "control" | {"plan": "premium"} | {"experiment_id": ""} | + + Scenario: An identity a running experiment does not enroll is not exposed + Given the environment has the following features: + | feature | enabled | value | + | checkout | true | "control" | + And a running experiment on feature "checkout" enrolling 0% of identities, serving variant "treatment" with value "new checkout" to 100% of them + When the application gets the experiment flag "checkout" for identity "user_123" with traits: + | trait | value | + | plan | "premium" | + Then the application receives the following flags: + | feature | enabled | value | + | checkout | true | "control" | + And Flagsmith has received no events + + Scenario: An identity outside a running experiment is not exposed + Given the environment has the following features: + | feature | enabled | value | + | checkout | true | "control" | + When the application gets the experiment flag "checkout" for identity "user_123" with traits: + | trait | value | + | plan | "premium" | + Then the application receives the following flags: + | feature | enabled | value | + | checkout | true | "control" | + And Flagsmith has received no events diff --git a/runner/sdk_spec/flagsmith_admin.py b/runner/sdk_spec/flagsmith_admin.py index c5fc855..b00e493 100644 --- a/runner/sdk_spec/flagsmith_admin.py +++ b/runner/sdk_spec/flagsmith_admin.py @@ -103,12 +103,19 @@ def create_feature( environment.features[name] = Feature(id=feature["id"], value=value) def start_experiment( - self, environment: Environment, feature: str, variant: str, value: Any - ) -> None: - """Runs an experiment on the feature, serving `variant` to every identity. - - A single variant at 100% keeps the scenario deterministic: every identity is - enrolled, and every one of them is served that variant. + self, + environment: Environment, + feature: str, + variant: str, + value: Any, + rollout_percentage: int, + variant_percentage: int, + ) -> int: + """Runs an experiment on the feature, and returns its id. + + It enrolls `rollout_percentage` of identities, and serves `variant` to + `variant_percentage` of those; the rest are served the feature's value as control. + Percentages of 100% or 0% keep the scenario deterministic. """ control = environment.features[feature] option = self._request( @@ -131,11 +138,14 @@ def start_experiment( "hypothesis": f"Serving {variant} changes what identities do", "experiment_rollout": { "enabled": True, - "rollout_percentage": 100, + "rollout_percentage": rollout_percentage, # What an enrolled identity is served when it is allocated no variant. "feature_state_value": _rollout_value(control.value), "multivariate_feature_state_values": [ - {"multivariate_feature_option": option["id"], "percentage_allocation": 100} + { + "multivariate_feature_option": option["id"], + "percentage_allocation": variant_percentage, + } ], # No segments: the experiment targets every identity in the environment. "audience": {"match": "any", "segment_ids": []}, @@ -145,6 +155,7 @@ def start_experiment( self._request( "POST", f"/environments/{environment.api_key}/experiments/{experiment['id']}/start/" ) + return experiment["id"] def create_trait_segment( self, environment: Environment, name: str, trait: str, operator: str, value: Any diff --git a/runner/tests/conftest.py b/runner/tests/conftest.py index ed3c9c0..dbc8022 100644 --- a/runner/tests/conftest.py +++ b/runner/tests/conftest.py @@ -68,6 +68,7 @@ class ScenarioState: client: str | None = None events: bool = False local_evaluation: bool = False + experiment_id: int | None = None result: Any = None @@ -162,6 +163,10 @@ def require_environment(state: ScenarioState) -> Environment: CREATE_CLIENT_WITH_EVENTS = "Create the SDK client with events enabled" # A client evaluating flags against the environment document, for server-side SDKs only. CREATE_CLIENT_WITH_LOCAL_EVALUATION = "Create the SDK client with local evaluation" +# Both of the above, for scenarios about the events an SDK sends under local evaluation. +CREATE_CLIENT_WITH_LOCAL_EVALUATION_AND_EVENTS = ( + "Create the SDK client with local evaluation and events enabled" +) SDK_STEPS: list[str] = [] PLACEHOLDER = re.compile(r'("?)\{(\w+)\}\1') @@ -174,7 +179,11 @@ def sdk_client(state: ScenarioState, sdk: SDK) -> str: args = [environment.server_side_key, f"{FLAGSMITH_SDK_URL.rstrip('/')}/api/v1/"] if state.events: args.append(f"{EVENTS_SDK_URL.rstrip('/')}/") - step = CREATE_CLIENT_WITH_EVENTS + step = ( + CREATE_CLIENT_WITH_LOCAL_EVALUATION_AND_EVENTS + if state.local_evaluation + else CREATE_CLIENT_WITH_EVENTS + ) elif state.local_evaluation: step = CREATE_CLIENT_WITH_LOCAL_EVALUATION else: @@ -249,6 +258,7 @@ def sdk_sends_events(state: ScenarioState, flagsmith: FlagsmithAdmin) -> None: @given("the SDK evaluates flags locally") def sdk_evaluates_flags_locally(state: ScenarioState) -> None: """Asks for a client with local evaluation, which the runner creates for the first SDK step.""" + assert state.client is None, "The SDK client has already been created without local evaluation" state.local_evaluation = True @@ -267,15 +277,26 @@ def environment_features( @given( - parsers.parse( - 'a running experiment on feature "{feature}" serving variant "{variant}" with value {value}' - ) + parsers.re( + r'a running experiment on feature "(?P[^"]+)"' + r" enrolling (?P\d+)% of identities," + r' serving variant "(?P[^"]+)" with value (?P.+)' + r" to (?P\d+)% of them" + ), + converters={"rollout": int, "allocation": int}, ) def running_experiment( - state: ScenarioState, flagsmith: FlagsmithAdmin, feature: str, variant: str, value: str + state: ScenarioState, + flagsmith: FlagsmithAdmin, + feature: str, + variant: str, + value: str, + rollout: int, + allocation: int, ) -> None: - flagsmith.start_experiment( - require_environment(state), feature, variant, parse_value(value) + """Runs an experiment on the feature. Identities not served the variant are its control.""" + state.experiment_id = flagsmith.start_experiment( + require_environment(state), feature, variant, parse_value(value), rollout, allocation ) @@ -347,6 +368,19 @@ def persisted_traits( EVENT_VALUES = ("value", "traits", "metadata") +# A metadata value standing for the id of the scenario's experiment, which Flagsmith assigns. +EXPERIMENT_ID = "" + + +def with_experiment_id(metadata: Any, state: ScenarioState) -> Any: + if not isinstance(metadata, dict): + return metadata + return { + key: state.experiment_id if value == EXPERIMENT_ID else value + for key, value in metadata.items() + } + + def event_as_expected(event: dict[str, Any], expectation: dict[str, Any]) -> dict[str, Any]: """The part of a delivered event its expectation names, in the expectation's terms.""" actual = {column: event.get(EVENT_FIELDS.get(column, column)) for column in expectation} @@ -366,6 +400,9 @@ def received_events( {key: parse_value(value) if key in EVENT_VALUES else value for key, value in row.items()} for row in table_rows(datatable) ] + for expectation in expected: + if "metadata" in expectation: + expectation["metadata"] = with_experiment_id(expectation["metadata"], state) delivered = warehouse.wait_for_events(require_environment(state).api_key, len(expected)) assert len(delivered) == len(expected), f"Flagsmith received {delivered}" for event, expectation in zip(delivered, expected):