diff --git a/docs/api-integrations.rst b/docs/api-integrations.rst index 8d8146ff..594d7ed8 100644 --- a/docs/api-integrations.rst +++ b/docs/api-integrations.rst @@ -8,3 +8,13 @@ ldclient.integrations module :members: :special-members: __init__ :show-inheritance: + +ldclient.integrations.overrides module +-------------------------------------- + +The entry point for this feature is :class:`ldclient.integrations.overrides.FileOverrideSourceBuilder`. + +.. automodule:: ldclient.integrations.overrides + :members: + :special-members: __init__ + :show-inheritance: diff --git a/ldclient/async_client.py b/ldclient/async_client.py index 54f1ada0..11c1461a 100644 --- a/ldclient/async_client.py +++ b/ldclient/async_client.py @@ -486,7 +486,12 @@ async def evaluate(): tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': stage, 'tracker': tracker}) - detail = EvaluationDetail(default_stage.value, None, error_reason('WRONG_TYPE')) + # The type mismatch replaces the reason. The evaluation read the same definitions, + # so the new reason keeps the override-affected marking. + reason = error_reason('WRONG_TYPE') + if detail.reason.get('overrideAffected') is True: + reason['overrideAffected'] = True + detail = EvaluationDetail(default_stage.value, None, reason) tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': default_stage, 'tracker': tracker}) @@ -547,17 +552,19 @@ def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: detail = result.detail if detail.is_default_value(): detail = EvaluationDetail(default, None, detail.reason) - self._send_event(event_factory.new_eval_event(flag, context, detail, default)) + self._send_event(event_factory.new_eval_event(flag, context, detail, default, None, result.override_affected)) return detail, flag except Exception as e: log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e))) log.debug(traceback.format_exc()) reason = error_reason('EXCEPTION') - # The evaluation read the flag definition before it failed, so a definition from - # the override layer marks the result. - if is_override_definition(flag): + # The evaluation read the flag definition before it failed. A definition from + # the override layer marks the result, so the failure is reported in the + # summary counters only, like any other override-affected evaluation. + override_affected = is_override_definition(flag) + if override_affected: reason['overrideAffected'] = True - self._send_event(event_factory.new_default_event(flag, context, default, reason)) + self._send_event(event_factory.new_default_event(flag, context, default, reason, override_affected)) return EvaluationDetail(default, None, reason), flag async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: @@ -647,17 +654,29 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState result = await self._evaluator.evaluate(flag, context, self._event_factory_default) detail = result.detail prerequisites = result.prerequisites + override_affected = result.override_affected except Exception as e: log.error("Error evaluating flag \"%s\" in all_flags_state: %s" % (key, repr(e))) log.debug(traceback.format_exc()) reason = error_reason('EXCEPTION') # The evaluation read the flag definition before it failed, so a definition from # the override layer marks the result. - if is_override_definition(flag): + override_affected = is_override_definition(flag) + if override_affected: reason['overrideAffected'] = True detail = EvaluationDetail(None, None, reason) prerequisites = [] requires_experiment_data = EventFactory.is_experiment(flag, detail.reason) + track_events = flag.get('trackEvents', False) or requires_experiment_data + track_reason = requires_experiment_data + debug_events_until_date = flag.get('debugEventsUntilDate', None) + if override_affected: + # A consumer of this state sends individual events according to these fields. + # An override-affected evaluation produces no individual events, so the state + # turns them off for this flag. The flag, its value, and its reason stay. + track_events = False + track_reason = False + debug_events_until_date = None flag_state = { 'key': flag['key'], 'value': detail.value, @@ -665,9 +684,9 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState 'reason': detail.reason, 'version': flag['version'], 'prerequisites': prerequisites, - 'trackEvents': flag.get('trackEvents', False) or requires_experiment_data, - 'trackReason': requires_experiment_data, - 'debugEventsUntilDate': flag.get('debugEventsUntilDate', None), + 'trackEvents': track_events, + 'trackReason': track_reason, + 'debugEventsUntilDate': debug_events_until_date, } state.add_flag(flag_state, with_reasons, details_only_if_tracked) diff --git a/ldclient/client.py b/ldclient/client.py index 0866e8ea..6361512b 100644 --- a/ldclient/client.py +++ b/ldclient/client.py @@ -480,7 +480,12 @@ def evaluate(): tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': stage, 'tracker': tracker}) - detail = EvaluationDetail(default_stage.value, None, error_reason('WRONG_TYPE')) + # The type mismatch replaces the reason. The evaluation read the same definitions, + # so the new reason keeps the override-affected marking. + reason = error_reason('WRONG_TYPE') + if detail.reason.get('overrideAffected') is True: + reason['overrideAffected'] = True + detail = EvaluationDetail(default_stage.value, None, reason) tracker = OpTracker(key, flag, context, detail, default_stage) return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': default_stage, 'tracker': tracker}) @@ -543,17 +548,19 @@ def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]: detail = result.detail if detail.is_default_value(): detail = EvaluationDetail(default, None, detail.reason) - self._send_event(event_factory.new_eval_event(flag, context, detail, default)) + self._send_event(event_factory.new_eval_event(flag, context, detail, default, None, result.override_affected)) return detail, flag except Exception as e: log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e))) log.debug(traceback.format_exc()) reason = error_reason('EXCEPTION') - # The evaluation read the flag definition before it failed, so a definition from - # the override layer marks the result. - if is_override_definition(flag): + # The evaluation read the flag definition before it failed. A definition from + # the override layer marks the result, so the failure is reported in the + # summary counters only, like any other override-affected evaluation. + override_affected = is_override_definition(flag) + if override_affected: reason['overrideAffected'] = True - self._send_event(event_factory.new_default_event(flag, context, default, reason)) + self._send_event(event_factory.new_default_event(flag, context, default, reason, override_affected)) return EvaluationDetail(default, None, reason), flag def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: @@ -649,18 +656,30 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: result = self._evaluator.evaluate(flag, context, self._event_factory_default) detail = result.detail prerequisites = result.prerequisites + override_affected = result.override_affected except Exception as e: log.error("Error evaluating flag \"%s\" in all_flags_state: %s" % (key, repr(e))) log.debug(traceback.format_exc()) reason = error_reason('EXCEPTION') # The evaluation read the flag definition before it failed, so a definition from # the override layer marks the result. - if is_override_definition(flag): + override_affected = is_override_definition(flag) + if override_affected: reason['overrideAffected'] = True detail = EvaluationDetail(None, None, reason) prerequisites = [] requires_experiment_data = EventFactory.is_experiment(flag, detail.reason) + track_events = flag.get('trackEvents', False) or requires_experiment_data + track_reason = requires_experiment_data + debug_events_until_date = flag.get('debugEventsUntilDate', None) + if override_affected: + # A consumer of this state sends individual events according to these fields. + # An override-affected evaluation produces no individual events, so the state + # turns them off for this flag. The flag, its value, and its reason stay. + track_events = False + track_reason = False + debug_events_until_date = None flag_state = { 'key': flag['key'], 'value': detail.value, @@ -668,9 +687,9 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState: 'reason': detail.reason, 'version': flag['version'], 'prerequisites': prerequisites, - 'trackEvents': flag.get('trackEvents', False) or requires_experiment_data, - 'trackReason': requires_experiment_data, - 'debugEventsUntilDate': flag.get('debugEventsUntilDate', None), + 'trackEvents': track_events, + 'trackReason': track_reason, + 'debugEventsUntilDate': debug_events_until_date, } state.add_flag(flag_state, with_reasons, details_only_if_tracked) diff --git a/ldclient/impl/async_evaluator.py b/ldclient/impl/async_evaluator.py index 935fc84d..31f3466c 100644 --- a/ldclient/impl/async_evaluator.py +++ b/ldclient/impl/async_evaluator.py @@ -148,7 +148,7 @@ async def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: # off variation was. But we still need to evaluate it in order to generate an event. if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation: failed_prereq = prereq - event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag) + event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag, prereq_override_affected) state.add_event(event) if failed_prereq: return {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': failed_prereq.key} diff --git a/ldclient/impl/datasystem/async_fdv2.py b/ldclient/impl/datasystem/async_fdv2.py index 7592d02c..a0db7c3d 100644 --- a/ldclient/impl/datasystem/async_fdv2.py +++ b/ldclient/impl/datasystem/async_fdv2.py @@ -347,7 +347,12 @@ def start(self, set_on_ready: AsyncEvent): listeners = self._flag_change_listeners def notify(key: str) -> None: - loop.call_soon_threadsafe(listeners.notify, FlagChange(key)) + try: + loop.call_soon_threadsafe(listeners.notify, FlagChange(key)) + except RuntimeError: + # A reload can finish after the client closed. The loop is closed then, + # and the notification has nobody left to reach. + log.debug("Dropped the flag change notification for %s because the event loop is closed", key) self._override_sink = OverrideSinkImpl(self._override_layer, self._store._memory_store, notify, listeners.has_listeners) try: diff --git a/ldclient/impl/evaluator.py b/ldclient/impl/evaluator.py index f6a14a25..bbce2260 100644 --- a/ldclient/impl/evaluator.py +++ b/ldclient/impl/evaluator.py @@ -146,7 +146,7 @@ def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: EvalR # off variation was. But we still need to evaluate it in order to generate an event. if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation: failed_prereq = prereq - event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag) + event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag, prereq_override_affected) state.add_event(event) if failed_prereq: return {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': failed_prereq.key} diff --git a/ldclient/impl/events/event_processor_common.py b/ldclient/impl/events/event_processor_common.py index 95ea278f..75f634b1 100644 --- a/ldclient/impl/events/event_processor_common.py +++ b/ldclient/impl/events/event_processor_common.py @@ -181,7 +181,7 @@ def make_summary_event(self, summary: EventSummary): } counters = [] for ckey, cval in flag_data.counters.items(): - variation, version = ckey + variation, version, override_affected = ckey counter = {'count': cval.count, 'value': cval.value} if variation is not None: counter['variation'] = variation @@ -189,6 +189,9 @@ def make_summary_event(self, summary: EventSummary): counter['unknown'] = True else: counter['version'] = version + # The marker is present only when true, like the unknown marker. + if override_affected: + counter['overrideAffected'] = True counters.append(counter) flag_data_out['counters'] = counters flags_out[key] = flag_data_out @@ -266,9 +269,11 @@ def _process_event(self, event: EventInput): context = event.context if not event.exclude_from_summaries: self._outbox.add_to_summary(event) - if event.track_events: + # An override-affected evaluation appears only in the summary counters. It produces + # no individual feature event and no debug event, even when the flag requests them. + if event.track_events and not event.override_affected: full_event = event - if self._should_debug_event(event): + if not event.override_affected and self._should_debug_event(event): debug_event = DebugEvent(event) elif isinstance(event, EventInputIdentify): if self._omit_anonymous_contexts: diff --git a/ldclient/impl/events/event_summarizer.py b/ldclient/impl/events/event_summarizer.py index acf3a88f..ca2b5b5e 100644 --- a/ldclient/impl/events/event_summarizer.py +++ b/ldclient/impl/events/event_summarizer.py @@ -22,10 +22,16 @@ def __repr__(self) -> str: # used only in test debugging return "EventSummaryCounter(%d, %s)" % (self.count, self.value) +# A counter is keyed by variation index, flag version, and whether the evaluations it counts +# were override-affected. The marker is part of the key, so override-affected and ordinary +# evaluations of the same flag, variation, and version accumulate into separate counters. +CounterKey = Tuple[Optional[int], Optional[int], bool] + + class EventSummaryFlag: __slots__ = ['context_kinds', 'default', 'counters'] - def __init__(self, context_kinds: Set[str], default: Any, counters: Dict[Tuple[Optional[int], Optional[int]], EventSummaryCounter]): + def __init__(self, context_kinds: Set[str], default: Any, counters: Dict[CounterKey, EventSummaryCounter]): self.context_kinds = context_kinds self.counters = counters self.default = default @@ -71,7 +77,7 @@ def summarize_event(self, event: EventInputEvaluation): if c is not None: flag_data.context_kinds.add(c.kind) - counter_key = (event.variation, None if event.flag is None else event.flag.version) + counter_key = (event.variation, None if event.flag is None else event.flag.version, event.override_affected) counter = flag_data.counters.get(counter_key) if counter is None: counter = EventSummaryCounter(1, event.value) diff --git a/ldclient/impl/events/types.py b/ldclient/impl/events/types.py index d0883c47..6f754a23 100644 --- a/ldclient/impl/events/types.py +++ b/ldclient/impl/events/types.py @@ -52,6 +52,7 @@ class EventInputEvaluation(EventInput): "track_events", "sampling_ratio", "exclude_from_summaries", + "override_affected", ] def __init__( @@ -66,6 +67,7 @@ def __init__( default_value: Any, prereq_of: Optional[FeatureFlag] = None, track_events: bool = False, + override_affected: bool = False, ): super().__init__(timestamp, context, 1 if flag is None else flag.sampling_ratio) self.key = key @@ -79,6 +81,11 @@ def __init__( self.exclude_from_summaries = ( False if flag is None else flag.exclude_from_summaries ) + # True when a flag override affected this evaluation, directly or through a + # prerequisite or segment. The event processor keys on this scalar alone, never on the + # reason: such an evaluation produces no individual feature or debug event and is + # counted in a summary counter that carries the override-affected marker. + self.override_affected = override_affected def to_debugging_dict(self) -> dict: return { @@ -94,6 +101,7 @@ def to_debugging_dict(self) -> dict: "track_events": self.track_events, "exclude_from_summaries": self.exclude_from_summaries, "sampling_ratio": self.sampling_ratio, + "override_affected": self.override_affected, } @@ -155,6 +163,7 @@ def new_eval_event( detail: EvaluationDetail, default_value: Any, prereq_of_flag: Optional[FeatureFlag] = None, + override_affected: bool = False, ) -> EventInputEvaluation: add_experiment_data = self.is_experiment(flag, detail.reason) return EventInputEvaluation( @@ -168,6 +177,7 @@ def new_eval_event( default_value, prereq_of_flag, flag.track_events or add_experiment_data, + override_affected, ) def new_default_event( @@ -176,6 +186,7 @@ def new_default_event( context: Context, default_value: Any, reason: Optional[dict], + override_affected: bool = False, ) -> EventInputEvaluation: return EventInputEvaluation( self._timestamp_fn(), @@ -188,6 +199,7 @@ def new_default_event( default_value, None, flag.track_events, + override_affected, ) def new_unknown_flag_event( diff --git a/ldclient/impl/integrations/overrides/__init__.py b/ldclient/impl/integrations/overrides/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ldclient/impl/integrations/overrides/file_override_source.py b/ldclient/impl/integrations/overrides/file_override_source.py new file mode 100644 index 00000000..220a73b8 --- /dev/null +++ b/ldclient/impl/integrations/overrides/file_override_source.py @@ -0,0 +1,153 @@ +""" +The file-based override source. It reads flag and segment overrides from one or more local +files and reloads them when the files change. +""" + +import threading +from enum import Enum +from typing import List, Optional, Union + +from ldclient.impl.integrations.files.filedata import ( + DEFAULT_DEBOUNCE_DELAY, + DEFAULT_RETRY_DELAY, + DuplicateKeysHandling, + FileSummary, + MergeResult, + Poller, + Reloader, + Watcher +) +from ldclient.impl.util import log +from ldclient.interfaces import OverrideSink, OverrideSource + + +class ChangeDetection(str, Enum): + """ + Selects how the file-based override source learns that a file changed. The two modes are + alternatives. Flag overrides are currently experimental and subject to change. + """ + + POLLING = "polling" + """ + The source examines the files on a fixed interval and reloads when the modification time + or the size of a file changes. Polling works on every file system, including network + mounts and directories whose contents are swapped through symbolic links, as Kubernetes + does for mounted ConfigMaps. It is the default. + """ + + WATCHING = "watching" + """ + The source reloads in response to file system change notifications, using the ``watchdog`` + package. It reacts faster than polling. It depends on notifications, which some file + systems do not deliver reliably. + """ + + +class _FileOverrideSource(OverrideSource): + """ + Reads overrides from the configured files and supplies each successful load to the sink as + a full snapshot. A configured file that does not exist contributes no overrides. A file + that cannot be read or parsed fails that load, the last good snapshot stays in effect, and + the load is retried. A file whose content is in effect and that is found with no entries is + held back until it has stayed so for the retry delay, because it may be in the middle of an + in-place rewrite. + """ + + def __init__(self, paths: List[str], duplicate_keys_handling: DuplicateKeysHandling, change_detection: ChangeDetection, poll_interval: float): + self._paths = paths + self._duplicate_keys_handling = duplicate_keys_handling + self._change_detection = change_detection + self._poll_interval = poll_interval + self._reloader: Optional[Reloader] = None + self._change_detector: Optional[Union[Poller, Watcher]] = None + self._lock = threading.Lock() + self._closed = False + + def start(self, sink: OverrideSink) -> None: + def apply(merged: MergeResult) -> None: + sink.set_overrides(merged.flags, merged.segments) + _log_overrides_in_effect(merged) + + reloader = Reloader( + self._paths, + self._duplicate_keys_handling, + apply=apply, + skip_missing_paths=True, + debounce_delay=DEFAULT_DEBOUNCE_DELAY, + retry_delay=DEFAULT_RETRY_DELAY, + skip_unchanged=True, + ) + # Change detection is set up before the initial load, so an edit made while the files + # are first read is picked up by the reload it triggers instead of waiting for a later + # change. + with self._lock: + if self._closed: + return + self._reloader = reloader + if self._change_detection == ChangeDetection.WATCHING: + self._change_detector = Watcher(self._paths, reloader.trigger) + else: + poller = Poller(self._paths, self._poll_interval, reloader.trigger) + poller.start() + self._change_detector = poller + + # The initial load runs synchronously, so overrides present in the files are in effect + # by the time the client constructor returns. A file that does not exist yet + # contributes no overrides. A file that cannot be read or parsed is not fatal: the + # client runs with no overrides, the failure is logged, and the retry recovers once + # the file is readable. + reloader.reload_now() + + def close(self) -> None: + with self._lock: + if self._closed: + return + self._closed = True + change_detector = self._change_detector + reloader = self._reloader + self._change_detector = None + self._reloader = None + try: + if change_detector is not None: + change_detector.close() + finally: + # The reloader stops even when the change detector fails to close, so no reload + # can run after close. + if reloader is not None: + reloader.close() + + +def _log_overrides_in_effect(merged: MergeResult) -> None: + """ + Reports the overrides now in effect and what each configured file supplied. The reloader + applies a snapshot only when the content changed, so this logs each change once. + """ + details = "; ".join(_file_summary_text(summary) for summary in merged.files) + if len(merged.flags) == 0 and len(merged.segments) == 0: + log.info("Flag overrides: none in effect (%s)", details) + return + log.info("Flag overrides in effect: %s (%s)", _counts_text(len(merged.flags), len(merged.segments)), details) + + +def _file_summary_text(summary: FileSummary) -> str: + if not summary.present: + return "%s: absent" % summary.path + if summary.flags == 0 and summary.segments == 0: + return "%s: no entries" % summary.path + return "%s: %s" % (summary.path, _counts_text(summary.flags, summary.segments)) + + +def _counts_text(flags: int, segments: int) -> str: + """Formats flag and segment counts, for example "2 flags, 1 segment".""" + parts = [] + if flags > 0: + parts.append(_pluralize(flags, "flag")) + if segments > 0: + parts.append(_pluralize(segments, "segment")) + return ", ".join(parts) + + +def _pluralize(count: int, noun: str) -> str: + if count == 1: + return "1 %s" % noun + return "%d %ss" % (count, noun) diff --git a/ldclient/integrations/overrides.py b/ldclient/integrations/overrides.py new file mode 100644 index 00000000..e0bfe586 --- /dev/null +++ b/ldclient/integrations/overrides.py @@ -0,0 +1,206 @@ +""" +Override sources for the SDK's flag override capability. Flag overrides are currently +experimental and subject to change. + +Overrides are flag and segment definitions that take precedence over data received from +LaunchDarkly at evaluation time, on a per-key basis. They exist for resilience during an +incident. An operator can force one or more flags to a known state on a running application, +whether or not the application can reach LaunchDarkly. The override stays in effect until the +operator removes it. Flags not present in the override data are completely unaffected. + +This module currently provides one source: :class:`FileOverrideSourceBuilder`, which reads +overrides from local files and reloads them as the files change. Configure it with the data +system builder: +:: + + from ldclient import Config, datasystem + from ldclient.integrations.overrides import FileOverrideSourceBuilder + + source = FileOverrideSourceBuilder(['/etc/launchdarkly/overrides.json']) + config = Config(sdk_key, datasystem_config=datasystem.default().overrides(source).build()) + +An evaluation that an override affects is marked. The marking is direct or transitive. It +applies when the evaluated flag, a prerequisite at any depth, or a segment read during the +evaluation came from the override layer. The evaluation reason's ``overrideAffected`` property +reports the marking. Marked evaluations appear in analytics summary events only, under separate +counters, so LaunchDarkly can distinguish them. They produce no individual evaluation events. +""" + +import math +import os +import threading +from typing import List, Optional, Union + +from ldclient.config import DataSourceBuilderConfig, OverrideSourceBuilder +from ldclient.impl.integrations.files.filedata import ( + DuplicateKeysHandling, + abs_file_paths, + have_watchdog, + have_yaml, + is_yaml_path +) +from ldclient.impl.integrations.overrides.file_override_source import ( + ChangeDetection, + _FileOverrideSource +) +from ldclient.impl.util import log +from ldclient.interfaces import OverrideSource + + +class FileOverrideSourceBuilder(OverrideSourceBuilder): + """ + A builder for a file-based override source. Flag overrides are currently experimental and + subject to change. + + The source reads flag and segment overrides from one or more local files and reloads them + as the files change. The files use the same document format as the file data source: + each file is a JSON or YAML document with optional ``flags``, ``flagValues``, and + ``segments`` members. ``flagValues`` entries are expanded into full flag definitions that + return the given value for every context. A ``flagValues`` entry is not available to + client-side SDKs, so :meth:`ldclient.client.LDClient.all_flags_state` with + ``client_side_only=True`` leaves it out, even when the flag it overrides is available to them. + To override such a flag, define it in ``flags`` with ``"clientSide": true``. + + The file name decides the format of each file: a file whose name ends in ``.yaml`` or + ``.yml`` is YAML, which requires the ``pyyaml`` package, and any other file is JSON. Files + must be UTF-8. + + When multiple files are configured, their entries are combined in the configured order. + The order determines which file wins under the duplicate keys handling. A reload replaces + the entire override set, so removing an entry from the files removes the override. A + configured file that does not exist contributes no overrides: deleting a file removes its + overrides, and deleting every file removes them all. A file that exists but cannot be read + or parsed makes that whole reload fail. The previously loaded overrides stay in effect, the + source logs the failure, retries after a short delay, and recovers on its own once the file + is readable again. + + Files can be rewritten in place: if the source finds a file empty while it is being + rewritten, it keeps the overrides in effect until the new content is written. To remove a + file's overrides, delete the file or write a document with no entries, such as ``{}``. An + empty JSON file is not a valid document. A partly written YAML file can still parse, so for a + YAML file, replacing its contents in one operation, such as by renaming a fully written + temporary file over it, avoids briefly applying a partly written file. + + Whenever the set of overrides in effect changes, including at startup, the source logs the + overrides in effect and what each configured file supplied, at Info level. + + By default the source polls the files for changes once per second. See + :meth:`change_detection` and :meth:`poll_interval`. + """ + + DEFAULT_POLL_INTERVAL = 1.0 + """ + The interval, in seconds, at which the source examines the files for changes in polling + mode when no interval was specified. Because the source reads local files rather than + contacting a service, a short interval keeps an override responsive during an incident at + negligible cost. + """ + + MINIMUM_POLL_INTERVAL = 1.0 + """ + The shortest allowed polling interval, in seconds. A configured interval below this is + raised to it. The minimum exists only to prevent a pathological tight loop over the file + system. + """ + + def __init__(self, paths: Union[str, List[str]]): + """ + :param paths: the files to load overrides from, as a single path or a list of paths. + The order is significant: it determines which file wins under the duplicate keys + handling when the same key appears in more than one file. Relative paths are + resolved against the current working directory when the source is built. + """ + self.__paths: List[str] = [paths] if isinstance(paths, str) else list(paths) + self.__duplicate_keys_handling = DuplicateKeysHandling.FAIL + self.__change_detection = ChangeDetection.POLLING + self.__poll_interval = self.DEFAULT_POLL_INTERVAL + + def duplicate_keys_handling(self, handling: Union[DuplicateKeysHandling, str]) -> 'FileOverrideSourceBuilder': + """ + Specifies how to handle the same key appearing in more than one file. The default is + :attr:`DuplicateKeysHandling.FAIL`, which treats the reload as failed and keeps the + previously loaded overrides. :attr:`DuplicateKeysHandling.IGNORE` keeps the entry from + the first configured file that defines the key and discards the others. + + :param handling: the handling, as the enum or its string value (``"fail"`` or ``"ignore"``) + """ + self.__duplicate_keys_handling = DuplicateKeysHandling(handling) + return self + + def change_detection(self, mode: Union[ChangeDetection, str]) -> 'FileOverrideSourceBuilder': + """ + Selects how the source detects file changes. The default is + :attr:`ChangeDetection.POLLING`. The two modes are alternatives, so setting one replaces + the other. :attr:`ChangeDetection.WATCHING` requires the ``watchdog`` package. + + Watching follows edits to the configured files and to the targets of configured symbolic + links, including the update of a mounted Kubernetes ConfigMap. It does not follow a + deployment that points a link further along the path, or a configured directory, at + another location, such as a ``current`` link to a release directory, or that replaces a + configured directory. Use polling for such deployments. + + :param mode: the mode, as the enum or its string value (``"polling"`` or ``"watching"``) + """ + self.__change_detection = ChangeDetection(mode) + return self + + def poll_interval(self, seconds: float) -> 'FileOverrideSourceBuilder': + """ + Sets the interval between examinations of the files in polling mode. Watching mode + ignores it. The default is :attr:`DEFAULT_POLL_INTERVAL`. An interval below + :attr:`MINIMUM_POLL_INTERVAL` is raised to the minimum. + + :param seconds: the interval in seconds + :raises TypeError: when the interval is not a number + :raises ValueError: when the interval is not a finite number, or is longer than the + runtime can wait + """ + if isinstance(seconds, bool) or not isinstance(seconds, (int, float)): + raise TypeError("the poll interval must be a number of seconds") + if not math.isfinite(seconds): + raise ValueError("the poll interval must be a finite number of seconds") + if seconds > threading.TIMEOUT_MAX: + raise ValueError("the poll interval must not exceed %s seconds" % threading.TIMEOUT_MAX) + self.__poll_interval = seconds + return self + + def build(self, config: DataSourceBuilderConfig) -> OverrideSource: # pylint: disable=unused-argument + """ + Builds the override source. This is called internally by the SDK. It raises + ``ValueError`` when no file paths were specified, when watching mode was selected + without the ``watchdog`` package, or when a YAML file was specified without the + ``pyyaml`` package. + """ + if len(self.__paths) == 0: + raise ValueError("no file paths were specified for the file-based override source") + if self.__change_detection == ChangeDetection.WATCHING and not have_watchdog: + raise ValueError("the file-based override source cannot watch files for changes because the watchdog package is not installed; install it or use polling") + if not have_yaml: + for path in self.__paths: + if is_yaml_path(path): + raise ValueError("the file-based override source cannot read the YAML file %s because the pyyaml package is not installed; install it or use JSON" % path) + + poll_interval = self.__poll_interval + if self.__change_detection == ChangeDetection.POLLING and poll_interval < self.MINIMUM_POLL_INTERVAL: + log.warning("Poll interval %s is below the minimum for the file-based override source; using %s", poll_interval, self.MINIMUM_POLL_INTERVAL) + poll_interval = self.MINIMUM_POLL_INTERVAL + + # A path listed more than once would make every entry of that file a duplicate of itself, + # so the repeats are dropped and the first occurrence keeps its place in the order. Paths + # are compared the way the platform compares file names: without regard to case on + # Windows, exactly elsewhere. + all_paths = abs_file_paths(self.__paths) + paths: List[str] = [] + seen = set() + for path in all_paths: + key = os.path.normcase(path) + if key not in seen: + seen.add(key) + paths.append(path) + if len(paths) != len(all_paths): + log.warning("The file-based override source lists the same file more than once; the repeats are ignored") + + return _FileOverrideSource(paths, self.__duplicate_keys_handling, self.__change_detection, poll_interval) + + +__all__ = ['ChangeDetection', 'DuplicateKeysHandling', 'FileOverrideSourceBuilder'] diff --git a/ldclient/testing/impl/events/test_event_overrides.py b/ldclient/testing/impl/events/test_event_overrides.py new file mode 100644 index 00000000..4f4345a3 --- /dev/null +++ b/ldclient/testing/impl/events/test_event_overrides.py @@ -0,0 +1,188 @@ +""" +Tests for how analytics events treat override-affected evaluations: they appear in summary +counters only, under a counter that carries the override-affected marker, and they produce no +individual feature event and no debug event. +""" +import json +import time +from typing import Any, Dict, List + +import pytest + +from ldclient.async_config import AsyncConfig +from ldclient.config import Config +from ldclient.context import Context +from ldclient.impl.events.async_event_processor import ( + DefaultAsyncEventProcessor +) +from ldclient.impl.events.event_processor import DefaultEventProcessor +from ldclient.impl.events.event_processor_common import EventOutputFormatter +from ldclient.impl.events.event_summarizer import ( + EventSummarizer, + EventSummaryCounter +) +from ldclient.impl.events.types import EventInputEvaluation +from ldclient.testing.builders import FlagBuilder +from ldclient.testing.impl.events.test_async_event_processor import MockAioHttp +from ldclient.testing.stub_util import MockHttp + +context = Context.builder('userkey').name('Red').build() +timestamp = 10000 + + +def tracked_flag(key: str = 'flagkey', version: int = 2): + """A flag that requests individual feature events and debug events far into the future.""" + return FlagBuilder(key).version(version).track_events(True).debug_events_until_date(int(time.time() * 1000) + 100000).build() + + +def evaluation(flag, value: str = 'value', variation: int = 1, override_affected: bool = False, track_events: bool = True) -> EventInputEvaluation: + return EventInputEvaluation(timestamp, context, flag.key, flag, variation, value, None, 'default', None, track_events, override_affected) + + +def events_of_kind(output: List[Dict[str, Any]], kind: str) -> List[Dict[str, Any]]: + return [e for e in output if e['kind'] == kind] + + +# --------------------------------------------------------------------------- +# Summarizer and output formatter +# --------------------------------------------------------------------------- + +def test_summarizer_keeps_override_affected_counters_separate(): + flag = tracked_flag() + es = EventSummarizer() + es.summarize_event(evaluation(flag)) + es.summarize_event(evaluation(flag, override_affected=True)) + es.summarize_event(evaluation(flag, override_affected=True)) + es.summarize_event(evaluation(flag)) + counters = es.snapshot().flags[flag.key].counters + assert counters == { + (1, flag.version, False): EventSummaryCounter(2, 'value'), + (1, flag.version, True): EventSummaryCounter(2, 'value'), + } + + +def test_summary_output_carries_the_marker_only_on_override_affected_counters(): + flag = tracked_flag() + es = EventSummarizer() + es.summarize_event(evaluation(flag)) + es.summarize_event(evaluation(flag, override_affected=True)) + es.summarize_event(EventInputEvaluation(timestamp, context, 'unknown-flag', None, None, 'default', None, 'default', None, False, False)) + output = EventOutputFormatter(Config('SDK_KEY')).make_summary_event(es.snapshot()) + + counters = output['features'][flag.key]['counters'] + assert len(counters) == 2 + by_marker = {c.get('overrideAffected'): c for c in counters} + assert by_marker[None] == {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version} + assert by_marker[True] == {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True} + assert 'overrideAffected' not in json.dumps(output['features']['unknown-flag']) + + +def test_event_input_evaluation_defaults_to_not_override_affected(): + event = EventInputEvaluation(timestamp, context, 'flag', None, None, 'default', None, 'default') + assert event.override_affected is False + assert event.to_debugging_dict()['override_affected'] is False + + +# --------------------------------------------------------------------------- +# Sync event processor +# --------------------------------------------------------------------------- + +def flush_and_get_events(ep: DefaultEventProcessor, mock_http: MockHttp) -> List[Dict[str, Any]]: + ep.flush() + ep._wait_until_inactive() + assert mock_http.request_data is not None, 'Expected to get an HTTP request but did not get one' + return json.loads(mock_http.request_data) + + +def make_processor(mock_http: MockHttp) -> DefaultEventProcessor: + return DefaultEventProcessor(Config('SDK_KEY', diagnostic_opt_out=True), mock_http) + + +def test_override_affected_evaluation_produces_no_feature_or_debug_event(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag, override_affected=True)) + output = flush_and_get_events(ep, mock_http) + + assert [e['kind'] for e in output] == ['index', 'summary'] + counters = output[1]['features'][flag.key]['counters'] + assert counters == [{'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True}] + + +def test_ordinary_evaluation_still_produces_feature_and_debug_events(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag)) + output = flush_and_get_events(ep, mock_http) + + assert sorted(e['kind'] for e in output) == ['debug', 'feature', 'index', 'summary'] + counters = events_of_kind(output, 'summary')[0]['features'][flag.key]['counters'] + assert counters == [{'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version}] + + +def test_mixed_evaluations_of_one_flag_split_into_two_counters_and_one_feature_event(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag, override_affected=True)) + ep.send_event(evaluation(flag, override_affected=True)) + ep.send_event(evaluation(flag)) + output = flush_and_get_events(ep, mock_http) + + assert len(events_of_kind(output, 'feature')) == 1 + assert len(events_of_kind(output, 'debug')) == 1 + counters = events_of_kind(output, 'summary')[0]['features'][flag.key]['counters'] + assert sorted(counters, key=lambda c: c['count']) == [ + {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version}, + {'count': 2, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True}, + ] + + +def test_override_affected_prerequisite_record_produces_no_feature_event(): + parent = tracked_flag('parent', 1) + prereq = tracked_flag('prereq', 7) + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(EventInputEvaluation(timestamp, context, prereq.key, prereq, 1, 'value', None, None, parent, True, True)) + output = flush_and_get_events(ep, mock_http) + + assert [e['kind'] for e in output] == ['index', 'summary'] + counters = output[1]['features'][prereq.key]['counters'] + assert counters == [{'count': 1, 'value': 'value', 'variation': 1, 'version': prereq.version, 'overrideAffected': True}] + + +def test_override_affected_evaluation_still_produces_an_index_event(): + flag = tracked_flag() + mock_http = MockHttp() + with make_processor(mock_http) as ep: + ep.send_event(evaluation(flag, override_affected=True)) + output = flush_and_get_events(ep, mock_http) + assert output[0]['kind'] == 'index' + assert output[0]['context'] == context.to_dict() + + +# --------------------------------------------------------------------------- +# Async event processor (shares the dispatch logic) +# --------------------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_async_processor_suppresses_individual_events_for_override_affected_evaluations(): + flag = tracked_flag() + mock_http = MockAioHttp() + ep = DefaultAsyncEventProcessor(AsyncConfig('SDK_KEY', diagnostic_opt_out=True), mock_http) + try: + ep.send_event(evaluation(flag, override_affected=True)) + ep.send_event(evaluation(flag)) + assert await ep.flush_and_wait(5) is True + output = json.loads(mock_http.request_data) + finally: + await ep.stop() + + assert sorted(e['kind'] for e in output) == ['debug', 'feature', 'index', 'summary'] + counters = events_of_kind(output, 'summary')[0]['features'][flag.key]['counters'] + assert sorted(counters, key=lambda c: 'overrideAffected' in c) == [ + {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version}, + {'count': 1, 'value': 'value', 'variation': 1, 'version': flag.version, 'overrideAffected': True}, + ] diff --git a/ldclient/testing/impl/events/test_event_summarizer.py b/ldclient/testing/impl/events/test_event_summarizer.py index d7cf4ad1..3b2a2f82 100644 --- a/ldclient/testing/impl/events/test_event_summarizer.py +++ b/ldclient/testing/impl/events/test_event_summarizer.py @@ -41,8 +41,8 @@ def test_summarize_event_increments_counters(): data = es.snapshot() expected = { - 'flag1': EventSummaryFlag({'user'}, 'default1', {(1, flag1.version): EventSummaryCounter(2, 'value1'), (2, flag1.version): EventSummaryCounter(1, 'value2')}), - 'flag2': EventSummaryFlag({'user'}, 'default2', {(1, flag2.version): EventSummaryCounter(1, 'value99')}), - 'badkey': EventSummaryFlag({'user'}, 'default3', {(None, None): EventSummaryCounter(1, 'default3')}), + 'flag1': EventSummaryFlag({'user'}, 'default1', {(1, flag1.version, False): EventSummaryCounter(2, 'value1'), (2, flag1.version, False): EventSummaryCounter(1, 'value2')}), + 'flag2': EventSummaryFlag({'user'}, 'default2', {(1, flag2.version, False): EventSummaryCounter(1, 'value99')}), + 'badkey': EventSummaryFlag({'user'}, 'default3', {(None, None, False): EventSummaryCounter(1, 'default3')}), } assert data.flags == expected diff --git a/ldclient/testing/integrations/test_file_override_source.py b/ldclient/testing/integrations/test_file_override_source.py new file mode 100644 index 00000000..abfff9b8 --- /dev/null +++ b/ldclient/testing/integrations/test_file_override_source.py @@ -0,0 +1,732 @@ +""" +Tests for the file-based override source: its builder, initial load, multi-file merging, +absent files, change detection in both modes, failure retention, and its Info log. +""" +import logging +import ntpath +import os +import threading +import time +from queue import Empty, Queue +from typing import Any, Dict, List, Optional, Set, Tuple + +import pytest + +from ldclient.client import Config, Context, LDClient +from ldclient.config import Config as SDKConfig +from ldclient.datasystem import custom +from ldclient.impl.integrations.files import filedata +from ldclient.impl.integrations.overrides import file_override_source +from ldclient.impl.integrations.overrides.file_override_source import ( + _FileOverrideSource +) +from ldclient.integrations.overrides import ( + ChangeDetection, + DuplicateKeysHandling, + FileOverrideSourceBuilder +) +from ldclient.testing.file_util import rewrite_in_place, truncate +from ldclient.testing.mock_components import HangingSynchronizer +from ldclient.testing.stub_util import MockEventProcessor + +TEST_TIMEOUT = 10.0 +QUIET_PERIOD = 0.3 + +watchdog_required = pytest.mark.skipif(not filedata.have_watchdog, reason="watchdog is not installed") +yaml_required = pytest.mark.skipif(not filedata.have_yaml, reason="pyyaml is not installed") + +Snapshot = Tuple[Dict[str, Any], Dict[str, Any]] + + +class CapturingSink: + """Records every override snapshot it receives.""" + + def __init__(self): + self.snapshots: Queue = Queue() + + def set_overrides(self, flags, segments) -> None: + self.snapshots.put((dict(flags), dict(segments))) + + def require_snapshot(self, timeout: float = TEST_TIMEOUT) -> Snapshot: + try: + return self.snapshots.get(timeout=timeout) + except Empty: + pytest.fail("timed out waiting for an override snapshot") + + def require_no_snapshot(self, duration: float = QUIET_PERIOD) -> None: + try: + snapshot = self.snapshots.get(timeout=duration) + except Empty: + return + pytest.fail("received an unexpected override snapshot: %r" % (snapshot,)) + + +def write_file(path: str, content: str) -> None: + with open(path, 'w') as f: + f.write(content) + + +def flag_values(snapshot: Snapshot) -> Dict[str, Any]: + """The single variation of each flag in a snapshot, keyed by flag key.""" + return {key: flag.variations[0] for key, flag in snapshot[0].items()} + + +@pytest.fixture +def sources(): + started: List[_FileOverrideSource] = [] + yield started + for source in started: + source.close() + + +def build_source(sources, paths, configure=None) -> Tuple[_FileOverrideSource, CapturingSink]: + builder = FileOverrideSourceBuilder(paths) + if configure is not None: + configure(builder) + source = builder.build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + sink = CapturingSink() + source.start(sink) + sources.append(source) + return source, sink + + +def info_lines(caplog) -> List[str]: + return [r.getMessage() for r in caplog.records if r.levelno == logging.INFO] + + +def require_info_line(caplog, expected: str) -> None: + """Waits for an Info line equal to the expected text. The source logs after it hands the snapshot to the sink.""" + deadline = time.time() + TEST_TIMEOUT + while True: + if expected in info_lines(caplog): + return + if time.time() > deadline: + pytest.fail("timed out waiting for the Info line %r; Info output: %r" % (expected, info_lines(caplog))) + time.sleep(0.01) + + +# --------------------------------------------------------------------------- +# Builder +# --------------------------------------------------------------------------- + +def test_builder_requires_paths(): + with pytest.raises(ValueError): + FileOverrideSourceBuilder([]).build(SDKConfig('SDK_KEY')) + + +def test_builder_accepts_a_single_path_string(tmp_path): + path = os.path.join(str(tmp_path), 'overrides.json') + source = FileOverrideSourceBuilder(path).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [path] + + +def test_builder_resolves_relative_paths(): + source = FileOverrideSourceBuilder(['relative/overrides.json']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [os.path.abspath('relative/overrides.json')] + + +def test_builder_drops_a_repeated_path_and_warns(caplog): + # Under the default fail handling the same file listed twice would make every one of its keys + # a duplicate of itself. The repeat is ignored and the first occurrence keeps its place. + with caplog.at_level(logging.WARNING): + source = FileOverrideSourceBuilder(['/a.json', '/b.json', '/a.json']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [os.path.abspath('/a.json'), os.path.abspath('/b.json')] + assert any('more than once' in r.getMessage() for r in caplog.records) + + +def test_builder_treats_a_path_repeated_in_different_case_as_a_repeat_only_where_file_names_ignore_case(): + source = FileOverrideSourceBuilder(['/a.json', '/A.json']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + if os.name == 'nt': + assert source._paths == [os.path.abspath('/a.json')] + else: + assert source._paths == [os.path.abspath('/a.json'), os.path.abspath('/A.json')] + + +def test_builder_drops_a_path_repeated_in_different_case_on_windows(monkeypatch): + # Windows file names ignore case, so two spellings of one path name the same file. The path + # comparison of Windows is used here on any platform. + monkeypatch.setattr(os.path, 'normcase', ntpath.normcase) + source = FileOverrideSourceBuilder(['/a.json', '/b.json', '/A.json']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._paths == [os.path.abspath('/a.json'), os.path.abspath('/b.json')] + + +def test_builder_polls_by_default(tmp_path): + path = os.path.join(str(tmp_path), 'overrides.json') + source = FileOverrideSourceBuilder([path]).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._change_detection == ChangeDetection.POLLING + assert source._poll_interval == FileOverrideSourceBuilder.DEFAULT_POLL_INTERVAL + assert source._duplicate_keys_handling == DuplicateKeysHandling.FAIL + + +def test_builder_rejects_unknown_change_detection(): + with pytest.raises(ValueError) as excinfo: + FileOverrideSourceBuilder(['a']).change_detection('notify') + assert 'notify' in str(excinfo.value) + + +def test_builder_rejects_unknown_duplicate_keys_handling(): + with pytest.raises(ValueError): + FileOverrideSourceBuilder(['a']).duplicate_keys_handling('merge') + + +def test_builder_accepts_string_values(): + builder = FileOverrideSourceBuilder(['a']).change_detection('polling').duplicate_keys_handling('ignore') + source = builder.build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._change_detection == ChangeDetection.POLLING + assert source._duplicate_keys_handling == DuplicateKeysHandling.IGNORE + + +def test_builder_clamps_poll_interval_to_the_minimum(caplog): + with caplog.at_level(logging.WARNING): + source = FileOverrideSourceBuilder(['a']).poll_interval(0.001).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._poll_interval == FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL + assert any('below the minimum' in r.getMessage() for r in caplog.records) + + source = FileOverrideSourceBuilder(['a']).poll_interval(2.5).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + assert source._poll_interval == 2.5 + + +@pytest.mark.parametrize('seconds', ['1', None, True], ids=['string', 'none', 'bool']) +def test_builder_rejects_a_poll_interval_that_is_not_a_number(seconds): + with pytest.raises(TypeError): + FileOverrideSourceBuilder(['a']).poll_interval(seconds) + + +def test_builder_rejects_a_poll_interval_longer_than_the_runtime_can_wait(): + with pytest.raises(ValueError) as excinfo: + FileOverrideSourceBuilder(['a']).poll_interval(1e20) + assert 'must not exceed' in str(excinfo.value) + + +@pytest.mark.parametrize('seconds', [float('nan'), float('inf'), float('-inf')], ids=['nan', 'inf', '-inf']) +def test_builder_rejects_a_poll_interval_that_is_not_finite(seconds): + with pytest.raises(ValueError): + FileOverrideSourceBuilder(['a']).poll_interval(seconds) + + +def test_builder_rejects_watching_without_watchdog(monkeypatch): + import ldclient.integrations.overrides as module + monkeypatch.setattr(module, 'have_watchdog', False) + with pytest.raises(ValueError) as excinfo: + FileOverrideSourceBuilder(['a']).change_detection(ChangeDetection.WATCHING).build(SDKConfig('SDK_KEY')) + assert 'watchdog' in str(excinfo.value) + + +@pytest.mark.parametrize('name', ['overrides.yaml', 'overrides.YML']) +def test_builder_rejects_a_yaml_file_without_pyyaml(monkeypatch, name): + # A YAML file can never load without a YAML parser, so it is a configuration error rather + # than a failure that every reload would report. + import ldclient.integrations.overrides as module + monkeypatch.setattr(module, 'have_yaml', False) + with pytest.raises(ValueError) as excinfo: + FileOverrideSourceBuilder(['/overrides.json', '/' + name]).build(SDKConfig('SDK_KEY')) + assert 'pyyaml' in str(excinfo.value) + assert name in str(excinfo.value) + + +def test_builder_accepts_json_files_without_pyyaml(monkeypatch): + import ldclient.integrations.overrides as module + monkeypatch.setattr(module, 'have_yaml', False) + source = FileOverrideSourceBuilder(['/overrides.json', '/overrides']).build(SDKConfig('SDK_KEY')) + assert isinstance(source, _FileOverrideSource) + + +# --------------------------------------------------------------------------- +# Loading +# --------------------------------------------------------------------------- + +def test_source_loads_initial_data_synchronously(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true, "flag2": "x"}, "segments": {"seg": {"key": "seg"}}}') + source, sink = build_source(sources, [path]) + # The snapshot was delivered before start returned. + snapshot = sink.snapshots.get_nowait() + assert flag_values(snapshot) == {'flag1': True, 'flag2': 'x'} + assert list(snapshot[1].keys()) == ['seg'] + assert snapshot[0]['flag1'].is_override is False, "the source supplies unmarked definitions; the SDK marks them" + + +@yaml_required +def test_source_loads_yaml(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.yaml') + write_file(path, 'flagValues:\n yaml-flag: "override-value"\n') + _, sink = build_source(sources, [path]) + assert flag_values(sink.require_snapshot()) == {'yaml-flag': 'override-value'} + + +def test_source_merges_files_in_configured_order(tmp_path, sources): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"shared": "first", "only-first": 1}}') + write_file(second, '{"flagValues": {"shared": "second", "only-second": 2}}') + _, sink = build_source(sources, [first, second], lambda b: b.duplicate_keys_handling(DuplicateKeysHandling.IGNORE)) + assert flag_values(sink.require_snapshot()) == {'shared': 'first', 'only-first': 1, 'only-second': 2} + + +def test_source_duplicate_keys_fail_by_default(tmp_path, sources, caplog): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"shared": "first"}}') + write_file(second, '{"flagValues": {"shared": "second"}}') + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [first, second]) + # The load failed, so no snapshot was supplied and the failure was logged. + sink.require_no_snapshot() + assert any("is specified by multiple files" in r.getMessage() for r in caplog.records) + + +def test_source_starts_with_missing_file(tmp_path, sources): + path = os.path.join(str(tmp_path), 'not-yet.json') + _, sink = build_source(sources, [path], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + # A missing file contributes no overrides. The initial snapshot is empty. + assert sink.require_snapshot() == ({}, {}) + # Once the file appears, the change detection picks it up. + write_file(path, '{"flagValues": {"flag1": true}}') + assert flag_values(sink.require_snapshot()) == {'flag1': True} + + +def test_source_missing_file_contributes_no_entries(tmp_path, sources): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"from-first": true}}') + + # Step 1: one configured file exists and one does not. The existing file applies. + _, sink = build_source(sources, [first, second], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert flag_values(sink.require_snapshot()) == {'from-first': True} + + # Step 2: the second file appears. Both apply. + write_file(second, '{"flagValues": {"from-second": true}}') + assert flag_values(sink.require_snapshot()) == {'from-first': True, 'from-second': True} + + # Step 3: the second file is deleted. Its overrides are removed. + os.remove(second) + assert flag_values(sink.require_snapshot()) == {'from-first': True} + + # Step 4: the last file is deleted. The layer is cleared. + os.remove(first) + assert sink.require_snapshot() == ({}, {}) + + +def test_source_logs_overrides_in_effect_on_each_change(tmp_path, sources, caplog): + first = os.path.join(str(tmp_path), 'first.json') + second = os.path.join(str(tmp_path), 'second.json') + write_file(first, '{"flagValues": {"flag1": true, "flag2": false}, "segments": {"seg": {"key": "seg"}}}') + + with caplog.at_level(logging.INFO): + # Step 1: at startup, one file supplies entries and the other is absent. + _, sink = build_source(sources, [first, second], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + sink.require_snapshot() + require_info_line(caplog, "Flag overrides in effect: 2 flags, 1 segment (%s: 2 flags, 1 segment; %s: absent)" % (first, second)) + + # Step 2: the absent file appears with one entry. + write_file(second, '{"flagValues": {"flag3": true}}') + sink.require_snapshot() + require_info_line(caplog, "Flag overrides in effect: 3 flags, 1 segment (%s: 2 flags, 1 segment; %s: 1 flag)" % (first, second)) + + # Step 3: both files are deleted. Nothing is in effect. + os.remove(first) + os.remove(second) + sink.require_snapshot() + require_info_line(caplog, "Flag overrides: none in effect (%s: absent; %s: absent)" % (first, second)) + + +def test_source_logs_none_in_effect_at_startup_without_files(tmp_path, sources, caplog): + path = os.path.join(str(tmp_path), 'overrides.json') + with caplog.at_level(logging.INFO): + _, sink = build_source(sources, [path]) + sink.require_snapshot() + require_info_line(caplog, "Flag overrides: none in effect (%s: absent)" % path) + + +def test_source_logs_a_present_file_with_no_entries(tmp_path, sources, caplog): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{}') + with caplog.at_level(logging.INFO): + _, sink = build_source(sources, [path]) + sink.require_snapshot() + require_info_line(caplog, "Flag overrides: none in effect (%s: no entries)" % path) + + +def wait_for_log(caplog, text: str) -> None: + deadline = time.time() + TEST_TIMEOUT + while not any(text in r.getMessage() for r in caplog.records): + assert time.time() < deadline, "timed out waiting for a log line containing %r" % text + time.sleep(0.01) + + +def test_source_keeps_the_overrides_when_a_json_file_is_emptied(tmp_path, sources, caplog): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [path], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert flag_values(sink.require_snapshot()) == {'flag1': True} + + # An empty JSON file is not a document. The overrides stay in effect and, once the file + # has been read again, the failure is logged. + truncate(path) + wait_for_log(caplog, 'no content') + sink.require_no_snapshot() + + # A document with no entries removes the overrides. + write_file(path, '{}') + assert sink.require_snapshot() == ({}, {}) + + +@yaml_required +@watchdog_required +def test_source_removes_the_overrides_of_an_emptied_yaml_file_after_the_retry_delay(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.yaml') + write_file(path, 'flagValues:\n flag1: true\n') + _, sink = build_source(sources, [path], lambda b: b.change_detection(ChangeDetection.WATCHING)) + assert flag_values(sink.require_snapshot()) == {'flag1': True} + + # An empty YAML file is a document with no entries. The watch notices the truncation at + # once, and the file is read again after the retry delay before it is applied, in case it + # is in the middle of an in-place rewrite. + emptied = time.monotonic() + truncate(path) + assert sink.require_snapshot() == ({}, {}) + assert time.monotonic() - emptied >= filedata.DEFAULT_RETRY_DELAY + + +# --------------------------------------------------------------------------- +# Change detection +# --------------------------------------------------------------------------- + +@watchdog_required +def test_watching_mode_is_quiet_when_file_is_absent(tmp_path, sources): + path = os.path.join(str(tmp_path), 'not-yet.json') + _, sink = build_source(sources, [path], lambda b: b.change_detection(ChangeDetection.WATCHING)) + assert sink.require_snapshot() == ({}, {}) + sink.require_no_snapshot() + write_file(path, '{"flagValues": {"flag1": true}}') + assert flag_values(sink.require_snapshot()) == {'flag1': True} + + +@watchdog_required +def test_watching_mode_reloads_on_change(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + source, sink = build_source(sources, [path], lambda b: b.change_detection(ChangeDetection.WATCHING)) + assert isinstance(source._change_detector, filedata.Watcher) + assert flag_values(sink.require_snapshot()) == {'flag1': True} + write_file(path, '{"flagValues": {"flag1": false}}') + assert flag_values(sink.require_snapshot()) == {'flag1': False} + + +def test_polling_mode_reloads_on_change(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + source, sink = build_source(sources, [path], lambda b: b.change_detection(ChangeDetection.POLLING).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert isinstance(source._change_detector, filedata.Poller) + assert flag_values(sink.require_snapshot()) == {'flag1': True} + write_file(path, '{"flagValues": {"flag1": false}}') + assert flag_values(sink.require_snapshot()) == {'flag1': False} + + +@pytest.mark.parametrize("mode", [ChangeDetection.POLLING, pytest.param(ChangeDetection.WATCHING, marks=watchdog_required)]) +def test_source_retains_last_good_data_across_malformed_edit(tmp_path, sources, caplog, mode): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [path], lambda b: b.change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + sink.require_snapshot() + + # A malformed edit produces no snapshot: the previously applied overrides stay in + # effect because the sink is never called. The failure is logged. + write_file(path, '{"flagValues"') + sink.require_no_snapshot(1.5) + deadline = time.time() + TEST_TIMEOUT + while not any('Unable to load flag data' in r.getMessage() for r in caplog.records): + assert time.time() < deadline, "the load failure was not logged" + time.sleep(0.01) + + # Fixing the file recovers, through the change notification or the failure retry. + write_file(path, '{"flagValues": {"flag1": false}}') + assert flag_values(sink.require_snapshot()) == {'flag1': False} + + +def test_source_retries_a_failed_load_without_a_change_signal(tmp_path, sources, caplog): + # The fixed content has the same size as the malformed content and the file keeps its + # modification time, so the poller sees no change. Only the automatic retry can recover. + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + with caplog.at_level(logging.ERROR): + _, sink = build_source(sources, [path], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + sink.require_snapshot() + + malformed = '{"flagValues": {"flag1": false}' + fixed = '{"flagValues": {"flag1":false}}' + assert len(malformed) == len(fixed) + write_file(path, malformed) + deadline = time.time() + TEST_TIMEOUT + while not any('Unable to load flag data' in r.getMessage() for r in caplog.records): + assert time.time() < deadline, "the load failure was not logged" + time.sleep(0.01) + observed = os.stat(path) + write_file(path, fixed) + os.utime(path, ns=(observed.st_atime_ns, observed.st_mtime_ns)) + assert os.stat(path).st_size == observed.st_size + assert flag_values(sink.require_snapshot()) == {'flag1': False} + + +@pytest.mark.parametrize("mode", [ChangeDetection.POLLING, pytest.param(ChangeDetection.WATCHING, marks=watchdog_required)]) +def test_source_detects_an_edit_made_during_the_initial_load(tmp_path, sources, monkeypatch, mode): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": "initial"}}') + original_reload_now = filedata.Reloader.reload_now + + def load_then_edit(reloader: filedata.Reloader) -> None: + # The edit lands after the files were read and before start returns. + original_reload_now(reloader) + write_file(path, '{"flagValues": {"flag1": "edited-after-the-first-read"}}') + + monkeypatch.setattr(filedata.Reloader, 'reload_now', load_then_edit) + _, sink = build_source(sources, [path], lambda b: b.change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + + # The initial snapshot holds what the first read saw. The edit is detected and applied + # without any further change to the file. + assert flag_values(sink.require_snapshot()) == {'flag1': 'initial'} + assert flag_values(sink.require_snapshot()) == {'flag1': 'edited-after-the-first-read'} + + +# --------------------------------------------------------------------------- +# In-place rewrites +# --------------------------------------------------------------------------- + +IN_PLACE_REWRITE_CASES = [ + pytest.param('json', ChangeDetection.POLLING, id='json-polling'), + pytest.param('yaml', ChangeDetection.POLLING, marks=yaml_required, id='yaml-polling'), + pytest.param('json', ChangeDetection.WATCHING, marks=watchdog_required, id='json-watching'), + pytest.param('yaml', ChangeDetection.WATCHING, marks=[watchdog_required, yaml_required], id='yaml-watching'), +] + + +def override_document(file_format: str, value: str) -> str: + if file_format == 'json': + return '{"flagValues": {"flag1": "%s"}}' % value + return 'flagValues:\n flag1: "%s"\n' % value + + +def configure_rewrite_timing(monkeypatch, mode: ChangeDetection) -> float: + """ + Returns the pause of a slow in-place rewrite and sets a retry delay longer than it, so that + the rewrite completes before the retry, even on a slow machine. A watch notices the + truncation at once. A poll notices it at the next examination, so in polling mode the pause + is longer than the poll interval, which makes a poll find the file empty. + """ + if mode == ChangeDetection.WATCHING: + pause, retry_delay = 0.3, 1.5 + else: + pause, retry_delay = FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL + 0.6, 3.0 + monkeypatch.setattr(file_override_source, 'DEFAULT_RETRY_DELAY', retry_delay) + return pause + + +@pytest.mark.parametrize('file_format, mode', IN_PLACE_REWRITE_CASES) +def test_source_never_supplies_a_snapshot_without_the_override_during_an_in_place_rewrite(tmp_path, sources, monkeypatch, file_format, mode): + pause = configure_rewrite_timing(monkeypatch, mode) + path = os.path.join(str(tmp_path), 'overrides.' + file_format) + write_file(path, override_document(file_format, 'a')) + _, sink = build_source(sources, [path], lambda b: b.change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert flag_values(sink.require_snapshot()) == {'flag1': 'a'} + + # The file is empty for the whole pause, and a load finds it empty. The next snapshot holds + # the rewritten override, and the retry that the empty file armed supplies nothing more. + rewrite_in_place(path, override_document(file_format, 'b'), pause) + assert flag_values(sink.require_snapshot()) == {'flag1': 'b'} + sink.require_no_snapshot(file_override_source.DEFAULT_RETRY_DELAY) + + +@pytest.mark.parametrize('file_format, mode', IN_PLACE_REWRITE_CASES) +def test_source_never_supplies_a_snapshot_without_the_override_during_successive_in_place_rewrites(tmp_path, sources, file_format, mode): + # The source's own timing: the default retry delay and a one-second poll interval. An + # operator rewrites the file several times in a row, in place. A watch sees each write and + # the next truncation together; a poll can see a truncation without the content written + # before it. No snapshot may lack the override. + path = os.path.join(str(tmp_path), 'overrides.' + file_format) + write_file(path, override_document(file_format, 'a')) + _, sink = build_source(sources, [path], lambda b: b.change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + assert flag_values(sink.require_snapshot()) == {'flag1': 'a'} + + values = ['b', 'c', 'd', 'e'] + for value in values: + rewrite_in_place(path, override_document(file_format, value), 0.3) + if mode == ChangeDetection.POLLING: + # The rewrites start about a second apart. + time.sleep(0.7) + + supplied: List[Dict[str, Any]] = [] + while len(supplied) == 0 or supplied[-1] != {'flag1': values[-1]}: + supplied.append(flag_values(sink.require_snapshot())) + assert all(list(snapshot.keys()) == ['flag1'] for snapshot in supplied), supplied + sink.require_no_snapshot(filedata.DEFAULT_RETRY_DELAY + 0.5) + + +@pytest.mark.parametrize('file_format, mode', IN_PLACE_REWRITE_CASES) +def test_client_serves_the_override_throughout_an_in_place_rewrite(tmp_path, monkeypatch, file_format, mode): + # An operator changes an override by rewriting its file in place. A client that evaluates the + # flag throughout serves the old value and then the new one, never the default. + pause = configure_rewrite_timing(monkeypatch, mode) + path = os.path.join(str(tmp_path), 'overrides.' + file_format) + write_file(path, override_document(file_format, 'a')) + source = FileOverrideSourceBuilder([path]).change_detection(mode).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + user = Context.create('user-key') + + with LDClient(config, start_wait=0) as client: + served: List[Any] = [] + stop = threading.Event() + + def sample() -> None: + while not stop.is_set(): + value = client.variation('flag1', user, 'default') + if len(served) == 0 or served[-1] != value: + served.append(value) + time.sleep(0.001) + + sampler = threading.Thread(target=sample, daemon=True) + sampler.start() + try: + rewrite_in_place(path, override_document(file_format, 'b'), pause) + deadline = time.time() + TEST_TIMEOUT + while served[-1] != 'b': + assert time.time() < deadline, "the rewritten override was not served; served %r" % served + time.sleep(0.01) + time.sleep(0.5) + finally: + stop.set() + sampler.join(TEST_TIMEOUT) + assert served == ['a', 'b'] + + +def reloader_threads() -> Set[threading.Thread]: + return {t for t in threading.enumerate() if t.name == 'ldclient.filedata.reloader'} + + +def test_source_close_stops_the_reloader_when_the_change_detector_fails_to_close(tmp_path, sources, monkeypatch): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{}') + before = reloader_threads() + source, sink = build_source(sources, [path]) + sink.require_snapshot() + started = reloader_threads() - before + assert len(started) == 1 + + detector: Any = source._change_detector + original_close = detector.close + + def failing_close() -> None: + original_close() + raise RuntimeError("detector close failure") + + monkeypatch.setattr(detector, 'close', failing_close) + + # The failure propagates, and the reloader is stopped all the same: its worker ends. + with pytest.raises(RuntimeError): + source.close() + worker = started.pop() + worker.join(TEST_TIMEOUT) + assert not worker.is_alive() + + +def test_source_does_not_supply_snapshots_after_close(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + source, sink = build_source(sources, [path], lambda b: b.poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL)) + sink.require_snapshot() + source.close() + write_file(path, '{"flagValues": {"flag1": false}}') + sink.require_no_snapshot(2.5) + + +def test_source_close_is_idempotent(tmp_path, sources): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{}') + source, _ = build_source(sources, [path]) + source.close() + source.close() + + +def test_source_start_after_close_does_nothing(tmp_path): + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"flag1": true}}') + source = FileOverrideSourceBuilder([path]).build(SDKConfig('SDK_KEY')) + source.close() + sink = CapturingSink() + source.start(sink) + sink.require_no_snapshot(0.2) + + +# --------------------------------------------------------------------------- +# Through the client +# --------------------------------------------------------------------------- + +def test_file_overrides_end_to_end(tmp_path): + # An operator writes, edits, and empties an override file. Evaluations follow without any + # client restart, even though the client never obtains data from LaunchDarkly. + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{}') + user = Context.create('user-key') + source = FileOverrideSourceBuilder([path]).poll_interval(FileOverrideSourceBuilder.MINIMUM_POLL_INTERVAL) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + + def eventually(predicate, message: str) -> None: + deadline = time.time() + TEST_TIMEOUT + while not predicate(): + assert time.time() < deadline, message + time.sleep(0.05) + + with LDClient(config, start_wait=0) as client: + # Not initialized and no override present: the default is served. + detail = client.variation_detail('overridden-flag', user, False) + assert detail.value is False + assert detail.reason == {'kind': 'ERROR', 'errorKind': 'CLIENT_NOT_READY'} + + # An operator adds an override. The running client picks it up. + write_file(path, '{"flagValues": {"overridden-flag": true}}') + eventually(lambda: client.variation('overridden-flag', user, False) is True, "the override was not picked up") + + # The override changes value. + write_file(path, '{"flagValues": {"overridden-flag": false}}') + + def changed() -> bool: + detail = client.variation_detail('overridden-flag', user, True) + return detail.value is False and detail.reason.get('overrideAffected') is True + + eventually(changed, "the changed override was not picked up") + + # The override is removed. The not-initialized short-circuit returns. + write_file(path, '{}') + eventually(lambda: client.variation_detail('overridden-flag', user, False).reason.get('errorKind') == 'CLIENT_NOT_READY', "the removed override was not picked up") + + +def test_client_side_all_flags_state_includes_a_full_definition_with_client_side_but_not_a_value_entry(tmp_path): + # A flagValues entry is not available to client-side SDKs. A full definition that sets + # clientSide is, so that is how an operator overrides a flag that client-side SDKs use. + path = os.path.join(str(tmp_path), 'overrides.json') + write_file(path, '{"flagValues": {"value-flag": true}, ' + '"flags": {"full-flag": {"key": "full-flag", "version": 1, "on": true, "variations": [true], ' + '"fallthrough": {"variation": 0}, "clientSide": true}}}') + source = FileOverrideSourceBuilder([path]) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor) + + with LDClient(config, start_wait=0) as client: + user = Context.create('user-key') + assert set(client.all_flags_state(user).to_values_map().keys()) == {'value-flag', 'full-flag'} + state = client.all_flags_state(user, client_side_only=True) + assert state.valid + assert state.to_values_map() == {'full-flag': True} diff --git a/ldclient/testing/test_async_client_override_vectors.py b/ldclient/testing/test_async_client_override_vectors.py new file mode 100644 index 00000000..3eab7e3f --- /dev/null +++ b/ldclient/testing/test_async_client_override_vectors.py @@ -0,0 +1,71 @@ +""" +Runs the OVERRIDE specification test vectors through the async client. The vectors and the +checks are those of the sync runner: each vector sets up LaunchDarkly data, an override layer, +and an initialization state, and the test evaluates one flag through the full client stack. +""" +from typing import Any, Dict + +import pytest + +from ldclient.async_client import AsyncLDClient +from ldclient.async_config import AsyncConfig, AsyncDataSystemConfig +from ldclient.context import Context +from ldclient.impl.events.types import EventInputEvaluation +from ldclient.testing.mock_async_components import MockAsyncEventProcessor +from ldclient.testing.mock_components import ( + MockDataSourceBuilder, + MockOverrideSource +) +from ldclient.testing.test_async_client_overrides import ( + AsyncHangingSynchronizer, + AsyncStaticInitializer +) +from ldclient.testing.test_ldclient_override_vectors import ( + assert_reason, + load_vectors, + override_flags, + vector_id +) + + +async def make_client(vector: Dict[str, Any]) -> AsyncLDClient: + source = MockOverrideSource(flags=override_flags(vector['overrides']), segments=dict(vector['overrides'].get('segments', {}))) + ld_data = vector['launchDarklyData'] + if ld_data['initialized']: + initializer = AsyncStaticInitializer(ld_data.get('flags', {}), ld_data.get('segments', {})) + datasystem = AsyncDataSystemConfig(initializers=[MockDataSourceBuilder(initializer)], override_source=source.builder) + start_wait = 5 + else: + # With no sources at all, the client would consider cached data available rather than + # applying its not-initialized handling. A synchronizer that never delivers avoids that. + datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=source.builder) + start_wait = 0 + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor()) + client = AsyncLDClient(config) + await client.start(start_wait=start_wait) + return client + + +@pytest.mark.asyncio +@pytest.mark.parametrize("vector", load_vectors(), ids=vector_id) +async def test_override_spec_vector(vector: Dict[str, Any]): + client = await make_client(vector) + try: + assert await client.is_initialized() is vector['launchDarklyData']['initialized'] + evaluate = vector['evaluate'] + detail = await client.variation_detail(evaluate['flagKey'], Context.from_dict(evaluate['context']), evaluate['defaultValue']) + + expect = vector['expect'] + assert detail.value == expect['value'], "value" + assert detail.variation_index == expect['variationIndex'], "variationIndex" + assert_reason(expect['reason'], detail.reason) + + # summaryOverrideAffected is the marking the client hands to the event processor for this + # evaluation. The event processor keys individual-event suppression and the summary + # counter marker on that scalar, not on the reason. + if 'summaryOverrideAffected' in expect: + records = [e for e in client._event_processor.events if isinstance(e, EventInputEvaluation) and e.key == evaluate['flagKey']] + assert len(records) == 1, "expected exactly one evaluation record for the flag" + assert records[0].override_affected is expect['summaryOverrideAffected'], "summaryOverrideAffected" + finally: + await client.close() diff --git a/ldclient/testing/test_async_client_overrides.py b/ldclient/testing/test_async_client_overrides.py index 18ed7e0b..8fe69458 100644 --- a/ldclient/testing/test_async_client_overrides.py +++ b/ldclient/testing/test_async_client_overrides.py @@ -3,8 +3,10 @@ client tests. """ import asyncio +import json import logging import threading +import time from typing import Any, Dict, List, Optional import pytest @@ -14,10 +16,17 @@ from ldclient.async_feature_store import AsyncInMemoryFeatureStore from ldclient.context import Context from ldclient.evaluation import EvaluationDetail +from ldclient.hook import AsyncHook, EvaluationSeriesContext, Metadata from ldclient.impl.aio.concurrency import AsyncEvent +from ldclient.impl.events.async_event_processor import ( + DefaultAsyncEventProcessor +) +from ldclient.impl.events.types import EventInputEvaluation from ldclient.impl.integrations.files.filedata import make_flag_with_value from ldclient.interfaces import AsyncFeatureStore, DataStoreMode +from ldclient.migrations import Stage from ldclient.testing.builders import FlagBuilder +from ldclient.testing.impl.events.test_async_event_processor import MockAioHttp from ldclient.testing.mock_async_components import MockAsyncEventProcessor from ldclient.testing.mock_components import ( FailingOverrideSourceBuilder, @@ -439,3 +448,203 @@ async def test_failed_evaluation_reason_is_marked_only_when_the_flag_came_from_t assert state.get_flag_reason('plain-flag') == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} finally: await client.close() + + +@pytest.mark.asyncio +async def test_override_evaluation_events_carry_override_affected_marking(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + client = await make_uninitialized_client(source) + try: + assert await client.variation('overridden-flag', user, False) is True + records = [e for e in client._event_processor.events if isinstance(e, EventInputEvaluation)] + assert [e.key for e in records] == ['overridden-flag'] + assert records[0].override_affected is True + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_turns_off_event_tracking_for_override_affected_flags(): + overridden = FlagBuilder('overridden-flag').version(7).on(False).off_variation(0).variations(True).track_events(True).debug_events_until_date(int(time.time() * 1000) + 100000).build().to_json_dict() + plain = FlagBuilder('plain-tracked').version(1).on(False).off_variation(0).variations(True).track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'overridden-flag': overridden}) + client = await make_initialized_client({'plain-tracked': plain}, source) + try: + state = await client.all_flags_state(user, with_reasons=True) + flags_state = state.to_json_dict()['$flagsState'] + assert flags_state['plain-tracked']['trackEvents'] is True + assert 'trackEvents' not in flags_state['overridden-flag'] + assert 'debugEventsUntilDate' not in flags_state['overridden-flag'] + assert flags_state['overridden-flag']['reason']['overrideAffected'] is True + assert state.get_flag_value('overridden-flag') is True + finally: + await client.close() + + +def test_override_update_after_the_client_and_its_loop_are_gone_is_dropped(): + source = MockOverrideSource() + + async def run_client() -> None: + client = await make_uninitialized_client(source) + client.flag_tracker.add_listener(lambda change: None) + await client.close() + + asyncio.run(run_client()) + # A reload that finishes after the client closed finds the event loop closed. The + # notification has nobody left to reach and is dropped. + source.set_overrides({'overridden-flag': single_value_flag('overridden-flag', True)}, {}) + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +def tracked_bool_flag(key: str) -> FlagBuilder: + return FlagBuilder(key).version(100).variations(False, True).off_variation(0).fallthrough_variation(1).track_events(True) + + +def evaluation_events_by_key(client: AsyncLDClient) -> Dict[str, EventInputEvaluation]: + """The evaluation records the client handed to the event processor, keyed by flag key.""" + processor: Any = client._event_processor + return {event.key: event for event in processor.events if isinstance(event, EventInputEvaluation)} + + +@pytest.mark.asyncio +async def test_overridden_prerequisite_marks_the_dependent_evaluation_records(): + # top-flag (LaunchDarkly) --> mid-flag (LaunchDarkly) --> leaf-flag (overridden) + # --> plain-flag (LaunchDarkly) + # The LaunchDarkly copy of leaf-flag is off, so the chain passes only through the override. + ld_data = { + 'top-flag': tracked_bool_flag('top-flag').on(True).prerequisite('mid-flag', 1).prerequisite('plain-flag', 1).build().to_json_dict(), + 'mid-flag': tracked_bool_flag('mid-flag').on(True).prerequisite('leaf-flag', 1).build().to_json_dict(), + 'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict(), + 'leaf-flag': tracked_bool_flag('leaf-flag').on(False).build().to_json_dict(), + } + source = MockOverrideSource(flags={'leaf-flag': tracked_bool_flag('leaf-flag').on(True).build().to_json_dict()}) + client = await make_initialized_client(ld_data, source) + try: + detail = await client.variation_detail('top-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + + records = evaluation_events_by_key(client) + assert sorted(records.keys()) == ['leaf-flag', 'mid-flag', 'plain-flag', 'top-flag'] + assert records['top-flag'].override_affected is True + assert records['mid-flag'].override_affected is True + assert records['leaf-flag'].override_affected is True + assert records['plain-flag'].override_affected is False + assert records['leaf-flag'].reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + assert records['plain-flag'].reason == {'kind': 'FALLTHROUGH'} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_wrong_type_result_of_overridden_flag_stays_marked(): + details = [] + + class CapturingHook(AsyncHook): + @property + def metadata(self) -> Metadata: + return Metadata(name='capturing-hook') + + async def before_evaluation(self, series_context: EvaluationSeriesContext, data: dict) -> dict: + return data + + async def after_evaluation(self, series_context: EvaluationSeriesContext, data: dict, detail: EvaluationDetail) -> dict: + details.append(detail) + return data + + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', 'not-a-stage')}) + datasystem = AsyncDataSystemConfig(synchronizers=[MockDataSourceBuilder(AsyncHangingSynchronizer())], override_source=source.builder) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, event_processor_class=lambda config: MockAsyncEventProcessor(), hooks=[CapturingHook()]) + client = AsyncLDClient(config) + await client.start(start_wait=0) + try: + stage, _ = await client.migration_variation('overridden-flag', user, Stage.OFF) + assert stage == Stage.OFF + assert len(details) == 1 + assert details[0].value == 'off' + assert details[0].reason == {'kind': 'ERROR', 'errorKind': 'WRONG_TYPE', 'overrideAffected': True} + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_failed_evaluation_is_marked_only_when_the_flag_came_from_the_override_layer(monkeypatch): + source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()}) + client = await make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) + try: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + + # The failure of the override flag is marked. The failure of the ordinary flag is not. + detail = await client.variation_detail('overridden-flag', user, 'default') + assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True}) + detail = await client.variation_detail('plain-flag', user, 'default') + assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}) + + # The marked record produces no individual event. The ordinary record keeps its tracking. + records = evaluation_events_by_key(client) + assert records['overridden-flag'].override_affected is True + assert records['plain-flag'].override_affected is False + assert records['plain-flag'].track_events is True + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_all_flags_state_turns_off_event_tracking_for_an_override_flag_whose_evaluation_fails(monkeypatch): + source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()}) + client = await make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) + try: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + state = await client.all_flags_state(user, with_reasons=True) + + # The failed override flag stays in the state with a marked reason and no tracking + # fields. The failed ordinary flag keeps its tracking fields. + assert state.valid is True + flags_state = state.to_json_dict()['$flagsState'] + assert flags_state['overridden-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True} + assert 'trackEvents' not in flags_state['overridden-flag'] + assert flags_state['plain-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + assert flags_state['plain-flag']['trackEvents'] is True + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_override_affected_evaluations_appear_only_in_summary_output(): + # End to end through the real event processor: the overridden flag requests individual + # feature events and debug events, and an ordinary flag requests feature events. + debug_until = int(time.time() * 1000) + 100000 + overridden = FlagBuilder('flag-tracked-override').version(300).on(False).off_variation(0).variations('override-value').track_events(True).debug_events_until_date(debug_until).build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'flag-tracked-override': overridden}) + mock_http = MockAioHttp() + + initializer = AsyncStaticInitializer({'flag-normal': normal}, {}) + datasystem = AsyncDataSystemConfig(initializers=[MockDataSourceBuilder(initializer)], override_source=source.builder) + config = AsyncConfig('SDK_KEY', datasystem_config=datasystem, diagnostic_opt_out=True, event_processor_class=lambda config: DefaultAsyncEventProcessor(config, mock_http)) + client = AsyncLDClient(config) + await client.start(start_wait=5) + try: + assert await client.is_initialized() is True + for _ in range(2): + assert await client.variation('flag-tracked-override', user, 'default1') == 'override-value' + assert await client.variation('flag-normal', user, 'default2') == 'normal-value' + assert await client._event_processor.flush_and_wait(5) is True + finally: + await client.close() + + assert mock_http.request_data is not None + output = json.loads(mock_http.request_data) + kinds = sorted(e['kind'] for e in output) + assert kinds == ['feature', 'index', 'summary'] + feature = [e for e in output if e['kind'] == 'feature'][0] + assert feature['key'] == 'flag-normal' + summary = [e for e in output if e['kind'] == 'summary'][0] + assert summary['features']['flag-tracked-override']['default'] == 'default1' + assert summary['features']['flag-tracked-override']['counters'] == [ + {'count': 2, 'value': 'override-value', 'variation': 0, 'version': 300, 'overrideAffected': True} + ] + assert summary['features']['flag-normal']['counters'] == [{'count': 1, 'value': 'normal-value', 'variation': 0, 'version': 100}] diff --git a/ldclient/testing/test_ldclient_override_vectors.py b/ldclient/testing/test_ldclient_override_vectors.py index d9533c3d..a55ea87d 100644 --- a/ldclient/testing/test_ldclient_override_vectors.py +++ b/ldclient/testing/test_ldclient_override_vectors.py @@ -1,7 +1,8 @@ """ Runs the OVERRIDE specification test vectors. Each vector sets up LaunchDarkly data, an override layer, and an initialization state. The test evaluates one flag through the full -client stack and checks the value, the variation index, and the reason. +client stack and checks the value, the variation index, the reason, and the marking handed to the +event processor. """ import json import os @@ -11,6 +12,7 @@ from ldclient.client import Config, Context, LDClient from ldclient.datasystem import custom +from ldclient.impl.events.types import EventInputEvaluation from ldclient.impl.integrations.files.filedata import make_flag_with_value from ldclient.testing.mock_components import ( HangingSynchronizer, @@ -83,3 +85,11 @@ def test_override_spec_vector(vector: Dict[str, Any]): assert detail.value == expect['value'], "value" assert detail.variation_index == expect['variationIndex'], "variationIndex" assert_reason(expect['reason'], detail.reason) + + # summaryOverrideAffected is the marking the client hands to the event processor for this + # evaluation. The event processor keys individual-event suppression and the summary + # counter marker on that scalar, not on the reason. + if 'summaryOverrideAffected' in expect: + records = [e for e in client._event_processor._events if isinstance(e, EventInputEvaluation) and e.key == evaluate['flagKey']] + assert len(records) == 1, "expected exactly one evaluation record for the flag" + assert records[0].override_affected is expect['summaryOverrideAffected'], "summaryOverrideAffected" diff --git a/ldclient/testing/test_ldclient_overrides.py b/ldclient/testing/test_ldclient_overrides.py index 6bce387e..cd595f29 100644 --- a/ldclient/testing/test_ldclient_overrides.py +++ b/ldclient/testing/test_ldclient_overrides.py @@ -2,7 +2,9 @@ Tests for flag overrides through the client: the override source lifecycle, the overlay at the store read boundary, the not-initialized gate, the all-flags state, and flag change notifications. """ +import json import logging +import time from queue import Empty, Queue from typing import Any, Dict, List, Optional @@ -12,9 +14,12 @@ from ldclient.datasystem import custom, daemon from ldclient.evaluation import EvaluationDetail from ldclient.feature_store import InMemoryFeatureStore +from ldclient.hook import EvaluationSeriesContext, Hook, Metadata from ldclient.impl.big_segments import BigSegmentStoreManager from ldclient.impl.datasystem.fdv1 import FDv1 from ldclient.impl.datasystem.fdv2 import FDv2 +from ldclient.impl.events.event_processor import DefaultEventProcessor +from ldclient.impl.events.types import EventInputEvaluation from ldclient.impl.integrations.files.filedata import make_flag_with_value from ldclient.integrations.test_datav2 import TestDataV2 from ldclient.interfaces import ( @@ -23,6 +28,7 @@ FeatureStore, FlagChange ) +from ldclient.migrations import Stage from ldclient.testing.builders import ( FlagBuilder, FlagRuleBuilder, @@ -35,7 +41,7 @@ MockOverrideSource, StaticInitializer ) -from ldclient.testing.stub_util import MockEventProcessor +from ldclient.testing.stub_util import MockEventProcessor, MockHttp from ldclient.versioned_data_kind import FEATURES user = Context.create('user-key') @@ -525,3 +531,205 @@ def failing_stop(self): assert stopped == ['data system', 'big segment store manager', 'event processor'] assert warnings_containing(caplog, 'Error stopping the data system after a failed start: stop failed') != [] + + +# --------------------------------------------------------------------------- +# Events +# --------------------------------------------------------------------------- + +def evaluation_events_by_key(client: LDClient) -> Dict[str, EventInputEvaluation]: + """The evaluation records the client handed to the event processor, keyed by flag key.""" + records = {} + processor: Any = client._event_processor + for event in processor._events: + if isinstance(event, EventInputEvaluation): + records[event.key] = event + return records + + +def test_override_evaluation_events_carry_override_affected_marking(): + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', True)}) + with make_uninitialized_client(source) as client: + assert client.variation('overridden-flag', user, False) is True + records = evaluation_events_by_key(client) + assert list(records.keys()) == ['overridden-flag'] + assert records['overridden-flag'].override_affected is True + + +def test_ordinary_evaluation_events_are_not_marked(): + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'other': single_value_flag('other', True)}) + with make_initialized_client({'flag-normal': normal}, source) as client: + client.variation('flag-normal', user, 'default') + records = evaluation_events_by_key(client) + assert records['flag-normal'].override_affected is False + assert records['flag-normal'].track_events is True + + +def tracked_bool_flag(key: str) -> FlagBuilder: + return FlagBuilder(key).version(100).variations(False, True).off_variation(0).fallthrough_variation(1).track_events(True) + + +def test_overridden_prerequisite_marks_the_dependent_evaluation_records(): + # top-flag (LaunchDarkly) --> mid-flag (LaunchDarkly) --> leaf-flag (overridden) + # --> plain-flag (LaunchDarkly) + # The LaunchDarkly copy of leaf-flag is off, so the chain passes only through the override. + ld_data = { + 'top-flag': tracked_bool_flag('top-flag').on(True).prerequisite('mid-flag', 1).prerequisite('plain-flag', 1).build().to_json_dict(), + 'mid-flag': tracked_bool_flag('mid-flag').on(True).prerequisite('leaf-flag', 1).build().to_json_dict(), + 'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict(), + 'leaf-flag': tracked_bool_flag('leaf-flag').on(False).build().to_json_dict(), + } + source = MockOverrideSource(flags={'leaf-flag': tracked_bool_flag('leaf-flag').on(True).build().to_json_dict()}) + with make_initialized_client(ld_data, source) as client: + detail = client.variation_detail('top-flag', user, False) + assert detail.value is True + assert detail.reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + + records = evaluation_events_by_key(client) + assert sorted(records.keys()) == ['leaf-flag', 'mid-flag', 'plain-flag', 'top-flag'] + assert records['top-flag'].override_affected is True + assert records['mid-flag'].override_affected is True + assert records['leaf-flag'].override_affected is True + assert records['plain-flag'].override_affected is False + assert records['mid-flag'].prereq_of is not None and records['mid-flag'].prereq_of.key == 'top-flag' + assert records['leaf-flag'].prereq_of is not None and records['leaf-flag'].prereq_of.key == 'mid-flag' + assert records['leaf-flag'].reason == {'kind': 'FALLTHROUGH', 'overrideAffected': True} + assert records['plain-flag'].reason == {'kind': 'FALLTHROUGH'} + + +def test_all_flags_state_turns_off_event_tracking_for_override_affected_flags(): + debug_until = int(time.time() * 1000) + 100000 + ld_data = { + 'plain-tracked': FlagBuilder('plain-tracked').version(1).on(False).off_variation(0).variations(True).track_events(True).debug_events_until_date(debug_until).build().to_json_dict(), + 'dependent-tracked': FlagBuilder('dependent-tracked').version(1).on(True).variations(False, True).fallthrough_variation(1).prerequisite('overridden-flag', 0).track_events(True).debug_events_until_date(debug_until).build().to_json_dict(), + } + # The overridden flag is on and serves variation 0, so the dependent flag's prerequisite passes. + overridden = FlagBuilder('overridden-flag').version(7).on(True).fallthrough_variation(0).variations(True).track_events(True).debug_events_until_date(debug_until).build().to_json_dict() + source = MockOverrideSource(flags={'overridden-flag': overridden}) + with make_initialized_client(ld_data, source) as client: + state = client.all_flags_state(user, with_reasons=True) + assert state.valid is True + flags_state = state.to_json_dict()['$flagsState'] + + # A flag with no override keeps its tracking fields. + assert flags_state['plain-tracked']['trackEvents'] is True + assert flags_state['plain-tracked']['debugEventsUntilDate'] == debug_until + + # The overridden flag and the flag that depends on it stay in the state with their values + # and marked reasons, but with no tracking fields. + for key in ('overridden-flag', 'dependent-tracked'): + assert flags_state[key]['reason']['overrideAffected'] is True, key + assert 'trackEvents' not in flags_state[key], key + assert 'trackReason' not in flags_state[key], key + assert 'debugEventsUntilDate' not in flags_state[key], key + assert state.get_flag_value('overridden-flag') is True + assert state.get_flag_value('dependent-tracked') is True + assert flags_state['overridden-flag']['version'] == 7 + + +def test_all_flags_state_omits_details_of_override_affected_flags_when_details_only_for_tracked_flags(): + # With details only for tracked flags, an override-affected flag counts as untracked, so its + # version and reason are omitted like any other untracked flag, and its value stays. + overridden = FlagBuilder('overridden-flag').version(7).on(False).off_variation(0).variations(True).track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'overridden-flag': overridden}) + with make_initialized_client({}, source) as client: + state = client.all_flags_state(user, with_reasons=True, details_only_for_tracked_flags=True) + flags_state = state.to_json_dict()['$flagsState'] + assert state.get_flag_value('overridden-flag') is True + assert 'version' not in flags_state['overridden-flag'] + assert 'reason' not in flags_state['overridden-flag'] + + +def test_wrong_type_result_of_overridden_flag_stays_marked(): + details = [] + + class CapturingHook(Hook): + @property + def metadata(self) -> Metadata: + return Metadata(name='capturing-hook') + + def before_evaluation(self, series_context: EvaluationSeriesContext, data: dict) -> dict: + return data + + def after_evaluation(self, series_context: EvaluationSeriesContext, data: dict, detail: EvaluationDetail) -> dict: + details.append(detail) + return data + + source = MockOverrideSource(flags={'overridden-flag': single_value_flag('overridden-flag', 'not-a-stage')}) + datasystem = custom().synchronizers(HangingSynchronizer().builder).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, event_processor_class=MockEventProcessor, hooks=[CapturingHook()]) + with LDClient(config, start_wait=0) as client: + stage, _ = client.migration_variation('overridden-flag', user, Stage.OFF) + assert stage == Stage.OFF + assert len(details) == 1 + assert details[0].value == 'off' + assert details[0].reason == {'kind': 'ERROR', 'errorKind': 'WRONG_TYPE', 'overrideAffected': True} + + +def test_failed_evaluation_is_marked_only_when_the_flag_came_from_the_override_layer(monkeypatch): + source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()}) + with make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) as client: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + + # The failure of the override flag is marked. The failure of the ordinary flag is not. + detail = client.variation_detail('overridden-flag', user, 'default') + assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True}) + detail = client.variation_detail('plain-flag', user, 'default') + assert detail == EvaluationDetail('default', None, {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}) + + # The marked record produces no individual event. The ordinary record keeps its tracking. + records = evaluation_events_by_key(client) + assert records['overridden-flag'].override_affected is True + assert records['plain-flag'].override_affected is False + assert records['plain-flag'].track_events is True + + +def test_all_flags_state_turns_off_event_tracking_for_an_override_flag_whose_evaluation_fails(monkeypatch): + source = MockOverrideSource(flags={'overridden-flag': tracked_bool_flag('overridden-flag').on(True).build().to_json_dict()}) + with make_initialized_client({'plain-flag': tracked_bool_flag('plain-flag').on(True).build().to_json_dict()}, source) as client: + monkeypatch.setattr(client._evaluator, 'evaluate', raise_evaluation_failure) + state = client.all_flags_state(user, with_reasons=True) + + # The failed override flag stays in the state with a marked reason and no tracking + # fields. The failed ordinary flag keeps its tracking fields. + assert state.valid is True + flags_state = state.to_json_dict()['$flagsState'] + assert flags_state['overridden-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION', 'overrideAffected': True} + assert 'trackEvents' not in flags_state['overridden-flag'] + assert flags_state['plain-flag']['reason'] == {'kind': 'ERROR', 'errorKind': 'EXCEPTION'} + assert flags_state['plain-flag']['trackEvents'] is True + + +def test_override_affected_evaluations_appear_only_in_summary_output(): + # End to end through the real event processor: the overridden flag requests individual + # feature events and debug events, and an ordinary flag requests feature events. + debug_until = int(time.time() * 1000) + 100000 + overridden = FlagBuilder('flag-tracked-override').version(300).on(False).off_variation(0).variations('override-value').track_events(True).debug_events_until_date(debug_until).build().to_json_dict() + normal = FlagBuilder('flag-normal').version(100).on(False).off_variation(0).variations('normal-value').track_events(True).build().to_json_dict() + source = MockOverrideSource(flags={'flag-tracked-override': overridden}) + mock_http = MockHttp() + + initializer = StaticInitializer({'flag-normal': normal}, {}) + datasystem = custom().initializers([initializer.builder]).overrides(source.builder).build() + config = Config(sdk_key='SDK_KEY', datasystem_config=datasystem, diagnostic_opt_out=True, event_processor_class=lambda config: DefaultEventProcessor(config, mock_http)) + with LDClient(config, start_wait=5) as client: + assert client.is_initialized() is True + for _ in range(2): + assert client.variation('flag-tracked-override', user, 'default1') == 'override-value' + assert client.variation('flag-normal', user, 'default2') == 'normal-value' + client.flush() + client._event_processor._wait_until_inactive() + + assert mock_http.request_data is not None + output = json.loads(mock_http.request_data) + kinds = sorted(e['kind'] for e in output) + assert kinds == ['feature', 'index', 'summary'] + feature = [e for e in output if e['kind'] == 'feature'][0] + assert feature['key'] == 'flag-normal' + summary = [e for e in output if e['kind'] == 'summary'][0] + assert summary['features']['flag-tracked-override']['default'] == 'default1' + assert summary['features']['flag-tracked-override']['counters'] == [ + {'count': 2, 'value': 'override-value', 'variation': 0, 'version': 300, 'overrideAffected': True} + ] + assert summary['features']['flag-normal']['counters'] == [{'count': 1, 'value': 'normal-value', 'variation': 0, 'version': 100}]