diff --git a/langfuse/api/dataset_run_items/types/create_dataset_run_item_request.py b/langfuse/api/dataset_run_items/types/create_dataset_run_item_request.py index 169888912..20397e162 100644 --- a/langfuse/api/dataset_run_items/types/create_dataset_run_item_request.py +++ b/langfuse/api/dataset_run_items/types/create_dataset_run_item_request.py @@ -41,7 +41,7 @@ class CreateDatasetRunItemRequest(UniversalBaseModel): ] = pydantic.Field(default=None) """ ISO 8601 timestamp (RFC 3339, Section 5.6) in UTC (e.g., "2026-01-21T14:35:42Z"). - Specifies the dataset version to use for this experiment run. + Specifies the dataset version to use for this experiment run. If provided, the experiment will use dataset items as they existed at or before this timestamp. If not provided, uses the latest version of dataset items. """ diff --git a/langfuse/api/unstable/__init__.py b/langfuse/api/unstable/__init__.py index 1b6c232dd..2a249200a 100644 --- a/langfuse/api/unstable/__init__.py +++ b/langfuse/api/unstable/__init__.py @@ -19,68 +19,8 @@ PublicApiValidationIssue, TooManyRequestsError, UnauthorizedError, - UnprocessableContentError, - ) - from . import ( - commons, - dashboard_widgets, - dashboards, - errors, - evaluation_rules, - evaluators, - ) - from .commons import ( - ArrayOptionsEvaluationRuleFilter, - BooleanEvaluationRuleFilter, - CategoryOptionsEvaluationRuleFilter, - CodeEvaluatorSourceCodeLanguage, - DateTimeEvaluationRuleFilter, - EvaluationRuleArrayOptionsFilterOperator, - EvaluationRuleBooleanFilterOperator, - EvaluationRuleFilter, - EvaluationRuleFilter_ArrayOptions, - EvaluationRuleFilter_Boolean, - EvaluationRuleFilter_CategoryOptions, - EvaluationRuleFilter_Datetime, - EvaluationRuleFilter_Null, - EvaluationRuleFilter_Number, - EvaluationRuleFilter_NumberObject, - EvaluationRuleFilter_String, - EvaluationRuleFilter_StringObject, - EvaluationRuleFilter_StringOptions, - EvaluationRuleNullFilterOperator, - EvaluationRuleNumberFilterOperator, - EvaluationRuleOptionsFilterOperator, - EvaluationRuleStatus, - EvaluationRuleStringFilterOperator, - EvaluationRuleTarget, - EvaluatorModelConfig, - EvaluatorOutputDataType, - EvaluatorOutputDefinition, - EvaluatorOutputDefinition_Boolean, - EvaluatorOutputDefinition_Categorical, - EvaluatorOutputDefinition_Numeric, - EvaluatorOutputFieldDefinition, - EvaluatorType, - NullEvaluationRuleFilter, - NumberEvaluationRuleFilter, - NumberObjectEvaluationRuleFilter, - PromptVariableMappingInput, - PromptVariableMappingRead, - PromptVariableMappingSource, - PublicBooleanEvaluatorOutputDefinition, - PublicCategoricalEvaluatorOutputDefinition, - PublicCategoricalEvaluatorOutputScoreDefinition, - PublicEvaluatorOutputDefinition, - PublicEvaluatorOutputDefinition_Boolean, - PublicEvaluatorOutputDefinition_Categorical, - PublicEvaluatorOutputDefinition_Numeric, - PublicNumericEvaluatorOutputDefinition, - PublicNumericEvaluatorOutputScoreDefinition, - StringEvaluationRuleFilter, - StringObjectEvaluationRuleFilter, - StringOptionsEvaluationRuleFilter, ) + from . import dashboard_widgets, dashboards, errors from .dashboard_widgets import ( CreateDashboardWidgetRequest, DashboardWidget, @@ -119,74 +59,15 @@ UpdateDashboardRequest, WidgetPlacement, ) - from .evaluation_rules import ( - CodeEvaluationRuleEvaluatorReference, - CreateCodeEvaluationRuleRequest, - CreateEvaluationRuleEvaluatorAssignment, - CreateEvaluationRuleEvaluatorReference, - CreateEvaluationRuleRequest, - CreateEvaluationRuleWithEvaluatorsRequest, - CreateLlmAsJudgeEvaluationRuleRequest, - DeleteEvaluationRuleResponse, - EvaluationRule, - EvaluationRuleBase, - EvaluationRuleEvaluator, - EvaluationRuleEvaluatorAssignment, - EvaluationRuleEvaluatorReference, - EvaluationRuleTimeScope, - EvaluationRules, - LegacyEvaluationObject, - LegacyEvaluationRule, - LegacyEvaluationRuleEvaluatorAssignment, - LegacyEvaluationRuleTarget, - LegacyPromptVariableMapping, - LlmAsJudgeEvaluationRuleEvaluatorReference, - LlmAsJudgeEvaluatorType, - ReadableEvaluationRule, - ReadableV2EvaluationRule, - UpdateEvaluationRuleRequest, - ) - from .evaluators import ( - CodeEvaluator, - CreateCodeEvaluatorRequest, - CreateEvaluatorRequest, - CreateEvaluatorRequest_Code, - CreateEvaluatorRequest_LlmAsJudge, - CreateLlmAsJudgeEvaluatorRequest, - DeleteEvaluatorResponse, - Evaluator, - EvaluatorBase, - Evaluator_Code, - Evaluator_LlmAsJudge, - Evaluators, - LlmAsJudgeEvaluator, - ) _dynamic_imports: typing.Dict[str, str] = { "AccessDeniedError": ".errors", - "ArrayOptionsEvaluationRuleFilter": ".commons", "BadRequestError": ".errors", - "BooleanEvaluationRuleFilter": ".commons", - "CategoryOptionsEvaluationRuleFilter": ".commons", - "CodeEvaluationRuleEvaluatorReference": ".evaluation_rules", - "CodeEvaluator": ".evaluators", - "CodeEvaluatorSourceCodeLanguage": ".commons", "ConflictError": ".errors", - "CreateCodeEvaluationRuleRequest": ".evaluation_rules", - "CreateCodeEvaluatorRequest": ".evaluators", "CreateDashboardPlacementRequest": ".dashboards", "CreateDashboardPlacementRequest_Preset": ".dashboards", "CreateDashboardPlacementRequest_Widget": ".dashboards", "CreateDashboardRequest": ".dashboards", "CreateDashboardWidgetRequest": ".dashboard_widgets", - "CreateEvaluationRuleEvaluatorAssignment": ".evaluation_rules", - "CreateEvaluationRuleEvaluatorReference": ".evaluation_rules", - "CreateEvaluationRuleRequest": ".evaluation_rules", - "CreateEvaluationRuleWithEvaluatorsRequest": ".evaluation_rules", - "CreateEvaluatorRequest": ".evaluators", - "CreateEvaluatorRequest_Code": ".evaluators", - "CreateEvaluatorRequest_LlmAsJudge": ".evaluators", - "CreateLlmAsJudgeEvaluationRuleRequest": ".evaluation_rules", - "CreateLlmAsJudgeEvaluatorRequest": ".evaluators", "CreatePresetPlacement": ".dashboards", "CreateWidgetPlacement": ".dashboards", "Dashboard": ".dashboards", @@ -208,101 +89,26 @@ "DashboardWidgetSortOrder": ".dashboard_widgets", "DashboardWidgetView": ".dashboard_widgets", "DashboardWidgetViewWithLegacy": ".dashboard_widgets", - "DateTimeEvaluationRuleFilter": ".commons", "DeleteDashboardPlacementResponse": ".dashboards", "DeleteDashboardResponse": ".dashboards", "DeleteDashboardWidgetResponse": ".dashboard_widgets", - "DeleteEvaluationRuleResponse": ".evaluation_rules", - "DeleteEvaluatorResponse": ".evaluators", - "EvaluationRule": ".evaluation_rules", - "EvaluationRuleArrayOptionsFilterOperator": ".commons", - "EvaluationRuleBase": ".evaluation_rules", - "EvaluationRuleBooleanFilterOperator": ".commons", - "EvaluationRuleEvaluator": ".evaluation_rules", - "EvaluationRuleEvaluatorAssignment": ".evaluation_rules", - "EvaluationRuleEvaluatorReference": ".evaluation_rules", - "EvaluationRuleFilter": ".commons", - "EvaluationRuleFilter_ArrayOptions": ".commons", - "EvaluationRuleFilter_Boolean": ".commons", - "EvaluationRuleFilter_CategoryOptions": ".commons", - "EvaluationRuleFilter_Datetime": ".commons", - "EvaluationRuleFilter_Null": ".commons", - "EvaluationRuleFilter_Number": ".commons", - "EvaluationRuleFilter_NumberObject": ".commons", - "EvaluationRuleFilter_String": ".commons", - "EvaluationRuleFilter_StringObject": ".commons", - "EvaluationRuleFilter_StringOptions": ".commons", - "EvaluationRuleNullFilterOperator": ".commons", - "EvaluationRuleNumberFilterOperator": ".commons", - "EvaluationRuleOptionsFilterOperator": ".commons", - "EvaluationRuleStatus": ".commons", - "EvaluationRuleStringFilterOperator": ".commons", - "EvaluationRuleTarget": ".commons", - "EvaluationRuleTimeScope": ".evaluation_rules", - "EvaluationRules": ".evaluation_rules", - "Evaluator": ".evaluators", - "EvaluatorBase": ".evaluators", - "EvaluatorModelConfig": ".commons", - "EvaluatorOutputDataType": ".commons", - "EvaluatorOutputDefinition": ".commons", - "EvaluatorOutputDefinition_Boolean": ".commons", - "EvaluatorOutputDefinition_Categorical": ".commons", - "EvaluatorOutputDefinition_Numeric": ".commons", - "EvaluatorOutputFieldDefinition": ".commons", - "EvaluatorType": ".commons", - "Evaluator_Code": ".evaluators", - "Evaluator_LlmAsJudge": ".evaluators", - "Evaluators": ".evaluators", "InternalServerError": ".errors", - "LegacyEvaluationObject": ".evaluation_rules", - "LegacyEvaluationRule": ".evaluation_rules", - "LegacyEvaluationRuleEvaluatorAssignment": ".evaluation_rules", - "LegacyEvaluationRuleTarget": ".evaluation_rules", - "LegacyPromptVariableMapping": ".evaluation_rules", - "LlmAsJudgeEvaluationRuleEvaluatorReference": ".evaluation_rules", - "LlmAsJudgeEvaluator": ".evaluators", - "LlmAsJudgeEvaluatorType": ".evaluation_rules", "MethodNotAllowedError": ".errors", "NotFoundError": ".errors", - "NullEvaluationRuleFilter": ".commons", - "NumberEvaluationRuleFilter": ".commons", - "NumberObjectEvaluationRuleFilter": ".commons", "PresetPlacement": ".dashboards", - "PromptVariableMappingInput": ".commons", - "PromptVariableMappingRead": ".commons", - "PromptVariableMappingSource": ".commons", "PublicApiError": ".errors", "PublicApiErrorCode": ".errors", "PublicApiErrorDetails": ".errors", "PublicApiValidationIssue": ".errors", - "PublicBooleanEvaluatorOutputDefinition": ".commons", - "PublicCategoricalEvaluatorOutputDefinition": ".commons", - "PublicCategoricalEvaluatorOutputScoreDefinition": ".commons", - "PublicEvaluatorOutputDefinition": ".commons", - "PublicEvaluatorOutputDefinition_Boolean": ".commons", - "PublicEvaluatorOutputDefinition_Categorical": ".commons", - "PublicEvaluatorOutputDefinition_Numeric": ".commons", - "PublicNumericEvaluatorOutputDefinition": ".commons", - "PublicNumericEvaluatorOutputScoreDefinition": ".commons", - "ReadableEvaluationRule": ".evaluation_rules", - "ReadableV2EvaluationRule": ".evaluation_rules", - "StringEvaluationRuleFilter": ".commons", - "StringObjectEvaluationRuleFilter": ".commons", - "StringOptionsEvaluationRuleFilter": ".commons", "TooManyRequestsError": ".errors", "UnauthorizedError": ".errors", - "UnprocessableContentError": ".errors", "UpdateDashboardPlacementRequest": ".dashboards", "UpdateDashboardRequest": ".dashboards", "UpdateDashboardWidgetRequest": ".dashboard_widgets", - "UpdateEvaluationRuleRequest": ".evaluation_rules", "WidgetPlacement": ".dashboards", - "commons": ".commons", "dashboard_widgets": ".dashboard_widgets", "dashboards": ".dashboards", "errors": ".errors", - "evaluation_rules": ".evaluation_rules", - "evaluators": ".evaluators", } @@ -335,30 +141,13 @@ def __dir__(): __all__ = [ "AccessDeniedError", - "ArrayOptionsEvaluationRuleFilter", "BadRequestError", - "BooleanEvaluationRuleFilter", - "CategoryOptionsEvaluationRuleFilter", - "CodeEvaluationRuleEvaluatorReference", - "CodeEvaluator", - "CodeEvaluatorSourceCodeLanguage", "ConflictError", - "CreateCodeEvaluationRuleRequest", - "CreateCodeEvaluatorRequest", "CreateDashboardPlacementRequest", "CreateDashboardPlacementRequest_Preset", "CreateDashboardPlacementRequest_Widget", "CreateDashboardRequest", "CreateDashboardWidgetRequest", - "CreateEvaluationRuleEvaluatorAssignment", - "CreateEvaluationRuleEvaluatorReference", - "CreateEvaluationRuleRequest", - "CreateEvaluationRuleWithEvaluatorsRequest", - "CreateEvaluatorRequest", - "CreateEvaluatorRequest_Code", - "CreateEvaluatorRequest_LlmAsJudge", - "CreateLlmAsJudgeEvaluationRuleRequest", - "CreateLlmAsJudgeEvaluatorRequest", "CreatePresetPlacement", "CreateWidgetPlacement", "Dashboard", @@ -380,99 +169,24 @@ def __dir__(): "DashboardWidgetSortOrder", "DashboardWidgetView", "DashboardWidgetViewWithLegacy", - "DateTimeEvaluationRuleFilter", "DeleteDashboardPlacementResponse", "DeleteDashboardResponse", "DeleteDashboardWidgetResponse", - "DeleteEvaluationRuleResponse", - "DeleteEvaluatorResponse", - "EvaluationRule", - "EvaluationRuleArrayOptionsFilterOperator", - "EvaluationRuleBase", - "EvaluationRuleBooleanFilterOperator", - "EvaluationRuleEvaluator", - "EvaluationRuleEvaluatorAssignment", - "EvaluationRuleEvaluatorReference", - "EvaluationRuleFilter", - "EvaluationRuleFilter_ArrayOptions", - "EvaluationRuleFilter_Boolean", - "EvaluationRuleFilter_CategoryOptions", - "EvaluationRuleFilter_Datetime", - "EvaluationRuleFilter_Null", - "EvaluationRuleFilter_Number", - "EvaluationRuleFilter_NumberObject", - "EvaluationRuleFilter_String", - "EvaluationRuleFilter_StringObject", - "EvaluationRuleFilter_StringOptions", - "EvaluationRuleNullFilterOperator", - "EvaluationRuleNumberFilterOperator", - "EvaluationRuleOptionsFilterOperator", - "EvaluationRuleStatus", - "EvaluationRuleStringFilterOperator", - "EvaluationRuleTarget", - "EvaluationRuleTimeScope", - "EvaluationRules", - "Evaluator", - "EvaluatorBase", - "EvaluatorModelConfig", - "EvaluatorOutputDataType", - "EvaluatorOutputDefinition", - "EvaluatorOutputDefinition_Boolean", - "EvaluatorOutputDefinition_Categorical", - "EvaluatorOutputDefinition_Numeric", - "EvaluatorOutputFieldDefinition", - "EvaluatorType", - "Evaluator_Code", - "Evaluator_LlmAsJudge", - "Evaluators", "InternalServerError", - "LegacyEvaluationObject", - "LegacyEvaluationRule", - "LegacyEvaluationRuleEvaluatorAssignment", - "LegacyEvaluationRuleTarget", - "LegacyPromptVariableMapping", - "LlmAsJudgeEvaluationRuleEvaluatorReference", - "LlmAsJudgeEvaluator", - "LlmAsJudgeEvaluatorType", "MethodNotAllowedError", "NotFoundError", - "NullEvaluationRuleFilter", - "NumberEvaluationRuleFilter", - "NumberObjectEvaluationRuleFilter", "PresetPlacement", - "PromptVariableMappingInput", - "PromptVariableMappingRead", - "PromptVariableMappingSource", "PublicApiError", "PublicApiErrorCode", "PublicApiErrorDetails", "PublicApiValidationIssue", - "PublicBooleanEvaluatorOutputDefinition", - "PublicCategoricalEvaluatorOutputDefinition", - "PublicCategoricalEvaluatorOutputScoreDefinition", - "PublicEvaluatorOutputDefinition", - "PublicEvaluatorOutputDefinition_Boolean", - "PublicEvaluatorOutputDefinition_Categorical", - "PublicEvaluatorOutputDefinition_Numeric", - "PublicNumericEvaluatorOutputDefinition", - "PublicNumericEvaluatorOutputScoreDefinition", - "ReadableEvaluationRule", - "ReadableV2EvaluationRule", - "StringEvaluationRuleFilter", - "StringObjectEvaluationRuleFilter", - "StringOptionsEvaluationRuleFilter", "TooManyRequestsError", "UnauthorizedError", - "UnprocessableContentError", "UpdateDashboardPlacementRequest", "UpdateDashboardRequest", "UpdateDashboardWidgetRequest", - "UpdateEvaluationRuleRequest", "WidgetPlacement", - "commons", "dashboard_widgets", "dashboards", "errors", - "evaluation_rules", - "evaluators", ] diff --git a/langfuse/api/unstable/client.py b/langfuse/api/unstable/client.py index c948dd77b..23050bc80 100644 --- a/langfuse/api/unstable/client.py +++ b/langfuse/api/unstable/client.py @@ -13,11 +13,6 @@ DashboardWidgetsClient, ) from .dashboards.client import AsyncDashboardsClient, DashboardsClient - from .evaluation_rules.client import ( - AsyncEvaluationRulesClient, - EvaluationRulesClient, - ) - from .evaluators.client import AsyncEvaluatorsClient, EvaluatorsClient class UnstableClient: @@ -26,8 +21,6 @@ def __init__(self, *, client_wrapper: SyncClientWrapper): self._client_wrapper = client_wrapper self._dashboard_widgets: typing.Optional[DashboardWidgetsClient] = None self._dashboards: typing.Optional[DashboardsClient] = None - self._evaluation_rules: typing.Optional[EvaluationRulesClient] = None - self._evaluators: typing.Optional[EvaluatorsClient] = None @property def with_raw_response(self) -> RawUnstableClient: @@ -58,24 +51,6 @@ def dashboards(self): self._dashboards = DashboardsClient(client_wrapper=self._client_wrapper) return self._dashboards - @property - def evaluation_rules(self): - if self._evaluation_rules is None: - from .evaluation_rules.client import EvaluationRulesClient # noqa: E402 - - self._evaluation_rules = EvaluationRulesClient( - client_wrapper=self._client_wrapper - ) - return self._evaluation_rules - - @property - def evaluators(self): - if self._evaluators is None: - from .evaluators.client import EvaluatorsClient # noqa: E402 - - self._evaluators = EvaluatorsClient(client_wrapper=self._client_wrapper) - return self._evaluators - class AsyncUnstableClient: def __init__(self, *, client_wrapper: AsyncClientWrapper): @@ -83,8 +58,6 @@ def __init__(self, *, client_wrapper: AsyncClientWrapper): self._client_wrapper = client_wrapper self._dashboard_widgets: typing.Optional[AsyncDashboardWidgetsClient] = None self._dashboards: typing.Optional[AsyncDashboardsClient] = None - self._evaluation_rules: typing.Optional[AsyncEvaluationRulesClient] = None - self._evaluators: typing.Optional[AsyncEvaluatorsClient] = None @property def with_raw_response(self) -> AsyncRawUnstableClient: @@ -116,23 +89,3 @@ def dashboards(self): client_wrapper=self._client_wrapper ) return self._dashboards - - @property - def evaluation_rules(self): - if self._evaluation_rules is None: - from .evaluation_rules.client import AsyncEvaluationRulesClient # noqa: E402 - - self._evaluation_rules = AsyncEvaluationRulesClient( - client_wrapper=self._client_wrapper - ) - return self._evaluation_rules - - @property - def evaluators(self): - if self._evaluators is None: - from .evaluators.client import AsyncEvaluatorsClient # noqa: E402 - - self._evaluators = AsyncEvaluatorsClient( - client_wrapper=self._client_wrapper - ) - return self._evaluators diff --git a/langfuse/api/unstable/commons/__init__.py b/langfuse/api/unstable/commons/__init__.py deleted file mode 100644 index 61d7a2a87..000000000 --- a/langfuse/api/unstable/commons/__init__.py +++ /dev/null @@ -1,193 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -# isort: skip_file - -import typing -from importlib import import_module - -if typing.TYPE_CHECKING: - from .types import ( - ArrayOptionsEvaluationRuleFilter, - BooleanEvaluationRuleFilter, - CategoryOptionsEvaluationRuleFilter, - CodeEvaluatorSourceCodeLanguage, - DateTimeEvaluationRuleFilter, - EvaluationRuleArrayOptionsFilterOperator, - EvaluationRuleBooleanFilterOperator, - EvaluationRuleFilter, - EvaluationRuleFilter_ArrayOptions, - EvaluationRuleFilter_Boolean, - EvaluationRuleFilter_CategoryOptions, - EvaluationRuleFilter_Datetime, - EvaluationRuleFilter_Null, - EvaluationRuleFilter_Number, - EvaluationRuleFilter_NumberObject, - EvaluationRuleFilter_String, - EvaluationRuleFilter_StringObject, - EvaluationRuleFilter_StringOptions, - EvaluationRuleNullFilterOperator, - EvaluationRuleNumberFilterOperator, - EvaluationRuleOptionsFilterOperator, - EvaluationRuleStatus, - EvaluationRuleStringFilterOperator, - EvaluationRuleTarget, - EvaluatorModelConfig, - EvaluatorOutputDataType, - EvaluatorOutputDefinition, - EvaluatorOutputDefinition_Boolean, - EvaluatorOutputDefinition_Categorical, - EvaluatorOutputDefinition_Numeric, - EvaluatorOutputFieldDefinition, - EvaluatorType, - NullEvaluationRuleFilter, - NumberEvaluationRuleFilter, - NumberObjectEvaluationRuleFilter, - PromptVariableMappingInput, - PromptVariableMappingRead, - PromptVariableMappingSource, - PublicBooleanEvaluatorOutputDefinition, - PublicCategoricalEvaluatorOutputDefinition, - PublicCategoricalEvaluatorOutputScoreDefinition, - PublicEvaluatorOutputDefinition, - PublicEvaluatorOutputDefinition_Boolean, - PublicEvaluatorOutputDefinition_Categorical, - PublicEvaluatorOutputDefinition_Numeric, - PublicNumericEvaluatorOutputDefinition, - PublicNumericEvaluatorOutputScoreDefinition, - StringEvaluationRuleFilter, - StringObjectEvaluationRuleFilter, - StringOptionsEvaluationRuleFilter, - ) -_dynamic_imports: typing.Dict[str, str] = { - "ArrayOptionsEvaluationRuleFilter": ".types", - "BooleanEvaluationRuleFilter": ".types", - "CategoryOptionsEvaluationRuleFilter": ".types", - "CodeEvaluatorSourceCodeLanguage": ".types", - "DateTimeEvaluationRuleFilter": ".types", - "EvaluationRuleArrayOptionsFilterOperator": ".types", - "EvaluationRuleBooleanFilterOperator": ".types", - "EvaluationRuleFilter": ".types", - "EvaluationRuleFilter_ArrayOptions": ".types", - "EvaluationRuleFilter_Boolean": ".types", - "EvaluationRuleFilter_CategoryOptions": ".types", - "EvaluationRuleFilter_Datetime": ".types", - "EvaluationRuleFilter_Null": ".types", - "EvaluationRuleFilter_Number": ".types", - "EvaluationRuleFilter_NumberObject": ".types", - "EvaluationRuleFilter_String": ".types", - "EvaluationRuleFilter_StringObject": ".types", - "EvaluationRuleFilter_StringOptions": ".types", - "EvaluationRuleNullFilterOperator": ".types", - "EvaluationRuleNumberFilterOperator": ".types", - "EvaluationRuleOptionsFilterOperator": ".types", - "EvaluationRuleStatus": ".types", - "EvaluationRuleStringFilterOperator": ".types", - "EvaluationRuleTarget": ".types", - "EvaluatorModelConfig": ".types", - "EvaluatorOutputDataType": ".types", - "EvaluatorOutputDefinition": ".types", - "EvaluatorOutputDefinition_Boolean": ".types", - "EvaluatorOutputDefinition_Categorical": ".types", - "EvaluatorOutputDefinition_Numeric": ".types", - "EvaluatorOutputFieldDefinition": ".types", - "EvaluatorType": ".types", - "NullEvaluationRuleFilter": ".types", - "NumberEvaluationRuleFilter": ".types", - "NumberObjectEvaluationRuleFilter": ".types", - "PromptVariableMappingInput": ".types", - "PromptVariableMappingRead": ".types", - "PromptVariableMappingSource": ".types", - "PublicBooleanEvaluatorOutputDefinition": ".types", - "PublicCategoricalEvaluatorOutputDefinition": ".types", - "PublicCategoricalEvaluatorOutputScoreDefinition": ".types", - "PublicEvaluatorOutputDefinition": ".types", - "PublicEvaluatorOutputDefinition_Boolean": ".types", - "PublicEvaluatorOutputDefinition_Categorical": ".types", - "PublicEvaluatorOutputDefinition_Numeric": ".types", - "PublicNumericEvaluatorOutputDefinition": ".types", - "PublicNumericEvaluatorOutputScoreDefinition": ".types", - "StringEvaluationRuleFilter": ".types", - "StringObjectEvaluationRuleFilter": ".types", - "StringOptionsEvaluationRuleFilter": ".types", -} - - -def __getattr__(attr_name: str) -> typing.Any: - module_name = _dynamic_imports.get(attr_name) - if module_name is None: - raise AttributeError( - f"No {attr_name} found in _dynamic_imports for module name -> {__name__}" - ) - try: - module = import_module(module_name, __package__) - if module_name == f".{attr_name}": - return module - else: - return getattr(module, attr_name) - except ImportError as e: - raise ImportError( - f"Failed to import {attr_name} from {module_name}: {e}" - ) from e - except AttributeError as e: - raise AttributeError( - f"Failed to get {attr_name} from {module_name}: {e}" - ) from e - - -def __dir__(): - lazy_attrs = list(_dynamic_imports.keys()) - return sorted(lazy_attrs) - - -__all__ = [ - "ArrayOptionsEvaluationRuleFilter", - "BooleanEvaluationRuleFilter", - "CategoryOptionsEvaluationRuleFilter", - "CodeEvaluatorSourceCodeLanguage", - "DateTimeEvaluationRuleFilter", - "EvaluationRuleArrayOptionsFilterOperator", - "EvaluationRuleBooleanFilterOperator", - "EvaluationRuleFilter", - "EvaluationRuleFilter_ArrayOptions", - "EvaluationRuleFilter_Boolean", - "EvaluationRuleFilter_CategoryOptions", - "EvaluationRuleFilter_Datetime", - "EvaluationRuleFilter_Null", - "EvaluationRuleFilter_Number", - "EvaluationRuleFilter_NumberObject", - "EvaluationRuleFilter_String", - "EvaluationRuleFilter_StringObject", - "EvaluationRuleFilter_StringOptions", - "EvaluationRuleNullFilterOperator", - "EvaluationRuleNumberFilterOperator", - "EvaluationRuleOptionsFilterOperator", - "EvaluationRuleStatus", - "EvaluationRuleStringFilterOperator", - "EvaluationRuleTarget", - "EvaluatorModelConfig", - "EvaluatorOutputDataType", - "EvaluatorOutputDefinition", - "EvaluatorOutputDefinition_Boolean", - "EvaluatorOutputDefinition_Categorical", - "EvaluatorOutputDefinition_Numeric", - "EvaluatorOutputFieldDefinition", - "EvaluatorType", - "NullEvaluationRuleFilter", - "NumberEvaluationRuleFilter", - "NumberObjectEvaluationRuleFilter", - "PromptVariableMappingInput", - "PromptVariableMappingRead", - "PromptVariableMappingSource", - "PublicBooleanEvaluatorOutputDefinition", - "PublicCategoricalEvaluatorOutputDefinition", - "PublicCategoricalEvaluatorOutputScoreDefinition", - "PublicEvaluatorOutputDefinition", - "PublicEvaluatorOutputDefinition_Boolean", - "PublicEvaluatorOutputDefinition_Categorical", - "PublicEvaluatorOutputDefinition_Numeric", - "PublicNumericEvaluatorOutputDefinition", - "PublicNumericEvaluatorOutputScoreDefinition", - "StringEvaluationRuleFilter", - "StringObjectEvaluationRuleFilter", - "StringOptionsEvaluationRuleFilter", -] diff --git a/langfuse/api/unstable/commons/types/__init__.py b/langfuse/api/unstable/commons/types/__init__.py deleted file mode 100644 index 894b1d731..000000000 --- a/langfuse/api/unstable/commons/types/__init__.py +++ /dev/null @@ -1,219 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -# isort: skip_file - -import typing -from importlib import import_module - -if typing.TYPE_CHECKING: - from .array_options_evaluation_rule_filter import ArrayOptionsEvaluationRuleFilter - from .boolean_evaluation_rule_filter import BooleanEvaluationRuleFilter - from .category_options_evaluation_rule_filter import ( - CategoryOptionsEvaluationRuleFilter, - ) - from .code_evaluator_source_code_language import CodeEvaluatorSourceCodeLanguage - from .date_time_evaluation_rule_filter import DateTimeEvaluationRuleFilter - from .evaluation_rule_array_options_filter_operator import ( - EvaluationRuleArrayOptionsFilterOperator, - ) - from .evaluation_rule_boolean_filter_operator import ( - EvaluationRuleBooleanFilterOperator, - ) - from .evaluation_rule_filter import ( - EvaluationRuleFilter, - EvaluationRuleFilter_ArrayOptions, - EvaluationRuleFilter_Boolean, - EvaluationRuleFilter_CategoryOptions, - EvaluationRuleFilter_Datetime, - EvaluationRuleFilter_Null, - EvaluationRuleFilter_Number, - EvaluationRuleFilter_NumberObject, - EvaluationRuleFilter_String, - EvaluationRuleFilter_StringObject, - EvaluationRuleFilter_StringOptions, - ) - from .evaluation_rule_null_filter_operator import EvaluationRuleNullFilterOperator - from .evaluation_rule_number_filter_operator import ( - EvaluationRuleNumberFilterOperator, - ) - from .evaluation_rule_options_filter_operator import ( - EvaluationRuleOptionsFilterOperator, - ) - from .evaluation_rule_status import EvaluationRuleStatus - from .evaluation_rule_string_filter_operator import ( - EvaluationRuleStringFilterOperator, - ) - from .evaluation_rule_target import EvaluationRuleTarget - from .evaluator_model_config import EvaluatorModelConfig - from .evaluator_output_data_type import EvaluatorOutputDataType - from .evaluator_output_definition import ( - EvaluatorOutputDefinition, - EvaluatorOutputDefinition_Boolean, - EvaluatorOutputDefinition_Categorical, - EvaluatorOutputDefinition_Numeric, - ) - from .evaluator_output_field_definition import EvaluatorOutputFieldDefinition - from .evaluator_type import EvaluatorType - from .null_evaluation_rule_filter import NullEvaluationRuleFilter - from .number_evaluation_rule_filter import NumberEvaluationRuleFilter - from .number_object_evaluation_rule_filter import NumberObjectEvaluationRuleFilter - from .prompt_variable_mapping_input import PromptVariableMappingInput - from .prompt_variable_mapping_read import PromptVariableMappingRead - from .prompt_variable_mapping_source import PromptVariableMappingSource - from .public_boolean_evaluator_output_definition import ( - PublicBooleanEvaluatorOutputDefinition, - ) - from .public_categorical_evaluator_output_definition import ( - PublicCategoricalEvaluatorOutputDefinition, - ) - from .public_categorical_evaluator_output_score_definition import ( - PublicCategoricalEvaluatorOutputScoreDefinition, - ) - from .public_evaluator_output_definition import ( - PublicEvaluatorOutputDefinition, - PublicEvaluatorOutputDefinition_Boolean, - PublicEvaluatorOutputDefinition_Categorical, - PublicEvaluatorOutputDefinition_Numeric, - ) - from .public_numeric_evaluator_output_definition import ( - PublicNumericEvaluatorOutputDefinition, - ) - from .public_numeric_evaluator_output_score_definition import ( - PublicNumericEvaluatorOutputScoreDefinition, - ) - from .string_evaluation_rule_filter import StringEvaluationRuleFilter - from .string_object_evaluation_rule_filter import StringObjectEvaluationRuleFilter - from .string_options_evaluation_rule_filter import StringOptionsEvaluationRuleFilter -_dynamic_imports: typing.Dict[str, str] = { - "ArrayOptionsEvaluationRuleFilter": ".array_options_evaluation_rule_filter", - "BooleanEvaluationRuleFilter": ".boolean_evaluation_rule_filter", - "CategoryOptionsEvaluationRuleFilter": ".category_options_evaluation_rule_filter", - "CodeEvaluatorSourceCodeLanguage": ".code_evaluator_source_code_language", - "DateTimeEvaluationRuleFilter": ".date_time_evaluation_rule_filter", - "EvaluationRuleArrayOptionsFilterOperator": ".evaluation_rule_array_options_filter_operator", - "EvaluationRuleBooleanFilterOperator": ".evaluation_rule_boolean_filter_operator", - "EvaluationRuleFilter": ".evaluation_rule_filter", - "EvaluationRuleFilter_ArrayOptions": ".evaluation_rule_filter", - "EvaluationRuleFilter_Boolean": ".evaluation_rule_filter", - "EvaluationRuleFilter_CategoryOptions": ".evaluation_rule_filter", - "EvaluationRuleFilter_Datetime": ".evaluation_rule_filter", - "EvaluationRuleFilter_Null": ".evaluation_rule_filter", - "EvaluationRuleFilter_Number": ".evaluation_rule_filter", - "EvaluationRuleFilter_NumberObject": ".evaluation_rule_filter", - "EvaluationRuleFilter_String": ".evaluation_rule_filter", - "EvaluationRuleFilter_StringObject": ".evaluation_rule_filter", - "EvaluationRuleFilter_StringOptions": ".evaluation_rule_filter", - "EvaluationRuleNullFilterOperator": ".evaluation_rule_null_filter_operator", - "EvaluationRuleNumberFilterOperator": ".evaluation_rule_number_filter_operator", - "EvaluationRuleOptionsFilterOperator": ".evaluation_rule_options_filter_operator", - "EvaluationRuleStatus": ".evaluation_rule_status", - "EvaluationRuleStringFilterOperator": ".evaluation_rule_string_filter_operator", - "EvaluationRuleTarget": ".evaluation_rule_target", - "EvaluatorModelConfig": ".evaluator_model_config", - "EvaluatorOutputDataType": ".evaluator_output_data_type", - "EvaluatorOutputDefinition": ".evaluator_output_definition", - "EvaluatorOutputDefinition_Boolean": ".evaluator_output_definition", - "EvaluatorOutputDefinition_Categorical": ".evaluator_output_definition", - "EvaluatorOutputDefinition_Numeric": ".evaluator_output_definition", - "EvaluatorOutputFieldDefinition": ".evaluator_output_field_definition", - "EvaluatorType": ".evaluator_type", - "NullEvaluationRuleFilter": ".null_evaluation_rule_filter", - "NumberEvaluationRuleFilter": ".number_evaluation_rule_filter", - "NumberObjectEvaluationRuleFilter": ".number_object_evaluation_rule_filter", - "PromptVariableMappingInput": ".prompt_variable_mapping_input", - "PromptVariableMappingRead": ".prompt_variable_mapping_read", - "PromptVariableMappingSource": ".prompt_variable_mapping_source", - "PublicBooleanEvaluatorOutputDefinition": ".public_boolean_evaluator_output_definition", - "PublicCategoricalEvaluatorOutputDefinition": ".public_categorical_evaluator_output_definition", - "PublicCategoricalEvaluatorOutputScoreDefinition": ".public_categorical_evaluator_output_score_definition", - "PublicEvaluatorOutputDefinition": ".public_evaluator_output_definition", - "PublicEvaluatorOutputDefinition_Boolean": ".public_evaluator_output_definition", - "PublicEvaluatorOutputDefinition_Categorical": ".public_evaluator_output_definition", - "PublicEvaluatorOutputDefinition_Numeric": ".public_evaluator_output_definition", - "PublicNumericEvaluatorOutputDefinition": ".public_numeric_evaluator_output_definition", - "PublicNumericEvaluatorOutputScoreDefinition": ".public_numeric_evaluator_output_score_definition", - "StringEvaluationRuleFilter": ".string_evaluation_rule_filter", - "StringObjectEvaluationRuleFilter": ".string_object_evaluation_rule_filter", - "StringOptionsEvaluationRuleFilter": ".string_options_evaluation_rule_filter", -} - - -def __getattr__(attr_name: str) -> typing.Any: - module_name = _dynamic_imports.get(attr_name) - if module_name is None: - raise AttributeError( - f"No {attr_name} found in _dynamic_imports for module name -> {__name__}" - ) - try: - module = import_module(module_name, __package__) - if module_name == f".{attr_name}": - return module - else: - return getattr(module, attr_name) - except ImportError as e: - raise ImportError( - f"Failed to import {attr_name} from {module_name}: {e}" - ) from e - except AttributeError as e: - raise AttributeError( - f"Failed to get {attr_name} from {module_name}: {e}" - ) from e - - -def __dir__(): - lazy_attrs = list(_dynamic_imports.keys()) - return sorted(lazy_attrs) - - -__all__ = [ - "ArrayOptionsEvaluationRuleFilter", - "BooleanEvaluationRuleFilter", - "CategoryOptionsEvaluationRuleFilter", - "CodeEvaluatorSourceCodeLanguage", - "DateTimeEvaluationRuleFilter", - "EvaluationRuleArrayOptionsFilterOperator", - "EvaluationRuleBooleanFilterOperator", - "EvaluationRuleFilter", - "EvaluationRuleFilter_ArrayOptions", - "EvaluationRuleFilter_Boolean", - "EvaluationRuleFilter_CategoryOptions", - "EvaluationRuleFilter_Datetime", - "EvaluationRuleFilter_Null", - "EvaluationRuleFilter_Number", - "EvaluationRuleFilter_NumberObject", - "EvaluationRuleFilter_String", - "EvaluationRuleFilter_StringObject", - "EvaluationRuleFilter_StringOptions", - "EvaluationRuleNullFilterOperator", - "EvaluationRuleNumberFilterOperator", - "EvaluationRuleOptionsFilterOperator", - "EvaluationRuleStatus", - "EvaluationRuleStringFilterOperator", - "EvaluationRuleTarget", - "EvaluatorModelConfig", - "EvaluatorOutputDataType", - "EvaluatorOutputDefinition", - "EvaluatorOutputDefinition_Boolean", - "EvaluatorOutputDefinition_Categorical", - "EvaluatorOutputDefinition_Numeric", - "EvaluatorOutputFieldDefinition", - "EvaluatorType", - "NullEvaluationRuleFilter", - "NumberEvaluationRuleFilter", - "NumberObjectEvaluationRuleFilter", - "PromptVariableMappingInput", - "PromptVariableMappingRead", - "PromptVariableMappingSource", - "PublicBooleanEvaluatorOutputDefinition", - "PublicCategoricalEvaluatorOutputDefinition", - "PublicCategoricalEvaluatorOutputScoreDefinition", - "PublicEvaluatorOutputDefinition", - "PublicEvaluatorOutputDefinition_Boolean", - "PublicEvaluatorOutputDefinition_Categorical", - "PublicEvaluatorOutputDefinition_Numeric", - "PublicNumericEvaluatorOutputDefinition", - "PublicNumericEvaluatorOutputScoreDefinition", - "StringEvaluationRuleFilter", - "StringObjectEvaluationRuleFilter", - "StringOptionsEvaluationRuleFilter", -] diff --git a/langfuse/api/unstable/commons/types/array_options_evaluation_rule_filter.py b/langfuse/api/unstable/commons/types/array_options_evaluation_rule_filter.py deleted file mode 100644 index c89ce8b16..000000000 --- a/langfuse/api/unstable/commons/types/array_options_evaluation_rule_filter.py +++ /dev/null @@ -1,26 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from .evaluation_rule_array_options_filter_operator import ( - EvaluationRuleArrayOptionsFilterOperator, -) - - -class ArrayOptionsEvaluationRuleFilter(UniversalBaseModel): - column: str = pydantic.Field() - """ - Column to filter on. - """ - - operator: EvaluationRuleArrayOptionsFilterOperator - value: typing.List[str] = pydantic.Field() - """ - One or more array elements to match. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/boolean_evaluation_rule_filter.py b/langfuse/api/unstable/commons/types/boolean_evaluation_rule_filter.py deleted file mode 100644 index 666b691bb..000000000 --- a/langfuse/api/unstable/commons/types/boolean_evaluation_rule_filter.py +++ /dev/null @@ -1,21 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from .evaluation_rule_boolean_filter_operator import EvaluationRuleBooleanFilterOperator - - -class BooleanEvaluationRuleFilter(UniversalBaseModel): - column: str = pydantic.Field() - """ - Column to filter on. - """ - - operator: EvaluationRuleBooleanFilterOperator - value: bool - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/category_options_evaluation_rule_filter.py b/langfuse/api/unstable/commons/types/category_options_evaluation_rule_filter.py deleted file mode 100644 index 97f13ae62..000000000 --- a/langfuse/api/unstable/commons/types/category_options_evaluation_rule_filter.py +++ /dev/null @@ -1,26 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from .evaluation_rule_options_filter_operator import EvaluationRuleOptionsFilterOperator - - -class CategoryOptionsEvaluationRuleFilter(UniversalBaseModel): - column: str = pydantic.Field() - """ - Object-valued column to filter on. - """ - - key: str = pydantic.Field() - """ - Key inside the object-valued column to filter on. - """ - - operator: EvaluationRuleOptionsFilterOperator - value: typing.List[str] - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/code_evaluator_source_code_language.py b/langfuse/api/unstable/commons/types/code_evaluator_source_code_language.py deleted file mode 100644 index 7071a317c..000000000 --- a/langfuse/api/unstable/commons/types/code_evaluator_source_code_language.py +++ /dev/null @@ -1,26 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core import enum - -T_Result = typing.TypeVar("T_Result") - - -class CodeEvaluatorSourceCodeLanguage(enum.StrEnum): - """ - Code evaluator runtime language. - """ - - PYTHON = "PYTHON" - TYPESCRIPT = "TYPESCRIPT" - - def visit( - self, - python: typing.Callable[[], T_Result], - typescript: typing.Callable[[], T_Result], - ) -> T_Result: - if self is CodeEvaluatorSourceCodeLanguage.PYTHON: - return python() - if self is CodeEvaluatorSourceCodeLanguage.TYPESCRIPT: - return typescript() diff --git a/langfuse/api/unstable/commons/types/date_time_evaluation_rule_filter.py b/langfuse/api/unstable/commons/types/date_time_evaluation_rule_filter.py deleted file mode 100644 index 9ee23b1fe..000000000 --- a/langfuse/api/unstable/commons/types/date_time_evaluation_rule_filter.py +++ /dev/null @@ -1,29 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import datetime as dt -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from .evaluation_rule_number_filter_operator import EvaluationRuleNumberFilterOperator - - -class DateTimeEvaluationRuleFilter(UniversalBaseModel): - column: str = pydantic.Field() - """ - Column to filter on. - """ - - operator: EvaluationRuleNumberFilterOperator = pydantic.Field() - """ - Comparison operator for datetime values. - """ - - value: dt.datetime = pydantic.Field() - """ - Datetime value to compare against. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/evaluation_rule_array_options_filter_operator.py b/langfuse/api/unstable/commons/types/evaluation_rule_array_options_filter_operator.py deleted file mode 100644 index ba8f49a13..000000000 --- a/langfuse/api/unstable/commons/types/evaluation_rule_array_options_filter_operator.py +++ /dev/null @@ -1,26 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core import enum - -T_Result = typing.TypeVar("T_Result") - - -class EvaluationRuleArrayOptionsFilterOperator(enum.StrEnum): - ANY_OF = "any of" - NONE_OF = "none of" - ALL_OF = "all of" - - def visit( - self, - any_of: typing.Callable[[], T_Result], - none_of: typing.Callable[[], T_Result], - all_of: typing.Callable[[], T_Result], - ) -> T_Result: - if self is EvaluationRuleArrayOptionsFilterOperator.ANY_OF: - return any_of() - if self is EvaluationRuleArrayOptionsFilterOperator.NONE_OF: - return none_of() - if self is EvaluationRuleArrayOptionsFilterOperator.ALL_OF: - return all_of() diff --git a/langfuse/api/unstable/commons/types/evaluation_rule_boolean_filter_operator.py b/langfuse/api/unstable/commons/types/evaluation_rule_boolean_filter_operator.py deleted file mode 100644 index 737d6063a..000000000 --- a/langfuse/api/unstable/commons/types/evaluation_rule_boolean_filter_operator.py +++ /dev/null @@ -1,22 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core import enum - -T_Result = typing.TypeVar("T_Result") - - -class EvaluationRuleBooleanFilterOperator(enum.StrEnum): - EQUALS = "=" - NOT_EQUALS = "<>" - - def visit( - self, - equals: typing.Callable[[], T_Result], - not_equals: typing.Callable[[], T_Result], - ) -> T_Result: - if self is EvaluationRuleBooleanFilterOperator.EQUALS: - return equals() - if self is EvaluationRuleBooleanFilterOperator.NOT_EQUALS: - return not_equals() diff --git a/langfuse/api/unstable/commons/types/evaluation_rule_filter.py b/langfuse/api/unstable/commons/types/evaluation_rule_filter.py deleted file mode 100644 index 9d70f822b..000000000 --- a/langfuse/api/unstable/commons/types/evaluation_rule_filter.py +++ /dev/null @@ -1,760 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -from __future__ import annotations - -import datetime as dt -import typing - -import pydantic -import typing_extensions -from ....core.pydantic_utilities import UniversalBaseModel -from .evaluation_rule_array_options_filter_operator import ( - EvaluationRuleArrayOptionsFilterOperator, -) -from .evaluation_rule_boolean_filter_operator import EvaluationRuleBooleanFilterOperator -from .evaluation_rule_null_filter_operator import EvaluationRuleNullFilterOperator -from .evaluation_rule_number_filter_operator import EvaluationRuleNumberFilterOperator -from .evaluation_rule_options_filter_operator import EvaluationRuleOptionsFilterOperator -from .evaluation_rule_string_filter_operator import EvaluationRuleStringFilterOperator - - -class EvaluationRuleFilter_Datetime(UniversalBaseModel): - """ - One filter condition used to decide whether a live-ingested target should be evaluated. - - An evaluation rule can include zero or more filter objects. All filters must be satisfied for the target to run. - - How to build a valid filter object: - - Pick the `target` first, because it changes the supported columns. - - Pick the filter `type`. That determines which fields are required. - - Use `key` only for object filters such as `metadata`. - - Use the correct `value` shape for the chosen filter `type`. - - Operator quick reference by filter `type`: - - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` - - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `stringOptions`: `any of`, `none of` - - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` - - `boolean`: `"="`, `"<>"` - - `null`: `is null`, `is not null` - - Supported columns by target: - - `target=observation` - - `type`: `stringOptions`, operators `any of` / `none of`, values `GENERATION`, `SPAN`, `EVENT` - - `name`: `stringOptions`, operators `any of` / `none of` - - `environment`: `stringOptions`, operators `any of` / `none of` - - `level`: `stringOptions`, operators `any of` / `none of`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` - - `version`: `string` - - `traceName`: `stringOptions`, operators `any of` / `none of` - - `userId`: `string` - - `sessionId`: `string` - - `tags`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `metadata`: `stringObject` with `key` - - `isRootObservation`: `boolean`, operators `=` / `<>`; true when the observation has no parent or is explicitly marked as an application root - - `parentObservationId`: `null`, operators `is null` / `is not null` - - `calledToolNames`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `toolCalls`: `number` - - `target=experiment` - - `datasetId`: `stringOptions`, operators `any of` / `none of` - Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - - Recovery guidance: - - `invalid_filter_value` with `details.column` but no `invalidValues`: the selected `column` is not supported for the chosen `target` - - `invalid_filter_value` with `details.invalidValues`: the selected values are not allowed for that column. Replace them with one of `details.allowedValues` when provided. - - `invalid_filter_value` for `column=datasetId`: call `GET /api/public/v2/datasets`, then retry with dataset `id` values from that response. - - Examples - -------- - from langfuse.unstable.commons import ( - EvaluationRuleFilter_StringOptions, - EvaluationRuleOptionsFilterOperator, - ) - - EvaluationRuleFilter_StringOptions( - column="type", - operator=EvaluationRuleOptionsFilterOperator.ANY_OF, - value=["GENERATION"], - ) - """ - - type: typing.Literal["datetime"] = "datetime" - column: str - operator: EvaluationRuleNumberFilterOperator - value: dt.datetime - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -class EvaluationRuleFilter_String(UniversalBaseModel): - """ - One filter condition used to decide whether a live-ingested target should be evaluated. - - An evaluation rule can include zero or more filter objects. All filters must be satisfied for the target to run. - - How to build a valid filter object: - - Pick the `target` first, because it changes the supported columns. - - Pick the filter `type`. That determines which fields are required. - - Use `key` only for object filters such as `metadata`. - - Use the correct `value` shape for the chosen filter `type`. - - Operator quick reference by filter `type`: - - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` - - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `stringOptions`: `any of`, `none of` - - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` - - `boolean`: `"="`, `"<>"` - - `null`: `is null`, `is not null` - - Supported columns by target: - - `target=observation` - - `type`: `stringOptions`, operators `any of` / `none of`, values `GENERATION`, `SPAN`, `EVENT` - - `name`: `stringOptions`, operators `any of` / `none of` - - `environment`: `stringOptions`, operators `any of` / `none of` - - `level`: `stringOptions`, operators `any of` / `none of`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` - - `version`: `string` - - `traceName`: `stringOptions`, operators `any of` / `none of` - - `userId`: `string` - - `sessionId`: `string` - - `tags`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `metadata`: `stringObject` with `key` - - `isRootObservation`: `boolean`, operators `=` / `<>`; true when the observation has no parent or is explicitly marked as an application root - - `parentObservationId`: `null`, operators `is null` / `is not null` - - `calledToolNames`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `toolCalls`: `number` - - `target=experiment` - - `datasetId`: `stringOptions`, operators `any of` / `none of` - Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - - Recovery guidance: - - `invalid_filter_value` with `details.column` but no `invalidValues`: the selected `column` is not supported for the chosen `target` - - `invalid_filter_value` with `details.invalidValues`: the selected values are not allowed for that column. Replace them with one of `details.allowedValues` when provided. - - `invalid_filter_value` for `column=datasetId`: call `GET /api/public/v2/datasets`, then retry with dataset `id` values from that response. - - Examples - -------- - from langfuse.unstable.commons import ( - EvaluationRuleFilter_StringOptions, - EvaluationRuleOptionsFilterOperator, - ) - - EvaluationRuleFilter_StringOptions( - column="type", - operator=EvaluationRuleOptionsFilterOperator.ANY_OF, - value=["GENERATION"], - ) - """ - - type: typing.Literal["string"] = "string" - column: str - operator: EvaluationRuleStringFilterOperator - value: str - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -class EvaluationRuleFilter_Number(UniversalBaseModel): - """ - One filter condition used to decide whether a live-ingested target should be evaluated. - - An evaluation rule can include zero or more filter objects. All filters must be satisfied for the target to run. - - How to build a valid filter object: - - Pick the `target` first, because it changes the supported columns. - - Pick the filter `type`. That determines which fields are required. - - Use `key` only for object filters such as `metadata`. - - Use the correct `value` shape for the chosen filter `type`. - - Operator quick reference by filter `type`: - - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` - - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `stringOptions`: `any of`, `none of` - - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` - - `boolean`: `"="`, `"<>"` - - `null`: `is null`, `is not null` - - Supported columns by target: - - `target=observation` - - `type`: `stringOptions`, operators `any of` / `none of`, values `GENERATION`, `SPAN`, `EVENT` - - `name`: `stringOptions`, operators `any of` / `none of` - - `environment`: `stringOptions`, operators `any of` / `none of` - - `level`: `stringOptions`, operators `any of` / `none of`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` - - `version`: `string` - - `traceName`: `stringOptions`, operators `any of` / `none of` - - `userId`: `string` - - `sessionId`: `string` - - `tags`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `metadata`: `stringObject` with `key` - - `isRootObservation`: `boolean`, operators `=` / `<>`; true when the observation has no parent or is explicitly marked as an application root - - `parentObservationId`: `null`, operators `is null` / `is not null` - - `calledToolNames`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `toolCalls`: `number` - - `target=experiment` - - `datasetId`: `stringOptions`, operators `any of` / `none of` - Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - - Recovery guidance: - - `invalid_filter_value` with `details.column` but no `invalidValues`: the selected `column` is not supported for the chosen `target` - - `invalid_filter_value` with `details.invalidValues`: the selected values are not allowed for that column. Replace them with one of `details.allowedValues` when provided. - - `invalid_filter_value` for `column=datasetId`: call `GET /api/public/v2/datasets`, then retry with dataset `id` values from that response. - - Examples - -------- - from langfuse.unstable.commons import ( - EvaluationRuleFilter_StringOptions, - EvaluationRuleOptionsFilterOperator, - ) - - EvaluationRuleFilter_StringOptions( - column="type", - operator=EvaluationRuleOptionsFilterOperator.ANY_OF, - value=["GENERATION"], - ) - """ - - type: typing.Literal["number"] = "number" - column: str - operator: EvaluationRuleNumberFilterOperator - value: float - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -class EvaluationRuleFilter_StringOptions(UniversalBaseModel): - """ - One filter condition used to decide whether a live-ingested target should be evaluated. - - An evaluation rule can include zero or more filter objects. All filters must be satisfied for the target to run. - - How to build a valid filter object: - - Pick the `target` first, because it changes the supported columns. - - Pick the filter `type`. That determines which fields are required. - - Use `key` only for object filters such as `metadata`. - - Use the correct `value` shape for the chosen filter `type`. - - Operator quick reference by filter `type`: - - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` - - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `stringOptions`: `any of`, `none of` - - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` - - `boolean`: `"="`, `"<>"` - - `null`: `is null`, `is not null` - - Supported columns by target: - - `target=observation` - - `type`: `stringOptions`, operators `any of` / `none of`, values `GENERATION`, `SPAN`, `EVENT` - - `name`: `stringOptions`, operators `any of` / `none of` - - `environment`: `stringOptions`, operators `any of` / `none of` - - `level`: `stringOptions`, operators `any of` / `none of`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` - - `version`: `string` - - `traceName`: `stringOptions`, operators `any of` / `none of` - - `userId`: `string` - - `sessionId`: `string` - - `tags`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `metadata`: `stringObject` with `key` - - `isRootObservation`: `boolean`, operators `=` / `<>`; true when the observation has no parent or is explicitly marked as an application root - - `parentObservationId`: `null`, operators `is null` / `is not null` - - `calledToolNames`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `toolCalls`: `number` - - `target=experiment` - - `datasetId`: `stringOptions`, operators `any of` / `none of` - Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - - Recovery guidance: - - `invalid_filter_value` with `details.column` but no `invalidValues`: the selected `column` is not supported for the chosen `target` - - `invalid_filter_value` with `details.invalidValues`: the selected values are not allowed for that column. Replace them with one of `details.allowedValues` when provided. - - `invalid_filter_value` for `column=datasetId`: call `GET /api/public/v2/datasets`, then retry with dataset `id` values from that response. - - Examples - -------- - from langfuse.unstable.commons import ( - EvaluationRuleFilter_StringOptions, - EvaluationRuleOptionsFilterOperator, - ) - - EvaluationRuleFilter_StringOptions( - column="type", - operator=EvaluationRuleOptionsFilterOperator.ANY_OF, - value=["GENERATION"], - ) - """ - - type: typing.Literal["stringOptions"] = "stringOptions" - column: str - operator: EvaluationRuleOptionsFilterOperator - value: typing.List[str] - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -class EvaluationRuleFilter_CategoryOptions(UniversalBaseModel): - """ - One filter condition used to decide whether a live-ingested target should be evaluated. - - An evaluation rule can include zero or more filter objects. All filters must be satisfied for the target to run. - - How to build a valid filter object: - - Pick the `target` first, because it changes the supported columns. - - Pick the filter `type`. That determines which fields are required. - - Use `key` only for object filters such as `metadata`. - - Use the correct `value` shape for the chosen filter `type`. - - Operator quick reference by filter `type`: - - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` - - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `stringOptions`: `any of`, `none of` - - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` - - `boolean`: `"="`, `"<>"` - - `null`: `is null`, `is not null` - - Supported columns by target: - - `target=observation` - - `type`: `stringOptions`, operators `any of` / `none of`, values `GENERATION`, `SPAN`, `EVENT` - - `name`: `stringOptions`, operators `any of` / `none of` - - `environment`: `stringOptions`, operators `any of` / `none of` - - `level`: `stringOptions`, operators `any of` / `none of`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` - - `version`: `string` - - `traceName`: `stringOptions`, operators `any of` / `none of` - - `userId`: `string` - - `sessionId`: `string` - - `tags`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `metadata`: `stringObject` with `key` - - `isRootObservation`: `boolean`, operators `=` / `<>`; true when the observation has no parent or is explicitly marked as an application root - - `parentObservationId`: `null`, operators `is null` / `is not null` - - `calledToolNames`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `toolCalls`: `number` - - `target=experiment` - - `datasetId`: `stringOptions`, operators `any of` / `none of` - Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - - Recovery guidance: - - `invalid_filter_value` with `details.column` but no `invalidValues`: the selected `column` is not supported for the chosen `target` - - `invalid_filter_value` with `details.invalidValues`: the selected values are not allowed for that column. Replace them with one of `details.allowedValues` when provided. - - `invalid_filter_value` for `column=datasetId`: call `GET /api/public/v2/datasets`, then retry with dataset `id` values from that response. - - Examples - -------- - from langfuse.unstable.commons import ( - EvaluationRuleFilter_StringOptions, - EvaluationRuleOptionsFilterOperator, - ) - - EvaluationRuleFilter_StringOptions( - column="type", - operator=EvaluationRuleOptionsFilterOperator.ANY_OF, - value=["GENERATION"], - ) - """ - - type: typing.Literal["categoryOptions"] = "categoryOptions" - column: str - key: str - operator: EvaluationRuleOptionsFilterOperator - value: typing.List[str] - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -class EvaluationRuleFilter_ArrayOptions(UniversalBaseModel): - """ - One filter condition used to decide whether a live-ingested target should be evaluated. - - An evaluation rule can include zero or more filter objects. All filters must be satisfied for the target to run. - - How to build a valid filter object: - - Pick the `target` first, because it changes the supported columns. - - Pick the filter `type`. That determines which fields are required. - - Use `key` only for object filters such as `metadata`. - - Use the correct `value` shape for the chosen filter `type`. - - Operator quick reference by filter `type`: - - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` - - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `stringOptions`: `any of`, `none of` - - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` - - `boolean`: `"="`, `"<>"` - - `null`: `is null`, `is not null` - - Supported columns by target: - - `target=observation` - - `type`: `stringOptions`, operators `any of` / `none of`, values `GENERATION`, `SPAN`, `EVENT` - - `name`: `stringOptions`, operators `any of` / `none of` - - `environment`: `stringOptions`, operators `any of` / `none of` - - `level`: `stringOptions`, operators `any of` / `none of`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` - - `version`: `string` - - `traceName`: `stringOptions`, operators `any of` / `none of` - - `userId`: `string` - - `sessionId`: `string` - - `tags`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `metadata`: `stringObject` with `key` - - `isRootObservation`: `boolean`, operators `=` / `<>`; true when the observation has no parent or is explicitly marked as an application root - - `parentObservationId`: `null`, operators `is null` / `is not null` - - `calledToolNames`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `toolCalls`: `number` - - `target=experiment` - - `datasetId`: `stringOptions`, operators `any of` / `none of` - Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - - Recovery guidance: - - `invalid_filter_value` with `details.column` but no `invalidValues`: the selected `column` is not supported for the chosen `target` - - `invalid_filter_value` with `details.invalidValues`: the selected values are not allowed for that column. Replace them with one of `details.allowedValues` when provided. - - `invalid_filter_value` for `column=datasetId`: call `GET /api/public/v2/datasets`, then retry with dataset `id` values from that response. - - Examples - -------- - from langfuse.unstable.commons import ( - EvaluationRuleFilter_StringOptions, - EvaluationRuleOptionsFilterOperator, - ) - - EvaluationRuleFilter_StringOptions( - column="type", - operator=EvaluationRuleOptionsFilterOperator.ANY_OF, - value=["GENERATION"], - ) - """ - - type: typing.Literal["arrayOptions"] = "arrayOptions" - column: str - operator: EvaluationRuleArrayOptionsFilterOperator - value: typing.List[str] - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -class EvaluationRuleFilter_StringObject(UniversalBaseModel): - """ - One filter condition used to decide whether a live-ingested target should be evaluated. - - An evaluation rule can include zero or more filter objects. All filters must be satisfied for the target to run. - - How to build a valid filter object: - - Pick the `target` first, because it changes the supported columns. - - Pick the filter `type`. That determines which fields are required. - - Use `key` only for object filters such as `metadata`. - - Use the correct `value` shape for the chosen filter `type`. - - Operator quick reference by filter `type`: - - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` - - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `stringOptions`: `any of`, `none of` - - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` - - `boolean`: `"="`, `"<>"` - - `null`: `is null`, `is not null` - - Supported columns by target: - - `target=observation` - - `type`: `stringOptions`, operators `any of` / `none of`, values `GENERATION`, `SPAN`, `EVENT` - - `name`: `stringOptions`, operators `any of` / `none of` - - `environment`: `stringOptions`, operators `any of` / `none of` - - `level`: `stringOptions`, operators `any of` / `none of`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` - - `version`: `string` - - `traceName`: `stringOptions`, operators `any of` / `none of` - - `userId`: `string` - - `sessionId`: `string` - - `tags`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `metadata`: `stringObject` with `key` - - `isRootObservation`: `boolean`, operators `=` / `<>`; true when the observation has no parent or is explicitly marked as an application root - - `parentObservationId`: `null`, operators `is null` / `is not null` - - `calledToolNames`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `toolCalls`: `number` - - `target=experiment` - - `datasetId`: `stringOptions`, operators `any of` / `none of` - Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - - Recovery guidance: - - `invalid_filter_value` with `details.column` but no `invalidValues`: the selected `column` is not supported for the chosen `target` - - `invalid_filter_value` with `details.invalidValues`: the selected values are not allowed for that column. Replace them with one of `details.allowedValues` when provided. - - `invalid_filter_value` for `column=datasetId`: call `GET /api/public/v2/datasets`, then retry with dataset `id` values from that response. - - Examples - -------- - from langfuse.unstable.commons import ( - EvaluationRuleFilter_StringOptions, - EvaluationRuleOptionsFilterOperator, - ) - - EvaluationRuleFilter_StringOptions( - column="type", - operator=EvaluationRuleOptionsFilterOperator.ANY_OF, - value=["GENERATION"], - ) - """ - - type: typing.Literal["stringObject"] = "stringObject" - column: str - key: str - operator: EvaluationRuleStringFilterOperator - value: str - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -class EvaluationRuleFilter_NumberObject(UniversalBaseModel): - """ - One filter condition used to decide whether a live-ingested target should be evaluated. - - An evaluation rule can include zero or more filter objects. All filters must be satisfied for the target to run. - - How to build a valid filter object: - - Pick the `target` first, because it changes the supported columns. - - Pick the filter `type`. That determines which fields are required. - - Use `key` only for object filters such as `metadata`. - - Use the correct `value` shape for the chosen filter `type`. - - Operator quick reference by filter `type`: - - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` - - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `stringOptions`: `any of`, `none of` - - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` - - `boolean`: `"="`, `"<>"` - - `null`: `is null`, `is not null` - - Supported columns by target: - - `target=observation` - - `type`: `stringOptions`, operators `any of` / `none of`, values `GENERATION`, `SPAN`, `EVENT` - - `name`: `stringOptions`, operators `any of` / `none of` - - `environment`: `stringOptions`, operators `any of` / `none of` - - `level`: `stringOptions`, operators `any of` / `none of`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` - - `version`: `string` - - `traceName`: `stringOptions`, operators `any of` / `none of` - - `userId`: `string` - - `sessionId`: `string` - - `tags`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `metadata`: `stringObject` with `key` - - `isRootObservation`: `boolean`, operators `=` / `<>`; true when the observation has no parent or is explicitly marked as an application root - - `parentObservationId`: `null`, operators `is null` / `is not null` - - `calledToolNames`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `toolCalls`: `number` - - `target=experiment` - - `datasetId`: `stringOptions`, operators `any of` / `none of` - Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - - Recovery guidance: - - `invalid_filter_value` with `details.column` but no `invalidValues`: the selected `column` is not supported for the chosen `target` - - `invalid_filter_value` with `details.invalidValues`: the selected values are not allowed for that column. Replace them with one of `details.allowedValues` when provided. - - `invalid_filter_value` for `column=datasetId`: call `GET /api/public/v2/datasets`, then retry with dataset `id` values from that response. - - Examples - -------- - from langfuse.unstable.commons import ( - EvaluationRuleFilter_StringOptions, - EvaluationRuleOptionsFilterOperator, - ) - - EvaluationRuleFilter_StringOptions( - column="type", - operator=EvaluationRuleOptionsFilterOperator.ANY_OF, - value=["GENERATION"], - ) - """ - - type: typing.Literal["numberObject"] = "numberObject" - column: str - key: str - operator: EvaluationRuleNumberFilterOperator - value: float - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -class EvaluationRuleFilter_Boolean(UniversalBaseModel): - """ - One filter condition used to decide whether a live-ingested target should be evaluated. - - An evaluation rule can include zero or more filter objects. All filters must be satisfied for the target to run. - - How to build a valid filter object: - - Pick the `target` first, because it changes the supported columns. - - Pick the filter `type`. That determines which fields are required. - - Use `key` only for object filters such as `metadata`. - - Use the correct `value` shape for the chosen filter `type`. - - Operator quick reference by filter `type`: - - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` - - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `stringOptions`: `any of`, `none of` - - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` - - `boolean`: `"="`, `"<>"` - - `null`: `is null`, `is not null` - - Supported columns by target: - - `target=observation` - - `type`: `stringOptions`, operators `any of` / `none of`, values `GENERATION`, `SPAN`, `EVENT` - - `name`: `stringOptions`, operators `any of` / `none of` - - `environment`: `stringOptions`, operators `any of` / `none of` - - `level`: `stringOptions`, operators `any of` / `none of`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` - - `version`: `string` - - `traceName`: `stringOptions`, operators `any of` / `none of` - - `userId`: `string` - - `sessionId`: `string` - - `tags`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `metadata`: `stringObject` with `key` - - `isRootObservation`: `boolean`, operators `=` / `<>`; true when the observation has no parent or is explicitly marked as an application root - - `parentObservationId`: `null`, operators `is null` / `is not null` - - `calledToolNames`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `toolCalls`: `number` - - `target=experiment` - - `datasetId`: `stringOptions`, operators `any of` / `none of` - Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - - Recovery guidance: - - `invalid_filter_value` with `details.column` but no `invalidValues`: the selected `column` is not supported for the chosen `target` - - `invalid_filter_value` with `details.invalidValues`: the selected values are not allowed for that column. Replace them with one of `details.allowedValues` when provided. - - `invalid_filter_value` for `column=datasetId`: call `GET /api/public/v2/datasets`, then retry with dataset `id` values from that response. - - Examples - -------- - from langfuse.unstable.commons import ( - EvaluationRuleFilter_StringOptions, - EvaluationRuleOptionsFilterOperator, - ) - - EvaluationRuleFilter_StringOptions( - column="type", - operator=EvaluationRuleOptionsFilterOperator.ANY_OF, - value=["GENERATION"], - ) - """ - - type: typing.Literal["boolean"] = "boolean" - column: str - operator: EvaluationRuleBooleanFilterOperator - value: bool - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -class EvaluationRuleFilter_Null(UniversalBaseModel): - """ - One filter condition used to decide whether a live-ingested target should be evaluated. - - An evaluation rule can include zero or more filter objects. All filters must be satisfied for the target to run. - - How to build a valid filter object: - - Pick the `target` first, because it changes the supported columns. - - Pick the filter `type`. That determines which fields are required. - - Use `key` only for object filters such as `metadata`. - - Use the correct `value` shape for the chosen filter `type`. - - Operator quick reference by filter `type`: - - `string`: `"="`, `contains`, `does not contain`, `starts with`, `ends with` - - `number`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `datetime`: `"="`, `">"`, `"<"`, `">="`, `"<="` - - `stringOptions`: `any of`, `none of` - - `arrayOptions`: `any of`, `none of`, `all of` - - `stringObject`: same operators as `string` - - `boolean`: `"="`, `"<>"` - - `null`: `is null`, `is not null` - - Supported columns by target: - - `target=observation` - - `type`: `stringOptions`, operators `any of` / `none of`, values `GENERATION`, `SPAN`, `EVENT` - - `name`: `stringOptions`, operators `any of` / `none of` - - `environment`: `stringOptions`, operators `any of` / `none of` - - `level`: `stringOptions`, operators `any of` / `none of`, values `DEBUG`, `DEFAULT`, `WARNING`, `ERROR` - - `version`: `string` - - `traceName`: `stringOptions`, operators `any of` / `none of` - - `userId`: `string` - - `sessionId`: `string` - - `tags`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `metadata`: `stringObject` with `key` - - `isRootObservation`: `boolean`, operators `=` / `<>`; true when the observation has no parent or is explicitly marked as an application root - - `parentObservationId`: `null`, operators `is null` / `is not null` - - `calledToolNames`: `arrayOptions`, operators `any of` / `none of` / `all of` - - `toolCalls`: `number` - - `target=experiment` - - `datasetId`: `stringOptions`, operators `any of` / `none of` - Use dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - - Recovery guidance: - - `invalid_filter_value` with `details.column` but no `invalidValues`: the selected `column` is not supported for the chosen `target` - - `invalid_filter_value` with `details.invalidValues`: the selected values are not allowed for that column. Replace them with one of `details.allowedValues` when provided. - - `invalid_filter_value` for `column=datasetId`: call `GET /api/public/v2/datasets`, then retry with dataset `id` values from that response. - - Examples - -------- - from langfuse.unstable.commons import ( - EvaluationRuleFilter_StringOptions, - EvaluationRuleOptionsFilterOperator, - ) - - EvaluationRuleFilter_StringOptions( - column="type", - operator=EvaluationRuleOptionsFilterOperator.ANY_OF, - value=["GENERATION"], - ) - """ - - type: typing.Literal["null"] = "null" - column: str - operator: EvaluationRuleNullFilterOperator - value: typing.Optional[str] = None - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -""" -from langfuse.unstable.commons import ( - EvaluationRuleFilter_StringOptions, - EvaluationRuleOptionsFilterOperator, -) - -EvaluationRuleFilter_StringOptions( - column="type", - operator=EvaluationRuleOptionsFilterOperator.ANY_OF, - value=["GENERATION"], -) -""" -EvaluationRuleFilter = typing_extensions.Annotated[ - typing.Union[ - EvaluationRuleFilter_Datetime, - EvaluationRuleFilter_String, - EvaluationRuleFilter_Number, - EvaluationRuleFilter_StringOptions, - EvaluationRuleFilter_CategoryOptions, - EvaluationRuleFilter_ArrayOptions, - EvaluationRuleFilter_StringObject, - EvaluationRuleFilter_NumberObject, - EvaluationRuleFilter_Boolean, - EvaluationRuleFilter_Null, - ], - pydantic.Field(discriminator="type"), -] diff --git a/langfuse/api/unstable/commons/types/evaluation_rule_null_filter_operator.py b/langfuse/api/unstable/commons/types/evaluation_rule_null_filter_operator.py deleted file mode 100644 index 833c8406f..000000000 --- a/langfuse/api/unstable/commons/types/evaluation_rule_null_filter_operator.py +++ /dev/null @@ -1,22 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core import enum - -T_Result = typing.TypeVar("T_Result") - - -class EvaluationRuleNullFilterOperator(enum.StrEnum): - IS_NULL = "is null" - IS_NOT_NULL = "is not null" - - def visit( - self, - is_null: typing.Callable[[], T_Result], - is_not_null: typing.Callable[[], T_Result], - ) -> T_Result: - if self is EvaluationRuleNullFilterOperator.IS_NULL: - return is_null() - if self is EvaluationRuleNullFilterOperator.IS_NOT_NULL: - return is_not_null() diff --git a/langfuse/api/unstable/commons/types/evaluation_rule_number_filter_operator.py b/langfuse/api/unstable/commons/types/evaluation_rule_number_filter_operator.py deleted file mode 100644 index 927523e04..000000000 --- a/langfuse/api/unstable/commons/types/evaluation_rule_number_filter_operator.py +++ /dev/null @@ -1,34 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core import enum - -T_Result = typing.TypeVar("T_Result") - - -class EvaluationRuleNumberFilterOperator(enum.StrEnum): - EQUALS = "=" - GREATER_THAN = ">" - LESS_THAN = "<" - GREATER_THAN_OR_EQUAL = ">=" - LESS_THAN_OR_EQUAL = "<=" - - def visit( - self, - equals: typing.Callable[[], T_Result], - greater_than: typing.Callable[[], T_Result], - less_than: typing.Callable[[], T_Result], - greater_than_or_equal: typing.Callable[[], T_Result], - less_than_or_equal: typing.Callable[[], T_Result], - ) -> T_Result: - if self is EvaluationRuleNumberFilterOperator.EQUALS: - return equals() - if self is EvaluationRuleNumberFilterOperator.GREATER_THAN: - return greater_than() - if self is EvaluationRuleNumberFilterOperator.LESS_THAN: - return less_than() - if self is EvaluationRuleNumberFilterOperator.GREATER_THAN_OR_EQUAL: - return greater_than_or_equal() - if self is EvaluationRuleNumberFilterOperator.LESS_THAN_OR_EQUAL: - return less_than_or_equal() diff --git a/langfuse/api/unstable/commons/types/evaluation_rule_options_filter_operator.py b/langfuse/api/unstable/commons/types/evaluation_rule_options_filter_operator.py deleted file mode 100644 index 01cd13ea3..000000000 --- a/langfuse/api/unstable/commons/types/evaluation_rule_options_filter_operator.py +++ /dev/null @@ -1,22 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core import enum - -T_Result = typing.TypeVar("T_Result") - - -class EvaluationRuleOptionsFilterOperator(enum.StrEnum): - ANY_OF = "any of" - NONE_OF = "none of" - - def visit( - self, - any_of: typing.Callable[[], T_Result], - none_of: typing.Callable[[], T_Result], - ) -> T_Result: - if self is EvaluationRuleOptionsFilterOperator.ANY_OF: - return any_of() - if self is EvaluationRuleOptionsFilterOperator.NONE_OF: - return none_of() diff --git a/langfuse/api/unstable/commons/types/evaluation_rule_status.py b/langfuse/api/unstable/commons/types/evaluation_rule_status.py deleted file mode 100644 index 4a313a962..000000000 --- a/langfuse/api/unstable/commons/types/evaluation_rule_status.py +++ /dev/null @@ -1,34 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core import enum - -T_Result = typing.TypeVar("T_Result") - - -class EvaluationRuleStatus(enum.StrEnum): - """ - Effective runtime status of the evaluation rule. - - - `active`: enabled and currently runnable. - - `inactive`: disabled by configuration. - - `paused`: enabled, but Langfuse has blocked execution until the underlying issue is resolved. - """ - - ACTIVE = "active" - INACTIVE = "inactive" - PAUSED = "paused" - - def visit( - self, - active: typing.Callable[[], T_Result], - inactive: typing.Callable[[], T_Result], - paused: typing.Callable[[], T_Result], - ) -> T_Result: - if self is EvaluationRuleStatus.ACTIVE: - return active() - if self is EvaluationRuleStatus.INACTIVE: - return inactive() - if self is EvaluationRuleStatus.PAUSED: - return paused() diff --git a/langfuse/api/unstable/commons/types/evaluation_rule_string_filter_operator.py b/langfuse/api/unstable/commons/types/evaluation_rule_string_filter_operator.py deleted file mode 100644 index 9955172b9..000000000 --- a/langfuse/api/unstable/commons/types/evaluation_rule_string_filter_operator.py +++ /dev/null @@ -1,34 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core import enum - -T_Result = typing.TypeVar("T_Result") - - -class EvaluationRuleStringFilterOperator(enum.StrEnum): - EQUALS = "=" - CONTAINS = "contains" - DOES_NOT_CONTAIN = "does not contain" - STARTS_WITH = "starts with" - ENDS_WITH = "ends with" - - def visit( - self, - equals: typing.Callable[[], T_Result], - contains: typing.Callable[[], T_Result], - does_not_contain: typing.Callable[[], T_Result], - starts_with: typing.Callable[[], T_Result], - ends_with: typing.Callable[[], T_Result], - ) -> T_Result: - if self is EvaluationRuleStringFilterOperator.EQUALS: - return equals() - if self is EvaluationRuleStringFilterOperator.CONTAINS: - return contains() - if self is EvaluationRuleStringFilterOperator.DOES_NOT_CONTAIN: - return does_not_contain() - if self is EvaluationRuleStringFilterOperator.STARTS_WITH: - return starts_with() - if self is EvaluationRuleStringFilterOperator.ENDS_WITH: - return ends_with() diff --git a/langfuse/api/unstable/commons/types/evaluation_rule_target.py b/langfuse/api/unstable/commons/types/evaluation_rule_target.py deleted file mode 100644 index b3cfbb5ae..000000000 --- a/langfuse/api/unstable/commons/types/evaluation_rule_target.py +++ /dev/null @@ -1,33 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core import enum - -T_Result = typing.TypeVar("T_Result") - - -class EvaluationRuleTarget(enum.StrEnum): - """ - The ingestion object type that should trigger evaluation runs. - - Choose the target first, because it changes both the valid filter columns and the valid variable-mapping sources: - - `observation` evaluates live-ingested observations such as generations, spans, and events. - It supports mapping from `input`, `output`, `metadata`, and `tool_calls`. - - `experiment` evaluates live experiment executions and can additionally map `expected_output` and `experiment_item_metadata`. - It currently supports filtering by `datasetId`. - Discover valid dataset IDs with `GET /api/public/v2/datasets`, then use the returned dataset `id` values in your filter. - """ - - OBSERVATION = "observation" - EXPERIMENT = "experiment" - - def visit( - self, - observation: typing.Callable[[], T_Result], - experiment: typing.Callable[[], T_Result], - ) -> T_Result: - if self is EvaluationRuleTarget.OBSERVATION: - return observation() - if self is EvaluationRuleTarget.EXPERIMENT: - return experiment() diff --git a/langfuse/api/unstable/commons/types/evaluator_model_config.py b/langfuse/api/unstable/commons/types/evaluator_model_config.py deleted file mode 100644 index 5473cca8f..000000000 --- a/langfuse/api/unstable/commons/types/evaluator_model_config.py +++ /dev/null @@ -1,46 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel - - -class EvaluatorModelConfig(UniversalBaseModel): - """ - Optional explicit model configuration for an evaluator. - - If omitted, Langfuse uses the project's default evaluation model. - If provided, the model must be available to the project when the evaluator or evaluation rule is enabled. - - To discover valid configured `provider` values for a project, call `GET /api/public/llm-connections` and read the `provider` field from the returned connections. - Use a `provider` value that matches one of the connections already configured in the same project. - - Recovery guidance: - - If evaluator creation returns `422` with `code=evaluator_preflight_failed`, either provide a valid explicit `modelConfig` here or configure the project's default evaluation model, then retry the same request. - - Examples - -------- - from langfuse.unstable.commons import EvaluatorModelConfig - - EvaluatorModelConfig( - provider="openai", - model="gpt-4.1-mini", - ) - """ - - provider: str = pydantic.Field() - """ - Provider identifier to use for this evaluator, for example `openai` or `anthropic`. - - To discover valid values for the current project, call `GET /api/public/llm-connections` and use one of the returned `provider` values. - """ - - model: str = pydantic.Field() - """ - Model identifier exposed by the provider, for example `gpt-4.1-mini`. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/evaluator_output_data_type.py b/langfuse/api/unstable/commons/types/evaluator_output_data_type.py deleted file mode 100644 index a6c309868..000000000 --- a/langfuse/api/unstable/commons/types/evaluator_output_data_type.py +++ /dev/null @@ -1,35 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core import enum - -T_Result = typing.TypeVar("T_Result") - - -class EvaluatorOutputDataType(enum.StrEnum): - """ - Structured score type returned by an evaluator. - - This controls the type of score value Langfuse stores for evaluation results: - - `NUMERIC`: a numeric score such as `0.82` - - `BOOLEAN`: a boolean score such as `true` - - `CATEGORICAL`: one or more category labels from a fixed list - """ - - NUMERIC = "NUMERIC" - BOOLEAN = "BOOLEAN" - CATEGORICAL = "CATEGORICAL" - - def visit( - self, - numeric: typing.Callable[[], T_Result], - boolean: typing.Callable[[], T_Result], - categorical: typing.Callable[[], T_Result], - ) -> T_Result: - if self is EvaluatorOutputDataType.NUMERIC: - return numeric() - if self is EvaluatorOutputDataType.BOOLEAN: - return boolean() - if self is EvaluatorOutputDataType.CATEGORICAL: - return categorical() diff --git a/langfuse/api/unstable/commons/types/evaluator_output_definition.py b/langfuse/api/unstable/commons/types/evaluator_output_definition.py deleted file mode 100644 index 661d142bb..000000000 --- a/langfuse/api/unstable/commons/types/evaluator_output_definition.py +++ /dev/null @@ -1,179 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -from __future__ import annotations - -import typing - -import pydantic -import typing_extensions -from ....core.pydantic_utilities import UniversalBaseModel -from ....core.serialization import FieldMetadata -from .evaluator_output_field_definition import EvaluatorOutputFieldDefinition -from .public_categorical_evaluator_output_score_definition import ( - PublicCategoricalEvaluatorOutputScoreDefinition, -) -from .public_numeric_evaluator_output_score_definition import ( - PublicNumericEvaluatorOutputScoreDefinition, -) - - -class EvaluatorOutputDefinition_Numeric(UniversalBaseModel): - """ - Structured output definition to send when creating an evaluator. - - Agent guidance: - - `dataType` is required. - - Do not send `version`; that is an internal storage detail and is not part of the public request contract. - - For `NUMERIC` and `BOOLEAN`, provide `reasoning.description` and `score.description`. - - For `NUMERIC`, `score.minValue` and `score.maxValue` optionally define inclusive bounds. If both are set, `minValue` must not exceed `maxValue`. - - For `CATEGORICAL`, also provide `score.categories` and `score.shouldAllowMultipleMatches`. - - Examples - -------- - from langfuse.unstable.commons import ( - EvaluatorOutputDataType, - EvaluatorOutputDefinition_Numeric, - EvaluatorOutputFieldDefinition, - PublicNumericEvaluatorOutputScoreDefinition, - ) - - EvaluatorOutputDefinition_Numeric( - data_type=EvaluatorOutputDataType.NUMERIC, - reasoning=EvaluatorOutputFieldDefinition( - description="Explain why the answer is correct or incorrect.", - ), - score=PublicNumericEvaluatorOutputScoreDefinition( - description="Return a score between 0 and 1.", - min_value=0.0, - max_value=1.0, - ), - ) - """ - - data_type: typing_extensions.Annotated[ - typing.Literal["NUMERIC"], FieldMetadata(alias="dataType") - ] = "NUMERIC" - reasoning: EvaluatorOutputFieldDefinition - score: PublicNumericEvaluatorOutputScoreDefinition - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -class EvaluatorOutputDefinition_Boolean(UniversalBaseModel): - """ - Structured output definition to send when creating an evaluator. - - Agent guidance: - - `dataType` is required. - - Do not send `version`; that is an internal storage detail and is not part of the public request contract. - - For `NUMERIC` and `BOOLEAN`, provide `reasoning.description` and `score.description`. - - For `NUMERIC`, `score.minValue` and `score.maxValue` optionally define inclusive bounds. If both are set, `minValue` must not exceed `maxValue`. - - For `CATEGORICAL`, also provide `score.categories` and `score.shouldAllowMultipleMatches`. - - Examples - -------- - from langfuse.unstable.commons import ( - EvaluatorOutputDataType, - EvaluatorOutputDefinition_Numeric, - EvaluatorOutputFieldDefinition, - PublicNumericEvaluatorOutputScoreDefinition, - ) - - EvaluatorOutputDefinition_Numeric( - data_type=EvaluatorOutputDataType.NUMERIC, - reasoning=EvaluatorOutputFieldDefinition( - description="Explain why the answer is correct or incorrect.", - ), - score=PublicNumericEvaluatorOutputScoreDefinition( - description="Return a score between 0 and 1.", - min_value=0.0, - max_value=1.0, - ), - ) - """ - - data_type: typing_extensions.Annotated[ - typing.Literal["BOOLEAN"], FieldMetadata(alias="dataType") - ] = "BOOLEAN" - reasoning: EvaluatorOutputFieldDefinition - score: EvaluatorOutputFieldDefinition - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -class EvaluatorOutputDefinition_Categorical(UniversalBaseModel): - """ - Structured output definition to send when creating an evaluator. - - Agent guidance: - - `dataType` is required. - - Do not send `version`; that is an internal storage detail and is not part of the public request contract. - - For `NUMERIC` and `BOOLEAN`, provide `reasoning.description` and `score.description`. - - For `NUMERIC`, `score.minValue` and `score.maxValue` optionally define inclusive bounds. If both are set, `minValue` must not exceed `maxValue`. - - For `CATEGORICAL`, also provide `score.categories` and `score.shouldAllowMultipleMatches`. - - Examples - -------- - from langfuse.unstable.commons import ( - EvaluatorOutputDataType, - EvaluatorOutputDefinition_Numeric, - EvaluatorOutputFieldDefinition, - PublicNumericEvaluatorOutputScoreDefinition, - ) - - EvaluatorOutputDefinition_Numeric( - data_type=EvaluatorOutputDataType.NUMERIC, - reasoning=EvaluatorOutputFieldDefinition( - description="Explain why the answer is correct or incorrect.", - ), - score=PublicNumericEvaluatorOutputScoreDefinition( - description="Return a score between 0 and 1.", - min_value=0.0, - max_value=1.0, - ), - ) - """ - - data_type: typing_extensions.Annotated[ - typing.Literal["CATEGORICAL"], FieldMetadata(alias="dataType") - ] = "CATEGORICAL" - reasoning: EvaluatorOutputFieldDefinition - score: PublicCategoricalEvaluatorOutputScoreDefinition - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -""" -from langfuse.unstable.commons import ( - EvaluatorOutputDataType, - EvaluatorOutputDefinition_Numeric, - EvaluatorOutputFieldDefinition, - PublicNumericEvaluatorOutputScoreDefinition, -) - -EvaluatorOutputDefinition_Numeric( - data_type=EvaluatorOutputDataType.NUMERIC, - reasoning=EvaluatorOutputFieldDefinition( - description="Explain why the answer is correct or incorrect.", - ), - score=PublicNumericEvaluatorOutputScoreDefinition( - description="Return a score between 0 and 1.", - min_value=0.0, - max_value=1.0, - ), -) -""" -EvaluatorOutputDefinition = typing_extensions.Annotated[ - typing.Union[ - EvaluatorOutputDefinition_Numeric, - EvaluatorOutputDefinition_Boolean, - EvaluatorOutputDefinition_Categorical, - ], - pydantic.Field(discriminator="data_type"), -] diff --git a/langfuse/api/unstable/commons/types/evaluator_output_field_definition.py b/langfuse/api/unstable/commons/types/evaluator_output_field_definition.py deleted file mode 100644 index 419610d0a..000000000 --- a/langfuse/api/unstable/commons/types/evaluator_output_field_definition.py +++ /dev/null @@ -1,17 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel - - -class EvaluatorOutputFieldDefinition(UniversalBaseModel): - description: str = pydantic.Field() - """ - Human-readable instructions for what the evaluator should return in this field. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/evaluator_type.py b/langfuse/api/unstable/commons/types/evaluator_type.py deleted file mode 100644 index 633eb30ff..000000000 --- a/langfuse/api/unstable/commons/types/evaluator_type.py +++ /dev/null @@ -1,28 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core import enum - -T_Result = typing.TypeVar("T_Result") - - -class EvaluatorType(enum.StrEnum): - """ - The evaluator type. - - The unstable public API supports LLM-as-a-judge and code evaluators. - """ - - LLM_AS_JUDGE = "llm_as_judge" - CODE = "code" - - def visit( - self, - llm_as_judge: typing.Callable[[], T_Result], - code: typing.Callable[[], T_Result], - ) -> T_Result: - if self is EvaluatorType.LLM_AS_JUDGE: - return llm_as_judge() - if self is EvaluatorType.CODE: - return code() diff --git a/langfuse/api/unstable/commons/types/null_evaluation_rule_filter.py b/langfuse/api/unstable/commons/types/null_evaluation_rule_filter.py deleted file mode 100644 index d224d7590..000000000 --- a/langfuse/api/unstable/commons/types/null_evaluation_rule_filter.py +++ /dev/null @@ -1,24 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from .evaluation_rule_null_filter_operator import EvaluationRuleNullFilterOperator - - -class NullEvaluationRuleFilter(UniversalBaseModel): - column: str = pydantic.Field() - """ - Column to filter on. In the unstable public API this is currently `parentObservationId`. - """ - - operator: EvaluationRuleNullFilterOperator - value: typing.Optional[str] = pydantic.Field(default=None) - """ - Ignored placeholder value. Clients may omit it or send an empty string. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/number_evaluation_rule_filter.py b/langfuse/api/unstable/commons/types/number_evaluation_rule_filter.py deleted file mode 100644 index f9c489291..000000000 --- a/langfuse/api/unstable/commons/types/number_evaluation_rule_filter.py +++ /dev/null @@ -1,21 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from .evaluation_rule_number_filter_operator import EvaluationRuleNumberFilterOperator - - -class NumberEvaluationRuleFilter(UniversalBaseModel): - column: str = pydantic.Field() - """ - Column to filter on. - """ - - operator: EvaluationRuleNumberFilterOperator - value: float - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/number_object_evaluation_rule_filter.py b/langfuse/api/unstable/commons/types/number_object_evaluation_rule_filter.py deleted file mode 100644 index fd9462174..000000000 --- a/langfuse/api/unstable/commons/types/number_object_evaluation_rule_filter.py +++ /dev/null @@ -1,26 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from .evaluation_rule_number_filter_operator import EvaluationRuleNumberFilterOperator - - -class NumberObjectEvaluationRuleFilter(UniversalBaseModel): - column: str = pydantic.Field() - """ - Object-valued column to filter on. - """ - - key: str = pydantic.Field() - """ - Key inside the object-valued column to filter on. - """ - - operator: EvaluationRuleNumberFilterOperator - value: float - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/prompt_variable_mapping_input.py b/langfuse/api/unstable/commons/types/prompt_variable_mapping_input.py deleted file mode 100644 index 478eabe52..000000000 --- a/langfuse/api/unstable/commons/types/prompt_variable_mapping_input.py +++ /dev/null @@ -1,76 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ....core.pydantic_utilities import UniversalBaseModel -from ....core.serialization import FieldMetadata -from .prompt_variable_mapping_source import PromptVariableMappingSource - - -class PromptVariableMappingInput(UniversalBaseModel): - """ - Connects one prompt variable to the available evaluation data. - - Manual mappings are used for `llm_as_judge` evaluators. `code` evaluators use a fixed runtime mapping managed by Langfuse. - - How to build a valid mapping list: - 1. Create the evaluator or fetch it with `GET /evaluators/{id}`. - 2. Read the evaluator `variables` array. - 3. Add exactly one mapping object for each variable in that array. - 4. Use the variable name exactly as returned, without braces such as `{{` or `}}`. - 5. Choose a `source` that is valid for the selected `target`. - - `jsonPath` is optional. Use it only when the selected source is a JSON object and you want to extract one nested field before inserting it into the evaluator prompt. - - Recovery guidance: - - `invalid_variable_mapping`: the variable name is unknown for this evaluator, or the selected `source` is not valid for the chosen `target` - - `missing_variable_mapping`: one or more LLM-as-judge prompt variables are not mapped yet - - `duplicate_variable_mapping`: the same prompt variable appears more than once - - `invalid_json_path`: the JSONPath expression is malformed. Remove it or correct it. - - Examples - -------- - from langfuse.unstable.commons import ( - PromptVariableMappingInput, - PromptVariableMappingSource, - ) - - PromptVariableMappingInput( - variable="input", - source=PromptVariableMappingSource.INPUT, - ) - """ - - variable: str = pydantic.Field() - """ - Prompt variable name without braces. - - Example: for the prompt `Judge {{input}} against {{output}}`, use `input` and `output`. - """ - - source: PromptVariableMappingSource = pydantic.Field() - """ - Source field that should populate the prompt variable. - - Quick reference: - - `target=observation`: `input`, `output`, `metadata`, `tool_calls` - - `target=experiment`: `input`, `output`, `metadata`, `tool_calls`, `expected_output`, `experiment_item_metadata` - """ - - json_path: typing_extensions.Annotated[ - typing.Optional[str], FieldMetadata(alias="jsonPath") - ] = pydantic.Field(default=None) - """ - Optional JSONPath selector applied to the selected source before it is passed to the evaluator prompt. - - Requirements: - - Must start with `$` - - Must be a syntactically valid JSONPath expression - - Most useful with `source=metadata` - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/prompt_variable_mapping_read.py b/langfuse/api/unstable/commons/types/prompt_variable_mapping_read.py deleted file mode 100644 index 370903955..000000000 --- a/langfuse/api/unstable/commons/types/prompt_variable_mapping_read.py +++ /dev/null @@ -1,50 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ....core.pydantic_utilities import UniversalBaseModel -from ....core.serialization import FieldMetadata -from .prompt_variable_mapping_source import PromptVariableMappingSource - - -class PromptVariableMappingRead(UniversalBaseModel): - """ - Connects one prompt variable to data from a target. - - `source` is `null` when the mapping is incomplete. - - Examples - -------- - from langfuse.unstable.commons import ( - PromptVariableMappingRead, - PromptVariableMappingSource, - ) - - PromptVariableMappingRead( - variable="input", - source=PromptVariableMappingSource.INPUT, - ) - """ - - variable: str = pydantic.Field() - """ - Prompt variable name without braces. - """ - - source: typing.Optional[PromptVariableMappingSource] = pydantic.Field(default=None) - """ - Source field populating the variable, or `null` when the mapping is incomplete. - """ - - json_path: typing_extensions.Annotated[ - typing.Optional[str], FieldMetadata(alias="jsonPath") - ] = pydantic.Field(default=None) - """ - Optional JSONPath selector applied to the selected source. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/prompt_variable_mapping_source.py b/langfuse/api/unstable/commons/types/prompt_variable_mapping_source.py deleted file mode 100644 index 37e5e19c4..000000000 --- a/langfuse/api/unstable/commons/types/prompt_variable_mapping_source.py +++ /dev/null @@ -1,56 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core import enum - -T_Result = typing.TypeVar("T_Result") - - -class PromptVariableMappingSource(enum.StrEnum): - """ - Source field used to populate a prompt variable. - - Use these values when mapping evaluator prompt variables to live data. - - Target-specific rules: - - `target=observation` supports `input`, `output`, `metadata`, and `tool_calls` - - `target=experiment` supports `input`, `output`, `metadata`, `tool_calls`, `expected_output`, and `experiment_item_metadata` - - Source semantics: - - `input`: the observation or experiment input payload - - `output`: the observation or experiment output payload - - `metadata`: the metadata object for the target. Combine with `jsonPath` when you need one nested field instead of the whole object. - - `tool_calls`: the tool calls recorded on the observation, as an array of `{id, name, arguments, type, index}` objects in the order the model emitted them. Combine with `jsonPath` (for example `$[*].name`) to select parts of each call. - - `expected_output`: the experiment item's expected output. Only valid for `target=experiment`. - - `experiment_item_metadata`: the experiment item's metadata object. Only valid for `target=experiment`. - """ - - INPUT = "input" - OUTPUT = "output" - METADATA = "metadata" - TOOL_CALLS = "tool_calls" - EXPECTED_OUTPUT = "expected_output" - EXPERIMENT_ITEM_METADATA = "experiment_item_metadata" - - def visit( - self, - input: typing.Callable[[], T_Result], - output: typing.Callable[[], T_Result], - metadata: typing.Callable[[], T_Result], - tool_calls: typing.Callable[[], T_Result], - expected_output: typing.Callable[[], T_Result], - experiment_item_metadata: typing.Callable[[], T_Result], - ) -> T_Result: - if self is PromptVariableMappingSource.INPUT: - return input() - if self is PromptVariableMappingSource.OUTPUT: - return output() - if self is PromptVariableMappingSource.METADATA: - return metadata() - if self is PromptVariableMappingSource.TOOL_CALLS: - return tool_calls() - if self is PromptVariableMappingSource.EXPECTED_OUTPUT: - return expected_output() - if self is PromptVariableMappingSource.EXPERIMENT_ITEM_METADATA: - return experiment_item_metadata() diff --git a/langfuse/api/unstable/commons/types/public_boolean_evaluator_output_definition.py b/langfuse/api/unstable/commons/types/public_boolean_evaluator_output_definition.py deleted file mode 100644 index 7baaf209a..000000000 --- a/langfuse/api/unstable/commons/types/public_boolean_evaluator_output_definition.py +++ /dev/null @@ -1,26 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ....core.pydantic_utilities import UniversalBaseModel -from ....core.serialization import FieldMetadata -from .evaluator_output_data_type import EvaluatorOutputDataType -from .evaluator_output_field_definition import EvaluatorOutputFieldDefinition - - -class PublicBooleanEvaluatorOutputDefinition(UniversalBaseModel): - data_type: typing_extensions.Annotated[ - EvaluatorOutputDataType, FieldMetadata(alias="dataType") - ] = pydantic.Field() - """ - Always `BOOLEAN`. - """ - - reasoning: EvaluatorOutputFieldDefinition - score: EvaluatorOutputFieldDefinition - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/public_categorical_evaluator_output_definition.py b/langfuse/api/unstable/commons/types/public_categorical_evaluator_output_definition.py deleted file mode 100644 index 30d4673bb..000000000 --- a/langfuse/api/unstable/commons/types/public_categorical_evaluator_output_definition.py +++ /dev/null @@ -1,29 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ....core.pydantic_utilities import UniversalBaseModel -from ....core.serialization import FieldMetadata -from .evaluator_output_data_type import EvaluatorOutputDataType -from .evaluator_output_field_definition import EvaluatorOutputFieldDefinition -from .public_categorical_evaluator_output_score_definition import ( - PublicCategoricalEvaluatorOutputScoreDefinition, -) - - -class PublicCategoricalEvaluatorOutputDefinition(UniversalBaseModel): - data_type: typing_extensions.Annotated[ - EvaluatorOutputDataType, FieldMetadata(alias="dataType") - ] = pydantic.Field() - """ - Always `CATEGORICAL`. - """ - - reasoning: EvaluatorOutputFieldDefinition - score: PublicCategoricalEvaluatorOutputScoreDefinition - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/public_categorical_evaluator_output_score_definition.py b/langfuse/api/unstable/commons/types/public_categorical_evaluator_output_score_definition.py deleted file mode 100644 index 81deadb93..000000000 --- a/langfuse/api/unstable/commons/types/public_categorical_evaluator_output_score_definition.py +++ /dev/null @@ -1,20 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ....core.pydantic_utilities import UniversalBaseModel -from ....core.serialization import FieldMetadata - - -class PublicCategoricalEvaluatorOutputScoreDefinition(UniversalBaseModel): - description: str - categories: typing.List[str] - should_allow_multiple_matches: typing_extensions.Annotated[ - bool, FieldMetadata(alias="shouldAllowMultipleMatches") - ] - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/public_evaluator_output_definition.py b/langfuse/api/unstable/commons/types/public_evaluator_output_definition.py deleted file mode 100644 index 52331e6c1..000000000 --- a/langfuse/api/unstable/commons/types/public_evaluator_output_definition.py +++ /dev/null @@ -1,182 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -from __future__ import annotations - -import typing - -import pydantic -import typing_extensions -from ....core.pydantic_utilities import UniversalBaseModel -from ....core.serialization import FieldMetadata -from .evaluator_output_field_definition import EvaluatorOutputFieldDefinition -from .public_categorical_evaluator_output_score_definition import ( - PublicCategoricalEvaluatorOutputScoreDefinition, -) -from .public_numeric_evaluator_output_score_definition import ( - PublicNumericEvaluatorOutputScoreDefinition, -) - - -class PublicEvaluatorOutputDefinition_Numeric(UniversalBaseModel): - """ - Evaluator output definition returned by the public API. - - This response always includes `dataType` and never includes an internal output-definition `version`. - Legacy stored evaluator definitions are normalized into this shape before they are returned. - - Use this response shape when deciding how to interpret future evaluation scores: - - `NUMERIC`: expect numeric score values - - `BOOLEAN`: expect `true` / `false` - - `CATEGORICAL`: expect one or more values from `score.categories` - - Examples - -------- - from langfuse.unstable.commons import ( - EvaluatorOutputDataType, - EvaluatorOutputFieldDefinition, - PublicEvaluatorOutputDefinition_Numeric, - PublicNumericEvaluatorOutputScoreDefinition, - ) - - PublicEvaluatorOutputDefinition_Numeric( - data_type=EvaluatorOutputDataType.NUMERIC, - reasoning=EvaluatorOutputFieldDefinition( - description="Explain why the answer is correct or incorrect.", - ), - score=PublicNumericEvaluatorOutputScoreDefinition( - description="Return a score between 0 and 1.", - min_value=0.0, - max_value=1.0, - ), - ) - """ - - data_type: typing_extensions.Annotated[ - typing.Literal["NUMERIC"], FieldMetadata(alias="dataType") - ] = "NUMERIC" - reasoning: EvaluatorOutputFieldDefinition - score: PublicNumericEvaluatorOutputScoreDefinition - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -class PublicEvaluatorOutputDefinition_Boolean(UniversalBaseModel): - """ - Evaluator output definition returned by the public API. - - This response always includes `dataType` and never includes an internal output-definition `version`. - Legacy stored evaluator definitions are normalized into this shape before they are returned. - - Use this response shape when deciding how to interpret future evaluation scores: - - `NUMERIC`: expect numeric score values - - `BOOLEAN`: expect `true` / `false` - - `CATEGORICAL`: expect one or more values from `score.categories` - - Examples - -------- - from langfuse.unstable.commons import ( - EvaluatorOutputDataType, - EvaluatorOutputFieldDefinition, - PublicEvaluatorOutputDefinition_Numeric, - PublicNumericEvaluatorOutputScoreDefinition, - ) - - PublicEvaluatorOutputDefinition_Numeric( - data_type=EvaluatorOutputDataType.NUMERIC, - reasoning=EvaluatorOutputFieldDefinition( - description="Explain why the answer is correct or incorrect.", - ), - score=PublicNumericEvaluatorOutputScoreDefinition( - description="Return a score between 0 and 1.", - min_value=0.0, - max_value=1.0, - ), - ) - """ - - data_type: typing_extensions.Annotated[ - typing.Literal["BOOLEAN"], FieldMetadata(alias="dataType") - ] = "BOOLEAN" - reasoning: EvaluatorOutputFieldDefinition - score: EvaluatorOutputFieldDefinition - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -class PublicEvaluatorOutputDefinition_Categorical(UniversalBaseModel): - """ - Evaluator output definition returned by the public API. - - This response always includes `dataType` and never includes an internal output-definition `version`. - Legacy stored evaluator definitions are normalized into this shape before they are returned. - - Use this response shape when deciding how to interpret future evaluation scores: - - `NUMERIC`: expect numeric score values - - `BOOLEAN`: expect `true` / `false` - - `CATEGORICAL`: expect one or more values from `score.categories` - - Examples - -------- - from langfuse.unstable.commons import ( - EvaluatorOutputDataType, - EvaluatorOutputFieldDefinition, - PublicEvaluatorOutputDefinition_Numeric, - PublicNumericEvaluatorOutputScoreDefinition, - ) - - PublicEvaluatorOutputDefinition_Numeric( - data_type=EvaluatorOutputDataType.NUMERIC, - reasoning=EvaluatorOutputFieldDefinition( - description="Explain why the answer is correct or incorrect.", - ), - score=PublicNumericEvaluatorOutputScoreDefinition( - description="Return a score between 0 and 1.", - min_value=0.0, - max_value=1.0, - ), - ) - """ - - data_type: typing_extensions.Annotated[ - typing.Literal["CATEGORICAL"], FieldMetadata(alias="dataType") - ] = "CATEGORICAL" - reasoning: EvaluatorOutputFieldDefinition - score: PublicCategoricalEvaluatorOutputScoreDefinition - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -""" -from langfuse.unstable.commons import ( - EvaluatorOutputDataType, - EvaluatorOutputFieldDefinition, - PublicEvaluatorOutputDefinition_Numeric, - PublicNumericEvaluatorOutputScoreDefinition, -) - -PublicEvaluatorOutputDefinition_Numeric( - data_type=EvaluatorOutputDataType.NUMERIC, - reasoning=EvaluatorOutputFieldDefinition( - description="Explain why the answer is correct or incorrect.", - ), - score=PublicNumericEvaluatorOutputScoreDefinition( - description="Return a score between 0 and 1.", - min_value=0.0, - max_value=1.0, - ), -) -""" -PublicEvaluatorOutputDefinition = typing_extensions.Annotated[ - typing.Union[ - PublicEvaluatorOutputDefinition_Numeric, - PublicEvaluatorOutputDefinition_Boolean, - PublicEvaluatorOutputDefinition_Categorical, - ], - pydantic.Field(discriminator="data_type"), -] diff --git a/langfuse/api/unstable/commons/types/public_numeric_evaluator_output_definition.py b/langfuse/api/unstable/commons/types/public_numeric_evaluator_output_definition.py deleted file mode 100644 index db83d0121..000000000 --- a/langfuse/api/unstable/commons/types/public_numeric_evaluator_output_definition.py +++ /dev/null @@ -1,29 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ....core.pydantic_utilities import UniversalBaseModel -from ....core.serialization import FieldMetadata -from .evaluator_output_data_type import EvaluatorOutputDataType -from .evaluator_output_field_definition import EvaluatorOutputFieldDefinition -from .public_numeric_evaluator_output_score_definition import ( - PublicNumericEvaluatorOutputScoreDefinition, -) - - -class PublicNumericEvaluatorOutputDefinition(UniversalBaseModel): - data_type: typing_extensions.Annotated[ - EvaluatorOutputDataType, FieldMetadata(alias="dataType") - ] = pydantic.Field() - """ - Always `NUMERIC`. - """ - - reasoning: EvaluatorOutputFieldDefinition - score: PublicNumericEvaluatorOutputScoreDefinition - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/public_numeric_evaluator_output_score_definition.py b/langfuse/api/unstable/commons/types/public_numeric_evaluator_output_score_definition.py deleted file mode 100644 index 7f8c39597..000000000 --- a/langfuse/api/unstable/commons/types/public_numeric_evaluator_output_score_definition.py +++ /dev/null @@ -1,29 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ....core.pydantic_utilities import UniversalBaseModel -from ....core.serialization import FieldMetadata - - -class PublicNumericEvaluatorOutputScoreDefinition(UniversalBaseModel): - description: str - min_value: typing_extensions.Annotated[ - typing.Optional[float], FieldMetadata(alias="minValue") - ] = pydantic.Field(default=None) - """ - Inclusive minimum value for numeric evaluator scores. - """ - - max_value: typing_extensions.Annotated[ - typing.Optional[float], FieldMetadata(alias="maxValue") - ] = pydantic.Field(default=None) - """ - Inclusive maximum value for numeric evaluator scores. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/string_evaluation_rule_filter.py b/langfuse/api/unstable/commons/types/string_evaluation_rule_filter.py deleted file mode 100644 index bd9332092..000000000 --- a/langfuse/api/unstable/commons/types/string_evaluation_rule_filter.py +++ /dev/null @@ -1,21 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from .evaluation_rule_string_filter_operator import EvaluationRuleStringFilterOperator - - -class StringEvaluationRuleFilter(UniversalBaseModel): - column: str = pydantic.Field() - """ - Column to filter on. - """ - - operator: EvaluationRuleStringFilterOperator - value: str - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/string_object_evaluation_rule_filter.py b/langfuse/api/unstable/commons/types/string_object_evaluation_rule_filter.py deleted file mode 100644 index 6c287aad6..000000000 --- a/langfuse/api/unstable/commons/types/string_object_evaluation_rule_filter.py +++ /dev/null @@ -1,26 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from .evaluation_rule_string_filter_operator import EvaluationRuleStringFilterOperator - - -class StringObjectEvaluationRuleFilter(UniversalBaseModel): - column: str = pydantic.Field() - """ - Object-valued column to filter on. In the unstable public API this is currently `metadata`. - """ - - key: str = pydantic.Field() - """ - Top-level key inside the object-valued column to filter on. - """ - - operator: EvaluationRuleStringFilterOperator - value: str - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/commons/types/string_options_evaluation_rule_filter.py b/langfuse/api/unstable/commons/types/string_options_evaluation_rule_filter.py deleted file mode 100644 index a830e5ad9..000000000 --- a/langfuse/api/unstable/commons/types/string_options_evaluation_rule_filter.py +++ /dev/null @@ -1,24 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from .evaluation_rule_options_filter_operator import EvaluationRuleOptionsFilterOperator - - -class StringOptionsEvaluationRuleFilter(UniversalBaseModel): - column: str = pydantic.Field() - """ - Column to filter on. - """ - - operator: EvaluationRuleOptionsFilterOperator - value: typing.List[str] = pydantic.Field() - """ - One or more allowed string values. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/errors/__init__.py b/langfuse/api/unstable/errors/__init__.py index 42f230c41..da8bdfd78 100644 --- a/langfuse/api/unstable/errors/__init__.py +++ b/langfuse/api/unstable/errors/__init__.py @@ -21,7 +21,6 @@ NotFoundError, TooManyRequestsError, UnauthorizedError, - UnprocessableContentError, ) _dynamic_imports: typing.Dict[str, str] = { "AccessDeniedError": ".errors", @@ -36,7 +35,6 @@ "PublicApiValidationIssue": ".types", "TooManyRequestsError": ".errors", "UnauthorizedError": ".errors", - "UnprocessableContentError": ".errors", } @@ -80,5 +78,4 @@ def __dir__(): "PublicApiValidationIssue", "TooManyRequestsError", "UnauthorizedError", - "UnprocessableContentError", ] diff --git a/langfuse/api/unstable/errors/errors/__init__.py b/langfuse/api/unstable/errors/errors/__init__.py index 510e3beb1..0651c1d6f 100644 --- a/langfuse/api/unstable/errors/errors/__init__.py +++ b/langfuse/api/unstable/errors/errors/__init__.py @@ -14,7 +14,6 @@ from .not_found_error import NotFoundError from .too_many_requests_error import TooManyRequestsError from .unauthorized_error import UnauthorizedError - from .unprocessable_content_error import UnprocessableContentError _dynamic_imports: typing.Dict[str, str] = { "AccessDeniedError": ".access_denied_error", "BadRequestError": ".bad_request_error", @@ -24,7 +23,6 @@ "NotFoundError": ".not_found_error", "TooManyRequestsError": ".too_many_requests_error", "UnauthorizedError": ".unauthorized_error", - "UnprocessableContentError": ".unprocessable_content_error", } @@ -64,5 +62,4 @@ def __dir__(): "NotFoundError", "TooManyRequestsError", "UnauthorizedError", - "UnprocessableContentError", ] diff --git a/langfuse/api/unstable/errors/errors/unprocessable_content_error.py b/langfuse/api/unstable/errors/errors/unprocessable_content_error.py deleted file mode 100644 index a701ef9c5..000000000 --- a/langfuse/api/unstable/errors/errors/unprocessable_content_error.py +++ /dev/null @@ -1,15 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core.api_error import ApiError -from ..types.public_api_error import PublicApiError - - -class UnprocessableContentError(ApiError): - def __init__( - self, - body: PublicApiError, - headers: typing.Optional[typing.Dict[str, str]] = None, - ): - super().__init__(status_code=422, headers=headers, body=body) diff --git a/langfuse/api/unstable/errors/types/public_api_error.py b/langfuse/api/unstable/errors/types/public_api_error.py index 5d1384e7c..d7b34c366 100644 --- a/langfuse/api/unstable/errors/types/public_api_error.py +++ b/langfuse/api/unstable/errors/types/public_api_error.py @@ -10,32 +10,7 @@ class PublicApiError(UniversalBaseModel): """ - Standard error envelope for the unstable evaluators API. - - Response handling guidance: - - Use the HTTP status code for the broad class of failure. - - Use `code` for precise branching in SDKs, CLIs, or agents. - - Inspect `details` for field-level validation context such as invalid filter values, malformed JSONPath expressions, or missing variable mappings. - - Retry only after fixing the specific issue described by `code` and `details`. - - Examples - -------- - from langfuse.unstable.errors import ( - PublicApiError, - PublicApiErrorCode, - PublicApiErrorDetails, - ) - - PublicApiError( - message='Filter column "type" contains unsupported value(s): INVALID', - code=PublicApiErrorCode.INVALID_FILTER_VALUE, - details=PublicApiErrorDetails( - field="filter[0].value", - column="type", - invalid_values=["INVALID"], - allowed_values=["GENERATION", "SPAN", "EVENT"], - ), - ) + Standard error envelope for unstable public API endpoints. """ message: str = pydantic.Field() @@ -45,12 +20,12 @@ class PublicApiError(UniversalBaseModel): code: PublicApiErrorCode = pydantic.Field() """ - Stable machine-readable error code. + Machine-readable error code. """ details: typing.Optional[PublicApiErrorDetails] = pydantic.Field(default=None) """ - Optional structured error context. Inspect the populated fields based on `code`. + Optional structured error context. """ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( diff --git a/langfuse/api/unstable/errors/types/public_api_error_code.py b/langfuse/api/unstable/errors/types/public_api_error_code.py index fe8f67f83..c968d1f1a 100644 --- a/langfuse/api/unstable/errors/types/public_api_error_code.py +++ b/langfuse/api/unstable/errors/types/public_api_error_code.py @@ -9,10 +9,7 @@ class PublicApiErrorCode(enum.StrEnum): """ - Machine-readable error code returned by the unstable evaluators API. - - SDKs, CLIs, and agents should branch on `code` rather than parsing the human-readable `message`. - The HTTP status still indicates the broad error class, while `code` gives the specific failure reason. + Machine-readable error code returned by unstable public API endpoints. """ AUTHENTICATION_FAILED = "authentication_failed" @@ -20,16 +17,8 @@ class PublicApiErrorCode(enum.StrEnum): INVALID_REQUEST = "invalid_request" INVALID_QUERY = "invalid_query" INVALID_BODY = "invalid_body" - INVALID_FILTER_VALUE = "invalid_filter_value" - INVALID_JSON_PATH = "invalid_json_path" - INVALID_VARIABLE_MAPPING = "invalid_variable_mapping" - MISSING_VARIABLE_MAPPING = "missing_variable_mapping" - DUPLICATE_VARIABLE_MAPPING = "duplicate_variable_mapping" RESOURCE_NOT_FOUND = "resource_not_found" - NAME_CONFLICT = "name_conflict" - EVALUATOR_PREFLIGHT_FAILED = "evaluator_preflight_failed" CONFLICT = "conflict" - UNPROCESSABLE_CONTENT = "unprocessable_content" RATE_LIMITED = "rate_limited" METHOD_NOT_ALLOWED = "method_not_allowed" INTERNAL_ERROR = "internal_error" @@ -41,16 +30,8 @@ def visit( invalid_request: typing.Callable[[], T_Result], invalid_query: typing.Callable[[], T_Result], invalid_body: typing.Callable[[], T_Result], - invalid_filter_value: typing.Callable[[], T_Result], - invalid_json_path: typing.Callable[[], T_Result], - invalid_variable_mapping: typing.Callable[[], T_Result], - missing_variable_mapping: typing.Callable[[], T_Result], - duplicate_variable_mapping: typing.Callable[[], T_Result], resource_not_found: typing.Callable[[], T_Result], - name_conflict: typing.Callable[[], T_Result], - evaluator_preflight_failed: typing.Callable[[], T_Result], conflict: typing.Callable[[], T_Result], - unprocessable_content: typing.Callable[[], T_Result], rate_limited: typing.Callable[[], T_Result], method_not_allowed: typing.Callable[[], T_Result], internal_error: typing.Callable[[], T_Result], @@ -65,26 +46,10 @@ def visit( return invalid_query() if self is PublicApiErrorCode.INVALID_BODY: return invalid_body() - if self is PublicApiErrorCode.INVALID_FILTER_VALUE: - return invalid_filter_value() - if self is PublicApiErrorCode.INVALID_JSON_PATH: - return invalid_json_path() - if self is PublicApiErrorCode.INVALID_VARIABLE_MAPPING: - return invalid_variable_mapping() - if self is PublicApiErrorCode.MISSING_VARIABLE_MAPPING: - return missing_variable_mapping() - if self is PublicApiErrorCode.DUPLICATE_VARIABLE_MAPPING: - return duplicate_variable_mapping() if self is PublicApiErrorCode.RESOURCE_NOT_FOUND: return resource_not_found() - if self is PublicApiErrorCode.NAME_CONFLICT: - return name_conflict() - if self is PublicApiErrorCode.EVALUATOR_PREFLIGHT_FAILED: - return evaluator_preflight_failed() if self is PublicApiErrorCode.CONFLICT: return conflict() - if self is PublicApiErrorCode.UNPROCESSABLE_CONTENT: - return unprocessable_content() if self is PublicApiErrorCode.RATE_LIMITED: return rate_limited() if self is PublicApiErrorCode.METHOD_NOT_ALLOWED: diff --git a/langfuse/api/unstable/errors/types/public_api_error_details.py b/langfuse/api/unstable/errors/types/public_api_error_details.py index 803378164..5e534222c 100644 --- a/langfuse/api/unstable/errors/types/public_api_error_details.py +++ b/langfuse/api/unstable/errors/types/public_api_error_details.py @@ -11,103 +11,27 @@ class PublicApiErrorDetails(UniversalBaseModel): """ - Optional structured context attached to an unstable-evals error. - - The populated fields depend on the error `code`: - - request parsing failures populate `issues` - - filter validation failures populate `field`, `column`, `invalidValues`, and `allowedValues` - - variable mapping failures populate `field`, `variable`, or `variables` - - JSONPath validation failures populate `field`, `variable`, and `value` - - evaluator preflight failures populate `evaluatorName`, `provider`, and `model` - - rate limiting populates `retryAfterSeconds`, `limit`, `remaining`, and `resetAt` - """ - - issues: typing.Optional[typing.List[PublicApiValidationIssue]] = pydantic.Field( - default=None - ) - """ - Validation issues for malformed request bodies or query parameters. - """ - - field: typing.Optional[str] = pydantic.Field(default=None) - """ - Path-like reference to the failing field, for example `mapping[1].jsonPath`. - """ - - column: typing.Optional[str] = pydantic.Field(default=None) - """ - Filter column that failed validation. + Optional structured context attached to an error. """ + issues: typing.Optional[typing.List[PublicApiValidationIssue]] = None + field: typing.Optional[str] = None + column: typing.Optional[str] = None invalid_values: typing_extensions.Annotated[ typing.Optional[typing.List[str]], FieldMetadata(alias="invalidValues") - ] = pydantic.Field(default=None) - """ - Unsupported values supplied by the caller. - """ - + ] = None allowed_values: typing_extensions.Annotated[ typing.Optional[typing.List[str]], FieldMetadata(alias="allowedValues") - ] = pydantic.Field(default=None) - """ - Allowed values for the failing filter column. - """ - - variable: typing.Optional[str] = pydantic.Field(default=None) - """ - Evaluator variable involved in the failure. - """ - - variables: typing.Optional[typing.List[str]] = pydantic.Field(default=None) - """ - Multiple evaluator variables involved in the failure, for example missing mappings. - """ - - value: typing.Optional[str] = pydantic.Field(default=None) - """ - Raw invalid value supplied by the caller. - """ - - evaluator_name: typing_extensions.Annotated[ - typing.Optional[str], FieldMetadata(alias="evaluatorName") - ] = pydantic.Field(default=None) - """ - Evaluator name used during preflight validation. - """ - - provider: typing.Optional[str] = pydantic.Field(default=None) - """ - Provider resolved during evaluator preflight, if any. - """ - - model: typing.Optional[str] = pydantic.Field(default=None) - """ - Model resolved during evaluator preflight, if any. - """ - + ] = None + value: typing.Optional[str] = None retry_after_seconds: typing_extensions.Annotated[ typing.Optional[int], FieldMetadata(alias="retryAfterSeconds") - ] = pydantic.Field(default=None) - """ - Suggested retry delay for rate-limited requests. - """ - - limit: typing.Optional[int] = pydantic.Field(default=None) - """ - Numeric limit associated with the failure, for example the active evaluation-rule cap or the current rate-limit window. - """ - - remaining: typing.Optional[int] = pydantic.Field(default=None) - """ - Remaining requests in the current rate-limit window. - """ - + ] = None + limit: typing.Optional[int] = None + remaining: typing.Optional[int] = None reset_at: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="resetAt") - ] = pydantic.Field(default=None) - """ - ISO-8601 timestamp when the current rate-limit window resets. - """ + ] = None model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( extra="allow", frozen=True diff --git a/langfuse/api/unstable/errors/types/public_api_validation_issue.py b/langfuse/api/unstable/errors/types/public_api_validation_issue.py index 877d0376a..d948a9f2d 100644 --- a/langfuse/api/unstable/errors/types/public_api_validation_issue.py +++ b/langfuse/api/unstable/errors/types/public_api_validation_issue.py @@ -8,26 +8,12 @@ class PublicApiValidationIssue(UniversalBaseModel): """ - One validation issue returned for malformed request bodies or query parameters. - - This mirrors the most important parts of a Zod issue: a machine-readable `code`, - a human-readable `message`, and a structured `path`. + A validation issue returned for malformed request bodies or query parameters. """ - code: str = pydantic.Field() - """ - Machine-readable validation issue code emitted by the server validator. - """ - - message: str = pydantic.Field() - """ - Human-readable explanation of the validation failure. - """ - - path: typing.List[typing.Any] = pydantic.Field() - """ - Path to the invalid field, for example `["mapping", 0, "jsonPath"]`. - """ + code: str + message: str + path: typing.List[typing.Any] model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( extra="allow", frozen=True diff --git a/langfuse/api/unstable/evaluation_rules/__init__.py b/langfuse/api/unstable/evaluation_rules/__init__.py deleted file mode 100644 index 34995580f..000000000 --- a/langfuse/api/unstable/evaluation_rules/__init__.py +++ /dev/null @@ -1,118 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -# isort: skip_file - -import typing -from importlib import import_module - -if typing.TYPE_CHECKING: - from .types import ( - CodeEvaluationRuleEvaluatorReference, - CreateCodeEvaluationRuleRequest, - CreateEvaluationRuleEvaluatorAssignment, - CreateEvaluationRuleEvaluatorReference, - CreateEvaluationRuleRequest, - CreateEvaluationRuleWithEvaluatorsRequest, - CreateLlmAsJudgeEvaluationRuleRequest, - DeleteEvaluationRuleResponse, - EvaluationRule, - EvaluationRuleBase, - EvaluationRuleEvaluator, - EvaluationRuleEvaluatorAssignment, - EvaluationRuleEvaluatorReference, - EvaluationRuleTimeScope, - EvaluationRules, - LegacyEvaluationObject, - LegacyEvaluationRule, - LegacyEvaluationRuleEvaluatorAssignment, - LegacyEvaluationRuleTarget, - LegacyPromptVariableMapping, - LlmAsJudgeEvaluationRuleEvaluatorReference, - LlmAsJudgeEvaluatorType, - ReadableEvaluationRule, - ReadableV2EvaluationRule, - UpdateEvaluationRuleRequest, - ) -_dynamic_imports: typing.Dict[str, str] = { - "CodeEvaluationRuleEvaluatorReference": ".types", - "CreateCodeEvaluationRuleRequest": ".types", - "CreateEvaluationRuleEvaluatorAssignment": ".types", - "CreateEvaluationRuleEvaluatorReference": ".types", - "CreateEvaluationRuleRequest": ".types", - "CreateEvaluationRuleWithEvaluatorsRequest": ".types", - "CreateLlmAsJudgeEvaluationRuleRequest": ".types", - "DeleteEvaluationRuleResponse": ".types", - "EvaluationRule": ".types", - "EvaluationRuleBase": ".types", - "EvaluationRuleEvaluator": ".types", - "EvaluationRuleEvaluatorAssignment": ".types", - "EvaluationRuleEvaluatorReference": ".types", - "EvaluationRuleTimeScope": ".types", - "EvaluationRules": ".types", - "LegacyEvaluationObject": ".types", - "LegacyEvaluationRule": ".types", - "LegacyEvaluationRuleEvaluatorAssignment": ".types", - "LegacyEvaluationRuleTarget": ".types", - "LegacyPromptVariableMapping": ".types", - "LlmAsJudgeEvaluationRuleEvaluatorReference": ".types", - "LlmAsJudgeEvaluatorType": ".types", - "ReadableEvaluationRule": ".types", - "ReadableV2EvaluationRule": ".types", - "UpdateEvaluationRuleRequest": ".types", -} - - -def __getattr__(attr_name: str) -> typing.Any: - module_name = _dynamic_imports.get(attr_name) - if module_name is None: - raise AttributeError( - f"No {attr_name} found in _dynamic_imports for module name -> {__name__}" - ) - try: - module = import_module(module_name, __package__) - if module_name == f".{attr_name}": - return module - else: - return getattr(module, attr_name) - except ImportError as e: - raise ImportError( - f"Failed to import {attr_name} from {module_name}: {e}" - ) from e - except AttributeError as e: - raise AttributeError( - f"Failed to get {attr_name} from {module_name}: {e}" - ) from e - - -def __dir__(): - lazy_attrs = list(_dynamic_imports.keys()) - return sorted(lazy_attrs) - - -__all__ = [ - "CodeEvaluationRuleEvaluatorReference", - "CreateCodeEvaluationRuleRequest", - "CreateEvaluationRuleEvaluatorAssignment", - "CreateEvaluationRuleEvaluatorReference", - "CreateEvaluationRuleRequest", - "CreateEvaluationRuleWithEvaluatorsRequest", - "CreateLlmAsJudgeEvaluationRuleRequest", - "DeleteEvaluationRuleResponse", - "EvaluationRule", - "EvaluationRuleBase", - "EvaluationRuleEvaluator", - "EvaluationRuleEvaluatorAssignment", - "EvaluationRuleEvaluatorReference", - "EvaluationRuleTimeScope", - "EvaluationRules", - "LegacyEvaluationObject", - "LegacyEvaluationRule", - "LegacyEvaluationRuleEvaluatorAssignment", - "LegacyEvaluationRuleTarget", - "LegacyPromptVariableMapping", - "LlmAsJudgeEvaluationRuleEvaluatorReference", - "LlmAsJudgeEvaluatorType", - "ReadableEvaluationRule", - "ReadableV2EvaluationRule", - "UpdateEvaluationRuleRequest", -] diff --git a/langfuse/api/unstable/evaluation_rules/client.py b/langfuse/api/unstable/evaluation_rules/client.py deleted file mode 100644 index 7fe839ffe..000000000 --- a/langfuse/api/unstable/evaluation_rules/client.py +++ /dev/null @@ -1,866 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import typing_extensions -from ...core.client_wrapper import AsyncClientWrapper, SyncClientWrapper -from ...core.request_options import RequestOptions -from ..commons.types.evaluation_rule_filter import EvaluationRuleFilter -from ..commons.types.evaluation_rule_target import EvaluationRuleTarget -from ..commons.types.prompt_variable_mapping_input import PromptVariableMappingInput -from .raw_client import AsyncRawEvaluationRulesClient, RawEvaluationRulesClient -from .types.create_evaluation_rule_evaluator_assignment import ( - CreateEvaluationRuleEvaluatorAssignment, -) -from .types.create_evaluation_rule_request import CreateEvaluationRuleRequest -from .types.delete_evaluation_rule_response import DeleteEvaluationRuleResponse -from .types.evaluation_rule import EvaluationRule -from .types.evaluation_rule_evaluator_reference import EvaluationRuleEvaluatorReference -from .types.evaluation_rules import EvaluationRules -from .types.readable_evaluation_rule import ReadableEvaluationRule -from .types.readable_v2evaluation_rule import ReadableV2EvaluationRule - -# this is used as the default value for optional parameters -OMIT = typing.cast(typing.Any, ...) - - -class EvaluationRulesClient: - def __init__(self, *, client_wrapper: SyncClientWrapper): - self._raw_client = RawEvaluationRulesClient(client_wrapper=client_wrapper) - - @property - def with_raw_response(self) -> RawEvaluationRulesClient: - """ - Retrieves a raw implementation of this client that returns raw responses. - - Returns - ------- - RawEvaluationRulesClient - """ - return self._raw_client - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def create( - self, - *, - request: CreateEvaluationRuleRequest, - request_options: typing.Optional[RequestOptions] = None, - ) -> EvaluationRule: - """ - Create an evaluation rule. - - An evaluation rule defines **what** incoming data should be evaluated and **how prompt variables should be populated** from that data. - - Use this resource after choosing an evaluator from the evaluator endpoints. - - Key rules: - - `name` must be unique within the project for public evaluation rules - - `target` must be `observation` or `experiment` - - provide either the compatibility `evaluator` field or the new `evaluators` array, never both - - each evaluator `name` + `type` must identify an existing evaluator family returned by the evaluator endpoints - - Langfuse resolves that family to its latest version before saving the evaluation rule - - for `target=experiment`, use dataset `id` values from `GET /api/public/v2/datasets` when filtering by `datasetId` - - an omitted LLM-as-judge assignment mapping inherits the evaluator version's default mapping - - the effective mapping must map every evaluator prompt variable exactly once - - for `code` evaluators, Langfuse uses the fixed code runtime mapping; omit `mapping` in create and update requests - - for user-provided `llm_as_judge` mappings, `expected_output` and `experiment_item_metadata` are only valid for `target=experiment` - - if `enabled=true`, Langfuse validates that the referenced evaluator can currently run - - at most 500 evaluation rules can be effectively active in one project at the same time (enforced identically by the API, the MCP tools, and the app) - - If an evaluation rule with the same `name` already exists in the project, the API returns `409`. - In that case, update the existing resource with `PATCH /api/public/unstable/evaluation-rules/{evaluationRuleId}` instead of creating a second one. - - If enabling this resource would exceed the 500-active limit, the API also returns `409`. - In that case, disable or pause another active evaluation rule before enabling a new one. - - Current scope: - - evaluation rules are live-ingestion rules only - - they do not trigger historical backfills - - Recovery guidance: - - `400 invalid_filter_value`: fix the filter `column` or `value` using `details.column`, `details.invalidValues`, and `details.allowedValues` - - `400 invalid_filter_value` with `details.column=datasetId`: call `GET /api/public/v2/datasets`, then retry with dataset `id` values from that response - - `400 missing_variable_mapping`: for `llm_as_judge` evaluators, fetch the evaluator again and make sure every variable in `variables` appears exactly once in `mapping` - - `400 duplicate_variable_mapping`: remove repeated mappings for the same variable - - `400 invalid_variable_mapping`: for `llm_as_judge`, switch to a valid `source` for the selected `target`, or fix the variable name - - `400 invalid_json_path`: remove or correct the `jsonPath` - - `422 evaluator_preflight_failed`: the selected evaluator cannot run with the resolved model configuration. Fix the evaluator/default model setup, then retry the create request. - - Parameters - ---------- - request : CreateEvaluationRuleRequest - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - EvaluationRule - - Examples - -------- - from langfuse import LangfuseAPI - from langfuse.unstable.commons import ( - EvaluationRuleFilter_StringOptions, - EvaluationRuleOptionsFilterOperator, - EvaluationRuleTarget, - PromptVariableMappingInput, - PromptVariableMappingSource, - ) - from langfuse.unstable.evaluation_rules import ( - CreateLlmAsJudgeEvaluationRuleRequest, - LlmAsJudgeEvaluationRuleEvaluatorReference, - LlmAsJudgeEvaluatorType, - ) - - client = LangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - client.unstable.evaluation_rules.create( - request=CreateLlmAsJudgeEvaluationRuleRequest( - name="answer-correctness-live", - evaluator=LlmAsJudgeEvaluationRuleEvaluatorReference( - name="answer-correctness", - type=LlmAsJudgeEvaluatorType.LLM_AS_JUDGE, - ), - target=EvaluationRuleTarget.OBSERVATION, - enabled=True, - sampling=1.0, - filter=[ - EvaluationRuleFilter_StringOptions( - column="type", - operator=EvaluationRuleOptionsFilterOperator.ANY_OF, - value=["GENERATION"], - ) - ], - mapping=[ - PromptVariableMappingInput( - variable="input", - source=PromptVariableMappingSource.INPUT, - ), - PromptVariableMappingInput( - variable="output", - source=PromptVariableMappingSource.OUTPUT, - ), - ], - ), - ) - """ - _response = self._raw_client.create( - request=request, request_options=request_options - ) - return _response.data - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def list( - self, - *, - page: typing.Optional[int] = None, - limit: typing.Optional[int] = None, - request_options: typing.Optional[RequestOptions] = None, - ) -> EvaluationRules: - """ - List evaluation rules in the authenticated project. - - This includes legacy `trace` and `dataset` rules so they can be inspected and migrated to v4 rules. Legacy rules are read-only through this API; create, update, and delete continue to support only `observation` and `experiment` rules. - - Parameters - ---------- - page : typing.Optional[int] - 1-based page number. Defaults to `1`. - - limit : typing.Optional[int] - Maximum number of items per page. Defaults to `50`. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - EvaluationRules - - Examples - -------- - from langfuse import LangfuseAPI - - client = LangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - client.unstable.evaluation_rules.list() - """ - _response = self._raw_client.list( - page=page, limit=limit, request_options=request_options - ) - return _response.data - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def get( - self, - evaluation_rule_id: str, - *, - request_options: typing.Optional[RequestOptions] = None, - ) -> ReadableEvaluationRule: - """ - Get one evaluation rule by its identifier. - - Use this endpoint to inspect the current evaluator, target, mapping, filters, execution timing, and effective runtime status. Legacy `trace` and `dataset` rules are returned for migration and are read-only through this API. - - Parameters - ---------- - evaluation_rule_id : str - Evaluation rule identifier returned by the evaluation rule endpoints. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - ReadableEvaluationRule - - Examples - -------- - from langfuse import LangfuseAPI - - client = LangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - client.unstable.evaluation_rules.get( - evaluation_rule_id="evaluationRuleId", - ) - """ - _response = self._raw_client.get( - evaluation_rule_id, request_options=request_options - ) - return _response.data - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def update( - self, - evaluation_rule_id: str, - *, - name: typing.Optional[str] = OMIT, - evaluators: typing.Optional[ - typing.Sequence[CreateEvaluationRuleEvaluatorAssignment] - ] = OMIT, - evaluator: typing.Optional[EvaluationRuleEvaluatorReference] = OMIT, - target: typing.Optional[EvaluationRuleTarget] = OMIT, - enabled: typing.Optional[bool] = OMIT, - sampling: typing.Optional[float] = OMIT, - filter: typing.Optional[typing.Sequence[EvaluationRuleFilter]] = OMIT, - mapping: typing.Optional[typing.Sequence[PromptVariableMappingInput]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> ReadableV2EvaluationRule: - """ - Update an evaluation rule. - - Typical uses: - - enable or disable live execution - - switch to another evaluator - - adjust sampling - - change filters - - update LLM-as-judge variable mappings - - Important behavior: - - provide only the fields you want to change - - if you provide `evaluator`, Langfuse resolves that evaluator family to its latest version before saving - - changing `target`, `filter`, or an LLM-as-judge `mapping` must still produce a valid target-specific configuration - - if you change `target` for an LLM-as-judge rule, also send a compatible `filter` and `mapping` in the same request unless the existing ones are still valid for the new target - - for `code` evaluator rules, omit `mapping`; Langfuse stores the fixed code runtime mapping automatically - - if the resulting config is enabled, Langfuse re-validates that the selected evaluator can run - - if the update would move a non-active evaluation rule into the active state and the project already has 500 active evaluation rules, the API returns `409` - - Recovery guidance: - - if an LLM-as-judge update fails with `missing_variable_mapping` or `invalid_variable_mapping` after changing `evaluator` or `target`, resend the request with a complete new `mapping` - - if the update fails with `invalid_filter_value` after changing `target`, resend the request with a target-compatible `filter` - - Parameters - ---------- - evaluation_rule_id : str - Evaluation rule identifier. - - name : typing.Optional[str] - Updated deployment name. - - evaluators : typing.Optional[typing.Sequence[CreateEvaluationRuleEvaluatorAssignment]] - Full replacement of the rule's evaluator assignments: entries that are - not listed are detached. - - Mutually exclusive with the deprecated `evaluator` and `mapping` fields. - - evaluator : typing.Optional[EvaluationRuleEvaluatorReference] - Deprecated single-evaluator alias: updates the first assignment only. Prefer `evaluators`. - - Langfuse resolves the provided evaluator family to its latest version before saving the rule. - A rule's evaluator type cannot be changed: provide `name` for an evaluator family of the rule's current type. To use a different evaluator type, create a new rule. - - target : typing.Optional[EvaluationRuleTarget] - Updated target object type. - - enabled : typing.Optional[bool] - Updated desired enabled state. - - sampling : typing.Optional[float] - Updated sampling fraction. - - filter : typing.Optional[typing.Sequence[EvaluationRuleFilter]] - Updated filter list. - - For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - - mapping : typing.Optional[typing.Sequence[PromptVariableMappingInput]] - Updated LLM-as-judge variable mappings. - - Do not send this field for code evaluator rules. Langfuse stores the fixed code runtime mapping automatically and returns it in the response. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - ReadableV2EvaluationRule - - Examples - -------- - from langfuse import LangfuseAPI - - client = LangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - client.unstable.evaluation_rules.update( - evaluation_rule_id="evaluationRuleId", - ) - """ - _response = self._raw_client.update( - evaluation_rule_id, - name=name, - evaluators=evaluators, - evaluator=evaluator, - target=target, - enabled=enabled, - sampling=sampling, - filter=filter, - mapping=mapping, - request_options=request_options, - ) - return _response.data - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def delete( - self, - evaluation_rule_id: str, - *, - request_options: typing.Optional[RequestOptions] = None, - ) -> DeleteEvaluationRuleResponse: - """ - Delete an evaluation rule. - - This removes the live-ingestion rule only. It does not delete the referenced evaluator. - - Parameters - ---------- - evaluation_rule_id : str - Evaluation rule identifier. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - DeleteEvaluationRuleResponse - - Examples - -------- - from langfuse import LangfuseAPI - - client = LangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - client.unstable.evaluation_rules.delete( - evaluation_rule_id="evaluationRuleId", - ) - """ - _response = self._raw_client.delete( - evaluation_rule_id, request_options=request_options - ) - return _response.data - - -class AsyncEvaluationRulesClient: - def __init__(self, *, client_wrapper: AsyncClientWrapper): - self._raw_client = AsyncRawEvaluationRulesClient(client_wrapper=client_wrapper) - - @property - def with_raw_response(self) -> AsyncRawEvaluationRulesClient: - """ - Retrieves a raw implementation of this client that returns raw responses. - - Returns - ------- - AsyncRawEvaluationRulesClient - """ - return self._raw_client - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def create( - self, - *, - request: CreateEvaluationRuleRequest, - request_options: typing.Optional[RequestOptions] = None, - ) -> EvaluationRule: - """ - Create an evaluation rule. - - An evaluation rule defines **what** incoming data should be evaluated and **how prompt variables should be populated** from that data. - - Use this resource after choosing an evaluator from the evaluator endpoints. - - Key rules: - - `name` must be unique within the project for public evaluation rules - - `target` must be `observation` or `experiment` - - provide either the compatibility `evaluator` field or the new `evaluators` array, never both - - each evaluator `name` + `type` must identify an existing evaluator family returned by the evaluator endpoints - - Langfuse resolves that family to its latest version before saving the evaluation rule - - for `target=experiment`, use dataset `id` values from `GET /api/public/v2/datasets` when filtering by `datasetId` - - an omitted LLM-as-judge assignment mapping inherits the evaluator version's default mapping - - the effective mapping must map every evaluator prompt variable exactly once - - for `code` evaluators, Langfuse uses the fixed code runtime mapping; omit `mapping` in create and update requests - - for user-provided `llm_as_judge` mappings, `expected_output` and `experiment_item_metadata` are only valid for `target=experiment` - - if `enabled=true`, Langfuse validates that the referenced evaluator can currently run - - at most 500 evaluation rules can be effectively active in one project at the same time (enforced identically by the API, the MCP tools, and the app) - - If an evaluation rule with the same `name` already exists in the project, the API returns `409`. - In that case, update the existing resource with `PATCH /api/public/unstable/evaluation-rules/{evaluationRuleId}` instead of creating a second one. - - If enabling this resource would exceed the 500-active limit, the API also returns `409`. - In that case, disable or pause another active evaluation rule before enabling a new one. - - Current scope: - - evaluation rules are live-ingestion rules only - - they do not trigger historical backfills - - Recovery guidance: - - `400 invalid_filter_value`: fix the filter `column` or `value` using `details.column`, `details.invalidValues`, and `details.allowedValues` - - `400 invalid_filter_value` with `details.column=datasetId`: call `GET /api/public/v2/datasets`, then retry with dataset `id` values from that response - - `400 missing_variable_mapping`: for `llm_as_judge` evaluators, fetch the evaluator again and make sure every variable in `variables` appears exactly once in `mapping` - - `400 duplicate_variable_mapping`: remove repeated mappings for the same variable - - `400 invalid_variable_mapping`: for `llm_as_judge`, switch to a valid `source` for the selected `target`, or fix the variable name - - `400 invalid_json_path`: remove or correct the `jsonPath` - - `422 evaluator_preflight_failed`: the selected evaluator cannot run with the resolved model configuration. Fix the evaluator/default model setup, then retry the create request. - - Parameters - ---------- - request : CreateEvaluationRuleRequest - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - EvaluationRule - - Examples - -------- - import asyncio - - from langfuse import AsyncLangfuseAPI - from langfuse.unstable.commons import ( - EvaluationRuleFilter_StringOptions, - EvaluationRuleOptionsFilterOperator, - EvaluationRuleTarget, - PromptVariableMappingInput, - PromptVariableMappingSource, - ) - from langfuse.unstable.evaluation_rules import ( - CreateLlmAsJudgeEvaluationRuleRequest, - LlmAsJudgeEvaluationRuleEvaluatorReference, - LlmAsJudgeEvaluatorType, - ) - - client = AsyncLangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - - - async def main() -> None: - await client.unstable.evaluation_rules.create( - request=CreateLlmAsJudgeEvaluationRuleRequest( - name="answer-correctness-live", - evaluator=LlmAsJudgeEvaluationRuleEvaluatorReference( - name="answer-correctness", - type=LlmAsJudgeEvaluatorType.LLM_AS_JUDGE, - ), - target=EvaluationRuleTarget.OBSERVATION, - enabled=True, - sampling=1.0, - filter=[ - EvaluationRuleFilter_StringOptions( - column="type", - operator=EvaluationRuleOptionsFilterOperator.ANY_OF, - value=["GENERATION"], - ) - ], - mapping=[ - PromptVariableMappingInput( - variable="input", - source=PromptVariableMappingSource.INPUT, - ), - PromptVariableMappingInput( - variable="output", - source=PromptVariableMappingSource.OUTPUT, - ), - ], - ), - ) - - - asyncio.run(main()) - """ - _response = await self._raw_client.create( - request=request, request_options=request_options - ) - return _response.data - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def list( - self, - *, - page: typing.Optional[int] = None, - limit: typing.Optional[int] = None, - request_options: typing.Optional[RequestOptions] = None, - ) -> EvaluationRules: - """ - List evaluation rules in the authenticated project. - - This includes legacy `trace` and `dataset` rules so they can be inspected and migrated to v4 rules. Legacy rules are read-only through this API; create, update, and delete continue to support only `observation` and `experiment` rules. - - Parameters - ---------- - page : typing.Optional[int] - 1-based page number. Defaults to `1`. - - limit : typing.Optional[int] - Maximum number of items per page. Defaults to `50`. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - EvaluationRules - - Examples - -------- - import asyncio - - from langfuse import AsyncLangfuseAPI - - client = AsyncLangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - - - async def main() -> None: - await client.unstable.evaluation_rules.list() - - - asyncio.run(main()) - """ - _response = await self._raw_client.list( - page=page, limit=limit, request_options=request_options - ) - return _response.data - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def get( - self, - evaluation_rule_id: str, - *, - request_options: typing.Optional[RequestOptions] = None, - ) -> ReadableEvaluationRule: - """ - Get one evaluation rule by its identifier. - - Use this endpoint to inspect the current evaluator, target, mapping, filters, execution timing, and effective runtime status. Legacy `trace` and `dataset` rules are returned for migration and are read-only through this API. - - Parameters - ---------- - evaluation_rule_id : str - Evaluation rule identifier returned by the evaluation rule endpoints. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - ReadableEvaluationRule - - Examples - -------- - import asyncio - - from langfuse import AsyncLangfuseAPI - - client = AsyncLangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - - - async def main() -> None: - await client.unstable.evaluation_rules.get( - evaluation_rule_id="evaluationRuleId", - ) - - - asyncio.run(main()) - """ - _response = await self._raw_client.get( - evaluation_rule_id, request_options=request_options - ) - return _response.data - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def update( - self, - evaluation_rule_id: str, - *, - name: typing.Optional[str] = OMIT, - evaluators: typing.Optional[ - typing.Sequence[CreateEvaluationRuleEvaluatorAssignment] - ] = OMIT, - evaluator: typing.Optional[EvaluationRuleEvaluatorReference] = OMIT, - target: typing.Optional[EvaluationRuleTarget] = OMIT, - enabled: typing.Optional[bool] = OMIT, - sampling: typing.Optional[float] = OMIT, - filter: typing.Optional[typing.Sequence[EvaluationRuleFilter]] = OMIT, - mapping: typing.Optional[typing.Sequence[PromptVariableMappingInput]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> ReadableV2EvaluationRule: - """ - Update an evaluation rule. - - Typical uses: - - enable or disable live execution - - switch to another evaluator - - adjust sampling - - change filters - - update LLM-as-judge variable mappings - - Important behavior: - - provide only the fields you want to change - - if you provide `evaluator`, Langfuse resolves that evaluator family to its latest version before saving - - changing `target`, `filter`, or an LLM-as-judge `mapping` must still produce a valid target-specific configuration - - if you change `target` for an LLM-as-judge rule, also send a compatible `filter` and `mapping` in the same request unless the existing ones are still valid for the new target - - for `code` evaluator rules, omit `mapping`; Langfuse stores the fixed code runtime mapping automatically - - if the resulting config is enabled, Langfuse re-validates that the selected evaluator can run - - if the update would move a non-active evaluation rule into the active state and the project already has 500 active evaluation rules, the API returns `409` - - Recovery guidance: - - if an LLM-as-judge update fails with `missing_variable_mapping` or `invalid_variable_mapping` after changing `evaluator` or `target`, resend the request with a complete new `mapping` - - if the update fails with `invalid_filter_value` after changing `target`, resend the request with a target-compatible `filter` - - Parameters - ---------- - evaluation_rule_id : str - Evaluation rule identifier. - - name : typing.Optional[str] - Updated deployment name. - - evaluators : typing.Optional[typing.Sequence[CreateEvaluationRuleEvaluatorAssignment]] - Full replacement of the rule's evaluator assignments: entries that are - not listed are detached. - - Mutually exclusive with the deprecated `evaluator` and `mapping` fields. - - evaluator : typing.Optional[EvaluationRuleEvaluatorReference] - Deprecated single-evaluator alias: updates the first assignment only. Prefer `evaluators`. - - Langfuse resolves the provided evaluator family to its latest version before saving the rule. - A rule's evaluator type cannot be changed: provide `name` for an evaluator family of the rule's current type. To use a different evaluator type, create a new rule. - - target : typing.Optional[EvaluationRuleTarget] - Updated target object type. - - enabled : typing.Optional[bool] - Updated desired enabled state. - - sampling : typing.Optional[float] - Updated sampling fraction. - - filter : typing.Optional[typing.Sequence[EvaluationRuleFilter]] - Updated filter list. - - For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - - mapping : typing.Optional[typing.Sequence[PromptVariableMappingInput]] - Updated LLM-as-judge variable mappings. - - Do not send this field for code evaluator rules. Langfuse stores the fixed code runtime mapping automatically and returns it in the response. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - ReadableV2EvaluationRule - - Examples - -------- - import asyncio - - from langfuse import AsyncLangfuseAPI - - client = AsyncLangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - - - async def main() -> None: - await client.unstable.evaluation_rules.update( - evaluation_rule_id="evaluationRuleId", - ) - - - asyncio.run(main()) - """ - _response = await self._raw_client.update( - evaluation_rule_id, - name=name, - evaluators=evaluators, - evaluator=evaluator, - target=target, - enabled=enabled, - sampling=sampling, - filter=filter, - mapping=mapping, - request_options=request_options, - ) - return _response.data - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def delete( - self, - evaluation_rule_id: str, - *, - request_options: typing.Optional[RequestOptions] = None, - ) -> DeleteEvaluationRuleResponse: - """ - Delete an evaluation rule. - - This removes the live-ingestion rule only. It does not delete the referenced evaluator. - - Parameters - ---------- - evaluation_rule_id : str - Evaluation rule identifier. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - DeleteEvaluationRuleResponse - - Examples - -------- - import asyncio - - from langfuse import AsyncLangfuseAPI - - client = AsyncLangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - - - async def main() -> None: - await client.unstable.evaluation_rules.delete( - evaluation_rule_id="evaluationRuleId", - ) - - - asyncio.run(main()) - """ - _response = await self._raw_client.delete( - evaluation_rule_id, request_options=request_options - ) - return _response.data diff --git a/langfuse/api/unstable/evaluation_rules/raw_client.py b/langfuse/api/unstable/evaluation_rules/raw_client.py deleted file mode 100644 index 4f531d118..000000000 --- a/langfuse/api/unstable/evaluation_rules/raw_client.py +++ /dev/null @@ -1,2258 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing -import typing_extensions -from json.decoder import JSONDecodeError - -from ...commons.errors.access_denied_error import ( - AccessDeniedError as commons_errors_access_denied_error_AccessDeniedError, -) -from ...commons.errors.error import Error -from ...commons.errors.method_not_allowed_error import ( - MethodNotAllowedError as commons_errors_method_not_allowed_error_MethodNotAllowedError, -) -from ...commons.errors.not_found_error import ( - NotFoundError as commons_errors_not_found_error_NotFoundError, -) -from ...commons.errors.unauthorized_error import ( - UnauthorizedError as commons_errors_unauthorized_error_UnauthorizedError, -) -from ...core.api_error import ApiError -from ...core.client_wrapper import AsyncClientWrapper, SyncClientWrapper -from ...core.http_response import AsyncHttpResponse, HttpResponse -from ...core.jsonable_encoder import jsonable_encoder -from ...core.pydantic_utilities import parse_obj_as -from ...core.request_options import RequestOptions -from ...core.serialization import convert_and_respect_annotation_metadata -from ..commons.types.evaluation_rule_filter import EvaluationRuleFilter -from ..commons.types.evaluation_rule_target import EvaluationRuleTarget -from ..commons.types.prompt_variable_mapping_input import PromptVariableMappingInput -from ..errors.errors.access_denied_error import ( - AccessDeniedError as unstable_errors_errors_access_denied_error_AccessDeniedError, -) -from ..errors.errors.bad_request_error import BadRequestError -from ..errors.errors.conflict_error import ConflictError -from ..errors.errors.internal_server_error import InternalServerError -from ..errors.errors.method_not_allowed_error import ( - MethodNotAllowedError as unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError, -) -from ..errors.errors.not_found_error import ( - NotFoundError as unstable_errors_errors_not_found_error_NotFoundError, -) -from ..errors.errors.too_many_requests_error import TooManyRequestsError -from ..errors.errors.unauthorized_error import ( - UnauthorizedError as unstable_errors_errors_unauthorized_error_UnauthorizedError, -) -from ..errors.errors.unprocessable_content_error import UnprocessableContentError -from ..errors.types.public_api_error import PublicApiError -from .types.create_evaluation_rule_evaluator_assignment import ( - CreateEvaluationRuleEvaluatorAssignment, -) -from .types.create_evaluation_rule_request import CreateEvaluationRuleRequest -from .types.delete_evaluation_rule_response import DeleteEvaluationRuleResponse -from .types.evaluation_rule import EvaluationRule -from .types.evaluation_rule_evaluator_reference import EvaluationRuleEvaluatorReference -from .types.evaluation_rules import EvaluationRules -from .types.readable_evaluation_rule import ReadableEvaluationRule -from .types.readable_v2evaluation_rule import ReadableV2EvaluationRule - -# this is used as the default value for optional parameters -OMIT = typing.cast(typing.Any, ...) - - -class RawEvaluationRulesClient: - def __init__(self, *, client_wrapper: SyncClientWrapper): - self._client_wrapper = client_wrapper - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def create( - self, - *, - request: CreateEvaluationRuleRequest, - request_options: typing.Optional[RequestOptions] = None, - ) -> HttpResponse[EvaluationRule]: - """ - Create an evaluation rule. - - An evaluation rule defines **what** incoming data should be evaluated and **how prompt variables should be populated** from that data. - - Use this resource after choosing an evaluator from the evaluator endpoints. - - Key rules: - - `name` must be unique within the project for public evaluation rules - - `target` must be `observation` or `experiment` - - provide either the compatibility `evaluator` field or the new `evaluators` array, never both - - each evaluator `name` + `type` must identify an existing evaluator family returned by the evaluator endpoints - - Langfuse resolves that family to its latest version before saving the evaluation rule - - for `target=experiment`, use dataset `id` values from `GET /api/public/v2/datasets` when filtering by `datasetId` - - an omitted LLM-as-judge assignment mapping inherits the evaluator version's default mapping - - the effective mapping must map every evaluator prompt variable exactly once - - for `code` evaluators, Langfuse uses the fixed code runtime mapping; omit `mapping` in create and update requests - - for user-provided `llm_as_judge` mappings, `expected_output` and `experiment_item_metadata` are only valid for `target=experiment` - - if `enabled=true`, Langfuse validates that the referenced evaluator can currently run - - at most 500 evaluation rules can be effectively active in one project at the same time (enforced identically by the API, the MCP tools, and the app) - - If an evaluation rule with the same `name` already exists in the project, the API returns `409`. - In that case, update the existing resource with `PATCH /api/public/unstable/evaluation-rules/{evaluationRuleId}` instead of creating a second one. - - If enabling this resource would exceed the 500-active limit, the API also returns `409`. - In that case, disable or pause another active evaluation rule before enabling a new one. - - Current scope: - - evaluation rules are live-ingestion rules only - - they do not trigger historical backfills - - Recovery guidance: - - `400 invalid_filter_value`: fix the filter `column` or `value` using `details.column`, `details.invalidValues`, and `details.allowedValues` - - `400 invalid_filter_value` with `details.column=datasetId`: call `GET /api/public/v2/datasets`, then retry with dataset `id` values from that response - - `400 missing_variable_mapping`: for `llm_as_judge` evaluators, fetch the evaluator again and make sure every variable in `variables` appears exactly once in `mapping` - - `400 duplicate_variable_mapping`: remove repeated mappings for the same variable - - `400 invalid_variable_mapping`: for `llm_as_judge`, switch to a valid `source` for the selected `target`, or fix the variable name - - `400 invalid_json_path`: remove or correct the `jsonPath` - - `422 evaluator_preflight_failed`: the selected evaluator cannot run with the resolved model configuration. Fix the evaluator/default model setup, then retry the create request. - - Parameters - ---------- - request : CreateEvaluationRuleRequest - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[EvaluationRule] - """ - _response = self._client_wrapper.httpx_client.request( - "api/public/unstable/evaluation-rules", - method="POST", - json=convert_and_respect_annotation_metadata( - object_=request, - annotation=CreateEvaluationRuleRequest, - direction="write", - ), - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - EvaluationRule, - parse_obj_as( - type_=EvaluationRule, # type: ignore - object_=_response.json(), - ), - ) - return HttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise unstable_errors_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 409: - raise ConflictError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 422: - raise UnprocessableContentError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def list( - self, - *, - page: typing.Optional[int] = None, - limit: typing.Optional[int] = None, - request_options: typing.Optional[RequestOptions] = None, - ) -> HttpResponse[EvaluationRules]: - """ - List evaluation rules in the authenticated project. - - This includes legacy `trace` and `dataset` rules so they can be inspected and migrated to v4 rules. Legacy rules are read-only through this API; create, update, and delete continue to support only `observation` and `experiment` rules. - - Parameters - ---------- - page : typing.Optional[int] - 1-based page number. Defaults to `1`. - - limit : typing.Optional[int] - Maximum number of items per page. Defaults to `50`. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[EvaluationRules] - """ - _response = self._client_wrapper.httpx_client.request( - "api/public/unstable/evaluation-rules", - method="GET", - params={ - "page": page, - "limit": limit, - }, - request_options=request_options, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - EvaluationRules, - parse_obj_as( - type_=EvaluationRules, # type: ignore - object_=_response.json(), - ), - ) - return HttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def get( - self, - evaluation_rule_id: str, - *, - request_options: typing.Optional[RequestOptions] = None, - ) -> HttpResponse[ReadableEvaluationRule]: - """ - Get one evaluation rule by its identifier. - - Use this endpoint to inspect the current evaluator, target, mapping, filters, execution timing, and effective runtime status. Legacy `trace` and `dataset` rules are returned for migration and are read-only through this API. - - Parameters - ---------- - evaluation_rule_id : str - Evaluation rule identifier returned by the evaluation rule endpoints. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[ReadableEvaluationRule] - """ - _response = self._client_wrapper.httpx_client.request( - f"api/public/unstable/evaluation-rules/{jsonable_encoder(evaluation_rule_id)}", - method="GET", - request_options=request_options, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - ReadableEvaluationRule, - parse_obj_as( - type_=ReadableEvaluationRule, # type: ignore - object_=_response.json(), - ), - ) - return HttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise unstable_errors_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def update( - self, - evaluation_rule_id: str, - *, - name: typing.Optional[str] = OMIT, - evaluators: typing.Optional[ - typing.Sequence[CreateEvaluationRuleEvaluatorAssignment] - ] = OMIT, - evaluator: typing.Optional[EvaluationRuleEvaluatorReference] = OMIT, - target: typing.Optional[EvaluationRuleTarget] = OMIT, - enabled: typing.Optional[bool] = OMIT, - sampling: typing.Optional[float] = OMIT, - filter: typing.Optional[typing.Sequence[EvaluationRuleFilter]] = OMIT, - mapping: typing.Optional[typing.Sequence[PromptVariableMappingInput]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> HttpResponse[ReadableV2EvaluationRule]: - """ - Update an evaluation rule. - - Typical uses: - - enable or disable live execution - - switch to another evaluator - - adjust sampling - - change filters - - update LLM-as-judge variable mappings - - Important behavior: - - provide only the fields you want to change - - if you provide `evaluator`, Langfuse resolves that evaluator family to its latest version before saving - - changing `target`, `filter`, or an LLM-as-judge `mapping` must still produce a valid target-specific configuration - - if you change `target` for an LLM-as-judge rule, also send a compatible `filter` and `mapping` in the same request unless the existing ones are still valid for the new target - - for `code` evaluator rules, omit `mapping`; Langfuse stores the fixed code runtime mapping automatically - - if the resulting config is enabled, Langfuse re-validates that the selected evaluator can run - - if the update would move a non-active evaluation rule into the active state and the project already has 500 active evaluation rules, the API returns `409` - - Recovery guidance: - - if an LLM-as-judge update fails with `missing_variable_mapping` or `invalid_variable_mapping` after changing `evaluator` or `target`, resend the request with a complete new `mapping` - - if the update fails with `invalid_filter_value` after changing `target`, resend the request with a target-compatible `filter` - - Parameters - ---------- - evaluation_rule_id : str - Evaluation rule identifier. - - name : typing.Optional[str] - Updated deployment name. - - evaluators : typing.Optional[typing.Sequence[CreateEvaluationRuleEvaluatorAssignment]] - Full replacement of the rule's evaluator assignments: entries that are - not listed are detached. - - Mutually exclusive with the deprecated `evaluator` and `mapping` fields. - - evaluator : typing.Optional[EvaluationRuleEvaluatorReference] - Deprecated single-evaluator alias: updates the first assignment only. Prefer `evaluators`. - - Langfuse resolves the provided evaluator family to its latest version before saving the rule. - A rule's evaluator type cannot be changed: provide `name` for an evaluator family of the rule's current type. To use a different evaluator type, create a new rule. - - target : typing.Optional[EvaluationRuleTarget] - Updated target object type. - - enabled : typing.Optional[bool] - Updated desired enabled state. - - sampling : typing.Optional[float] - Updated sampling fraction. - - filter : typing.Optional[typing.Sequence[EvaluationRuleFilter]] - Updated filter list. - - For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - - mapping : typing.Optional[typing.Sequence[PromptVariableMappingInput]] - Updated LLM-as-judge variable mappings. - - Do not send this field for code evaluator rules. Langfuse stores the fixed code runtime mapping automatically and returns it in the response. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[ReadableV2EvaluationRule] - """ - _response = self._client_wrapper.httpx_client.request( - f"api/public/unstable/evaluation-rules/{jsonable_encoder(evaluation_rule_id)}", - method="PATCH", - json={ - "name": name, - "evaluators": convert_and_respect_annotation_metadata( - object_=evaluators, - annotation=typing.Sequence[CreateEvaluationRuleEvaluatorAssignment], - direction="write", - ), - "evaluator": convert_and_respect_annotation_metadata( - object_=evaluator, - annotation=EvaluationRuleEvaluatorReference, - direction="write", - ), - "target": target, - "enabled": enabled, - "sampling": sampling, - "filter": convert_and_respect_annotation_metadata( - object_=filter, - annotation=typing.Sequence[EvaluationRuleFilter], - direction="write", - ), - "mapping": convert_and_respect_annotation_metadata( - object_=mapping, - annotation=typing.Sequence[PromptVariableMappingInput], - direction="write", - ), - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - ReadableV2EvaluationRule, - parse_obj_as( - type_=ReadableV2EvaluationRule, # type: ignore - object_=_response.json(), - ), - ) - return HttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise unstable_errors_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 422: - raise UnprocessableContentError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def delete( - self, - evaluation_rule_id: str, - *, - request_options: typing.Optional[RequestOptions] = None, - ) -> HttpResponse[DeleteEvaluationRuleResponse]: - """ - Delete an evaluation rule. - - This removes the live-ingestion rule only. It does not delete the referenced evaluator. - - Parameters - ---------- - evaluation_rule_id : str - Evaluation rule identifier. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[DeleteEvaluationRuleResponse] - """ - _response = self._client_wrapper.httpx_client.request( - f"api/public/unstable/evaluation-rules/{jsonable_encoder(evaluation_rule_id)}", - method="DELETE", - request_options=request_options, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - DeleteEvaluationRuleResponse, - parse_obj_as( - type_=DeleteEvaluationRuleResponse, # type: ignore - object_=_response.json(), - ), - ) - return HttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise unstable_errors_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) - - -class AsyncRawEvaluationRulesClient: - def __init__(self, *, client_wrapper: AsyncClientWrapper): - self._client_wrapper = client_wrapper - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def create( - self, - *, - request: CreateEvaluationRuleRequest, - request_options: typing.Optional[RequestOptions] = None, - ) -> AsyncHttpResponse[EvaluationRule]: - """ - Create an evaluation rule. - - An evaluation rule defines **what** incoming data should be evaluated and **how prompt variables should be populated** from that data. - - Use this resource after choosing an evaluator from the evaluator endpoints. - - Key rules: - - `name` must be unique within the project for public evaluation rules - - `target` must be `observation` or `experiment` - - provide either the compatibility `evaluator` field or the new `evaluators` array, never both - - each evaluator `name` + `type` must identify an existing evaluator family returned by the evaluator endpoints - - Langfuse resolves that family to its latest version before saving the evaluation rule - - for `target=experiment`, use dataset `id` values from `GET /api/public/v2/datasets` when filtering by `datasetId` - - an omitted LLM-as-judge assignment mapping inherits the evaluator version's default mapping - - the effective mapping must map every evaluator prompt variable exactly once - - for `code` evaluators, Langfuse uses the fixed code runtime mapping; omit `mapping` in create and update requests - - for user-provided `llm_as_judge` mappings, `expected_output` and `experiment_item_metadata` are only valid for `target=experiment` - - if `enabled=true`, Langfuse validates that the referenced evaluator can currently run - - at most 500 evaluation rules can be effectively active in one project at the same time (enforced identically by the API, the MCP tools, and the app) - - If an evaluation rule with the same `name` already exists in the project, the API returns `409`. - In that case, update the existing resource with `PATCH /api/public/unstable/evaluation-rules/{evaluationRuleId}` instead of creating a second one. - - If enabling this resource would exceed the 500-active limit, the API also returns `409`. - In that case, disable or pause another active evaluation rule before enabling a new one. - - Current scope: - - evaluation rules are live-ingestion rules only - - they do not trigger historical backfills - - Recovery guidance: - - `400 invalid_filter_value`: fix the filter `column` or `value` using `details.column`, `details.invalidValues`, and `details.allowedValues` - - `400 invalid_filter_value` with `details.column=datasetId`: call `GET /api/public/v2/datasets`, then retry with dataset `id` values from that response - - `400 missing_variable_mapping`: for `llm_as_judge` evaluators, fetch the evaluator again and make sure every variable in `variables` appears exactly once in `mapping` - - `400 duplicate_variable_mapping`: remove repeated mappings for the same variable - - `400 invalid_variable_mapping`: for `llm_as_judge`, switch to a valid `source` for the selected `target`, or fix the variable name - - `400 invalid_json_path`: remove or correct the `jsonPath` - - `422 evaluator_preflight_failed`: the selected evaluator cannot run with the resolved model configuration. Fix the evaluator/default model setup, then retry the create request. - - Parameters - ---------- - request : CreateEvaluationRuleRequest - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - AsyncHttpResponse[EvaluationRule] - """ - _response = await self._client_wrapper.httpx_client.request( - "api/public/unstable/evaluation-rules", - method="POST", - json=convert_and_respect_annotation_metadata( - object_=request, - annotation=CreateEvaluationRuleRequest, - direction="write", - ), - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - EvaluationRule, - parse_obj_as( - type_=EvaluationRule, # type: ignore - object_=_response.json(), - ), - ) - return AsyncHttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise unstable_errors_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 409: - raise ConflictError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 422: - raise UnprocessableContentError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def list( - self, - *, - page: typing.Optional[int] = None, - limit: typing.Optional[int] = None, - request_options: typing.Optional[RequestOptions] = None, - ) -> AsyncHttpResponse[EvaluationRules]: - """ - List evaluation rules in the authenticated project. - - This includes legacy `trace` and `dataset` rules so they can be inspected and migrated to v4 rules. Legacy rules are read-only through this API; create, update, and delete continue to support only `observation` and `experiment` rules. - - Parameters - ---------- - page : typing.Optional[int] - 1-based page number. Defaults to `1`. - - limit : typing.Optional[int] - Maximum number of items per page. Defaults to `50`. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - AsyncHttpResponse[EvaluationRules] - """ - _response = await self._client_wrapper.httpx_client.request( - "api/public/unstable/evaluation-rules", - method="GET", - params={ - "page": page, - "limit": limit, - }, - request_options=request_options, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - EvaluationRules, - parse_obj_as( - type_=EvaluationRules, # type: ignore - object_=_response.json(), - ), - ) - return AsyncHttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def get( - self, - evaluation_rule_id: str, - *, - request_options: typing.Optional[RequestOptions] = None, - ) -> AsyncHttpResponse[ReadableEvaluationRule]: - """ - Get one evaluation rule by its identifier. - - Use this endpoint to inspect the current evaluator, target, mapping, filters, execution timing, and effective runtime status. Legacy `trace` and `dataset` rules are returned for migration and are read-only through this API. - - Parameters - ---------- - evaluation_rule_id : str - Evaluation rule identifier returned by the evaluation rule endpoints. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - AsyncHttpResponse[ReadableEvaluationRule] - """ - _response = await self._client_wrapper.httpx_client.request( - f"api/public/unstable/evaluation-rules/{jsonable_encoder(evaluation_rule_id)}", - method="GET", - request_options=request_options, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - ReadableEvaluationRule, - parse_obj_as( - type_=ReadableEvaluationRule, # type: ignore - object_=_response.json(), - ), - ) - return AsyncHttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise unstable_errors_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def update( - self, - evaluation_rule_id: str, - *, - name: typing.Optional[str] = OMIT, - evaluators: typing.Optional[ - typing.Sequence[CreateEvaluationRuleEvaluatorAssignment] - ] = OMIT, - evaluator: typing.Optional[EvaluationRuleEvaluatorReference] = OMIT, - target: typing.Optional[EvaluationRuleTarget] = OMIT, - enabled: typing.Optional[bool] = OMIT, - sampling: typing.Optional[float] = OMIT, - filter: typing.Optional[typing.Sequence[EvaluationRuleFilter]] = OMIT, - mapping: typing.Optional[typing.Sequence[PromptVariableMappingInput]] = OMIT, - request_options: typing.Optional[RequestOptions] = None, - ) -> AsyncHttpResponse[ReadableV2EvaluationRule]: - """ - Update an evaluation rule. - - Typical uses: - - enable or disable live execution - - switch to another evaluator - - adjust sampling - - change filters - - update LLM-as-judge variable mappings - - Important behavior: - - provide only the fields you want to change - - if you provide `evaluator`, Langfuse resolves that evaluator family to its latest version before saving - - changing `target`, `filter`, or an LLM-as-judge `mapping` must still produce a valid target-specific configuration - - if you change `target` for an LLM-as-judge rule, also send a compatible `filter` and `mapping` in the same request unless the existing ones are still valid for the new target - - for `code` evaluator rules, omit `mapping`; Langfuse stores the fixed code runtime mapping automatically - - if the resulting config is enabled, Langfuse re-validates that the selected evaluator can run - - if the update would move a non-active evaluation rule into the active state and the project already has 500 active evaluation rules, the API returns `409` - - Recovery guidance: - - if an LLM-as-judge update fails with `missing_variable_mapping` or `invalid_variable_mapping` after changing `evaluator` or `target`, resend the request with a complete new `mapping` - - if the update fails with `invalid_filter_value` after changing `target`, resend the request with a target-compatible `filter` - - Parameters - ---------- - evaluation_rule_id : str - Evaluation rule identifier. - - name : typing.Optional[str] - Updated deployment name. - - evaluators : typing.Optional[typing.Sequence[CreateEvaluationRuleEvaluatorAssignment]] - Full replacement of the rule's evaluator assignments: entries that are - not listed are detached. - - Mutually exclusive with the deprecated `evaluator` and `mapping` fields. - - evaluator : typing.Optional[EvaluationRuleEvaluatorReference] - Deprecated single-evaluator alias: updates the first assignment only. Prefer `evaluators`. - - Langfuse resolves the provided evaluator family to its latest version before saving the rule. - A rule's evaluator type cannot be changed: provide `name` for an evaluator family of the rule's current type. To use a different evaluator type, create a new rule. - - target : typing.Optional[EvaluationRuleTarget] - Updated target object type. - - enabled : typing.Optional[bool] - Updated desired enabled state. - - sampling : typing.Optional[float] - Updated sampling fraction. - - filter : typing.Optional[typing.Sequence[EvaluationRuleFilter]] - Updated filter list. - - For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - - mapping : typing.Optional[typing.Sequence[PromptVariableMappingInput]] - Updated LLM-as-judge variable mappings. - - Do not send this field for code evaluator rules. Langfuse stores the fixed code runtime mapping automatically and returns it in the response. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - AsyncHttpResponse[ReadableV2EvaluationRule] - """ - _response = await self._client_wrapper.httpx_client.request( - f"api/public/unstable/evaluation-rules/{jsonable_encoder(evaluation_rule_id)}", - method="PATCH", - json={ - "name": name, - "evaluators": convert_and_respect_annotation_metadata( - object_=evaluators, - annotation=typing.Sequence[CreateEvaluationRuleEvaluatorAssignment], - direction="write", - ), - "evaluator": convert_and_respect_annotation_metadata( - object_=evaluator, - annotation=EvaluationRuleEvaluatorReference, - direction="write", - ), - "target": target, - "enabled": enabled, - "sampling": sampling, - "filter": convert_and_respect_annotation_metadata( - object_=filter, - annotation=typing.Sequence[EvaluationRuleFilter], - direction="write", - ), - "mapping": convert_and_respect_annotation_metadata( - object_=mapping, - annotation=typing.Sequence[PromptVariableMappingInput], - direction="write", - ), - }, - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - ReadableV2EvaluationRule, - parse_obj_as( - type_=ReadableV2EvaluationRule, # type: ignore - object_=_response.json(), - ), - ) - return AsyncHttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise unstable_errors_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 422: - raise UnprocessableContentError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluation-rules` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def delete( - self, - evaluation_rule_id: str, - *, - request_options: typing.Optional[RequestOptions] = None, - ) -> AsyncHttpResponse[DeleteEvaluationRuleResponse]: - """ - Delete an evaluation rule. - - This removes the live-ingestion rule only. It does not delete the referenced evaluator. - - Parameters - ---------- - evaluation_rule_id : str - Evaluation rule identifier. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - AsyncHttpResponse[DeleteEvaluationRuleResponse] - """ - _response = await self._client_wrapper.httpx_client.request( - f"api/public/unstable/evaluation-rules/{jsonable_encoder(evaluation_rule_id)}", - method="DELETE", - request_options=request_options, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - DeleteEvaluationRuleResponse, - parse_obj_as( - type_=DeleteEvaluationRuleResponse, # type: ignore - object_=_response.json(), - ), - ) - return AsyncHttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise unstable_errors_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/__init__.py b/langfuse/api/unstable/evaluation_rules/types/__init__.py deleted file mode 100644 index 2540c08f5..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/__init__.py +++ /dev/null @@ -1,130 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -# isort: skip_file - -import typing -from importlib import import_module - -if typing.TYPE_CHECKING: - from .code_evaluation_rule_evaluator_reference import ( - CodeEvaluationRuleEvaluatorReference, - ) - from .create_code_evaluation_rule_request import CreateCodeEvaluationRuleRequest - from .create_evaluation_rule_evaluator_assignment import ( - CreateEvaluationRuleEvaluatorAssignment, - ) - from .create_evaluation_rule_evaluator_reference import ( - CreateEvaluationRuleEvaluatorReference, - ) - from .create_evaluation_rule_request import CreateEvaluationRuleRequest - from .create_evaluation_rule_with_evaluators_request import ( - CreateEvaluationRuleWithEvaluatorsRequest, - ) - from .create_llm_as_judge_evaluation_rule_request import ( - CreateLlmAsJudgeEvaluationRuleRequest, - ) - from .delete_evaluation_rule_response import DeleteEvaluationRuleResponse - from .evaluation_rule import EvaluationRule - from .evaluation_rule_base import EvaluationRuleBase - from .evaluation_rule_evaluator import EvaluationRuleEvaluator - from .evaluation_rule_evaluator_assignment import EvaluationRuleEvaluatorAssignment - from .evaluation_rule_evaluator_reference import EvaluationRuleEvaluatorReference - from .evaluation_rule_time_scope import EvaluationRuleTimeScope - from .evaluation_rules import EvaluationRules - from .legacy_evaluation_object import LegacyEvaluationObject - from .legacy_evaluation_rule import LegacyEvaluationRule - from .legacy_evaluation_rule_evaluator_assignment import ( - LegacyEvaluationRuleEvaluatorAssignment, - ) - from .legacy_evaluation_rule_target import LegacyEvaluationRuleTarget - from .legacy_prompt_variable_mapping import LegacyPromptVariableMapping - from .llm_as_judge_evaluation_rule_evaluator_reference import ( - LlmAsJudgeEvaluationRuleEvaluatorReference, - ) - from .llm_as_judge_evaluator_type import LlmAsJudgeEvaluatorType - from .readable_evaluation_rule import ReadableEvaluationRule - from .readable_v2evaluation_rule import ReadableV2EvaluationRule - from .update_evaluation_rule_request import UpdateEvaluationRuleRequest -_dynamic_imports: typing.Dict[str, str] = { - "CodeEvaluationRuleEvaluatorReference": ".code_evaluation_rule_evaluator_reference", - "CreateCodeEvaluationRuleRequest": ".create_code_evaluation_rule_request", - "CreateEvaluationRuleEvaluatorAssignment": ".create_evaluation_rule_evaluator_assignment", - "CreateEvaluationRuleEvaluatorReference": ".create_evaluation_rule_evaluator_reference", - "CreateEvaluationRuleRequest": ".create_evaluation_rule_request", - "CreateEvaluationRuleWithEvaluatorsRequest": ".create_evaluation_rule_with_evaluators_request", - "CreateLlmAsJudgeEvaluationRuleRequest": ".create_llm_as_judge_evaluation_rule_request", - "DeleteEvaluationRuleResponse": ".delete_evaluation_rule_response", - "EvaluationRule": ".evaluation_rule", - "EvaluationRuleBase": ".evaluation_rule_base", - "EvaluationRuleEvaluator": ".evaluation_rule_evaluator", - "EvaluationRuleEvaluatorAssignment": ".evaluation_rule_evaluator_assignment", - "EvaluationRuleEvaluatorReference": ".evaluation_rule_evaluator_reference", - "EvaluationRuleTimeScope": ".evaluation_rule_time_scope", - "EvaluationRules": ".evaluation_rules", - "LegacyEvaluationObject": ".legacy_evaluation_object", - "LegacyEvaluationRule": ".legacy_evaluation_rule", - "LegacyEvaluationRuleEvaluatorAssignment": ".legacy_evaluation_rule_evaluator_assignment", - "LegacyEvaluationRuleTarget": ".legacy_evaluation_rule_target", - "LegacyPromptVariableMapping": ".legacy_prompt_variable_mapping", - "LlmAsJudgeEvaluationRuleEvaluatorReference": ".llm_as_judge_evaluation_rule_evaluator_reference", - "LlmAsJudgeEvaluatorType": ".llm_as_judge_evaluator_type", - "ReadableEvaluationRule": ".readable_evaluation_rule", - "ReadableV2EvaluationRule": ".readable_v2evaluation_rule", - "UpdateEvaluationRuleRequest": ".update_evaluation_rule_request", -} - - -def __getattr__(attr_name: str) -> typing.Any: - module_name = _dynamic_imports.get(attr_name) - if module_name is None: - raise AttributeError( - f"No {attr_name} found in _dynamic_imports for module name -> {__name__}" - ) - try: - module = import_module(module_name, __package__) - if module_name == f".{attr_name}": - return module - else: - return getattr(module, attr_name) - except ImportError as e: - raise ImportError( - f"Failed to import {attr_name} from {module_name}: {e}" - ) from e - except AttributeError as e: - raise AttributeError( - f"Failed to get {attr_name} from {module_name}: {e}" - ) from e - - -def __dir__(): - lazy_attrs = list(_dynamic_imports.keys()) - return sorted(lazy_attrs) - - -__all__ = [ - "CodeEvaluationRuleEvaluatorReference", - "CreateCodeEvaluationRuleRequest", - "CreateEvaluationRuleEvaluatorAssignment", - "CreateEvaluationRuleEvaluatorReference", - "CreateEvaluationRuleRequest", - "CreateEvaluationRuleWithEvaluatorsRequest", - "CreateLlmAsJudgeEvaluationRuleRequest", - "DeleteEvaluationRuleResponse", - "EvaluationRule", - "EvaluationRuleBase", - "EvaluationRuleEvaluator", - "EvaluationRuleEvaluatorAssignment", - "EvaluationRuleEvaluatorReference", - "EvaluationRuleTimeScope", - "EvaluationRules", - "LegacyEvaluationObject", - "LegacyEvaluationRule", - "LegacyEvaluationRuleEvaluatorAssignment", - "LegacyEvaluationRuleTarget", - "LegacyPromptVariableMapping", - "LlmAsJudgeEvaluationRuleEvaluatorReference", - "LlmAsJudgeEvaluatorType", - "ReadableEvaluationRule", - "ReadableV2EvaluationRule", - "UpdateEvaluationRuleRequest", -] diff --git a/langfuse/api/unstable/evaluation_rules/types/code_evaluation_rule_evaluator_reference.py b/langfuse/api/unstable/evaluation_rules/types/code_evaluation_rule_evaluator_reference.py deleted file mode 100644 index 813c065b0..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/code_evaluation_rule_evaluator_reference.py +++ /dev/null @@ -1,26 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel - - -class CodeEvaluationRuleEvaluatorReference(UniversalBaseModel): - """ - Code evaluator family reference used when creating an evaluation rule. - """ - - name: str = pydantic.Field() - """ - Evaluator family name. - """ - - type: typing.Literal["code"] = pydantic.Field(default="code") - """ - Must be `code`. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/create_code_evaluation_rule_request.py b/langfuse/api/unstable/evaluation_rules/types/create_code_evaluation_rule_request.py deleted file mode 100644 index fe3edb606..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/create_code_evaluation_rule_request.py +++ /dev/null @@ -1,56 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from ...commons.types.evaluation_rule_filter import EvaluationRuleFilter -from ...commons.types.evaluation_rule_target import EvaluationRuleTarget -from .code_evaluation_rule_evaluator_reference import ( - CodeEvaluationRuleEvaluatorReference, -) - - -class CreateCodeEvaluationRuleRequest(UniversalBaseModel): - name: str = pydantic.Field() - """ - Human-readable deployment name. - """ - - evaluator: CodeEvaluationRuleEvaluatorReference = pydantic.Field() - """ - Code evaluator family to use. - - Use `name` and `type` from the evaluator endpoints. - Langfuse resolves that family to its latest version before saving the rule. - """ - - target: EvaluationRuleTarget = pydantic.Field() - """ - Target object type to evaluate. - """ - - enabled: bool = pydantic.Field() - """ - Whether the deployment should be active immediately after creation. - """ - - sampling: typing.Optional[float] = pydantic.Field(default=None) - """ - Optional sampling fraction. Defaults to `1`. - """ - - filter: typing.Optional[typing.List[EvaluationRuleFilter]] = pydantic.Field( - default=None - ) - """ - Optional filter list. - - Omit or pass an empty list to evaluate all matching targets for the selected `target`. - Each filter object must use a column that is valid for that `target`. - For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_evaluator_assignment.py b/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_evaluator_assignment.py deleted file mode 100644 index a69c59072..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_evaluator_assignment.py +++ /dev/null @@ -1,24 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from ...commons.types.prompt_variable_mapping_input import PromptVariableMappingInput -from .create_evaluation_rule_evaluator_reference import ( - CreateEvaluationRuleEvaluatorReference, -) - - -class CreateEvaluationRuleEvaluatorAssignment(UniversalBaseModel): - evaluator: CreateEvaluationRuleEvaluatorReference - mapping: typing.Optional[typing.List[PromptVariableMappingInput]] = pydantic.Field( - default=None - ) - """ - Optional rule-specific override. Omit to inherit the evaluator version's default mapping. Omit for code evaluators. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_evaluator_reference.py b/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_evaluator_reference.py deleted file mode 100644 index 37a16c62d..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_evaluator_reference.py +++ /dev/null @@ -1,19 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from ...commons.types.evaluator_type import EvaluatorType - - -class CreateEvaluationRuleEvaluatorReference(UniversalBaseModel): - name: str - type: typing.Optional[EvaluatorType] = pydantic.Field(default=None) - """ - Defaults to `llm_as_judge`. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_request.py b/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_request.py deleted file mode 100644 index 4e77864cd..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_request.py +++ /dev/null @@ -1,17 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from .create_code_evaluation_rule_request import CreateCodeEvaluationRuleRequest -from .create_evaluation_rule_with_evaluators_request import ( - CreateEvaluationRuleWithEvaluatorsRequest, -) -from .create_llm_as_judge_evaluation_rule_request import ( - CreateLlmAsJudgeEvaluationRuleRequest, -) - -CreateEvaluationRuleRequest = typing.Union[ - CreateLlmAsJudgeEvaluationRuleRequest, - CreateCodeEvaluationRuleRequest, - CreateEvaluationRuleWithEvaluatorsRequest, -] diff --git a/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_with_evaluators_request.py b/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_with_evaluators_request.py deleted file mode 100644 index 1594bd9b3..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/create_evaluation_rule_with_evaluators_request.py +++ /dev/null @@ -1,34 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from ...commons.types.evaluation_rule_filter import EvaluationRuleFilter -from ...commons.types.evaluation_rule_target import EvaluationRuleTarget -from .create_evaluation_rule_evaluator_assignment import ( - CreateEvaluationRuleEvaluatorAssignment, -) - - -class CreateEvaluationRuleWithEvaluatorsRequest(UniversalBaseModel): - name: str = pydantic.Field() - """ - Human-readable deployment name. - """ - - evaluators: typing.List[CreateEvaluationRuleEvaluatorAssignment] = pydantic.Field() - """ - One or more evaluator assignments. Providing the deprecated top-level `evaluator` or `mapping` fields alongside this is rejected with `400`. - - Multiple assignments are supported on writable targets. - """ - - target: EvaluationRuleTarget - enabled: bool - sampling: typing.Optional[float] = None - filter: typing.Optional[typing.List[EvaluationRuleFilter]] = None - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/create_llm_as_judge_evaluation_rule_request.py b/langfuse/api/unstable/evaluation_rules/types/create_llm_as_judge_evaluation_rule_request.py deleted file mode 100644 index b8347bc30..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/create_llm_as_judge_evaluation_rule_request.py +++ /dev/null @@ -1,65 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from ...commons.types.evaluation_rule_filter import EvaluationRuleFilter -from ...commons.types.evaluation_rule_target import EvaluationRuleTarget -from ...commons.types.prompt_variable_mapping_input import PromptVariableMappingInput -from .llm_as_judge_evaluation_rule_evaluator_reference import ( - LlmAsJudgeEvaluationRuleEvaluatorReference, -) - - -class CreateLlmAsJudgeEvaluationRuleRequest(UniversalBaseModel): - name: str = pydantic.Field() - """ - Human-readable deployment name. - """ - - evaluator: LlmAsJudgeEvaluationRuleEvaluatorReference = pydantic.Field() - """ - LLM-as-judge evaluator family to use. - - Use `name` and `type` from the evaluator endpoints. If `type` is omitted, Langfuse defaults it to `llm_as_judge` for backwards compatibility. - Langfuse resolves that family to its latest version before saving the rule. - """ - - target: EvaluationRuleTarget = pydantic.Field() - """ - Target object type to evaluate. - """ - - enabled: bool = pydantic.Field() - """ - Whether the deployment should be active immediately after creation. - """ - - sampling: typing.Optional[float] = pydantic.Field(default=None) - """ - Optional sampling fraction. Defaults to `1`. - """ - - filter: typing.Optional[typing.List[EvaluationRuleFilter]] = pydantic.Field( - default=None - ) - """ - Optional filter list. - - Omit or pass an empty list to evaluate all matching targets for the selected `target`. - Each filter object must use a column that is valid for that `target`. - For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - """ - - mapping: typing.List[PromptVariableMappingInput] = pydantic.Field() - """ - LLM-as-judge variable mappings. - - Every prompt variable must appear exactly once. - Build this list from the evaluator's `variables` array. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/delete_evaluation_rule_response.py b/langfuse/api/unstable/evaluation_rules/types/delete_evaluation_rule_response.py deleted file mode 100644 index 42423c3dc..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/delete_evaluation_rule_response.py +++ /dev/null @@ -1,21 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel - - -class DeleteEvaluationRuleResponse(UniversalBaseModel): - """ - Confirmation response returned after successful deletion. - """ - - message: str = pydantic.Field() - """ - Always `Evaluation rule successfully deleted`. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule.py b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule.py deleted file mode 100644 index 653e163dc..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule.py +++ /dev/null @@ -1,36 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ...commons.types.evaluation_rule_filter import EvaluationRuleFilter -from ...commons.types.evaluation_rule_target import EvaluationRuleTarget -from ...commons.types.prompt_variable_mapping_read import PromptVariableMappingRead -from .evaluation_rule_base import EvaluationRuleBase -from .evaluation_rule_evaluator_assignment import EvaluationRuleEvaluatorAssignment - - -class EvaluationRule(EvaluationRuleBase): - evaluators: typing.List[EvaluationRuleEvaluatorAssignment] = pydantic.Field() - """ - Evaluators attached to this rule in deterministic assignment order. A `null` mapping inherits the evaluator version's default mapping. - """ - - target: EvaluationRuleTarget = pydantic.Field() - """ - Target object type that should trigger scoring. - """ - - filter: typing.List[EvaluationRuleFilter] = pydantic.Field() - """ - List of filter conditions used to decide whether a target should be evaluated. - """ - - mapping: typing.List[PromptVariableMappingRead] = pydantic.Field() - """ - Deprecated compatibility alias containing the effective mapping for `evaluators[0]`. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_base.py b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_base.py deleted file mode 100644 index 9ee19d860..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_base.py +++ /dev/null @@ -1,99 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import datetime as dt -import typing - -import pydantic -import typing_extensions -from ....core.pydantic_utilities import UniversalBaseModel -from ....core.serialization import FieldMetadata -from ...commons.types.evaluation_rule_status import EvaluationRuleStatus -from .evaluation_rule_evaluator import EvaluationRuleEvaluator - - -class EvaluationRuleBase(UniversalBaseModel): - """ - Live evaluation rule for incoming data. - - An evaluation rule answers: - - which evaluator should be used - - which target objects should trigger scoring - - how often scoring should run - - which target fields should populate each prompt variable - - whether the deployment is active, inactive, or paused - - Important status semantics: - - `enabled` is the desired on/off setting from the client - - `status` is the effective runtime state after Langfuse applies validation and blocking rules - - `enabled=true` with `status=paused` means the rule should run, but Langfuse has paused it until the underlying problem is fixed - """ - - id: str = pydantic.Field() - """ - Stable evaluation rule identifier. - """ - - name: str = pydantic.Field() - """ - Human-readable deployment name. This is independent from the evaluator name. - """ - - evaluator: typing.Optional[EvaluationRuleEvaluator] = pydantic.Field(default=None) - """ - Deprecated compatibility alias for `evaluators[0].evaluator`, or `null` - when the rule currently has no evaluator assignments. - - `id` identifies the evaluator family. The rule automatically uses the - latest available version of that evaluator. - """ - - enabled: bool = pydantic.Field() - """ - Desired enabled state configured by the client. - """ - - status: EvaluationRuleStatus = pydantic.Field() - """ - Effective runtime status after Langfuse applies validation and blocking rules. - """ - - paused_reason: typing_extensions.Annotated[ - typing.Optional[str], FieldMetadata(alias="pausedReason") - ] = pydantic.Field(default=None) - """ - Machine-readable reason when `status=paused`, otherwise `null`. - """ - - paused_message: typing_extensions.Annotated[ - typing.Optional[str], FieldMetadata(alias="pausedMessage") - ] = pydantic.Field(default=None) - """ - Human-readable explanation when `status=paused`, otherwise `null`. - """ - - sampling: float = pydantic.Field() - """ - Fraction of matching target objects that should be evaluated. - - Must be greater than `0` and less than or equal to `1`. - - `1` means evaluate every matching target. - - `0.25` means evaluate approximately 25% of matching targets. - """ - - created_at: typing_extensions.Annotated[ - dt.datetime, FieldMetadata(alias="createdAt") - ] = pydantic.Field() - """ - Timestamp when the evaluation rule was created. - """ - - updated_at: typing_extensions.Annotated[ - dt.datetime, FieldMetadata(alias="updatedAt") - ] = pydantic.Field() - """ - Timestamp when the evaluation rule was last updated. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator.py b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator.py deleted file mode 100644 index 57f8378ee..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator.py +++ /dev/null @@ -1,35 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from ...commons.types.evaluator_type import EvaluatorType - - -class EvaluationRuleEvaluator(UniversalBaseModel): - """ - Resolved evaluator currently used by the evaluation rule. - - `id` identifies the evaluator family. Evaluation runs automatically use - the latest available evaluator version. - """ - - id: str = pydantic.Field() - """ - Identifier of the exact evaluator version currently used by the rule. - """ - - name: str = pydantic.Field() - """ - Evaluator family name. - """ - - type: EvaluatorType = pydantic.Field() - """ - Evaluator type. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator_assignment.py b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator_assignment.py deleted file mode 100644 index b299fd5a9..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator_assignment.py +++ /dev/null @@ -1,22 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from ...commons.types.prompt_variable_mapping_read import PromptVariableMappingRead -from .evaluation_rule_evaluator import EvaluationRuleEvaluator - - -class EvaluationRuleEvaluatorAssignment(UniversalBaseModel): - evaluator: EvaluationRuleEvaluator - mapping: typing.Optional[typing.List[PromptVariableMappingRead]] = pydantic.Field( - default=None - ) - """ - Rule-specific override, or `null` to inherit the evaluator version's default mapping. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator_reference.py b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator_reference.py deleted file mode 100644 index 367ae6bc5..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_evaluator_reference.py +++ /dev/null @@ -1,24 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel - - -class EvaluationRuleEvaluatorReference(UniversalBaseModel): - """ - Evaluator family reference used when updating an evaluation rule. - - `name` identifies the evaluator family in the authenticated project context. - A rule's evaluator type cannot be changed, so this reference does not accept a `type`; the family must match the rule's current evaluator type. - """ - - name: str = pydantic.Field() - """ - Evaluator family name. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_time_scope.py b/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_time_scope.py deleted file mode 100644 index 887769bbb..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/evaluation_rule_time_scope.py +++ /dev/null @@ -1,22 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core import enum - -T_Result = typing.TypeVar("T_Result") - - -class EvaluationRuleTimeScope(enum.StrEnum): - NEW = "NEW" - EXISTING = "EXISTING" - - def visit( - self, - new: typing.Callable[[], T_Result], - existing: typing.Callable[[], T_Result], - ) -> T_Result: - if self is EvaluationRuleTimeScope.NEW: - return new() - if self is EvaluationRuleTimeScope.EXISTING: - return existing() diff --git a/langfuse/api/unstable/evaluation_rules/types/evaluation_rules.py b/langfuse/api/unstable/evaluation_rules/types/evaluation_rules.py deleted file mode 100644 index b6d8ddc01..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/evaluation_rules.py +++ /dev/null @@ -1,28 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from ....utils.pagination.types.meta_response import MetaResponse -from .readable_evaluation_rule import ReadableEvaluationRule - - -class EvaluationRules(UniversalBaseModel): - """ - Paginated list of evaluation rules. - """ - - data: typing.List[ReadableEvaluationRule] = pydantic.Field() - """ - Evaluation rules in the current page. - """ - - meta: MetaResponse = pydantic.Field() - """ - Standard pagination metadata. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_object.py b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_object.py deleted file mode 100644 index 7f952d7b9..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_object.py +++ /dev/null @@ -1,66 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core import enum - -T_Result = typing.TypeVar("T_Result") - - -class LegacyEvaluationObject(enum.StrEnum): - """ - **Deprecated:** Legacy Langfuse object used by a trace or dataset evaluation rule. - """ - - TRACE = "trace" - SPAN = "span" - GENERATION = "generation" - EVENT = "event" - AGENT = "agent" - TOOL = "tool" - CHAIN = "chain" - RETRIEVER = "retriever" - EVALUATOR = "evaluator" - EMBEDDING = "embedding" - GUARDRAIL = "guardrail" - DATASET_ITEM = "dataset_item" - - def visit( - self, - trace: typing.Callable[[], T_Result], - span: typing.Callable[[], T_Result], - generation: typing.Callable[[], T_Result], - event: typing.Callable[[], T_Result], - agent: typing.Callable[[], T_Result], - tool: typing.Callable[[], T_Result], - chain: typing.Callable[[], T_Result], - retriever: typing.Callable[[], T_Result], - evaluator: typing.Callable[[], T_Result], - embedding: typing.Callable[[], T_Result], - guardrail: typing.Callable[[], T_Result], - dataset_item: typing.Callable[[], T_Result], - ) -> T_Result: - if self is LegacyEvaluationObject.TRACE: - return trace() - if self is LegacyEvaluationObject.SPAN: - return span() - if self is LegacyEvaluationObject.GENERATION: - return generation() - if self is LegacyEvaluationObject.EVENT: - return event() - if self is LegacyEvaluationObject.AGENT: - return agent() - if self is LegacyEvaluationObject.TOOL: - return tool() - if self is LegacyEvaluationObject.CHAIN: - return chain() - if self is LegacyEvaluationObject.RETRIEVER: - return retriever() - if self is LegacyEvaluationObject.EVALUATOR: - return evaluator() - if self is LegacyEvaluationObject.EMBEDDING: - return embedding() - if self is LegacyEvaluationObject.GUARDRAIL: - return guardrail() - if self is LegacyEvaluationObject.DATASET_ITEM: - return dataset_item() diff --git a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule.py b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule.py deleted file mode 100644 index 7ce85a386..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule.py +++ /dev/null @@ -1,55 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ....core.serialization import FieldMetadata -from ...commons.types.evaluation_rule_filter import EvaluationRuleFilter -from .evaluation_rule_base import EvaluationRuleBase -from .evaluation_rule_time_scope import EvaluationRuleTimeScope -from .legacy_evaluation_rule_evaluator_assignment import ( - LegacyEvaluationRuleEvaluatorAssignment, -) -from .legacy_evaluation_rule_target import LegacyEvaluationRuleTarget -from .legacy_prompt_variable_mapping import LegacyPromptVariableMapping - - -class LegacyEvaluationRule(EvaluationRuleBase): - """ - **Deprecated:** Legacy trace- or dataset-level evaluation rule returned by list and get for migration. - - This resource is read-only through the unstable public API. Its mapping preserves the trace, dataset item, or named observation selected for each prompt variable. - """ - - evaluators: typing.List[LegacyEvaluationRuleEvaluatorAssignment] = pydantic.Field() - """ - Evaluators attached to this rule in deterministic assignment order. - """ - - target: LegacyEvaluationRuleTarget - delay: int = pydantic.Field() - """ - Delay in milliseconds before the legacy evaluation job runs. - """ - - time_scope: typing_extensions.Annotated[ - typing.List[EvaluationRuleTimeScope], FieldMetadata(alias="timeScope") - ] = pydantic.Field() - """ - Whether the legacy rule evaluates newly ingested data, existing data, or both. - """ - - filter: typing.List[EvaluationRuleFilter] = pydantic.Field() - """ - Stored filters used by the legacy trace or dataset rule. - """ - - mapping: typing.List[LegacyPromptVariableMapping] = pydantic.Field() - """ - Stored variable mappings, including the trace, dataset item, or named observation selected for each variable. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_evaluator_assignment.py b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_evaluator_assignment.py deleted file mode 100644 index 00233e0c8..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_evaluator_assignment.py +++ /dev/null @@ -1,21 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from .evaluation_rule_evaluator import EvaluationRuleEvaluator -from .legacy_prompt_variable_mapping import LegacyPromptVariableMapping - - -class LegacyEvaluationRuleEvaluatorAssignment(UniversalBaseModel): - """ - **Deprecated:** Evaluator assignment on a legacy trace or dataset rule. - """ - - evaluator: EvaluationRuleEvaluator - mapping: typing.Optional[typing.List[LegacyPromptVariableMapping]] = None - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_target.py b/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_target.py deleted file mode 100644 index 24603666d..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/legacy_evaluation_rule_target.py +++ /dev/null @@ -1,26 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core import enum - -T_Result = typing.TypeVar("T_Result") - - -class LegacyEvaluationRuleTarget(enum.StrEnum): - """ - **Deprecated:** Legacy trace or dataset evaluation target. - """ - - TRACE = "trace" - DATASET = "dataset" - - def visit( - self, - trace: typing.Callable[[], T_Result], - dataset: typing.Callable[[], T_Result], - ) -> T_Result: - if self is LegacyEvaluationRuleTarget.TRACE: - return trace() - if self is LegacyEvaluationRuleTarget.DATASET: - return dataset() diff --git a/langfuse/api/unstable/evaluation_rules/types/legacy_prompt_variable_mapping.py b/langfuse/api/unstable/evaluation_rules/types/legacy_prompt_variable_mapping.py deleted file mode 100644 index 238694d9e..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/legacy_prompt_variable_mapping.py +++ /dev/null @@ -1,35 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ....core.pydantic_utilities import UniversalBaseModel -from ....core.serialization import FieldMetadata -from .legacy_evaluation_object import LegacyEvaluationObject - - -class LegacyPromptVariableMapping(UniversalBaseModel): - """ - **Deprecated:** Connects one prompt variable to data from a trace, dataset item, or named observation in a legacy rule. - """ - - variable: str - langfuse_object: typing_extensions.Annotated[ - LegacyEvaluationObject, FieldMetadata(alias="langfuseObject") - ] - object_name: typing_extensions.Annotated[ - typing.Optional[str], FieldMetadata(alias="objectName") - ] = pydantic.Field(default=None) - """ - Observation name to match, or `null` when `langfuseObject` is `trace` or `dataset_item`. - """ - - source: str - json_path: typing_extensions.Annotated[ - typing.Optional[str], FieldMetadata(alias="jsonPath") - ] = None - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/llm_as_judge_evaluation_rule_evaluator_reference.py b/langfuse/api/unstable/evaluation_rules/types/llm_as_judge_evaluation_rule_evaluator_reference.py deleted file mode 100644 index 14062c1d3..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/llm_as_judge_evaluation_rule_evaluator_reference.py +++ /dev/null @@ -1,27 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from .llm_as_judge_evaluator_type import LlmAsJudgeEvaluatorType - - -class LlmAsJudgeEvaluationRuleEvaluatorReference(UniversalBaseModel): - """ - LLM-as-judge evaluator family reference used when creating an evaluation rule. - """ - - name: str = pydantic.Field() - """ - Evaluator family name. - """ - - type: typing.Optional[LlmAsJudgeEvaluatorType] = pydantic.Field(default=None) - """ - Evaluator type. Defaults to `llm_as_judge` when omitted. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/llm_as_judge_evaluator_type.py b/langfuse/api/unstable/evaluation_rules/types/llm_as_judge_evaluator_type.py deleted file mode 100644 index b18856d22..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/llm_as_judge_evaluator_type.py +++ /dev/null @@ -1,15 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from ....core import enum - -T_Result = typing.TypeVar("T_Result") - - -class LlmAsJudgeEvaluatorType(enum.StrEnum): - LLM_AS_JUDGE = "llm_as_judge" - - def visit(self, llm_as_judge: typing.Callable[[], T_Result]) -> T_Result: - if self is LlmAsJudgeEvaluatorType.LLM_AS_JUDGE: - return llm_as_judge() diff --git a/langfuse/api/unstable/evaluation_rules/types/readable_evaluation_rule.py b/langfuse/api/unstable/evaluation_rules/types/readable_evaluation_rule.py deleted file mode 100644 index f4eba5936..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/readable_evaluation_rule.py +++ /dev/null @@ -1,8 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -from .legacy_evaluation_rule import LegacyEvaluationRule -from .readable_v2evaluation_rule import ReadableV2EvaluationRule - -ReadableEvaluationRule = typing.Union[ReadableV2EvaluationRule, LegacyEvaluationRule] diff --git a/langfuse/api/unstable/evaluation_rules/types/readable_v2evaluation_rule.py b/langfuse/api/unstable/evaluation_rules/types/readable_v2evaluation_rule.py deleted file mode 100644 index 2f9fe6ffa..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/readable_v2evaluation_rule.py +++ /dev/null @@ -1,115 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....evaluation_commons.types.evaluation_rule_read_filter import ( - EvaluationRuleReadFilter, -) -from ...commons.types.evaluation_rule_target import EvaluationRuleTarget -from ...commons.types.prompt_variable_mapping_read import PromptVariableMappingRead -from .evaluation_rule_base import EvaluationRuleBase -from .evaluation_rule_evaluator_assignment import EvaluationRuleEvaluatorAssignment - - -class ReadableV2EvaluationRule(EvaluationRuleBase): - """ - Examples - -------- - import datetime - - from langfuse.evaluation_commons import EvaluationRuleReadFilterBase - from langfuse.unstable.commons import ( - EvaluationRuleStatus, - EvaluationRuleTarget, - EvaluatorType, - PromptVariableMappingRead, - PromptVariableMappingSource, - ) - from langfuse.unstable.evaluation_rules import ( - EvaluationRuleEvaluator, - EvaluationRuleEvaluatorAssignment, - ReadableV2EvaluationRule, - ) - - ReadableV2EvaluationRule( - id="erule_123", - name="answer-correctness-live", - evaluator=EvaluationRuleEvaluator( - id="evaltmpl_123", - name="answer-correctness", - type=EvaluatorType.LLM_AS_JUDGE, - ), - evaluators=[ - EvaluationRuleEvaluatorAssignment( - evaluator=EvaluationRuleEvaluator( - id="evaltmpl_123", - name="answer-correctness", - type=EvaluatorType.LLM_AS_JUDGE, - ), - mapping=[ - PromptVariableMappingRead( - variable="input", - source=PromptVariableMappingSource.INPUT, - ), - PromptVariableMappingRead( - variable="output", - source=PromptVariableMappingSource.OUTPUT, - ), - ], - ) - ], - target=EvaluationRuleTarget.OBSERVATION, - enabled=True, - status=EvaluationRuleStatus.ACTIVE, - sampling=1.0, - filter=[ - EvaluationRuleReadFilterBase( - type="stringOptions", - column="type", - operator="any of", - value=["GENERATION"], - ) - ], - mapping=[ - PromptVariableMappingRead( - variable="input", - source=PromptVariableMappingSource.INPUT, - ), - PromptVariableMappingRead( - variable="output", - source=PromptVariableMappingSource.OUTPUT, - ), - ], - created_at=datetime.datetime.fromisoformat( - "2026-03-30 09:20:00+00:00", - ), - updated_at=datetime.datetime.fromisoformat( - "2026-03-30 09:20:00+00:00", - ), - ) - """ - - evaluators: typing.List[EvaluationRuleEvaluatorAssignment] = pydantic.Field() - """ - Evaluators attached to this rule in deterministic assignment order. A `null` mapping inherits the evaluator version's default mapping. - """ - - target: EvaluationRuleTarget = pydantic.Field() - """ - Target object type that should trigger scoring. - """ - - filter: typing.List[EvaluationRuleReadFilter] = pydantic.Field() - """ - List of stored filter conditions returned verbatim. Historical filters remain readable even when the current write contract no longer accepts their shape. - """ - - mapping: typing.List[PromptVariableMappingRead] = pydantic.Field() - """ - Deprecated compatibility alias containing the effective mapping for `evaluators[0]`. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluation_rules/types/update_evaluation_rule_request.py b/langfuse/api/unstable/evaluation_rules/types/update_evaluation_rule_request.py deleted file mode 100644 index 567ab59f3..000000000 --- a/langfuse/api/unstable/evaluation_rules/types/update_evaluation_rule_request.py +++ /dev/null @@ -1,93 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from ...commons.types.evaluation_rule_filter import EvaluationRuleFilter -from ...commons.types.evaluation_rule_target import EvaluationRuleTarget -from ...commons.types.prompt_variable_mapping_input import PromptVariableMappingInput -from .create_evaluation_rule_evaluator_assignment import ( - CreateEvaluationRuleEvaluatorAssignment, -) -from .evaluation_rule_evaluator_reference import EvaluationRuleEvaluatorReference - - -class UpdateEvaluationRuleRequest(UniversalBaseModel): - """ - Partial update body for an evaluation rule. - - Provide only the fields you want to change. - An empty body is rejected. - - Practical guidance: - - If you only want to rename the rule or change sampling, send just those fields. - - To add, remove, or remap evaluators, send `evaluators`. It replaces the whole assignment set, so include every evaluator the rule should keep. - - `evaluators` cannot be combined with the deprecated `evaluator`/`mapping` pair, which only ever addressed the first assignment. - - If you change to an LLM-as-judge `evaluator`, send a fresh `mapping` unless you are certain the existing mapping still matches the prompt variables. - - If you change `target` for an LLM-as-judge rule, usually send both `filter` and `mapping` in the same request. - - For code evaluator rules, omit `mapping`; Langfuse stores the fixed code runtime mapping automatically. - - If you change an experiment `datasetId` filter, call `GET /api/public/v2/datasets` and use dataset `id` values from that response. - """ - - name: typing.Optional[str] = pydantic.Field(default=None) - """ - Updated deployment name. - """ - - evaluators: typing.Optional[ - typing.List[CreateEvaluationRuleEvaluatorAssignment] - ] = pydantic.Field(default=None) - """ - Full replacement of the rule's evaluator assignments: entries that are - not listed are detached. - - Mutually exclusive with the deprecated `evaluator` and `mapping` fields. - """ - - evaluator: typing.Optional[EvaluationRuleEvaluatorReference] = pydantic.Field( - default=None - ) - """ - Deprecated single-evaluator alias: updates the first assignment only. Prefer `evaluators`. - - Langfuse resolves the provided evaluator family to its latest version before saving the rule. - A rule's evaluator type cannot be changed: provide `name` for an evaluator family of the rule's current type. To use a different evaluator type, create a new rule. - """ - - target: typing.Optional[EvaluationRuleTarget] = pydantic.Field(default=None) - """ - Updated target object type. - """ - - enabled: typing.Optional[bool] = pydantic.Field(default=None) - """ - Updated desired enabled state. - """ - - sampling: typing.Optional[float] = pydantic.Field(default=None) - """ - Updated sampling fraction. - """ - - filter: typing.Optional[typing.List[EvaluationRuleFilter]] = pydantic.Field( - default=None - ) - """ - Updated filter list. - - For `target=experiment`, `column=datasetId` expects dataset `id` values from `GET /api/public/v2/datasets`, not dataset names. - """ - - mapping: typing.Optional[typing.List[PromptVariableMappingInput]] = pydantic.Field( - default=None - ) - """ - Updated LLM-as-judge variable mappings. - - Do not send this field for code evaluator rules. Langfuse stores the fixed code runtime mapping automatically and returns it in the response. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluators/__init__.py b/langfuse/api/unstable/evaluators/__init__.py deleted file mode 100644 index ec54f35c9..000000000 --- a/langfuse/api/unstable/evaluators/__init__.py +++ /dev/null @@ -1,82 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -# isort: skip_file - -import typing -from importlib import import_module - -if typing.TYPE_CHECKING: - from .types import ( - CodeEvaluator, - CreateCodeEvaluatorRequest, - CreateEvaluatorRequest, - CreateEvaluatorRequest_Code, - CreateEvaluatorRequest_LlmAsJudge, - CreateLlmAsJudgeEvaluatorRequest, - DeleteEvaluatorResponse, - Evaluator, - EvaluatorBase, - Evaluator_Code, - Evaluator_LlmAsJudge, - Evaluators, - LlmAsJudgeEvaluator, - ) -_dynamic_imports: typing.Dict[str, str] = { - "CodeEvaluator": ".types", - "CreateCodeEvaluatorRequest": ".types", - "CreateEvaluatorRequest": ".types", - "CreateEvaluatorRequest_Code": ".types", - "CreateEvaluatorRequest_LlmAsJudge": ".types", - "CreateLlmAsJudgeEvaluatorRequest": ".types", - "DeleteEvaluatorResponse": ".types", - "Evaluator": ".types", - "EvaluatorBase": ".types", - "Evaluator_Code": ".types", - "Evaluator_LlmAsJudge": ".types", - "Evaluators": ".types", - "LlmAsJudgeEvaluator": ".types", -} - - -def __getattr__(attr_name: str) -> typing.Any: - module_name = _dynamic_imports.get(attr_name) - if module_name is None: - raise AttributeError( - f"No {attr_name} found in _dynamic_imports for module name -> {__name__}" - ) - try: - module = import_module(module_name, __package__) - if module_name == f".{attr_name}": - return module - else: - return getattr(module, attr_name) - except ImportError as e: - raise ImportError( - f"Failed to import {attr_name} from {module_name}: {e}" - ) from e - except AttributeError as e: - raise AttributeError( - f"Failed to get {attr_name} from {module_name}: {e}" - ) from e - - -def __dir__(): - lazy_attrs = list(_dynamic_imports.keys()) - return sorted(lazy_attrs) - - -__all__ = [ - "CodeEvaluator", - "CreateCodeEvaluatorRequest", - "CreateEvaluatorRequest", - "CreateEvaluatorRequest_Code", - "CreateEvaluatorRequest_LlmAsJudge", - "CreateLlmAsJudgeEvaluatorRequest", - "DeleteEvaluatorResponse", - "Evaluator", - "EvaluatorBase", - "Evaluator_Code", - "Evaluator_LlmAsJudge", - "Evaluators", - "LlmAsJudgeEvaluator", -] diff --git a/langfuse/api/unstable/evaluators/client.py b/langfuse/api/unstable/evaluators/client.py deleted file mode 100644 index 516182dd9..000000000 --- a/langfuse/api/unstable/evaluators/client.py +++ /dev/null @@ -1,599 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import typing_extensions -from ...core.client_wrapper import AsyncClientWrapper, SyncClientWrapper -from ...core.request_options import RequestOptions -from .raw_client import AsyncRawEvaluatorsClient, RawEvaluatorsClient -from .types.create_evaluator_request import CreateEvaluatorRequest -from .types.delete_evaluator_response import DeleteEvaluatorResponse -from .types.evaluator import Evaluator -from .types.evaluators import Evaluators - -# this is used as the default value for optional parameters -OMIT = typing.cast(typing.Any, ...) - - -class EvaluatorsClient: - def __init__(self, *, client_wrapper: SyncClientWrapper): - self._raw_client = RawEvaluatorsClient(client_wrapper=client_wrapper) - - @property - def with_raw_response(self) -> RawEvaluatorsClient: - """ - Retrieves a raw implementation of this client that returns raw responses. - - Returns - ------- - RawEvaluatorsClient - """ - return self._raw_client - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def create( - self, - *, - request: CreateEvaluatorRequest, - request_options: typing.Optional[RequestOptions] = None, - ) -> Evaluator: - """ - Create an evaluator in the authenticated project. - - Use evaluators to define **how** Langfuse should score data. - LLM-as-a-judge evaluators define a prompt, expected structured output, and optional model configuration. - Code evaluators define source code and a runtime language. - - Naming behavior: - - If this is a new evaluator name in your project, Langfuse creates version `1`. - - If the name already exists in your project, Langfuse creates the next version and returns it. - - The evaluator `id` remains stable across versions. - - Existing evaluation rules automatically use the latest evaluator version; no rule update is required. - - Recommended workflow: - 1. Create the evaluator. - 2. Read the returned `variables` array. - 3. Read the returned `outputDefinition.dataType` so the client knows whether future scores will be numeric, boolean, or categorical. - 4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `type`. - - Code evaluator validation: - - At creation, Langfuse only validates the request shape - - The `sourceCode` itself is not executed here. It is first run (preflight-tested against a sample observation) when you link the evaluator to an evaluation rule, so runtime errors in the code surface at evaluation-rule creation, not at evaluator creation. - - Recovery guidance: - - `422` with `code=evaluator_preflight_failed`: the evaluator cannot run with the resolved model configuration. Add a valid explicit `modelConfig`, or configure the project's default evaluation model, then retry the same request. - - `400` with `code=invalid_body`: the request shape is malformed. Use the structured `details.issues` array to fix the specific fields and retry. - - `400` with `code=invalid_body` on `outputDefinition`: for `type=llm_as_judge`, send `dataType`, `reasoning.description`, and `score.description`. Do not send `version`; it is not part of the public request shape. - - If `type` is omitted, Langfuse treats the request as `type=llm_as_judge` for backwards compatibility. New clients should send `type` explicitly. - - Unstable API note: - - This surface may evolve while the underlying evaluation data model is being redesigned. - - Parameters - ---------- - request : CreateEvaluatorRequest - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - Evaluator - - Examples - -------- - from langfuse import LangfuseAPI - from langfuse.unstable.commons import ( - EvaluatorModelConfig, - EvaluatorOutputDataType, - EvaluatorOutputDefinition_Numeric, - EvaluatorOutputFieldDefinition, - PromptVariableMappingInput, - PromptVariableMappingSource, - PublicNumericEvaluatorOutputScoreDefinition, - ) - from langfuse.unstable.evaluators import CreateEvaluatorRequest_LlmAsJudge - - client = LangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - client.unstable.evaluators.create( - request=CreateEvaluatorRequest_LlmAsJudge( - name="answer-correctness", - prompt="You are grading an answer.\n\nInput:\n{{input}}\n\nOutput:\n{{output}}\n\nReturn a score between 0 and 1.\n", - output_definition=EvaluatorOutputDefinition_Numeric( - data_type=EvaluatorOutputDataType.NUMERIC, - reasoning=EvaluatorOutputFieldDefinition( - description="Explain why the score was assigned.", - ), - score=PublicNumericEvaluatorOutputScoreDefinition( - description="Correctness score between 0 and 1.", - ), - ), - model_config=EvaluatorModelConfig( - provider="openai", - model="gpt-4.1-mini", - ), - mapping=[ - PromptVariableMappingInput( - variable="input", - source=PromptVariableMappingSource.INPUT, - ), - PromptVariableMappingInput( - variable="output", - source=PromptVariableMappingSource.OUTPUT, - ), - ], - ), - ) - """ - _response = self._raw_client.create( - request=request, request_options=request_options - ) - return _response.data - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def list( - self, - *, - page: typing.Optional[int] = None, - limit: typing.Optional[int] = None, - request_options: typing.Optional[RequestOptions] = None, - ) -> Evaluators: - """ - List the evaluators available to the authenticated project. - - Important behavior: - - This endpoint returns the latest version of each available evaluator. - - Every evaluator is owned by the authenticated project. - - Parameters - ---------- - page : typing.Optional[int] - 1-based page number. Defaults to `1`. - - limit : typing.Optional[int] - Maximum number of items per page. Defaults to `50`. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - Evaluators - - Examples - -------- - from langfuse import LangfuseAPI - - client = LangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - client.unstable.evaluators.list() - """ - _response = self._raw_client.list( - page=page, limit=limit, request_options=request_options - ) - return _response.data - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def get( - self, - evaluator_id: str, - *, - request_options: typing.Optional[RequestOptions] = None, - ) -> Evaluator: - """ - Get one evaluator by `id`. - - This endpoint always returns the evaluator's latest version. Use it when you want the current prompt, output definition, model configuration, and derived variables for the evaluator you plan to use in an evaluation rule. - - Parameters - ---------- - evaluator_id : str - Evaluator identifier returned by the evaluator endpoints. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - Evaluator - - Examples - -------- - from langfuse import LangfuseAPI - - client = LangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - client.unstable.evaluators.get( - evaluator_id="evaluatorId", - ) - """ - _response = self._raw_client.get(evaluator_id, request_options=request_options) - return _response.data - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def delete( - self, - evaluator_id: str, - *, - request_options: typing.Optional[RequestOptions] = None, - ) -> DeleteEvaluatorResponse: - """ - Delete an evaluator. - - Important behavior: - - This deletes the evaluator including all of its stored versions. - - Evaluation rule assignments referencing the evaluator are also deleted. - - Scores already produced by the evaluator are not deleted. - - Parameters - ---------- - evaluator_id : str - Evaluator identifier returned by the evaluator endpoints. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - DeleteEvaluatorResponse - - Examples - -------- - from langfuse import LangfuseAPI - - client = LangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - client.unstable.evaluators.delete( - evaluator_id="evaluatorId", - ) - """ - _response = self._raw_client.delete( - evaluator_id, request_options=request_options - ) - return _response.data - - -class AsyncEvaluatorsClient: - def __init__(self, *, client_wrapper: AsyncClientWrapper): - self._raw_client = AsyncRawEvaluatorsClient(client_wrapper=client_wrapper) - - @property - def with_raw_response(self) -> AsyncRawEvaluatorsClient: - """ - Retrieves a raw implementation of this client that returns raw responses. - - Returns - ------- - AsyncRawEvaluatorsClient - """ - return self._raw_client - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def create( - self, - *, - request: CreateEvaluatorRequest, - request_options: typing.Optional[RequestOptions] = None, - ) -> Evaluator: - """ - Create an evaluator in the authenticated project. - - Use evaluators to define **how** Langfuse should score data. - LLM-as-a-judge evaluators define a prompt, expected structured output, and optional model configuration. - Code evaluators define source code and a runtime language. - - Naming behavior: - - If this is a new evaluator name in your project, Langfuse creates version `1`. - - If the name already exists in your project, Langfuse creates the next version and returns it. - - The evaluator `id` remains stable across versions. - - Existing evaluation rules automatically use the latest evaluator version; no rule update is required. - - Recommended workflow: - 1. Create the evaluator. - 2. Read the returned `variables` array. - 3. Read the returned `outputDefinition.dataType` so the client knows whether future scores will be numeric, boolean, or categorical. - 4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `type`. - - Code evaluator validation: - - At creation, Langfuse only validates the request shape - - The `sourceCode` itself is not executed here. It is first run (preflight-tested against a sample observation) when you link the evaluator to an evaluation rule, so runtime errors in the code surface at evaluation-rule creation, not at evaluator creation. - - Recovery guidance: - - `422` with `code=evaluator_preflight_failed`: the evaluator cannot run with the resolved model configuration. Add a valid explicit `modelConfig`, or configure the project's default evaluation model, then retry the same request. - - `400` with `code=invalid_body`: the request shape is malformed. Use the structured `details.issues` array to fix the specific fields and retry. - - `400` with `code=invalid_body` on `outputDefinition`: for `type=llm_as_judge`, send `dataType`, `reasoning.description`, and `score.description`. Do not send `version`; it is not part of the public request shape. - - If `type` is omitted, Langfuse treats the request as `type=llm_as_judge` for backwards compatibility. New clients should send `type` explicitly. - - Unstable API note: - - This surface may evolve while the underlying evaluation data model is being redesigned. - - Parameters - ---------- - request : CreateEvaluatorRequest - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - Evaluator - - Examples - -------- - import asyncio - - from langfuse import AsyncLangfuseAPI - from langfuse.unstable.commons import ( - EvaluatorModelConfig, - EvaluatorOutputDataType, - EvaluatorOutputDefinition_Numeric, - EvaluatorOutputFieldDefinition, - PromptVariableMappingInput, - PromptVariableMappingSource, - PublicNumericEvaluatorOutputScoreDefinition, - ) - from langfuse.unstable.evaluators import CreateEvaluatorRequest_LlmAsJudge - - client = AsyncLangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - - - async def main() -> None: - await client.unstable.evaluators.create( - request=CreateEvaluatorRequest_LlmAsJudge( - name="answer-correctness", - prompt="You are grading an answer.\n\nInput:\n{{input}}\n\nOutput:\n{{output}}\n\nReturn a score between 0 and 1.\n", - output_definition=EvaluatorOutputDefinition_Numeric( - data_type=EvaluatorOutputDataType.NUMERIC, - reasoning=EvaluatorOutputFieldDefinition( - description="Explain why the score was assigned.", - ), - score=PublicNumericEvaluatorOutputScoreDefinition( - description="Correctness score between 0 and 1.", - ), - ), - model_config=EvaluatorModelConfig( - provider="openai", - model="gpt-4.1-mini", - ), - mapping=[ - PromptVariableMappingInput( - variable="input", - source=PromptVariableMappingSource.INPUT, - ), - PromptVariableMappingInput( - variable="output", - source=PromptVariableMappingSource.OUTPUT, - ), - ], - ), - ) - - - asyncio.run(main()) - """ - _response = await self._raw_client.create( - request=request, request_options=request_options - ) - return _response.data - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def list( - self, - *, - page: typing.Optional[int] = None, - limit: typing.Optional[int] = None, - request_options: typing.Optional[RequestOptions] = None, - ) -> Evaluators: - """ - List the evaluators available to the authenticated project. - - Important behavior: - - This endpoint returns the latest version of each available evaluator. - - Every evaluator is owned by the authenticated project. - - Parameters - ---------- - page : typing.Optional[int] - 1-based page number. Defaults to `1`. - - limit : typing.Optional[int] - Maximum number of items per page. Defaults to `50`. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - Evaluators - - Examples - -------- - import asyncio - - from langfuse import AsyncLangfuseAPI - - client = AsyncLangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - - - async def main() -> None: - await client.unstable.evaluators.list() - - - asyncio.run(main()) - """ - _response = await self._raw_client.list( - page=page, limit=limit, request_options=request_options - ) - return _response.data - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def get( - self, - evaluator_id: str, - *, - request_options: typing.Optional[RequestOptions] = None, - ) -> Evaluator: - """ - Get one evaluator by `id`. - - This endpoint always returns the evaluator's latest version. Use it when you want the current prompt, output definition, model configuration, and derived variables for the evaluator you plan to use in an evaluation rule. - - Parameters - ---------- - evaluator_id : str - Evaluator identifier returned by the evaluator endpoints. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - Evaluator - - Examples - -------- - import asyncio - - from langfuse import AsyncLangfuseAPI - - client = AsyncLangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - - - async def main() -> None: - await client.unstable.evaluators.get( - evaluator_id="evaluatorId", - ) - - - asyncio.run(main()) - """ - _response = await self._raw_client.get( - evaluator_id, request_options=request_options - ) - return _response.data - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def delete( - self, - evaluator_id: str, - *, - request_options: typing.Optional[RequestOptions] = None, - ) -> DeleteEvaluatorResponse: - """ - Delete an evaluator. - - Important behavior: - - This deletes the evaluator including all of its stored versions. - - Evaluation rule assignments referencing the evaluator are also deleted. - - Scores already produced by the evaluator are not deleted. - - Parameters - ---------- - evaluator_id : str - Evaluator identifier returned by the evaluator endpoints. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - DeleteEvaluatorResponse - - Examples - -------- - import asyncio - - from langfuse import AsyncLangfuseAPI - - client = AsyncLangfuseAPI( - x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", - x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", - x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", - username="YOUR_USERNAME", - password="YOUR_PASSWORD", - base_url="https://yourhost.com/path/to/api", - ) - - - async def main() -> None: - await client.unstable.evaluators.delete( - evaluator_id="evaluatorId", - ) - - - asyncio.run(main()) - """ - _response = await self._raw_client.delete( - evaluator_id, request_options=request_options - ) - return _response.data diff --git a/langfuse/api/unstable/evaluators/raw_client.py b/langfuse/api/unstable/evaluators/raw_client.py deleted file mode 100644 index ac62620ad..000000000 --- a/langfuse/api/unstable/evaluators/raw_client.py +++ /dev/null @@ -1,1643 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing -import typing_extensions -from json.decoder import JSONDecodeError - -from ...commons.errors.access_denied_error import ( - AccessDeniedError as commons_errors_access_denied_error_AccessDeniedError, -) -from ...commons.errors.error import Error -from ...commons.errors.method_not_allowed_error import ( - MethodNotAllowedError as commons_errors_method_not_allowed_error_MethodNotAllowedError, -) -from ...commons.errors.not_found_error import ( - NotFoundError as commons_errors_not_found_error_NotFoundError, -) -from ...commons.errors.unauthorized_error import ( - UnauthorizedError as commons_errors_unauthorized_error_UnauthorizedError, -) -from ...core.api_error import ApiError -from ...core.client_wrapper import AsyncClientWrapper, SyncClientWrapper -from ...core.http_response import AsyncHttpResponse, HttpResponse -from ...core.jsonable_encoder import jsonable_encoder -from ...core.pydantic_utilities import parse_obj_as -from ...core.request_options import RequestOptions -from ...core.serialization import convert_and_respect_annotation_metadata -from ..errors.errors.access_denied_error import ( - AccessDeniedError as unstable_errors_errors_access_denied_error_AccessDeniedError, -) -from ..errors.errors.bad_request_error import BadRequestError -from ..errors.errors.conflict_error import ConflictError -from ..errors.errors.internal_server_error import InternalServerError -from ..errors.errors.method_not_allowed_error import ( - MethodNotAllowedError as unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError, -) -from ..errors.errors.not_found_error import ( - NotFoundError as unstable_errors_errors_not_found_error_NotFoundError, -) -from ..errors.errors.too_many_requests_error import TooManyRequestsError -from ..errors.errors.unauthorized_error import ( - UnauthorizedError as unstable_errors_errors_unauthorized_error_UnauthorizedError, -) -from ..errors.errors.unprocessable_content_error import UnprocessableContentError -from ..errors.types.public_api_error import PublicApiError -from .types.create_evaluator_request import CreateEvaluatorRequest -from .types.delete_evaluator_response import DeleteEvaluatorResponse -from .types.evaluator import Evaluator -from .types.evaluators import Evaluators - -# this is used as the default value for optional parameters -OMIT = typing.cast(typing.Any, ...) - - -class RawEvaluatorsClient: - def __init__(self, *, client_wrapper: SyncClientWrapper): - self._client_wrapper = client_wrapper - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def create( - self, - *, - request: CreateEvaluatorRequest, - request_options: typing.Optional[RequestOptions] = None, - ) -> HttpResponse[Evaluator]: - """ - Create an evaluator in the authenticated project. - - Use evaluators to define **how** Langfuse should score data. - LLM-as-a-judge evaluators define a prompt, expected structured output, and optional model configuration. - Code evaluators define source code and a runtime language. - - Naming behavior: - - If this is a new evaluator name in your project, Langfuse creates version `1`. - - If the name already exists in your project, Langfuse creates the next version and returns it. - - The evaluator `id` remains stable across versions. - - Existing evaluation rules automatically use the latest evaluator version; no rule update is required. - - Recommended workflow: - 1. Create the evaluator. - 2. Read the returned `variables` array. - 3. Read the returned `outputDefinition.dataType` so the client knows whether future scores will be numeric, boolean, or categorical. - 4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `type`. - - Code evaluator validation: - - At creation, Langfuse only validates the request shape - - The `sourceCode` itself is not executed here. It is first run (preflight-tested against a sample observation) when you link the evaluator to an evaluation rule, so runtime errors in the code surface at evaluation-rule creation, not at evaluator creation. - - Recovery guidance: - - `422` with `code=evaluator_preflight_failed`: the evaluator cannot run with the resolved model configuration. Add a valid explicit `modelConfig`, or configure the project's default evaluation model, then retry the same request. - - `400` with `code=invalid_body`: the request shape is malformed. Use the structured `details.issues` array to fix the specific fields and retry. - - `400` with `code=invalid_body` on `outputDefinition`: for `type=llm_as_judge`, send `dataType`, `reasoning.description`, and `score.description`. Do not send `version`; it is not part of the public request shape. - - If `type` is omitted, Langfuse treats the request as `type=llm_as_judge` for backwards compatibility. New clients should send `type` explicitly. - - Unstable API note: - - This surface may evolve while the underlying evaluation data model is being redesigned. - - Parameters - ---------- - request : CreateEvaluatorRequest - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[Evaluator] - """ - _response = self._client_wrapper.httpx_client.request( - "api/public/unstable/evaluators", - method="POST", - json=convert_and_respect_annotation_metadata( - object_=request, annotation=CreateEvaluatorRequest, direction="write" - ), - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - Evaluator, - parse_obj_as( - type_=Evaluator, # type: ignore - object_=_response.json(), - ), - ) - return HttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 409: - raise ConflictError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 422: - raise UnprocessableContentError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def list( - self, - *, - page: typing.Optional[int] = None, - limit: typing.Optional[int] = None, - request_options: typing.Optional[RequestOptions] = None, - ) -> HttpResponse[Evaluators]: - """ - List the evaluators available to the authenticated project. - - Important behavior: - - This endpoint returns the latest version of each available evaluator. - - Every evaluator is owned by the authenticated project. - - Parameters - ---------- - page : typing.Optional[int] - 1-based page number. Defaults to `1`. - - limit : typing.Optional[int] - Maximum number of items per page. Defaults to `50`. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[Evaluators] - """ - _response = self._client_wrapper.httpx_client.request( - "api/public/unstable/evaluators", - method="GET", - params={ - "page": page, - "limit": limit, - }, - request_options=request_options, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - Evaluators, - parse_obj_as( - type_=Evaluators, # type: ignore - object_=_response.json(), - ), - ) - return HttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def get( - self, - evaluator_id: str, - *, - request_options: typing.Optional[RequestOptions] = None, - ) -> HttpResponse[Evaluator]: - """ - Get one evaluator by `id`. - - This endpoint always returns the evaluator's latest version. Use it when you want the current prompt, output definition, model configuration, and derived variables for the evaluator you plan to use in an evaluation rule. - - Parameters - ---------- - evaluator_id : str - Evaluator identifier returned by the evaluator endpoints. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[Evaluator] - """ - _response = self._client_wrapper.httpx_client.request( - f"api/public/unstable/evaluators/{jsonable_encoder(evaluator_id)}", - method="GET", - request_options=request_options, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - Evaluator, - parse_obj_as( - type_=Evaluator, # type: ignore - object_=_response.json(), - ), - ) - return HttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise unstable_errors_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - def delete( - self, - evaluator_id: str, - *, - request_options: typing.Optional[RequestOptions] = None, - ) -> HttpResponse[DeleteEvaluatorResponse]: - """ - Delete an evaluator. - - Important behavior: - - This deletes the evaluator including all of its stored versions. - - Evaluation rule assignments referencing the evaluator are also deleted. - - Scores already produced by the evaluator are not deleted. - - Parameters - ---------- - evaluator_id : str - Evaluator identifier returned by the evaluator endpoints. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - HttpResponse[DeleteEvaluatorResponse] - """ - _response = self._client_wrapper.httpx_client.request( - f"api/public/unstable/evaluators/{jsonable_encoder(evaluator_id)}", - method="DELETE", - request_options=request_options, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - DeleteEvaluatorResponse, - parse_obj_as( - type_=DeleteEvaluatorResponse, # type: ignore - object_=_response.json(), - ), - ) - return HttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise unstable_errors_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) - - -class AsyncRawEvaluatorsClient: - def __init__(self, *, client_wrapper: AsyncClientWrapper): - self._client_wrapper = client_wrapper - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def create( - self, - *, - request: CreateEvaluatorRequest, - request_options: typing.Optional[RequestOptions] = None, - ) -> AsyncHttpResponse[Evaluator]: - """ - Create an evaluator in the authenticated project. - - Use evaluators to define **how** Langfuse should score data. - LLM-as-a-judge evaluators define a prompt, expected structured output, and optional model configuration. - Code evaluators define source code and a runtime language. - - Naming behavior: - - If this is a new evaluator name in your project, Langfuse creates version `1`. - - If the name already exists in your project, Langfuse creates the next version and returns it. - - The evaluator `id` remains stable across versions. - - Existing evaluation rules automatically use the latest evaluator version; no rule update is required. - - Recommended workflow: - 1. Create the evaluator. - 2. Read the returned `variables` array. - 3. Read the returned `outputDefinition.dataType` so the client knows whether future scores will be numeric, boolean, or categorical. - 4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `type`. - - Code evaluator validation: - - At creation, Langfuse only validates the request shape - - The `sourceCode` itself is not executed here. It is first run (preflight-tested against a sample observation) when you link the evaluator to an evaluation rule, so runtime errors in the code surface at evaluation-rule creation, not at evaluator creation. - - Recovery guidance: - - `422` with `code=evaluator_preflight_failed`: the evaluator cannot run with the resolved model configuration. Add a valid explicit `modelConfig`, or configure the project's default evaluation model, then retry the same request. - - `400` with `code=invalid_body`: the request shape is malformed. Use the structured `details.issues` array to fix the specific fields and retry. - - `400` with `code=invalid_body` on `outputDefinition`: for `type=llm_as_judge`, send `dataType`, `reasoning.description`, and `score.description`. Do not send `version`; it is not part of the public request shape. - - If `type` is omitted, Langfuse treats the request as `type=llm_as_judge` for backwards compatibility. New clients should send `type` explicitly. - - Unstable API note: - - This surface may evolve while the underlying evaluation data model is being redesigned. - - Parameters - ---------- - request : CreateEvaluatorRequest - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - AsyncHttpResponse[Evaluator] - """ - _response = await self._client_wrapper.httpx_client.request( - "api/public/unstable/evaluators", - method="POST", - json=convert_and_respect_annotation_metadata( - object_=request, annotation=CreateEvaluatorRequest, direction="write" - ), - request_options=request_options, - omit=OMIT, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - Evaluator, - parse_obj_as( - type_=Evaluator, # type: ignore - object_=_response.json(), - ), - ) - return AsyncHttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 409: - raise ConflictError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 422: - raise UnprocessableContentError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def list( - self, - *, - page: typing.Optional[int] = None, - limit: typing.Optional[int] = None, - request_options: typing.Optional[RequestOptions] = None, - ) -> AsyncHttpResponse[Evaluators]: - """ - List the evaluators available to the authenticated project. - - Important behavior: - - This endpoint returns the latest version of each available evaluator. - - Every evaluator is owned by the authenticated project. - - Parameters - ---------- - page : typing.Optional[int] - 1-based page number. Defaults to `1`. - - limit : typing.Optional[int] - Maximum number of items per page. Defaults to `50`. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - AsyncHttpResponse[Evaluators] - """ - _response = await self._client_wrapper.httpx_client.request( - "api/public/unstable/evaluators", - method="GET", - params={ - "page": page, - "limit": limit, - }, - request_options=request_options, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - Evaluators, - parse_obj_as( - type_=Evaluators, # type: ignore - object_=_response.json(), - ), - ) - return AsyncHttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def get( - self, - evaluator_id: str, - *, - request_options: typing.Optional[RequestOptions] = None, - ) -> AsyncHttpResponse[Evaluator]: - """ - Get one evaluator by `id`. - - This endpoint always returns the evaluator's latest version. Use it when you want the current prompt, output definition, model configuration, and derived variables for the evaluator you plan to use in an evaluation rule. - - Parameters - ---------- - evaluator_id : str - Evaluator identifier returned by the evaluator endpoints. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - AsyncHttpResponse[Evaluator] - """ - _response = await self._client_wrapper.httpx_client.request( - f"api/public/unstable/evaluators/{jsonable_encoder(evaluator_id)}", - method="GET", - request_options=request_options, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - Evaluator, - parse_obj_as( - type_=Evaluator, # type: ignore - object_=_response.json(), - ), - ) - return AsyncHttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise unstable_errors_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) - - @typing_extensions.deprecated( - "On Langfuse Cloud, this unstable endpoint is deprecated and will be removed on September 4, 2026. Use the stable `/api/public/v2/evaluators` API instead. Self-hosted deployments are unaffected by this date; the endpoint becomes unavailable when they upgrade to Langfuse v4.", - category=None, - ) - async def delete( - self, - evaluator_id: str, - *, - request_options: typing.Optional[RequestOptions] = None, - ) -> AsyncHttpResponse[DeleteEvaluatorResponse]: - """ - Delete an evaluator. - - Important behavior: - - This deletes the evaluator including all of its stored versions. - - Evaluation rule assignments referencing the evaluator are also deleted. - - Scores already produced by the evaluator are not deleted. - - Parameters - ---------- - evaluator_id : str - Evaluator identifier returned by the evaluator endpoints. - - request_options : typing.Optional[RequestOptions] - Request-specific configuration. - - Returns - ------- - AsyncHttpResponse[DeleteEvaluatorResponse] - """ - _response = await self._client_wrapper.httpx_client.request( - f"api/public/unstable/evaluators/{jsonable_encoder(evaluator_id)}", - method="DELETE", - request_options=request_options, - ) - try: - if 200 <= _response.status_code < 300: - _data = typing.cast( - DeleteEvaluatorResponse, - parse_obj_as( - type_=DeleteEvaluatorResponse, # type: ignore - object_=_response.json(), - ), - ) - return AsyncHttpResponse(response=_response, data=_data) - if _response.status_code == 400: - raise BadRequestError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise unstable_errors_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise unstable_errors_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise unstable_errors_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 429: - raise TooManyRequestsError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 500: - raise InternalServerError( - headers=dict(_response.headers), - body=typing.cast( - PublicApiError, - parse_obj_as( - type_=PublicApiError, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 400: - raise Error( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 401: - raise commons_errors_unauthorized_error_UnauthorizedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 403: - raise commons_errors_access_denied_error_AccessDeniedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 405: - raise commons_errors_method_not_allowed_error_MethodNotAllowedError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - if _response.status_code == 404: - raise commons_errors_not_found_error_NotFoundError( - headers=dict(_response.headers), - body=typing.cast( - typing.Any, - parse_obj_as( - type_=typing.Any, # type: ignore - object_=_response.json(), - ), - ), - ) - _response_json = _response.json() - except JSONDecodeError: - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response.text, - ) - raise ApiError( - status_code=_response.status_code, - headers=dict(_response.headers), - body=_response_json, - ) diff --git a/langfuse/api/unstable/evaluators/types/__init__.py b/langfuse/api/unstable/evaluators/types/__init__.py deleted file mode 100644 index fd031d61d..000000000 --- a/langfuse/api/unstable/evaluators/types/__init__.py +++ /dev/null @@ -1,80 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -# isort: skip_file - -import typing -from importlib import import_module - -if typing.TYPE_CHECKING: - from .code_evaluator import CodeEvaluator - from .create_code_evaluator_request import CreateCodeEvaluatorRequest - from .create_evaluator_request import ( - CreateEvaluatorRequest, - CreateEvaluatorRequest_Code, - CreateEvaluatorRequest_LlmAsJudge, - ) - from .create_llm_as_judge_evaluator_request import CreateLlmAsJudgeEvaluatorRequest - from .delete_evaluator_response import DeleteEvaluatorResponse - from .evaluator import Evaluator, Evaluator_Code, Evaluator_LlmAsJudge - from .evaluator_base import EvaluatorBase - from .evaluators import Evaluators - from .llm_as_judge_evaluator import LlmAsJudgeEvaluator -_dynamic_imports: typing.Dict[str, str] = { - "CodeEvaluator": ".code_evaluator", - "CreateCodeEvaluatorRequest": ".create_code_evaluator_request", - "CreateEvaluatorRequest": ".create_evaluator_request", - "CreateEvaluatorRequest_Code": ".create_evaluator_request", - "CreateEvaluatorRequest_LlmAsJudge": ".create_evaluator_request", - "CreateLlmAsJudgeEvaluatorRequest": ".create_llm_as_judge_evaluator_request", - "DeleteEvaluatorResponse": ".delete_evaluator_response", - "Evaluator": ".evaluator", - "EvaluatorBase": ".evaluator_base", - "Evaluator_Code": ".evaluator", - "Evaluator_LlmAsJudge": ".evaluator", - "Evaluators": ".evaluators", - "LlmAsJudgeEvaluator": ".llm_as_judge_evaluator", -} - - -def __getattr__(attr_name: str) -> typing.Any: - module_name = _dynamic_imports.get(attr_name) - if module_name is None: - raise AttributeError( - f"No {attr_name} found in _dynamic_imports for module name -> {__name__}" - ) - try: - module = import_module(module_name, __package__) - if module_name == f".{attr_name}": - return module - else: - return getattr(module, attr_name) - except ImportError as e: - raise ImportError( - f"Failed to import {attr_name} from {module_name}: {e}" - ) from e - except AttributeError as e: - raise AttributeError( - f"Failed to get {attr_name} from {module_name}: {e}" - ) from e - - -def __dir__(): - lazy_attrs = list(_dynamic_imports.keys()) - return sorted(lazy_attrs) - - -__all__ = [ - "CodeEvaluator", - "CreateCodeEvaluatorRequest", - "CreateEvaluatorRequest", - "CreateEvaluatorRequest_Code", - "CreateEvaluatorRequest_LlmAsJudge", - "CreateLlmAsJudgeEvaluatorRequest", - "DeleteEvaluatorResponse", - "Evaluator", - "EvaluatorBase", - "Evaluator_Code", - "Evaluator_LlmAsJudge", - "Evaluators", - "LlmAsJudgeEvaluator", -] diff --git a/langfuse/api/unstable/evaluators/types/code_evaluator.py b/langfuse/api/unstable/evaluators/types/code_evaluator.py deleted file mode 100644 index f8648603d..000000000 --- a/langfuse/api/unstable/evaluators/types/code_evaluator.py +++ /dev/null @@ -1,31 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ....core.serialization import FieldMetadata -from ...commons.types.code_evaluator_source_code_language import ( - CodeEvaluatorSourceCodeLanguage, -) -from .evaluator_base import EvaluatorBase - - -class CodeEvaluator(EvaluatorBase): - source_code: typing_extensions.Annotated[str, FieldMetadata(alias="sourceCode")] = ( - pydantic.Field() - ) - """ - Source code executed for each matched observation. - """ - - source_code_language: typing_extensions.Annotated[ - CodeEvaluatorSourceCodeLanguage, FieldMetadata(alias="sourceCodeLanguage") - ] = pydantic.Field() - """ - Runtime language for `sourceCode`. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluators/types/create_code_evaluator_request.py b/langfuse/api/unstable/evaluators/types/create_code_evaluator_request.py deleted file mode 100644 index 860c15f9a..000000000 --- a/langfuse/api/unstable/evaluators/types/create_code_evaluator_request.py +++ /dev/null @@ -1,36 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ....core.pydantic_utilities import UniversalBaseModel -from ....core.serialization import FieldMetadata -from ...commons.types.code_evaluator_source_code_language import ( - CodeEvaluatorSourceCodeLanguage, -) - - -class CreateCodeEvaluatorRequest(UniversalBaseModel): - name: str = pydantic.Field() - """ - Evaluator name within the authenticated project. - """ - - source_code: typing_extensions.Annotated[str, FieldMetadata(alias="sourceCode")] = ( - pydantic.Field() - ) - """ - Code executed for each matched observation. - """ - - source_code_language: typing_extensions.Annotated[ - CodeEvaluatorSourceCodeLanguage, FieldMetadata(alias="sourceCodeLanguage") - ] = pydantic.Field() - """ - Runtime language for `sourceCode`. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluators/types/create_evaluator_request.py b/langfuse/api/unstable/evaluators/types/create_evaluator_request.py deleted file mode 100644 index ea7e3c23c..000000000 --- a/langfuse/api/unstable/evaluators/types/create_evaluator_request.py +++ /dev/null @@ -1,68 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -from __future__ import annotations - -import typing - -import pydantic -import typing_extensions -from ....core.pydantic_utilities import UniversalBaseModel -from ....core.serialization import FieldMetadata -from ...commons.types.code_evaluator_source_code_language import ( - CodeEvaluatorSourceCodeLanguage, -) -from ...commons.types.evaluator_model_config import EvaluatorModelConfig -from ...commons.types.evaluator_output_definition import EvaluatorOutputDefinition -from ...commons.types.prompt_variable_mapping_input import PromptVariableMappingInput - - -class CreateEvaluatorRequest_LlmAsJudge(UniversalBaseModel): - """ - Request body for creating an evaluator. - - If the same `name` already exists in your project, Langfuse creates the next version and returns it. - Existing evaluation rules automatically use the latest evaluator version. - If `type` is omitted, Langfuse defaults it to `llm_as_judge` for backwards compatibility. - """ - - type: typing.Literal["llm_as_judge"] = "llm_as_judge" - name: str - prompt: str - output_definition: typing_extensions.Annotated[ - EvaluatorOutputDefinition, FieldMetadata(alias="outputDefinition") - ] - model_config_: typing_extensions.Annotated[ - typing.Optional[EvaluatorModelConfig], FieldMetadata(alias="modelConfig") - ] = None - mapping: typing.Optional[typing.List[PromptVariableMappingInput]] = None - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -class CreateEvaluatorRequest_Code(UniversalBaseModel): - """ - Request body for creating an evaluator. - - If the same `name` already exists in your project, Langfuse creates the next version and returns it. - Existing evaluation rules automatically use the latest evaluator version. - If `type` is omitted, Langfuse defaults it to `llm_as_judge` for backwards compatibility. - """ - - type: typing.Literal["code"] = "code" - name: str - source_code: typing_extensions.Annotated[str, FieldMetadata(alias="sourceCode")] - source_code_language: typing_extensions.Annotated[ - CodeEvaluatorSourceCodeLanguage, FieldMetadata(alias="sourceCodeLanguage") - ] - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -CreateEvaluatorRequest = typing_extensions.Annotated[ - typing.Union[CreateEvaluatorRequest_LlmAsJudge, CreateEvaluatorRequest_Code], - pydantic.Field(discriminator="type"), -] diff --git a/langfuse/api/unstable/evaluators/types/create_llm_as_judge_evaluator_request.py b/langfuse/api/unstable/evaluators/types/create_llm_as_judge_evaluator_request.py deleted file mode 100644 index a90f71c72..000000000 --- a/langfuse/api/unstable/evaluators/types/create_llm_as_judge_evaluator_request.py +++ /dev/null @@ -1,51 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ....core.pydantic_utilities import UniversalBaseModel -from ....core.serialization import FieldMetadata -from ...commons.types.evaluator_model_config import EvaluatorModelConfig -from ...commons.types.evaluator_output_definition import EvaluatorOutputDefinition -from ...commons.types.prompt_variable_mapping_input import PromptVariableMappingInput - - -class CreateLlmAsJudgeEvaluatorRequest(UniversalBaseModel): - name: str = pydantic.Field() - """ - Evaluator name within the authenticated project. - """ - - prompt: str = pydantic.Field() - """ - Prompt template used by the evaluator. - """ - - output_definition: typing_extensions.Annotated[ - EvaluatorOutputDefinition, FieldMetadata(alias="outputDefinition") - ] = pydantic.Field() - """ - Structured output schema the evaluator must return. - - Always send `dataType`. - Do not send `version`; it is an internal storage detail and not part of the public request contract. - """ - - model_config_: typing_extensions.Annotated[ - typing.Optional[EvaluatorModelConfig], FieldMetadata(alias="modelConfig") - ] = pydantic.Field(default=None) - """ - Optional explicit model configuration. Omit or set to `null` to use the project default evaluation model. - """ - - mapping: typing.Optional[typing.List[PromptVariableMappingInput]] = pydantic.Field( - default=None - ) - """ - Optional default variable mapping inherited by rule assignments that do not provide an override. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluators/types/delete_evaluator_response.py b/langfuse/api/unstable/evaluators/types/delete_evaluator_response.py deleted file mode 100644 index 1cbcc050f..000000000 --- a/langfuse/api/unstable/evaluators/types/delete_evaluator_response.py +++ /dev/null @@ -1,21 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel - - -class DeleteEvaluatorResponse(UniversalBaseModel): - """ - Confirmation response returned after successful deletion. - """ - - message: str = pydantic.Field() - """ - Always `Evaluator successfully deleted`. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluators/types/evaluator.py b/langfuse/api/unstable/evaluators/types/evaluator.py deleted file mode 100644 index 2b8387bc2..000000000 --- a/langfuse/api/unstable/evaluators/types/evaluator.py +++ /dev/null @@ -1,116 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -from __future__ import annotations - -import datetime as dt -import typing - -import pydantic -import typing_extensions -from ....core.pydantic_utilities import UniversalBaseModel -from ....core.serialization import FieldMetadata -from ...commons.types.code_evaluator_source_code_language import ( - CodeEvaluatorSourceCodeLanguage, -) -from ...commons.types.evaluator_model_config import EvaluatorModelConfig -from ...commons.types.prompt_variable_mapping_read import PromptVariableMappingRead -from ...commons.types.public_evaluator_output_definition import ( - PublicEvaluatorOutputDefinition, -) - - -class Evaluator_LlmAsJudge(UniversalBaseModel): - """ - One evaluator that can be used for scoring. - - An evaluator describes **how** to score data. - - It does not define **which** live objects are evaluated. That is the job of `evaluation-rules`. - - For agent clients, the most important fields are: - - `type`: determines which evaluator fields are present - - `variables`: for LLM evaluators, use these exact names when building the evaluation-rule `mapping` array. LLM evaluators require every variable to be mapped. Code evaluators always expose the fixed runtime payload fields and Langfuse maps them automatically. - - Versioning behavior: - - `GET /evaluators` returns the latest version of each available evaluator. - - `GET /evaluators/{id}` returns the latest version. - - `id` identifies the evaluator family and remains stable when a new version is created. - - Evaluation rules always run against the latest version for the selected evaluator name within the same source (`project` or `managed`). - """ - - type: typing.Literal["llm_as_judge"] = "llm_as_judge" - prompt: str - output_definition: typing_extensions.Annotated[ - PublicEvaluatorOutputDefinition, FieldMetadata(alias="outputDefinition") - ] - model_config_: typing_extensions.Annotated[ - typing.Optional[EvaluatorModelConfig], FieldMetadata(alias="modelConfig") - ] = None - id: str - name: str - version: int - variables: typing.List[str] - mapping: typing.Optional[typing.List[PromptVariableMappingRead]] = None - evaluation_rule_count: typing_extensions.Annotated[ - int, FieldMetadata(alias="evaluationRuleCount") - ] - created_at: typing_extensions.Annotated[ - dt.datetime, FieldMetadata(alias="createdAt") - ] - updated_at: typing_extensions.Annotated[ - dt.datetime, FieldMetadata(alias="updatedAt") - ] - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -class Evaluator_Code(UniversalBaseModel): - """ - One evaluator that can be used for scoring. - - An evaluator describes **how** to score data. - - It does not define **which** live objects are evaluated. That is the job of `evaluation-rules`. - - For agent clients, the most important fields are: - - `type`: determines which evaluator fields are present - - `variables`: for LLM evaluators, use these exact names when building the evaluation-rule `mapping` array. LLM evaluators require every variable to be mapped. Code evaluators always expose the fixed runtime payload fields and Langfuse maps them automatically. - - Versioning behavior: - - `GET /evaluators` returns the latest version of each available evaluator. - - `GET /evaluators/{id}` returns the latest version. - - `id` identifies the evaluator family and remains stable when a new version is created. - - Evaluation rules always run against the latest version for the selected evaluator name within the same source (`project` or `managed`). - """ - - type: typing.Literal["code"] = "code" - source_code: typing_extensions.Annotated[str, FieldMetadata(alias="sourceCode")] - source_code_language: typing_extensions.Annotated[ - CodeEvaluatorSourceCodeLanguage, FieldMetadata(alias="sourceCodeLanguage") - ] - id: str - name: str - version: int - variables: typing.List[str] - mapping: typing.Optional[typing.List[PromptVariableMappingRead]] = None - evaluation_rule_count: typing_extensions.Annotated[ - int, FieldMetadata(alias="evaluationRuleCount") - ] - created_at: typing_extensions.Annotated[ - dt.datetime, FieldMetadata(alias="createdAt") - ] - updated_at: typing_extensions.Annotated[ - dt.datetime, FieldMetadata(alias="updatedAt") - ] - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) - - -Evaluator = typing_extensions.Annotated[ - typing.Union[Evaluator_LlmAsJudge, Evaluator_Code], - pydantic.Field(discriminator="type"), -] diff --git a/langfuse/api/unstable/evaluators/types/evaluator_base.py b/langfuse/api/unstable/evaluators/types/evaluator_base.py deleted file mode 100644 index 6bf6484cd..000000000 --- a/langfuse/api/unstable/evaluators/types/evaluator_base.py +++ /dev/null @@ -1,70 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import datetime as dt -import typing - -import pydantic -import typing_extensions -from ....core.pydantic_utilities import UniversalBaseModel -from ....core.serialization import FieldMetadata -from ...commons.types.prompt_variable_mapping_read import PromptVariableMappingRead - - -class EvaluatorBase(UniversalBaseModel): - id: str = pydantic.Field() - """ - Stable identifier of this evaluator across all versions. - """ - - name: str = pydantic.Field() - """ - Evaluator name. - """ - - version: int = pydantic.Field() - """ - Version number of this evaluator. - """ - - variables: typing.List[str] = pydantic.Field() - """ - Variables that can be mapped when creating an evaluation rule. - - LLM evaluators require every variable to be mapped exactly once. Code evaluators always expose the fixed runtime payload fields and Langfuse maps them automatically. - """ - - mapping: typing.Optional[typing.List[PromptVariableMappingRead]] = pydantic.Field( - default=None - ) - """ - Default variable mapping for this evaluator version, or `null` when no default is configured. - - An entry's `source` is `null` when that variable was never fully configured, and sources - are not restricted by rule `target` here, because the default is stored on the evaluator - rather than on any one rule. - """ - - evaluation_rule_count: typing_extensions.Annotated[ - int, FieldMetadata(alias="evaluationRuleCount") - ] = pydantic.Field() - """ - Number of evaluation rules in the project that currently use this evaluator. - """ - - created_at: typing_extensions.Annotated[ - dt.datetime, FieldMetadata(alias="createdAt") - ] = pydantic.Field() - """ - Timestamp when this evaluator was created. - """ - - updated_at: typing_extensions.Annotated[ - dt.datetime, FieldMetadata(alias="updatedAt") - ] = pydantic.Field() - """ - Timestamp when this evaluator was last updated. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluators/types/evaluators.py b/langfuse/api/unstable/evaluators/types/evaluators.py deleted file mode 100644 index 51247a66e..000000000 --- a/langfuse/api/unstable/evaluators/types/evaluators.py +++ /dev/null @@ -1,17 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -from ....core.pydantic_utilities import UniversalBaseModel -from ....utils.pagination.types.meta_response import MetaResponse -from .evaluator import Evaluator - - -class Evaluators(UniversalBaseModel): - data: typing.List[Evaluator] - meta: MetaResponse - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - ) diff --git a/langfuse/api/unstable/evaluators/types/llm_as_judge_evaluator.py b/langfuse/api/unstable/evaluators/types/llm_as_judge_evaluator.py deleted file mode 100644 index 0cf186f47..000000000 --- a/langfuse/api/unstable/evaluators/types/llm_as_judge_evaluator.py +++ /dev/null @@ -1,40 +0,0 @@ -# This file was auto-generated by Fern from our API Definition. - -import typing - -import pydantic -import typing_extensions -from ....core.serialization import FieldMetadata -from ...commons.types.evaluator_model_config import EvaluatorModelConfig -from ...commons.types.public_evaluator_output_definition import ( - PublicEvaluatorOutputDefinition, -) -from .evaluator_base import EvaluatorBase - - -class LlmAsJudgeEvaluator(EvaluatorBase): - prompt: str = pydantic.Field() - """ - Prompt template used during evaluation. - """ - - output_definition: typing_extensions.Annotated[ - PublicEvaluatorOutputDefinition, FieldMetadata(alias="outputDefinition") - ] = pydantic.Field() - """ - Structured output schema returned by this evaluator. - - Responses always include `dataType` and omit the internal output-definition `version`. - Use `dataType` to decide how future scores should be interpreted. - """ - - model_config_: typing_extensions.Annotated[ - typing.Optional[EvaluatorModelConfig], FieldMetadata(alias="modelConfig") - ] = pydantic.Field(default=None) - """ - Explicit model configuration, or `null` when the project default evaluation model is used. - """ - - model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict( - extra="allow", frozen=True - )