diff --git a/pyproject.toml b/pyproject.toml index e29eb473f..186b130b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath-langchain" -version = "0.20.7" +version = "0.20.8" description = "Python SDK that enables developers to build and deploy LangGraph agents to the UiPath Cloud Platform" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/src/uipath_langchain/guardrails/middlewares/_base.py b/src/uipath_langchain/guardrails/middlewares/_base.py index afadd2c84..03579d6de 100644 --- a/src/uipath_langchain/guardrails/middlewares/_base.py +++ b/src/uipath_langchain/guardrails/middlewares/_base.py @@ -55,6 +55,11 @@ registry_references, tool_result_references, ) +from ._grouping import ( + guardrail_stage, + guardrails_in_use, + register_guardrail_node, +) from ._spans import guardrail_action_span from ._utils import ( convert_block_exception, @@ -416,16 +421,17 @@ async def _run_tool_guardrail( tool_args=input_data, ) try: - result = await asyncio.to_thread( - self._evaluate_guardrail, input_data, attachments - ) - modified_input = self._handle_validation_result( - result, - input_data, - scope=GuardrailScope.TOOL, - stage=GuardrailExecutionStage.PRE, - component=tool_name, - ) + with guardrail_stage(GuardrailScope.TOOL, GuardrailExecutionStage.PRE): + result = await asyncio.to_thread( + self._evaluate_guardrail, input_data, attachments + ) + modified_input = self._handle_validation_result( + result, + input_data, + scope=GuardrailScope.TOOL, + stage=GuardrailExecutionStage.PRE, + component=tool_name, + ) if modified_input is not None: request = create_modified_tool_request(request, modified_input) except GuardrailBlockException as exc: @@ -457,19 +463,22 @@ async def _run_tool_guardrail( tool_result=output_data, ) try: - result = await asyncio.to_thread( - self._evaluate_guardrail, output_data, attachments - ) - modified_output = self._handle_validation_result( - result, - output_data, - scope=GuardrailScope.TOOL, - stage=GuardrailExecutionStage.POST, - component=tool_name, - input_payload=json.dumps( - self._extract_tool_input_data(request) - ), - ) + with guardrail_stage( + GuardrailScope.TOOL, GuardrailExecutionStage.POST + ): + result = await asyncio.to_thread( + self._evaluate_guardrail, output_data, attachments + ) + modified_output = self._handle_validation_result( + result, + output_data, + scope=GuardrailScope.TOOL, + stage=GuardrailExecutionStage.POST, + component=tool_name, + input_payload=json.dumps( + self._extract_tool_input_data(request) + ), + ) if modified_output is not None: tool_result = create_modified_tool_result( tool_result, modified_output @@ -492,6 +501,7 @@ async def _run_tool_guardrail( def _create_tool_wrap_hook(self, guardrail_name: str) -> AgentMiddleware: """Create a wrap_tool_call hook that delegates to _run_tool_guardrail.""" + guardrails_in_use() middleware_instance = self async def _wrap_tool_call_func( @@ -547,6 +557,11 @@ async def _before_agent_func( ) _before_agent_func.__name__ = f"{guardrail_name}_before_agent" + register_guardrail_node( + f"{_before_agent_func.__name__}.before_agent", + GuardrailScope.AGENT, + GuardrailExecutionStage.PRE, + ) hooks.append(before_agent(_before_agent_func)) if include_post: @@ -575,6 +590,11 @@ async def _after_agent_func( ) _after_agent_func.__name__ = f"{guardrail_name}_after_agent" + register_guardrail_node( + f"{_after_agent_func.__name__}.after_agent", + GuardrailScope.AGENT, + GuardrailExecutionStage.POST, + ) hooks.append(after_agent(_after_agent_func)) elif scope == GuardrailScope.LLM: @@ -596,6 +616,11 @@ async def _before_model_func( ) _before_model_func.__name__ = f"{guardrail_name}_before_model" + register_guardrail_node( + f"{_before_model_func.__name__}.before_model", + GuardrailScope.LLM, + GuardrailExecutionStage.PRE, + ) hooks.append(before_model(_before_model_func)) if include_post: @@ -619,6 +644,11 @@ async def _after_model_func( ) _after_model_func.__name__ = f"{guardrail_name}_after_model" + register_guardrail_node( + f"{_after_model_func.__name__}.after_model", + GuardrailScope.LLM, + GuardrailExecutionStage.POST, + ) hooks.append(after_model(_after_model_func)) return hooks @@ -656,15 +686,16 @@ def _check_messages( payload = text or NO_TEXT_PLACEHOLDER try: - result = self._evaluate_guardrail(payload, attachments) - modified_text = self._handle_validation_result( - result, - payload, - scope=scope, - stage=stage, - component=component_label(scope), - input_payload=json.dumps(input_text) if input_text else None, - ) + with guardrail_stage(scope, stage): + result = self._evaluate_guardrail(payload, attachments) + modified_text = self._handle_validation_result( + result, + payload, + scope=scope, + stage=stage, + component=component_label(scope), + input_payload=json.dumps(input_text) if input_text else None, + ) if text: _apply_text_modification(messages, text, modified_text) except GuardrailBlockException as exc: diff --git a/src/uipath_langchain/guardrails/middlewares/_grouping.py b/src/uipath_langchain/guardrails/middlewares/_grouping.py new file mode 100644 index 000000000..3236e4e27 --- /dev/null +++ b/src/uipath_langchain/guardrails/middlewares/_grouping.py @@ -0,0 +1,311 @@ +"""Group a stage's guardrail checks into one container span. + +Each guardrail hook runs as its own LangGraph node, so the checks of one stage +(e.g. LLM input) are spread over several nodes. The first check of a stage +starts a container next to those nodes (inside ``tools`` for tool checks); +later checks reuse it. ``GuardrailStageCloser`` ends it when something else +starts beside it or its parent ends, and ``is_exported`` hides the now-empty +guardrail node spans. +""" + +import logging +import threading +import time +from collections.abc import Callable, Generator +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +from typing import Any, NamedTuple, TypeGuard + +from langgraph.errors import GraphBubbleUp +from opentelemetry import context as otel_context +from opentelemetry import trace +from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor +from opentelemetry.trace import NonRecordingSpan, Span, SpanContext, StatusCode +from uipath.core.guardrails import GuardrailScope +from uipath.core.tracing.span_utils import UiPathSpanUtils +from uipath.platform.guardrails.decorators import GuardrailExecutionStage + +from ._span_name import SpanName, SpanType +from ._spans import error_text, new_span + +logger = logging.getLogger(__name__) + + +class _Stage(NamedTuple): + name: str + span_type: str + + +_PRE = GuardrailExecutionStage.PRE +_POST = GuardrailExecutionStage.POST + +_STAGES: dict[tuple[GuardrailScope, GuardrailExecutionStage], _Stage] = { + (GuardrailScope.AGENT, _PRE): _Stage( + SpanName.AGENT_PRE_GUARDRAILS, SpanType.AGENT_PRE_GUARDRAILS + ), + (GuardrailScope.AGENT, _POST): _Stage( + SpanName.AGENT_POST_GUARDRAILS, SpanType.AGENT_POST_GUARDRAILS + ), + (GuardrailScope.LLM, _PRE): _Stage( + SpanName.LLM_PRE_GUARDRAILS, SpanType.LLM_PRE_GUARDRAILS + ), + (GuardrailScope.LLM, _POST): _Stage( + SpanName.LLM_POST_GUARDRAILS, SpanType.LLM_POST_GUARDRAILS + ), + (GuardrailScope.TOOL, _PRE): _Stage( + SpanName.TOOL_PRE_GUARDRAILS, SpanType.TOOL_PRE_GUARDRAILS + ), + (GuardrailScope.TOOL, _POST): _Stage( + SpanName.TOOL_POST_GUARDRAILS, SpanType.TOOL_POST_GUARDRAILS + ), +} + + +@dataclass +class _Container: + span: Span + span_type: str + last_check_end: int = 0 # ns + + def end(self, end_time: int | None) -> None: + status = getattr(self.span, "status", None) + if status is None or status.status_code is StatusCode.UNSET: + self.span.set_status(StatusCode.OK) + # Never before its last check: coarse clocks (Windows before Python + # 3.13) can stamp the next sibling's start a tick earlier. + self.span.end(end_time=max(end_time, self.last_check_end) if end_time else None) + + +class _GroupingState: + """Process-wide state. Grouping is on while a closer is attached.""" + + def __init__(self) -> None: + self.in_use = False # a guardrail middleware was built + self.trace_manager: Any = None # offered by the runtime factory + self.closer: GuardrailStageCloser | None = None + self.guardrail_nodes: dict[str, str] = {} # node name -> stage span type + # Parent span id -> its open container. One at a time: starting a + # stage's container ends the other stage's one under the same parent. + self.containers: dict[int, _Container] = {} + # Re-entrant: starting a container runs the closer's on_start, which + # takes it again on the same thread. + self.lock = threading.RLock() + + @property + def enabled(self) -> bool: + return self.closer is not None + + +_state = _GroupingState() + +_current_container: ContextVar[Span | None] = ContextVar( + "uipath_guardrail_container", default=None +) + + +def enable_guardrail_grouping( + trace_manager: Any, current_span_provider: Callable[[], Span | None] +) -> None: + """Register the span provider and offer the trace manager for grouping. + + Grouping turns on once a guardrail middleware is also in use. + """ + UiPathSpanUtils.register_current_span_provider( + guardrail_aware(current_span_provider) + ) + _state.trace_manager = trace_manager + _activate() + + +def guardrails_in_use() -> None: + """Called by guardrail middlewares when built; grouping needs one in use.""" + _state.in_use = True + _activate() + + +def register_guardrail_node( + node_name: str, scope: GuardrailScope, stage: GuardrailExecutionStage +) -> None: + """Record which stage a guardrail hook's LangGraph node belongs to.""" + _state.guardrail_nodes[node_name] = _STAGES[(scope, stage)].span_type + guardrails_in_use() + + +def _activate() -> None: + """Attach a closer to the offered trace manager if guardrails are in use.""" + with _state.lock: + manager = _state.trace_manager + if manager is None or not _state.in_use: + return + if _state.closer is None or _state.closer.trace_manager is not manager: + _state.closer = GuardrailStageCloser(manager) + manager.add_span_processor(_state.closer) + + +@contextmanager +def guardrail_stage( + scope: GuardrailScope | None, stage: GuardrailExecutionStage | None +) -> Generator[None]: + """Run one check inside its stage's shared container.""" + try: + container = _shared_container(scope, stage) + except Exception: + logger.warning("Could not group a guardrail check", exc_info=True) + container = None + if container is None: + yield + return + token = _current_container.set(container.span) + try: + yield + except GraphBubbleUp: + raise + except Exception as exc: + container.span.set_status(StatusCode.ERROR, error_text(exc)) + raise + finally: + _current_container.reset(token) + container.last_check_end = time.time_ns() + + +def guardrail_aware( + provider: Callable[[], Span | None], +) -> Callable[[], Span | None]: + """Report the running check's container as the current span.""" + + def current_span() -> Span | None: + return _current_container.get() or provider() + + return current_span + + +def _shared_container( + scope: GuardrailScope | None, stage: GuardrailExecutionStage | None +) -> _Container | None: + """Get the stage's open container, or start one. ``None`` if not grouped.""" + stage_spans = _STAGES.get((scope, stage)) # type: ignore[arg-type] + if not _state.enabled or stage_spans is None: + return None + place = _container_place(scope) + if place is None: + return None + node, parent = place + return _open_container(stage_spans, parent, node) + + +def _container_place(scope: GuardrailScope | None) -> tuple[Span, SpanContext] | None: + """The check's node and the container's parent, if a container there would close. + + Tool checks go inside the ``tools`` node; others beside their own node. + """ + node = UiPathSpanUtils.get_external_current_span() + if node is None or not node.is_recording(): + return None # a sampled-out node never reports its end + if scope == GuardrailScope.TOOL: + parent: SpanContext | None = node.get_span_context() + else: + parent = getattr(node, "parent", None) + if not _can_group_under(parent): + return None + return node, parent + + +def _open_container(stage: _Stage, parent: SpanContext, node: Span) -> _Container: + """Reuse ``stage``'s open container under ``parent``, or start it there. + + Starting it ends the other stage's container under the same parent, even if + no sibling started in between (a middleware that answers a tool call itself). + """ + with _state.lock: + previous = _state.containers.get(parent.span_id) + if previous is not None and previous.span_type == stage.span_type: + return previous + # Out while the new one starts, so the closer doesn't end it under the + # lock; back in if the start fails. + _state.containers.pop(parent.span_id, None) + try: + container = _start_container(stage, parent, node) + except Exception: + if previous is not None: + _state.containers[parent.span_id] = previous + raise + _state.containers[parent.span_id] = container + if previous is not None: + previous.end(getattr(container.span, "start_time", None)) + return container + + +def _can_group_under(parent: SpanContext | None) -> TypeGuard[SpanContext]: + """Whether a container under ``parent`` would ever be closed.""" + return parent is not None and parent.is_valid and parent.trace_flags.sampled + + +def _start_container(stage: _Stage, parent: SpanContext, node: Span) -> _Container: + """Start ``stage``'s container under ``parent``.""" + span = new_span( + stage.name, + stage.span_type, + context=trace.set_span_in_context(NonRecordingSpan(parent)), + **{ + # Its parent is non-recording, so this isn't inherited. + "execution.id": (getattr(node, "attributes", None) or {}).get( + "execution.id" + ), + }, + ) + return _Container(span, stage.span_type) + + +class GuardrailStageCloser(SpanProcessor): + """End a container when a non-guardrail sibling starts or its parent ends.""" + + def __init__(self, trace_manager: Any = None) -> None: + self.trace_manager = trace_manager + + def on_start( + self, span: ReadableSpan, parent_context: otel_context.Context | None = None + ) -> None: + if not _state.containers or span.parent is None: + return + with _state.lock: + container = _state.containers.get(span.parent.span_id) + # A guardrail node of the same stage joins the container; anything + # else starting beside it ends the stage. + if container is None or container.span_type == _state.guardrail_nodes.get( + span.name + ): + return + del _state.containers[span.parent.span_id] + container.end(span.start_time) + + def on_end(self, span: ReadableSpan) -> None: + if not _state.containers or span.context is None: + return + with _state.lock: + container = _state.containers.pop(span.context.span_id, None) + if container is not None: + container.end(span.end_time) + + def shutdown(self) -> None: + """Called when its trace manager shuts down: detach, end what's open.""" + with _state.lock: + if self is not _state.closer: + return + _state.closer = None + if _state.trace_manager is self.trace_manager: + _state.trace_manager = None + still_open = list(_state.containers.values()) + _state.containers.clear() + for container in still_open: + container.end(None) + + +def is_exported(span: ReadableSpan) -> bool: + """Hide guardrail node spans. Decided by name: a span uploads when it starts.""" + if not _state.enabled or span.name not in _state.guardrail_nodes: + return True + if not _can_group_under(span.parent): + return True # its checks stay under it, ungrouped + scope = getattr(span, "instrumentation_scope", None) + return "openinference" not in (getattr(scope, "name", "") or "") diff --git a/src/uipath_langchain/guardrails/middlewares/_span_name.py b/src/uipath_langchain/guardrails/middlewares/_span_name.py new file mode 100644 index 000000000..9e91ba7b4 --- /dev/null +++ b/src/uipath_langchain/guardrails/middlewares/_span_name.py @@ -0,0 +1,26 @@ +"""Guardrail span types and display names.""" + + +class SpanType: + """Values of a guardrail span's ``span_type``.""" + + GUARDRAIL_ACTION = "guardrailAction" + GUARDRAIL_EVALUATION = "guardrailEvaluation" + + AGENT_PRE_GUARDRAILS = "agentPreGuardrails" + AGENT_POST_GUARDRAILS = "agentPostGuardrails" + LLM_PRE_GUARDRAILS = "llmPreGuardrails" + LLM_POST_GUARDRAILS = "llmPostGuardrails" + TOOL_PRE_GUARDRAILS = "toolPreGuardrails" + TOOL_POST_GUARDRAILS = "toolPostGuardrails" + + +class SpanName: + """Display names of the stage containers.""" + + AGENT_PRE_GUARDRAILS = "Agent input guardrail check" + AGENT_POST_GUARDRAILS = "Agent output guardrail check" + LLM_PRE_GUARDRAILS = "LLM input guardrail check" + LLM_POST_GUARDRAILS = "LLM output guardrail check" + TOOL_PRE_GUARDRAILS = "Tool input guardrail check" + TOOL_POST_GUARDRAILS = "Tool output guardrail check" diff --git a/src/uipath_langchain/guardrails/middlewares/_spans.py b/src/uipath_langchain/guardrails/middlewares/_spans.py index 36c8eba19..7cf0626f1 100644 --- a/src/uipath_langchain/guardrails/middlewares/_spans.py +++ b/src/uipath_langchain/guardrails/middlewares/_spans.py @@ -1,26 +1,34 @@ -"""Guardrail spans shaped like low-code agents'.""" +"""Spans for one guardrail check: the rule evaluation and the action taken on a +violation. ``_grouping`` puts them in their stage's container.""" from collections.abc import Generator from contextlib import contextmanager from typing import Any from langgraph.errors import GraphBubbleUp +from opentelemetry import context as otel_context from opentelemetry import trace from opentelemetry.trace import Span, StatusCode from uipath.core.guardrails import GuardrailValidationResult from uipath.core.tracing.span_utils import UiPathSpanUtils -from uipath.platform.guardrails.decorators import LoggingSeverityLevel +from uipath.platform.guardrails.decorators import ( + LoggingSeverityLevel, +) from uipath.platform.guardrails.decorators._exceptions import GuardrailBlockException -GUARDRAIL_ACTION = "guardrailAction" -GUARDRAIL_EVALUATION = "guardrailEvaluation" +from ._span_name import SpanType -def _new_span(name: str, span_type: str, **attributes: Any) -> Span: +def new_span( + name: str, + span_type: str, + context: otel_context.Context | None = None, + **attributes: Any, +) -> Span: """Start a span under the deepest active span, as ``@traced`` does.""" return trace.get_tracer(__name__).start_span( name, - context=UiPathSpanUtils.get_parent_context(), + context=context or UiPathSpanUtils.get_parent_context(), attributes={ "type": span_type, "span_type": span_type, @@ -42,12 +50,12 @@ def _active(span: Span) -> Generator[Span]: span.set_status(StatusCode.OK) raise except Exception as exc: - span.set_status(StatusCode.ERROR, _error_text(exc)) + span.set_status(StatusCode.ERROR, error_text(exc)) raise span.set_status(StatusCode.OK) -def _error_text(exc: BaseException) -> str: +def error_text(exc: BaseException) -> str: if isinstance(exc, GuardrailBlockException): return f"Guardrail violation: {exc.detail}" return str(exc) @@ -74,9 +82,9 @@ def guardrail_action_span( ) -> Generator[Span]: """Record the action taken on a failed ``result``.""" severity = getattr(action, "severity_level", None) - span = _new_span( + span = new_span( action_name(action), - GUARDRAIL_ACTION, + SpanType.GUARDRAIL_ACTION, guardrailName=guardrail_name, action=action_name(action), guardrailSpanId=evaluation_span_id or result.span_id, @@ -96,9 +104,9 @@ def guardrail_evaluation_span( guardrail_name: str, description: str | None = None ) -> Generator[Span]: """Record a rule check that runs in the agent.""" - span = _new_span( + span = new_span( guardrail_name, - GUARDRAIL_EVALUATION, + SpanType.GUARDRAIL_EVALUATION, guardrailName=guardrail_name, guardrailDescription=description, guardrailType="custom", diff --git a/src/uipath_langchain/guardrails/middlewares/deterministic.py b/src/uipath_langchain/guardrails/middlewares/deterministic.py index 9a65af3d2..b578a483a 100644 --- a/src/uipath_langchain/guardrails/middlewares/deterministic.py +++ b/src/uipath_langchain/guardrails/middlewares/deterministic.py @@ -12,6 +12,7 @@ from langgraph.prebuilt.tool_node import ToolCallRequest from langgraph.types import Command from uipath.core.guardrails import ( + GuardrailScope, GuardrailValidationResult, GuardrailValidationResultType, ) @@ -21,6 +22,7 @@ from ..enums import GuardrailExecutionStage from ..models import GuardrailAction +from ._grouping import guardrail_stage, guardrails_in_use from ._spans import ( guardrail_action_span, guardrail_evaluation_span, @@ -162,6 +164,7 @@ def __init__( def _create_middleware_instances(self) -> list[AgentMiddleware]: """Create middleware instances from decorated functions.""" + guardrails_in_use() instances = [] middleware_instance = self guardrail_name = self._name.replace(" ", "_") @@ -240,16 +243,16 @@ def _evaluate_and_apply_modifications( ToolCallRequest | None, dict[str, Any] | None, ToolMessage | Command[Any] | None ]: """Evaluate rules and apply modifications if validation fails.""" - with guardrail_evaluation_span(self._name, self._description) as span: - result = self._evaluate_rules(rules, stage, input_data, output_data) - set_validation_result(span, result) - if result.result != GuardrailValidationResultType.VALIDATION_FAILED: - return request, input_data, tool_result - try: - modified_data = self._handle_validation_result( - result, data_for_action, evaluation_span_id=span_guid(span) - ) + with guardrail_stage(GuardrailScope.TOOL, stage): + with guardrail_evaluation_span(self._name, self._description) as span: + result = self._evaluate_rules(rules, stage, input_data, output_data) + set_validation_result(span, result) + if result.result != GuardrailValidationResultType.VALIDATION_FAILED: + return request, input_data, tool_result + modified_data = self._handle_validation_result( + result, data_for_action, evaluation_span_id=span_guid(span) + ) except GuardrailBlockException as exc: raise convert_block_exception(exc) from exc except AgentRuntimeError: diff --git a/src/uipath_langchain/runtime/factory.py b/src/uipath_langchain/runtime/factory.py index d983cedae..864868700 100644 --- a/src/uipath_langchain/runtime/factory.py +++ b/src/uipath_langchain/runtime/factory.py @@ -11,7 +11,7 @@ get_current_span, ) from uipath.core.adapters import EvaluatorProtocol -from uipath.core.tracing import UiPathSpanUtils, UiPathTraceManager +from uipath.core.tracing import UiPathSpanUtils, UiPathTraceManager, UiPathTraceSettings from uipath.platform.resume_triggers import ( UiPathResumeTriggerHandler, ) @@ -28,6 +28,10 @@ from uipath_langchain._tracing import _instrument_traceable_attributes from uipath_langchain.chat.chat_model_factory import _TraceContextHeadersCallback from uipath_langchain.governance import GovernanceCallbackHandler +from uipath_langchain.guardrails.middlewares._grouping import ( + enable_guardrail_grouping, + is_exported, +) from uipath_langchain.runtime.config import LangGraphConfig from uipath_langchain.runtime.errors import LangGraphErrorCode, LangGraphRuntimeError from uipath_langchain.runtime.graph import LangGraphLoader @@ -67,7 +71,7 @@ def _setup_instrumentation(self, trace_manager: UiPathTraceManager | None) -> No """Setup tracing and instrumentation.""" _instrument_traceable_attributes() LangChainInstrumentor().instrument() - UiPathSpanUtils.register_current_span_provider(get_current_span) + enable_guardrail_grouping(trace_manager, get_current_span) UiPathSpanUtils.register_current_span_ancestors_provider(get_ancestor_spans) def _get_connection_string(self) -> str: @@ -251,6 +255,7 @@ async def get_settings(self) -> UiPathRuntimeFactorySettings | None: return UiPathRuntimeFactorySettings( agent_type=_AGENT_TYPE_CODED, agent_framework=_AGENT_FRAMEWORK, + trace_settings=UiPathTraceSettings(span_filter=is_exported), ) async def get_storage(self) -> UiPathRuntimeStorageProtocol | None: diff --git a/tests/guardrails/middlewares/test_action_span.py b/tests/guardrails/middlewares/test_action_span.py index 7d68c8896..a5d3c1875 100644 --- a/tests/guardrails/middlewares/test_action_span.py +++ b/tests/guardrails/middlewares/test_action_span.py @@ -32,6 +32,7 @@ from uipath_langchain.guardrails.middlewares import ( UiPathDeterministicGuardrailMiddleware, UiPathPIIDetectionMiddleware, + _grouping, ) from uipath_langchain.guardrails.middlewares._spans import action_name from uipath_langchain.guardrails.models import PIIDetectionEntity @@ -49,6 +50,7 @@ @pytest.fixture def recorded_spans(monkeypatch: pytest.MonkeyPatch) -> Iterator[InMemorySpanExporter]: + monkeypatch.setattr(_grouping, "_state", _grouping._GroupingState()) provider = TracerProvider() exporter = InMemorySpanExporter() provider.add_span_processor(SimpleSpanProcessor(exporter)) diff --git a/tests/guardrails/middlewares/test_grouping.py b/tests/guardrails/middlewares/test_grouping.py new file mode 100644 index 000000000..c08bfc639 --- /dev/null +++ b/tests/guardrails/middlewares/test_grouping.py @@ -0,0 +1,801 @@ +"""Separate guardrail middlewares share one container span per stage.""" + +import asyncio +import threading +import time +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from langchain.agents import create_agent +from langchain_core.language_models.fake_chat_models import GenericFakeChatModel +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from langchain_core.tools import tool +from langgraph.prebuilt.tool_node import ToolCallRequest +from openinference.instrumentation.langchain import ( + LangChainInstrumentor, + get_ancestor_spans, + get_current_span, +) +from opentelemetry import trace +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.sdk.trace.sampling import ALWAYS_ON +from opentelemetry.trace import ( + NonRecordingSpan, + Span, + SpanContext, + StatusCode, + TraceFlags, +) +from uipath.core.guardrails import ( + GuardrailScope, + GuardrailValidationResult, + GuardrailValidationResultType, +) +from uipath.core.tracing.span_utils import UiPathSpanUtils +from uipath.platform.guardrails.decorators import BlockAction, LogAction + +from uipath_langchain.agent.exceptions import AgentRuntimeError +from uipath_langchain.guardrails import GuardrailExecutionStage +from uipath_langchain.guardrails.enums import PIIDetectionEntityType +from uipath_langchain.guardrails.middlewares import ( + UiPathDeterministicGuardrailMiddleware, + UiPathPIIDetectionMiddleware, + UiPathPromptInjectionMiddleware, + _base, + _grouping, +) +from uipath_langchain.guardrails.middlewares._grouping import ( + enable_guardrail_grouping, + guardrail_stage, + is_exported, +) +from uipath_langchain.guardrails.models import PIIDetectionEntity + +_FAILED = GuardrailValidationResult( + result=GuardrailValidationResultType.VALIDATION_FAILED, reason="found" +) +_STATE = {"messages": [HumanMessage(content="mail me at a@b.com")]} +_OPENINFERENCE = "openinference.instrumentation.langchain" + + +@pytest.fixture +def exporter(monkeypatch: pytest.MonkeyPatch) -> Iterator[InMemorySpanExporter]: + monkeypatch.setattr(_grouping, "_state", _grouping._GroupingState()) + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + monkeypatch.setattr(trace, "get_tracer", provider.get_tracer) + exporter.provider = provider # type: ignore[attr-defined] + yield exporter + UiPathSpanUtils.register_current_span_provider(None) + UiPathSpanUtils.register_current_span_ancestors_provider(None) + provider.shutdown() + + +class _Graph: + """Plays LangGraph: a graph span and node spans started in order.""" + + def __init__(self, exporter: InMemorySpanExporter) -> None: + self.exporter = exporter + self._tracer = exporter.provider.get_tracer(_OPENINFERENCE) # type: ignore[attr-defined] + self.current: Span | None = None + self.span = self._tracer.start_span("LangGraph") + + @contextmanager + def node( + self, + name: str, + parent: Span | None = None, + attributes: dict[str, Any] | None = None, + start_time: int | None = None, + ) -> Iterator[Span]: + span = self._tracer.start_span( + name, + context=trace.set_span_in_context(parent or self.span), + attributes=attributes, + start_time=start_time, + ) + previous, self.current = self.current, span + try: + yield span + finally: + self.current = previous + span.end() + + def finish(self) -> list[ReadableSpan]: + self.span.end() + return list(self.exporter.get_finished_spans()) + + +@pytest.fixture +def graph(exporter: InMemorySpanExporter) -> _Graph: + fake = _Graph(exporter) + enable_guardrail_grouping(exporter.provider, lambda: fake.current) # type: ignore[attr-defined] + return fake + + +def _hook(middleware: Any, name: str) -> Any: + [hook] = [h for h in middleware if h.name.endswith(name)] + return hook + + +def _named(spans: list[ReadableSpan], name: str) -> list[ReadableSpan]: + return [s for s in spans if s.name == name] + + +def _children(spans: list[ReadableSpan], parent: ReadableSpan) -> list[str]: + return [ + s.name + for s in spans + if s.parent is not None and s.parent.span_id == parent.context.span_id + ] + + +def _pii(**kwargs: Any) -> UiPathPIIDetectionMiddleware: + return UiPathPIIDetectionMiddleware( + scopes=kwargs.pop("scopes", [GuardrailScope.LLM]), + action=kwargs.pop("action", LogAction()), + entities=[PIIDetectionEntity(PIIDetectionEntityType.EMAIL)], + **kwargs, + ) + + +def _rule(stage: GuardrailExecutionStage) -> UiPathDeterministicGuardrailMiddleware: + return UiPathDeterministicGuardrailMiddleware( + tools=["get_weather"], + rules=[lambda *args: True], + action=LogAction(), + stage=stage, + name="Rule", + ) + + +def _tool_request() -> ToolCallRequest: + return ToolCallRequest( + tool_call={"id": "tc1", "name": "get_weather", "args": {"city": "London"}}, + tool=MagicMock(), + state={}, + runtime=MagicMock(), + ) + + +class TestStages: + async def test_guardrail_nodes_of_one_stage_share_a_container( + self, graph: _Graph + ) -> None: + pii = _pii() + injection = UiPathPromptInjectionMiddleware(action=LogAction()) + pii_hook = _hook(pii, "_before_model") + injection_hook = _hook(injection, "_before_model") + + with ( + patch.object(pii, "_evaluate_guardrail", return_value=_FAILED), + patch.object(injection, "_evaluate_guardrail", return_value=_FAILED), + ): + with graph.node(f"{pii_hook.name}.before_model"): + await pii_hook.abefore_model(_STATE, MagicMock()) + with graph.node(f"{injection_hook.name}.before_model"): + await injection_hook.abefore_model(_STATE, MagicMock()) + with graph.node("model") as model: + model_start = model.start_time # type: ignore[attr-defined] + spans = graph.finish() + + [container] = _named(spans, "LLM input guardrail check") + assert (container.attributes or {})["span_type"] == "llmPreGuardrails" + assert container.parent is not None + assert container.parent.span_id == graph.span.get_span_context().span_id + assert _children(spans, container) == ["Log", "Log"] + assert container.end_time == model_start + assert container.status.status_code == StatusCode.OK + + async def test_a_container_never_ends_before_its_last_check( + self, graph: _Graph + ) -> None: + pii = _pii() + hook = _hook(pii, "_before_model") + + with ( + patch.object(pii, "_evaluate_guardrail", return_value=_FAILED), + graph.node(f"{hook.name}.before_model"), + ): + await hook.abefore_model(_STATE, MagicMock()) + # A coarse clock stamps the next node's start before the check ended. + with graph.node("model", start_time=time.time_ns() - 10_000_000): + pass + spans = graph.finish() + + [container] = _named(spans, "LLM input guardrail check") + [log] = _named(spans, "Log") + assert container.end_time is not None + assert log.end_time is not None + assert container.end_time >= log.end_time + + async def test_the_container_carries_the_execution_id(self, graph: _Graph) -> None: + pii = _pii() + hook = _hook(pii, "_before_model") + + with ( + patch.object(pii, "_evaluate_guardrail", return_value=_FAILED), + graph.node( + f"{hook.name}.before_model", attributes={"execution.id": "exec-1"} + ), + ): + await hook.abefore_model(_STATE, MagicMock()) + spans = graph.finish() + + [container] = _named(spans, "LLM input guardrail check") + assert (container.attributes or {})["execution.id"] == "exec-1" + + async def test_a_new_stage_gets_its_own_container(self, graph: _Graph) -> None: + pii = _pii(scopes=[GuardrailScope.AGENT, GuardrailScope.LLM]) + agent_hook = _hook(pii, "_before_agent") + model_hook = _hook(pii, "_before_model") + + with patch.object(pii, "_evaluate_guardrail", return_value=_FAILED): + with graph.node(f"{agent_hook.name}.before_agent"): + await agent_hook.abefore_agent(_STATE, MagicMock()) + with graph.node(f"{model_hook.name}.before_model"): + await model_hook.abefore_model(_STATE, MagicMock()) + spans = graph.finish() + + assert len(_named(spans, "Agent input guardrail check")) == 1 + assert len(_named(spans, "LLM input guardrail check")) == 1 + + async def test_another_node_between_guardrails_splits_the_stage( + self, graph: _Graph + ) -> None: + pii = _pii() + injection = UiPathPromptInjectionMiddleware(action=LogAction()) + pii_hook = _hook(pii, "_before_model") + injection_hook = _hook(injection, "_before_model") + + with ( + patch.object(pii, "_evaluate_guardrail", return_value=_FAILED), + patch.object(injection, "_evaluate_guardrail", return_value=_FAILED), + ): + with graph.node(f"{pii_hook.name}.before_model"): + await pii_hook.abefore_model(_STATE, MagicMock()) + with graph.node("SummarizationMiddleware.before_model"): + pass + with graph.node(f"{injection_hook.name}.before_model"): + await injection_hook.abefore_model(_STATE, MagicMock()) + spans = graph.finish() + + assert len(_named(spans, "LLM input guardrail check")) == 2 + + async def test_llm_output_checks_get_their_container(self, graph: _Graph) -> None: + pii = _pii() + hook = _hook(pii, "_after_model") + state = {"messages": [HumanMessage("hi"), AIMessage("mail a@b.com")]} + + with ( + patch.object(pii, "_evaluate_guardrail", return_value=_FAILED), + graph.node(f"{hook.name}.after_model"), + ): + await hook.aafter_model(state, MagicMock()) + spans = graph.finish() + + [container] = _named(spans, "LLM output guardrail check") + assert _children(spans, container) == ["Log"] + + async def test_a_block_marks_the_container_as_failed(self, graph: _Graph) -> None: + pii = _pii(action=BlockAction()) + hook = _hook(pii, "_before_model") + + with ( + patch.object(pii, "_evaluate_guardrail", return_value=_FAILED), + pytest.raises(AgentRuntimeError), + graph.node(f"{hook.name}.before_model"), + ): + await hook.abefore_model(_STATE, MagicMock()) + spans = graph.finish() + + [container] = _named(spans, "LLM input guardrail check") + assert container.status.status_code == StatusCode.ERROR + + async def test_a_grouping_failure_still_runs_the_check(self, graph: _Graph) -> None: + pii = _pii(action=BlockAction()) + hook = _hook(pii, "_before_model") + + with ( + patch.object(pii, "_evaluate_guardrail", return_value=_FAILED), + patch.object(_grouping, "_shared_container", side_effect=RuntimeError), + pytest.raises(AgentRuntimeError), + graph.node(f"{hook.name}.before_model"), + ): + await hook.abefore_model(_STATE, MagicMock()) + spans = graph.finish() + + assert _named(spans, "LLM input guardrail check") == [] + assert len(_named(spans, "Block")) == 1 + + async def test_a_stage_with_nothing_to_check_leaves_no_container( + self, graph: _Graph + ) -> None: + pii = _pii() + hook = _hook(pii, "_before_model") + + with ( + patch.object(pii, "_evaluate_guardrail") as evaluate, + graph.node(f"{hook.name}.before_model"), + ): + await hook.abefore_model({"messages": []}, MagicMock()) + spans = graph.finish() + + evaluate.assert_not_called() + assert not _named(spans, "LLM input guardrail check") + + +class TestTools: + async def _call_tool( + self, graph: _Graph, stage: GuardrailExecutionStage + ) -> list[ReadableSpan]: + [wrapper] = list(_rule(stage)) + with graph.node("tools") as tools: + + async def run_tool(request: ToolCallRequest) -> ToolMessage: + with graph.node("get_weather", parent=tools): + return ToolMessage(content="sunny", tool_call_id="tc1") + + await wrapper.awrap_tool_call(_tool_request(), run_tool) + return graph.finish() + + async def test_input_checks_close_when_the_tool_starts(self, graph: _Graph) -> None: + spans = await self._call_tool(graph, GuardrailExecutionStage.PRE) + + [container] = _named(spans, "Tool input guardrail check") + [tools] = _named(spans, "tools") + [tool_span] = _named(spans, "get_weather") + assert container.parent is not None + assert container.parent.span_id == tools.context.span_id + assert _children(spans, container) == ["Rule", "Log"] + assert container.end_time == tool_span.start_time + + async def test_output_checks_close_when_the_tools_node_ends( + self, graph: _Graph + ) -> None: + spans = await self._call_tool(graph, GuardrailExecutionStage.POST) + + [container] = _named(spans, "Tool output guardrail check") + [tools] = _named(spans, "tools") + [tool_span] = _named(spans, "get_weather") + assert container.start_time is not None + assert tool_span.end_time is not None + assert container.start_time >= tool_span.end_time + assert container.end_time == tools.end_time + + def test_a_container_closed_during_a_check_stays_closed( + self, graph: _Graph + ) -> None: + _pii() + with graph.node("tools") as tools: + with guardrail_stage(GuardrailScope.TOOL, GuardrailExecutionStage.PRE): + with graph.node("get_weather", parent=tools): # closes it + pass + spans = graph.finish() + + [container] = _named(spans, "Tool input guardrail check") + [tool_span] = _named(spans, "get_weather") + assert container.end_time == tool_span.start_time + assert _grouping._state.containers == {} + + async def test_input_checks_close_when_the_output_stage_starts( + self, graph: _Graph + ) -> None: + [pre] = list(_rule(GuardrailExecutionStage.PRE)) + [post] = list(_rule(GuardrailExecutionStage.POST)) + with graph.node("tools"): + + async def answer_without_the_tool(request: ToolCallRequest) -> ToolMessage: + return ToolMessage(content="cached", tool_call_id="tc1") + + async def inner(request: ToolCallRequest) -> ToolMessage: + return await post.awrap_tool_call(request, answer_without_the_tool) + + await pre.awrap_tool_call(_tool_request(), inner) + spans = graph.finish() + + [pre_container] = _named(spans, "Tool input guardrail check") + [post_container] = _named(spans, "Tool output guardrail check") + assert pre_container.end_time is not None + assert post_container.start_time is not None + assert pre_container.end_time <= post_container.start_time + + async def test_a_failed_new_stage_keeps_the_previous_container_closable( + self, graph: _Graph + ) -> None: + _pii() # a guardrail in use turns grouping on + with graph.node("tools") as tools: + graph.current = tools + with guardrail_stage(GuardrailScope.TOOL, GuardrailExecutionStage.PRE): + pass + with ( + patch.object(_grouping, "new_span", side_effect=RuntimeError), + guardrail_stage(GuardrailScope.TOOL, GuardrailExecutionStage.POST), + ): + pass + spans = graph.finish() + + [container] = _named(spans, "Tool input guardrail check") + assert container.end_time == _named(spans, "tools")[0].end_time + assert _grouping._state.containers == {} + + +class TestEnabling: + async def test_nothing_is_grouped_without_a_closer( + self, exporter: InMemorySpanExporter + ) -> None: + node = exporter.provider.get_tracer(_OPENINFERENCE).start_span("node") # type: ignore[attr-defined] + enable_guardrail_grouping(None, lambda: node) + + with guardrail_stage(GuardrailScope.LLM, GuardrailExecutionStage.PRE): + pass + node.end() + + assert _grouping._state.containers == {} + assert not _grouping._state.enabled + + async def test_nothing_is_grouped_under_an_invalid_node_span( + self, exporter: InMemorySpanExporter + ) -> None: + enable_guardrail_grouping(exporter.provider, lambda: trace.INVALID_SPAN) # type: ignore[attr-defined] + _pii() # a guardrail in use turns grouping on + + with guardrail_stage(GuardrailScope.TOOL, GuardrailExecutionStage.PRE): + pass + + assert _grouping._state.enabled + assert _grouping._state.containers == {} + + def test_the_closer_is_added_once_per_trace_manager( + self, exporter: InMemorySpanExporter + ) -> None: + manager = MagicMock() + enable_guardrail_grouping(manager, lambda: None) + _pii() + enable_guardrail_grouping(manager, lambda: None) + _pii() + + manager.add_span_processor.assert_called_once() + + def test_the_closer_comes_back_after_its_trace_manager_shuts_down( + self, exporter: InMemorySpanExporter + ) -> None: + manager = MagicMock() + enable_guardrail_grouping(manager, lambda: None) + _pii() + [closer] = [call.args[0] for call in manager.add_span_processor.call_args_list] + + closer.shutdown() # what UiPathTraceManager.shutdown() does + assert not _grouping._state.enabled + enable_guardrail_grouping(manager, lambda: None) + + assert manager.add_span_processor.call_count == 2 + assert _grouping._state.enabled + + def test_a_shut_down_trace_manager_is_not_reused( + self, exporter: InMemorySpanExporter + ) -> None: + manager = MagicMock() + enable_guardrail_grouping(manager, lambda: None) + _pii() + [closer] = [call.args[0] for call in manager.add_span_processor.call_args_list] + + closer.shutdown() + _pii() # a middleware built before the next runtime factory + + manager.add_span_processor.assert_called_once() + assert not _grouping._state.enabled + + async def test_nothing_is_grouped_under_a_sampled_out_node( + self, exporter: InMemorySpanExporter + ) -> None: + node = NonRecordingSpan( + SpanContext( + trace_id=1, span_id=2, is_remote=False, trace_flags=TraceFlags(0) + ) + ) + enable_guardrail_grouping(exporter.provider, lambda: node) # type: ignore[attr-defined] + _pii() + + with guardrail_stage(GuardrailScope.TOOL, GuardrailExecutionStage.PRE): + pass + + assert _grouping._state.enabled + assert _grouping._state.containers == {} + + def test_nothing_is_grouped_under_a_sampled_out_parent( + self, exporter: InMemorySpanExporter + ) -> None: + # A sampler that keeps the node although its parent was dropped. + provider = TracerProvider(sampler=ALWAYS_ON) + parent = NonRecordingSpan( + SpanContext( + trace_id=1, span_id=2, is_remote=False, trace_flags=TraceFlags(0) + ) + ) + node = provider.get_tracer(_OPENINFERENCE).start_span( + "pii_detection_before_model", context=trace.set_span_in_context(parent) + ) + enable_guardrail_grouping(exporter.provider, lambda: node) # type: ignore[attr-defined] + _pii() + + with guardrail_stage(GuardrailScope.LLM, GuardrailExecutionStage.PRE): + pass + + assert node.is_recording() + assert _grouping._state.containers == {} + node.end() + provider.shutdown() + + def test_the_closer_skips_the_lock_with_nothing_open( + self, graph: _Graph, monkeypatch: pytest.MonkeyPatch + ) -> None: + lock = MagicMock() + monkeypatch.setattr(_grouping._state, "lock", lock) + closer = _grouping.GuardrailStageCloser() + + with graph.node("model") as span: + closer.on_start(span) # type: ignore[arg-type] + closer.on_end(span) # type: ignore[arg-type] + + lock.__enter__.assert_not_called() + + def test_starting_a_container_does_not_deadlock( + self, exporter: InMemorySpanExporter, monkeypatch: pytest.MonkeyPatch + ) -> None: + # A container starts under the lock, and with another container open + # the closer's on_start takes it again. Its own provider, so a deadlock + # fails the test instead of hanging the fixture's shutdown. + provider = TracerProvider() + monkeypatch.setattr(trace, "get_tracer", provider.get_tracer) + tracer = provider.get_tracer(_OPENINFERENCE) + other_tools, tools = tracer.start_span("tools"), tracer.start_span("tools") + current = [other_tools] + enable_guardrail_grouping(provider, lambda: current[0]) + _pii() + with guardrail_stage(GuardrailScope.TOOL, GuardrailExecutionStage.PRE): + pass # stays open under other_tools + current[0] = tools + + def check() -> None: + with guardrail_stage(GuardrailScope.TOOL, GuardrailExecutionStage.PRE): + pass + + worker = threading.Thread(target=check, daemon=True) + worker.start() + worker.join(timeout=5) + + assert not worker.is_alive() + assert len(_grouping._state.containers) == 2 + tools.end() + other_tools.end() + provider.shutdown() + + def test_a_new_stage_ends_the_previous_one_without_the_closer( + self, graph: _Graph, monkeypatch: pytest.MonkeyPatch + ) -> None: + # Containers from a provider the closer isn't attached to, so it never + # sees their start (as when another manager's shutdown detached it). + hidden = TracerProvider() + finished = InMemorySpanExporter() + hidden.add_span_processor(SimpleSpanProcessor(finished)) + monkeypatch.setattr(trace, "get_tracer", hidden.get_tracer) + _pii() + + with graph.node("tools"): + with guardrail_stage(GuardrailScope.TOOL, GuardrailExecutionStage.PRE): + pass + with guardrail_stage(GuardrailScope.TOOL, GuardrailExecutionStage.POST): + pass + + ended = [s.name for s in finished.get_finished_spans()] + assert ended == ["Tool input guardrail check"] + graph.finish() + hidden.shutdown() + + def test_tool_only_guardrails_turn_grouping_on( + self, exporter: InMemorySpanExporter + ) -> None: + manager = MagicMock() + enable_guardrail_grouping(manager, lambda: None) + _rule(GuardrailExecutionStage.PRE) + + manager.add_span_processor.assert_called_once() + assert _grouping._state.enabled + + def test_grouping_stays_off_without_guardrail_middlewares( + self, exporter: InMemorySpanExporter + ) -> None: + manager = MagicMock() + enable_guardrail_grouping(manager, lambda: None) + + manager.add_span_processor.assert_not_called() + assert not _grouping._state.enabled + + async def test_only_guardrail_node_spans_are_hidden(self, graph: _Graph) -> None: + pii = _pii() + hook = _hook(pii, "_before_model") + node_name = f"{hook.name}.before_model" + user_span = trace.get_tracer("my_app").start_span(node_name) + user_span.end() + + with ( + patch.object(pii, "_evaluate_guardrail", return_value=_FAILED), + graph.node(node_name), + ): + await hook.abefore_model(_STATE, MagicMock()) + spans = graph.finish() + + hidden = [s for s in spans if not is_exported(s)] + assert [s.instrumentation_scope.name for s in hidden] == [_OPENINFERENCE] # type: ignore[union-attr] + assert is_exported(_named(spans, "LLM input guardrail check")[0]) + + def test_a_node_that_cannot_be_grouped_stays_visible(self, graph: _Graph) -> None: + hook = _hook(_pii(), "_before_model") + node_name = f"{hook.name}.before_model" + # A sampler that keeps the nodes although one's parent was dropped. + provider = TracerProvider(sampler=ALWAYS_ON) + tracer = provider.get_tracer(_OPENINFERENCE) + unsampled = NonRecordingSpan( + SpanContext( + trace_id=1, span_id=2, is_remote=False, trace_flags=TraceFlags(0) + ) + ) + under_unsampled = tracer.start_span( + node_name, context=trace.set_span_in_context(unsampled) + ) + without_parent = tracer.start_span(node_name) + + assert is_exported(under_unsampled) # type: ignore[arg-type] + assert is_exported(without_parent) # type: ignore[arg-type] + under_unsampled.end() + without_parent.end() + provider.shutdown() + + +class _ToolCallingModel(GenericFakeChatModel): + def bind_tools(self, *args: Any, **kwargs: Any) -> "_ToolCallingModel": + return self + + +@tool +def get_weather(city: str) -> str: + """Weather for a city.""" + return "sunny" + + +class TestRealAgent: + """A real create_agent graph traced by OpenInference.""" + + @pytest.fixture + def instrumented(self, exporter: InMemorySpanExporter) -> Iterator[None]: + provider = exporter.provider # type: ignore[attr-defined] + instrumentor = LangChainInstrumentor() + # Instrumentation is global: re-instrument on this provider, then restore. + was_instrumented = instrumentor.is_instrumented_by_opentelemetry + if was_instrumented: + instrumentor.uninstrument() + instrumentor.instrument(tracer_provider=provider) + enable_guardrail_grouping(provider, get_current_span) + UiPathSpanUtils.register_current_span_ancestors_provider(get_ancestor_spans) + yield + instrumentor.uninstrument() + if was_instrumented: + instrumentor.instrument() + + def _agent(self, parallel_calls: int) -> Any: + calls = [ + {"id": f"t{i}", "name": "get_weather", "args": {"city": "London"}} + for i in range(parallel_calls) + ] + model = _ToolCallingModel( + messages=iter([AIMessage(content="", tool_calls=calls), AIMessage("ok")]) + ) + guardrails = [ + *_pii( + scopes=[GuardrailScope.LLM, GuardrailScope.TOOL], tools=[get_weather] + ), + *UiPathPromptInjectionMiddleware(action=LogAction()), + *_rule(GuardrailExecutionStage.PRE_AND_POST), + ] + return create_agent(model, tools=[get_weather], middleware=guardrails) + + @staticmethod + def _assert_well_formed(spans: list[ReadableSpan]) -> None: + by_id = {s.context.span_id: s for s in spans} + exported = [s for s in spans if is_exported(s)] + orphans = [ + s.name + for s in exported + if s.parent + and s.parent.span_id in by_id + and not is_exported(by_id[s.parent.span_id]) + ] + assert orphans == [] + assert _grouping._state.containers == {} + + async def test_parallel_tool_calls_group_per_tools_span( + self, exporter: InMemorySpanExporter, instrumented: None + ) -> None: + with patch.object( + _base.BuiltInGuardrailMiddlewareMixin, + "_evaluate_guardrail", + return_value=_FAILED, + ): + await self._agent(parallel_calls=2).ainvoke( + {"messages": [HumanMessage("a@b.com weather")]} + ) + spans = list(exporter.get_finished_spans()) + + self._assert_well_formed(spans) + assert len(_named(spans, "LLM input guardrail check")) == 2 + assert len(_named(spans, "LLM output guardrail check")) == 1 + assert len(_named(spans, "Tool input guardrail check")) == 2 + assert len(_named(spans, "Tool output guardrail check")) == 2 + for container in _named(spans, "LLM input guardrail check"): + assert _children(spans, container) == ["Log", "Log"] + + async def test_overlapping_checks_on_parallel_tool_calls_stay_in_their_container( + self, exporter: InMemorySpanExporter, instrumented: None + ) -> None: + def slow_check(*args: Any, **kwargs: Any) -> GuardrailValidationResult: + time.sleep(0.05) # long enough for the calls' checks to overlap + return _FAILED + + with patch.object( + _base.BuiltInGuardrailMiddlewareMixin, "_evaluate_guardrail", slow_check + ): + await self._agent(parallel_calls=3).ainvoke( + {"messages": [HumanMessage("a@b.com weather")]} + ) + spans = list(exporter.get_finished_spans()) + + # Each tool call runs in its own tools span, so each has its own container, + # and every check ends inside it. + containers = _named(spans, "Tool input guardrail check") + assert len(containers) == 3 + assert len({c.parent.span_id for c in containers if c.parent}) == 3 + for container in containers: + children = [ + s + for s in spans + if s.parent and s.parent.span_id == container.context.span_id + ] + assert children + assert all( + c.end_time is not None + and container.end_time is not None + and c.end_time <= container.end_time + for c in children + ) + + async def test_concurrent_runs_keep_their_own_containers( + self, exporter: InMemorySpanExporter, instrumented: None + ) -> None: + with patch.object( + _base.BuiltInGuardrailMiddlewareMixin, + "_evaluate_guardrail", + return_value=_FAILED, + ): + await asyncio.gather( + self._agent(parallel_calls=1).ainvoke( + {"messages": [HumanMessage("a@b.com")]} + ), + self._agent(parallel_calls=1).ainvoke( + {"messages": [HumanMessage("a@b.com")]} + ), + ) + spans = list(exporter.get_finished_spans()) + + self._assert_well_formed(spans) + containers = _named(spans, "LLM input guardrail check") + assert len(containers) == 4 + assert all(_children(spans, c) for c in containers) diff --git a/uv.lock b/uv.lock index 4267b7d4e..960bd9fe6 100644 --- a/uv.lock +++ b/uv.lock @@ -5067,7 +5067,7 @@ wheels = [ [[package]] name = "uipath-langchain" -version = "0.20.7" +version = "0.20.8" source = { editable = "." } dependencies = [ { name = "a2a-sdk" },