From 0a297df8530dc1d80a3f6aed29f03490ae8ca250 Mon Sep 17 00:00:00 2001 From: Daniel Joica Date: Wed, 18 Nov 2026 11:57:23 +0200 Subject: [PATCH 1/3] feat(guardrails): group coded-agent guardrail spans like low-code agents Each guardrail middleware hook is its own LangGraph node, so a coded agent's trace showed one node per guardrail per hook, with no grouping, no action spans and nothing for deterministic rules. - Stage containers ("LLM input guardrail check", ...) shared across the separate guardrail nodes: the first check of a stage opens one, later guardrail nodes of that stage reuse it, the next unrelated node closes it. The empty guardrail node spans are hidden from the export. - A guardrailAction span (Block, Log, ...) per action, linked to its evaluation. - A client guardrailEvaluation span for deterministic rules. The API is unchanged. Grouping only turns on when a guardrail middleware is in use and a trace manager is available. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 2 +- .../guardrails/middlewares/_base.py | 101 ++-- .../guardrails/middlewares/_spans.py | 322 +++++++++++ .../guardrails/middlewares/deterministic.py | 57 +- src/uipath_langchain/runtime/factory.py | 9 +- .../middlewares/test_action_span.py | 163 ++++++ tests/guardrails/middlewares/test_grouping.py | 517 ++++++++++++++++++ uv.lock | 2 +- 8 files changed, 1119 insertions(+), 54 deletions(-) create mode 100644 src/uipath_langchain/guardrails/middlewares/_spans.py create mode 100644 tests/guardrails/middlewares/test_action_span.py create mode 100644 tests/guardrails/middlewares/test_grouping.py diff --git a/pyproject.toml b/pyproject.toml index 61a77791e..86d210195 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath-langchain" -version = "0.20.2" +version = "0.20.3" description = "Python SDK that enables developers to build and deploy LangGraph agents to the UiPath Cloud Platform" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/src/uipath_langchain/guardrails/middlewares/_base.py b/src/uipath_langchain/guardrails/middlewares/_base.py index 8ed5640c2..b65c970fb 100644 --- a/src/uipath_langchain/guardrails/middlewares/_base.py +++ b/src/uipath_langchain/guardrails/middlewares/_base.py @@ -55,6 +55,12 @@ registry_references, tool_result_references, ) +from ._spans import ( + guardrail_action_span, + guardrail_stage, + guardrails_in_use, + register_guardrail_node, +) from ._utils import ( convert_block_exception, create_modified_tool_request, @@ -362,7 +368,10 @@ def _handle_validation_result( ) ) try: - return self.action.handle_validation_result(result, input_data, self._name) + with guardrail_action_span(self._name, self.action, result): + return self.action.handle_validation_result( + result, input_data, self._name + ) finally: _action_context.reset(token) @@ -412,16 +421,17 @@ async def _run_tool_guardrail( tool_args=input_data, ) try: - result = await asyncio.to_thread( - self._evaluate_guardrail, input_data, attachments - ) - modified_input = self._handle_validation_result( - result, - input_data, - scope=GuardrailScope.TOOL, - stage=GuardrailExecutionStage.PRE, - component=tool_name, - ) + with guardrail_stage(GuardrailScope.TOOL, GuardrailExecutionStage.PRE): + result = await asyncio.to_thread( + self._evaluate_guardrail, input_data, attachments + ) + modified_input = self._handle_validation_result( + result, + input_data, + scope=GuardrailScope.TOOL, + stage=GuardrailExecutionStage.PRE, + component=tool_name, + ) if modified_input is not None: request = create_modified_tool_request(request, modified_input) except GuardrailBlockException as exc: @@ -453,19 +463,22 @@ async def _run_tool_guardrail( tool_result=output_data, ) try: - result = await asyncio.to_thread( - self._evaluate_guardrail, output_data, attachments - ) - modified_output = self._handle_validation_result( - result, - output_data, - scope=GuardrailScope.TOOL, - stage=GuardrailExecutionStage.POST, - component=tool_name, - input_payload=json.dumps( - self._extract_tool_input_data(request) - ), - ) + with guardrail_stage( + GuardrailScope.TOOL, GuardrailExecutionStage.POST + ): + result = await asyncio.to_thread( + self._evaluate_guardrail, output_data, attachments + ) + modified_output = self._handle_validation_result( + result, + output_data, + scope=GuardrailScope.TOOL, + stage=GuardrailExecutionStage.POST, + component=tool_name, + input_payload=json.dumps( + self._extract_tool_input_data(request) + ), + ) if modified_output is not None: tool_result = create_modified_tool_result( tool_result, modified_output @@ -488,6 +501,7 @@ async def _run_tool_guardrail( def _create_tool_wrap_hook(self, guardrail_name: str) -> AgentMiddleware: """Create a wrap_tool_call hook that delegates to _run_tool_guardrail.""" + guardrails_in_use() middleware_instance = self async def _wrap_tool_call_func( @@ -543,6 +557,11 @@ async def _before_agent_func( ) _before_agent_func.__name__ = f"{guardrail_name}_before_agent" + register_guardrail_node( + f"{_before_agent_func.__name__}.before_agent", + GuardrailScope.AGENT, + GuardrailExecutionStage.PRE, + ) hooks.append(before_agent(_before_agent_func)) if include_post: @@ -571,6 +590,11 @@ async def _after_agent_func( ) _after_agent_func.__name__ = f"{guardrail_name}_after_agent" + register_guardrail_node( + f"{_after_agent_func.__name__}.after_agent", + GuardrailScope.AGENT, + GuardrailExecutionStage.POST, + ) hooks.append(after_agent(_after_agent_func)) elif scope == GuardrailScope.LLM: @@ -592,6 +616,11 @@ async def _before_model_func( ) _before_model_func.__name__ = f"{guardrail_name}_before_model" + register_guardrail_node( + f"{_before_model_func.__name__}.before_model", + GuardrailScope.LLM, + GuardrailExecutionStage.PRE, + ) hooks.append(before_model(_before_model_func)) if include_post: @@ -615,6 +644,11 @@ async def _after_model_func( ) _after_model_func.__name__ = f"{guardrail_name}_after_model" + register_guardrail_node( + f"{_after_model_func.__name__}.after_model", + GuardrailScope.LLM, + GuardrailExecutionStage.POST, + ) hooks.append(after_model(_after_model_func)) return hooks @@ -652,15 +686,16 @@ def _check_messages( payload = text or NO_TEXT_PLACEHOLDER try: - result = self._evaluate_guardrail(payload, attachments) - modified_text = self._handle_validation_result( - result, - payload, - scope=scope, - stage=stage, - component=component_label(scope), - input_payload=json.dumps(input_text) if input_text else None, - ) + with guardrail_stage(scope, stage): + result = self._evaluate_guardrail(payload, attachments) + modified_text = self._handle_validation_result( + result, + payload, + scope=scope, + stage=stage, + component=component_label(scope), + input_payload=json.dumps(input_text) if input_text else None, + ) if text: _apply_text_modification(messages, text, modified_text) except GuardrailBlockException as exc: diff --git a/src/uipath_langchain/guardrails/middlewares/_spans.py b/src/uipath_langchain/guardrails/middlewares/_spans.py new file mode 100644 index 000000000..e419ec2fe --- /dev/null +++ b/src/uipath_langchain/guardrails/middlewares/_spans.py @@ -0,0 +1,322 @@ +"""Guardrail spans shaped like low-code agents': actions, rule evaluations and +stage containers that group a stage's checks across separate middleware nodes.""" + +import threading +import weakref +from collections.abc import Callable, Generator +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Any + +from langgraph.errors import GraphBubbleUp +from opentelemetry import context as otel_context +from opentelemetry import trace +from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor +from opentelemetry.trace import NonRecordingSpan, Span, SpanContext, StatusCode +from uipath.core.guardrails import GuardrailScope, GuardrailValidationResult +from uipath.core.tracing.span_utils import UiPathSpanUtils +from uipath.platform.guardrails.decorators import ( + GuardrailExecutionStage, + LoggingSeverityLevel, +) +from uipath.platform.guardrails.decorators._exceptions import GuardrailBlockException + +GUARDRAIL_ACTION = "guardrailAction" +GUARDRAIL_EVALUATION = "guardrailEvaluation" +_CONTAINER_MARK = "uipath.guardrail_container" + + +def _new_span( + name: str, + span_type: str, + context: otel_context.Context | None = None, + **attributes: Any, +) -> Span: + """Start a span under the deepest active span, as ``@traced`` does.""" + return trace.get_tracer(__name__).start_span( + name, + context=context or UiPathSpanUtils.get_parent_context(), + attributes={ + "type": span_type, + "span_type": span_type, + "uipath.custom_instrumentation": True, + **{key: value for key, value in attributes.items() if value is not None}, + }, + ) + + +@contextmanager +def _active(span: Span) -> Generator[Span]: + """Make ``span`` current and end it; errors (not interrupts) mark it failed.""" + with trace.use_span( + span, end_on_exit=True, record_exception=False, set_status_on_exception=False + ): + try: + yield span + except GraphBubbleUp: + raise + except Exception as exc: + span.set_status(StatusCode.ERROR, _error_text(exc)) + raise + + +def _error_text(exc: BaseException) -> str: + if isinstance(exc, GuardrailBlockException): + return f"Guardrail violation: {exc.detail}" + return str(exc) + + +def span_guid(span: Span) -> str: + """Span id in the GUID form LLMOps uses.""" + span_id = format(span.get_span_context().span_id, "016x") + return f"00000000-0000-0000-{span_id[:4]}-{span_id[4:]}" + + +def action_name(action: Any) -> str: + """``BlockAction`` -> ``Block``.""" + name = type(action).__name__ + return name.removesuffix("Action") or name + + +@contextmanager +def guardrail_action_span( + guardrail_name: str, action: Any, result: GuardrailValidationResult +) -> Generator[Span]: + """Record the action taken on a failed ``result``.""" + severity = getattr(action, "severity_level", None) + span = _new_span( + action_name(action), + GUARDRAIL_ACTION, + guardrailName=guardrail_name, + action=action_name(action), + guardrailSpanId=result.span_id, + reason=result.reason or None, + severityLevel=( + severity.name.title() + if isinstance(severity, LoggingSeverityLevel) + else None + ), + ) + with _active(span): + yield span + + +@contextmanager +def guardrail_evaluation_span( + guardrail_name: str, description: str | None = None +) -> Generator[Span]: + """Record a client-side check; set its outcome with ``set_validation_result``.""" + span = _new_span( + guardrail_name, + GUARDRAIL_EVALUATION, + guardrailName=guardrail_name, + guardrailDescription=description, + guardrailType="custom", + ) + with _active(span): + yield span + + +def set_validation_result(span: Span, result: GuardrailValidationResult) -> None: + """Record ``result`` the way the Guardrails Service does.""" + span.set_attribute("validationResult", result.result.name) + if result.reason: + span.set_attribute("details", [result.reason]) + + +_PRE = GuardrailExecutionStage.PRE +_POST = GuardrailExecutionStage.POST + +# (scope, stage) -> (container name, span type), as in low-code agents. +_STAGES: dict[tuple[GuardrailScope, GuardrailExecutionStage], tuple[str, str]] = { + (GuardrailScope.AGENT, _PRE): ("Agent input guardrail check", "agentPreGuardrails"), + (GuardrailScope.AGENT, _POST): ( + "Agent output guardrail check", + "agentPostGuardrails", + ), + (GuardrailScope.LLM, _PRE): ("LLM input guardrail check", "llmPreGuardrails"), + (GuardrailScope.LLM, _POST): ("LLM output guardrail check", "llmPostGuardrails"), + (GuardrailScope.TOOL, _PRE): ("Tool input guardrail check", "toolPreGuardrails"), + (GuardrailScope.TOOL, _POST): ("Tool output guardrail check", "toolPostGuardrails"), +} + + +class _GroupingState: + """Process-wide state. Grouping stays off until a closer is registered.""" + + def __init__(self) -> None: + self.enabled = False + self.in_use = False # a guardrail middleware was built + self.trace_manager: Any = None # offered by the runtime factory + self.closers: weakref.WeakKeyDictionary[Any, GuardrailStageCloser] = ( + weakref.WeakKeyDictionary() + ) # trace manager -> its closer + self.guardrail_nodes: dict[str, str] = {} # node name -> stage span type + self.containers: dict[tuple[int, str], Span] = {} # (parent id, type) + self.lock = threading.Lock() + + +_state = _GroupingState() + +_current_container: ContextVar[Span | None] = ContextVar( + "uipath_guardrail_container", default=None +) + + +def register_guardrail_node( + node_name: str, scope: GuardrailScope, stage: GuardrailExecutionStage +) -> None: + """Record which stage a guardrail hook's LangGraph node belongs to.""" + _state.guardrail_nodes[node_name] = _STAGES[(scope, stage)][1] + guardrails_in_use() + + +def guardrails_in_use() -> None: + """Called by guardrail middlewares when built; grouping needs one in use.""" + _state.in_use = True + _activate() + + +@contextmanager +def guardrail_stage( + scope: GuardrailScope | None, stage: GuardrailExecutionStage | None +) -> Generator[None]: + """Run one check inside its stage's shared container.""" + container = _shared_container(scope, stage) + if container is None: + yield + return + token = _current_container.set(container) + try: + yield + except GraphBubbleUp: + raise + except Exception as exc: + container.set_status(StatusCode.ERROR, _error_text(exc)) + raise + finally: + _current_container.reset(token) + + +def _shared_container( + scope: GuardrailScope | None, stage: GuardrailExecutionStage | None +) -> Span | None: + """Get or create the stage container: beside message nodes, inside ``tools``.""" + if not _state.enabled or (scope, stage) not in _STAGES: + return None + node = UiPathSpanUtils.get_external_current_span() + if node is None: + return None + if scope == GuardrailScope.TOOL: + parent: SpanContext | None = node.get_span_context() + else: + parent = getattr(node, "parent", None) + if parent is None or not parent.is_valid: + return None # nothing would ever close it + + name, span_type = _STAGES[(scope, stage)] # type: ignore[index] + key = (parent.span_id, span_type) + with _state.lock: + container = _state.containers.get(key) + if container is None: + container = _new_span( + name, + span_type, + context=trace.set_span_in_context(NonRecordingSpan(parent)), + **{ + _CONTAINER_MARK: True, + # Its parent is non-recording, so this isn't inherited. + "execution.id": (getattr(node, "attributes", None) or {}).get( + "execution.id" + ), + }, + ) + _state.containers[key] = container + return container + + +def guardrail_aware( + provider: Callable[[], Span | None], +) -> Callable[[], Span | None]: + """Report the running check's container as the current span.""" + + def current_span() -> Span | None: + return _current_container.get() or provider() + + return current_span + + +class GuardrailStageCloser(SpanProcessor): + """End a container when a non-guardrail sibling starts or its parent ends.""" + + def on_start( + self, span: Span, parent_context: otel_context.Context | None = None + ) -> None: + attributes = getattr(span, "attributes", None) or {} + parent = getattr(span, "parent", None) + if attributes.get(_CONTAINER_MARK) or parent is None: + return + same_stage = _state.guardrail_nodes.get(getattr(span, "name", "")) + self._close( + lambda key: key[0] == parent.span_id and key[1] != same_stage, + end_time=getattr(span, "start_time", None), + ) + + def on_end(self, span: ReadableSpan) -> None: + span_id = span.context.span_id if span.context else None + self._close(lambda key: key[0] == span_id, end_time=span.end_time) + + def shutdown(self) -> None: + """Called when its trace manager shuts down: unregister, end what's open.""" + with _state.lock: + for manager, closer in list(_state.closers.items()): + if closer is self: + del _state.closers[manager] + _state.enabled = bool(_state.closers) + if not _state.enabled: + self._close(lambda key: True, end_time=None) + + @staticmethod + def _close( + matches: Callable[[tuple[int, str]], bool], end_time: int | None + ) -> None: + with _state.lock: + done = [key for key in _state.containers if matches(key)] + spans = [_state.containers.pop(key) for key in done] + for span in spans: + span.end(end_time=end_time) + + +def is_exported(span: ReadableSpan) -> bool: + """Span filter hiding guardrail node spans (by name: spans upload on start).""" + if not _state.enabled or span.name not in _state.guardrail_nodes: + return True + scope = getattr(span, "instrumentation_scope", None) + return "openinference" not in (getattr(scope, "name", "") or "") + + +def enable_guardrail_grouping( + trace_manager: Any, current_span_provider: Callable[[], Span | None] +) -> None: + """Register the span provider and offer the trace manager for grouping. + + Grouping turns on once a guardrail middleware is also in use. + """ + UiPathSpanUtils.register_current_span_provider( + guardrail_aware(current_span_provider) + ) + _state.trace_manager = trace_manager + _activate() + + +def _activate() -> None: + """Attach a closer to the offered trace manager if guardrails are in use.""" + with _state.lock: + manager = _state.trace_manager + if manager is None or not _state.in_use: + return + if manager not in _state.closers: + closer = GuardrailStageCloser() + manager.add_span_processor(closer) + _state.closers[manager] = closer + _state.enabled = True diff --git a/src/uipath_langchain/guardrails/middlewares/deterministic.py b/src/uipath_langchain/guardrails/middlewares/deterministic.py index 46ff5efbd..90d05cbc2 100644 --- a/src/uipath_langchain/guardrails/middlewares/deterministic.py +++ b/src/uipath_langchain/guardrails/middlewares/deterministic.py @@ -12,15 +12,22 @@ from langgraph.prebuilt.tool_node import ToolCallRequest from langgraph.types import Command from uipath.core.guardrails import ( + GuardrailScope, GuardrailValidationResult, GuardrailValidationResultType, ) from uipath.platform.guardrails.decorators._exceptions import GuardrailBlockException -from uipath_langchain.agent.exceptions import AgentRuntimeError - from ..enums import GuardrailExecutionStage from ..models import GuardrailAction +from ._spans import ( + guardrail_action_span, + guardrail_evaluation_span, + guardrail_stage, + guardrails_in_use, + set_validation_result, + span_guid, +) from ._utils import ( convert_block_exception, create_modified_tool_request, @@ -156,6 +163,7 @@ def __init__( def _create_middleware_instances(self) -> list[AgentMiddleware]: """Create middleware instances from decorated functions.""" + guardrails_in_use() instances = [] middleware_instance = self guardrail_name = self._name.replace(" ", "_") @@ -234,22 +242,15 @@ def _evaluate_and_apply_modifications( ToolCallRequest | None, dict[str, Any] | None, ToolMessage | Command[Any] | None ]: """Evaluate rules and apply modifications if validation fails.""" - if not rules: - result = GuardrailValidationResult( - result=GuardrailValidationResultType.VALIDATION_FAILED, - reason="Empty rules - always apply action", - ) - else: - result = self._evaluate_rules(rules, stage, input_data, output_data) - if result.result != GuardrailValidationResultType.VALIDATION_FAILED: - return request, input_data, tool_result - try: - modified_data = self._handle_validation_result(result, data_for_action) + with guardrail_stage(GuardrailScope.TOOL, stage): + failed, modified_data = self._check( + stage, rules, input_data, output_data, data_for_action + ) except GuardrailBlockException as exc: raise convert_block_exception(exc) from exc - except AgentRuntimeError: - raise + if not failed: + return request, input_data, tool_result if stage == GuardrailExecutionStage.PRE: if modified_data is not None and isinstance(modified_data, dict): @@ -268,6 +269,27 @@ def _evaluate_and_apply_modifications( return request, input_data, tool_result + def _check( + self, + stage: GuardrailExecutionStage, + rules: Sequence[RuleFunction], + input_data: dict[str, Any] | None, + output_data: dict[str, Any] | None, + data_for_action: dict[str, Any] | str, + ) -> tuple[bool, str | dict[str, Any] | None]: + """Evaluate the rules and apply the action on a violation. + + Returns: + Whether the rules failed, and the data the action modified. + """ + with guardrail_evaluation_span(self._name, self._description) as span: + result = self._evaluate_rules(rules, stage, input_data, output_data) + set_validation_result(span, result) + if result.result != GuardrailValidationResultType.VALIDATION_FAILED: + return False, None + result = result.model_copy(update={"span_id": span_guid(span)}) + return True, self._handle_validation_result(result, data_for_action) + def _evaluate_rules( self, rules: Sequence[RuleFunction], @@ -385,6 +407,7 @@ def _handle_validation_result( self, result: GuardrailValidationResult, input_data: str | dict[str, Any] ) -> str | dict[str, Any] | None: """Handle guardrail validation result.""" - if result.result == GuardrailValidationResultType.VALIDATION_FAILED: + if result.result != GuardrailValidationResultType.VALIDATION_FAILED: + return None + with guardrail_action_span(self._name, self.action, result): return self.action.handle_validation_result(result, input_data, self._name) - return None diff --git a/src/uipath_langchain/runtime/factory.py b/src/uipath_langchain/runtime/factory.py index d983cedae..755b4c5fb 100644 --- a/src/uipath_langchain/runtime/factory.py +++ b/src/uipath_langchain/runtime/factory.py @@ -11,7 +11,7 @@ get_current_span, ) from uipath.core.adapters import EvaluatorProtocol -from uipath.core.tracing import UiPathSpanUtils, UiPathTraceManager +from uipath.core.tracing import UiPathSpanUtils, UiPathTraceManager, UiPathTraceSettings from uipath.platform.resume_triggers import ( UiPathResumeTriggerHandler, ) @@ -28,6 +28,10 @@ from uipath_langchain._tracing import _instrument_traceable_attributes from uipath_langchain.chat.chat_model_factory import _TraceContextHeadersCallback from uipath_langchain.governance import GovernanceCallbackHandler +from uipath_langchain.guardrails.middlewares._spans import ( + enable_guardrail_grouping, + is_exported, +) from uipath_langchain.runtime.config import LangGraphConfig from uipath_langchain.runtime.errors import LangGraphErrorCode, LangGraphRuntimeError from uipath_langchain.runtime.graph import LangGraphLoader @@ -67,7 +71,7 @@ def _setup_instrumentation(self, trace_manager: UiPathTraceManager | None) -> No """Setup tracing and instrumentation.""" _instrument_traceable_attributes() LangChainInstrumentor().instrument() - UiPathSpanUtils.register_current_span_provider(get_current_span) + enable_guardrail_grouping(trace_manager, get_current_span) UiPathSpanUtils.register_current_span_ancestors_provider(get_ancestor_spans) def _get_connection_string(self) -> str: @@ -251,6 +255,7 @@ async def get_settings(self) -> UiPathRuntimeFactorySettings | None: return UiPathRuntimeFactorySettings( agent_type=_AGENT_TYPE_CODED, agent_framework=_AGENT_FRAMEWORK, + trace_settings=UiPathTraceSettings(span_filter=is_exported), ) async def get_storage(self) -> UiPathRuntimeStorageProtocol | None: diff --git a/tests/guardrails/middlewares/test_action_span.py b/tests/guardrails/middlewares/test_action_span.py new file mode 100644 index 000000000..fa885dd63 --- /dev/null +++ b/tests/guardrails/middlewares/test_action_span.py @@ -0,0 +1,163 @@ +"""The middleware records the action it takes on a violation as a guardrailAction span.""" + +from collections.abc import Iterator +from typing import Any +from unittest.mock import patch + +import pytest +from langchain_core.messages import HumanMessage +from langgraph.errors import GraphInterrupt +from opentelemetry import trace +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.trace import StatusCode +from uipath.core.guardrails import ( + GuardrailScope, + GuardrailValidationResult, + GuardrailValidationResultType, +) +from uipath.platform.guardrails.decorators import ( + BlockAction, + LogAction, + LoggingSeverityLevel, +) + +from uipath_langchain.agent.exceptions import AgentRuntimeError +from uipath_langchain.guardrails import GuardrailAction +from uipath_langchain.guardrails.enums import PIIDetectionEntityType +from uipath_langchain.guardrails.middlewares import UiPathPIIDetectionMiddleware, _spans +from uipath_langchain.guardrails.middlewares._spans import action_name +from uipath_langchain.guardrails.models import PIIDetectionEntity + +_SERVICE_SPAN_ID = "00000000-0000-0000-1234-567890abcdef" +_FAILED = GuardrailValidationResult( + result=GuardrailValidationResultType.VALIDATION_FAILED, + reason="PII detected", + span_id=_SERVICE_SPAN_ID, +) +_PASSED = GuardrailValidationResult( + result=GuardrailValidationResultType.PASSED, reason="" +) + + +@pytest.fixture +def recorded_spans(monkeypatch: pytest.MonkeyPatch) -> Iterator[InMemorySpanExporter]: + monkeypatch.setattr(_spans, "_state", _spans._GroupingState()) + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + monkeypatch.setattr(trace, "get_tracer", provider.get_tracer) + yield exporter + provider.shutdown() + + +def _check(action: GuardrailAction, result: GuardrailValidationResult) -> None: + """Run an LLM-input check under a span named ``caller``.""" + middleware = UiPathPIIDetectionMiddleware( + scopes=[GuardrailScope.LLM], + action=action, + entities=[PIIDetectionEntity(PIIDetectionEntityType.EMAIL)], + ) + with ( + patch.object(middleware, "_evaluate_guardrail", return_value=result), + trace.get_tracer("test").start_as_current_span("caller"), + ): + middleware._check_messages([HumanMessage(content="mail me at a@b.com")]) + + +def _action_spans(exporter: InMemorySpanExporter) -> list[ReadableSpan]: + return [ + span + for span in exporter.get_finished_spans() + if (span.attributes or {}).get("span_type") == "guardrailAction" + ] + + +class _Recording(GuardrailAction): + def __init__(self, raise_exc: BaseException | None = None) -> None: + self._raise = raise_exc + + def handle_validation_result( + self, result: Any, data: Any, guardrail_name: str + ) -> Any: + if self._raise is not None: + raise self._raise + return None + + +class TestGuardrailActionSpan: + def test_block_records_an_error_span_linked_to_the_service_span( + self, recorded_spans: InMemorySpanExporter + ) -> None: + with pytest.raises(AgentRuntimeError): + _check(BlockAction(), _FAILED) + + [span] = _action_spans(recorded_spans) + assert span.name == "Block" + attributes = dict(span.attributes or {}) + assert attributes["type"] == "guardrailAction" + assert attributes["uipath.custom_instrumentation"] is True + assert attributes["guardrailName"] == "PII Detection" + assert attributes["action"] == "Block" + assert attributes["reason"] == "PII detected" + assert attributes["guardrailSpanId"] == _SERVICE_SPAN_ID + assert span.status.status_code == StatusCode.ERROR + assert span.status.description == "Guardrail violation: PII detected" + caller = next( + s for s in recorded_spans.get_finished_spans() if s.name == "caller" + ) + assert span.parent is not None + assert span.parent.span_id == caller.context.span_id + + def test_log_records_an_ok_span_with_its_severity( + self, recorded_spans: InMemorySpanExporter + ) -> None: + _check(LogAction(severity_level=LoggingSeverityLevel.ERROR), _FAILED) + + [span] = _action_spans(recorded_spans) + assert span.name == "Log" + attributes = dict(span.attributes or {}) + assert attributes["action"] == "Log" + assert attributes["severityLevel"] == "Error" + assert attributes["reason"] == "PII detected" + assert span.status.status_code != StatusCode.ERROR + + def test_passed_result_records_no_action_span( + self, recorded_spans: InMemorySpanExporter + ) -> None: + _check(BlockAction(), _PASSED) + + assert _action_spans(recorded_spans) == [] + + def test_suspending_escalation_ends_the_span_without_error( + self, recorded_spans: InMemorySpanExporter + ) -> None: + with pytest.raises(GraphInterrupt): + _check(_Recording(raise_exc=GraphInterrupt()), _FAILED) + + [span] = _action_spans(recorded_spans) + assert span.status.status_code != StatusCode.ERROR + + def test_missing_service_span_id_is_omitted( + self, recorded_spans: InMemorySpanExporter + ) -> None: + without_span_id = _FAILED.model_copy(update={"span_id": None}) + _check(LogAction(), without_span_id) + + [span] = _action_spans(recorded_spans) + assert "guardrailSpanId" not in (span.attributes or {}) + + +@pytest.mark.parametrize( + ("action", "expected"), + [ + (BlockAction(), "Block"), + (LogAction(), "Log"), + (_Recording(), "_Recording"), + ], +) +def test_action_name(action: Any, expected: str) -> None: + assert action_name(action) == expected diff --git a/tests/guardrails/middlewares/test_grouping.py b/tests/guardrails/middlewares/test_grouping.py new file mode 100644 index 000000000..3baf228dc --- /dev/null +++ b/tests/guardrails/middlewares/test_grouping.py @@ -0,0 +1,517 @@ +"""Separate guardrail middlewares share one container span per stage.""" + +import asyncio +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from langchain.agents import create_agent +from langchain_core.language_models.fake_chat_models import GenericFakeChatModel +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from langchain_core.tools import tool +from langgraph.prebuilt.tool_node import ToolCallRequest +from openinference.instrumentation.langchain import ( + LangChainInstrumentor, + get_ancestor_spans, + get_current_span, +) +from opentelemetry import trace +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) +from opentelemetry.trace import Span, StatusCode +from uipath.core.guardrails import ( + GuardrailScope, + GuardrailValidationResult, + GuardrailValidationResultType, +) +from uipath.core.tracing.span_utils import UiPathSpanUtils +from uipath.platform.guardrails.decorators import BlockAction, LogAction + +from uipath_langchain.agent.exceptions import AgentRuntimeError +from uipath_langchain.guardrails import GuardrailExecutionStage +from uipath_langchain.guardrails.enums import PIIDetectionEntityType +from uipath_langchain.guardrails.middlewares import ( + UiPathDeterministicGuardrailMiddleware, + UiPathPIIDetectionMiddleware, + UiPathPromptInjectionMiddleware, + _base, + _spans, +) +from uipath_langchain.guardrails.middlewares._spans import ( + enable_guardrail_grouping, + guardrail_stage, + is_exported, +) +from uipath_langchain.guardrails.models import PIIDetectionEntity + +_FAILED = GuardrailValidationResult( + result=GuardrailValidationResultType.VALIDATION_FAILED, reason="found" +) +_STATE = {"messages": [HumanMessage(content="mail me at a@b.com")]} +_OPENINFERENCE = "openinference.instrumentation.langchain" + + +@pytest.fixture +def exporter(monkeypatch: pytest.MonkeyPatch) -> Iterator[InMemorySpanExporter]: + monkeypatch.setattr(_spans, "_state", _spans._GroupingState()) + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + monkeypatch.setattr(trace, "get_tracer", provider.get_tracer) + exporter.provider = provider # type: ignore[attr-defined] + yield exporter + UiPathSpanUtils.register_current_span_provider(None) + UiPathSpanUtils.register_current_span_ancestors_provider(None) + provider.shutdown() + + +class _Graph: + """Plays LangGraph: a graph span and node spans started in order.""" + + def __init__(self, exporter: InMemorySpanExporter) -> None: + self.exporter = exporter + self._tracer = exporter.provider.get_tracer(_OPENINFERENCE) # type: ignore[attr-defined] + self.current: Span | None = None + self.span = self._tracer.start_span("LangGraph") + + @contextmanager + def node( + self, + name: str, + parent: Span | None = None, + attributes: dict[str, Any] | None = None, + ) -> Iterator[Span]: + span = self._tracer.start_span( + name, + context=trace.set_span_in_context(parent or self.span), + attributes=attributes, + ) + previous, self.current = self.current, span + try: + yield span + finally: + self.current = previous + span.end() + + def finish(self) -> list[ReadableSpan]: + self.span.end() + return list(self.exporter.get_finished_spans()) + + +@pytest.fixture +def graph(exporter: InMemorySpanExporter) -> _Graph: + fake = _Graph(exporter) + enable_guardrail_grouping(exporter.provider, lambda: fake.current) # type: ignore[attr-defined] + return fake + + +def _hook(middleware: Any, name: str) -> Any: + [hook] = [h for h in middleware if h.name.endswith(name)] + return hook + + +def _named(spans: list[ReadableSpan], name: str) -> list[ReadableSpan]: + return [s for s in spans if s.name == name] + + +def _children(spans: list[ReadableSpan], parent: ReadableSpan) -> list[str]: + return [ + s.name + for s in spans + if s.parent is not None and s.parent.span_id == parent.context.span_id + ] + + +def _pii(**kwargs: Any) -> UiPathPIIDetectionMiddleware: + return UiPathPIIDetectionMiddleware( + scopes=kwargs.pop("scopes", [GuardrailScope.LLM]), + action=kwargs.pop("action", LogAction()), + entities=[PIIDetectionEntity(PIIDetectionEntityType.EMAIL)], + **kwargs, + ) + + +def _rule(stage: GuardrailExecutionStage) -> UiPathDeterministicGuardrailMiddleware: + return UiPathDeterministicGuardrailMiddleware( + tools=["get_weather"], + rules=[lambda *args: True], + action=LogAction(), + stage=stage, + name="Rule", + ) + + +def _tool_request() -> ToolCallRequest: + return ToolCallRequest( + tool_call={"id": "tc1", "name": "get_weather", "args": {"city": "London"}}, + tool=MagicMock(), + state={}, + runtime=MagicMock(), + ) + + +class TestStages: + async def test_guardrail_nodes_of_one_stage_share_a_container( + self, graph: _Graph + ) -> None: + pii = _pii() + injection = UiPathPromptInjectionMiddleware(action=LogAction()) + pii_hook = _hook(pii, "_before_model") + injection_hook = _hook(injection, "_before_model") + + with ( + patch.object(pii, "_evaluate_guardrail", return_value=_FAILED), + patch.object(injection, "_evaluate_guardrail", return_value=_FAILED), + ): + with graph.node(f"{pii_hook.name}.before_model"): + await pii_hook.abefore_model(_STATE, MagicMock()) + with graph.node(f"{injection_hook.name}.before_model"): + await injection_hook.abefore_model(_STATE, MagicMock()) + with graph.node("model") as model: + model_start = model.start_time # type: ignore[attr-defined] + spans = graph.finish() + + [container] = _named(spans, "LLM input guardrail check") + assert (container.attributes or {})["span_type"] == "llmPreGuardrails" + assert container.parent is not None + assert container.parent.span_id == graph.span.get_span_context().span_id + assert _children(spans, container) == ["Log", "Log"] + assert container.end_time == model_start + + async def test_the_container_carries_the_execution_id(self, graph: _Graph) -> None: + pii = _pii() + hook = _hook(pii, "_before_model") + + with ( + patch.object(pii, "_evaluate_guardrail", return_value=_FAILED), + graph.node( + f"{hook.name}.before_model", attributes={"execution.id": "exec-1"} + ), + ): + await hook.abefore_model(_STATE, MagicMock()) + spans = graph.finish() + + [container] = _named(spans, "LLM input guardrail check") + assert (container.attributes or {})["execution.id"] == "exec-1" + + async def test_a_new_stage_gets_its_own_container(self, graph: _Graph) -> None: + pii = _pii(scopes=[GuardrailScope.AGENT, GuardrailScope.LLM]) + agent_hook = _hook(pii, "_before_agent") + model_hook = _hook(pii, "_before_model") + + with patch.object(pii, "_evaluate_guardrail", return_value=_FAILED): + with graph.node(f"{agent_hook.name}.before_agent"): + await agent_hook.abefore_agent(_STATE, MagicMock()) + with graph.node(f"{model_hook.name}.before_model"): + await model_hook.abefore_model(_STATE, MagicMock()) + spans = graph.finish() + + assert len(_named(spans, "Agent input guardrail check")) == 1 + assert len(_named(spans, "LLM input guardrail check")) == 1 + + async def test_another_node_between_guardrails_splits_the_stage( + self, graph: _Graph + ) -> None: + pii = _pii() + injection = UiPathPromptInjectionMiddleware(action=LogAction()) + pii_hook = _hook(pii, "_before_model") + injection_hook = _hook(injection, "_before_model") + + with ( + patch.object(pii, "_evaluate_guardrail", return_value=_FAILED), + patch.object(injection, "_evaluate_guardrail", return_value=_FAILED), + ): + with graph.node(f"{pii_hook.name}.before_model"): + await pii_hook.abefore_model(_STATE, MagicMock()) + with graph.node("SummarizationMiddleware.before_model"): + pass + with graph.node(f"{injection_hook.name}.before_model"): + await injection_hook.abefore_model(_STATE, MagicMock()) + spans = graph.finish() + + assert len(_named(spans, "LLM input guardrail check")) == 2 + + async def test_llm_output_checks_get_their_container(self, graph: _Graph) -> None: + pii = _pii() + hook = _hook(pii, "_after_model") + state = {"messages": [HumanMessage("hi"), AIMessage("mail a@b.com")]} + + with ( + patch.object(pii, "_evaluate_guardrail", return_value=_FAILED), + graph.node(f"{hook.name}.after_model"), + ): + await hook.aafter_model(state, MagicMock()) + spans = graph.finish() + + [container] = _named(spans, "LLM output guardrail check") + assert _children(spans, container) == ["Log"] + + async def test_a_block_marks_the_container_as_failed(self, graph: _Graph) -> None: + pii = _pii(action=BlockAction()) + hook = _hook(pii, "_before_model") + + with ( + patch.object(pii, "_evaluate_guardrail", return_value=_FAILED), + pytest.raises(AgentRuntimeError), + graph.node(f"{hook.name}.before_model"), + ): + await hook.abefore_model(_STATE, MagicMock()) + spans = graph.finish() + + [container] = _named(spans, "LLM input guardrail check") + assert container.status.status_code == StatusCode.ERROR + + async def test_a_stage_with_nothing_to_check_leaves_no_container( + self, graph: _Graph + ) -> None: + pii = _pii() + hook = _hook(pii, "_before_model") + + with ( + patch.object(pii, "_evaluate_guardrail") as evaluate, + graph.node(f"{hook.name}.before_model"), + ): + await hook.abefore_model({"messages": []}, MagicMock()) + spans = graph.finish() + + evaluate.assert_not_called() + assert not _named(spans, "LLM input guardrail check") + + +class TestTools: + async def _call_tool( + self, graph: _Graph, stage: GuardrailExecutionStage + ) -> list[ReadableSpan]: + [wrapper] = list(_rule(stage)) + with graph.node("tools") as tools: + + async def run_tool(request: ToolCallRequest) -> ToolMessage: + with graph.node("get_weather", parent=tools): + return ToolMessage(content="sunny", tool_call_id="tc1") + + await wrapper.awrap_tool_call(_tool_request(), run_tool) + return graph.finish() + + async def test_input_checks_close_when_the_tool_starts(self, graph: _Graph) -> None: + spans = await self._call_tool(graph, GuardrailExecutionStage.PRE) + + [container] = _named(spans, "Tool input guardrail check") + [tools] = _named(spans, "tools") + [tool_span] = _named(spans, "get_weather") + assert container.parent is not None + assert container.parent.span_id == tools.context.span_id + assert _children(spans, container) == ["Rule", "Log"] + assert container.end_time == tool_span.start_time + + async def test_output_checks_close_when_the_tools_node_ends( + self, graph: _Graph + ) -> None: + spans = await self._call_tool(graph, GuardrailExecutionStage.POST) + + [container] = _named(spans, "Tool output guardrail check") + [tools] = _named(spans, "tools") + [tool_span] = _named(spans, "get_weather") + assert container.start_time is not None and tool_span.end_time is not None + assert container.start_time >= tool_span.end_time + assert container.end_time == tools.end_time + + +class TestEnabling: + async def test_nothing_is_grouped_without_a_closer( + self, exporter: InMemorySpanExporter + ) -> None: + node = exporter.provider.get_tracer(_OPENINFERENCE).start_span("node") # type: ignore[attr-defined] + enable_guardrail_grouping(None, lambda: node) + + with guardrail_stage(GuardrailScope.LLM, GuardrailExecutionStage.PRE): + pass + node.end() + + assert _spans._state.containers == {} + assert not _spans._state.enabled + + async def test_nothing_is_grouped_under_an_invalid_node_span( + self, exporter: InMemorySpanExporter + ) -> None: + enable_guardrail_grouping(exporter.provider, lambda: trace.INVALID_SPAN) # type: ignore[attr-defined] + _pii() # a guardrail in use turns grouping on + + with guardrail_stage(GuardrailScope.TOOL, GuardrailExecutionStage.PRE): + pass + + assert _spans._state.enabled + assert _spans._state.containers == {} + + def test_the_closer_is_added_once_per_trace_manager( + self, exporter: InMemorySpanExporter + ) -> None: + manager = MagicMock() + enable_guardrail_grouping(manager, lambda: None) + _pii() + enable_guardrail_grouping(manager, lambda: None) + _pii() + + manager.add_span_processor.assert_called_once() + + def test_the_closer_comes_back_after_its_trace_manager_shuts_down( + self, exporter: InMemorySpanExporter + ) -> None: + manager = MagicMock() + enable_guardrail_grouping(manager, lambda: None) + _pii() + [closer] = [call.args[0] for call in manager.add_span_processor.call_args_list] + + closer.shutdown() # what UiPathTraceManager.shutdown() does + assert not _spans._state.enabled + enable_guardrail_grouping(manager, lambda: None) + + assert manager.add_span_processor.call_count == 2 + assert _spans._state.enabled + + def test_tool_only_guardrails_turn_grouping_on( + self, exporter: InMemorySpanExporter + ) -> None: + manager = MagicMock() + enable_guardrail_grouping(manager, lambda: None) + _rule(GuardrailExecutionStage.PRE) + + manager.add_span_processor.assert_called_once() + assert _spans._state.enabled + + def test_grouping_stays_off_without_guardrail_middlewares( + self, exporter: InMemorySpanExporter + ) -> None: + manager = MagicMock() + enable_guardrail_grouping(manager, lambda: None) + + manager.add_span_processor.assert_not_called() + assert not _spans._state.enabled + + async def test_only_guardrail_node_spans_are_hidden(self, graph: _Graph) -> None: + pii = _pii() + hook = _hook(pii, "_before_model") + node_name = f"{hook.name}.before_model" + user_span = trace.get_tracer("my_app").start_span(node_name) + user_span.end() + + with ( + patch.object(pii, "_evaluate_guardrail", return_value=_FAILED), + graph.node(node_name), + ): + await hook.abefore_model(_STATE, MagicMock()) + spans = graph.finish() + + hidden = [s for s in spans if not is_exported(s)] + assert [s.instrumentation_scope.name for s in hidden] == [_OPENINFERENCE] # type: ignore[union-attr] + assert is_exported(_named(spans, "LLM input guardrail check")[0]) + + +class _ToolCallingModel(GenericFakeChatModel): + def bind_tools(self, *args: Any, **kwargs: Any) -> "_ToolCallingModel": + return self + + +@tool +def get_weather(city: str) -> str: + """Weather for a city.""" + return "sunny" + + +class TestRealAgent: + """A real create_agent graph traced by OpenInference.""" + + @pytest.fixture + def instrumented(self, exporter: InMemorySpanExporter) -> Iterator[None]: + provider = exporter.provider # type: ignore[attr-defined] + instrumentor = LangChainInstrumentor() + # Instrumentation is global: re-instrument on this provider, then restore. + was_instrumented = instrumentor.is_instrumented_by_opentelemetry + if was_instrumented: + instrumentor.uninstrument() + instrumentor.instrument(tracer_provider=provider) + enable_guardrail_grouping(provider, get_current_span) + UiPathSpanUtils.register_current_span_ancestors_provider(get_ancestor_spans) + yield + instrumentor.uninstrument() + if was_instrumented: + instrumentor.instrument() + + def _agent(self, parallel_calls: int) -> Any: + calls = [ + {"id": f"t{i}", "name": "get_weather", "args": {"city": "London"}} + for i in range(parallel_calls) + ] + model = _ToolCallingModel( + messages=iter([AIMessage(content="", tool_calls=calls), AIMessage("ok")]) + ) + guardrails = [ + *_pii( + scopes=[GuardrailScope.LLM, GuardrailScope.TOOL], tools=[get_weather] + ), + *UiPathPromptInjectionMiddleware(action=LogAction()), + *_rule(GuardrailExecutionStage.PRE_AND_POST), + ] + return create_agent(model, tools=[get_weather], middleware=guardrails) + + @staticmethod + def _assert_well_formed(spans: list[ReadableSpan]) -> None: + by_id = {s.context.span_id: s for s in spans} + exported = [s for s in spans if is_exported(s)] + orphans = [ + s.name + for s in exported + if s.parent + and s.parent.span_id in by_id + and not is_exported(by_id[s.parent.span_id]) + ] + assert orphans == [] + assert _spans._state.containers == {} + + async def test_parallel_tool_calls_group_per_tools_span( + self, exporter: InMemorySpanExporter, instrumented: None + ) -> None: + with patch.object( + _base.BuiltInGuardrailMiddlewareMixin, + "_evaluate_guardrail", + return_value=_FAILED, + ): + await self._agent(parallel_calls=2).ainvoke( + {"messages": [HumanMessage("a@b.com weather")]} + ) + spans = list(exporter.get_finished_spans()) + + self._assert_well_formed(spans) + assert len(_named(spans, "LLM input guardrail check")) == 2 + assert len(_named(spans, "LLM output guardrail check")) == 1 + assert len(_named(spans, "Tool input guardrail check")) == 2 + assert len(_named(spans, "Tool output guardrail check")) == 2 + for container in _named(spans, "LLM input guardrail check"): + assert _children(spans, container) == ["Log", "Log"] + + async def test_concurrent_runs_keep_their_own_containers( + self, exporter: InMemorySpanExporter, instrumented: None + ) -> None: + with patch.object( + _base.BuiltInGuardrailMiddlewareMixin, + "_evaluate_guardrail", + return_value=_FAILED, + ): + await asyncio.gather( + self._agent(parallel_calls=1).ainvoke( + {"messages": [HumanMessage("a@b.com")]} + ), + self._agent(parallel_calls=1).ainvoke( + {"messages": [HumanMessage("a@b.com")]} + ), + ) + spans = list(exporter.get_finished_spans()) + + self._assert_well_formed(spans) + containers = _named(spans, "LLM input guardrail check") + assert len(containers) == 4 + assert all(_children(spans, c) for c in containers) diff --git a/uv.lock b/uv.lock index febf27a60..e7caa5f91 100644 --- a/uv.lock +++ b/uv.lock @@ -4861,7 +4861,7 @@ wheels = [ [[package]] name = "uipath-langchain" -version = "0.20.2" +version = "0.20.3" source = { editable = "." } dependencies = [ { name = "a2a-sdk" }, From 8bd36fc6b3e4e43d4916d5130f6122a10e739639 Mon Sep 17 00:00:00 2001 From: Daniel Joica Date: Wed, 18 Nov 2026 12:51:20 +0200 Subject: [PATCH 2/3] test(guardrails): cover overlapping checks on parallel tool calls [AL-712] Addresses review on #1219: a regression test with three parallel tool calls and deliberately overlapping checks (each call has its own tools span, so its own container). Also Sonar: collect the closer's owners before deleting them, one throwing call per pytest.raises, and split a composite assertion. Co-Authored-By: Claude Opus 5.5 --- .../guardrails/middlewares/_spans.py | 6 +-- .../middlewares/test_action_span.py | 3 +- tests/guardrails/middlewares/test_grouping.py | 38 ++++++++++++++++++- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/src/uipath_langchain/guardrails/middlewares/_spans.py b/src/uipath_langchain/guardrails/middlewares/_spans.py index e419ec2fe..f07556f94 100644 --- a/src/uipath_langchain/guardrails/middlewares/_spans.py +++ b/src/uipath_langchain/guardrails/middlewares/_spans.py @@ -269,9 +269,9 @@ def on_end(self, span: ReadableSpan) -> None: def shutdown(self) -> None: """Called when its trace manager shuts down: unregister, end what's open.""" with _state.lock: - for manager, closer in list(_state.closers.items()): - if closer is self: - del _state.closers[manager] + owners = [m for m, closer in _state.closers.items() if closer is self] + for manager in owners: + del _state.closers[manager] _state.enabled = bool(_state.closers) if not _state.enabled: self._close(lambda key: True, end_time=None) diff --git a/tests/guardrails/middlewares/test_action_span.py b/tests/guardrails/middlewares/test_action_span.py index fa885dd63..df7747d0b 100644 --- a/tests/guardrails/middlewares/test_action_span.py +++ b/tests/guardrails/middlewares/test_action_span.py @@ -135,8 +135,9 @@ def test_passed_result_records_no_action_span( def test_suspending_escalation_ends_the_span_without_error( self, recorded_spans: InMemorySpanExporter ) -> None: + escalation = _Recording(raise_exc=GraphInterrupt()) with pytest.raises(GraphInterrupt): - _check(_Recording(raise_exc=GraphInterrupt()), _FAILED) + _check(escalation, _FAILED) [span] = _action_spans(recorded_spans) assert span.status.status_code != StatusCode.ERROR diff --git a/tests/guardrails/middlewares/test_grouping.py b/tests/guardrails/middlewares/test_grouping.py index 3baf228dc..f5f02c73d 100644 --- a/tests/guardrails/middlewares/test_grouping.py +++ b/tests/guardrails/middlewares/test_grouping.py @@ -1,6 +1,7 @@ """Separate guardrail middlewares share one container span per stage.""" import asyncio +import time from collections.abc import Iterator from contextlib import contextmanager from typing import Any @@ -316,7 +317,8 @@ async def test_output_checks_close_when_the_tools_node_ends( [container] = _named(spans, "Tool output guardrail check") [tools] = _named(spans, "tools") [tool_span] = _named(spans, "get_weather") - assert container.start_time is not None and tool_span.end_time is not None + assert container.start_time is not None + assert tool_span.end_time is not None assert container.start_time >= tool_span.end_time assert container.end_time == tools.end_time @@ -493,6 +495,40 @@ async def test_parallel_tool_calls_group_per_tools_span( for container in _named(spans, "LLM input guardrail check"): assert _children(spans, container) == ["Log", "Log"] + async def test_overlapping_checks_on_parallel_tool_calls_stay_in_their_container( + self, exporter: InMemorySpanExporter, instrumented: None + ) -> None: + def slow_check(*args: Any, **kwargs: Any) -> GuardrailValidationResult: + time.sleep(0.05) # long enough for the calls' checks to overlap + return _FAILED + + with patch.object( + _base.BuiltInGuardrailMiddlewareMixin, "_evaluate_guardrail", slow_check + ): + await self._agent(parallel_calls=3).ainvoke( + {"messages": [HumanMessage("a@b.com weather")]} + ) + spans = list(exporter.get_finished_spans()) + + # Each tool call runs in its own tools span, so each has its own container, + # and every check ends inside it. + containers = _named(spans, "Tool input guardrail check") + assert len(containers) == 3 + assert len({c.parent.span_id for c in containers if c.parent}) == 3 + for container in containers: + children = [ + s + for s in spans + if s.parent and s.parent.span_id == container.context.span_id + ] + assert children + assert all( + c.end_time is not None + and container.end_time is not None + and c.end_time <= container.end_time + for c in children + ) + async def test_concurrent_runs_keep_their_own_containers( self, exporter: InMemorySpanExporter, instrumented: None ) -> None: From b1d9dc3a574b1446fed7087c506db62e74cf56fb Mon Sep 17 00:00:00 2001 From: Daniel Joica Date: Wed, 18 Nov 2026 13:43:38 +0200 Subject: [PATCH 3/3] fix(guardrails): never end a stage container before its last check [AL-712] On Windows before Python 3.13 the clock advances about every 15 ms, so the next sibling's start can be stamped a tick before the last check ended. Each stage now records when its last check finished, and the closer ends the container at the later of the two. Co-Authored-By: Claude Opus 5.5 --- .../guardrails/middlewares/_spans.py | 14 +++++++++-- tests/guardrails/middlewares/test_grouping.py | 24 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/uipath_langchain/guardrails/middlewares/_spans.py b/src/uipath_langchain/guardrails/middlewares/_spans.py index f07556f94..945af5126 100644 --- a/src/uipath_langchain/guardrails/middlewares/_spans.py +++ b/src/uipath_langchain/guardrails/middlewares/_spans.py @@ -2,6 +2,7 @@ stage containers that group a stage's checks across separate middleware nodes.""" import threading +import time import weakref from collections.abc import Callable, Generator from contextlib import contextmanager @@ -153,6 +154,7 @@ def __init__(self) -> None: ) # trace manager -> its closer self.guardrail_nodes: dict[str, str] = {} # node name -> stage span type self.containers: dict[tuple[int, str], Span] = {} # (parent id, type) + self.last_check_end: dict[int, int] = {} # container span id -> ns self.lock = threading.Lock() @@ -196,6 +198,8 @@ def guardrail_stage( raise finally: _current_container.reset(token) + with _state.lock: + _state.last_check_end[container.get_span_context().span_id] = time.time_ns() def _shared_container( @@ -283,8 +287,14 @@ def _close( with _state.lock: done = [key for key in _state.containers if matches(key)] spans = [_state.containers.pop(key) for key in done] - for span in spans: - span.end(end_time=end_time) + last_ends = [ + _state.last_check_end.pop(span.get_span_context().span_id, 0) + for span in spans + ] + for span, last_end in zip(spans, last_ends, strict=True): + # Never before its last check: coarse clocks (Windows before Python + # 3.13) can stamp the next sibling's start a tick earlier. + span.end(end_time=max(end_time, last_end) if end_time else None) def is_exported(span: ReadableSpan) -> bool: diff --git a/tests/guardrails/middlewares/test_grouping.py b/tests/guardrails/middlewares/test_grouping.py index f5f02c73d..ad7a1a613 100644 --- a/tests/guardrails/middlewares/test_grouping.py +++ b/tests/guardrails/middlewares/test_grouping.py @@ -86,11 +86,13 @@ def node( name: str, parent: Span | None = None, attributes: dict[str, Any] | None = None, + start_time: int | None = None, ) -> Iterator[Span]: span = self._tracer.start_span( name, context=trace.set_span_in_context(parent or self.span), attributes=attributes, + start_time=start_time, ) previous, self.current = self.current, span try: @@ -184,6 +186,28 @@ async def test_guardrail_nodes_of_one_stage_share_a_container( assert _children(spans, container) == ["Log", "Log"] assert container.end_time == model_start + async def test_a_container_never_ends_before_its_last_check( + self, graph: _Graph + ) -> None: + pii = _pii() + hook = _hook(pii, "_before_model") + + with ( + patch.object(pii, "_evaluate_guardrail", return_value=_FAILED), + graph.node(f"{hook.name}.before_model"), + ): + await hook.abefore_model(_STATE, MagicMock()) + # A coarse clock stamps the next node's start before the check ended. + with graph.node("model", start_time=time.time_ns() - 10_000_000): + pass + spans = graph.finish() + + [container] = _named(spans, "LLM input guardrail check") + [log] = _named(spans, "Log") + assert container.end_time is not None + assert log.end_time is not None + assert container.end_time >= log.end_time + async def test_the_container_carries_the_execution_id(self, graph: _Graph) -> None: pii = _pii() hook = _hook(pii, "_before_model")