Skip to content

Commit 59dacd2

Browse files
fix(eval): raise the simulator's output token limit and make its tool schema work with Gemini (#1974)
1 parent 1392211 commit 59dacd2

10 files changed

Lines changed: 430 additions & 18 deletions

File tree

‎packages/uipath/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "uipath"
3-
version = "2.15.0"
3+
version = "2.15.1"
44
description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
55
readme = { file = "README.md", content-type = "text/markdown" }
66
requires-python = ">=3.11"

‎packages/uipath/src/uipath/_cli/cli_debug.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
from uipath._cli._utils._tracing import create_trace_manager
1313
from uipath.eval.mocks import SimulationConfig, UiPathMockRuntime, build_mocking_context
1414
from uipath.eval.mocks._mock_runtime import load_simulation_config
15+
from uipath.eval.mocks._simulation_model import get_agent_max_tokens
1516
from uipath.platform.common import (
1617
ExecutionSourceContext,
1718
ResourceOverwritesContext,
@@ -249,13 +250,14 @@ async def execute_debug_runtime():
249250
agent_model = None
250251
if schema.metadata and "settings" in schema.metadata:
251252
agent_model = schema.metadata["settings"].get("model")
253+
agent_max_tokens = get_agent_max_tokens(schema)
252254

253255
delegate_runtime: UiPathDebugRuntime | UiPathMockRuntime = (
254256
debug_runtime
255257
)
256258
if simulation_config:
257259
mocking_context = build_mocking_context(
258-
simulation_config, agent_model
260+
simulation_config, agent_model, agent_max_tokens
259261
)
260262
if mocking_context:
261263
delegate_runtime = UiPathMockRuntime(
@@ -264,7 +266,8 @@ async def execute_debug_runtime():
264266
)
265267
else:
266268
mocking_context = load_simulation_config(
267-
agent_model=agent_model
269+
agent_model=agent_model,
270+
agent_max_tokens=agent_max_tokens,
268271
)
269272
delegate_runtime = UiPathMockRuntime(
270273
delegate=debug_runtime,

‎packages/uipath/src/uipath/_cli/cli_run.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from uipath._cli._utils._debug import setup_debugging
1111
from uipath._cli._utils._tracing import create_trace_manager
1212
from uipath.eval.mocks import SimulationConfig, UiPathMockRuntime, build_mocking_context
13+
from uipath.eval.mocks._simulation_model import get_agent_max_tokens
1314
from uipath.platform.common import (
1415
ExecutionSourceContext,
1516
ResourceOverwritesContext,
@@ -316,7 +317,9 @@ async def execute() -> None:
316317
"model"
317318
)
318319
mocking_context = build_mocking_context(
319-
simulation_config, agent_model
320+
simulation_config,
321+
agent_model,
322+
get_agent_max_tokens(schema),
320323
)
321324
if mocking_context:
322325
runtime = UiPathMockRuntime(

‎packages/uipath/src/uipath/eval/mocks/_mock_runtime.py‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
)
2424
from ._mock_context import mocker_context, mocking_context
2525
from ._mocker_factory import MockerFactory
26+
from ._simulation_model import simulation_model_settings
2627
from ._types import (
2728
LLMMockingStrategy,
2829
MockingContext,
@@ -35,9 +36,15 @@
3536

3637

3738
def build_mocking_context(
38-
config: SimulationConfig, agent_model: str | None = None
39+
config: SimulationConfig,
40+
agent_model: str | None = None,
41+
agent_max_tokens: int | None = None,
3942
) -> MockingContext | None:
40-
"""Build a MockingContext from a validated SimulationConfig."""
43+
"""Build a MockingContext from a validated SimulationConfig.
44+
45+
Without a model in the config, the simulation uses the agent's model and
46+
its max tokens (capped, see ``simulation_model_settings``).
47+
"""
4148
if not config.enabled:
4249
return None
4350

@@ -68,7 +75,7 @@ def build_mocking_context(
6875
model = (
6976
ModelSettings(model=config.model)
7077
if config.model
71-
else ModelSettings(model=agent_model)
78+
else simulation_model_settings(agent_model, agent_max_tokens)
7279
if agent_model
7380
else None
7481
)
@@ -91,7 +98,9 @@ def build_mocking_context(
9198

9299

93100
def build_mocking_context_from_dict(
94-
simulation_data: dict[str, Any], agent_model: str | None = None
101+
simulation_data: dict[str, Any],
102+
agent_model: str | None = None,
103+
agent_max_tokens: int | None = None,
95104
) -> MockingContext | None:
96105
"""Build a MockingContext from a simulation config dictionary.
97106
@@ -100,15 +109,18 @@ def build_mocking_context_from_dict(
100109
Args:
101110
simulation_data: Parsed simulation config (same schema as simulation.json).
102111
agent_model: Optional agent model name to use as fallback.
112+
agent_max_tokens: Optional max tokens of the agent, used with agent_model.
103113
104114
Returns:
105115
MockingContext if valid and enabled, None otherwise.
106116
"""
107117
config = SimulationConfig.model_validate(simulation_data)
108-
return build_mocking_context(config, agent_model)
118+
return build_mocking_context(config, agent_model, agent_max_tokens)
109119

110120

111-
def load_simulation_config(agent_model: str | None = None) -> MockingContext | None:
121+
def load_simulation_config(
122+
agent_model: str | None = None, agent_max_tokens: int | None = None
123+
) -> MockingContext | None:
112124
"""Load simulation.json from current directory and convert to MockingContext.
113125
114126
Returns:
@@ -124,7 +136,7 @@ def load_simulation_config(agent_model: str | None = None) -> MockingContext | N
124136
config = SimulationConfig.model_validate_json(
125137
simulation_path.read_text(encoding="utf-8")
126138
)
127-
return build_mocking_context(config, agent_model)
139+
return build_mocking_context(config, agent_model, agent_max_tokens)
128140

129141
except Exception as e:
130142
logger.warning(f"Failed to load simulation.json: {e}")

‎packages/uipath/src/uipath/eval/mocks/_simulation_model.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,47 @@
44
from typing import Any
55

66
from uipath.platform.chat._llm_gateway_service import ChatModels
7+
from uipath.runtime.schema import UiPathRuntimeSchema
78

89
from ._types import ModelSettings
910

1011
SIMULATION_MODEL_ENV = "UIPATH_SIMULATION_MODEL"
12+
# Output token limit of a simulation call. Reasoning models count their
13+
# reasoning against it, so the chat service default (4096) is too low for long
14+
# simulated responses; the cap bounds a runaway reply.
15+
SIMULATION_MAX_TOKENS = 16384
16+
17+
18+
def get_agent_max_tokens(schema: UiPathRuntimeSchema) -> int | None:
19+
"""Get the agent's max tokens from the runtime schema metadata.
20+
21+
Read from ``schema.metadata["settings"]``, where the low-code agents runtime
22+
puts the agent's settings (the model is read from there too).
23+
24+
Returns:
25+
The agent's max tokens, or None if not found or not a positive integer.
26+
"""
27+
settings = (schema.metadata or {}).get("settings")
28+
if not isinstance(settings, dict):
29+
return None
30+
max_tokens = settings.get("max_tokens", settings.get("maxTokens"))
31+
if isinstance(max_tokens, int) and not isinstance(max_tokens, bool):
32+
return max_tokens if max_tokens > 0 else None
33+
return None
34+
35+
36+
def simulation_model_settings(
37+
model: str, agent_max_tokens: int | None = None
38+
) -> ModelSettings:
39+
"""Build the settings for a simulation that uses the agent's model.
40+
41+
The output limit is the agent's own max tokens, a value the model is known
42+
to accept, capped at ``SIMULATION_MAX_TOKENS``.
43+
"""
44+
max_tokens = SIMULATION_MAX_TOKENS
45+
if agent_max_tokens and agent_max_tokens > 0:
46+
max_tokens = min(agent_max_tokens, SIMULATION_MAX_TOKENS)
47+
return ModelSettings(model=model, maxTokens=max_tokens)
1148

1249

1350
def simulation_completion_kwargs(
@@ -24,4 +61,5 @@ def simulation_completion_kwargs(
2461
"model",
2562
os.environ.get(SIMULATION_MODEL_ENV) or ChatModels.gpt_4_1_mini_2025_04_14,
2663
)
64+
completion_kwargs.setdefault("max_tokens", SIMULATION_MAX_TOKENS)
2765
return completion_kwargs

‎packages/uipath/src/uipath/eval/mocks/_structured_output.py‎

Lines changed: 104 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,11 @@
2424
RESPONSE_TOOL_NAME = "submit_tool_response"
2525
RESPONSE_KEY = "response"
2626
_DEFS_PREFIX = "#/$defs/"
27+
# Keys whose value maps names to schemas, and keys whose value is data.
28+
_SCHEMA_MAP_KEYS = frozenset(
29+
{"properties", "patternProperties", "$defs", "definitions"}
30+
)
31+
_DATA_KEYS = frozenset({"default", "examples", "const", "enum", "required"})
2732

2833
logger = logging.getLogger(__name__)
2934

@@ -81,6 +86,81 @@ def resolve(node: Any, active: frozenset[str]) -> Any:
8186
return inlined, leftover
8287

8388

89+
def _narrowed_type(branch_type: Any, types: list[str]) -> Any:
90+
"""Return the part of ``branch_type`` that ``types`` allows, or None."""
91+
if branch_type in types or (branch_type == "integer" and "number" in types):
92+
return branch_type
93+
if branch_type == "number" and "integer" in types:
94+
return "integer"
95+
return None
96+
97+
98+
def _restrict_to_types(branches: list[Any], types: list[str]) -> list[Any]:
99+
"""Narrow ``anyOf`` branches to ``types``, for a node that has both.
100+
101+
A branch with its own type is kept, narrowed, only when ``types`` allows
102+
it; a branch without one is copied once per type in ``types``.
103+
"""
104+
restricted: list[Any] = []
105+
for branch in branches:
106+
if not isinstance(branch, dict):
107+
restricted.append(branch)
108+
elif "type" not in branch:
109+
restricted.extend({**branch, "type": t} for t in types)
110+
elif (narrowed := _narrowed_type(branch["type"], types)) is not None:
111+
restricted.append({**branch, "type": narrowed})
112+
# No branch fits the types: keep the branches rather than send an empty anyOf.
113+
return restricted or branches
114+
115+
116+
def _to_gemini_schema(schema: Any) -> Any:
117+
"""Rewrite JSON-schema type lists into the form Gemini accepts.
118+
119+
Gemini's function declarations take a single ``type`` and mark nullable
120+
fields with ``nullable: true``; a type list such as ``["string", "null"]``
121+
(common in MCP tool schemas) fails the request with a 400. So
122+
``{"type": ["string", "null"]}`` becomes
123+
``{"type": "string", "nullable": true}``, and a list with several non-null
124+
types becomes an ``anyOf`` of single types. When the node already has an
125+
``anyOf``, the types are applied to its branches instead, so both
126+
constraints still hold. Values that are data rather than schemas
127+
(``default``, ``enum``, ...) are left untouched.
128+
"""
129+
if isinstance(schema, list):
130+
return [_to_gemini_schema(item) for item in schema]
131+
if not isinstance(schema, dict):
132+
return schema
133+
134+
converted: dict[str, Any] = {}
135+
for key, value in schema.items():
136+
if key in _SCHEMA_MAP_KEYS and isinstance(value, dict):
137+
converted[key] = {
138+
name: _to_gemini_schema(sub) for name, sub in value.items()
139+
}
140+
elif key in _DATA_KEYS:
141+
converted[key] = value
142+
else:
143+
converted[key] = _to_gemini_schema(value)
144+
145+
types = converted.get("type")
146+
if isinstance(types, list):
147+
non_null = [t for t in types if t != "null"]
148+
if len(non_null) == 1:
149+
converted["type"] = non_null[0]
150+
else:
151+
del converted["type"]
152+
if len(non_null) > 1:
153+
branches = converted.get("anyOf")
154+
converted["anyOf"] = (
155+
_restrict_to_types(branches, non_null)
156+
if isinstance(branches, list)
157+
else [{"type": t} for t in non_null]
158+
)
159+
if "null" in types:
160+
converted["nullable"] = True
161+
return converted
162+
163+
84164
def build_response_tool(schema: dict[str, Any], description: str) -> dict[str, Any]:
85165
"""Build a normalized-API function tool that wraps ``schema`` under ``response``.
86166
@@ -223,7 +303,30 @@ class AnthropicStructuredOutput(ToolCallStructuredOutput):
223303

224304

225305
class GeminiStructuredOutput(ToolCallStructuredOutput):
226-
"""Gemini returns empty content for ``response_format``; go straight to tools."""
306+
"""Gemini returns empty content for ``response_format``; go straight to tools.
307+
308+
The tool schema is rewritten into the form Gemini accepts first.
309+
"""
310+
311+
async def generate(
312+
self,
313+
llm: Any,
314+
messages: list[dict[str, str]],
315+
*,
316+
schema: dict[str, Any],
317+
response_format_name: str,
318+
description: str,
319+
completion_kwargs: dict[str, Any],
320+
) -> Any:
321+
"""Force a tool call with a Gemini-compatible schema."""
322+
return await super().generate(
323+
llm,
324+
messages,
325+
schema=_to_gemini_schema(schema),
326+
response_format_name=response_format_name,
327+
description=description,
328+
completion_kwargs=completion_kwargs,
329+
)
227330

228331

229332
def _strategy_for_model(model: str | None) -> ToolCallStructuredOutput:

‎packages/uipath/src/uipath/eval/runtime/runtime.py‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,11 @@
6565
from ..mocks._mock_context import cache_manager_context
6666
from ..mocks._mock_runtime import UiPathMockRuntime
6767
from ..mocks._mocker import UiPathInputMockingError, format_exception_message
68-
from ..mocks._types import LLMMockingStrategy, MockingContext, ModelSettings
68+
from ..mocks._simulation_model import (
69+
get_agent_max_tokens,
70+
simulation_model_settings,
71+
)
72+
from ..mocks._types import LLMMockingStrategy, MockingContext
6973
from ..models import EvaluationResult
7074
from ..models.evaluation_set import (
7175
EvaluationItem,
@@ -662,7 +666,14 @@ async def _execute_eval(
662666
mocking_model = get_agent_model(self.context.runtime_schema)
663667
if mocking_model:
664668
mocking_strategy = mocking_strategy.model_copy(
665-
update={"model": ModelSettings(model=mocking_model)}
669+
update={
670+
"model": simulation_model_settings(
671+
mocking_model,
672+
get_agent_max_tokens(
673+
self.context.runtime_schema
674+
),
675+
)
676+
}
666677
)
667678

668679
agent_execution_output = await self.execute_runtime(
@@ -956,7 +967,12 @@ async def _generate_input_for_eval(
956967
input_generation_model = get_agent_model(self.context.runtime_schema)
957968
if input_generation_model:
958969
input_strategy = input_strategy.model_copy(
959-
update={"model": ModelSettings(model=input_generation_model)}
970+
update={
971+
"model": simulation_model_settings(
972+
input_generation_model,
973+
get_agent_max_tokens(self.context.runtime_schema),
974+
)
975+
}
960976
)
961977

962978
generated_input = await generate_llm_input(

0 commit comments

Comments
 (0)