diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index 2e5404478..ec813ed17 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath" -version = "2.13.13" +version = "2.13.14" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py index 67aaee0c3..8cb8da0de 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py @@ -1,6 +1,7 @@ """Helper functions for legacy LLM evaluators using function calling.""" import logging +import math from typing import Any from uipath.platform.chat.llm_gateway import ( @@ -89,9 +90,32 @@ def extract_tool_call_response(response: Any, model: str) -> LLMResponse: logger.debug(f"Arguments: {arguments}") raise ValueError(error_msg) - score = float(arguments["score"]) + try: + score = float(arguments["score"]) + except (ValueError, TypeError) as e: + error_msg = ( + f"Non-numeric score {arguments['score']!r} in tool call arguments " + f"from model {model}: expected a number between 0 and 100" + ) + logger.error(f"❌ {error_msg}") + logger.debug(f"Arguments: {arguments}") + raise ValueError(error_msg) from e + justification = str(arguments["justification"]) + # Models occasionally emit corrupted numeric tool arguments despite the + # 0-100 range stated in the tool schema (e.g. gemini-2.5-flash returning + # 989898 or 950). Unvalidated, such a value poisons every run-level + # aggregate downstream, so reject it and let the evaluation surface as + # an error instead of recording a fabricated score. + if not math.isfinite(score) or score < 0.0 or score > 100.0: + error_msg = ( + f"Invalid score {score!r} in tool call arguments from model {model}: " + f"expected a number between 0 and 100" + ) + logger.error(f"❌ {error_msg}") + raise ValueError(error_msg) + logger.debug( f"✅ Extracted score: {score}, justification length: {len(justification)} chars" ) diff --git a/packages/uipath/tests/evaluators/test_legacy_llm_helpers.py b/packages/uipath/tests/evaluators/test_legacy_llm_helpers.py new file mode 100644 index 000000000..c44f8d855 --- /dev/null +++ b/packages/uipath/tests/evaluators/test_legacy_llm_helpers.py @@ -0,0 +1,84 @@ +"""Tests for legacy LLM helper functions (submit_evaluation tool-call parsing).""" + +from types import SimpleNamespace +from typing import Any + +import pytest + +from uipath.eval.evaluators.legacy_llm_helpers import extract_tool_call_response + + +def _make_response(arguments: dict[str, Any]) -> Any: + """Build a minimal chat-completions response carrying a submit_evaluation tool call.""" + tool_call = SimpleNamespace(arguments=arguments) + message = SimpleNamespace(tool_calls=[tool_call]) + choice = SimpleNamespace(message=message) + return SimpleNamespace(choices=[choice]) + + +class TestExtractToolCallResponse: + """Test extract_tool_call_response score validation.""" + + def test_valid_score_passes_through(self) -> None: + response = _make_response({"score": 88, "justification": "ok"}) + + result = extract_tool_call_response(response, "gemini-2.5-flash") + + assert result.score == 88.0 + assert result.justification == "ok" + + def test_out_of_range_score_is_rejected(self) -> None: + # Real payload observed in production: gemini-2.5-flash returned + # score=989898 in its submit_evaluation tool call while the justification + # said the outputs "match perfectly". Unvalidated, this single value blew + # a 64-item run-level average up to 15559.13%. The evaluation must surface + # as an error rather than record a fabricated score. + response = _make_response( + {"score": 989898, "justification": "matches perfectly"} + ) + + with pytest.raises(ValueError, match="Invalid score 989898"): + extract_tool_call_response(response, "gemini-2.5-flash") + + def test_out_of_range_950_is_rejected(self) -> None: + # Second production occurrence from the same eval run: score=950 + # (the model most likely intended 95). + response = _make_response({"score": 950, "justification": "equivalent"}) + + with pytest.raises(ValueError, match="Invalid score 950"): + extract_tool_call_response(response, "gemini-2.5-flash") + + def test_negative_score_is_rejected(self) -> None: + response = _make_response({"score": -5, "justification": "bad"}) + + with pytest.raises(ValueError, match="Invalid score -5"): + extract_tool_call_response(response, "gpt-4o") + + @pytest.mark.parametrize("boundary", [0, 100]) + def test_boundary_scores_accepted(self, boundary: int) -> None: + response = _make_response({"score": boundary, "justification": "j"}) + + result = extract_tool_call_response(response, "m") + + assert result.score == float(boundary) + assert result.justification == "j" + + @pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) + def test_non_finite_score_is_rejected(self, value: float) -> None: + response = _make_response({"score": value, "justification": "j"}) + + with pytest.raises(ValueError, match="Invalid score"): + extract_tool_call_response(response, "m") + + def test_missing_score_raises(self) -> None: + response = _make_response({"justification": "j"}) + + with pytest.raises(ValueError, match="Missing 'score'"): + extract_tool_call_response(response, "m") + + @pytest.mark.parametrize("value", ["not-a-number", None, [95]]) + def test_non_numeric_score_is_rejected(self, value: Any) -> None: + response = _make_response({"score": value, "justification": "j"}) + + with pytest.raises(ValueError, match="Non-numeric score"): + extract_tool_call_response(response, "m") diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index 1314ef227..cca93a5f9 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2598,7 +2598,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.13.13" +version = "2.13.14" source = { editable = "." } dependencies = [ { name = "applicationinsights" },