chore: add JsonSimilarityEvaluator

Author: radu-mocanu

src/uipath/_cli/_evals/_evaluators/__init__.py

@@ -3,18 +3,20 @@
 This package contains all evaluator types and the factory for creating them.
 """
 
-from ._agent_scorer_evaluator import AgentScorerEvaluator
-from ._deterministic_evaluator import DeterministicEvaluator
+from ._deterministic_evaluator_base import DeterministicEvaluatorBase
 from ._evaluator_base import EvaluatorBase
 from ._evaluator_factory import EvaluatorFactory
+from ._exact_match_evaluator import ExactMatchEvaluator
+from ._json_similarity_evaluator import JsonSimilarityEvaluator
 from ._llm_as_judge_evaluator import LlmAsAJudgeEvaluator
 from ._trajectory_evaluator import TrajectoryEvaluator
 
 __all__ = [
     "EvaluatorBase",
+    "DeterministicEvaluatorBase",
     "EvaluatorFactory",
-    "DeterministicEvaluator",
+    "JsonSimilarityEvaluator",
+    "ExactMatchEvaluator",
     "LlmAsAJudgeEvaluator",
-    "AgentScorerEvaluator",
     "TrajectoryEvaluator",
 ]

src/uipath/_cli/_evals/_evaluators/_agent_scorer_evaluator.py

@@ -0,0 +1,46 @@
+import copy
+import json
+from abc import ABC
+from typing import Any, Dict, Tuple
+
+from ._evaluator_base import EvaluatorBase
+
+
+class DeterministicEvaluatorBase(EvaluatorBase, ABC):
+    def __init__(self, target_output_key: str = "*"):
+        super().__init__()
+        self.target_output_key = target_output_key
+
+    def _select_targets(
+        self, expected_output: Dict[str, Any], actual_output: Dict[str, Any]
+    ) -> Tuple[Any, Any]:
+        actual_output_copy = copy.deepcopy(actual_output)
+        expected_output_copy = copy.deepcopy(expected_output)
+        if self.target_output_key != "*":
+            if (
+                self.target_output_key not in actual_output
+                or self.target_output_key not in expected_output
+            ):
+                raise ValueError(
+                    f"Field '{self.target_output_key}' missing from expected or actual output"
+                )
+            actual_output_copy = actual_output_copy[self.target_output_key]
+            expected_output_copy = expected_output[self.target_output_key]
+        return actual_output_copy, expected_output_copy
+
+    def _canonical_json(self, obj: Any) -> str:
+        return json.dumps(
+            self._normalize_numbers(obj),
+            sort_keys=True,
+            separators=(",", ":"),
+            ensure_ascii=False,
+        )
+
+    def _normalize_numbers(self, obj: Any) -> Any:
+        if isinstance(obj, dict):
+            return {k: self._normalize_numbers(v) for k, v in obj.items()}
+        if isinstance(obj, (list, tuple)):
+            return [self._normalize_numbers(v) for v in obj]
+        if isinstance(obj, (int, float)) and not isinstance(obj, bool):
+            return float(obj)
+        return obj

src/uipath/_cli/_evals/_evaluators/_deterministic_evaluator.py

@@ -1,9 +1,9 @@
 from typing import Any, Dict
 
 from .._models import EvaluatorCategory, EvaluatorType
-from ._agent_scorer_evaluator import AgentScorerEvaluator
-from ._deterministic_evaluator import DeterministicEvaluator
 from ._evaluator_base import EvaluatorBase, EvaluatorBaseParams
+from ._exact_match_evaluator import ExactMatchEvaluator
+from ._json_similarity_evaluator import JsonSimilarityEvaluator
 from ._llm_as_judge_evaluator import LlmAsAJudgeEvaluator
 from ._trajectory_evaluator import TrajectoryEvaluator
 
@@ -50,23 +50,47 @@ def create_evaluator(data: Dict[str, Any]) -> EvaluatorBase:
         )
 
         # Create evaluator based on category
-        if category == EvaluatorCategory.Deterministic:
-            return EvaluatorFactory._create_deterministic_evaluator(base_params, data)
-        elif category == EvaluatorCategory.LlmAsAJudge:
-            return EvaluatorFactory._create_llm_as_judge_evaluator(base_params, data)
-        elif category == EvaluatorCategory.AgentScorer:
-            return EvaluatorFactory._create_agent_scorer_evaluator(base_params, data)
-        elif category == EvaluatorCategory.Trajectory:
-            return EvaluatorFactory._create_trajectory_evaluator(base_params, data)
-        else:
-            raise ValueError(f"Unknown evaluator category: {category}")
+        match category:
+            case EvaluatorCategory.Deterministic:
+                if evaluator_type == evaluator_type.Equals:
+                    return EvaluatorFactory._create_exact_match_evaluator(
+                        base_params, data
+                    )
+                elif evaluator_type == evaluator_type.JsonSimilarity:
+                    return EvaluatorFactory._create_json_similarity_evaluator(
+                        base_params, data
+                    )
+                else:
+                    raise ValueError(
+                        f"Unknown evaluator type {evaluator_type} for category {category}"
+                    )
+            case EvaluatorCategory.LlmAsAJudge:
+                return EvaluatorFactory._create_llm_as_judge_evaluator(
+                    base_params, data
+                )
+            case EvaluatorCategory.AgentScorer:
+                raise NotImplementedError()
+            case EvaluatorCategory.Trajectory:
+                return EvaluatorFactory._create_trajectory_evaluator(base_params, data)
+            case _:
+                raise ValueError(f"Unknown evaluator category: {category}")
 
     @staticmethod
-    def _create_deterministic_evaluator(
+    def _create_exact_match_evaluator(
         base_params: EvaluatorBaseParams, data: Dict[str, Any]
-    ) -> DeterministicEvaluator:
+    ) -> ExactMatchEvaluator:
         """Create a deterministic evaluator."""
-        return DeterministicEvaluator.from_params(
+        return ExactMatchEvaluator.from_params(
+            base_params,
+            target_output_key=data.get("targetOutputKey", "*"),
+        )
+
+    @staticmethod
+    def _create_json_similarity_evaluator(
+        base_params: EvaluatorBaseParams, data: Dict[str, Any]
+    ) -> JsonSimilarityEvaluator:
+        """Create a deterministic evaluator."""
+        return JsonSimilarityEvaluator.from_params(
             base_params,
             target_output_key=data.get("targetOutputKey", "*"),
         )
@@ -91,13 +115,6 @@ def _create_llm_as_judge_evaluator(
             target_output_key=data.get("targetOutputKey", "*"),
         )
 
-    @staticmethod
-    def _create_agent_scorer_evaluator(
-        base_params: EvaluatorBaseParams, data: Dict[str, Any]
-    ) -> AgentScorerEvaluator:
-        """Create an agent scorer evaluator."""
-        raise NotImplementedError()
-
     @staticmethod
     def _create_trajectory_evaluator(
         base_params: EvaluatorBaseParams, data: Dict[str, Any]

src/uipath/_cli/_evals/_evaluators/_deterministic_evaluator_base.py

@@ -0,0 +1,40 @@
+import copy
+from typing import Any, Dict
+
+from uipath._cli._evals._evaluators._deterministic_evaluator_base import (
+    DeterministicEvaluatorBase,
+)
+from uipath._cli._evals._models import EvaluationResult
+from uipath._cli._evals._models._evaluators import ScoreType
+
+
+class ExactMatchEvaluator(DeterministicEvaluatorBase):
+    async def evaluate(
+        self,
+        evaluation_id: str,
+        evaluation_name: str,
+        input_data: Dict[str, Any],
+        expected_output: Dict[str, Any],
+        actual_output: Dict[str, Any],
+    ) -> EvaluationResult:
+        actual_output_copy = copy.deepcopy(actual_output)
+        expected_output_copy = copy.deepcopy(expected_output)
+
+        actual_output, expected_output = self._select_targets(
+            expected_output, actual_output
+        )
+        are_equal = self._canonical_json(actual_output) == self._canonical_json(
+            expected_output
+        )
+
+        return EvaluationResult(
+            evaluation_id=evaluation_id,
+            evaluation_name=evaluation_name,
+            evaluator_id=self.id,
+            evaluator_name=self.name,
+            score=are_equal,
+            input=input_data,
+            expected_output=expected_output_copy,
+            actual_output=actual_output_copy,
+            score_type=ScoreType.BOOLEAN,
+        )