Added tool dependency metric

Author: adivate2021

src/demo/multi_agent/multi_agent.py

@@ -2,7 +2,7 @@
 from pydantic import BaseModel
 from judgeval.common.tracer import Tracer, wrap
 from judgeval import JudgmentClient
-from judgeval.scorers import ToolOrderScorer
+from judgeval.scorers import ToolOrderScorer, ToolDependencyScorer
 from judgeval.common.tracer import Tracer
 import os
 
@@ -56,12 +56,13 @@ def run_simple_task(self, prompt: str):
         # Create two agents
         alice = self.add_agent("Alice")
         bob = self.add_agent("Bob")
+        charles = self.add_agent("Charles")
 
         # Have them exchange messages
-        alice.send_message("Hello Bob, how are you?", "Bob")
-        bob.send_message("I'm good Alice, thanks for asking!", "Alice")
-        alice.send_message("Great to hear! Let's work together on a task.", "Bob")
 
+        bob.send_message("I'm good Alice, thanks for asking!", "Alice")
+        alice.send_message("Great to hear! What about you, Charles?", "Charles")
+        charles.send_message("I'm good Alice, thanks for asking!", "Alice")
         # Print the conversation
         print("\nAlice's messages:")
         for msg in alice.get_all_messages():
@@ -70,17 +71,29 @@ def run_simple_task(self, prompt: str):
         print("\nBob's messages:")
         for msg in bob.get_all_messages():
             print(f"From {msg.sender}: {msg.content}")
+        
+        print("\nCharles's messages:")
+        for msg in charles.get_all_messages():
+            print(f"From {msg.sender}: {msg.content}")
 
 # Example usage
 if __name__ == "__main__":
     system = MultiAgentSystem()
 
-    test_file = os.path.join(os.path.dirname(__file__), "tests.yaml")
+    # test_file = os.path.join(os.path.dirname(__file__), "tests.yaml")
+    # judgment_client.assert_test(
+    #     scorers=[ToolOrderScorer(threshold=0.5)],
+    #     function=system.run_simple_task,
+    #     tracer=judgment,
+    #     override=True,
+    #     test_file=test_file
+    # )
+
+    test_file2 = os.path.join(os.path.dirname(__file__), "tests2.yaml")
     judgment_client.assert_test(
-        scorers=[ToolOrderScorer(threshold=0.5)],
+        scorers=[ToolDependencyScorer(threshold=0.5)],
         function=system.run_simple_task,
         tracer=judgment,
         override=True,
-        test_file=test_file
+        test_file=test_file2
     )
-

src/demo/multi_agent/tests.yaml

@@ -16,6 +16,8 @@ examples:
         #     name: "Random Tool Agent"
       - tool_name: "send_message"
         agent: Alice
+      - tool_name: "send_message"
+        agent: Charles
         # parameters:
         #   self:
         #     name: "Random Tool Agent"

src/demo/multi_agent/tests2.yaml

@@ -1,7 +1,12 @@
-expected_tools:
-    - tool_name: "send_message"
-      agent: Alice
-    - tool_name: "send_message"
-      agent: Bob
-    - tool_name: "send_message"
-      agent: Alice
+examples:
+  - input:
+      prompt: "Do something random"
+    expected_tools:
+      - tool_name: "send_message"
+        agent: Bob
+        dependencies:
+          - tool_name: "send_message"
+            agent: Alice
+          - tool_name: "send_message"
+            agent: Charles
+        # require_all: true

src/judgeval/constants.py

@@ -28,6 +28,7 @@ class APIScorer(str, Enum):
     GROUNDEDNESS = "groundedness"
     DERAILMENT = "derailment"
     TOOL_ORDER = "tool_order"
+    TOOL_DEPENDENCY = "tool_dependency"
     @classmethod
     def _missing_(cls, value):
         # Handle case-insensitive lookup

src/judgeval/run_evaluation.py

@@ -387,7 +387,6 @@ def run_trace_eval(trace_run: TraceRun, override: bool = False, ignore_errors: b
             trace_run.organization_id,
             True
         )
-
     if function and tracer:
         new_traces: List[Trace] = []
         tracer.offline_mode = True

src/judgeval/scorers/__init__.py

@@ -17,6 +17,7 @@
     GroundednessScorer,
     DerailmentScorer,
     ToolOrderScorer,
+    ToolDependencyScorer,
 )
 from judgeval.scorers.judgeval_scorers.classifiers import (
     Text2SQLScorer,
@@ -43,4 +44,5 @@
     "GroundednessScorer",
     "DerailmentScorer",
     "ToolOrderScorer",
+    "ToolDependencyScorer",
 ]

src/judgeval/scorers/judgeval_scorers/api_scorers/__init__.py

@@ -13,6 +13,7 @@
 from judgeval.scorers.judgeval_scorers.api_scorers.groundedness import GroundednessScorer
 from judgeval.scorers.judgeval_scorers.api_scorers.derailment_scorer import DerailmentScorer
 from judgeval.scorers.judgeval_scorers.api_scorers.tool_order import ToolOrderScorer
+from judgeval.scorers.judgeval_scorers.api_scorers.tool_dependency import ToolDependencyScorer
 __all__ = [
     "ExecutionOrderScorer",
     "JSONCorrectnessScorer",
@@ -29,4 +30,5 @@
     "GroundednessScorer",
     "DerailmentScorer",
     "ToolOrderScorer",
+    "ToolDependencyScorer",
 ]

src/judgeval/scorers/judgeval_scorers/api_scorers/tool_dependency.py

@@ -0,0 +1,18 @@
+"""
+`judgeval` tool dependency scorer
+"""
+
+# Internal imports
+from judgeval.scorers.api_scorer import APIJudgmentScorer
+from judgeval.constants import APIScorer
+
+class ToolDependencyScorer(APIJudgmentScorer):
+    def __init__(self, threshold: float=1.0):
+        super().__init__(
+            threshold=threshold, 
+            score_type=APIScorer.TOOL_DEPENDENCY,
+        )
+
+    @property
+    def __name__(self):
+        return "Tool Dependency"