Merge remote-tracking branch 'origin/az-sequence-to-trace' into add_agent_names

Author: adivate2021

README.md

@@ -17,6 +17,7 @@ Judgeval is an open-source tool for testing, monitoring, and optimizing AI agent
 **🔍 Tracing**
 * Automatic agent tracing for common agent frameworks and SDKs (LangGraph, OpenAI, Anthropic, etc.)
 * Track input/output, latency, cost, token usage at every step
+    * Granular cost tracking per customer/per task
 * Function tracing with `@judgment.observe` decorator
 
 **🧪 Evals**

docs/api_reference/judgment_client.mdx

@@ -52,7 +52,7 @@ example = Example(
 results = client.run_evaluation(
     examples=[example],
     scorers=[FaithfulnessScorer(threshold=0.5)],
-    model="gpt-4o",
+    model="gpt-4.1",
 )
 ```
 ```Typescript Typescript
@@ -70,7 +70,7 @@ async function runEval() {
     const results = await client.evaluate({
         examples: [example],
         scorers: [new FaithfulnessScorer(0.5)],
-        model: "gpt-4o",
+        model: "gpt-4.1",
         projectName: "client-api-ref-proj", // Optional: Provide a project name
         evalName: "client-api-ref-eval"   // Optional: Provide an eval name
     });
@@ -83,7 +83,7 @@ runEval();
 
 The `run_evaluation` (Python) / `evaluate` (Typescript) method accepts the following arguments/options:
 - `examples`: A list/array of [Example](/evaluation/data_examples) objects to evaluate.
-- `model`: The model to use for the evaluation, such as `gpt-4o` or `Qwen/Qwen2.5-72B-Instruct-Turbo`.
+- `model`: The model to use for the evaluation, such as `gpt-4.1` or `Qwen/Qwen2.5-72B-Instruct-Turbo`.
 - `scorers`: A list/array of [Scorer](/evaluation/scorers) objects to use for the evaluation.
 - `log_results` (Python) / `logResults` (Typescript): Whether to log the results of the evaluation to the Judgment platform. Defaults to `true`.
 - `override`: Whether to override an existing evaluation with the same name. Defaults to `false`.
@@ -135,7 +135,7 @@ airline_sequence = Sequence(
 results = client.run_sequence_evaluation(
     sequences=[airline_sequence],
     scorers=[DerailmentScorer(threshold=0.5)],
-    model="gpt-4o",
+    model="gpt-4.1",
     log_results=True,
     override=True,
 )

docs/evaluation/data_datasets.mdx

@@ -253,7 +253,7 @@ from judgeval.scorers import FaithfulnessScorer # Added import
 res = client.run_evaluation(
     examples=dataset.examples,
     scorers=[FaithfulnessScorer(threshold=0.9)],
-    model="gpt-4o",
+    model="gpt-4.1",
 )
 ```
 ```Typescript Typescript
@@ -270,7 +270,7 @@ const dataset: Example[] = [
 const results = await client.evaluate({
     examples: dataset,
     scorers: [new FaithfulnessScorer(0.9)],
-    model: "gpt-4o",
+    model: "gpt-4.1",
     projectName: "dataset-eval-ts-proj",
     evalName: "dataset-eval-ts-run"
 });

docs/evaluation/introduction.mdx

@@ -70,14 +70,14 @@ faithfulness_scorer = FaithfulnessScorer(threshold=0.5)
 results = client.run_evaluation(
     examples=[example],
     scorers=[faithfulness_scorer],
-    model="gpt-4o",
+    model="gpt-4.1",
 )
 
 # You also run evaluations asynchronously like so:
 results = client.a_run_evaluation(
     examples=[example],
     scorers=[faithfulness_scorer],
-    model="gpt-4o",
+    model="gpt-4.1",
 )
 print(results)
 ```
@@ -102,7 +102,7 @@ const faithfulnessScorer = new FaithfulnessScorer(0.5);
 const results = await client.evaluate({
   examples: [example],
   scorers: [faithfulnessScorer],
-  model: "gpt-4o",
+  model: "gpt-4.1",
   projectName: "my-intro-project",
   evalName: "intro-evaluation-run"
 });

docs/evaluation/judges.mdx

@@ -15,7 +15,7 @@ Both `judgeval` (Python) and `judgeval-js` (TypeScript) support OpenAI models (l
 
 In Python, this is handled via LiteLLM integration. In TypeScript, the built-in `DefaultJudge` is used.
 
-You simply pass the model name (e.g., "gpt-4o") to the `model` parameter in your evaluation call:
+You simply pass the model name (e.g., "gpt-4.1") to the `model` parameter in your evaluation call:
 
 <CodeGroup>
 ```Python Python
@@ -29,7 +29,7 @@ example1 = Example(input="Q1", actual_output="A1")
 results = client.run_evaluation(
     examples=[example1],
     scorers=[AnswerRelevancyScorer(threshold=0.5)],
-    model="gpt-4o"  # Uses LiteLLM
+    model="gpt-4.1"  # Uses LiteLLM
 )
 ```
 ```Typescript Typescript
@@ -42,7 +42,7 @@ async function runOpenAIJudge() {
     const results = await client.evaluate({
         examples: [example1],
         scorers: [new AnswerRelevancyScorer(0.5)],
-        model: "gpt-4o", // Uses DefaultJudge internally
+        model: "gpt-4.1", // Uses DefaultJudge internally
         projectName: "openai-judge-ts-proj",
         evalName: "openai-judge-ts-eval"
     });
@@ -205,5 +205,5 @@ useCustomJudge();
 </CodeGroup>
 
 <Note>
-When providing a custom judge instance (like `VertexAIJudge` in Python or `MyCustomJudge` in TypeScript), pass the instance directly to the `model` parameter (Python) or the `judge` option (TypeScript) in the evaluation call. The built-in judges (`DefaultJudge`, `TogetherJudge`) are used automatically when you pass a model *name* string (like "gpt-4o" or "meta-llama/...") to the `model` option in TypeScript.
+When providing a custom judge instance (like `VertexAIJudge` in Python or `MyCustomJudge` in TypeScript), pass the instance directly to the `model` parameter (Python) or the `judge` option (TypeScript) in the evaluation call. The built-in judges (`DefaultJudge`, `TogetherJudge`) are used automatically when you pass a model *name* string (like "gpt-4.1" or "meta-llama/...") to the `model` option in TypeScript.
 </Note>

docs/evaluation/scorers/agent/derailment.mdx

@@ -43,7 +43,7 @@ airline_sequence = Sequence(
 results = client.run_sequence_evaluation(
     sequences=[airline_sequence],
     scorers=[DerailmentScorer(threshold=0.5)],
-    model="gpt-4o",
+    model="gpt-4.1",
     log_results=True,
     override=True,
 )

docs/evaluation/scorers/classifier_scorer.mdx

@@ -70,7 +70,7 @@ They can also be run in conjunction with other scorers in a single evaluation ru
 results = client.run_evaluation(
     examples=[example1],
     scorers=[friendliness_scorer],
-    model="gpt-4o"
+    model="gpt-4.1"
 )
 ```
 

docs/evaluation/scorers/custom_scorers.mdx

@@ -318,7 +318,7 @@ sample_scorer = SampleScorer()
 results = client.run_evaluation(
     examples=[example1],
     scorers=[sample_scorer],
-    model="gpt-4o"
+    model="gpt-4.1"
 )
 ```
 ## Custom Scorers with Custom Examples
@@ -350,7 +350,7 @@ scorer = CustomScorer(threshold=0.5) # Your custom scorer
 results = client.run_evaluation(
     examples=[custom_example],
     scorers=[scorer],
-    model="gpt-4o-mini",
+    model="gpt-4.1-mini",
 )
 ```
 

docs/evaluation/scorers/default/answer_correctness.mdx

@@ -46,7 +46,7 @@ scorer = AnswerCorrectnessScorer(threshold=0.8)
 results = client.run_evaluation(
     examples=[example],
     scorers=[scorer],
-    model="gpt-4o",
+    model="gpt-4.1",
 )
 print(results)
 ```
@@ -69,7 +69,7 @@ async function runAnswerCorrectness() {
     const results = await client.evaluate({
         examples: [example],
         scorers: [scorer],
-        model: "gpt-4o",
+        model: "gpt-4.1",
         projectName: "ans-correct-ts-proj",
         evalName: "ans-correct-ts-eval"
     });

docs/evaluation/scorers/default/answer_relevancy.mdx

@@ -47,7 +47,7 @@ scorer = AnswerRelevancyScorer(threshold=0.8)
 results = client.run_evaluation(
     examples=[example],
     scorers=[scorer],
-    model="gpt-4o",
+    model="gpt-4.1",
 )
 print(results)
 ```
@@ -68,7 +68,7 @@ async function runAnswerRelevancy() {
     const results = await client.evaluate({
         examples: [example],
         scorers: [scorer],
-        model: "gpt-4o",
+        model: "gpt-4.1",
         projectName: "ans-relevancy-ts-proj",
         evalName: "ans-relevancy-ts-eval"
     });