Skip to content

Ahh/new error format #273

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
May 26, 2025
Merged

Ahh/new error format #273

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4d8484e
feat: start of new record_error
abhishekg999 May 21, 2025
b93b65f
feat: error property on TraceSpan
abhishekg999 May 22, 2025
0adb534
chore(docs): update observe also records Error
abhishekg999 May 24, 2025
1918bf7
chore: cleanup capture exception impl
abhishekg999 May 24, 2025
781644f
chore: better comment on _serialize_value
abhishekg999 May 24, 2025
c973649
Merge branch 'main' into ahh/new-error-format
Mandolaro May 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 59 additions & 9 deletions src/judgeval/common/tracer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,15 @@ def record_output(self, output: Any):
return span # Return the created entry
# Removed else block - original didn't have one
return None # Return None if no span_id found


def record_error(self, error: Any):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

The new record_error method is a good addition for specifically handling errors within a span. Could we add a docstring to explain its purpose, parameters, and what it returns? This would improve code clarity and maintainability for other developers.

current_span_id = current_span_var.get()
if current_span_id:
span = self.span_id_to_span[current_span_id]
span.error = error
return span
return None

def add_span(self, span: TraceSpan):
"""Add a trace span to this trace context"""
self.trace_spans.append(span)
Expand Down Expand Up @@ -876,9 +884,7 @@ def _trace(self, frame: types.FrameType, event: str, arg: Any):
"traceback": traceback.format_tb(exc_traceback)
}
current_trace = current_trace_var.get()
current_trace.record_output({
"error": formatted_exception
})
current_trace.record_error(formatted_exception)

return self._trace

Expand Down Expand Up @@ -1154,8 +1160,19 @@ async def async_wrapper(*args, **kwargs):
with _DeepTracer():
result = await func(*args, **kwargs)
else:
result = await func(*args, **kwargs)

try:
result = await func(*args, **kwargs)
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
formatted_exception = {
"type": exc_type.__name__,
"message": str(exc_value),
"traceback": traceback.format_tb(exc_traceback)
}
current_trace = current_trace_var.get()
current_trace.record_error(formatted_exception)
raise e

# Record output
span.record_output(result)
return result
Expand All @@ -1175,7 +1192,18 @@ async def async_wrapper(*args, **kwargs):
with _DeepTracer():
result = await func(*args, **kwargs)
else:
result = await func(*args, **kwargs)
try:
result = await func(*args, **kwargs)
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
formatted_exception = {
"type": exc_type.__name__,
"message": str(exc_value),
"traceback": traceback.format_tb(exc_traceback)
}
current_trace = current_trace_var.get()
current_trace.record_error(formatted_exception)
raise e

span.record_output(result)
return result
Expand Down Expand Up @@ -1221,7 +1249,18 @@ def wrapper(*args, **kwargs):
with _DeepTracer():
result = func(*args, **kwargs)
else:
result = func(*args, **kwargs)
try:
result = func(*args, **kwargs)
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
formatted_exception = {
"type": exc_type.__name__,
"message": str(exc_value),
"traceback": traceback.format_tb(exc_traceback)
}
current_trace = current_trace_var.get()
current_trace.record_error(formatted_exception)
raise e

# Record output
span.record_output(result)
Expand All @@ -1243,7 +1282,18 @@ def wrapper(*args, **kwargs):
with _DeepTracer():
result = func(*args, **kwargs)
else:
result = func(*args, **kwargs)
try:
result = func(*args, **kwargs)
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
formatted_exception = {
"type": exc_type.__name__,
"message": str(exc_value),
"traceback": traceback.format_tb(exc_traceback)
}
current_trace = current_trace_var.get()
current_trace.record_error(formatted_exception)
raise e

span.record_output(result)
return result
Expand Down
39 changes: 8 additions & 31 deletions src/judgeval/data/trace.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ class TraceSpan(BaseModel):
parent_span_id: Optional[str] = None
span_type: Optional[str] = "span"
inputs: Optional[Dict[str, Any]] = None
error: Optional[Dict[str, Any]] = None
output: Optional[Any] = None
duration: Optional[float] = None
annotation: Optional[List[Dict[str, Any]]] = None
Expand All @@ -26,10 +27,10 @@ def model_dump(self, **kwargs):
"span_id": self.span_id,
"trace_id": self.trace_id,
"depth": self.depth,
# "created_at": datetime.fromtimestamp(self.created_at).isoformat(),
"created_at": datetime.fromtimestamp(self.created_at, tz=timezone.utc).isoformat(),
"inputs": self._serialize_inputs(),
"output": self._serialize_output(),
"inputs": self._serialize_value(self.inputs),
"output": self._serialize_value(self.output),
"error": self._serialize_value(self.error),
"evaluation_runs": [run.model_dump() for run in self.evaluation_runs] if self.evaluation_runs else [],
"parent_span_id": self.parent_span_id,
"function": self.function,
Expand All @@ -42,30 +43,6 @@ def print_span(self):
indent = " " * self.depth
parent_info = f" (parent_id: {self.parent_span_id})" if self.parent_span_id else ""
print(f"{indent}→ {self.function} (id: {self.span_id}){parent_info}")

def _serialize_inputs(self) -> dict:
"""Helper method to serialize input data safely."""
if self.inputs is None:
return {}

serialized_inputs = {}
for key, value in self.inputs.items():
if isinstance(value, BaseModel):
serialized_inputs[key] = value.model_dump()
elif isinstance(value, (list, tuple)):
# Handle lists/tuples of arguments
serialized_inputs[key] = [
item.model_dump() if isinstance(item, BaseModel)
else None if not self._is_json_serializable(item)
else item
for item in value
]
else:
if self._is_json_serializable(value):
serialized_inputs[key] = value
else:
serialized_inputs[key] = self.safe_stringify(value, self.function)
return serialized_inputs

def _is_json_serializable(self, obj: Any) -> bool:
"""Helper method to check if an object is JSON serializable."""
Expand Down Expand Up @@ -94,9 +71,9 @@ def safe_stringify(self, output, function_name):
)
return None

def _serialize_output(self) -> Any:
def _serialize_value(self, value: Any) -> Any:
"""Helper method to serialize output data safely."""
if self.output is None:
if value is None:
return None

def serialize_value(value):
Expand All @@ -117,8 +94,8 @@ def serialize_value(value):
# Fallback to safe stringification
return self.safe_stringify(value, self.function)

# Start serialization with the top-level output
return serialize_value(self.output)
# Start serialization with the top-level value
return serialize_value(value)

class Trace(BaseModel):
trace_id: str
Expand Down
Loading