diff --git a/judgeval/common/telemetry.py b/judgeval/common/telemetry.py deleted file mode 100644 index 22fd05db..00000000 --- a/judgeval/common/telemetry.py +++ /dev/null @@ -1,123 +0,0 @@ -from contextlib import contextmanager -import logging -import os -import socket -import sys -import uuid -import sentry_sdk -from opentelemetry import trace -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( - OTLPSpanExporter, -) - - -def get_unique_id(): - unique_id = os.getenv("judgeval_UNIQUE_ID") - if unique_id is None: - unique_id = str(uuid.uuid4()) - os.environ["judgeval_UNIQUE_ID"] = unique_id - return unique_id - - -def telemetry_opt_out(): - return os.getenv("judgeval_TELEMETRY_OPT_OUT") == "YES" - - -def blocked_by_firewall(): - try: - socket.create_connection(("www.google.com", 80)) - return False - except OSError: - return True - - -if not telemetry_opt_out(): - sentry_sdk.init( - dsn="https://5ef587d58109ee45d6544f3657efdd1f@o4506098477236224.ingest.sentry.io/4506098479136768", - profiles_sample_rate=1.0, - traces_sample_rate=1.0, # For performance monitoring - send_default_pii=False, # Don't send personally identifiable information - attach_stacktrace=False, # Don't attach stack traces to messages - default_integrations=False, # Disable Sentry's default integrations - ) - - # Set up the Tracer Provider - trace.set_tracer_provider(TracerProvider()) - tracer_provider = trace.get_tracer_provider() - - # New Relic License Key and OTLP Endpoint - NEW_RELIC_LICENSE_KEY = "1711c684db8a30361a7edb0d0398772cFFFFNRAL" - NEW_RELIC_OTLP_ENDPOINT = "https://otlp.nr-data.net:4317" - otlp_exporter = OTLPSpanExporter( - endpoint=NEW_RELIC_OTLP_ENDPOINT, - headers={"api-key": NEW_RELIC_LICENSE_KEY}, - ) - - # Add the OTLP exporter to the span processor - span_processor = BatchSpanProcessor(otlp_exporter) - tracer_provider.add_span_processor(span_processor) - - logging.getLogger("opentelemetry.exporter.otlp").setLevel(logging.CRITICAL) - - # Create a tracer for your application - tracer = trace.get_tracer(__name__) - - -if ( - os.getenv("ERROR_REPORTING") == "YES" - and not blocked_by_firewall() - and not os.getenv("TELEMETRY_OPT_OUT") -): - - def handle_exception(exc_type, exc_value, exc_traceback): - print({"exc_type": exc_type, "exc_value": exc_value}) - sentry_sdk.capture_exception(exc_value) - sys.__excepthook__(exc_type, exc_value, exc_traceback) - - sys.excepthook = handle_exception - - -@contextmanager -def capture_evaluation_run(type: str): - if not telemetry_opt_out(): - with tracer.start_as_current_span(f"Evaluation run: {type}") as span: - span.set_attribute("user.unique_id", get_unique_id()) - yield span - else: - yield - - -@contextmanager -def capture_metric_type(metric_name: str, _track: bool = True): - if not telemetry_opt_out() and _track: - with tracer.start_as_current_span(metric_name) as span: - span.set_attribute("user.unique_id", get_unique_id()) - yield span - else: - yield - - -@contextmanager -def capture_synthesizer_run(max_generations: int = None, method: str = None): - if not telemetry_opt_out() and max_generations is not None: - with tracer.start_as_current_span( - f"Invoked synthesizer ({max_generations}) | Method: {method}" - ) as span: - span.set_attribute("user.unique_id", get_unique_id()) - yield span - else: - yield - - -@contextmanager -def capture_red_teamer_run(task: str): - if not telemetry_opt_out(): - with tracer.start_as_current_span( - f"Invoked red teamer: ({task})" - ) as span: - span.set_attribute("user.unique_id", get_unique_id()) - yield span - else: - yield diff --git a/judgeval/playground.py b/judgeval/playground.py index c5d065c6..19db5809 100644 --- a/judgeval/playground.py +++ b/judgeval/playground.py @@ -15,7 +15,6 @@ from judgeval.judges.utils import create_judge from judgeval.scorers.custom_scorer import CustomScorer from judgeval.scorers.score import * -from judgeval.common.telemetry import capture_metric_type """ Testing implementation of CustomFaithfulness @@ -195,22 +194,21 @@ def metric_progress_indicator( total: int = 9999, transient: bool = True, ): - with capture_metric_type(metric.__name__): - console = Console(file=sys.stderr) # Direct output to standard error - if _show_indicator: - with Progress( - SpinnerColumn(style="rgb(106,0,255)"), - TextColumn("[progress.description]{task.description}"), - console=console, # Use the custom console - transient=transient, - ) as progress: - progress.add_task( - description=scorer_console_msg(metric, async_mode), - total=total, - ) - yield - else: + console = Console(file=sys.stderr) # Direct output to standard error + if _show_indicator: + with Progress( + SpinnerColumn(style="rgb(106,0,255)"), + TextColumn("[progress.description]{task.description}"), + console=console, # Use the custom console + transient=transient, + ) as progress: + progress.add_task( + description=scorer_console_msg(metric, async_mode), + total=total, + ) yield + else: + yield def prettify_list(lst: List[Any]): diff --git a/judgeval/scorers/score.py b/judgeval/scorers/score.py index b16352e8..6e2f7f0d 100644 --- a/judgeval/scorers/score.py +++ b/judgeval/scorers/score.py @@ -18,7 +18,6 @@ ) from judgeval.scorers import CustomScorer from judgeval.scorers.utils import clone_scorers, scorer_console_msg -from judgeval.common.telemetry import capture_evaluation_run from judgeval.common.exceptions import MissingTestCaseParamsError from judgeval.common.logger import example_logging_context, debug, error, warning, info from judgeval.judges import judgevalJudge @@ -312,36 +311,10 @@ async def execute_with_semaphore(func: Callable, *args, **kwargs): debug(f"Scorer threshold: {scorer.threshold}") if hasattr(scorer, 'model'): debug(f"Scorer model: {type(scorer.model).__name__}") - with capture_evaluation_run("Example"): - if isinstance(ex, Example): - if len(scorers) == 0: - pbar.update(1) - continue - - cloned_scorers: List[CustomScorer] = clone_scorers( - scorers - ) - task = execute_with_semaphore( - func=a_eval_examples_helper, - scorers=cloned_scorers, - example=ex, - scoring_results=scoring_results, - score_index=i, - ignore_errors=ignore_errors, - skip_on_missing_params=skip_on_missing_params, - show_indicator=show_indicator, - _use_bar_indicator=_use_bar_indicator, - pbar=pbar, - ) - tasks.append(asyncio.create_task(task)) - - await asyncio.sleep(throttle_value) - await asyncio.gather(*tasks) - else: - for i, ex in enumerate(examples): - with capture_evaluation_run("Example"): + if isinstance(ex, Example): if len(scorers) == 0: + pbar.update(1) continue cloned_scorers: List[CustomScorer] = clone_scorers( @@ -355,12 +328,37 @@ async def execute_with_semaphore(func: Callable, *args, **kwargs): score_index=i, ignore_errors=ignore_errors, skip_on_missing_params=skip_on_missing_params, - _use_bar_indicator=_use_bar_indicator, show_indicator=show_indicator, + _use_bar_indicator=_use_bar_indicator, + pbar=pbar, ) - tasks.append(asyncio.create_task((task))) + tasks.append(asyncio.create_task(task)) await asyncio.sleep(throttle_value) + await asyncio.gather(*tasks) + else: + for i, ex in enumerate(examples): + if isinstance(ex, Example): + if len(scorers) == 0: + continue + + cloned_scorers: List[CustomScorer] = clone_scorers( + scorers + ) + task = execute_with_semaphore( + func=a_eval_examples_helper, + scorers=cloned_scorers, + example=ex, + scoring_results=scoring_results, + score_index=i, + ignore_errors=ignore_errors, + skip_on_missing_params=skip_on_missing_params, + _use_bar_indicator=_use_bar_indicator, + show_indicator=show_indicator, + ) + tasks.append(asyncio.create_task((task))) + + await asyncio.sleep(throttle_value) await asyncio.gather(*tasks) return scoring_results