formatted

Author: hrshdhgd

src/llm_change_agent/evaluations/evaluator.py

@@ -28,7 +28,6 @@
 logger.info("Evaluating the LLM Change Agent.")
 
 
-
 def download_document(url, input_dir):
     """Download the document from the URL."""
     if not os.path.exists(input_dir):
@@ -174,44 +173,39 @@ def generate_changes_via_llm(eval_dir, output_dir, provider, model):
     print(f"Predicted changes saved to {output_sub_dir}")
 
 
-def compare_changes(expected_dir:Path, output_dir:Path):
+def compare_changes(expected_dir: Path, output_dir: Path):
     """Compare the actual changes with the predicted changes."""
     # For each document in the expected directory, there is a corresponding document in the output directory
 
     output_files = list(output_dir.rglob("*.yaml"))
 
     # output_files_dict is : {provider_model: {filename: file_path}}
-    output_files_list_of_dicts = [
-        {f"{file.parts[-3]}_{file.parts[-2]}": {file.name:file}} for file in output_files
-    ]
-    
+    output_files_list_of_dicts = [{f"{file.parts[-3]}_{file.parts[-2]}": {file.name: file}} for file in output_files]
+
     for model_output in output_files_list_of_dicts:
         for provider_model, file_info in model_output.items():
             for filename, filepath in file_info.items():
                 filename = filepath.name
                 expected_file = expected_dir / filename
                 output_file = filepath
-                with open(expected_file, "r") as ex , open(output_file, "r") as out:
+                with open(expected_file, "r") as ex, open(output_file, "r") as out:
                     expected_yaml = yaml.safe_load(ex)
                     output_yaml = yaml.safe_load(out)
                 expected_yaml_subset = {k: v for k, v in expected_yaml.items() if k in output_yaml}
                 for pr_id, output_changes in output_yaml.items():
                     expected_change = expected_yaml_subset.get(pr_id)
                     if len(output_changes) > 0:
                         compare_output_vs_expected(expected_change, output_changes)
+        logger.info(f"Finished comparing changes for {provider_model}")
 
 
-
-def compare_output_vs_expected(expected_changes, output_changes:List):
+def compare_output_vs_expected(expected_changes, output_changes: List):
     """Compare the expected changes with the output changes."""
     output_changes = normalize_changes(output_changes)
     accuracy = 0.0
     total = len(expected_changes)
     correct = 0
     import pdb; pdb.set_trace()
-    
-        
-
 
 
 def run_evaluate(model: str, provider: str):
@@ -230,4 +224,3 @@ def run_evaluate(model: str, provider: str):
     generate_changes_via_llm(model=model, provider=provider, eval_dir=eval_dir, output_dir=output_dir)
 
     compare_changes(expected_dir=expected_dir, output_dir=output_dir)
-

src/llm_change_agent/utils/general_utils.py

@@ -46,4 +46,4 @@ def jaccard_similarity(statement1, statement2):
     union = set1.union(set2)
 
     # Calculate the Jaccard similarity coefficient
-    return len(intersection) / len(union)
+    return len(intersection) / len(union)

src/llm_change_agent/utils/llm_utils.py

@@ -10,10 +10,10 @@
 from langchain.agents import AgentExecutor
 from langchain.agents.react.agent import create_react_agent
 from langchain.tools.retriever import create_retriever_tool
-from langchain_core.tools import tool
 from langchain_chroma import Chroma
 from langchain_community.document_loaders import WebBaseLoader
 from langchain_core.documents import Document
+from langchain_core.tools import tool
 from langchain_openai import OpenAIEmbeddings
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from openai import OpenAI
@@ -319,21 +319,24 @@ def extract_commands(command):
     else:
         return cleaned_command
 
+
 def normalize_changes(changes):
+    """Convert IRIs to CURIEs in change statements."""
     for idx, change in enumerate(changes):
         if any(string.startswith("<http") or string.startswith("http") for string in change.split()):
-            iri = [string for string in change.split() if string.startswith("<http")or string.startswith("http")]
+            iri = [string for string in change.split() if string.startswith("<http") or string.startswith("http")]
             # Replace the strings in the list with the curie using converter.compress(item)
             for _, item in enumerate(iri):
-                stripped_item = item.strip('<>')
+                stripped_item = item.strip("<>")
                 compressed_item = compress_iri(stripped_item) if compress_iri(stripped_item) else item
                 # Update the original change list with the compressed item
                 change = change.replace(item, compressed_item)
                 changes[idx] = change
     return changes
 
+
 @tool
 def compress_iri(iri: str) -> str:
     """Compress the IRI."""
     converter = curies.get_obo_converter()
-    return converter.compress(iri)
+    return converter.compress(iri)