refactor: streamline configuration variable access

- Replace YAML-based configuration handling with pydantic-settings and TOML for enhanced reliability and ease of use.
- Refactor file path operations to use pathlib.Path, adopting a more modern and object-oriented approach.
- Rebuild the CLI entry point using Click to improve maintainability and user experience.
- Version bumped to 0.1.0 to mark the significant updates in configuration management and CLI design.

Author: Umpire2018

.gitignore

@@ -1,6 +1,6 @@
 # 忽略私人配置文件，包含有api_key等信息
-config private.yml
-private.yml
+config private.toml
+private.toml
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -181,4 +181,5 @@ prompt_output/
 log.txt
 .chroma_db
 config.yml
+config.toml
 .pre-commit-config.yaml

README.md

@@ -104,7 +104,6 @@ repo_path: /path/to/your/repo
 project_hierarchy: .project_hierarchy # This is a folder, where we store the project hierarchy and metainfo. This can be shared with your team members.
 Markdown_Docs_folder: Markdown_Docs # The folder in the root directory of your target repository to store the documentation.
 ignore_list: ["ignore_file1.py", "ignore_file2.py", "ignore_directory"] # Ignore some py files or folders that you don't want to generate documentation for by giving relative paths in ignore_list.
-whitelist_path: /path/of/whitelist_path_json #if you provide the whitelist json with the same structure in Metainfo, RepoAgent will only process the given part. This is useful in a very big project, like "higgingface Transformers"
 
 language: en # Two-letter language codes (ISO 639-1 codes), e.g. `language: en` for English. Refer to Supported Language for more languages.
 max_thread_count: 10 # We support multiprocessing to speedup the process

config.yml.template

@@ -3,7 +3,7 @@ authors = [
     {name = "Qinyu Luo", email = "qinyuluo123@gmail.com"},
 ]
 maintainers = [
-    {name = "Arno Edwards", email = "Arno.Edwards@outlook.com"},
+    {name = "Edwards Arno", email = "Edwards.Arno@outlook.com"},
 ]
 license = {text = "Apache-2.0"}
 requires-python = ">=3.10,<4.0"
@@ -24,15 +24,15 @@ dependencies = [
     "tomli-w>=1.0.0",
 ]
 name = "repoagent"
-version = "0.0.6"
+version = "0.1.0"
 description = "An Agent designed to offer an intelligent approach for generating project documentation."
 readme = "README.md"
 
 [project.urls]
 repository = "https://github.com/LOGIC-10/RepoAgent"
 
 [project.scripts]
-repoagent = "repo_agent.main:app"
+repoagent = "repo_agent.main:cli"
 
 [tool.pdm]
 [tool.pdm.dev-dependencies]

pdm.lock

@@ -1,38 +1,4 @@
-import os
-import sys
+from .main import cli
 
-os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))   #修改当前工作目录
-
-from repo_agent.runner import Runner, delete_fake_files
-from repo_agent.log import logger
-from repo_agent.config import CONFIG
-from repo_agent.doc_meta_info import MetaInfo, DocItem
-from repo_agent.utils.meta_info_utils import  make_fake_files, delete_fake_files
-
-if len(sys.argv) == 1:
-    runner = Runner()
-    runner.run()
-    logger.info("文档任务完成。")
-elif len(sys.argv) == 2:
-    if sys.argv[1] == "clean":
-        delete_fake_files()
-    elif sys.argv[1] == "print":
-        runner = Runner()
-        runner.meta_info.target_repo_hierarchical_tree.print_recursive()
-    elif sys.argv[1] == "diff":
-        runner = Runner()
-        if runner.meta_info.in_generation_process: # 如果不是在生成过程中，就开始检测变更
-            print("this command only support pre-check")
-            exit()
-        file_path_reflections, jump_files = make_fake_files()
-        new_meta_info = MetaInfo.init_meta_info(file_path_reflections, jump_files)
-        new_meta_info.load_doc_from_older_meta(runner.meta_info)
-        delete_fake_files()
-
-        ignore_list = CONFIG.get("ignore_list", [])
-        DocItem.check_has_task(new_meta_info.target_repo_hierarchical_tree, ignore_list)
-        if new_meta_info.target_repo_hierarchical_tree.has_task:
-            print("the following docs will be generated/updated:")
-            new_meta_info.target_repo_hierarchical_tree.print_recursive(diff_status = True, ignore_list = ignore_list)
-        else:
-            print("no docs will be generated/updated, check your source-code update")
+if __name__ == "__main__":
+    cli()

pyproject.toml

@@ -1,10 +1,12 @@
-import git
-import re, os
+import os
+import re
 import subprocess
+
+import git
 from colorama import Fore, Style
 
-from repo_agent.config import CONFIG
 from repo_agent.file_handler import FileHandler
+from repo_agent.settings import setting
 
 
 class ChangeDetector:
@@ -166,7 +168,7 @@ def get_to_be_staged_files(self):
         print(f"{Fore.LIGHTYELLOW_EX}target_repo_path{Style.RESET_ALL}: {self.repo_path}")
         print(f"{Fore.LIGHTMAGENTA_EX}already_staged_files{Style.RESET_ALL}:{staged_files}")
 
-        project_hierarchy = CONFIG["project_hierarchy"]
+        project_hierarchy = setting.project.hierarchy_name
         # diffs是所有未暂存更改文件的列表。这些更改文件是相对于工作区（working directory）的，也就是说，它们是自上次提交（commit）以来在工作区发生的更改，但还没有被添加到暂存区（staging area）
         # 比如原本存在的md文件现在由于代码的变更发生了更新，就会标记为未暂存diff
         diffs = self.repo.index.diff(None)
@@ -178,7 +180,7 @@ def get_to_be_staged_files(self):
         # 处理untrack_files中的内容
         for untracked_file in untracked_files:
             # 连接repo_path和untracked_file以获取完整的绝对路径
-            if untracked_file.startswith(CONFIG["Markdown_Docs_folder"]):
+            if untracked_file.startswith(setting.project.markdown_docs_name):
                 to_be_staged_files.append(untracked_file)
             continue
             print(f"rel_untracked_file:{rel_untracked_file}")
@@ -187,7 +189,7 @@ def get_to_be_staged_files(self):
             if rel_untracked_file.endswith(".md"):
                 # 把rel_untracked_file从CONFIG['Markdown_Docs_folder']中拆离出来。判断是否能跟暂存区中的某一个.py文件对应上
                 rel_untracked_file = os.path.relpath(
-                    rel_untracked_file, CONFIG["Markdown_Docs_folder"]
+                    rel_untracked_file, setting.project.markdown_docs_name
                 )
                 corresponding_py_file = os.path.splitext(rel_untracked_file)[0] + ".py"
                 print(
@@ -198,7 +200,7 @@ def get_to_be_staged_files(self):
                     to_be_staged_files.append(
                         os.path.join(
                             self.repo_path.lstrip("/"),
-                            CONFIG["Markdown_Docs_folder"],
+                            setting.project.markdown_docs_name,
                             rel_untracked_file,
                         )
                     )
@@ -211,7 +213,7 @@ def get_to_be_staged_files(self):
 
         for unstaged_file in unstaged_files:
             # 连接repo_path和unstaged_file以获取完整的绝对路径
-            if unstaged_file.startswith(CONFIG["Markdown_Docs_folder"]):
+            if unstaged_file.startswith(setting.project.markdown_docs_name):
                 # abs_unstaged_file = os.path.join(self.repo_path, unstaged_file)
                 # # # 获取相对于仓库根目录的相对路径
                 # # rel_unstaged_file = os.path.relpath(abs_unstaged_file, self.repo_path)
@@ -227,7 +229,7 @@ def get_to_be_staged_files(self):
             if unstaged_file.endswith(".md"):
                 # 把rel_unstaged_file从CONFIG['Markdown_Docs_folder']中拆离出来。判断是否能跟暂存区中的某一个.py文件对应上
                 rel_unstaged_file = os.path.relpath(
-                    rel_unstaged_file, CONFIG["Markdown_Docs_folder"]
+                    rel_unstaged_file, setting.project.markdown_docs_name
                 )
                 corresponding_py_file = os.path.splitext(rel_unstaged_file)[0] + ".py"
                 print(f"corresponding_py_file:{corresponding_py_file}")
@@ -236,7 +238,7 @@ def get_to_be_staged_files(self):
                     to_be_staged_files.append(
                         os.path.join(
                             self.repo_path.lstrip("/"),
-                            CONFIG["Markdown_Docs_folder"],
+                            setting.project.markdown_docs_name,
                             rel_unstaged_file,
                         )
                     )

repo_agent/__main__.py

@@ -1,14 +1,17 @@
+import inspect
+import os
 import sys
-import tiktoken
 import time
-import inspect
-from repo_agent.log import logger
-from repo_agent.settings import setting, max_input_tokens_map
-from repo_agent.prompt import SYS_PROMPT, USR_PROMPT
+from dataclasses import dataclass
+
+import tiktoken
+from openai import APIConnectionError, OpenAI
+
 from repo_agent.doc_meta_info import DocItem
 from repo_agent.log import logger
-from openai import OpenAI, APIConnectionError
-from dataclasses import dataclass
+from repo_agent.prompt import SYS_PROMPT, USR_PROMPT
+from repo_agent.settings import max_input_tokens_map, setting
+
 
 def get_import_statements():
     source_lines = inspect.getsourcelines(sys.modules[__name__])[0]
@@ -255,7 +258,7 @@ def get_relationship_description(referencer_content, reference_letter):
                 for k, v in max_input_tokens_map.items()
                 if (v - max_tokens) > total_tokens
             }  # 抽取出所有上下文长度大于当前总输入tokens的模型
-            for model_name, max_input_length in larger_models:
+            for model_name, max_input_length in larger_models.items():
                 if max_input_length - max_tokens > total_tokens:
                     try:
                         # Attempt to make a request with the larger model

repo_agent/change_detector.py

@@ -1,5 +1,6 @@
 import gradio as gr
 import markdown
+
 from repo_agent.log import logger