Merge pull request #32 from daeisbae/31-working-repository-analysis-demo

Repository add and analysis working demo

Author: daeisbae

.env.example

@@ -0,0 +1,14 @@
+# Postgres Database Configuration
+DB_HOST=
+DB_PORT=
+DB_NAME=
+DB_USER=
+DB_PASSWORD=
+
+# Github Token for increasing rate limit of reading the repository
+GITHUB_TOKEN=
+
+# LLM_PROVIDER=deepseek | google (for google ai studio)
+LLM_PROVIDER=
+LLM_APIKEY=
+LLM_MODELNAME=

components.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "https://ui.shadcn.com/schema.json",
+  "style": "new-york",
+  "rsc": true,
+  "tsx": false,
+  "tailwind": {
+    "config": "tailwind.config.mjs",
+    "css": "src/app/globals.css",
+    "baseColor": "neutral",
+    "cssVariables": false,
+    "prefix": ""
+  },
+  "aliases": {
+    "components": "@/components",
+    "utils": "@/lib/utils",
+    "ui": "@/components/ui",
+    "lib": "@/lib",
+    "hooks": "@/hooks"
+  },
+  "iconLibrary": "lucide"
+}

package.json

@@ -23,6 +23,7 @@
     "lucide-react": "^0.469.0",
     "next": "15.1.0",
     "next-mdx-remote": "^5.0.0",
+    "openai": "^4.77.0",
     "pg": "^8.13.1",
     "react": "^19.0.0",
     "react-dom": "^19.0.0",

src/agent/document-splitter/code-splitter.ts

@@ -7,7 +7,10 @@ enum Language {
     RUBY = "ruby",
     RUST = "rust",
     PHP = "php",
-    CPP = "cpp"
+    CPP = "cpp",
+    JAVA = "java",
+    SCALA = "scala",
+    MARKDOWN = "markdown"
   }
 
 /**
@@ -46,6 +49,9 @@ export default class CodeSplitter {
             'hpp': Language.CPP,
             'hxx': Language.CPP,
             'h': Language.CPP,
+            'java': Language.JAVA,
+            'scala': Language.SCALA,
+            'md': Language.MARKDOWN
         };
         return extensionToLanguageMap[extension.toLowerCase()] || null;
     }
@@ -54,7 +60,7 @@ export default class CodeSplitter {
      * Splits the provided code into chunks based on the file extension.
      * @param {string} fileExtension - The file extension indicating the programming language.
      * @param {string} code - The code content to be split.
-     * @returns {Promise<string | null>}  - A promise that resolves to an array of document chunks.
+     * @returns {Promise<string | null>}  - A promise that returns the code with line numbers.
      */
     async splitCode(fileExtension: string, code: string): Promise<string | null> {
         const language : SupportedTextSplitterLanguage | null = this.getLanguageFromExtension(fileExtension);

src/agent/structured-output/index.ts

@@ -7,6 +7,7 @@ import {
     RepoInfo,
 } from '@/agent/structured-output/prompt-generator'
 import { CodePrompt, FolderPrompt } from '@/agent/structured-output/prompt'
+import CodeSplitter from '@/agent/document-splitter/code-splitter'
 
 /**
  * Abstract base class for processing prompts using LLM
@@ -26,16 +27,22 @@ abstract class BaseProcessor {
 
     /**
      * Process the given prompt using LLM and parse the response into JSON format
-     * @param {any} prompt - The prompt to process
-     * @param {RepoInfo | null} repoInfo - Information about the repository
+     * @param {string} prompt - The prompt to process
      * @returns Parsed object from the LLM response
      */
-    protected async process(prompt: any, repoInfo: RepoInfo | null): Promise<object> {
+    protected async process(prompt: string): Promise<any> {
         const response = await this.llm.run(prompt, [])
         return await this.schemaParser.parse(response)
     }
 }
 
+interface CodeSummaryOutput {
+    name: string
+    path: string
+    summary: string
+    usage: string
+}
+
 /**
  * Processes code-related prompts with specific schema and formatting
  * @example
@@ -45,12 +52,15 @@ abstract class BaseProcessor {
  * const result = await codeProcessor.process(sourceCode)
  */
 export class CodeProcessor extends BaseProcessor {
+    private codeSplitter: CodeSplitter
+
     /**
      * Creates a new CodeProcessor instance with file schema type
      * @param llm - The LLM provider instance
      */
     constructor(llm: LLMProvider) {
         super(llm)
+        this.codeSplitter = new CodeSplitter(200, 25)
         this.schemaParser = new SchemaParser(getSchema(SchemaType.FILE))
         this.promptGenerator = new PromptGenerator(
             {
@@ -67,23 +77,43 @@ export class CodeProcessor extends BaseProcessor {
 
     /**
      * Process the given code string
-     * @param {string }code - Source code to process
+     * @param {string} path - Path of the file
+     * @param {string} code - Source code to process
      * @param {RepoInfo} repoInfo - Information about the repository
      * @returns Processed and parsed JSON object defined in schema factory
      */
-    async process(code: string, repoInfo: RepoInfo): Promise<object> {
+    async generate(code: string, repoInfo: RepoInfo): Promise<CodeSummaryOutput | null> {
+        const extension = repoInfo.path.split('.').pop()
+        if (!extension) {
+            console.warn('No extension found in the file path')
+            return null
+        }
+        const splittedDoc = await this.codeSplitter.splitCode(extension, code)
+        if (!splittedDoc) {
+            console.warn('Code splitting failed')
+            return null
+        }
+
         const prompt = await this.promptGenerator.generate(
             {
                 requirements: CodePrompt,
                 formatInstructions: this.schemaParser.formalInstructions,
                 ...repoInfo,
             },
-            code
+            splittedDoc
         )
-        return await super.process(prompt, null)
+
+        return await super.process(prompt)
     }
 }
 
+interface FolderSummaryOutput {
+    name: string
+    usage: string
+    path: string
+    summary: string
+}
+
 /**
  * Processes folder-related prompts with specific schema and formatting
  * @example
@@ -119,7 +149,7 @@ export class FolderProcessor extends BaseProcessor {
      * @param {RepoInfo} repoInfo - Information about the repository
      * @returns Processed and parsed JSON object defined in schema factory
      */
-    async process(folder: string[], repoInfo: RepoInfo): Promise<object> {
+    async generate(folder: string[], repoInfo: RepoInfo): Promise<FolderSummaryOutput | null> {
         const prompt = await this.promptGenerator.generate(
             {
                 requirements: FolderPrompt,
@@ -129,6 +159,6 @@ export class FolderProcessor extends BaseProcessor {
             undefined,
             folder
         )
-        return await super.process(prompt, null)
+        return await super.process(prompt)
     }
 }

src/agent/structured-output/prompt.ts

@@ -21,7 +21,7 @@ You will receive the following information, extracted from a GitHub repository:
     *   Explain its role in the overall system.
     *   Identify its dependencies on other modules/components.
     *   Highlight any important classes, functions, or data structures.
-    *   If you need to refer to a specific code, mention the path to that file by using the following markdown link format: [\`Description of Code Block\`](Full github url of the file including the start line with optional ending line#L{startLine}-L{endLine}). This is in the form of "https://github.com/{owner}/{repo}/blob/{commitSha}/{path}#L{lineStart}-L{lineEnd}".
+    *   Link all the code blocks (Class,Function,Enum,Exception) that are referenced using the following markdown link format: [\`Description of Code Block\`](Full github url of the file including the start line with optional ending line#L{startLine}-L{endLine}). This is in the form of "https://github.com/{owner}/{repo}/blob/{commitSha}/{path}#L{lineStart}-L{lineEnd}".
 2. **Code-Level Insights:**
     *   Analyze the code files to understand the implementation details.
     *   Identify core algorithms, data structures, and design patterns used.
@@ -56,7 +56,7 @@ You will receive the following information, summarized from the expert software
     *   Explain its role in the overall system.
     *   Identify its dependencies on other modules/components/folder.
     *   Highlight any important classes, functions, or data structures in it's sub-files and sub-folders.
-    *   If you need to refer to a specific code, mention the path to that file by using the following markdown link format: [\`Description of Code Block\`]({Full github url of the file including the start line with optional ending line#L{startLine}-L{endLine}). This is in the form of "https://github.com/{owner}/{repo}/blob/{commitSha}/{path}#L{lineStart}-L{lineEnd}".
+    *   Link all the code blocks that are referenced using the following markdown link format: [\`Description of Code Block\`](Full github url of the file including the start line with optional ending line#L{startLine}-L{endLine}). This is in the form of "https://github.com/{owner}/{repo}/blob/{commitSha}/{path}#L{lineStart}-L{lineEnd}".
 2. **Dependencies and Relationships:**
     *   Clearly document the relationships between different folders and files.
     *   Explain how different parts of the codebase interact with each other.

src/agent/structured-output/schema-factory.ts

@@ -46,6 +46,11 @@ File Schema Example Output:
                 path: z
                     .string()
                     .describe('Path to the file within the repository.'),
+                usage: z
+                    .string()
+                    .describe(
+                        'What the file is used for. Describe less than 10 words (ex. Data Parsing, API Requests, etc.)'
+                    ),
                 summary: z
                     .string()
                     .describe(
@@ -82,30 +87,13 @@ Folder Schema Example Output:
         case SchemaType.FOLDER:
             return z.object({
                 name: z.string().describe('Name of the folder.'),
-                usage: z.string().describe('What the folder is used for.'),
+                usage: z.string().describe('What the folder is used for. Describe less than 10 words (ex. Server Lifecycle Management, API Utility Functions, etc.)'),
                 path: z.string().describe('Path to the folder.'),
                 summary: z
                     .string()
                     .describe(
                         'Summary of the folder, its main purpose, and its role in the project. Include Markdown links to important code blocks within the file using the format `[{Description of Code Block}]({Full github url of the file including the start line with optional ending line}#L{startLine}-L{endLine})` where applicable.'
-                    ),
-                files: z
-                    .array(
-                        z.object({
-                            fileName: z
-                                .string()
-                                .describe(
-                                    'Name of the file mentioned in the summary'
-                                ),
-                            filePath: z
-                                .string()
-                                .describe(
-                                    'Path to the file mentioned in the summary'
-                                ),
-                        })
                     )
-                    .optional()
-                    .describe('List of files in the folder.'),
             })
 
         default:

src/app/[ownerSlug]/[repoSlug]/formatter.ts

@@ -0,0 +1,44 @@
+import { FullFolder, FullRepository } from '@/service/get-db'
+
+export function formatToMarkdown(fullRepo: FullRepository): string {
+    const repo = fullRepo.repository
+    const branches = fullRepo.branch
+    const folders = fullRepo.folders
+
+    const url = `https://github.com/${repo.owner}/${repo.repo}/blob/${branches?.last_commit_sha}`
+
+    let markdown = recursiveFolderToMarkdown(folders[0], url, '#')
+
+    return markdown
+}
+
+function recursiveFolderToMarkdown(
+    folder: FullFolder,
+    url: string,
+    subfolderHeading: string
+): string {
+    let markdown = `${subfolderHeading} ${folder.usage}\n `
+    markdown += `---\n`
+    markdown += folder.path
+        ? `- Reference: [\`${folder.path}\`](${url}/${folder.path}) \n`
+        : ''
+    markdown += `\n${folder.ai_summary}\n`
+    folder.files.forEach((file) => {
+        markdown += `###### ${file.usage}\n`
+        markdown += `---\n`
+        markdown += file.name
+            ? `- Reference: [\`${file.name}\`](${url}/${file.name}) \n`
+            : ''
+        markdown += `\n${file.ai_summary}\n`
+    })
+    folder.subfolders.forEach((subfolder) => {
+        markdown += recursiveFolderToMarkdown(
+            subfolder,
+            url,
+            subfolderHeading.length >= 5
+                ? subfolderHeading
+                : subfolderHeading + '#'
+        )
+    })
+    return markdown
+}

src/app/[ownerSlug]/[repoSlug]/page.tsx

@@ -1,60 +1,42 @@
 import { JSX, Suspense } from 'react'
 import { RepoCard } from '@/components/RepoCard'
-import Loading from '@/app/[ownerSlug]/[repoSlug]/loading'
-import { FetchRepoService, FullRepository } from '@/service/get-db'
+import { FetchRepoService } from '@/service/get-db'
 import { MarkdownContent } from '@/components/MarkdownContent'
 import { notFound } from 'next/navigation'
+import { formatToMarkdown } from '@/app/[ownerSlug]/[repoSlug]/formatter'
+import Loading from '@/app/[ownerSlug]/[repoSlug]/loading'
 
+// Define the correct page props type for Next.js 13+
 interface PageProps {
-    params: {
-        ownerSlug: string
-        repoSlug: string
-    }
+    params: Promise<{ ownerSlug: string; repoSlug: string }>
 }
 
-interface RepoPageProps {
-    ownerSlug: string
-    repoSlug: string
-}
-
-const placeholder = `
-# parser.cpp\n\n
-
-- Reference \`parser/parser.cpp\`\n
-
-This file defines the \`Parser\` class, responsible for transforming a stream of tokens into an Abstract Syntax Tree (AST). The parser utilizes a queue of \`TokenPtr\` objects (\`tok_queue_\`) and provides methods for consuming tokens (\`Eat\`), peeking at the next token (\`Peek\`), and expecting specific token types (\`ExpectedTokenType\`). The core functionality resides in \`ProduceAST\` which drives the parsing process by repeatedly calling \`ParseStatement\` until an end-of-line token is encountered. Different parsing methods are present to handle various expressions such as \`ParsePrimaryExpression\`, \`ParseAdditionExpression\`, \`ParseMultiplicationExpression\`, and \`ParseComparisonExpression\`. It supports variable declarations and assignments, and also handles whitespace using \`ParseWhitespaceExpression\`. The parser uses recursive descent parsing strategy with helper functions for each type of expression. It throws \`UnexpectedTokenParsedException\` when unexpected tokens are encountered. The \`Parser\` class depends on the \`TokenPtr\` and the \`ast.hpp\` module for the AST node definitions. Key methods include: [\`Eat\`](https://github.com/daeisbae/AParser/blob/9bbea84efa9f8eeed5576f53e4a65ca87a7f023c/parser/parser.cpp#L11-L15) for consuming tokens, [\`Peek\`](https://github.com/daeisbae/AParser/blob/9bbea84efa9f8eeed5576f53e4a65ca87a7f023c/parser/parser.cpp#L17-L17) for peeking at tokens, [\`ExpectedTokenType\`](https://github.com/daeisbae/AParser/blob/9bbea84efa9f8eeed5576f53e4a65ca87a7f023c/parser/parser.cpp#L19-L39) for validating token types, [\`ProduceAST\`](https://github.com/daeisbae/AParser/blob/9bbea84efa9f8eeed5576f53e4a65ca87a7f023c/parser/parser.cpp#L41-L53) for generating the AST, [\`ParseStatement\`](https://github.com/daeisbae/AParser/blob/9bbea84efa9f8eeed5576f53e4a65ca87a7f023c/parser/parser.cpp#L55-L62) for parsing statements, and [\`ParseExpression\`](https://github.com/daeisbae/AParser/blob/9bbea84efa9f8eeed5576f53e4a65ca87a7f023c/parser/parser.cpp#L64-L66) for parsing expressions. The file also handles different types of expressions using separate parsing methods like [\`ParsePrimaryExpression\`](https://github.com/daeisbae/AParser/blob/9bbea84efa9f8eeed5576f53e4a65ca87a7f023c/parser/parser.cpp#L68-L145), [\`ParseAdditionExpression\`](https://github.com/daeisbae/AParser/blob/9bbea84efa9f8eeed5576f53e4a65ca87a7f023c/parser/parser.cpp#L147-L165), [\`ParseMultiplicationExpression\`](https://github.com/daeisbae/AParser/blob/9bbea84efa9f8eeed5576f53e4a65ca87a7f023c/parser/parser.cpp#L167-L185), [\`ParseWhitespaceExpression\`](https://github.com/daeisbae/AParser/blob/9bbea84efa9f8eeed5576f53e4a65ca87a7f023c/parser/parser.cpp#L187-L192), [\`ParseIdentifierDeclarationExpression\`](https://github.com/daeisbae/AParser/blob/9bbea84efa9f8eeed5576f53e4a65ca87a7f023c/parser/parser.cpp#L194-L217), [\`ParseIdentifierAssignmentExpression\`](https://github.com/daeisbae/AParser/blob/9bbea84efa9f8eeed5576f53e4a65ca87a7f023c/parser/parser.cpp#L219-L243), and [\`ParseComparisonExpression\`](https://github.com/daeisbae/AParser/blob/9bbea84efa9f8eeed5576f53e4a65ca87a7f023c/parser/parser.cpp#L245-L263).
-`
+// Main page component
+export default async function Page({
+    params,
+}: PageProps): Promise<JSX.Element> {
+    const { ownerSlug, repoSlug } = await params
 
-async function RepoPage({
-    ownerSlug,
-    repoSlug,
-}: RepoPageProps): Promise<JSX.Element> {
     const fetchRepoService = new FetchRepoService()
-    const repoDetails: FullRepository | null =
-        await fetchRepoService.getFullRepositoryTree(ownerSlug, repoSlug)
+    const repoDetails = await fetchRepoService.getFullRepositoryTree(
+        ownerSlug,
+        repoSlug
+    )
 
     if (!repoDetails) {
         notFound()
     }
 
     return (
         <div className="flex gap-6 p-6">
-            <div className="flex-1">
-                <MarkdownContent content={placeholder} />
-            </div>
-            <div className="w-[300px]">
-                <RepoCard repoInfo={repoDetails.repository} />
-            </div>
+            <Suspense fallback={<Loading />}>
+                <div className="flex-1">
+                    <MarkdownContent content={formatToMarkdown(repoDetails)} />
+                </div>
+                <div className="w-[300px]">
+                    <RepoCard repoInfo={repoDetails.repository} />
+                </div>
+            </Suspense>
         </div>
     )
 }
-
-export default function DocumentationPage({ params }: PageProps) {
-    const { ownerSlug, repoSlug } = params
-
-    return (
-        <Suspense fallback={<Loading />}>
-            <RepoPage ownerSlug={ownerSlug} repoSlug={repoSlug} />
-        </Suspense>
-    )
-}

src/app/add-repositories/insert-repo.tsx

@@ -1,9 +1,12 @@
 'use server'
 
+import LLMConfig from '@/llm/llm-config'
+import LLMFactory from '@/app/add-repositories/llm-factory'
 import { InsertRepoService } from '@/service/insert-db'
 
 export async function insertRepository(owner: string, repo: string) {
-    const insertRepoService = new InsertRepoService()
+    const llmConfig = new LLMConfig(1, 0.95, 40, 8192)
+    const insertRepoService = new InsertRepoService(LLMFactory.createProvider(llmConfig))
     const result = await insertRepoService.insertRepository(owner, repo)
     if (!result) {
         return { success: false, error: 'Repository already exists' }