Merge pull request #22 from guy-hartstein/feat/crawl

Feat/crawl

Author: guy-hartstein

.env.example

@@ -0,0 +1,3 @@
+TAVILY_API_KEY=
+OPENAI_API_KEY=
+GEMINI_API_KEY=

backend/nodes/grounding.py

@@ -51,8 +51,14 @@ async def initial_search(self, state: InputState) -> ResearchState:
                     )
 
             try:
-                logger.info("Initiating Tavily extraction")
-                site_extraction = await self.tavily_client.extract(url, extract_depth="basic")
+                logger.info("Initiating Tavily crawl")
+                site_extraction = await self.tavily_client.crawl(
+                    url=url, 
+                    instructions="Find any pages that will help us understand the company's business, products, services, and any other relevant information.",
+                    max_depth=1, 
+                    max_breadth=50, 
+                    extract_depth="advanced"
+                )
 
                 raw_contents = []
                 for item in site_extraction.get("results", []):
@@ -64,19 +70,19 @@ async def initial_search(self, state: InputState) -> ResearchState:
                         'title': company,
                         'raw_content': "\n\n".join(raw_contents)
                     }
-                    logger.info(f"Successfully extracted {len(raw_contents)} content sections")
-                    msg += "\n✅ Successfully extracted content from website"
+                    logger.info(f"Successfully crawled {len(raw_contents)} content sections")
+                    msg += "\n✅ Successfully crawled content from website"
                     if websocket_manager := state.get('websocket_manager'):
                         if job_id := state.get('job_id'):
                             await websocket_manager.send_status_update(
                                 job_id=job_id,
                                 status="processing",
-                                message="Successfully extracted content from website",
+                                message="Successfully crawled and extracted content from website",
                                 result={"step": "Initial Site Scrape"}
                             )
                 else:
-                    logger.warning("No content found in extraction results")
-                    msg += "\n⚠️ No content found in website extraction"
+                    logger.warning("No content found in crawl results")
+                    msg += "\n⚠️ No content found in website crawl"
                     if websocket_manager := state.get('websocket_manager'):
                         if job_id := state.get('job_id'):
                             await websocket_manager.send_status_update(
@@ -87,8 +93,8 @@ async def initial_search(self, state: InputState) -> ResearchState:
                             )
             except Exception as e:
                 error_str = str(e)
-                logger.error(f"Website extraction error: {error_str}", exc_info=True)
-                error_msg = f"⚠️ Error extracting website content: {error_str}"
+                logger.error(f"Website crawl error: {error_str}", exc_info=True)
+                error_msg = f"⚠️ Error crawling website content: {error_str}"
                 print(error_msg)
                 msg += f"\n{error_msg}"
                 if websocket_manager := state.get('websocket_manager'):
@@ -137,8 +143,8 @@ async def initial_search(self, state: InputState) -> ResearchState:
             "job_id": state.get('job_id')
         }
 
-        # If there was an error in the initial extraction, store it in the state
-        if "⚠️ Error extracting website content:" in msg:
+        # If there was an error in the initial crawl, store it in the state
+        if "⚠️ Error crawling website content:" in msg:
             research_state["error"] = error_str
 
         return research_state

requirements.txt

@@ -7,7 +7,7 @@ protobuf~=4.25.0
 pydantic==2.10.6
 pymongo==4.6.3
 reportlab==4.3.1
-tavily_python==0.5.1
+tavily_python==0.7.1
 uvicorn[standard]==0.34.0
 websockets==12.0
 google-generativeai==0.8.4

ui/src/components/LocationInput.tsx

@@ -14,97 +14,6 @@ declare global {
   }
 }
 
-// Global state to track script loading
-let isScriptLoaded = false;
-let isScriptLoading = false;
-let scriptLoadPromise: Promise<void> | null = null;
-
-const loadGoogleMapsScript = (): Promise<void> => {
-  // If already loaded, return resolved promise
-  if (isScriptLoaded && window.google?.maps?.places) {
-    return Promise.resolve();
-  }
-
-  // If already loading, return existing promise
-  if (isScriptLoading && scriptLoadPromise) {
-    return scriptLoadPromise;
-  }
-
-  // Check if script is already in the document
-  const existingScript = document.querySelector('script[src*="maps.googleapis.com/maps/api/js"]');
-  if (existingScript && window.google?.maps?.places) {
-    isScriptLoaded = true;
-    return Promise.resolve();
-  }
-
-  isScriptLoading = true;
-  scriptLoadPromise = new Promise<void>((resolve, reject) => {
-  // If API is already available, resolve immediately
-    if (window.google?.maps?.places) {
-      isScriptLoaded = true;
-      isScriptLoading = false;
-      resolve();
-      return;
-    }
-
-    // Define the callback function
-    window.initGoogleMapsCallback = () => {
-      isScriptLoaded = true;
-      isScriptLoading = false;
-      resolve();
-    };
-
-    // Create script element only if it doesn't exist
-    if (!existingScript) {
-      const script = document.createElement('script');
-      const apiKey = import.meta.env.VITE_GOOGLE_MAPS_API_KEY;
-
-      if (!apiKey) {
-        console.error('Google Maps API key is not defined. Please set VITE_GOOGLE_MAPS_API_KEY in your environment variables.');
-        isScriptLoading = false;
-        reject(new Error('Google Maps API key is not defined'));
-        return;
-      }
-
-      script.src = `https://maps.googleapis.com/maps/api/js?key=${apiKey}&libraries=places&loading=async&callback=initGoogleMapsCallback`;
-      script.async = true;
-      script.defer = true;
-
-      // Handle errors
-      script.onerror = (error) => {
-        console.error('Error loading Google Maps script:', error);
-        isScriptLoading = false;
-        scriptLoadPromise = null;
-        reject(error);
-      };
-
-      // Append to document
-      document.head.appendChild(script);
-    } else {
-      // Script exists but API might not be ready yet
-      const checkInterval = setInterval(() => {
-        if (window.google?.maps?.places) {
-          isScriptLoaded = true;
-          isScriptLoading = false;
-          clearInterval(checkInterval);
-          resolve();
-        }
-      }, 100);
-
-      // Clear interval after 10 seconds to prevent infinite checking
-      setTimeout(() => {
-        clearInterval(checkInterval);
-        if (!isScriptLoaded) {
-          isScriptLoading = false;
-          reject(new Error('Google Maps API failed to load within timeout'));
-        }
-      }, 10000);
-    }
-  });
-
-  return scriptLoadPromise;
-};
-
 const LocationInput: React.FC<LocationInputProps> = ({ value, onChange, className }) => {
   const inputRef = useRef<HTMLInputElement>(null);
   const autocompleteElementRef = useRef<any>(null);
@@ -125,6 +34,10 @@ const LocationInput: React.FC<LocationInputProps> = ({ value, onChange, classNam
         setIsApiLoaded(true);
       } catch (error) {
         console.error('Failed to load Google Maps API:', error);
+        // Ensure input is visible when Google Maps fails to load
+        if (inputRef.current) {
+          inputRef.current.style.display = '';
+        }
       }
     };