Updated to version 0.1.9 after adding feature for using proxy

Author: shaikhsajid1111

README.MD

@@ -18,7 +18,7 @@
 <h3> Installing from source: </h3>
 
 ```
-git clone https://github.com/shaikhsajid1111/facebook_page_scraper 
+git clone https://github.com/shaikhsajid1111/facebook_page_scraper
 ```
 
 <h4> Inside project's directory </h4>
@@ -47,12 +47,12 @@ from facebook_page_scraper import Facebook_scraper
 page_name = "facebookai"
 posts_count = 10
 browser = "firefox"
-
-facebook_ai = Facebook_scraper(page_name,posts_count,browser)
+proxy = "IP:PORT" #if proxy requires authentication then user:password@IP:PORT
+facebook_ai = Facebook_scraper(page_name,posts_count,browser,proxy=proxy)
 
 ```
 
-<h3> Parameters for  <code>Facebook_scraper(page_name,posts_count,browser) </code> class </h3>
+<h3> Parameters for  <code>Facebook_scraper(page_name,posts_count,browser,proxy) </code> class </h3>
 <table>
 <th>
 <tr>
@@ -98,6 +98,18 @@ which browser to use, either chrome or firefox. if not passed,default is chrome
 </td>
 </tr>
 
+<tr>
+<td>
+proxy(optional)
+</td>
+<td>
+string
+</td>
+<td>
+optional argument, if user wants to set proxy, if proxy requires authentication then the format will be <code> user:password@IP:PORT </code>
+</td>
+</tr>
+
 </table>
 <br>
 <hr>
@@ -152,7 +164,7 @@ Output Structure for JSON format:
 ``` javascript
 {
     "id": {
-        "name": string,    
+        "name": string,
         "shares": integer,
         "reactions": {
             "likes": integer,
@@ -321,7 +333,7 @@ reactions
 dictionary
 </td>
 <td>
-dictionary containing reactions as keys and its count as value. Keys => <code> ["likes","loves","wow","cares","sad","angry","haha"] </code> 
+dictionary containing reactions as keys and its count as value. Keys => <code> ["likes","loves","wow","cares","sad","angry","haha"] </code>
 </td>
 </tr>
 

changelog.MD

@@ -1,4 +1,11 @@
 <h1> Changelog </h1>
+<section>
+<h2> 0.1.9 </h2>
+<h3>Added</h3>
+<li>Added feature for using proxy while scraping</li>
+
+<br>
+
 
 <section>
 <h2> 0.1.8 </h2>

facebook_page_scraper/__init__.py

@@ -3,4 +3,5 @@
 from .driver_utilities import Utilities
 from .element_finder import Finder
 from .scraping_utilities import Scraping_utilities
+
 __all__ = ["Initializer","Facebook_scraper","Utilities","Finder","Scraping_utilities"]

facebook_page_scraper/driver_initialization.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python3
 try:
-    from selenium import webdriver
-    #to add capabilities for chrome and firefox, import their Options with different aliases
-    from selenium.webdriver.chrome.options import Options as ChromeOptions   
+    from seleniumwire import webdriver
+    # to add capabilities for chrome and firefox, import their Options with different aliases
+    from selenium.webdriver.chrome.options import Options as ChromeOptions
     from selenium.webdriver.firefox.options import Options as FirefoxOptions
-    #import webdriver for downloading respective driver for the browser
+    # import webdriver for downloading respective driver for the browser
     from webdriver_manager.chrome import ChromeDriverManager
     from webdriver_manager.firefox import GeckoDriverManager
 except Exception as ex:
@@ -13,36 +13,62 @@
 
 class Initializer:
 
-    def __init__(self,browser_name):
+    def __init__(self, browser_name, proxy=None):
         self.browser_name = browser_name
+        self.proxy = proxy
 
-    def set_properties(self,browser_option):
+    def set_properties(self, browser_option):
         """adds capabilities to the driver"""
-        browser_option.add_argument('--headless')   #runs browser in headless mode
-        browser_option.add_argument('--no-sandbox') 
-
+        browser_option.add_argument(
+            '--headless')  # runs browser in headless mode
+        browser_option.add_argument('--no-sandbox')
         browser_option.add_argument("--disable-dev-shm-usage")
         browser_option.add_argument('--ignore-certificate-errors')
         browser_option.add_argument('--disable-gpu')
         browser_option.add_argument('--log-level=3')
         browser_option.add_argument('--disable-notifications')
         browser_option.add_argument('--disable-popup-blocking')
+
+        # browser_option.add_argument(
+        #   "--proxy-server=http://{}".format(self.proxy.replace(" ", "")))
+
         return browser_option
 
-    def set_driver_for_browser(self,browser_name):
+    def set_driver_for_browser(self, browser_name):
         """expects browser name and returns a driver instance"""
-        #if browser is suppose to be chrome 
+        # if browser is suppose to be chrome
         if browser_name.lower() == "chrome":
             browser_option = ChromeOptions()
-            #automatically installs chromedriver and initialize it and returns the instance
-            return webdriver.Chrome(executable_path=ChromeDriverManager().install(),options=self.set_properties(browser_option))
+            # automatically installs chromedriver and initialize it and returns the instance
+            if self.proxy is not None:
+                options = {
+                    'https': 'https://{}'.format(self.proxy.replace(" ", "")),
+                    'http': 'http://{}'.format(self.proxy.replace(" ", "")),
+                    'no_proxy': 'localhost, 127.0.0.1'
+                }
+                print("Using: {}".format(self.proxy))
+                return webdriver.Chrome(executable_path=ChromeDriverManager().install(),
+                                        options=self.set_properties(browser_option), seleniumwire_options=options)
+
+            return webdriver.Chrome(executable_path=ChromeDriverManager().install(), options=self.set_properties(browser_option))
         elif browser_name.lower() == "firefox":
             browser_option = FirefoxOptions()
-            #automatically installs geckodriver and initialize it and returns the instance
-            return webdriver.Firefox(executable_path=GeckoDriverManager().install(),options=self.set_properties(browser_option))
+            if self.proxy is not None:
+                options = {
+                    'https': 'https://{}'.format(self.proxy.replace(" ", "")),
+                    'http': 'http://{}'.format(self.proxy.replace(" ", "")),
+                    'no_proxy': 'localhost, 127.0.0.1'
+                }
+                print("Using: {}".format(self.proxy))
+                return webdriver.Firefox(executable_path=GeckoDriverManager().install(),
+                                         options=self.set_properties(browser_option), seleniumwire_options=options)
+
+            # automatically installs geckodriver and initialize it and returns the instance
+            return webdriver.Firefox(executable_path=GeckoDriverManager().install(), options=self.set_properties(browser_option))
         else:
-            #if browser_name is not chrome neither firefox than raise an exception 
+            # if browser_name is not chrome neither firefox than raise an exception
             raise Exception("Browser not supported!")
+
     def init(self):
         """returns driver instance"""
         driver = self.set_driver_for_browser(self.browser_name)