fix retrieve bugs

Author: stuartthomas25

jparty/retrieve.py

@@ -49,33 +49,75 @@ def get_Gsheet_game(file_id):
         r3 = csv.reader(lines)
         return list_to_game(list(r3))
 
+class RetrievalException(Exception):
+    pass
+
+class IncompleteException(Exception):
+    pass
 
 def get_game(game_id):
     if len(str(game_id)) < 7:
         try:
+            logging.info("trying jarchive")
             return get_jarchive_game(game_id)
-        except Exception as e:
+        except RetrievalException as e:
             logging.error(e)
+            logging.info("trying wayback")
             return get_wayback_game(game_id)
     else:
         return get_Gsheet_game(str(game_id))
 
 
-def findanswer(clue):
-    return re.findall(r'correct_response">(.*?)</em', unescape(str(clue)))[0]
 
 def get_jarchive_game(game_id):
     return get_generic_game(game_id, f"http://www.j-archive.com/showgame.php?game_id={game_id}")
 
+def get_wayback_game(game_id):
+    # kudos to Abhi Kumbar: https://medium.com/analytics-vidhya/the-wayback-machine-scraper-63238f6abb66
+    # this query's the wayback cdx api for possible instances of the saved jarchive page with the specified game id & returns the latest one
+    JArchive_url = f"j-archive.com/showgame.php?game_id={str(game_id)}"  # use the url w/o the http:// or https:// to include both in query
+    url = f'http://web.archive.org/cdx/search/cdx?url={JArchive_url}&collapse=digest&limit=-2&fastLatest=true&output=json'  # for some reason, using limit=-1 does not work
+    urls = requests.get(url).text
+    parse_url = json.loads(urls)  # parses the JSON from urls.
+    if len(parse_url) == 0:  # if no results, return None
+        logging.info("no games found in wayback")
+        # alternative: use fallback to get game from scraping j-archive directly
+        raise RetrievalException("no games found in wayback")
+
+    ## Extracts timestamp and original columns from urls and compiles a url list.
+    url_list = []
+    for i in range(1, len(parse_url)): # gets the wayback url
+        orig_url = parse_url[i][2]
+        tstamp = parse_url[i][1]
+        waylink = tstamp + '/' + orig_url
+        final_url = f'http://web.archive.org/web/{waylink}'
+        url_list.append(final_url)
+    latest_url = url_list[-1]
+    return get_generic_game(game_id, latest_url)
+
+def findanswer(clue):
+    return re.findall(r'correct_response">(.*?)</em', unescape(str(clue)))[0]
+
 def get_generic_game(game_id, url):
     logging.info(f"getting game {game_id} from url {url}")
-    r = requests.get(url)
+    try:
+        r = requests.get(url, timeout=5)
+    except requests.exceptions.ConnectTimeout as e:
+        logging.info(repr(type(e)))
+        raise RetrievalException(repr(e))
+
+    logging.info(f"returned status {r.status_code}")
+    if r.status_code != 200:
+        raise RetrievalException(f"{url} returned HTTP code {r.status_code}")
+
     soup = BeautifulSoup(r.text, "html.parser")
     datesearch = re.search(
-        r"- \w+, (.*?)$", soup.select("#game_title > h1")[0].contents[0]
+        r"<h1>(.*?)</h1>", str(soup.select("#game_title > h1")[0])
     )
+
     if datesearch is None:
-        return None
+        raise RetrievalException("Cannot get game summary")
+
     date = datesearch.groups()[0]
     comments = soup.select("#game_comments")[0].contents
     comments = comments[0] if len(comments) > 0 else ""
@@ -90,8 +132,7 @@ def get_generic_game(game_id, url):
         for clue in ro.find_all(class_="clue"):
             text_obj = clue.find(class_="clue_text")
             if text_obj is None:
-                logging.info("this game is incomplete")
-                return None
+                raise IncompleteException()
 
             text = text_obj.text
             index_key = text_obj["id"]
@@ -114,8 +155,7 @@ def get_generic_game(game_id, url):
     clue = final_round_obj.find_all(class_="clue")[0]
     text_obj = clue.find(class_="clue_text")
     if text_obj is None:
-        logging.info("this game is incomplete")
-        return None
+        raise IncompleteException()
 
     text = text_obj.text
     answer = findanswer(final_round_obj)
@@ -125,29 +165,6 @@ def get_generic_game(game_id, url):
 
     return GameData(boards, date, comments)
 
-def get_wayback_game(game_id):
-    # kudos to Abhi Kumbar: https://medium.com/analytics-vidhya/the-wayback-machine-scraper-63238f6abb66
-    # this query's the wayback cdx api for possible instances of the saved jarchive page with the specified game id & returns the latest one
-    JArchive_url = f"j-archive.com/showgame.php?game_id={str(game_id)}"  # use the url w/o the http:// or https:// to include both in query
-    url = f'http://web.archive.org/cdx/search/cdx?url={JArchive_url}&collapse=digest&limit=-2&fastLatest=true&output=json'  # for some reason, using limit=-1 does not work
-    urls = requests.get(url).text
-    parse_url = json.loads(urls)  # parses the JSON from urls.
-    if len(parse_url) == 0:  # if no results, return None
-        logging.info("no games found in wayback")
-        # alternative: use fallback to get game from scraping j-archive directly
-        raise Exception("no games found in wayback")
-
-    ## Extracts timestamp and original columns from urls and compiles a url list.
-    url_list = []
-    for i in range(1, len(parse_url)): # gets the wayback url
-        orig_url = parse_url[i][2]
-        tstamp = parse_url[i][1]
-        waylink = tstamp + '/' + orig_url
-        final_url = f'http://web.archive.org/web/{waylink}'
-        url_list.append(final_url)
-    latest_url = url_list[-1]
-    return get_generic_game(game_id, latest_url)
-
 
 def get_game_sum(soup):
     date = re.search(

jparty/welcome_widget.py

@@ -13,10 +13,11 @@
 import qrcode
 import time
 from threading import Thread
+import threading
 import logging
 
 from jparty.version import version
-from jparty.retrieve import get_game, get_random_game
+from jparty.retrieve import get_game, get_random_game, IncompleteException
 from jparty.utils import resource_path, add_shadow, DynamicLabel, DynamicButton
 from jparty.helpmsg import helpmsg
 from jparty.style import WINDOWPAL
@@ -117,7 +118,7 @@ def __init__(self, game, parent=None):
         self.gameid_label.setOpenExternalLinks(True)
 
         self.textbox = QLineEdit(self)
-        self.textbox.textChanged.connect(self.show_summary)
+        self.textbox.textEdited.connect(self.load_game)
         f = self.textbox.font()
         self.textbox.setFont(f)
 
@@ -204,50 +205,62 @@ def __random(self):
             while True:
                 game_id = get_random_game()
                 logging.info(f"GAMEID {game_id}")
-                self.game.data = get_game(game_id)
-                if self.game.valid_game():
+                try:
+                    self.game.data = get_game(game_id)
                     break
-                else:
+                except IncompleteException as e:
+                    logging.error("this game is complete")
                     time.sleep(0.25)
 
             self.gameid_trigger.emit(str(game_id))
-            self.summary_trigger.emit(self.game.data.date + "\n" + self.game.data.comments)
+            summary_string = self.game.data.date + "\n" + self.game.data.comments
+
 
         except Exception as e:
             logging.error(e)
-            self.summary_trigger.emit("Cannot get game")
+            summary_string = "Cannot get game"
+
+        self.summary_trigger.emit(summary_string)
+        self.check_start()
 
     def random(self, checked):
         self.summary_trigger.emit("Loading...")
         t = Thread(target=self.__random)
         t.start()
 
-    def __show_summary(self):
+    def __load_game(self):
+        logging.info("textbox changed, reading ID")
         game_id = self.textbox.text()
         try:
-            self.game.data = get_game(game_id)
-            if self.game.valid_game():
-                self.summary_trigger.emit(
-                    self.game.data.date + "\n" + self.game.data.comments
-                )
-            else:
-                self.summary_trigger.emit("Game has blank questions")
+            game_data = get_game(game_id)
+            summary_string = game_data.date + "\n" + game_data.comments
+
+        except IncompleteException as e:
+            game_data = None
+            summary_string = "Game is incomplete"
 
         except Exception as e:
-            logging.error(e)
-            self.summary_trigger.emit("Cannot get game")
+            logging.error(repr(e))
+            game_data = None
+            summary_string = "Cannot get game"
+
+        # make sure the game is the one in the textbox
+        if self.textbox.text() == game_id:
+            self.game.data = game_data
+            self.summary_trigger.emit(summary_string)
+            self.check_start()
+
 
-        self.check_start()
 
     def set_summary(self, text):
         self.summary_label.setText(text)
 
     def set_gameid(self, text):
         self.textbox.setText(text)
 
-    def show_summary(self, text=None):
+    def load_game(self, text=None):
         self.summary_trigger.emit("Loading...")
-        t = Thread(target=self.__show_summary)
+        t = Thread(target=self.__load_game, name="retrieve")
         t.start()
 
         self.check_start()
@@ -259,7 +272,7 @@ def check_start(self):
             self.start_button.setEnabled(False)
 
     def restart(self):
-        self.show_summary(self)
+        self.load_game(self)
 
 
 class QRWidget(StartWidget):