added stashed_files method, new logic to try upload error recovery

Author: fastily

pwiki/gquery.py

@@ -244,6 +244,19 @@ def search(wiki: Wiki, phrase: str, ns: list[Union[NS, str]] = [], limit: Union[
         """
         return GQuery._list_cont(wiki, limit, ListCont.SEARCH, {"srsearch": phrase} | ({"srnamespace": wiki.ns_manager.create_filter(ns)} if ns else {}))
 
+    @staticmethod
+    def stashed_files(wiki: Wiki, limit: Union[int, str] = 1) -> Generator[list[tuple[str, int, str]], None, None]:
+        """Fetch the user's stashed files.  PRECONDITION: You must be logged in for this to work
+
+        Args:
+            wiki (Wiki): The Wiki object to use
+            limit (Union[int, str], optional): The maxmimum number of elements to fetch each iteration.  Defaults to 1.
+
+        Returns:
+            Generator[list[tuple[str, int]], None, None]: A `Generator` which yields a `list` of 3-`tuple` where each tuple is of the form (file key, file size, status).   Known values for status: `"finished"`, `"chunks"`
+        """
+        return GQuery._list_cont(wiki, limit, ListCont.STASHED_FILES)
+
     @staticmethod
     def user_uploads(wiki: Wiki, user: str, limit: Union[int, str] = 1) -> Generator[list[str], None, None]:
         """Gets the uploads of a user.

pwiki/query_constants.py

@@ -75,8 +75,9 @@ class ListCont:
     CATEGORY_MEMBERS = QConstant("categorymembers", limit_key="cmlimit")
     CONTRIBS = QConstant("usercontribs", limit_key="uclimit", retrieve_results=lambda l: [Contrib(e) for e in l])
     DUPLICATE_FILES = QConstant("querypage", {"qppage": "ListDuplicatedFiles"}, "qplimit", lambda q: [e["title"] for e in q["results"]])
-    LOGS = QConstant("logevents", {"leprop": "title|type|user|timestamp|comment|tags"}, "lelimit", retrieve_results=lambda l: [Log(e) for e in l])
+    LOGS = QConstant("logevents", {"leprop": "title|type|user|timestamp|comment|tags"}, "lelimit", lambda l: [Log(e) for e in l])
     PREFIX_INDEX = QConstant("allpages", limit_key="aplimit")
     RANDOM = QConstant("random", {"rnfilterredir": "nonredirects"}, "rnlimit")
     SEARCH = QConstant("search", {"srprop": ""}, "srlimit")
+    STASHED_FILES = QConstant("mystashedfiles", {"msfprop": "size"}, "msflimit", lambda l: [(e["filekey"], e["size"], e["status"]) for e in l])
     USER_UPLOADS = QConstant("allimages", {"aisort": "timestamp"}, "ailimit")

pwiki/waction.py

@@ -8,6 +8,7 @@
 from time import sleep
 from typing import TYPE_CHECKING
 
+from .ns import NS
 from .oquery import OQuery
 from .query_utils import chunker
 from .utils import has_error, make_params, mine_for, read_error
@@ -167,7 +168,7 @@ def purge(wiki: Wiki, titles: Iterable[str]) -> bool:
         return all(WAction._post_action(wiki, "purge", {"titles": "|".join(chunk)}) for chunk in chunker(titles, wiki.prop_title_max))
 
     @staticmethod
-    def unstash(wiki: Wiki, filekey: str, title: str, desc: str = "", summary: str = "", max_retries: int = 5, retry_interval: int = 5) -> bool:
+    def unstash(wiki: Wiki, filekey: str, title: str, desc: str = "", summary: str = "", max_retries: int = 5, retry_interval: int = 30) -> bool:
         """Attempt to unstash a file uploaded to the file stash.
 
         Args:
@@ -177,7 +178,7 @@ def unstash(wiki: Wiki, filekey: str, title: str, desc: str = "", summary: str =
             desc (str, optional): The text to go on the file description page. Defaults to "".
             summary (str, optional): The upload log summary to use. Defaults to "".
             max_retries (int, optional): The maximum number of retry in the event of failure (assuming the server expereinced an error). Defaults to 5.
-            retry_interval (int, optional): The number of seconds to wait in between retries.  Set 0 to disable. Defaults to 5.
+            retry_interval (int, optional): The number of seconds to wait in between retries.  Set 0 to disable. Defaults to 30.
 
         Returns:
             bool: True if unstashing was successful
@@ -186,7 +187,7 @@ def unstash(wiki: Wiki, filekey: str, title: str, desc: str = "", summary: str =
 
         tries = 0
         status = False
-        while tries < max_retries and not (status := bool(WAction._action_and_validate(wiki, "upload", {"filename": title, "text": desc, "comment": summary, "filekey": filekey, "ignorewarnings": 1}, timeout=360))):
+        while tries < max_retries and not (status := bool(WAction._action_and_validate(wiki, "upload", {"filename": title, "text": desc, "comment": summary, "filekey": filekey, "ignorewarnings": 1}, timeout=360)) or wiki.exists(wiki.convert_ns(title, NS.FILE))):
             log.warning("%s: Unstash failed, this is a attempt %d of %d. Sleeping %ds...", wiki, tries + 1, max_retries, retry_interval)
             sleep(retry_interval)
             tries += 1
@@ -224,12 +225,24 @@ def upload_only(wiki: Wiki, path: Path, title: str, max_retries: int = 5) -> str
                     chunk_count += 1
                     pl["offset"] = _CHUNKSIZE * chunk_count
                     pl["filekey"] = filekey
-                    continue
-
-                err_count += 1
-                log.warning("%s: Encountered error while uploading, this was %d/%d", wiki, err_count, max_retries)
-                if err_count > max_retries:
-                    log.error("%s: Exceeded error threshold, abort.", wiki)
+                else:
+                    err_count += 1
+                    log.warning("%s: Encountered error while uploading, this was %d/%d", wiki, err_count, max_retries)
+                    if err_count > max_retries:
+                        log.error("%s: Exceeded error threshold, abort.", wiki)
+                        return
+
+        if chunk_count == total_chunks - 1:  # a poorly configured MediaWiki installation may fail to acknowledge the final chunk, but we can attempt recovery on our end
+            for i in range(max_retries):
+                log.info("%s: Attempting to unmangle filekey, '%s'.  Attempt %d/%d, but first sleeping 30s...", wiki, pl["filekey"], i+1, max_retries)
+                sleep(30)
+
+                if t := next((e for e in wiki.stashed_files() if e[0] == pl["filekey"] or e[1] == fsize), None):
+                    if t[2] == "finished":
+                        log.info("%s: Found a matching filekey: '%s'", wiki, t[0])
+                        return t[0]
+                else:
+                    log.error("No matching filekey found, unable to recover from MediaWiki error!")
                     return
 
         return pl.get("filekey")

pwiki/wiki.py

@@ -697,6 +697,15 @@ def search(self, phrase: str, ns: list[Union[NS, str]] = []) -> list[str]:
         log.info("%s: Searching the wiki for '%s'", self, phrase)
         return flatten_generator(GQuery.search(self, phrase, ns, MAX))
 
+    def stashed_files(self) -> list[tuple[str, int, str]]:
+        """Fetch the user's stashed files.  PRECONDITION: You must be logged in for this to work
+
+        Returns:
+            list[tuple[str, int]]: a `list` of 3-`tuple` where each tuple is of the form (file key, file size, status).  Known values for status: `"finished"`, `"chunks"`
+        """
+        log.info("%s: fetching user's stashed files...", self)
+        return flatten_generator(GQuery.stashed_files(self, MAX))
+
     def templates_on_page(self, title: str) -> list[str]:
         """Fetch templates transcluded on a page.