@pcrespov use old implementation of pagination generator

Author: bisgaard-itis

clients/python/src/osparc/_api_files_api.py

@@ -31,7 +31,7 @@
 import shutil
 from ._utils import (
     DEFAULT_TIMEOUT_SECONDS,
-    PaginationIterator,
+    PaginationIterable,
     compute_sha256,
     file_chunk_generator,
     Chunk,
@@ -273,7 +273,7 @@ def _search_files(
         file_id: Optional[str] = None,
         sha256_checksum: Optional[str] = None,
         timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS,
-    ) -> PaginationIterator:
+    ) -> PaginationIterable:
         kwargs = {
             "file_id": file_id,
             "sha256_checksum": sha256_checksum,
@@ -285,7 +285,7 @@ def _pagination_method():
                 **{k: v for k, v in kwargs.items() if v is not None}
             )
 
-        return PaginationIterator(
+        return PaginationIterable(
             first_page_callback=_pagination_method,
             api_client=self.api_client,
             base_url=self.api_client.configuration.host,

clients/python/src/osparc/_api_solvers_api.py

@@ -11,7 +11,7 @@
 from ._utils import (
     _DEFAULT_PAGINATION_LIMIT,
     _DEFAULT_PAGINATION_OFFSET,
-    PaginationIterator,
+    PaginationIterable,
 )
 
 import warnings
@@ -47,7 +47,7 @@ def list_solver_ports(
         )
         return page.items if page.items else []
 
-    def iter_jobs(self, solver_key: str, version: str, **kwargs) -> PaginationIterator:
+    def iter_jobs(self, solver_key: str, version: str, **kwargs) -> PaginationIterable:
         """Returns an iterator through which one can iterate over
         all Jobs submitted to the solver
 
@@ -72,14 +72,14 @@ def _pagination_method():
                 **kwargs,
             )
 
-        return PaginationIterator(
+        return PaginationIterable(
             first_page_callback=_pagination_method,
             api_client=self.api_client,
             base_url=self.api_client.configuration.host,
             auth=self._auth,
         )
 
-    def jobs(self, solver_key: str, version: str, **kwargs) -> PaginationIterator:
+    def jobs(self, solver_key: str, version: str, **kwargs) -> PaginationIterable:
         warnings.warn(
             "The 'jobs' method is deprecated and will be removed in a future version. "
             "Please use 'iter_jobs' instead.",

clients/python/src/osparc/_api_studies_api.py

@@ -17,7 +17,7 @@
 from ._utils import (
     _DEFAULT_PAGINATION_LIMIT,
     _DEFAULT_PAGINATION_OFFSET,
-    PaginationIterator,
+    PaginationIterable,
 )
 import warnings
 
@@ -65,7 +65,7 @@ def clone_study(self, study_id: str, **kwargs):
         kwargs = {**kwargs, **ParentProjectInfo().model_dump(exclude_none=True)}
         return super().clone_study(study_id, **kwargs)
 
-    def iter_studies(self, **kwargs) -> PaginationIterator:
+    def iter_studies(self, **kwargs) -> PaginationIterable:
         def _pagination_method():
             page_study = self.list_studies(
                 limit=_DEFAULT_PAGINATION_LIMIT,
@@ -75,14 +75,14 @@ def _pagination_method():
             assert isinstance(page_study, PageStudy)  # nosec
             return page_study
 
-        return PaginationIterator(
+        return PaginationIterable(
             first_page_callback=_pagination_method,
             api_client=self.api_client,
             base_url=self.api_client.configuration.host,
             auth=self._auth,
         )
 
-    def studies(self, **kwargs) -> PaginationIterator:
+    def studies(self, **kwargs) -> PaginationIterable:
         warnings.warn(
             "The 'studies' method is deprecated and will be removed in a future version. "
             "Please use 'iter_studies' instead.",

clients/python/src/osparc/_utils.py

@@ -7,11 +7,9 @@
     Optional,
     TypeVar,
     Union,
-    cast,
-    List,
     NamedTuple,
 )
-from collections.abc import Iterator
+from collections.abc import Iterable, Generator, Sized
 import httpx
 from osparc_client import (
     ApiClient,
@@ -24,6 +22,7 @@
     Study,
 )
 import aiofiles
+from .exceptions import RequestError
 
 _KB = 1024  # in bytes
 _MB = _KB * 1024  # in bytes
@@ -38,7 +37,7 @@
 T = TypeVar("T", Job, File, Solver, Study)
 
 
-class PaginationIterator(Iterator):
+class PaginationIterable(Iterable, Sized):
     """Class for wrapping paginated http methods as iterables. It supports three simple operations:
     - for elm in pagination_iterable
     - elm = next(pagination_iterable)
@@ -53,46 +52,37 @@ def __init__(
     ):
         self._first_page_callback: Callable[[], Page] = first_page_callback
         self._api_client: ApiClient = api_client
+        self._next_page_url: Optional[str] = None
         self._client: httpx.Client = httpx.Client(
             auth=auth, base_url=base_url, follow_redirects=True
         )
-        self._page: Optional[Page] = None
-        self._items_iterator: Optional[Iterator] = None
 
     def __del__(self):
         self._client.close()
 
-    def __next__(self):
-        if self._page is None:
-            self._page = self._first_page_callback()
-            self._items_iterator = iter(cast(List, self._page.items))
-            if self._items_iterator is None:
-                raise StopIteration
-        assert self._items_iterator is not None  # nosec
-        try:
-            return next(self._items_iterator)
-        except StopIteration as exc:
-            next_page_url = self._page.links.next
-            if next_page_url is None:
-                self._page = None
-                self._items_iterator = None
-                raise StopIteration from exc
-            response = self._client.get(next_page_url)
-            self._page = self._api_client._ApiClient__deserialize(
-                response.json(), type(self._page)
-            )
-            assert self._page is not None  # nosec
-            self._items_iterator = iter(cast(List, self._page.items))
-            return next(self._items_iterator)
-
     def __len__(self) -> int:
         """Number of elements which the iterator can produce"""
-        if self._page is not None:
-            return cast(int, self._page.total)
-        self._page = self._first_page_callback()
-        assert self._page is not None  # nosec
-        self._items_iterator = iter(cast(List, self._page.items))
-        return cast(int, self._page.total)
+        page: Page = self._first_page_callback()
+        assert isinstance(page.total, int)
+        return page.total
+
+    def __iter__(self) -> Generator[T, None, None]:
+        """Returns the generator"""
+        if len(self) == 0:
+            return
+        page: Page = self._first_page_callback()
+        while True:
+            assert page.items is not None
+            assert isinstance(page.total, int)
+            yield from page.items
+            if page.links.next is None:
+                return
+            response: httpx.Response = self._client.get(page.links.next)
+            try:
+                response.raise_for_status()
+            except httpx.HTTPStatusError as e:
+                raise RequestError(f"{e}") from e
+            page = self._api_client._ApiClient__deserialize(response.json(), type(page))
 
 
 class Chunk(NamedTuple):

clients/python/test/e2e/test_files_api.py

@@ -8,7 +8,7 @@
 from pathlib import Path
 
 import osparc
-from osparc._utils import PaginationIterator
+from osparc._utils import PaginationIterable
 import pytest
 from memory_profiler import memory_usage
 from typing import Final, Callable
@@ -90,7 +90,7 @@ def test_search_files(
     use_id: bool,
     faker: Faker,
 ) -> None:
-    results: PaginationIterator = files_api._search_files(
+    results: PaginationIterable = files_api._search_files(
         sha256_checksum=f"{faker.sha256()}"
     )
     assert len(results) == 0, "Found file which shouldn't be there"
@@ -102,5 +102,5 @@ def test_search_files(
         else None,
     )
     assert len(results) == 1, "Could not find file after it had been uploaded"
-    file = next(results)
+    file = next(iter(results))
     assert file.checksum == large_server_file.server_file.checksum

clients/python/test/test_osparc/test_basic.py

@@ -12,7 +12,7 @@
 from faker import Faker
 import httpx
 from urllib.parse import urlparse
-from osparc._utils import PaginationIterator
+from osparc._utils import PaginationIterable
 from functools import partial
 from copy import deepcopy
 
@@ -124,14 +124,14 @@ def _sideeffect(all_items: List, request: httpx.Request):
             side_effect=partial(_sideeffect, server_items)
         )
 
-        pagination_iterator = PaginationIterator(
+        pagination_iterator = PaginationIterable(
             lambda: page_file, api_client=api_client, base_url=_base_url, auth=_auth
         )
         client_items = [item for item in pagination_iterator]
         assert len(server_items) > 0
         assert server_items == client_items
 
-        first_client_item = next(pagination_iterator)
+        first_client_item = next(iter(pagination_iterator))
         assert first_client_item == server_items[0]
 
         assert len(pagination_iterator) == page_file.total