port-labs
diff --git a/‎CHANGELOG.md
Lines changed: 9 additions & 1 deletion b/‎CHANGELOG.md
Lines changed: 9 additions & 1 deletion
diff --git a/‎port_ocean/helpers/async_client.py
Lines changed: 7 additions & 8 deletions b/‎port_ocean/helpers/async_client.py
Lines changed: 7 additions & 8 deletions
diff --git a/‎port_ocean/helpers/retry.py
Lines changed: 162 additions & 72 deletions b/‎port_ocean/helpers/retry.py
Lines changed: 162 additions & 72 deletions
@@ -6,6 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 <!-- towncrier release notes start -->
+
+## 0.28.0 (2025-08-19)
+
+### Improvements
+
+- Made HTTP retry config extensible with new RetryConfig class and callback to apply per-integration policies without code changes.
+- Added rate-limit aware retries through configurable retry-after headers (e.g., X-RateLimit-Reset) and additional retry status codes that extend safe defaults.
+- Control max_attempts/base_delay/jitter_ratio/max_backoff_wait and override retryable_methods via callback.
+
 ## 0.27.10 (2025-08-24)
 
 ### Improvements
@@ -20,7 +29,6 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 - Removed premature cleanup of Prometheus metrics after subprocess finish to fix reconciliation stuck on pending
 - Enhanced sync state tracking across different phases
 
-
 ## 0.27.8 (2025-08-18)
 
 ### Improvements
 
@@ -3,7 +3,7 @@
 import httpx
 from loguru import logger
 
-from port_ocean.helpers.retry import RetryTransport
+from port_ocean.helpers.retry import RetryTransport, RetryConfig
 from port_ocean.helpers.stream import Stream
 
 
@@ -18,10 +18,12 @@ def __init__(
         self,
         transport_class: Type[RetryTransport] = RetryTransport,
         transport_kwargs: dict[str, Any] | None = None,
+        retry_config: RetryConfig | None = None,
         **kwargs: Any,
     ):
         self._transport_kwargs = transport_kwargs
         self._transport_class = transport_class
+        self._retry_config = retry_config
         super().__init__(**kwargs)
 
     def _init_transport(  # type: ignore[override]
@@ -33,9 +35,8 @@ def _init_transport(  # type: ignore[override]
             return super()._init_transport(transport=transport, **kwargs)
 
         return self._transport_class(
-            wrapped_transport=httpx.AsyncHTTPTransport(
-                **kwargs,
-            ),
+            wrapped_transport=httpx.AsyncHTTPTransport(**kwargs),
+            retry_config=self._retry_config,
             logger=logger,
             **(self._transport_kwargs or {}),
         )
@@ -44,10 +45,8 @@ def _init_proxy_transport(  # type: ignore[override]
         self, proxy: httpx.Proxy, **kwargs: Any
     ) -> httpx.AsyncBaseTransport:
         return self._transport_class(
-            wrapped_transport=httpx.AsyncHTTPTransport(
-                proxy=proxy,
-                **kwargs,
-            ),
+            wrapped_transport=httpx.AsyncHTTPTransport(proxy=proxy, **kwargs),
+            retry_config=self._retry_config,
             logger=logger,
             **(self._transport_kwargs or {}),
         )
 
@@ -4,12 +4,24 @@
 from datetime import datetime
 from functools import partial
 from http import HTTPStatus
-from typing import Any, Callable, Coroutine, Iterable, Mapping, Union, cast
+from typing import (
+    Any,
+    Callable,
+    Coroutine,
+    Iterable,
+    Mapping,
+    Union,
+    cast,
+    Optional,
+    List,
+)
 import httpx
 from dateutil.parser import isoparse
 import logging
 
+MAX_BACKOFF_WAIT_IN_SECONDS = 60
 _ON_RETRY_CALLBACK: Callable[[httpx.Request], httpx.Request] | None = None
+_RETRY_CONFIG_CALLBACK: Callable[[], "RetryConfig"] | None = None
 
 
 def register_on_retry_callback(
@@ -19,6 +31,92 @@ def register_on_retry_callback(
     _ON_RETRY_CALLBACK = _on_retry_callback
 
 
+def register_retry_config_callback(
+    retry_config_callback: Callable[[], "RetryConfig"]
+) -> None:
+    """Register a callback function that returns a RetryConfig instance.
+
+    The callback will be called when a RetryTransport needs to be created.
+
+    Args:
+        retry_config_callback: A function that returns a RetryConfig instance
+    """
+    global _RETRY_CONFIG_CALLBACK
+    _RETRY_CONFIG_CALLBACK = retry_config_callback
+
+
+class RetryConfig:
+    """Configuration class for retry behavior that can be customized per integration."""
+
+    def __init__(
+        self,
+        max_attempts: int = 10,
+        max_backoff_wait: float = MAX_BACKOFF_WAIT_IN_SECONDS,
+        base_delay: float = 0.1,
+        jitter_ratio: float = 0.1,
+        respect_retry_after_header: bool = True,
+        retryable_methods: Optional[Iterable[str]] = None,
+        retry_status_codes: Optional[Iterable[int]] = None,
+        retry_after_headers: Optional[List[str]] = None,
+        additional_retry_status_codes: Optional[Iterable[int]] = None,
+    ):
+        """
+        Initialize retry configuration.
+
+        Args:
+            max_attempts: Maximum number of retry attempts
+            max_backoff_wait: Maximum backoff wait time in seconds
+            base_delay: Base delay for exponential backoff
+            jitter_ratio: Jitter ratio for backoff (0-0.5)
+            respect_retry_after_header: Whether to respect Retry-After header
+            retryable_methods: HTTP methods that can be retried (overrides defaults if provided)
+            retry_status_codes: DEPRECATED - use additional_retry_status_codes instead
+            retry_after_headers: Custom headers to check for retry timing (e.g., ['X-RateLimit-Reset', 'Retry-After'])
+            additional_retry_status_codes: Additional status codes to retry (extends system defaults)
+        """
+        self.max_attempts = max_attempts
+        self.max_backoff_wait = max_backoff_wait
+        self.base_delay = base_delay
+        self.jitter_ratio = jitter_ratio
+        self.respect_retry_after_header = respect_retry_after_header
+
+        # Default retryable methods - always include these unless explicitly overridden
+        default_methods = frozenset(
+            ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"]
+        )
+        self.retryable_methods = (
+            frozenset(retryable_methods) if retryable_methods else default_methods
+        )
+
+        # Default retry status codes - always include these for system reliability
+        default_status_codes = frozenset(
+            [
+                HTTPStatus.TOO_MANY_REQUESTS,
+                HTTPStatus.BAD_GATEWAY,
+                HTTPStatus.SERVICE_UNAVAILABLE,
+                HTTPStatus.GATEWAY_TIMEOUT,
+                HTTPStatus.UNAUTHORIZED,
+                HTTPStatus.BAD_REQUEST,
+            ]
+        )
+
+        # Additional status codes to retry (extends defaults)
+        additional_codes = (
+            frozenset(additional_retry_status_codes)
+            if additional_retry_status_codes
+            else frozenset()
+        )
+
+        # Combine defaults with additional codes for extensibility
+        self.retry_status_codes = default_status_codes | additional_codes
+        self.retry_after_headers = retry_after_headers or ["Retry-After"]
+
+        if jitter_ratio < 0 or jitter_ratio > 0.5:
+            raise ValueError(
+                f"Jitter ratio should be between 0 and 0.5, actual {jitter_ratio}"
+            )
+
+
 # Adapted from https://github.com/encode/httpx/issues/108#issuecomment-1434439481
 class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
     """
@@ -41,32 +139,16 @@ class RetryTransport(httpx.AsyncBaseTransport, httpx.BaseTransport):
             ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"].
         retry_status_codes (Iterable[int], optional): The HTTP status codes that can be retried. Defaults to
             [429, 502, 503, 504].
+        retry_config (RetryConfig, optional): Configuration for retry behavior. If not provided, uses defaults.
+        logger (Any, optional): The logger to use for logging retries.
 
     Attributes:
         _wrapped_transport (Union[httpx.BaseTransport, httpx.AsyncBaseTransport]): The underlying HTTP transport
             being wrapped.
-        _max_attempts (int): The maximum number of times to retry a request.
-        _backoff_factor (float): The factor by which the wait time increases with each retry attempt.
-        _respect_retry_after_header (bool): Whether to respect the Retry-After header in HTTP responses.
-        _retryable_methods (frozenset): The HTTP methods that can be retried.
-        _retry_status_codes (frozenset): The HTTP status codes that can be retried.
-        _jitter_ratio (float): The amount of jitter to add to the backoff time.
-        _max_backoff_wait (float): The maximum time to wait between retries in seconds.
+        _retry_config (RetryConfig): The retry configuration object.
+        _logger (Any): The logger to use for logging retries.
     """
 
-    RETRYABLE_METHODS = frozenset(["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"])
-    RETRYABLE_STATUS_CODES = frozenset(
-        [
-            HTTPStatus.TOO_MANY_REQUESTS,
-            HTTPStatus.BAD_GATEWAY,
-            HTTPStatus.SERVICE_UNAVAILABLE,
-            HTTPStatus.GATEWAY_TIMEOUT,
-            HTTPStatus.UNAUTHORIZED,
-            HTTPStatus.BAD_REQUEST,
-        ]
-    )
-    MAX_BACKOFF_WAIT_IN_SECONDS = 60
-
     def __init__(
         self,
         wrapped_transport: Union[httpx.BaseTransport, httpx.AsyncBaseTransport],
@@ -77,6 +159,7 @@ def __init__(
         respect_retry_after_header: bool = True,
         retryable_methods: Iterable[str] | None = None,
         retry_status_codes: Iterable[int] | None = None,
+        retry_config: Optional[RetryConfig] = None,
         logger: Any | None = None,
     ) -> None:
         """
@@ -106,29 +189,27 @@ def __init__(
             retry_status_codes (Iterable[int], optional):
                 The HTTP status codes that can be retried.
                 Defaults to [429, 502, 503, 504].
+            retry_config (RetryConfig, optional):
+                Configuration for retry behavior. If not provided, uses default configuration.
             logger (Any): The logger to use for logging retries.
         """
         self._wrapped_transport = wrapped_transport
-        if jitter_ratio < 0 or jitter_ratio > 0.5:
-            raise ValueError(
-                f"Jitter ratio should be between 0 and 0.5, actual {jitter_ratio}"
+
+        if retry_config is not None:
+            self._retry_config = retry_config
+        elif _RETRY_CONFIG_CALLBACK is not None:
+            self._retry_config = _RETRY_CONFIG_CALLBACK()
+        else:
+            self._retry_config = RetryConfig(
+                max_attempts=max_attempts,
+                max_backoff_wait=max_backoff_wait,
+                base_delay=base_delay,
+                jitter_ratio=jitter_ratio,
+                respect_retry_after_header=respect_retry_after_header,
+                retryable_methods=retryable_methods,
+                retry_status_codes=retry_status_codes,
             )
 
-        self._max_attempts = max_attempts
-        self._base_delay = base_delay
-        self._respect_retry_after_header = respect_retry_after_header
-        self._retryable_methods = (
-            frozenset(retryable_methods)
-            if retryable_methods
-            else self.RETRYABLE_METHODS
-        )
-        self._retry_status_codes = (
-            frozenset(retry_status_codes)
-            if retry_status_codes
-            else self.RETRYABLE_STATUS_CODES
-        )
-        self._jitter_ratio = jitter_ratio
-        self._max_backoff_wait = max_backoff_wait
         self._logger = logger
 
     def handle_request(self, request: httpx.Request) -> httpx.Response:
@@ -206,12 +287,13 @@ def close(self) -> None:
         transport.close()
 
     def _is_retryable_method(self, request: httpx.Request) -> bool:
-        return request.method in self._retryable_methods or request.extensions.get(
-            "retryable", False
+        return (
+            request.method in self._retry_config.retryable_methods
+            or request.extensions.get("retryable", False)
         )
 
     def _should_retry(self, response: httpx.Response) -> bool:
-        return response.status_code in self._retry_status_codes
+        return response.status_code in self._retry_config.retry_status_codes
 
     def _log_error(
         self,
@@ -314,46 +396,54 @@ def _log_response_size(
         )
 
     async def _should_retry_async(self, response: httpx.Response) -> bool:
-        return response.status_code in self._retry_status_codes
+        return response.status_code in self._retry_config.retry_status_codes
 
     def _calculate_sleep(
         self, attempts_made: int, headers: Union[httpx.Headers, Mapping[str, str]]
     ) -> float:
-        # Retry-After
-        # The Retry-After response HTTP header indicates how long the user agent should wait before
-        # making a follow-up request. There are three main cases this header is used:
-        # - When sent with a 503 (Service Unavailable) response, this indicates how long the service
-        #   is expected to be unavailable.
-        # - When sent with a 429 (Too Many Requests) response, this indicates how long to wait before
-        #   making a new request.
-        # - When sent with a redirect response, such as 301 (Moved Permanently), this indicates the
-        #   minimum time that the user agent is asked to wait before issuing the redirected request.
-        retry_after_header = (headers.get("Retry-After") or "").strip()
-        if self._respect_retry_after_header and retry_after_header:
-            if retry_after_header.isdigit():
-                return float(retry_after_header)
-
-            try:
-                parsed_date = isoparse(
-                    retry_after_header
-                ).astimezone()  # converts to local time
-                diff = (parsed_date - datetime.now().astimezone()).total_seconds()
-                if diff > 0:
-                    return min(diff, self._max_backoff_wait)
-            except ValueError:
-                pass
-
-        backoff = self._base_delay * (2 ** (attempts_made - 1))
-        jitter = (backoff * self._jitter_ratio) * random.choice([1, -1])
+        # Check custom retry headers first, then fall back to Retry-After
+        if self._retry_config.respect_retry_after_header:
+            for header_name in self._retry_config.retry_after_headers:
+                if header_value := (headers.get(header_name) or "").strip():
+                    sleep_time = self._parse_retry_header(header_value)
+                    if sleep_time is not None:
+                        return min(sleep_time, self._retry_config.max_backoff_wait)
+
+        # Fall back to exponential backoff
+        backoff = self._retry_config.base_delay * (2 ** (attempts_made - 1))
+        jitter = (backoff * self._retry_config.jitter_ratio) * random.choice([1, -1])
         total_backoff = backoff + jitter
-        return min(total_backoff, self._max_backoff_wait)
+        return min(total_backoff, self._retry_config.max_backoff_wait)
+
+    def _parse_retry_header(self, header_value: str) -> Optional[float]:
+        """Parse retry header value and return sleep time in seconds.
+
+        Args:
+            header_value: The header value to parse (e.g., "30", "2023-12-01T12:00:00Z")
+
+        Returns:
+            Sleep time in seconds if parsing succeeds, None if the header value cannot be parsed
+        """
+        if header_value.isdigit():
+            return float(header_value)
+
+        try:
+            # Try to parse as ISO date (common for rate limit headers like X-RateLimit-Reset)
+            parsed_date = isoparse(header_value).astimezone()
+            diff = (parsed_date - datetime.now().astimezone()).total_seconds()
+            if diff > 0:
+                return diff
+        except ValueError:
+            pass
+
+        return None
 
     async def _retry_operation_async(
         self,
         request: httpx.Request,
         send_method: Callable[..., Coroutine[Any, Any, httpx.Response]],
     ) -> httpx.Response:
-        remaining_attempts = self._max_attempts
+        remaining_attempts = self._retry_config.max_attempts
         attempts_made = 0
         response: httpx.Response | None = None
         error: Exception | None = None
@@ -403,7 +493,7 @@ def _retry_operation(
         request: httpx.Request,
         send_method: Callable[..., httpx.Response],
     ) -> httpx.Response:
-        remaining_attempts = self._max_attempts
+        remaining_attempts = self._retry_config.max_attempts
         attempts_made = 0
         response: httpx.Response | None = None
         error: Exception | None = None