feat: adding the retries argument for all methods of Fetcher class

scrapling/engines/static.py

@@ -8,7 +8,7 @@ from .toolbelt import Response, generate_convincing_referer, generate_headers
 
 
 class StaticEngine:
-    def __init__(self, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, adaptor_arguments: Dict = None):
+    def __init__(self, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, retries: Optional[int] = 3, adaptor_arguments: Dict = None):
         """An engine that utilizes httpx library, check the `Fetcher` class for more documentation.
 
         :param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
@@ -17,6 +17,7 @@ class StaticEngine:
         """
         self.timeout = timeout
         self.follow_redirects = bool(follow_redirects)
+        self.retries = retries
         self._extra_headers = generate_headers(browser_mode=False)
         self.adaptor_arguments = adaptor_arguments if adaptor_arguments else {}
 
@@ -75,7 +76,7 @@ class StaticEngine:
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
         headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers)
-        with httpx.Client(proxy=proxy) as client:
+        with httpx.Client(proxy=proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:
             request = client.get(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
 
         return self._prepare_response(request)
@@ -91,7 +92,7 @@ class StaticEngine:
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
         headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers)
-        with httpx.Client(proxy=proxy) as client:
+        with httpx.Client(proxy=proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:
             request = client.post(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
 
         return self._prepare_response(request)
@@ -107,7 +108,7 @@ class StaticEngine:
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
         headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers)
-        with httpx.Client(proxy=proxy) as client:
+        with httpx.Client(proxy=proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:
             request = client.delete(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
 
         return self._prepare_response(request)
@@ -123,7 +124,7 @@ class StaticEngine:
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
         headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers)
-        with httpx.Client(proxy=proxy) as client:
+        with httpx.Client(proxy=proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client:
             request = client.put(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs)
 
         return self._prepare_response(request)

scrapling/fetchers.py

@@ -10,7 +10,9 @@ class Fetcher(BaseFetcher):
 
     Any additional keyword arguments passed to the methods below are passed to the respective httpx's method directly.
     """
-    def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, proxy: Optional[str] = None, **kwargs: Dict) -> Response:
+    def get(
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP GET request for you but with some added flavors.
 
         :param url: Target url.
@@ -19,13 +21,16 @@ class Fetcher(BaseFetcher):
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request had came from Google's search of this URL's domain.
         :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
+        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).get(url, proxy, stealthy_headers, **kwargs)
+        response_object = StaticEngine(follow_redirects, timeout, retries, adaptor_arguments=self.adaptor_arguments).get(url, proxy, stealthy_headers, **kwargs)
         return response_object
 
-    def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, proxy: Optional[str] = None, **kwargs: Dict) -> Response:
+    def post(
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP POST request for you but with some added flavors.
 
         :param url: Target url.
@@ -34,13 +39,16 @@ class Fetcher(BaseFetcher):
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request came from Google's search of this URL's domain.
         :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
+        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).post(url, proxy, stealthy_headers, **kwargs)
+        response_object = StaticEngine(follow_redirects, timeout, retries, adaptor_arguments=self.adaptor_arguments).post(url, proxy, stealthy_headers, **kwargs)
         return response_object
 
-    def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, proxy: Optional[str] = None, **kwargs: Dict) -> Response:
+    def put(
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP PUT request for you but with some added flavors.
 
         :param url: Target url
@@ -49,14 +57,17 @@ class Fetcher(BaseFetcher):
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request came from Google's search of this URL's domain.
         :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
+        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
 
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).put(url, proxy, stealthy_headers, **kwargs)
+        response_object = StaticEngine(follow_redirects, timeout, retries, adaptor_arguments=self.adaptor_arguments).put(url, proxy, stealthy_headers, **kwargs)
         return response_object
 
-    def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, proxy: Optional[str] = None, **kwargs: Dict) -> Response:
+    def delete(
+            self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True,
+            proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response:
         """Make basic HTTP DELETE request for you but with some added flavors.
 
         :param url: Target url
@@ -65,10 +76,13 @@ class Fetcher(BaseFetcher):
         :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
             create a referer header as if this request came from Google's search of this URL's domain.
         :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
+        :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
         :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
         :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
         """
-        response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).delete(url, proxy, stealthy_headers, **kwargs)
+        response_object = StaticEngine(follow_redirects, timeout, retries, adaptor_arguments=self.adaptor_arguments).delete(url, proxy, stealthy_headers, **kwargs)
+        return response_object
+
         return response_object