From b10cfd3643593b0eef9b08758be46adf7cda769f Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 15 Dec 2024 16:51:58 +0200 Subject: [PATCH] feat: Adding `AsyncFetcher` class version of `Fetcher` The first step in fully supporting async --- .flake8 | 1 - scrapling/engines/static.py | 40 +++++++++++++------ scrapling/fetchers.py | 76 +++++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 13 deletions(-) diff --git a/.flake8 b/.flake8 index cd2987a..fae58af 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,3 @@ [flake8] ignore = E501, F401 -extend-ignore = E999 exclude = .git,.venv,__pycache__,docs,.github,build,dist,tests,benchmarks.py \ No newline at end of file diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 8dd67fa..9d5bed7 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -87,11 +87,15 @@ class StaticEngine: return self._prepare_response(request) + async def async_get(self, **kwargs: Dict) -> Response: + """Make basic async HTTP GET request for you but with some added flavors. + + :param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details. :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` """ - headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers) - with httpx.Client(proxy=proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client: - request = client.get(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs) + headers = self._headers_job(kwargs.pop('headers', {})) + async with httpx.AsyncClient(proxy=self.proxy) as client: + request = await client.get(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs) return self._prepare_response(request) @@ -107,11 +111,15 @@ class StaticEngine: return self._prepare_response(request) + async def async_post(self, **kwargs: Dict) -> Response: + """Make basic async HTTP POST request for you but with some added flavors. + + :param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details. :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` """ - headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers) - with httpx.Client(proxy=proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client: - request = client.post(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs) + headers = self._headers_job(kwargs.pop('headers', {})) + async with httpx.AsyncClient(proxy=self.proxy) as client: + request = await client.post(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs) return self._prepare_response(request) @@ -127,11 +135,15 @@ class StaticEngine: return self._prepare_response(request) + async def async_delete(self, **kwargs: Dict) -> Response: + """Make basic async HTTP DELETE request for you but with some added flavors. + + :param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details. :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` """ - headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers) - with httpx.Client(proxy=proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client: - request = client.delete(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs) + headers = self._headers_job(kwargs.pop('headers', {})) + async with httpx.AsyncClient(proxy=self.proxy) as client: + request = await client.delete(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs) return self._prepare_response(request) @@ -147,10 +159,14 @@ class StaticEngine: return self._prepare_response(request) + async def async_put(self, **kwargs: Dict) -> Response: + """Make basic async HTTP PUT request for you but with some added flavors. + + :param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details. :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` """ - headers = self._headers_job(kwargs.pop('headers', {}), url, stealthy_headers) - with httpx.Client(proxy=proxy, transport=httpx.HTTPTransport(retries=self.retries)) as client: - request = client.put(url=url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs) + headers = self._headers_job(kwargs.pop('headers', {})) + async with httpx.AsyncClient(proxy=self.proxy) as client: + request = await client.put(url=self.url, headers=headers, follow_redirects=self.follow_redirects, timeout=self.timeout, **kwargs) return self._prepare_response(request) diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index 8ad6a78..6f6e4a5 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -87,6 +87,82 @@ class Fetcher(BaseFetcher): response_object = StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries, adaptor_arguments=adaptor_arguments).delete(**kwargs) return response_object + +class AsyncFetcher(Fetcher): + async def get( + self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, + proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response: + """Make basic HTTP GET request for you but with some added flavors. + + :param url: Target url. + :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. + :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. + :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and + create a referer header as if this request had came from Google's search of this URL's domain. + :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` + :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. + :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details. + :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` + """ + adaptor_arguments = tuple(self.adaptor_arguments.items()) + response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_get(**kwargs) + return response_object + + async def post( + self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, + proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response: + """Make basic HTTP POST request for you but with some added flavors. + + :param url: Target url. + :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. + :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. + :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and + create a referer header as if this request came from Google's search of this URL's domain. + :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` + :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. + :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details. + :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` + """ + adaptor_arguments = tuple(self.adaptor_arguments.items()) + response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_post(**kwargs) + return response_object + + async def put( + self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, + proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response: + """Make basic HTTP PUT request for you but with some added flavors. + + :param url: Target url + :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. + :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. + :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and + create a referer header as if this request came from Google's search of this URL's domain. + :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` + :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. + :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details. + :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` + """ + adaptor_arguments = tuple(self.adaptor_arguments.items()) + response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_post(**kwargs) + return response_object + + async def delete( + self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, + proxy: Optional[str] = None, retries: Optional[int] = 3, **kwargs: Dict) -> Response: + """Make basic HTTP DELETE request for you but with some added flavors. + + :param url: Target url + :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. + :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. + :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and + create a referer header as if this request came from Google's search of this URL's domain. + :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` + :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. + :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details. + :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` + """ + adaptor_arguments = tuple(self.adaptor_arguments.items()) + response_object = await StaticEngine(url, proxy, stealthy_headers, follow_redirects, timeout, retries=retries, adaptor_arguments=adaptor_arguments).async_delete(**kwargs) return response_object