From 2528d4e023eb4af542e73d2106a439bec33cd205 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sun, 3 Nov 2024 14:39:24 +0200 Subject: [PATCH] Add documentation for the Fetcher class --- scrapling/fetcher.py | 44 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/scrapling/fetcher.py b/scrapling/fetcher.py index 55c59b2..687cadc 100644 --- a/scrapling/fetcher.py +++ b/scrapling/fetcher.py @@ -5,19 +5,55 @@ from scrapling.engines import CamoufoxEngine, PlaywrightEngine, StaticEngine, ch class Fetcher(BaseFetcher): - def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: + """A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on httpx. + + Any additional keyword arguments passed to the methods below are passed to the respective httpx's method directly. + """ + def get(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: + """Make basic HTTP GET request for you but with some added flavors. + :param url: Target url. + :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. + :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds. + :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and create a referer header as if this request had came from Google's search. + :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details. + :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course. + """ response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).get(url, stealthy_headers, **kwargs) return response_object - def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: + def post(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: + """Make basic HTTP POST request for you but with some added flavors. + :param url: Target url. + :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. + :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds. + :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and create a referer header as if this request had came from Google's search. + :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details. + :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course. + """ response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).post(url, stealthy_headers, **kwargs) return response_object - def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: + def put(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: + """Make basic HTTP PUT request for you but with some added flavors. + :param url: Target url + :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. + :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds. + :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and create a referer header as if this request had came from Google's search. + :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details. + :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course. + """ response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).put(url, stealthy_headers, **kwargs) return response_object - def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = None, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: + def delete(self, url: str, follow_redirects: bool = True, timeout: Optional[Union[int, float]] = 10, stealthy_headers: Optional[bool] = True, **kwargs: Dict) -> Response: + """Make basic HTTP DELETE request for you but with some added flavors. + :param url: Target url + :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. + :param timeout: The time to wait for the request to finish in seconds. Default is 10 seconds. + :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and create a referer header as if this request had came from Google's search. + :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details. + :return: A Response object with `url`, `text`, `content`, `status`, `reason`, `encoding`, `cookies`, `headers`, `request_headers`, and the `adaptor` class for parsing, of course. + """ response_object = StaticEngine(follow_redirects, timeout, adaptor_arguments=self.adaptor_arguments).delete(url, stealthy_headers, **kwargs) return response_object