diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py index d4cfb37..da6574a 100644 --- a/scrapling/core/_types.py +++ b/scrapling/core/_types.py @@ -18,8 +18,11 @@ from typing import ( TypeVar, Union, Match, + Mapping, + Awaitable, ) +SUPPORTED_HTTP_METHODS = Literal["GET", "POST", "PUT", "DELETE"] SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"] StrOrBytes = Union[str, bytes] diff --git a/scrapling/engines/__init__.py b/scrapling/engines/__init__.py index db9de24..5d0c240 100644 --- a/scrapling/engines/__init__.py +++ b/scrapling/engines/__init__.py @@ -1,7 +1,7 @@ from .camo import CamoufoxEngine from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS from .pw import PlaywrightEngine -from .static import StaticEngine +from .static import FetcherSession, FetcherClient, AsyncFetcherClient from .toolbelt import check_if_engine_usable __all__ = ["CamoufoxEngine", "PlaywrightEngine"] diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py index 187d36e..58f87d8 100644 --- a/scrapling/engines/static.py +++ b/scrapling/engines/static.py @@ -1,61 +1,138 @@ -import httpx -from httpx._models import Response as httpxResponse +from time import sleep as time_sleep +from asyncio import sleep as asyncio_sleep -from scrapling.core._types import Dict, Optional, Tuple, Union -from scrapling.core.utils import log, lru_cache +from curl_cffi.requests.session import CurlError +from curl_cffi.requests import ( + ProxySpec, + CookieTypes, + BrowserTypeLiteral, + Session as CurlSession, + AsyncSession as AsyncCurlSession, +) -from .toolbelt import Response, generate_convincing_referer, generate_headers +from scrapling.core.utils import log +from scrapling.core._types import ( + Dict, + Optional, + Tuple, + Union, + Mapping, + SUPPORTED_HTTP_METHODS, + Awaitable, + List, + Any, +) + +from .toolbelt import ( + Response, + generate_convincing_referer, + generate_headers, + ResponseFactory, +) + +__default_useragent__ = generate_headers(browser_mode=False).get("User-Agent") -@lru_cache(2, typed=True) # Singleton easily -class StaticEngine: +class FetcherSession: + """ + A context manager that provides configured Fetcher sessions. + + When this manager is used in a 'with' or 'async with' block, + it yields a new session configured with the manager's defaults. + A single instance of this manager should ideally be used for one active + session at a time (or sequentially). Re-entering a context with the + same manager instance while a session is already active is disallowed. + """ + def __init__( self, - url: str, + impersonate: Optional[str] = "chrome136", + stealthy_headers: Optional[bool] = True, + proxies: Optional[Dict[str, str]] = None, proxy: Optional[str] = None, - stealthy_headers: bool = True, - follow_redirects: bool = True, - timeout: Optional[Union[int, float]] = None, + proxy_auth: Optional[Tuple[str, str]] = None, + timeout: Optional[Union[int, float]] = 30, + headers: Optional[Dict[str, str]] = None, retries: Optional[int] = 3, - cookies: Optional[Tuple] = None, - adaptor_arguments: Tuple = None, + retry_delay: Optional[int] = 1, + follow_redirects: bool = True, + max_redirects: int = 30, + verify: bool = True, + cert: Optional[Union[str, Tuple[str, str]]] = None, + adaptor_arguments: Optional[Dict] = None, ): - """An engine that utilizes httpx library, check the `Fetcher` class for more documentation. - - :param url: Target url. - :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and - create a referer header as if this request had came from Google's search of this URL's domain. - :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` - :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. - :param cookies: Set cookies for the next request. - :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. - :param adaptor_arguments: The arguments that will be passed in the end while creating the final Adaptor's class. """ - self.url = url - self.proxy = proxy + :param impersonate: Browser version to impersonate. Defaults to "chrome136". + :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. + :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param timeout: Number of seconds to wait before timing out. + :param headers: Headers to include in the session with every request. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param verify: Whether to verify HTTPS certificates. Defaults to True. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param adaptor_arguments: Arguments passed when creating the final Adaptor class. + """ + self.default_impersonate = impersonate self.stealth = stealthy_headers - self.timeout = timeout - self.follow_redirects = bool(follow_redirects) - self.retries = retries - self.cookies = dict(cookies) if cookies else {} - self._extra_headers = generate_headers(browser_mode=False) - # Because we are using `lru_cache` for a slight optimization but both dict/dict_items are not hashable so they can't be cached - # So my solution here was to convert it to tuple then convert it back to dictionary again here as tuples are hashable, ofc `tuple().__hash__()` - self.adaptor_arguments = dict(adaptor_arguments) if adaptor_arguments else {} + self.default_proxies = proxies or {} + self.default_proxy = proxy or None + self.default_proxy_auth = proxy_auth or None + self.default_timeout = timeout + self.default_headers = headers or {} + self.default_retries = retries + self.default_retry_delay = retry_delay + self.default_follow_redirects = follow_redirects + self.default_max_redirects = max_redirects + self.default_verify = verify + self.default_cert = cert + self.adaptor_arguments = adaptor_arguments or {} - def _headers_job(self, headers: Optional[Dict]) -> Dict: + self._curl_session: Optional[CurlSession] = None + self._async_curl_session: Optional[AsyncCurlSession] = None + + def _merge_request_args(self, **kwargs) -> Dict[str, Any]: + """Merge request-specific arguments with default session arguments.""" + request_args = { + "headers": self._headers_job( + kwargs["url"], kwargs.get("headers"), kwargs.pop("stealth") + ), + "proxies": kwargs.get("proxies", self.default_proxies), + "proxy": kwargs.get("proxy", self.default_proxy), + "proxy_auth": kwargs.get("proxy_auth", self.default_proxy_auth), + "timeout": kwargs.get("timeout", self.default_timeout), + "allow_redirects": kwargs.get( + "follow_redirects", self.default_follow_redirects + ), + "max_redirects": kwargs.get("max_redirects", self.default_max_redirects), + "verify": kwargs.get("verify", self.default_verify), + "cert": kwargs.get("cert", self.default_cert), + "impersonate": kwargs.get("impersonate", self.default_impersonate), + **kwargs, + } + return request_args + + def _headers_job( + self, url, headers: Optional[Dict], stealth: Optional[bool] + ) -> Dict: """Adds useragent to headers if it doesn't exist, generates real headers and append it to current headers, and finally generates a referer header that looks like if this request came from Google's search of the current URL's domain. :param headers: Current headers in the request if the user passed any + :param stealth: Whether to enable the `stealthy_headers` argument to this request or not. If `None`, it defaults to the session default value. :return: A dictionary of the new headers. """ - headers = headers or {} + headers = {**self.default_headers, **(headers or {})} headers_keys = set(map(str.lower, headers.keys())) - if self.stealth: + if stealth: extra_headers = generate_headers(browser_mode=False) - # Don't overwrite user supplied headers + # Don't overwrite user-supplied headers extra_headers = { key: value for key, value in extra_headers.items() @@ -63,133 +140,772 @@ class StaticEngine: } headers.update(extra_headers) if "referer" not in headers_keys: - headers.update({"referer": generate_convincing_referer(self.url)}) + headers.update({"referer": generate_convincing_referer(url)}) elif "user-agent" not in headers_keys: - headers["User-Agent"] = generate_headers(browser_mode=False).get( - "User-Agent" - ) + headers["User-Agent"] = __default_useragent__ log.debug( f"Can't find useragent in headers so '{headers['User-Agent']}' was used." ) return headers - def _prepare_response(self, response: httpxResponse) -> Response: - """Takes httpx response and generates `Response` object from it. + def __enter__(self): + """Creates and returns a new synchronous Fetcher Session""" + if self._curl_session: + raise RuntimeError( + "This FetcherSession instance already has an active synchronous session. " + "Create a new FetcherSession instance for a new independent session, " + "or use the current instance sequentially after the previous context has exited." + ) + if ( + self._async_curl_session + ): # Prevent mixing if async is active from this instance + raise RuntimeError( + "This FetcherSession instance has an active asynchronous session. " + "Cannot enter a synchronous context simultaneously with the same manager instance." + ) - :param response: httpx response object - :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` + self._curl_session = CurlSession() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Closes the active synchronous session managed by this instance, if any.""" + if self._curl_session: + self._curl_session.close() + self._curl_session = None + + async def __aenter__(self): + """Creates and returns a new asynchronous Session.""" + if self._async_curl_session: + raise RuntimeError( + "This FetcherSession instance already has an active asynchronous session. " + "Create a new FetcherSession instance for a new independent session, " + "or use the current instance sequentially after the previous context has exited." + ) + if self._curl_session: # Prevent mixing if sync is active from this instance + raise RuntimeError( + "This FetcherSession instance has an active synchronous session. " + "Cannot enter an asynchronous context simultaneously with the same manager instance." + ) + + self._async_curl_session = AsyncCurlSession() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + """Closes the active asynchronous session managed by this instance, if any.""" + if self._async_curl_session: + await self._async_curl_session.close() + self._async_curl_session = None + + def __make_request( + self, + method: SUPPORTED_HTTP_METHODS, + request_args: Dict[str, Any], + max_retries: int, + retry_delay: int, + adaptor_arguments: Optional[Dict] = None, + ) -> Response: """ - return Response( - url=str(response.url), - text=response.text, - body=response.content, - status=response.status_code, - reason=response.reason_phrase, - encoding=response.encoding or "utf-8", - cookies=dict(response.cookies), - headers=dict(response.headers), - request_headers=dict(response.request.headers), - method=response.request.method, - history=[ - self._prepare_response(redirection) for redirection in response.history - ], - **self.adaptor_arguments, + Perform an HTTP request using the configured session. + + :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"] + :param url: Target URL for the request. + :param request_args: Arguments to be passed to the session's `request()` method. + :param max_retries: Maximum number of retries for the request. + :param retry_delay: Number of seconds to wait between retries. + :param adaptor_arguments: Arguments passed when creating the final Adaptor class. + :return: A `Response` object for synchronous requests or an awaitable for asynchronous. + """ + if self._curl_session: + for attempt in range(max_retries): + try: + response = self._curl_session.request(method, **request_args) + # response.raise_for_status() # Retry responses with a status code between 200-400 + return ResponseFactory.from_http_request( + response, adaptor_arguments + ) + except CurlError as e: + if attempt < max_retries - 1: + log.error( + f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..." + ) + time_sleep(retry_delay) + else: + log.error(f"Failed after {max_retries} attempts: {e}") + raise # Raise the exception if all retries fail + + raise RuntimeError("No active session available.") + + async def __make_async_request( + self, + method: SUPPORTED_HTTP_METHODS, + request_args: Dict[str, Any], + max_retries: int, + retry_delay: int, + adaptor_arguments: Optional[Dict] = None, + ) -> Response: + """ + Perform an HTTP request using the configured session. + + :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"] + :param url: Target URL for the request. + :param request_args: Arguments to be passed to the session's `request()` method. + :param max_retries: Maximum number of retries for the request. + :param retry_delay: Number of seconds to wait between retries. + :param adaptor_arguments: Arguments passed when creating the final Adaptor class. + :return: A `Response` object for synchronous requests or an awaitable for asynchronous. + """ + if self._async_curl_session: + for attempt in range(max_retries): + try: + response = await self._async_curl_session.request( + method, **request_args + ) + # response.raise_for_status() # Retry responses with a status code between 200-400 + return ResponseFactory.from_http_request( + response, adaptor_arguments + ) + except CurlError as e: + if attempt < max_retries - 1: + log.error( + f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..." + ) + await asyncio_sleep(retry_delay) + else: + log.error(f"Failed after {max_retries} attempts: {e}") + raise # Raise the exception if all retries fail + + raise RuntimeError("No active session available.") + + def __prepare_and_dispatch( + self, + method: SUPPORTED_HTTP_METHODS, + stealth: Optional[bool] = None, + **kwargs, + ) -> Union[Response, Awaitable[Response]]: + """ + Internal dispatcher. Prepares arguments and calls sync or async request helper. + + :param method: HTTP method to be used, supported methods are ["GET", "POST", "PUT", "DELETE"] + :param stealth: Whether to enable the `stealthy_headers` argument to this request or not. If `None`, it defaults to the session default value. + :param url: Target URL for the request. + :param kwargs: Additional request-specific arguments. + :return: A `Response` object for synchronous requests or an awaitable for asynchronous. + """ + stealth = self.stealth if stealth is None else stealth + + adaptor_arguments = ( + kwargs.pop("adaptor_arguments", {}) or self.adaptor_arguments + ) + max_retries = kwargs.pop("retries", self.default_retries) + retry_delay = kwargs.pop("retry_delay", self.default_retry_delay) + request_args = self._merge_request_args(stealth=stealth, **kwargs) + if self._curl_session: + return self.__make_request( + method, request_args, max_retries, retry_delay, adaptor_arguments + ) + elif self._async_curl_session: + # The returned value is a Coroutine + return self.__make_async_request( + method, request_args, max_retries, retry_delay, adaptor_arguments + ) + + raise RuntimeError("No active session available.") + + def get( + self, + url: str, + params: Optional[Union[Dict, List, Tuple]] = None, # <-- + headers: Optional[Mapping[str, Optional[str]]] = None, + cookies: Optional[CookieTypes] = None, # <-- + timeout: Optional[Union[int, float]] = 30, # <-- + follow_redirects: Optional[bool] = True, # <-- + max_redirects: Optional[int] = 30, # <-- + retries: Optional[int] = 3, + retry_delay: Optional[int] = 1, # <-- + proxies: Optional[ProxySpec] = None, # <-- + proxy: Optional[str] = None, # <-- + proxy_auth: Optional[Tuple[str, str]] = None, + auth: Optional[Tuple[str, str]] = None, + verify: Optional[bool] = True, # <-- + cert: Optional[Union[str, Tuple[str, str]]] = None, + impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <-- + stealthy_headers: Optional[bool] = True, + **kwargs, + ) -> Union[Response, Awaitable[Response]]: + """ + Perform a GET request. + + :param url: Target URL for the request. + :param params: Query string parameters for the request. + :param headers: Headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Defaults to "chrome136". + :param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :return: A `Response` object or an awaitable for async. + """ + request_args = { + "url": url, + "params": params, + "headers": headers, + "cookies": cookies, + "timeout": timeout, + "retry_delay": retry_delay, + "allow_redirects": follow_redirects, + "max_redirects": max_redirects, + "retries": retries, + "proxies": proxies, + "proxy": proxy, + "proxy_auth": proxy_auth, + "auth": auth, + "verify": verify, + "cert": cert, + "impersonate": impersonate, + **kwargs, + } + return self.__prepare_and_dispatch( + "GET", stealth=stealthy_headers, **request_args ) - def _make_request(self, method: str, **kwargs) -> Response: - headers = self._headers_job(kwargs.pop("headers", {})) - with httpx.Client( - proxy=self.proxy, - transport=httpx.HTTPTransport(retries=self.retries), - cookies=self.cookies, - ) as client: - request = getattr(client, method)( - url=self.url, - headers=headers, - follow_redirects=self.follow_redirects, - timeout=self.timeout, - **kwargs, - ) - return self._prepare_response(request) - - async def _async_make_request(self, method: str, **kwargs) -> Response: - headers = self._headers_job(kwargs.pop("headers", {})) - async with httpx.AsyncClient( - proxy=self.proxy, - transport=httpx.AsyncHTTPTransport(retries=self.retries), - cookies=self.cookies, - ) as client: - request = await getattr(client, method)( - url=self.url, - headers=headers, - follow_redirects=self.follow_redirects, - timeout=self.timeout, - **kwargs, - ) - return self._prepare_response(request) - - def get(self, **kwargs: Dict) -> Response: - """Make basic HTTP GET request for you but with some added flavors. - - :param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details. - :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` + def post( + self, + url: str, + data: Optional[Union[Dict, str]] = None, + json: Optional[Union[Dict, List]] = None, + headers: Optional[Mapping[str, Optional[str]]] = None, + params: Optional[Union[Dict, List, Tuple]] = None, # <-- + cookies: Optional[CookieTypes] = None, # <-- + timeout: Optional[Union[int, float]] = 30, # <-- + follow_redirects: Optional[bool] = True, # <-- + max_redirects: Optional[int] = 30, # <-- + retries: Optional[int] = 3, + retry_delay: Optional[int] = 1, # <-- + proxies: Optional[ProxySpec] = None, # <-- + proxy: Optional[str] = None, # <-- + proxy_auth: Optional[Tuple[str, str]] = None, + auth: Optional[Tuple[str, str]] = None, + verify: Optional[bool] = True, # <-- + cert: Optional[Union[str, Tuple[str, str]]] = None, + impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <-- + stealthy_headers: Optional[bool] = True, + **kwargs, + ) -> Union[Response, Awaitable[Response]]: """ - return self._make_request("get", **kwargs) + Perform a POST request. - async def async_get(self, **kwargs: Dict) -> Response: - """Make basic async HTTP GET request for you but with some added flavors. - - :param kwargs: Any keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details. - :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` + :param url: Target URL for the request. + :param data: Form data to include in the request body. + :param json: A JSON serializable object to include in the body of the request. + :param headers: Headers to include in the request. + :param params: Query string parameters for the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. Defaults to True. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Defaults to "chrome136". + :param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :return: A `Response` object or an awaitable for async. """ - return await self._async_make_request("get", **kwargs) + request_args = { + "url": url, + "data": data, + "json": json, + "headers": headers, + "params": params, + "cookies": cookies, + "timeout": timeout, + "retry_delay": retry_delay, + "proxy": proxy, + "impersonate": impersonate, + "allow_redirects": follow_redirects, + "max_redirects": max_redirects, + "retries": retries, + "proxies": proxies, + "proxy_auth": proxy_auth, + "auth": auth, + "verify": verify, + "cert": cert, + **kwargs, + } + return self.__prepare_and_dispatch( + "POST", stealth=stealthy_headers, **request_args + ) - def post(self, **kwargs: Dict) -> Response: - """Make basic HTTP POST request for you but with some added flavors. - - :param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details. - :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` + def put( + self, + url: str, + data: Optional[Union[Dict, str]] = None, + json: Optional[Union[Dict, List]] = None, + headers: Optional[Mapping[str, Optional[str]]] = None, + params: Optional[Union[Dict, List, Tuple]] = None, # <-- + cookies: Optional[CookieTypes] = None, # <-- + timeout: Optional[Union[int, float]] = 30, # <-- + follow_redirects: Optional[bool] = True, # <-- + max_redirects: Optional[int] = 30, # <-- + retries: Optional[int] = 3, + retry_delay: Optional[int] = 1, # <-- + proxies: Optional[ProxySpec] = None, # <-- + proxy: Optional[str] = None, # <-- + proxy_auth: Optional[Tuple[str, str]] = None, + auth: Optional[Tuple[str, str]] = None, + verify: Optional[bool] = True, # <-- + cert: Optional[Union[str, Tuple[str, str]]] = None, + impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <-- + stealthy_headers: Optional[bool] = True, + **kwargs, + ) -> Union[Response, Awaitable[Response]]: """ - return self._make_request("post", **kwargs) + Perform a PUT request. - async def async_post(self, **kwargs: Dict) -> Response: - """Make basic async HTTP POST request for you but with some added flavors. - - :param kwargs: Any keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details. - :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` + :param url: Target URL for the request. + :param data: Form data to include in the request body. + :param json: A JSON serializable object to include in the body of the request. + :param headers: Headers to include in the request. + :param params: Query string parameters for the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. Defaults to True. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Defaults to "chrome136". + :param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :return: A `Response` object or an awaitable for async. """ - return await self._async_make_request("post", **kwargs) + request_args = { + "url": url, + "data": data, + "json": json, + "headers": headers, + "params": params, + "cookies": cookies, + "timeout": timeout, + "retry_delay": retry_delay, + "proxy": proxy, + "impersonate": impersonate, + "allow_redirects": follow_redirects, + "max_redirects": max_redirects, + "retries": retries, + "proxies": proxies, + "proxy_auth": proxy_auth, + "auth": auth, + "verify": verify, + "cert": cert, + **kwargs, + } + return self.__prepare_and_dispatch( + "PUT", stealth=stealthy_headers, **request_args + ) - def delete(self, **kwargs: Dict) -> Response: - """Make basic HTTP DELETE request for you but with some added flavors. - - :param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details. - :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` + def delete( + self, + url: str, + data: Optional[Union[Dict, str]] = None, + json: Optional[Union[Dict, List]] = None, + headers: Optional[Mapping[str, Optional[str]]] = None, + params: Optional[Union[Dict, List, Tuple]] = None, # <-- + cookies: Optional[CookieTypes] = None, # <-- + timeout: Optional[Union[int, float]] = 30, # <-- + follow_redirects: Optional[bool] = True, # <-- + max_redirects: Optional[int] = 30, # <-- + retries: Optional[int] = 3, + retry_delay: Optional[int] = 1, # <-- + proxies: Optional[ProxySpec] = None, # <-- + proxy: Optional[str] = None, # <-- + proxy_auth: Optional[Tuple[str, str]] = None, + auth: Optional[Tuple[str, str]] = None, + verify: Optional[bool] = True, # <-- + cert: Optional[Union[str, Tuple[str, str]]] = None, + impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <-- + stealthy_headers: Optional[bool] = True, + **kwargs, + ) -> Union[Response, Awaitable[Response]]: """ - return self._make_request("delete", **kwargs) + Perform a DELETE request. - async def async_delete(self, **kwargs: Dict) -> Response: - """Make basic async HTTP DELETE request for you but with some added flavors. - - :param kwargs: Any keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details. - :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` + :param url: Target URL for the request. + :param data: Form data to include in the request body. + :param json: A JSON serializable object to include in the body of the request. + :param headers: Headers to include in the request. + :param params: Query string parameters for the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. Defaults to True. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Defaults to "chrome136". + :param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the [`curl_cffi.requests.Session().request()`, `curl_cffi.requests.AsyncSession().request()`] method. + :return: A `Response` object or an awaitable for async. """ - return await self._async_make_request("delete", **kwargs) + request_args = { + "url": url, + # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5, + # But some websites accept it, it depends on the implementation used. + "data": data, + "json": json, + "headers": headers, + "params": params, + "cookies": cookies, + "timeout": timeout, + "retry_delay": retry_delay, + "proxy": proxy, + "impersonate": impersonate, + "allow_redirects": follow_redirects, + "max_redirects": max_redirects, + "retries": retries, + "proxies": proxies, + "proxy_auth": proxy_auth, + "auth": auth, + "verify": verify, + "cert": cert, + **kwargs, + } + return self.__prepare_and_dispatch( + "DELETE", stealth=stealthy_headers, **request_args + ) - def put(self, **kwargs: Dict) -> Response: - """Make basic HTTP PUT request for you but with some added flavors. - :param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details. - :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` +class FetcherClient(FetcherSession): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # Using one session for all requests is faster than using stateless `curl_cffi.get` + self.__enter__ = None + self.__exit__ = None + self.__aenter__ = None + self.__aexit__ = None + self._curl_session = CurlSession() + + +class AsyncFetcherClient: + # Since curl_cffi doesn't support making async requests without sessions + # And using a single session for many requests at the same time in async doesn't sit well with curl_cffi. + # We do this + + @staticmethod + async def get( + url: str, + params: Optional[Union[Dict, List, Tuple]] = None, # <-- + headers: Optional[Mapping[str, Optional[str]]] = None, + cookies: Optional[CookieTypes] = None, # <-- + timeout: Optional[Union[int, float]] = 30, # <-- + follow_redirects: Optional[bool] = True, # <-- + max_redirects: Optional[int] = 30, # <-- + retries: Optional[int] = 3, + retry_delay: Optional[int] = 1, # <-- + proxies: Optional[ProxySpec] = None, # <-- + proxy: Optional[str] = None, # <-- + proxy_auth: Optional[Tuple[str, str]] = None, + auth: Optional[Tuple[str, str]] = None, + verify: Optional[bool] = True, # <-- + cert: Optional[Union[str, Tuple[str, str]]] = None, + impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <-- + stealthy_headers: Optional[bool] = True, + **kwargs, + ) -> Response: """ - return self._make_request("put", **kwargs) + Perform a GET request. - async def async_put(self, **kwargs: Dict) -> Response: - """Make basic async HTTP PUT request for you but with some added flavors. - - :param kwargs: Any keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details. - :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` + :param url: Target URL for the request. + :param params: Query string parameters for the request. + :param headers: Headers to include in the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Defaults to "chrome136". + :param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method. + :return: An awaitable `Response` object. """ - return await self._async_make_request("put", **kwargs) + request_args = { + "url": url, + "params": params, + "headers": headers, + "cookies": cookies, + "timeout": timeout, + "retry_delay": retry_delay, + "allow_redirects": follow_redirects, + "max_redirects": max_redirects, + "retries": retries, + "proxies": proxies, + "proxy": proxy, + "proxy_auth": proxy_auth, + "auth": auth, + "verify": verify, + "cert": cert, + "impersonate": impersonate, + **kwargs, + } + async with FetcherSession(stealthy_headers=stealthy_headers) as client: + return await client.get(**request_args) + + @staticmethod + async def post( + url: str, + data: Optional[Union[Dict, str]] = None, + json: Optional[Union[Dict, List]] = None, + headers: Optional[Mapping[str, Optional[str]]] = None, + params: Optional[Union[Dict, List, Tuple]] = None, # <-- + cookies: Optional[CookieTypes] = None, # <-- + timeout: Optional[Union[int, float]] = 30, # <-- + follow_redirects: Optional[bool] = True, # <-- + max_redirects: Optional[int] = 30, # <-- + retries: Optional[int] = 3, + retry_delay: Optional[int] = 1, # <-- + proxies: Optional[ProxySpec] = None, # <-- + proxy: Optional[str] = None, # <-- + proxy_auth: Optional[Tuple[str, str]] = None, + auth: Optional[Tuple[str, str]] = None, + verify: Optional[bool] = True, # <-- + cert: Optional[Union[str, Tuple[str, str]]] = None, + impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <-- + stealthy_headers: Optional[bool] = True, + **kwargs, + ) -> Response: + """ + Perform a POST request. + + :param url: Target URL for the request. + :param data: Form data to include in the request body. + :param json: A JSON serializable object to include in the body of the request. + :param headers: Headers to include in the request. + :param params: Query string parameters for the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. Defaults to True. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Defaults to "chrome136". + :param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method. + :return: An awaitable `Response` object. + """ + request_args = { + "url": url, + "data": data, + "json": json, + "headers": headers, + "params": params, + "cookies": cookies, + "timeout": timeout, + "retry_delay": retry_delay, + "proxy": proxy, + "impersonate": impersonate, + "allow_redirects": follow_redirects, + "max_redirects": max_redirects, + "retries": retries, + "proxies": proxies, + "proxy_auth": proxy_auth, + "auth": auth, + "verify": verify, + "cert": cert, + **kwargs, + } + async with FetcherSession(stealthy_headers=stealthy_headers) as client: + return await client.post(**request_args) + + @staticmethod + async def put( + url: str, + data: Optional[Union[Dict, str]] = None, + json: Optional[Union[Dict, List]] = None, + headers: Optional[Mapping[str, Optional[str]]] = None, + params: Optional[Union[Dict, List, Tuple]] = None, # <-- + cookies: Optional[CookieTypes] = None, # <-- + timeout: Optional[Union[int, float]] = 30, # <-- + follow_redirects: Optional[bool] = True, # <-- + max_redirects: Optional[int] = 30, # <-- + retries: Optional[int] = 3, + retry_delay: Optional[int] = 1, # <-- + proxies: Optional[ProxySpec] = None, # <-- + proxy: Optional[str] = None, # <-- + proxy_auth: Optional[Tuple[str, str]] = None, + auth: Optional[Tuple[str, str]] = None, + verify: Optional[bool] = True, # <-- + cert: Optional[Union[str, Tuple[str, str]]] = None, + impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <-- + stealthy_headers: Optional[bool] = True, + **kwargs, + ) -> Response: + """ + Perform a PUT request. + + :param url: Target URL for the request. + :param data: Form data to include in the request body. + :param json: A JSON serializable object to include in the body of the request. + :param headers: Headers to include in the request. + :param params: Query string parameters for the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. Defaults to True. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Defaults to "chrome136". + :param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method. + :return: An awaitable `Response` object. + """ + request_args = { + "url": url, + "data": data, + "json": json, + "headers": headers, + "params": params, + "cookies": cookies, + "timeout": timeout, + "retry_delay": retry_delay, + "proxy": proxy, + "impersonate": impersonate, + "allow_redirects": follow_redirects, + "max_redirects": max_redirects, + "retries": retries, + "proxies": proxies, + "proxy_auth": proxy_auth, + "auth": auth, + "verify": verify, + "cert": cert, + **kwargs, + } + async with FetcherSession(stealthy_headers=stealthy_headers) as client: + return await client.put(**request_args) + + @staticmethod + async def delete( + url: str, + data: Optional[Union[Dict, str]] = None, + json: Optional[Union[Dict, List]] = None, + headers: Optional[Mapping[str, Optional[str]]] = None, + params: Optional[Union[Dict, List, Tuple]] = None, # <-- + cookies: Optional[CookieTypes] = None, # <-- + timeout: Optional[Union[int, float]] = 30, # <-- + follow_redirects: Optional[bool] = True, # <-- + max_redirects: Optional[int] = 30, # <-- + retries: Optional[int] = 3, + retry_delay: Optional[int] = 1, # <-- + proxies: Optional[ProxySpec] = None, # <-- + proxy: Optional[str] = None, # <-- + proxy_auth: Optional[Tuple[str, str]] = None, + auth: Optional[Tuple[str, str]] = None, + verify: Optional[bool] = True, # <-- + cert: Optional[Union[str, Tuple[str, str]]] = None, + impersonate: Optional[BrowserTypeLiteral] = "chrome136", # <-- + stealthy_headers: Optional[bool] = True, + **kwargs, + ) -> Response: + """ + Perform a DELETE request. + + :param url: Target URL for the request. + :param data: Form data to include in the request body. + :param json: A JSON serializable object to include in the body of the request. + :param headers: Headers to include in the request. + :param params: Query string parameters for the request. + :param cookies: Cookies to use in the request. + :param timeout: Number of seconds to wait before timing out. + :param follow_redirects: Whether to follow redirects. Defaults to True. + :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited. + :param retries: Number of retry attempts. Defaults to 3. + :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second. + :param proxies: Dict of proxies to use. Format: {"http": proxy_url, "https": proxy_url}. + :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030". + Cannot be used together with the `proxies` parameter. + :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password). + :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported. + :param verify: Whether to verify HTTPS certificates. Defaults to True. + :param cert: Tuple of (cert, key) filenames for the client certificate. + :param impersonate: Browser version to impersonate. Defaults to "chrome136". + :param stealthy_headers: If enabled for this request (default), it creates and adds real browser headers. It also referer header as if it is from a Google search of URL's domain. + :param kwargs: Additional keyword arguments to pass to the `curl_cffi.requests.AsyncSession().request()` method. + :return: An awaitable `Response` object. + """ + request_args = { + "url": url, + # Careful of sending a body in a DELETE request, it might cause some websites to reject the request as per https://www.rfc-editor.org/rfc/rfc7231#section-4.3.5, + # But some websites accept it, it depends on the implementation used. + "data": data, + "json": json, + "headers": headers, + "params": params, + "cookies": cookies, + "timeout": timeout, + "retry_delay": retry_delay, + "proxy": proxy, + "impersonate": impersonate, + "allow_redirects": follow_redirects, + "max_redirects": max_redirects, + "retries": retries, + "proxies": proxies, + "proxy_auth": proxy_auth, + "auth": auth, + "verify": verify, + "cert": cert, + **kwargs, + } + async with FetcherSession(stealthy_headers=stealthy_headers) as client: + return await client.delete(**request_args) diff --git a/scrapling/fetchers.py b/scrapling/fetchers.py index bbd4b9f..b20b0a1 100644 --- a/scrapling/fetchers.py +++ b/scrapling/fetchers.py @@ -9,462 +9,34 @@ from scrapling.core._types import ( Iterable, ) from scrapling.engines import ( + FetcherSession, CamoufoxEngine, PlaywrightEngine, - StaticEngine, check_if_engine_usable, + FetcherClient as _FetcherClient, + AsyncFetcherClient as _AsyncFetcherClient, ) from scrapling.engines.toolbelt import BaseFetcher, Response +__FetcherClientInstance__ = _FetcherClient() + class Fetcher(BaseFetcher): - """A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on httpx. + """A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`.""" - Any additional keyword arguments passed to the methods below are passed to the respective httpx's method directly. - """ - - @classmethod - def get( - cls, - url: str, - follow_redirects: bool = True, - timeout: Optional[Union[int, float]] = 10, - stealthy_headers: bool = True, - proxy: Optional[str] = None, - retries: Optional[int] = 3, - cookies: Optional[Dict] = None, - custom_config: Dict = None, - **kwargs: Dict, - ) -> Response: - """Make basic HTTP GET request for you but with some added flavors. - - :param url: Target url. - :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. - :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. - :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and - create a referer header as if this request had came from Google's search of this URL's domain. - :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` - :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. - :param cookies: Set cookies for the next request. - :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. - :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details. - :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` - """ - if not custom_config: - custom_config = {} - elif not isinstance(custom_config, dict): - ValueError( - f"The custom parser config must be of type dictionary, got {cls.__class__}" - ) - - adaptor_arguments = tuple( - {**cls._generate_parser_arguments(), **custom_config}.items() - ) - - if not cookies: - cookies = {} - elif not isinstance(cookies, dict): - ValueError(f"The cookies must be of type dictionary, got {cls.__class__}") - - response_object = StaticEngine( - url, - proxy, - stealthy_headers, - follow_redirects, - timeout, - retries, - tuple(cookies.items()), - adaptor_arguments=adaptor_arguments, - ).get(**kwargs) - return response_object - - @classmethod - def post( - cls, - url: str, - follow_redirects: bool = True, - timeout: Optional[Union[int, float]] = 10, - stealthy_headers: bool = True, - proxy: Optional[str] = None, - retries: Optional[int] = 3, - cookies: Optional[Dict] = None, - custom_config: Dict = None, - **kwargs: Dict, - ) -> Response: - """Make basic HTTP POST request for you but with some added flavors. - - :param url: Target url. - :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. - :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. - :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and - create a referer header as if this request came from Google's search of this URL's domain. - :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` - :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. - :param cookies: Set cookies for the next request. - :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. - :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details. - :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` - """ - if not custom_config: - custom_config = {} - elif not isinstance(custom_config, dict): - ValueError( - f"The custom parser config must be of type dictionary, got {cls.__class__}" - ) - - adaptor_arguments = tuple( - {**cls._generate_parser_arguments(), **custom_config}.items() - ) - - if not cookies: - cookies = {} - elif not isinstance(cookies, dict): - ValueError(f"The cookies must be of type dictionary, got {cls.__class__}") - - response_object = StaticEngine( - url, - proxy, - stealthy_headers, - follow_redirects, - timeout, - retries, - tuple(cookies.items()), - adaptor_arguments=adaptor_arguments, - ).post(**kwargs) - return response_object - - @classmethod - def put( - cls, - url: str, - follow_redirects: bool = True, - timeout: Optional[Union[int, float]] = 10, - stealthy_headers: bool = True, - proxy: Optional[str] = None, - retries: Optional[int] = 3, - cookies: Optional[Dict] = None, - custom_config: Dict = None, - **kwargs: Dict, - ) -> Response: - """Make basic HTTP PUT request for you but with some added flavors. - - :param url: Target url - :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. - :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. - :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and - create a referer header as if this request came from Google's search of this URL's domain. - :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` - :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. - :param cookies: Set cookies for the next request. - :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. - :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details. - - :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` - """ - if not custom_config: - custom_config = {} - elif not isinstance(custom_config, dict): - ValueError( - f"The custom parser config must be of type dictionary, got {cls.__class__}" - ) - - adaptor_arguments = tuple( - {**cls._generate_parser_arguments(), **custom_config}.items() - ) - - if not cookies: - cookies = {} - elif not isinstance(cookies, dict): - ValueError(f"The cookies must be of type dictionary, got {cls.__class__}") - - response_object = StaticEngine( - url, - proxy, - stealthy_headers, - follow_redirects, - timeout, - retries, - tuple(cookies.items()), - adaptor_arguments=adaptor_arguments, - ).put(**kwargs) - return response_object - - @classmethod - def delete( - cls, - url: str, - follow_redirects: bool = True, - timeout: Optional[Union[int, float]] = 10, - stealthy_headers: bool = True, - proxy: Optional[str] = None, - retries: Optional[int] = 3, - cookies: Optional[Dict] = None, - custom_config: Dict = None, - **kwargs: Dict, - ) -> Response: - """Make basic HTTP DELETE request for you but with some added flavors. - - :param url: Target url - :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. - :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. - :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and - create a referer header as if this request came from Google's search of this URL's domain. - :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` - :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. - :param cookies: Set cookies for the next request. - :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. - :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details. - :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` - """ - if not custom_config: - custom_config = {} - elif not isinstance(custom_config, dict): - ValueError( - f"The custom parser config must be of type dictionary, got {cls.__class__}" - ) - - adaptor_arguments = tuple( - {**cls._generate_parser_arguments(), **custom_config}.items() - ) - - if not cookies: - cookies = {} - elif not isinstance(cookies, dict): - ValueError(f"The cookies must be of type dictionary, got {cls.__class__}") - - response_object = StaticEngine( - url, - proxy, - stealthy_headers, - follow_redirects, - timeout, - retries, - tuple(cookies.items()), - adaptor_arguments=adaptor_arguments, - ).delete(**kwargs) - return response_object + get = __FetcherClientInstance__.get + post = __FetcherClientInstance__.post + put = __FetcherClientInstance__.put + delete = __FetcherClientInstance__.delete -class AsyncFetcher(Fetcher): - @classmethod - async def get( - cls, - url: str, - follow_redirects: bool = True, - timeout: Optional[Union[int, float]] = 10, - stealthy_headers: bool = True, - proxy: Optional[str] = None, - retries: Optional[int] = 3, - cookies: Optional[Dict] = None, - custom_config: Dict = None, - **kwargs: Dict, - ) -> Response: - """Make basic HTTP GET request for you but with some added flavors. +class AsyncFetcher(BaseFetcher): + """A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`.""" - :param url: Target url. - :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. - :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. - :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and - create a referer header as if this request had came from Google's search of this URL's domain. - :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` - :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. - :param cookies: Set cookies for the next request. - :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. - :param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details. - :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` - """ - if not custom_config: - custom_config = {} - elif not isinstance(custom_config, dict): - ValueError( - f"The custom parser config must be of type dictionary, got {cls.__class__}" - ) - - adaptor_arguments = tuple( - {**cls._generate_parser_arguments(), **custom_config}.items() - ) - - if not cookies: - cookies = {} - elif not isinstance(cookies, dict): - ValueError(f"The cookies must be of type dictionary, got {cls.__class__}") - - response_object = await StaticEngine( - url, - proxy, - stealthy_headers, - follow_redirects, - timeout, - retries=retries, - cookies=tuple(cookies.items()), - adaptor_arguments=adaptor_arguments, - ).async_get(**kwargs) - return response_object - - @classmethod - async def post( - cls, - url: str, - follow_redirects: bool = True, - timeout: Optional[Union[int, float]] = 10, - stealthy_headers: bool = True, - proxy: Optional[str] = None, - retries: Optional[int] = 3, - cookies: Optional[Dict] = None, - custom_config: Dict = None, - **kwargs: Dict, - ) -> Response: - """Make basic HTTP POST request for you but with some added flavors. - - :param url: Target url. - :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. - :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. - :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and - create a referer header as if this request came from Google's search of this URL's domain. - :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` - :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. - :param cookies: Set cookies for the next request. - :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. - :param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details. - :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` - """ - if not custom_config: - custom_config = {} - elif not isinstance(custom_config, dict): - ValueError( - f"The custom parser config must be of type dictionary, got {cls.__class__}" - ) - - adaptor_arguments = tuple( - {**cls._generate_parser_arguments(), **custom_config}.items() - ) - - if not cookies: - cookies = {} - elif not isinstance(cookies, dict): - ValueError(f"The cookies must be of type dictionary, got {cls.__class__}") - - response_object = await StaticEngine( - url, - proxy, - stealthy_headers, - follow_redirects, - timeout, - retries=retries, - cookies=tuple(cookies.items()), - adaptor_arguments=adaptor_arguments, - ).async_post(**kwargs) - return response_object - - @classmethod - async def put( - cls, - url: str, - follow_redirects: bool = True, - timeout: Optional[Union[int, float]] = 10, - stealthy_headers: bool = True, - proxy: Optional[str] = None, - retries: Optional[int] = 3, - cookies: Optional[Dict] = None, - custom_config: Dict = None, - **kwargs: Dict, - ) -> Response: - """Make basic HTTP PUT request for you but with some added flavors. - - :param url: Target url - :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. - :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. - :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and - create a referer header as if this request came from Google's search of this URL's domain. - :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` - :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. - :param cookies: Set cookies for the next request. - :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. - :param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details. - :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` - """ - if not custom_config: - custom_config = {} - elif not isinstance(custom_config, dict): - ValueError( - f"The custom parser config must be of type dictionary, got {cls.__class__}" - ) - - adaptor_arguments = tuple( - {**cls._generate_parser_arguments(), **custom_config}.items() - ) - - if not cookies: - cookies = {} - elif not isinstance(cookies, dict): - ValueError(f"The cookies must be of type dictionary, got {cls.__class__}") - - response_object = await StaticEngine( - url, - proxy, - stealthy_headers, - follow_redirects, - timeout, - retries=retries, - cookies=tuple(cookies.items()), - adaptor_arguments=adaptor_arguments, - ).async_put(**kwargs) - return response_object - - @classmethod - async def delete( - cls, - url: str, - follow_redirects: bool = True, - timeout: Optional[Union[int, float]] = 10, - stealthy_headers: bool = True, - proxy: Optional[str] = None, - retries: Optional[int] = 3, - cookies: Optional[Dict] = None, - custom_config: Dict = None, - **kwargs: Dict, - ) -> Response: - """Make basic HTTP DELETE request for you but with some added flavors. - - :param url: Target url - :param follow_redirects: As the name says -- if enabled (default), redirects will be followed. - :param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds. - :param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and - create a referer header as if this request came from Google's search of this URL's domain. - :param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030` - :param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries. - :param cookies: Set cookies for the next request. - :param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values. - :param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details. - :return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers` - """ - if not custom_config: - custom_config = {} - elif not isinstance(custom_config, dict): - ValueError( - f"The custom parser config must be of type dictionary, got {cls.__class__}" - ) - - adaptor_arguments = tuple( - {**cls._generate_parser_arguments(), **custom_config}.items() - ) - - if not cookies: - cookies = {} - elif not isinstance(cookies, dict): - ValueError(f"The cookies must be of type dictionary, got {cls.__class__}") - - response_object = await StaticEngine( - url, - proxy, - stealthy_headers, - follow_redirects, - timeout, - retries=retries, - cookies=tuple(cookies.items()), - adaptor_arguments=adaptor_arguments, - ).async_delete(**kwargs) - return response_object + get = _AsyncFetcherClient.get + post = _AsyncFetcherClient.post + put = _AsyncFetcherClient.put + delete = _AsyncFetcherClient.delete class StealthyFetcher(BaseFetcher): diff --git a/setup.py b/setup.py index 28e1d54..ec5fe67 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,6 @@ setup( "Programming Language :: Python :: Implementation :: CPython", "Typing :: Typed", ], - # Instead of using requirements file to dodge possible errors from tox? install_requires=[ "lxml>=5.0", "cssselect>=1.2", @@ -56,7 +55,7 @@ setup( "click", "orjson>=3", "tldextract", - "httpx[brotli,zstd, socks]", + "curl_cffi>=0.11.1", "playwright>=1.49.1", "rebrowser-playwright>=1.49.1", "camoufox[geoip]>=0.4.11",