freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(Fetcher): Replacing httpx + Adding FetcherSession

Browse Source 
Check out the discord server for details
This commit is contained in:
Karim shoair
2025-05-25 21:10:20 +03:00
parent 5c420b7384
commit 9e68e60161
5 changed files with 876 additions and 586 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/core/_types.py
+3
View File
@@ -18,8 +18,11 @@ from typing import (
TypeVar,
Union,
Match,
Mapping,
Awaitable,
)
SUPPORTED_HTTP_METHODS = Literal["GET", "POST", "PUT", "DELETE"]
SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"]
StrOrBytes = Union[str, bytes]
scrapling/engines/__init__.py
+1 -1
View File
@@ -1,7 +1,7 @@
from .camo import CamoufoxEngine
from .constants import DEFAULT_DISABLED_RESOURCES, DEFAULT_STEALTH_FLAGS
from .pw import PlaywrightEngine
from .static import StaticEngine
from .static import FetcherSession, FetcherClient, AsyncFetcherClient
from .toolbelt import check_if_engine_usable
__all__ = ["CamoufoxEngine", "PlaywrightEngine"]
scrapling/engines/static.py
+855 -139
View File
File diff suppressed because it is too large Load Diff
scrapling/fetchers.py
+16 -444
View File
@@ -9,462 +9,34 @@ from scrapling.core._types import (
Iterable,
)
from scrapling.engines import (
FetcherSession,
CamoufoxEngine,
PlaywrightEngine,
StaticEngine,
check_if_engine_usable,
FetcherClient as _FetcherClient,
AsyncFetcherClient as _AsyncFetcherClient,
)
from scrapling.engines.toolbelt import BaseFetcher, Response
__FetcherClientInstance__ = _FetcherClient()
class Fetcher(BaseFetcher):
"""A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on httpx.
"""A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`."""
Any additional keyword arguments passed to the methods below are passed to the respective httpx's method directly.
"""
@classmethod
def get(
cls,
url: str,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = 10,
stealthy_headers: bool = True,
proxy: Optional[str] = None,
retries: Optional[int] = 3,
cookies: Optional[Dict] = None,
custom_config: Dict = None,
**kwargs: Dict,
) -> Response:
"""Make basic HTTP GET request for you but with some added flavors.
:param url: Target url.
:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
:param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
create a referer header as if this request had came from Google's search of this URL's domain.
:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
:param cookies: Set cookies for the next request.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
adaptor_arguments = tuple(
{**cls._generate_parser_arguments(), **custom_config}.items()
)
if not cookies:
cookies = {}
elif not isinstance(cookies, dict):
ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
response_object = StaticEngine(
url,
proxy,
stealthy_headers,
follow_redirects,
timeout,
retries,
tuple(cookies.items()),
adaptor_arguments=adaptor_arguments,
).get(**kwargs)
return response_object
@classmethod
def post(
cls,
url: str,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = 10,
stealthy_headers: bool = True,
proxy: Optional[str] = None,
retries: Optional[int] = 3,
cookies: Optional[Dict] = None,
custom_config: Dict = None,
**kwargs: Dict,
) -> Response:
"""Make basic HTTP POST request for you but with some added flavors.
:param url: Target url.
:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
:param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
create a referer header as if this request came from Google's search of this URL's domain.
:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
:param cookies: Set cookies for the next request.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
adaptor_arguments = tuple(
{**cls._generate_parser_arguments(), **custom_config}.items()
)
if not cookies:
cookies = {}
elif not isinstance(cookies, dict):
ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
response_object = StaticEngine(
url,
proxy,
stealthy_headers,
follow_redirects,
timeout,
retries,
tuple(cookies.items()),
adaptor_arguments=adaptor_arguments,
).post(**kwargs)
return response_object
@classmethod
def put(
cls,
url: str,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = 10,
stealthy_headers: bool = True,
proxy: Optional[str] = None,
retries: Optional[int] = 3,
cookies: Optional[Dict] = None,
custom_config: Dict = None,
**kwargs: Dict,
) -> Response:
"""Make basic HTTP PUT request for you but with some added flavors.
:param url: Target url
:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
:param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
create a referer header as if this request came from Google's search of this URL's domain.
:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
:param cookies: Set cookies for the next request.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
adaptor_arguments = tuple(
{**cls._generate_parser_arguments(), **custom_config}.items()
)
if not cookies:
cookies = {}
elif not isinstance(cookies, dict):
ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
response_object = StaticEngine(
url,
proxy,
stealthy_headers,
follow_redirects,
timeout,
retries,
tuple(cookies.items()),
adaptor_arguments=adaptor_arguments,
).put(**kwargs)
return response_object
@classmethod
def delete(
cls,
url: str,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = 10,
stealthy_headers: bool = True,
proxy: Optional[str] = None,
retries: Optional[int] = 3,
cookies: Optional[Dict] = None,
custom_config: Dict = None,
**kwargs: Dict,
) -> Response:
"""Make basic HTTP DELETE request for you but with some added flavors.
:param url: Target url
:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
:param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
create a referer header as if this request came from Google's search of this URL's domain.
:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
:param cookies: Set cookies for the next request.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
adaptor_arguments = tuple(
{**cls._generate_parser_arguments(), **custom_config}.items()
)
if not cookies:
cookies = {}
elif not isinstance(cookies, dict):
ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
response_object = StaticEngine(
url,
proxy,
stealthy_headers,
follow_redirects,
timeout,
retries,
tuple(cookies.items()),
adaptor_arguments=adaptor_arguments,
).delete(**kwargs)
return response_object
get = __FetcherClientInstance__.get
post = __FetcherClientInstance__.post
put = __FetcherClientInstance__.put
delete = __FetcherClientInstance__.delete
class AsyncFetcher(Fetcher):
@classmethod
async def get(
cls,
url: str,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = 10,
stealthy_headers: bool = True,
proxy: Optional[str] = None,
retries: Optional[int] = 3,
cookies: Optional[Dict] = None,
custom_config: Dict = None,
**kwargs: Dict,
) -> Response:
"""Make basic HTTP GET request for you but with some added flavors.
class AsyncFetcher(BaseFetcher):
"""A basic `Fetcher` class type that can only do basic GET, POST, PUT, and DELETE HTTP requests based on `curl_cffi`."""
:param url: Target url.
:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
:param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
create a referer header as if this request had came from Google's search of this URL's domain.
:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
:param cookies: Set cookies for the next request.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param kwargs: Any additional keyword arguments are passed directly to `httpx.get()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
adaptor_arguments = tuple(
{**cls._generate_parser_arguments(), **custom_config}.items()
)
if not cookies:
cookies = {}
elif not isinstance(cookies, dict):
ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
response_object = await StaticEngine(
url,
proxy,
stealthy_headers,
follow_redirects,
timeout,
retries=retries,
cookies=tuple(cookies.items()),
adaptor_arguments=adaptor_arguments,
).async_get(**kwargs)
return response_object
@classmethod
async def post(
cls,
url: str,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = 10,
stealthy_headers: bool = True,
proxy: Optional[str] = None,
retries: Optional[int] = 3,
cookies: Optional[Dict] = None,
custom_config: Dict = None,
**kwargs: Dict,
) -> Response:
"""Make basic HTTP POST request for you but with some added flavors.
:param url: Target url.
:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
:param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
create a referer header as if this request came from Google's search of this URL's domain.
:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
:param cookies: Set cookies for the next request.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param kwargs: Any additional keyword arguments are passed directly to `httpx.post()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
adaptor_arguments = tuple(
{**cls._generate_parser_arguments(), **custom_config}.items()
)
if not cookies:
cookies = {}
elif not isinstance(cookies, dict):
ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
response_object = await StaticEngine(
url,
proxy,
stealthy_headers,
follow_redirects,
timeout,
retries=retries,
cookies=tuple(cookies.items()),
adaptor_arguments=adaptor_arguments,
).async_post(**kwargs)
return response_object
@classmethod
async def put(
cls,
url: str,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = 10,
stealthy_headers: bool = True,
proxy: Optional[str] = None,
retries: Optional[int] = 3,
cookies: Optional[Dict] = None,
custom_config: Dict = None,
**kwargs: Dict,
) -> Response:
"""Make basic HTTP PUT request for you but with some added flavors.
:param url: Target url
:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
:param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
create a referer header as if this request came from Google's search of this URL's domain.
:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
:param cookies: Set cookies for the next request.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param kwargs: Any additional keyword arguments are passed directly to `httpx.put()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
adaptor_arguments = tuple(
{**cls._generate_parser_arguments(), **custom_config}.items()
)
if not cookies:
cookies = {}
elif not isinstance(cookies, dict):
ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
response_object = await StaticEngine(
url,
proxy,
stealthy_headers,
follow_redirects,
timeout,
retries=retries,
cookies=tuple(cookies.items()),
adaptor_arguments=adaptor_arguments,
).async_put(**kwargs)
return response_object
@classmethod
async def delete(
cls,
url: str,
follow_redirects: bool = True,
timeout: Optional[Union[int, float]] = 10,
stealthy_headers: bool = True,
proxy: Optional[str] = None,
retries: Optional[int] = 3,
cookies: Optional[Dict] = None,
custom_config: Dict = None,
**kwargs: Dict,
) -> Response:
"""Make basic HTTP DELETE request for you but with some added flavors.
:param url: Target url
:param follow_redirects: As the name says -- if enabled (default), redirects will be followed.
:param timeout: The time to wait for the request to finish in seconds. The default is 10 seconds.
:param stealthy_headers: If enabled (default), Fetcher will create and add real browser's headers and
create a referer header as if this request came from Google's search of this URL's domain.
:param proxy: A string of a proxy to use for http and https requests, the format accepted is `http://username:password@localhost:8030`
:param retries: The number of retries to do through httpx if the request failed for any reason. The default is 3 retries.
:param cookies: Set cookies for the next request.
:param custom_config: A dictionary of custom parser arguments to use with this request. Any argument passed will override any class parameters values.
:param kwargs: Any additional keyword arguments are passed directly to `httpx.delete()` function so check httpx documentation for details.
:return: A `Response` object that is the same as `Adaptor` object except it has these added attributes: `status`, `reason`, `cookies`, `headers`, and `request_headers`
"""
if not custom_config:
custom_config = {}
elif not isinstance(custom_config, dict):
ValueError(
f"The custom parser config must be of type dictionary, got {cls.__class__}"
)
adaptor_arguments = tuple(
{**cls._generate_parser_arguments(), **custom_config}.items()
)
if not cookies:
cookies = {}
elif not isinstance(cookies, dict):
ValueError(f"The cookies must be of type dictionary, got {cls.__class__}")
response_object = await StaticEngine(
url,
proxy,
stealthy_headers,
follow_redirects,
timeout,
retries=retries,
cookies=tuple(cookies.items()),
adaptor_arguments=adaptor_arguments,
).async_delete(**kwargs)
return response_object
get = _AsyncFetcherClient.get
post = _AsyncFetcherClient.post
put = _AsyncFetcherClient.put
delete = _AsyncFetcherClient.delete
class StealthyFetcher(BaseFetcher):
setup.py
+1 -2
View File
@@ -48,7 +48,6 @@ setup(
"Programming Language :: Python :: Implementation :: CPython",
"Typing :: Typed",
],
# Instead of using requirements file to dodge possible errors from tox?
install_requires=[
"lxml>=5.0",
"cssselect>=1.2",
@@ -56,7 +55,7 @@ setup(
"click",
"orjson>=3",
"tldextract",
"httpx[brotli,zstd, socks]",
"curl_cffi>=0.11.1",
"playwright>=1.49.1",
"rebrowser-playwright>=1.49.1",
"camoufox[geoip]>=0.4.11",