freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: applying the new ruff rules to all files

Browse Source
This commit is contained in:
Karim shoair
2025-09-13 03:22:53 +03:00
parent 60be9dc816
commit 330d03559c
18 changed files with 176 additions and 572 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/_browsers/_base.py
+7 -25
View File
@@ -80,9 +80,7 @@ class SyncSession:
return self.page_pool.add_page(page)
@staticmethod
def _get_with_precedence(
request_value: Any, session_value: Any, sentinel_value: object
) -> Any:
def _get_with_precedence(request_value: Any, session_value: Any, sentinel_value: object) -> Any:
"""Get value with request-level priority over session-level"""
return request_value if request_value is not sentinel_value else session_value
@@ -169,11 +167,7 @@ class DynamicSessionMixin:
self.wait_selector_state = config.wait_selector_state
self.selector_config = config.selector_config
self.page_action = config.page_action
self._headers_keys = (
set(map(str.lower, self.extra_headers.keys()))
if self.extra_headers
else set()
)
self._headers_keys = set(map(str.lower, self.extra_headers.keys())) if self.extra_headers else set()
self.__initiate_browser_options__()
def __initiate_browser_options__(self):
@@ -184,9 +178,7 @@ class DynamicSessionMixin:
self.headless,
self.proxy,
self.locale,
tuple(self.extra_headers.items())
if self.extra_headers
else tuple(),
tuple(self.extra_headers.items()) if self.extra_headers else tuple(),
self.useragent,
self.real_chrome,
self.stealth,
@@ -194,9 +186,7 @@ class DynamicSessionMixin:
self.disable_webgl,
)
)
self.launch_options["extra_http_headers"] = dict(
self.launch_options["extra_http_headers"]
)
self.launch_options["extra_http_headers"] = dict(self.launch_options["extra_http_headers"])
self.launch_options["proxy"] = dict(self.launch_options["proxy"]) or None
self.context_options = dict()
else:
@@ -206,16 +196,12 @@ class DynamicSessionMixin:
_context_kwargs(
self.proxy,
self.locale,
tuple(self.extra_headers.items())
if self.extra_headers
else tuple(),
tuple(self.extra_headers.items()) if self.extra_headers else tuple(),
self.useragent,
self.stealth,
)
)
self.context_options["extra_http_headers"] = dict(
self.context_options["extra_http_headers"]
)
self.context_options["extra_http_headers"] = dict(self.context_options["extra_http_headers"])
self.context_options["proxy"] = dict(self.context_options["proxy"]) or None
@@ -249,11 +235,7 @@ class StealthySessionMixin:
self.selector_config = config.selector_config
self.additional_args = config.additional_args
self.page_action = config.page_action
self._headers_keys = (
set(map(str.lower, self.extra_headers.keys()))
if self.extra_headers
else set()
)
self._headers_keys = set(map(str.lower, self.extra_headers.keys())) if self.extra_headers else set()
self.__initiate_browser_options__()
def __initiate_browser_options__(self):
scrapling/engines/_browsers/_page.py
+1 -3
View File
@@ -6,9 +6,7 @@ from playwright.async_api import Page as AsyncPage
from scrapling.core._types import Optional, List, Literal
PageState = Literal[
"finished", "ready", "busy", "error"
] # States that a page can be in
PageState = Literal["finished", "ready", "busy", "error"] # States that a page can be in
@dataclass
scrapling/engines/_browsers/_validators.py
+6 -14
View File
@@ -25,9 +25,7 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
stealth: bool = False
wait: int | float = 0
page_action: Optional[Callable] = None
proxy: Optional[str | Dict[str, str]] = (
None # The default value for proxy in Playwright's source is `None`
)
proxy: Optional[str | Dict[str, str]] = None # The default value for proxy in Playwright's source is `None`
locale: str = "en-US"
extra_headers: Optional[Dict[str, str]] = None
useragent: Optional[str] = None
@@ -46,10 +44,8 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False):
raise ValueError("max_pages must be between 1 and 50")
if self.timeout < 0:
raise ValueError("timeout must be >= 0")
if self.page_action is not None and not callable(self.page_action):
raise TypeError(
f"page_action must be callable, got {type(self.page_action).__name__}"
)
if self.page_action and not callable(self.page_action):
raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
if self.proxy:
self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
if self.cdp_url:
@@ -108,9 +104,7 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
cookies: Optional[List[Dict]] = None
google_search: bool = True
extra_headers: Optional[Dict[str, str]] = None
proxy: Optional[str | Dict[str, str]] = (
None # The default value for proxy in Playwright's source is `None`
)
proxy: Optional[str | Dict[str, str]] = None # The default value for proxy in Playwright's source is `None`
os_randomize: bool = False
disable_ads: bool = False
geoip: bool = False
@@ -123,10 +117,8 @@ class CamoufoxConfig(Struct, kw_only=True, frozen=False):
raise ValueError("max_pages must be between 1 and 50")
if self.timeout < 0:
raise ValueError("timeout must be >= 0")
if self.page_action is not None and not callable(self.page_action):
raise TypeError(
f"page_action must be callable, got {type(self.page_action).__name__}"
)
if self.page_action and not callable(self.page_action):
raise TypeError(f"page_action must be callable, got {type(self.page_action).__name__}")
if self.proxy:
self.proxy = construct_proxy_dict(self.proxy, as_tuple=True)
scrapling/engines/static.py
+22 -68
View File
@@ -108,13 +108,9 @@ class FetcherSession:
headers = self.get_with_precedence(kwargs, "headers", self.default_headers)
stealth = self.get_with_precedence(kwargs, "stealth", self.stealth)
impersonate = self.get_with_precedence(
kwargs, "impersonate", self.default_impersonate
)
impersonate = self.get_with_precedence(kwargs, "impersonate", self.default_impersonate)
if self.get_with_precedence(
kwargs, "http3", self.default_http3
): # pragma: no cover
if self.get_with_precedence(kwargs, "http3", self.default_http3): # pragma: no cover
request_args["http_version"] = CurlHttpVersion.V3ONLY
if impersonate:
log.warning(
@@ -126,25 +122,13 @@ class FetcherSession:
"url": url,
# Curl automatically generates the suitable browser headers when you use `impersonate`
"headers": self._headers_job(url, headers, stealth, bool(impersonate)),
"proxies": self.get_with_precedence(
kwargs, "proxies", self.default_proxies
),
"proxies": self.get_with_precedence(kwargs, "proxies", self.default_proxies),
"proxy": self.get_with_precedence(kwargs, "proxy", self.default_proxy),
"proxy_auth": self.get_with_precedence(
kwargs, "proxy_auth", self.default_proxy_auth
),
"timeout": self.get_with_precedence(
kwargs, "timeout", self.default_timeout
),
"allow_redirects": self.get_with_precedence(
kwargs, "allow_redirects", self.default_follow_redirects
),
"max_redirects": self.get_with_precedence(
kwargs, "max_redirects", self.default_max_redirects
),
"verify": self.get_with_precedence(
kwargs, "verify", self.default_verify
),
"proxy_auth": self.get_with_precedence(kwargs, "proxy_auth", self.default_proxy_auth),
"timeout": self.get_with_precedence(kwargs, "timeout", self.default_timeout),
"allow_redirects": self.get_with_precedence(kwargs, "allow_redirects", self.default_follow_redirects),
"max_redirects": self.get_with_precedence(kwargs, "max_redirects", self.default_max_redirects),
"verify": self.get_with_precedence(kwargs, "verify", self.default_verify),
"cert": self.get_with_precedence(kwargs, "cert", self.default_cert),
"impersonate": impersonate,
**{
@@ -192,18 +176,12 @@ class FetcherSession:
extra_headers = generate_headers(browser_mode=False)
# Don't overwrite user-supplied headers
extra_headers = {
key: value
for key, value in extra_headers.items()
if key.lower() not in headers_keys
}
extra_headers = {key: value for key, value in extra_headers.items() if key.lower() not in headers_keys}
headers.update(extra_headers)
elif "user-agent" not in headers_keys and not impersonate_enabled:
headers["User-Agent"] = __default_useragent__
log.debug(
f"Can't find useragent in headers so '{headers['User-Agent']}' was used."
)
log.debug(f"Can't find useragent in headers so '{headers['User-Agent']}' was used.")
return headers
@@ -215,9 +193,7 @@ class FetcherSession:
"Create a new FetcherSession instance for a new independent session, "
"or use the current instance sequentially after the previous context has exited."
)
if (
self._async_curl_session
): # Prevent mixing if async is active from this instance
if self._async_curl_session: # Prevent mixing if async is active from this instance
raise RuntimeError(
"This FetcherSession instance has an active asynchronous session. "
"Cannot enter a synchronous context simultaneously with the same manager instance."
@@ -275,9 +251,7 @@ class FetcherSession:
:return: A `Response` object for synchronous requests or an awaitable for asynchronous.
"""
session = self._curl_session
if session is True and not any(
(self.__enter__, self.__exit__, self.__aenter__, self.__aexit__)
):
if session is True and not any((self.__enter__, self.__exit__, self.__aenter__, self.__aexit__)):
# For usage inside FetcherClient
# It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time.
session = CurlSession()
@@ -290,9 +264,7 @@ class FetcherSession:
return ResponseFactory.from_http_request(response, selector_config)
except CurlError as e: # pragma: no cover
if attempt < max_retries - 1:
log.error(
f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..."
)
log.error(f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds...")
time_sleep(retry_delay)
else:
log.error(f"Failed after {max_retries} attempts: {e}")
@@ -320,9 +292,7 @@ class FetcherSession:
:return: A `Response` object for synchronous requests or an awaitable for asynchronous.
"""
session = self._async_curl_session
if session is True and not any(
(self.__enter__, self.__exit__, self.__aenter__, self.__aexit__)
):
if session is True and not any((self.__enter__, self.__exit__, self.__aenter__, self.__aexit__)):
# For usage inside the ` AsyncFetcherClient ` class, and that's for several reasons
# 1. It turns out `curl_cffi` caches impersonation state, so if you turned it off, then on then off, it won't be off on the last time.
# 2. `curl_cffi` doesn't support making async requests without sessions
@@ -337,9 +307,7 @@ class FetcherSession:
return ResponseFactory.from_http_request(response, selector_config)
except CurlError as e: # pragma: no cover
if attempt < max_retries - 1:
log.error(
f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds..."
)
log.error(f"Attempt {attempt + 1} failed: {e}. Retrying in {retry_delay} seconds...")
await asyncio_sleep(retry_delay)
else:
log.error(f"Failed after {max_retries} attempts: {e}")
@@ -372,19 +340,13 @@ class FetcherSession:
selector_config = kwargs.pop("selector_config", {}) or self.selector_config
max_retries = self.get_with_precedence(kwargs, "retries", self.default_retries)
retry_delay = self.get_with_precedence(
kwargs, "retry_delay", self.default_retry_delay
)
retry_delay = self.get_with_precedence(kwargs, "retry_delay", self.default_retry_delay)
request_args = self._merge_request_args(stealth=stealth, **kwargs)
if self._curl_session:
return self.__make_request(
method, request_args, max_retries, retry_delay, selector_config
)
return self.__make_request(method, request_args, max_retries, retry_delay, selector_config)
elif self._async_curl_session:
# The returned value is a Coroutine
return self.__make_async_request(
method, request_args, max_retries, retry_delay, selector_config
)
return self.__make_async_request(method, request_args, max_retries, retry_delay, selector_config)
raise RuntimeError("No active session available.")
@@ -455,9 +417,7 @@ class FetcherSession:
"http3": http3,
**kwargs,
}
return self.__prepare_and_dispatch(
"GET", stealth=stealthy_headers, **request_args
)
return self.__prepare_and_dispatch("GET", stealth=stealthy_headers, **request_args)
def post(
self,
@@ -532,9 +492,7 @@ class FetcherSession:
"http3": http3,
**kwargs,
}
return self.__prepare_and_dispatch(
"POST", stealth=stealthy_headers, **request_args
)
return self.__prepare_and_dispatch("POST", stealth=stealthy_headers, **request_args)
def put(
self,
@@ -609,9 +567,7 @@ class FetcherSession:
"http3": http3,
**kwargs,
}
return self.__prepare_and_dispatch(
"PUT", stealth=stealthy_headers, **request_args
)
return self.__prepare_and_dispatch("PUT", stealth=stealthy_headers, **request_args)
def delete(
self,
@@ -688,9 +644,7 @@ class FetcherSession:
"http3": http3,
**kwargs,
}
return self.__prepare_and_dispatch(
"DELETE", stealth=stealthy_headers, **request_args
)
return self.__prepare_and_dispatch("DELETE", stealth=stealthy_headers, **request_args)
class FetcherClient(FetcherSession):
scrapling/engines/toolbelt/convertor.py
+13 -39
View File
@@ -18,9 +18,7 @@ class ResponseFactory:
"""
@classmethod
def _process_response_history(
cls, first_response: SyncResponse, parser_arguments: Dict
) -> list[Response]:
def _process_response_history(cls, first_response: SyncResponse, parser_arguments: Dict) -> list[Response]:
"""Process response history to build a list of `Response` objects"""
history = []
current_request = first_response.request.redirected_from
@@ -36,18 +34,12 @@ class ResponseFactory:
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
content="",
status=current_response.status if current_response else 301,
reason=(
current_response.status_text
or StatusText.get(current_response.status)
)
reason=(current_response.status_text or StatusText.get(current_response.status))
if current_response
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
encoding=current_response.headers.get("content-type", "") or "utf-8",
cookies=tuple(),
headers=current_response.all_headers()
if current_response
else {},
headers=current_response.all_headers() if current_response else {},
request_headers=current_request.all_headers(),
**parser_arguments,
),
@@ -94,13 +86,9 @@ class ResponseFactory:
raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response`
encoding = (
final_response.headers.get("content-type", "") or "utf-8"
) # default encoding
encoding = final_response.headers.get("content-type", "") or "utf-8" # default encoding
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(
final_response.status
)
status_text = final_response.status_text or StatusText.get(final_response.status)
history = cls._process_response_history(first_response, parser_arguments)
try:
@@ -141,18 +129,12 @@ class ResponseFactory:
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
content="",
status=current_response.status if current_response else 301,
reason=(
current_response.status_text
or StatusText.get(current_response.status)
)
reason=(current_response.status_text or StatusText.get(current_response.status))
if current_response
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
encoding=current_response.headers.get("content-type", "") or "utf-8",
cookies=tuple(),
headers=await current_response.all_headers()
if current_response
else {},
headers=await current_response.all_headers() if current_response else {},
request_headers=await current_request.all_headers(),
**parser_arguments,
),
@@ -199,17 +181,11 @@ class ResponseFactory:
raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response`
encoding = (
final_response.headers.get("content-type", "") or "utf-8"
) # default encoding
encoding = final_response.headers.get("content-type", "") or "utf-8" # default encoding
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(
final_response.status
)
status_text = final_response.status_text or StatusText.get(final_response.status)
history = await cls._async_process_response_history(
first_response, parser_arguments
)
history = await cls._async_process_response_history(first_response, parser_arguments)
try:
page_content = await page.content()
except Exception as e: # pragma: no cover
@@ -239,9 +215,7 @@ class ResponseFactory:
"""
return Response(
url=response.url,
content=response.content
if isinstance(response.content, bytes)
else response.content.encode(),
content=response.content if isinstance(response.content, bytes) else response.content.encode(),
status=response.status_code,
reason=response.reason,
encoding=response.encoding or "utf-8",
scrapling/engines/toolbelt/custom.py
+7 -21
View File
@@ -49,9 +49,7 @@ class ResponseEncoding:
@classmethod
@lru_cache(maxsize=128)
def get_value(
cls, content_type: Optional[str], text: Optional[str] = "test"
) -> str:
def get_value(cls, content_type: Optional[str], text: Optional[str] = "test") -> str:
"""Determine the appropriate character encoding from a content-type header.
The encoding is determined by these rules in order:
@@ -84,9 +82,7 @@ class ResponseEncoding:
encoding = cls.__DEFAULT_ENCODING
if encoding:
_ = text.encode(
encoding
) # Validate encoding and validate it can encode the given text
_ = text.encode(encoding) # Validate encoding and validate it can encode the given text
return encoding
return cls.__DEFAULT_ENCODING
@@ -129,9 +125,7 @@ class Response(Selector):
**selector_config,
)
# For easier debugging while working from a Python shell
log.info(
f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})"
)
log.info(f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})")
class BaseFetcher:
@@ -190,18 +184,12 @@ class BaseFetcher:
setattr(cls, key, value)
else:
# Yup, no fun allowed LOL
raise AttributeError(
f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?'
)
raise AttributeError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
else:
raise ValueError(
f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?'
)
raise ValueError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
if not kwargs:
raise AttributeError(
f"You must pass a keyword to configure, current keywords: {cls.parser_keywords}?"
)
raise AttributeError(f"You must pass a keyword to configure, current keywords: {cls.parser_keywords}?")
@classmethod
def _generate_parser_arguments(cls) -> Dict:
@@ -217,9 +205,7 @@ class BaseFetcher:
)
if cls.adaptive_domain:
if not isinstance(cls.adaptive_domain, str):
log.warning(
'[Ignored] The argument "adaptive_domain" must be of string type'
)
log.warning('[Ignored] The argument "adaptive_domain" must be of string type')
else:
parser_arguments.update({"adaptive_domain": cls.adaptive_domain})
scrapling/engines/toolbelt/navigation.py
+4 -13
View File
@@ -30,9 +30,7 @@ def intercept_route(route: Route):
:return: PlayWright `Route` object
"""
if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
log.debug(
f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"'
)
log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
route.abort()
else:
route.continue_()
@@ -45,17 +43,13 @@ async def async_intercept_route(route: async_Route):
:return: PlayWright `Route` object
"""
if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
log.debug(
f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"'
)
log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
await route.abort()
else:
await route.continue_()
def construct_proxy_dict(
proxy_string: str | Dict[str, str], as_tuple=False
) -> Optional[Dict | Tuple]:
def construct_proxy_dict(proxy_string: str | Dict[str, str], as_tuple=False) -> Optional[Dict | Tuple]:
"""Validate a proxy and return it in the acceptable format for Playwright
Reference: https://playwright.dev/python/docs/network#http-proxy
@@ -65,10 +59,7 @@ def construct_proxy_dict(
"""
if isinstance(proxy_string, str):
proxy = urlparse(proxy_string)
if (
proxy.scheme not in ("http", "https", "socks4", "socks5")
or not proxy.hostname
):
if proxy.scheme not in ("http", "https", "socks4", "socks5") or not proxy.hostname:
raise ValueError("Invalid proxy string!")
try: