freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: applying the new ruff rules to all files

Browse Source
This commit is contained in:
Karim shoair
2025-09-13 03:22:53 +03:00
parent 60be9dc816
commit 330d03559c
18 changed files with 176 additions and 572 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/toolbelt/convertor.py
+13 -39
View File
@@ -18,9 +18,7 @@ class ResponseFactory:
"""
@classmethod
def _process_response_history(
cls, first_response: SyncResponse, parser_arguments: Dict
) -> list[Response]:
def _process_response_history(cls, first_response: SyncResponse, parser_arguments: Dict) -> list[Response]:
"""Process response history to build a list of `Response` objects"""
history = []
current_request = first_response.request.redirected_from
@@ -36,18 +34,12 @@ class ResponseFactory:
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
content="",
status=current_response.status if current_response else 301,
reason=(
current_response.status_text
or StatusText.get(current_response.status)
)
reason=(current_response.status_text or StatusText.get(current_response.status))
if current_response
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
encoding=current_response.headers.get("content-type", "") or "utf-8",
cookies=tuple(),
headers=current_response.all_headers()
if current_response
else {},
headers=current_response.all_headers() if current_response else {},
request_headers=current_request.all_headers(),
**parser_arguments,
),
@@ -94,13 +86,9 @@ class ResponseFactory:
raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response`
encoding = (
final_response.headers.get("content-type", "") or "utf-8"
) # default encoding
encoding = final_response.headers.get("content-type", "") or "utf-8" # default encoding
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(
final_response.status
)
status_text = final_response.status_text or StatusText.get(final_response.status)
history = cls._process_response_history(first_response, parser_arguments)
try:
@@ -141,18 +129,12 @@ class ResponseFactory:
# using current_response.text() will trigger "Error: Response.text: Response body is unavailable for redirect responses"
content="",
status=current_response.status if current_response else 301,
reason=(
current_response.status_text
or StatusText.get(current_response.status)
)
reason=(current_response.status_text or StatusText.get(current_response.status))
if current_response
else StatusText.get(301),
encoding=current_response.headers.get("content-type", "")
or "utf-8",
encoding=current_response.headers.get("content-type", "") or "utf-8",
cookies=tuple(),
headers=await current_response.all_headers()
if current_response
else {},
headers=await current_response.all_headers() if current_response else {},
request_headers=await current_request.all_headers(),
**parser_arguments,
),
@@ -199,17 +181,11 @@ class ResponseFactory:
raise ValueError("Failed to get a response from the page")
# This will be parsed inside `Response`
encoding = (
final_response.headers.get("content-type", "") or "utf-8"
) # default encoding
encoding = final_response.headers.get("content-type", "") or "utf-8" # default encoding
# PlayWright API sometimes give empty status text for some reason!
status_text = final_response.status_text or StatusText.get(
final_response.status
)
status_text = final_response.status_text or StatusText.get(final_response.status)
history = await cls._async_process_response_history(
first_response, parser_arguments
)
history = await cls._async_process_response_history(first_response, parser_arguments)
try:
page_content = await page.content()
except Exception as e: # pragma: no cover
@@ -239,9 +215,7 @@ class ResponseFactory:
"""
return Response(
url=response.url,
content=response.content
if isinstance(response.content, bytes)
else response.content.encode(),
content=response.content if isinstance(response.content, bytes) else response.content.encode(),
status=response.status_code,
reason=response.reason,
encoding=response.encoding or "utf-8",
scrapling/engines/toolbelt/custom.py
+7 -21
View File
@@ -49,9 +49,7 @@ class ResponseEncoding:
@classmethod
@lru_cache(maxsize=128)
def get_value(
cls, content_type: Optional[str], text: Optional[str] = "test"
) -> str:
def get_value(cls, content_type: Optional[str], text: Optional[str] = "test") -> str:
"""Determine the appropriate character encoding from a content-type header.
The encoding is determined by these rules in order:
@@ -84,9 +82,7 @@ class ResponseEncoding:
encoding = cls.__DEFAULT_ENCODING
if encoding:
_ = text.encode(
encoding
) # Validate encoding and validate it can encode the given text
_ = text.encode(encoding) # Validate encoding and validate it can encode the given text
return encoding
return cls.__DEFAULT_ENCODING
@@ -129,9 +125,7 @@ class Response(Selector):
**selector_config,
)
# For easier debugging while working from a Python shell
log.info(
f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})"
)
log.info(f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})")
class BaseFetcher:
@@ -190,18 +184,12 @@ class BaseFetcher:
setattr(cls, key, value)
else:
# Yup, no fun allowed LOL
raise AttributeError(
f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?'
)
raise AttributeError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
else:
raise ValueError(
f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?'
)
raise ValueError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
if not kwargs:
raise AttributeError(
f"You must pass a keyword to configure, current keywords: {cls.parser_keywords}?"
)
raise AttributeError(f"You must pass a keyword to configure, current keywords: {cls.parser_keywords}?")
@classmethod
def _generate_parser_arguments(cls) -> Dict:
@@ -217,9 +205,7 @@ class BaseFetcher:
)
if cls.adaptive_domain:
if not isinstance(cls.adaptive_domain, str):
log.warning(
'[Ignored] The argument "adaptive_domain" must be of string type'
)
log.warning('[Ignored] The argument "adaptive_domain" must be of string type')
else:
parser_arguments.update({"adaptive_domain": cls.adaptive_domain})
scrapling/engines/toolbelt/navigation.py
+4 -13
View File
@@ -30,9 +30,7 @@ def intercept_route(route: Route):
:return: PlayWright `Route` object
"""
if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
log.debug(
f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"'
)
log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
route.abort()
else:
route.continue_()
@@ -45,17 +43,13 @@ async def async_intercept_route(route: async_Route):
:return: PlayWright `Route` object
"""
if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
log.debug(
f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"'
)
log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
await route.abort()
else:
await route.continue_()
def construct_proxy_dict(
proxy_string: str | Dict[str, str], as_tuple=False
) -> Optional[Dict | Tuple]:
def construct_proxy_dict(proxy_string: str | Dict[str, str], as_tuple=False) -> Optional[Dict | Tuple]:
"""Validate a proxy and return it in the acceptable format for Playwright
Reference: https://playwright.dev/python/docs/network#http-proxy
@@ -65,10 +59,7 @@ def construct_proxy_dict(
"""
if isinstance(proxy_string, str):
proxy = urlparse(proxy_string)
if (
proxy.scheme not in ("http", "https", "socks4", "socks5")
or not proxy.hostname
):
if proxy.scheme not in ("http", "https", "socks4", "socks5") or not proxy.hostname:
raise ValueError("Invalid proxy string!")
try: