freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: migrating to ruff and updating pre-commit hooks

Browse Source
This commit is contained in:
Karim shoair
2025-04-13 17:32:00 +02:00
parent f34b42ea33
commit 0c8dd63f87
35 changed files with 2324 additions and 1182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/toolbelt/navigation.py
+29 -14
View File
@@ -1,6 +1,7 @@
"""
Functions related to files and URLs
"""
import os
from urllib.parse import urlencode, urlparse
@@ -19,7 +20,9 @@ def intercept_route(route: Route):
:return: PlayWright `Route` object
"""
if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
log.debug(
f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"'
)
route.abort()
else:
route.continue_()
@@ -32,7 +35,9 @@ async def async_intercept_route(route: async_Route):
:return: PlayWright `Route` object
"""
if route.request.resource_type in DEFAULT_DISABLED_RESOURCES:
log.debug(f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"')
log.debug(
f'Blocking background resource "{route.request.url}" of type "{route.request.resource_type}"'
)
await route.abort()
else:
await route.continue_()
@@ -50,23 +55,33 @@ def construct_proxy_dict(proxy_string: Union[str, Dict[str, str]]) -> Union[Dict
proxy = urlparse(proxy_string)
try:
return {
'server': f'{proxy.scheme}://{proxy.hostname}:{proxy.port}',
'username': proxy.username or '',
'password': proxy.password or '',
"server": f"{proxy.scheme}://{proxy.hostname}:{proxy.port}",
"username": proxy.username or "",
"password": proxy.password or "",
}
except ValueError:
# Urllib will say that one of the parameters above can't be casted to the correct type like `int` for port etc...
raise TypeError('The proxy argument\'s string is in invalid format!')
raise TypeError("The proxy argument's string is in invalid format!")
elif isinstance(proxy_string, dict):
valid_keys = ('server', 'username', 'password', )
if all(key in valid_keys for key in proxy_string.keys()) and not any(key not in valid_keys for key in proxy_string.keys()):
valid_keys = (
"server",
"username",
"password",
)
if all(key in valid_keys for key in proxy_string.keys()) and not any(
key not in valid_keys for key in proxy_string.keys()
):
return proxy_string
else:
raise TypeError(f'A proxy dictionary must have only these keys: {valid_keys}')
raise TypeError(
f"A proxy dictionary must have only these keys: {valid_keys}"
)
else:
raise TypeError(f'Invalid type of proxy ({type(proxy_string)}), the proxy argument must be a string or a dictionary!')
raise TypeError(
f"Invalid type of proxy ({type(proxy_string)}), the proxy argument must be a string or a dictionary!"
)
# The default value for proxy in Playwright's source is `None`
return None
@@ -84,7 +99,7 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
parsed = urlparse(cdp_url)
# Check scheme
if parsed.scheme not in ('ws', 'wss'):
if parsed.scheme not in ("ws", "wss"):
raise ValueError("CDP URL must use 'ws://' or 'wss://' scheme")
# Validate hostname and port
@@ -93,8 +108,8 @@ def construct_cdp_url(cdp_url: str, query_params: Optional[Dict] = None) -> str:
# Ensure path starts with /
path = parsed.path
if not path.startswith('/'):
path = '/' + path
if not path.startswith("/"):
path = "/" + path
# Reconstruct the base URL with validated parts
validated_base = f"{parsed.scheme}://{parsed.netloc}{path}"
@@ -118,4 +133,4 @@ def js_bypass_path(filename: str) -> str:
:return: The full path of the JS file.
"""
current_directory = os.path.dirname(__file__)
return os.path.join(current_directory, 'bypasses', filename)
return os.path.join(current_directory, "bypasses", filename)