freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: migrating to ruff and updating pre-commit hooks

Browse Source
This commit is contained in:
Karim shoair
2025-04-13 17:32:00 +02:00
parent f34b42ea33
commit 0c8dd63f87
35 changed files with 2324 additions and 1182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/engines/toolbelt/custom.py
+167 -95
View File
@@ -1,11 +1,20 @@
"""
Functions related to custom types or type checking
"""
import inspect
from email.message import Message
from scrapling.core._types import (Any, Callable, Dict, List, Optional, Tuple,
Type, Union)
from scrapling.core._types import (
Any,
Callable,
Dict,
List,
Optional,
Tuple,
Type,
Union,
)
from scrapling.core.custom_types import MappingProxyType
from scrapling.core.utils import log, lru_cache
from scrapling.parser import Adaptor, SQLiteStorageSystem
@@ -13,7 +22,12 @@ from scrapling.parser import Adaptor, SQLiteStorageSystem
class ResponseEncoding:
__DEFAULT_ENCODING = "utf-8"
__ISO_8859_1_CONTENT_TYPES = {"text/plain", "text/html", "text/css", "text/javascript"}
__ISO_8859_1_CONTENT_TYPES = {
"text/plain",
"text/html",
"text/css",
"text/javascript",
}
@classmethod
@lru_cache(maxsize=128)
@@ -27,19 +41,21 @@ class ResponseEncoding:
"""
# Create a Message object and set the Content-Type header then get the content type and parameters
msg = Message()
msg['content-type'] = header_value
msg["content-type"] = header_value
content_type = msg.get_content_type()
params = dict(msg.get_params(failobj=[]))
# Remove the content-type from params if present somehow
params.pop('content-type', None)
params.pop("content-type", None)
return content_type, params
@classmethod
@lru_cache(maxsize=128)
def get_value(cls, content_type: Optional[str], text: Optional[str] = 'test') -> str:
def get_value(
cls, content_type: Optional[str], text: Optional[str] = "test"
) -> str:
"""Determine the appropriate character encoding from a content-type header.
The encoding is determined by these rules in order:
@@ -72,7 +88,9 @@ class ResponseEncoding:
encoding = cls.__DEFAULT_ENCODING
if encoding:
_ = text.encode(encoding) # Validate encoding and validate it can encode the given text
_ = text.encode(
encoding
) # Validate encoding and validate it can encode the given text
return encoding
return cls.__DEFAULT_ENCODING
@@ -84,9 +102,22 @@ class ResponseEncoding:
class Response(Adaptor):
"""This class is returned by all engines as a way to unify response type between different libraries."""
def __init__(self, url: str, text: str, body: bytes, status: int, reason: str, cookies: Dict, headers: Dict, request_headers: Dict,
encoding: str = 'utf-8', method: str = 'GET', history: List = None, **adaptor_arguments: Dict):
automatch_domain = adaptor_arguments.pop('automatch_domain', None)
def __init__(
self,
url: str,
text: str,
body: bytes,
status: int,
reason: str,
cookies: Dict,
headers: Dict,
request_headers: Dict,
encoding: str = "utf-8",
method: str = "GET",
history: List = None,
**adaptor_arguments: Dict,
):
automatch_domain = adaptor_arguments.pop("automatch_domain", None)
self.status = status
self.reason = reason
self.cookies = cookies
@@ -94,11 +125,19 @@ class Response(Adaptor):
self.request_headers = request_headers
self.history = history or []
encoding = ResponseEncoding.get_value(encoding, text)
super().__init__(text=text, body=body, url=automatch_domain or url, encoding=encoding, **adaptor_arguments)
super().__init__(
text=text,
body=body,
url=automatch_domain or url,
encoding=encoding,
**adaptor_arguments,
)
# For back-ward compatibility
self.adaptor = self
# For easier debugging while working from a Python shell
log.info(f'Fetched ({status}) <{method} {url}> (referer: {request_headers.get("referer")})')
log.info(
f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})"
)
# def __repr__(self):
# return f'<{self.__class__.__name__} [{self.status} {self.reason}]>'
@@ -113,16 +152,26 @@ class BaseFetcher:
storage_args: Optional[Dict] = None
keep_comments: Optional[bool] = False
automatch_domain: Optional[str] = None
parser_keywords: Tuple = ('huge_tree', 'auto_match', 'storage', 'keep_cdata', 'storage_args', 'keep_comments', 'automatch_domain',) # Left open for the user
parser_keywords: Tuple = (
"huge_tree",
"auto_match",
"storage",
"keep_cdata",
"storage_args",
"keep_comments",
"automatch_domain",
) # Left open for the user
def __init__(self, *args, **kwargs):
# For backward-compatibility before 0.2.99
args_str = ", ".join(args) or ''
kwargs_str = ", ".join(f'{k}={v}' for k, v in kwargs.items()) or ''
args_str = ", ".join(args) or ""
kwargs_str = ", ".join(f"{k}={v}" for k, v in kwargs.items()) or ""
if args_str:
args_str += ', '
args_str += ", "
log.warning(f'This logic is deprecated now, and have no effect; It will be removed with v0.3. Use `{self.__class__.__name__}.configure({args_str}{kwargs_str})` instead before fetching')
log.warning(
f"This logic is deprecated now, and have no effect; It will be removed with v0.3. Use `{self.__class__.__name__}.configure({args_str}{kwargs_str})` instead before fetching"
)
pass
@classmethod
@@ -150,12 +199,18 @@ class BaseFetcher:
setattr(cls, key, value)
else:
# Yup, no fun allowed LOL
raise AttributeError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
raise AttributeError(
f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?'
)
else:
raise ValueError(f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?')
raise ValueError(
f'Unknown parser argument: "{key}"; maybe you meant {cls.parser_keywords}?'
)
if not kwargs:
raise AttributeError(f'You must pass a keyword to configure, current keywords: {cls.parser_keywords}?')
raise AttributeError(
f"You must pass a keyword to configure, current keywords: {cls.parser_keywords}?"
)
@classmethod
def _generate_parser_arguments(cls) -> Dict:
@@ -167,13 +222,15 @@ class BaseFetcher:
keep_cdata=cls.keep_cdata,
auto_match=cls.auto_match,
storage=cls.storage,
storage_args=cls.storage_args
storage_args=cls.storage_args,
)
if cls.automatch_domain:
if type(cls.automatch_domain) is not str:
log.warning('[Ignored] The argument "automatch_domain" must be of string type')
log.warning(
'[Ignored] The argument "automatch_domain" must be of string type'
)
else:
parser_arguments.update({'automatch_domain': cls.automatch_domain})
parser_arguments.update({"automatch_domain": cls.automatch_domain})
return parser_arguments
@@ -181,72 +238,75 @@ class BaseFetcher:
class StatusText:
"""A class that gets the status text of response status code.
Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
"""
_phrases = MappingProxyType({
100: "Continue",
101: "Switching Protocols",
102: "Processing",
103: "Early Hints",
200: "OK",
201: "Created",
202: "Accepted",
203: "Non-Authoritative Information",
204: "No Content",
205: "Reset Content",
206: "Partial Content",
207: "Multi-Status",
208: "Already Reported",
226: "IM Used",
300: "Multiple Choices",
301: "Moved Permanently",
302: "Found",
303: "See Other",
304: "Not Modified",
305: "Use Proxy",
307: "Temporary Redirect",
308: "Permanent Redirect",
400: "Bad Request",
401: "Unauthorized",
402: "Payment Required",
403: "Forbidden",
404: "Not Found",
405: "Method Not Allowed",
406: "Not Acceptable",
407: "Proxy Authentication Required",
408: "Request Timeout",
409: "Conflict",
410: "Gone",
411: "Length Required",
412: "Precondition Failed",
413: "Payload Too Large",
414: "URI Too Long",
415: "Unsupported Media Type",
416: "Range Not Satisfiable",
417: "Expectation Failed",
418: "I'm a teapot",
421: "Misdirected Request",
422: "Unprocessable Entity",
423: "Locked",
424: "Failed Dependency",
425: "Too Early",
426: "Upgrade Required",
428: "Precondition Required",
429: "Too Many Requests",
431: "Request Header Fields Too Large",
451: "Unavailable For Legal Reasons",
500: "Internal Server Error",
501: "Not Implemented",
502: "Bad Gateway",
503: "Service Unavailable",
504: "Gateway Timeout",
505: "HTTP Version Not Supported",
506: "Variant Also Negotiates",
507: "Insufficient Storage",
508: "Loop Detected",
510: "Not Extended",
511: "Network Authentication Required"
})
_phrases = MappingProxyType(
{
100: "Continue",
101: "Switching Protocols",
102: "Processing",
103: "Early Hints",
200: "OK",
201: "Created",
202: "Accepted",
203: "Non-Authoritative Information",
204: "No Content",
205: "Reset Content",
206: "Partial Content",
207: "Multi-Status",
208: "Already Reported",
226: "IM Used",
300: "Multiple Choices",
301: "Moved Permanently",
302: "Found",
303: "See Other",
304: "Not Modified",
305: "Use Proxy",
307: "Temporary Redirect",
308: "Permanent Redirect",
400: "Bad Request",
401: "Unauthorized",
402: "Payment Required",
403: "Forbidden",
404: "Not Found",
405: "Method Not Allowed",
406: "Not Acceptable",
407: "Proxy Authentication Required",
408: "Request Timeout",
409: "Conflict",
410: "Gone",
411: "Length Required",
412: "Precondition Failed",
413: "Payload Too Large",
414: "URI Too Long",
415: "Unsupported Media Type",
416: "Range Not Satisfiable",
417: "Expectation Failed",
418: "I'm a teapot",
421: "Misdirected Request",
422: "Unprocessable Entity",
423: "Locked",
424: "Failed Dependency",
425: "Too Early",
426: "Upgrade Required",
428: "Precondition Required",
429: "Too Many Requests",
431: "Request Header Fields Too Large",
451: "Unavailable For Legal Reasons",
500: "Internal Server Error",
501: "Not Implemented",
502: "Bad Gateway",
503: "Service Unavailable",
504: "Gateway Timeout",
505: "HTTP Version Not Supported",
506: "Variant Also Negotiates",
507: "Insufficient Storage",
508: "Loop Detected",
510: "Not Extended",
511: "Network Authentication Required",
}
)
@classmethod
@lru_cache(maxsize=128)
@@ -265,20 +325,26 @@ def check_if_engine_usable(engine: Callable) -> Union[Callable, None]:
# if isinstance(engine, type):
# raise TypeError("Expected an engine instance, not a class definition of the engine")
if hasattr(engine, 'fetch'):
if hasattr(engine, "fetch"):
fetch_function = getattr(engine, "fetch")
if callable(fetch_function):
if len(inspect.signature(fetch_function).parameters) > 0:
return engine
else:
# raise TypeError("Engine class instance must have a callable method 'fetch' with the first argument used for the url.")
raise TypeError("Engine class must have a callable method 'fetch' with the first argument used for the url.")
raise TypeError(
"Engine class must have a callable method 'fetch' with the first argument used for the url."
)
else:
# raise TypeError("Invalid engine instance! Engine class must have a callable method 'fetch'")
raise TypeError("Invalid engine class! Engine class must have a callable method 'fetch'")
raise TypeError(
"Invalid engine class! Engine class must have a callable method 'fetch'"
)
else:
# raise TypeError("Invalid engine instance! Engine class must have the method 'fetch'")
raise TypeError("Invalid engine class! Engine class must have the method 'fetch'")
raise TypeError(
"Invalid engine class! Engine class must have the method 'fetch'"
)
def get_variable_name(var: Any) -> Optional[str]:
@@ -293,7 +359,13 @@ def get_variable_name(var: Any) -> Optional[str]:
return None
def check_type_validity(variable: Any, valid_types: Union[List[Type], None], default_value: Any = None, critical: bool = False, param_name: Optional[str] = None) -> Any:
def check_type_validity(
variable: Any,
valid_types: Union[List[Type], None],
default_value: Any = None,
critical: bool = False,
param_name: Optional[str] = None,
) -> Any:
"""Check if a variable matches the specified type constraints.
:param variable: The variable to check
:param valid_types: List of valid types for the variable
@@ -316,7 +388,7 @@ def check_type_validity(variable: Any, valid_types: Union[List[Type], None], def
error_msg = f'Argument "{var_name}" cannot be None'
if critical:
raise TypeError(error_msg)
log.error(f'[Ignored] {error_msg}')
log.error(f"[Ignored] {error_msg}")
return default_value
# If no valid_types specified and variable has a value, return it
@@ -329,7 +401,7 @@ def check_type_validity(variable: Any, valid_types: Union[List[Type], None], def
error_msg = f'Argument "{var_name}" must be of type {" or ".join(type_names)}'
if critical:
raise TypeError(error_msg)
log.error(f'[Ignored] {error_msg}')
log.error(f"[Ignored] {error_msg}")
return default_value
return variable