freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: migrating to ruff and updating pre-commit hooks

Browse Source
This commit is contained in:
Karim shoair
2025-04-13 17:32:00 +02:00
parent f34b42ea33
commit 0c8dd63f87
35 changed files with 2324 additions and 1182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/core/_types.py
+16 -3
View File
@@ -2,9 +2,22 @@
Type definitions for type checking purposes.
"""
from typing import (TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable,
List, Literal, Optional, Pattern, Tuple, Type, TypeVar,
Union)
from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
Generator,
Iterable,
List,
Literal,
Optional,
Pattern,
Tuple,
Type,
TypeVar,
Union,
)
SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"]
scrapling/core/custom_types.py
+122 -55
View File
@@ -6,16 +6,26 @@ from types import MappingProxyType
from orjson import dumps, loads
from w3lib.html import replace_entities as _replace_entities
from scrapling.core._types import (Dict, Iterable, List, Literal, Optional,
Pattern, SupportsIndex, TypeVar, Union)
from scrapling.core._types import (
Dict,
Iterable,
List,
Literal,
Optional,
Pattern,
SupportsIndex,
TypeVar,
Union,
)
from scrapling.core.utils import _is_iterable, flatten
# Define type variable for AttributeHandler value type
_TextHandlerType = TypeVar('_TextHandlerType', bound='TextHandler')
_TextHandlerType = TypeVar("_TextHandlerType", bound="TextHandler")
class TextHandler(str):
"""Extends standard Python string by adding more functionality"""
__slots__ = ()
def __new__(cls, string):
@@ -25,77 +35,89 @@ class TextHandler(str):
lst = super().__getitem__(key)
return typing.cast(_TextHandlerType, TextHandler(lst))
def split(self, sep: str = None, maxsplit: SupportsIndex = -1) -> 'TextHandlers':
def split(self, sep: str = None, maxsplit: SupportsIndex = -1) -> "TextHandlers":
return TextHandlers(
typing.cast(List[_TextHandlerType], [TextHandler(s) for s in super().split(sep, maxsplit)])
typing.cast(
List[_TextHandlerType],
[TextHandler(s) for s in super().split(sep, maxsplit)],
)
)
def strip(self, chars: str = None) -> Union[str, 'TextHandler']:
def strip(self, chars: str = None) -> Union[str, "TextHandler"]:
return TextHandler(super().strip(chars))
def lstrip(self, chars: str = None) -> Union[str, 'TextHandler']:
def lstrip(self, chars: str = None) -> Union[str, "TextHandler"]:
return TextHandler(super().lstrip(chars))
def rstrip(self, chars: str = None) -> Union[str, 'TextHandler']:
def rstrip(self, chars: str = None) -> Union[str, "TextHandler"]:
return TextHandler(super().rstrip(chars))
def capitalize(self) -> Union[str, 'TextHandler']:
def capitalize(self) -> Union[str, "TextHandler"]:
return TextHandler(super().capitalize())
def casefold(self) -> Union[str, 'TextHandler']:
def casefold(self) -> Union[str, "TextHandler"]:
return TextHandler(super().casefold())
def center(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']:
def center(
self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]:
return TextHandler(super().center(width, fillchar))
def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, 'TextHandler']:
def expandtabs(self, tabsize: SupportsIndex = 8) -> Union[str, "TextHandler"]:
return TextHandler(super().expandtabs(tabsize))
def format(self, *args: str, **kwargs: str) -> Union[str, 'TextHandler']:
def format(self, *args: str, **kwargs: str) -> Union[str, "TextHandler"]:
return TextHandler(super().format(*args, **kwargs))
def format_map(self, mapping) -> Union[str, 'TextHandler']:
def format_map(self, mapping) -> Union[str, "TextHandler"]:
return TextHandler(super().format_map(mapping))
def join(self, iterable: Iterable[str]) -> Union[str, 'TextHandler']:
def join(self, iterable: Iterable[str]) -> Union[str, "TextHandler"]:
return TextHandler(super().join(iterable))
def ljust(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']:
def ljust(
self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]:
return TextHandler(super().ljust(width, fillchar))
def rjust(self, width: SupportsIndex, fillchar: str = ' ') -> Union[str, 'TextHandler']:
def rjust(
self, width: SupportsIndex, fillchar: str = " "
) -> Union[str, "TextHandler"]:
return TextHandler(super().rjust(width, fillchar))
def swapcase(self) -> Union[str, 'TextHandler']:
def swapcase(self) -> Union[str, "TextHandler"]:
return TextHandler(super().swapcase())
def title(self) -> Union[str, 'TextHandler']:
def title(self) -> Union[str, "TextHandler"]:
return TextHandler(super().title())
def translate(self, table) -> Union[str, 'TextHandler']:
def translate(self, table) -> Union[str, "TextHandler"]:
return TextHandler(super().translate(table))
def zfill(self, width: SupportsIndex) -> Union[str, 'TextHandler']:
def zfill(self, width: SupportsIndex) -> Union[str, "TextHandler"]:
return TextHandler(super().zfill(width))
def replace(self, old: str, new: str, count: SupportsIndex = -1) -> Union[str, 'TextHandler']:
def replace(
self, old: str, new: str, count: SupportsIndex = -1
) -> Union[str, "TextHandler"]:
return TextHandler(super().replace(old, new, count))
def upper(self) -> Union[str, 'TextHandler']:
def upper(self) -> Union[str, "TextHandler"]:
return TextHandler(super().upper())
def lower(self) -> Union[str, 'TextHandler']:
def lower(self) -> Union[str, "TextHandler"]:
return TextHandler(super().lower())
##############
def sort(self, reverse: bool = False) -> Union[str, 'TextHandler']:
def sort(self, reverse: bool = False) -> Union[str, "TextHandler"]:
"""Return a sorted version of the string"""
return self.__class__("".join(sorted(self, reverse=reverse)))
def clean(self) -> Union[str, 'TextHandler']:
def clean(self) -> Union[str, "TextHandler"]:
"""Return a new version of the string after removing all white spaces and consecutive spaces"""
data = re.sub(r'[\t|\r|\n]', '', self)
data = re.sub(' +', ' ', data)
data = re.sub(r"[\t|\r|\n]", "", self)
data = re.sub(" +", " ", data)
return self.__class__(data.strip())
# For easy copy-paste from Scrapy/parsel code when needed :)
@@ -122,8 +144,7 @@ class TextHandler(str):
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
) -> bool:
...
) -> bool: ...
@typing.overload
def re(
@@ -133,12 +154,15 @@ class TextHandler(str):
clean_match: bool = False,
case_sensitive: bool = True,
check_match: Literal[False] = False,
) -> "TextHandlers[TextHandler]":
...
) -> "TextHandlers[TextHandler]": ...
def re(
self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False,
case_sensitive: bool = True, check_match: bool = False
self,
regex: Union[str, Pattern[str]],
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
check_match: bool = False,
) -> Union["TextHandlers[TextHandler]", bool]:
"""Apply the given regex to the current text and return a list of strings with the matches.
@@ -164,12 +188,27 @@ class TextHandler(str):
results = flatten(results)
if not replace_entities:
return TextHandlers(typing.cast(List[_TextHandlerType], [TextHandler(string) for string in results]))
return TextHandlers(
typing.cast(
List[_TextHandlerType], [TextHandler(string) for string in results]
)
)
return TextHandlers(typing.cast(List[_TextHandlerType], [TextHandler(_replace_entities(s)) for s in results]))
return TextHandlers(
typing.cast(
List[_TextHandlerType],
[TextHandler(_replace_entities(s)) for s in results],
)
)
def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True,
clean_match: bool = False, case_sensitive: bool = True) -> "TextHandler":
def re_first(
self,
regex: Union[str, Pattern[str]],
default=None,
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
) -> "TextHandler":
"""Apply the given regex to text and return the first match if found, otherwise return the default value.
:param regex: Can be either a compiled regular expression or a string.
@@ -179,7 +218,12 @@ class TextHandler(str):
:param case_sensitive: if disabled, function will set the regex to ignore letters case while compiling it
"""
result = self.re(regex, replace_entities, clean_match=clean_match, case_sensitive=case_sensitive)
result = self.re(
regex,
replace_entities,
clean_match=clean_match,
case_sensitive=case_sensitive,
)
return result[0] if result else default
@@ -187,6 +231,7 @@ class TextHandlers(List[TextHandler]):
"""
The :class:`TextHandlers` class is a subclass of the builtin ``List`` class, which provides a few additional methods.
"""
__slots__ = ()
@typing.overload
@@ -197,15 +242,22 @@ class TextHandlers(List[TextHandler]):
def __getitem__(self, pos: slice) -> "TextHandlers":
pass
def __getitem__(self, pos: Union[SupportsIndex, slice]) -> Union[TextHandler, "TextHandlers"]:
def __getitem__(
self, pos: Union[SupportsIndex, slice]
) -> Union[TextHandler, "TextHandlers"]:
lst = super().__getitem__(pos)
if isinstance(pos, slice):
lst = [TextHandler(s) for s in lst]
return TextHandlers(typing.cast(List[_TextHandlerType], lst))
return typing.cast(_TextHandlerType, TextHandler(lst))
def re(self, regex: Union[str, Pattern[str]], replace_entities: bool = True, clean_match: bool = False,
case_sensitive: bool = True) -> 'TextHandlers[TextHandler]':
def re(
self,
regex: Union[str, Pattern[str]],
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
) -> "TextHandlers[TextHandler]":
"""Call the ``.re()`` method for each element in this list and return
their results flattened as TextHandlers.
@@ -219,8 +271,14 @@ class TextHandlers(List[TextHandler]):
]
return TextHandlers(flatten(results))
def re_first(self, regex: Union[str, Pattern[str]], default=None, replace_entities: bool = True,
clean_match: bool = False, case_sensitive: bool = True) -> TextHandler:
def re_first(
self,
regex: Union[str, Pattern[str]],
default=None,
replace_entities: bool = True,
clean_match: bool = False,
case_sensitive: bool = True,
) -> TextHandler:
"""Call the ``.re_first()`` method for each element in this list and return
the first result or the default value otherwise.
@@ -251,26 +309,35 @@ class TextHandlers(List[TextHandler]):
class AttributesHandler(Mapping[str, _TextHandlerType]):
"""A read-only mapping to use instead of the standard dictionary for the speed boost but at the same time I use it to add more functionalities.
If standard dictionary is needed, just convert this class to dictionary with `dict` function
If standard dictionary is needed, just convert this class to dictionary with `dict` function
"""
__slots__ = ('_data',)
__slots__ = ("_data",)
def __init__(self, mapping=None, **kwargs):
mapping = {
key: TextHandler(value) if type(value) is str else value
for key, value in mapping.items()
} if mapping is not None else {}
mapping = (
{
key: TextHandler(value) if type(value) is str else value
for key, value in mapping.items()
}
if mapping is not None
else {}
)
if kwargs:
mapping.update({
key: TextHandler(value) if type(value) is str else value
for key, value in kwargs.items()
})
mapping.update(
{
key: TextHandler(value) if type(value) is str else value
for key, value in kwargs.items()
}
)
# Fastest read-only mapping type
self._data = MappingProxyType(mapping)
def get(self, key: str, default: Optional[str] = None) -> Union[_TextHandlerType, None]:
def get(
self, key: str, default: Optional[str] = None
) -> Union[_TextHandlerType, None]:
"""Acts like standard dictionary `.get()` method"""
return self._data.get(key, default)
scrapling/core/mixins.py
+20 -16
View File
@@ -1,32 +1,33 @@
class SelectorsGeneration:
"""Selectors generation functions
Trying to generate selectors like Firefox or maybe cleaner ones!? Ehm
Inspiration: https://searchfox.org/mozilla-central/source/devtools/shared/inspector/css-logic.js#591"""
def __general_selection(self, selection: str = 'css', full_path=False) -> str:
def __general_selection(self, selection: str = "css", full_path=False) -> str:
"""Generate a selector for the current element.
:return: A string of the generated selector.
"""
selectorPath = []
target = self
css = selection.lower() == 'css'
css = selection.lower() == "css"
while target is not None:
if target.parent:
if target.attrib.get('id'):
if target.attrib.get("id"):
# id is enough
part = (
f'#{target.attrib["id"]}' if css
f"#{target.attrib['id']}"
if css
else f"[@id='{target.attrib['id']}']"
)
selectorPath.append(part)
if not full_path:
return (
" > ".join(reversed(selectorPath)) if css
else '//*' + "/".join(reversed(selectorPath))
" > ".join(reversed(selectorPath))
if css
else "//*" + "/".join(reversed(selectorPath))
)
else:
part = f'{target.tag}'
part = f"{target.tag}"
# We won't use classes anymore because I some websites share exact classes between elements
# classes = target.attrib.get('class', '').split()
# if classes and css:
@@ -41,23 +42,26 @@ class SelectorsGeneration:
if counter[target.tag] > 1:
part += (
f":nth-of-type({counter[target.tag]})" if css
f":nth-of-type({counter[target.tag]})"
if css
else f"[{counter[target.tag]}]"
)
selectorPath.append(part)
target = target.parent
if target is None or target.tag == 'html':
if target is None or target.tag == "html":
return (
" > ".join(reversed(selectorPath)) if css
else '//' + "/".join(reversed(selectorPath))
" > ".join(reversed(selectorPath))
if css
else "//" + "/".join(reversed(selectorPath))
)
else:
break
return (
" > ".join(reversed(selectorPath)) if css
else '//' + "/".join(reversed(selectorPath))
" > ".join(reversed(selectorPath))
if css
else "//" + "/".join(reversed(selectorPath))
)
@property
@@ -79,11 +83,11 @@ class SelectorsGeneration:
"""Generate a XPath selector for the current element
:return: A string of the generated selector.
"""
return self.__general_selection('xpath')
return self.__general_selection("xpath")
@property
def generate_full_xpath_selector(self) -> str:
"""Generate a complete XPath selector for the current element
:return: A string of the generated selector.
"""
return self.__general_selection('xpath', full_path=True)
return self.__general_selection("xpath", full_path=True)
scrapling/core/storage_adaptors.py
+11 -7
View File
@@ -20,7 +20,7 @@ class StorageSystemMixin(ABC):
self.url = url
@lru_cache(64, typed=True)
def _get_base_url(self, default_value: str = 'default') -> str:
def _get_base_url(self, default_value: str = "default") -> str:
if not self.url or type(self.url) is not str:
return default_value
@@ -38,7 +38,7 @@ class StorageSystemMixin(ABC):
:param identifier: This is the identifier that will be used to retrieve the element later from the storage. See
the docs for more info.
"""
raise NotImplementedError('Storage system must implement `save` method')
raise NotImplementedError("Storage system must implement `save` method")
@abstractmethod
def retrieve(self, identifier: str) -> Optional[Dict]:
@@ -48,7 +48,7 @@ class StorageSystemMixin(ABC):
the docs for more info.
:return: A dictionary of the unique properties
"""
raise NotImplementedError('Storage system must implement `save` method')
raise NotImplementedError("Storage system must implement `save` method")
@staticmethod
@lru_cache(128, typed=True)
@@ -57,7 +57,7 @@ class StorageSystemMixin(ABC):
identifier = identifier.lower().strip()
if isinstance(identifier, str):
# Hash functions have to take bytes
identifier = identifier.encode('utf-8')
identifier = identifier.encode("utf-8")
hash_value = sha256(identifier).hexdigest()
return f"{hash_value}_{len(identifier)}" # Length to reduce collision chance
@@ -68,6 +68,7 @@ class SQLiteStorageSystem(StorageSystemMixin):
"""The recommended system to use, it's race condition safe and thread safe.
Mainly built so the library can run in threaded frameworks like scrapy or threaded tools
> It's optimized for threaded applications but running it without threads shouldn't make it slow."""
def __init__(self, storage_file: str, url: Union[str, None] = None):
"""
:param storage_file: File to be used to store elements
@@ -111,10 +112,13 @@ class SQLiteStorageSystem(StorageSystemMixin):
url = self._get_base_url()
element_data = _StorageTools.element_to_dict(element)
with self.lock:
self.cursor.execute("""
self.cursor.execute(
"""
INSERT OR REPLACE INTO storage (url, identifier, element_data)
VALUES (?, ?, ?)
""", (url, identifier, orjson.dumps(element_data)))
""",
(url, identifier, orjson.dumps(element_data)),
)
self.cursor.fetchall()
self.connection.commit()
@@ -129,7 +133,7 @@ class SQLiteStorageSystem(StorageSystemMixin):
with self.lock:
self.cursor.execute(
"SELECT element_data FROM storage WHERE url = ? AND identifier = ?",
(url, identifier)
(url, identifier),
)
result = self.cursor.fetchone()
if result:
scrapling/core/translator.py
+1 -2
View File
@@ -24,7 +24,6 @@ replace_html5_whitespaces = re.compile(regex).sub
class XPathExpr(OriginalXPathExpr):
textnode: bool = False
attribute: Optional[str] = None
@@ -123,7 +122,7 @@ class TranslatorMixin:
@staticmethod
def xpath_attr_functional_pseudo_element(
xpath: OriginalXPathExpr, function: FunctionalPseudoElement
xpath: OriginalXPathExpr, function: FunctionalPseudoElement
) -> XPathExpr:
"""Support selecting attribute values using ::attr() pseudo-element"""
if function.argument_types() not in (["STRING"], ["IDENT"]):
scrapling/core/utils.py
+44 -25
View File
@@ -11,7 +11,9 @@ from scrapling.core._types import Any, Dict, Iterable, Union
# functools.cache is available on Python 3.9+ only so let's keep lru_cache
from functools import lru_cache # isort:skip
html_forbidden = {html.HtmlComment, }
html_forbidden = {
html.HtmlComment,
}
@lru_cache(1, typed=True)
@@ -20,12 +22,11 @@ def setup_logger():
:returns: logging.Logger: Configured logger instance
"""
logger = logging.getLogger('scrapling')
logger = logging.getLogger("scrapling")
logger.setLevel(logging.INFO)
formatter = logging.Formatter(
fmt="[%(asctime)s] %(levelname)s: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S"
fmt="[%(asctime)s] %(levelname)s: %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
)
console_handler = logging.StreamHandler()
@@ -58,7 +59,13 @@ def flatten(lst: Iterable):
def _is_iterable(s: Any):
# This will be used only in regex functions to make sure it's iterable but not string/bytes
return isinstance(s, (list, tuple,))
return isinstance(
s,
(
list,
tuple,
),
)
class _StorageTools:
@@ -66,31 +73,43 @@ class _StorageTools:
def __clean_attributes(element: html.HtmlElement, forbidden: tuple = ()) -> Dict:
if not element.attrib:
return {}
return {k: v.strip() for k, v in element.attrib.items() if v and v.strip() and k not in forbidden}
return {
k: v.strip()
for k, v in element.attrib.items()
if v and v.strip() and k not in forbidden
}
@classmethod
def element_to_dict(cls, element: html.HtmlElement) -> Dict:
parent = element.getparent()
result = {
'tag': str(element.tag),
'attributes': cls.__clean_attributes(element),
'text': element.text.strip() if element.text else None,
'path': cls._get_element_path(element)
"tag": str(element.tag),
"attributes": cls.__clean_attributes(element),
"text": element.text.strip() if element.text else None,
"path": cls._get_element_path(element),
}
if parent is not None:
result.update({
'parent_name': parent.tag,
'parent_attribs': dict(parent.attrib),
'parent_text': parent.text.strip() if parent.text else None
})
result.update(
{
"parent_name": parent.tag,
"parent_attribs": dict(parent.attrib),
"parent_text": parent.text.strip() if parent.text else None,
}
)
siblings = [child.tag for child in parent.iterchildren() if child != element]
siblings = [
child.tag for child in parent.iterchildren() if child != element
]
if siblings:
result.update({'siblings': tuple(siblings)})
result.update({"siblings": tuple(siblings)})
children = [child.tag for child in element.iterchildren() if type(child) not in html_forbidden]
children = [
child.tag
for child in element.iterchildren()
if type(child) not in html_forbidden
]
if children:
result.update({'children': tuple(children)})
result.update({"children": tuple(children)})
return result
@@ -98,9 +117,9 @@ class _StorageTools:
def _get_element_path(cls, element: html.HtmlElement):
parent = element.getparent()
return tuple(
(element.tag,) if parent is None else (
cls._get_element_path(parent) + (element.tag,)
)
(element.tag,)
if parent is None
else (cls._get_element_path(parent) + (element.tag,))
)
@@ -117,6 +136,6 @@ class _StorageTools:
@lru_cache(128, typed=True)
def clean_spaces(string):
string = string.replace('\t', ' ')
string = re.sub('[\n|\r]', '', string)
return re.sub(' +', ' ', string)
string = string.replace("\t", " ")
string = re.sub("[\n|\r]", "", string)
return re.sub(" +", " ", string)