freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(parser): More on correcting type hints and unifying return type

Browse Source
This commit is contained in:
Karim shoair
2025-01-30 02:03:09 +02:00
parent 7ee0f6114d
commit 4e2e7a4b22
1 changed files with 10 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/parser.py
+10 -15
View File
@@ -285,18 +285,18 @@ class Adaptor(SelectorsGeneration):
return self.__handle_element(self._root.getparent()) return self.__handle_element(self._root.getparent())
@property @property
def children(self) -> Union['Adaptors[Adaptor]', List]: def children(self) -> 'Adaptors[Adaptor]':
"""Return the children elements of the current element or empty list otherwise""" """Return the children elements of the current element or empty list otherwise"""
return Adaptors([ return Adaptors([
self.__element_convertor(child) for child in self._root.iterchildren() if type(child) not in html_forbidden self.__element_convertor(child) for child in self._root.iterchildren() if type(child) not in html_forbidden
]) ])
@property @property
def siblings(self) -> Union['Adaptors[Adaptor]', List]: def siblings(self) -> 'Adaptors[Adaptor]':
"""Return other children of the current element's parent or empty list otherwise""" """Return other children of the current element's parent or empty list otherwise"""
if self.parent: if self.parent:
return Adaptors([child for child in self.parent.children if child._root != self._root]) return Adaptors([child for child in self.parent.children if child._root != self._root])
return [] return Adaptors([])
def iterancestors(self) -> Generator['Adaptor', None, None]: def iterancestors(self) -> Generator['Adaptor', None, None]:
"""Return a generator that loops over all ancestors of the element, starting with element's parent.""" """Return a generator that loops over all ancestors of the element, starting with element's parent."""
@@ -442,8 +442,6 @@ class Adaptor(SelectorsGeneration):
:param percentage: The minimum percentage to accept while auto-matching and not going lower than that. :param percentage: The minimum percentage to accept while auto-matching and not going lower than that.
Be aware that the percentage calculation depends solely on the page structure so don't play with this Be aware that the percentage calculation depends solely on the page structure so don't play with this
number unless you must know what you are doing! number unless you must know what you are doing!
:return: List as :class:`Adaptors`
""" """
for element in self.css(selector, identifier, auto_match, auto_save, percentage): for element in self.css(selector, identifier, auto_match, auto_save, percentage):
return element return element
@@ -468,8 +466,6 @@ class Adaptor(SelectorsGeneration):
:param percentage: The minimum percentage to accept while auto-matching and not going lower than that. :param percentage: The minimum percentage to accept while auto-matching and not going lower than that.
Be aware that the percentage calculation depends solely on the page structure so don't play with this Be aware that the percentage calculation depends solely on the page structure so don't play with this
number unless you must know what you are doing! number unless you must know what you are doing!
:return: List as :class:`Adaptors`
""" """
for element in self.xpath(selector, identifier, auto_match, auto_save, percentage, **kwargs): for element in self.xpath(selector, identifier, auto_match, auto_save, percentage, **kwargs):
return element return element
@@ -899,7 +895,7 @@ class Adaptor(SelectorsGeneration):
def find_by_text( def find_by_text(
self, text: str, first_match: bool = True, partial: bool = False, self, text: str, first_match: bool = True, partial: bool = False,
case_sensitive: bool = False, clean_match: bool = True case_sensitive: bool = False, clean_match: bool = True
) -> Union['Adaptors[Adaptor]', 'Adaptor', List]: ) -> Union['Adaptors[Adaptor]', 'Adaptor']:
"""Find elements that its text content fully/partially matches input. """Find elements that its text content fully/partially matches input.
:param text: Text query to match :param text: Text query to match
:param first_match: Return first element that matches conditions, enabled by default :param first_match: Return first element that matches conditions, enabled by default
@@ -946,7 +942,7 @@ class Adaptor(SelectorsGeneration):
def find_by_regex( def find_by_regex(
self, query: Union[str, Pattern[str]], first_match: bool = True, case_sensitive: bool = False, clean_match: bool = True self, query: Union[str, Pattern[str]], first_match: bool = True, case_sensitive: bool = False, clean_match: bool = True
) -> Union['Adaptors[Adaptor]', 'Adaptor', List]: ) -> Union['Adaptors[Adaptor]', 'Adaptor']:
"""Find elements that its text content matches the input regex pattern. """Find elements that its text content matches the input regex pattern.
:param query: Regex query/pattern to match :param query: Regex query/pattern to match
:param first_match: Return first element that matches conditions, enabled by default :param first_match: Return first element that matches conditions, enabled by default
@@ -1001,7 +997,7 @@ class Adaptors(List[Adaptor]):
def xpath( def xpath(
self, selector: str, identifier: str = '', auto_save: bool = False, percentage: int = 0, **kwargs: Any self, selector: str, identifier: str = '', auto_save: bool = False, percentage: int = 0, **kwargs: Any
) -> Union["Adaptors[Adaptor]", List]: ) -> "Adaptors[Adaptor]":
""" """
Call the ``.xpath()`` method for each element in this list and return Call the ``.xpath()`` method for each element in this list and return
their results as another :class:`Adaptors`. their results as another :class:`Adaptors`.
@@ -1027,7 +1023,7 @@ class Adaptors(List[Adaptor]):
] ]
return self.__class__(flatten(results)) return self.__class__(flatten(results))
def css(self, selector: str, identifier: str = '', auto_save: bool = False, percentage: int = 0) -> Union["Adaptors[Adaptor]", List]: def css(self, selector: str, identifier: str = '', auto_save: bool = False, percentage: int = 0) -> "Adaptors[Adaptor]":
""" """
Call the ``.css()`` method for each element in this list and return Call the ``.css()`` method for each element in this list and return
their results flattened as another :class:`Adaptors`. their results flattened as another :class:`Adaptors`.
@@ -1092,15 +1088,14 @@ class Adaptors(List[Adaptor]):
return element return element
return None return None
def filter(self, func: Callable[['Adaptor'], bool]) -> Union['Adaptors', List]: def filter(self, func: Callable[['Adaptor'], bool]) -> 'Adaptors[Adaptor]':
"""Filter current elements based on the passed function """Filter current elements based on the passed function
:param func: A function that takes each element as an argument and returns True/False :param func: A function that takes each element as an argument and returns True/False
:return: The new `Adaptors` object or empty list otherwise. :return: The new `Adaptors` object or empty list otherwise.
""" """
results = [ return self.__class__([
element for element in self if func(element) element for element in self if func(element)
] ])
return self.__class__(results) if results else results
# For easy copy-paste from Scrapy/parsel code when needed :) # For easy copy-paste from Scrapy/parsel code when needed :)
def get(self, default=None): def get(self, default=None):