style: Adjustments to the translator

This commit is contained in:
Karim shoair
2025-07-30 03:28:16 +03:00
parent 5bb1266fa5
commit 32cb76604c
2 changed files with 10 additions and 15 deletions
+7 -12
View File
@@ -8,7 +8,7 @@ So you don't have to learn a new selectors/api method like what bs4 done with so
If you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement If you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement
""" """
import re from functools import lru_cache
from cssselect import HTMLTranslator as OriginalHTMLTranslator from cssselect import HTMLTranslator as OriginalHTMLTranslator
from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement from cssselect.parser import Element, FunctionalPseudoElement, PseudoElement
@@ -16,11 +16,6 @@ from cssselect.xpath import ExpressionError
from cssselect.xpath import XPathExpr as OriginalXPathExpr from cssselect.xpath import XPathExpr as OriginalXPathExpr
from scrapling.core._types import Any, Optional, Protocol, Self from scrapling.core._types import Any, Optional, Protocol, Self
from scrapling.core.utils import lru_cache
HTML5_WHITESPACE = " \t\n\r\x0c" # From w3lib.html.HTML5_WHITESPACE
regex = f"[{HTML5_WHITESPACE}]+"
replace_html5_whitespaces = re.compile(regex).sub
class XPathExpr(OriginalXPathExpr): class XPathExpr(OriginalXPathExpr):
@@ -33,7 +28,7 @@ class XPathExpr(OriginalXPathExpr):
xpath: OriginalXPathExpr, xpath: OriginalXPathExpr,
textnode: bool = False, textnode: bool = False,
attribute: Optional[str] = None, attribute: Optional[str] = None,
) -> "Self": ) -> Self:
x = cls(path=xpath.path, element=xpath.element, condition=xpath.condition) x = cls(path=xpath.path, element=xpath.element, condition=xpath.condition)
x.textnode = textnode x.textnode = textnode
x.attribute = attribute x.attribute = attribute
@@ -57,12 +52,12 @@ class XPathExpr(OriginalXPathExpr):
return path return path
def join( def join(
self: "Self", self: Self,
combiner: str, combiner: str,
other: OriginalXPathExpr, other: OriginalXPathExpr,
*args: Any, *args: Any,
**kwargs: Any, **kwargs: Any,
) -> "Self": ) -> Self:
if not isinstance(other, XPathExpr): if not isinstance(other, XPathExpr):
raise ValueError( raise ValueError(
f"Expressions of type {__name__}.XPathExpr can ony join expressions" f"Expressions of type {__name__}.XPathExpr can ony join expressions"
@@ -90,7 +85,7 @@ class TranslatorMixin:
""" """
def xpath_element(self: TranslatorProtocol, selector: Element) -> XPathExpr: def xpath_element(self: TranslatorProtocol, selector: Element) -> XPathExpr:
# https://github.com/python/mypy/issues/12344 # https://github.com/python/mypy/issues/14757
xpath = super().xpath_element(selector) # type: ignore[safe-super] xpath = super().xpath_element(selector) # type: ignore[safe-super]
return XPathExpr.from_xpath(xpath) return XPathExpr.from_xpath(xpath)
@@ -98,7 +93,7 @@ class TranslatorMixin:
self, xpath: OriginalXPathExpr, pseudo_element: PseudoElement self, xpath: OriginalXPathExpr, pseudo_element: PseudoElement
) -> OriginalXPathExpr: ) -> OriginalXPathExpr:
""" """
Dispatch method that transforms XPath to support pseudo-elements. Dispatch method that transforms XPath to support the pseudo-element.
""" """
if isinstance(pseudo_element, FunctionalPseudoElement): if isinstance(pseudo_element, FunctionalPseudoElement):
method_name = f"xpath_{pseudo_element.name.replace('-', '_')}_functional_pseudo_element" method_name = f"xpath_{pseudo_element.name.replace('-', '_')}_functional_pseudo_element"
@@ -143,4 +138,4 @@ class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator):
return super().css_to_xpath(css, prefix) return super().css_to_xpath(css, prefix)
translator_instance = HTMLTranslator() translator = HTMLTranslator()
+3 -3
View File
@@ -36,7 +36,7 @@ from scrapling.core.storage import (
StorageSystemMixin, StorageSystemMixin,
_StorageTools, _StorageTools,
) )
from scrapling.core.translator import translator_instance from scrapling.core.translator import translator as _translator
from scrapling.core.utils import clean_spaces, flatten, html_forbidden, is_jsonable, log from scrapling.core.utils import clean_spaces, flatten, html_forbidden, is_jsonable, log
__DEFAULT_DB_FILE__ = str(Path(__file__).parent / "elements_storage.db") __DEFAULT_DB_FILE__ = str(Path(__file__).parent / "elements_storage.db")
@@ -600,7 +600,7 @@ class Selector(SelectorsGeneration):
try: try:
if not self.__adaptive_enabled or "," not in selector: if not self.__adaptive_enabled or "," not in selector:
# No need to split selectors in this case, let's save some CPU cycles :) # No need to split selectors in this case, let's save some CPU cycles :)
xpath_selector = translator_instance.css_to_xpath(selector) xpath_selector = _translator.css_to_xpath(selector)
return self.xpath( return self.xpath(
xpath_selector, xpath_selector,
identifier or selector, identifier or selector,
@@ -614,7 +614,7 @@ class Selector(SelectorsGeneration):
for single_selector in split_selectors(selector): for single_selector in split_selectors(selector):
# I'm doing this only so the `save` function saves data correctly for combined selectors # I'm doing this only so the `save` function saves data correctly for combined selectors
# Like using the ',' to combine two different selectors that point to different elements. # Like using the ',' to combine two different selectors that point to different elements.
xpath_selector = translator_instance.css_to_xpath( xpath_selector = _translator.css_to_xpath(
single_selector.canonical() single_selector.canonical()
) )
results += self.xpath( results += self.xpath(