From e785cf26ce41b2e5f4e3ef3a4c70a573d45a8459 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Sat, 14 Feb 2026 03:34:13 +0200 Subject: [PATCH] fix(parser)!: Optimize parser for repeated operations --- scrapling/parser.py | 100 +++++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 61 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index c8d04af..166e6ea 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -4,7 +4,7 @@ from urllib.parse import urljoin from difflib import SequenceMatcher from re import Pattern as re_Pattern -from lxml.html import HtmlElement, HtmlMixin, HTMLParser +from lxml.html import HtmlElement, HTMLParser from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors from lxml.etree import ( XPath, @@ -76,9 +76,6 @@ class Selector(SelectorsGeneration): "_raw_body", ) - if TYPE_CHECKING: - _storage: StorageSystemMixin - def __init__( self, content: Optional[str | bytes] = None, @@ -120,9 +117,17 @@ class Selector(SelectorsGeneration): if root is None and content is None: raise ValueError("Selector class needs HTML content, or root arguments to work") + self.url = url + self._raw_body: str | bytes = "" + self.encoding = encoding + self.__keep_cdata = keep_cdata + self.__huge_tree_enabled = huge_tree + self.__keep_comments = keep_comments + # For selector stuff self.__text: Optional[TextHandler] = None - self.__tag: Optional[str] = None self.__attributes: Optional[AttributesHandler] = None + self.__tag: Optional[str] = None + self._storage: Optional[StorageSystemMixin] = None if root is None: body: str | bytes if isinstance(content, str): @@ -148,27 +153,11 @@ class Selector(SelectorsGeneration): self._raw_body = content else: - if self._is_text_node(root): - # Text node (from ::text, /text(), ::attr(), /@attr, etc.) - self._root = root - self._raw_body = "" - self.__adaptive_enabled = False - self.__keep_comments = keep_comments - self.__keep_cdata = keep_cdata - self.__huge_tree_enabled = huge_tree - self.encoding = encoding - self.url = url - self.__attributes = None - self.__tag = None - return - # All HTML types inherit from HtmlMixin so this to check for all at once - elif not issubclass(type(root), HtmlMixin): - raise TypeError( - f"Root have to be a valid element of `html` module types to work, not of type {type(root)}" - ) - self._root = cast(HtmlElement, root) - self._raw_body = "" + + if self._is_text_node(root): + self.__adaptive_enabled = False + return self.__adaptive_enabled = bool(adaptive) @@ -190,36 +179,6 @@ class Selector(SelectorsGeneration): self._storage = storage(**storage_args) - self.__keep_comments = keep_comments - self.__keep_cdata = keep_cdata - self.__huge_tree_enabled = huge_tree - self.encoding = encoding - self.url = url - # For selector stuff - self.__attributes = None - self.__tag = None - - @property - def __response_data(self): - # No need to check if all response attributes exist or not because if `status` exist, then the rest exist (Save some CPU cycles for speed) - if not hasattr(self, "_cached_response_data"): - self._cached_response_data = ( - { - key: getattr(self, key) - for key in ( - "status", - "reason", - "cookies", - "history", - "headers", - "request_headers", - ) - } - if hasattr(self, "status") - else {} - ) - return self._cached_response_data - def __getitem__(self, key: str) -> TextHandler: if self._is_text_node(self._root): raise TypeError("Text nodes do not have attributes") @@ -245,21 +204,40 @@ class Selector(SelectorsGeneration): def __element_convertor(self, element: HtmlElement | _ElementUnicodeResult) -> "Selector": """Used internally to convert a single HtmlElement or text node to Selector directly without checks""" - db_instance = self._storage if (hasattr(self, "_storage") and self._storage) else None return Selector( root=element, url=self.url, encoding=self.encoding, adaptive=self.__adaptive_enabled, - _storage=db_instance, # Reuse existing storage if it exists otherwise it won't be checked if `adaptive` is turned off + _storage=self._storage, keep_comments=self.__keep_comments, keep_cdata=self.__keep_cdata, huge_tree=self.__huge_tree_enabled, - **self.__response_data, ) def __elements_convertor(self, elements: List[HtmlElement | _ElementUnicodeResult]) -> "Selectors": - return Selectors(map(self.__element_convertor, elements)) + # Store them for non-repeated call-ups + url = self.url + encoding = self.encoding + adaptive = self.__adaptive_enabled + storage = self._storage + comments = self.__keep_comments + cdata = self.__keep_cdata + huge_tree = self.__huge_tree_enabled + + return Selectors( + Selector( + root=el, + url=url, + encoding=encoding, + adaptive=adaptive, + _storage=storage, + keep_comments=comments, + keep_cdata=cdata, + huge_tree=huge_tree, + ) + for el in elements + ) def __handle_elements(self, result: List[HtmlElement | _ElementUnicodeResult]) -> "Selectors": """Used internally in all functions to convert results to Selectors in bulk""" @@ -889,7 +867,7 @@ class Selector(SelectorsGeneration): :param identifier: This is the identifier that will be used to retrieve the element later from the storage. See the docs for more info. """ - if self.__adaptive_enabled: + if self.__adaptive_enabled and self._storage: target_element: Any = element if isinstance(target_element, self.__class__): target_element = target_element._root @@ -910,7 +888,7 @@ class Selector(SelectorsGeneration): the docs for more info. :return: A dictionary of the unique properties """ - if self.__adaptive_enabled: + if self.__adaptive_enabled and self._storage: return self._storage.retrieve(identifier) raise RuntimeError(