freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(parser): Improving CSS selecting memory usage by ~144%

Browse Source 
and speed by ~2%
This commit is contained in:
Karim shoair
2025-03-16 22:33:46 +02:00
parent ed89161cd9
commit e5a022ec6a
2 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/core/translator.py
+3
View File
@@ -142,3 +142,6 @@ class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator):
@lru_cache(maxsize=256) @lru_cache(maxsize=256)
def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str: def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str:
return super().css_to_xpath(css, prefix) return super().css_to_xpath(css, prefix)
translator_instance = HTMLTranslator()
scrapling/parser.py
+3 -3
View File
@@ -17,7 +17,7 @@ from scrapling.core.custom_types import (AttributesHandler, TextHandler,
from scrapling.core.mixins import SelectorsGeneration from scrapling.core.mixins import SelectorsGeneration
from scrapling.core.storage_adaptors import (SQLiteStorageSystem, from scrapling.core.storage_adaptors import (SQLiteStorageSystem,
StorageSystemMixin, _StorageTools) StorageSystemMixin, _StorageTools)
from scrapling.core.translator import HTMLTranslator from scrapling.core.translator import translator_instance
from scrapling.core.utils import (clean_spaces, flatten, html_forbidden, from scrapling.core.utils import (clean_spaces, flatten, html_forbidden,
is_jsonable, log) is_jsonable, log)
@@ -476,7 +476,7 @@ class Adaptor(SelectorsGeneration):
try: try:
if not self.__auto_match_enabled or ',' not in selector: if not self.__auto_match_enabled or ',' not in selector:
# No need to split selectors in this case, let's save some CPU cycles :) # No need to split selectors in this case, let's save some CPU cycles :)
xpath_selector = HTMLTranslator().css_to_xpath(selector) xpath_selector = translator_instance.css_to_xpath(selector)
return self.xpath(xpath_selector, identifier or selector, auto_match, auto_save, percentage) return self.xpath(xpath_selector, identifier or selector, auto_match, auto_save, percentage)
results = [] results = []
@@ -484,7 +484,7 @@ class Adaptor(SelectorsGeneration):
for single_selector in split_selectors(selector): for single_selector in split_selectors(selector):
# I'm doing this only so the `save` function save data correctly for combined selectors # I'm doing this only so the `save` function save data correctly for combined selectors
# Like using the ',' to combine two different selectors that point to different elements. # Like using the ',' to combine two different selectors that point to different elements.
xpath_selector = HTMLTranslator().css_to_xpath(single_selector.canonical()) xpath_selector = translator_instance.css_to_xpath(single_selector.canonical())
results += self.xpath( results += self.xpath(
xpath_selector, identifier or single_selector.canonical(), auto_match, auto_save, percentage xpath_selector, identifier or single_selector.canonical(), auto_match, auto_save, percentage
) )