freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(parser): Lowering memory with replacing body property with html_content

Browse Source
This commit is contained in:
Karim shoair
2025-03-17 05:58:40 +02:00
parent e5a022ec6a
commit 68ac9d4d05
1 changed files with 2 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/parser.py
+2 -8
View File
@@ -26,7 +26,7 @@ class Adaptor(SelectorsGeneration):
__slots__ = ( __slots__ = (
'url', 'encoding', '__auto_match_enabled', '_root', '_storage', 'url', 'encoding', '__auto_match_enabled', '_root', '_storage',
'__keep_comments', '__huge_tree_enabled', '__attributes', '__text', '__tag', '__keep_comments', '__huge_tree_enabled', '__attributes', '__text', '__tag',
'__keep_cdata', '__raw_body' '__keep_cdata'
) )
def __init__( def __init__(
@@ -72,20 +72,17 @@ class Adaptor(SelectorsGeneration):
raise ValueError("Adaptor class needs text, body, or root arguments to work") raise ValueError("Adaptor class needs text, body, or root arguments to work")
self.__text = '' self.__text = ''
self.__raw_body = ''
if root is None: if root is None:
if text is None: if text is None:
if not body or not isinstance(body, bytes): if not body or not isinstance(body, bytes):
raise TypeError(f"body argument must be valid and of type bytes, got {body.__class__}") raise TypeError(f"body argument must be valid and of type bytes, got {body.__class__}")
body = body.replace(b"\x00", b"").strip() body = body.replace(b"\x00", b"").strip()
self.__raw_body = body.replace(b"\x00", b"").strip().decode()
else: else:
if not isinstance(text, str): if not isinstance(text, str):
raise TypeError(f"text argument must be of type str, got {text.__class__}") raise TypeError(f"text argument must be of type str, got {text.__class__}")
body = text.strip().replace("\x00", "").encode(encoding) or b"<html/>" body = text.strip().replace("\x00", "").encode(encoding) or b"<html/>"
self.__raw_body = text.strip()
# https://lxml.de/api/lxml.etree.HTMLParser-class.html # https://lxml.de/api/lxml.etree.HTMLParser-class.html
parser = html.HTMLParser( parser = html.HTMLParser(
@@ -250,10 +247,7 @@ class Adaptor(SelectorsGeneration):
"""Return the inner html code of the element""" """Return the inner html code of the element"""
return TextHandler(etree.tostring(self._root, encoding='unicode', method='html', with_tail=False)) return TextHandler(etree.tostring(self._root, encoding='unicode', method='html', with_tail=False))
@property body = html_content
def body(self) -> TextHandler:
"""Return raw HTML code of the element/page without any processing when possible or return `Adaptor.html_content`"""
return TextHandler(self.__raw_body) or self.html_content
def prettify(self) -> TextHandler: def prettify(self) -> TextHandler:
"""Return a prettified version of the element's inner html-code""" """Return a prettified version of the element's inner html-code"""