freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Handling an edge case and adding more commentary

Browse Source
This commit is contained in:
Karim shoair
2024-10-16 19:32:01 +03:00
parent a01cfe66db
commit d546b3499b
1 changed files with 14 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/parser.py
+14 -10
View File
@@ -188,17 +188,21 @@ class Adaptor(SelectorsGeneration):
"""Get text content of the element""" """Get text content of the element"""
if not self.__text: if not self.__text:
if self.__keep_comments: if self.__keep_comments:
# If use chose to keep comments, remove comments from text if not self.children:
# Escape lxml default behaviour and remove comments like this `<span>CONDITION: <!-- -->Excellent</span>` # If use chose to keep comments, remove comments from text
# This issue is present in parsel/scrapy as well so no need to repeat it here so the user can run regex on the full text. # Escape lxml default behaviour and remove comments like this `<span>CONDITION: <!-- -->Excellent</span>`
code = self.html_content # This issue is present in parsel/scrapy as well so no need to repeat it here so the user can run regex on the full text.
parser = html.HTMLParser( code = self.html_content
recover=True, remove_blank_text=True, remove_comments=True, encoding=self.encoding, parser = html.HTMLParser(
compact=True, huge_tree=self.__huge_tree_enabled, default_doctype=True recover=True, remove_blank_text=True, remove_comments=True, encoding=self.encoding,
) compact=True, huge_tree=self.__huge_tree_enabled, default_doctype=True
fragment_root = html.fragment_fromstring(code, parser=parser) )
self.__text = TextHandler(fragment_root.text) fragment_root = html.fragment_fromstring(code, parser=parser)
self.__text = TextHandler(fragment_root.text)
else:
self.__text = TextHandler(self._root.text)
else: else:
# If user already chose to not keep comments then all is good
self.__text = TextHandler(self._root.text) self.__text = TextHandler(self._root.text)
return self.__text return self.__text