From d546b3499b0d6e6f06d85bfb310b49b97ab89b00 Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Wed, 16 Oct 2024 19:32:01 +0300 Subject: [PATCH] Handling an edge case and adding more commentary --- scrapling/parser.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index d117b23..a517112 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -188,17 +188,21 @@ class Adaptor(SelectorsGeneration): """Get text content of the element""" if not self.__text: if self.__keep_comments: - # If use chose to keep comments, remove comments from text - # Escape lxml default behaviour and remove comments like this `CONDITION: Excellent` - # This issue is present in parsel/scrapy as well so no need to repeat it here so the user can run regex on the full text. - code = self.html_content - parser = html.HTMLParser( - recover=True, remove_blank_text=True, remove_comments=True, encoding=self.encoding, - compact=True, huge_tree=self.__huge_tree_enabled, default_doctype=True - ) - fragment_root = html.fragment_fromstring(code, parser=parser) - self.__text = TextHandler(fragment_root.text) + if not self.children: + # If use chose to keep comments, remove comments from text + # Escape lxml default behaviour and remove comments like this `CONDITION: Excellent` + # This issue is present in parsel/scrapy as well so no need to repeat it here so the user can run regex on the full text. + code = self.html_content + parser = html.HTMLParser( + recover=True, remove_blank_text=True, remove_comments=True, encoding=self.encoding, + compact=True, huge_tree=self.__huge_tree_enabled, default_doctype=True + ) + fragment_root = html.fragment_fromstring(code, parser=parser) + self.__text = TextHandler(fragment_root.text) + else: + self.__text = TextHandler(self._root.text) else: + # If user already chose to not keep comments then all is good self.__text = TextHandler(self._root.text) return self.__text