diff --git a/scrapling/parser.py b/scrapling/parser.py index cac7fef..a9945d0 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -10,6 +10,7 @@ from scrapling.core.storage_adaptors import SQLiteStorageSystem, StorageSystemMi from scrapling.core.utils import setup_basic_logging, logging, clean_spaces, flatten, html_forbidden from scrapling.core._types import Any, Dict, List, Tuple, Optional, Pattern, Union, Callable, Generator, SupportsIndex, Iterable from lxml import etree, html +from lxml.etree import XMLSyntaxError from cssselect import SelectorError, SelectorSyntaxError, parse as split_selectors @@ -60,6 +61,7 @@ class Adaptor(SelectorsGeneration): if root is None and not body and text is None: raise ValueError("Adaptor class needs text, body, or root arguments to work") + self.__text = None if root is None: if text is None: if not body or not isinstance(body, bytes): @@ -72,12 +74,21 @@ class Adaptor(SelectorsGeneration): body = text.strip().replace("\x00", "").encode(encoding) or b"" - parser = html.HTMLParser( - # https://lxml.de/api/lxml.etree.HTMLParser-class.html - recover=True, remove_blank_text=True, remove_comments=(keep_comments is False), encoding=encoding, - compact=True, huge_tree=huge_tree, default_doctype=True - ) - self._root = etree.fromstring(body, parser=parser, base_url=url) + # https://lxml.de/api/lxml.etree.HTMLParser-class.html + try: + # Test with recover set to False first so if this is a text body like a json response, we get error + parser = html.HTMLParser( + recover=False, remove_blank_text=True, remove_comments=(keep_comments is False), encoding=encoding, + compact=True, huge_tree=huge_tree, default_doctype=True + ) + self._root = etree.fromstring(body, parser=parser, base_url=url) + except XMLSyntaxError: + parser = html.HTMLParser( + recover=True, remove_blank_text=True, remove_comments=(keep_comments is False), encoding=encoding, + compact=True, huge_tree=huge_tree, default_doctype=True + ) + self._root = etree.fromstring(body, parser=parser, base_url=url) + self.__text = TextHandler(text or body.decode()) else: # All html types inherits from HtmlMixin so this to check for all at once @@ -112,7 +123,6 @@ class Adaptor(SelectorsGeneration): self.url = url # For selector stuff self.__attributes = None - self.__text = None self.__tag = None self.__debug = debug