Adding keep_cdata argument for Adaptor and Response classes
This will force lxml to keep cdata while parsing html if you want
This commit is contained in:
@@ -219,7 +219,7 @@ You might be slightly confused by now so let me clear things up. All fetcher-typ
|
|||||||
```python
|
```python
|
||||||
from scrapling import Fetcher, StealthyFetcher, PlayWrightFetcher
|
from scrapling import Fetcher, StealthyFetcher, PlayWrightFetcher
|
||||||
```
|
```
|
||||||
All of them can take these initialization arguments: `auto_match`, `huge_tree`, `keep_comments`, `storage`, `storage_args`, and `debug`, which are the same ones you give to the `Adaptor` class.
|
All of them can take these initialization arguments: `auto_match`, `huge_tree`, `keep_comments`, `keep_cdata`, `storage`, `storage_args`, and `debug`, which are the same ones you give to the `Adaptor` class.
|
||||||
|
|
||||||
If you don't want to pass arguments to the generated `Adaptor` object and want to use the default values, you can use this import instead for cleaner code:
|
If you don't want to pass arguments to the generated `Adaptor` object and want to use the default values, you can use this import instead for cleaner code:
|
||||||
```python
|
```python
|
||||||
|
|||||||
@@ -105,7 +105,7 @@ class BaseFetcher:
|
|||||||
def __init__(
|
def __init__(
|
||||||
self, huge_tree: bool = True, keep_comments: Optional[bool] = False, auto_match: Optional[bool] = True,
|
self, huge_tree: bool = True, keep_comments: Optional[bool] = False, auto_match: Optional[bool] = True,
|
||||||
storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None, debug: Optional[bool] = False,
|
storage: Any = SQLiteStorageSystem, storage_args: Optional[Dict] = None, debug: Optional[bool] = False,
|
||||||
automatch_domain: Optional[str] = None,
|
automatch_domain: Optional[str] = None, keep_cdata: Optional[bool] = False,
|
||||||
):
|
):
|
||||||
"""Arguments below are the same from the Adaptor class so you can pass them directly, the rest of Adaptor's arguments
|
"""Arguments below are the same from the Adaptor class so you can pass them directly, the rest of Adaptor's arguments
|
||||||
are detected and passed automatically from the Fetcher based on the response for accessibility.
|
are detected and passed automatically from the Fetcher based on the response for accessibility.
|
||||||
@@ -113,6 +113,7 @@ class BaseFetcher:
|
|||||||
:param huge_tree: Enabled by default, should always be enabled when parsing large HTML documents. This controls
|
:param huge_tree: Enabled by default, should always be enabled when parsing large HTML documents. This controls
|
||||||
libxml2 feature that forbids parsing certain large documents to protect from possible memory exhaustion.
|
libxml2 feature that forbids parsing certain large documents to protect from possible memory exhaustion.
|
||||||
:param keep_comments: While parsing the HTML body, drop comments or not. Disabled by default for obvious reasons
|
:param keep_comments: While parsing the HTML body, drop comments or not. Disabled by default for obvious reasons
|
||||||
|
:param keep_cdata: While parsing the HTML body, drop cdata or not. Disabled by default for cleaner HTML.
|
||||||
:param auto_match: Globally turn-off the auto-match feature in all functions, this argument takes higher
|
:param auto_match: Globally turn-off the auto-match feature in all functions, this argument takes higher
|
||||||
priority over all auto-match related arguments/functions in the class.
|
priority over all auto-match related arguments/functions in the class.
|
||||||
:param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.
|
:param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.
|
||||||
@@ -127,6 +128,7 @@ class BaseFetcher:
|
|||||||
self.adaptor_arguments = dict(
|
self.adaptor_arguments = dict(
|
||||||
huge_tree=huge_tree,
|
huge_tree=huge_tree,
|
||||||
keep_comments=keep_comments,
|
keep_comments=keep_comments,
|
||||||
|
keep_cdata=keep_cdata,
|
||||||
auto_match=auto_match,
|
auto_match=auto_match,
|
||||||
storage=storage,
|
storage=storage,
|
||||||
storage_args=storage_args,
|
storage_args=storage_args,
|
||||||
|
|||||||
+7
-3
@@ -25,6 +25,7 @@ class Adaptor(SelectorsGeneration):
|
|||||||
__slots__ = (
|
__slots__ = (
|
||||||
'url', 'encoding', '__auto_match_enabled', '_root', '_storage', '__debug',
|
'url', 'encoding', '__auto_match_enabled', '_root', '_storage', '__debug',
|
||||||
'__keep_comments', '__huge_tree_enabled', '__attributes', '__text', '__tag',
|
'__keep_comments', '__huge_tree_enabled', '__attributes', '__text', '__tag',
|
||||||
|
'__keep_cdata',
|
||||||
)
|
)
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -36,6 +37,7 @@ class Adaptor(SelectorsGeneration):
|
|||||||
huge_tree: bool = True,
|
huge_tree: bool = True,
|
||||||
root: Optional[html.HtmlElement] = None,
|
root: Optional[html.HtmlElement] = None,
|
||||||
keep_comments: Optional[bool] = False,
|
keep_comments: Optional[bool] = False,
|
||||||
|
keep_cdata: Optional[bool] = False,
|
||||||
auto_match: Optional[bool] = True,
|
auto_match: Optional[bool] = True,
|
||||||
storage: Any = SQLiteStorageSystem,
|
storage: Any = SQLiteStorageSystem,
|
||||||
storage_args: Optional[Dict] = None,
|
storage_args: Optional[Dict] = None,
|
||||||
@@ -59,6 +61,7 @@ class Adaptor(SelectorsGeneration):
|
|||||||
:param root: Used internally to pass etree objects instead of text/body arguments, it takes highest priority.
|
:param root: Used internally to pass etree objects instead of text/body arguments, it takes highest priority.
|
||||||
Don't use it unless you know what you are doing!
|
Don't use it unless you know what you are doing!
|
||||||
:param keep_comments: While parsing the HTML body, drop comments or not. Disabled by default for obvious reasons
|
:param keep_comments: While parsing the HTML body, drop comments or not. Disabled by default for obvious reasons
|
||||||
|
:param keep_cdata: While parsing the HTML body, drop cdata or not. Disabled by default for cleaner HTML.
|
||||||
:param auto_match: Globally turn-off the auto-match feature in all functions, this argument takes higher
|
:param auto_match: Globally turn-off the auto-match feature in all functions, this argument takes higher
|
||||||
priority over all auto-match related arguments/functions in the class.
|
priority over all auto-match related arguments/functions in the class.
|
||||||
:param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.
|
:param storage: The storage class to be passed for auto-matching functionalities, see ``Docs`` for more info.
|
||||||
@@ -84,8 +87,8 @@ class Adaptor(SelectorsGeneration):
|
|||||||
|
|
||||||
# https://lxml.de/api/lxml.etree.HTMLParser-class.html
|
# https://lxml.de/api/lxml.etree.HTMLParser-class.html
|
||||||
parser = html.HTMLParser(
|
parser = html.HTMLParser(
|
||||||
recover=True, remove_blank_text=True, remove_comments=(keep_comments is False), encoding=encoding,
|
recover=True, remove_blank_text=True, remove_comments=(not keep_comments), encoding=encoding,
|
||||||
compact=True, huge_tree=huge_tree, default_doctype=True
|
compact=True, huge_tree=huge_tree, default_doctype=True, strip_cdata=(not keep_cdata),
|
||||||
)
|
)
|
||||||
self._root = etree.fromstring(body, parser=parser, base_url=url)
|
self._root = etree.fromstring(body, parser=parser, base_url=url)
|
||||||
if is_jsonable(text or body.decode()):
|
if is_jsonable(text or body.decode()):
|
||||||
@@ -119,6 +122,7 @@ class Adaptor(SelectorsGeneration):
|
|||||||
self._storage = storage(**storage_args)
|
self._storage = storage(**storage_args)
|
||||||
|
|
||||||
self.__keep_comments = keep_comments
|
self.__keep_comments = keep_comments
|
||||||
|
self.__keep_cdata = keep_cdata
|
||||||
self.__huge_tree_enabled = huge_tree
|
self.__huge_tree_enabled = huge_tree
|
||||||
self.encoding = encoding
|
self.encoding = encoding
|
||||||
self.url = url
|
self.url = url
|
||||||
@@ -156,7 +160,7 @@ class Adaptor(SelectorsGeneration):
|
|||||||
root=element,
|
root=element,
|
||||||
text='', body=b'', # Since root argument is provided, both `text` and `body` will be ignored so this is just a filler
|
text='', body=b'', # Since root argument is provided, both `text` and `body` will be ignored so this is just a filler
|
||||||
url=self.url, encoding=self.encoding, auto_match=self.__auto_match_enabled,
|
url=self.url, encoding=self.encoding, auto_match=self.__auto_match_enabled,
|
||||||
keep_comments=True, # if the comments are already removed in initialization, no need to try to delete them in sub-elements
|
keep_comments=self.__keep_comments, keep_cdata=self.__keep_cdata,
|
||||||
huge_tree=self.__huge_tree_enabled, debug=self.__debug,
|
huge_tree=self.__huge_tree_enabled, debug=self.__debug,
|
||||||
**self.__response_data
|
**self.__response_data
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user