refactor(parser/get_all_text): Cleaner and a bit faster implementation
This commit is contained in:
+3
-16
@@ -223,29 +223,16 @@ class Adaptor(SelectorsGeneration):
|
|||||||
:return: A TextHandler
|
:return: A TextHandler
|
||||||
"""
|
"""
|
||||||
_all_strings = []
|
_all_strings = []
|
||||||
|
for node in self._root.xpath('.//*'):
|
||||||
def _traverse(node: html.HtmlElement) -> None:
|
|
||||||
"""Traverse element children and get text content of each
|
|
||||||
|
|
||||||
:param node: Current node in the tree structure
|
|
||||||
:return:
|
|
||||||
"""
|
|
||||||
if node.tag not in ignore_tags:
|
if node.tag not in ignore_tags:
|
||||||
text = node.text
|
text = node.text
|
||||||
if text and type(text) is str:
|
if text and type(text) is str:
|
||||||
if valid_values:
|
if valid_values and text.strip():
|
||||||
if text.strip():
|
|
||||||
_all_strings.append(text if not strip else text.strip())
|
_all_strings.append(text if not strip else text.strip())
|
||||||
else:
|
else:
|
||||||
_all_strings.append(text if not strip else text.strip())
|
_all_strings.append(text if not strip else text.strip())
|
||||||
|
|
||||||
for branch in node.iterchildren():
|
return TextHandler(separator.join(_all_strings))
|
||||||
_traverse(branch)
|
|
||||||
|
|
||||||
# We will start using Lxml directly for the speed boost
|
|
||||||
_traverse(self._root)
|
|
||||||
|
|
||||||
return TextHandler(separator.join([s for s in _all_strings]))
|
|
||||||
|
|
||||||
def urljoin(self, relative_url: str) -> str:
|
def urljoin(self, relative_url: str) -> str:
|
||||||
"""Join this Adaptor's url with a relative url to form an absolute full URL."""
|
"""Join this Adaptor's url with a relative url to form an absolute full URL."""
|
||||||
|
|||||||
Reference in New Issue
Block a user