From 7c35341d85cf78f4ad825985ee9f5b7d6b0dcb3c Mon Sep 17 00:00:00 2001 From: Karim shoair Date: Fri, 31 Jan 2025 02:47:49 +0200 Subject: [PATCH] refactor(parser/get_all_text): Cleaner and a bit faster implementation --- scrapling/parser.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index eac0d64..5a4d4f5 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -223,29 +223,16 @@ class Adaptor(SelectorsGeneration): :return: A TextHandler """ _all_strings = [] - - def _traverse(node: html.HtmlElement) -> None: - """Traverse element children and get text content of each - - :param node: Current node in the tree structure - :return: - """ + for node in self._root.xpath('.//*'): if node.tag not in ignore_tags: text = node.text if text and type(text) is str: - if valid_values: - if text.strip(): - _all_strings.append(text if not strip else text.strip()) + if valid_values and text.strip(): + _all_strings.append(text if not strip else text.strip()) else: _all_strings.append(text if not strip else text.strip()) - for branch in node.iterchildren(): - _traverse(branch) - - # We will start using Lxml directly for the speed boost - _traverse(self._root) - - return TextHandler(separator.join([s for s in _all_strings])) + return TextHandler(separator.join(_all_strings)) def urljoin(self, relative_url: str) -> str: """Join this Adaptor's url with a relative url to form an absolute full URL."""