diff --git a/scrapling/parser.py b/scrapling/parser.py index 166e6ea..eced0a9 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -58,6 +58,7 @@ _find_all_elements = XPath(".//*") _find_all_elements_with_spaces = XPath( ".//*[normalize-space(text())]" ) # This selector gets all elements with text content +_find_all_text_nodes = XPath(".//text()") class Selector(SelectorsGeneration): @@ -299,18 +300,31 @@ class Selector(SelectorsGeneration): ignored_elements: set[Any] = set() if ignore_tags: - for element in self._root.iter(*ignore_tags): - ignored_elements.add(element) - ignored_elements.update(cast(list, _find_all_elements(element))) + ignored_elements.update(self._root.iter(*ignore_tags)) _all_strings = [] - for node in self._root.iter(): - if node not in ignored_elements: - text = node.text - if text and isinstance(text, str): - processed_text = text.strip() if strip else text - if not valid_values or processed_text.strip(): - _all_strings.append(processed_text) + + def append_text(text: str) -> None: + processed_text = text.strip() if strip else text + if not valid_values or processed_text.strip(): + _all_strings.append(processed_text) + + def is_visible_text_node(text_node: _ElementUnicodeResult) -> bool: + parent = text_node.getparent() + if parent is None: + return False + + owner = parent.getparent() if text_node.is_tail else parent + while owner is not None: + if owner in ignored_elements: + return False + owner = owner.getparent() + return True + + for text_node in cast(list[_ElementUnicodeResult], _find_all_text_nodes(self._root)): + text = str(text_node) + if text and is_visible_text_node(text_node): + append_text(text) return cast(TextHandler, TextHandler(separator).join(_all_strings)) diff --git a/tests/parser/test_parser_advanced.py b/tests/parser/test_parser_advanced.py index 57086ba..f34ba5c 100644 --- a/tests/parser/test_parser_advanced.py +++ b/tests/parser/test_parser_advanced.py @@ -183,6 +183,33 @@ class TestAdvancedSelectors: text = page.get_all_text(valid_values=False) assert text != "" + def test_get_all_text_preserves_interleaved_text_nodes(self): + """Test get_all_text preserves interleaved text nodes""" + html = """ + + +
+ string1 + string2 + string3 +
+ string4 +
+ string5 + + string6 + + string7 +
+ + + """ + + page = Selector(html, adaptive=False) + node = page.css("main")[0] + + assert node.get_all_text("\n", strip=True) == "string1\nstring2\nstring3\nstring4\nstring5\nstring6\nstring7" + class TestTextHandlerAdvanced: """Test advanced TextHandler functionality"""