diff --git a/scrapling/parser.py b/scrapling/parser.py index 166e6ea..e510fdf 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -304,13 +304,23 @@ class Selector(SelectorsGeneration): ignored_elements.update(cast(list, _find_all_elements(element))) _all_strings = [] - for node in self._root.iter(): - if node not in ignored_elements: - text = node.text - if text and isinstance(text, str): - processed_text = text.strip() if strip else text - if not valid_values or processed_text.strip(): - _all_strings.append(processed_text) + + def append_text(text: Any) -> None: + if text and isinstance(text, str): + processed_text = text.strip() if strip else text + if not valid_values or processed_text.strip(): + _all_strings.append(processed_text) + + def walk(node: Any) -> None: + if node in ignored_elements: + return + + append_text(node.text) + for child in node: + walk(child) + append_text(child.tail) + + walk(self._root) return cast(TextHandler, TextHandler(separator).join(_all_strings)) diff --git a/tests/parser/test_general.py b/tests/parser/test_general.py index 26f6e6c..d459d44 100644 --- a/tests/parser/test_general.py +++ b/tests/parser/test_general.py @@ -327,6 +327,30 @@ def test_getting_all_text(page): assert page.get_all_text() != "" +def test_getting_all_text_from_nested_content(): + """Test getting all text preserves interleaved text nodes""" + html = """ + + +
+ string1 + string2 + string3 +
+ string4 +
+ string5 +
+ + + """ + + page = Selector(html, adaptive=False) + node = page.css("main")[0] + + assert node.get_all_text("\n", strip=True) == "string1\nstring2\nstring3\nstring4\nstring5" + + def test_regex_on_text(page): """Test regex operations on text""" element = page.css('[data-id="1"] .price')[0]