From 5457a697faec1ea76015c022ba5630a5cb27f95c Mon Sep 17 00:00:00 2001 From: Matt Hillebrand Date: Thu, 5 Mar 2026 20:34:44 -0800 Subject: [PATCH 1/4] fix: Selector.get_all_text() doesn't get all text #167 - Updated text extraction using recursion - Added new unit test --- scrapling/parser.py | 24 +++++++++++++++++------- tests/parser/test_general.py | 24 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index 166e6ea..e510fdf 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -304,13 +304,23 @@ class Selector(SelectorsGeneration): ignored_elements.update(cast(list, _find_all_elements(element))) _all_strings = [] - for node in self._root.iter(): - if node not in ignored_elements: - text = node.text - if text and isinstance(text, str): - processed_text = text.strip() if strip else text - if not valid_values or processed_text.strip(): - _all_strings.append(processed_text) + + def append_text(text: Any) -> None: + if text and isinstance(text, str): + processed_text = text.strip() if strip else text + if not valid_values or processed_text.strip(): + _all_strings.append(processed_text) + + def walk(node: Any) -> None: + if node in ignored_elements: + return + + append_text(node.text) + for child in node: + walk(child) + append_text(child.tail) + + walk(self._root) return cast(TextHandler, TextHandler(separator).join(_all_strings)) diff --git a/tests/parser/test_general.py b/tests/parser/test_general.py index 26f6e6c..d459d44 100644 --- a/tests/parser/test_general.py +++ b/tests/parser/test_general.py @@ -327,6 +327,30 @@ def test_getting_all_text(page): assert page.get_all_text() != "" +def test_getting_all_text_from_nested_content(): + """Test getting all text preserves interleaved text nodes""" + html = """ + + +
+ string1 + string2 + string3 +
+ string4 +
+ string5 +
+ + + """ + + page = Selector(html, adaptive=False) + node = page.css("main")[0] + + assert node.get_all_text("\n", strip=True) == "string1\nstring2\nstring3\nstring4\nstring5" + + def test_regex_on_text(page): """Test regex operations on text""" element = page.css('[data-id="1"] .price')[0] From 5806b8e01489e4d03168ae4e7890ab49b41aabd0 Mon Sep 17 00:00:00 2001 From: Matt Hillebrand Date: Fri, 6 Mar 2026 09:01:32 -0800 Subject: [PATCH 2/4] - Add pre-compiled XPath text selector - Avoid recursion --- scrapling/parser.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/scrapling/parser.py b/scrapling/parser.py index e510fdf..eced0a9 100644 --- a/scrapling/parser.py +++ b/scrapling/parser.py @@ -58,6 +58,7 @@ _find_all_elements = XPath(".//*") _find_all_elements_with_spaces = XPath( ".//*[normalize-space(text())]" ) # This selector gets all elements with text content +_find_all_text_nodes = XPath(".//text()") class Selector(SelectorsGeneration): @@ -299,28 +300,31 @@ class Selector(SelectorsGeneration): ignored_elements: set[Any] = set() if ignore_tags: - for element in self._root.iter(*ignore_tags): - ignored_elements.add(element) - ignored_elements.update(cast(list, _find_all_elements(element))) + ignored_elements.update(self._root.iter(*ignore_tags)) _all_strings = [] - def append_text(text: Any) -> None: - if text and isinstance(text, str): - processed_text = text.strip() if strip else text - if not valid_values or processed_text.strip(): - _all_strings.append(processed_text) + def append_text(text: str) -> None: + processed_text = text.strip() if strip else text + if not valid_values or processed_text.strip(): + _all_strings.append(processed_text) - def walk(node: Any) -> None: - if node in ignored_elements: - return + def is_visible_text_node(text_node: _ElementUnicodeResult) -> bool: + parent = text_node.getparent() + if parent is None: + return False - append_text(node.text) - for child in node: - walk(child) - append_text(child.tail) + owner = parent.getparent() if text_node.is_tail else parent + while owner is not None: + if owner in ignored_elements: + return False + owner = owner.getparent() + return True - walk(self._root) + for text_node in cast(list[_ElementUnicodeResult], _find_all_text_nodes(self._root)): + text = str(text_node) + if text and is_visible_text_node(text_node): + append_text(text) return cast(TextHandler, TextHandler(separator).join(_all_strings)) From 00ad6b4c5f3dc0f219cf7151d14f383b23df2a41 Mon Sep 17 00:00:00 2001 From: Matt Hillebrand Date: Fri, 6 Mar 2026 11:39:00 -0800 Subject: [PATCH 3/4] Add two ignored elements to unit test --- tests/parser/test_general.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/parser/test_general.py b/tests/parser/test_general.py index d459d44..36241a6 100644 --- a/tests/parser/test_general.py +++ b/tests/parser/test_general.py @@ -340,6 +340,8 @@ def test_getting_all_text_from_nested_content(): string4 string5 + + From 7cbe566acdbc3cfc78a375b454bf6c226e10a665 Mon Sep 17 00:00:00 2001 From: Matt Hillebrand Date: Sun, 8 Mar 2026 08:14:35 -0700 Subject: [PATCH 4/4] Move test for get_all_text to test_parser_advanced.py --- tests/parser/test_general.py | 26 -------------------------- tests/parser/test_parser_advanced.py | 27 +++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/tests/parser/test_general.py b/tests/parser/test_general.py index 36241a6..26f6e6c 100644 --- a/tests/parser/test_general.py +++ b/tests/parser/test_general.py @@ -327,32 +327,6 @@ def test_getting_all_text(page): assert page.get_all_text() != "" -def test_getting_all_text_from_nested_content(): - """Test getting all text preserves interleaved text nodes""" - html = """ - - -
- string1 - string2 - string3 -
- string4 -
- string5 - - -
- - - """ - - page = Selector(html, adaptive=False) - node = page.css("main")[0] - - assert node.get_all_text("\n", strip=True) == "string1\nstring2\nstring3\nstring4\nstring5" - - def test_regex_on_text(page): """Test regex operations on text""" element = page.css('[data-id="1"] .price')[0] diff --git a/tests/parser/test_parser_advanced.py b/tests/parser/test_parser_advanced.py index 57086ba..f34ba5c 100644 --- a/tests/parser/test_parser_advanced.py +++ b/tests/parser/test_parser_advanced.py @@ -183,6 +183,33 @@ class TestAdvancedSelectors: text = page.get_all_text(valid_values=False) assert text != "" + def test_get_all_text_preserves_interleaved_text_nodes(self): + """Test get_all_text preserves interleaved text nodes""" + html = """ + + +
+ string1 + string2 + string3 +
+ string4 +
+ string5 + + string6 + + string7 +
+ + + """ + + page = Selector(html, adaptive=False) + node = page.css("main")[0] + + assert node.get_all_text("\n", strip=True) == "string1\nstring2\nstring3\nstring4\nstring5\nstring6\nstring7" + class TestTextHandlerAdvanced: """Test advanced TextHandler functionality"""