fix: Selector.get_all_text() doesn't get all text #167

- Updated text extraction using recursion - Added new unit test
2026-03-05 20:34:44 -08:00
parent 54f2c96adf
commit 5457a697fa
2 changed files with 41 additions and 7 deletions
@@ -304,13 +304,23 @@ class Selector(SelectorsGeneration):
                ignored_elements.update(cast(list, _find_all_elements(element)))

        _all_strings = []
-        for node in self._root.iter():
-            if node not in ignored_elements:
-                text = node.text
-                if text and isinstance(text, str):
-                    processed_text = text.strip() if strip else text
-                    if not valid_values or processed_text.strip():
-                        _all_strings.append(processed_text)
+
+        def append_text(text: Any) -> None:
+            if text and isinstance(text, str):
+                processed_text = text.strip() if strip else text
+                if not valid_values or processed_text.strip():
+                    _all_strings.append(processed_text)
+
+        def walk(node: Any) -> None:
+            if node in ignored_elements:
+                return
+
+            append_text(node.text)
+            for child in node:
+                walk(child)
+                append_text(child.tail)
+
+        walk(self._root)

        return cast(TextHandler, TextHandler(separator).join(_all_strings))

@@ -327,6 +327,30 @@ def test_getting_all_text(page):
    assert page.get_all_text() != ""


+def test_getting_all_text_from_nested_content():
+    """Test getting all text preserves interleaved text nodes"""
+    html = """
+    <html>
+    <body>
+        <main>
+            string1
+            <b>string2</b>
+            string3
+            <div>
+                <span>string4</span>
+            </div>
+            string5
+        </main>
+    </body>
+    </html>
+    """
+
+    page = Selector(html, adaptive=False)
+    node = page.css("main")[0]
+
+    assert node.get_all_text("\n", strip=True) == "string1\nstring2\nstring3\nstring4\nstring5"
+
+
 def test_regex_on_text(page):
    """Test regex operations on text"""
    element = page.css('[data-id="1"] .price')[0]