Merge branch 'dev' into fix/google-referrer-spoof

2026-03-08 19:01:20 +02:00
parent 68b9e128bc 11bc08fbaa
commit b0c8448f81
2 changed files with 51 additions and 10 deletions
@@ -58,6 +58,7 @@ _find_all_elements = XPath(".//*")
 _find_all_elements_with_spaces = XPath(
    ".//*[normalize-space(text())]"
 )  # This selector gets all elements with text content
+_find_all_text_nodes = XPath(".//text()")


 class Selector(SelectorsGeneration):
@@ -299,18 +300,31 @@ class Selector(SelectorsGeneration):

        ignored_elements: set[Any] = set()
        if ignore_tags:
-            for element in self._root.iter(*ignore_tags):
-                ignored_elements.add(element)
-                ignored_elements.update(cast(list, _find_all_elements(element)))
+            ignored_elements.update(self._root.iter(*ignore_tags))

        _all_strings = []
-        for node in self._root.iter():
-            if node not in ignored_elements:
-                text = node.text
-                if text and isinstance(text, str):
-                    processed_text = text.strip() if strip else text
-                    if not valid_values or processed_text.strip():
-                        _all_strings.append(processed_text)
+
+        def append_text(text: str) -> None:
+            processed_text = text.strip() if strip else text
+            if not valid_values or processed_text.strip():
+                _all_strings.append(processed_text)
+
+        def is_visible_text_node(text_node: _ElementUnicodeResult) -> bool:
+            parent = text_node.getparent()
+            if parent is None:
+                return False
+
+            owner = parent.getparent() if text_node.is_tail else parent
+            while owner is not None:
+                if owner in ignored_elements:
+                    return False
+                owner = owner.getparent()
+            return True
+
+        for text_node in cast(list[_ElementUnicodeResult], _find_all_text_nodes(self._root)):
+            text = str(text_node)
+            if text and is_visible_text_node(text_node):
+                append_text(text)

        return cast(TextHandler, TextHandler(separator).join(_all_strings))

@@ -183,6 +183,33 @@ class TestAdvancedSelectors:
        text = page.get_all_text(valid_values=False)
        assert text != ""

+    def test_get_all_text_preserves_interleaved_text_nodes(self):
+        """Test get_all_text preserves interleaved text nodes"""
+        html = """
+        <html>
+        <body>
+            <main>
+                string1
+                <b>string2</b>
+                string3
+                <div>
+                    <span>string4</span>
+                </div>
+                string5
+                <script>ignored</script>
+                string6
+                <style>ignored</style>
+                string7
+            </main>
+        </body>
+        </html>
+        """
+
+        page = Selector(html, adaptive=False)
+        node = page.css("main")[0]
+
+        assert node.get_all_text("\n", strip=True) == "string1\nstring2\nstring3\nstring4\nstring5\nstring6\nstring7"
+

 class TestTextHandlerAdvanced:
    """Test advanced TextHandler functionality"""