freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Selector.get_all_text() doesn't get all text #167 (#168)

Browse Source 
Fixes #167
This commit is contained in:
Karim shoair
2026-03-08 19:01:04 +02:00
committed by GitHub
parent 49db2b643b 7cbe566acd
commit 11bc08fbaa
2 changed files with 51 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/parser.py
+24 -10
View File
@@ -58,6 +58,7 @@ _find_all_elements = XPath(".//*")
_find_all_elements_with_spaces = XPath( _find_all_elements_with_spaces = XPath(
".//*[normalize-space(text())]" ".//*[normalize-space(text())]"
) # This selector gets all elements with text content ) # This selector gets all elements with text content
_find_all_text_nodes = XPath(".//text()")
class Selector(SelectorsGeneration): class Selector(SelectorsGeneration):
@@ -299,18 +300,31 @@ class Selector(SelectorsGeneration):
ignored_elements: set[Any] = set() ignored_elements: set[Any] = set()
if ignore_tags: if ignore_tags:
for element in self._root.iter(*ignore_tags): ignored_elements.update(self._root.iter(*ignore_tags))
ignored_elements.add(element)
ignored_elements.update(cast(list, _find_all_elements(element)))
_all_strings = [] _all_strings = []
for node in self._root.iter():
if node not in ignored_elements: def append_text(text: str) -> None:
text = node.text processed_text = text.strip() if strip else text
if text and isinstance(text, str): if not valid_values or processed_text.strip():
processed_text = text.strip() if strip else text _all_strings.append(processed_text)
if not valid_values or processed_text.strip():
_all_strings.append(processed_text) def is_visible_text_node(text_node: _ElementUnicodeResult) -> bool:
parent = text_node.getparent()
if parent is None:
return False
owner = parent.getparent() if text_node.is_tail else parent
while owner is not None:
if owner in ignored_elements:
return False
owner = owner.getparent()
return True
for text_node in cast(list[_ElementUnicodeResult], _find_all_text_nodes(self._root)):
text = str(text_node)
if text and is_visible_text_node(text_node):
append_text(text)
return cast(TextHandler, TextHandler(separator).join(_all_strings)) return cast(TextHandler, TextHandler(separator).join(_all_strings))
tests/parser/test_parser_advanced.py
+27
View File
@@ -183,6 +183,33 @@ class TestAdvancedSelectors:
text = page.get_all_text(valid_values=False) text = page.get_all_text(valid_values=False)
assert text != "" assert text != ""
def test_get_all_text_preserves_interleaved_text_nodes(self):
"""Test get_all_text preserves interleaved text nodes"""
html = """
<html>
<body>
<main>
string1
<b>string2</b>
string3
<div>
<span>string4</span>
</div>
string5
<script>ignored</script>
string6
<style>ignored</style>
string7
</main>
</body>
</html>
"""
page = Selector(html, adaptive=False)
node = page.css("main")[0]
assert node.get_all_text("\n", strip=True) == "string1\nstring2\nstring3\nstring4\nstring5\nstring6\nstring7"
class TestTextHandlerAdvanced: class TestTextHandlerAdvanced:
"""Test advanced TextHandler functionality""" """Test advanced TextHandler functionality"""