freedak/Scrapling
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Selector.get_all_text() doesn't get all text #167

Browse Source 
- Updated text extraction using recursion
- Added new unit test
This commit is contained in:
Matt Hillebrand
2026-03-05 20:34:44 -08:00
committed by mhillebrand
parent 54f2c96adf
commit 5457a697fa
2 changed files with 41 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapling/parser.py
+17 -7
View File
@@ -304,13 +304,23 @@ class Selector(SelectorsGeneration):
ignored_elements.update(cast(list, _find_all_elements(element))) ignored_elements.update(cast(list, _find_all_elements(element)))
_all_strings = [] _all_strings = []
for node in self._root.iter():
if node not in ignored_elements: def append_text(text: Any) -> None:
text = node.text if text and isinstance(text, str):
if text and isinstance(text, str): processed_text = text.strip() if strip else text
processed_text = text.strip() if strip else text if not valid_values or processed_text.strip():
if not valid_values or processed_text.strip(): _all_strings.append(processed_text)
_all_strings.append(processed_text)
def walk(node: Any) -> None:
if node in ignored_elements:
return
append_text(node.text)
for child in node:
walk(child)
append_text(child.tail)
walk(self._root)
return cast(TextHandler, TextHandler(separator).join(_all_strings)) return cast(TextHandler, TextHandler(separator).join(_all_strings))
tests/parser/test_general.py
+24
View File
@@ -327,6 +327,30 @@ def test_getting_all_text(page):
assert page.get_all_text() != "" assert page.get_all_text() != ""
def test_getting_all_text_from_nested_content():
"""Test getting all text preserves interleaved text nodes"""
html = """
<html>
<body>
<main>
string1
<b>string2</b>
string3
<div>
<span>string4</span>
</div>
string5
</main>
</body>
</html>
"""
page = Selector(html, adaptive=False)
node = page.css("main")[0]
assert node.get_all_text("\n", strip=True) == "string1\nstring2\nstring3\nstring4\nstring5"
def test_regex_on_text(page): def test_regex_on_text(page):
"""Test regex operations on text""" """Test regex operations on text"""
element = page.css('[data-id="1"] .price')[0] element = page.css('[data-id="1"] .price')[0]